diff --git a/.cargo/config.toml b/.cargo/config.toml index ce65ee3f3a..618cc64d9d 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -70,8 +70,8 @@ rustflags = ["-C", "split-debuginfo=unpacked"] # results, 8 threads, slow-timeout termination), the root fixture feature and # `search-eval`, which is what compiles the evaluator acceptance suites (the # root crate's dependency on the evaluator is optional so the Linux transport -# partition does not build the eval-only lexical projection; a `--workspace` -# run resolves the evaluator anyway). `.github/workflows/ci.yml` runs this +# partition does not link it; a `--workspace` run resolves the evaluator +# anyway). `.github/workflows/ci.yml` runs this # exact alias, and its support binaries/archives are built with `--profile # perf` so they share artifacts with it. Extra flags append: `cargo test-ci # --locked -E 'binary(=x)'`. diff --git a/.gitattributes b/.gitattributes index 0f81578631..44627bb93d 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,9 +1,10 @@ # Force LF line endings for test fixtures. Tree-sitter grammars # expect Unix line endings and produce wrong parse trees with CRLF. tests/fixtures/** text eol=lf -# The embedded SQL fixture's definitions feed the exact configuration schema -# digest; checkout line-ending conversion must not change those definitions. -crates/tracedecay-global-db/src/configuration/store/tests/fixtures/configuration-registry-revision3.sql text eol=lf +# Embedded SQL is executed verbatim and its stored definitions feed exact +# schema-shape digests; a CRLF checkout rewrites those definitions and makes +# a released shape read as foreign. +*.sql text eol=lf # The search-quality candidate workload pins each corpus document against the # blob in the pinned source tree, and the packaged evaluator assets are pulled # into the binary with include_bytes! under a pinned digest. One corpus diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8e68675c24..e74da881a5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -461,9 +461,8 @@ jobs: # `tracedecay-search-eval`) with the root fixture feature. Journeys also # enable `tracedecay/search-eval`, the only link from the root crate to the # evaluator library. It stays off every other partition: an unconditional - # dependency unifies `tracedecay-query/search-eval` into every test - # target of the package, and the transport suites then compile the eval-only - # lexical projection. Every cargo invocation inside a job (the test build, + # dependency links the evaluator into every test target of the package. + # Every cargo invocation inside a job (the test build, # the executables the suites spawn) is a cache hit against that job's # resolution. `scripts/linux-test-partitions.py check` proves, from `cargo # metadata`, that every test target in the workspace is selected by exactly @@ -859,7 +858,7 @@ jobs: # Acceptance tests execute the ordinary evaluator binaries, and # `tracedecay/search-eval` is what compiles the suites that check the # CLI receipt against the library. This lane builds `--workspace`, so - # the evaluator and `tracedecay-query/search-eval` already resolve here + # the evaluator already resolves here # and the feature adds no compilation; the Linux partitions split the # package selection, which is why it is per-partition there. Match the # archive's features and perf profile to reuse dependency artifacts; diff --git a/Cargo.lock b/Cargo.lock index eafc4e7cd9..778fcb2027 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3280,6 +3280,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "jsonc-parser" +version = "0.33.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ff5a48f48971be8e762a6ff955725a0802b6e46c441057992da5a673db9fd3a" +dependencies = [ + "serde_json", +] + [[package]] name = "jsonschema" version = "0.46.10" @@ -5455,6 +5464,7 @@ dependencies = [ "gix", "hex", "hotpath", + "jsonc-parser", "same-file", "serde", "serde_json", @@ -5568,6 +5578,7 @@ name = "tracedecay-automation" version = "0.1.0" dependencies = [ "hotpath", + "schemars", "serde", "serde_json", "sha2", @@ -5585,6 +5596,7 @@ dependencies = [ "hex", "hotpath", "rustls", + "schemars", "serde", "serde_json", "sha2", @@ -5679,6 +5691,7 @@ dependencies = [ "tracedecay-maintenance", "tracedecay-mcp", "tracedecay-private-fs", + "tracedecay-project", "tracedecay-runtime-core", "tracedecay-sdk", "tracedecay-session-memory", @@ -5719,6 +5732,7 @@ version = "0.1.0" dependencies = [ "ast-grep-core", "criterion", + "flate2", "hex", "hotpath", "ignore", @@ -5732,6 +5746,7 @@ dependencies = [ "thiserror 2.0.20", "toml", "tracedecay-code-extraction", + "tracedecay-code-index", "tracedecay-contracts", "tracedecay-domain", "tracedecay-graph-db", @@ -5758,6 +5773,7 @@ dependencies = [ "thiserror 2.0.20", "tracedecay-code-index", "tracedecay-domain", + "tracedecay-graph-db", "tracedecay-private-fs", "tracedecay-runtime-core", "tracedecay-store", @@ -5810,7 +5826,6 @@ dependencies = [ name = "tracedecay-configuration" version = "0.1.0" dependencies = [ - "glob", "hotpath", "serde", "serde_json", @@ -5891,6 +5906,7 @@ dependencies = [ "semver", "serde", "serde_json", + "serde_path_to_error", "tempfile", "thiserror 2.0.20", "tokio", @@ -6366,6 +6382,7 @@ dependencies = [ "hotpath", "libc", "tempfile", + "tracing", "windows-sys 0.61.2", ] @@ -6416,6 +6433,7 @@ dependencies = [ name = "tracedecay-query" version = "0.1.0" dependencies = [ + "flate2", "fst", "gix", "hex", @@ -6436,7 +6454,6 @@ dependencies = [ "tracedecay-domain", "tracedecay-graph-db", "tracedecay-private-fs", - "tracedecay-query", "tracedecay-runtime-core", "tracedecay-temporal-query", "tracing", @@ -6544,7 +6561,7 @@ dependencies = [ "hotpath", "memmap2", "regex", - "rusqlite", + "schemars", "serde", "serde_json", "sha2", @@ -6584,6 +6601,7 @@ dependencies = [ "tracedecay-application", "tracedecay-code-index", "tracedecay-code-index-runtime", + "tracedecay-configuration", "tracedecay-contracts", "tracedecay-daemon-identity", "tracedecay-daemon-protocol", @@ -6700,6 +6718,7 @@ name = "tracedecay-store" version = "0.1.0" dependencies = [ "hotpath", + "schemars", "serde", "serde_json", "sha2", @@ -6722,7 +6741,6 @@ dependencies = [ "tempfile", "thiserror 2.0.20", "tokio", - "tracedecay-automation-runtime", "tracedecay-code-index", "tracedecay-code-index-retention", "tracedecay-code-index-runtime", diff --git a/Cargo.toml b/Cargo.toml index 488a19d264..74f194fc77 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -85,6 +85,7 @@ rusqlite = { version = "0.40.1", default-features = false, features = ["backup", # resolve the same object database and status implementation, so a feature # added for one is never missing for the other. gix = { version = "=0.87.1", default-features = false, features = ["revision", "blob-diff", "parallel", "sha1", "sha256", "status"] } +schemars = "1.2.1" [profile.bench] opt-level = 3 diff --git a/README.md b/README.md index e38d65eccb..d89a74316b 100644 --- a/README.md +++ b/README.md @@ -102,7 +102,6 @@ the installed cache is loaded. tracedecay daemon install-service # install + start the daemon (required before init) tracedecay init [path] # enroll a project and publish its first generation tracedecay sync [path] # explicit administrative refresh -tracedecay sync --force [path] # explicit full generation refresh tracedecay status [path] # graph stats, freshness, savings, cost tracedecay tool # list every MCP tool tracedecay tool search "" # CLI symbol search @@ -124,7 +123,7 @@ list and `tracedecay tool --help` for one tool's parameters. The MCP server exposes tools grouped around normal coding workflows: -- Discovery: `tracedecay_context`, `tracedecay_search`, `tracedecay_outline`, `tracedecay_files` +- Discovery: `tracedecay_context`, `tracedecay_search`, `tracedecay_source_outline`, `tracedecay_files` - Graph traversal: `tracedecay_callers`, `tracedecay_callees`, `tracedecay_impact`, `tracedecay_affected` - Code health: `tracedecay_complexity`, `tracedecay_dead_code`, `tracedecay_unmounted_files`, `tracedecay_coupling`, `tracedecay_test_risk` - Git workflow: `tracedecay_diff_context`, `tracedecay_pr_context`, `tracedecay_changelog`, `tracedecay_test_map` diff --git a/SECURITY.md b/SECURITY.md index 0f2f179107..4afba2b9ee 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -31,7 +31,7 @@ tracedecay builds a **local** code graph stored in the active project store. Rep - Call relationships and dependency edges - FTS5 search index - Cross-session memory: durable facts, named entities, code-area notes, decisions, and feedback events in the holographic fact store. Those rows are local-only project data. -- A response cache for `tracedecay_read` (`read_cache` table): the rendered output served to the agent, stored as a BLOB keyed by file path, mode, and arguments. For full/line-range reads this rendered output contains source text. Rows are freshness-gated by file mtime and swept after a period of inactivity. +- A response cache for mode-aware source reads (`read_cache` table): the rendered output served to the agent, stored as a BLOB keyed by file path, mode, and arguments. For full/line-range reads this rendered output contains source text. Rows are freshness-gated by file mtime and swept after a period of inactivity. Aside from the `read_cache`, the graph itself does **not** persist raw source code. It stores structural metadata only. The active project store is local-only. There is no cloud sync, remote database, or server-side storage. @@ -148,7 +148,7 @@ The Windows-elevation `unsafe` documented in earlier versions was removed alongs ## Best Practices - Add `.tracedecay/` to your `.gitignore` to avoid committing local store markers or repo-local databases. -- If your project contains sensitive code, be aware that the database stores symbol names and signatures, and the `read_cache` table can hold rendered source text from `tracedecay_read` responses. Keeping repo-local store directories ignored and treating profile-sharded stores as private user data keeps both out of version control. +- If your project contains sensitive code, be aware that the database stores symbol names and signatures, and the `read_cache` table can hold rendered source text from source-read responses. Keeping repo-local store directories ignored and treating profile-sharded stores as private user data keeps both out of version control. - Keep tracedecay updated (`tracedecay upgrade`) to receive security fixes. - Review the [CHANGELOG](CHANGELOG.md) before upgrading to understand what changed. diff --git a/benchmark_data/run_benchmarks.py b/benchmark_data/run_benchmarks.py index c33d4fe6d6..6482095be8 100644 --- a/benchmark_data/run_benchmarks.py +++ b/benchmark_data/run_benchmarks.py @@ -342,14 +342,20 @@ def avg_ms(samples: list[int]) -> float | None: ] out["find_symbol_avg_ms"] = avg_ms(find_us) - body_us = [ - d - for q in queries - if ( - d := handler_us(mcp.call_tool("tracedecay_body", {"symbol": q, "limit": 1})) + # Name-to-source is an exact lookup followed by `tracedecay_source_body`. + body_us: list[int] = [] + for q in queries: + fresp = mcp.call_tool( + "tracedecay_find_exact_symbol", {"name": q, "limit": 1, "format": "json"} ) - is not None - ] + try: + matches = json.loads(fresp["result"]["content"][0]["text"])["matches"] + node_id = matches[0]["id"] + except (KeyError, IndexError, TypeError, json.JSONDecodeError): + continue + bus = handler_us(mcp.call_tool("tracedecay_source_body", {"node_id": node_id})) + if bus is not None: + body_us.append((handler_us(fresp) or 0) + bus) out["get_function_source_avg_ms"] = avg_ms(body_us) impact_us: list[int] = [] diff --git a/benchmark_data/transport-boundary/README.md b/benchmark_data/transport-boundary/README.md index 7ed2e027cc..14c82862a0 100644 --- a/benchmark_data/transport-boundary/README.md +++ b/benchmark_data/transport-boundary/README.md @@ -1,26 +1,10 @@ -# Transport-boundary direct test fixture +# Transport-boundary compile baseline -> **Historical evidence only.** Preserve the real fixture and provenance in -> this directory. Current requirements come only from the -> `docs/plans/tracedecay-v2/` hierarchy; exact commands, test names/counts, -> snapshots, receipts, attestations, PR packets, and gate fields below are not -> rebuild instructions. Validate current transport behavior directly. - -This directory supplies a checked-in real fixture to the direct product test -for the callable application transport boundary. It is not an acceptance packet, -snapshot, gate manifest, or separate evidence authority. Product acceptance -comes from direct tests and normal CI, with pending work reported as pending. - -`goldens/application-surface-parity.json` is consumed by -`tests/api_application_parity.rs`. Together they require: - -- identical request/result contracts and binding identities for Git preview, - Git apply, feedback diagnostics, feedback get, feedback expand, and feedback - list across CLI, MCP, and HTTP dispatch; -- all four feedback reads to reach a callable owner and conceal an unknown - request handle as `not_found_or_not_authorized`, never `unavailable`; -- HTTP and SSE to project the same canonical feedback payload; and -- concealed HTTP problems to omit binding identity. +> **Historical evidence only.** Current requirements come only from the +> `docs/plans/tracedecay-v2/` hierarchy; exact commands and fields below are +> not rebuild instructions. +> `crates/tracedecay/tests/product_surface_suite/api_application_parity.rs` +> verifies transport parity directly. The compile workload remains executable, but its measurement is explicitly pending execution. Static validation does not run Cargo: diff --git a/benchmark_data/transport-boundary/goldens/application-surface-parity.json b/benchmark_data/transport-boundary/goldens/application-surface-parity.json deleted file mode 100644 index 3a50671f63..0000000000 --- a/benchmark_data/transport-boundary/goldens/application-surface-parity.json +++ /dev/null @@ -1,456 +0,0 @@ -{ - "schema_version": 1, - "operations": { - "git_preview": { - "request_schema": "schema.application.git.preview.request", - "result_schema": "schema.application.git.preview.result", - "bindings": { - "cli": "binding.cli.git_preview.v1", - "mcp": "binding.mcp.git_preview.v1" - } - }, - "git_apply": { - "request_schema": "schema.application.git.apply.request", - "result_schema": "schema.application.git.apply.result", - "bindings": { - "cli": "binding.cli.git_apply.v1", - "mcp": "binding.mcp.git_apply.v1" - } - }, - "feedback_diagnostics": { - "request_handle": "rh_missing-application-parity", - "request_schema": "schema.application.feedback.diagnostics.request", - "result_schema": "schema.application.feedback.diagnostics.result", - "bindings": { - "cli": "binding.cli.feedback_diagnostics.v1", - "mcp": "binding.mcp.feedback_diagnostics.v1", - "http": "binding.http.feedback_diagnostics.v1" - } - }, - "feedback_get": { - "request_handle": "rh_missing-application-parity", - "request_schema": "schema.application.feedback.get.request", - "result_schema": "schema.application.feedback.get.result", - "bindings": { - "cli": "binding.cli.feedback_get.v1", - "mcp": "binding.mcp.feedback_get.v1", - "http": "binding.http.feedback_get.v1" - } - }, - "feedback_expand": { - "request_handle": "rh_missing-application-parity", - "request_schema": "schema.application.feedback.expand.request", - "result_schema": "schema.application.feedback.expand.result", - "bindings": { - "cli": "binding.cli.feedback_expand.v1", - "mcp": "binding.mcp.feedback_expand.v1", - "http": "binding.http.feedback_expand.v1" - } - }, - "feedback_list": { - "request_handle": "rh_missing-application-parity", - "request_schema": "schema.application.feedback.list.request", - "result_schema": "schema.application.feedback.list.result", - "bindings": { - "cli": "binding.cli.feedback_list.v1", - "mcp": "binding.mcp.feedback_list.v1", - "http": "binding.http.feedback_list.v1" - } - }, - "feedback_proximity": { - "request": { - "observed_at": 0 - }, - "request_schema": "schema.application.feedback.proximity.request", - "result_schema": "schema.application.feedback.proximity.result", - "bindings": { - "cli": "binding.cli.feedback_proximity.v1", - "mcp": "binding.mcp.feedback_proximity.v1", - "http": "binding.http.feedback_proximity.v1" - } - }, - "feedback_impact": { - "request": { - "node_id": "node.application-parity.fixture" - }, - "request_schema": "schema.application.feedback.impact.request", - "result_schema": "schema.application.feedback.impact.result", - "bindings": { - "cli": "binding.cli.feedback_impact.v1", - "mcp": "binding.mcp.feedback_impact.v1", - "http": "binding.http.feedback_impact.v1" - } - }, - "affected_tests": { - "request": { - "files": [ - "src/lib.rs" - ] - }, - "request_schema": "schema.application.feedback.affected-tests.request", - "result_schema": "schema.application.feedback.affected-tests.result", - "bindings": { - "cli": "binding.cli.affected_tests.v1", - "mcp": "binding.mcp.affected_tests.v1", - "http": "binding.http.affected_tests.v1" - } - }, - "test_results": { - "request": {}, - "request_schema": "schema.application.feedback.test-results.request", - "result_schema": "schema.application.feedback.test-results.result", - "bindings": { - "cli": "binding.cli.test_results.v1", - "mcp": "binding.mcp.test_results.v1", - "http": "binding.http.test_results.v1" - } - }, - "code_callees": { - "bindings": { - "cli": "binding.cli.code_callees.v1", - "http": "binding.http.code_callees.v1", - "mcp": "binding.mcp.code_callees.v1" - }, - "request": { - "maximum_depth": 3, - "meta": { - "order": "source_position", - "projection": "evidence" - }, - "node_id": "node.application-parity", - "resolve_trait_dispatch": true, - "scope": { - "generation": "generation.application-parity", - "path_prefix": "src" - } - }, - "request_schema": "schema.application.code-query.callees.request", - "result_schema": "schema.application.code-query.callees.result" - }, - "code_callers": { - "bindings": { - "cli": "binding.cli.code_callers.v1", - "http": "binding.http.code_callers.v1", - "mcp": "binding.mcp.code_callers.v1" - }, - "request": { - "maximum_depth": 3, - "meta": { - "order": "source_position", - "projection": "evidence" - }, - "node_id": "node.application-parity", - "scope": { - "path_prefix": "src" - } - }, - "request_schema": "schema.application.primitive.code-callers.request", - "result_schema": "schema.application.primitive.code-callers.result" - }, - "code_declaration": { - "bindings": { - "cli": "binding.cli.code_declaration.v1", - "http": "binding.http.code_declaration.v1", - "mcp": "binding.mcp.code_declaration.v1" - }, - "request": { - "meta": { - "order": "source_position", - "projection": "evidence" - }, - "node_id": "symbol.application-parity", - "scope": { - "generation": "generation.application-parity", - "path_prefix": "src" - } - }, - "request_schema": "schema.application.code-query.declaration.request", - "result_schema": "schema.application.code-query.declaration.result" - }, - "code_exact_occurrence": { - "bindings": { - "cli": "binding.cli.code_exact_occurrence.v1", - "http": "binding.http.code_exact_occurrence.v1", - "mcp": "binding.mcp.code_exact_occurrence.v1" - }, - "request": { - "kind": "whole_symbol", - "literal": "ApplicationSurfaceOperation", - "meta": { - "order": "source_position", - "projection": "evidence" - }, - "scope": { - "generation": "generation.application-parity", - "path_prefix": "src" - } - }, - "request_schema": "schema.application.code-query.exact-occurrence.request", - "result_schema": "schema.application.code-query.exact-occurrence.result" - }, - "code_facets": { - "bindings": { - "cli": "binding.cli.code_facets.v1", - "http": "binding.http.code_facets.v1", - "mcp": "binding.mcp.code_facets.v1" - }, - "request": { - "dimension": "language", - "meta": { - "order": "source_position", - "projection": "evidence" - }, - "scope": { - "generation": "generation.application-parity", - "path_prefix": "src" - } - }, - "request_schema": "schema.application.code-query.facets.request", - "result_schema": "schema.application.code-query.facets.result" - }, - "code_implementations": { - "bindings": { - "cli": "binding.cli.code_implementations.v1", - "http": "binding.http.code_implementations.v1", - "mcp": "binding.mcp.code_implementations.v1" - }, - "request": { - "meta": { - "order": "source_position", - "projection": "evidence" - }, - "scope": { - "path_prefix": "src" - }, - "selector": { - "name": "HttpApplicationOwners", - "selector": "trait" - } - }, - "request_schema": "schema.application.primitive.code-implementations.request", - "result_schema": "schema.application.primitive.code-implementations.result" - }, - "code_phrase_search": { - "bindings": { - "cli": "binding.cli.code_phrase_search.v1", - "http": "binding.http.code_phrase_search.v1", - "mcp": "binding.mcp.code_phrase_search.v1" - }, - "request": { - "field_filters": [ - { - "field": "path", - "include": true - } - ], - "fuzzy_budget": 7, - "meta": { - "order": "source_position", - "projection": "evidence" - }, - "phrases": [ - "callable application", - "surface" - ], - "query": "callable application surface", - "scope": { - "generation": "generation.application-parity", - "path_prefix": "src" - } - }, - "request_schema": "schema.application.code-query.phrase-search.request", - "result_schema": "schema.application.code-query.phrase-search.result" - }, - "code_references": { - "bindings": { - "cli": "binding.cli.code_references.v1", - "http": "binding.http.code_references.v1", - "mcp": "binding.mcp.code_references.v1" - }, - "request": { - "meta": { - "order": "source_position", - "projection": "evidence" - }, - "node_id": "symbol.application-parity", - "scope": { - "generation": "generation.application-parity", - "path_prefix": "src" - } - }, - "request_schema": "schema.application.code-query.references.request", - "result_schema": "schema.application.code-query.references.result" - }, - "code_signature_search": { - "bindings": { - "cli": "binding.cli.code_signature_search.v1", - "http": "binding.http.code_signature_search.v1", - "mcp": "binding.mcp.code_signature_search.v1" - }, - "request": { - "is_async": true, - "meta": { - "order": "source_position", - "projection": "evidence" - }, - "params": [ - "RequestContext" - ], - "returns": "ApplicationResult", - "scope": { - "path_prefix": "src" - } - }, - "request_schema": "schema.application.primitive.code-signature-search.request", - "result_schema": "schema.application.primitive.code-signature-search.result" - }, - "code_symbol_search": { - "bindings": { - "cli": "binding.cli.code_symbol_search.v1", - "http": "binding.http.code_symbol_search.v1", - "mcp": "binding.mcp.code_symbol_search.v1" - }, - "request": { - "lazy_index_ignored_dependencies": false, - "meta": { - "order": "source_position", - "projection": "evidence" - }, - "query": "ApplicationSurfaceOperation", - "scope": { - "path_prefix": "src" - } - }, - "request_schema": "schema.application.symbol-search.request", - "result_schema": "schema.application.symbol-search.result" - }, - "code_timeline": { - "bindings": { - "cli": "binding.cli.code_timeline.v1", - "http": "binding.http.code_timeline.v1", - "mcp": "binding.mcp.code_timeline.v1" - }, - "request": { - "meta": { - "order": "source_position", - "projection": "evidence" - }, - "scope": { - "generation": "generation.application-parity", - "path_prefix": "src" - } - }, - "request_schema": "schema.application.code-query.timeline.request", - "result_schema": "schema.application.code-query.timeline.result" - }, - "code_type_definition": { - "bindings": { - "cli": "binding.cli.code_type_definition.v1", - "http": "binding.http.code_type_definition.v1", - "mcp": "binding.mcp.code_type_definition.v1" - }, - "request": { - "meta": { - "order": "source_position", - "projection": "evidence" - }, - "node_id": "symbol.application-parity", - "scope": { - "generation": "generation.application-parity", - "path_prefix": "src" - } - }, - "request_schema": "schema.application.code-query.type-definition.request", - "result_schema": "schema.application.code-query.type-definition.result" - }, - "code_type_hierarchy": { - "bindings": { - "cli": "binding.cli.code_type_hierarchy.v1", - "http": "binding.http.code_type_hierarchy.v1", - "mcp": "binding.mcp.code_type_hierarchy.v1" - }, - "request": { - "maximum_depth": 3, - "meta": { - "order": "source_position", - "projection": "evidence" - }, - "node_id": "node.application-parity", - "scope": { - "path_prefix": "src" - } - }, - "request_schema": "schema.application.primitive.code-type-hierarchy.request", - "result_schema": "schema.application.primitive.code-type-hierarchy.result" - } - }, - "http_sse": { - "http_binding": "binding.http.feedback_list.v1", - "result_schema": "schema.application.feedback.list.result", - "sequence": 7, - "item": { - "finding_id": "feedback-finding.application-parity.fixture", - "summary": "Canonical feedback evidence remains transport-neutral" - } - }, - "authorization_concealment": { - "problem_kind": "not_found_or_not_authorized", - "binding_identity": "omitted", - "unknown_handle_behavior": "same_as_unauthorized" - }, - "unpinned_operations": [ - "git_status", - "git_diff", - "git_history", - "git_blame", - "git_hunks", - "stack_snapshot", - "github_stack_signal_expand", - "preflight_native_integration", - "approve_native_integration", - "apply_native_integration", - "native_integration_status", - "cancel_native_integration", - "worktree_inventory", - "worktree_cleanup_inspect", - "worktree_cleanup_confirm", - "worktree_cleanup_remove", - "worktree_cleanup_reconcile", - "feedback_advisory_cycle", - "session_lookup", - "qualified_name", - "call_chain", - "file_dependents", - "source_lines", - "source_body", - "source_outline", - "module_api", - "health_read", - "health_delta", - "storage_status", - "diagnostics_read", - "observatory_read", - "configuration_list", - "configuration_get", - "configuration_set", - "configuration_unset", - "configuration_batch", - "configuration_observed_state", - "configuration_protected_preview", - "configuration_protected_apply", - "configuration_rollback_preview", - "configuration_rollback_apply", - "configuration_audit", - "context_scout_status", - "context_scout_recent", - "context_scout_explain", - "context_scout_capability", - "context_scout_budget", - "context_scout_pause", - "context_scout_resume", - "context_scout_cancel", - "context_scout_claim", - "context_scout_delivery", - "context_scout_feedback" - ] -} diff --git a/benchmark_data/tsbench/README.md b/benchmark_data/tsbench/README.md index dff32d1988..86df20a77c 100644 --- a/benchmark_data/tsbench/README.md +++ b/benchmark_data/tsbench/README.md @@ -42,7 +42,7 @@ PY `token_savior.server` over Python stdio. - **System prompt**, rewrites `SYSTEM_PROMPT_TS` to map each token-savior tool to its tracedecay equivalent (`find_symbol` → - `tracedecay_find_exact_symbol`, `get_function_source` → `tracedecay_body`, + `tracedecay_find_exact_symbol`, `get_function_source` → `tracedecay_source_body`, `get_full_context` → `tracedecay_context`, etc.). Where no tracedecay equivalent exists (`add_field_to_model`, `move_symbol`, `analyze_config`, `analyze_docker`), the prompt explicitly allows diff --git a/benchmark_data/tsbench/bench_tracedecay.patch b/benchmark_data/tsbench/bench_tracedecay.patch index 5faa657fb1..f6a39853ef 100644 --- a/benchmark_data/tsbench/bench_tracedecay.patch +++ b/benchmark_data/tsbench/bench_tracedecay.patch @@ -40,12 +40,12 @@ -- After empty find_symbol: try search_codebase. After empty search_codebase: Read/Grep are allowed for non-indexed files (.prisma, .sql, .graphql, .proto). +- Locate (exact name): `tracedecay_find_exact_symbol(name)`. One call. O(log n) index probe. +- Locate (ranked, fuzzy): `tracedecay_search(query, limit)`. Use when you don't know the exact name. -+- Read source of a function/class/struct/etc: `tracedecay_body(symbol, limit=1)`. One call. Returns the full body text. ++- Read source of a function/class/struct/etc: resolve with `tracedecay_find_exact_symbol(name)`, then `tracedecay_source_body(node_id)`. Returns the full body text. +- Whole-task context (loc + related code): `tracedecay_context(task="")`. Use this when the prompt is open-ended. +- Edit a function/method/class body: `tracedecay_replace_symbol(symbol, new_source)`. The new_source MUST include the symbol's own declaration line (e.g. `def foo(...):`). Reindex is automatic. +- Insert near a symbol: `tracedecay_insert_at_symbol(symbol, content, position="before"|"after")`. For non-symbol locations use `tracedecay_insert_at(path, anchor, content)`. +- Add a field to a Prisma model or TS interface/type: NO direct tracedecay tool — use `Edit` or `tracedecay_str_replace(path, old_str, new_str)`. Re-run after to verify the edit landed. -+- Move a symbol across files: NO direct tracedecay tool — combine `tracedecay_body` (read), `tracedecay_str_replace` (delete from source file + paste in target file), and `Edit` for import-site fixups. ++- Move a symbol across files: NO direct tracedecay tool — combine `tracedecay_source_body` (read), `tracedecay_str_replace` (delete from source file + paste in target file), and `Edit` for import-site fixups. +- Detect cycles: `tracedecay_circular`. Do not infer manually. +- Detect duplicates: `tracedecay_similar(name|file)`. Enumerate the pairs returned. The MCP tool surfaces semantic neighbors, not strict duplicates — interpret accordingly. +- Diff between refs/branches: `tracedecay_branch_diff(base, head)` for tracked branches, or `tracedecay_changelog` for commit-based diffs. diff --git a/crates/tracedecay-agent-hosts/Cargo.toml b/crates/tracedecay-agent-hosts/Cargo.toml index 521f41e202..eb0cd7d1f1 100644 --- a/crates/tracedecay-agent-hosts/Cargo.toml +++ b/crates/tracedecay-agent-hosts/Cargo.toml @@ -31,6 +31,7 @@ same-file = "1.0.6" getrandom = "0.2" hex = "0.4" hotpath.workspace = true +jsonc-parser = { version = "0.33.2", features = ["serde_json"] } serde = { version = "1", features = ["derive"] } serde_json = "1" sha2 = "0.11" diff --git a/crates/tracedecay-agent-hosts/src/agents/antigravity.rs b/crates/tracedecay-agent-hosts/src/agents/antigravity.rs index 4eee058b31..da5c95a51d 100644 --- a/crates/tracedecay-agent-hosts/src/agents/antigravity.rs +++ b/crates/tracedecay-agent-hosts/src/agents/antigravity.rs @@ -29,7 +29,7 @@ use tracedecay_domain::errors::{Result, TraceDecayError}; use super::host_bundle::{HostBundleRegistrationStateV1, HostComponentV1}; use super::{ AgentIntegration, DoctorCounters, HealthcheckContext, InstallContext, JsonConfigDialect, - McpDoctorLabels, TextFileMutation, config_backup_path, report_mcp_registration, + McpDoctorLabels, TextFileMutation, report_mcp_registration, update_two_config_files_transactionally, }; @@ -45,10 +45,6 @@ fn cli_plugin_path(home: &Path) -> PathBuf { home.join(".gemini/antigravity-cli/plugins/tracedecay.json") } -fn original_config_path(config: &Path) -> PathBuf { - PathBuf::from(format!("{}.tracedecay-original", config.display())) -} - impl AgentIntegration for AntigravityIntegration { fn name(&self) -> &'static str { "Antigravity" @@ -115,7 +111,7 @@ impl AgentIntegration for AntigravityIntegration { if components != [HostComponentV1::ContextMcp] { return Vec::new(); } - registration_paths(home) + vec![mcp_config_path(home), cli_plugin_path(home)] } #[hotpath::measure(label = "antigravity_mcp_install")] @@ -144,19 +140,6 @@ impl AgentIntegration for AntigravityIntegration { } } -fn registration_paths(home: &Path) -> Vec { - let ide = mcp_config_path(home); - let cli = cli_plugin_path(home); - vec![ - ide.clone(), - config_backup_path(&ide), - original_config_path(&ide), - cli.clone(), - config_backup_path(&cli), - original_config_path(&cli), - ] -} - fn document_registration_state( config: &Path, expected_binary: Option<&str>, @@ -270,21 +253,9 @@ fn add_registration(config: &Path, existing: &str, binary: &str) -> Result Result<()> { - let original = original_config_path(config); - let settings = parse_document(config, existing)?; - if settings.pointer("/mcpServers/tracedecay").is_none() - && config.is_file() - && !original.exists() - { - super::safe_write_bytes_file(&original, existing.as_bytes(), None)?; - } - Ok(()) + Ok(TextFileMutation::Write( + JsonConfigDialect::Json.render_edit(config, existing, &settings)?, + )) } fn install_mcp_if_selected(components: &[HostComponentV1], ctx: &InstallContext) -> Result<()> { @@ -293,51 +264,16 @@ fn install_mcp_if_selected(components: &[HostComponentV1], ctx: &InstallContext) } let ide = mcp_config_path(&ctx.home); let cli = cli_plugin_path(&ctx.home); - let ide_original = original_config_path(&ide); - let cli_original = original_config_path(&cli); - let ide_original_existed = ide_original.exists(); - let cli_original_existed = cli_original.exists(); - let result = - update_two_config_files_transactionally(&ide, &cli, |ide_existing, cli_existing| { - save_original_if_needed(&ide, ide_existing)?; - save_original_if_needed(&cli, cli_existing)?; - Ok(( - (), - add_registration(&ide, ide_existing, &ctx.tracedecay_bin)?, - add_registration(&cli, cli_existing, &ctx.tracedecay_bin)?, - )) - }); - if result.is_err() { - remove_new_original(&ide_original, ide_original_existed)?; - remove_new_original(&cli_original, cli_original_existed)?; - } - result + update_two_config_files_transactionally(&ide, &cli, |ide_existing, cli_existing| { + Ok(( + (), + add_registration(&ide, ide_existing, &ctx.tracedecay_bin)?, + add_registration(&cli, cli_existing, &ctx.tracedecay_bin)?, + )) + }) } -fn remove_new_original(path: &Path, existed_before: bool) -> Result<()> { - if !existed_before && path.exists() { - super::safe_remove_host_file(path).map_err(|error| TraceDecayError::Config { - message: format!( - "failed to remove {} after rollback: {error}", - path.display() - ), - })?; - } - Ok(()) -} - -#[derive(Clone, Copy)] -enum DocumentRemoval { - NoEntry, - RestoredOriginal, - RemovedFile, - Rewritten, -} - -fn remove_registration( - config: &Path, - existing: &str, -) -> Result<(DocumentRemoval, TextFileMutation)> { +fn remove_registration(config: &Path, existing: &str) -> Result { let mut settings = parse_document(config, existing)?; let Some(root) = settings.as_object_mut() else { return Err(TraceDecayError::Config { @@ -348,33 +284,19 @@ fn remove_registration( .get_mut("mcpServers") .and_then(serde_json::Value::as_object_mut) else { - return Ok((DocumentRemoval::NoEntry, TextFileMutation::Unchanged)); + return Ok(TextFileMutation::Unchanged); }; if servers.remove("tracedecay").is_none() { - return Ok((DocumentRemoval::NoEntry, TextFileMutation::Unchanged)); + return Ok(TextFileMutation::Unchanged); } if servers.is_empty() { root.remove("mcpServers"); } - let root_is_empty = root.is_empty(); - let original = original_config_path(config); - if let Ok(bytes) = std::fs::read(&original) - && serde_json::from_slice::(&bytes).ok() == Some(settings.clone()) - { - let bytes = String::from_utf8(bytes).map_err(|error| TraceDecayError::Config { - message: format!("{} is not valid UTF-8: {error}", original.display()), - })?; - return Ok(( - DocumentRemoval::RestoredOriginal, - TextFileMutation::Write(bytes), - )); + if root.is_empty() { + return Ok(TextFileMutation::Remove); } - if root_is_empty { - return Ok((DocumentRemoval::RemovedFile, TextFileMutation::Remove)); - } - Ok(( - DocumentRemoval::Rewritten, - TextFileMutation::Write(super::render_json_config(config, &settings)?), + Ok(TextFileMutation::Write( + JsonConfigDialect::Json.render_edit(config, existing, &settings)?, )) } @@ -384,24 +306,13 @@ fn uninstall_mcp_if_selected(components: &[HostComponentV1], home: &Path) -> Res } let ide = mcp_config_path(home); let cli = cli_plugin_path(home); - let (ide_outcome, cli_outcome) = - update_two_config_files_transactionally(&ide, &cli, |ide_existing, cli_existing| { - let (ide_outcome, ide_mutation) = remove_registration(&ide, ide_existing)?; - let (cli_outcome, cli_mutation) = remove_registration(&cli, cli_existing)?; - Ok(((ide_outcome, cli_outcome), ide_mutation, cli_mutation)) - })?; - remove_restored_original(&ide, ide_outcome)?; - remove_restored_original(&cli, cli_outcome) -} - -fn remove_restored_original(config: &Path, outcome: DocumentRemoval) -> Result<()> { - if matches!(outcome, DocumentRemoval::RestoredOriginal) { - let original = original_config_path(config); - super::safe_remove_host_file(&original).map_err(|error| TraceDecayError::Config { - message: format!("failed to remove {}: {error}", original.display()), - })?; - } - Ok(()) + update_two_config_files_transactionally(&ide, &cli, |ide_existing, cli_existing| { + Ok(( + (), + remove_registration(&ide, ide_existing)?, + remove_registration(&cli, cli_existing)?, + )) + }) } #[cfg(test)] @@ -414,7 +325,6 @@ mod tests { InstallContext { home: home.to_path_buf(), tracedecay_bin: binary.to_string(), - tool_permissions: Vec::new(), project_root: None, dashboard: false, } @@ -426,7 +336,7 @@ mod tests { } #[test] - fn antigravity_lifecycle_preserves_peers_and_restores_both_documents() { + fn antigravity_lifecycle_preserves_peers_and_keeps_no_copies() { let home = tempfile::tempdir().unwrap(); let ide = mcp_config_path(home.path()); let cli = cli_plugin_path(home.path()); @@ -458,8 +368,17 @@ mod tests { .deactivate_deployed_host_component_registration(&components, &install) .unwrap(); - assert_eq!(std::fs::read(&ide).unwrap(), ide_original); - assert_eq!(std::fs::read(&cli).unwrap(), cli_original); + for (path, original) in [(&ide, &ide_original[..]), (&cli, &cli_original[..])] { + assert_eq!( + super::super::load_json_file(path), + serde_json::from_slice::(original).unwrap() + ); + let siblings: Vec<_> = std::fs::read_dir(path.parent().unwrap()) + .unwrap() + .map(|entry| entry.unwrap().file_name()) + .collect(); + assert_eq!(siblings, vec![path.file_name().unwrap().to_owned()]); + } } #[test] diff --git a/crates/tracedecay-agent-hosts/src/agents/claude.rs b/crates/tracedecay-agent-hosts/src/agents/claude.rs index 3e4210c5db..e7c8dcdc7e 100644 --- a/crates/tracedecay-agent-hosts/src/agents/claude.rs +++ b/crates/tracedecay-agent-hosts/src/agents/claude.rs @@ -7,11 +7,11 @@ //! commands. TraceDecay stages the source; Claude Code owns registration, //! enabled state, cache, and trust through its native plugin commands. //! -//! 1. Deploy the embedded bundle to a stable marketplace dir -//! (`~/.claude/plugins/marketplaces/tracedecay/`), stamping the plugin -//! version and substituting the resolved tracedecay binary path. -//! 2. The operator runs Claude Code's native `claude plugin` command against -//! that source and then retries TraceDecay so the receipt can be tracked. +//! 1. The receipt-backed component transaction deploys the rendered bundle to +//! a stable marketplace dir (`~/.claude/plugins/marketplaces/tracedecay/`), +//! stamping the plugin version and substituting the resolved binary path. +//! 2. Its activation step then drives Claude Code's native `claude plugin` +//! commands against that source, inside the same rollback boundary. use std::path::{Path, PathBuf}; @@ -21,9 +21,8 @@ use tracedecay_domain::errors::{Result, TraceDecayError}; pub(super) use super::plugin_bundle::TRACEDECAY_BIN_PLACEHOLDER; use super::{ - AgentIntegration, DeferredUserAction, DoctorCounters, HealthcheckContext, InstallContext, - JsonConfigDialect, JsonConfigMutation, NonInteractiveInstallOutcome, UpdatePluginOutcome, - expected_tool_perms, load_json_file, safe_write_text_file, update_json_config_transactionally, + AgentIntegration, DoctorCounters, HealthcheckContext, InstallContext, JsonConfigDialect, + JsonConfigMutation, load_json_file, update_json_config_transactionally, }; pub struct ClaudeIntegration; @@ -41,21 +40,6 @@ impl AgentIntegration for ClaudeIntegration { true } - fn preflight_non_interactive_install( - &self, - ctx: &InstallContext, - ) -> Result { - claude_non_interactive_install_state(&ctx.home, &ctx.tracedecay_bin, Vec::new()) - } - - fn prepare_non_interactive_install( - &self, - ctx: &InstallContext, - ) -> Result { - let deploy_dir = deploy_plugin_bundle(&ctx.home, &ctx.tracedecay_bin)?; - claude_non_interactive_install_state(&ctx.home, &ctx.tracedecay_bin, vec![deploy_dir]) - } - // Claude Code exposes a first-party plugin lifecycle CLI, so TraceDecay // drives that CLI rather than deferring to the operator. Reporting // interactive activation/removal guidance here would re-enter the @@ -92,12 +76,11 @@ impl AgentIntegration for ClaudeIntegration { fn deactivate_project_host_component_registration( &self, _components: &[super::host_bundle::HostComponentV1], - ctx: &InstallContext, + _ctx: &InstallContext, project_path: &Path, ) -> Result<()> { let claude_md_path = project_path.join(".claude/CLAUDE.md"); super::remove_managed_skill_prompt_index( - &ctx.home, &claude_md_path, tracedecay_automation_runtime::automation::skill_targets::SkillInstallTarget::Claude, )?; @@ -120,26 +103,6 @@ impl AgentIntegration for ClaudeIntegration { claude_plugin_deactivate_with(&claude, &ctx.home) } - fn update_plugin(&self, ctx: &InstallContext) -> Result { - if !plugin_marketplace_manifest_path(&ctx.home).exists() { - return Ok(UpdatePluginOutcome::NotInstalled); - } - - // The marketplace source is TraceDecay-owned, but Claude Code activates - // a versioned cache through its own CLI. Refreshing only this source - // cannot honestly report an activated plugin, so stage it and defer - // the host-native cache update to the operator. - let deploy_dir = deploy_plugin_bundle(&ctx.home, &ctx.tracedecay_bin)?; - Ok(UpdatePluginOutcome::DeferredUserAction( - super::DeferredUserAction { - remediation: format!( - "Claude Code plugin source is staged. Run `claude plugin update {PLUGIN_IDENTIFIER}`, then restart Claude Code." - ), - staged_paths: vec![deploy_dir], - }, - )) - } - fn healthcheck(&self, dc: &mut DoctorCounters, ctx: &HealthcheckContext) { eprintln!("\n\x1b[1mClaude Code integration\x1b[0m"); doctor_check_plugin(dc, &ctx.home); @@ -287,29 +250,6 @@ impl AgentIntegration for ClaudeIntegration { } } -fn claude_non_interactive_install_state( - home: &Path, - tracedecay_bin: &str, - staged_paths: Vec, -) -> Result { - if claude_plugin_is_natively_active(home, Some(tracedecay_bin))? { - Ok(NonInteractiveInstallOutcome::Ready) - } else if claude_plugin_registration_is_active(home)? { - Ok(NonInteractiveInstallOutcome::DeferredUserAction( - DeferredUserAction { - remediation: format!( - "Claude Code's loaded TraceDecay cache is stale. Run `claude plugin update {PLUGIN_IDENTIFIER}`, restart Claude Code, then retry the TraceDecay lifecycle." - ), - staged_paths, - }, - )) - } else { - Ok(NonInteractiveInstallOutcome::DeferredUserAction( - claude_native_install_action(staged_paths.first().map(PathBuf::as_path)), - )) - } -} - fn claude_plugin_is_natively_active(home: &Path, tracedecay_bin: Option<&str>) -> Result { let active = claude_plugin_registration_is_active(home)?; let cache_current = claude_loaded_cache_matches_rendered_bundle(home, tracedecay_bin)?; @@ -403,19 +343,6 @@ fn claude_loaded_cache_matches_rendered_bundle( ) } -fn claude_native_install_action(staged_dir: Option<&Path>) -> DeferredUserAction { - let register = staged_dir.map_or_else( - || "Claude Code's native marketplace command".to_string(), - |path| format!("`claude plugin marketplace add {}`", path.display()), - ); - DeferredUserAction { - remediation: format!( - "Claude Code owns marketplace registration, cache, and enabled state. Run {register}, then `claude plugin install {PLUGIN_IDENTIFIER}` and re-run TraceDecay to record the staged source." - ), - staged_paths: staged_dir.into_iter().map(Path::to_path_buf).collect(), - } -} - /// Name of Claude Code's lifecycle binary. const CLAUDE_CLI: &str = "claude"; @@ -528,37 +455,9 @@ fn known_marketplaces_path(home: &Path) -> PathBuf { home.join(".claude/plugins/known_marketplaces.json") } -/// Deploy every embedded bundle file into the stable marketplace dir, -/// stamping the plugin version and substituting the binary path. -#[hotpath::measure(label = "hosts.agent.claude.plugin_deploy")] -fn deploy_plugin_bundle(home: &Path, tracedecay_bin: &str) -> Result { - if std::fs::symlink_metadata(home.join(".claude")) - .is_ok_and(|metadata| metadata.file_type().is_symlink()) - { - return Err(TraceDecayError::Config { - message: super::host_bundle::HostBundleError::UnsafeClaudeHomeSymlink.to_string(), - }); - } - let deploy_dir = plugin_deploy_dir(home); - write_rendered_plugin_bundle(&deploy_dir, tracedecay_bin)?; - eprintln!( - "\x1b[32m✔\x1b[0m Deployed tracedecay plugin bundle to {}", - deploy_dir.display() - ); - Ok(deploy_dir) -} - -fn write_rendered_plugin_bundle(deploy_dir: &Path, tracedecay_bin: &str) -> Result<()> { - clean_replace_owned_deploy_dir(deploy_dir)?; - for (relative, rendered) in rendered_plugin_files(tracedecay_bin)? { - safe_write_text_file(&deploy_dir.join(relative), &rendered, None)?; - } - Ok(()) -} - -/// Canonical rendered Claude plugin inventory shared by native-activation -/// staging and the receipt-backed first-party catalog. One renderer keeps the -/// staged source byte-identical to the later component transaction. +/// Canonical rendered Claude plugin inventory: the receipt-backed first-party +/// catalog deploys exactly these bytes as the marketplace source, and the +/// native-activation probe compares Claude's loaded cache against them. pub(crate) fn rendered_plugin_files(tracedecay_bin: &str) -> Result> { claude_embedded_plugin_files() .into_iter() @@ -569,40 +468,6 @@ pub(crate) fn rendered_plugin_files(tracedecay_bin: &str) -> Result bool { - let names_tracedecay = |manifest: &Path| { - load_json_file(manifest) - .get("name") - .and_then(|v| v.as_str()) - == Some("tracedecay") - }; - names_tracedecay(&deploy_dir.join(".claude-plugin/plugin.json")) - || names_tracedecay(&deploy_dir.join(".claude-plugin/marketplace.json")) -} - -/// Remove the tracedecay-owned deploy dir so the next write is a clean replace. -/// No-op when the dir is missing. Refuses (errors) when the dir exists but is -/// not tracedecay-owned, so an unrelated directory is never deleted. -fn clean_replace_owned_deploy_dir(deploy_dir: &Path) -> Result<()> { - if !deploy_dir.exists() { - return Ok(()); - } - if !deploy_dir_is_tracedecay(deploy_dir) { - return Err(TraceDecayError::Config { - message: format!( - "refusing to replace non-tracedecay plugin directory {}", - deploy_dir.display() - ), - }); - } - std::fs::remove_dir_all(deploy_dir).map_err(|e| TraceDecayError::Config { - message: format!("failed to remove {}: {e}", deploy_dir.display()), - }) -} - /// Apply per-file deploy-time substitutions: /// - `plugin.json`: stamp `version` from the crate version. /// - `.lsp.json`: set the configured-language bridge command. @@ -688,13 +553,9 @@ fn ensure_claude_dir(claude_dir: &Path) -> Result<()> { }) } -/// Permission-allowlist prefixes, shared with usage classification so the -/// installer and the analytics reader agree on which namespaces are ours. The -/// legacy and prior-plugin prefixes are read only to detect and mirror existing -/// entries onto the current plugin namespace; they are never removed. -use crate::tool_name::{ - LEGACY_TOOL_PREFIX as LEGACY_TOOL_PERM_PREFIX, PLUGIN_TOOL_PREFIX as PLUGIN_TOOL_PERM_PREFIX, -}; +/// Permission-allowlist prefix, shared with usage classification so the +/// installer and the analytics reader agree on which namespace is ours. +use crate::tool_name::PLUGIN_TOOL_PREFIX as PLUGIN_TOOL_PERM_PREFIX; /// Every managed tracedecay tool's plugin-namespace permission entry. fn plugin_tool_perms() -> tracedecay_domain::errors::Result> { @@ -715,8 +576,7 @@ fn plugin_wildcard_perm() -> String { /// Add the one documented plugin-namespace allow rule without replacing any /// other Claude setting. The receipt-backed lifecycle snapshots settings.json -/// before this registration effect, while the config transaction also leaves -/// the normal recoverable `.bak` used by every shared-config edit. +/// before this registration effect. fn ensure_claude_plugin_permission(home: &Path) -> Result<()> { let settings_path = home.join(".claude/settings.json"); ensure_claude_dir( @@ -806,23 +666,6 @@ const CLAUDE_MD_SENTINELS: super::prompt_rules::OwnedBlockSentinels = start: "", end: "", }; -/// Heading markers shipped releases (through v0.1.0-beta.37) used as the -/// block's identity: the steady heading, its display-case product-name -/// variant, and the Codegraph-era fragment (matched as a substring because -/// historical heading prefixes varied). Update and uninstall must recognize -/// them so an existing install converges instead of stranding a stale block. -const CLAUDE_MD_HISTORICAL_MARKERS: [&str; 3] = [ - "## MANDATORY: No Explore Agents When Tracedecay Is Available", - "## MANDATORY: No Explore Agents When TraceDecay Is Available", - "No Explore Agents When Codegraph Is Available", -]; -/// The one `## ` sub-heading historical blocks owned. A historical block range -/// extends across exactly this heading, never any arbitrary line containing -/// "tracedecay", which would wrongly absorb a user's own `## …tracedecay…` -/// heading on uninstall. -const CLAUDE_MD_HISTORICAL_OWNED_SUBHEADING: &str = - "## When you spawn an Explore agent in a tracedecay-enabled project"; - /// True when a `CLAUDE.md` is a tracedecay-managed Claude config (references /// tracedecay), so a lifecycle skill export may refresh it. An unrelated /// project `CLAUDE.md` must not become an export destination. @@ -830,55 +673,11 @@ fn claude_md_references_tracedecay(claude_md_path: &Path) -> bool { std::fs::read_to_string(claude_md_path).is_ok_and(|contents| contents.contains("tracedecay")) } -/// Every tracedecay-owned CLAUDE.md range in document order: current -/// sentinel-delimited blocks plus historical heading-marked ones. +/// Every tracedecay-owned CLAUDE.md block in document order. fn owned_claude_md_ranges(contents: &str) -> Vec> { - super::prompt_rules::owned_block_ranges(contents, first_owned_claude_md_range) -} - -/// Earliest owned block at or after `from`. -fn first_owned_claude_md_range(contents: &str, from: usize) -> Option> { - let current = CLAUDE_MD_SENTINELS.block_range(contents, from); - let historical = historical_claude_md_range(contents, from); - match (current, historical) { - (Some(current), Some(historical)) if historical.start < current.start => Some(historical), - (Some(current), _) => Some(current), - (None, historical) => historical, - } -} - -/// Byte range of the earliest historical heading-marked block at or after -/// `from`: from the start of the marker's line across its owned sub-heading to -/// the next foreign `## ` heading, the managed skill index, a current start -/// sentinel, or EOF. -fn historical_claude_md_range(contents: &str, from: usize) -> Option> { - let (start, mut search_from) = CLAUDE_MD_HISTORICAL_MARKERS - .iter() - .filter_map(|marker| { - contents[from..].find(marker).map(|at| { - let pos = from + at; - let line_start = contents[..pos].rfind('\n').map_or(0, |nl| nl + 1); - (line_start, pos + marker.len()) - }) - }) - .min_by_key(|(start, _)| *start)?; - loop { - let boundary = super::prompt_rules::historical_heading_block_end( - contents, - search_from, - CLAUDE_MD_SENTINELS, - ); - // Only extend across the block's own known sub-heading; any other - // boundary closes the block. - let heading_line = contents[boundary..] - .strip_prefix('\n') - .and_then(|rest| rest.lines().next()); - if heading_line.map(str::trim_end) == Some(CLAUDE_MD_HISTORICAL_OWNED_SUBHEADING) { - search_from = boundary + 1 + CLAUDE_MD_HISTORICAL_OWNED_SUBHEADING.len(); - continue; - } - return Some(start..boundary); - } + super::prompt_rules::owned_block_ranges(contents, |contents, from| { + CLAUDE_MD_SENTINELS.block_range(contents, from) + }) } /// The full tracedecay-managed CLAUDE.md block. @@ -919,9 +718,9 @@ fn claude_md_guidance_text() -> String { ) } -/// Install or refresh the CLAUDE.md block: every owned range, current or -/// historical, converges onto exactly one copy of the current block in place -/// while operator text around it is preserved. +/// Install or refresh the CLAUDE.md block: every owned range converges onto +/// exactly one copy of the current block in place while operator text around +/// it is preserved. fn install_claude_md_rules(claude_md_path: &Path) -> Result<()> { let block = claude_md_rules_text(); super::prompt_rules::reconcile_prompt_rules_with(claude_md_path, |existing| { @@ -932,7 +731,7 @@ fn install_claude_md_rules(claude_md_path: &Path) -> Result<()> { }) } -/// Remove every tracedecay-owned CLAUDE.md block, current or historical. +/// Remove every tracedecay-owned CLAUDE.md block. fn uninstall_claude_md_rules(claude_md_path: &Path) -> Result<()> { super::prompt_rules::remove_prompt_rules_with(claude_md_path, |contents| { let ranges = owned_claude_md_ranges(contents); @@ -1112,43 +911,6 @@ fn doctor_check_permissions_json(dc: &mut DoctorCounters, home: &Path) { settings_path.display() )); } - - let expected = match expected_tool_perms() { - Ok(expected) => expected, - Err(error) => { - dc.fail(&format!( - "Could not read the advertised tool catalog: {error}" - )); - return; - } - }; - let missing: Vec<&String> = expected - .iter() - .filter(|p| !installed.contains(&p.as_str())) - .collect(); - - if missing.is_empty() { - dc.pass(&format!( - "All {} legacy tool permissions granted", - expected.len() - )); - } else { - dc.info(&format!( - "{} legacy tool permission(s) not present (harmless, plugin namespace is authoritative)", - missing.len() - )); - } - - let stale: Vec<&&str> = installed - .iter() - .filter(|p| p.starts_with(LEGACY_TOOL_PERM_PREFIX) && !expected.contains(&p.to_string())) - .collect(); - if !stale.is_empty() { - dc.warn(&format!( - "{} stale permission(s) from older version (harmless)", - stale.len() - )); - } } /// Report local project config without rewriting host-owned files. @@ -1200,10 +962,9 @@ fn warn_missing_permissions(settings: &serde_json::Value) { .unwrap_or_default(); // Check the plugin namespace, the entries the plugin MCP server matches. - // A machine mid-upgrade may carry legacy `mcp__tracedecay__*` entries but - // lack coverage of the `mcp__plugin_tracedecay_graph__*` namespace, which - // is exactly what causes per-call prompts, so that is the gap worth - // warning about, with the one-rule remedy, not a tool census. + // Missing coverage of `mcp__plugin_tracedecay_graph__*` is exactly what + // causes per-call prompts, so that is the gap worth warning about, with + // the one-rule remedy. match plugin_perms_satisfied(&installed) { Ok(true) => {} Ok(false) => eprintln!( diff --git a/crates/tracedecay-agent-hosts/src/agents/claude/tests.rs b/crates/tracedecay-agent-hosts/src/agents/claude/tests.rs index bb4382ab1e..aed65e2484 100644 --- a/crates/tracedecay-agent-hosts/src/agents/claude/tests.rs +++ b/crates/tracedecay-agent-hosts/src/agents/claude/tests.rs @@ -1,7 +1,17 @@ -use super::super::{load_json_file_strict, safe_write_json_file}; +use super::super::{load_json_file_strict, safe_write_json_file, safe_write_text_file}; use super::*; use serde_json::json; +/// Writes the rendered marketplace source exactly where the component +/// catalog deploys it. +fn deploy_rendered_bundle(home: &Path, tracedecay_bin: &str) -> PathBuf { + let deploy_dir = plugin_deploy_dir(home); + for (relative, rendered) in rendered_plugin_files(tracedecay_bin).unwrap() { + safe_write_text_file(&deploy_dir.join(relative), &rendered).unwrap(); + } + deploy_dir +} + fn copy_rendered_bundle_to_native_cache(home: &Path, tracedecay_bin: &str) { let source = plugin_deploy_dir(home); let cache = claude_current_cached_plugin_root(home); @@ -18,7 +28,6 @@ fn write_native_activation(home: &Path, tracedecay_bin: &str) { safe_write_json_file( &settings, &json!({"enabledPlugins": {"tracedecay@tracedecay": true}}), - None, ) .unwrap(); safe_write_json_file( @@ -32,7 +41,6 @@ fn write_native_activation(home: &Path, tracedecay_bin: &str) { "installLocation": plugin_deploy_dir(home), } }), - None, ) .unwrap(); copy_rendered_bundle_to_native_cache(home, tracedecay_bin); @@ -41,7 +49,7 @@ fn write_native_activation(home: &Path, tracedecay_bin: &str) { #[test] fn native_activation_requires_exact_catalog_mount_and_versioned_cache() { let home = tempfile::tempdir().unwrap(); - deploy_plugin_bundle(home.path(), "/bin/tracedecay").unwrap(); + deploy_rendered_bundle(home.path(), "/bin/tracedecay"); write_native_activation(home.path(), "/bin/tracedecay"); assert!(claude_plugin_is_natively_active(home.path(), Some("/bin/tracedecay")).unwrap()); @@ -49,14 +57,14 @@ fn native_activation_requires_exact_catalog_mount_and_versioned_cache() { let mut state: serde_json::Value = serde_json::from_slice(&std::fs::read(&marketplace).unwrap()).unwrap(); state["tracedecay"]["installLocation"] = json!("/different/marketplace"); - safe_write_json_file(&marketplace, &state, None).unwrap(); + safe_write_json_file(&marketplace, &state).unwrap(); assert!(!claude_plugin_is_natively_active(home.path(), Some("/bin/tracedecay")).unwrap()); } #[test] fn native_activation_rejects_current_version_manifest_in_unbound_cache_directory() { let home = tempfile::tempdir().unwrap(); - deploy_plugin_bundle(home.path(), "/bin/tracedecay").unwrap(); + deploy_rendered_bundle(home.path(), "/bin/tracedecay"); write_native_activation(home.path(), "/bin/tracedecay"); let exact = claude_current_cached_plugin_manifest_path(home.path()); let unbound = exact @@ -76,77 +84,31 @@ fn native_cache_content_drift_and_binary_relocation_require_refresh() { let home = tempfile::tempdir().unwrap(); let old_bin = "/old/bin/tracedecay"; let new_bin = "/relocated/bin/tracedecay"; - deploy_plugin_bundle(home.path(), old_bin).unwrap(); + deploy_rendered_bundle(home.path(), old_bin); write_native_activation(home.path(), old_bin); - let old_ctx = InstallContext { - home: home.path().to_path_buf(), - tracedecay_bin: old_bin.to_string(), - tool_permissions: Vec::new(), - project_root: None, - dashboard: true, - }; - assert!(matches!( - ClaudeIntegration - .preflight_non_interactive_install(&old_ctx) - .unwrap(), - NonInteractiveInstallOutcome::Ready - )); + assert!(claude_plugin_is_natively_active(home.path(), Some(old_bin)).unwrap()); let retired_command = claude_current_cached_plugin_root(home.path()).join("commands/retired.md"); std::fs::create_dir_all(retired_command.parent().unwrap()).unwrap(); std::fs::write(&retired_command, "# stale auto-discovered command\n").unwrap(); - assert!(matches!( - ClaudeIntegration - .preflight_non_interactive_install(&old_ctx) - .unwrap(), - NonInteractiveInstallOutcome::DeferredUserAction(_) - )); + assert!(!claude_plugin_is_natively_active(home.path(), Some(old_bin)).unwrap()); std::fs::remove_file(retired_command).unwrap(); - assert!(matches!( - ClaudeIntegration - .preflight_non_interactive_install(&old_ctx) - .unwrap(), - NonInteractiveInstallOutcome::Ready - )); + assert!(claude_plugin_is_natively_active(home.path(), Some(old_bin)).unwrap()); std::fs::write( claude_current_cached_plugin_root(home.path()).join(".mcp.json"), "{}\n", ) .unwrap(); - assert!(matches!( - ClaudeIntegration - .preflight_non_interactive_install(&old_ctx) - .unwrap(), - NonInteractiveInstallOutcome::DeferredUserAction(_) - )); + assert!(!claude_plugin_is_natively_active(home.path(), Some(old_bin)).unwrap()); copy_rendered_bundle_to_native_cache(home.path(), old_bin); - assert!(matches!( - ClaudeIntegration - .preflight_non_interactive_install(&old_ctx) - .unwrap(), - NonInteractiveInstallOutcome::Ready - )); - - deploy_plugin_bundle(home.path(), new_bin).unwrap(); - let relocated_ctx = InstallContext { - tracedecay_bin: new_bin.to_string(), - ..old_ctx - }; - assert!(matches!( - ClaudeIntegration - .preflight_non_interactive_install(&relocated_ctx) - .unwrap(), - NonInteractiveInstallOutcome::DeferredUserAction(_) - )); + assert!(claude_plugin_is_natively_active(home.path(), Some(old_bin)).unwrap()); + + deploy_rendered_bundle(home.path(), new_bin); + assert!(!claude_plugin_is_natively_active(home.path(), Some(new_bin)).unwrap()); copy_rendered_bundle_to_native_cache(home.path(), new_bin); - assert!(matches!( - ClaudeIntegration - .preflight_non_interactive_install(&relocated_ctx) - .unwrap(), - NonInteractiveInstallOutcome::Ready - )); + assert!(claude_plugin_is_natively_active(home.path(), Some(new_bin)).unwrap()); } #[test] @@ -165,7 +127,6 @@ fn missing_manifest_with_stale_registration_is_repairable() { "source": { "source": "directory", "path": "/stale" } } }), - None, ) .unwrap(); let state = ClaudeIntegration.host_component_registration( @@ -188,7 +149,6 @@ fn project_only_legacy_residue_does_not_claim_plugin_registration() { safe_write_json_file( &project.path().join(".mcp.json"), &json!({ "mcpServers": { "tracedecay": { "command": "old" } } }), - None, ) .unwrap(); let state = ClaudeIntegration.host_component_registration( @@ -206,7 +166,7 @@ fn project_only_legacy_residue_does_not_claim_plugin_registration() { #[test] fn deploy_stamps_version_and_binary_path() { let home = tempfile::tempdir().unwrap(); - let deploy_dir = deploy_plugin_bundle(home.path(), "/abs/bin/tracedecay").unwrap(); + let deploy_dir = deploy_rendered_bundle(home.path(), "/abs/bin/tracedecay"); let plugin: serde_json::Value = serde_json::from_str( &std::fs::read_to_string(deploy_dir.join(".claude-plugin/plugin.json")).unwrap(), @@ -237,7 +197,7 @@ fn deploy_stamps_version_and_binary_path() { fn deploy_escapes_special_chars_in_binary_path() { let home = tempfile::tempdir().unwrap(); let weird_bin = "/opt/td \"quote\"/tracedecay"; - let deploy_dir = deploy_plugin_bundle(home.path(), weird_bin).unwrap(); + let deploy_dir = deploy_rendered_bundle(home.path(), weird_bin); let hooks_raw = std::fs::read_to_string(deploy_dir.join("hooks/hooks.json")).unwrap(); // Must parse, a raw replace would have produced invalid JSON here. @@ -253,58 +213,6 @@ fn deploy_escapes_special_chars_in_binary_path() { assert_eq!(command, weird_bin, "command must be the exact binary path"); } -/// Redeploy must be a CLEAN REPLACE of the owned marketplace dir: a stale -/// file the current bundle no longer ships (e.g. a retired skill dir) is -/// gone after a redeploy, while the fresh bundle is present. -#[test] -fn deploy_is_a_clean_replace_dropping_stale_files() { - let home = tempfile::tempdir().unwrap(); - let deploy_dir = deploy_plugin_bundle(home.path(), "/bin/tracedecay").unwrap(); - // A stale skill dir the current bundle does not ship. - let stale = deploy_dir.join("skills/totally-retired-skill"); - std::fs::create_dir_all(&stale).unwrap(); - std::fs::write(stale.join("SKILL.md"), "stale skill").unwrap(); - - // Redeploy (the install/update path). - deploy_plugin_bundle(home.path(), "/bin/tracedecay").unwrap(); - - assert!( - !stale.exists(), - "a stale skill dir must be gone after a clean-replace redeploy" - ); - assert!( - deploy_dir.join(".claude-plugin/plugin.json").exists(), - "the fresh bundle must be present after redeploy" - ); -} - -/// The clean replace must refuse to delete a marketplace dir tracedecay -/// does not own (no tracedecay plugin/marketplace manifest), so an -/// unrelated dir squatting on the path is never nuked. -#[test] -fn deploy_refuses_to_replace_non_tracedecay_dir() { - let home = tempfile::tempdir().unwrap(); - let deploy_dir = plugin_deploy_dir(home.path()); - std::fs::create_dir_all(deploy_dir.join(".claude-plugin")).unwrap(); - std::fs::write( - deploy_dir.join(".claude-plugin/plugin.json"), - r#"{"name":"someone-elses-plugin"}"#, - ) - .unwrap(); - std::fs::write(deploy_dir.join("user-file.txt"), "keep me").unwrap(); - - let err = deploy_plugin_bundle(home.path(), "/bin/tracedecay") - .expect_err("must refuse a non-tracedecay dir"); - assert!( - err.to_string().contains("non-tracedecay"), - "unexpected error: {err}" - ); - assert!( - deploy_dir.join("user-file.txt").exists(), - "an unowned dir must be left untouched" - ); -} - /// The managed-block range must extend across only its own owned /// sub-heading, not a user's own `## …tracedecay…` heading placed after /// the block, otherwise uninstall would swallow the user's section. @@ -337,126 +245,6 @@ fn uninstall_preserves_user_tracedecay_heading_after_block() { ); } -/// The heading shipped releases through v0.1.0-beta.37 wrote as the block's -/// identity, with the sub-heading those blocks owned. -const SHIPPED_HEADING: &str = "## MANDATORY: No Explore Agents When Tracedecay Is Available"; -const SHIPPED_DISPLAY_HEADING: &str = - "## MANDATORY: No Explore Agents When TraceDecay Is Available"; -const SHIPPED_SUBHEADING: &str = - "## When you spawn an Explore agent in a tracedecay-enabled project"; - -fn shipped_block(heading: &str) -> String { - format!( - "{heading}\n\n**NEVER use Agent(subagent_type=Explore).** No exceptions. No rationalizing.\n\n\ - {SHIPPED_SUBHEADING}\n\nUse `tracedecay_context` as your ONLY exploration tool." - ) -} - -#[test] -fn every_historical_claude_md_shape_converges_on_update_and_preserves_peers() { - let block = claude_md_rules_text(); - let historical_shapes = [ - ("shipped heading", shipped_block(SHIPPED_HEADING)), - ( - "display-case heading", - shipped_block(SHIPPED_DISPLAY_HEADING), - ), - ( - "codegraph-era heading", - "## IMPORTANT: No Explore Agents When Codegraph Is Available\n\nNever explore." - .to_string(), - ), - ]; - for (shape, stale) in historical_shapes { - let root = tempfile::tempdir().unwrap(); - let claude_md = root.path().join("CLAUDE.md"); - let original = - format!("# Project\n\nkeep me\n\n{stale}\n\n## Using tracedecay in CI\n\nand me\n"); - std::fs::write(&claude_md, &original).unwrap(); - - install_claude_md_rules(&claude_md).unwrap(); - - let updated = std::fs::read_to_string(&claude_md).unwrap(); - assert_eq!( - updated, - format!("# Project\n\nkeep me\n\n{block}\n\n## Using tracedecay in CI\n\nand me\n"), - "{shape}: update must replace the whole owned block (including its owned \ - sub-heading) in place and keep both peers" - ); - assert!( - !updated.contains("NEVER") && !updated.contains("rationaliz"), - "{shape}: no historical forcing may survive the migration" - ); - - install_claude_md_rules(&claude_md).unwrap(); - assert_eq!( - std::fs::read_to_string(&claude_md).unwrap(), - updated, - "{shape}: a current reinstall is idempotent" - ); - } -} - -#[test] -fn every_historical_claude_md_shape_is_removed_on_uninstall() { - for stale in [ - shipped_block(SHIPPED_HEADING), - shipped_block(SHIPPED_DISPLAY_HEADING), - "## IMPORTANT: No Explore Agents When Codegraph Is Available\n\nNever explore.".to_string(), - claude_md_rules_text(), - ] { - let root = tempfile::tempdir().unwrap(); - let claude_md = root.path().join("CLAUDE.md"); - std::fs::write( - &claude_md, - format!("keep me\n\n{stale}\n\n## Using tracedecay in CI\n\nand me\n"), - ) - .unwrap(); - - uninstall_claude_md_rules(&claude_md).unwrap(); - - assert_eq!( - std::fs::read_to_string(&claude_md).unwrap(), - "keep me\n\n## Using tracedecay in CI\n\nand me\n", - "uninstall must remove the owned block and only that block" - ); - } -} - -#[test] -fn duplicate_and_mixed_claude_md_blocks_converge_deterministically() { - let block = claude_md_rules_text(); - // The display-case block directly precedes the current one: a heading-marked - // historical block must stop at the current start sentinel rather than - // swallow it. - let mixed = format!( - "keep me\n\n{}\n\n## Operator section\n\nand me\n\n{}\n\n{block}\n\ntail peer\n", - shipped_block(SHIPPED_HEADING), - shipped_block(SHIPPED_DISPLAY_HEADING), - ); - let root = tempfile::tempdir().unwrap(); - let claude_md = root.path().join("CLAUDE.md"); - std::fs::write(&claude_md, &mixed).unwrap(); - - install_claude_md_rules(&claude_md).unwrap(); - - let converged = std::fs::read_to_string(&claude_md).unwrap(); - assert_eq!( - converged, - format!("keep me\n\n{block}\n\n## Operator section\n\nand me\n\ntail peer\n"), - "mixed markers must collapse onto one current block at the first owned position" - ); - assert_eq!(owned_claude_md_ranges(&converged).len(), 1); - - std::fs::write(&claude_md, &mixed).unwrap(); - uninstall_claude_md_rules(&claude_md).unwrap(); - assert_eq!( - std::fs::read_to_string(&claude_md).unwrap(), - "keep me\n\n## Operator section\n\nand me\n\ntail peer\n", - "uninstall must remove every owned block, historical and current" - ); -} - #[test] fn claude_md_ownership_is_the_sentinel_not_prose() { let prose = "Use tracedecay MCP tools and never spawn Explore agents.\n"; @@ -640,7 +428,7 @@ fn activation_drives_the_hosts_own_marketplace_and_install_commands() { let bin_dir = tempfile::tempdir().unwrap(); let log = bin_dir.path().join("invocations.log"); let claude = bin_dir.path().join("claude"); - deploy_plugin_bundle(home.path(), "/bin/tracedecay").unwrap(); + deploy_rendered_bundle(home.path(), "/bin/tracedecay"); fake_claude_cli(&claude, &log, "exit 0"); claude_plugin_activate_with(&claude, home.path()) @@ -717,7 +505,7 @@ fn plugin_permission_coverage_accepts_wildcard_or_full_per_tool_grants() { fn activation_adds_wildcard_permission_without_replacing_user_settings() { let home = tempfile::tempdir().unwrap(); let tracedecay_bin = "/bin/tracedecay"; - deploy_plugin_bundle(home.path(), tracedecay_bin).unwrap(); + deploy_rendered_bundle(home.path(), tracedecay_bin); write_native_activation(home.path(), tracedecay_bin); let settings_path = home.path().join(".claude/settings.json"); @@ -732,11 +520,10 @@ fn activation_adds_wildcard_permission_without_replacing_user_settings() { "deny": ["Bash(rm:*)"] } }); - safe_write_json_file(&settings_path, &existing, None).unwrap(); + safe_write_json_file(&settings_path, &existing).unwrap(); let ctx = InstallContext { home: home.path().to_path_buf(), tracedecay_bin: tracedecay_bin.to_string(), - tool_permissions: Vec::new(), project_root: None, dashboard: true, }; diff --git a/crates/tracedecay-agent-hosts/src/agents/cline.rs b/crates/tracedecay-agent-hosts/src/agents/cline.rs index d7f0e9059b..02bbdc9421 100644 --- a/crates/tracedecay-agent-hosts/src/agents/cline.rs +++ b/crates/tracedecay-agent-hosts/src/agents/cline.rs @@ -13,9 +13,8 @@ use tracedecay_domain::errors::Result; use super::{ AgentIntegration, DoctorCounters, HealthcheckContext, InstallContext, JsonConfigDialect, - McpDoctorLabels, McpUninstallPolicy, config_backup_path, install_mcp_server_entry, - load_json_file, mcp_servers_registration_state, report_mcp_registration, - uninstall_mcp_server_entry, + McpDoctorLabels, McpUninstallPolicy, install_mcp_server_entry, load_json_file, + mcp_servers_registration_state, report_mcp_registration, uninstall_mcp_server_entry, }; pub struct ClineIntegration; @@ -25,20 +24,6 @@ fn cline_mcp_settings_path(home: &Path) -> PathBuf { home.join(".cline/mcp.json") } -/// Legacy VS Code extension storage path retained only for migration diagnosis. -fn legacy_cline_mcp_settings_path(home: &Path) -> PathBuf { - super::vscode_data_dir(home) - .join("User/globalStorage/saoudrizwan.claude-dev") - .join("settings/cline_mcp_settings.json") -} - -fn cline_settings_paths(home: &Path) -> [PathBuf; 2] { - [ - cline_mcp_settings_path(home), - legacy_cline_mcp_settings_path(home), - ] -} - /// Cline accepts any `mcpServers.tracedecay` entry, so this deliberately skips /// the object-shape filter [`super::doctor_check_mcp_registration`] applies. fn settings_have_tracedecay(path: &Path) -> bool { @@ -72,9 +57,6 @@ impl AgentIntegration for ClineIntegration { fn is_detected(&self, home: &Path) -> bool { home.join(".cline").is_dir() - || legacy_cline_mcp_settings_path(home) - .parent() - .is_some_and(Path::is_dir) } fn primary_config_path(&self, home: &Path) -> Option { @@ -87,8 +69,7 @@ impl AgentIntegration for ClineIntegration { home: &Path, ) -> Vec { if components == [super::host_bundle::HostComponentV1::ContextMcp] { - let path = cline_mcp_settings_path(home); - vec![path.clone(), config_backup_path(&path)] + vec![cline_mcp_settings_path(home)] } else { Vec::new() } @@ -135,9 +116,7 @@ impl AgentIntegration for ClineIntegration { } fn has_tracedecay(&self, home: &Path) -> bool { - cline_settings_paths(home) - .iter() - .any(|path| settings_have_tracedecay(path)) + settings_have_tracedecay(&cline_mcp_settings_path(home)) } } @@ -145,24 +124,9 @@ impl AgentIntegration for ClineIntegration { // Healthcheck helpers // --------------------------------------------------------------------------- -/// Unlike the plain [`super::doctor_check_mcp_registration`] flow, an absent -/// primary settings file is not a warning on its own: Cline falls through to -/// the legacy VS Code extension path first and only then reports a failure. fn doctor_check_settings(dc: &mut DoctorCounters, home: &Path) { let settings_path = cline_mcp_settings_path(home); let registered = settings_have_tracedecay(&settings_path); - - if !registered { - let legacy_path = legacy_cline_mcp_settings_path(home); - if settings_have_tracedecay(&legacy_path) { - dc.warn(&format!( - "legacy Cline MCP registration found in {}, configure or remove it through Cline's supported flow", - legacy_path.display() - )); - return; - } - } - report_mcp_registration( dc, &settings_path, diff --git a/crates/tracedecay-agent-hosts/src/agents/codex.rs b/crates/tracedecay-agent-hosts/src/agents/codex.rs index f8e65b5fcd..9706f05ac8 100644 --- a/crates/tracedecay-agent-hosts/src/agents/codex.rs +++ b/crates/tracedecay-agent-hosts/src/agents/codex.rs @@ -25,9 +25,9 @@ //! standalone server: the plugin bundle already carries `.mcp.json`. //! //! Note on rollback ownership: `CodexIntegration::host_registration_paths` -//! already lists `~/.codex/config.toml` and its backup, so the component-set -//! transaction stages that file before the registry command runs and can -//! restore the pre-command document if the effect is rejected. +//! already lists `~/.codex/config.toml`, so the component-set transaction +//! observes that file before the registry command runs and can restore the +//! pre-command document if the effect is rejected. use std::collections::HashSet; use std::path::{Path, PathBuf}; @@ -38,10 +38,9 @@ use tracedecay_domain::canonical_sha256; use tracedecay_domain::errors::{Result, TraceDecayError}; use super::{ - AgentIntegration, DeferredUserAction, DoctorCounters, HealthcheckContext, InstallContext, - InstallScope, NonInteractiveInstallOutcome, TextFileMutation, UpdatePluginOutcome, - config_backup_path, load_json_file, load_json_file_strict, load_toml_file, - safe_write_json_file, safe_write_text_file, update_toml_config_transactionally, + AgentIntegration, DoctorCounters, HealthcheckContext, InstallContext, InstallScope, + JsonConfigDialect, JsonConfigMutation, load_json_file, load_json_file_strict, load_toml_file, + safe_write_text_file, update_json_config_transactionally, update_toml_config_transactionally, }; /// The prefix every Codex activation key for this plugin starts with. @@ -52,7 +51,6 @@ const CODEX_PLUGIN_ACTIVATION_KEY_PREFIX: &str = "tracedecay@"; mod mcp_registry; mod plugin_registry; -mod retired_entrypoints; pub struct CodexIntegration; @@ -69,46 +67,6 @@ impl AgentIntegration for CodexIntegration { true } - fn preflight_non_interactive_install( - &self, - ctx: &InstallContext, - ) -> Result { - codex_non_interactive_install_state(&ctx.home, &ctx.tracedecay_bin, Vec::new()) - } - - fn interactive_activation_guidance(&self) -> Option { - None - } - - fn interactive_removal_guidance(&self) -> Option { - None - } - - fn prepare_non_interactive_install( - &self, - ctx: &InstallContext, - ) -> Result { - install_codex_plugin(&ctx.home, &ctx.tracedecay_bin)?; - // Core apply drives `codex plugin add` when the host CLI is present. - // When it is not, stop with the same backtick remediation preflight - // uses so operators (and lifecycle tests) can activate natively. - if plugin_registry::require_codex_plugin_cli().is_err() { - let marketplace_name = codex_exact_personal_marketplace_name(&ctx.home) - .ok() - .flatten() - .unwrap_or_else(|| codex_cached_marketplace_name(&ctx.home)); - return Ok(NonInteractiveInstallOutcome::DeferredUserAction( - DeferredUserAction { - remediation: format!( - "Codex activates plugins through its native cache. Run `codex plugin add tracedecay@{marketplace_name}` after TraceDecay stages the source package." - ), - staged_paths: Vec::new(), - }, - )); - } - Ok(NonInteractiveInstallOutcome::Ready) - } - #[hotpath::measure(label = "hosts.agent.codex.project_install")] fn activate_project_host_component_registration( &self, @@ -144,51 +102,12 @@ impl AgentIntegration for CodexIntegration { let local = InstallContext { home: ctx.home.clone(), tracedecay_bin: ctx.tracedecay_bin.clone(), - tool_permissions: ctx.tool_permissions.clone(), project_root: Some(project_path.to_path_buf()), dashboard: ctx.dashboard, }; uninstall_codex_repo_plugin_if_present(&local) } - fn update_plugin(&self, ctx: &InstallContext) -> Result { - let cached_install_present = - codex_exact_cache_manifest_path(&ctx.home)?.is_some_and(|path| path.is_file()); - let source_present = codex_plugin_manifest_path(&ctx.home).exists(); - let mut staged = Vec::new(); - if cached_install_present || source_present { - // Codex owns its cache lifecycle. Refresh the marketplace source - // it will consume, but never materialise or replace a cache entry - // on the host's behalf. - staged.push(install_codex_personal_bootstrap( - &ctx.home, - &ctx.tracedecay_bin, - )?); - } - - if let Some(project_path) = codex_update_project_path(ctx) { - let repo_dir = codex_repo_plugin_install_dir(&project_path); - if repo_dir.join(".codex-plugin/plugin.json").exists() - && codex_plugin_dir_is_tracedecay(&repo_dir) - { - install_codex_plugin_bundle( - &repo_dir, - &ctx.tracedecay_bin, - InstallScope::ProjectLocal, - &ctx.home, - )?; - staged.push(repo_dir); - } - } - - if staged.is_empty() { - return Ok(UpdatePluginOutcome::NotInstalled); - } - // Activation also re-pins hook trust for the refreshed bundle. - self.activate_deployed_host_registration(ctx)?; - Ok(UpdatePluginOutcome::Refreshed(staged)) - } - fn export_managed_skills( &self, home: &Path, @@ -354,10 +273,6 @@ impl AgentIntegration for CodexIntegration { codex_config_path(home), codex_personal_marketplace_path(home), ]; - paths.extend([ - config_backup_path(&codex_config_path(home)), - config_backup_path(&codex_personal_marketplace_path(home)), - ]); let current_cache = codex_plugin_current_cached_install_dir(home); paths.extend(codex_plugin_managed_paths(¤t_cache)); paths.sort(); @@ -376,8 +291,8 @@ impl AgentIntegration for CodexIntegration { // transaction that retires stale exports can still roll them back. if components.contains(&super::host_bundle::HostComponentV1::Core) { // `agent_targets` lives in automation-runtime and reads agent - // bytes through the host I/O bundle this crate owns, so preview, - // backup, and activate all inventory the same surface. + // bytes through the host I/O bundle this crate owns, so preview + // and activate inventory the same surface. if let Ok(managed) = tracedecay_automation_runtime::automation::agent_targets::managed_agent_transaction_paths( &crate::host_io(), home, @@ -401,8 +316,16 @@ impl AgentIntegration for CodexIntegration { &ctx.home, )?; if !codex_plugin_is_natively_active(&ctx.home, Some(&ctx.tracedecay_bin))? { - let marketplace_name = codex_cached_marketplace_name(&ctx.home); let codex_cli = plugin_registry::require_codex_plugin_cli()?; + // `codex plugin add` resolves the catalog-deployed source through + // this entry; it is a registration path, so rollback restores it. + install_codex_marketplace_entry( + &codex_personal_marketplace_path(&ctx.home), + "personal", + "Personal", + CODEX_GLOBAL_PLUGIN_SOURCE_PATH, + )?; + let marketplace_name = codex_cached_marketplace_name(&ctx.home); plugin_registry::codex_plugin_add_with(&codex_cli, &ctx.home, &marketplace_name)?; } // Auto-trust the personal bundle's hooks whenever one is present: @@ -574,21 +497,6 @@ fn record_codex_cached_plugin_registration_intents(home: &Path) -> Result<()> { Ok(()) } -fn codex_exact_cache_manifest_path(home: &Path) -> Result> { - let marketplace_name = - codex_exact_personal_marketplace_name(home).map_err(|()| TraceDecayError::Config { - message: format!( - "could not read exact Codex marketplace identity at {}", - codex_personal_marketplace_path(home).display() - ), - })?; - Ok(marketplace_name.map(|marketplace_name| { - codex_plugin_cached_root(home, &marketplace_name) - .join(crate::PRODUCT_VERSION) - .join(".codex-plugin/plugin.json") - })) -} - fn codex_plugin_cached_install_dirs(home: &Path) -> Vec { let mut dirs = Vec::new(); let mut marketplace_names = vec![ @@ -641,28 +549,6 @@ fn codex_update_project_path(ctx: &InstallContext) -> Option { .or_else(|| std::env::current_dir().ok()) } -#[hotpath::measure(label = "hosts.agent.codex.plugin_install")] -fn install_codex_plugin(home: &Path, tracedecay_bin: &str) -> Result<()> { - let install_dir = install_codex_personal_bootstrap(home, tracedecay_bin)?; - eprintln!( - "\x1b[32m✔\x1b[0m Staged Codex plugin source at {}", - install_dir.display() - ); - Ok(()) -} - -fn install_codex_personal_bootstrap(home: &Path, tracedecay_bin: &str) -> Result { - let install_dir = codex_plugin_install_dir(home); - install_codex_plugin_bundle(&install_dir, tracedecay_bin, InstallScope::Global, home)?; - install_codex_marketplace_entry( - &codex_personal_marketplace_path(home), - "personal", - "Personal", - CODEX_GLOBAL_PLUGIN_SOURCE_PATH, - )?; - Ok(install_dir) -} - #[hotpath::measure(label = "hosts.agent.codex.repo_plugin_install")] fn install_codex_repo_plugin(home: &Path, project_path: &Path, tracedecay_bin: &str) -> Result<()> { let install_dir = codex_repo_plugin_install_dir(project_path); @@ -865,7 +751,6 @@ fn install_codex_managed_skill_overlay( tracedecay_automation_runtime::automation::skill_targets::profile_root_for_agent_home( profile_home, ); - super::retired_memory_digest::remove_state(&profile_root)?; tracedecay_automation_runtime::automation::skill_targets::install_managed_skills( &crate::host_io(), &profile_root, @@ -880,7 +765,7 @@ fn write_codex_plugin_files( policy: CodexBundlePolicy, ) -> Result<()> { for (relative, rendered) in rendered_plugin_files(tracedecay_bin, policy)? { - safe_write_text_file(&install_dir.join(relative), &rendered, None)?; + safe_write_text_file(&install_dir.join(relative), &rendered)?; } Ok(()) } @@ -1291,7 +1176,7 @@ struct CodexHookTrustSyncOutcome { /// `~/.codex/config.toml` so Codex runs them without a manual `/hooks` approval. /// /// Uses the marketplace identity and hook payload actually installed on disk, -/// pruning stale active/legacy-personal entries while preserving every other +/// pruning stale active-marketplace entries while preserving every other /// plugin's and the user's own config. Hooks whose command does not exactly /// match a generated `TraceDecay` command are skipped (see /// [`codex_hook_command_invokes_tracedecay`]). The rewrite runs as a config @@ -1300,105 +1185,98 @@ struct CodexHookTrustSyncOutcome { fn sync_codex_hook_trust(home: &Path, tracedecay_bin: &str) -> Result { let (marketplace_name, entries) = codex_installed_hook_trust_entries(home)?; let config_path = codex_config_path(home); - let outcome = update_toml_config_transactionally(&config_path, |mut config| { - let table = config - .as_table_mut() - .ok_or_else(|| TraceDecayError::Config { - message: format!("{} is not a TOML table", config_path.display()), - })?; - let hooks = table - .entry("hooks") - .or_insert_with(|| toml::Value::Table(toml::value::Table::new())); - let hooks = hooks - .as_table_mut() - .ok_or_else(|| TraceDecayError::Config { - message: format!("[hooks] in {} is not a table", config_path.display()), - })?; - let state = hooks - .entry("state") - .or_insert_with(|| toml::Value::Table(toml::value::Table::new())); - let state = state - .as_table_mut() - .ok_or_else(|| TraceDecayError::Config { - message: format!("[hooks.state] in {} is not a table", config_path.display()), - })?; + let outcome = update_toml_config_transactionally(&config_path, |config| { + let state = codex_hook_trust_state(config, &config_path)?; - // Drop trust for the active marketplace plus the legacy hard-coded - // `personal` identity before adding the exact installed payload. - // Foreign plugin and repo-local marketplace records remain untouched. + // Drop trust for the active marketplace that the installed payload no + // longer carries, and rewrite only records whose hash moved. Foreign + // plugin and repo-local marketplace records remain untouched. let current_prefix = codex_plugin_hook_trust_prefix(&marketplace_name); - let legacy_prefix = codex_plugin_hook_trust_prefix(CODEX_DEFAULT_MARKETPLACE_NAME); + let (trusted_entries, skipped_entries): (Vec<_>, Vec<_>) = + entries.iter().partition(|entry| { + codex_hook_command_invokes_tracedecay(&entry.command, tracedecay_bin) + }); state.retain(|key, _| { !key.starts_with(¤t_prefix) - && (current_prefix == legacy_prefix || !key.starts_with(&legacy_prefix)) + || trusted_entries.iter().any(|entry| entry.trust_key == key) }); - - let mut trusted = 0usize; - let mut skipped = Vec::new(); - for entry in &entries { - if !codex_hook_command_invokes_tracedecay(&entry.command, tracedecay_bin) { - skipped.push(entry.event_label.clone()); + for entry in &trusted_entries { + let recorded = state + .get(&entry.trust_key) + .and_then(|record| record.get("trusted_hash")) + .and_then(toml_edit::Item::as_str); + if recorded == Some(entry.hash.as_str()) { continue; } - let mut record = toml::value::Table::new(); - record.insert( - "trusted_hash".to_string(), - toml::Value::String(entry.hash.clone()), - ); - state.insert(entry.trust_key.clone(), toml::Value::Table(record)); - trusted += 1; - } - - let outcome = CodexHookTrustSyncOutcome { trusted, skipped }; - // A truthful all-skip (or empty hook payload) leaves no trust records. - // That is not a serializer failure, announce treats it as Ok + guidance. - // Drop hollow `[hooks.state]`/`[hooks]` tables the same way prune does. - if state.is_empty() { - if let Some(hooks) = table.get_mut("hooks").and_then(toml::Value::as_table_mut) { - hooks.remove("state"); - if hooks.is_empty() { - table.remove("hooks"); - } - } - let contents = render_codex_config(&config_path, &config)?; - return Ok((outcome, TextFileMutation::Write(contents))); + let mut record = toml_edit::Table::new(); + record.insert("trusted_hash", toml_edit::value(entry.hash.clone())); + state.insert(&entry.trust_key, toml_edit::Item::Table(record)); } - - let contents = render_codex_config(&config_path, &config)?; - // Child trust records exist: Codex requires an explicit `[hooks.state]` - // parent. Missing child headers here means the serializer dropped - // entries we just inserted, a real contract breach. - let Some(updated) = with_explicit_hooks_state_parent(&contents) else { - return Err(TraceDecayError::Config { - message: "Codex hook trust state serialized without hook entries".to_string(), - }); + let outcome = CodexHookTrustSyncOutcome { + trusted: trusted_entries.len(), + skipped: skipped_entries + .iter() + .map(|entry| entry.event_label.clone()) + .collect(), }; - Ok((outcome, TextFileMutation::Write(updated))) + // A truthful all-skip (or empty hook payload) leaves no trust records, + // and no hollow `[hooks.state]`/`[hooks]` tables, the same as prune. + drop_hollow_codex_hook_tables(config); + Ok(outcome) })?; eprintln!("\x1b[32m✔\x1b[0m Wrote {}", config_path.display()); Ok(outcome) } -fn render_codex_config(config_path: &Path, config: &toml::Value) -> Result { - toml::to_string_pretty(config).map_err(|error| TraceDecayError::Config { - message: format!("failed to serialize {}: {error}", config_path.display()), - }) -} - -/// Codex's hook loader requires the parent table to be explicit on disk. The -/// `toml` serializer otherwise emits only `[hooks.state."..."]` child tables, -/// which parses equivalently but still triggers Codex's hook-review prompt. -/// Returns `None` when no hook trust child tables are present, callers that -/// just inserted records treat that as a serializer contract breach; callers -/// that intentionally cleared state (prune / all-skip) fall back to the -/// unshaped document. -fn with_explicit_hooks_state_parent(contents: &str) -> Option { - let child_offset = contents.find("[hooks.state.\"")?; - let mut updated = String::with_capacity(contents.len() + "[hooks.state]\n\n".len()); - updated.push_str(&contents[..child_offset]); - updated.push_str("[hooks.state]\n\n"); - updated.push_str(&contents[child_offset..]); - Some(updated) +/// The `[hooks.state]` table, created when absent. Codex's hook loader +/// requires that parent header explicitly on disk, while `[hooks]` itself +/// stays implicit unless the operator wrote it. +fn codex_hook_trust_state<'a>( + config: &'a mut toml_edit::DocumentMut, + config_path: &Path, +) -> Result<&'a mut toml_edit::Table> { + let not_a_table = |name: &str| TraceDecayError::Config { + message: format!("[{name}] in {} is not a table", config_path.display()), + }; + let hooks = config + .as_table_mut() + .entry("hooks") + .or_insert_with(|| { + let mut hooks = toml_edit::Table::new(); + hooks.set_implicit(true); + toml_edit::Item::Table(hooks) + }) + .as_table_mut() + .ok_or_else(|| not_a_table("hooks"))?; + let state = hooks + .entry("state") + .or_insert_with(|| toml_edit::Item::Table(toml_edit::Table::new())) + .as_table_mut() + .ok_or_else(|| not_a_table("hooks.state"))?; + state.set_implicit(false); + Ok(state) +} + +/// Remove an emptied `[hooks.state]`, then a `[hooks]` left holding nothing +/// that the operator never wrote a header for. +fn drop_hollow_codex_hook_tables(config: &mut toml_edit::DocumentMut) { + let root = config.as_table_mut(); + let Some(hooks) = root + .get_mut("hooks") + .and_then(toml_edit::Item::as_table_mut) + else { + return; + }; + if hooks + .get("state") + .and_then(toml_edit::Item::as_table) + .is_some_and(toml_edit::Table::is_empty) + { + hooks.remove("state"); + } + if hooks.is_empty() && hooks.is_implicit() { + root.remove("hooks"); + } } /// Remove every TraceDecay-managed `[hooks.state]` trust record from @@ -1414,39 +1292,21 @@ fn prune_codex_hook_trust_records(home: &Path) -> Result<()> { if !config_path.exists() { return Ok(()); } - let pruned = update_toml_config_transactionally(&config_path, |mut config| { - let Some(table) = config.as_table_mut() else { - return Err(TraceDecayError::Config { - message: format!("{} is not a TOML table", config_path.display()), - }); - }; - let Some(state) = table + let pruned = update_toml_config_transactionally(&config_path, |config| { + let Some(state) = config .get_mut("hooks") - .and_then(toml::Value::as_table_mut) .and_then(|hooks| hooks.get_mut("state")) - .and_then(toml::Value::as_table_mut) + .and_then(toml_edit::Item::as_table_mut) else { - return Ok((false, TextFileMutation::Unchanged)); + return Ok(false); }; let before = state.len(); state.retain(|key, _| !key.starts_with(CODEX_PLUGIN_ACTIVATION_KEY_PREFIX)); if state.len() == before { - return Ok((false, TextFileMutation::Unchanged)); + return Ok(false); } - let state_empty = state.is_empty(); - if state_empty - && let Some(hooks) = table.get_mut("hooks").and_then(toml::Value::as_table_mut) - { - hooks.remove("state"); - if hooks.is_empty() { - table.remove("hooks"); - } - } - let contents = render_codex_config(&config_path, &config)?; - // Foreign trust records that remain still need the explicit parent - // table Codex's hook loader requires. - let updated = with_explicit_hooks_state_parent(&contents).unwrap_or(contents); - Ok((true, TextFileMutation::Write(updated))) + drop_hollow_codex_hook_tables(config); + Ok(true) })?; if pruned { eprintln!( @@ -1699,56 +1559,6 @@ fn codex_plugin_is_natively_active(home: &Path, tracedecay_bin: Option<&str>) -> }) } -fn codex_non_interactive_install_state( - home: &Path, - tracedecay_bin: &str, - staged_paths: Vec, -) -> Result { - if codex_plugin_is_natively_active(home, Some(tracedecay_bin))? { - return Ok(NonInteractiveInstallOutcome::Ready); - } - let exact_marketplace_name = - codex_exact_personal_marketplace_name(home).map_err(|()| TraceDecayError::Config { - message: format!( - "could not read Codex marketplace identity at {}", - codex_personal_marketplace_path(home).display() - ), - })?; - let marketplace_name = exact_marketplace_name - .clone() - .unwrap_or_else(|| codex_cached_marketplace_name(home)); - let exact_cache_present = exact_marketplace_name.is_some_and(|marketplace_name| { - codex_plugin_cached_root(home, &marketplace_name) - .join(crate::PRODUCT_VERSION) - .join(".codex-plugin/plugin.json") - .is_file() - }); - if codex_plugin_enabled(home).map_err(|()| TraceDecayError::Config { - message: format!( - "could not read Codex native plugin activation state at {}", - codex_config_path(home).display() - ), - })? && exact_cache_present - { - return Ok(NonInteractiveInstallOutcome::DeferredUserAction( - DeferredUserAction { - remediation: format!( - "Codex's loaded TraceDecay cache is stale. Run `codex plugin add tracedecay@{marketplace_name}` to reinstall it, re-trust changed hooks, then retry the TraceDecay lifecycle." - ), - staged_paths, - }, - )); - } - Ok(NonInteractiveInstallOutcome::DeferredUserAction( - DeferredUserAction { - remediation: format!( - "Codex activates plugins through its native cache. Run `codex plugin add tracedecay@{marketplace_name}` after TraceDecay stages the source package." - ), - staged_paths, - }, - )) -} - fn codex_hook_state_table_is_explicit(contents: &str) -> bool { contents.lines().any(|line| line.trim() == "[hooks.state]") } @@ -1812,76 +1622,85 @@ fn install_codex_marketplace_entry( display_name: &str, source_path: &str, ) -> Result<()> { - let mut marketplace = load_json_file_strict(marketplace_path)?; - if !marketplace.is_object() { - marketplace = json!({}); - } - let existing_name = marketplace.get("name").and_then(|value| value.as_str()); - if let Some(existing_name) = existing_name { - validate_codex_marketplace_name(existing_name)?; - } - let has_tracedecay_entry = marketplace - .get("plugins") - .and_then(serde_json::Value::as_array) - .is_some_and(|plugins| { - plugins.iter().any(|entry| { - entry.get("name").and_then(|value| value.as_str()) == Some("tracedecay") - }) - }); - let should_write_identity = - existing_name.is_none() || (existing_name == Some("caveman-home") && has_tracedecay_entry); - if should_write_identity { - marketplace["name"] = json!(marketplace_name); - } - if !marketplace - .get("interface") - .is_some_and(serde_json::Value::is_object) - { - marketplace["interface"] = json!({}); - } - if should_write_identity - || marketplace["interface"] - .get("displayName") - .and_then(|value| value.as_str()) - .is_none() - { - marketplace["interface"]["displayName"] = json!(display_name); - } - if !marketplace - .get("plugins") - .is_some_and(serde_json::Value::is_array) - { - marketplace["plugins"] = json!([]); - } - let Some(plugins) = marketplace["plugins"].as_array_mut() else { - return Err(TraceDecayError::Config { - message: "failed to normalize Codex marketplace plugins to an array".to_string(), - }); - }; - plugins.retain(|entry| { - !matches!( - entry.get("name").and_then(|value| value.as_str()), - Some("tracedecay") - ) - }); - plugins.push(json!({ - "name": "tracedecay", - "source": { - "source": "local", - "path": source_path, - }, - "policy": { - "installation": "AVAILABLE", - "authentication": "ON_INSTALL", + let effective_marketplace_name = update_json_config_transactionally( + marketplace_path, + JsonConfigDialect::Json, + |mut marketplace| { + if !marketplace.is_object() { + marketplace = json!({}); + } + let existing_name = marketplace.get("name").and_then(|value| value.as_str()); + if let Some(existing_name) = existing_name { + validate_codex_marketplace_name(existing_name)?; + } + let has_tracedecay_entry = marketplace + .get("plugins") + .and_then(serde_json::Value::as_array) + .is_some_and(|plugins| { + plugins.iter().any(|entry| { + entry.get("name").and_then(|value| value.as_str()) == Some("tracedecay") + }) + }); + let should_write_identity = existing_name.is_none() + || (existing_name == Some("caveman-home") && has_tracedecay_entry); + if should_write_identity { + marketplace["name"] = json!(marketplace_name); + } + if !marketplace + .get("interface") + .is_some_and(serde_json::Value::is_object) + { + marketplace["interface"] = json!({}); + } + if should_write_identity + || marketplace["interface"] + .get("displayName") + .and_then(|value| value.as_str()) + .is_none() + { + marketplace["interface"]["displayName"] = json!(display_name); + } + if !marketplace + .get("plugins") + .is_some_and(serde_json::Value::is_array) + { + marketplace["plugins"] = json!([]); + } + let Some(plugins) = marketplace["plugins"].as_array_mut() else { + return Err(TraceDecayError::Config { + message: "failed to normalize Codex marketplace plugins to an array" + .to_string(), + }); + }; + plugins.retain(|entry| { + !matches!( + entry.get("name").and_then(|value| value.as_str()), + Some("tracedecay") + ) + }); + plugins.push(json!({ + "name": "tracedecay", + "source": { + "source": "local", + "path": source_path, + }, + "policy": { + "installation": "AVAILABLE", + "authentication": "ON_INSTALL", + }, + "category": "Productivity", + })); + let effective_marketplace_name = marketplace + .get("name") + .and_then(serde_json::Value::as_str) + .unwrap_or(marketplace_name) + .to_string(); + Ok(( + effective_marketplace_name, + JsonConfigMutation::Write(marketplace), + )) }, - "category": "Productivity", - })); - let effective_marketplace_name = marketplace - .get("name") - .and_then(serde_json::Value::as_str) - .unwrap_or(marketplace_name) - .to_string(); - safe_write_json_file(marketplace_path, &marketplace, None)?; + )?; eprintln!( "\x1b[32m✔\x1b[0m Added tracedecay to Codex {effective_marketplace_name} marketplace at {}", marketplace_path.display() @@ -1919,64 +1738,6 @@ fn remove_codex_plugin_skills_dir(install_dir: &Path) -> Result<()> { Ok(()) } -fn remove_codex_retired_autodiscovered_files(install_dir: &Path) -> Result<()> { - let managed = codex_plugin_managed_paths(install_dir) - .into_iter() - .collect::>(); - for relative_root in ["agents", "commands", "hooks", "skills"] { - let root = install_dir.join(relative_root); - let Ok(metadata) = std::fs::symlink_metadata(&root) else { - continue; - }; - if metadata.file_type().is_symlink() || !metadata.is_dir() { - continue; - } - let mut files = - super::collect_regular_files(&root).map_err(|error| TraceDecayError::Config { - message: format!( - "failed to inventory retired Codex plugin files under {}: {error}", - root.display() - ), - })?; - files.sort(); - for file in files { - if managed.contains(&file) { - continue; - } - let Some(relative) = file - .strip_prefix(install_dir) - .ok() - .and_then(Path::to_str) - .map(|relative| relative.replace(std::path::MAIN_SEPARATOR, "/")) - else { - continue; - }; - if !super::is_auto_discovered_entrypoint(&relative) { - continue; - } - let Ok(contents) = std::fs::read(&file) else { - continue; - }; - if !retired_entrypoints::has_exact_identity(&relative, &contents) { - continue; - } - super::safe_remove_host_file(&file).map_err(|error| TraceDecayError::Config { - message: format!( - "failed to remove retired TraceDecay plugin file {}: {error}", - file.display() - ), - })?; - } - prune_empty_dirs(&root).map_err(|error| TraceDecayError::Config { - message: format!( - "failed to prune retired Codex plugin directories under {}: {error}", - root.display() - ), - })?; - } - Ok(()) -} - fn remove_codex_managed_skill_overlay(install_dir: &Path) { std::fs::remove_dir_all(install_dir.join("skills/agent-managed")).ok(); } @@ -2043,7 +1804,6 @@ fn remove_codex_plugin_install(install_dir: &Path) -> Result<()> { }); } remove_codex_plugin_skills_dir(install_dir)?; - remove_codex_retired_autodiscovered_files(install_dir)?; if codex_plugin_dir_has_only_managed_files(install_dir) { std::fs::remove_dir_all(install_dir).map_err(|e| TraceDecayError::Config { message: format!("failed to remove {}: {e}", install_dir.display()), @@ -2085,24 +1845,32 @@ fn remove_codex_marketplace_entry_at(marketplace_path: &Path, label: &str) -> Re if !marketplace_path.exists() { return Ok(()); } - let mut marketplace = load_json_file_strict(marketplace_path)?; - let Some(plugins) = marketplace - .get_mut("plugins") - .and_then(|value| value.as_array_mut()) - else { - return Ok(()); - }; - let before = plugins.len(); - plugins.retain(|entry| { - !matches!( - entry.get("name").and_then(|value| value.as_str()), - Some("tracedecay") - ) - }); - if plugins.len() == before { + let removed = update_json_config_transactionally( + marketplace_path, + JsonConfigDialect::Json, + |mut marketplace| { + let Some(plugins) = marketplace + .get_mut("plugins") + .and_then(|value| value.as_array_mut()) + else { + return Ok((false, JsonConfigMutation::Unchanged)); + }; + let before = plugins.len(); + plugins.retain(|entry| { + !matches!( + entry.get("name").and_then(|value| value.as_str()), + Some("tracedecay") + ) + }); + if plugins.len() == before { + return Ok((false, JsonConfigMutation::Unchanged)); + } + Ok((true, JsonConfigMutation::Write(marketplace))) + }, + )?; + if !removed { return Ok(()); } - safe_write_json_file(marketplace_path, &marketplace, None)?; eprintln!( "\x1b[32m✔\x1b[0m Removed tracedecay from Codex {label} marketplace at {}", marketplace_path.display() diff --git a/crates/tracedecay-agent-hosts/src/agents/codex/retired_entrypoints.rs b/crates/tracedecay-agent-hosts/src/agents/codex/retired_entrypoints.rs deleted file mode 100644 index e3b6990b43..0000000000 --- a/crates/tracedecay-agent-hosts/src/agents/codex/retired_entrypoints.rs +++ /dev/null @@ -1,137 +0,0 @@ -use tracedecay_domain::canonical_text::sha256_hex; - -/// Exact identities of auto-discovered Codex skills retired from previously -/// shipped plugin bundles. Cleanup is deliberately closed over these -/// path-and-content pairs: a name, frontmatter field, or TraceDecay tool -/// reference is not ownership evidence. -pub(super) const CODEX_RETIRED_ENTRYPOINT_IDENTITIES: &[(&str, &str)] = &[ - ( - "skills/recalling-session-context/SKILL.md", - "8baea6f4050fdecbd273eab8ff4131179ffb3ae8c9d6bfe64a88a73ce62acbb7", - ), - ( - "skills/retrieving-cached-context/SKILL.md", - "fb1b2fa3e50d7f6e5a472259ed37c1e8135e29ced0ebb3e3cbb3bb2c5eab7300", - ), - ( - "skills/retrieving-project-memory/SKILL.md", - "e3a2fd2d24836e9319e8e177d4eebb06f15d8253d42d85bfd8b911fb27b488d5", - ), - ( - "skills/storing-project-memory/SKILL.md", - "0ac3f8a41bb88c61bcac0121aebe1b8ff239adfaee4ccaeb9904850c66521bfd", - ), - ( - "skills/tracedecay-audit-safety/SKILL.md", - "d31189e7edcd510ddf574b8a0406f8f87b95fed980bc833911b946d3389d9985", - ), - ( - "skills/tracedecay-audit-safety/SKILL.md", - "bb196c87c451d75a148c47cfad3f50f3c8a0eff887d79de14169fa57df453d8e", - ), - ( - "skills/tracedecay-check-health/SKILL.md", - "34ce491ee8ff4d5a887b4d9ddad7589734f548221d7014e4585aff3c07025537", - ), - ( - "skills/tracedecay-check-health/SKILL.md", - "14a48887d0b0759053031f5a7f5ca04ca5c47e41a4b01b6e21c8109bb58cadb5", - ), - ( - "skills/tracedecay-clean-dead-code/SKILL.md", - "e45b29417511f335a395d10985e646c4eb8ce104f772148418c95cbb62304d7f", - ), - ( - "skills/tracedecay-clean-dead-code/SKILL.md", - "3fdfaa6bf98063f988b4daba7710c7329d97e8f2b50b199058647528c4a006c2", - ), - ( - "skills/tracedecay-compare-branches/SKILL.md", - "4540d196228e8cebfcd6211fe616fefa664ad7f0347fe6cdef575b1643509d74", - ), - ( - "skills/tracedecay-compare-branches/SKILL.md", - "cbf0a7a1a3b40deead079008d5a263a021ec4a7deffea6e65c5f1c022b1cbea7", - ), - ( - "skills/tracedecay-curate-memory/SKILL.md", - "6b05eec049940a25f28eb0186853fb0b95747707750c2e3cbaa04643213b6fee", - ), - ( - "skills/tracedecay-curate-memory/SKILL.md", - "662615971acb8286540ce4b788ff765e1042deae83e7e329222a2e187f6ccedf", - ), - ( - "skills/tracedecay-draft-commit/SKILL.md", - "9a17774f196d2840a14824144273cc1981d5ef01a72f254fa3a270df493ada52", - ), - ( - "skills/tracedecay-draft-commit/SKILL.md", - "ac6d44274d05a9e55f2ab6c215ccead1db9eebe69c3926eb958f2bd06df29ff3", - ), - ( - "skills/tracedecay-find-impact/SKILL.md", - "184540ec2da7673dd535868f659fb2eff48bcd07abc5f651aa1272795a2bf640", - ), - ( - "skills/tracedecay-find-impact/SKILL.md", - "349da2f8b490085b7515f106cc052980bbad9d98c09cc0815171a83e0a7f418f", - ), - ( - "skills/tracedecay-fix-build/SKILL.md", - "64a876ea2615f1b940116bf3570d4cf722a03af06bdcf1faf716601eb72f9739", - ), - ( - "skills/tracedecay-fix-build/SKILL.md", - "c22aafad7165101fa012e92419eda3e18ad7ecf5859259e65636ce1431e3f75a", - ), - ( - "skills/tracedecay-map-architecture/SKILL.md", - "6bf5800ace17a547bc6d2e7c2112a1140923f9890102c653c35c4659f6ea0fef", - ), - ( - "skills/tracedecay-map-architecture/SKILL.md", - "2242ed70968d832fc40f917784e7c4571690ec07850ba1edfbbdfd69b344e0ac", - ), - ( - "skills/tracedecay-port-code/SKILL.md", - "08c5b95fb3ca8abdb2a077645a4c97a805518f1ca98d15d4a90751801097a546", - ), - ( - "skills/tracedecay-port-code/SKILL.md", - "9c222985b6bd7d9f1a4c60c4844587dfa86c5a8c77bc90ea71b67eb201df1a44", - ), - ( - "skills/tracedecay-recall-memory/SKILL.md", - "6481a0780a031108e2a8084d7da926d06d524f8ab7f3b1428ae8f7c236cfd4a4", - ), - ( - "skills/tracedecay-recall-memory/SKILL.md", - "6bc0fa8061bd2aa704ac8a28811b433297411bbf63aabc39522009417ffb3717", - ), - ( - "skills/tracedecay-review-diff/SKILL.md", - "f977e9bd0dec65569905a7c360bd232c6652e1644d870ad15fe48b80d70e71cf", - ), - ( - "skills/tracedecay-review-diff/SKILL.md", - "9ad0572e2abbd1cd5784848042fc5c16fb44b6d1a57b2bdead89f6ad8e9ae018", - ), - ( - "skills/tracedecay-test-changes/SKILL.md", - "04481d6cd6c110efddd8db93129767003443ef57687d6e6dd775186b437bb85b", - ), - ( - "skills/tracedecay-test-changes/SKILL.md", - "aec63d5a77d238eae959c579bd490068b283a859769c0a11d111ce257eeaf38f", - ), -]; - -pub(super) fn has_exact_identity(relative: &str, contents: &[u8]) -> bool { - let digest = sha256_hex(contents); - CODEX_RETIRED_ENTRYPOINT_IDENTITIES - .iter() - .any(|(owned_relative, owned_digest)| { - relative == *owned_relative && digest == *owned_digest - }) -} diff --git a/crates/tracedecay-agent-hosts/src/agents/codex/tests.rs b/crates/tracedecay-agent-hosts/src/agents/codex/tests.rs index bacb3d3374..2d57f4bd58 100644 --- a/crates/tracedecay-agent-hosts/src/agents/codex/tests.rs +++ b/crates/tracedecay-agent-hosts/src/agents/codex/tests.rs @@ -1,5 +1,5 @@ use super::*; -use sha2::{Digest, Sha256}; +use crate::agents::safe_write_json_file; /// The repo-local `hooks-codex.json` ships only an empty `hooks` object. /// Rendering the global bundle must fill the object from `CODEX_MANAGED_HOOKS` @@ -292,16 +292,9 @@ trusted_hash = "sha256:foreign" & 0o777, 0o600 ); - // The backup carries the same secrets as the original, so it must - // inherit the restrictive mode instead of the umask default. - assert_eq!( - std::fs::metadata(crate::agents::config_backup_path(&config_path)) - .unwrap() - .permissions() - .mode() - & 0o777, - 0o600, - "config.toml.bak must keep the original 0600 mode" + assert!( + !config_path.with_extension("toml.bak").exists(), + "hook trust must not keep a copy of the secret-bearing config" ); } @@ -428,7 +421,7 @@ fn sync_codex_hook_trust_hashes_the_installed_hook_payload() { let hooks_path = plugin_dir.join("hooks/hooks.json"); let mut hooks = load_json_file_strict(&hooks_path).unwrap(); hooks["hooks"]["SessionStart"][0]["hooks"][0]["timeout"] = json!(9); - safe_write_json_file(&hooks_path, &hooks, None).unwrap(); + safe_write_json_file(&hooks_path, &hooks).unwrap(); let changed_entries = codex_hook_trust_entries(&hooks).unwrap(); std::fs::create_dir_all(home.path().join(".codex")).unwrap(); @@ -464,7 +457,7 @@ fn sync_codex_hook_trust_reads_a_custom_marketplace_cache() { let hooks_path = plugin_dir.join("hooks/hooks.json"); let mut hooks = load_json_file_strict(&hooks_path).unwrap(); hooks["hooks"]["SessionStart"][0]["hooks"][0]["timeout"] = json!(9); - safe_write_json_file(&hooks_path, &hooks, None).unwrap(); + safe_write_json_file(&hooks_path, &hooks).unwrap(); let changed_entries = codex_hook_trust_entries_for_marketplace(&hooks, "my-marketplace").unwrap(); std::fs::create_dir_all(home.path().join(".codex")).unwrap(); @@ -490,7 +483,7 @@ fn sync_codex_hook_trust_rejects_tampered_installed_command() { .to_string(); hooks["hooks"]["SessionStart"][0]["hooks"][0]["command"] = json!(format!("{command} && /tmp/untrusted-payload")); - safe_write_json_file(&hooks_path, &hooks, None).unwrap(); + safe_write_json_file(&hooks_path, &hooks).unwrap(); std::fs::create_dir_all(home.path().join(".codex")).unwrap(); let outcome = sync_codex_hook_trust(home.path(), TEST_BIN).unwrap(); @@ -529,7 +522,7 @@ fn sync_codex_hook_trust_all_skipped_is_ok_without_hollow_state() { } } } - safe_write_json_file(&hooks_path, &hooks, None).unwrap(); + safe_write_json_file(&hooks_path, &hooks).unwrap(); let config_path = codex_config_path(home.path()); std::fs::create_dir_all(config_path.parent().unwrap()).unwrap(); std::fs::write( @@ -683,12 +676,25 @@ fn install_ctx(home: &Path) -> InstallContext { InstallContext { home: home.to_path_buf(), tracedecay_bin: TEST_BIN.to_string(), - tool_permissions: Vec::new(), project_root: None, dashboard: false, } } +/// The personal plugin source plus its marketplace entry, as a completed +/// Core install leaves them. +fn install_codex_personal_bootstrap(home: &Path, tracedecay_bin: &str) -> Result { + let install_dir = codex_plugin_install_dir(home); + install_codex_plugin_bundle(&install_dir, tracedecay_bin, InstallScope::Global, home)?; + install_codex_marketplace_entry( + &codex_personal_marketplace_path(home), + "personal", + "Personal", + CODEX_GLOBAL_PLUGIN_SOURCE_PATH, + )?; + Ok(install_dir) +} + fn copy_rendered_bundle_to_native_cache(home: &Path, tracedecay_bin: &str) { let source = codex_plugin_install_dir(home); let cache = codex_plugin_current_cached_install_dir(home); @@ -775,140 +781,35 @@ fn native_cache_content_drift_and_binary_relocation_require_refresh() { let old_bin = "/old/bin/tracedecay"; let new_bin = "/relocated/bin/tracedecay"; write_exact_native_activation(home.path(), old_bin); - let old_ctx = install_ctx(home.path()); - let old_ctx = InstallContext { - tracedecay_bin: old_bin.to_string(), - ..old_ctx - }; - assert!(matches!( - CodexIntegration - .preflight_non_interactive_install(&old_ctx) - .unwrap(), - NonInteractiveInstallOutcome::Ready - )); + assert!(codex_plugin_is_natively_active(home.path(), Some(old_bin)).unwrap()); let retired_skill = codex_plugin_current_cached_install_dir(home.path()).join("skills/retired/SKILL.md"); std::fs::create_dir_all(retired_skill.parent().unwrap()).unwrap(); std::fs::write(&retired_skill, "# stale auto-discovered skill\n").unwrap(); - assert!(matches!( - CodexIntegration - .preflight_non_interactive_install(&old_ctx) - .unwrap(), - NonInteractiveInstallOutcome::DeferredUserAction(_) - )); + assert!(!codex_plugin_is_natively_active(home.path(), Some(old_bin)).unwrap()); std::fs::remove_file(retired_skill).unwrap(); - assert!(matches!( - CodexIntegration - .preflight_non_interactive_install(&old_ctx) - .unwrap(), - NonInteractiveInstallOutcome::Ready - )); + assert!(codex_plugin_is_natively_active(home.path(), Some(old_bin)).unwrap()); std::fs::write( codex_plugin_current_cached_install_dir(home.path()).join(".mcp.json"), "{}\n", ) .unwrap(); - assert!(matches!( - CodexIntegration - .preflight_non_interactive_install(&old_ctx) - .unwrap(), - NonInteractiveInstallOutcome::DeferredUserAction(_) - )); + assert!(!codex_plugin_is_natively_active(home.path(), Some(old_bin)).unwrap()); copy_rendered_bundle_to_native_cache(home.path(), old_bin); - assert!(matches!( - CodexIntegration - .preflight_non_interactive_install(&old_ctx) - .unwrap(), - NonInteractiveInstallOutcome::Ready - )); + assert!(codex_plugin_is_natively_active(home.path(), Some(old_bin)).unwrap()); install_codex_personal_bootstrap(home.path(), new_bin).unwrap(); - let relocated_ctx = InstallContext { - tracedecay_bin: new_bin.to_string(), - ..old_ctx - }; - assert!(matches!( - CodexIntegration - .preflight_non_interactive_install(&relocated_ctx) - .unwrap(), - NonInteractiveInstallOutcome::DeferredUserAction(_) - )); + assert!(!codex_plugin_is_natively_active(home.path(), Some(new_bin)).unwrap()); copy_rendered_bundle_to_native_cache(home.path(), new_bin); - assert!(matches!( - CodexIntegration - .preflight_non_interactive_install(&relocated_ctx) - .unwrap(), - NonInteractiveInstallOutcome::Ready - )); -} - -#[test] -fn every_published_retired_discovery_identity_converges_on_redeploy() { - #[derive(serde::Deserialize)] - struct PublishedRetiredEntrypoint { - path: String, - digest: String, - releases: Vec, - contents: String, - } - - let variants = serde_json::from_str::>(include_str!(concat!( - env!("CARGO_MANIFEST_DIR"), - "/../../tests/fixtures/host_integrations/codex_retired_entrypoints.json" - ))) - .unwrap(); - assert_eq!(variants.len(), 30); - let published_identities = variants - .iter() - .map(|variant| (variant.path.as_str(), variant.digest.as_str())) - .collect::>(); - let production_identities = retired_entrypoints::CODEX_RETIRED_ENTRYPOINT_IDENTITIES - .iter() - .copied() - .collect::>(); - assert_eq!(production_identities, published_identities); - - for variant in variants { - let observed_digest = hex::encode(Sha256::digest(variant.contents.as_bytes())); - assert_eq!(observed_digest, variant.digest); - assert!( - retired_entrypoints::has_exact_identity(&variant.path, variant.contents.as_bytes()), - "published identity missing for {} from {:?}", - variant.path, - variant.releases - ); - - let home = tempfile::tempdir().unwrap(); - write_exact_native_activation(home.path(), TEST_BIN); - let ctx = install_ctx(home.path()); - let retired = codex_plugin_install_dir(home.path()).join(&variant.path); - std::fs::create_dir_all(retired.parent().unwrap()).unwrap(); - std::fs::write(&retired, variant.contents).unwrap(); - assert!(matches!( - CodexIntegration - .preflight_non_interactive_install(&ctx) - .unwrap(), - NonInteractiveInstallOutcome::DeferredUserAction(_) - )); - - install_codex_personal_bootstrap(home.path(), TEST_BIN).unwrap(); - assert!(!retired.exists(), "retained {}", variant.path); - assert!(matches!( - CodexIntegration - .preflight_non_interactive_install(&ctx) - .unwrap(), - NonInteractiveInstallOutcome::Ready - )); - } + assert!(codex_plugin_is_natively_active(home.path(), Some(new_bin)).unwrap()); } #[test] fn redeploy_preserves_foreign_discovery_and_support_bytes() { let home = tempfile::tempdir().unwrap(); write_exact_native_activation(home.path(), TEST_BIN); - let ctx = install_ctx(home.path()); let source = codex_plugin_install_dir(home.path()); let operator_skill = source.join("skills/operator-owned/SKILL.md"); std::fs::create_dir_all(operator_skill.parent().unwrap()).unwrap(); @@ -929,12 +830,7 @@ fn redeploy_preserves_foreign_discovery_and_support_bytes() { let helper = source.join("hooks/helper.py"); let helper_bytes = b"# operator helper for tracedecay_lcm_describe\n"; std::fs::write(&helper, helper_bytes).unwrap(); - assert!(matches!( - CodexIntegration - .preflight_non_interactive_install(&ctx) - .unwrap(), - NonInteractiveInstallOutcome::DeferredUserAction(_) - )); + assert!(!codex_plugin_is_natively_active(home.path(), Some(TEST_BIN)).unwrap()); install_codex_personal_bootstrap(home.path(), TEST_BIN).unwrap(); assert_eq!( @@ -951,50 +847,26 @@ fn redeploy_preserves_foreign_discovery_and_support_bytes() { ); assert_eq!(std::fs::read(&reference).unwrap(), reference_bytes); assert_eq!(std::fs::read(&helper).unwrap(), helper_bytes); - assert!(matches!( - CodexIntegration - .preflight_non_interactive_install(&ctx) - .unwrap(), - NonInteractiveInstallOutcome::DeferredUserAction(_) - )); -} - -/// Preflight still reports that the cache is not yet active; activation itself -/// is no longer an interactive deferral. Codex CLI 0.147 drives `plugin add`. -#[test] -fn codex_preflight_reports_inactive_cache_without_interactive_guidance() { - let home = tempfile::tempdir().unwrap(); - let NonInteractiveInstallOutcome::DeferredUserAction(deferred) = CodexIntegration - .preflight_non_interactive_install(&install_ctx(home.path())) - .unwrap() - else { - panic!("inactive Codex cache must still be a typed preflight deferral"); - }; - assert!( - deferred - .remediation - .contains("codex plugin add tracedecay@personal") - ); - assert!(CodexIntegration.interactive_activation_guidance().is_none()); - assert!(CodexIntegration.interactive_removal_guidance().is_none()); + assert!(!codex_plugin_is_natively_active(home.path(), Some(TEST_BIN)).unwrap()); } /// Install an executable `codex` on the host-program search path only. /// -/// Preparation is `Ready` exactly when Codex's own plugin CLI is present, so -/// the outcome under test is a property of the environment, not of the host -/// integration. CI runners carry no `codex` binary while a developer box -/// usually does; pin it here instead of reading whichever the machine has. -/// Only host program resolution sees this directory, the process `PATH` is -/// untouched. +/// CI runners carry no `codex` binary while a developer box usually does; +/// pin it here instead of reading whichever the machine has. Only host +/// program resolution sees this directory, the process `PATH` is untouched. fn install_fake_codex_cli( dir: &Path, ) -> tracedecay_runtime_core::config::HostProgramSearchPathGuard { - let binary = dir.join(format!("codex{}", std::env::consts::EXE_SUFFIX)); - std::fs::write(&binary, "#!/bin/sh\nexit 0\n").unwrap(); + // Windows runs neither a shebang script nor one named `.exe`; a batch + // file is the native spelling the host-program lookup admits. + #[cfg(windows)] + std::fs::write(dir.join("codex.cmd"), "@exit /b 0\r\n").unwrap(); #[cfg(unix)] { use std::os::unix::fs::PermissionsExt; + let binary = dir.join("codex"); + std::fs::write(&binary, "#!/bin/sh\nexit 0\n").unwrap(); let mut permissions = std::fs::metadata(&binary).unwrap().permissions(); permissions.set_mode(0o755); std::fs::set_permissions(&binary, permissions).unwrap(); @@ -1002,65 +874,30 @@ fn install_fake_codex_cli( tracedecay_runtime_core::config::HostProgramSearchPathGuard::set(dir) } +/// `codex plugin add tracedecay@personal` resolves the plugin through the +/// personal marketplace, so activation writes that entry itself, inside the +/// transaction's registration boundary, before driving the host CLI. #[test] -fn prepare_stages_the_source_and_returns_ready_for_cli_activation() { +fn activation_registers_the_personal_marketplace_before_plugin_add() { let home = tempfile::tempdir().unwrap(); let cli_dir = tempfile::tempdir().unwrap(); let _codex_cli = install_fake_codex_cli(cli_dir.path()); - // Pre-existing user config: preparation runs before the component - // transaction stages `config.toml`, so it must not write there, hook - // trust is recorded by activation, inside the rollback boundary. - let config_path = codex_config_path(home.path()); - std::fs::create_dir_all(config_path.parent().unwrap()).unwrap(); - std::fs::write(&config_path, "model = \"gpt-5\"\n").unwrap(); + let marketplace_path = codex_personal_marketplace_path(home.path()); + assert!(!marketplace_path.exists()); - let outcome = CodexIntegration - .prepare_non_interactive_install(&install_ctx(home.path())) + CodexIntegration + .activate_deployed_host_registration(&install_ctx(home.path())) .unwrap(); - assert!(matches!(outcome, NonInteractiveInstallOutcome::Ready)); - assert!(codex_plugin_manifest_path(home.path()).is_file()); - assert!(codex_personal_marketplace_path(home.path()).is_file()); - assert_eq!( - std::fs::read_to_string(&config_path).unwrap(), - "model = \"gpt-5\"\n", - "preparation must leave config.toml untouched" - ); -} -/// `Ready` promises Core apply can drive `codex plugin add`, so an -/// unresolvable plugin CLI must defer instead. -/// -/// Answering `Ready` opens a component transaction that can only die in -/// activation with `HostCliUnavailable`. Its rollback leaves a `RolledBack` -/// journal pinning `config.toml` and the versioned cache as they were before -/// the operator runs the remediation the failure prints, and the next -/// lifecycle command's `recover_host` then refuses the drifted host with -/// `StalePreview`. -#[test] -fn prepare_defers_when_no_plugin_cli_resolves() { - let home = tempfile::tempdir().unwrap(); - // Resolution sees only this empty directory; the process `PATH` (which on - // a developer box usually does carry `codex`) is untouched. - let empty = tempfile::tempdir().unwrap(); - let _host_programs = - tracedecay_runtime_core::config::HostProgramSearchPathGuard::set(empty.path()); - - let outcome = CodexIntegration - .prepare_non_interactive_install(&install_ctx(home.path())) - .unwrap(); - let NonInteractiveInstallOutcome::DeferredUserAction(deferred) = outcome else { - panic!("staging Codex without a resolvable plugin CLI must defer, got {outcome:?}"); - }; - assert!( - deferred - .remediation - .contains("`codex plugin add tracedecay@personal`"), - "the deferral must print the executable remediation: {}", - deferred.remediation + let marketplace: serde_json::Value = + serde_json::from_slice(&std::fs::read(&marketplace_path).unwrap()).unwrap(); + assert_eq!(marketplace["name"], "personal"); + assert_eq!( + marketplace + .pointer("/plugins/0/source/path") + .and_then(serde_json::Value::as_str), + Some(CODEX_GLOBAL_PLUGIN_SOURCE_PATH) ); - // The source is still staged: the operator's `codex plugin add` consumes it. - assert!(codex_plugin_manifest_path(home.path()).is_file()); - assert!(codex_personal_marketplace_path(home.path()).is_file()); } /// Activation must record hook trust even when Codex already reports the @@ -1151,10 +988,9 @@ trusted_hash = "sha256:foreign" CodexIntegration .deactivate_deployed_host_registration(&install_ctx(home.path())) .unwrap(); - let cleaned = load_toml_file(&config_path).unwrap(); - assert_eq!(cleaned["model"].as_str().unwrap(), "gpt-5"); - assert!( - cleaned.get("hooks").is_none(), + assert_eq!( + std::fs::read_to_string(&config_path).unwrap(), + "model = \"gpt-5\"\n", "an emptied [hooks] tree is dropped rather than left hollow" ); @@ -1166,55 +1002,6 @@ trusted_hash = "sha256:foreign" assert_eq!(std::fs::read(&config_path).unwrap(), before); } -#[test] -fn codex_update_plugin_refreshes_bundle_and_records_hook_trust() { - let home = tempfile::tempdir().unwrap(); - write_exact_native_activation(home.path(), TEST_BIN); - // Re-seed config.toml with the native activation record plus an unrelated - // user key the trust write must preserve. - let config_path = codex_config_path(home.path()); - std::fs::write( - &config_path, - "model = \"gpt-5\"\n\n[plugins.\"tracedecay@personal\"]\nenabled = true\n", - ) - .unwrap(); - let project_root = home.path().join("workspace"); - let ctx = InstallContext { - project_root: Some(project_root), - ..install_ctx(home.path()) - }; - - let outcome = CodexIntegration.update_plugin(&ctx).unwrap(); - let UpdatePluginOutcome::Refreshed(paths) = outcome else { - panic!("expected codex update_plugin to refresh the bundle"); - }; - assert_eq!(paths, vec![codex_plugin_install_dir(home.path())]); - - // update-plugin auto-trusts the refreshed hooks by recording their content - // hashes in config.toml, while leaving the user's unrelated keys intact. - let updated = load_toml_file(&config_path).unwrap(); - assert_eq!(updated["model"].as_str().unwrap(), "gpt-5"); - assert_eq!( - updated["plugins"]["tracedecay@personal"]["enabled"].as_bool(), - Some(true), - "update-plugin must preserve Codex's own activation record" - ); - assert!( - updated["hooks"]["state"] - .as_table() - .unwrap() - .keys() - .any(|key| key.starts_with("tracedecay@personal:hooks/hooks.json:")), - "update-plugin should record tracedecay hook trust entries" - ); - let entries = managed_entries(TEST_BIN); - assert_eq!( - codex_plugin_hook_trust_state(&updated, &entries), - CodexHookTrustState::Trusted - ); - assert_eq!(codex_hook_trust_followup(home.path()), None); -} - #[test] fn deactivation_fails_on_corrupt_plugins_table() { let home = tempfile::tempdir().unwrap(); @@ -1243,3 +1030,65 @@ fn deactivation_fails_on_corrupt_plugins_table() { "plugins = \"corrupt\"\n" ); } + +/// Hook trust edits only TraceDecay's `[hooks.state]` records: the operator's +/// comments, spacing, key order and inline tables survive install, a repeat +/// sync is byte-stable, and deactivation restores the original bytes. +#[test] +fn hook_trust_install_and_uninstall_restore_operator_config_bytes() { + let home = tempfile::tempdir().unwrap(); + install_codex_personal_bootstrap(home.path(), TEST_BIN).unwrap(); + let config_path = codex_config_path(home.path()); + std::fs::create_dir_all(config_path.parent().unwrap()).unwrap(); + let original = "# operator header\n\ +zeta = \"last-alphabetically\" # inline note\n\ +model = 'o4-mini'\n\ +sandbox = { mode = \"workspace-write\", network = false }\n\ +\n\ +[hooks.state]\n\ +[hooks.state.\"other@plugin:hooks/hooks.json:session_start:0:0\"]\n\ +trusted_hash = \"sha256:foreign\"\n\ +\n\ +# servers below\n\ +[mcp_servers.foreign]\n\ +args = [ \"--stdio\" ]\n\ +command = \"foreign-bin\""; + std::fs::write(&config_path, original).unwrap(); + + sync_codex_hook_trust(home.path(), TEST_BIN).unwrap(); + let installed = std::fs::read_to_string(&config_path).unwrap(); + let config = load_toml_file(&config_path).unwrap(); + assert_eq!( + codex_plugin_hook_trust_state(&config, &managed_entries(TEST_BIN)), + CodexHookTrustState::Trusted + ); + for line in original.lines() { + assert!( + installed.lines().any(|installed| installed == line), + "install rewrote operator line {line:?}:\n{installed}" + ); + } + + sync_codex_hook_trust(home.path(), TEST_BIN).unwrap(); + assert_eq!(std::fs::read_to_string(&config_path).unwrap(), installed); + + CodexIntegration + .deactivate_deployed_host_registration(&install_ctx(home.path())) + .unwrap(); + assert_eq!(std::fs::read_to_string(&config_path).unwrap(), original); +} + +/// A config TraceDecay created only to hold hook trust is removed again. +#[test] +fn hook_trust_uninstall_removes_a_config_it_created() { + let home = tempfile::tempdir().unwrap(); + install_codex_personal_bootstrap(home.path(), TEST_BIN).unwrap(); + let config_path = codex_config_path(home.path()); + + sync_codex_hook_trust(home.path(), TEST_BIN).unwrap(); + assert!(config_path.exists()); + CodexIntegration + .deactivate_deployed_host_registration(&install_ctx(home.path())) + .unwrap(); + assert!(!config_path.exists()); +} diff --git a/crates/tracedecay-agent-hosts/src/agents/context_scout.rs b/crates/tracedecay-agent-hosts/src/agents/context_scout.rs index cc3db60238..c80d243a27 100644 --- a/crates/tracedecay-agent-hosts/src/agents/context_scout.rs +++ b/crates/tracedecay-agent-hosts/src/agents/context_scout.rs @@ -30,9 +30,9 @@ use tracedecay_domain::{ActorId, ManifestDigest, RetrievalAnchorId, UtcMicros}; use tracedecay_hooks::{HookEventEnvelopeV2, HookScopedFeedbackV1}; use tracedecay_runtime_core::cancellation::{CancellationToken, MonotonicDeadline}; +pub mod address_registry; pub mod model; pub mod owner; -pub mod ports; const MAX_SCOUT_TEXT_BYTES: usize = 4 * 1024; const MAX_SCOUT_CANDIDATES: usize = 32; diff --git a/crates/tracedecay-agent-hosts/src/agents/context_scout/ports.rs b/crates/tracedecay-agent-hosts/src/agents/context_scout/address_registry.rs similarity index 99% rename from crates/tracedecay-agent-hosts/src/agents/context_scout/ports.rs rename to crates/tracedecay-agent-hosts/src/agents/context_scout/address_registry.rs index 826a8cf72a..6128fbddfb 100644 --- a/crates/tracedecay-agent-hosts/src/agents/context_scout/ports.rs +++ b/crates/tracedecay-agent-hosts/src/agents/context_scout/address_registry.rs @@ -1,4 +1,5 @@ -//! Daemon-owned canonical ports for Context Scout orchestration. +//! Daemon-owned durable Context Scout address registry, the configuration and +//! scope pins that gate it, and canonical input assembly for bound addresses. //! //! Opaque fixed-size values in [`ContextScoutAddressV1`] are locators only. //! Exact identity remains the typed lifecycle tuple retained in the durable @@ -706,7 +707,7 @@ impl ProjectContextScoutAddressRegistryV1 { } // The shared decode/validate funnel behind every resolve and authorize - // port operation on the durable address ledger. + // operation on the durable address ledger. #[hotpath::measure( label = "context_scout_address_ledger_read", impl_type = "ProjectContextScoutAddressRegistryV1" @@ -1097,12 +1098,13 @@ mod tests { use std::collections::BTreeMap; use tempfile::TempDir; + use tracedecay_domain::NativeHostIdentityV1; use tracedecay_domain::configuration::{ CandidateDispositionV1, ConfigurationCandidateV1, ConfigurationLayerIdV1, ContextScoutConfigurationLimitsV1, ContextScoutSettingsV1, }; use tracedecay_hooks::{ - HookCapabilityV1, HookEventFamily, HookHostV1, NativeEnvelopeMaterialV1, + HookCapabilityV1, HookEventFamily, NativeEnvelopeMaterialV1, decode_bound_native_hook_event, stock_event_support, }; @@ -1195,7 +1197,7 @@ mod tests { fn binding() -> HookScopeBindingV1 { HookScopeBindingV1 { - host: HookHostV1::ClaudeCode, + host: NativeHostIdentityV1::ClaudeCode, project_id: [1; 16], repository_id: [2; 16], worktree_id: [3; 16], @@ -1211,7 +1213,7 @@ mod tests { .into_iter() .map(|family| HookCapabilityV1 { family, - support: stock_event_support(HookHostV1::ClaudeCode, family), + support: stock_event_support(NativeHostIdentityV1::ClaudeCode, family), }) .collect(), } @@ -1220,7 +1222,7 @@ mod tests { fn admitted_hook() -> AdmittedContextScoutHookV1 { let binding = binding(); let envelope = decode_bound_native_hook_event( - HookHostV1::ClaudeCode, + NativeHostIdentityV1::ClaudeCode, include_bytes!( "../../../../../tests/fixtures/packaged_host_events/claude/post_tool_use_write.json" ), diff --git a/crates/tracedecay-agent-hosts/src/agents/context_scout/model.rs b/crates/tracedecay-agent-hosts/src/agents/context_scout/model.rs index 03c466aef2..8ed1085c8c 100644 --- a/crates/tracedecay-agent-hosts/src/agents/context_scout/model.rs +++ b/crates/tracedecay-agent-hosts/src/agents/context_scout/model.rs @@ -8,12 +8,21 @@ use super::{ ContextScoutModelExecutionV1, ContextScoutModelFuture, ContextScoutModelProposalV1, ContextScoutModelRequestV1, serialized_token_count, warm_token_counter, }; -use crate::ports::pricing::cost_of_turn; use tracedecay_automation_runtime::automation::backend::{ AgentTaskBackend, AgentTaskContract, AgentTaskError, AgentTaskKind, AgentTaskRequest, AgentTaskResponse, CodexAppServerBackend, backend_availability, }; use tracedecay_automation_runtime::automation::config::{AutomationBackend, AutomationConfig}; +use tracedecay_domain::configuration::LcmSummarizerExecutableV1; +use tracedecay_session_memory::provider_pricing::{cost_of_usage, load_table}; + +/// The automation settings and the configured `codex` executable one project +/// configuration snapshot binds for the Scout model route. +#[derive(Clone, Copy)] +pub struct ContextScoutModelConfig<'a> { + pub automation: &'a AutomationConfig, + pub codex: &'a LcmSummarizerExecutableV1, +} const CONTEXT_SCOUT_PROMPT_V1: &str = "\ Select one supplied candidate and return only the JSON object required by the response schema. \ @@ -35,22 +44,25 @@ pub fn context_scout_backend_from_automation_config( #[hotpath::measure(label = "agent_hosts.context_scout.model_route")] pub fn context_scout_model_assistant_from_automation_config( - config: &AutomationConfig, + config: ContextScoutModelConfig<'_>, ) -> Arc { - let route = context_scout_backend_from_automation_config(config); + let route = context_scout_backend_from_automation_config(config.automation); if route != ContextScoutModelBackendV1::CodexAppServer - || !backend_availability(config).available + || !backend_availability(config.automation, config.codex).available { return Arc::new(UnavailableContextScoutModelAssistantV1 { route }); } Arc::new(ProductionContextScoutModelAssistantV1::new( - Arc::new(CodexAppServerBackend::from_automation_config(config)), + Arc::new(CodexAppServerBackend::from_automation_config( + config.automation, + config.codex, + )), route, )) } pub fn context_scout_model_assistant_from_project_config( - config: Option<&AutomationConfig>, + config: Option>, ) -> Arc { config.map_or_else( || { @@ -278,7 +290,15 @@ fn estimated_cost_microusd( input_tokens: Option, output_tokens: Option, ) -> Option { - let cost = cost_of_turn(provider?, model?, input_tokens?, output_tokens?, 0, 0)?; + let cost = cost_of_usage( + load_table(), + provider?, + model?, + input_tokens?, + output_tokens?, + Some(0), + Some(0), + )?; Some((cost * 1_000_000.0).round().clamp(0.0, u64::MAX as f64) as u64) } @@ -316,6 +336,10 @@ mod tests { output_tokens: Some(16), }) } + + fn executable(&self) -> Option<&std::path::Path> { + None + } } #[cfg(feature = "token-counting")] @@ -338,6 +362,10 @@ mod tests { output_tokens: None, }) } + + fn executable(&self) -> Option<&std::path::Path> { + None + } } fn request() -> ContextScoutModelRequestV1 { @@ -427,6 +455,10 @@ mod tests { ) -> Result { Err(self.error.clone()) } + + fn executable(&self) -> Option<&std::path::Path> { + None + } } #[cfg(feature = "token-counting")] diff --git a/crates/tracedecay-agent-hosts/src/agents/context_scout/owner.rs b/crates/tracedecay-agent-hosts/src/agents/context_scout/owner.rs index 92a12c933f..11b3ce5e56 100644 --- a/crates/tracedecay-agent-hosts/src/agents/context_scout/owner.rs +++ b/crates/tracedecay-agent-hosts/src/agents/context_scout/owner.rs @@ -5,7 +5,6 @@ use std::sync::{Arc, OnceLock}; use std::time::{SystemTime, UNIX_EPOCH}; use tokio::sync::{Mutex, RwLock}; -use tracedecay_automation_runtime::automation::config::AutomationConfig; use tracedecay_contracts::RequestContext; use tracedecay_contracts::context_scout::{ ContextScoutAddressV1, ContextScoutClaimHandleV1, ContextScoutClaimRequestV1, @@ -20,11 +19,12 @@ use tracedecay_hooks::{ }; use tracedecay_runtime_core::cancellation::{CancellationToken, MonotonicDeadline}; -use super::model::context_scout_model_assistant_from_project_config; -use super::ports::{ - AdmittedContextScoutHookV1, ContextScoutAuthorityPinV1, ContextScoutConfigurationPinV1, - ContextScoutLifecycleAddressV1, ProjectContextScoutAddressRegistryV1, +use super::address_registry::{ + AdmittedContextScoutHookV1, ContextScoutAddressResolveOutcomeV1, ContextScoutAuthorityPinV1, + ContextScoutConfigurationPinV1, ContextScoutLifecycleAddressV1, + ProjectContextScoutAddressRegistryV1, }; +use super::model::{ContextScoutModelConfig, context_scout_model_assistant_from_project_config}; use super::{ ContextScoutBudgetStateV1, ContextScoutCapabilityStateV1, ContextScoutControlV1, ContextScoutDurableClaimOutcomeV1, ContextScoutDurableRuntimeV1, @@ -145,7 +145,7 @@ impl ProjectContextScoutOwnerV1 { database: Database, project_id: [u8; 16], now: UtcMicros, - model_config: Option<&AutomationConfig>, + model_config: Option>, ) -> Option> { if let Some(existing) = lookup_registered_context_scout_owners(project_id) .into_iter() @@ -212,7 +212,7 @@ impl ProjectContextScoutOwnerV1 { || registry .resolve_current_exact(hook, &pin, &lifecycle, &context, observed_at) .await - != super::ports::ContextScoutAddressResolveOutcomeV1::Resolved(address) + != ContextScoutAddressResolveOutcomeV1::Resolved(address) { return ContextScoutClaimAdmissionV1::Rejected; } @@ -263,7 +263,7 @@ impl ProjectContextScoutOwnerV1 { .registry .resolve_current_exact(hook, &mounted.pin, lifecycle, &mounted.context, observed_at) .await; - (resolved == super::ports::ContextScoutAddressResolveOutcomeV1::Resolved(mounted.address)) + (resolved == ContextScoutAddressResolveOutcomeV1::Resolved(mounted.address)) .then_some((mounted.address, mounted.input_watermark)) } @@ -329,7 +329,7 @@ impl ProjectContextScoutOwnerV1 { observed_at, ) .await; - (resolved == super::ports::ContextScoutAddressResolveOutcomeV1::Resolved(mounted.address)) + (resolved == ContextScoutAddressResolveOutcomeV1::Resolved(mounted.address)) .then_some((mounted.address, mounted.input_watermark)) } @@ -612,7 +612,7 @@ impl ProjectContextScoutOwnerV1 { pub async fn install_configuration( &self, pin: ContextScoutConfigurationPinV1, - model_config: Option<&AutomationConfig>, + model_config: Option>, ) -> Result<(), ContextScoutErrorV1> { let control = pin.control(); let model = model_config.map_or_else( @@ -1122,13 +1122,14 @@ impl ContextScoutModelAssistantV1 for UnavailableConfiguredContextScoutModelV1 { #[cfg(test)] mod tests { - use super::super::ports::ContextScoutAddressBindOutcomeV1; + use super::super::address_registry::ContextScoutAddressBindOutcomeV1; use super::*; use std::collections::{BTreeMap, BTreeSet}; use tracedecay_contracts::{ CancellationContext, CapabilityGrantId, CapabilityGrantSnapshot, Deadline, DisclosureClass, RequestId, ResolvedScope, }; + use tracedecay_domain::NativeHostIdentityV1; use tracedecay_domain::canonical_sha256; use tracedecay_domain::configuration::{ CONTEXT_SCOUT_SETTINGS_SETTING_KEY, CandidateDispositionV1, ConfigurationCandidateV1, @@ -1139,8 +1140,8 @@ mod tests { use tracedecay_domain::{ActorId, RepositoryId, WorktreeId}; use tracedecay_global_db::configuration::contracts::ConfigurationCurrentStateV1; use tracedecay_hooks::{ - HookCapabilityV1, HookEventFamily, HookHostV1, HookScopeBindingV1, - NativeEnvelopeMaterialV1, decode_bound_native_hook_event, stock_event_support, + HookCapabilityV1, HookEventFamily, HookScopeBindingV1, NativeEnvelopeMaterialV1, + decode_bound_native_hook_event, stock_event_support, }; use tracedecay_tool_catalog::{CapabilityId, UseCaseId}; @@ -1402,7 +1403,7 @@ mod tests { ContextScoutAuthorityPinV1::new(&context, feedback_scope, configuration, observed_at) .expect("authority pin"); let binding = HookScopeBindingV1 { - host: HookHostV1::ClaudeCode, + host: NativeHostIdentityV1::ClaudeCode, project_id: [1; 16], repository_id: [2; 16], worktree_id: [3; 16], @@ -1418,12 +1419,12 @@ mod tests { .into_iter() .map(|family| HookCapabilityV1 { family, - support: stock_event_support(HookHostV1::ClaudeCode, family), + support: stock_event_support(NativeHostIdentityV1::ClaudeCode, family), }) .collect(), }; let envelope = decode_bound_native_hook_event( - HookHostV1::ClaudeCode, + NativeHostIdentityV1::ClaudeCode, include_bytes!( "../../../../../tests/fixtures/packaged_host_events/claude/post_tool_use_write.json" ), diff --git a/crates/tracedecay-agent-hosts/src/agents/copilot.rs b/crates/tracedecay-agent-hosts/src/agents/copilot.rs index 52aa2fd43d..6bab18f386 100644 --- a/crates/tracedecay-agent-hosts/src/agents/copilot.rs +++ b/crates/tracedecay-agent-hosts/src/agents/copilot.rs @@ -27,8 +27,8 @@ use std::path::{Path, PathBuf}; use tracedecay_domain::errors::{Result, TraceDecayError}; use super::{ - AgentIntegration, DoctorCounters, HealthcheckContext, InstallContext, config_backup_path, - load_json_file, load_jsonc_file, + AgentIntegration, DoctorCounters, HealthcheckContext, InstallContext, load_json_file, + load_jsonc_file, }; /// Name of GitHub Copilot's own CLI, which owns `~/.copilot/mcp-config.json`. @@ -121,8 +121,8 @@ impl AgentIntegration for CopilotIntegration { /// /// `ContextMcp` is the CLI-driven half of this integration: the only file /// it mutates is Copilot's own `~/.copilot/mcp-config.json`, and the writer - /// is `copilot mcp`, not TraceDecay. Naming that file (and its staged - /// backup) here is what gives the component-set transaction rollback + /// is `copilot mcp`, not TraceDecay. Naming that file here is what gives + /// the component-set transaction rollback /// authority over the host command's effect; without it the observation /// recorded in `run_mcp_registry_step` would have nothing to restore. /// Any other component set keeps the default inventory, which is the @@ -133,8 +133,7 @@ impl AgentIntegration for CopilotIntegration { home: &Path, ) -> Vec { if components == [super::host_bundle::HostComponentV1::ContextMcp] { - let path = copilot_cli_mcp_config_path(home); - vec![path.clone(), config_backup_path(&path)] + vec![copilot_cli_mcp_config_path(home)] } else { self.host_registration_paths(home) } diff --git a/crates/tracedecay-agent-hosts/src/agents/cursor.rs b/crates/tracedecay-agent-hosts/src/agents/cursor.rs index 38708661bc..34d19005c1 100644 --- a/crates/tracedecay-agent-hosts/src/agents/cursor.rs +++ b/crates/tracedecay-agent-hosts/src/agents/cursor.rs @@ -7,14 +7,11 @@ use std::path::{Path, PathBuf}; use serde_json::{Value, json}; -use tracedecay_domain::errors::{Result, TraceDecayError}; +use tracedecay_domain::errors::Result; use super::host_bundle::{HostBundleRegistrationStateV1, HostComponentV1}; use super::{ - AgentIntegration, DoctorCounters, HealthcheckContext, InstallContext, JsonConfigDialect, - JsonConfigMutation, McpUninstallPolicy, UpdatePluginOutcome, load_json_file, - load_jsonc_file_strict, mcp_config_has_tracedecay, safe_remove_host_file, safe_write_text_file, - uninstall_mcp_server_entry, update_json_config_transactionally, + AgentIntegration, DoctorCounters, HealthcheckContext, load_json_file, load_jsonc_file_strict, }; pub struct CursorIntegration; @@ -56,22 +53,6 @@ impl AgentIntegration for CursorIntegration { true } - fn update_plugin(&self, ctx: &InstallContext) -> Result { - // The whole plugin directory is a tracedecay-generated bundle (its - // mcp.json / hooks.json are rendered artifacts, not user config), so - // refreshing it is exactly the install path. User config such as - // `~/.cursor/mcp.json` is never written by `install_cursor_plugin`, - // and unmanaged files inside the plugin dir are preserved. - if !cursor_plugin_manifest_path(&ctx.home).exists() { - return Ok(UpdatePluginOutcome::NotInstalled); - } - install_cursor_plugin(&ctx.home, &ctx.tracedecay_bin)?; - sweep_legacy_project_artifacts_at_cwd(&ctx.home); - Ok(UpdatePluginOutcome::Refreshed(vec![ - cursor_plugin_install_dir(&ctx.home), - ])) - } - fn export_managed_skills( &self, home: &Path, @@ -93,16 +74,8 @@ impl AgentIntegration for CursorIntegration { fn healthcheck(&self, dc: &mut DoctorCounters, ctx: &HealthcheckContext) { eprintln!("\n\x1b[1mCursor integration\x1b[0m"); - let project_cursor = ctx.project_path.join(".cursor"); doctor_check_plugin(dc, &ctx.home); doctor_check_native_extension(dc, &ctx.home); - if legacy_project_cursor_has_tracedecay(&project_cursor) { - dc.warn( - "legacy project Cursor MCP/hooks/rule files are present; rerun \ - `tracedecay install --agent cursor` from this project to remove \ - tracedecay-owned entries", - ); - } super::cursor_diagnostics::report_cursor_mcp_log_findings(dc, &ctx.home); } @@ -314,87 +287,8 @@ fn stale_native_extension_dirs(home: &Path) -> Vec { stale } -const RETIRED_CURSOR_MEMORY_RULE_MARKER: &str = - ""; - -fn remove_retired_global_cursor_memory_rule(home: &Path) -> Result { - let rule_path = home.join(".cursor/rules/tracedecay-memory.mdc"); - let metadata = match std::fs::symlink_metadata(&rule_path) { - Ok(metadata) => metadata, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(false), - Err(error) => { - return Err(TraceDecayError::Config { - message: format!( - "failed to inspect retired Cursor memory rule {}: {error}", - rule_path.display() - ), - }); - } - }; - if metadata.file_type().is_symlink() || !metadata.is_file() { - return Ok(false); - } - let contents = - std::fs::read_to_string(&rule_path).map_err(|error| TraceDecayError::Config { - message: format!( - "failed to read retired Cursor memory rule {}: {error}", - rule_path.display() - ), - })?; - if !contents.contains(RETIRED_CURSOR_MEMORY_RULE_MARKER) { - return Ok(false); - } - std::fs::remove_file(&rule_path).map_err(|error| TraceDecayError::Config { - message: format!( - "failed to remove retired Cursor memory rule {}: {error}", - rule_path.display() - ), - })?; - Ok(true) -} - -#[hotpath::measure(label = "hosts.agent.cursor.plugin_install")] -fn install_cursor_plugin(home: &Path, tracedecay_bin: &str) -> Result<()> { - remove_retired_global_cursor_memory_rule(home)?; - let install_dir = cursor_plugin_install_dir(home); - if let Some(parent) = install_dir.parent() { - std::fs::create_dir_all(parent).map_err(|e| TraceDecayError::Config { - message: format!("failed to create {}: {e}", parent.display()), - })?; - } - remove_cursor_plugin_install(&install_dir)?; - - write_embedded_plugin(&install_dir, tracedecay_bin)?; - install_cursor_managed_skill_overlay(home, &install_dir)?; - eprintln!( - "\x1b[32m✔\x1b[0m Installed Cursor plugin at {}", - install_dir.display() - ); - Ok(()) -} - -fn install_cursor_managed_skill_overlay(home: &Path, install_dir: &Path) -> Result<()> { - let profile_root = - tracedecay_automation_runtime::automation::skill_targets::profile_root_for_agent_home(home); - super::retired_memory_digest::remove_state(&profile_root)?; - tracedecay_automation_runtime::automation::skill_targets::install_managed_skills( - &crate::host_io(), - &profile_root, - tracedecay_automation_runtime::automation::skill_targets::SkillInstallTarget::Cursor, - install_dir, - )?; - Ok(()) -} - -fn write_embedded_plugin(install_dir: &Path, tracedecay_bin: &str) -> Result<()> { - for (relative, rendered) in rendered_plugin_files(tracedecay_bin)? { - safe_write_text_file(&install_dir.join(relative), &rendered, None)?; - } - Ok(()) -} - -/// Canonical rendered Cursor plugin inventory shared by explicit artifact -/// refresh and the receipt-backed first-party catalog. +/// Canonical rendered Cursor plugin inventory the receipt-backed +/// first-party catalog deploys. pub(crate) fn rendered_plugin_files(tracedecay_bin: &str) -> Result> { embedded_plugin_files() .into_iter() @@ -436,447 +330,6 @@ fn cursor_plugin_hooks(raw: &str, tracedecay_bin: &str) -> Result { Ok(rendered) } -fn remove_cursor_plugin_install(install_dir: &Path) -> Result<()> { - super::sweep_superseded_plugin_siblings(install_dir, &[".cursor-plugin/plugin.json"])?; - let Ok(metadata) = std::fs::symlink_metadata(install_dir) else { - return Ok(()); - }; - if metadata.file_type().is_symlink() || metadata.is_file() { - safe_remove_host_file(install_dir).map_err(|error| TraceDecayError::Config { - message: format!("failed to remove {}: {error}", install_dir.display()), - })?; - return Ok(()); - } - if !metadata.is_dir() { - return Err(TraceDecayError::Config { - message: format!( - "refusing to replace non-directory Cursor plugin path {}", - install_dir.display() - ), - }); - } - if !cursor_plugin_dir_is_tracedecay(install_dir) { - return Err(TraceDecayError::Config { - message: format!( - "refusing to replace unmanaged Cursor plugin directory {}", - install_dir.display() - ), - }); - } - // The directory is tracedecay-owned. Sweep every skill dir the *current* - // bundle no longer ships (retired dispatcher/workflow/memory skills), then - // remove the managed skill overlay. Deriving the keep-set from the live - // bundle means a newly retired skill is swept automatically, no - // hand-maintained legacy list to fall out of date. User-added files - // outside `skills/` (and any non-tracedecay skill dir) are preserved. - sweep_retired_bundle_skill_dirs(install_dir)?; - remove_cursor_managed_skill_overlay(install_dir)?; - for path in cursor_plugin_managed_paths(install_dir) { - remove_cursor_plugin_file(&path)?; - } - if cursor_plugin_dir_has_only_managed_files(install_dir) { - match std::fs::remove_dir_all(install_dir) { - Ok(()) => {} - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} - Err(error) => { - return Err(TraceDecayError::Config { - message: format!("failed to remove {}: {error}", install_dir.display()), - }); - } - } - } - Ok(()) -} - -fn remove_cursor_plugin_file(path: &Path) -> Result<()> { - match safe_remove_host_file(path) { - Ok(()) => Ok(()), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), - Err(error) => Err(TraceDecayError::Config { - message: format!("failed to remove {}: {error}", path.display()), - }), - } -} - -fn remove_cursor_managed_skill_overlay(install_dir: &Path) -> Result<()> { - let overlay = install_dir.join("skills/agent-managed"); - match super::collect_regular_files(&overlay) { - Ok(files) => { - for path in files { - remove_cursor_plugin_file(&path)?; - } - } - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), - Err(error) => { - return Err(TraceDecayError::Config { - message: format!("failed to inspect {}: {error}", overlay.display()), - }); - } - } - match std::fs::remove_dir_all(&overlay) { - Ok(()) => Ok(()), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), - Err(error) => Err(TraceDecayError::Config { - message: format!("failed to remove {}: {error}", overlay.display()), - }), - } -} - -/// Recognize a receiptless Cursor deployment as a prior first-party install. -/// -/// Pre-receipt releases deployed the plugin bundle without host-bundle v2 -/// receipts, so receipt evidence alone cannot tell those files from an -/// operator's own. The recognizable provenance is the bundle's own durable -/// anchor, exactly the evidence the legacy installer trusts before its clean -/// replace: the plugin directory's `.cursor-plugin/plugin.json` naming -/// `tracedecay` (plugin components), or the versioned native-extension -/// directory carrying tracedecay's own `package.json` (the Agent component). -/// Arbitrary bytes parked at a cataloged deploy path carry neither anchor and -/// stay refused without the operator's explicit `--yes --adopt`. -pub(crate) fn receiptless_component_provenance(home: &Path, component: HostComponentV1) -> bool { - if component == HostComponentV1::Agent { - return matches!( - cursor_native_extension_registration(home), - HostBundleRegistrationStateV1::Current | HostBundleRegistrationStateV1::Repairable - ); - } - cursor_plugin_dir_is_tracedecay(&cursor_plugin_install_dir(home)) -} - -/// Sweep retired artifacts only after the current plugin manifest proves the -/// directory is TraceDecay-owned. Receiptless upgrades call this after their -/// component-set receipt commits, so a failed cleanup is safely retryable by -/// the next install, update, or repair. -pub fn sweep_retired_cursor_plugin_artifacts(home: &Path) -> Result<()> { - let install_dir = cursor_plugin_install_dir(home); - if !cursor_plugin_dir_is_tracedecay(&install_dir) { - return Ok(()); - } - - sweep_retired_bundle_skill_dirs(&install_dir)?; - for relative in [ - "rules/tracedecay-memory.mdc", - "rules/tracedecay-memory-digest.mdc", - ] { - let path = install_dir.join(relative); - let metadata = match std::fs::symlink_metadata(&path) { - Ok(metadata) => metadata, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, - Err(error) => { - return Err(TraceDecayError::Config { - message: format!( - "failed to inspect retired Cursor plugin artifact {}: {error}", - path.display() - ), - }); - } - }; - if metadata.file_type().is_symlink() || !metadata.is_file() { - continue; - } - let contents = std::fs::read_to_string(&path).map_err(|error| TraceDecayError::Config { - message: format!( - "failed to read retired Cursor plugin artifact {}: {error}", - path.display() - ), - })?; - if contents.contains(RETIRED_CURSOR_MEMORY_RULE_MARKER) { - std::fs::remove_file(&path).map_err(|error| TraceDecayError::Config { - message: format!( - "failed to remove retired Cursor plugin artifact {}: {error}", - path.display() - ), - })?; - } - } - remove_retired_global_cursor_memory_rule(home)?; - Ok(()) -} - -/// Remove every `skills/` under the tracedecay plugin dir that the current -/// bundle does not ship. The keep-set is derived from the live embedded bundle, -/// so any retired skill (dispatcher, workflow, or merged-away memory skill) is -/// swept on upgrade without a hand-maintained legacy list. The `agent-managed` -/// overlay is preserved here (removed separately) and never counted as retired. -/// -/// Only tracedecay-owned skill dirs are swept: a same-name user-authored skill -/// whose `SKILL.md` carries no tracedecay marker is left untouched, so an -/// upgrade never deletes a user's private workflow that happens to collide with -/// a retired bundle slug. -fn sweep_retired_bundle_skill_dirs(install_dir: &Path) -> Result<()> { - let skills_root = install_dir.join("skills"); - let entries = match std::fs::read_dir(&skills_root) { - Ok(entries) => entries, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), - Err(error) => { - return Err(TraceDecayError::Config { - message: format!( - "failed to inspect retired Cursor plugin skills at {}: {error}", - skills_root.display() - ), - }); - } - }; - let shipped: std::collections::BTreeSet = embedded_plugin_files() - .into_iter() - .filter_map(|(relative, _)| { - relative - .strip_prefix("skills/") - .and_then(|rest| rest.split('/').next()) - .map(str::to_string) - }) - .collect(); - for entry in entries { - let entry = entry.map_err(|error| TraceDecayError::Config { - message: format!( - "failed to inspect retired Cursor plugin skills at {}: {error}", - skills_root.display() - ), - })?; - if !entry - .file_type() - .map_err(|error| TraceDecayError::Config { - message: format!( - "failed to inspect retired Cursor plugin skill at {}: {error}", - entry.path().display() - ), - })? - .is_dir() - { - continue; - } - let name = entry.file_name().to_string_lossy().into_owned(); - // The managed overlay is handled separately; never treat it as retired. - if name == "agent-managed" || shipped.contains(&name) { - continue; - } - // Preserve user-authored skills that reuse a retired slug: only sweep a - // non-shipped dir that is demonstrably tracedecay-owned. - if !skill_file_has_tracedecay_marker(&entry.path().join("SKILL.md")) { - continue; - } - std::fs::remove_dir_all(entry.path()).map_err(|error| TraceDecayError::Config { - message: format!( - "failed to remove retired Cursor plugin skill at {}: {error}", - entry.path().display() - ), - })?; - } - Ok(()) -} - -/// True when a Cursor `SKILL.md` carries a tracedecay authorship marker, marking -/// the skill dir as tracedecay-owned (and therefore safe to sweep when retired). -fn skill_file_has_tracedecay_marker(skill_file: &Path) -> bool { - std::fs::read_to_string(skill_file) - .is_ok_and(|contents| super::skill_contents_have_tracedecay_marker(&contents)) -} - -fn cursor_plugin_dir_is_tracedecay(install_dir: &Path) -> bool { - let manifest = load_json_file(&install_dir.join(".cursor-plugin/plugin.json")); - matches!( - manifest.get("name").and_then(|v| v.as_str()), - Some("tracedecay") - ) -} - -fn cursor_plugin_dir_has_only_managed_files(install_dir: &Path) -> bool { - let Ok(entries) = super::collect_regular_files(install_dir) else { - return false; - }; - let managed = cursor_plugin_managed_paths(install_dir); - entries.iter().all(|entry| managed.contains(entry)) -} - -fn cursor_plugin_managed_paths(install_dir: &Path) -> Vec { - let mut paths: Vec = embedded_plugin_files() - .into_iter() - .map(|(relative, _)| install_dir.join(relative)) - .collect(); - // Retired in 0.0.66 when dynamic memory moved to ~/.cursor/rules. Keep the - // old receipt-owned path in the sweep inventory so install and uninstall - // can remove artifacts written by older bundles. - paths.push(install_dir.join("rules/tracedecay-memory.mdc")); - paths.push(install_dir.join("rules/tracedecay-memory-digest.mdc")); - paths -} - -fn legacy_mcp_has_tracedecay(mcp_path: &Path) -> bool { - mcp_config_has_tracedecay(mcp_path, "mcpServers", load_json_file) -} - -fn legacy_project_cursor_has_tracedecay(cursor_dir: &Path) -> bool { - legacy_mcp_has_tracedecay(&cursor_dir.join("mcp.json")) - || legacy_hooks_have_tracedecay(&cursor_dir.join("hooks.json")) - || legacy_rule_has_tracedecay(&cursor_dir.join("rules/tracedecay.mdc")) -} - -/// Removes legacy PROJECT-local tracedecay artifacts. Pre-plugin versions of -/// `tracedecay install --local` wrote the MCP server entry, lifecycle hooks, -/// and the steering rule into `/.cursor/`; the user-level plugin -/// owns all three surfaces now. This is the project-level counterpart of the -/// user-level plugin-dir clean replace: detection-gated so projects without -/// legacy artifacts are untouched, and only tracedecay-owned entries are removed, -/// user-authored config (other MCP servers, custom hooks and rules, and -/// `permissions.json` allowlists, which the plugin README still recommends -/// per-repo) is preserved. -fn sweep_legacy_project_artifacts(project_path: &Path) -> Result<()> { - let cursor_dir = project_path.join(".cursor"); - let mcp_path = cursor_dir.join("mcp.json"); - let hooks_path = cursor_dir.join("hooks.json"); - let rule_paths = [cursor_dir.join("rules/tracedecay.mdc")]; - let legacy_mcp = legacy_mcp_has_tracedecay(&mcp_path); - let legacy_hooks = legacy_hooks_have_tracedecay(&hooks_path); - let legacy_rule = rule_paths - .iter() - .any(|path| legacy_rule_has_tracedecay(path)); - if !legacy_mcp && !legacy_hooks && !legacy_rule { - return Ok(()); - } - for path in [&mcp_path, &hooks_path] { - super::ensure_project_local_safe_path(project_path, path)?; - } - for path in &rule_paths { - super::ensure_project_local_safe_path(project_path, path)?; - } - if legacy_mcp { - uninstall_mcp_server_entry( - &mcp_path, - "mcpServers", - JsonConfigDialect::Json, - McpUninstallPolicy { - prune_empty_root: true, - remove_empty_file: true, - }, - )?; - } - if legacy_hooks { - remove_legacy_project_hooks(&hooks_path)?; - } - if legacy_rule { - for path in &rule_paths { - remove_legacy_project_rule(path)?; - } - } - Ok(()) -} - -/// The project directory a cwd-based legacy sweep should target, or `None` -/// when the cwd *is* the home directory, there `.cursor/` is Cursor's -/// user-level config tree, not a project workspace. -fn cwd_sweep_target(cwd: PathBuf, home: &Path) -> Option { - let canonical = |path: &Path| path.canonicalize().unwrap_or_else(|_| path.to_path_buf()); - (canonical(&cwd) != canonical(home)).then_some(cwd) -} - -/// Best-effort [`sweep_legacy_project_artifacts`] for global install / -/// update-plugin / uninstall flows, which have no explicit project path: the -/// current working directory is treated as the project. Failures only warn so -/// a malformed `.cursor/` in an unrelated cwd can never block plugin -/// management. -fn sweep_legacy_project_artifacts_at_cwd(home: &Path) { - let Some(project_path) = std::env::current_dir() - .ok() - .and_then(|cwd| cwd_sweep_target(cwd, home)) - else { - return; - }; - if let Err(err) = sweep_legacy_project_artifacts(&project_path) { - eprintln!( - "\x1b[33mwarning:\x1b[0m could not remove legacy project Cursor artifacts in {}: {err}", - project_path.display() - ); - } -} - -/// A Cursor hook entry is tracedecay-owned when its `command` runs a -/// `hook-cursor-*` subcommand. -fn is_legacy_tracedecay_hook(entry: &serde_json::Value) -> bool { - entry - .get("command") - .and_then(|value| value.as_str()) - .is_some_and(|command| command.contains("hook-cursor-")) -} - -fn legacy_hooks_have_tracedecay(hooks_path: &Path) -> bool { - load_json_file(hooks_path) - .get("hooks") - .and_then(|value| value.as_object()) - .is_some_and(|events| { - events.values().any(|value| { - value - .as_array() - .is_some_and(|entries| entries.iter().any(is_legacy_tracedecay_hook)) - }) - }) -} - -fn legacy_rule_has_tracedecay(rule_path: &Path) -> bool { - std::fs::read_to_string(rule_path) - .is_ok_and(|contents| contents.contains("tracedecay MCP tools")) -} - -fn remove_legacy_project_hooks(hooks_path: &Path) -> Result<()> { - if !hooks_path.exists() { - return Ok(()); - } - let removed = - update_json_config_transactionally(hooks_path, JsonConfigDialect::Jsonc, |mut hooks| { - let Some(events) = hooks - .get_mut("hooks") - .and_then(|value| value.as_object_mut()) - else { - return Ok((false, JsonConfigMutation::Unchanged)); - }; - - let mut removed = false; - for value in events.values_mut() { - let Some(entries) = value.as_array_mut() else { - continue; - }; - let before = entries.len(); - entries.retain(|entry| !is_legacy_tracedecay_hook(entry)); - removed |= entries.len() != before; - } - events.retain(|_, value| value.as_array().is_none_or(|entries| !entries.is_empty())); - - if !removed { - return Ok((false, JsonConfigMutation::Unchanged)); - } - if events.is_empty() { - Ok((true, JsonConfigMutation::Remove)) - } else { - Ok((true, JsonConfigMutation::Write(hooks))) - } - })?; - if removed { - eprintln!( - "\x1b[32m✔\x1b[0m Removed legacy Cursor hooks from {}", - hooks_path.display() - ); - } - Ok(()) -} - -fn remove_legacy_project_rule(rule_path: &Path) -> Result<()> { - if !rule_path.exists() { - return Ok(()); - } - let contents = std::fs::read_to_string(rule_path).map_err(|e| TraceDecayError::Config { - message: format!("failed to read {}: {e}", rule_path.display()), - })?; - if contents.contains("tracedecay MCP tools") { - std::fs::remove_file(rule_path).map_err(|e| TraceDecayError::Config { - message: format!("failed to remove {}: {e}", rule_path.display()), - })?; - eprintln!( - "\x1b[32m✔\x1b[0m Removed legacy Cursor rule from {}", - rule_path.display() - ); - } - Ok(()) -} - // --------------------------------------------------------------------------- // Healthcheck helpers // --------------------------------------------------------------------------- @@ -889,11 +342,6 @@ fn doctor_check_plugin(dc: &mut DoctorCounters, home: &Path) { "{} not found, run `tracedecay install --agent cursor` if you use Cursor", manifest_path.display() )); - if legacy_mcp_has_tracedecay(&home.join(".cursor/mcp.json")) { - dc.warn( - "legacy Cursor MCP config is installed; rerun install to use the Cursor plugin", - ); - } return; } @@ -1197,8 +645,13 @@ mod tests { use tempfile::TempDir; use tracedecay_host_integration::HostCapabilityUnavailableReasonV1; - const RETIRED_CURSOR_MEMORY_RULE_FIXTURE: &str = - ""; + /// Writes the rendered bundle exactly as the component catalog deploys it. + fn write_embedded_plugin(install_dir: &Path, tracedecay_bin: &str) -> Result<()> { + for (relative, rendered) in rendered_plugin_files(tracedecay_bin)? { + super::super::safe_write_text_file(&install_dir.join(relative), &rendered)?; + } + Ok(()) + } /// The doctor's expected-hooks list is parsed from the embedded bundle /// template; a parse regression would silently disable the hook checks. @@ -1283,19 +736,10 @@ mod tests { .exists(), "the retired dispatcher skill must not ship" ); - - // Every embedded file is also a managed path so uninstall can clean it. - let managed = cursor_plugin_managed_paths(&install_dir); - for (relative, _) in embedded_plugin_files() { - assert!( - managed.contains(&install_dir.join(relative)), - "{relative} should be a managed path" - ); - } } #[test] - fn clean_cursor_install_update_and_doctor_preserve_user_config() { + fn catalog_cursor_bundle_passes_doctor_and_preserves_user_config() { let home = TempDir::new().unwrap(); let project = TempDir::new().unwrap(); let user_config = home.path().join(".cursor/mcp.json"); @@ -1304,55 +748,17 @@ mod tests { std::fs::write(&user_config, user_config_bytes).unwrap(); let integration = CursorIntegration; - let context = InstallContext { - home: home.path().to_path_buf(), - tracedecay_bin: "/opt/tracedecay-previous".to_string(), - tool_permissions: super::super::expected_tool_perms().expect("tool catalog"), - project_root: None, - dashboard: true, - }; - assert_eq!( - integration.update_plugin(&context).unwrap(), - UpdatePluginOutcome::NotInstalled, - "update must refuse to create a Cursor installation that was never installed" - ); - assert!(!cursor_plugin_install_dir(home.path()).exists()); - - install_cursor_plugin(home.path(), &context.tracedecay_bin).unwrap(); let plugin_dir = cursor_plugin_install_dir(home.path()); - assert!(plugin_dir.join("agents/code-explorer.md").is_file()); - let mcp_path = plugin_dir.join("mcp.json"); + write_embedded_plugin(&plugin_dir, "/opt/tracedecay-next").unwrap(); let installed_mcp: serde_json::Value = - serde_json::from_slice(&std::fs::read(&mcp_path).unwrap()).unwrap(); + serde_json::from_slice(&std::fs::read(plugin_dir.join("mcp.json")).unwrap()).unwrap(); assert_eq!( installed_mcp["mcpServers"]["tracedecay"]["command"], - "/opt/tracedecay-previous" - ); - - let updated_context = InstallContext { - tracedecay_bin: "/opt/tracedecay-next".to_string(), - ..context - }; - assert!(matches!( - integration.update_plugin(&updated_context).unwrap(), - UpdatePluginOutcome::Refreshed(paths) if paths == vec![plugin_dir.clone()] - )); - let updated_mcp: serde_json::Value = - serde_json::from_slice(&std::fs::read(&mcp_path).unwrap()).unwrap(); - assert_eq!( - updated_mcp["mcpServers"]["tracedecay"]["command"], "/opt/tracedecay-next", - "the in-composer MCP registration must refresh for the new binary" + "/opt/tracedecay-next" ); assert!(plugin_dir.join("agents/code-explorer.md").is_file()); assert_eq!(std::fs::read(&user_config).unwrap(), user_config_bytes); - let before_idempotent_update = std::fs::read(&mcp_path).unwrap(); - assert!(matches!( - integration.update_plugin(&updated_context).unwrap(), - UpdatePluginOutcome::Refreshed(paths) if paths == vec![plugin_dir.clone()] - )); - assert_eq!(std::fs::read(&mcp_path).unwrap(), before_idempotent_update); - let mut doctor = DoctorCounters::new(); integration.healthcheck( &mut doctor, @@ -1433,7 +839,6 @@ mod tests { let mut registration = crate::agents::host_component_registration::CatalogHostComponentRegistrationAuthority::new_with_tracedecay_bin( "cursor", home.path(), - lifecycle.path(), install.lifecycle.operation, "/opt/tracedecay-v1".to_string(), ) @@ -1462,7 +867,6 @@ mod tests { let mut registration = crate::agents::host_component_registration::CatalogHostComponentRegistrationAuthority::new_with_tracedecay_bin( "cursor", home.path(), - lifecycle.path(), update.lifecycle.operation, current_bin.clone(), ) @@ -1483,7 +887,6 @@ mod tests { let mut repeat_registration = crate::agents::host_component_registration::CatalogHostComponentRegistrationAuthority::new_with_tracedecay_bin( "cursor", home.path(), - lifecycle.path(), update.lifecycle.operation, current_bin.clone(), ) @@ -1504,7 +907,6 @@ mod tests { let mut denied_registration = crate::agents::host_component_registration::CatalogHostComponentRegistrationAuthority::new_with_tracedecay_bin( "cursor", home.path(), - lifecycle.path(), denied.lifecycle.operation, "/opt/tracedecay-v3".to_string(), ) @@ -1589,115 +991,10 @@ mod tests { ); } - /// A live pre-receipt Cursor bundle, deployed by a release that predates - /// host-bundle receipts, stamped with an older product version and binary - /// path, and recorded by no receipt, must be taken over by the production - /// component transaction. `Install` (the operator's - /// `install --agent cursor`) adopts it, and `Update` (`update-plugin`) - /// restamps it to the running binary, instead of refusing with a - /// "recorded by no receipt" ownership conflict that no command can clear. - #[test] - fn cursor_component_transaction_takes_over_a_pre_receipt_bundle() { - for operation in [ - HostBundleLifecycleOpV1::Install, - HostBundleLifecycleOpV1::Update, - ] { - let home = TempDir::new().unwrap(); - let lifecycle = TempDir::new().unwrap(); - let project = TempDir::new().unwrap(); - // The pre-receipt live bundle: rendered for an older binary, with - // a version-stale manifest, and no v2 receipt anywhere. - let install_dir = cursor_plugin_install_dir(home.path()); - write_embedded_plugin(&install_dir, "/opt/tracedecay-previous") - .expect("the simulated pre-receipt bundle must deploy"); - let manifest_path = cursor_plugin_manifest_path(home.path()); - let stamped = std::fs::read_to_string(&manifest_path).unwrap(); - assert!( - stamped.contains(crate::PRODUCT_VERSION), - "the rendered manifest must carry the product version to re-stamp: {stamped}" - ); - std::fs::write( - &manifest_path, - stamped.replace(crate::PRODUCT_VERSION, "0.1.0-beta.4"), - ) - .unwrap(); - - // Doctor re-renders the expected catalog with the resolved - // binary, so the takeover must deploy exactly that binary for the - // receipts to read as the current release afterwards. - let current_bin = - super::super::which_tracedecay().unwrap_or_else(|| "tracedecay".to_string()); - let current = cursor_component_set(¤t_bin); - let request = cursor_component_request(operation, [64; 16], true); - let mut writer = - HostBundleWriterV1::open_with_lifecycle_root(home.path(), lifecycle.path()) - .unwrap(); - let mut registration = crate::agents::host_component_registration::CatalogHostComponentRegistrationAuthority::new_with_tracedecay_bin( - "cursor", - home.path(), - lifecycle.path(), - operation, - current_bin.clone(), - ) - .unwrap(); - HostComponentSetTransactionV1::new(&mut writer) - .execute( - ¤t.component_set, - &request, - ¤t, - &mut registration, - ) - .unwrap_or_else(|error| { - panic!("{operation:?} must take over the pre-receipt Cursor bundle: {error}") - }); - - let manifest: Value = - serde_json::from_slice(&std::fs::read(&manifest_path).unwrap()).unwrap(); - assert_eq!( - manifest["version"], - crate::PRODUCT_VERSION, - "{operation:?} must restamp the plugin manifest to the running product version" - ); - let mcp: Value = - serde_json::from_slice(&std::fs::read(install_dir.join("mcp.json")).unwrap()) - .unwrap(); - assert_eq!( - mcp["mcpServers"]["tracedecay"]["command"], - Value::String(current_bin.clone()), - "{operation:?} must re-point the MCP registration at the running binary" - ); - - // The takeover recorded durable receipts: Doctor now reports the - // whole Cursor Desktop set as receipt-backed and current. - let report = crate::agents::inspect_receipt_backed_host_components( - &HealthcheckContext { - home: home.path().to_path_buf(), - project_path: project.path().to_path_buf(), - }, - lifecycle.path(), - crate::agents::TEST_GENERATOR_COMMIT, - ) - .expect("Doctor must inspect the adopted bundle through its new receipts"); - assert_eq!( - report.components.len(), - current.component_set.components.len(), - "{operation:?} must leave every Cursor Desktop component receipt-backed: {report:#?}" - ); - for component in &report.components { - assert_eq!( - component.state, - HostBundleComponentDoctorStateV1::Current, - "{operation:?} left a non-current component: {component:#?}" - ); - } - } - } - /// A cataloged deploy path proves nothing about ownership: bytes an - /// operator parked at the plugin manifest path carry no recognizable - /// legacy provenance, so Install and Update refuse them untouched unless - /// the operator explicitly adopts. Explicit adoption then takes them - /// over, backing the previous bytes up first. + /// operator parked at the plugin manifest path are refused untouched by + /// Install and Update unless the operator explicitly adopts. Explicit + /// adoption then takes them over. #[test] fn cursor_transaction_refuses_unrecognized_receiptless_bytes_without_adoption() { for operation in [ @@ -1722,7 +1019,6 @@ mod tests { let mut registration = crate::agents::host_component_registration::CatalogHostComponentRegistrationAuthority::new_with_tracedecay_bin( "cursor", home.path(), - lifecycle.path(), operation, current_bin.clone(), ) @@ -1753,7 +1049,6 @@ mod tests { let mut adopting_registration = crate::agents::host_component_registration::CatalogHostComponentRegistrationAuthority::new_with_tracedecay_bin( "cursor", home.path(), - lifecycle.path(), operation, current_bin.clone(), ) @@ -1774,103 +1069,6 @@ mod tests { } } - /// The provenance recognizer trusts exactly the durable first-party - /// anchors: the plugin directory's own manifest naming tracedecay, and - /// tracedecay's versioned native-extension `package.json`. Arbitrary - /// bytes at those paths recognize nothing. - #[test] - fn receiptless_provenance_requires_a_first_party_anchor() { - let home = TempDir::new().unwrap(); - for component in [ - HostComponentV1::Core, - HostComponentV1::ContextMcp, - HostComponentV1::Agent, - ] { - assert!( - !receiptless_component_provenance(home.path(), component), - "an empty home recognizes no {component:?} provenance" - ); - } - - let manifest_path = cursor_plugin_manifest_path(home.path()); - std::fs::create_dir_all(manifest_path.parent().unwrap()).unwrap(); - std::fs::write(&manifest_path, b"not a tracedecay manifest").unwrap(); - assert!(!receiptless_component_provenance( - home.path(), - HostComponentV1::Core - )); - - std::fs::write( - &manifest_path, - serde_json::to_vec(&json!({ "name": "tracedecay", "version": "0.1.0-beta.4" })) - .unwrap(), - ) - .unwrap(); - assert!(receiptless_component_provenance( - home.path(), - HostComponentV1::Core - )); - assert!(receiptless_component_provenance( - home.path(), - HostComponentV1::ContextMcp - )); - // The plugin anchor says nothing about the native extension. - assert!(!receiptless_component_provenance( - home.path(), - HostComponentV1::Agent - )); - - let extension_dir = home.path().join(cursor_native_extension_relative_dir()); - std::fs::create_dir_all(&extension_dir).unwrap(); - std::fs::write( - extension_dir.join("package.json"), - br#"{"name":"cursor-native","publisher":"tracedecay","main":"./dist/extension.js"}"#, - ) - .unwrap(); - assert!(receiptless_component_provenance( - home.path(), - HostComponentV1::Agent - )); - } - - /// The bounded legacy-inventory sweep removes retired first-party - /// artifacts after adoption validated the bundle, and nothing else: a - /// user's own files survive, and a directory without the tracedecay - /// manifest anchor is never touched. - #[test] - fn retired_artifact_sweep_is_bounded_and_ownership_gated() { - let home = TempDir::new().unwrap(); - let install_dir = cursor_plugin_install_dir(home.path()); - write_embedded_plugin(&install_dir, "tracedecay").expect("bundle should deploy"); - let retired = install_dir.join("rules/tracedecay-memory.mdc"); - std::fs::write(&retired, RETIRED_CURSOR_MEMORY_RULE_FIXTURE).unwrap(); - let user_rule = install_dir.join("rules/my-own-notes.mdc"); - std::fs::write(&user_rule, b"operator notes").unwrap(); - - sweep_retired_cursor_plugin_artifacts(home.path()).expect("sweep should succeed"); - assert!( - !retired.exists(), - "the retired memory rule must be swept from an owned bundle" - ); - assert!(user_rule.exists(), "user files must survive the sweep"); - - // A same-name rule without the tracedecay marker is a user file too. - std::fs::write(&retired, b"my own memory rule").unwrap(); - sweep_retired_cursor_plugin_artifacts(home.path()).expect("sweep should succeed"); - assert_eq!(std::fs::read(&retired).unwrap(), b"my own memory rule"); - std::fs::remove_file(&retired).unwrap(); - - // Without the manifest anchor the sweep never touches the directory. - std::fs::write( - install_dir.join(".cursor-plugin/plugin.json"), - b"not tracedecay", - ) - .unwrap(); - std::fs::write(&retired, RETIRED_CURSOR_MEMORY_RULE_FIXTURE).unwrap(); - sweep_retired_cursor_plugin_artifacts(home.path()).expect("sweep should succeed"); - assert!(retired.exists(), "an unowned directory must never be swept"); - } - #[test] fn native_extension_registration_is_receipt_doctor_ready() { let tmp = TempDir::new().unwrap(); @@ -1933,354 +1131,6 @@ mod tests { ); } - #[test] - fn embedded_install_uninstalls_completely() { - let tmp = TempDir::new().unwrap(); - let install_dir = tmp.path().join("tracedecay"); - write_embedded_plugin(&install_dir, "tracedecay").expect("embedded install should succeed"); - assert!(install_dir.join("skills/exploring-code/SKILL.md").exists()); - - // Because managed paths cover every embedded file, uninstall recognises a - // tracedecay-only directory and removes it entirely. - remove_cursor_plugin_install(&install_dir).expect("uninstall should succeed"); - assert!( - !install_dir.exists(), - "embedded install should be fully removed on uninstall" - ); - } - - #[test] - fn uninstall_sweeps_retired_plugin_memory_rule() { - let tmp = TempDir::new().unwrap(); - let install_dir = tmp.path().join("tracedecay"); - write_embedded_plugin(&install_dir, "tracedecay").expect("embedded install should succeed"); - let retired_rule = install_dir.join("rules/tracedecay-memory.mdc"); - std::fs::write(&retired_rule, RETIRED_CURSOR_MEMORY_RULE_FIXTURE).unwrap(); - - remove_cursor_plugin_install(&install_dir).expect("uninstall should succeed"); - - assert!( - !retired_rule.exists(), - "uninstall must remove the memory rule retired from the plugin inventory" - ); - assert!( - !install_dir.exists(), - "the retired managed rule must not strand the owned plugin directory" - ); - } - - #[test] - fn install_sweeps_retired_plugin_memory_rule() { - let home = TempDir::new().unwrap(); - let install_dir = cursor_plugin_install_dir(home.path()); - write_embedded_plugin(&install_dir, "old-tracedecay") - .expect("old embedded install should succeed"); - let retired_rule = install_dir.join("rules/tracedecay-memory.mdc"); - std::fs::write(&retired_rule, RETIRED_CURSOR_MEMORY_RULE_FIXTURE).unwrap(); - - install_cursor_plugin(home.path(), "new-tracedecay").expect("install should refresh"); - - assert!( - !retired_rule.exists(), - "install must remove the memory rule retired from the plugin inventory" - ); - assert!(install_dir.join("rules/tracedecay.mdc").exists()); - } - - #[test] - fn retired_global_memory_rule_is_removed_only_when_tracedecay_managed() { - let home = TempDir::new().unwrap(); - let rule = home.path().join(".cursor/rules/tracedecay-memory.mdc"); - std::fs::create_dir_all(rule.parent().unwrap()).unwrap(); - std::fs::write( - &rule, - format!("{RETIRED_CURSOR_MEMORY_RULE_MARKER}\nmanaged memory"), - ) - .unwrap(); - - assert!(remove_retired_global_cursor_memory_rule(home.path()).unwrap()); - assert!(!rule.exists()); - - std::fs::write(&rule, "my own Cursor rule").unwrap(); - assert!(!remove_retired_global_cursor_memory_rule(home.path()).unwrap()); - assert_eq!( - std::fs::read_to_string(&rule).unwrap(), - "my own Cursor rule" - ); - } - - #[test] - fn install_sweeps_owned_superseded_plugin_siblings_only() { - let home = TempDir::new().unwrap(); - let plugins = home.path().join(".cursor/plugins/local"); - let retired = plugins.join("tracedecay.pre-v2-adopt"); - let foreign = plugins.join("tracedecay.personal"); - for dir in [&retired, &foreign] { - std::fs::create_dir_all(dir.join(".cursor-plugin")).unwrap(); - std::fs::write( - dir.join(".cursor-plugin/plugin.json"), - serde_json::to_vec(&json!({ "name": "tracedecay" })).unwrap(), - ) - .unwrap(); - } - - install_cursor_plugin(home.path(), "tracedecay").expect("install should succeed"); - - assert!( - !retired.exists(), - "a manifest-owned superseded tracedecay sibling must be swept" - ); - assert!( - foreign.exists(), - "an owned-looking sibling without an explicitly retired suffix must be preserved" - ); - } - - /// Upgrading over an install that shipped the `tracedecay-*` dispatcher - /// *skills* (now re-expressed as native `commands/` slash commands) must - /// sweep those retired skill dirs instead of stranding them as unmanaged - /// leftovers, so Cursor does not list both the retired dispatcher skill and - /// the new native command. - #[test] - fn reinstall_sweeps_retired_dispatcher_skill_dirs() { - let tmp = TempDir::new().unwrap(); - let install_dir = tmp.path().join("tracedecay"); - write_embedded_plugin(&install_dir, "tracedecay").expect("embedded install should succeed"); - // Simulate a pre-migration install that shipped the dispatcher skill. - std::fs::create_dir_all(install_dir.join("skills/tracedecay-review-diff")).unwrap(); - std::fs::write( - install_dir.join("skills/tracedecay-review-diff/SKILL.md"), - "---\nname: tracedecay-review-diff\n---\nApply the `tracedecay:reviewing-changes` skill.\n", - ) - .unwrap(); - // Also simulate a released install that still ships one of the retired - // memory skills merged into `project-memory`; the clean replace must - // sweep it too. - std::fs::create_dir_all(install_dir.join("skills/recalling-project-memory")).unwrap(); - std::fs::write( - install_dir.join("skills/recalling-project-memory/SKILL.md"), - "---\nname: recalling-project-memory\n---\nRecall facts with `tracedecay_fact_store`.\n", - ) - .unwrap(); - - remove_cursor_plugin_install(&install_dir).expect("replace should succeed"); - assert!( - !install_dir.exists(), - "retired dispatcher skill dirs must be swept so the tracedecay-only dir is fully removed" - ); - } - - /// A reinstall must be a CLEAN REPLACE of the tracedecay-owned dir: a stale - /// file the current bundle no longer ships is gone afterward, while the - /// fresh bundle is present. Exercises the full write → remove → write path. - #[test] - fn reinstall_is_a_clean_replace_dropping_stale_files() { - let tmp = TempDir::new().unwrap(); - let install_dir = tmp.path().join("tracedecay"); - write_embedded_plugin(&install_dir, "tracedecay").expect("first install should succeed"); - // A stale skill dir the current bundle does not ship. - std::fs::create_dir_all(install_dir.join("skills/totally-retired-skill")).unwrap(); - std::fs::write( - install_dir.join("skills/totally-retired-skill/SKILL.md"), - "---\nname: totally-retired-skill\n---\nRun `tracedecay_search` first.\n", - ) - .unwrap(); - - // A clean replace: remove the owned dir, then write the fresh bundle. - remove_cursor_plugin_install(&install_dir).expect("clean replace should succeed"); - write_embedded_plugin(&install_dir, "tracedecay").expect("re-install should succeed"); - - assert!( - !install_dir.join("skills/totally-retired-skill").exists(), - "a stale skill dir must be gone after a clean-replace reinstall" - ); - assert!( - install_dir.join("skills/exploring-code/SKILL.md").exists(), - "the current bundle must be present after reinstall" - ); - } - - /// The clean replace must refuse to delete a directory tracedecay does not - /// own (no tracedecay plugin manifest), so it never nukes an unrelated dir. - #[test] - fn clean_replace_refuses_unmanaged_dir() { - let tmp = TempDir::new().unwrap(); - let install_dir = tmp.path().join("tracedecay"); - std::fs::create_dir_all(&install_dir).unwrap(); - std::fs::write(install_dir.join("user-file.txt"), "not tracedecay").unwrap(); - - let err = remove_cursor_plugin_install(&install_dir) - .expect_err("must refuse an unmanaged directory"); - assert!( - err.to_string().contains("unmanaged"), - "unexpected error: {err}" - ); - assert!( - install_dir.join("user-file.txt").exists(), - "an unmanaged dir must be left untouched" - ); - } - - #[test] - fn uninstall_removes_managed_files_and_preserves_user_files() { - let tmp = TempDir::new().unwrap(); - let install_dir = tmp.path().join("tracedecay"); - write_embedded_plugin(&install_dir, "tracedecay").expect("embedded install should succeed"); - std::fs::write(install_dir.join("user-keep.txt"), "keep").unwrap(); - - remove_cursor_plugin_install(&install_dir).expect("uninstall should succeed"); - - assert_eq!( - std::fs::read_to_string(install_dir.join("user-keep.txt")).unwrap(), - "keep" - ); - assert!( - !install_dir.join(".cursor-plugin/plugin.json").exists(), - "managed plugin files must be removed beside operator files" - ); - assert!( - !install_dir.join("rules/tracedecay.mdc").exists(), - "managed rule files must be removed beside operator files" - ); - } - - #[test] - fn leftover_managed_file_removal_propagates_errors() { - let tmp = TempDir::new().unwrap(); - let install_dir = tmp.path().join("tracedecay"); - write_embedded_plugin(&install_dir, "tracedecay").expect("embedded install should succeed"); - std::fs::write(install_dir.join("user-keep.txt"), "keep").unwrap(); - let managed = install_dir.join("rules/tracedecay.mdc"); - std::fs::remove_file(&managed).unwrap(); - std::fs::create_dir(&managed).unwrap(); - std::fs::write(managed.join("nested"), "blocked").unwrap(); - - let error = remove_cursor_plugin_install(&install_dir) - .expect_err("a leftover managed path that is not a file must fail uninstall"); - assert!(error.to_string().contains("failed to remove"), "{error}"); - assert_eq!( - std::fs::read_to_string(install_dir.join("user-keep.txt")).unwrap(), - "keep" - ); - } - - /// The project-local legacy sweep must remove exactly the tracedecay-owned - /// entries pre-plugin installs wrote (`mcp.json` server entry, - /// `hook-cursor-*` hooks, the steering rule) while preserving everything - /// the user authored alongside them. - #[test] - fn sweep_removes_legacy_project_artifacts_preserving_user_config() { - let project = TempDir::new().unwrap(); - let cursor_dir = project.path().join(".cursor"); - std::fs::create_dir_all(cursor_dir.join("rules")).unwrap(); - std::fs::write( - cursor_dir.join("mcp.json"), - serde_json::to_string_pretty(&json!({ - "mcpServers": { - "tracedecay": { "command": "tracedecay", "args": ["serve"] }, - "other": { "url": "https://example.com/mcp" } - } - })) - .unwrap(), - ) - .unwrap(); - std::fs::write( - cursor_dir.join("hooks.json"), - serde_json::to_string_pretty(&json!({ - "hooks": { - "sessionStart": [ - { "command": "tracedecay hook-cursor-session-start" }, - { "command": "./my-hook.sh" } - ] - } - })) - .unwrap(), - ) - .unwrap(); - std::fs::write( - cursor_dir.join("rules/tracedecay.mdc"), - "Prefer tracedecay MCP tools", - ) - .unwrap(); - std::fs::write( - cursor_dir.join("permissions.json"), - serde_json::to_string_pretty(&json!({ - "mcpAllowlist": ["tracedecay:tracedecay_search"] - })) - .unwrap(), - ) - .unwrap(); - - sweep_legacy_project_artifacts(project.path()).expect("sweep should succeed"); - - let mcp = load_json_file(&cursor_dir.join("mcp.json")); - assert!( - mcp["mcpServers"].get("tracedecay").is_none(), - "project-local MCP entry must be removed" - ); - assert!( - mcp["mcpServers"].get("other").is_some(), - "user-authored MCP servers must be preserved" - ); - let hooks = load_json_file(&cursor_dir.join("hooks.json")); - let entries = hooks["hooks"]["sessionStart"].as_array().unwrap(); - assert_eq!( - entries, - &[json!({ "command": "./my-hook.sh" })], - "only hook-cursor-* entries may be removed" - ); - assert!( - !cursor_dir.join("rules/tracedecay.mdc").exists(), - "the legacy steering rule must be removed" - ); - let permissions = load_json_file(&cursor_dir.join("permissions.json")); - assert_eq!( - permissions["mcpAllowlist"], - json!(["tracedecay:tracedecay_search"]), - "per-repo permissions.json allowlists are README-endorsed user config" - ); - } - - /// A project whose `.cursor/` only holds user-authored config (no legacy - /// tracedecay artifacts) must come through the sweep byte-identical, no - /// rewrites, no backups, no deletions. - #[test] - fn sweep_is_noop_without_legacy_tracedecay_artifacts() { - let project = TempDir::new().unwrap(); - let cursor_dir = project.path().join(".cursor"); - std::fs::create_dir_all(cursor_dir.join("rules")).unwrap(); - let mcp = serde_json::to_string_pretty(&json!({ - "mcpServers": { "other": { "url": "https://example.com/mcp" } } - })) - .unwrap(); - std::fs::write(cursor_dir.join("mcp.json"), &mcp).unwrap(); - // A user file that happens to use the legacy rule filename but not - // the tracedecay-generated contents stays untouched. - let rule = "---\ndescription: my own rule\n---\nFollow project conventions.\n"; - std::fs::write(cursor_dir.join("rules/tracedecay.mdc"), rule).unwrap(); - - sweep_legacy_project_artifacts(project.path()).expect("sweep should succeed"); - - assert_eq!( - std::fs::read_to_string(cursor_dir.join("mcp.json")).unwrap(), - mcp - ); - assert_eq!( - std::fs::read_to_string(cursor_dir.join("rules/tracedecay.mdc")).unwrap(), - rule - ); - let mut files = crate::agents::collect_regular_files(&cursor_dir).unwrap(); - files.sort(); - assert_eq!( - files, - vec![ - cursor_dir.join("mcp.json"), - cursor_dir.join("rules/tracedecay.mdc") - ], - "a no-op sweep must not create backups or new files" - ); - } - #[test] fn doctor_plugin_rule_distinguishes_unreadable_from_incomplete() { let tmp = TempDir::new().unwrap(); @@ -2327,22 +1177,6 @@ mod tests { assert_eq!(counters.warnings, 0); } - /// The cwd-based sweep must never treat the home directory as a project: - /// `~/.cursor` is Cursor's user-level config tree. - #[test] - fn cwd_sweep_target_skips_home_dir() { - let home = TempDir::new().unwrap(); - let project = TempDir::new().unwrap(); - assert_eq!( - cwd_sweep_target(home.path().to_path_buf(), home.path()), - None - ); - assert_eq!( - cwd_sweep_target(project.path().to_path_buf(), home.path()), - Some(project.path().to_path_buf()) - ); - } - fn session_ingest_status(placeholder_paths: Value) -> Value { json!({ "cursor_session_ingest": { diff --git a/crates/tracedecay-agent-hosts/src/agents/cursor_diagnostics.rs b/crates/tracedecay-agent-hosts/src/agents/cursor_diagnostics.rs index 3c69490432..83b2f1a71f 100644 --- a/crates/tracedecay-agent-hosts/src/agents/cursor_diagnostics.rs +++ b/crates/tracedecay-agent-hosts/src/agents/cursor_diagnostics.rs @@ -17,16 +17,6 @@ use std::path::{Path, PathBuf}; use super::DoctorCounters; -/// Legacy marker recognized in existing Cursor logs by doctor diagnostics. -/// -/// Proxy-only `serve` no longer emits it, but older logs remain actionable, so -/// the scanner below still has to recognize it. The literal moved down here -/// with the Cursor diagnostics that are its only reader; `serve` itself stays -/// in the root crate. Root wiring: `src/serve.rs` re-exports this constant -/// instead of declaring its own copy. -pub const DEGRADED_SERVE_STDERR_MARKER: &str = - "[tracedecay] serve: staying alive in degraded MCP mode"; - /// How many of the newest Cursor log sessions to scan. Each session directory /// corresponds to one Cursor launch; older sessions describe long-fixed runs. const MAX_SESSIONS_SCANNED: usize = 3; @@ -48,8 +38,6 @@ pub(crate) struct CursorMcpLogFindings { /// Connection failures where the stdio shim reached the managed daemon, /// but the daemon socket reset or broke before initialization completed. pub daemon_transport_failures: usize, - /// Legacy degraded-mode marker lines retained in recent Cursor logs. - pub degraded_mode_notices: usize, /// Log files (newest session first) that contained at least one finding. pub affected_logs: Vec, /// Whether any Cursor MCP log was found at all (distinguishes "clean" @@ -62,7 +50,6 @@ impl CursorMcpLogFindings { self.literal_placeholder_lines > 0 || self.connection_failures > 0 || self.daemon_transport_failures > 0 - || self.degraded_mode_notices > 0 } } @@ -94,7 +81,6 @@ pub(crate) fn scan_cursor_mcp_logs(logs_root: &Path) -> CursorMcpLogFindings { .and_then(|name| name.to_str()) .is_some_and(|name| !name.contains("tracedecay")); let mut affected = false; - let stale_ambiguity = stale_degraded_ambiguity(&contents); for line in contents.lines() { if require_tracedecay_mention && !line.contains("tracedecay") { continue; @@ -114,10 +100,6 @@ pub(crate) fn scan_cursor_mcp_logs(logs_root: &Path) -> CursorMcpLogFindings { if daemon_transport_failure { findings.daemon_transport_failures += 1; } - if line.contains(DEGRADED_SERVE_STDERR_MARKER) && !stale_ambiguity { - findings.degraded_mode_notices += 1; - affected = true; - } } if affected { findings.affected_logs.push(log_path); @@ -127,25 +109,6 @@ pub(crate) fn scan_cursor_mcp_logs(logs_root: &Path) -> CursorMcpLogFindings { findings } -fn stale_degraded_ambiguity(contents: &str) -> bool { - let Some(ambiguity_start) = contents.find("Multiple tracedecay projects found") else { - return false; - }; - let ambiguity = &contents[ambiguity_start..]; - let mut paths = Vec::new(); - for line in ambiguity.lines().skip(1) { - if line.contains(DEGRADED_SERVE_STDERR_MARKER) { - break; - } - let trimmed = line.trim(); - let path = Path::new(trimmed); - if path.is_absolute() { - paths.push(path); - } - } - !paths.is_empty() && paths.iter().any(|path| !path.exists()) -} - /// Reports Cursor MCP log findings through the doctor counters, with the /// concrete remediation for Cursor's no-retry behavior. pub(crate) fn report_cursor_mcp_log_findings(dc: &mut DoctorCounters, home: &Path) { @@ -155,7 +118,6 @@ pub(crate) fn report_cursor_mcp_log_findings(dc: &mut DoctorCounters, home: &Pat findings.literal_placeholder_lines += scanned.literal_placeholder_lines; findings.connection_failures += scanned.connection_failures; findings.daemon_transport_failures += scanned.daemon_transport_failures; - findings.degraded_mode_notices += scanned.degraded_mode_notices; findings.scanned_any_log |= scanned.scanned_any_log; findings.affected_logs.extend(scanned.affected_logs); } @@ -192,14 +154,6 @@ pub(crate) fn report_cursor_mcp_log_findings(dc: &mut DoctorCounters, home: &Pat findings.daemon_transport_failures )); } - if findings.degraded_mode_notices > 0 { - dc.warn(&format!( - "found {} legacy tracedecay serve degraded-mode notice(s) in recent Cursor logs \ - (an older version failed project resolution at startup); run `tracedecay init` \ - in the affected project", - findings.degraded_mode_notices - )); - } dc.info( " After fixing the cause, toggle the tracedecay MCP server in Cursor Settings → MCP \ or reload the Cursor window. Cursor does not retry a failed MCP scope on its own.", @@ -323,7 +277,6 @@ mod tests { assert!(findings.scanned_any_log); assert_eq!(findings.literal_placeholder_lines, 1); assert_eq!(findings.connection_failures, 1); - assert_eq!(findings.degraded_mode_notices, 0); assert_eq!(findings.affected_logs.len(), 1); assert!(findings.has_findings()); } @@ -346,80 +299,6 @@ mod tests { assert!(findings.has_findings()); } - /// The scanner must match the exact marker older `serve` versions emitted; - /// [`DEGRADED_SERVE_STDERR_MARKER`] retains that legacy log contract. - #[test] - fn scan_detects_degraded_mode_notice() { - let logs = TempDir::new().unwrap(); - write_session_log( - logs.path(), - "20260702T030000", - "mcp-server-plugin-tracedecay-tracedecay.log", - &format!( - "2026-07-02 03:00:00.000 [warning] {DEGRADED_SERVE_STDERR_MARKER} — MCP \ - handshake will complete\n" - ), - ); - - let findings = scan_cursor_mcp_logs(logs.path()); - assert_eq!(findings.degraded_mode_notices, 1); - assert!(findings.has_findings()); - } - - #[test] - fn scan_ignores_stale_degraded_ambiguity_after_worktree_removed() { - let logs = TempDir::new().unwrap(); - let repo = logs.path().join("repo"); - let stale_worktree = logs.path().join("repo/.worktrees/codex-read-context"); - std::fs::create_dir_all(&repo).unwrap(); - write_session_log( - logs.path(), - "20260702T030000", - "mcp-server-plugin-tracedecay-tracedecay.log", - &format!( - "2026-07-02 03:00:00.000 [error] Error: config error: Multiple tracedecay \ - projects found — pass -p to select one:\n\ - {}\n\ - {}\n\ - {DEGRADED_SERVE_STDERR_MARKER} — MCP handshake will complete\n", - repo.display(), - stale_worktree.display() - ), - ); - - let findings = scan_cursor_mcp_logs(logs.path()); - assert!(findings.scanned_any_log); - assert_eq!(findings.degraded_mode_notices, 0); - assert!(!findings.has_findings(), "{findings:?}"); - } - - #[test] - fn scan_keeps_degraded_ambiguity_when_paths_still_exist() { - let logs = TempDir::new().unwrap(); - let repo = logs.path().join("repo"); - let worktree = logs.path().join("repo/.worktrees/codex-read-context"); - std::fs::create_dir_all(&repo).unwrap(); - std::fs::create_dir_all(&worktree).unwrap(); - write_session_log( - logs.path(), - "20260702T030000", - "mcp-server-plugin-tracedecay-tracedecay.log", - &format!( - "2026-07-02 03:00:00.000 [error] Error: config error: Multiple tracedecay \ - projects found — pass -p to select one:\n\ - {}\n\ - {}\n\ - {DEGRADED_SERVE_STDERR_MARKER} — MCP handshake will complete\n", - repo.display(), - worktree.display() - ), - ); - - let findings = scan_cursor_mcp_logs(logs.path()); - assert_eq!(findings.degraded_mode_notices, 1); - assert!(findings.has_findings()); - } - #[test] fn mcpprocess_log_only_counts_tracedecay_lines() { let logs = TempDir::new().unwrap(); diff --git a/crates/tracedecay-agent-hosts/src/agents/devin.rs b/crates/tracedecay-agent-hosts/src/agents/devin.rs index 31c8ee8f98..e77d79d18c 100644 --- a/crates/tracedecay-agent-hosts/src/agents/devin.rs +++ b/crates/tracedecay-agent-hosts/src/agents/devin.rs @@ -16,8 +16,8 @@ use tracedecay_domain::errors::{Result, TraceDecayError}; use super::host_bundle::HostBundleRegistrationStateV1; use super::{ AgentIntegration, DoctorCounters, HealthcheckContext, InstallContext, JsonConfigDialect, - McpDoctorLabels, TextFileMutation, config_backup_path, load_json_file, report_mcp_registration, - update_config_file_transactionally, + McpDoctorLabels, TextFileMutation, load_json_file, report_mcp_registration, + update_text_file_transactionally, }; pub struct DevinIntegration; @@ -36,10 +36,6 @@ fn devin_project_mcp_config_path(project_path: &Path) -> PathBuf { project_path.join(".devin/mcp_config.json") } -fn devin_original_config_path(config_path: &Path) -> PathBuf { - PathBuf::from(format!("{}.tracedecay-original", config_path.display())) -} - impl AgentIntegration for DevinIntegration { fn name(&self) -> &'static str { "Devin" @@ -120,12 +116,7 @@ impl AgentIntegration for DevinIntegration { home: &Path, ) -> Vec { if components == [super::host_bundle::HostComponentV1::ContextMcp] { - let path = devin_mcp_config_path(home); - vec![ - path.clone(), - config_backup_path(&path), - devin_original_config_path(&path), - ] + vec![devin_mcp_config_path(home)] } else { Vec::new() } @@ -138,12 +129,7 @@ impl AgentIntegration for DevinIntegration { project_path: &Path, ) -> Result> { if components == [super::host_bundle::HostComponentV1::ContextMcp] { - let path = devin_project_mcp_config_path(project_path); - Ok(vec![ - path.clone(), - config_backup_path(&path), - devin_original_config_path(&path), - ]) + Ok(vec![devin_project_mcp_config_path(project_path)]) } else { Ok(Vec::new()) } @@ -173,11 +159,7 @@ impl AgentIntegration for DevinIntegration { project_path: &Path, ) -> Result<()> { let config_path = devin_project_mcp_config_path(project_path); - let original_path = devin_original_config_path(&config_path); - super::ensure_project_local_safe_paths( - project_path, - [config_path.as_path(), original_path.as_path()], - )?; + super::ensure_project_local_safe_path(project_path, &config_path)?; install_mcp_if_selected(components, &config_path, ctx) } @@ -188,11 +170,7 @@ impl AgentIntegration for DevinIntegration { project_path: &Path, ) -> Result<()> { let config_path = devin_project_mcp_config_path(project_path); - let original_path = devin_original_config_path(&config_path); - super::ensure_project_local_safe_paths( - project_path, - [config_path.as_path(), original_path.as_path()], - )?; + super::ensure_project_local_safe_path(project_path, &config_path)?; uninstall_mcp_if_selected(components, &config_path) } @@ -278,8 +256,7 @@ fn install_mcp_if_selected( ), })?; } - let original_path = devin_original_config_path(config_path); - update_config_file_transactionally(config_path, |existing| { + update_text_file_transactionally(config_path, |existing| { let mut settings = JsonConfigDialect::Json.parse_for_edit(config_path, existing)?; if !settings.is_object() { return Err(TraceDecayError::Config { @@ -294,10 +271,6 @@ fn install_mcp_if_selected( message: format!("{}.mcpServers must be a JSON object", config_path.display()), }); } - let has_tracedecay = settings.pointer("/mcpServers/tracedecay").is_some(); - if !has_tracedecay && config_path.is_file() && !original_path.exists() { - super::safe_write_bytes_file(&original_path, existing.as_bytes(), None)?; - } settings["mcpServers"]["tracedecay"] = json!({ "command": ctx.tracedecay_bin.clone(), "args": ["serve"], @@ -306,7 +279,11 @@ fn install_mcp_if_selected( }); Ok(( (), - TextFileMutation::Write(super::render_json_config(config_path, &settings)?), + TextFileMutation::Write(JsonConfigDialect::Json.render_edit( + config_path, + existing, + &settings, + )?), )) })?; eprintln!( @@ -317,12 +294,6 @@ fn install_mcp_if_selected( Ok(()) } -enum DevinMcpRemoval { - NoEntry, - RestoredOriginal, - Rewritten, -} - fn uninstall_mcp_if_selected( components: &[super::host_bundle::HostComponentV1], config_path: &Path, @@ -332,56 +303,36 @@ fn uninstall_mcp_if_selected( eprintln!(" {} not found, skipping", config_path.display()); return Ok(()); } - let original_path = devin_original_config_path(config_path); - let outcome = update_config_file_transactionally(config_path, |existing| { + let removed = update_text_file_transactionally(config_path, |existing| { let mut settings = JsonConfigDialect::Json.parse_for_edit(config_path, existing)?; let Some(servers) = settings .get_mut("mcpServers") .and_then(serde_json::Value::as_object_mut) else { - return Ok((DevinMcpRemoval::NoEntry, TextFileMutation::Unchanged)); + return Ok((false, TextFileMutation::Unchanged)); }; if servers.remove("tracedecay").is_none() { - return Ok((DevinMcpRemoval::NoEntry, TextFileMutation::Unchanged)); - } - if let Ok(original) = std::fs::read(&original_path) - && serde_json::from_slice::(&original).ok() - == Some(settings.clone()) - { - let original = - String::from_utf8(original).map_err(|error| TraceDecayError::Config { - message: format!("{} is not valid UTF-8: {error}", original_path.display()), - })?; - return Ok(( - DevinMcpRemoval::RestoredOriginal, - TextFileMutation::Write(original), - )); + return Ok((false, TextFileMutation::Unchanged)); } Ok(( - DevinMcpRemoval::Rewritten, - TextFileMutation::Write(super::render_json_config(config_path, &settings)?), + true, + TextFileMutation::Write(JsonConfigDialect::Json.render_edit( + config_path, + existing, + &settings, + )?), )) })?; - match outcome { - DevinMcpRemoval::NoEntry => eprintln!( - " No tracedecay MCP server in {}, skipping", - config_path.display() - ), - DevinMcpRemoval::RestoredOriginal => { - super::safe_remove_host_file(&original_path).map_err(|error| { - TraceDecayError::Config { - message: format!("failed to remove {}: {error}", original_path.display()), - } - })?; - eprintln!( - "\x1b[32m✔\x1b[0m Restored original Devin configuration in {}", - config_path.display() - ); - } - DevinMcpRemoval::Rewritten => eprintln!( + if removed { + eprintln!( "\x1b[32m✔\x1b[0m Removed tracedecay MCP server from {}", config_path.display() - ), + ); + } else { + eprintln!( + " No tracedecay MCP server in {}, skipping", + config_path.display() + ); } } Ok(()) @@ -425,7 +376,6 @@ mod tests { let install = InstallContext { home: home.path().to_path_buf(), tracedecay_bin: "/tmp/tracedecay".to_string(), - tool_permissions: Vec::new(), project_root: None, dashboard: false, }; @@ -482,7 +432,6 @@ mod tests { let install = InstallContext { home: home.path().to_path_buf(), tracedecay_bin: "/tmp/tracedecay".to_string(), - tool_permissions: Vec::new(), project_root: Some(project.path().to_path_buf()), dashboard: false, }; @@ -513,7 +462,6 @@ mod tests { let install = InstallContext { home: home.path().to_path_buf(), tracedecay_bin: "/tmp/tracedecay-a".to_string(), - tool_permissions: Vec::new(), project_root: Some(project.path().to_path_buf()), dashboard: false, }; @@ -521,10 +469,6 @@ mod tests { DevinIntegration .activate_project_host_component_registration(&components, &install, project.path()) .unwrap(); - assert_eq!( - std::fs::read(devin_original_config_path(&config)).unwrap(), - original - ); let installed = load_json_file(&config); assert_eq!(installed["ui"]["theme"], "dark"); assert_eq!(installed["mcpServers"]["other"]["command"], "other-mcp"); @@ -540,7 +484,10 @@ mod tests { assert_eq!(removed["ui"]["theme"], "dark"); assert_eq!(removed["mcpServers"]["other"]["command"], "other-mcp"); assert!(removed["mcpServers"].get("tracedecay").is_none()); - assert_eq!(std::fs::read(&config).unwrap(), original); - assert!(!devin_original_config_path(&config).exists()); + let siblings: Vec<_> = std::fs::read_dir(config.parent().unwrap()) + .unwrap() + .map(|entry| entry.unwrap().file_name()) + .collect(); + assert_eq!(siblings, vec![std::ffi::OsString::from("mcp_config.json")]); } } diff --git a/crates/tracedecay-agent-hosts/src/agents/gemini.rs b/crates/tracedecay-agent-hosts/src/agents/gemini.rs index 7b8f0053e6..1c687d2c79 100644 --- a/crates/tracedecay-agent-hosts/src/agents/gemini.rs +++ b/crates/tracedecay-agent-hosts/src/agents/gemini.rs @@ -6,7 +6,8 @@ //! server entry, a context file, and commands. TraceDecay therefore **adopts //! that lifecycle** instead of configuring Gemini by hand: //! -//! 1. TraceDecay stages an extension source it owns outright under +//! 1. The receipt-backed component transaction deploys an extension source +//! TraceDecay owns outright under //! `~/.gemini/tracedecay-extension/`, a `gemini-extension.json` manifest //! naming the tracedecay MCP server (`args: ["serve"]`, `trust: true`, the //! resolved binary substituted through a placeholder) plus the extension's @@ -23,22 +24,16 @@ //! requirement for the lifecycle for the same reason: there is no //! config-editing fallback, only a typed refusal naming the missing binary. //! -//! Because the doctor previously failed on a *missing* `mcpServers.tracedecay` -//! entry in `~/.gemini/settings.json`, it would now report a defect for the -//! correct state. Its checks below were re-pointed at what is actually true -//! under the extension model: the staged source, the host's installed -//! extension copy, and, when the binary is present, Gemini's own -//! `gemini extensions list`. A settings entry is now reported as *legacy -//! residue*, not as the required registration. +//! The doctor therefore checks what the extension model makes true: the +//! staged source, the host's installed extension copy, and, when the binary +//! is present, Gemini's own `gemini extensions list`. It never requires an +//! `mcpServers.tracedecay` entry in `~/.gemini/settings.json`. use std::path::{Path, PathBuf}; use tracedecay_domain::errors::Result; -use super::{ - AgentIntegration, DeferredUserAction, DoctorCounters, HealthcheckContext, InstallContext, - NonInteractiveInstallOutcome, UpdatePluginOutcome, load_json_file, -}; +use super::{AgentIntegration, DoctorCounters, HealthcheckContext, InstallContext, load_json_file}; mod extension; @@ -49,12 +44,11 @@ mod extension; pub(crate) use extension::{GEMINI_STAGED_EXTENSION_RELATIVE, rendered_extension_files}; use extension::{ - EXTENSION_CONTEXT_FILE, EXTENSION_NAME, InstalledExtensionV1, MCP_SERVER_NAME, - deploy_extension_bundle, extension_stage_dir, gemini_extension_activate_with, - gemini_extension_deactivate_with, host_reported_extensions, installed_extension_dir, + EXTENSION_NAME, InstalledExtensionV1, MCP_SERVER_NAME, extension_stage_dir, + gemini_extension_activate_with, gemini_extension_deactivate_with, host_reported_extensions, installed_extension_is_current, installed_extension_is_present, installed_manifest_path, manifest_declares_current_server, read_installed_extension, require_gemini_cli, settings_path, - stage_dir_is_tracedecay, staged_context_path, staged_manifest_path, user_context_path, + stage_dir_is_tracedecay, staged_context_path, staged_manifest_path, }; pub struct GeminiIntegration; @@ -81,41 +75,6 @@ impl AgentIntegration for GeminiIntegration { false } - /// Read-only readiness: has Gemini already adopted an extension matching - /// what this version would stage? Nothing is written here, and the host - /// CLI is not required, an absent binary only becomes a hard failure once - /// a lifecycle actually needs to run. - fn preflight_non_interactive_install( - &self, - ctx: &InstallContext, - ) -> Result { - Ok(gemini_extension_install_state( - &ctx.home, - &ctx.tracedecay_bin, - Vec::new(), - )) - } - - /// Stage the extension source and drive Gemini's own install command. - /// - /// The returned outcome is recomputed from the host's installed copy - /// *after* the command: a clean exit that did not leave an installed - /// extension where Gemini keeps them is reported as still-deferred rather - /// than claimed as an activation TraceDecay never observed. - fn prepare_non_interactive_install( - &self, - ctx: &InstallContext, - ) -> Result { - let stage_dir = deploy_extension_bundle(&ctx.home, &ctx.tracedecay_bin)?; - let gemini = require_gemini_cli()?; - gemini_extension_activate_with(&gemini, &ctx.home)?; - Ok(gemini_extension_install_state( - &ctx.home, - &ctx.tracedecay_bin, - vec![stage_dir], - )) - } - fn activate_deployed_host_registration(&self, ctx: &InstallContext) -> Result<()> { if installed_extension_is_current(&ctx.home, Some(&ctx.tracedecay_bin)) { return Ok(()); @@ -135,36 +94,11 @@ impl AgentIntegration for GeminiIntegration { gemini_extension_deactivate_with(&gemini, &ctx.home) } - /// Refresh the staged extension source (the only generated artifact: it - /// bakes the crate version and the resolved binary path). - /// - /// Gemini owns the installed copy, so refreshing the source alone cannot - /// honestly report an updated extension, the adoption step is reported as - /// a deferred host action instead of silently claimed. - fn update_plugin(&self, ctx: &InstallContext) -> Result { - if !staged_manifest_path(&ctx.home).exists() { - return Ok(UpdatePluginOutcome::NotInstalled); - } - let stage_dir = deploy_extension_bundle(&ctx.home, &ctx.tracedecay_bin)?; - Ok(UpdatePluginOutcome::DeferredUserAction( - DeferredUserAction { - remediation: format!( - "Gemini CLI extension source is staged. Run \ - `gemini extensions update {EXTENSION_NAME}` (or re-run \ - `tracedecay install --agent gemini`) so Gemini CLI adopts the refreshed source." - ), - staged_paths: vec![stage_dir], - }, - )) - } - fn healthcheck(&self, dc: &mut DoctorCounters, ctx: &HealthcheckContext) { eprintln!("\n\x1b[1mGemini CLI integration\x1b[0m"); doctor_check_staged_extension(dc, &ctx.home); doctor_check_installed_extension(dc, &ctx.home); doctor_check_host_reported_extensions(dc, &ctx.home); - doctor_check_settings(dc, &ctx.home); - doctor_check_prompt(dc, &ctx.home); } /// Read-only registration state, observed from the host's installed @@ -241,30 +175,6 @@ impl AgentIntegration for GeminiIntegration { // Lifecycle state // --------------------------------------------------------------------------- -/// Whether the host has adopted a current tracedecay extension, expressed as -/// the non-interactive install outcome the lifecycle expects. -fn gemini_extension_install_state( - home: &Path, - tracedecay_bin: &str, - staged_paths: Vec, -) -> NonInteractiveInstallOutcome { - if installed_extension_is_current(home, Some(tracedecay_bin)) { - return NonInteractiveInstallOutcome::Ready; - } - let stage_dir = extension_stage_dir(home); - NonInteractiveInstallOutcome::DeferredUserAction(DeferredUserAction { - remediation: format!( - "Gemini CLI owns extension registration and the installed copy. TraceDecay could not \ - observe a current tracedecay extension at {}. Run `gemini extensions install {}` \ - (uninstall an older one first with `gemini extensions uninstall {EXTENSION_NAME}`), \ - then re-run TraceDecay.", - installed_extension_dir(home).display(), - stage_dir.display() - ), - staged_paths, - }) -} - /// Registration state from the host's installed extension alone. fn gemini_extension_registration_state( home: &Path, @@ -463,60 +373,6 @@ fn doctor_check_host_reported_extensions(dc: &mut DoctorCounters, home: &Path) { } } -/// `~/.gemini/settings.json` is no longer where tracedecay is registered: the -/// extension supplies the MCP server. A surviving `mcpServers.tracedecay` -/// entry is pre-extension residue, and reporting its *absence* as a failure, -/// as this check once did, would now be a lie. -fn doctor_check_settings(dc: &mut DoctorCounters, home: &Path) { - let settings = settings_path(home); - if !settings.exists() { - dc.pass(&format!( - "{} has no legacy tracedecay MCP entry (the extension supplies the server)", - settings.display() - )); - return; - } - let has_legacy_entry = load_json_file(&settings) - .get("mcpServers") - .and_then(|servers| servers.get("tracedecay")) - .is_some(); - if has_legacy_entry { - dc.warn(&format!( - "{} still declares mcpServers.tracedecay from the pre-extension install; the \ - extension now supplies that server. Remove the entry so Gemini does not load two \ - tracedecay servers", - settings.display() - )); - } else { - dc.pass(&format!( - "{} has no legacy tracedecay MCP entry (the extension supplies the server)", - settings.display() - )); - } -} - -/// The extension carries its own context file, so the operator's -/// `~/.gemini/GEMINI.md` is expected *not* to contain tracedecay rules. A -/// managed block there is residue from the marker-append era. -fn doctor_check_prompt(dc: &mut DoctorCounters, home: &Path) { - let user_context = user_context_path(home); - let has_legacy_block = std::fs::read_to_string(&user_context) - .is_ok_and(|contents| contents.contains(super::prompt_rules::PROMPT_RULE_MARKER)); - if has_legacy_block { - dc.warn(&format!( - "{} still contains the tracedecay rules block appended by the pre-extension \ - install; the extension now ships its own {EXTENSION_CONTEXT_FILE}. Remove the block \ - to avoid duplicated rules", - user_context.display() - )); - } else { - dc.pass(&format!( - "{} carries no TraceDecay-managed block (the extension ships its own context file)", - user_context.display() - )); - } -} - #[cfg(test)] #[allow(clippy::unwrap_used, clippy::expect_used)] mod tests; diff --git a/crates/tracedecay-agent-hosts/src/agents/gemini/extension.rs b/crates/tracedecay-agent-hosts/src/agents/gemini/extension.rs index ed6fd2f95a..8245e2f17c 100644 --- a/crates/tracedecay-agent-hosts/src/agents/gemini/extension.rs +++ b/crates/tracedecay-agent-hosts/src/agents/gemini/extension.rs @@ -31,9 +31,7 @@ use serde_json::json; use tracedecay_domain::errors::{Result, TraceDecayError}; -use crate::agents::{ - host_cli, load_json_file, record_host_config_observation_bytes, safe_write_text_file, -}; +use crate::agents::{host_cli, load_json_file, record_host_config_observation_bytes}; /// Name of Gemini CLI's lifecycle binary. pub(super) const GEMINI_CLI: &str = "gemini"; @@ -140,12 +138,6 @@ pub(super) fn settings_path(home: &Path) -> PathBuf { gemini_home(home).join("settings.json") } -/// The operator's own global context file. Read-only for this integration; a -/// tracedecay block here is legacy residue from the pre-extension model. -pub(super) fn user_context_path(home: &Path) -> PathBuf { - gemini_home(home).join(EXTENSION_CONTEXT_FILE) -} - // --------------------------------------------------------------------------- // Staging // --------------------------------------------------------------------------- @@ -218,24 +210,6 @@ fn context_file_text() -> String { ) } -/// Render the extension source into its stable staging directory and report -/// that directory. A clean replace, so a file a previous version staged but -/// this one no longer ships cannot linger into the next `gemini extensions -/// install`. -#[hotpath::measure(label = "hosts.agent.gemini.extension_deploy")] -pub(super) fn deploy_extension_bundle(home: &Path, tracedecay_bin: &str) -> Result { - let stage_dir = extension_stage_dir(home); - clean_replace_owned_stage_dir(&stage_dir)?; - for (relative, rendered) in rendered_extension_files(tracedecay_bin)? { - safe_write_text_file(&stage_dir.join(relative), &rendered, None)?; - } - eprintln!( - "\x1b[32m✔\x1b[0m Staged tracedecay Gemini extension in {}", - stage_dir.display() - ); - Ok(stage_dir) -} - /// True when a staging directory is tracedecay-owned: its manifest names the /// tracedecay extension. A missing directory is trivially safe to write into. pub(super) fn stage_dir_is_tracedecay(stage_dir: &Path) -> bool { @@ -245,27 +219,6 @@ pub(super) fn stage_dir_is_tracedecay(stage_dir: &Path) -> bool { == Some(EXTENSION_NAME) } -/// Remove the tracedecay-owned staging directory so the next write is a clean -/// replace. No-op when it is missing; refuses when it exists but is not -/// tracedecay-owned, so a directory squatting on the path, an operator's own -/// hand-written extension source, say, is never deleted. -fn clean_replace_owned_stage_dir(stage_dir: &Path) -> Result<()> { - if !stage_dir.exists() { - return Ok(()); - } - if !stage_dir_is_tracedecay(stage_dir) { - return Err(TraceDecayError::Config { - message: format!( - "refusing to replace non-tracedecay Gemini extension directory {}", - stage_dir.display() - ), - }); - } - std::fs::remove_dir_all(stage_dir).map_err(|error| TraceDecayError::Config { - message: format!("failed to remove {}: {error}", stage_dir.display()), - }) -} - // --------------------------------------------------------------------------- // Installed-extension observation // --------------------------------------------------------------------------- diff --git a/crates/tracedecay-agent-hosts/src/agents/gemini/tests.rs b/crates/tracedecay-agent-hosts/src/agents/gemini/tests.rs index b09a844cbb..0b41050f37 100644 --- a/crates/tracedecay-agent-hosts/src/agents/gemini/tests.rs +++ b/crates/tracedecay-agent-hosts/src/agents/gemini/tests.rs @@ -7,7 +7,7 @@ use super::extension::{ EXTENSION_CONTEXT_FILE, EXTENSION_MANIFEST_FILE, TRACEDECAY_BIN_PLACEHOLDER, - rendered_extension_files, + installed_extension_dir, rendered_extension_files, }; use super::*; @@ -17,6 +17,16 @@ use tracedecay_domain::errors::TraceDecayError; // Fixtures // --------------------------------------------------------------------------- +/// Writes the rendered extension source exactly where the component catalog +/// deploys it. +fn stage_rendered_extension(home: &Path, tracedecay_bin: &str) -> PathBuf { + let stage_dir = extension_stage_dir(home); + for (relative, rendered) in rendered_extension_files(tracedecay_bin).unwrap() { + crate::agents::safe_write_text_file(&stage_dir.join(relative), &rendered).unwrap(); + } + stage_dir +} + /// Install a fake `gemini` that appends each invocation's argv to `log` and /// then performs `body`. #[cfg(unix)] @@ -68,7 +78,6 @@ fn install_context(home: &Path, tracedecay_bin: &str) -> InstallContext { InstallContext { home: home.to_path_buf(), tracedecay_bin: tracedecay_bin.to_string(), - tool_permissions: Vec::new(), project_root: None, dashboard: true, } @@ -99,7 +108,7 @@ fn simulate_host_install(home: &Path, tracedecay_bin: &str) { fn staging_renders_the_manifest_with_the_admitted_binary_serve_args_and_trust() { let home = tempfile::tempdir().unwrap(); - let stage_dir = deploy_extension_bundle(home.path(), "/abs/bin/tracedecay").unwrap(); + let stage_dir = stage_rendered_extension(home.path(), "/abs/bin/tracedecay"); assert_eq!(stage_dir, extension_stage_dir(home.path())); let raw = std::fs::read_to_string(stage_dir.join(EXTENSION_MANIFEST_FILE)).unwrap(); @@ -142,7 +151,7 @@ fn staging_escapes_special_chars_in_the_binary_path() { let home = tempfile::tempdir().unwrap(); let weird_bin = "/opt/td \"quote\"/tracedecay"; - let stage_dir = deploy_extension_bundle(home.path(), weird_bin).unwrap(); + let stage_dir = stage_rendered_extension(home.path(), weird_bin); let manifest: serde_json::Value = serde_json::from_str( &std::fs::read_to_string(stage_dir.join(EXTENSION_MANIFEST_FILE)) @@ -152,56 +161,6 @@ fn staging_escapes_special_chars_in_the_binary_path() { assert_eq!(manifest["mcpServers"]["tracedecay"]["command"], weird_bin); } -/// Re-staging is a clean replace: a file an older version staged but this one -/// no longer ships must not survive into the next `gemini extensions install`. -#[test] -fn staging_is_a_clean_replace_dropping_stale_files() { - let home = tempfile::tempdir().unwrap(); - let stage_dir = deploy_extension_bundle(home.path(), "/bin/tracedecay").unwrap(); - let stale = stage_dir.join("commands/retired.toml"); - std::fs::create_dir_all(stale.parent().unwrap()).unwrap(); - std::fs::write(&stale, "stale command").unwrap(); - - deploy_extension_bundle(home.path(), "/bin/tracedecay").unwrap(); - - assert!( - !stale.exists(), - "a stale staged file must be gone after a clean-replace restage" - ); - assert!(stage_dir.join(EXTENSION_MANIFEST_FILE).exists()); -} - -/// The clean replace must refuse a directory TraceDecay does not own, so an -/// operator's own extension source squatting on the path is never deleted. -#[test] -fn staging_refuses_to_replace_a_directory_tracedecay_does_not_own() { - let home = tempfile::tempdir().unwrap(); - let stage_dir = extension_stage_dir(home.path()); - std::fs::create_dir_all(&stage_dir).unwrap(); - std::fs::write( - stage_dir.join(EXTENSION_MANIFEST_FILE), - r#"{"name":"someone-elses-extension"}"#, - ) - .unwrap(); - std::fs::write(stage_dir.join("user-file.txt"), "keep me").unwrap(); - - let error = deploy_extension_bundle(home.path(), "/bin/tracedecay") - .expect_err("a non-tracedecay extension directory must not be replaced"); - - assert!( - error.to_string().contains("non-tracedecay"), - "unexpected error: {error}" - ); - assert!( - stage_dir.join("user-file.txt").exists(), - "an unowned directory must be left untouched" - ); - assert_eq!( - std::fs::read_to_string(stage_dir.join(EXTENSION_MANIFEST_FILE)).unwrap(), - r#"{"name":"someone-elses-extension"}"# - ); -} - // --------------------------------------------------------------------------- // Host-CLI-driven lifecycle // --------------------------------------------------------------------------- @@ -215,7 +174,7 @@ fn activation_drives_the_hosts_own_extension_install_against_the_staged_source() let bin_dir = tempfile::tempdir().unwrap(); let log = bin_dir.path().join("invocations.log"); let gemini = bin_dir.path().join("gemini"); - let stage_dir = deploy_extension_bundle(home.path(), "/bin/tracedecay").unwrap(); + let stage_dir = stage_rendered_extension(home.path(), "/bin/tracedecay"); fake_gemini_cli(&gemini, &log, FAKE_EXTENSION_LIFECYCLE_BODY); gemini_extension_activate_with(&gemini, home.path()) @@ -247,7 +206,7 @@ fn activation_removes_an_existing_extension_through_the_host_before_reinstalling let bin_dir = tempfile::tempdir().unwrap(); let log = bin_dir.path().join("invocations.log"); let gemini = bin_dir.path().join("gemini"); - let stage_dir = deploy_extension_bundle(home.path(), "/relocated/tracedecay").unwrap(); + let stage_dir = stage_rendered_extension(home.path(), "/relocated/tracedecay"); simulate_host_install(home.path(), "/old/tracedecay"); fake_gemini_cli(&gemini, &log, FAKE_EXTENSION_LIFECYCLE_BODY); @@ -298,7 +257,7 @@ fn deactivation_drives_the_hosts_own_uninstall_by_extension_name() { let bin_dir = tempfile::tempdir().unwrap(); let log = bin_dir.path().join("invocations.log"); let gemini = bin_dir.path().join("gemini"); - deploy_extension_bundle(home.path(), "/bin/tracedecay").unwrap(); + stage_rendered_extension(home.path(), "/bin/tracedecay"); simulate_host_install(home.path(), "/bin/tracedecay"); fake_gemini_cli(&gemini, &log, FAKE_EXTENSION_LIFECYCLE_BODY); @@ -335,7 +294,7 @@ fn registration_state_follows_the_hosts_installed_extension() { home: home.path().to_path_buf(), project_path: home.path().to_path_buf(), }; - deploy_extension_bundle(home.path(), "/bin/tracedecay").unwrap(); + stage_rendered_extension(home.path(), "/bin/tracedecay"); assert!( !GeminiIntegration.has_tracedecay(home.path()), "a staged source the host never installed is not an installation" @@ -351,11 +310,9 @@ fn registration_state_follows_the_hosts_installed_extension() { GeminiIntegration.host_component_registration(HostComponentV1::ContextMcp, &health), HostBundleRegistrationStateV1::Current ); - assert!(matches!( - GeminiIntegration - .preflight_non_interactive_install(&install_context(home.path(), "/bin/tracedecay")) - .unwrap(), - NonInteractiveInstallOutcome::Ready + assert!(installed_extension_is_current( + home.path(), + Some("/bin/tracedecay") )); // A relocated binary is only visible to the lifecycle-aware readback. @@ -367,14 +324,9 @@ fn registration_state_follows_the_hosts_installed_extension() { ), HostBundleRegistrationStateV1::Repairable ); - assert!(matches!( - GeminiIntegration - .preflight_non_interactive_install(&install_context( - home.path(), - "/relocated/tracedecay" - )) - .unwrap(), - NonInteractiveInstallOutcome::DeferredUserAction(_) + assert!(!installed_extension_is_current( + home.path(), + Some("/relocated/tracedecay") )); std::fs::write(installed_manifest_path(home.path()), b"{not json").unwrap(); @@ -384,21 +336,17 @@ fn registration_state_follows_the_hosts_installed_extension() { ); } -/// The doctor must not fail on the *correct* post-adoption state: under the -/// extension model `~/.gemini/settings.json` carries no tracedecay entry and -/// `~/.gemini/GEMINI.md` carries no managed block, because the extension -/// supplies both. +/// The doctor must not fail on the *correct* post-adoption state: the +/// extension supplies the server and its context file. #[test] fn doctor_reports_no_issue_when_the_extension_supplies_the_server() { let home = tempfile::tempdir().unwrap(); - deploy_extension_bundle(home.path(), "/bin/tracedecay").unwrap(); + stage_rendered_extension(home.path(), "/bin/tracedecay"); simulate_host_install(home.path(), "/bin/tracedecay"); let mut dc = DoctorCounters::new(); doctor_check_staged_extension(&mut dc, home.path()); doctor_check_installed_extension(&mut dc, home.path()); - doctor_check_settings(&mut dc, home.path()); - doctor_check_prompt(&mut dc, home.path()); assert_eq!( dc.issues, 0, @@ -407,38 +355,6 @@ fn doctor_reports_no_issue_when_the_extension_supplies_the_server() { assert_eq!(dc.warnings, 0); } -/// Pre-extension state is reported as residue, a warning about duplication, -/// never as the registration the doctor is looking for. -#[test] -fn doctor_reports_pre_extension_state_as_residue() { - let home = tempfile::tempdir().unwrap(); - let settings = settings_path(home.path()); - std::fs::create_dir_all(settings.parent().unwrap()).unwrap(); - std::fs::write( - &settings, - br#"{"mcpServers":{"tracedecay":{"command":"/old/tracedecay","args":["serve"]}}}"#, - ) - .unwrap(); - std::fs::write( - user_context_path(home.path()), - format!( - "{}\n\nold rules\n", - super::super::prompt_rules::PROMPT_RULE_MARKER - ), - ) - .unwrap(); - - let mut dc = DoctorCounters::new(); - doctor_check_settings(&mut dc, home.path()); - doctor_check_prompt(&mut dc, home.path()); - - assert_eq!( - dc.issues, 0, - "legacy residue is a duplication warning, not a failed registration" - ); - assert_eq!(dc.warnings, 2, "both residues must be reported"); -} - /// A staged source that is missing its manifest is reported as "not staged", /// and a not-yet-adopted extension is reported as not installed, neither is /// silently upgraded into a claim that Gemini has the extension. @@ -500,35 +416,3 @@ fn doctor_fails_when_a_present_gemini_cli_is_not_executable() { assert_eq!(dc.issues, 1); assert_eq!(dc.warnings, 0); } - -/// `update_plugin` refreshes only the TraceDecay-owned source and says so: -/// Gemini owns the installed copy, so an unadopted refresh must not be -/// reported as an updated extension. -#[test] -fn update_refreshes_the_staged_source_and_defers_host_adoption() { - let home = tempfile::tempdir().unwrap(); - assert!(matches!( - GeminiIntegration - .update_plugin(&install_context(home.path(), "/bin/tracedecay")) - .unwrap(), - UpdatePluginOutcome::NotInstalled - )); - - deploy_extension_bundle(home.path(), "/old/tracedecay").unwrap(); - let outcome = GeminiIntegration - .update_plugin(&install_context(home.path(), "/new/tracedecay")) - .unwrap(); - - let UpdatePluginOutcome::DeferredUserAction(deferred) = outcome else { - panic!("a refreshed source the host has not adopted is a deferred action"); - }; - assert!(deferred.remediation.contains("gemini extensions update")); - assert_eq!( - deferred.staged_paths, - vec![extension_stage_dir(home.path())] - ); - assert_eq!( - read_json(&staged_manifest_path(home.path()))["mcpServers"]["tracedecay"]["command"], - "/new/tracedecay" - ); -} diff --git a/crates/tracedecay-agent-hosts/src/agents/hermes.rs b/crates/tracedecay-agent-hosts/src/agents/hermes.rs index 7c3ccf65a9..83017b6d23 100644 --- a/crates/tracedecay-agent-hosts/src/agents/hermes.rs +++ b/crates/tracedecay-agent-hosts/src/agents/hermes.rs @@ -12,7 +12,6 @@ use std::io::ErrorKind; use std::path::{Path, PathBuf}; use crate::ports::mcp_tools::{AdvertisedToolV1, advertised_tools}; -pub use profile_config::read_config_pinned_project_root; use profile_config::{disable_plugin, enable_plugin}; use tracedecay_domain::errors::{Result, TraceDecayError}; @@ -76,9 +75,7 @@ impl AgentIntegration for HermesIntegration { let Some(profile_dir) = plugin_dir.parent().and_then(Path::parent) else { continue; }; - let config = profile_dir.join("config.yaml"); - paths.push(config.clone()); - paths.push(profile_config::original_config_path(&config)); + paths.push(profile_dir.join("config.yaml")); paths.extend(dashboard_wrapper::managed_paths(&plugin_dir)); if plugin_dir != default_plugin { paths.extend(managed_plugin_paths(&plugin_dir)); @@ -252,7 +249,7 @@ pub(super) fn activate_deployed_plugin_profile( .into_iter() .zip(deployed_files) { - super::safe_write_bytes_file(&path, &contents, None)?; + super::safe_write_bytes_file(&path, &contents)?; } } dashboard_wrapper::apply_install_policy(plugin_dir, tracedecay_bin, deploy_dashboard)?; @@ -568,7 +565,7 @@ fn reconcile_managed_skill_overlay(profile_root: &Path, plugin_dir: &Path) -> Re .into_iter() .collect::>(); for (path, bytes) in &desired { - super::safe_write_bytes_file(path, bytes, None)?; + super::safe_write_bytes_file(path, bytes)?; } for path in existing { if !desired.contains_key(&path) { @@ -632,7 +629,7 @@ pub(super) fn write_text_file(path: &Path, contents: &str) -> Result<()> { if current == contents { return Ok(()); } - super::safe_write_text_file(path, contents, None) + super::safe_write_text_file(path, contents) } pub(super) fn remove_generated_file(path: &Path) -> Result<()> { diff --git a/crates/tracedecay-agent-hosts/src/agents/hermes/dashboard_wrapper.rs b/crates/tracedecay-agent-hosts/src/agents/hermes/dashboard_wrapper.rs index 7b0306c36e..1af804ee2d 100644 --- a/crates/tracedecay-agent-hosts/src/agents/hermes/dashboard_wrapper.rs +++ b/crates/tracedecay-agent-hosts/src/agents/hermes/dashboard_wrapper.rs @@ -33,16 +33,6 @@ const WRAPPER_ENTRY_JS: &str = include_str!("../../../../../dashboard/hermes-wra /// Placeholder line in `plugin_api.py` rewritten with the installed binary. const BIN_PLACEHOLDER: &str = "DEPLOYED_TRACEDECAY_BIN = None"; -/// Basenames of retired generated `dist/` assets: current installs must not -/// contain them, and deploy/uninstall both remove them. -const RETIRED_DIST_FILES: [&str; 5] = [ - "holographic.js", - "lcm.js", - "graph.js", - "savings.js", - "style.css", -]; - pub(super) fn is_current(plugin_dir: &Path) -> bool { [ "dashboard/manifest.json", @@ -51,9 +41,6 @@ pub(super) fn is_current(plugin_dir: &Path) -> bool { ] .into_iter() .all(|relative| plugin_dir.join(relative).is_file()) - && RETIRED_DIST_FILES - .into_iter() - .all(|retired| !plugin_dir.join("dashboard/dist").join(retired).exists()) } pub(super) fn is_absent(plugin_dir: &Path) -> bool { @@ -78,11 +65,6 @@ pub(super) fn managed_paths(plugin_dir: &Path) -> Vec { ] .into_iter() .map(|relative| plugin_dir.join(relative)) - .chain( - RETIRED_DIST_FILES - .into_iter() - .map(|retired| plugin_dir.join("dashboard/dist").join(retired)), - ) .collect() } @@ -120,9 +102,6 @@ fn deploy(plugin_dir: &Path, tracedecay_bin: &str) -> Result<()> { &plugin_api(tracedecay_bin)?, )?; super::write_text_file(&dist_dir.join("index.js"), WRAPPER_ENTRY_JS)?; - for retired in RETIRED_DIST_FILES { - super::remove_generated_file(&dist_dir.join(retired))?; - } tracing::debug!( dashboard_dir = %dashboard_dir.display(), @@ -140,9 +119,7 @@ pub(super) fn uninstall(plugin_dir: &Path) -> Result<()> { return Ok(()); } let dist_dir = dashboard_dir.join("dist"); - for file in std::iter::once("index.js").chain(RETIRED_DIST_FILES) { - super::remove_generated_file(&dist_dir.join(file))?; - } + super::remove_generated_file(&dist_dir.join("index.js"))?; super::remove_empty_dir(&dist_dir)?; super::remove_generated_file(&dashboard_dir.join("manifest.json"))?; super::remove_generated_file(&dashboard_dir.join("plugin_api.py"))?; @@ -334,8 +311,6 @@ mod tests { let temp = TempDir::new().unwrap(); let plugin_dir = temp.path().join(".hermes/plugins/tracedecay"); apply_install_policy(&plugin_dir, "/old/bin/tracedecay", true).unwrap(); - let retired = plugin_dir.join("dashboard/dist/holographic.js"); - std::fs::write(&retired, "retired generated asset").unwrap(); apply_install_policy(&plugin_dir, "/new/bin/tracedecay", true).unwrap(); @@ -343,7 +318,6 @@ mod tests { assert!(api.contains("/new/bin/tracedecay")); assert!(!api.contains("/old/bin/tracedecay")); assert!(!api.contains("DEPLOYED_PROJECT_ROOT")); - assert!(!retired.exists()); } #[test] diff --git a/crates/tracedecay-agent-hosts/src/agents/hermes/lifecycle.rs b/crates/tracedecay-agent-hosts/src/agents/hermes/lifecycle.rs index 8be5b8aa8d..381023623e 100644 --- a/crates/tracedecay-agent-hosts/src/agents/hermes/lifecycle.rs +++ b/crates/tracedecay-agent-hosts/src/agents/hermes/lifecycle.rs @@ -51,7 +51,6 @@ mod tests { InstallContext { home: home.to_path_buf(), tracedecay_bin: tracedecay_bin.to_string(), - tool_permissions: crate::agents::expected_tool_perms().expect("tool catalog"), project_root: None, dashboard, } diff --git a/crates/tracedecay-agent-hosts/src/agents/hermes/profile_config.rs b/crates/tracedecay-agent-hosts/src/agents/hermes/profile_config.rs index 3bf96234ee..52206e6e91 100644 --- a/crates/tracedecay-agent-hosts/src/agents/hermes/profile_config.rs +++ b/crates/tracedecay-agent-hosts/src/agents/hermes/profile_config.rs @@ -6,12 +6,12 @@ //! flows have explicit inputs and preserve the historical error messages. use std::io::ErrorKind; -use std::path::{Path, PathBuf}; +use std::path::Path; use std::str::FromStr; use yaml_edit::{Document, Mapping, Sequence, YamlNode}; -use crate::agents::{backup_config_file, safe_write_bytes_file}; +use crate::agents::safe_write_bytes_file; use tracedecay_domain::errors::{Result, TraceDecayError}; #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -68,16 +68,6 @@ impl ProfileConfigDocument { } } -/// Reads the removed `plugins.tracedecay.project_root` setting solely as -/// provenance for one-time data migration and transcript import. -pub fn read_config_pinned_project_root(config_path: &Path) -> Option { - let contents = std::fs::read_to_string(config_path).ok()?; - let config = ProfileConfigDocument::parse(&contents).ok()?; - let plugins = config.root().get_mapping("plugins")?; - let tracedecay = plugins.get_mapping("tracedecay")?; - string_value(&tracedecay, "project_root") -} - pub(super) fn registration_state( config_path: &Path, ) -> crate::agents::host_bundle::HostBundleRegistrationStateV1 { @@ -121,10 +111,6 @@ pub(super) fn enable_plugin(config_path: &Path) -> Result { }); } }; - let original_path = original_config_path(config_path); - if !existing.is_empty() && config_path.is_file() && !original_path.exists() { - crate::agents::safe_write_bytes_file(&original_path, existing.as_bytes(), None)?; - } let updated = enable_plugin_config(&existing).map_err(|message| TraceDecayError::Config { message: format!( "{message} in {}.\nFix the config by hand, then re-run: tracedecay install --agent hermes", @@ -147,29 +133,12 @@ pub(super) fn disable_plugin(config_path: &Path) -> Result<()> { config_path.display() ), })?; - let original_path = original_config_path(config_path); - if let Ok(original) = std::fs::read(&original_path) - && std::str::from_utf8(&original) - .is_ok_and(|original| updated.trim_end() == original.trim_end()) - { - crate::agents::safe_write_bytes_file(config_path, &original, None)?; - crate::agents::safe_remove_host_file(&original_path).map_err(|error| { - TraceDecayError::Config { - message: format!("failed to remove {}: {error}", original_path.display()), - } - })?; - return Ok(()); - } if updated != existing { write_config_file(config_path, &updated)?; } Ok(()) } -pub(super) fn original_config_path(config_path: &Path) -> PathBuf { - PathBuf::from(format!("{}.tracedecay-original", config_path.display())) -} - // Error messages preserved from the historical line-oriented implementation so // install/update surfaces stay stable. const PLUGINS_ERR: &str = "unsupported Hermes plugins config"; @@ -204,8 +173,7 @@ fn disable_plugin_config(existing: &str) -> std::result::Result } fn enable_normalized(existing: &str) -> std::result::Result { - let text = remove_legacy_project_pin(existing)?; - let text = remove_seq_item(&text, &["plugins", "disabled"], "tracedecay")?; + let text = remove_seq_item(existing, &["plugins", "disabled"], "tracedecay")?; let text = ensure_enabled(&text)?; let text = ensure_scalar( &text, @@ -230,7 +198,6 @@ fn enable_normalized(existing: &str) -> std::result::Result { fn disable_normalized(existing: &str) -> std::result::Result { let text = remove_seq_item(existing, &["plugins", "enabled"], "tracedecay")?; - let text = remove_legacy_project_pin(&text)?; let text = disable_scalar(&text, "context", "engine")?; let text = disable_scalar(&text, "memory", "provider")?; Ok(text) @@ -478,43 +445,6 @@ fn insert_block_child( // ---- removals ---- -/// Remove the legacy `plugins.tracedecay.project_root` pin, collapsing an -/// otherwise-empty `tracedecay` mapping when it carries no comments/anchors. -fn remove_legacy_project_pin(text: &str) -> std::result::Result { - let document = parse_profile(text)?; - let root = document - .as_mapping() - .unwrap_or_else(|| panic!("parse_profile guarantees a mapping")); - let Some(plugins) = root - .get("plugins") - .and_then(|node| node.as_mapping().cloned()) - else { - return Ok(text.to_string()); - }; - let Some(tracedecay) = plugins - .get("tracedecay") - .and_then(|node| node.as_mapping().cloned()) - else { - return Ok(text.to_string()); - }; - if !tracedecay.contains_key("project_root") { - return Ok(text.to_string()); - } - let after = remove_map_entry(text, &tracedecay, "project_root")?; - - let document = parse_profile(&after)?; - let root = document - .as_mapping() - .unwrap_or_else(|| panic!("parse_profile guarantees a mapping")); - if let Some(plugins) = root - .get("plugins") - .and_then(|node| node.as_mapping().cloned()) - { - return collapse_if_empty(&after, &plugins, "tracedecay"); - } - Ok(after) -} - fn remove_seq_item(text: &str, path: &[&str], value: &str) -> std::result::Result { let document = parse_profile(text)?; let root = document @@ -733,19 +663,7 @@ fn write_config_file(path: &Path, contents: &str) -> Result<()> { message: format!("failed to create {}: {error}", parent.display()), })?; } - let backup = backup_config_file(path)?; - safe_write_bytes_file(path, contents.as_bytes(), backup.as_deref()).map_err(|error| { - let backup_hint = backup - .as_ref() - .map(|path| format!(" Backup is at {}.", path.display())) - .unwrap_or_default(); - TraceDecayError::Config { - message: format!( - "failed to atomically replace {}: {error}.{backup_hint}", - path.display() - ), - } - }) + safe_write_bytes_file(path, contents.as_bytes()) } #[cfg(test)] @@ -799,29 +717,6 @@ mod tests { ); } - #[test] - fn collapse_after_removal_keeps_following_authored_lines() { - let text = concat!( - "plugins:\n", - " enabled:\n", - " - tracedecay\n", - " tracedecay:\n", - " project_root: /legacy\n", - "\n", - "memory: keep\n", - ); - assert_eq!( - remove_legacy_project_pin(text).unwrap(), - concat!( - "plugins:\n", - " enabled:\n", - " - tracedecay\n", - "\n", - "memory: keep\n", - ), - ); - } - #[derive(Debug, Clone, Copy)] enum Mutation { Enable, @@ -885,11 +780,6 @@ mod tests { .get_sequence("enabled") .is_none_or(|enabled| !sequence_contains(&enabled, "tracedecay")) ); - assert!( - plugins - .get_mapping("tracedecay") - .is_none_or(|plugin| !plugin.contains_key("project_root")) - ); } assert_ne!( root.get_mapping("memory") @@ -914,7 +804,7 @@ mod tests { input: concat!( "# leading comment\n", "\"plugins\": {enabled: [other], disabled: [tracedecay, blocked], ", - "tracedecay: {project_root: \"/legacy\", keep: yes}}\n", + "tracedecay: {keep: yes}}\n", "memory: {note: \"keep me\"}\n", "context: {note: 'keep me too'}\n", "unknown: {quoted: \"value\"}\n", @@ -928,7 +818,7 @@ mod tests { "note: 'keep me too'", "unknown: {quoted: \"value\"}", ], - removed: &["project_root:"], + removed: &[], crlf: false, }, CorpusCase { @@ -939,7 +829,6 @@ mod tests { "plugins:\n", " enabled: [other]\n", " tracedecay:\n", - " project_root: /legacy\n", " options: *defaults\n", "consumer:\n", " <<: *defaults\n", @@ -951,7 +840,7 @@ mod tests { "color: blue", "retries: 3", ], - removed: &["project_root:"], + removed: &[], crlf: false, }, CorpusCase { @@ -977,7 +866,6 @@ mod tests { "plugins:\n", " enabled: [tracedecay, other]\n", " tracedecay:\n", - " project_root: /legacy\n", " summary_model: glm-5\n", "memory: {provider: tracedecay, keep: true}\n", "context: {engine: tracedecay, budget: 42}\n", @@ -993,7 +881,7 @@ mod tests { "hooks: &hooks", "mcp: {servers: *hooks}", ], - removed: &["project_root:"], + removed: &[], crlf: false, }, ]; @@ -1077,17 +965,21 @@ mod tests { } #[test] - fn enable_plugin_backs_up_existing_config_before_atomic_write() { + fn enable_plugin_keeps_no_copy_of_the_prior_config() { let dir = TempDir::new().unwrap(); let config = dir.path().join("config.yaml"); let original = "theme: dark\nplugins:\n enabled:\n - other\n"; std::fs::write(&config, original).unwrap(); enable_plugin(&config).unwrap(); + disable_plugin(&config).unwrap(); - let backup = dir.path().join("config.yaml.bak"); - assert!(backup.exists()); - assert_eq!(read(&backup), original); + assert_eq!(read(&config), original); + let entries: Vec<_> = std::fs::read_dir(dir.path()) + .unwrap() + .map(|entry| entry.unwrap().file_name()) + .collect(); + assert_eq!(entries, vec![std::ffi::OsString::from("config.yaml")]); } #[test] @@ -1098,6 +990,5 @@ mod tests { let error = enable_plugin(&config).unwrap_err(); assert!(error.to_string().contains("failed to read")); - assert!(!original_config_path(&config).exists()); } } diff --git a/crates/tracedecay-agent-hosts/src/agents/hermes/templates/plugin_init.py b/crates/tracedecay-agent-hosts/src/agents/hermes/templates/plugin_init.py index c5ade411b6..d2cac65b0e 100644 --- a/crates/tracedecay-agent-hosts/src/agents/hermes/templates/plugin_init.py +++ b/crates/tracedecay-agent-hosts/src/agents/hermes/templates/plugin_init.py @@ -1656,7 +1656,7 @@ def _translate_lcm_args(native_name: str, args: dict) -> dict: translated.setdefault("scope", "current") translated.setdefault("sort", "relevance") translated.setdefault("include_summaries", False) - translated.setdefault("temporal_mode", "current") + translated.setdefault("temporal_mode", {"kind": "current"}) if "time_from" in translated: translated["start_time"] = translated.pop("time_from") if "time_to" in translated: @@ -1665,7 +1665,7 @@ def _translate_lcm_args(native_name: str, args: dict) -> dict: if native_name == "lcm_load_session": translated.setdefault("limit", 100) translated.setdefault("content_limit", 4000) - translated.setdefault("temporal_mode", "forensic") + translated.setdefault("temporal_mode", {"kind": "forensic"}) if "max_content_chars" in translated: translated["content_limit"] = translated.pop("max_content_chars") if "time_from" in translated: diff --git a/crates/tracedecay-agent-hosts/src/agents/hermes/templates/skill.md b/crates/tracedecay-agent-hosts/src/agents/hermes/templates/skill.md index 460908448f..fcec735af8 100644 --- a/crates/tracedecay-agent-hosts/src/agents/hermes/templates/skill.md +++ b/crates/tracedecay-agent-hosts/src/agents/hermes/templates/skill.md @@ -40,9 +40,9 @@ compresses a session. 1. Start with `tracedecay_message_search`. Its defaults are `provider=all`, `include_subagents=true`, `scope=all`, `message_type=all`, `limit=10`, and - `catch_up=false`. It finds stored message evidence and session ids; an - explicit freshness request can return `refresh_required`, but never catches - data up itself. + `require_fresh=false`. It finds stored message evidence and session ids; + `require_fresh=true` can return `refresh_required`, but never refreshes + data itself. 2. Hermes exposes native aliases `lcm_grep`, `lcm_load_session`, `lcm_describe`, `lcm_expand`, `lcm_expand_query`, `lcm_status`, and `lcm_doctor`. They dispatch to their matching `tracedecay_lcm_*` commands; @@ -50,10 +50,10 @@ compresses a session. canonical command and its schema elsewhere. Do not invent fields by mixing the two surfaces. 3. Narrow temporal evidence with `lcm_grep` / `tracedecay_lcm_grep` (default - `temporal_mode=current`; Hermes starts its native alias at the current - session), replay one session with `lcm_load_session` / - `tracedecay_lcm_load_session` (default `temporal_mode=forensic`), then use - `lcm_describe` / `tracedecay_lcm_describe` and `lcm_expand` / + `temporal_mode={"kind":"current"}`; Hermes starts its native alias at the + current session), replay one session with `lcm_load_session` / + `tracedecay_lcm_load_session` (default `temporal_mode={"kind":"forensic"}`), + then use `lcm_describe` / `tracedecay_lcm_describe` and `lcm_expand` / `tracedecay_lcm_expand` to open only the needed DAG node or payload. Summary node IDs are opaque strings, not integers. `source_limit` and the opaque continuation cursor apply only to summary source pages; raw and diff --git a/crates/tracedecay-agent-hosts/src/agents/host_bundle/component_set.rs b/crates/tracedecay-agent-hosts/src/agents/host_bundle/component_set.rs index 68437f79a9..e8ed9369ff 100644 --- a/crates/tracedecay-agent-hosts/src/agents/host_bundle/component_set.rs +++ b/crates/tracedecay-agent-hosts/src/agents/host_bundle/component_set.rs @@ -1,45 +1,35 @@ -//! Aggregate component-set transaction: one journal, one registration -//! adapter, and one rollback boundary spanning every component of a host. +//! Aggregate component-set transaction: one registration adapter and one +//! in-memory rollback boundary spanning every component of a host. use std::collections::BTreeMap; use std::path::Path; -use cap_std::fs::Dir; use sha2::{Digest, Sha256}; -use tracedecay_host_integration::host_bundle_recovery_required; use tracedecay_host_integration::host_bundle_stale_preview; -use tracedecay_host_integration::host_bundle_storage_failure; use super::control::{ - backup_name, component_set_from_journal, component_set_receipt_matches, - component_set_receipt_matches_preview, component_set_stage_name, - validate_component_set_journal, validate_component_set_receipt, validate_component_set_request, + component_set_receipt_matches, component_set_receipt_matches_preview, + validate_component_set_receipt, validate_component_set_request, }; use super::model::{ HostComponentSetExecutionRequestV1, HostComponentSetLifecyclePreviewV1, - HostComponentSetLifecycleRequestV1, HostComponentSetRegistrationV1, HostComponentSetV1, + HostComponentSetRegistrationV1, HostComponentSetV1, }; use super::planner::{ HostArtifactActionV1, HostBundleLifecycleRequestV1, HostBundleMutationPlanV1, - component_receiptless_adoption, dry_run_host_component_set_lifecycle_with_lifecycle_root_at, - observe_artifact_at, plan_verified_complete_lifecycle_mutation, - validate_artifact_contents_for_operation, -}; -use super::writer::{ - HostBundleWriterV1, atomic_write_nofollow, move_regular_to_backup, read_regular_nofollow, - regular_file_exists, remove_if_digest_matches, sync_cap_dir, + dry_run_host_component_set_lifecycle_with_lifecycle_root_at, observe_artifact_at, + plan_verified_complete_lifecycle_mutation, validate_artifact_contents_for_operation, }; +use super::writer::{ArtifactUndo, HostBundleWriterV1, read_regular_nofollow}; use super::{ HOST_BUNDLE_RECEIPT_SCHEMA_VERSION, HostBundleError, HostBundleInstallReceiptV1, - HostBundleJournalEntryV1, HostBundleLifecycleOpV1, HostBundleManifestV1, - HostBundleReceiptArtifactV1, HostBundleRollbackBoundaryV1, HostBundleVerificationAdapterV1, - HostComponentSetJournalComponentV1, HostComponentSetJournalStateV1, HostComponentSetJournalV1, - HostComponentSetReceiptV1, HostComponentV1, HostKindV1, stock_host_kinds, + HostBundleLifecycleOpV1, HostBundleManifestV1, HostBundleReceiptArtifactV1, + HostBundleVerificationAdapterV1, HostComponentSetReceiptV1, }; /// Public component-set lifecycle façade over the capability-rooted writer. /// It keeps the existing per-component receipt API intact while ensuring the -/// default host lifecycle has one aggregate recovery boundary. +/// default host lifecycle has one aggregate rollback boundary. pub struct HostComponentSetTransactionV1<'a> { writer: &'a mut HostBundleWriterV1, } @@ -49,42 +39,6 @@ impl<'a> HostComponentSetTransactionV1<'a> { Self { writer } } - /// Recover whichever single component-set journal is outstanding. Callers - /// that know the host should prefer [`Self::recover_host`], which never - /// hands another host's journal to this registration authority. - pub fn recover( - &mut self, - registration: &mut R, - ) -> Result<(), HostBundleError> { - let Some(journal) = self.writer.load_component_set_journal()? else { - for host in stock_host_kinds() { - if self.writer.load_journal_for(host)?.is_some() { - self.writer.recover_interrupted_operation(host)?; - } - } - return Ok(()); - }; - let host = journal.host; - self.writer.ensure_host_lock(host)?; - self.writer - .recover_component_set_operation(Some(host), registration)?; - self.writer.recover_interrupted_operation(host) - } - - /// Recover only `host`'s pending component-set journal. Other hosts' - /// journals are left untouched: their artifact path spaces are disjoint, - /// and their registration state belongs to a different adapter. - pub fn recover_host( - &mut self, - host: HostKindV1, - registration: &mut R, - ) -> Result<(), HostBundleError> { - self.writer.ensure_host_lock(host)?; - self.writer - .recover_component_set_operation(Some(host), registration)?; - self.writer.recover_interrupted_operation(host) - } - pub fn preview( &mut self, component_set: &HostComponentSetV1, @@ -92,17 +46,7 @@ impl<'a> HostComponentSetTransactionV1<'a> { verifier: &V, registration: &mut R, ) -> Result { - // Only this host's own pending journal blocks the preview. A wedged - // transaction for an unrelated host mutates a disjoint path space and - // is not a reason to refuse work here. - if self.writer.load_journal_for(component_set.host)?.is_some() - || self - .writer - .load_component_set_journal_for(component_set.host)? - .is_some() - { - return Err(host_bundle_recovery_required!()); - } + self.writer.discard_stale_receipts_for_reinstall(request)?; dry_run_host_component_set_lifecycle_with_lifecycle_root_at( &self.writer.root_path, &self.writer.lifecycle_root_path, @@ -176,10 +120,6 @@ impl<'a> HostComponentSetTransactionV1<'a> { verifier: &V, registration: &mut R, ) -> Result { - // Host-scoped: a pending journal for an unrelated host governs a - // disjoint artifact subtree and belongs to a different registration - // adapter, so it must neither be recovered here nor block this work. - self.recover_host(component_set.host, registration)?; self.writer .execute_component_set(component_set, request, verifier, registration) } @@ -193,10 +133,25 @@ struct PreparedHostComponentSetComponentV1 { } impl HostBundleWriterV1 { - /// Execute a complete canonical host component set under one aggregate - /// journal. Every component is preflighted and staged before any owned - /// file is moved; receipts are published only after all artifacts and the - /// host registration authority verify successfully. + /// An explicitly adopting install, reinstall, or update replaces receipts + /// written under an older schema; every other request leaves them to fail + /// as [`HostBundleError::ReinstallRequired`]. + fn discard_stale_receipts_for_reinstall( + &mut self, + request: &HostComponentSetExecutionRequestV1, + ) -> Result<(), HostBundleError> { + if request.lifecycle.explicit_adoption + && request.lifecycle.operation != HostBundleLifecycleOpV1::Uninstall + { + self.discard_stale_receipts(request.lifecycle.expected_host)?; + } + Ok(()) + } + + /// Execute a complete canonical host component set as one operation. + /// Every component is preflighted before any owned file changes; receipts + /// are published only after all artifacts and the host registration + /// authority verify successfully. pub fn execute_component_set< V: HostBundleVerificationAdapterV1, R: HostComponentSetRegistrationV1, @@ -250,17 +205,7 @@ impl HostBundleWriterV1 { ) -> Result { validate_component_set_request(component_set, request)?; self.ensure_host_lock(component_set.host)?; - if self.load_journal_for(component_set.host)?.is_some() { - return Err(host_bundle_recovery_required!()); - } - // Never clobber this host's own outstanding journal: it is the only - // durable record of how to roll the earlier transaction back. - if self - .load_component_set_journal_for(component_set.host)? - .is_some() - { - return Err(host_bundle_recovery_required!()); - } + self.discard_stale_receipts_for_reinstall(request)?; if let Some(receipt) = self.load_component_set_receipt(request.operation_id)? { if !component_set_receipt_matches(&receipt, component_set, request)? { return Err(HostBundleError::ReceiptCorrupted); @@ -273,24 +218,7 @@ impl HostBundleWriterV1 { return Ok(receipt); } - // Resolve receiptless-adoption authority through the same adapter the - // preview used, so the replanned mutations match the confirmed plan. - let adoption_by_component: BTreeMap = component_set - .components - .iter() - .map(|component| { - ( - component.manifest.component, - component_receiptless_adoption( - request, - registration, - component.manifest.component, - ), - ) - }) - .collect(); - let prepared = - self.preflight_component_set(component_set, request, verifier, &adoption_by_component)?; + let prepared = self.preflight_component_set(component_set, request, verifier)?; // Declare the exact write set before any adapter observes state, so a // registration surface that is also one of these artifacts can tell // this transaction's own write apart from a foreign edit. @@ -302,86 +230,21 @@ impl HostBundleWriterV1 { registration.declare_artifact_writes(component_set, request, &declared_writes)?; registration.preflight(component_set, request)?; - let mut journal = HostComponentSetJournalV1 { - schema_version: HOST_BUNDLE_RECEIPT_SCHEMA_VERSION, - operation_id: request.operation_id, - host: component_set.host, - operation: request.lifecycle.operation, - explicit_confirmation: request.lifecycle.explicit_confirmation, - hermes_profile_bindings: request.lifecycle.hermes_profile_bindings, - confirmed_plan_digest: confirmed_preview.map(|preview| preview.plan_digest), - base_registration_revision: confirmed_preview - .map(|preview| preview.base_registration_revision), - current_registration_revision: confirmed_preview - .map(|preview| preview.current_registration_revision), - artifact_state_revision: confirmed_preview - .map(|preview| preview.artifact_state_revision), - state: HostComponentSetJournalStateV1::Prepared, - registration_staged: false, - registration_applied: false, - components: prepared - .iter() - .map(|component| HostComponentSetJournalComponentV1 { - manifest: component.manifest.clone(), - previous_receipt: component.previous_receipt.clone(), - entries: component - .plan - .mutations - .iter() - .map(|mutation| HostBundleJournalEntryV1 { - relative_path: mutation.relative_path.clone(), - backup_name: matches!( - mutation.action, - HostArtifactActionV1::BackupThenReplace - | HostArtifactActionV1::BackupThenRemove - ) - .then(|| backup_name(request.operation_id, &mutation.relative_path)), - backup_created: false, - wrote_new: false, - installed_digest: component - .manifest - .artifacts - .iter() - .find(|artifact| artifact.relative_path == mutation.relative_path) - .map(|artifact| artifact.artifact_digest) - .filter(|_| { - !matches!( - mutation.action, - HostArtifactActionV1::BackupThenRemove - ) - }), - }) - .collect(), - }) - .collect(), - }; - self.write_component_set_journal(&journal)?; - + let mut undo = Vec::new(); let result = (|| { - self.stage_component_set_assets(&prepared, request.operation_id)?; - journal.registration_staged = true; - self.write_component_set_journal(&journal)?; registration.stage(component_set, request)?; - journal.state = HostComponentSetJournalStateV1::Staged; - self.write_component_set_journal(&journal)?; - - let backup_dir = self.open_or_create_backup_dir(request.operation_id)?; - self.backup_component_set_entries(&prepared, &mut journal, &backup_dir)?; - drop(backup_dir); - self.write_component_set_entries(&prepared, &mut journal)?; - - // Mark this before calling into host registration: a failing - // adapter can still have made a partial native mutation. - journal.registration_applied = true; - self.write_component_set_journal(&journal)?; + for component in &prepared { + for mutation in &component.plan.mutations { + if let Some(record) = + self.apply_artifact_mutation(mutation, &component.content_by_path)? + { + undo.push(record); + } + } + } registration.apply(component_set, request)?; - journal.state = HostComponentSetJournalStateV1::Applied; - self.write_component_set_journal(&journal)?; - - self.verify_component_set_artifacts(&journal)?; + self.verify_component_set_artifacts(&prepared)?; registration.verify(component_set, request)?; - journal.state = HostComponentSetJournalStateV1::Verified; - self.write_component_set_journal(&journal)?; let receipt = component_set_receipt_from_prepared(&prepared, request, confirmed_preview)?; @@ -389,110 +252,34 @@ impl HostBundleWriterV1 { self.write_receipt(component_receipt)?; } self.write_component_set_receipt(&receipt)?; - journal.state = HostComponentSetJournalStateV1::Committed; - self.write_component_set_journal(&journal)?; - - // Registration cleanup and backup retirement happen only after the - // aggregate and every component receipt have crossed commit. - registration.commit(component_set, request)?; - self.cleanup_component_set_boundary(request.operation_id)?; - self.remove_component_set_journal(component_set.host)?; Ok(receipt) })(); match result { - Ok(receipt) => Ok(receipt), - Err(error) if journal.state == HostComponentSetJournalStateV1::Committed => { - // The durable receipts prove commit. Keep the journal for a - // restarted transaction to finish registration/backup cleanup. - Err(error) - } - Err(error) => { - if self - .rollback_component_set(component_set, request, registration, &mut journal) - .is_err() - { - Err(host_bundle_recovery_required!()) - } else { - Err(error) - } + Ok(receipt) => { + registration.commit(component_set, request)?; + Ok(receipt) } + Err(error) => Err( + match self.rollback_component_set( + component_set, + request, + registration, + &prepared, + &undo, + ) { + Ok(()) => error, + Err(rollback_error) => rollback_error, + }, + ), } } - /// Resume a component-set operation left by a failed apply or a process - /// interruption. A fully published aggregate receipt wins; any other - /// state is rolled back in reverse component and artifact order. - fn recover_component_set_operation( - &mut self, - host: Option, - registration: &mut R, - ) -> Result<(), HostBundleError> { - let loaded = match host { - Some(host) => self.load_component_set_journal_for(host)?, - None => self.load_component_set_journal()?, - }; - let Some(mut journal) = loaded else { - return Ok(()); - }; - validate_component_set_journal(&journal)?; - let component_set = component_set_from_journal(&journal); - let request = HostComponentSetExecutionRequestV1 { - lifecycle: HostComponentSetLifecycleRequestV1 { - operation: journal.operation, - expected_host: journal.host, - expected_components: journal - .components - .iter() - .map(|component| component.manifest.component) - .collect(), - explicit_confirmation: journal.explicit_confirmation, - hermes_profile_bindings: journal.hermes_profile_bindings, - // Recovery replays or rolls back the journaled mutations; it - // never re-plans, so it can never adopt anything new. - explicit_adoption: false, - }, - operation_id: journal.operation_id, - }; - - if journal.state == HostComponentSetJournalStateV1::Committed - || self.component_set_commit_is_complete(&journal)? - { - registration.commit(&component_set, &request)?; - self.cleanup_component_set_boundary(journal.operation_id)?; - self.remove_component_set_journal(journal.host)?; - return Ok(()); - } - - if journal.state == HostComponentSetJournalStateV1::RolledBack { - // A rolled-back journal keeps whichever flags the failed attempt - // had reached, so they describe the interrupted work rather than - // the compensation still owed. Re-attempt it unconditionally: the - // adapter contract is idempotent and no-ops when it finds no staged - // registration backup, while skipping it would strand a mutated - // native host configuration with nothing left to compensate it. - registration.rollback(&component_set, &request)?; - self.cleanup_component_set_boundary(journal.operation_id)?; - self.remove_component_set_journal(journal.host)?; - return Ok(()); - } - - if journal.registration_compensation_required() { - registration.rollback(&component_set, &request)?; - } - self.restore_component_set_artifacts(&journal)?; - journal.state = HostComponentSetJournalStateV1::RolledBack; - self.write_component_set_journal(&journal)?; - self.cleanup_component_set_boundary(journal.operation_id)?; - self.remove_component_set_journal(journal.host) - } - fn preflight_component_set( &self, component_set: &HostComponentSetV1, request: &HostComponentSetExecutionRequestV1, verifier: &V, - adoption_by_component: &BTreeMap, ) -> Result, HostBundleError> { let mut prepared = Vec::with_capacity(component_set.components.len()); let mut claimed_paths = BTreeMap::new(); @@ -571,10 +358,7 @@ impl HostBundleWriterV1 { expected_component: component.manifest.component, explicit_confirmation: request.lifecycle.explicit_confirmation, hermes_profile_bindings: request.lifecycle.hermes_profile_bindings, - adopt_receiptless: adoption_by_component - .get(&component.manifest.component) - .copied() - .unwrap_or(false), + adopt_receiptless: request.lifecycle.explicit_adoption, }; let plan = plan_verified_complete_lifecycle_mutation( &component.manifest, @@ -602,90 +386,25 @@ impl HostBundleWriterV1 { Ok(prepared) } - fn stage_component_set_assets( + fn verify_component_set_artifacts( &self, prepared: &[PreparedHostComponentSetComponentV1], - operation_id: [u8; 16], ) -> Result<(), HostBundleError> { - let stage = self.open_or_create_component_set_stage_dir(operation_id)?; for component in prepared { - for (relative_path, bytes) in &component.content_by_path { - let stage_name = - component_set_stage_name(component.manifest.component, relative_path); - atomic_write_nofollow(&stage, &stage_name, bytes, false)?; - } - } - sync_cap_dir(&stage) - } - - fn backup_component_set_entries( - &self, - prepared: &[PreparedHostComponentSetComponentV1], - journal: &mut HostComponentSetJournalV1, - backup_dir: &Dir, - ) -> Result<(), HostBundleError> { - for (component_index, prepared_component) in prepared.iter().enumerate() { - for (entry_index, mutation) in prepared_component.plan.mutations.iter().enumerate() { - if !matches!( - mutation.action, - HostArtifactActionV1::BackupThenReplace - | HostArtifactActionV1::BackupThenRemove - ) { - continue; - } - let backup_name = journal.components[component_index].entries[entry_index] - .backup_name - .clone() - .ok_or(HostBundleError::ReceiptCorrupted)?; - let (parent, name) = - self.open_parent_nofollow(Path::new(&mutation.relative_path))?; - move_regular_to_backup(&parent, &name, backup_dir, &backup_name)?; - journal.components[component_index].entries[entry_index].backup_created = true; - self.write_component_set_journal(journal)?; - } - } - Ok(()) - } - - fn write_component_set_entries( - &self, - prepared: &[PreparedHostComponentSetComponentV1], - journal: &mut HostComponentSetJournalV1, - ) -> Result<(), HostBundleError> { - for (component_index, prepared_component) in prepared.iter().enumerate() { - for (entry_index, mutation) in prepared_component.plan.mutations.iter().enumerate() { + for mutation in &component.plan.mutations { + let expected = (mutation.action != HostArtifactActionV1::Remove) + .then(|| { + component + .manifest + .artifacts + .iter() + .find(|artifact| artifact.relative_path == mutation.relative_path) + .map(|artifact| artifact.artifact_digest) + }) + .flatten(); let (parent, name) = self.open_parent_nofollow(Path::new(&mutation.relative_path))?; - match mutation.action { - HostArtifactActionV1::Noop | HostArtifactActionV1::BackupThenRemove => {} - HostArtifactActionV1::WriteNew | HostArtifactActionV1::BackupThenReplace => { - journal.components[component_index].entries[entry_index].wrote_new = true; - self.write_component_set_journal(journal)?; - atomic_write_nofollow( - &parent, - &name, - prepared_component - .content_by_path - .get(&mutation.relative_path) - .ok_or(HostBundleError::ArtifactContentMismatch)?, - false, - )?; - } - } - } - } - Ok(()) - } - - fn verify_component_set_artifacts( - &self, - journal: &HostComponentSetJournalV1, - ) -> Result<(), HostBundleError> { - for component in &journal.components { - for entry in &component.entries { - let (parent, name) = self.open_parent_nofollow(Path::new(&entry.relative_path))?; - let observed = read_regular_nofollow(&parent, &name)?; - match (entry.installed_digest, observed) { + match (expected, read_regular_nofollow(&parent, &name)?) { (Some(expected), Some(bytes)) => { let digest: [u8; 32] = Sha256::digest(&bytes).into(); if digest != expected { @@ -697,7 +416,7 @@ impl HostBundleWriterV1 { (None, Some(_)) => { return Err(HostBundleError::OwnershipConflict(format!( "{}: a file appeared at a path this transaction removed", - entry.relative_path + mutation.relative_path ))); } } @@ -711,171 +430,18 @@ impl HostBundleWriterV1 { component_set: &HostComponentSetV1, request: &HostComponentSetExecutionRequestV1, registration: &mut R, - journal: &mut HostComponentSetJournalV1, - ) -> Result<(), HostBundleError> { - if journal.registration_compensation_required() { - registration.rollback(component_set, request)?; - } - self.restore_component_set_artifacts(journal)?; - self.remove_component_set_receipt(journal.operation_id)?; - journal.state = HostComponentSetJournalStateV1::RolledBack; - // Leave the completed rollback journal and its backups for an explicit - // restart reconciliation boundary; a new transaction invokes recover. - self.write_component_set_journal(journal) - } - - fn restore_component_set_artifacts( - &self, - journal: &HostComponentSetJournalV1, + prepared: &[PreparedHostComponentSetComponentV1], + undo: &[ArtifactUndo], ) -> Result<(), HostBundleError> { - let backup_dir = self.open_existing_backup_dir(journal.operation_id)?; - for component in journal.components.iter().rev() { - for entry in component.entries.iter().rev() { - self.restore_component_set_entry(entry, backup_dir.as_ref())?; - } - } - for component in journal.components.iter().rev() { + registration.rollback(component_set, request)?; + self.undo_artifact_mutations(undo)?; + for component in prepared.iter().rev() { match &component.previous_receipt { Some(receipt) => self.write_receipt(receipt)?, - None => self.remove_receipt(journal.host, component.manifest.component)?, - } - } - Ok(()) - } - - /// Restore one journal entry to its pre-transaction state. - /// - /// Rollback must be able to CONVERGE when a second writer touched a - /// deployed path after this transaction wrote it. A post-apply fault can - /// leave live bytes that are neither the backup nor this transaction's - /// cataloged output. Before the convergence rules below, that state was - /// unrecoverable: rollback - /// returned `RecoveryRequired` forever, the journal stayed behind, and - /// every later host transaction failed up front. - /// - /// Two content equalities are provably safe to converge on, because in both - /// cases the operator-visible end state is byte-identical to a successful - /// restore: - /// - /// 1. **Live bytes equal the pre-transaction backup.** The end state - /// rollback wants is already true; renaming the backup over it would - /// produce the same bytes. Treat the path as restored. - /// 2. **Live bytes equal this entry's cataloged install target - /// (`installed_digest`).** Those bytes are provably this transaction's - /// own output, so removing them is a restore and not third-party data - /// loss. This also closes the crash window between the artifact write - /// and the `wrote_new` journal update. - /// - /// Anything else, foreign bytes that match neither, stays fail-closed - /// with `RecoveryRequired`, and the operator resolves it explicitly with - /// `tracedecay host-bundle recover`. - fn restore_component_set_entry( - &self, - entry: &HostBundleJournalEntryV1, - backup_dir: Option<&Dir>, - ) -> Result<(), HostBundleError> { - let (parent, name) = self.open_parent_nofollow(Path::new(&entry.relative_path))?; - if let Some(backup_name) = &entry.backup_name { - let backup_bytes = match backup_dir { - Some(backups) => read_regular_nofollow(backups, backup_name)?, - None => None, - }; - let backup_exists = backup_bytes.is_some(); - // Convergence rule 1: the live file already holds the exact - // pre-transaction bytes, so this path needs no mutation at all. - // The backup stays until the boundary cleanup retires the whole - // operation directory, which keeps a repeated restore idempotent. - if let (Some(backup), Some(live)) = ( - backup_bytes.as_ref(), - read_regular_nofollow(&parent, &name)?, - ) && live == *backup - { - return Ok(()); - } - if !entry.backup_created { - if !backup_exists { - return Ok(()); - } - if regular_file_exists(&parent, &name)? { - return Err(host_bundle_recovery_required!()); - } - } - let backups = backup_dir - .filter(|_| backup_exists) - .ok_or(host_bundle_recovery_required!())?; - if entry.wrote_new { - remove_if_digest_matches( - &parent, - &name, - entry - .installed_digest - .ok_or(HostBundleError::ReceiptCorrupted)?, - )?; - } else if let Some(live) = read_regular_nofollow(&parent, &name)? { - // Convergence rule 2. `installed_digest` is `None` for a - // BackupThenRemove entry, which has no cataloged target and - // therefore stays fail-closed. - let installed = entry - .installed_digest - .ok_or(host_bundle_recovery_required!())?; - if <[u8; 32]>::from(Sha256::digest(&live)) != installed { - return Err(host_bundle_recovery_required!()); - } - parent - .remove_file(&name) - .map_err(|_| host_bundle_storage_failure!())?; + None => self.remove_receipt(component_set.host, component.manifest.component)?, } - backups - .rename(backup_name, &parent, &name) - .map_err(|_| host_bundle_storage_failure!())?; - sync_cap_dir(backups)?; - sync_cap_dir(&parent) - } else if entry.wrote_new { - // No backup: the path did not exist before the transaction, so - // rollback wants it gone. `remove_if_digest_matches` already - // converges on the two safe outcomes (already absent, or holding - // this transaction's cataloged bytes). Foreign bytes at a path this - // transaction created are genuinely ambiguous, removing them could - // destroy another writer's file, so that case stays fail-closed. - remove_if_digest_matches( - &parent, - &name, - entry - .installed_digest - .ok_or(HostBundleError::ReceiptCorrupted)?, - )?; - sync_cap_dir(&parent) - } else { - Ok(()) } - } - - fn component_set_commit_is_complete( - &self, - journal: &HostComponentSetJournalV1, - ) -> Result { - let Some(receipt) = self.load_component_set_receipt(journal.operation_id)? else { - return Ok(false); - }; - let component_set = component_set_from_journal(journal); - let request = HostComponentSetExecutionRequestV1 { - lifecycle: HostComponentSetLifecycleRequestV1 { - operation: journal.operation, - expected_host: journal.host, - expected_components: component_set - .components - .iter() - .map(|component| component.manifest.component) - .collect(), - explicit_confirmation: true, - hermes_profile_bindings: u8::from(journal.host == HostKindV1::Hermes), - // Receipt matching compares durable identity; adoption - // authority is a planning input and plays no part here. - explicit_adoption: false, - }, - operation_id: journal.operation_id, - }; - component_set_receipt_matches(&receipt, &component_set, &request) + self.remove_component_set_receipt(request.operation_id) } } @@ -924,30 +490,6 @@ fn component_set_receipt_from_prepared( { return Ok(previous_receipt.clone()); } - let mut rollback_history = component - .previous_receipt - .as_ref() - .map(|receipt| receipt.rollback_history.clone()) - .unwrap_or_default(); - // A Repair that overwrites a receipt-owned path whose bytes drifted - // from the catalog backs up genuinely foreign content, a user edit, - // never tracedecay's own prior output, because Repair replaces a - // path only when its observed digest differs from the cataloged one, - // which for an unchanged Repair manifest is also the previously - // owned digest. Referencing this operation from the receipt keeps the - // commit boundary from retiring that backup, so an operator can still - // recover the overwritten bytes. Ordinary Update backups hold - // tracedecay's own output and stay retired on commit. - if request.lifecycle.operation == HostBundleLifecycleOpV1::Repair - && component - .plan - .mutations - .iter() - .any(|mutation| mutation.action == HostArtifactActionV1::BackupThenReplace) - && !rollback_history.contains(&request.operation_id) - { - rollback_history.push(request.operation_id); - } Ok(HostBundleInstallReceiptV1 { schema_version: HOST_BUNDLE_RECEIPT_SCHEMA_VERSION, operation_id: request.operation_id, @@ -969,8 +511,6 @@ fn component_set_receipt_from_prepared( }) .collect() }, - rollback_boundary: HostBundleRollbackBoundaryV1::Passed, - rollback_history, }) }) .collect::, HostBundleError>>()?; diff --git a/crates/tracedecay-agent-hosts/src/agents/host_bundle/control.rs b/crates/tracedecay-agent-hosts/src/agents/host_bundle/control.rs index e54417e7aa..64db82f63a 100644 --- a/crates/tracedecay-agent-hosts/src/agents/host_bundle/control.rs +++ b/crates/tracedecay-agent-hosts/src/agents/host_bundle/control.rs @@ -1,43 +1,27 @@ //! Layout of the `.tracedecay-host-bundle-v1` control directory: file names, -//! path-rooted receipt readers, and receipt/journal validators. +//! path-rooted receipt readers, and receipt validators. use std::collections::BTreeMap; use std::fs; use std::io; -use std::path::{Path, PathBuf}; +use std::path::Path; -use sha2::{Digest, Sha256}; +use serde::Deserialize; +use serde::de::DeserializeOwned; use tracedecay_host_integration::host_bundle_storage_failure; use super::model::{ - HostComponentSetEntryV1, HostComponentSetExecutionRequestV1, - HostComponentSetLifecyclePreviewV1, HostComponentSetV1, + HostComponentSetExecutionRequestV1, HostComponentSetLifecyclePreviewV1, HostComponentSetV1, }; use super::planner::inspect_install_target; use super::{ - HOST_BUNDLE_RECEIPT_SCHEMA_VERSION, HostBundleBackupReceiptV1, HostBundleError, - HostBundleInstallReceiptV1, HostBundleJournalV1, HostBundleLifecycleOpV1, - HostBundleRestoreReceiptV1, HostBundleRollbackBoundaryV1, HostComponentSetJournalV1, - HostComponentSetReceiptV1, HostComponentV1, HostKindV1, MAX_HOST_COMPONENTS, - MAX_MANIFEST_ARTIFACTS, stock_host_kinds, validate_identifier, validate_relative_install_path, + HOST_BUNDLE_RECEIPT_SCHEMA_VERSION, HostBundleError, HostBundleInstallReceiptV1, + HostBundleLifecycleOpV1, HostComponentSetReceiptV1, HostComponentV1, HostKindV1, + MAX_HOST_COMPONENTS, MAX_MANIFEST_ARTIFACTS, stock_host_kinds, validate_identifier, + validate_relative_install_path, }; pub(super) const HOST_BUNDLE_CONTROL_DIR: &str = ".tracedecay-host-bundle-v1"; -/// Legacy shared single-component journal. One file per lifecycle root meant -/// recovering host Y rolled back host X, and a wedged journal blocked every -/// other host. Journals are host-scoped now; this name is still read (and -/// retired) so a journal left by an older binary is recovered rather than -/// orphaned. -pub(super) const HOST_BUNDLE_JOURNAL_FILE: &str = "journal.v1.json"; -/// Legacy shared component-set journal name. One journal per lifecycle root -/// meant an interrupted transaction for any host blocked every other host. -/// Journals are host-scoped now; this name is still read (and retired) so a -/// journal left by an older binary is recovered rather than orphaned. -pub(super) const HOST_COMPONENT_SET_JOURNAL_FILE: &str = "component-set-journal.v1.json"; -pub(super) const HOST_COMPONENT_SET_STAGE_DIR: &str = "component-set-staging"; -/// Set-aside directory for journals an operator explicitly abandoned with -/// `tracedecay host-bundle recover --quarantine --yes`. Backups stay in place. -pub(super) const HOST_BUNDLE_QUARANTINE_DIR: &str = "quarantine"; /// Retired lifecycle-root lock. Hosts do not share a write target, so each /// host owns `writer.{slug}.v1.lock`. This name is not acquired; a new binary /// must not recreate it or independent hosts serialize again. @@ -74,8 +58,14 @@ pub fn latest_host_component_set_receipt_at( continue; } let bytes = fs::read(entry.path()).map_err(|_| host_bundle_storage_failure!())?; - let Ok(receipt) = serde_json::from_slice::(&bytes) else { - continue; + let receipt = match parse_receipt::(&bytes) { + Ok(receipt) => receipt, + Err(HostBundleError::ReinstallRequired) + if receipt_schema_probe(&bytes).is_some_and(|probe| probe.host == Some(host)) => + { + return Err(HostBundleError::ReinstallRequired); + } + Err(_) => continue, }; if receipt.host != host || receipt.operation == HostBundleLifecycleOpV1::Uninstall @@ -94,16 +84,6 @@ pub fn latest_host_component_set_receipt_at( Ok(latest.map(|(_, receipt)| receipt)) } -/// Where rollback backups are written, one subdirectory per applied operation -/// id. Exposed so a dry run can tell the operator where the bytes it is about -/// to replace will be preserved, without the CLI reconstructing a -/// control-directory layout it does not own. The operation id is minted when -/// the mutation actually runs, so only the root is knowable during a preview. -#[must_use] -pub fn host_bundle_backup_root(lifecycle_root: &Path) -> PathBuf { - lifecycle_root.join(HOST_BUNDLE_CONTROL_DIR).join("backups") -} - pub fn latest_host_component_receipt_at( lifecycle_root: &Path, host: HostKindV1, @@ -131,7 +111,7 @@ pub(super) fn read_receipt_at( Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None), Err(_) => return Err(host_bundle_storage_failure!()), }; - let receipt = serde_json::from_slice(&bytes).map_err(|_| HostBundleError::ReceiptCorrupted)?; + let receipt: HostBundleInstallReceiptV1 = parse_receipt(&bytes)?; validate_receipt(&receipt)?; if receipt.host != host || receipt.component != component { return Err(HostBundleError::ReceiptCorrupted); @@ -139,6 +119,33 @@ pub(super) fn read_receipt_at( Ok(Some(receipt)) } +#[derive(Deserialize)] +pub(super) struct ReceiptSchemaProbe { + schema_version: u16, + #[serde(default)] + pub(super) host: Option, +} + +impl ReceiptSchemaProbe { + pub(super) fn is_current(&self) -> bool { + self.schema_version == HOST_BUNDLE_RECEIPT_SCHEMA_VERSION + } +} + +pub(super) fn receipt_schema_probe(bytes: &[u8]) -> Option { + serde_json::from_slice(bytes).ok() +} + +/// Parse a receipt of the current schema. A receipt from an older schema is a +/// typed [`HostBundleError::ReinstallRequired`], never migrated in place. +pub(super) fn parse_receipt(bytes: &[u8]) -> Result { + let probe = receipt_schema_probe(bytes).ok_or(HostBundleError::ReceiptCorrupted)?; + if !probe.is_current() { + return Err(HostBundleError::ReinstallRequired); + } + serde_json::from_slice(bytes).map_err(|_| HostBundleError::ReceiptCorrupted) +} + pub(super) fn is_safe_component(value: &str) -> bool { !value.is_empty() && value != "." @@ -154,7 +161,6 @@ pub(super) fn validate_receipt( || receipt.operation_id == [0; 16] || receipt.manifest_digest == [0; 32] || receipt.artifacts.len() > MAX_MANIFEST_ARTIFACTS - || receipt.rollback_history.len() > MAX_MANIFEST_ARTIFACTS || (receipt.operation == HostBundleLifecycleOpV1::Uninstall) != receipt.artifacts.is_empty() { return Err(HostBundleError::ReceiptCorrupted); @@ -172,97 +178,6 @@ pub(super) fn validate_receipt( return Err(HostBundleError::ReceiptCorrupted); } } - for (index, operation_id) in receipt.rollback_history.iter().enumerate() { - if *operation_id == [0; 16] || receipt.rollback_history[..index].contains(operation_id) { - return Err(HostBundleError::ReceiptCorrupted); - } - } - Ok(()) -} - -pub(super) fn validate_backup_receipt( - receipt: &HostBundleBackupReceiptV1, -) -> Result<(), HostBundleError> { - if receipt.schema_version != HOST_BUNDLE_RECEIPT_SCHEMA_VERSION - || receipt.operation_id == [0; 16] - || receipt.source_receipt_digest == [0; 32] - || receipt.host != receipt.manifest.host - || receipt.component != receipt.manifest.component - || receipt.artifacts.len() != receipt.manifest.artifacts.len() - { - return Err(HostBundleError::ReceiptCorrupted); - } - receipt - .manifest - .validate_structure() - .map_err(|_| HostBundleError::ReceiptCorrupted)?; - for (index, artifact) in receipt.artifacts.iter().enumerate() { - validate_relative_install_path(Path::new(&artifact.relative_path))?; - validate_identifier(&artifact.ownership_marker)?; - if artifact.artifact_digest == [0; 32] - || !is_safe_component(&artifact.snapshot_name) - || receipt.artifacts[..index] - .iter() - .any(|existing| existing.relative_path == artifact.relative_path) - || !receipt.manifest.artifacts.iter().any(|expected| { - expected.relative_path == artifact.relative_path - && expected.artifact_digest == artifact.artifact_digest - && expected.ownership_marker == artifact.ownership_marker - }) - { - return Err(HostBundleError::ReceiptCorrupted); - } - } - Ok(()) -} - -pub(super) fn validate_restore_receipt( - receipt: &HostBundleRestoreReceiptV1, -) -> Result<(), HostBundleError> { - validate_receipt(&receipt.restored_receipt)?; - if receipt.schema_version != HOST_BUNDLE_RECEIPT_SCHEMA_VERSION - || receipt.operation_id == [0; 16] - || receipt.backup_operation_id == [0; 16] - || receipt.restored_receipt.operation_id != receipt.operation_id - || receipt.restored_receipt.operation != HostBundleLifecycleOpV1::Repair - || receipt.restored_receipt.rollback_boundary != HostBundleRollbackBoundaryV1::Passed - { - return Err(HostBundleError::ReceiptCorrupted); - } - Ok(()) -} - -pub(super) fn validate_journal(journal: &HostBundleJournalV1) -> Result<(), HostBundleError> { - if journal.schema_version != HOST_BUNDLE_RECEIPT_SCHEMA_VERSION - || journal.operation_id == [0; 16] - || journal.manifest_digest == [0; 32] - || (journal.entries.is_empty() && journal.operation != HostBundleLifecycleOpV1::Uninstall) - || journal.entries.len() > MAX_MANIFEST_ARTIFACTS - { - return Err(HostBundleError::ReceiptCorrupted); - } - if let Some(receipt) = &journal.previous_receipt { - validate_receipt(receipt)?; - if receipt.host != journal.host || receipt.component != journal.component { - return Err(HostBundleError::ReceiptCorrupted); - } - } - for (index, entry) in journal.entries.iter().enumerate() { - validate_relative_install_path(Path::new(&entry.relative_path))?; - if entry - .backup_name - .as_deref() - .is_some_and(|backup| !is_safe_component(backup)) - || journal.entries[..index] - .iter() - .any(|existing| existing.relative_path == entry.relative_path) - || (entry.backup_created && entry.backup_name.is_none()) - || (entry.backup_name.is_some() && entry.wrote_new && !entry.backup_created) - || (entry.wrote_new && entry.installed_digest.is_none()) - { - return Err(HostBundleError::ReceiptCorrupted); - } - } Ok(()) } @@ -344,7 +259,6 @@ pub(super) fn component_set_receipt_matches( component_receipt.host == component.manifest.host && component_receipt.component == component.manifest.component && component_receipt.manifest_digest == manifest_digest - && component_receipt.rollback_boundary == HostBundleRollbackBoundaryV1::Passed }); if !receipt_matches { return Ok(false); @@ -399,7 +313,6 @@ pub(super) fn validate_component_set_receipt( if component_receipt.host != receipt.host || manifest.host != receipt.host || manifest.canonical_digest()? != component_receipt.manifest_digest - || component_receipt.rollback_boundary != HostBundleRollbackBoundaryV1::Passed || receipt.component_receipts[..index] .iter() .any(|previous| previous.component == component_receipt.component) @@ -410,145 +323,6 @@ pub(super) fn validate_component_set_receipt( Ok(()) } -pub(super) fn component_set_from_journal( - journal: &HostComponentSetJournalV1, -) -> HostComponentSetV1 { - HostComponentSetV1 { - host: journal.host, - components: journal - .components - .iter() - .map(|component| HostComponentSetEntryV1 { - manifest: component.manifest.clone(), - contents: Vec::new(), - }) - .collect(), - } -} - -pub(super) fn validate_component_set_journal( - journal: &HostComponentSetJournalV1, -) -> Result<(), HostBundleError> { - if journal.schema_version != HOST_BUNDLE_RECEIPT_SCHEMA_VERSION - || journal.operation_id == [0; 16] - || journal.components.is_empty() - || journal.components.len() > MAX_HOST_COMPONENTS - || !journal.explicit_confirmation - || matches!( - journal.host, - HostKindV1::Hermes if journal.hermes_profile_bindings != 1 - ) - || matches!( - journal.host, - host if host != HostKindV1::Hermes && journal.hermes_profile_bindings != 0 - ) - { - return Err(HostBundleError::ReceiptCorrupted); - } - let preview_authority = [ - journal.confirmed_plan_digest, - journal.base_registration_revision, - journal.current_registration_revision, - journal.artifact_state_revision, - ]; - if preview_authority.iter().any(Option::is_some) - && preview_authority.iter().any(Option::is_none) - { - return Err(HostBundleError::ReceiptCorrupted); - } - // The recorded phase and the two registration flags are not independent: - // the writer raises each flag before the hook it names and advances the - // phase after that hook returns. A journal claiming a phase its flags - // cannot support was never written by this lifecycle, so recovery must not - // act on its registration story at all. - if !journal.registration_flags_match_state() { - return Err(HostBundleError::ReceiptCorrupted); - } - let mut components = BTreeMap::new(); - let mut paths = BTreeMap::new(); - let mut configuration_authority = None; - for component in &journal.components { - component.manifest.validate_structure()?; - let authority = ( - component.manifest.configuration_snapshot_id.as_str(), - component.manifest.integration_manifest_digest, - component.manifest.catalog_digest, - ); - if let Some(expected) = configuration_authority { - if authority != expected { - return Err(HostBundleError::ReceiptCorrupted); - } - } else { - configuration_authority = Some(authority); - } - if component.manifest.host != journal.host - || components - .insert(component.manifest.component, ()) - .is_some() - || (component.entries.is_empty() - && journal.operation != HostBundleLifecycleOpV1::Uninstall) - || component.entries.len() > MAX_MANIFEST_ARTIFACTS - { - return Err(HostBundleError::ReceiptCorrupted); - } - if let Some(receipt) = &component.previous_receipt { - validate_receipt(receipt)?; - if receipt.host != journal.host || receipt.component != component.manifest.component { - return Err(HostBundleError::ReceiptCorrupted); - } - } - for (index, entry) in component.entries.iter().enumerate() { - validate_relative_install_path(Path::new(&entry.relative_path))?; - if entry - .backup_name - .as_deref() - .is_some_and(|backup| !is_safe_component(backup)) - || component.entries[..index] - .iter() - .any(|previous| previous.relative_path == entry.relative_path) - || paths.insert(entry.relative_path.clone(), ()).is_some() - || (entry.backup_created && entry.backup_name.is_none()) - || (entry.backup_name.is_some() && entry.wrote_new && !entry.backup_created) - || (entry.wrote_new && entry.installed_digest.is_none()) - { - return Err(HostBundleError::ReceiptCorrupted); - } - } - } - Ok(()) -} - -pub(super) fn backup_name(operation_id: [u8; 16], relative_path: &str) -> String { - let mut hasher = Sha256::new(); - hasher.update(operation_id); - hasher.update(relative_path.as_bytes()); - format!("artifact-{}", hex::encode(hasher.finalize())) -} - -pub(super) fn host_bundle_snapshot_name(index: usize, relative_path: &str) -> String { - let digest = Sha256::digest(relative_path.as_bytes()); - format!("{index:03}-{}", hex::encode(&digest[..16])) -} - -pub(super) fn host_bundle_backup_receipt_file(operation_id: [u8; 16]) -> String { - format!("backup-receipt.{}.v1.json", hex::encode(operation_id)) -} - -pub(super) fn host_bundle_restore_receipt_file(operation_id: [u8; 16]) -> String { - format!("restore-receipt.{}.v1.json", hex::encode(operation_id)) -} - -pub(super) fn component_set_stage_name(component: HostComponentV1, relative_path: &str) -> String { - let mut hasher = Sha256::new(); - hasher.update(component_slug(component).as_bytes()); - hasher.update(relative_path.as_bytes()); - format!( - "{}-{}", - component_slug(component), - hex::encode(hasher.finalize()) - ) -} - pub(super) fn receipt_file(host: HostKindV1, component: HostComponentV1) -> String { format!( "receipt.{}.{}.v1.json", @@ -557,29 +331,14 @@ pub(super) fn receipt_file(host: HostKindV1, component: HostComponentV1) -> Stri ) } -/// Host-scoped component-set journal name. +/// Host-scoped writer lock name. /// -/// Blast-radius argument for per-host isolation: every host deploys its -/// artifacts under its own disjoint subtree of the artifact root -/// (`.claude/…`, `.codex/…`, `.cursor/…`, `.config/opencode/…`, +/// Every host deploys its artifacts under its own disjoint subtree of the +/// artifact root (`.claude/…`, `.codex/…`, `.cursor/…`, `.config/opencode/…`, /// `.kimi-code/…`, `.hermes/…`, `.kiro/…`, `.cline/…`, `.roo/…`, -/// `.config/kilo/…`), and backups plus staging directories are keyed by -/// `operation_id`. A pending transaction for host X therefore shares no -/// mutable path with a transaction for host Y, so X awaiting recovery is not a -/// reason to refuse Y. `first_party_host_artifact_prefixes_are_disjoint` -/// pins that premise as a test, so a future host that violates it fails the -/// suite rather than silently widening the blast radius. The receipt namespace -/// is already host-scoped (`receipt_file`). The writer lock is host-scoped -/// too (`writer_lock_file`): one host's in-flight mutation is a real -/// invariant, a second host's is not. -pub(super) fn component_set_journal_file(host: HostKindV1) -> String { - format!("component-set-journal.{}.v1.json", host.descriptor().slug()) -} - -pub(super) fn journal_file(host: HostKindV1) -> String { - format!("journal.{}.v1.json", host.descriptor().slug()) -} - +/// `.config/kilo/…`), so one host's in-flight mutation shares no mutable path +/// with another's. `first_party_host_artifact_prefixes_are_disjoint` pins that +/// premise as a test. pub(super) fn writer_lock_file(host: HostKindV1) -> String { format!("writer.{}.v1.lock", host.descriptor().slug()) } diff --git a/crates/tracedecay-agent-hosts/src/agents/host_bundle/doctor.rs b/crates/tracedecay-agent-hosts/src/agents/host_bundle/doctor.rs index 09eb031ce3..abb2e314b8 100644 --- a/crates/tracedecay-agent-hosts/src/agents/host_bundle/doctor.rs +++ b/crates/tracedecay-agent-hosts/src/agents/host_bundle/doctor.rs @@ -9,18 +9,14 @@ use serde::Serialize; use tracedecay_host_integration::host_bundle_storage_failure; use super::control::{ - HOST_BUNDLE_CONTROL_DIR, HOST_BUNDLE_JOURNAL_FILE, HOST_COMPONENT_SET_JOURNAL_FILE, - MAX_CONTROL_FILE_BYTES, component_set_journal_file, component_slug, journal_file, receipt_file, - receipt_identity_from_file_name, validate_component_set_journal, validate_journal, - validate_receipt, + HOST_BUNDLE_CONTROL_DIR, MAX_CONTROL_FILE_BYTES, component_slug, parse_receipt, receipt_file, + receipt_identity_from_file_name, validate_receipt, }; use super::planner::{ObservedArtifactKindV1, ObservedHostArtifactV1, observe_artifact_at}; use super::{ - HostBundleArtifactV1, HostBundleError, HostBundleInstallReceiptV1, HostBundleJournalV1, - HostBundleLifecycleOpV1, HostBundleRollbackBoundaryV1, HostComponentSetJournalV1, + HostBundleArtifactV1, HostBundleError, HostBundleInstallReceiptV1, HostBundleLifecycleOpV1, HostComponentV1, HostEditStopConformanceEvidenceV1, HostKindV1, HostNativeFixtureEvidenceV1, - native_host_edit_stop_conformance_evidence, stock_host_kinds, - supported_host_edit_stop_conformance_evidence, + native_host_edit_stop_conformance_evidence, supported_host_edit_stop_conformance_evidence, }; #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] @@ -55,7 +51,7 @@ pub trait HostBundleRegistrationInspectorV1 { /// Read-only classification of one installed component (or one of its /// artifacts). This type is `Serialize`-only and is never persisted into a -/// receipt, journal, or any other durable control file, it exists solely for +/// receipt or any other durable control file, it exists solely for /// the transient [`HostBundleDoctorReportV1`]. Adding a variant therefore /// widens the doctor's reported vocabulary without making any previously /// written artifact unreadable. @@ -67,7 +63,7 @@ pub enum HostBundleComponentDoctorStateV1 { /// A receipt-owned artifact whose ownership marker is still this /// component's own but whose bytes moved away from the recorded digest. /// This is ordinary content drift, not a contested path: `Repair` plans it - /// as `BackupThenReplace` (see `plan_artifact_action`), so reinstall + /// as `Replace` (see `plan_artifact_action`), so reinstall /// converges without an operator first resolving a foreign claim. Drifted, OwnershipConflict, @@ -87,6 +83,9 @@ pub enum HostBundleComponentDoctorStateV1 { ActivationDeferred, Missing, Corrupt, + /// The receipt was written under an older receipt schema. It is not + /// migrated; an adopting install replaces it. + ReinstallRequired, } #[derive(Clone, Debug, PartialEq, Eq, Serialize)] @@ -192,17 +191,22 @@ pub fn inspect_installed_host_bundle_components_at( continue; } }; - let receipt = - if let Ok(receipt) = serde_json::from_slice::(&bytes) { - receipt - } else { - components.push(corrupt_component_result( + let receipt = match parse_receipt::(&bytes) { + Ok(receipt) => receipt, + Err(error) => { + let mut result = corrupt_component_result( receipt_path, receipt_identity.map(|identity| identity.0), receipt_identity.map(|identity| identity.1), - )); + ); + if error == HostBundleError::ReinstallRequired { + result.state = HostBundleComponentDoctorStateV1::ReinstallRequired; + result.repair_action = "run `tracedecay install --yes --adopt`".to_string(); + } + components.push(result); continue; - }; + } + }; if validate_receipt(&receipt).is_err() { components.push(corrupt_component_result( receipt_path, @@ -305,10 +309,9 @@ pub fn inspect_installed_host_bundle_components_at( state, }); } - let state = if receipt.rollback_boundary != HostBundleRollbackBoundaryV1::Passed - || artifacts - .iter() - .any(|artifact| artifact.state == HostBundleComponentDoctorStateV1::Corrupt) + let state = if artifacts + .iter() + .any(|artifact| artifact.state == HostBundleComponentDoctorStateV1::Corrupt) { HostBundleComponentDoctorStateV1::Corrupt } else if artifacts @@ -365,109 +368,6 @@ pub fn inspect_installed_host_bundle_components_at( repair_action: component_repair_action, }); } - // Single-component journals are host-scoped; the legacy shared name is - // still inspected so a journal left by an older binary stays visible. - let journal_paths = std::iter::once(HOST_BUNDLE_JOURNAL_FILE.to_string()) - .chain(stock_host_kinds().into_iter().map(journal_file)) - .map(|file| control_root.join(file)); - for journal_path in journal_paths { - if journal_path.exists() { - let journal = fs::read(&journal_path) - .ok() - .filter(|bytes| !bytes.is_empty() && bytes.len() <= MAX_CONTROL_FILE_BYTES) - .and_then(|bytes| serde_json::from_slice::(&bytes).ok()) - .filter(|journal| validate_journal(journal).is_ok()); - match journal { - Some(journal) => { - if let Some(component) = components.iter_mut().find(|component| { - component.host == Some(journal.host) - && component.component == Some(journal.component) - }) { - component.state = HostBundleComponentDoctorStateV1::Repairable; - component.repair_action = repair_action( - journal.host, - journal.component, - HostBundleComponentDoctorStateV1::Repairable, - HostBundleRegistrationStateV1::Current, - ); - } else { - components.push(HostBundleComponentDoctorResultV1 { - receipt_path: journal_path.clone(), - host: Some(journal.host), - component: Some(journal.component), - state: HostBundleComponentDoctorStateV1::Repairable, - registration: None, - artifacts: Vec::new(), - repair_action: repair_action( - journal.host, - journal.component, - HostBundleComponentDoctorStateV1::Repairable, - HostBundleRegistrationStateV1::Current, - ), - }); - } - } - None => components.push(corrupt_component_result(journal_path, None, None)), - } - } - } - // Component-set journals are host-scoped; the legacy shared name is still - // inspected so a journal left by an older binary stays visible to doctor. - let component_set_journal_paths = std::iter::once(HOST_COMPONENT_SET_JOURNAL_FILE.to_string()) - .chain( - stock_host_kinds() - .into_iter() - .map(component_set_journal_file), - ) - .map(|file| control_root.join(file)); - for component_set_journal_path in component_set_journal_paths { - if component_set_journal_path.exists() { - let journal = fs::read(&component_set_journal_path) - .ok() - .filter(|bytes| !bytes.is_empty() && bytes.len() <= MAX_CONTROL_FILE_BYTES) - .and_then(|bytes| serde_json::from_slice::(&bytes).ok()) - .filter(|journal| validate_component_set_journal(journal).is_ok()); - match journal { - Some(journal) => { - for set_component in journal.components { - let host = set_component.manifest.host; - let component = set_component.manifest.component; - if let Some(result) = components.iter_mut().find(|result| { - result.host == Some(host) && result.component == Some(component) - }) { - result.state = HostBundleComponentDoctorStateV1::Repairable; - result.repair_action = repair_action( - host, - component, - HostBundleComponentDoctorStateV1::Repairable, - HostBundleRegistrationStateV1::Current, - ); - } else { - components.push(HostBundleComponentDoctorResultV1 { - receipt_path: component_set_journal_path.clone(), - host: Some(host), - component: Some(component), - state: HostBundleComponentDoctorStateV1::Repairable, - registration: None, - artifacts: Vec::new(), - repair_action: repair_action( - host, - component, - HostBundleComponentDoctorStateV1::Repairable, - HostBundleRegistrationStateV1::Current, - ), - }); - } - } - } - None => components.push(corrupt_component_result( - component_set_journal_path, - None, - None, - )), - } - } - } for entry in fs::read_dir(&control_root) .map_err(|_| host_bundle_storage_failure!())? .filter_map(Result::ok) @@ -486,9 +386,6 @@ pub fn inspect_installed_host_bundle_components_at( components.push(corrupt_component_result(path, None, None)); continue; }; - if value.get("status").and_then(serde_json::Value::as_str) == Some("restored") { - continue; - } let Some(host) = value .get("host") .cloned() @@ -572,7 +469,7 @@ fn receipt_ownership_claims(receipt_paths: &[PathBuf]) -> BTreeMap format!( - "run `tracedecay reinstall --component {component}` (backs up and refreshes tracedecay-owned files)" + "run `tracedecay reinstall --component {component}` (refreshes tracedecay-owned files)" ), + HostBundleComponentDoctorStateV1::ReinstallRequired => { + "run `tracedecay install --yes --adopt`".to_string() + } HostBundleComponentDoctorStateV1::OrphanedRegistration => format!( "{host} still registers {component} with no owning receipt; run `tracedecay uninstall --agent {host} --component {component} --yes` to finish removing it, or `tracedecay reinstall --component {component} --yes` to re-own it" ), diff --git a/crates/tracedecay-agent-hosts/src/agents/host_bundle/mod.rs b/crates/tracedecay-agent-hosts/src/agents/host_bundle/mod.rs index de2053c572..40627cfa45 100644 --- a/crates/tracedecay-agent-hosts/src/agents/host_bundle/mod.rs +++ b/crates/tracedecay-agent-hosts/src/agents/host_bundle/mod.rs @@ -6,7 +6,7 @@ //! daemon lifecycle, product semantics, or host-specific business authority. //! //! The lifecycle is split along its seams: `planner` observes and plans, -//! `writer` and `component_set` mutate under a recoverable journal, +//! `writer` and `component_set` mutate with in-memory rollback, //! `doctor` discovers installed state, `control` owns the control //! directory layout and validators, and `runtime` composes injected //! verifier and storage authorities. Every public item is re-exported here so @@ -18,17 +18,14 @@ pub use tracedecay_host_integration::{ ClineFamilyAdmissionV1, ClineFamilyEvidenceV1, ClineFamilyProviderV1, EmbeddedHostIntegrationEvidenceV1, EmbeddedNativeHostFixtureV1, HOST_BUNDLE_RECEIPT_SCHEMA_VERSION, HOST_BUNDLE_SCHEMA_VERSION, HostBundleArtifactContentV1, - HostBundleArtifactV1, HostBundleBackupArtifactV1, HostBundleBackupReceiptV1, HostBundleError, - HostBundleInstallReceiptV1, HostBundleJournalEntryV1, HostBundleJournalStateV1, - HostBundleJournalV1, HostBundleLifecycleOpV1, HostBundleManifestV1, - HostBundleReceiptArtifactV1, HostBundleRestoreReceiptV1, HostBundleRollbackBoundaryV1, - HostBundleVerificationAdapterV1, HostCapabilityRecordV1, HostCapabilityStateV1, - HostCapabilityUnavailableReasonV1, HostCapabilityV1, HostComponentSetJournalComponentV1, - HostComponentSetJournalStateV1, HostComponentSetJournalV1, HostComponentSetReceiptV1, - HostComponentV1, HostEditStopConformanceEvidenceV1, HostFeedbackBoundaryEvidenceV1, - HostFeedbackBoundaryV1, HostKindV1, HostNativeFixtureEvidenceV1, HostRegistrationEvidenceV1, - HostRegistrationRouteV1, MAX_ARTIFACT_CONTENT_BYTES, MAX_HOST_COMPONENTS, - MAX_MANIFEST_ARTIFACTS, MAX_RELATIVE_PATH_BYTES, stock_host_capabilities, validate_identifier, + HostBundleArtifactV1, HostBundleError, HostBundleInstallReceiptV1, HostBundleLifecycleOpV1, + HostBundleManifestV1, HostBundleReceiptArtifactV1, HostBundleVerificationAdapterV1, + HostCapabilityRecordV1, HostCapabilityStateV1, HostCapabilityUnavailableReasonV1, + HostCapabilityV1, HostComponentSetReceiptV1, HostComponentV1, + HostEditStopConformanceEvidenceV1, HostFeedbackBoundaryEvidenceV1, HostFeedbackBoundaryV1, + HostKindV1, HostNativeFixtureEvidenceV1, HostRegistrationEvidenceV1, HostRegistrationRouteV1, + MAX_ARTIFACT_CONTENT_BYTES, MAX_HOST_COMPONENTS, MAX_MANIFEST_ARTIFACTS, + MAX_RELATIVE_PATH_BYTES, stock_host_capabilities, validate_identifier, validate_relative_install_path, }; use tracedecay_host_integration::{ @@ -52,9 +49,7 @@ mod writer; pub use capability_admission::{require_capability, require_component_capabilities}; pub use component_set::HostComponentSetTransactionV1; -pub use control::{ - host_bundle_backup_root, latest_host_component_receipt_at, latest_host_component_set_receipt_at, -}; +pub use control::{latest_host_component_receipt_at, latest_host_component_set_receipt_at}; pub use doctor::{ HostBundleArtifactDoctorResultV1, HostBundleComponentDoctorResultV1, HostBundleComponentDoctorStateV1, HostBundleDoctorReportV1, HostBundleRegistrationInspectorV1, @@ -82,8 +77,8 @@ pub use runtime::{ pub use writer::HostBundleWriterV1; /// Resolve the lifecycle authority from the active `TraceDecay` user profile. -/// Host homes contain deployed artifacts only; receipts, journals, locks, and -/// rollback backups are owned by this profile-scoped root. +/// Host homes contain deployed artifacts only; receipts and locks are owned by +/// this profile-scoped root. pub fn resolved_host_bundle_lifecycle_root() -> tracedecay_domain::errors::Result { Ok(tracedecay_runtime_core::storage::default_profile_root()?.join("host-components")) } diff --git a/crates/tracedecay-agent-hosts/src/agents/host_bundle/model.rs b/crates/tracedecay-agent-hosts/src/agents/host_bundle/model.rs index b87bfc2e01..65b1b0b594 100644 --- a/crates/tracedecay-agent-hosts/src/agents/host_bundle/model.rs +++ b/crates/tracedecay-agent-hosts/src/agents/host_bundle/model.rs @@ -23,8 +23,8 @@ pub struct HostBundleExecutionRequestV1 { } /// One verified component in the canonical set for a host lifecycle operation. -/// The content remains outside receipts and journals; it is staged and checked -/// against the embedded manifest before any host path is changed. +/// The content remains outside receipts; it is checked against the embedded +/// manifest before any host path is changed. #[derive(Clone, Debug, PartialEq, Eq)] pub struct HostComponentSetEntryV1 { pub manifest: HostBundleManifestV1, @@ -55,8 +55,8 @@ pub struct HostComponentSetLifecycleRequestV1 { pub explicit_adoption: bool, } -/// One operation id spans every component, registration mutation, receipt, -/// backup, and recovery record in a component-set transaction. +/// One operation id spans every component, registration mutation, and receipt +/// in a component-set transaction. #[derive(Clone, Debug, PartialEq, Eq)] pub struct HostComponentSetExecutionRequestV1 { pub lifecycle: HostComponentSetLifecycleRequestV1, @@ -96,13 +96,12 @@ pub struct HostBundleRollbackSeamV1 { pub operation_id: [u8; 16], pub host: HostKindV1, pub component: HostComponentV1, - pub backup_relative_paths: Vec, - pub interrupted_recovery_required: bool, + pub replaced_relative_paths: Vec, } /// Read-only lifecycle result. Producing this value verifies the embedded /// first-party manifest and exact ownership observations but never opens a -/// writer, creates a control directory, writes a receipt, or recovers a journal. +/// writer, creates a control directory, or writes a receipt. #[derive(Clone, Debug, PartialEq, Eq)] pub struct HostBundleLifecyclePreviewV1 { pub plan: HostBundleMutationPlanV1, @@ -115,8 +114,6 @@ pub struct HostBundleLifecyclePreviewV1 { /// implements this trait, while daemon wiring can provide its opened authority /// without exposing a filesystem path or mutation capability to callers. pub trait HostBundleLifecycleStorageV1 { - fn recover_lifecycle(&mut self) -> Result<(), HostBundleError>; - fn execute_lifecycle( &mut self, manifest: &HostBundleManifestV1, @@ -127,9 +124,9 @@ pub trait HostBundleLifecycleStorageV1 { } /// Host-native registration boundary coordinated with an artifact component -/// set. Implementations persist their own bounded registration backups during -/// `stage`; the aggregate writer records the state transition in its recovery -/// journal and invokes these hooks in reverse on failure or restart. +/// set. Implementations snapshot their bounded registration state in memory +/// during `stage`; the aggregate writer invokes these hooks in reverse when the +/// running operation fails. pub trait HostComponentSetRegistrationV1 { /// Exact revision of the host registration state that this adapter may /// mutate. Concrete host adapters hash their bounded native config; @@ -142,18 +139,6 @@ pub trait HostComponentSetRegistrationV1 { Ok(Sha256::digest(b"tracedecay.host-registration.none.v1").into()) } - /// Recognize this host component's receiptless deployment as a prior - /// first-party install ("legacy provenance"). Pre-receipt installers - /// wrote cataloged deploy paths without v2 receipts, so receipt evidence - /// alone cannot tell their files from a user's; a cataloged path alone - /// must never be treated as ownership. Implementations inspect durable - /// host state (for example a bundle's own manifest naming tracedecay) - /// and fail closed: the default recognizes nothing, so adoption then - /// requires the operator's explicit `--yes --adopt`. - fn receiptless_component_provenance(&self, _component: HostComponentV1) -> bool { - false - } - /// Bounded read-only discovery of third-party extensions that already /// claim a surface this component set would register. Discovery reports; /// it never grants authority to disable, replace, or adopt the competing @@ -169,7 +154,7 @@ pub trait HostComponentSetRegistrationV1 { /// Bind the adapter to the confirmed preview immediately before staging. /// Implementations may retain the revision and recheck it while capturing - /// their rollback backup. + /// their rollback snapshot. fn confirm_preview( &mut self, component_set: &HostComponentSetV1, diff --git a/crates/tracedecay-agent-hosts/src/agents/host_bundle/planner.rs b/crates/tracedecay-agent-hosts/src/agents/host_bundle/planner.rs index 6a40cb510d..106ec88de6 100644 --- a/crates/tracedecay-agent-hosts/src/agents/host_bundle/planner.rs +++ b/crates/tracedecay-agent-hosts/src/agents/host_bundle/planner.rs @@ -1,7 +1,7 @@ //! Ownership-aware lifecycle planning and read-only previews. //! //! Everything here observes host state and produces immutable mutation plans; -//! nothing writes a host path, a receipt, or a journal. +//! nothing writes a host path or a receipt. use std::collections::BTreeMap; use std::fs; @@ -107,8 +107,8 @@ pub struct ObservedHostArtifactV1 { pub enum HostArtifactActionV1 { Noop, WriteNew, - BackupThenReplace, - BackupThenRemove, + Replace, + Remove, } #[derive(Clone, Debug, PartialEq, Eq, Serialize)] @@ -128,11 +128,9 @@ pub struct HostBundleLifecycleRequestV1 { pub hermes_profile_bindings: u8, /// Authorization to adopt receiptless observations at this component's /// cataloged deploy paths. True only when the operator explicitly - /// confirmed adoption (`--yes --adopt`) or the host adapter recognized - /// the receiptless deployment as a prior first-party bundle - /// ([`HostComponentSetRegistrationV1::receiptless_component_provenance`]). - /// A cataloged deploy path alone never grants this; byte-identical - /// staged deploys are adoptable without it. + /// confirmed adoption (`--yes --adopt`). A cataloged deploy path alone + /// never grants this; byte-identical staged deploys are adoptable without + /// it. pub adopt_receiptless: bool, } @@ -200,7 +198,7 @@ pub fn plan_lifecycle_mutation( let rollback_required = mutations.iter().any(|mutation| { matches!( mutation.action, - HostArtifactActionV1::BackupThenReplace | HostArtifactActionV1::BackupThenRemove + HostArtifactActionV1::Replace | HostArtifactActionV1::Remove ) }); Ok(HostBundleMutationPlanV1 { @@ -286,7 +284,7 @@ pub fn plan_complete_lifecycle_mutation( plan.rollback_required = plan.mutations.iter().any(|mutation| { matches!( mutation.action, - HostArtifactActionV1::BackupThenReplace | HostArtifactActionV1::BackupThenRemove + HostArtifactActionV1::Replace | HostArtifactActionV1::Remove ) }); Ok(plan) @@ -317,9 +315,8 @@ pub(super) fn plan_artifact_action( } if state.ownership_marker.as_deref() != Some(artifact.ownership_marker.as_str()) { // Receiptless artifacts are adoptable only inside the boundary - // `adopts_pre_receipt_artifact` defines: byte-identical staged bytes, - // host-recognized legacy provenance, or the operator's explicit - // adoption. Everything else with a foreign or absent marker conflicts. + // `adopts_pre_receipt_artifact` defines: byte-identical staged bytes + // or the operator's explicit adoption. Everything else with a foreign or absent marker conflicts. if !adopts_pre_receipt_artifact(operation, artifact, state, adopt_receiptless) { let reason = if let Some(marker) = state.ownership_marker.as_deref() { format!( @@ -338,8 +335,7 @@ pub(super) fn plan_artifact_action( ) } else { "a receiptless file at a cataloged deploy path is adopted only when it matches \ - the staged bytes, carries recognizable legacy first-party provenance, or the \ - operator re-runs with `--yes --adopt`" + the staged bytes or the operator re-runs with `--yes --adopt`" .to_string() }; return Err(HostBundleError::OwnershipConflict(format!( @@ -350,7 +346,7 @@ pub(super) fn plan_artifact_action( return Ok(if state.artifact_digest == Some(artifact.artifact_digest) { HostArtifactActionV1::Noop } else { - HostArtifactActionV1::BackupThenReplace + HostArtifactActionV1::Replace }); } let owned_digest = state @@ -359,7 +355,7 @@ pub(super) fn plan_artifact_action( match operation { HostBundleLifecycleOpV1::Uninstall => { if state.artifact_digest == Some(owned_digest) { - Ok(HostArtifactActionV1::BackupThenRemove) + Ok(HostArtifactActionV1::Remove) } else { Err(HostBundleError::OwnershipConflict(format!( "{}: deployed bytes no longer match the receipt-owned content; refusing to \ @@ -377,7 +373,7 @@ pub(super) fn plan_artifact_action( if state.artifact_digest == Some(artifact.artifact_digest) { Ok(HostArtifactActionV1::Noop) } else if state.artifact_digest == Some(owned_digest) { - Ok(HostArtifactActionV1::BackupThenReplace) + Ok(HostArtifactActionV1::Replace) } else { Err(HostBundleError::OwnershipConflict(format!( "{}: deployed file was modified outside TraceDecay since its receipt was \ @@ -390,7 +386,7 @@ pub(super) fn plan_artifact_action( if state.artifact_digest == Some(artifact.artifact_digest) { Ok(HostArtifactActionV1::Noop) } else { - Ok(HostArtifactActionV1::BackupThenReplace) + Ok(HostArtifactActionV1::Replace) } } } @@ -413,13 +409,6 @@ pub(super) fn plan_artifact_action( /// TraceDecay staged. Paths inside TraceDecay's own staging namespace /// ([`HOST_BUNDLE_STAGE_ROOT_RELATIVE`]) extend this to divergent bytes, /// because everything there is TraceDecay-staged by construction; -/// * recognizable legacy provenance: the host adapter inspected the -/// receiptless deployment and recognized a prior first-party bundle -/// ([`HostComponentSetRegistrationV1::receiptless_component_provenance`]), -/// e.g. a Cursor plugin directory whose own manifest names tracedecay. -/// Live pre-receipt bundles restamp versions and binary paths every -/// release, so they are never byte-identical, provenance is what lets -/// `install`/`update-plugin` converge them without wedging; /// * explicit operator adoption: `--yes --adopt` claimed the path knowingly. /// /// `Uninstall` never adopts: it must not delete a file whose ownership it @@ -606,8 +595,8 @@ fn component_set_plan_digest( /// Read-only host-root preview used by the official CLI. It verifies the /// manifest, reads existing receipts and artifact digests, and produces the -/// same immutable plan as apply without creating control files, backups, or -/// directories and without recovering an interrupted journal. +/// same immutable plan as apply without creating control files or +/// directories. pub fn dry_run_host_bundle_lifecycle_at( root: &Path, manifest: &HostBundleManifestV1, @@ -705,13 +694,13 @@ pub fn dry_run_host_bundle_lifecycle_with_lifecycle_root_at( &orphan_observed, verifier, )?; - let backup_relative_paths = plan + let replaced_relative_paths = plan .mutations .iter() .filter(|mutation| { matches!( mutation.action, - HostArtifactActionV1::BackupThenReplace | HostArtifactActionV1::BackupThenRemove + HostArtifactActionV1::Replace | HostArtifactActionV1::Remove ) }) .map(|mutation| mutation.relative_path.clone()) @@ -724,8 +713,7 @@ pub fn dry_run_host_bundle_lifecycle_with_lifecycle_root_at( operation_id: request.operation_id, host: manifest.host, component: manifest.component, - backup_relative_paths, - interrupted_recovery_required: plan.rollback_required, + replaced_relative_paths, }, plan, }) @@ -773,11 +761,7 @@ pub fn dry_run_host_component_set_lifecycle_with_lifecycle_root_at< expected_component: component.manifest.component, explicit_confirmation: true, hermes_profile_bindings: planning_request.lifecycle.hermes_profile_bindings, - adopt_receiptless: component_receiptless_adoption( - &planning_request, - registration, - component.manifest.component, - ), + adopt_receiptless: planning_request.lifecycle.explicit_adoption, }, operation_id: planning_request.operation_id, }; @@ -832,19 +816,6 @@ pub fn dry_run_host_component_set_lifecycle_with_lifecycle_root_at< }) } -/// Resolve per-component receiptless-adoption authority for a set request: -/// the operator's explicit `--adopt` or the adapter's recognized legacy -/// provenance. Preview and confirmed execute both resolve through this, so -/// their plans agree; provenance drift between them surfaces as the ordinary -/// plan/`StalePreview` mismatch. -pub(super) fn component_receiptless_adoption( - request: &HostComponentSetExecutionRequestV1, - registration: &R, - component: HostComponentV1, -) -> bool { - request.lifecycle.explicit_adoption || registration.receiptless_component_provenance(component) -} - /// Collect and normalise the adapter's claim discovery so preview, plan /// digest, and apply all compare the same canonical ordering. fn discovered_competing_extension_claims( diff --git a/crates/tracedecay-agent-hosts/src/agents/host_bundle/runtime.rs b/crates/tracedecay-agent-hosts/src/agents/host_bundle/runtime.rs index 7ae6e52018..514ab2087b 100644 --- a/crates/tracedecay-agent-hosts/src/agents/host_bundle/runtime.rs +++ b/crates/tracedecay-agent-hosts/src/agents/host_bundle/runtime.rs @@ -14,13 +14,12 @@ use super::planner::{ }; use super::{ HostBundleArtifactContentV1, HostBundleError, HostBundleInstallReceiptV1, - HostBundleLifecycleOpV1, HostBundleManifestV1, HostBundleRollbackBoundaryV1, - HostBundleVerificationAdapterV1, HostKindV1, + HostBundleLifecycleOpV1, HostBundleManifestV1, HostBundleVerificationAdapterV1, HostKindV1, }; /// Production-composition seam for independently injected cryptographic and -/// filesystem authorities. It verifies before it asks storage to recover or -/// mutate, so an incompatible catalog entry cannot trigger filesystem access. +/// filesystem authorities. It verifies before it asks storage to mutate, so an +/// incompatible catalog entry cannot trigger filesystem access. pub struct HostBundleLifecycleRuntimeV1 { verifier: V, storage: S, @@ -66,14 +65,13 @@ where orphan_observed, &self.verifier, )?; - let backup_relative_paths = plan + let replaced_relative_paths = plan .mutations .iter() .filter(|mutation| { matches!( mutation.action, - HostArtifactActionV1::BackupThenReplace - | HostArtifactActionV1::BackupThenRemove + HostArtifactActionV1::Replace | HostArtifactActionV1::Remove ) }) .map(|mutation| mutation.relative_path.clone()) @@ -86,17 +84,12 @@ where operation_id: request.operation_id, host: manifest.host, component: manifest.component, - backup_relative_paths, - interrupted_recovery_required: plan.rollback_required, + replaced_relative_paths, }, plan, }) } - pub fn recover(&mut self) -> Result<(), HostBundleError> { - self.storage.recover_lifecycle() - } - pub fn execute( &mut self, manifest: &HostBundleManifestV1, @@ -104,8 +97,6 @@ where contents: &[HostBundleArtifactContentV1], ) -> Result { self.verifier.verify_manifest(manifest)?; - // execute_lifecycle recovers this manifest's host. Recovering every - // host here would roll back an unrelated host's journal. self.storage .execute_lifecycle(manifest, request, contents, &self.verifier) } @@ -230,9 +221,6 @@ where return Err(HostBundleError::ConfirmationRequired); } validate_receipt(&switch_receipt.apply_receipt)?; - if switch_receipt.apply_receipt.rollback_boundary != HostBundleRollbackBoundaryV1::Passed { - return Err(HostBundleError::ReceiptCorrupted); - } if previous_manifest.host != switch_receipt.host || previous_manifest.canonical_digest()? != switch_receipt.previous_manifest_digest || request.lifecycle.operation != HostBundleLifecycleOpV1::Repair diff --git a/crates/tracedecay-agent-hosts/src/agents/host_bundle/tests.rs b/crates/tracedecay-agent-hosts/src/agents/host_bundle/tests.rs index c4adf6b45d..9e7f2c78fc 100644 --- a/crates/tracedecay-agent-hosts/src/agents/host_bundle/tests.rs +++ b/crates/tracedecay-agent-hosts/src/agents/host_bundle/tests.rs @@ -5,10 +5,8 @@ use sha2::{Digest, Sha256}; use tracedecay_host_integration::host_bundle_storage_failure; use super::control::{ - HOST_BUNDLE_CONTROL_DIR, HOST_BUNDLE_JOURNAL_FILE, HOST_BUNDLE_LOCK_FILE, - HOST_COMPONENT_SET_JOURNAL_FILE, component_set_journal_file, expected_ownership_marker, - host_bundle_backup_receipt_file, host_bundle_restore_receipt_file, journal_file, receipt_file, - validate_component_set_journal, writer_lock_file, + HOST_BUNDLE_CONTROL_DIR, HOST_BUNDLE_LOCK_FILE, component_set_receipt_file, + expected_ownership_marker, receipt_file, writer_lock_file, }; use super::doctor::doctor_artifact_state; use super::planner::plan_artifact_action; @@ -250,89 +248,6 @@ impl HostComponentSetRegistrationV1 for FailingSetRegistration { } } -#[test] -fn component_backup_restore_is_durable_idempotent_and_rollback_safe() { - let root = tempfile::tempdir().unwrap(); - let mut writer = HostBundleWriterV1::open(root.path()).unwrap(); - let original = manifest(HostKindV1::Codex, b"original"); - let original_verifier = verifier(&original); - writer - .execute( - &original, - &execution( - HostKindV1::Codex, - HostBundleLifecycleOpV1::Install, - 71, - true, - ), - &content(b"original"), - &original_verifier, - ) - .unwrap(); - assert_eq!( - writer.backup_component(&original, [72; 16], false, &original_verifier), - Err(HostBundleError::ConfirmationRequired) - ); - - let backup = writer - .backup_component(&original, [72; 16], true, &original_verifier) - .unwrap(); - assert_eq!( - writer - .backup_component(&original, [72; 16], true, &original_verifier) - .unwrap(), - backup - ); - - let updated = manifest(HostKindV1::Codex, b"updated"); - writer - .execute( - &updated, - &execution(HostKindV1::Codex, HostBundleLifecycleOpV1::Update, 73, true), - &content(b"updated"), - &verifier(&updated), - ) - .unwrap(); - assert_eq!( - std::fs::read(root.path().join("plugins/tracedecay.json")).unwrap(), - b"updated" - ); - assert_eq!( - writer.restore_component_backup([72; 16], [74; 16], false, &original_verifier), - Err(HostBundleError::ConfirmationRequired) - ); - - let restored = writer - .restore_component_backup([72; 16], [74; 16], true, &original_verifier) - .unwrap(); - assert_eq!( - writer - .restore_component_backup([72; 16], [74; 16], true, &original_verifier) - .unwrap(), - restored - ); - assert_eq!( - std::fs::read(root.path().join("plugins/tracedecay.json")).unwrap(), - b"original" - ); - assert_eq!( - restored.restored_receipt.rollback_boundary, - HostBundleRollbackBoundaryV1::Passed - ); - assert!( - root.path() - .join(HOST_BUNDLE_CONTROL_DIR) - .join(host_bundle_backup_receipt_file([72; 16])) - .is_file() - ); - assert!( - root.path() - .join(HOST_BUNDLE_CONTROL_DIR) - .join(host_bundle_restore_receipt_file([74; 16])) - .is_file() - ); -} - #[test] fn component_set_transaction_is_idempotent_and_rolls_back_every_component() { let root = tempfile::tempdir().unwrap(); @@ -409,278 +324,73 @@ fn component_set_transaction_is_idempotent_and_rolls_back_every_component() { .operation_id, [21; 16] ); - assert!( - root.path() - .join(HOST_BUNDLE_CONTROL_DIR) - .join(component_set_journal_file(HostKindV1::OpenCode)) - .is_file(), - "a rollback journal must remain available for restart reconciliation" - ); - let journal = writer - .load_component_set_journal_for(HostKindV1::OpenCode) - .unwrap() - .expect("interrupted rollback retains exact lifecycle authority"); - assert!(journal.explicit_confirmation); - assert_eq!(journal.hermes_profile_bindings, 0); - assert_eq!(journal.confirmed_plan_digest, Some(preview.plan_digest)); - assert_eq!( - journal.base_registration_revision, - Some(preview.base_registration_revision) - ); - assert_eq!( - journal.current_registration_revision, - Some(preview.current_registration_revision) - ); - assert_eq!( - journal.artifact_state_revision, - Some(preview.artifact_state_revision) - ); - let doctor = inspect_installed_host_bundle_components_at( - root.path(), - root.path(), - &CurrentRegistration, - crate::agents::TEST_GENERATOR_COMMIT, - ) - .unwrap(); - assert!( - doctor.components.iter().any(|component| { - component.host == Some(HostKindV1::OpenCode) - && component.component == Some(HostComponentV1::Core) - && component.state == HostBundleComponentDoctorStateV1::Repairable - }), - "Doctor keeps the component receipt API while surfacing the aggregate recovery boundary" - ); - // Explicit recover clears the completed rollback journal. Re-open then - // proves a restarted writer can take the lock once recovery finished. - HostComponentSetTransactionV1::new(&mut writer) - .recover(&mut failing_registration) - .unwrap(); - assert!( - !root - .path() - .join(HOST_BUNDLE_CONTROL_DIR) - .join(component_set_journal_file(HostKindV1::OpenCode)) - .exists(), - "restart recovery clears only a completed rollback boundary" - ); + let prior_set_receipt = component_set_receipt_file([21; 16]); + for entry in fs::read_dir(root.path().join(HOST_BUNDLE_CONTROL_DIR)).unwrap() { + let name = entry.unwrap().file_name().to_string_lossy().into_owned(); + assert!( + name.starts_with("receipt.") + || name.starts_with("writer.") + || name == prior_set_receipt, + "a rolled-back operation leaves no journal, backup, staging, or aggregate receipt: {name}" + ); + } drop(writer); - HostBundleWriterV1::open(root.path()).expect("reopen after recovery"); + HostBundleWriterV1::open(root.path()).expect("reopen after rollback"); } -/// Drive the two-component `OpenCode` set to a durable `RolledBack` -/// journal: install, then fail the update at registration verification so -/// the completed rollback boundary is left behind for restart recovery. -fn wedged_rolled_back_component_set_journal(root: &Path) { +/// Rollback puts back every path it can, but never overwrites bytes a second +/// writer placed at a path mid-operation; that path is a typed conflict. +#[test] +fn component_set_rollback_leaves_a_second_writers_bytes_in_place() { + let root = tempfile::tempdir().unwrap(); + let mut writer = HostBundleWriterV1::open(root.path()).unwrap(); let initial = component_set(HostKindV1::OpenCode, b"core-v1", b"agent-v1"); - let initial_request = - component_set_request(HostKindV1::OpenCode, HostBundleLifecycleOpV1::Install, 81); - let initial_verifier = ComponentSetVerifier::from_set(&initial); - let mut writer = HostBundleWriterV1::open(root).unwrap(); HostComponentSetTransactionV1::new(&mut writer) .execute( &initial, - &initial_request, - &initial_verifier, + &component_set_request(HostKindV1::OpenCode, HostBundleLifecycleOpV1::Install, 41), + &ComponentSetVerifier::from_set(&initial), &mut ArtifactOnlyTestRegistration, ) .unwrap(); + let repaired = component_set(HostKindV1::OpenCode, b"core-v2", b"agent-v2"); + let mut registration = SecondWriterRegistration { + artifact_root: root.path().to_path_buf(), + relative_path: "plugins/core.json", + bytes: b"foreign".to_vec(), + rolled_back: false, + }; - let updated = component_set(HostKindV1::OpenCode, b"core-v2", b"agent-v2"); - let update_request = - component_set_request(HostKindV1::OpenCode, HostBundleLifecycleOpV1::Update, 82); - let updated_verifier = ComponentSetVerifier::from_set(&updated); - let mut failing = FailingSetRegistration::default(); - assert_eq!( - HostComponentSetTransactionV1::new(&mut writer).execute( - &updated, - &update_request, - &updated_verifier, - &mut failing, - ), - Err(FAILING_SET_REGISTRATION_VERIFY) - ); - assert_eq!( - writer - .load_component_set_journal_for(HostKindV1::OpenCode) - .unwrap() - .expect("a completed rollback journal remains") - .state, - HostComponentSetJournalStateV1::RolledBack - ); - drop(writer); -} - -/// Rewrite the pending `OpenCode` journal on disk under `file_name`, as a -/// differently shaped binary or a corrupted control file would leave it. -fn reshape_component_set_journal( - root: &Path, - file_name: &str, - reshape: impl FnOnce(&mut HostComponentSetJournalV1), -) { - let control = root.join(HOST_BUNDLE_CONTROL_DIR); - let host_scoped = control.join(component_set_journal_file(HostKindV1::OpenCode)); - let mut journal: HostComponentSetJournalV1 = - serde_json::from_slice(&fs::read(&host_scoped).unwrap()).unwrap(); - reshape(&mut journal); - if file_name != component_set_journal_file(HostKindV1::OpenCode) { - fs::remove_file(&host_scoped).unwrap(); - } - fs::write( - control.join(file_name), - serde_json::to_vec(&journal).unwrap(), - ) - .unwrap(); -} - -/// Defect: recovery read `registration_staged`/`registration_applied` as -/// proof that a rolled-back journal owed no host-native compensation. Those -/// flags describe the interrupted attempt, not the outstanding work, so a -/// `RolledBack` journal with both flags clear silently skipped -/// `registration.rollback` and left the native host configuration mutated. -#[test] -fn a_rolled_back_component_set_journal_compensates_registration_without_its_flags() { - let root = tempfile::tempdir().unwrap(); - wedged_rolled_back_component_set_journal(root.path()); - reshape_component_set_journal( - root.path(), - &component_set_journal_file(HostKindV1::OpenCode), - |journal| { - journal.registration_staged = false; - journal.registration_applied = false; - }, - ); + let error = HostComponentSetTransactionV1::new(&mut writer) + .execute( + &repaired, + &component_set_request(HostKindV1::OpenCode, HostBundleLifecycleOpV1::Repair, 42), + &ComponentSetVerifier::from_set(&repaired), + &mut registration, + ) + .unwrap_err(); - let mut writer = HostBundleWriterV1::open(root.path()).unwrap(); - let mut registration = FailingSetRegistration::default(); - HostComponentSetTransactionV1::new(&mut writer) - .recover_host(HostKindV1::OpenCode, &mut registration) - .unwrap(); - assert!( - registration.rolled_back, - "a rolled-back journal must always re-attempt registration compensation" - ); assert!( - !root - .path() - .join(HOST_BUNDLE_CONTROL_DIR) - .join(component_set_journal_file(HostKindV1::OpenCode)) - .exists(), - "recovery still clears the completed rollback boundary" + matches!(error, HostBundleError::OwnershipConflict(_)), + "{error}" ); -} - -/// The recorded phase and the registration flags are not independent, so a -/// journal claiming a phase its flags cannot support was never written by -/// this lifecycle. It must be refused at load rather than recovered from. -#[test] -fn an_unrepresentable_component_set_journal_phase_and_flag_pair_is_rejected() { - use HostComponentSetJournalStateV1 as State; - - let root = tempfile::tempdir().unwrap(); - wedged_rolled_back_component_set_journal(root.path()); - reshape_component_set_journal( - root.path(), - &component_set_journal_file(HostKindV1::OpenCode), - |journal| { - journal.state = HostComponentSetJournalStateV1::Applied; - journal.registration_staged = false; - journal.registration_applied = false; - }, - ); - - let mut writer = HostBundleWriterV1::open(root.path()).unwrap(); - let mut registration = FailingSetRegistration::default(); + assert!(registration.rolled_back); assert_eq!( - HostComponentSetTransactionV1::new(&mut writer) - .recover_host(HostKindV1::OpenCode, &mut registration), - Err(HostBundleError::ReceiptCorrupted) + fs::read(root.path().join("plugins/core.json")).unwrap(), + b"foreign" ); assert_eq!( - writer.pending_component_set_journal_operation(HostKindV1::OpenCode), - Err(HostBundleError::ReceiptCorrupted) - ); - assert!( - !registration.rolled_back, - "a journal no writer could produce never drives host-native compensation" + fs::read(root.path().join("plugins/agent.json")).unwrap(), + b"agent-v1", + "the untouched path is still put back" ); - - let journal: HostComponentSetJournalV1 = serde_json::from_slice( - &fs::read( - root.path() - .join(HOST_BUNDLE_CONTROL_DIR) - .join(component_set_journal_file(HostKindV1::OpenCode)), - ) - .unwrap(), - ) - .unwrap(); - for (state, staged, applied, representable) in [ - (State::Prepared, false, false, true), - (State::Prepared, true, false, true), - (State::Prepared, false, true, false), - (State::Staged, true, false, true), - (State::Staged, false, false, false), - (State::Applied, true, true, true), - (State::Applied, true, false, false), - (State::Verified, false, true, false), - (State::Committed, true, true, true), - // Rollback preserves whichever flags the failed attempt reached, so - // every combination is authentic in this state. - (State::RolledBack, false, false, true), - (State::RolledBack, true, false, true), - (State::RolledBack, true, true, true), - ] { - let candidate = HostComponentSetJournalV1 { - state, - registration_staged: staged, - registration_applied: applied, - ..journal.clone() - }; - assert_eq!( - candidate.registration_flags_match_state(), - representable, - "{state:?} staged={staged} applied={applied}" - ); - assert_eq!( - validate_component_set_journal(&candidate).is_ok(), - representable, - "{state:?} staged={staged} applied={applied}" - ); - } -} - -/// A journal written by an older binary lives under the shared legacy name -/// and may carry any flag combination its rollback happened to reach. It -/// must still load, and its compensation must still run. -#[test] -fn a_legacy_named_rolled_back_component_set_journal_still_recovers() { - let root = tempfile::tempdir().unwrap(); - wedged_rolled_back_component_set_journal(root.path()); - reshape_component_set_journal(root.path(), HOST_COMPONENT_SET_JOURNAL_FILE, |journal| { - journal.registration_staged = false; - journal.registration_applied = false; - }); - - let mut writer = HostBundleWriterV1::open(root.path()).unwrap(); assert_eq!( - writer.pending_component_set_journal_hosts().unwrap(), - vec![HostKindV1::OpenCode], - "a legacy-named journal is still discovered" - ); - let mut registration = FailingSetRegistration::default(); - HostComponentSetTransactionV1::new(&mut writer) - .recover(&mut registration) - .unwrap(); - assert!( - registration.rolled_back, - "a legacy rolled-back journal still owes registration compensation" - ); - assert!( - !root - .path() - .join(HOST_BUNDLE_CONTROL_DIR) - .join(HOST_COMPONENT_SET_JOURNAL_FILE) - .exists(), - "recovery retires the legacy boundary it just resolved" + writer + .load_receipt(HostKindV1::OpenCode, HostComponentV1::Core) + .unwrap() + .expect("the prior receipt was never replaced") + .operation_id, + [41; 16] ); } @@ -714,171 +424,6 @@ impl HostComponentSetRegistrationV1 for SecondWriterRegistration { } } -/// Install the two-component `OpenCode` set, then attempt a repair whose -/// registration authority rewrites `plugins/core.json` with `second_bytes`. -fn wedge_repair_with_second_writer( - root: &Path, - second_bytes: &[u8], -) -> ( - HostBundleWriterV1, - Result, -) { - let initial = component_set(HostKindV1::OpenCode, b"core-v1", b"agent-v1"); - let initial_request = - component_set_request(HostKindV1::OpenCode, HostBundleLifecycleOpV1::Install, 41); - let initial_verifier = ComponentSetVerifier::from_set(&initial); - let mut writer = HostBundleWriterV1::open(root).unwrap(); - HostComponentSetTransactionV1::new(&mut writer) - .execute( - &initial, - &initial_request, - &initial_verifier, - &mut ArtifactOnlyTestRegistration, - ) - .unwrap(); - - let repair = component_set(HostKindV1::OpenCode, b"core-v2", b"agent-v2"); - let repair_request = - component_set_request(HostKindV1::OpenCode, HostBundleLifecycleOpV1::Repair, 42); - let repair_verifier = ComponentSetVerifier::from_set(&repair); - let mut registration = SecondWriterRegistration { - artifact_root: root.to_path_buf(), - relative_path: "plugins/core.json", - bytes: second_bytes.to_vec(), - rolled_back: false, - }; - let outcome = HostComponentSetTransactionV1::new(&mut writer).execute( - &repair, - &repair_request, - &repair_verifier, - &mut registration, - ); - assert!(registration.rolled_back, "registration rollback must run"); - (writer, outcome) -} - -/// Defect: a second writer that left the deployed path holding the exact -/// pre-transaction bytes used to make rollback unconvergeable forever, -/// `remove_if_digest_matches` refused to touch a file that no longer -/// matched the installed digest, so the journal stayed behind and wedged -/// every later host transaction. -#[test] -fn component_set_rollback_converges_when_a_second_writer_left_the_backup_bytes() { - let root = tempfile::tempdir().unwrap(); - let (mut writer, outcome) = wedge_repair_with_second_writer(root.path(), b"core-v1"); - - assert_eq!( - outcome.err(), - Some(HostBundleError::ArtifactContentMismatch), - "the failure must surface as the real content mismatch, not RecoveryRequired" - ); - assert_eq!( - fs::read(root.path().join("plugins/core.json")).unwrap(), - b"core-v1", - "the pre-transaction bytes are the converged end state" - ); - assert_eq!( - fs::read(root.path().join("plugins/agent.json")).unwrap(), - b"agent-v1" - ); - assert_eq!( - writer - .load_receipt(HostKindV1::OpenCode, HostComponentV1::Core) - .unwrap() - .expect("the pre-transaction receipt is restored") - .operation_id, - [41; 16] - ); - - // A completed rollback leaves the journal for an explicit restart - // boundary; recovery clears it and the host is usable again. - HostComponentSetTransactionV1::new(&mut writer) - .recover_host(HostKindV1::OpenCode, &mut ArtifactOnlyTestRegistration) - .unwrap(); - assert!( - writer - .pending_component_set_journal_hosts() - .unwrap() - .is_empty() - ); - let next = component_set(HostKindV1::OpenCode, b"core-v3", b"agent-v3"); - let next_request = - component_set_request(HostKindV1::OpenCode, HostBundleLifecycleOpV1::Repair, 43); - let next_verifier = ComponentSetVerifier::from_set(&next); - HostComponentSetTransactionV1::new(&mut writer) - .execute( - &next, - &next_request, - &next_verifier, - &mut ArtifactOnlyTestRegistration, - ) - .expect("the host is no longer wedged"); -} - -/// Genuinely foreign bytes stay fail-closed: converging would silently -/// destroy content this transaction can not account for. The operator -/// resolves it with the explicit recovery verb instead. -#[test] -fn component_set_rollback_stays_fail_closed_for_foreign_bytes() { - let root = tempfile::tempdir().unwrap(); - let (mut writer, outcome) = - wedge_repair_with_second_writer(root.path(), b"foreign-third-party-bytes"); - - assert!(matches!( - outcome.err(), - Some(HostBundleError::RecoveryRequired(_)) - )); - assert_eq!( - fs::read(root.path().join("plugins/core.json")).unwrap(), - b"foreign-third-party-bytes", - "unaccountable content is preserved, never silently discarded" - ); - assert_eq!( - writer.pending_component_set_journal_hosts().unwrap(), - vec![HostKindV1::OpenCode] - ); - // Convergent recovery cannot resolve this, so it still fails closed. - assert!(matches!( - HostComponentSetTransactionV1::new(&mut writer) - .recover_host(HostKindV1::OpenCode, &mut ArtifactOnlyTestRegistration) - .err(), - Some(HostBundleError::RecoveryRequired(_)) - )); - - // The recovery verb's escape hatch: the journal is set aside (not - // deleted) and the immutable backups stay on disk. - let quarantined = writer - .quarantine_component_set_journal(HostKindV1::OpenCode, 1_700_000_000) - .unwrap() - .expect("the pending journal is quarantined"); - assert!(quarantined.is_file()); - assert!( - writer - .pending_component_set_journal_hosts() - .unwrap() - .is_empty() - ); - assert!( - root.path() - .join(HOST_BUNDLE_CONTROL_DIR) - .join("backups") - .exists(), - "quarantine preserves the rollback backups" - ); - let next = component_set(HostKindV1::OpenCode, b"core-v4", b"agent-v4"); - let next_request = - component_set_request(HostKindV1::OpenCode, HostBundleLifecycleOpV1::Repair, 44); - let next_verifier = ComponentSetVerifier::from_set(&next); - HostComponentSetTransactionV1::new(&mut writer) - .execute( - &next, - &next_request, - &next_verifier, - &mut ArtifactOnlyTestRegistration, - ) - .expect("the recovery verb unblocks the host without hand-deleting a journal"); -} - fn host_scoped_component_set(host: HostKindV1, slug: &str, tag: &[u8]) -> HostComponentSetV1 { let core_path = format!("{slug}/core.json"); let agent_path = format!("{slug}/agent.json"); @@ -897,10 +442,8 @@ fn host_scoped_component_set(host: HostKindV1, slug: &str, tag: &[u8]) -> HostCo } } -/// Defect: one `writer.v1.lock` and one `journal.v1.json` still serialized -/// every host after component-set journals were split. A writer that has -/// already admitted OpenCode must not stop Codex, and must not roll back -/// OpenCode's legacy single-component journal. +/// Defect: one `writer.v1.lock` serialized every host. A writer that has +/// already admitted OpenCode must not stop Codex. #[test] fn a_host_lock_does_not_exclude_an_unrelated_host() { let root = tempfile::tempdir().unwrap(); @@ -961,180 +504,12 @@ fn a_host_lock_does_not_exclude_an_unrelated_host() { &mut ArtifactOnlyTestRegistration, ) .err(), - Some(HostBundleError::RecoveryRequired(_)) + Some(HostBundleError::HostWriterBusy) ), "the same host still has exactly one writer" ); } -/// Defect: `journal.v1.json` was still one file for every host. Recovering or -/// installing Codex must not roll back an OpenCode journal left by an older -/// binary. OpenCode's own recovery retires that legacy name. -#[test] -fn a_legacy_single_component_journal_is_attributed_to_its_own_host() { - let root = tempfile::tempdir().unwrap(); - let artifact = root.path().join("opencode/core.json"); - fs::create_dir_all(artifact.parent().unwrap()).unwrap(); - fs::write(&artifact, b"opencode-bytes").unwrap(); - let digest: [u8; 32] = Sha256::digest(b"opencode-bytes").into(); - let journal = HostBundleJournalV1 { - schema_version: HOST_BUNDLE_RECEIPT_SCHEMA_VERSION, - operation_id: [71; 16], - host: HostKindV1::OpenCode, - component: HostComponentV1::Core, - operation: HostBundleLifecycleOpV1::Install, - manifest_digest: digest, - state: HostBundleJournalStateV1::Prepared, - previous_receipt: None, - entries: vec![HostBundleJournalEntryV1 { - relative_path: "opencode/core.json".to_string(), - backup_name: None, - backup_created: false, - wrote_new: true, - installed_digest: Some(digest), - }], - }; - let control = root.path().join(HOST_BUNDLE_CONTROL_DIR); - fs::create_dir_all(&control).unwrap(); - fs::write( - control.join(HOST_BUNDLE_JOURNAL_FILE), - serde_json::to_vec(&journal).unwrap(), - ) - .unwrap(); - - let codex = host_scoped_component_set(HostKindV1::Codex, "codex", b"v1"); - let codex_request = - component_set_request(HostKindV1::Codex, HostBundleLifecycleOpV1::Install, 72); - let mut writer = HostBundleWriterV1::open(root.path()).unwrap(); - HostComponentSetTransactionV1::new(&mut writer) - .execute( - &codex, - &codex_request, - &ComponentSetVerifier::from_set(&codex), - &mut ArtifactOnlyTestRegistration, - ) - .expect("codex install must not refuse on opencode's legacy journal"); - assert_eq!(fs::read(&artifact).unwrap(), b"opencode-bytes"); - assert!( - control.join(HOST_BUNDLE_JOURNAL_FILE).is_file(), - "codex must not retire opencode's legacy journal" - ); - - let doctor = inspect_installed_host_bundle_components_at( - root.path(), - root.path(), - &CurrentRegistration, - crate::agents::TEST_GENERATOR_COMMIT, - ) - .unwrap(); - assert!( - doctor.components.iter().any(|component| { - component.host == Some(HostKindV1::OpenCode) - && component.component == Some(HostComponentV1::Core) - && component.state == HostBundleComponentDoctorStateV1::Repairable - }), - "doctor still reports the legacy single-component journal" - ); - - HostComponentSetTransactionV1::new(&mut writer) - .recover_host(HostKindV1::OpenCode, &mut ArtifactOnlyTestRegistration) - .expect("opencode recovery owns the legacy journal"); - assert!( - !artifact.exists(), - "opencode recovery rolls its own interrupted install back" - ); - assert!(!control.join(HOST_BUNDLE_JOURNAL_FILE).exists()); - assert!(!control.join(journal_file(HostKindV1::OpenCode)).exists()); - assert_eq!( - fs::read(root.path().join("codex/core.json")).unwrap(), - b"v1" - ); -} - -/// Defect: one shared journal per lifecycle root meant a wedged opencode -/// repair blocked codex, cursor, cline, roo-code, kilo, kiro, and kimi in -/// the same `tracedecay reinstall`. Journals are host-scoped now, and the -/// hosts' artifact path spaces are disjoint, so an unrelated host proceeds. -#[test] -fn a_wedged_host_journal_does_not_block_an_unrelated_host() { - let root = tempfile::tempdir().unwrap(); - let wedged = host_scoped_component_set(HostKindV1::OpenCode, "opencode", b"v1"); - let wedged_request = - component_set_request(HostKindV1::OpenCode, HostBundleLifecycleOpV1::Install, 51); - let wedged_verifier = ComponentSetVerifier::from_set(&wedged); - let mut writer = HostBundleWriterV1::open(root.path()).unwrap(); - let mut second_writer = SecondWriterRegistration { - artifact_root: root.path().to_path_buf(), - relative_path: "opencode/core.json", - bytes: b"foreign".to_vec(), - rolled_back: false, - }; - assert!(matches!( - HostComponentSetTransactionV1::new(&mut writer) - .execute( - &wedged, - &wedged_request, - &wedged_verifier, - &mut second_writer, - ) - .err(), - Some(HostBundleError::RecoveryRequired(_)) - )); - assert_eq!( - writer.pending_component_set_journal_hosts().unwrap(), - vec![HostKindV1::OpenCode] - ); - - let unrelated = host_scoped_component_set(HostKindV1::Codex, "codex", b"v1"); - let unrelated_request = - component_set_request(HostKindV1::Codex, HostBundleLifecycleOpV1::Install, 52); - let unrelated_verifier = ComponentSetVerifier::from_set(&unrelated); - HostComponentSetTransactionV1::new(&mut writer) - .execute( - &unrelated, - &unrelated_request, - &unrelated_verifier, - &mut ArtifactOnlyTestRegistration, - ) - .expect("an unrelated host's disjoint path space is not blocked"); - assert_eq!( - fs::read(root.path().join("codex/core.json")).unwrap(), - b"v1" - ); - assert_eq!( - writer.pending_component_set_journal_hosts().unwrap(), - vec![HostKindV1::OpenCode], - "the wedged host still awaits its own recovery" - ); - - // Recovering one host must not touch another host's journal. - let mut also_wedged = SecondWriterRegistration { - artifact_root: root.path().to_path_buf(), - relative_path: "codex/core.json", - bytes: b"foreign".to_vec(), - rolled_back: false, - }; - let repair = host_scoped_component_set(HostKindV1::Codex, "codex", b"v2"); - let repair_request = - component_set_request(HostKindV1::Codex, HostBundleLifecycleOpV1::Repair, 53); - let repair_verifier = ComponentSetVerifier::from_set(&repair); - assert!(matches!( - HostComponentSetTransactionV1::new(&mut writer) - .execute(&repair, &repair_request, &repair_verifier, &mut also_wedged) - .err(), - Some(HostBundleError::RecoveryRequired(_)) - )); - writer - .quarantine_component_set_journal(HostKindV1::Codex, 1_700_000_000) - .unwrap() - .expect("codex journal quarantined"); - assert_eq!( - writer.pending_component_set_journal_hosts().unwrap(), - vec![HostKindV1::OpenCode], - "quarantining one host leaves every other host's journal intact" - ); -} - #[test] fn unchanged_companion_receipt_keeps_original_operation_provenance() { let root = tempfile::tempdir().unwrap(); @@ -1183,8 +558,7 @@ fn unchanged_companion_receipt_keeps_original_operation_provenance() { // still earns a fresh receipt. The change must keep the set's shared // configuration authority (`configuration_snapshot_id`, // `integration_manifest_digest`, `catalog_digest`) uniform across - // components. `validate_component_set_journal` rejects a set whose - // components disagree on it, so bump a per-component manifest field + // components, so bump a per-component manifest field // (`effective_behavior_digest`) that shifts only the agent's canonical // digest and leaves the core component entirely unchanged. let mut metadata_only_change = core_only_change.clone(); @@ -1216,70 +590,6 @@ fn unchanged_companion_receipt_keeps_original_operation_provenance() { ); } -/// A journal written by an older binary lives under the shared legacy name. -/// It must still be discoverable, attributable to exactly one host, and -/// retired once its host-scoped successor is durable. -#[test] -fn a_legacy_shared_component_set_journal_is_attributed_to_its_own_host() { - let root = tempfile::tempdir().unwrap(); - let wedged = host_scoped_component_set(HostKindV1::OpenCode, "opencode", b"v1"); - let wedged_request = - component_set_request(HostKindV1::OpenCode, HostBundleLifecycleOpV1::Install, 61); - let wedged_verifier = ComponentSetVerifier::from_set(&wedged); - let mut writer = HostBundleWriterV1::open(root.path()).unwrap(); - let mut second_writer = SecondWriterRegistration { - artifact_root: root.path().to_path_buf(), - relative_path: "opencode/core.json", - bytes: b"foreign".to_vec(), - rolled_back: false, - }; - assert!( - HostComponentSetTransactionV1::new(&mut writer) - .execute( - &wedged, - &wedged_request, - &wedged_verifier, - &mut second_writer, - ) - .is_err() - ); - let control = root.path().join(HOST_BUNDLE_CONTROL_DIR); - fs::rename( - control.join(component_set_journal_file(HostKindV1::OpenCode)), - control.join(HOST_COMPONENT_SET_JOURNAL_FILE), - ) - .unwrap(); - drop(writer); - - let mut writer = HostBundleWriterV1::open(root.path()).unwrap(); - assert_eq!( - writer.pending_component_set_journal_hosts().unwrap(), - vec![HostKindV1::OpenCode], - "a legacy journal is discovered and attributed to its recorded host" - ); - let unrelated = host_scoped_component_set(HostKindV1::Codex, "codex", b"v1"); - let unrelated_request = - component_set_request(HostKindV1::Codex, HostBundleLifecycleOpV1::Install, 62); - let unrelated_verifier = ComponentSetVerifier::from_set(&unrelated); - HostComponentSetTransactionV1::new(&mut writer) - .execute( - &unrelated, - &unrelated_request, - &unrelated_verifier, - &mut ArtifactOnlyTestRegistration, - ) - .expect("a legacy journal blocks only its own host"); - assert!( - control.join(HOST_COMPONENT_SET_JOURNAL_FILE).is_file(), - "another host's transaction never retires the legacy journal" - ); - writer - .quarantine_component_set_journal(HostKindV1::OpenCode, 1_700_000_000) - .unwrap() - .expect("the legacy journal is quarantined for its own host"); - assert!(!control.join(HOST_COMPONENT_SET_JOURNAL_FILE).exists()); -} - #[test] fn component_set_preflights_cross_component_path_conflicts_before_artifact_writes() { let root = tempfile::tempdir().unwrap(); @@ -1434,7 +744,7 @@ fn confirmed_component_set_rejects_changed_artifact_state_without_overwrite() { } #[test] -fn feedback_switch_apply_restore_and_aggregate_receipt_share_writer_recovery() { +fn feedback_switch_apply_restore_and_aggregate_receipt_share_one_writer() { let root = tempfile::tempdir().unwrap(); let previous = manifest(HostKindV1::KimiCode, b"previous"); let target = manifest(HostKindV1::KimiCode, b"target"); @@ -1685,7 +995,7 @@ fn lifecycle_preserves_ownership_receipts_and_rollback_plan() { .unwrap(); assert!(preview.confirmation_required); assert!(preview.plan.rollback_required); - assert_eq!(preview.rollback.backup_relative_paths.len(), 1); + assert_eq!(preview.rollback.replaced_relative_paths.len(), 1); std::fs::write(root.path().join("plugins/tracedecay.json"), b"foreign").unwrap(); let mut writer = HostBundleWriterV1::open(root.path()).unwrap(); @@ -1722,9 +1032,8 @@ fn pre_v2_artifact( /// The receiptless-adoption boundary: a cataloged deploy path alone never /// authorizes taking a file over. Divergent bytes are refused without -/// adoption authority (recognized legacy provenance or the operator's -/// explicit `--adopt`), adopted with backup when that authority is -/// present, byte-identical bytes are recorded without authority (the +/// adoption authority (the operator's explicit `--adopt`), adopted when that +/// authority is present, byte-identical bytes are recorded without authority (the /// staged hand-over journey), and Uninstall never adopts anything. #[test] fn receiptless_adoption_requires_provenance_or_explicit_authority() { @@ -1755,8 +1064,7 @@ fn receiptless_adoption_requires_provenance_or_explicit_authority() { "the refusal must name the explicit adoption remedy: {refused}" ); - // With adoption authority the stale bytes are adopted, but only - // after they are backed up. + // With adoption authority the stale bytes are replaced. assert_eq!( plan_artifact_action( operation, @@ -1764,7 +1072,7 @@ fn receiptless_adoption_requires_provenance_or_explicit_authority() { Some(&pre_v2_artifact(artifact, b"pre-v2", marker.clone())), true, ), - Ok(HostArtifactActionV1::BackupThenReplace), + Ok(HostArtifactActionV1::Replace), "{operation:?} must adopt a receiptless cataloged deploy path when authorized" ); // Bytes identical to the staged catalog are the recorded staged @@ -1821,7 +1129,7 @@ fn receiptless_adoption_requires_provenance_or_explicit_authority() { )), false, ), - Ok(HostArtifactActionV1::BackupThenReplace), + Ok(HostArtifactActionV1::Replace), "a divergent first-party staging must converge without explicit adoption" ); } @@ -2016,7 +1324,7 @@ fn profile_lifecycle_dry_run_does_not_create_missing_control_root() { } #[test] -fn profile_owned_receipts_enumerate_only_installed_components_and_retire_backups() { +fn profile_owned_receipts_enumerate_only_installed_components() { let artifacts = tempfile::tempdir().unwrap(); let lifecycle = tempfile::tempdir().unwrap(); let first = manifest(HostKindV1::Hermes, b"first"); @@ -2037,7 +1345,7 @@ fn profile_owned_receipts_enumerate_only_installed_components_and_retire_backups &verifier(&first), ) .unwrap(); - let receipt = writer + writer .execute( &second, &execution( @@ -2051,10 +1359,6 @@ fn profile_owned_receipts_enumerate_only_installed_components_and_retire_backups ) .unwrap(); - assert_eq!( - receipt.rollback_boundary, - HostBundleRollbackBoundaryV1::Passed - ); assert!( lifecycle .path() @@ -2066,39 +1370,6 @@ fn profile_owned_receipts_enumerate_only_installed_components_and_retire_backups !artifacts.path().join(HOST_BUNDLE_CONTROL_DIR).exists(), "receipts must be profile-owned rather than ambient-home-owned" ); - assert!( - !lifecycle - .path() - .join(HOST_BUNDLE_CONTROL_DIR) - .join("backups") - .join(hex::encode([12; 16])) - .exists(), - "a committed receipt that passed the rollback boundary retires its backups" - ); - - let referenced_operation = [14; 16]; - let referenced_backup = lifecycle - .path() - .join(HOST_BUNDLE_CONTROL_DIR) - .join("backups") - .join(hex::encode(referenced_operation)); - drop( - writer - .open_or_create_backup_dir(referenced_operation) - .unwrap(), - ); - let mut receipt_with_history = receipt.clone(); - receipt_with_history - .rollback_history - .push(referenced_operation); - writer.write_receipt(&receipt_with_history).unwrap(); - writer - .cleanup_unreferenced_backup_dir(referenced_operation) - .unwrap(); - assert!( - referenced_backup.is_dir(), - "receipt-referenced rollback history must be preserved" - ); let report = inspect_installed_host_bundle_components_at( artifacts.path(), @@ -2244,7 +1515,7 @@ fn receipt_doctor_classifies_missing_conflicting_and_corrupt_components() { ); // Same ownership marker, different bytes: ordinary content drift. The - // planner would converge this with `BackupThenReplace` under `Repair`, + // planner would converge this with `Replace` under `Repair`, // so discovery must not report a contested path. std::fs::write(&artifact, b"drifted").unwrap(); let report = inspect_installed_host_bundle_components_at( @@ -2264,7 +1535,7 @@ fn receipt_doctor_classifies_missing_conflicting_and_corrupt_components() { ); assert_eq!( report.components[0].repair_action, - "run `tracedecay reinstall --component core` (backs up and refreshes tracedecay-owned files)" + "run `tracedecay reinstall --component core` (refreshes tracedecay-owned files)" ); // A second receipt claiming the same deploy path with a different @@ -2285,8 +1556,6 @@ fn receipt_doctor_classifies_missing_conflicting_and_corrupt_components() { artifact_digest: Sha256::digest(b"foreign").into(), ownership_marker: expected_ownership_marker(HostKindV1::Hermes, HostComponentV1::Core), }], - rollback_boundary: HostBundleRollbackBoundaryV1::Passed, - rollback_history: Vec::new(), }; let foreign_receipt_path = lifecycle .path() @@ -2393,8 +1662,6 @@ fn write_component_receipt( ownership_marker: expected_ownership_marker(host, component), }) .collect(), - rollback_boundary: HostBundleRollbackBoundaryV1::Passed, - rollback_history: Vec::new(), }; for (relative_path, bytes) in artifacts { let Some(bytes) = bytes else { @@ -2585,3 +1852,73 @@ fn doctor_surfaces_restart_safe_feedback_rollback_state() { .contains("feedback-rollback restore") ); } + +/// A receipt written under an older schema is never migrated: it reads as +/// `ReinstallRequired` everywhere until an explicitly adopting install +/// discards it and records a current receipt over the deployed bytes. +#[test] +fn stale_schema_receipts_require_an_adopting_reinstall() { + let root = tempfile::tempdir().unwrap(); + let set = component_set(HostKindV1::OpenCode, b"core-v1", b"agent-v1"); + let control = root.path().join(HOST_BUNDLE_CONTROL_DIR); + fs::create_dir_all(&control).unwrap(); + let stale = control.join(receipt_file(HostKindV1::OpenCode, HostComponentV1::Core)); + fs::write( + &stale, + br#"{"schema_version":1,"rollback_boundary":"passed","rollback_history":[]}"#, + ) + .unwrap(); + let mut writer = HostBundleWriterV1::open(root.path()).unwrap(); + + assert_eq!( + writer.load_receipt(HostKindV1::OpenCode, HostComponentV1::Core), + Err(HostBundleError::ReinstallRequired) + ); + let doctor = inspect_installed_host_bundle_components_at( + root.path(), + root.path(), + &CurrentRegistration, + crate::agents::TEST_GENERATOR_COMMIT, + ) + .unwrap(); + assert_eq!( + doctor.components[0].state, + HostBundleComponentDoctorStateV1::ReinstallRequired + ); + let plain = component_set_request(HostKindV1::OpenCode, HostBundleLifecycleOpV1::Install, 51); + assert_eq!( + HostComponentSetTransactionV1::new(&mut writer) + .execute( + &set, + &plain, + &ComponentSetVerifier::from_set(&set), + &mut ArtifactOnlyTestRegistration, + ) + .err(), + Some(HostBundleError::ReinstallRequired) + ); + assert!( + stale.is_file(), + "a non-adopting install leaves the stale receipt" + ); + + let mut adopting = + component_set_request(HostKindV1::OpenCode, HostBundleLifecycleOpV1::Install, 52); + adopting.lifecycle.explicit_adoption = true; + HostComponentSetTransactionV1::new(&mut writer) + .execute( + &set, + &adopting, + &ComponentSetVerifier::from_set(&set), + &mut ArtifactOnlyTestRegistration, + ) + .expect("an adopting install replaces the stale receipt"); + assert_eq!( + writer + .load_receipt(HostKindV1::OpenCode, HostComponentV1::Core) + .unwrap() + .expect("a current receipt replaced the stale one") + .operation_id, + [52; 16] + ); +} diff --git a/crates/tracedecay-agent-hosts/src/agents/host_bundle/writer.rs b/crates/tracedecay-agent-hosts/src/agents/host_bundle/writer.rs index 6b2daf4ecf..2dfc1b88e1 100644 --- a/crates/tracedecay-agent-hosts/src/agents/host_bundle/writer.rs +++ b/crates/tracedecay-agent-hosts/src/agents/host_bundle/writer.rs @@ -1,5 +1,5 @@ -//! Atomic, capability-rooted single-component writer with a recoverable -//! journal, component backup/restore, and the no-follow filesystem primitives. +//! Atomic, capability-rooted single-component writer with in-memory rollback, +//! and the no-follow filesystem primitives. use std::collections::BTreeMap; use std::fs; @@ -11,33 +11,24 @@ use cap_fs_ext::{DirExt, FollowSymlinks, OpenOptionsFollowExt}; use cap_std::ambient_authority; use cap_std::fs::{Dir, OpenOptions as CapOpenOptions}; use sha2::{Digest, Sha256}; -use tracedecay_domain::canonical_json_bytes; -use tracedecay_host_integration::host_bundle_recovery_required; use tracedecay_host_integration::host_bundle_storage_failure; use super::control::{ - HOST_BUNDLE_CONTROL_DIR, HOST_BUNDLE_JOURNAL_FILE, HOST_BUNDLE_LOCK_FILE, - HOST_BUNDLE_QUARANTINE_DIR, HOST_COMPONENT_SET_JOURNAL_FILE, HOST_COMPONENT_SET_STAGE_DIR, - MAX_CONTROL_FILE_BYTES, backup_name, component_set_journal_file, component_set_receipt_file, - host_bundle_backup_receipt_file, host_bundle_restore_receipt_file, host_bundle_snapshot_name, - is_safe_component, journal_file, latest_host_component_set_receipt_at, receipt_file, - validate_backup_receipt, validate_component_set_journal, validate_component_set_receipt, - validate_journal, validate_receipt, validate_restore_receipt, writer_lock_file, + HOST_BUNDLE_CONTROL_DIR, HOST_BUNDLE_LOCK_FILE, MAX_CONTROL_FILE_BYTES, + component_set_receipt_file, is_safe_component, latest_host_component_set_receipt_at, + parse_receipt, receipt_file, receipt_identity_from_file_name, receipt_schema_probe, + validate_component_set_receipt, validate_receipt, writer_lock_file, }; use super::model::{HostBundleExecutionRequestV1, HostBundleLifecycleStorageV1}; use super::planner::{ - HostArtifactActionV1, HostBundleLifecycleRequestV1, ObservedArtifactKindV1, - ObservedHostArtifactV1, plan_verified_complete_lifecycle_mutation, - validate_artifact_contents_for_operation, + HostArtifactActionV1, HostArtifactMutationV1, ObservedArtifactKindV1, ObservedHostArtifactV1, + plan_verified_complete_lifecycle_mutation, validate_artifact_contents_for_operation, }; use super::{ - HOST_BUNDLE_RECEIPT_SCHEMA_VERSION, HostBundleArtifactContentV1, HostBundleBackupArtifactV1, - HostBundleBackupReceiptV1, HostBundleError, HostBundleInstallReceiptV1, - HostBundleJournalEntryV1, HostBundleJournalStateV1, HostBundleJournalV1, - HostBundleLifecycleOpV1, HostBundleManifestV1, HostBundleReceiptArtifactV1, - HostBundleRestoreReceiptV1, HostBundleRollbackBoundaryV1, HostBundleVerificationAdapterV1, - HostComponentSetJournalV1, HostComponentSetReceiptV1, HostComponentV1, HostKindV1, - MAX_ARTIFACT_CONTENT_BYTES, stock_host_kinds, validate_relative_install_path, + HOST_BUNDLE_RECEIPT_SCHEMA_VERSION, HostBundleArtifactContentV1, HostBundleError, + HostBundleInstallReceiptV1, HostBundleLifecycleOpV1, HostBundleManifestV1, + HostBundleReceiptArtifactV1, HostBundleVerificationAdapterV1, HostComponentSetReceiptV1, + HostComponentV1, HostKindV1, MAX_ARTIFACT_CONTENT_BYTES, validate_relative_install_path, }; static HOST_BUNDLE_TEMP_NONCE: AtomicU64 = AtomicU64::new(1); @@ -45,8 +36,8 @@ static HOST_BUNDLE_TEMP_NONCE: AtomicU64 = AtomicU64::new(1); /// Exclusive owner of one host's mutable bundle state. /// /// The lock is released when the writer switches hosts or is dropped. A -/// second host does not share this file: their artifact trees, journals, and -/// receipts are already disjoint. +/// second host does not share this file: their artifact trees and receipts are +/// already disjoint. struct HostWriterLock { host: HostKindV1, file: fs::File, @@ -64,13 +55,23 @@ impl Drop for HostWriterLock { } } +/// Pre-mutation state of one artifact path, held only for the running +/// operation so a failure can put the path back. +pub(super) struct ArtifactUndo { + relative_path: String, + previous: Option>, + written_digest: Option<[u8; 32]>, +} + /// Atomic, capability-rooted host-bundle writer. Every descendant directory /// is opened without following symlinks; files are staged, fsynced, renamed, /// and followed by a directory sync before receipt publication. /// -/// Opening does not take a lifecycle-root lock and does not recover another -/// host's journal. Mutation acquires `writer.{slug}.v1.lock` for the host -/// being written and holds it until the writer switches hosts or drops. +/// Rollback bytes live only in memory for the running operation. A process +/// killed mid-operation leaves whatever it wrote; the next install or repair +/// converges from the observed state. Mutation acquires +/// `writer.{slug}.v1.lock` for the host being written and holds it until the +/// writer switches hosts or drops. pub struct HostBundleWriterV1 { pub(super) root_path: PathBuf, pub(super) lifecycle_root_path: PathBuf, @@ -128,99 +129,8 @@ impl HostBundleWriterV1 { Ok(()) } - /// Recover by rolling an incomplete transaction back from its immutable - /// backups. A receipt matching the journal operation is a durable commit - /// marker and is never rolled back after a crash between receipt/journal - /// cleanup. - pub fn recover_interrupted_operation( - &mut self, - host: HostKindV1, - ) -> Result<(), HostBundleError> { - self.ensure_host_lock(host)?; - self.recover_host_journal_locked(host) - } - - fn recover_host_journal_locked(&mut self, host: HostKindV1) -> Result<(), HostBundleError> { - let Some(journal) = self.load_journal_for(host)? else { - return Ok(()); - }; - if journal.host != host { - return Err(HostBundleError::ReceiptCorrupted); - } - validate_journal(&journal)?; - if let Some(receipt) = - self.load_receipt(journal.host, journal.component)? - .filter(|receipt| { - receipt.operation_id == journal.operation_id - && receipt.operation == journal.operation - && receipt.manifest_digest == journal.manifest_digest - }) - { - self.remove_journal(host)?; - if receipt.rollback_boundary == HostBundleRollbackBoundaryV1::Passed { - self.cleanup_unreferenced_backup_dir(journal.operation_id)?; - } - return Ok(()); - } - - let backup_dir = self.open_existing_backup_dir(journal.operation_id)?; - for entry in journal.entries.iter().rev() { - let (parent, name) = self.open_parent_nofollow(Path::new(&entry.relative_path))?; - if let Some(backup_name) = &entry.backup_name { - let backup_exists = match &backup_dir { - Some(backups) => regular_file_exists(backups, backup_name)?, - None => false, - }; - if !entry.backup_created { - if !backup_exists { - continue; - } - if regular_file_exists(&parent, &name)? { - return Err(host_bundle_recovery_required!()); - } - } - let backups = backup_dir - .as_ref() - .filter(|_| backup_exists) - .ok_or(host_bundle_recovery_required!())?; - if entry.wrote_new { - remove_if_digest_matches( - &parent, - &name, - entry - .installed_digest - .ok_or(HostBundleError::ReceiptCorrupted)?, - )?; - } else if regular_file_exists(&parent, &name)? { - return Err(host_bundle_recovery_required!()); - } - backups - .rename(backup_name, &parent, &name) - .map_err(|_| host_bundle_storage_failure!())?; - sync_cap_dir(backups)?; - sync_cap_dir(&parent)?; - } else if entry.wrote_new { - remove_if_digest_matches( - &parent, - &name, - entry - .installed_digest - .ok_or(HostBundleError::ReceiptCorrupted)?, - )?; - sync_cap_dir(&parent)?; - } - } - drop(backup_dir); - match journal.previous_receipt { - Some(receipt) => self.write_receipt(&receipt)?, - None => self.remove_receipt(journal.host, journal.component)?, - } - self.remove_journal(host)?; - self.cleanup_unreferenced_backup_dir(journal.operation_id) - } - /// Verify first-party catalog identity, validate artifact bytes, plan ownership-aware - /// mutations, then execute them atomically with a recoverable journal. + /// mutations, then execute them, putting every touched path back on failure. #[hotpath::measure(label = "hosts.agent.host_bundle.execute")] pub fn execute( &mut self, @@ -235,15 +145,6 @@ impl HostBundleWriterV1 { self.ensure_host_lock(manifest.host)?; verifier.verify_manifest(manifest)?; let content_by_path = validate_artifact_contents(manifest, request, contents)?; - // Scoped to this manifest's own host: another host's pending - // component-set journal governs a disjoint artifact subtree. - if self - .load_component_set_journal_for(manifest.host)? - .is_some() - { - return Err(host_bundle_recovery_required!()); - } - self.recover_host_journal_locked(manifest.host)?; let previous_receipt = self.load_receipt(manifest.host, manifest.component)?; let manifest_digest = manifest.canonical_digest()?; if let Some(receipt) = previous_receipt.as_ref() @@ -295,89 +196,16 @@ impl HostBundleWriterV1 { &orphan_observed, verifier, )?; - let mut journal = HostBundleJournalV1 { - schema_version: HOST_BUNDLE_RECEIPT_SCHEMA_VERSION, - operation_id: request.operation_id, - host: manifest.host, - component: manifest.component, - operation: request.lifecycle.operation, - manifest_digest, - state: HostBundleJournalStateV1::Prepared, - previous_receipt: previous_receipt.clone(), - entries: plan - .mutations - .iter() - .map(|mutation| HostBundleJournalEntryV1 { - relative_path: mutation.relative_path.clone(), - backup_name: matches!( - mutation.action, - HostArtifactActionV1::BackupThenReplace - | HostArtifactActionV1::BackupThenRemove - ) - .then(|| backup_name(request.operation_id, &mutation.relative_path)), - backup_created: false, - wrote_new: false, - installed_digest: manifest - .artifacts - .iter() - .find(|artifact| artifact.relative_path == mutation.relative_path) - .map(|artifact| artifact.artifact_digest) - .filter(|_| { - !matches!(mutation.action, HostArtifactActionV1::BackupThenRemove) - }), - }) - .collect(), - }; - self.write_journal(&journal)?; - let backup_dir = self.open_or_create_backup_dir(request.operation_id)?; - - for (index, mutation) in plan.mutations.iter().enumerate() { - let (parent, name) = self.open_parent_nofollow(Path::new(&mutation.relative_path))?; - match mutation.action { - HostArtifactActionV1::Noop => {} - HostArtifactActionV1::WriteNew => { - journal.entries[index].wrote_new = true; - self.write_journal(&journal)?; - atomic_write_nofollow( - &parent, - &name, - content_by_path - .get(&mutation.relative_path) - .ok_or(HostBundleError::ArtifactContentMismatch)?, - false, - )?; - } - HostArtifactActionV1::BackupThenReplace => { - let backup_name = journal.entries[index] - .backup_name - .as_deref() - .ok_or(HostBundleError::ReceiptCorrupted)?; - move_regular_to_backup(&parent, &name, &backup_dir, backup_name)?; - journal.entries[index].backup_created = true; - self.write_journal(&journal)?; - journal.entries[index].wrote_new = true; - self.write_journal(&journal)?; - atomic_write_nofollow( - &parent, - &name, - content_by_path - .get(&mutation.relative_path) - .ok_or(HostBundleError::ArtifactContentMismatch)?, - false, - )?; - } - HostArtifactActionV1::BackupThenRemove => { - let backup_name = journal.entries[index] - .backup_name - .as_deref() - .ok_or(HostBundleError::ReceiptCorrupted)?; - move_regular_to_backup(&parent, &name, &backup_dir, backup_name)?; - journal.entries[index].backup_created = true; - self.write_journal(&journal)?; - } + let mut undo = Vec::new(); + let applied = plan.mutations.iter().try_for_each(|mutation| { + if let Some(record) = self.apply_artifact_mutation(mutation, &content_by_path)? { + undo.push(record); } + Ok(()) + }); + if let Err(error) = applied { + return Err(self.undo_after_failure(error, &undo)); } - drop(backup_dir); let receipt = HostBundleInstallReceiptV1 { schema_version: HOST_BUNDLE_RECEIPT_SCHEMA_VERSION, @@ -399,22 +227,145 @@ impl HostBundleWriterV1 { }) .collect() }, - rollback_boundary: HostBundleRollbackBoundaryV1::Passed, - rollback_history: previous_receipt - .as_ref() - .map(|receipt| receipt.rollback_history.clone()) - .unwrap_or_default(), }; - self.write_receipt(&receipt)?; - journal.state = HostBundleJournalStateV1::Committed; - self.write_journal(&journal)?; - self.remove_journal(manifest.host)?; - if receipt.rollback_boundary == HostBundleRollbackBoundaryV1::Passed { - self.cleanup_unreferenced_backup_dir(request.operation_id)?; + if let Err(error) = self.write_receipt(&receipt) { + return Err(self.undo_after_failure(error, &undo)); } Ok(receipt) } + /// Apply one planned mutation, returning what the path held before when + /// the mutation changed it. + pub(super) fn apply_artifact_mutation( + &self, + mutation: &HostArtifactMutationV1, + content_by_path: &BTreeMap>, + ) -> Result, HostBundleError> { + let (parent, name) = self.open_parent_nofollow(Path::new(&mutation.relative_path))?; + let content = || { + content_by_path + .get(&mutation.relative_path) + .ok_or(HostBundleError::ArtifactContentMismatch) + }; + let (previous, written_digest) = match mutation.action { + HostArtifactActionV1::Noop => return Ok(None), + HostArtifactActionV1::WriteNew => { + let bytes = content()?; + atomic_write_nofollow(&parent, &name, bytes, false)?; + (None, Some(Sha256::digest(bytes).into())) + } + HostArtifactActionV1::Replace => { + let previous = read_regular_nofollow(&parent, &name)? + .ok_or(HostBundleError::InvalidObservedState)?; + let bytes = content()?; + atomic_write_nofollow(&parent, &name, bytes, true)?; + (Some(previous), Some(Sha256::digest(bytes).into())) + } + HostArtifactActionV1::Remove => { + let previous = read_regular_nofollow(&parent, &name)? + .ok_or(HostBundleError::InvalidObservedState)?; + parent + .remove_file(&name) + .map_err(|_| host_bundle_storage_failure!())?; + sync_cap_dir(&parent)?; + (Some(previous), None) + } + }; + Ok(Some(ArtifactUndo { + relative_path: mutation.relative_path.clone(), + previous, + written_digest, + })) + } + + /// Put every recorded path back in reverse order. A path that holds + /// neither its prior bytes nor this operation's output was changed by + /// another writer: it is left alone and reported once every other path has + /// been put back. + pub(super) fn undo_artifact_mutations( + &self, + undo: &[ArtifactUndo], + ) -> Result<(), HostBundleError> { + let mut conflict = None; + for record in undo.iter().rev() { + let (parent, name) = self.open_parent_nofollow(Path::new(&record.relative_path))?; + let live = read_regular_nofollow(&parent, &name)?; + if live == record.previous { + continue; + } + let live_digest = live + .as_ref() + .map(|bytes| <[u8; 32]>::from(Sha256::digest(bytes))); + if live.is_some() && live_digest != record.written_digest { + conflict.get_or_insert_with(|| { + HostBundleError::OwnershipConflict(format!( + "{}: changed by another writer while this operation rolled back", + record.relative_path + )) + }); + continue; + } + match &record.previous { + Some(bytes) => atomic_write_nofollow(&parent, &name, bytes, live.is_some())?, + None => { + parent + .remove_file(&name) + .map_err(|_| host_bundle_storage_failure!())?; + sync_cap_dir(&parent)?; + } + } + } + conflict.map_or(Ok(()), Err) + } + + fn undo_after_failure(&self, error: HostBundleError, undo: &[ArtifactUndo]) -> HostBundleError { + match self.undo_artifact_mutations(undo) { + Ok(()) => error, + Err(undo_error) => undo_error, + } + } + + /// Delete this host's receipts written under an older receipt schema. + /// Only an explicitly adopting lifecycle calls this; every other lifecycle + /// reports [`HostBundleError::ReinstallRequired`] for them. + pub(super) fn discard_stale_receipts( + &mut self, + host: HostKindV1, + ) -> Result<(), HostBundleError> { + self.ensure_host_lock(host)?; + let control_path = self.lifecycle_root_path.join(HOST_BUNDLE_CONTROL_DIR); + let entries = match fs::read_dir(&control_path) { + Ok(entries) => entries, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()), + Err(_) => return Err(host_bundle_storage_failure!()), + }; + for entry in entries { + let entry = entry.map_err(|_| host_bundle_storage_failure!())?; + let name = entry.file_name(); + let Some(name) = name.to_str() else { + continue; + }; + let owned_by_host = if name.starts_with("component-set-receipt.") { + None + } else if let Some((receipt_host, _)) = receipt_identity_from_file_name(name) { + Some(receipt_host == host) + } else { + continue; + }; + let Some(bytes) = read_control_json(&self.control, name)? else { + continue; + }; + let Some(probe) = receipt_schema_probe(&bytes) else { + continue; + }; + if probe.is_current() || !owned_by_host.unwrap_or(probe.host == Some(host)) { + continue; + } + self.remove_control_file(name)?; + } + Ok(()) + } + fn observe_artifacts( &self, manifest: &HostBundleManifestV1, @@ -504,48 +455,13 @@ impl HostBundleWriterV1 { )) } - pub(super) fn open_or_create_backup_dir( - &self, - operation_id: [u8; 16], - ) -> Result { - let backups = open_or_create_nofollow_dir(&self.control, "backups")?; - open_or_create_nofollow_dir(&backups, &hex::encode(operation_id)) - } - - pub(super) fn open_or_create_component_set_stage_dir( - &self, - operation_id: [u8; 16], - ) -> Result { - let stages = open_or_create_nofollow_dir(&self.control, HOST_COMPONENT_SET_STAGE_DIR)?; - open_or_create_nofollow_dir(&stages, &hex::encode(operation_id)) - } - - pub(super) fn open_existing_backup_dir( - &self, - operation_id: [u8; 16], - ) -> Result, HostBundleError> { - let backups = match self.control.open_dir_nofollow("backups") { - Ok(backups) => backups, - Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None), - Err(_) => return Err(HostBundleError::UnsafeInstallPath), - }; - match backups.open_dir_nofollow(hex::encode(operation_id)) { - Ok(directory) => Ok(Some(directory)), - Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(None), - Err(_) => Err(HostBundleError::UnsafeInstallPath), - } - } - pub(super) fn load_receipt( &self, host: HostKindV1, component: HostComponentV1, ) -> Result, HostBundleError> { - let receipt = read_control_json(&self.control, &receipt_file(host, component))?; - let receipt = receipt - .map(|bytes| { - serde_json::from_slice(&bytes).map_err(|_| HostBundleError::ReceiptCorrupted) - }) + let receipt = read_control_json(&self.control, &receipt_file(host, component))? + .map(|bytes| parse_receipt::(&bytes)) .transpose()?; if let Some(receipt) = &receipt { validate_receipt(receipt)?; @@ -584,9 +500,7 @@ impl HostBundleWriterV1 { operation_id: [u8; 16], ) -> Result, HostBundleError> { let receipt = read_control_json(&self.control, &component_set_receipt_file(operation_id))? - .map(|bytes| { - serde_json::from_slice(&bytes).map_err(|_| HostBundleError::ReceiptCorrupted) - }) + .map(|bytes| parse_receipt::(&bytes)) .transpose()?; if let Some(receipt) = &receipt { validate_component_set_receipt(receipt)?; @@ -616,461 +530,11 @@ impl HostBundleWriterV1 { self.remove_control_file(&component_set_receipt_file(operation_id)) } - fn read_journal_file( - &self, - file_name: &str, - ) -> Result, HostBundleError> { - read_control_json(&self.control, file_name)? - .map(|bytes| { - serde_json::from_slice(&bytes).map_err(|_| HostBundleError::ReceiptCorrupted) - }) - .transpose() - } - - /// Load the pending single-component journal for one host. - /// - /// A journal written by an older binary lives under the shared legacy name - /// and carries its own `host` field, so it is attributed to exactly one - /// host. Recovering a different host must not see it. - pub(super) fn load_journal_for( - &self, - host: HostKindV1, - ) -> Result, HostBundleError> { - if let Some(journal) = self.read_journal_file(&journal_file(host))? { - if journal.host != host { - return Err(HostBundleError::ReceiptCorrupted); - } - return Ok(Some(journal)); - } - Ok(self - .read_journal_file(HOST_BUNDLE_JOURNAL_FILE)? - .filter(|journal| journal.host == host)) - } - - fn read_component_set_journal_file( - &self, - file_name: &str, - ) -> Result, HostBundleError> { - read_control_json(&self.control, file_name)? - .map(|bytes| { - serde_json::from_slice(&bytes).map_err(|_| HostBundleError::ReceiptCorrupted) - }) - .transpose() - } - - /// Load the pending component-set journal for one host. - /// - /// Journals are host-scoped so an interrupted transaction for host X never - /// blocks an unrelated host Y. Journals written by an older binary live - /// under the shared legacy name; they carry their own `host` field, so they - /// are readable here and are attributed to exactly one host. - pub(super) fn load_component_set_journal_for( - &self, - host: HostKindV1, - ) -> Result, HostBundleError> { - if let Some(journal) = - self.read_component_set_journal_file(&component_set_journal_file(host))? - { - return Ok(Some(journal)); - } - Ok(self - .read_component_set_journal_file(HOST_COMPONENT_SET_JOURNAL_FILE)? - .filter(|journal| journal.host == host)) - } - - /// Load any pending component-set journal, host-scoped or legacy. Used by - /// the host-blind recovery entry point, which must still be able to find a - /// single outstanding transaction. - pub(super) fn load_component_set_journal( - &self, - ) -> Result, HostBundleError> { - for host in stock_host_kinds() { - if let Some(journal) = - self.read_component_set_journal_file(&component_set_journal_file(host))? - { - return Ok(Some(journal)); - } - } - self.read_component_set_journal_file(HOST_COMPONENT_SET_JOURNAL_FILE) - } - - /// Every host with a pending component-set journal. The recovery verb - /// reports these; `--agent` narrows the set. - pub fn pending_component_set_journal_hosts(&self) -> Result, HostBundleError> { - let mut hosts = Vec::new(); - for host in stock_host_kinds() { - if self.load_component_set_journal_for(host)?.is_some() { - hosts.push(host); - } - } - Ok(hosts) - } - - pub fn pending_component_set_journal_operation( - &self, - host: HostKindV1, - ) -> Result, HostBundleError> { - let Some(journal) = self.load_component_set_journal_for(host)? else { - return Ok(None); - }; - validate_component_set_journal(&journal)?; - Ok(Some(journal.operation)) - } - - #[hotpath::measure(label = "hosts.agent.host_bundle.journal_persist")] - fn write_journal(&self, journal: &HostBundleJournalV1) -> Result<(), HostBundleError> { - validate_journal(journal)?; - let bytes = serde_json::to_vec(journal).map_err(|_| HostBundleError::ReceiptCorrupted)?; - atomic_write_nofollow(&self.control, &journal_file(journal.host), &bytes, true)?; - // A journal written by an older binary lives under the shared legacy - // name. Once its host-scoped successor is durable, retire it so the - // legacy file can never shadow or double-recover this transaction. - // Never unlink a legacy journal that belongs to a different host. - if self - .read_journal_file(HOST_BUNDLE_JOURNAL_FILE)? - .is_some_and(|legacy| legacy.host == journal.host) - { - self.remove_control_file(HOST_BUNDLE_JOURNAL_FILE)?; - } - Ok(()) - } - - fn remove_journal(&self, host: HostKindV1) -> Result<(), HostBundleError> { - self.remove_control_file(&journal_file(host))?; - if self - .read_journal_file(HOST_BUNDLE_JOURNAL_FILE)? - .is_some_and(|legacy| legacy.host == host) - { - self.remove_control_file(HOST_BUNDLE_JOURNAL_FILE)?; - } - Ok(()) - } - - #[hotpath::measure(label = "hosts.agent.host_bundle.component_set_journal_persist")] - pub(super) fn write_component_set_journal( - &self, - journal: &HostComponentSetJournalV1, - ) -> Result<(), HostBundleError> { - validate_component_set_journal(journal)?; - let bytes = serde_json::to_vec(journal).map_err(|_| HostBundleError::ReceiptCorrupted)?; - atomic_write_nofollow( - &self.control, - &component_set_journal_file(journal.host), - &bytes, - true, - )?; - // A journal written by an older binary lives under the shared legacy - // name. Once its host-scoped successor is durable, retire it so the - // legacy file can never shadow or double-recover this transaction. - if self - .read_component_set_journal_file(HOST_COMPONENT_SET_JOURNAL_FILE)? - .is_some_and(|legacy| legacy.host == journal.host) - { - self.remove_control_file(HOST_COMPONENT_SET_JOURNAL_FILE)?; - } - Ok(()) - } - fn remove_control_file(&self, name: &str) -> Result<(), HostBundleError> { remove_regular_if_exists(&self.control, name)?; sync_cap_dir(&self.control) } - /// Last-resort operator escape when convergent recovery still cannot - /// resolve a host's component-set journal (genuinely foreign bytes at a - /// path the transaction created, for example). - /// - /// The journal is *moved* into a quarantine directory rather than deleted: - /// the transaction's immutable backups stay on disk beside it, so the - /// pre-transaction bytes remain recoverable by hand and nothing about the - /// failure is destroyed. Only the authority file that blocks further - /// mutation of this host is set aside. This replaces the previous recovery - /// path, which was hand-deleting the journal. - /// - /// Returns the quarantined path, or `None` when no journal was pending. - pub fn quarantine_component_set_journal( - &mut self, - host: HostKindV1, - now_unix: u64, - ) -> Result, HostBundleError> { - self.ensure_host_lock(host)?; - let mut moved = None; - for file in [ - component_set_journal_file(host), - HOST_COMPONENT_SET_JOURNAL_FILE.to_string(), - ] { - // The legacy shared file belongs to whichever host wrote it; never - // quarantine another host's journal from under it. - if file == HOST_COMPONENT_SET_JOURNAL_FILE - && self - .read_component_set_journal_file(&file)? - .is_none_or(|journal| journal.host != host) - { - continue; - } - if !regular_file_exists(&self.control, &file)? { - continue; - } - let quarantine = - open_or_create_nofollow_dir(&self.control, HOST_BUNDLE_QUARANTINE_DIR)?; - let target = format!("{now_unix}.{file}"); - if !is_safe_component(&target) { - return Err(HostBundleError::UnsafeInstallPath); - } - self.control - .rename(&file, &quarantine, &target) - .map_err(|_| host_bundle_storage_failure!())?; - sync_cap_dir(&quarantine)?; - sync_cap_dir(&self.control)?; - moved = Some( - self.lifecycle_root_path - .join(HOST_BUNDLE_CONTROL_DIR) - .join(HOST_BUNDLE_QUARANTINE_DIR) - .join(target), - ); - } - Ok(moved) - } - - pub(super) fn remove_component_set_journal( - &self, - host: HostKindV1, - ) -> Result<(), HostBundleError> { - self.remove_control_file(&component_set_journal_file(host))?; - if self - .read_component_set_journal_file(HOST_COMPONENT_SET_JOURNAL_FILE)? - .is_some_and(|legacy| legacy.host == host) - { - self.remove_control_file(HOST_COMPONENT_SET_JOURNAL_FILE)?; - } - Ok(()) - } - - /// Retires an operation's rollback backups once no receipt still names it. - /// - /// Every caller must drop its `Dir` capability on the operation's backup - /// directory first: `cap_std` opens directories without `FILE_SHARE_DELETE`, - /// so on Windows a live handle makes the removal below fail with a sharing - /// violation and turns a completed transaction into a storage failure. - pub(super) fn cleanup_unreferenced_backup_dir( - &self, - operation_id: [u8; 16], - ) -> Result<(), HostBundleError> { - let control_path = self.lifecycle_root_path.join(HOST_BUNDLE_CONTROL_DIR); - let mut referenced = false; - for entry in fs::read_dir(&control_path).map_err(|_| host_bundle_storage_failure!())? { - let Ok(entry) = entry else { - return Ok(()); - }; - let name = entry.file_name(); - let Some(name) = name.to_str() else { - continue; - }; - if !name.starts_with("receipt.") || !name.ends_with(".v1.json") { - continue; - } - let Ok(bytes) = fs::read(entry.path()) else { - return Ok(()); - }; - let Ok(receipt) = serde_json::from_slice::(&bytes) else { - return Ok(()); - }; - if validate_receipt(&receipt).is_err() { - return Ok(()); - } - referenced |= receipt.rollback_history.contains(&operation_id); - } - if referenced { - return Ok(()); - } - let backup_path = control_path.join("backups").join(hex::encode(operation_id)); - match fs::symlink_metadata(&backup_path) { - Ok(metadata) if metadata.is_dir() && !metadata.file_type().is_symlink() => { - fs::remove_dir_all(&backup_path).map_err(|_| host_bundle_storage_failure!())?; - } - Ok(_) => return Err(HostBundleError::UnsafeInstallPath), - Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()), - Err(_) => return Err(host_bundle_storage_failure!()), - } - if let Some(backups) = backup_path.parent() { - let _ = fs::remove_dir(backups); - } - Ok(()) - } - - pub(super) fn cleanup_component_set_boundary( - &self, - operation_id: [u8; 16], - ) -> Result<(), HostBundleError> { - self.cleanup_unreferenced_backup_dir(operation_id)?; - self.remove_component_set_stage_dir(operation_id) - } - - fn remove_component_set_stage_dir( - &self, - operation_id: [u8; 16], - ) -> Result<(), HostBundleError> { - let stage_path = self - .lifecycle_root_path - .join(HOST_BUNDLE_CONTROL_DIR) - .join(HOST_COMPONENT_SET_STAGE_DIR) - .join(hex::encode(operation_id)); - match fs::symlink_metadata(&stage_path) { - Ok(metadata) if metadata.is_dir() && !metadata.file_type().is_symlink() => { - fs::remove_dir_all(&stage_path).map_err(|_| host_bundle_storage_failure!())?; - } - Ok(_) => return Err(HostBundleError::UnsafeInstallPath), - Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()), - Err(_) => return Err(host_bundle_storage_failure!()), - } - if let Some(stages) = stage_path.parent() { - let _ = fs::remove_dir(stages); - } - Ok(()) - } - - /// Snapshot one installed component without mutating host state. Replaying - /// the same operation id returns the existing receipt after revalidation. - /// A missing, edited, or foreign artifact fails before receipt publication. - pub fn backup_component( - &mut self, - manifest: &HostBundleManifestV1, - operation_id: [u8; 16], - explicit_confirmation: bool, - verifier: &V, - ) -> Result { - if operation_id == [0; 16] { - return Err(HostBundleError::InvalidManifest); - } - if !explicit_confirmation { - return Err(HostBundleError::ConfirmationRequired); - } - self.ensure_host_lock(manifest.host)?; - manifest.validate_structure()?; - verifier.verify_manifest(manifest)?; - if let Some(receipt) = self.load_backup_receipt(operation_id)? { - validate_backup_receipt(&receipt)?; - self.read_backup_contents(&receipt)?; - return (receipt.manifest == *manifest) - .then_some(receipt) - .ok_or(HostBundleError::ReceiptCorrupted); - } - - let source_receipt = self - .load_receipt(manifest.host, manifest.component)? - .filter(|receipt| receipt.operation != HostBundleLifecycleOpV1::Uninstall) - .ok_or(HostBundleError::InvalidObservedState)?; - if source_receipt.manifest_digest != manifest.canonical_digest()? - || source_receipt.artifacts.len() != manifest.artifacts.len() - { - return Err(HostBundleError::InvalidObservedState); - } - let source_receipt_digest: [u8; 32] = Sha256::digest( - canonical_json_bytes(&source_receipt) - .map_err(|_| HostBundleError::CanonicalizationFailed)?, - ) - .into(); - let snapshot_dir = self.open_or_create_snapshot_dir(operation_id)?; - let mut artifacts = Vec::with_capacity(source_receipt.artifacts.len()); - for (index, owned) in source_receipt.artifacts.iter().enumerate() { - let expected = manifest - .artifacts - .iter() - .find(|artifact| artifact.relative_path == owned.relative_path) - .filter(|artifact| { - artifact.artifact_digest == owned.artifact_digest - && artifact.ownership_marker == owned.ownership_marker - }) - .ok_or(HostBundleError::InvalidObservedState)?; - let (parent, name) = self.open_parent_nofollow(Path::new(&expected.relative_path))?; - let bytes = read_regular_nofollow(&parent, &name)? - .ok_or(HostBundleError::InvalidObservedState)?; - if <[u8; 32]>::from(Sha256::digest(&bytes)) != expected.artifact_digest { - return Err(HostBundleError::OwnershipConflict(format!( - "{}: deployed bytes no longer match the receipt-owned content (marker {:?})", - expected.relative_path, expected.ownership_marker - ))); - } - let snapshot_name = host_bundle_snapshot_name(index, &expected.relative_path); - match read_regular_nofollow(&snapshot_dir, &snapshot_name)? { - Some(existing) if existing == bytes => {} - Some(_) => return Err(HostBundleError::ReceiptCorrupted), - None => atomic_write_nofollow(&snapshot_dir, &snapshot_name, &bytes, false)?, - } - artifacts.push(HostBundleBackupArtifactV1 { - relative_path: expected.relative_path.clone(), - artifact_digest: expected.artifact_digest, - ownership_marker: expected.ownership_marker.clone(), - snapshot_name, - }); - } - sync_cap_dir(&snapshot_dir)?; - let receipt = HostBundleBackupReceiptV1 { - schema_version: HOST_BUNDLE_RECEIPT_SCHEMA_VERSION, - operation_id, - host: manifest.host, - component: manifest.component, - manifest: manifest.clone(), - source_receipt_digest, - artifacts, - }; - self.write_backup_receipt(&receipt)?; - Ok(receipt) - } - - /// Restore a named component backup through the ordinary Repair - /// transaction. Any failure rolls the host files back to their pre-restore - /// bytes; replaying `operation_id` returns the durable terminal receipt. - pub fn restore_component_backup( - &mut self, - backup_operation_id: [u8; 16], - operation_id: [u8; 16], - explicit_confirmation: bool, - verifier: &V, - ) -> Result { - if backup_operation_id == [0; 16] || operation_id == [0; 16] { - return Err(HostBundleError::InvalidManifest); - } - if !explicit_confirmation { - return Err(HostBundleError::ConfirmationRequired); - } - if let Some(receipt) = self.load_restore_receipt(operation_id)? { - validate_restore_receipt(&receipt)?; - return (receipt.backup_operation_id == backup_operation_id) - .then_some(receipt) - .ok_or(HostBundleError::ReceiptCorrupted); - } - let backup = self - .load_backup_receipt(backup_operation_id)? - .ok_or(HostBundleError::InvalidObservedState)?; - validate_backup_receipt(&backup)?; - verifier.verify_manifest(&backup.manifest)?; - let contents = self.read_backup_contents(&backup)?; - let request = HostBundleExecutionRequestV1 { - lifecycle: HostBundleLifecycleRequestV1 { - operation: HostBundleLifecycleOpV1::Repair, - expected_host: backup.host, - expected_component: backup.component, - explicit_confirmation: true, - hermes_profile_bindings: u8::from(backup.host == HostKindV1::Hermes), - // The operator explicitly confirmed restoring this exact - // named backup, which is adoption authority over the backup's - // recorded deploy paths whatever bytes sit there now. - adopt_receiptless: true, - }, - operation_id, - }; - let restored_receipt = self.execute(&backup.manifest, &request, &contents, verifier)?; - let receipt = HostBundleRestoreReceiptV1 { - schema_version: HOST_BUNDLE_RECEIPT_SCHEMA_VERSION, - operation_id, - backup_operation_id, - restored_receipt, - }; - self.write_restore_receipt(&receipt)?; - Ok(receipt) - } - pub fn publish_feedback_component_set_receipt( &mut self, manifest: &HostBundleManifestV1, @@ -1123,118 +587,9 @@ impl HostBundleWriterV1 { pub fn lifecycle_root_path(&self) -> &Path { &self.lifecycle_root_path } - - fn open_or_create_snapshot_dir(&self, operation_id: [u8; 16]) -> Result { - let snapshots = open_or_create_nofollow_dir(&self.control, "snapshots")?; - open_or_create_nofollow_dir(&snapshots, &hex::encode(operation_id)) - } - - fn open_existing_snapshot_dir( - &self, - operation_id: [u8; 16], - ) -> Result, HostBundleError> { - let snapshots = match self.control.open_dir_nofollow("snapshots") { - Ok(directory) => directory, - Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None), - Err(_) => return Err(HostBundleError::UnsafeInstallPath), - }; - match snapshots.open_dir_nofollow(hex::encode(operation_id)) { - Ok(directory) => Ok(Some(directory)), - Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(None), - Err(_) => Err(HostBundleError::UnsafeInstallPath), - } - } - - fn read_backup_contents( - &self, - receipt: &HostBundleBackupReceiptV1, - ) -> Result, HostBundleError> { - validate_backup_receipt(receipt)?; - let snapshot_dir = self - .open_existing_snapshot_dir(receipt.operation_id)? - .ok_or(HostBundleError::ReceiptCorrupted)?; - receipt - .artifacts - .iter() - .map(|artifact| { - let bytes = read_regular_nofollow(&snapshot_dir, &artifact.snapshot_name)? - .ok_or(HostBundleError::ReceiptCorrupted)?; - if <[u8; 32]>::from(Sha256::digest(&bytes)) != artifact.artifact_digest { - return Err(HostBundleError::ReceiptCorrupted); - } - Ok(HostBundleArtifactContentV1 { - relative_path: artifact.relative_path.clone(), - bytes, - }) - }) - .collect() - } - - fn load_backup_receipt( - &self, - operation_id: [u8; 16], - ) -> Result, HostBundleError> { - read_control_json( - &self.control, - &host_bundle_backup_receipt_file(operation_id), - )? - .map(|bytes| serde_json::from_slice(&bytes).map_err(|_| HostBundleError::ReceiptCorrupted)) - .transpose() - } - - fn write_backup_receipt( - &self, - receipt: &HostBundleBackupReceiptV1, - ) -> Result<(), HostBundleError> { - validate_backup_receipt(receipt)?; - let bytes = serde_json::to_vec(receipt).map_err(|_| HostBundleError::ReceiptCorrupted)?; - atomic_write_nofollow( - &self.control, - &host_bundle_backup_receipt_file(receipt.operation_id), - &bytes, - false, - ) - } - - fn load_restore_receipt( - &self, - operation_id: [u8; 16], - ) -> Result, HostBundleError> { - read_control_json( - &self.control, - &host_bundle_restore_receipt_file(operation_id), - )? - .map(|bytes| serde_json::from_slice(&bytes).map_err(|_| HostBundleError::ReceiptCorrupted)) - .transpose() - } - - fn write_restore_receipt( - &self, - receipt: &HostBundleRestoreReceiptV1, - ) -> Result<(), HostBundleError> { - validate_restore_receipt(receipt)?; - let bytes = serde_json::to_vec(receipt).map_err(|_| HostBundleError::ReceiptCorrupted)?; - atomic_write_nofollow( - &self.control, - &host_bundle_restore_receipt_file(receipt.operation_id), - &bytes, - false, - ) - } } impl HostBundleLifecycleStorageV1 for HostBundleWriterV1 { - fn recover_lifecycle(&mut self) -> Result<(), HostBundleError> { - // Explicit recover-all. Each host owns its journal, so each recovery - // takes only that host's lock and releases it before the next host. - for host in stock_host_kinds() { - if self.load_journal_for(host)?.is_some() { - self.recover_interrupted_operation(host)?; - } - } - Ok(()) - } - fn execute_lifecycle( &mut self, manifest: &HostBundleManifestV1, @@ -1279,7 +634,7 @@ fn open_or_create_nofollow_dir(parent: &Dir, name: &str) -> Result { match parent.create_dir(name) { Ok(()) => {} - // Two hosts may create a shared parent (`backups/`, `.config/`) + // Two hosts may create a shared parent (`.config/`) // at once. The directory is a namespace, not a shared state // object; the loser retries the open instead of failing. Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {} @@ -1312,8 +667,7 @@ fn open_host_writer_lock( .map_err(|_| HostBundleError::UnsafeInstallPath)? .into_std(); file.try_lock() - .map_err(std::io::Error::from) - .map_err(|_| host_bundle_recovery_required!())?; + .map_err(|_| HostBundleError::HostWriterBusy)?; Ok(HostWriterLock { host, file }) } @@ -1365,42 +719,6 @@ fn remove_regular_if_exists(parent: &Dir, name: &str) -> Result<(), HostBundleEr Ok(()) } -pub(super) fn remove_if_digest_matches( - parent: &Dir, - name: &str, - expected_digest: [u8; 32], -) -> Result<(), HostBundleError> { - let Some(bytes) = read_regular_nofollow(parent, name)? else { - return Ok(()); - }; - let actual: [u8; 32] = Sha256::digest(&bytes).into(); - if actual != expected_digest { - return Err(host_bundle_recovery_required!()); - } - parent - .remove_file(name) - .map_err(|_| host_bundle_storage_failure!()) -} - -pub(super) fn move_regular_to_backup( - parent: &Dir, - name: &str, - backup_dir: &Dir, - backup_name: &str, -) -> Result<(), HostBundleError> { - if !regular_file_exists(parent, name)? || !is_safe_component(backup_name) { - return Err(HostBundleError::UnsafeInstallPath); - } - if regular_file_exists(backup_dir, backup_name)? { - return Err(host_bundle_recovery_required!()); - } - parent - .rename(name, backup_dir, backup_name) - .map_err(|_| host_bundle_storage_failure!())?; - sync_cap_dir(parent)?; - sync_cap_dir(backup_dir) -} - pub(super) fn atomic_write_nofollow( parent: &Dir, name: &str, diff --git a/crates/tracedecay-agent-hosts/src/agents/host_bundle_registry.rs b/crates/tracedecay-agent-hosts/src/agents/host_bundle_registry.rs index 14d570bdb4..afb1d0ba89 100644 --- a/crates/tracedecay-agent-hosts/src/agents/host_bundle_registry.rs +++ b/crates/tracedecay-agent-hosts/src/agents/host_bundle_registry.rs @@ -1390,7 +1390,14 @@ mod tests { .find(|asset| asset.relative_path.ends_with("plugins/tracedecay.ts")) .map(|asset| String::from_utf8(asset.bytes.clone()).unwrap()) .expect("OpenCode set includes Hook V2 plugin"); - assert!(!plugin.is_empty(), "OpenCode set includes a plugin payload"); + for marker in [ + r#"dispatchAfterAck("hook-opencode-event", event, deliver)"#, + r#"dispatchAfterAck("hook-opencode-tool-after", { input, output }, deliver)"#, + r#""tool.execute.after": ("#, + r#"id: "tracedecay-hooks""#, + ] { + assert!(plugin.contains(marker), "OpenCode plugin lacks {marker}"); + } } #[test] diff --git a/crates/tracedecay-agent-hosts/src/agents/host_component_registration.rs b/crates/tracedecay-agent-hosts/src/agents/host_component_registration.rs index e7a1acdf1b..36cb46e0e3 100644 --- a/crates/tracedecay-agent-hosts/src/agents/host_component_registration.rs +++ b/crates/tracedecay-agent-hosts/src/agents/host_component_registration.rs @@ -2,91 +2,35 @@ use std::collections::BTreeSet; use std::fs; -#[cfg(unix)] -use std::os::unix::fs::MetadataExt; use std::path::{Path, PathBuf}; -use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use tracedecay_host_integration::host_bundle_stale_preview; use tracedecay_host_integration::host_bundle_storage_failure; -const REGISTRATION_BACKUP_IDENTITY_SCHEMA_VERSION: u16 = 2; - -#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)] -struct RegistrationBackupIdentityV1 { - schema_version: u16, - integration_id: String, - canonical_home: PathBuf, - canonical_profile: PathBuf, -} - -#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)] +#[derive(Clone, Debug, PartialEq, Eq)] struct RegistrationObservedStateV1 { present: bool, digest: [u8; 32], metadata: Option, } -#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)] -struct RegistrationDirectoryAppliedStateV2 { - metadata: crate::agents::HostFileMetadataIdentityV1, - unix_identity: Option<(u64, u64)>, -} - -#[derive(Deserialize)] -struct HostConfigWriteIntentV2 { - schema_version: u16, - digest: [u8; 32], - metadata: Option, -} - -#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)] -struct RegistrationMutationPlanV1 { - schema_version: u16, - integration_id: String, - operation: crate::agents::host_bundle::HostBundleLifecycleOpV1, - paths: Vec, - directories: Vec, -} +type RegistrationFileBytes = Option<(Vec, crate::agents::HostFileMetadataIdentityV1)>; -impl RegistrationBackupIdentityV1 { - fn new( - integration_id: &str, - home: &Path, - profile: &Path, - ) -> Result { - Ok(Self { - schema_version: REGISTRATION_BACKUP_IDENTITY_SCHEMA_VERSION, - integration_id: integration_id.to_string(), - canonical_home: canonical_path(home)?, - canonical_profile: canonical_path(profile)?, - }) - } - - fn validate( - &self, - integration_id: &str, - home: &Path, - profile: &Path, - ) -> Result<(), crate::agents::host_bundle::HostBundleError> { - if self.schema_version != REGISTRATION_BACKUP_IDENTITY_SCHEMA_VERSION { - return Err(crate::agents::host_bundle::HostBundleError::UnsupportedRecoveryFormat); - } - let observed = Self::new(integration_id, home, profile)?; - (self.integration_id == observed.integration_id - && self.canonical_home == observed.canonical_home - && self.canonical_profile == observed.canonical_profile) - .then_some(()) - .ok_or(crate::agents::host_bundle::HostBundleError::WrongTarget) - } +/// Pre-effect registration bytes for the one in-flight operation, held only in +/// this authority. Nothing is persisted: a restarted process has no copy to +/// restore, and the next lifecycle run reconciles the host registration. +struct StagedRegistration { + operation_id: [u8; 16], + files: Vec<(PathBuf, RegistrationFileBytes)>, + applied: Option>, + effect_started: bool, } pub struct CatalogHostComponentRegistrationAuthority { integration: Box, context: crate::agents::InstallContext, health_context: crate::agents::HealthcheckContext, - lifecycle_root: PathBuf, registration_path: Option, operation: crate::agents::host_bundle::HostBundleLifecycleOpV1, should_apply: bool, @@ -101,6 +45,11 @@ pub struct CatalogHostComponentRegistrationAuthority { /// writes its own artifacts. `apply` compares against this so that only a /// genuinely foreign edit invalidates the transaction. staged_foreign_registration_revision: Option<[u8; 32]>, + staged: Option, + /// Remediation for a host whose only activation route is interactive: + /// the transaction commits the staged source that route consumes and + /// leaves the host registration untouched. + deferred_activation: Option, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -125,34 +74,24 @@ impl CatalogHostComponentRegistrationAuthority { pub fn new( agent_id: &str, home: &Path, - lifecycle_root: &Path, operation: crate::agents::host_bundle::HostBundleLifecycleOpV1, ) -> tracedecay_domain::errors::Result { let tracedecay_bin = current_tracedecay_binary()?; - Self::new_with_tracedecay_bin(agent_id, home, lifecycle_root, operation, tracedecay_bin) + Self::new_with_tracedecay_bin(agent_id, home, operation, tracedecay_bin) } pub fn new_with_tracedecay_bin( agent_id: &str, home: &Path, - lifecycle_root: &Path, operation: crate::agents::host_bundle::HostBundleLifecycleOpV1, tracedecay_bin: String, ) -> tracedecay_domain::errors::Result { - Self::new_with_tracedecay_bin_and_dashboard( - agent_id, - home, - lifecycle_root, - operation, - tracedecay_bin, - true, - ) + Self::new_with_tracedecay_bin_and_dashboard(agent_id, home, operation, tracedecay_bin, true) } pub fn new_with_tracedecay_bin_and_dashboard( agent_id: &str, home: &Path, - lifecycle_root: &Path, operation: crate::agents::host_bundle::HostBundleLifecycleOpV1, tracedecay_bin: String, dashboard: bool, @@ -169,7 +108,6 @@ impl CatalogHostComponentRegistrationAuthority { context: crate::agents::InstallContext { home: home.to_path_buf(), tracedecay_bin, - tool_permissions: crate::agents::expected_tool_perms()?, project_root: None, dashboard, }, @@ -177,16 +115,23 @@ impl CatalogHostComponentRegistrationAuthority { home: home.to_path_buf(), project_path, }, - lifecycle_root: lifecycle_root.to_path_buf(), registration_path, operation, should_apply: false, confirmed_registration_revision: None, declared_artifact_writes: BTreeSet::new(), staged_foreign_registration_revision: None, + staged: None, + deferred_activation: None, }) } + /// The host action still required after this transaction committed its + /// staged source, or `None` when the registration was fully applied. + pub fn deferred_activation(&self) -> Option<&str> { + self.deferred_activation.as_deref() + } + fn registration_error( host: crate::agents::host_bundle::HostKindV1, error: tracedecay_domain::errors::TraceDecayError, @@ -205,85 +150,6 @@ impl CatalogHostComponentRegistrationAuthority { host_bundle_storage_failure!() } - fn backup_dir(&self, operation_id: [u8; 16]) -> PathBuf { - self.lifecycle_root - .join(".tracedecay-host-bundle-v1") - .join("registration-backups") - .join(hex::encode(operation_id)) - .join(self.integration.id()) - } - - fn backup_path(&self, operation_id: [u8; 16], index: usize) -> PathBuf { - self.backup_dir(operation_id) - .join(format!("registration-{index}")) - } - - fn missing_marker_path(&self, operation_id: [u8; 16], index: usize) -> PathBuf { - self.backup_dir(operation_id) - .join(format!("registration-{index}.missing")) - } - - fn registration_path_marker(&self, operation_id: [u8; 16], index: usize) -> PathBuf { - self.backup_dir(operation_id) - .join(format!("registration-{index}.path.json")) - } - - fn registration_permission_marker(&self, operation_id: [u8; 16], index: usize) -> PathBuf { - self.backup_dir(operation_id) - .join(format!("registration-{index}.permissions.json")) - } - - fn directory_path_marker(&self, operation_id: [u8; 16], index: usize) -> PathBuf { - self.backup_dir(operation_id) - .join(format!("directory-{index}.path.json")) - } - - fn directory_metadata_marker(&self, operation_id: [u8; 16], index: usize) -> PathBuf { - self.backup_dir(operation_id) - .join(format!("directory-{index}.metadata.json")) - } - - fn directory_missing_marker(&self, operation_id: [u8; 16], index: usize) -> PathBuf { - self.backup_dir(operation_id) - .join(format!("directory-{index}.missing")) - } - - fn directory_applied_metadata_marker(&self, operation_id: [u8; 16], index: usize) -> PathBuf { - self.backup_dir(operation_id) - .join(format!("directory-{index}.applied.metadata.json")) - } - - fn directory_recovery_metadata_marker(&self, operation_id: [u8; 16], index: usize) -> PathBuf { - self.backup_dir(operation_id) - .join(format!("directory-{index}.recovery.metadata.json")) - } - - fn applied_state_marker(&self, operation_id: [u8; 16], index: usize) -> PathBuf { - self.backup_dir(operation_id) - .join(format!("registration-{index}.applied.json")) - } - - fn identity_path(&self, operation_id: [u8; 16]) -> PathBuf { - self.backup_dir(operation_id).join("identity.v1.json") - } - - fn backup_complete_path(&self, operation_id: [u8; 16]) -> PathBuf { - self.backup_dir(operation_id).join("backup.complete") - } - - fn registration_effect_path(&self, operation_id: [u8; 16]) -> PathBuf { - self.backup_dir(operation_id) - .join("registration-effect.started") - } - - fn mutation_plan_path(&self, operation_id: [u8; 16]) -> PathBuf { - self.backup_dir(operation_id).join("mutation-plan.v1.json") - } - - fn write_intent_root(&self, operation_id: [u8; 16]) -> PathBuf { - self.backup_dir(operation_id).join("write-intents") - } - fn registration_mode( &self, component_set: &crate::agents::host_bundle::HostComponentSetV1, @@ -304,7 +170,7 @@ impl CatalogHostComponentRegistrationAuthority { // Copilot's deployed artifact is a receipt-owned component // descriptor; the host carries nothing until `copilot mcp add` // writes its own registry, so the deployed bytes alone are not the - // lifecycle and artifact-only backup/restore must refuse it. + // lifecycle. || component_set.host == crate::agents::host_bundle::HostKindV1::Copilot || component_set.host == crate::agents::host_bundle::HostKindV1::Cline || component_set.host == crate::agents::host_bundle::HostKindV1::RooCode @@ -327,16 +193,6 @@ impl CatalogHostComponentRegistrationAuthority { } } - /// Whether managed artifact bytes are the component set's complete host - /// lifecycle. Artifact backup/restore must refuse every other mode because - /// it intentionally does not snapshot or reconcile native registration. - pub fn supports_artifact_only_backup_restore( - &self, - component_set: &crate::agents::host_bundle::HostComponentSetV1, - ) -> bool { - self.registration_mode(component_set) == CatalogRegistrationMode::ArtifactOnly - } - fn requires_competing_analyzer_preflight( &self, component_set: &crate::agents::host_bundle::HostComponentSetV1, @@ -503,13 +359,6 @@ impl CatalogHostComponentRegistrationAuthority { Ok(paths) } - fn registration_directories( - &self, - _component_set: &crate::agents::host_bundle::HostComponentSetV1, - ) -> Result, crate::agents::host_bundle::HostBundleError> { - Ok(Vec::new()) - } - fn current_registration_revision( &self, component_set: &crate::agents::host_bundle::HostComponentSetV1, @@ -602,698 +451,125 @@ impl CatalogHostComponentRegistrationAuthority { Ok(digest.finalize().into()) } - fn backup_registration( + fn stage_registration( &self, component_set: &crate::agents::host_bundle::HostComponentSetV1, operation_id: [u8; 16], - ) -> Result<(), crate::agents::host_bundle::HostBundleError> { - let backup_dir = self.backup_dir(operation_id); - fs::create_dir_all(&backup_dir).map_err(|_| host_bundle_storage_failure!())?; - tracedecay_private_fs::framed_log::sync_parent_directory( - &backup_dir, - tracedecay_private_fs::framed_log::DirectorySyncPolicy::TolerateUnsupported, - ) - .map_err(|_| host_bundle_storage_failure!())?; - let identity = RegistrationBackupIdentityV1::new( - self.integration.id(), - &self.context.home, - &self.lifecycle_root, - )?; - let identity_bytes = - serde_json::to_vec(&identity).map_err(|_| host_bundle_storage_failure!())?; - write_registration_backup(&self.identity_path(operation_id), &identity_bytes)?; - let registration_paths = self.registration_paths(component_set)?; - let mut registration_directories = Vec::new(); - for path in self.registration_directories(component_set)? { - match fs::symlink_metadata(&path) { - Ok(metadata) - if metadata.file_type().is_symlink() - && path == self.context.home.join(".claude") => - { - return Err( - crate::agents::host_bundle::HostBundleError::UnsafeClaudeHomeSymlink, - ); - } - Ok(metadata) if metadata.file_type().is_symlink() || !metadata.is_dir() => { - return Err(crate::agents::host_bundle::HostBundleError::UnsafeInstallPath); - } - Ok(_) => registration_directories.push(path), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => { - registration_directories.push(path); - } - Err(_) => { - return Err(host_bundle_storage_failure!()); - } - } - } - let mutation_plan = RegistrationMutationPlanV1 { - schema_version: REGISTRATION_BACKUP_IDENTITY_SCHEMA_VERSION, - integration_id: self.integration.id().to_string(), - operation: self.operation, - paths: registration_paths.clone(), - directories: registration_directories.clone(), - }; - let mutation_plan = - serde_json::to_vec(&mutation_plan).map_err(|_| host_bundle_storage_failure!())?; - write_registration_backup(&self.mutation_plan_path(operation_id), &mutation_plan)?; - for (index, path) in registration_directories.iter().enumerate() { - let path_bytes = - serde_json::to_vec(path).map_err(|_| host_bundle_storage_failure!())?; - write_registration_backup( - &self.directory_path_marker(operation_id, index), - &path_bytes, - )?; - match fs::symlink_metadata(path) { - Ok(metadata) if metadata.file_type().is_symlink() || !metadata.is_dir() => { - return Err(crate::agents::host_bundle::HostBundleError::UnsafeInstallPath); - } - Ok(_) => { - let metadata = crate::agents::capture_host_file_metadata(path) - .map_err(|_| host_bundle_storage_failure!())?; - let metadata = serde_json::to_vec(&metadata) - .map_err(|_| host_bundle_storage_failure!())?; - write_registration_backup( - &self.directory_metadata_marker(operation_id, index), - &metadata, - )?; - } - Err(error) if error.kind() == std::io::ErrorKind::NotFound => { - write_registration_backup( - &self.directory_missing_marker(operation_id, index), - b"missing", - )?; - } - Err(_) => { - return Err(host_bundle_storage_failure!()); - } - } - } - for (index, path) in registration_paths.iter().enumerate() { - let path_bytes = - serde_json::to_vec(path).map_err(|_| host_bundle_storage_failure!())?; - write_registration_backup( - &self.registration_path_marker(operation_id, index), - &path_bytes, - )?; - match fs::symlink_metadata(path) { + ) -> Result { + let mut files = Vec::new(); + for path in self.registration_paths(component_set)? { + let original = match fs::symlink_metadata(&path) { Ok(metadata) if metadata.file_type().is_symlink() || !metadata.is_file() => { return Err(crate::agents::host_bundle::HostBundleError::UnsafeInstallPath); } - Ok(_) => { - let bytes = fs::read(path).map_err(|_| host_bundle_storage_failure!())?; - write_registration_backup(&self.backup_path(operation_id, index), &bytes)?; - let permissions = crate::agents::capture_host_file_metadata(path) - .map_err(|_| host_bundle_storage_failure!())?; - let permissions = serde_json::to_vec(&permissions) - .map_err(|_| host_bundle_storage_failure!())?; - write_registration_backup( - &self.registration_permission_marker(operation_id, index), - &permissions, - )?; - } - Err(error) if error.kind() == std::io::ErrorKind::NotFound => { - write_registration_backup( - &self.missing_marker_path(operation_id, index), - b"missing", - )?; - } - Err(_) => { - return Err(host_bundle_storage_failure!()); - } - } + Ok(_) => Some(( + fs::read(&path).map_err(|_| host_bundle_storage_failure!())?, + crate::agents::capture_host_file_metadata(&path) + .map_err(|_| host_bundle_storage_failure!())?, + )), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => None, + Err(_) => return Err(host_bundle_storage_failure!()), + }; + files.push((path, original)); } - write_registration_backup(&self.backup_complete_path(operation_id), b"complete")?; - Ok(()) + Ok(StagedRegistration { + operation_id, + files, + applied: None, + effect_started: false, + }) } - fn capture_applied_registration( - &self, - component_set: &crate::agents::host_bundle::HostComponentSetV1, + fn staged_registration( + &mut self, operation_id: [u8; 16], - ) -> Result<(), crate::agents::host_bundle::HostBundleError> { - for (index, path) in self.registration_paths(component_set)?.iter().enumerate() { - let observed = registration_observed_state(path)?; - let bytes = - serde_json::to_vec(&observed).map_err(|_| host_bundle_storage_failure!())?; - write_registration_backup(&self.applied_state_marker(operation_id, index), &bytes)?; - } - Ok(()) + ) -> Result<&mut StagedRegistration, crate::agents::host_bundle::HostBundleError> { + self.staged + .as_mut() + .filter(|staged| staged.operation_id == operation_id) + .ok_or(crate::agents::host_bundle::HostBundleError::WrongTarget) } - fn prepare_missing_registration_directories( - &self, + fn capture_applied_registration( + &mut self, operation_id: [u8; 16], ) -> Result<(), crate::agents::host_bundle::HostBundleError> { - let mutation_plan = fs::read(self.mutation_plan_path(operation_id)) - .map_err(|_| crate::agents::host_bundle::HostBundleError::WrongTarget)?; - let mutation_plan: RegistrationMutationPlanV1 = serde_json::from_slice(&mutation_plan) - .map_err(|_| crate::agents::host_bundle::HostBundleError::WrongTarget)?; - for (index, path) in mutation_plan.directories.iter().enumerate() { - if !self.directory_missing_marker(operation_id, index).is_file() { - continue; - } - match fs::symlink_metadata(path) { - // Project registration directories are disjoint from catalog - // artifact paths. If an absent directory appears before this - // authority creates it, that state is foreign drift. - Ok(_) => return Err(host_bundle_stale_preview!()), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} - Err(_) => { - return Err(host_bundle_storage_failure!()); - } - } - let parent = path - .parent() - .ok_or(crate::agents::host_bundle::HostBundleError::RecoveryDirectoryUnavailable)?; - let staging_path = parent.join(format!( - ".tracedecay-registration-apply-{}-{index}", - hex::encode(operation_id) - )); - match fs::symlink_metadata(&staging_path) { - Ok(metadata) if metadata.file_type().is_symlink() || !metadata.is_dir() => { - return Err( - crate::agents::host_bundle::HostBundleError::RecoveryDirectoryUnavailable, - ); - } - Ok(_) => { - if fs::read_dir(&staging_path) - .map_err(|_| host_bundle_storage_failure!())? - .next() - .is_some() - { - return Err( - crate::agents::host_bundle::HostBundleError::RecoveryDirectoryUnavailable, - ); - } - } - Err(error) if error.kind() == std::io::ErrorKind::NotFound => { - fs::create_dir(&staging_path).map_err(|_| { - crate::agents::host_bundle::HostBundleError::RecoveryDirectoryUnavailable - })?; - } - Err(_) => { - return Err( - crate::agents::host_bundle::HostBundleError::RecoveryDirectoryUnavailable, - ); - } - } - let applied = registration_directory_applied_state(&staging_path)?; - write_registration_backup( - &self.directory_applied_metadata_marker(operation_id, index), - &serde_json::to_vec(&applied).map_err(|_| host_bundle_storage_failure!())?, - )?; - sync_registration_metadata(&staging_path)?; - fs::rename(&staging_path, path).map_err(|_| { - crate::agents::host_bundle::HostBundleError::RecoveryDirectoryUnavailable - })?; - tracedecay_private_fs::framed_log::sync_parent_directory( - path, - tracedecay_private_fs::framed_log::DirectorySyncPolicy::TolerateUnsupported, - ) - .map_err(|_| host_bundle_storage_failure!())?; - } + let staged = self.staged_registration(operation_id)?; + let applied = staged + .files + .iter() + .map(|(path, _)| registration_observed_state(path)) + .collect::, _>>()?; + staged.applied = Some(applied); Ok(()) } fn validate_applied_registration( &self, - component_set: &crate::agents::host_bundle::HostComponentSetV1, operation_id: [u8; 16], ) -> Result<(), crate::agents::host_bundle::HostBundleError> { - for (index, path) in self.registration_paths(component_set)?.iter().enumerate() { - let expected = fs::read(self.applied_state_marker(operation_id, index)) - .map_err(|_| crate::agents::host_bundle::HostBundleError::WrongTarget)?; - let expected: RegistrationObservedStateV1 = serde_json::from_slice(&expected) - .map_err(|_| crate::agents::host_bundle::HostBundleError::WrongTarget)?; - if registration_observed_state(path)? != expected { + let staged = self + .staged + .as_ref() + .filter(|staged| staged.operation_id == operation_id) + .ok_or(crate::agents::host_bundle::HostBundleError::WrongTarget)?; + let applied = staged + .applied + .as_ref() + .ok_or(crate::agents::host_bundle::HostBundleError::WrongTarget)?; + for ((path, _), expected) in staged.files.iter().zip(applied) { + if registration_observed_state(path)? != *expected { return Err(host_bundle_stale_preview!()); } } Ok(()) } - fn original_registration_state( - &self, - operation_id: [u8; 16], - index: usize, - ) -> Result { - let backup = self.backup_path(operation_id, index); - if backup.is_file() { - let bytes = fs::read(backup).map_err(|_| host_bundle_storage_failure!())?; - return Ok(RegistrationObservedStateV1 { - present: true, - digest: Sha256::digest(bytes).into(), - metadata: Some(self.original_registration_permissions(operation_id, index)?), - }); - } - if self.missing_marker_path(operation_id, index).is_file() { - return Ok(RegistrationObservedStateV1 { - present: false, - digest: [0; 32], - metadata: None, - }); - } - Err(crate::agents::host_bundle::HostBundleError::WrongTarget) - } - - fn original_registration_permissions( - &self, - operation_id: [u8; 16], - index: usize, - ) -> Result< - crate::agents::HostFileMetadataIdentityV1, - crate::agents::host_bundle::HostBundleError, - > { - let bytes = fs::read(self.registration_permission_marker(operation_id, index)) - .map_err(|_| host_bundle_storage_failure!())?; - serde_json::from_slice(&bytes).map_err(|_| host_bundle_storage_failure!()) - } - - fn intended_registration_state( - &self, - operation_id: [u8; 16], - path: &Path, - ) -> Result, crate::agents::host_bundle::HostBundleError> - { - match fs::read( - crate::agents::host_config_write_intent_path( - &self.write_intent_root(operation_id), - path, - ) - .map_err(|_| host_bundle_storage_failure!())?, - ) { - Ok(intent) if intent != [0] => { - let intent: HostConfigWriteIntentV2 = serde_json::from_slice(&intent) - .map_err(|_| crate::agents::host_bundle::HostBundleError::WrongTarget)?; - if intent.schema_version != 2 { - return Err(crate::agents::host_bundle::HostBundleError::WrongTarget); - } - Ok(Some(RegistrationObservedStateV1 { - present: true, - digest: intent.digest, - metadata: intent.metadata, - })) - } - Ok(intent) if intent == [0] => Ok(Some(RegistrationObservedStateV1 { - present: false, - digest: [0; 32], - metadata: None, - })), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), - _ => Err(crate::agents::host_bundle::HostBundleError::WrongTarget), - } - } - + /// Restore the staged pre-effect bytes. Every path must still hold either + /// its original or its just-applied state before any byte is rewritten, so + /// a foreign edit made during the operation is never overwritten. fn restore_registration( - &self, - component_set: &crate::agents::host_bundle::HostComponentSetV1, - operation_id: [u8; 16], + staged: &StagedRegistration, ) -> Result<(), crate::agents::host_bundle::HostBundleError> { - let identity_bytes = fs::read(self.identity_path(operation_id)) - .map_err(|_| crate::agents::host_bundle::HostBundleError::WrongTarget)?; - let identity: RegistrationBackupIdentityV1 = serde_json::from_slice(&identity_bytes) - .map_err(|_| crate::agents::host_bundle::HostBundleError::WrongTarget)?; - identity.validate( - self.integration.id(), - &self.context.home, - &self.lifecycle_root, - )?; - let mutation_plan = fs::read(self.mutation_plan_path(operation_id)) - .map_err(|_| crate::agents::host_bundle::HostBundleError::WrongTarget)?; - let mutation_plan: RegistrationMutationPlanV1 = serde_json::from_slice(&mutation_plan) - .map_err(|_| crate::agents::host_bundle::HostBundleError::WrongTarget)?; - if mutation_plan.schema_version != REGISTRATION_BACKUP_IDENTITY_SCHEMA_VERSION { - return Err(crate::agents::host_bundle::HostBundleError::UnsupportedRecoveryFormat); - } - if mutation_plan.integration_id != self.integration.id() - || mutation_plan.operation != self.operation - { - return Err(crate::agents::host_bundle::HostBundleError::WrongTarget); - } - let mut persisted_paths = Vec::new(); - for index in 0.. { - match fs::read(self.registration_path_marker(operation_id, index)) { - Ok(bytes) => { - let path = serde_json::from_slice::(&bytes) - .map_err(|_| host_bundle_storage_failure!())?; - persisted_paths.push(path); - } - Err(error) if error.kind() == std::io::ErrorKind::NotFound => break, - Err(_) => { - return Err(host_bundle_storage_failure!()); - } - } - } - let current_registration_paths = self.registration_paths(component_set)?; - // The inventory recorded at backup time is what the backup markers are - // indexed by, so it -- not a fresh recomputation -- is the set to - // restore. A newer binary may legitimately add version-scoped paths to - // that recomputation while recovering an older journal. Admit those - // paths only while they are absent: there are no bytes to restore, and - // the new transaction will snapshot them after recovery. Any live, - // symlinked, or unreadable path outside the persisted inventory stays - // fail-closed as a different target. - for path in current_registration_paths - .iter() - .filter(|path| !persisted_paths.contains(path)) - { - if !matches!( - fs::symlink_metadata(path), - Err(error) if error.kind() == std::io::ErrorKind::NotFound - ) { - return Err(crate::agents::host_bundle::HostBundleError::WrongTarget); - } - } - let registration_paths = persisted_paths.clone(); - let registration_directories = mutation_plan.directories.clone(); - let mut persisted_directories = Vec::new(); - for index in 0.. { - match fs::read(self.directory_path_marker(operation_id, index)) { - Ok(bytes) => { - persisted_directories.push( - serde_json::from_slice::(&bytes) - .map_err(|_| host_bundle_storage_failure!())?, - ); - } - Err(error) if error.kind() == std::io::ErrorKind::NotFound => break, - Err(_) => { - return Err(host_bundle_storage_failure!()); - } - } - } - if mutation_plan.paths != registration_paths - || persisted_paths != registration_paths - || persisted_directories != registration_directories - || !registration_directories.is_empty() - { - return Err(crate::agents::host_bundle::HostBundleError::WrongTarget); - } - let mut vanished_directories = Vec::new(); - let mut recovery_owned_directories = Vec::new(); - for (index, path) in registration_directories.iter().enumerate() { - let metadata_marker = self.directory_metadata_marker(operation_id, index); - let missing_marker = self.directory_missing_marker(operation_id, index); - if metadata_marker.is_file() == missing_marker.is_file() { - return Err(crate::agents::host_bundle::HostBundleError::UnsupportedRecoveryFormat); - } - if metadata_marker.is_file() { - // Parse every metadata record before restoring any file. The - // original identity is also the only permitted identity for a - // pre-existing directory: registration never changes its - // permissions or ACLs, so any other metadata is foreign drift. - let original_metadata: crate::agents::HostFileMetadataIdentityV1 = - serde_json::from_slice( - &fs::read(metadata_marker).map_err(|_| host_bundle_storage_failure!())?, - ) - .map_err(|_| { - crate::agents::host_bundle::HostBundleError::UnsupportedRecoveryFormat - })?; - match fs::symlink_metadata(path) { - Ok(metadata) if metadata.file_type().is_symlink() || !metadata.is_dir() => { - return Err(crate::agents::host_bundle::HostBundleError::UnsafeInstallPath); - } - Ok(_) => { - let observed = crate::agents::capture_host_file_metadata(path) - .map_err(|_| host_bundle_storage_failure!())?; - if observed != original_metadata { - return Err(host_bundle_stale_preview!()); - } - let recovery_marker = - self.directory_recovery_metadata_marker(operation_id, index); - if recovery_marker.is_file() { - let recovery_metadata: crate::agents::HostFileMetadataIdentityV1 = - serde_json::from_slice(&fs::read(recovery_marker).map_err(|_| { - host_bundle_storage_failure!() - })?) - .map_err(|_| { - crate::agents::host_bundle::HostBundleError::UnsupportedRecoveryFormat - })?; - if recovery_metadata != original_metadata { - return Err(host_bundle_stale_preview!()); - } - recovery_owned_directories.push(path); - } - } - Err(error) if error.kind() == std::io::ErrorKind::NotFound => { - vanished_directories.push((index, path)); - recovery_owned_directories.push(path); - } - Err(_) => { - return Err(host_bundle_storage_failure!()); - } - } - } else { - match fs::symlink_metadata(path) { - Ok(metadata) if metadata.file_type().is_symlink() || !metadata.is_dir() => { - return Err(crate::agents::host_bundle::HostBundleError::UnsafeInstallPath); - } - Ok(_) => { - let applied_marker = - self.directory_applied_metadata_marker(operation_id, index); - if !applied_marker.is_file() { - return Err(host_bundle_stale_preview!()); - } - let applied: RegistrationDirectoryAppliedStateV2 = serde_json::from_slice( - &fs::read(applied_marker) - .map_err(|_| host_bundle_storage_failure!())?, - ) - .map_err(|_| { - crate::agents::host_bundle::HostBundleError::UnsupportedRecoveryFormat - })?; - if registration_directory_applied_state(path)? != applied { - return Err(host_bundle_stale_preview!()); - } - recovery_owned_directories.push(path); - } - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} - Err(_) => { - return Err(host_bundle_storage_failure!()); - } - } - } - } - for (index, path) in registration_paths.iter().enumerate() { + for (index, (path, original)) in staged.files.iter().enumerate() { let observed = registration_observed_state(path)?; - let original = self.original_registration_state(operation_id, index)?; - let intended = self.intended_registration_state(operation_id, path)?; - let parent_will_be_recreated = !observed.present - && recovery_owned_directories - .iter() - .any(|directory| path.starts_with(directory)); - if observed != original - && intended.as_ref() != Some(&observed) - && !parent_will_be_recreated + if observed != original_observed_state(original.as_ref()) + && staged.applied.as_ref().map(|applied| &applied[index]) != Some(&observed) { return Err(host_bundle_stale_preview!()); } } - vanished_directories.sort_by_key(|(_, path)| path.components().count()); - for (index, path) in vanished_directories { - let metadata: crate::agents::HostFileMetadataIdentityV1 = serde_json::from_slice( - &fs::read(self.directory_metadata_marker(operation_id, index)) - .map_err(|_| host_bundle_storage_failure!())?, - ) - .map_err(|_| crate::agents::host_bundle::HostBundleError::UnsupportedRecoveryFormat)?; - let parent = path - .parent() - .ok_or(crate::agents::host_bundle::HostBundleError::RecoveryDirectoryUnavailable)?; - let staging_path = parent.join(format!( - ".tracedecay-registration-recovery-{}-{index}", - hex::encode(operation_id) - )); - match fs::symlink_metadata(&staging_path) { - Ok(staging_metadata) - if staging_metadata.file_type().is_symlink() || !staging_metadata.is_dir() => - { - return Err( - crate::agents::host_bundle::HostBundleError::RecoveryDirectoryUnavailable, - ); - } - Ok(_) => { - if fs::read_dir(&staging_path) - .map_err(|_| host_bundle_storage_failure!())? - .next() - .is_some() - { - return Err( - crate::agents::host_bundle::HostBundleError::RecoveryDirectoryUnavailable, - ); - } - } - Err(error) if error.kind() == std::io::ErrorKind::NotFound => { - fs::create_dir(&staging_path).map_err(|_| { - crate::agents::host_bundle::HostBundleError::RecoveryDirectoryUnavailable - })?; - } - Err(_) => { - return Err( - crate::agents::host_bundle::HostBundleError::RecoveryDirectoryUnavailable, - ); - } - } - crate::agents::restore_host_file_metadata(&staging_path, &metadata).map_err(|_| { - crate::agents::host_bundle::HostBundleError::RecoveryDirectoryUnavailable - })?; - sync_registration_metadata(&staging_path)?; - write_registration_backup( - &self.directory_recovery_metadata_marker(operation_id, index), - &serde_json::to_vec(&metadata).map_err(|_| host_bundle_storage_failure!())?, - )?; - fs::rename(&staging_path, path).map_err(|_| { - crate::agents::host_bundle::HostBundleError::RecoveryDirectoryUnavailable - })?; - tracedecay_private_fs::framed_log::sync_parent_directory( - path, - tracedecay_private_fs::framed_log::DirectorySyncPolicy::TolerateUnsupported, - ) - .map_err(|_| host_bundle_storage_failure!())?; - } - for (index, path) in registration_paths.iter().enumerate() { + for (path, original) in &staged.files { let observed = registration_observed_state(path)?; - let original = self.original_registration_state(operation_id, index)?; - let backup = self.backup_path(operation_id, index); - let missing = self.missing_marker_path(operation_id, index); - if backup.is_file() { - let permissions = - fs::read(self.registration_permission_marker(operation_id, index)) - .map_err(|_| host_bundle_storage_failure!())?; - let permissions: crate::agents::HostFileMetadataIdentityV1 = - serde_json::from_slice(&permissions) - .map_err(|_| host_bundle_storage_failure!())?; - if observed != original { - let bytes = fs::read(&backup).map_err(|_| host_bundle_storage_failure!())?; - crate::agents::safe_write_bytes_file_with_metadata( - path, - &bytes, - None, - Some(&permissions), - ) - .map_err(|_| host_bundle_storage_failure!())?; - #[cfg(feature = "test-transport")] - if std::env::var_os("TRACEDECAY_TEST_ABORT_AFTER_REGISTRATION_ROLLBACK_WRITE") - .is_some() - || std::env::var_os( - "TRACEDECAY_TEST_ABORT_AFTER_REGISTRATION_ROLLBACK_WRITE_PATH", + match original { + Some((bytes, metadata)) => { + if observed != original_observed_state(original.as_ref()) { + crate::agents::safe_write_bytes_file_with_metadata( + path, + bytes, + Some(metadata), ) - .is_some_and(|expected| Path::new(&expected) == path) - { - std::process::abort(); + .map_err(|_| host_bundle_storage_failure!())?; } + crate::agents::restore_host_file_metadata(path, metadata) + .map_err(|_| host_bundle_storage_failure!())?; + sync_registration_metadata(path)?; } - crate::agents::restore_host_file_metadata(path, &permissions) - .map_err(|_| host_bundle_storage_failure!())?; - sync_registration_metadata(path)?; - } else if missing.is_file() { - if observed == original { - continue; - } - match fs::symlink_metadata(path) { + None if observed.present => match fs::symlink_metadata(path) { Ok(metadata) if metadata.file_type().is_file() => { - fs::remove_file(path).map_err(|_| host_bundle_storage_failure!())? + fs::remove_file(path).map_err(|_| host_bundle_storage_failure!())?; } Ok(_) => { return Err(crate::agents::host_bundle::HostBundleError::UnsafeInstallPath); } Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} - Err(_) => { - return Err(host_bundle_storage_failure!()); - } - } - } - } - let mut directory_restore_order = registration_directories - .iter() - .enumerate() - .collect::>(); - directory_restore_order - .sort_by_key(|(_, path)| std::cmp::Reverse(path.components().count())); - for (index, path) in directory_restore_order { - let metadata_marker = self.directory_metadata_marker(operation_id, index); - if metadata_marker.is_file() { - let metadata: crate::agents::HostFileMetadataIdentityV1 = serde_json::from_slice( - &fs::read(metadata_marker).map_err(|_| host_bundle_storage_failure!())?, - ) - .map_err(|_| { - crate::agents::host_bundle::HostBundleError::UnsupportedRecoveryFormat - })?; - if crate::agents::capture_host_file_metadata(path) - .map_err(|_| host_bundle_storage_failure!())? - != metadata - { - return Err(host_bundle_stale_preview!()); - } - crate::agents::restore_host_file_metadata(path, &metadata) - .map_err(|_| host_bundle_storage_failure!())?; - } else { - let applied_marker = self.directory_applied_metadata_marker(operation_id, index); - match fs::symlink_metadata(path) { - Ok(metadata) if metadata.file_type().is_symlink() || !metadata.is_dir() => { - return Err(crate::agents::host_bundle::HostBundleError::UnsafeInstallPath); - } - Ok(_) => {} - Err(error) if error.kind() == std::io::ErrorKind::NotFound => { - let parent = path.parent().ok_or( - crate::agents::host_bundle::HostBundleError::RecoveryDirectoryUnavailable, - )?; - let staging_path = parent.join(format!( - ".tracedecay-registration-apply-{}-{index}", - hex::encode(operation_id) - )); - match fs::remove_dir(&staging_path) { - Ok(()) => {} - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} - Err(_) => { - return Err(host_bundle_storage_failure!()); - } - } - continue; - } - Err(_) => { - return Err(host_bundle_storage_failure!()); - } - } - if !applied_marker.is_file() { - return Err(host_bundle_stale_preview!()); - } - let applied: RegistrationDirectoryAppliedStateV2 = serde_json::from_slice( - &fs::read(applied_marker).map_err(|_| host_bundle_storage_failure!())?, - ) - .map_err(|_| { - crate::agents::host_bundle::HostBundleError::UnsupportedRecoveryFormat - })?; - if registration_directory_applied_state(path)? != applied { - return Err(host_bundle_stale_preview!()); - } - match fs::remove_dir(path) { - Ok(()) => {} - Err(error) - if matches!( - error.kind(), - std::io::ErrorKind::NotFound | std::io::ErrorKind::DirectoryNotEmpty - ) => {} - Err(_) => { - return Err(host_bundle_storage_failure!()); - } - } + Err(_) => return Err(host_bundle_storage_failure!()), + }, + None => {} } } Ok(()) } - - fn retire_backup( - &self, - operation_id: [u8; 16], - ) -> Result<(), crate::agents::host_bundle::HostBundleError> { - let backup_dir = self.backup_dir(operation_id); - match fs::symlink_metadata(&backup_dir) { - Ok(metadata) if metadata.is_dir() && !metadata.file_type().is_symlink() => { - fs::remove_dir_all(backup_dir).map_err(|_| host_bundle_storage_failure!()) - } - Ok(_) => Err(crate::agents::host_bundle::HostBundleError::UnsafeInstallPath), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), - Err(_) => Err(host_bundle_storage_failure!()), - } - } } fn current_tracedecay_binary() -> tracedecay_domain::errors::Result { @@ -1333,21 +609,6 @@ impl crate::agents::host_bundle::HostComponentSetRegistrationV1 self.competing_opencode_analyzer_claims(component_set) } - /// Cursor is the one host whose pre-receipt bundles carry a durable - /// first-party anchor the adapter can verify; every other integration - /// stays fail-closed on the trait default, so receiptless adoption there - /// requires the operator's explicit `--yes --adopt`. - fn receiptless_component_provenance( - &self, - component: crate::agents::host_bundle::HostComponentV1, - ) -> bool { - self.integration.id() == "cursor" - && crate::agents::cursor::receiptless_component_provenance( - &self.context.home, - component, - ) - } - fn confirm_preview( &mut self, component_set: &crate::agents::host_bundle::HostComponentSetV1, @@ -1453,6 +714,7 @@ impl crate::agents::host_bundle::HostComponentSetRegistrationV1 crate::agents::host_bundle::HostBundleLifecycleOpV1::Update | crate::agents::host_bundle::HostBundleLifecycleOpV1::Repair => true, }; + self.deferred_activation = None; let interactive_guidance = match self.operation { crate::agents::host_bundle::HostBundleLifecycleOpV1::Uninstall => { self.integration.interactive_removal_guidance() @@ -1467,8 +729,13 @@ impl crate::agents::host_bundle::HostComponentSetRegistrationV1 .map_err(|error| Self::registration_error(component_set.host, error))? { crate::agents::NonInteractiveInstallOutcome::Ready => None, + // Kimi's interactive `/plugins install` consumes the + // staged source, so the transaction must still commit + // it; the registration stays untouched. crate::agents::NonInteractiveInstallOutcome::DeferredUserAction(action) => { - Some(action.remediation) + self.should_apply = false; + self.deferred_activation = Some(action.remediation); + return Ok(()); } } } else { @@ -1548,7 +815,7 @@ impl crate::agents::host_bundle::HostComponentSetRegistrationV1 )?), None => None, }; - self.backup_registration(component_set, request.operation_id)?; + self.staged = Some(self.stage_registration(component_set, request.operation_id)?); Ok(()) } @@ -1593,49 +860,27 @@ impl crate::agents::host_bundle::HostComponentSetRegistrationV1 } } if !self.should_apply { - return self.capture_applied_registration(component_set, request.operation_id); + return self.capture_applied_registration(request.operation_id); } - write_registration_backup( - &self.registration_effect_path(request.operation_id), - b"started", - )?; - if request.lifecycle.operation - != crate::agents::host_bundle::HostBundleLifecycleOpV1::Uninstall - { - self.prepare_missing_registration_directories(request.operation_id)?; + self.staged_registration(request.operation_id)? + .effect_started = true; + let components = component_set + .components + .iter() + .map(|component| component.manifest.component) + .collect::>(); + let result = match request.lifecycle.operation { + crate::agents::host_bundle::HostBundleLifecycleOpV1::Uninstall => self + .integration + .deactivate_deployed_host_component_registration(&components, &self.context), + crate::agents::host_bundle::HostBundleLifecycleOpV1::Install + | crate::agents::host_bundle::HostBundleLifecycleOpV1::Update + | crate::agents::host_bundle::HostBundleLifecycleOpV1::Repair => self + .integration + .activate_deployed_host_component_registration(&components, &self.context), } - let result = crate::agents::with_host_config_write_intents( - self.write_intent_root(request.operation_id), - || match mode { - CatalogRegistrationMode::ArtifactOnly => Ok(()), - CatalogRegistrationMode::DeployedActivation => { - let components = component_set - .components - .iter() - .map(|component| component.manifest.component) - .collect::>(); - match request.lifecycle.operation { - crate::agents::host_bundle::HostBundleLifecycleOpV1::Uninstall => self - .integration - .deactivate_deployed_host_component_registration( - &components, - &self.context, - ) - .map_err(|error| Self::registration_error(component_set.host, error)), - crate::agents::host_bundle::HostBundleLifecycleOpV1::Install - | crate::agents::host_bundle::HostBundleLifecycleOpV1::Update - | crate::agents::host_bundle::HostBundleLifecycleOpV1::Repair => self - .integration - .activate_deployed_host_component_registration( - &components, - &self.context, - ) - .map_err(|error| Self::registration_error(component_set.host, error)), - } - } - }, - ); - let captured = self.capture_applied_registration(component_set, request.operation_id); + .map_err(|error| Self::registration_error(component_set.host, error)); + let captured = self.capture_applied_registration(request.operation_id); match (result, captured) { (_, Err(error)) => Err(error), (Err(error), Ok(())) => Err(error), @@ -1656,7 +901,10 @@ impl crate::agents::host_bundle::HostComponentSetRegistrationV1 if self.registration_mode(component_set) == CatalogRegistrationMode::ArtifactOnly { return Ok(()); } - self.validate_applied_registration(component_set, request.operation_id)?; + self.validate_applied_registration(request.operation_id)?; + if self.deferred_activation.is_some() { + return Ok(()); + } let expected = if request.lifecycle.operation == crate::agents::host_bundle::HostBundleLifecycleOpV1::Uninstall { @@ -1678,9 +926,10 @@ impl crate::agents::host_bundle::HostComponentSetRegistrationV1 fn commit( &mut self, _component_set: &crate::agents::host_bundle::HostComponentSetV1, - request: &crate::agents::host_bundle::HostComponentSetExecutionRequestV1, + _request: &crate::agents::host_bundle::HostComponentSetExecutionRequestV1, ) -> Result<(), crate::agents::host_bundle::HostBundleError> { - self.retire_backup(request.operation_id) + self.staged = None; + Ok(()) } fn rollback( @@ -1692,14 +941,19 @@ impl crate::agents::host_bundle::HostComponentSetRegistrationV1 if self.registration_mode(component_set) == CatalogRegistrationMode::ArtifactOnly { return Ok(()); } - if !self.backup_complete_path(request.operation_id).is_file() - || !self - .registration_effect_path(request.operation_id) - .is_file() - { + // A rollback before `stage` ran finds nothing staged; there is no + // pre-effect copy outside this process by design. + let Some(staged) = self + .staged + .take() + .filter(|staged| staged.operation_id == request.operation_id) + else { + return Ok(()); + }; + if !staged.effect_started { return Ok(()); } - self.restore_registration(component_set, request.operation_id) + Self::restore_registration(&staged) } } @@ -1755,10 +1009,6 @@ fn claim_identifier(name: &str) -> String { format!("opaque-{}", hex::encode(&Sha256::digest(name)[..8])) } -fn canonical_path(path: &Path) -> Result { - fs::canonicalize(path).map_err(|_| host_bundle_storage_failure!()) -} - fn registration_observed_state( path: &Path, ) -> Result { @@ -1788,22 +1038,21 @@ fn registration_observed_state( } } -fn registration_directory_applied_state( - path: &Path, -) -> Result { - let metadata = fs::symlink_metadata(path).map_err(|_| host_bundle_storage_failure!())?; - if metadata.file_type().is_symlink() || !metadata.is_dir() { - return Err(crate::agents::host_bundle::HostBundleError::UnsafeInstallPath); +fn original_observed_state( + original: Option<&(Vec, crate::agents::HostFileMetadataIdentityV1)>, +) -> RegistrationObservedStateV1 { + match original { + Some((bytes, metadata)) => RegistrationObservedStateV1 { + present: true, + digest: Sha256::digest(bytes).into(), + metadata: Some(metadata.clone()), + }, + None => RegistrationObservedStateV1 { + present: false, + digest: [0; 32], + metadata: None, + }, } - #[cfg(unix)] - let unix_identity = Some((metadata.dev(), metadata.ino())); - #[cfg(not(unix))] - let unix_identity = None; - Ok(RegistrationDirectoryAppliedStateV2 { - metadata: crate::agents::capture_host_file_metadata(path) - .map_err(|_| host_bundle_storage_failure!())?, - unix_identity, - }) } fn sync_registration_metadata( @@ -1819,19 +1068,6 @@ fn sync_registration_metadata( .map_err(|_| host_bundle_storage_failure!()) } -fn write_registration_backup( - path: &Path, - bytes: &[u8], -) -> Result<(), crate::agents::host_bundle::HostBundleError> { - tracedecay_private_fs::framed_log::atomic_write( - path, - "host-registration-state", - bytes, - tracedecay_private_fs::framed_log::DirectorySyncPolicy::TolerateUnsupported, - ) - .map_err(|_| host_bundle_storage_failure!()) -} - #[cfg(test)] mod tests { use super::*; @@ -1869,16 +1105,13 @@ mod tests { /// Gemini's deployed artifacts are an extension *source*: the host carries /// nothing until `gemini extensions install` adopts them. Classifying the /// set as artifact-only would let a lifecycle report an activation that - /// never happened, and would let artifact backup/restore claim it can - /// reverse a host registration it never snapshots. + /// never happened. #[test] fn gemini_component_sets_are_not_artifact_only_lifecycles() { let home = tempfile::tempdir().expect("home"); - let lifecycle_root = tempfile::tempdir().expect("lifecycle root"); let authority = CatalogHostComponentRegistrationAuthority::new( "gemini", home.path(), - lifecycle_root.path(), crate::agents::host_bundle::HostBundleLifecycleOpV1::Install, ) .expect("catalog registration authority"); @@ -1891,7 +1124,8 @@ mod tests { .expect("Gemini has a compiled default set"); assert!( - !authority.supports_artifact_only_backup_restore(&component_set.component_set), + authority.registration_mode(&component_set.component_set) + != CatalogRegistrationMode::ArtifactOnly, "the Gemini lifecycle drives `gemini extensions install`, so its deployed \ bytes are not the whole lifecycle" ); @@ -1901,7 +1135,6 @@ mod tests { let cursor = CatalogHostComponentRegistrationAuthority::new( "cursor", home.path(), - lifecycle_root.path(), crate::agents::host_bundle::HostBundleLifecycleOpV1::Install, ) .expect("catalog registration authority"); @@ -1912,7 +1145,10 @@ mod tests { crate::agents::TEST_GENERATOR_COMMIT, ) .expect("Cursor has a compiled default set"); - assert!(cursor.supports_artifact_only_backup_restore(&cursor_set.component_set)); + assert_eq!( + cursor.registration_mode(&cursor_set.component_set), + CatalogRegistrationMode::ArtifactOnly + ); } /// The live reinstall journey: TraceDecay's own staging residue (a @@ -1926,7 +1162,6 @@ mod tests { use crate::agents::host_bundle::HostComponentSetRegistrationV1; let home = tempfile::tempdir().unwrap(); - let lifecycle_root = tempfile::tempdir().unwrap(); let marketplace = home.path().join(".agents/plugins/marketplace.json"); std::fs::create_dir_all(marketplace.parent().unwrap()).unwrap(); std::fs::write( @@ -1952,7 +1187,6 @@ mod tests { let mut authority = CatalogHostComponentRegistrationAuthority::new( "codex", home.path(), - lifecycle_root.path(), crate::agents::host_bundle::HostBundleLifecycleOpV1::Install, ) .expect("catalog registration authority"); @@ -1980,122 +1214,4 @@ mod tests { "the converging install must re-activate the host registration" ); } - - #[test] - fn rollback_identity_rejects_other_home_profile_and_integration() { - let home = tempfile::tempdir().unwrap(); - let profile = tempfile::tempdir().unwrap(); - let other = tempfile::tempdir().unwrap(); - let identity = - RegistrationBackupIdentityV1::new("codex", home.path(), profile.path()).unwrap(); - - assert_eq!( - identity.validate("codex", home.path(), profile.path()), - Ok(()) - ); - for result in [ - identity.validate("codex", other.path(), profile.path()), - identity.validate("codex", home.path(), other.path()), - identity.validate("cursor", home.path(), profile.path()), - ] { - assert_eq!(result, Err(HostBundleError::WrongTarget)); - } - let mut future_identity = identity; - future_identity.schema_version = REGISTRATION_BACKUP_IDENTITY_SCHEMA_VERSION + 1; - assert_eq!( - future_identity.validate("codex", home.path(), profile.path()), - Err(HostBundleError::UnsupportedRecoveryFormat) - ); - } - - #[test] - fn rollback_accepts_absent_registration_paths_added_by_a_new_binary() { - let home = tempfile::tempdir().unwrap(); - let lifecycle_root = tempfile::tempdir().unwrap(); - let component_set = - crate::agents::host_bundle_registry::verified_embedded_default_host_component_set( - HostKindV1::Codex, - 0, - crate::agents::TEST_GENERATOR_COMMIT, - ) - .expect("Codex has a compiled default set"); - let authority = CatalogHostComponentRegistrationAuthority::new( - "codex", - home.path(), - lifecycle_root.path(), - crate::agents::host_bundle::HostBundleLifecycleOpV1::Update, - ) - .expect("catalog registration authority"); - let operation_id = [19; 16]; - let current_paths = authority - .registration_paths(&component_set.component_set) - .unwrap(); - let current_cache = home - .path() - .join(".codex/plugins/cache/personal/tracedecay") - .join(crate::PRODUCT_VERSION); - let persisted_paths = current_paths - .iter() - .filter(|path| !path.starts_with(¤t_cache)) - .cloned() - .collect::>(); - assert!( - persisted_paths.len() < current_paths.len(), - "the fixture must omit the newer binary's version-scoped cache paths" - ); - - let backup_dir = authority.backup_dir(operation_id); - std::fs::create_dir_all(&backup_dir).unwrap(); - let identity = RegistrationBackupIdentityV1::new( - authority.integration.id(), - home.path(), - lifecycle_root.path(), - ) - .unwrap(); - write_registration_backup( - &authority.identity_path(operation_id), - &serde_json::to_vec(&identity).unwrap(), - ) - .unwrap(); - let mutation_plan = RegistrationMutationPlanV1 { - schema_version: REGISTRATION_BACKUP_IDENTITY_SCHEMA_VERSION, - integration_id: authority.integration.id().to_string(), - operation: crate::agents::host_bundle::HostBundleLifecycleOpV1::Update, - paths: persisted_paths.clone(), - directories: Vec::new(), - }; - write_registration_backup( - &authority.mutation_plan_path(operation_id), - &serde_json::to_vec(&mutation_plan).unwrap(), - ) - .unwrap(); - for (index, path) in persisted_paths.iter().enumerate() { - write_registration_backup( - &authority.registration_path_marker(operation_id, index), - &serde_json::to_vec(path).unwrap(), - ) - .unwrap(); - write_registration_backup( - &authority.missing_marker_path(operation_id, index), - b"missing", - ) - .unwrap(); - } - - authority - .restore_registration(&component_set.component_set, operation_id) - .expect("absent paths introduced by a newer binary are not rollback targets"); - - let live_unjournaled = current_paths - .iter() - .find(|path| path.starts_with(¤t_cache)) - .expect("fixture has a current-version cache path"); - std::fs::create_dir_all(live_unjournaled.parent().unwrap()).unwrap(); - std::fs::write(live_unjournaled, b"not covered by the older journal").unwrap(); - assert_eq!( - authority.restore_registration(&component_set.component_set, operation_id), - Err(HostBundleError::WrongTarget), - "a live path outside the persisted inventory remains fail-closed" - ); - } } diff --git a/crates/tracedecay-agent-hosts/src/agents/host_config_io.rs b/crates/tracedecay-agent-hosts/src/agents/host_config_io.rs index 3a01962b68..d590c1a651 100644 --- a/crates/tracedecay-agent-hosts/src/agents/host_config_io.rs +++ b/crates/tracedecay-agent-hosts/src/agents/host_config_io.rs @@ -1,9 +1,8 @@ //! Host configuration file IO shared by every agent integration: lenient and -//! strict JSON/JSONC/TOML loaders, backup-then-atomic-replace writers with +//! strict JSON/JSONC/TOML loaders, atomic-replace writers with //! durable write intents, host file metadata capture, and the binary and //! host-directory probes installers embed into generated config. -use std::borrow::Cow; use std::cell::RefCell; use std::path::{Path, PathBuf}; @@ -12,8 +11,9 @@ use sha2::{Digest, Sha256}; use tracedecay_domain::canonical_text::sha256_hex; use tracedecay_domain::errors::{Result, TraceDecayError}; -use super::text_file_transaction::{self, TextFileMutation, update_config_file_transactionally}; +use super::text_file_transaction::{self, TextFileMutation, update_text_file_transactionally}; +mod json_edit; #[cfg(test)] mod tests; @@ -39,6 +39,10 @@ pub enum JsonConfigDialect { } impl JsonConfigDialect { + fn parse_options(self) -> jsonc_parser::ParseOptions { + json_edit::parse_options(self == Self::Jsonc) + } + /// Strict parse of already-observed config contents for a write path. /// Missing or blank content is a fresh `{}`; anything unparseable is a /// typed error so a transform never runs against fabricated state. @@ -46,19 +50,44 @@ impl JsonConfigDialect { if contents.trim().is_empty() { return Ok(serde_json::json!({})); } - let (dialect_label, parseable) = match self { - Self::Json => ("JSON", Cow::Borrowed(contents)), - Self::Jsonc => ("JSONC", Cow::Owned(strip_jsonc_comments(contents))), + let parsed = match self { + Self::Json => serde_json::from_str(contents).map_err(|e| e.to_string()), + Self::Jsonc => json_edit::parse_json_text(contents, &self.parse_options()), }; - serde_json::from_str(&parseable).map_err(|e| TraceDecayError::Config { + parsed.map_err(|e| TraceDecayError::Config { message: format!( - "cannot parse {} as {dialect_label}: {e}\n \ + "cannot parse {} as {}: {e}\n \ Hint: fix the JSON syntax manually and re-run the command,\n \ or delete the file to start fresh", - path.display() + path.display(), + match self { + Self::Json => "JSON", + Self::Jsonc => "JSONC", + } ), }) } + + /// Replacement text for a config whose observed contents are `existing` + /// and whose intended value is `value`. Only members whose values differ + /// are rewritten; the operator's comments, key order and formatting + /// around them are kept byte-for-byte, so removing what an install added + /// restores the original file exactly. Blank contents are a fresh file. + pub(super) fn render_edit( + self, + path: &Path, + existing: &str, + value: &serde_json::Value, + ) -> Result { + if existing.trim().is_empty() { + return render_json_config(path, value); + } + json_edit::edit_json_text(existing, value, &self.parse_options()).map_err(|e| { + TraceDecayError::Config { + message: format!("cannot update {}: {e}", path.display()), + } + }) + } } /// Load a JSON file for **editing**. Unlike [`load_json_file`], this returns @@ -82,99 +111,8 @@ pub fn load_json_file_strict(path: &Path) -> Result { JsonConfigDialect::Json.parse_for_edit(path, &contents) } -pub fn config_backup_path(path: &Path) -> PathBuf { - PathBuf::from(format!("{}.bak", path.display())) -} - -/// Create a backup copy of a config file before modifying it. -/// -/// The backup itself is written atomically: content is first written to a -/// staging file (`.bak.new`), then renamed to `.bak`. This ensures the -/// `.bak` file is never half-written even if the process is killed. -/// -/// Returns `Ok(Some(backup_path))` when a backup was created, or `Ok(None)` -/// when the file did not exist (nothing to back up). -/// -/// # Error conditions -/// - File exists but cannot be read (permissions, I/O error). -/// - Staging file cannot be written (disk full, permissions). -/// - Staging file cannot be renamed to `.bak` (cross-device, permissions). -#[hotpath::measure(label = "agent_hosts.agents.host_config.backup")] -pub fn backup_config_file(path: &Path) -> Result> { - if !path.exists() { - return Ok(None); - } - let backup_path = config_backup_path(path); - let staging_path = PathBuf::from(format!("{}.bak.new", path.display())); - - let content = std::fs::read(path).map_err(|e| TraceDecayError::Config { - message: format!( - "failed to read {} for backup: {e}\n \ - Hint: check file permissions", - path.display() - ), - })?; - std::fs::write(&staging_path, &content).map_err(|e| { - std::fs::remove_file(&staging_path).ok(); - TraceDecayError::Config { - message: format!( - "failed to write backup staging file {}: {e}\n \ - Hint: check available disk space and permissions", - staging_path.display() - ), - } - })?; - // The backup holds the same secrets as the original (host configs can - // carry credential env values), so it must not be published with the - // umask-default mode: copy the original's permission identity onto the - // staging file before it becomes `.bak`. - let original_metadata = - capture_host_file_metadata(path).map_err(|error| TraceDecayError::Config { - message: format!( - "failed to capture metadata for {} before backup: {error}", - path.display() - ), - })?; - restore_host_file_metadata(&staging_path, &original_metadata).map_err(|error| { - std::fs::remove_file(&staging_path).ok(); - TraceDecayError::Config { - message: format!( - "failed to apply original permissions to backup staging file {}: {error}", - staging_path.display() - ), - } - })?; - let backup_metadata = - capture_host_file_metadata(&staging_path).map_err(|error| TraceDecayError::Config { - message: format!( - "failed to inspect backup staging file {}: {error}", - staging_path.display() - ), - })?; - persist_host_config_write_intent(&backup_path, &content, Some(&backup_metadata))?; - - // Atomic rename staging → .bak - std::fs::rename(&staging_path, &backup_path).map_err(|e| { - std::fs::remove_file(&staging_path).ok(); - TraceDecayError::Config { - message: format!( - "failed to create backup {}: {e}\n \ - Hint: check file permissions", - backup_path.display() - ), - } - })?; - - Ok(Some(backup_path)) -} - /// Write a JSON value to a file via atomic rename. /// -/// The caller is responsible for creating the backup via -/// [`backup_config_file`] before loading the config. Pass the backup path -/// here so that it can be mentioned in error messages and used for restore -/// if the rename somehow leaves the target in a bad state. -/// /// # Strategy /// /// Serialize and re-validate, then use the shared durable atomic writer to @@ -187,18 +125,14 @@ pub fn backup_config_file(path: &Path) -> Result> { /// - Atomic staging or publication failure (permissions, disk full). /// /// In every error case the original file remains intact. -pub fn safe_write_json_file( - path: &Path, - value: &serde_json::Value, - backup: Option<&Path>, -) -> Result<()> { +pub fn safe_write_json_file(path: &Path, value: &serde_json::Value) -> Result<()> { let content = render_json_config(path, value)?; - safe_write_bytes_file(path, content.as_bytes(), backup) + safe_write_bytes_file(path, content.as_bytes()) } -/// Serialize a JSON config value for publication: pretty-printed, re-parse +/// Serialize a JSON config value for a fresh file: pretty-printed, re-parse /// validated, trailing newline. -pub(super) fn render_json_config(path: &Path, value: &serde_json::Value) -> Result { +fn render_json_config(path: &Path, value: &serde_json::Value) -> Result { let pretty = serde_json::to_string_pretty(value).map_err(|e| TraceDecayError::Config { message: format!("failed to serialize JSON for {}: {e}", path.display()), })?; @@ -228,20 +162,19 @@ pub(crate) enum JsonConfigMutation { /// config. The transform sees the value parsed strictly from the exact bytes /// the write lock observed, so a concurrent writer can no longer slip between /// load and publish, and a corrupt config is a typed error instead of a -/// silently-empty object. Rewrites and removals of an existing file leave a -/// `.bak` (issue #63). +/// silently-empty object. pub(crate) fn update_json_config_transactionally( path: &Path, dialect: JsonConfigDialect, update: impl FnOnce(serde_json::Value) -> Result<(T, JsonConfigMutation)>, ) -> Result { - update_config_file_transactionally(path, |existing| { + update_text_file_transactionally(path, |existing| { let settings = dialect.parse_for_edit(path, existing)?; let (output, mutation) = update(settings)?; let mutation = match mutation { JsonConfigMutation::Unchanged => TextFileMutation::Unchanged, JsonConfigMutation::Write(value) => { - TextFileMutation::Write(render_json_config(path, &value)?) + TextFileMutation::Write(dialect.render_edit(path, existing, &value)?) } JsonConfigMutation::Remove => TextFileMutation::Remove, }; @@ -250,16 +183,42 @@ pub(crate) fn update_json_config_transactionally( } /// TOML sibling of [`update_json_config_transactionally`]. The transform -/// returns the serialized replacement text itself because TOML publication -/// may need post-serialization shaping (Codex's explicit `[hooks.state]` -/// parent table). +/// edits the document parsed from the exact bytes the write lock observed; +/// every table, key, comment and blank line it leaves alone publishes +/// byte-for-byte, so removing what an install added restores the original +/// file. A document edited down to nothing removes the file. pub(crate) fn update_toml_config_transactionally( path: &Path, - update: impl FnOnce(toml::Value) -> Result<(T, TextFileMutation)>, + update: impl FnOnce(&mut toml_edit::DocumentMut) -> Result, ) -> Result { - update_config_file_transactionally(path, |existing| { - let value = parse_toml_config(path, existing)?; - update(value) + update_text_file_transactionally(path, |existing| { + let mut document = + existing + .parse::() + .map_err(|e| TraceDecayError::Config { + message: format!( + "failed to parse {} as TOML: {e}. Refusing to overwrite, fix the file or remove it manually.", + path.display() + ), + })?; + let output = update(&mut document)?; + let mut rendered = document.to_string(); + // Rendering terminates the last line; keep a file that had no final + // newline without one. + if !existing.is_empty() && !existing.ends_with('\n') && rendered.ends_with('\n') { + rendered.pop(); + if rendered.ends_with('\r') { + rendered.pop(); + } + } + let mutation = if rendered == existing { + TextFileMutation::Unchanged + } else if rendered.trim().is_empty() { + TextFileMutation::Remove + } else { + TextFileMutation::Write(rendered) + }; + Ok((output, mutation)) }) } @@ -268,8 +227,8 @@ pub(crate) fn update_toml_config_transactionally( /// Mirrors [`safe_write_json_file`] for generated prompt/rule files that are /// plain text rather than structured JSON. The target is not opened for writing /// until the final rename, so a failed write leaves the original untouched. -pub fn safe_write_text_file(path: &Path, contents: &str, backup: Option<&Path>) -> Result<()> { - safe_write_bytes_file(path, contents.as_bytes(), backup) +pub fn safe_write_text_file(path: &Path, contents: &str) -> Result<()> { + safe_write_bytes_file(path, contents.as_bytes()) } #[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)] @@ -371,18 +330,17 @@ pub fn restore_host_file_metadata( /// that metadata before returning. This authority is shared by every host: /// existing config symlinks are always refused so no integration can redirect /// a lifecycle write outside its inventoried path. -pub fn safe_write_bytes_file(path: &Path, contents: &[u8], backup: Option<&Path>) -> Result<()> { - safe_write_bytes_file_with_metadata(path, contents, backup, None) +pub fn safe_write_bytes_file(path: &Path, contents: &[u8]) -> Result<()> { + safe_write_bytes_file_with_metadata(path, contents, None) } #[hotpath::measure(label = "agent_hosts.agents.host_config.write")] pub fn safe_write_bytes_file_with_metadata( path: &Path, contents: &[u8], - backup: Option<&Path>, replacement_metadata: Option<&HostFileMetadataIdentityV1>, ) -> Result<()> { - text_file_transaction::write_bytes_file_locked(path, contents, backup, replacement_metadata) + text_file_transaction::write_bytes_file_locked(path, contents, replacement_metadata) } #[cfg(test)] @@ -526,11 +484,7 @@ pub(super) fn test_pause_host_config_write(path: &Path, boundary: TestHostConfig /// exercised against a real torn install. #[cfg(feature = "test-transport")] pub(super) fn test_abort_after_host_config_write(path: &Path) { - if (std::env::var_os("TRACEDECAY_TEST_ABORT_AFTER_HOST_CONFIG_WRITE").is_some() - && !path - .file_name() - .and_then(std::ffi::OsStr::to_str) - .is_some_and(|name| name.ends_with(".bak") || name.ends_with(".tracedecay-original"))) + if std::env::var_os("TRACEDECAY_TEST_ABORT_AFTER_HOST_CONFIG_WRITE").is_some() || std::env::var_os("TRACEDECAY_TEST_ABORT_AFTER_HOST_CONFIG_WRITE_PATH") .is_some_and(|expected| Path::new(&expected) == path) { @@ -829,77 +783,6 @@ fn path_component_eq(actual: &std::ffi::OsStr, expected: impl AsRef Result<()> { - const RETIRED_SUFFIXES: &[&str] = &["pre-v2-adopt"]; - - let Some(parent) = current_dir.parent() else { - return Ok(()); - }; - let Some(current_name) = current_dir.file_name().and_then(|name| name.to_str()) else { - return Ok(()); - }; - let prefix = format!("{current_name}."); - let entries = match std::fs::read_dir(parent) { - Ok(entries) => entries, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), - Err(error) => { - return Err(TraceDecayError::Config { - message: format!( - "failed to inspect plugin siblings in {}: {error}", - parent.display() - ), - }); - } - }; - - for entry in entries { - let entry = entry.map_err(|error| TraceDecayError::Config { - message: format!( - "failed to inspect a plugin sibling in {}: {error}", - parent.display() - ), - })?; - let file_type = entry.file_type().map_err(|error| TraceDecayError::Config { - message: format!("failed to inspect {}: {error}", entry.path().display()), - })?; - let name = entry.file_name(); - let name = name.to_string_lossy(); - let retired = name - .strip_prefix(&prefix) - .is_some_and(|suffix| RETIRED_SUFFIXES.contains(&suffix)); - if !file_type.is_dir() || !retired { - continue; - } - let sibling = entry.path(); - let owned = ownership_manifests.iter().any(|relative| { - load_json_file(&sibling.join(relative)) - .get("name") - .and_then(serde_json::Value::as_str) - == Some("tracedecay") - }); - if !owned { - continue; - } - std::fs::remove_dir_all(&sibling).map_err(|error| TraceDecayError::Config { - message: format!( - "failed to remove superseded tracedecay plugin {}: {error}", - sibling.display() - ), - })?; - } - Ok(()) -} - /// Recursively collect every regular file under `root` (following the same /// hand-rolled walk both the Cursor and Codex installers rely on). #[hotpath::measure(label = "agent_hosts.agents.fs.collect_regular_files")] @@ -1061,104 +944,11 @@ pub fn home_dir() -> Option { .map(PathBuf::from) } -/// Strip `//` line comments, `/* */` block comments, and trailing commas -/// before `}` / `]` from a JSONC string, then parse with `serde_json`. -/// Falls back to `serde_json::json!({})` on any parse failure. +/// Parse a JSONC string (comments and trailing commas allowed), falling back +/// to `serde_json::json!({})` on any parse failure. Read-only paths only. pub fn parse_jsonc(input: &str) -> serde_json::Value { - let stripped = strip_jsonc_comments(input); - serde_json::from_str(&stripped).unwrap_or_else(|_| serde_json::json!({})) -} - -/// Internal helper: removes JSONC comments and trailing commas. -pub(super) fn strip_jsonc_comments(input: &str) -> String { - let mut out = String::with_capacity(input.len()); - let chars: Vec = input.chars().collect(); - let len = chars.len(); - let mut i = 0; - let mut in_string = false; - - while i < len { - // Handle string literals (skip comment stripping inside strings). - if in_string { - if chars[i] == '\\' && i + 1 < len { - out.push(chars[i]); - out.push(chars[i + 1]); - i += 2; - continue; - } - if chars[i] == '"' { - in_string = false; - } - out.push(chars[i]); - i += 1; - continue; - } - - // Start of string. - if chars[i] == '"' { - in_string = true; - out.push(chars[i]); - i += 1; - continue; - } - - // Line comment `//`. - if chars[i] == '/' && i + 1 < len && chars[i + 1] == '/' { - // Skip until newline. - while i < len && chars[i] != '\n' { - i += 1; - } - continue; - } - - // Block comment `/* ... */`. - if chars[i] == '/' && i + 1 < len && chars[i + 1] == '*' { - i += 2; - while i + 1 < len && !(chars[i] == '*' && chars[i + 1] == '/') { - i += 1; - } - i += 2; // consume `*/` - continue; - } - - out.push(chars[i]); - i += 1; - } - - // Remove trailing commas before `}` or `]`. - // Simple regex-free approach: repeatedly collapse ", }" patterns. - remove_trailing_commas(&out) -} - -/// Removes trailing commas that appear immediately before `}` or `]` (with -/// optional whitespace/newlines in between). -fn remove_trailing_commas(input: &str) -> String { - // We scan for comma, optional whitespace, then `}` or `]`. - let bytes = input.as_bytes(); - let len = bytes.len(); - let mut out = Vec::with_capacity(len); - let mut i = 0; - - while i < len { - if bytes[i] == b',' { - // Peek ahead past whitespace. - let mut j = i + 1; - while j < len - && (bytes[j] == b' ' || bytes[j] == b'\t' || bytes[j] == b'\n' || bytes[j] == b'\r') - { - j += 1; - } - if j < len && (bytes[j] == b'}' || bytes[j] == b']') { - // Skip the comma; whitespace will be included normally. - i += 1; - continue; - } - } - out.push(bytes[i]); - i += 1; - } - - String::from_utf8(out).unwrap_or_else(|_| input.to_string()) + json_edit::parse_json_text(input, &JsonConfigDialect::Jsonc.parse_options()) + .unwrap_or_else(|_| serde_json::json!({})) } /// Read a file and parse it as JSONC. Falls back to `json!({})` if the file diff --git a/crates/tracedecay-agent-hosts/src/agents/host_config_io/json_edit.rs b/crates/tracedecay-agent-hosts/src/agents/host_config_io/json_edit.rs new file mode 100644 index 0000000000..e6fe042076 --- /dev/null +++ b/crates/tracedecay-agent-hosts/src/agents/host_config_io/json_edit.rs @@ -0,0 +1,391 @@ +//! Format-preserving edits of JSON and JSONC host configs. +//! +//! Installers compute the config value they want; [`edit_json_text`] moves +//! the operator's text to that value by splicing only the members that +//! differ. Comments, key order, indentation, trailing commas and line endings +//! outside those members stay byte-for-byte. Every splice has an exact +//! inverse: removing a member deletes precisely the text appending it added, +//! so install followed by uninstall restores the original bytes without any +//! copy of them being kept. + +use jsonc_parser::ast::{Object, Value as Node}; +use jsonc_parser::common::{Range, Ranged}; +use jsonc_parser::tokens::{Token, TokenAndRange}; +use jsonc_parser::{CollectOptions, CommentCollectionStrategy, ParseOptions, parse_to_ast}; +use serde_json::Value; + +/// Parse options that accept exactly the host config dialect: standard JSON, +/// optionally with comments and trailing commas. +pub(super) fn parse_options(comments_and_trailing_commas: bool) -> ParseOptions { + ParseOptions { + allow_comments: comments_and_trailing_commas, + allow_trailing_commas: comments_and_trailing_commas, + allow_loose_object_property_names: false, + allow_missing_commas: false, + allow_single_quoted_strings: false, + allow_hexadecimal_numbers: false, + allow_unary_plus_numbers: false, + } +} + +pub(super) fn parse_json_text(text: &str, options: &ParseOptions) -> Result { + parse_to_ast(text, &CollectOptions::default(), options) + .map_err(|error| error.to_string())? + .value + .map(Value::from) + .ok_or_else(|| "no JSON value".to_string()) +} + +/// Rewrite `text` so it parses to `target`, touching only differing members. +pub(super) fn edit_json_text( + text: &str, + target: &Value, + options: &ParseOptions, +) -> Result { + let mut text = text.to_string(); + let mut remaining_edits = node_count(&parse_json_text(&text, options)?) + node_count(target); + loop { + let splices = { + let parsed = parse_to_ast( + &text, + &CollectOptions { + comments: CommentCollectionStrategy::AsTokens, + tokens: true, + }, + options, + ) + .map_err(|error| error.to_string())?; + let root = parsed.value.ok_or_else(|| "no JSON value".to_string())?; + let tokens = parsed + .tokens + .ok_or_else(|| "parser returned no tokens".to_string())?; + let doc = Doc::new(&text, &tokens); + match doc.diff(&root, target) { + Some(splices) => splices, + None if Value::from(root) == *target => return Ok(text), + None => return Err("edited config does not match the intended value".into()), + } + }; + if remaining_edits == 0 { + return Err("config edit did not converge".into()); + } + remaining_edits -= 1; + text = apply(text, splices); + } +} + +fn node_count(value: &Value) -> usize { + 1 + match value { + Value::Array(items) => items.iter().map(node_count).sum(), + Value::Object(members) => members.values().map(node_count).sum(), + _ => 0, + } +} + +/// Replace `text[start..end]` with the string; non-overlapping by construction. +type Splice = (usize, usize, String); + +fn apply(mut text: String, mut splices: Vec) -> String { + splices.sort_by_key(|(start, _, _)| std::cmp::Reverse(*start)); + for (start, end, replacement) in splices { + text.replace_range(start..end, &replacement); + } + text +} + +/// One comma-separated container: an object's properties or an array's +/// elements, by source range. +struct Container { + open: usize, + close: usize, + items: Vec, +} + +struct Doc<'a> { + text: &'a str, + tokens: &'a [TokenAndRange<'a>], + newline: &'static str, + unit: String, +} + +impl<'a> Doc<'a> { + fn new(text: &'a str, tokens: &'a [TokenAndRange<'a>]) -> Self { + let newline = if text.contains("\r\n") { "\r\n" } else { "\n" }; + let unit = text + .lines() + .map(|line| &line[..line.len() - line.trim_start().len()]) + .find(|indent| !indent.is_empty()) + .unwrap_or(" ") + .to_string(); + Self { + text, + tokens, + newline, + unit, + } + } + + /// The first splice set that moves `node` toward `target`, or `None` + /// when they already agree. + fn diff(&self, node: &Node, target: &Value) -> Option> { + if Value::from(node.clone()) == *target { + return None; + } + match (node, target) { + (Node::Object(object), Value::Object(members)) => self.diff_object(object, members), + (Node::Array(array), Value::Array(items)) => { + let container = Container { + open: array.range.start, + close: array.range.end - 1, + items: array.elements.iter().map(Ranged::range).collect(), + }; + let current: Vec = array.elements.iter().cloned().map(Value::from).collect(); + match first_array_change(¤t, items)? { + ArrayChange::Edit(old, new) => self.diff(&array.elements[old], &items[new]), + ArrayChange::Remove(old) => Some(self.remove_item(&container, old)), + ArrayChange::Insert(at, new) => { + Some(self.insert_item(&container, at, None, &items[new])) + } + } + } + _ => { + let range = node.range(); + let indent = self.line_indent(range.start); + let multiline = self.text.contains('\n'); + Some(vec![( + range.start, + range.end, + self.render(target, &indent, multiline), + )]) + } + } + } + + fn diff_object( + &self, + object: &Object, + members: &serde_json::Map, + ) -> Option> { + let container = Container { + open: object.range.start, + close: object.range.end - 1, + items: object.properties.iter().map(|prop| prop.range).collect(), + }; + if let Some(index) = object + .properties + .iter() + .position(|prop| !members.contains_key(prop.name.as_str())) + { + return Some(self.remove_item(&container, index)); + } + if let Some(splices) = object + .properties + .iter() + .find_map(|prop| self.diff(&prop.value, &members[prop.name.as_str()])) + { + return Some(splices); + } + let (key, value) = members.iter().find(|(key, _)| { + !object + .properties + .iter() + .any(|prop| prop.name.as_str() == key.as_str()) + })?; + let colon = object + .properties + .first() + .map(|prop| &self.text[prop.name.end()..prop.value.start()]) + .filter(|colon| colon.trim() == ":") + .unwrap_or(": "); + Some(self.insert_item(&container, container.items.len(), Some((key, colon)), value)) + } + + /// End of the last token (comments included) that ends at or before `pos`. + fn prev_token_end(&self, pos: usize) -> usize { + self.tokens + .iter() + .rev() + .find(|token| token.range.end <= pos) + .map_or(0, |token| token.range.end) + } + + /// Position of the comma that directly follows the item ending at `end`. + fn comma_after(&self, end: usize) -> Option { + self.tokens + .iter() + .filter(|token| token.range.start >= end) + .find(|token| !matches!(token.token, Token::CommentLine(_) | Token::CommentBlock(_))) + .filter(|token| token.token == Token::Comma) + .map(|token| token.range.start) + } + + /// Leading whitespace of the line containing `pos`. + fn line_indent(&self, pos: usize) -> String { + let line_start = self.text[..pos].rfind('\n').map_or(0, |index| index + 1); + let line = &self.text[line_start..]; + line[..line.len() - line.trim_start_matches([' ', '\t']).len()].to_string() + } + + /// Indentation for items of `container`: the existing items' own line + /// indentation when they start their lines, else one unit deeper than the + /// line that opens the container. + fn item_indent(&self, container: &Container, reference: Option) -> String { + reference + .map(|item| (self.line_indent(item.start), item.start)) + .filter(|(indent, start)| { + let line_start = self.text[..*start].rfind('\n').map_or(0, |index| index + 1); + line_start + indent.len() == *start + }) + .map_or_else( + || format!("{}{}", self.line_indent(container.open), self.unit), + |(indent, _)| indent, + ) + } + + fn remove_item(&self, container: &Container, index: usize) -> Vec { + let item = container.items[index]; + let start = self.prev_token_end(item.start); + if container.items.len() == 1 { + let end = self + .comma_after(item.end) + .map_or(item.end, |comma| comma + 1); + let untouched_is_blank = self.text[container.open + 1..start].trim().is_empty() + && self.text[end..container.close].trim().is_empty(); + return if untouched_is_blank { + vec![(container.open + 1, container.close, String::new())] + } else { + vec![(start, end, String::new())] + }; + } + match self.comma_after(item.end) { + Some(comma) => vec![(start, comma + 1, String::new())], + None => { + let previous = container.items[index - 1]; + let mut splices = vec![(start, item.end, String::new())]; + if let Some(comma) = self.comma_after(previous.end) { + splices.push((comma, comma + 1, String::new())); + } + splices + } + } + } + + fn insert_item( + &self, + container: &Container, + at: usize, + key: Option<(&str, &str)>, + value: &Value, + ) -> Vec { + let multiline = self.text[container.open..container.close].contains('\n'); + let render = |indent: &str, multiline: bool| { + let value = self.render(value, indent, multiline); + match key { + Some((key, colon)) => format!("{}{colon}{value}", Value::from(key)), + None => value, + } + }; + let items = &container.items; + if items.is_empty() { + if key.is_none() && !value.is_object() && !value.is_array() { + return vec![(container.open + 1, container.open + 1, render("", false))]; + } + let outer = self.line_indent(container.open); + let indent = format!("{outer}{}", self.unit); + let inner = &self.text[container.open + 1..container.close]; + let closing = if inner.contains('\n') { + String::new() + } else { + format!("{}{outer}", self.newline) + }; + let body = format!("{}{indent}{}{closing}", self.newline, render(&indent, true)); + return vec![(container.open + 1, container.open + 1, body)]; + } + let reference = items[at.min(items.len() - 1)]; + let indent = self.item_indent(container, Some(reference)); + let separator = if multiline { + format!("{}{indent}", self.newline) + } else { + let gap = match items.get(1) { + Some(second) => self + .comma_after(items[0].end) + .map_or("", |comma| &self.text[comma + 1..second.start]), + None => &self.text[container.open + 1..items[0].start], + }; + if gap.trim().is_empty() { gap } else { " " }.to_string() + }; + let body = render(&indent, multiline); + if at < items.len() { + let start = self.prev_token_end(items[at].start); + return vec![(start, start, format!("{separator}{body},"))]; + } + let last = items[items.len() - 1]; + let end = self.prev_token_end(container.close); + match self.comma_after(last.end) { + Some(_) => vec![(end, end, format!("{separator}{body},"))], + None if end == last.end => vec![(end, end, format!(",{separator}{body}"))], + None => vec![ + (last.end, last.end, ",".to_string()), + (end, end, format!("{separator}{body}")), + ], + } + } + + /// Serialize `value` for insertion at a line indented by `indent`. + fn render(&self, value: &Value, indent: &str, multiline: bool) -> String { + let nested = value.as_object().is_some_and(|members| !members.is_empty()) + || value.as_array().is_some_and(|items| !items.is_empty()); + if !multiline || !nested { + return value.to_string(); + } + // `{:#}` indents by two spaces per level and escapes every newline + // inside strings, so each line's leading spaces are pure nesting. + format!("{value:#}") + .lines() + .map(|line| { + let depth = (line.len() - line.trim_start_matches(' ').len()) / 2; + format!( + "{}{}", + self.unit.repeat(depth), + line.trim_start_matches(' ') + ) + }) + .collect::>() + .join(&format!("{}{indent}", self.newline)) + } +} + +enum ArrayChange { + Edit(usize, usize), + Remove(usize), + Insert(usize, usize), +} + +/// First change of a longest-common-subsequence alignment of `old` to `new`: +/// unmatched elements are removed or inserted, and an unmatched pair in the +/// same gap is edited in place so its formatting survives. +fn first_array_change(old: &[Value], new: &[Value]) -> Option { + let mut lcs = vec![vec![0usize; new.len() + 1]; old.len() + 1]; + for i in (0..old.len()).rev() { + for j in (0..new.len()).rev() { + lcs[i][j] = if old[i] == new[j] { + lcs[i + 1][j + 1] + 1 + } else { + lcs[i + 1][j].max(lcs[i][j + 1]) + }; + } + } + let (mut i, mut j) = (0, 0); + while i < old.len() && j < new.len() && old[i] == new[j] && lcs[i][j] == lcs[i + 1][j + 1] + 1 { + i += 1; + j += 1; + } + let old_unmatched = i < old.len() && lcs[i][j] == lcs[i + 1][j]; + let new_unmatched = j < new.len() && lcs[i][j] == lcs[i][j + 1]; + match (old_unmatched, new_unmatched) { + (true, true) => Some(ArrayChange::Edit(i, j)), + (true, false) => Some(ArrayChange::Remove(i)), + (false, true) => Some(ArrayChange::Insert(i, j)), + (false, false) => None, + } +} diff --git a/crates/tracedecay-agent-hosts/src/agents/host_config_io/tests.rs b/crates/tracedecay-agent-hosts/src/agents/host_config_io/tests.rs index 976c996041..b5299d4a83 100644 --- a/crates/tracedecay-agent-hosts/src/agents/host_config_io/tests.rs +++ b/crates/tracedecay-agent-hosts/src/agents/host_config_io/tests.rs @@ -43,7 +43,7 @@ mod jsonc_tests { } // --------------------------------------------------------------------------- -// Regression tests for safe config backup / load / write +// Regression tests for safe config load / write // --------------------------------------------------------------------------- #[allow(clippy::unwrap_used, clippy::expect_used)] mod safe_config_tests { @@ -55,28 +55,6 @@ mod safe_config_tests { tempfile::tempdir().expect("failed to create temp dir") } - // ----- backup_config_file ----- - - #[test] - fn backup_returns_none_when_file_missing() { - let dir = tmpdir(); - let path = dir.path().join("nonexistent.json"); - let result = backup_config_file(&path).unwrap(); - assert!(result.is_none()); - } - - #[test] - fn backup_staging_file_is_cleaned_up() { - let dir = tmpdir(); - let path = dir.path().join("config.json"); - fs::write(&path, "{}").unwrap(); - - backup_config_file(&path).unwrap(); - - let staging = dir.path().join("config.json.bak.new"); - assert!(!staging.exists(), ".bak.new staging file should be removed"); - } - // ----- load_json_file_strict ----- #[test] @@ -114,7 +92,7 @@ mod safe_config_tests { fn safe_write_cleans_up_new_file_on_success() { let dir = tmpdir(); let path = dir.path().join("config.json"); - safe_write_json_file(&path, &serde_json::json!({}), None).unwrap(); + safe_write_json_file(&path, &serde_json::json!({})).unwrap(); let new_path = dir.path().join("config.json.new"); assert!(!new_path.exists(), ".new staging file should be removed"); @@ -122,8 +100,8 @@ mod safe_config_tests { #[test] fn full_install_cycle_preserves_existing_config() { - // Simulate the full install cycle: backup → strict load → mutate → safe write. - // Existing keys must be preserved. + // Simulate the full install cycle: strict load → mutate → safe write. + // Existing keys must be preserved and no copy of the prior bytes kept. let dir = tmpdir(); let path = dir.path().join("config.json"); let original = serde_json::json!({ @@ -136,13 +114,12 @@ mod safe_config_tests { fs::write(&path, serde_json::to_string_pretty(&original).unwrap()).unwrap(); // Simulate install - let backup = backup_config_file(&path).unwrap(); let mut config = load_json_file_strict(&path).unwrap(); config["mcp"]["tracedecay"] = serde_json::json!({ "type": "local", "command": ["tracedecay", "serve"] }); - safe_write_json_file(&path, &config, backup.as_deref()).unwrap(); + safe_write_json_file(&path, &config).unwrap(); // Verify let result: serde_json::Value = @@ -156,12 +133,11 @@ mod safe_config_tests { "http://localhost:8080" ); assert_eq!(result["other_setting"], serde_json::json!([1, 2, 3])); - - // Backup exists with original content - let bak_content: serde_json::Value = - serde_json::from_str(&fs::read_to_string(backup.unwrap()).unwrap()).unwrap(); - assert!(bak_content.get("tracedecay").is_none()); - assert_eq!(bak_content["theme"], "dark"); + let entries: Vec<_> = fs::read_dir(dir.path()) + .unwrap() + .map(|entry| entry.unwrap().file_name()) + .collect(); + assert_eq!(entries, vec![std::ffi::OsString::from("config.json")]); } #[test] @@ -173,21 +149,12 @@ mod safe_config_tests { let corrupt_content = "{ this is not valid json at all }}}"; fs::write(&path, corrupt_content).unwrap(); - // Backup succeeds (it just copies bytes) - let backup = backup_config_file(&path).unwrap(); - assert!(backup.is_some()); - // Strict load fails let err = load_json_file_strict(&path); assert!(err.is_err()); // Original file is byte-for-byte unchanged assert_eq!(fs::read_to_string(&path).unwrap(), corrupt_content); - // Backup also has the same content - assert_eq!( - fs::read_to_string(backup.unwrap()).unwrap(), - corrupt_content - ); } } @@ -369,7 +336,7 @@ mod local_install_safety_tests { let pause = pause_next_host_config_write_at_publication(path); let path = path.to_path_buf(); let writer = std::thread::spawn(move || { - safe_write_bytes_file(&path, contents, None).map_err(|error| error.to_string()) + safe_write_bytes_file(&path, contents).map_err(|error| error.to_string()) }); pause.wait_until_reached(); (pause, writer) @@ -487,7 +454,7 @@ mod local_install_safety_tests { std::fs::write(&outside, b"operator bytes").unwrap(); symlink(&outside, &config).unwrap(); - let error = safe_write_bytes_file(&config, b"tracedecay bytes", None).unwrap_err(); + let error = safe_write_bytes_file(&config, b"tracedecay bytes").unwrap_err(); assert!( error.to_string().contains("unsafe host metadata path"), "shared writer must surface its cross-host symlink refusal: {error}" @@ -504,8 +471,8 @@ mod local_install_safety_tests { let (finished_tx, finished_rx) = std::sync::mpsc::channel(); let second_path = config.clone(); let second = std::thread::spawn(move || { - let result = safe_write_bytes_file(&second_path, b"second", None) - .map_err(|error| error.to_string()); + let result = + safe_write_bytes_file(&second_path, b"second").map_err(|error| error.to_string()); finished_tx.send(()).unwrap(); result }); @@ -530,8 +497,7 @@ mod local_install_safety_tests { let pause = pause_next_host_config_write_at_publication(&config); let writer_path = config.clone(); let writer = std::thread::spawn(move || { - safe_write_bytes_file(&writer_path, b"tracedecay", None) - .map_err(|error| error.to_string()) + safe_write_bytes_file(&writer_path, b"tracedecay").map_err(|error| error.to_string()) }); pause.wait_until_reached(); @@ -556,8 +522,7 @@ mod local_install_safety_tests { let published = pause_next_host_config_write_after_publication(&config); let writer_path = config.clone(); let writer = std::thread::spawn(move || { - safe_write_bytes_file(&writer_path, b"tracedecay", None) - .map_err(|error| error.to_string()) + safe_write_bytes_file(&writer_path, b"tracedecay").map_err(|error| error.to_string()) }); publication.wait_until_reached(); @@ -642,3 +607,123 @@ mod local_install_safety_tests { assert_eq!(std::fs::read(&config).unwrap(), b"foreign create"); } } + +/// Install and uninstall edit only TraceDecay's member of an operator config, +/// so the operator's bytes survive both without any copy being kept. +#[allow(clippy::unwrap_used)] +mod format_preserving_edit_tests { + use super::*; + use crate::agents::{McpUninstallPolicy, install_mcp_server_entry, uninstall_mcp_server_entry}; + + const PRUNE: McpUninstallPolicy = McpUninstallPolicy { + prune_empty_root: true, + remove_empty_file: true, + }; + + fn entry(binary: &str) -> serde_json::Value { + serde_json::json!({"command": binary, "args": ["serve"]}) + } + + /// Install, reinstall with a moved binary, then uninstall `original`, + /// asserting the operator's bytes at every step. + fn assert_lifecycle_preserves_bytes( + file_name: &str, + dialect: JsonConfigDialect, + original: &str, + ) { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join(file_name); + std::fs::write(&path, original).unwrap(); + + install_mcp_server_entry( + &path, + "mcpServers", + entry("/opt/a/tracedecay"), + "test", + dialect, + ) + .unwrap(); + let installed = std::fs::read_to_string(&path).unwrap(); + let mut parsed = dialect.parse_for_edit(&path, &installed).unwrap(); + let servers = parsed["mcpServers"].as_object_mut().unwrap(); + assert_eq!( + servers.remove("tracedecay"), + Some(entry("/opt/a/tracedecay")) + ); + if servers.is_empty() { + parsed.as_object_mut().unwrap().remove("mcpServers"); + } + assert_eq!(parsed, dialect.parse_for_edit(&path, original).unwrap()); + + install_mcp_server_entry( + &path, + "mcpServers", + entry("/opt/b/tracedecay"), + "test", + dialect, + ) + .unwrap(); + assert_eq!( + std::fs::read_to_string(&path).unwrap(), + installed.replace("/opt/a/tracedecay", "/opt/b/tracedecay"), + "a reinstall rewrote more than the moved command" + ); + + uninstall_mcp_server_entry(&path, "mcpServers", dialect, PRUNE).unwrap(); + assert_eq!(std::fs::read_to_string(&path).unwrap(), original); + let entries: Vec<_> = std::fs::read_dir(dir.path()) + .unwrap() + .map(|entry| entry.unwrap().file_name()) + .collect(); + assert_eq!(entries, vec![std::ffi::OsString::from(file_name)]); + } + + #[test] + fn json_lifecycle_restores_indentation_key_order_and_line_endings() { + assert_lifecycle_preserves_bytes( + "mcp.json", + JsonConfigDialect::Json, + "{\r\n \"zeta\": true,\r\n \"mcpServers\": {\r\n \"foreign\": {\"command\": \"foreign-bin\", \"args\": []}\r\n },\r\n \"alpha\": [1, 2]\r\n}", + ); + assert_lifecycle_preserves_bytes( + "minified.json", + JsonConfigDialect::Json, + r#"{"zeta":1,"mcpServers":{"foreign":{"command":"foreign-bin"}}}"#, + ); + } + + #[test] + fn jsonc_lifecycle_restores_comments_and_trailing_commas() { + assert_lifecycle_preserves_bytes( + "settings.json", + JsonConfigDialect::Jsonc, + "// operator header\n{\n\t\"theme\": \"dark\", // same-line note\n\t/* servers */\n\t\"mcpServers\": {\n\t\t\"foreign\": {\"command\": \"foreign-bin\",},\n\t},\n\t\"alpha\": [1, 2,],\n}\n", + ); + // The root key TraceDecay creates is pruned again on uninstall. + assert_lifecycle_preserves_bytes( + "absent-root.json", + JsonConfigDialect::Jsonc, + "{\n // keep me\n \"theme\": \"dark\" // tail\n}\n", + ); + } + + #[test] + fn edit_refuses_a_config_it_cannot_parse() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("settings.json"); + let original = "{\n // a comment is not JSON\n \"a\": 1\n}\n"; + std::fs::write(&path, original).unwrap(); + + let error = install_mcp_server_entry( + &path, + "mcpServers", + entry("/opt/a/tracedecay"), + "test", + JsonConfigDialect::Json, + ) + .unwrap_err(); + + assert!(error.to_string().contains("cannot parse"), "{error}"); + assert_eq!(std::fs::read_to_string(&path).unwrap(), original); + } +} diff --git a/crates/tracedecay-agent-hosts/src/agents/kilo.rs b/crates/tracedecay-agent-hosts/src/agents/kilo.rs index 971c940f26..36d74cf103 100644 --- a/crates/tracedecay-agent-hosts/src/agents/kilo.rs +++ b/crates/tracedecay-agent-hosts/src/agents/kilo.rs @@ -15,8 +15,8 @@ use tracedecay_domain::errors::Result; use super::{ AgentIntegration, DoctorCounters, HealthcheckContext, InstallContext, JsonConfigDialect, - McpDoctorLabels, McpUninstallPolicy, config_backup_path, doctor_check_mcp_registration, - install_mcp_server_entry, load_jsonc_file, uninstall_mcp_server_entry, + McpDoctorLabels, McpUninstallPolicy, doctor_check_mcp_registration, install_mcp_server_entry, + load_jsonc_file, uninstall_mcp_server_entry, }; pub struct KiloIntegration; @@ -88,8 +88,7 @@ impl AgentIntegration for KiloIntegration { home: &Path, ) -> Vec { if components == [super::host_bundle::HostComponentV1::ContextMcp] { - let path = kilo_config_path(home); - vec![path.clone(), config_backup_path(&path)] + vec![kilo_config_path(home)] } else { Vec::new() } diff --git a/crates/tracedecay-agent-hosts/src/agents/kimi.rs b/crates/tracedecay-agent-hosts/src/agents/kimi.rs index 7795382e66..03ba8fc2f0 100644 --- a/crates/tracedecay-agent-hosts/src/agents/kimi.rs +++ b/crates/tracedecay-agent-hosts/src/agents/kimi.rs @@ -2,10 +2,12 @@ //! //! Kimi Code currently exposes plugin lifecycle only through its interactive //! `/plugins` host API. `TraceDecay` stages its first-party bundle under its -//! own profile, while registration in `plugins/installed.json` remains owned by Kimi's +//! own profile through the receipt-backed component transaction, while +//! registration in `plugins/installed.json` remains owned by Kimi's //! interactive host flow. Until Kimi ships a documented non-interactive -//! mutation API, global install/update/uninstall return an explicit -//! remediation instead of mutating the current registration. Project-local `--local` +//! mutation API, global install/update commit only that staged source and +//! return the `/plugins install` remediation, and uninstall refuses while the +//! registration stands. Project-local `--local` //! installs write //! `/.kimi-code/mcp.json` plus prompt rules in `/AGENTS.md`. //! Global installs register MCP in Kimi's user-level `mcp.json`; unlike plugin @@ -30,9 +32,9 @@ use tracedecay_domain::errors::{Result, TraceDecayError}; use super::{ AgentIntegration, DeferredUserAction, DoctorCounters, HealthcheckContext, InstallContext, - JsonConfigDialect, McpUninstallPolicy, NonInteractiveInstallOutcome, UpdatePluginOutcome, - host_home_override, install_mcp_server_entry, load_json_file, load_json_file_strict, - mcp_config_has_tracedecay, safe_write_text_file, uninstall_mcp_server_entry, + JsonConfigDialect, McpUninstallPolicy, NonInteractiveInstallOutcome, host_home_override, + install_mcp_server_entry, load_json_file, load_json_file_strict, mcp_config_has_tracedecay, + uninstall_mcp_server_entry, }; use super::prompt_rules::{PROMPT_RULE_MARKER, PromptRulesOptions}; @@ -73,27 +75,16 @@ impl AgentIntegration for KimiIntegration { )? { return Ok(NonInteractiveInstallOutcome::Ready); } + // The component transaction deploys the staged source; Kimi's own + // `/plugins install` must then copy it into the managed registry. Ok(NonInteractiveInstallOutcome::DeferredUserAction( - kimi_official_lifecycle_unavailable("install", None), + kimi_official_lifecycle_unavailable( + "install", + Some(&kimi_staged_plugin_dir(&ctx.home)), + ), )) } - fn prepare_non_interactive_install( - &self, - ctx: &InstallContext, - ) -> Result { - let deferred = stage_kimi_install_action(ctx)?; - if kimi_plugin_is_natively_active( - &ctx.home, - &kimi_code_home(&ctx.home), - &ctx.tracedecay_bin, - )? { - Ok(NonInteractiveInstallOutcome::Ready) - } else { - Ok(NonInteractiveInstallOutcome::DeferredUserAction(deferred)) - } - } - fn interactive_removal_guidance(&self) -> Option { Some(kimi_official_lifecycle_unavailable("remove", None).remediation) } @@ -149,7 +140,7 @@ impl AgentIntegration for KimiIntegration { fn deactivate_project_host_component_registration( &self, _components: &[super::host_bundle::HostComponentV1], - ctx: &InstallContext, + _ctx: &InstallContext, project_path: &Path, ) -> Result<()> { let mcp_path = project_path.join(".kimi-code/mcp.json"); @@ -164,7 +155,6 @@ impl AgentIntegration for KimiIntegration { )?; let agents_md = project_path.join("AGENTS.md"); super::remove_managed_skill_prompt_index( - &ctx.home, &agents_md, tracedecay_automation_runtime::automation::skill_targets::SkillInstallTarget::Kimi, )?; @@ -172,14 +162,6 @@ impl AgentIntegration for KimiIntegration { Ok(()) } - fn update_plugin(&self, ctx: &InstallContext) -> Result { - let code_home = kimi_code_home(&ctx.home); - if !installed_json_has_tracedecay(&code_home) { - return Ok(UpdatePluginOutcome::NotInstalled); - } - stage_kimi_install_action(ctx).map(UpdatePluginOutcome::DeferredUserAction) - } - fn healthcheck(&self, dc: &mut DoctorCounters, ctx: &HealthcheckContext) { eprintln!("\n\x1b[1mKimi CLI integration\x1b[0m"); doctor_check_plugin(dc, &ctx.home, &kimi_code_home(&ctx.home)); @@ -502,27 +484,6 @@ pub(crate) fn rendered_plugin_files(tracedecay_bin: &str) -> Result Result { - for (relative, rendered) in rendered_plugin_files(tracedecay_bin)? { - safe_write_text_file(&managed_dir.join(relative), &rendered, None)?; - } - eprintln!( - "\x1b[32m✔\x1b[0m Installed Kimi Code CLI plugin at {}", - managed_dir.display() - ); - Ok(managed_dir.to_path_buf()) -} - -fn stage_kimi_install_action(ctx: &InstallContext) -> Result { - let staged_dir = kimi_staged_plugin_dir(&ctx.home); - deploy_kimi_plugin_to(&staged_dir, &ctx.tracedecay_bin)?; - Ok(kimi_official_lifecycle_unavailable( - "install", - Some(&staged_dir), - )) -} - fn deferred_user_action_error(action: DeferredUserAction) -> TraceDecayError { TraceDecayError::Config { message: action.remediation, @@ -669,6 +630,13 @@ fn doctor_check_plugin(dc: &mut DoctorCounters, home: &Path, kimi_code_home: &Pa mod tests { use super::*; + fn deploy_kimi_plugin_to(dir: &Path, tracedecay_bin: &str) -> Result { + for (relative, rendered) in rendered_plugin_files(tracedecay_bin)? { + super::super::safe_write_text_file(&dir.join(relative), &rendered)?; + } + Ok(dir.to_path_buf()) + } + #[test] fn native_activation_waits_for_manager_to_copy_refreshed_staged_bundle() { let home = tempfile::tempdir().unwrap(); @@ -698,14 +666,13 @@ mod tests { let ctx = InstallContext { home: home.path().to_path_buf(), tracedecay_bin: "/new/tracedecay".to_string(), - tool_permissions: Vec::new(), project_root: None, dashboard: false, }; assert!(matches!( KimiIntegration - .prepare_non_interactive_install(&ctx) + .preflight_non_interactive_install(&ctx) .unwrap(), NonInteractiveInstallOutcome::DeferredUserAction(_) )); @@ -721,10 +688,19 @@ mod tests { super::super::host_bundle::HostBundleRegistrationStateV1::Repairable ); + // The component transaction refreshes the staged source; Kimi's + // manager then copies it into its managed root. + deploy_kimi_plugin_to(&staged_source, &ctx.tracedecay_bin).unwrap(); + assert!(matches!( + KimiIntegration + .preflight_non_interactive_install(&ctx) + .unwrap(), + NonInteractiveInstallOutcome::DeferredUserAction(_) + )); deploy_kimi_plugin_to(&managed_root, &ctx.tracedecay_bin).unwrap(); assert_eq!( KimiIntegration - .prepare_non_interactive_install(&ctx) + .preflight_non_interactive_install(&ctx) .unwrap(), NonInteractiveInstallOutcome::Ready ); @@ -803,7 +779,6 @@ mod tests { }; let home = tempfile::tempdir().unwrap(); - let lifecycle = tempfile::tempdir().unwrap(); let code_home = home.path().join(".kimi-code"); let staged_source = kimi_staged_plugin_dir(home.path()); let managed_root = kimi_managed_plugin_dir(&code_home); @@ -838,7 +813,6 @@ mod tests { let install = InstallContext { home: home.path().to_path_buf(), tracedecay_bin: tracedecay_bin.to_string(), - tool_permissions: Vec::new(), project_root: None, dashboard: false, }; @@ -872,7 +846,6 @@ mod tests { let mut registration = crate::agents::host_component_registration::CatalogHostComponentRegistrationAuthority::new_with_tracedecay_bin( "kimi", home.path(), - lifecycle.path(), HostBundleLifecycleOpV1::Install, tracedecay_bin.to_string(), ) @@ -919,7 +892,6 @@ mod tests { let mut uninstall = crate::agents::host_component_registration::CatalogHostComponentRegistrationAuthority::new_with_tracedecay_bin( "kimi", home.path(), - lifecycle.path(), HostBundleLifecycleOpV1::Uninstall, tracedecay_bin.to_string(), ) diff --git a/crates/tracedecay-agent-hosts/src/agents/kiro.rs b/crates/tracedecay-agent-hosts/src/agents/kiro.rs index 9e71bc2a05..bb8735d3b7 100644 --- a/crates/tracedecay-agent-hosts/src/agents/kiro.rs +++ b/crates/tracedecay-agent-hosts/src/agents/kiro.rs @@ -8,10 +8,7 @@ //! requirement for the global lifecycle, with no config-editing fallback. //! //! The canonical global integration is MCP-only and does not create global -//! steering, a managed agent, or a default-agent selection. Older releases -//! wrote those artifacts; doctor emits migration advisories when owned -//! leftovers remain, and activate still reconciles a leftover global steering -//! file onto the current owned block when present. Workspace-local registration +//! steering, a managed agent, or a default-agent selection. Workspace-local registration //! still writes its MCP entry, steering, and managed agent because Kiro has no //! project-path-aware registry operation. //! @@ -31,31 +28,19 @@ use tracedecay_domain::errors::{Result, TraceDecayError}; use super::{ AgentIntegration, DoctorCounters, HealthcheckContext, InstallContext, JsonConfigDialect, - McpUninstallPolicy, UpdatePluginOutcome, backup_config_file, config_backup_path, - install_mcp_server_entry, load_json_file, load_json_file_strict, mcp_config_has_tracedecay, + McpUninstallPolicy, install_mcp_server_entry, load_json_file, mcp_config_has_tracedecay, safe_write_json_file, uninstall_mcp_server_entry, }; pub struct KiroIntegration; -/// Ownership sentinels of the tracedecay steering block. The end sentinel is -/// the one shipped releases already wrote; the start sentinel replaces the -/// heading text as the block's identity so wording can change without another -/// marker migration. +/// Ownership sentinels of the tracedecay steering block, so wording can change +/// without another marker migration. const STEERING_SENTINELS: super::prompt_rules::OwnedBlockSentinels = super::prompt_rules::OwnedBlockSentinels { start: "", end: "", }; -/// Heading markers shipped releases (through v0.1.0-beta.37) used as the -/// block's identity. An existing install carries one of them, usually closed by -/// the same end sentinel, so update and uninstall must recognize them, -/// otherwise a reinstall appends the new block and strands the old one, and -/// uninstall never removes it. -const HISTORICAL_STEERING_HEADINGS: [&str; 2] = [ - "## TraceDecay: mandatory tool routing", - "## Prefer tracedecay MCP tools", -]; const KIRO_AGENT_NAME: &str = "tracedecay"; const OWNED_AGENT_DESCRIPTION: &str = "Default Kiro agent with tracedecay MCP tools and code-research guardrails."; @@ -150,18 +135,6 @@ pub(super) fn mcp_config_path(home: &Path) -> PathBuf { kiro_home(home).join("settings/mcp.json") } -fn cli_config_path(home: &Path) -> PathBuf { - kiro_home(home).join("settings/cli.json") -} - -fn managed_agent_path(home: &Path) -> PathBuf { - kiro_home(home).join("agents/tracedecay.json") -} - -fn steering_path(home: &Path) -> PathBuf { - kiro_home(home).join("steering/tracedecay.md") -} - pub(super) fn managed_skill_index_path(home: &Path) -> PathBuf { kiro_home(home).join("steering/tracedecay-managed-skills.md") } @@ -335,7 +308,7 @@ impl AgentIntegration for KiroIntegration { fn deactivate_project_host_component_registration( &self, _components: &[super::host_bundle::HostComponentV1], - ctx: &InstallContext, + _ctx: &InstallContext, project_path: &Path, ) -> Result<()> { let mcp_path = workspace_mcp_config_path(project_path); @@ -353,30 +326,11 @@ impl AgentIntegration for KiroIntegration { )?; uninstall_mcp_server(&mcp_path)?; remove_steering_rules(&steering)?; - remove_kiro_managed_skill_index(&ctx.home, &skill_index_path)?; + remove_kiro_managed_skill_index(&skill_index_path)?; uninstall_managed_agent(&agent_path); Ok(()) } - fn update_plugin(&self, ctx: &InstallContext) -> Result { - // Refresh only the owned managed-agent artifact that embeds the binary - // path. Migration cleanup owns retired global steering/default-agent - // state, and user-managed agent files are never rewritten. - let agent_path = managed_agent_path(&ctx.home); - if !is_owned_agent_file(&agent_path) { - return Ok(UpdatePluginOutcome::NotInstalled); - } - let skill_index_path = managed_skill_index_path(&ctx.home); - install_managed_agent( - &agent_path, - &ctx.tracedecay_bin, - &steering_path(&ctx.home), - &ctx.home, - Some(&skill_index_path), - )?; - Ok(UpdatePluginOutcome::Refreshed(vec![agent_path])) - } - fn healthcheck(&self, dc: &mut DoctorCounters, ctx: &HealthcheckContext) { eprintln!("\n\x1b[1mKiro integration\x1b[0m"); let host_home = kiro_home(&ctx.home); @@ -393,11 +347,6 @@ impl AgentIntegration for KiroIntegration { "Kiro is detected at {}, but TraceDecay is not installed, run `tracedecay install --agent kiro` if you use Kiro", host_home.display() )); - // Retired leftovers can exist without an MCP entry (for example - // after MCP-only uninstall, or a profile that never finished - // the MCP registration). Advise them independently of install - // presence so migration state is not silently omitted. - doctor_advise_retired_global_artifacts(dc, &ctx.home); return; } Ok(KiroDoctorInstallationState::Installed) => {} @@ -413,7 +362,6 @@ impl AgentIntegration for KiroIntegration { &ctx.project_path, global_server.as_ref(), ); - doctor_advise_retired_global_artifacts(dc, &ctx.home); super::doctor_check_managed_skill_prompt_indexes( dc, &ctx.home, @@ -452,13 +400,7 @@ impl AgentIntegration for KiroIntegration { } fn host_registration_paths(&self, home: &Path) -> Vec { - vec![ - mcp_config_path(home), - cli_config_path(home), - managed_agent_path(home), - steering_path(home), - managed_skill_index_path(home), - ] + vec![mcp_config_path(home), managed_skill_index_path(home)] } fn host_component_registration_paths( @@ -467,8 +409,7 @@ impl AgentIntegration for KiroIntegration { home: &Path, ) -> Vec { if components == [super::host_bundle::HostComponentV1::ContextMcp] { - let path = mcp_config_path(home); - vec![path.clone(), config_backup_path(&path)] + vec![mcp_config_path(home)] } else { self.host_registration_paths(home) } @@ -480,13 +421,6 @@ impl AgentIntegration for KiroIntegration { ctx: &InstallContext, ) -> Result<()> { if components.contains(&super::host_bundle::HostComponentV1::ContextMcp) { - // Catalog-native global install is MCP-only, so the steering file, - // the managed agent and the managed skill index prior releases - // wrote are retired by definition. Sweep them before the Kiro CLI - // is required: converging them is local file work, and welding it - // to uninstall left the only remedy one that also tears out the - // MCP registration this very pass is installing. - remove_retired_global_agent_artifacts(&ctx.home)?; let kiro_cli = require_kiro_cli()?; kiro_mcp_add_with(&kiro_cli, &ctx.home, &ctx.tracedecay_bin)?; } @@ -501,11 +435,6 @@ impl AgentIntegration for KiroIntegration { if components.contains(&super::host_bundle::HostComponentV1::ContextMcp) { let kiro_cli = require_kiro_cli()?; kiro_mcp_remove_with(&kiro_cli, &ctx.home)?; - // Canonical global install is MCP-only, but older releases left - // steering / managed agent / skill-index leftovers. Doctor advises - // `tracedecay uninstall --agent kiro` for those; sweep them here so - // that remediation actually clears the warned-about files. - remove_retired_global_artifacts(&ctx.home)?; } Ok(()) } @@ -696,9 +625,8 @@ fn install_managed_agent( Some(index_path) => install_kiro_managed_skill_index(profile_home, index_path)?, None => None, }; - let backup = backup_config_file(path)?; let config = managed_agent_config(tracedecay_bin, steering_path, managed_skill_index_path); - safe_write_json_file(path, &config, backup.as_deref())?; + safe_write_json_file(path, &config)?; eprintln!( "\x1b[32m✔\x1b[0m Wrote tracedecay Kiro agent to {}", path.display() @@ -711,8 +639,6 @@ fn install_kiro_managed_skill_index<'a>( index_path: &'a Path, ) -> Result> { let profile_root = profile_root_for_agent_home(home); - super::retired_memory_digest::remove_state(&profile_root)?; - super::retired_memory_digest::remove_prompt_block(index_path)?; let summary = install_managed_skills( &crate::host_io(), &profile_root, @@ -722,14 +648,13 @@ fn install_kiro_managed_skill_index<'a>( Ok((summary.exported_count > 0).then_some(index_path)) } -fn remove_kiro_managed_skill_index(home: &Path, index_path: &Path) -> Result<()> { - super::remove_managed_skill_prompt_index(home, index_path, SkillInstallTarget::Kiro) +fn remove_kiro_managed_skill_index(index_path: &Path) -> Result<()> { + super::remove_managed_skill_prompt_index(index_path, SkillInstallTarget::Kiro) } -/// Add or refresh tracedecay's steering resource. Every owned block, the -/// current sentinel-delimited shape or a historical heading-marked one, -/// converges onto exactly one copy of the current block in place; operator -/// text around it is preserved. +/// Add or refresh tracedecay's steering resource. Every owned block converges +/// onto exactly one copy of the current block in place; operator text around +/// it is preserved. fn install_steering_rules(path: &Path) -> Result<()> { let block = steering_block_text(); super::prompt_rules::reconcile_prompt_rules_with(path, |existing| { @@ -788,7 +713,7 @@ fn uninstall_mcp_server(path: &Path) -> Result<()> { ) } -/// Remove every tracedecay-owned steering block, current or historical. +/// Remove every tracedecay-owned steering block. fn remove_steering_rules(path: &Path) -> Result<()> { super::prompt_rules::remove_prompt_rules_with(path, |contents| { let ranges = owned_steering_ranges(contents); @@ -812,51 +737,6 @@ fn uninstall_managed_agent(path: &Path) { } } -/// Remove retired global artifacts that older non-MCP-only installs wrote. -/// -/// Called after the host MCP entry is removed so uninstall matches doctor -/// migration advisories for steering and the managed agent. `chat.defaultAgent` -/// is left alone: clearing it needs a strict cli.json rewrite that this -/// MCP-only lifecycle does not own; doctor tells the operator to clear it. -fn remove_retired_global_artifacts(home: &Path) -> Result<()> { - remove_retired_global_agent_artifacts(home)?; - let skill_index = managed_skill_index_path(home); - if skill_index.exists() { - remove_kiro_managed_skill_index(home, &skill_index)?; - } - Ok(()) -} - -/// The retired artifacts doctor advises removing: the steering block and the -/// managed agent. -/// -/// The managed skill index is deliberately not one of them. It is live state -/// the managed-skill export owns and rewrites, so sweeping it on install would -/// only fight that export on the next deploy. -fn remove_retired_global_agent_artifacts(home: &Path) -> Result<()> { - let steering = steering_path(home); - if steering.exists() { - remove_steering_rules(&steering)?; - } - uninstall_managed_agent(&managed_agent_path(home)); - Ok(()) -} - -/// True while a retired global artifact is still present. -/// -/// Global install is MCP-only, so an install carrying one is not in its -/// canonical shape. Saying so is what makes `tracedecay install --agent kiro` -/// run the sweep at all: an install whose MCP entry already reads `Current` -/// short-circuits before activation, which is precisely the state every -/// profile holding these leftovers is in. Self-limiting, the sweep clears it. -fn retired_global_agent_artifacts_present(home: &Path) -> bool { - if is_owned_agent_file(&managed_agent_path(home)) { - return true; - } - std::fs::read_to_string(steering_path(home)) - .is_ok_and(|contents| !owned_steering_ranges(&contents).is_empty()) -} - fn is_owned_agent_file(path: &Path) -> bool { if !path.exists() { return false; @@ -899,49 +779,18 @@ fn kiro_context_mcp_registration_state( .and_then(serde_json::Value::as_array) .is_some_and(|args| args.iter().any(|arg| arg.as_str() == Some("serve"))) && server.get("disabled").and_then(serde_json::Value::as_bool) != Some(true); - if !mcp_current || retired_global_agent_artifacts_present(home) { - return State::Repairable; + if mcp_current { + State::Current + } else { + State::Repairable } - State::Current } -/// Every tracedecay-owned steering range in document order: current -/// sentinel-delimited blocks plus historical heading-marked ones. +/// Every tracedecay-owned steering block in document order. fn owned_steering_ranges(contents: &str) -> Vec> { - super::prompt_rules::owned_block_ranges(contents, first_owned_steering_range) -} - -/// Earliest owned block at or after `from`. A historical heading block runs to -/// the shipped end sentinel when that sentinel closes it before any other -/// boundary; otherwise it ends at the next heading, the managed skill index, a -/// current start sentinel, or EOF, the shape the oldest installs wrote. -fn first_owned_steering_range(contents: &str, from: usize) -> Option> { - let current = STEERING_SENTINELS.block_range(contents, from); - let historical = HISTORICAL_STEERING_HEADINGS - .iter() - .filter_map(|heading| { - contents[from..] - .find(heading) - .map(|at| (from + at, heading)) - }) - .min_by_key(|(start, _)| *start) - .map(|(start, heading)| { - let body_from = start + heading.len(); - let boundary = super::prompt_rules::historical_heading_block_end( - contents, - body_from, - STEERING_SENTINELS, - ); - let end = contents[body_from..boundary] - .find(STEERING_SENTINELS.end) - .map_or(boundary, |at| body_from + at + STEERING_SENTINELS.end.len()); - start..end - }); - match (current, historical) { - (Some(current), Some(historical)) if historical.start < current.start => Some(historical), - (Some(current), _) => Some(current), - (None, historical) => historical, - } + super::prompt_rules::owned_block_ranges(contents, |contents, from| { + STEERING_SENTINELS.block_range(contents, from) + }) } // --------------------------------------------------------------------------- @@ -1074,97 +923,6 @@ fn doctor_check_workspace_mcp_override( } } -/// Emit migration advisories for retired global Kiro artifacts. -/// -/// Canonical global install is MCP-only. Older releases left -/// `~/.kiro/steering/tracedecay.md`, a managed agent, and -/// `chat.defaultAgent=tracedecay`. Doctor must not silently omit those -/// leftovers: warn so operators can clean them up without grading them as -/// current install health failures. -fn doctor_advise_retired_global_artifacts(dc: &mut DoctorCounters, home: &Path) { - doctor_advise_retired_steering(dc, home); - doctor_advise_retired_managed_agent(dc, home); - doctor_advise_retired_default_agent(dc, home); -} - -fn doctor_advise_retired_steering(dc: &mut DoctorCounters, home: &Path) { - let path = steering_path(home); - if !path.exists() { - return; - } - let contents = match std::fs::read_to_string(&path) { - Ok(contents) => contents, - Err(error) => { - dc.warn(&format!( - "migration advisory: retired Kiro global steering at {} is unreadable ({error}); \ - remove it manually or run `tracedecay install --agent kiro` after fixing permissions", - path.display() - )); - return; - } - }; - let ranges = owned_steering_ranges(&contents); - if ranges.is_empty() { - return; - } - dc.warn(&format!( - "migration advisory: retired Kiro global steering still present at {} \ - ({} owned block(s)); global install is MCP-only, remove with \ - `tracedecay install --agent kiro` or delete the owned block(s)", - path.display(), - ranges.len() - )); -} - -fn doctor_advise_retired_managed_agent(dc: &mut DoctorCounters, home: &Path) { - let path = managed_agent_path(home); - if !path.exists() { - return; - } - if !is_owned_agent_file(&path) { - return; - } - dc.warn(&format!( - "migration advisory: retired Kiro managed agent still present at {}; \ - global install is MCP-only, remove with `tracedecay install --agent kiro`", - path.display() - )); -} - -fn doctor_advise_retired_default_agent(dc: &mut DoctorCounters, home: &Path) { - let path = cli_config_path(home); - if !path.exists() { - return; - } - let config = match load_json_file_strict(&path) { - Ok(config) => config, - Err(error) => { - dc.warn(&format!( - "migration advisory: retired Kiro cli.json at {} is unreadable ({error}); \ - fix or delete the file so doctor can tell whether chat.defaultAgent still \ - points at `{KIRO_AGENT_NAME}`", - path.display() - )); - return; - } - }; - let Some(default_agent) = config - .pointer("/chat/defaultAgent") - .and_then(serde_json::Value::as_str) - else { - return; - }; - if default_agent != KIRO_AGENT_NAME { - return; - } - dc.warn(&format!( - "migration advisory: retired Kiro chat.defaultAgent still points at `{KIRO_AGENT_NAME}` in {}; \ - global install is MCP-only, clear or delete that setting manually \ - (`tracedecay uninstall --agent kiro` does not rewrite cli.json)", - path.display() - )); -} - #[cfg(test)] #[allow(clippy::unwrap_used, clippy::expect_used)] mod tests; diff --git a/crates/tracedecay-agent-hosts/src/agents/kiro/tests.rs b/crates/tracedecay-agent-hosts/src/agents/kiro/tests.rs index 3dd518704b..b0b83d2974 100644 --- a/crates/tracedecay-agent-hosts/src/agents/kiro/tests.rs +++ b/crates/tracedecay-agent-hosts/src/agents/kiro/tests.rs @@ -43,16 +43,6 @@ fn every_steering_mutation_branch_requires_a_persisted_write_intent() { } } -/// The heading shipped releases through v0.1.0-beta.37 wrote as the block's -/// identity, closed by the end sentinel those releases already emitted. -const SHIPPED_HEADING: &str = "## TraceDecay: mandatory tool routing"; -/// The heading the release before that used for the same block. -const OLDEST_HEADING: &str = "## Prefer tracedecay MCP tools"; - -fn shipped_block(heading: &str, body: &str) -> String { - format!("{heading}\n\n{body}\n\n{}", STEERING_SENTINELS.end) -} - fn steering_mutation_cases() -> Vec<(&'static str, Option>)> { vec![ ( @@ -65,181 +55,11 @@ fn steering_mutation_cases() -> Vec<(&'static str, Option>)> { .into_bytes(), ), ), - ( - "shipped-heading refresh", - Some( - format!( - "operator rules\n\n{}\n", - shipped_block(SHIPPED_HEADING, "stale rules") - ) - .into_bytes(), - ), - ), - ( - "heading fallback", - Some(format!("operator rules\n\n{SHIPPED_HEADING}\n\nstale rules\n").into_bytes()), - ), ("existing append", Some(b"operator rules\n".to_vec())), ("missing create", None), ] } -#[test] -fn every_historical_steering_shape_converges_on_update_and_preserves_peers() { - let block = steering_block_text(); - let historical_shapes = [ - ( - "shipped heading with end sentinel", - shipped_block( - SHIPPED_HEADING, - "You MUST use it. 1% chance. No rationalizing.", - ), - ), - ( - "oldest heading with end sentinel", - shipped_block( - OLDEST_HEADING, - "Before reading source files, use tracedecay.", - ), - ), - ( - "oldest heading without end sentinel", - format!("{OLDEST_HEADING}\n\nBefore reading source files, use tracedecay."), - ), - ]; - for (shape, stale) in historical_shapes { - let root = tempfile::tempdir().unwrap(); - let steering = root.path().join("tracedecay.md"); - let original = - format!("# Team steering\n\nkeep me\n\n{stale}\n\n## Operator section\n\nand me\n"); - std::fs::write(&steering, &original).unwrap(); - - install_steering_rules(&steering).unwrap(); - - let updated = std::fs::read_to_string(&steering).unwrap(); - assert_eq!( - updated, - format!("# Team steering\n\nkeep me\n\n{block}\n\n## Operator section\n\nand me\n"), - "{shape}: update must replace the whole owned block in place and keep both peers" - ); - assert!( - !updated.contains("MUST") && !updated.contains("rationaliz"), - "{shape}: no historical forcing may survive the migration" - ); - - install_steering_rules(&steering).unwrap(); - assert_eq!( - std::fs::read_to_string(&steering).unwrap(), - updated, - "{shape}: a current reinstall is idempotent" - ); - } -} - -#[test] -fn every_historical_steering_shape_is_removed_on_uninstall() { - for stale in [ - shipped_block(SHIPPED_HEADING, "stale mandate"), - shipped_block(OLDEST_HEADING, "stale mandate"), - format!("{OLDEST_HEADING}\n\nstale mandate"), - steering_block_text(), - ] { - let root = tempfile::tempdir().unwrap(); - let steering = root.path().join("tracedecay.md"); - std::fs::write( - &steering, - format!("keep me\n\n{stale}\n\n## Operator section\n\nand me\n"), - ) - .unwrap(); - - remove_steering_rules(&steering).unwrap(); - - assert_eq!( - std::fs::read_to_string(&steering).unwrap(), - "keep me\n\n## Operator section\n\nand me\n", - "uninstall must remove the owned block and only that block" - ); - } -} - -#[test] -fn duplicate_and_mixed_steering_blocks_converge_deterministically() { - let block = steering_block_text(); - let mixed = format!( - "keep me\n\n{}\n\n## Operator section\n\nand me\n\n{}\n\n{block}\n\ntail peer\n", - shipped_block(SHIPPED_HEADING, "stale mandate"), - shipped_block(OLDEST_HEADING, "older mandate"), - ); - let root = tempfile::tempdir().unwrap(); - let steering = root.path().join("tracedecay.md"); - std::fs::write(&steering, &mixed).unwrap(); - - install_steering_rules(&steering).unwrap(); - - let converged = std::fs::read_to_string(&steering).unwrap(); - assert_eq!( - converged, - format!("keep me\n\n{block}\n\n## Operator section\n\nand me\n\ntail peer\n"), - "mixed markers must collapse onto one current block at the first owned position" - ); - assert_eq!(owned_steering_ranges(&converged).len(), 1); - - std::fs::write(&steering, &mixed).unwrap(); - remove_steering_rules(&steering).unwrap(); - assert_eq!( - std::fs::read_to_string(&steering).unwrap(), - "keep me\n\n## Operator section\n\nand me\n\ntail peer\n", - "uninstall must remove every owned block, historical and current" - ); -} - -#[test] -fn steering_doctor_emits_migration_advisory_for_retired_owned_blocks() { - fn advise(home: &Path) -> DoctorCounters { - let mut counters = DoctorCounters::new(); - doctor_advise_retired_steering(&mut counters, home); - counters - } - let home = tempfile::tempdir().unwrap(); - let steering = steering_path(home.path()); - std::fs::create_dir_all(steering.parent().unwrap()).unwrap(); - - std::fs::write( - &steering, - "tracedecay MCP tools are great, use tracedecay_grep\n", - ) - .unwrap(); - let prose = advise(home.path()); - assert_eq!( - (prose.issues, prose.warnings), - (0, 0), - "prose without ownership sentinels is not a retired TraceDecay artifact" - ); - - std::fs::write(&steering, shipped_block(SHIPPED_HEADING, "stale mandate")).unwrap(); - let historical = advise(home.path()); - assert_eq!(historical.issues, 0); - assert_eq!( - historical.warnings, 1, - "owned historical steering must surface a migration advisory" - ); - - install_steering_rules(&steering).unwrap(); - let current_block = advise(home.path()); - assert_eq!(current_block.issues, 0); - assert_eq!( - current_block.warnings, 1, - "current owned steering is still retired globally and must advise migration" - ); - - std::fs::remove_file(&steering).unwrap(); - assert_eq!( - (advise(home.path()).issues, advise(home.path()).warnings), - (0, 0), - "absent retired steering emits no advisory" - ); -} - #[test] fn healthcheck_skips_steering_when_legacy_file_is_absent() { let home = tempfile::tempdir().unwrap(); @@ -266,198 +86,6 @@ fn healthcheck_skips_steering_when_legacy_file_is_absent() { ); } -#[test] -fn healthcheck_advises_shipped_heading_steering_as_retired() { - let home = tempfile::tempdir().unwrap(); - let mcp_path = mcp_config_path(home.path()); - std::fs::create_dir_all(mcp_path.parent().unwrap()).unwrap(); - std::fs::write( - &mcp_path, - br#"{"mcpServers":{"tracedecay":{"command":"/bin/tracedecay","args":["serve"],"disabled":false}}}"#, - ) - .unwrap(); - let steering = steering_path(home.path()); - std::fs::create_dir_all(steering.parent().unwrap()).unwrap(); - std::fs::write( - &steering, - shipped_block(SHIPPED_HEADING, "You MUST use tracedecay."), - ) - .unwrap(); - - let mut counters = DoctorCounters::new(); - KiroIntegration.healthcheck( - &mut counters, - &HealthcheckContext { - home: home.path().to_path_buf(), - project_path: home.path().to_path_buf(), - }, - ); - - assert_eq!(counters.issues, 0); - assert_eq!( - counters.warnings, 1, - "legacy heading-marked steering must surface as a migration advisory" - ); - - install_steering_rules(&steering).unwrap(); - let mut counters = DoctorCounters::new(); - KiroIntegration.healthcheck( - &mut counters, - &HealthcheckContext { - home: home.path().to_path_buf(), - project_path: home.path().to_path_buf(), - }, - ); - assert_eq!(counters.issues, 0); - assert_eq!( - counters.warnings, 1, - "converged owned steering is still retired globally and must keep advising migration" - ); -} - -#[cfg(unix)] -#[test] -fn global_activate_sweeps_retired_artifacts_and_clears_their_advisories() { - use crate::agents::host_bundle::HostComponentV1; - use crate::agents::{AgentIntegration, InstallContext}; - - let home = tempfile::tempdir().unwrap(); - let bin_dir = tempfile::tempdir().unwrap(); - let log = bin_dir.path().join("invocations.log"); - let kiro_cli = bin_dir.path().join("kiro-cli"); - fake_kiro_cli(&kiro_cli, &log, FAKE_REGISTRY_BODY); - let _path = tracedecay_runtime_core::config::HostProgramSearchPathGuard::set(bin_dir.path()); - - let steering = steering_path(home.path()); - std::fs::create_dir_all(steering.parent().unwrap()).unwrap(); - std::fs::write( - &steering, - shipped_block(SHIPPED_HEADING, "You MUST use tracedecay."), - ) - .unwrap(); - - let agent = managed_agent_path(home.path()); - std::fs::create_dir_all(agent.parent().unwrap()).unwrap(); - std::fs::write( - &agent, - serde_json::to_vec_pretty(&serde_json::json!({ - "name": KIRO_AGENT_NAME, - "description": OWNED_AGENT_DESCRIPTION, - })) - .unwrap(), - ) - .unwrap(); - - // Pretend MCP is already installed so doctor reaches retired-artifact advisories. - let mcp_path = mcp_config_path(home.path()); - std::fs::create_dir_all(mcp_path.parent().unwrap()).unwrap(); - std::fs::write( - &mcp_path, - br#"{"mcpServers":{"tracedecay":{"command":"/bin/tracedecay","args":["serve"],"disabled":false}}}"#, - ) - .unwrap(); - - let mut counters = DoctorCounters::new(); - KiroIntegration.healthcheck( - &mut counters, - &HealthcheckContext { - home: home.path().to_path_buf(), - project_path: home.path().to_path_buf(), - }, - ); - assert_eq!(counters.issues, 0); - assert_eq!( - counters.warnings, 2, - "precondition: retired steering and managed agent each emit an advisory" - ); - - KiroIntegration - .activate_deployed_host_component_registration( - &[HostComponentV1::ContextMcp], - &InstallContext { - home: home.path().to_path_buf(), - tracedecay_bin: "/bin/tracedecay".to_string(), - tool_permissions: Vec::new(), - project_root: None, - dashboard: false, - }, - ) - .expect("global activate must sweep retired artifacts"); - - assert!( - !agent.exists(), - "activate must remove the retired managed agent it advises removing" - ); - assert!( - !steering.exists() - || owned_steering_ranges(&std::fs::read_to_string(&steering).unwrap()).is_empty(), - "activate must leave no owned steering block behind" - ); - assert!( - mcp_registry_has_tracedecay(&mcp_path), - "sweeping retired artifacts must not disturb the MCP registration" - ); - - let mut counters = DoctorCounters::new(); - KiroIntegration.healthcheck( - &mut counters, - &HealthcheckContext { - home: home.path().to_path_buf(), - project_path: home.path().to_path_buf(), - }, - ); - assert_eq!(counters.issues, 0); - assert_eq!( - counters.warnings, 0, - "the advised remedy must actually clear every advisory it named" - ); -} - -#[test] -fn a_current_mcp_entry_reads_repairable_while_a_retired_artifact_remains() { - use crate::agents::host_bundle::HostBundleRegistrationStateV1 as State; - - let home = tempfile::tempdir().unwrap(); - let mcp_path = mcp_config_path(home.path()); - std::fs::create_dir_all(mcp_path.parent().unwrap()).unwrap(); - std::fs::write( - &mcp_path, - br#"{"mcpServers":{"tracedecay":{"command":"/bin/tracedecay","args":["serve"],"disabled":false}}}"#, - ) - .unwrap(); - assert_eq!( - kiro_context_mcp_registration_state(home.path()), - State::Current - ); - - let agent = managed_agent_path(home.path()); - std::fs::create_dir_all(agent.parent().unwrap()).unwrap(); - std::fs::write( - &agent, - serde_json::to_vec(&serde_json::json!({ - "name": KIRO_AGENT_NAME, - "description": OWNED_AGENT_DESCRIPTION, - })) - .unwrap(), - ) - .unwrap(); - - assert_eq!( - kiro_context_mcp_registration_state(home.path()), - State::Repairable, - "a current MCP entry must not mask a retired artifact: install short-circuits \ - on Current and never reaches the sweep, which is the state every profile \ - holding these leftovers is already in" - ); - - remove_retired_global_agent_artifacts(home.path()).unwrap(); - assert_eq!( - kiro_context_mcp_registration_state(home.path()), - State::Current, - "the sweep must settle the state rather than leave install repairing forever" - ); -} - #[cfg(unix)] #[test] fn global_activate_does_not_create_missing_legacy_steering() { @@ -471,7 +99,7 @@ fn global_activate_does_not_create_missing_legacy_steering() { fake_kiro_cli(&kiro_cli, &log, FAKE_REGISTRY_BODY); let _path = tracedecay_runtime_core::config::HostProgramSearchPathGuard::set(bin_dir.path()); - let steering = steering_path(home.path()); + let steering = home.path().join(".kiro/steering/tracedecay.md"); assert!(!steering.exists()); KiroIntegration @@ -480,7 +108,6 @@ fn global_activate_does_not_create_missing_legacy_steering() { &InstallContext { home: home.path().to_path_buf(), tracedecay_bin: "/bin/tracedecay".to_string(), - tool_permissions: Vec::new(), project_root: None, dashboard: false, }, @@ -895,7 +522,6 @@ fn failed_kiro_cli_effect_rolls_back_the_peer_containing_registry() { let mut registration = crate::agents::host_component_registration::CatalogHostComponentRegistrationAuthority::new_with_tracedecay_bin( "kiro", home.path(), - lifecycle.path(), request.lifecycle.operation, "/bin/tracedecay".to_string(), ) @@ -951,7 +577,6 @@ fn rollback_refuses_a_foreign_registry_write_after_cli_apply() { let mut registration = crate::agents::host_component_registration::CatalogHostComponentRegistrationAuthority::new_with_tracedecay_bin( "kiro", home.path(), - lifecycle.path(), request.lifecycle.operation, "/bin/tracedecay".to_string(), ) @@ -1027,104 +652,6 @@ fn detected_kiro_without_a_tracedecay_server_is_a_single_optional_warning() { assert_eq!(counters.warnings, 1); } -#[test] -fn absent_mcp_entry_still_advises_retired_global_artifacts() { - let home = tempfile::tempdir().unwrap(); - let mcp_path = mcp_config_path(home.path()); - std::fs::create_dir_all(mcp_path.parent().unwrap()).unwrap(); - std::fs::write( - &mcp_path, - br#"{"mcpServers":{"operator":{"command":"other","args":[]}}}"#, - ) - .unwrap(); - - let steering = steering_path(home.path()); - std::fs::create_dir_all(steering.parent().unwrap()).unwrap(); - std::fs::write( - &steering, - format!( - "{}\n", - shipped_block(SHIPPED_HEADING, "retired global steering") - ), - ) - .unwrap(); - - let agent = managed_agent_path(home.path()); - std::fs::create_dir_all(agent.parent().unwrap()).unwrap(); - std::fs::write( - &agent, - serde_json::to_vec(&serde_json::json!({ - "name": "tracedecay", - "description": OWNED_AGENT_DESCRIPTION, - "hooks": {} - })) - .unwrap(), - ) - .unwrap(); - - let mut counters = DoctorCounters::new(); - KiroIntegration.healthcheck( - &mut counters, - &HealthcheckContext { - home: home.path().to_path_buf(), - project_path: home.path().to_path_buf(), - }, - ); - - assert_eq!(counters.issues, 0); - assert_eq!( - counters.warnings, 3, - "absent MCP must still surface the not-installed warning plus steering and managed-agent advisories" - ); -} - -#[test] -fn unreadable_cli_json_emits_a_migration_advisory() { - let home = tempfile::tempdir().unwrap(); - let cli = cli_config_path(home.path()); - std::fs::create_dir_all(cli.parent().unwrap()).unwrap(); - std::fs::write(&cli, "{ not valid JSON").unwrap(); - - let mut counters = DoctorCounters::new(); - doctor_advise_retired_default_agent(&mut counters, home.path()); - - assert_eq!(counters.issues, 0); - assert_eq!(counters.warnings, 1); -} - -#[test] -fn remove_retired_global_artifacts_clears_owned_steering_and_managed_agent() { - let home = tempfile::tempdir().unwrap(); - let steering = steering_path(home.path()); - std::fs::create_dir_all(steering.parent().unwrap()).unwrap(); - std::fs::write( - &steering, - format!("keep me\n\n{}\n", shipped_block(SHIPPED_HEADING, "retired")), - ) - .unwrap(); - let agent = managed_agent_path(home.path()); - std::fs::create_dir_all(agent.parent().unwrap()).unwrap(); - std::fs::write( - &agent, - serde_json::to_vec(&serde_json::json!({ - "name": "tracedecay", - "description": OWNED_AGENT_DESCRIPTION, - "hooks": {} - })) - .unwrap(), - ) - .unwrap(); - - remove_retired_global_artifacts(home.path()).unwrap(); - - let remaining = std::fs::read_to_string(&steering).unwrap(); - assert!( - remaining.contains("keep me") && !remaining.contains(SHIPPED_HEADING), - "owned retired steering must be stripped while operator prose remains: {remaining:?}" - ); - assert!(!agent.exists(), "owned managed agent must be removed"); -} - #[test] fn malformed_kiro_mcp_config_remains_a_doctor_failure() { let home = tempfile::tempdir().unwrap(); @@ -1261,9 +788,9 @@ fn managed_agent_hook_entries_carry_only_documented_fields() { /// Kiro custom agents do not auto-include steering, so the managed agent's /// `resources` must reference the global steering file explicitly. #[test] -fn managed_agent_resources_reference_the_global_steering_file() { - let home = tempfile::tempdir().unwrap(); - let steering = steering_path(home.path()); +fn managed_agent_resources_reference_the_steering_file() { + let project = tempfile::tempdir().unwrap(); + let steering = project.path().join(".kiro/steering/tracedecay.md"); let config = managed_agent_config("/bin/tracedecay", &steering, None); let expected = file_resource_uri(&steering); assert!( @@ -1272,6 +799,6 @@ fn managed_agent_resources_reference_the_global_steering_file() { .expect("agent config has resources") .iter() .any(|value| value.as_str() == Some(expected.as_str())), - "managed agent must load global steering as an explicit resource" + "managed agent must load its steering as an explicit resource" ); } diff --git a/crates/tracedecay-agent-hosts/src/agents/mcp_registration.rs b/crates/tracedecay-agent-hosts/src/agents/mcp_registration.rs index 7e9dacee80..c7f04ad567 100644 --- a/crates/tracedecay-agent-hosts/src/agents/mcp_registration.rs +++ b/crates/tracedecay-agent-hosts/src/agents/mcp_registration.rs @@ -87,9 +87,8 @@ enum McpUninstallOutcome { /// config that exists but cannot be parsed is a typed error, reporting a /// clean uninstall over a corrupt config would fabricate state, and callers /// decide whether to keep going across the remaining hosts. Every rewrite or -/// removal of the existing file leaves a `.bak` (issue #63) and publishes -/// through the durable conditional write/remove shared by every host-file -/// transaction. +/// removal publishes through the durable conditional write/remove shared by +/// every host-file transaction. #[hotpath::measure(label = "agent_hosts.agents.mcp.uninstall")] pub fn uninstall_mcp_server_entry( config_path: &Path, @@ -322,11 +321,3 @@ pub fn read_only_tool_names() -> tracedecay_domain::errors::Result> .map(|tool| tool.name) .collect()) } - -/// Legacy-namespace permission entries for every advertised tool. -pub fn expected_tool_perms() -> tracedecay_domain::errors::Result> { - Ok(advertised_tools()? - .iter() - .map(|tool| format!("{}{}", crate::tool_name::LEGACY_TOOL_PREFIX, tool.name)) - .collect()) -} diff --git a/crates/tracedecay-agent-hosts/src/agents/mod.rs b/crates/tracedecay-agent-hosts/src/agents/mod.rs index 481998a2d4..2f1807b6bb 100644 --- a/crates/tracedecay-agent-hosts/src/agents/mod.rs +++ b/crates/tracedecay-agent-hosts/src/agents/mod.rs @@ -14,9 +14,6 @@ pub mod copilot; pub mod cursor; pub(crate) mod cursor_diagnostics; pub mod devin; -/// Legacy Cursor `serve` log marker; the root crate's `src/serve.rs` -/// re-exports this instead of declaring its own copy. -pub use cursor_diagnostics::DEGRADED_SERVE_STDERR_MARKER; pub mod gemini; mod git_post_commit_hook; pub mod hermes; @@ -34,10 +31,8 @@ pub mod plugin_bundle; pub mod prompt_rules; mod text_file_transaction; pub(crate) use text_file_transaction::{ - TextFileMutation, update_config_file_transactionally, update_text_file_transactionally, - update_two_config_files_transactionally, + TextFileMutation, update_text_file_transactionally, update_two_config_files_transactionally, }; -pub(crate) mod retired_memory_digest; pub mod roo_code; pub mod vibe; pub mod zed; @@ -51,8 +46,8 @@ use tracedecay_domain::errors::TraceDecayError; pub use antigravity::AntigravityIntegration; pub(crate) use bundle_identity::{ - is_auto_discovered_entrypoint, observed_bundle_content_digest, - observed_bundle_discovery_matches, rendered_bundle_content_digest, + observed_bundle_content_digest, observed_bundle_discovery_matches, + rendered_bundle_content_digest, }; pub use claude::ClaudeIntegration; pub use cline::ClineIntegration; @@ -72,10 +67,10 @@ pub use zed::ZedIntegration; pub use git_post_commit_hook::{install_git_post_commit_hook, report_git_post_commit_hook_status}; pub use host_config_io::{ - HostFileMetadataIdentityV1, JsonConfigDialect, backup_config_file, capture_host_file_metadata, - config_backup_path, copilot_cli_dir, home_dir, host_config_write_intent_path, kiro_data_dir, - load_json_file, load_json_file_strict, load_jsonc_file, load_jsonc_file_strict, load_toml_file, - parse_jsonc, restore_host_file_metadata, safe_remove_host_file, safe_write_bytes_file, + HostFileMetadataIdentityV1, JsonConfigDialect, capture_host_file_metadata, copilot_cli_dir, + home_dir, host_config_write_intent_path, kiro_data_dir, load_json_file, load_json_file_strict, + load_jsonc_file, load_jsonc_file_strict, load_toml_file, parse_jsonc, + restore_host_file_metadata, safe_remove_host_file, safe_write_bytes_file, safe_write_bytes_file_with_metadata, safe_write_json_file, safe_write_text_file, vscode_data_dir, vscode_insiders_data_dir, which_tracedecay, which_tracedecay_path, with_host_config_write_intents, @@ -83,22 +78,19 @@ pub use host_config_io::{ pub(crate) use host_config_io::{ JsonConfigMutation, collect_regular_files, ensure_project_local_safe_path, ensure_project_local_safe_paths, hook_command, host_home_override, - record_host_config_observation_bytes, sweep_superseded_plugin_siblings, - update_json_config_transactionally, update_toml_config_transactionally, + record_host_config_observation_bytes, update_json_config_transactionally, + update_toml_config_transactionally, }; -// Host adapters under this module reach these through `super::` / `crate::agents::`. #[cfg(test)] use host_config_io::{ TestHostConfigWritePauseController, pause_next_host_config_write_after_validation, pause_next_host_config_write_at_publication, }; -use host_config_io::{render_json_config, strip_jsonc_comments}; pub(crate) use mcp_registration::doctor_check_prompt_contains_tracedecay; pub use mcp_registration::{ - McpDoctorLabels, McpUninstallPolicy, doctor_check_mcp_registration, expected_tool_perms, - install_mcp_server_entry, mcp_config_has_tracedecay, mcp_registration_entry, - mcp_servers_registration_state, read_only_tool_names, report_mcp_registration, tool_names, - uninstall_mcp_server_entry, + McpDoctorLabels, McpUninstallPolicy, doctor_check_mcp_registration, install_mcp_server_entry, + mcp_config_has_tracedecay, mcp_registration_entry, mcp_servers_registration_state, + read_only_tool_names, report_mcp_registration, tool_names, uninstall_mcp_server_entry, }; #[hotpath::measure(label = "agent_hosts.agents.managed_skill.install_index")] @@ -111,8 +103,6 @@ pub(crate) fn install_managed_skill_prompt_index( tracedecay_automation_runtime::automation::skill_targets::profile_root_for_agent_home( profile_home, ); - retired_memory_digest::remove_state(&profile_root)?; - retired_memory_digest::remove_prompt_block(prompt_path)?; tracedecay_automation_runtime::automation::skill_targets::install_managed_skills( &crate::host_io(), &profile_root, @@ -124,21 +114,14 @@ pub(crate) fn install_managed_skill_prompt_index( #[hotpath::measure(label = "agent_hosts.agents.managed_skill.remove_index")] pub(crate) fn remove_managed_skill_prompt_index( - profile_home: &Path, prompt_path: &Path, target: tracedecay_automation_runtime::automation::skill_targets::SkillInstallTarget, ) -> Result<()> { - let profile_root = - tracedecay_automation_runtime::automation::skill_targets::profile_root_for_agent_home( - profile_home, - ); - retired_memory_digest::remove_state(&profile_root)?; tracedecay_automation_runtime::automation::skill_targets::remove_prompt_skill_index_for_target( &crate::host_io(), prompt_path, target, - )?; - retired_memory_digest::remove_prompt_block(prompt_path) + ) } /// Warns for each of a host's managed-skill prompt indexes that still @@ -282,11 +265,14 @@ pub trait AgentIntegration { false } - /// Validate non-interactive install readiness without changing host state. + /// Whether the host has already activated what this version deploys, + /// read without changing host state. /// - /// This is the read-only counterpart to - /// [`AgentIntegration::prepare_non_interactive_install`]. Hosts that need - /// manual activation report the same typed deferral without staging files. + /// Most integrations activate through their own host CLI inside the + /// component transaction and are always `Ready` here. A host whose only + /// activation route is interactive reports a typed deferral naming the + /// host action; the transaction then commits the staged source it owns + /// and surfaces that remediation instead of claiming activation. fn preflight_non_interactive_install( &self, _ctx: &InstallContext, @@ -294,27 +280,13 @@ pub trait AgentIntegration { Ok(NonInteractiveInstallOutcome::Ready) } - /// Prepare an install requested from a non-interactive orchestration path. - /// - /// Most integrations are immediately ready. Hosts whose official lifecycle - /// requires user interaction may stage verified artifacts and return a - /// typed deferral instead. Explicit install commands still surface that - /// deferral as an error, while maintenance can warn and continue. - fn prepare_non_interactive_install( - &self, - _ctx: &InstallContext, - ) -> Result { - Ok(NonInteractiveInstallOutcome::Ready) - } - /// Operator guidance for a host that activates deployed components only /// through an interactive UI, or `None` for a host TraceDecay can activate /// non-interactively. /// - /// This is the read-only capability twin of the typed deferral - /// [`AgentIntegration::prepare_non_interactive_install`] returns: doctor - /// needs the same fact without an `InstallContext` and without staging - /// anything. Every integration returning `Some` here must also return + /// This is the capability twin of the typed deferral + /// [`AgentIntegration::preflight_non_interactive_install`] returns: doctor + /// needs the same fact without an `InstallContext`. Every integration returning `Some` here must also return /// [`NonInteractiveInstallOutcome::DeferredUserAction`] from preflight, /// otherwise doctor would downgrade a state that an unattended reinstall /// could actually have repaired. @@ -342,20 +314,6 @@ pub trait AgentIntegration { None } - /// Refresh tracedecay-generated artifacts (plugin code, baked binary - /// paths, embedded assets) for every *detected* existing installation, - /// without writing to any agent config file. Pins, MCP registrations, - /// settings, and prompt rules are left byte-for-byte intact. - /// - /// The default reports [`UpdatePluginOutcome::ConfigOnly`]: most agents - /// keep their entire tracedecay integration inside shared config files - /// (MCP entries, hook blocks, prompt rules), so there is nothing to - /// refresh that would not be a config write. `tracedecay reinstall` - /// remains the path that reconciles those. - fn update_plugin(&self, _ctx: &InstallContext) -> Result { - Ok(UpdatePluginOutcome::ConfigOnly) - } - /// Re-export the profile's active managed skills into every export /// destination this agent's existing installation owns (native overlay /// or prompt index), without touching any other config. Returns one @@ -466,8 +424,9 @@ pub trait AgentIntegration { } /// Every mutable host registration/configuration path participating in an - /// aggregate component-set lifecycle. The transaction stages backups for - /// all returned paths before invoking the host registration authority. + /// aggregate component-set lifecycle. The transaction snapshots all + /// returned paths in memory before invoking the host registration + /// authority. fn host_registration_paths(&self, home: &Path) -> Vec { self.primary_config_path(home).into_iter().collect() } @@ -483,7 +442,7 @@ pub trait AgentIntegration { self.host_registration_paths(home) } - /// Fallible exact registration inventory used by the transaction backup. + /// Fallible exact registration inventory used by the transaction snapshot. /// /// Hosts whose paths depend on validated profile data override this rather /// than silently dropping files from rollback ownership. @@ -556,9 +515,10 @@ pub trait AgentIntegration { /// Apply only this host's project-scoped registration projection. /// - /// The component-set transaction calls this boundary after it has staged - /// exact registration backups. Implementations must mutate only bounded - /// project registration paths; they must not install global assets. + /// The component-set transaction calls this boundary after it has + /// snapshotted the exact registration paths. Implementations must mutate + /// only bounded project registration paths; they must not install global + /// assets. fn activate_project_host_component_registration( &self, _components: &[host_bundle::HostComponentV1], @@ -608,27 +568,10 @@ pub enum NonInteractiveInstallOutcome { DeferredUserAction(DeferredUserAction), } -/// Outcome of [`AgentIntegration::update_plugin`]. -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum UpdatePluginOutcome { - /// Generated artifacts were refreshed at these locations. - Refreshed(Vec), - /// The integration ships generated artifacts, but none were detected on - /// this machine, nothing was written. - NotInstalled, - /// The integration only writes shared config files; there are no - /// tracedecay-generated artifacts to refresh without touching config. - ConfigOnly, - /// Verified artifacts were staged, but the host requires explicit user - /// action before it can activate them. - DeferredUserAction(DeferredUserAction), -} - /// Context passed to catalog-backed host registration and refresh operations. pub struct InstallContext { pub home: PathBuf, pub tracedecay_bin: String, - pub tool_permissions: Vec, /// Codex update/uninstall can use this as an explicit repo-local plugin /// target. Other integrations ignore it. pub project_root: Option, @@ -860,19 +803,6 @@ Do not query private databases as a fallback. If the daemon is unavailable or in report that state rather than starting or replacing it." ); -/// True when a `SKILL.md` carries a TraceDecay authorship marker. Retired -/// plugin artifacts use this narrow check so same-name user workflows remain -/// outside TraceDecay's cleanup authority. -pub(crate) fn skill_contents_have_tracedecay_marker(contents: &str) -> bool { - contents.lines().map(str::trim).any(|line| { - line.starts_with("name: tracedecay:") - || line.starts_with("description: TraceDecay ") - || line.contains("TraceDecay MCP") - || line.contains("tracedecay_") - || line.contains("`tracedecay:") - }) -} - /// Choose which detected agents `tracedecay install` should configure, or /// `None` when this machine has no supported agent yet. /// diff --git a/crates/tracedecay-agent-hosts/src/agents/opencode.rs b/crates/tracedecay-agent-hosts/src/agents/opencode.rs index 5c733a3d69..7ddd6432bb 100644 --- a/crates/tracedecay-agent-hosts/src/agents/opencode.rs +++ b/crates/tracedecay-agent-hosts/src/agents/opencode.rs @@ -22,8 +22,7 @@ use tracedecay_domain::errors::{Result, TraceDecayError}; use super::{ AgentIntegration, DoctorCounters, HealthcheckContext, InstallContext, JsonConfigDialect, - TextFileMutation, UpdatePluginOutcome, load_json_file, render_json_config, - safe_write_text_file, update_config_file_transactionally, update_text_file_transactionally, + TextFileMutation, load_json_file, safe_write_text_file, update_text_file_transactionally, }; use super::prompt_rules::{PROMPT_RULE_MARKER, PromptRulesOptions}; @@ -102,14 +101,13 @@ impl AgentIntegration for OpenCodeIntegration { fn deactivate_project_host_component_registration( &self, _components: &[super::host_bundle::HostComponentV1], - ctx: &InstallContext, + _ctx: &InstallContext, project_path: &Path, ) -> Result<()> { uninstall_mcp_server(&project_path.join("opencode.json"))?; remove_opencode_plugin(&project_path.join(".opencode/plugins/tracedecay.ts"))?; let agents_md = project_path.join("AGENTS.md"); super::remove_managed_skill_prompt_index( - &ctx.home, &agents_md, tracedecay_automation_runtime::automation::skill_targets::SkillInstallTarget::OpenCode, )?; @@ -117,15 +115,6 @@ impl AgentIntegration for OpenCodeIntegration { Ok(()) } - fn update_plugin(&self, ctx: &InstallContext) -> Result { - let plugin_path = opencode_plugin_path(&ctx.home); - if !plugin_path.exists() { - return Ok(UpdatePluginOutcome::NotInstalled); - } - install_opencode_plugin(&plugin_path, &ctx.tracedecay_bin)?; - Ok(UpdatePluginOutcome::Refreshed(vec![plugin_path])) - } - fn healthcheck(&self, dc: &mut DoctorCounters, ctx: &HealthcheckContext) { eprintln!("\n\x1b[1mOpenCode integration\x1b[0m"); doctor_check_config(dc, &ctx.home); @@ -235,9 +224,7 @@ impl AgentIntegration for OpenCodeIntegration { if components.contains(&HostComponentV1::Core) || components.contains(&HostComponentV1::ContextMcp) { - let config = opencode_config_path(home); - paths.push(config.clone()); - paths.push(opencode_original_config_path(&config)); + paths.push(opencode_config_path(home)); } if components.contains(&HostComponentV1::Core) { paths.push(opencode_prompt_path(home)); @@ -264,7 +251,6 @@ impl AgentIntegration for OpenCodeIntegration { &ctx.tracedecay_bin, mcp, core, - false, )?; if core { let prompt = opencode_prompt_path(&ctx.home); @@ -288,11 +274,10 @@ impl AgentIntegration for OpenCodeIntegration { let core = components.contains(&HostComponentV1::Core); let mcp = components.contains(&HostComponentV1::ContextMcp); - remove_registration_entries(&opencode_config_path(&ctx.home), mcp, core, false)?; + remove_registration_entries(&opencode_config_path(&ctx.home), mcp, core)?; if core { let prompt = opencode_prompt_path(&ctx.home); super::remove_managed_skill_prompt_index( - &ctx.home, &prompt, tracedecay_automation_runtime::automation::skill_targets::SkillInstallTarget::OpenCode, )?; @@ -421,33 +406,18 @@ fn opencode_config_path_for(home: &Path, xdg: Option<&std::ffi::OsStr>) -> std:: .join("opencode/opencode.json") } -/// Resolution depends only on which prompt *file* exists, never on whether the -/// `~/.config/opencode` directory exists. The directory is created by -/// TraceDecay's own managed artifacts (`plugins/`, `agent/`, `command/`, -/// `skills/`), which a component-set transaction writes between the moment the -/// registration authority confirms a revision and the moment it applies. Keying -/// on the directory therefore moved this path, and with it the hashed -/// registration path list, mid-transaction, so every apply rechecked against a -/// different revision and rolled back with `StalePreview`. No managed artifact -/// ever writes an `AGENTS.md`, so file existence is stable across a deploy. -/// -/// A user whose rules already live in the legacy `~/AGENTS.md` keeps that file; -/// everyone else gets the modern config-dir path, whose parent the write path -/// creates on demand. +/// Resolution never depends on filesystem state. `~/.config/opencode` is +/// created by TraceDecay's own managed artifacts, which a component-set +/// transaction writes between the moment the registration authority confirms +/// a revision and the moment it applies; a path keyed on what exists would +/// move the hashed registration path list mid-transaction and roll every apply +/// back with `StalePreview`. The write path creates the parent on demand. pub(super) fn opencode_prompt_path(home: &Path) -> std::path::PathBuf { - if let Some(xdg) = ambient_xdg_config_home(home) + ambient_xdg_config_home(home) .map(std::path::PathBuf::from) .filter(|path| path.is_absolute()) - { - return xdg.join("opencode/AGENTS.md"); - } - let modern = home.join(".config/opencode/AGENTS.md"); - let legacy = home.join("AGENTS.md"); - if !modern.is_file() && legacy.is_file() { - legacy - } else { - modern - } + .unwrap_or_else(|| home.join(".config")) + .join("opencode/AGENTS.md") } fn opencode_asset_relative_paths( @@ -534,7 +504,7 @@ fn mirror_external_opencode_assets_to( source.display() ), })?; - super::safe_write_bytes_file(&destination, &bytes, None)?; + super::safe_write_bytes_file(&destination, &bytes)?; } Ok(()) } @@ -602,7 +572,7 @@ fn install_opencode_plugin(path: &Path, tracedecay_bin: &str) -> Result<()> { }); } for (_, rendered) in rendered_plugin_files(tracedecay_bin)? { - safe_write_text_file(path, &rendered, None)?; + safe_write_text_file(path, &rendered)?; } Ok(()) } @@ -637,11 +607,10 @@ fn remove_opencode_plugin(path: &Path) -> Result<()> { /// Register MCP server in opencode.json. /// -/// Safety: creates a `.bak` backup before writing and restores it on any -/// error. Uses strict JSON parsing so an existing file with invalid syntax -/// is never silently replaced with an empty object. +/// Uses strict JSON parsing so an existing file with invalid syntax is never +/// silently replaced with an empty object. fn install_mcp_server(config_path: &Path, tracedecay_bin: &str) -> Result<()> { - install_registration_entries(config_path, tracedecay_bin, true, true, true) + install_registration_entries(config_path, tracedecay_bin, true, true) } /// Merge TraceDecay's `mcp` and `lsp` registrations into `opencode.json`. @@ -663,12 +632,11 @@ fn install_registration_entries( tracedecay_bin: &str, install_mcp: bool, install_lsp: bool, - preserve_backup: bool, ) -> Result<()> { if !install_mcp && !install_lsp { return Ok(()); } - let merge = |existing: &str| { + update_text_file_transactionally(config_path, |existing: &str| { let config = merge_registration_entries( config_path, existing, @@ -678,17 +646,13 @@ fn install_registration_entries( )?; Ok(( (), - TextFileMutation::Write(render_json_config(config_path, &config)?), + TextFileMutation::Write(JsonConfigDialect::Json.render_edit( + config_path, + existing, + &config, + )?), )) - }; - // Component-set transactions (`preserve_backup: false`) already stage - // exact registration backups, so only the direct install path leaves the - // user-facing `.bak`. - if preserve_backup { - update_config_file_transactionally(config_path, merge)?; - } else { - update_text_file_transactionally(config_path, merge)?; - } + })?; eprintln!( "\x1b[32m✔\x1b[0m Added tracedecay MCP server to {}", config_path.display() @@ -710,12 +674,6 @@ fn merge_registration_entries( // so the write below can be proven not to have created, altered, or // dropped the key `opencode plugin` owns. let host_plugin_before = plugin_cli::host_owned_plugin_registration(&config); - let original_path = opencode_original_config_path(config_path); - let has_tracedecay = - config.pointer("/mcp/tracedecay").is_some() || config.pointer("/lsp/tracedecay").is_some(); - if !has_tracedecay && config_path.is_file() && !original_path.exists() { - super::safe_write_bytes_file(&original_path, existing.as_bytes(), None)?; - } let config_object = config .as_object_mut() @@ -836,13 +794,12 @@ fn install_prompt_rules(prompt_path: &Path) -> Result<()> { /// Remove MCP server from opencode.json. fn uninstall_mcp_server(config_path: &Path) -> Result<()> { - remove_registration_entries(config_path, true, true, true) + remove_registration_entries(config_path, true, true) } /// Outcome of the uninstall transform, reported after publication. enum OpenCodeRegistrationRemoval { NoEntry, - RestoredOriginal, RemovedFile, Rewritten, } @@ -851,29 +808,13 @@ fn remove_registration_entries( config_path: &Path, remove_mcp: bool, remove_lsp: bool, - preserve_backup: bool, ) -> Result<()> { if !config_path.exists() { return Ok(()); } - let original_path = opencode_original_config_path(config_path); - let strip = |existing: &str| { - strip_registration_entries( - config_path, - &original_path, - existing, - remove_mcp, - remove_lsp, - ) - }; - // Component-set transactions (`preserve_backup: false`) already stage - // exact registration backups, so only the direct uninstall path leaves - // the user-facing `.bak`. - let outcome = if preserve_backup { - update_config_file_transactionally(config_path, strip)? - } else { - update_text_file_transactionally(config_path, strip)? - }; + let outcome = update_text_file_transactionally(config_path, |existing: &str| { + strip_registration_entries(config_path, existing, remove_mcp, remove_lsp) + })?; match outcome { OpenCodeRegistrationRemoval::NoEntry => { eprintln!( @@ -881,13 +822,6 @@ fn remove_registration_entries( config_path.display() ); } - OpenCodeRegistrationRemoval::RestoredOriginal => { - super::safe_remove_host_file(&original_path).map_err(|error| { - TraceDecayError::Config { - message: format!("failed to remove {}: {error}", original_path.display()), - } - })?; - } OpenCodeRegistrationRemoval::RemovedFile => { eprintln!( "\x1b[32m✔\x1b[0m Removed {} (was empty)", @@ -905,11 +839,9 @@ fn remove_registration_entries( } /// Strip TraceDecay's registrations from the config bytes observed under the -/// write lock, deciding between a byte-exact pre-install restore, a rewrite, -/// and removal of an emptied file. +/// write lock, deciding between a rewrite and removal of an emptied file. fn strip_registration_entries( config_path: &Path, - original_path: &Path, existing: &str, remove_mcp: bool, remove_lsp: bool, @@ -952,17 +884,6 @@ fn strip_registration_entries( TextFileMutation::Unchanged, )); } - if let Ok(original) = std::fs::read(original_path) - && serde_json::from_slice::(&original).ok() == Some(config.clone()) - { - let original = String::from_utf8(original).map_err(|error| TraceDecayError::Config { - message: format!("{} is not valid UTF-8: {error}", original_path.display()), - })?; - return Ok(( - OpenCodeRegistrationRemoval::RestoredOriginal, - TextFileMutation::Write(original), - )); - } plugin_cli::ensure_host_owned_plugin_registration_untouched( host_plugin_before.as_ref(), &config, @@ -976,15 +897,15 @@ fn strip_registration_entries( } else { Ok(( OpenCodeRegistrationRemoval::Rewritten, - TextFileMutation::Write(render_json_config(config_path, &config)?), + TextFileMutation::Write(JsonConfigDialect::Json.render_edit( + config_path, + existing, + &config, + )?), )) } } -fn opencode_original_config_path(config_path: &Path) -> PathBuf { - PathBuf::from(format!("{}.tracedecay-original", config_path.display())) -} - fn uninstall_prompt_rules(prompt_path: &Path) -> Result<()> { super::prompt_rules::remove_standard_prompt_rules(prompt_path) } @@ -1350,7 +1271,6 @@ mod tests { let ctx = InstallContext { home: home.path().to_path_buf(), tracedecay_bin: "/usr/bin/tracedecay".to_string(), - tool_permissions: Vec::new(), project_root: Some(project.path().to_path_buf()), dashboard: false, }; diff --git a/crates/tracedecay-agent-hosts/src/agents/opencode/plugin_cli.rs b/crates/tracedecay-agent-hosts/src/agents/opencode/plugin_cli.rs index 464ba60d62..20e9d0640f 100644 --- a/crates/tracedecay-agent-hosts/src/agents/opencode/plugin_cli.rs +++ b/crates/tracedecay-agent-hosts/src/agents/opencode/plugin_cli.rs @@ -199,20 +199,14 @@ mod tests { ) .unwrap(); - super::super::install_registration_entries( - &config_path, - "/usr/bin/tracedecay", - true, - true, - true, - ) - .unwrap(); + super::super::install_registration_entries(&config_path, "/usr/bin/tracedecay", true, true) + .unwrap(); let config = crate::agents::load_json_file_strict(&config_path).unwrap(); assert_eq!(config["plugin"], host_written); assert!(config.pointer("/mcp/tracedecay").is_some()); - super::super::remove_registration_entries(&config_path, true, true, true).unwrap(); + super::super::remove_registration_entries(&config_path, true, true).unwrap(); let config = crate::agents::load_json_file_strict(&config_path).unwrap(); assert_eq!( @@ -228,14 +222,8 @@ mod tests { let home = tempfile::tempdir().unwrap(); let config_path = home.path().join("opencode.json"); - super::super::install_registration_entries( - &config_path, - "/usr/bin/tracedecay", - true, - true, - true, - ) - .unwrap(); + super::super::install_registration_entries(&config_path, "/usr/bin/tracedecay", true, true) + .unwrap(); let config = crate::agents::load_json_file_strict(&config_path).unwrap(); assert!( diff --git a/crates/tracedecay-agent-hosts/src/agents/prompt_rules.rs b/crates/tracedecay-agent-hosts/src/agents/prompt_rules.rs index 6a77f192e9..b47358e35f 100644 --- a/crates/tracedecay-agent-hosts/src/agents/prompt_rules.rs +++ b/crates/tracedecay-agent-hosts/src/agents/prompt_rules.rs @@ -41,8 +41,7 @@ impl OwnedBlockSentinels { } /// Every tracedecay-owned range in `contents` in document order. `locate_first` -/// returns the earliest owned block (current or historical shape) starting at -/// or after an offset; ranges never overlap because each search resumes at the +/// returns the earliest owned block starting at or after an offset; ranges never overlap because each search resumes at the /// previous block's end. pub(crate) fn owned_block_ranges( contents: &str, @@ -108,7 +107,7 @@ pub(crate) fn owned_block_is_current(contents: &str, ranges: &[Range], bl } /// Install `block` as the single owned block: unchanged when already current, -/// otherwise converge every current or historical owned range onto one copy +/// otherwise converge every owned range onto one copy /// (the first range's position, or appended when none exists). pub(crate) fn converge_owned_block( existing: &str, @@ -138,20 +137,6 @@ pub(crate) fn remove_owned_blocks(contents: &str, ranges: &[Range]) -> Pr } } -/// Earliest of the shipped boundaries that closes a heading-marked historical -/// block searched from `search_from`: the next `\n## ` heading, the managed -/// skill index, a current start sentinel, or EOF. -pub(crate) fn historical_heading_block_end( - contents: &str, - search_from: usize, - sentinels: OwnedBlockSentinels, -) -> usize { - let heading_end = heading_block_end(contents, search_from); - contents[search_from..] - .find(sentinels.start) - .map_or(heading_end, |offset| heading_end.min(search_from + offset)) -} - /// Managed-skill index marker prefix (the full marker carries a per-host /// suffix); strip heuristics stop here. const SKILL_INDEX_START_PREFIX: &str = ""; -const END: &str = ""; - -pub(crate) fn remove_state(profile_root: &Path) -> Result<()> { - for path in [ - profile_root.join("agent_managed/memory_digest.json"), - profile_root.join("agent_managed/memory_digest_targets.json"), - ] { - match std::fs::remove_file(&path) { - Ok(()) => {} - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} - Err(error) => return Err(error.into()), - } - } - Ok(()) -} - -pub(crate) fn remove_prompt_block(prompt_path: &Path) -> Result<()> { - let existing = match std::fs::read_to_string(prompt_path) { - Ok(contents) => contents, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), - Err(error) => return Err(error.into()), - }; - let starts = existing.match_indices(START).collect::>(); - let ends = existing.match_indices(END).collect::>(); - let (start, end) = match (starts.as_slice(), ends.as_slice()) { - ([], []) => return Ok(()), - ([(start, _)], [(end, _)]) if start <= end => (*start, *end + END.len()), - _ => { - return Err(TraceDecayError::Config { - message: format!( - "retired memory digest markers are ambiguous in {}", - prompt_path.display() - ), - }); - } - }; - let mut updated = existing[..start].trim_end().to_owned(); - let suffix = existing[end..].trim_start(); - if !updated.is_empty() && !suffix.is_empty() { - updated.push_str("\n\n"); - } - updated.push_str(suffix); - if !updated.is_empty() && !updated.ends_with('\n') { - updated.push('\n'); - } - if updated.trim().is_empty() { - std::fs::remove_file(prompt_path)?; - } else { - super::safe_write_text_file(prompt_path, &updated, None)?; - } - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn cleanup_preserves_operator_prompt_content() { - let dir = tempfile::tempdir().unwrap(); - let prompt = dir.path().join("AGENTS.md"); - std::fs::write( - &prompt, - format!("operator before\n\n{START}\nproject fact\n{END}\n\noperator after\n"), - ) - .unwrap(); - - remove_prompt_block(&prompt).unwrap(); - - assert_eq!( - std::fs::read_to_string(prompt).unwrap(), - "operator before\n\noperator after\n" - ); - } - - #[test] - fn ambiguous_markers_fail_without_writing() { - let dir = tempfile::tempdir().unwrap(); - let prompt = dir.path().join("AGENTS.md"); - let contents = format!("{START}\none\n{START}\ntwo\n{END}\n"); - std::fs::write(&prompt, &contents).unwrap(); - - let error = remove_prompt_block(&prompt).unwrap_err(); - - assert!(error.to_string().contains("ambiguous")); - assert_eq!(std::fs::read_to_string(prompt).unwrap(), contents); - } - - #[test] - fn state_cleanup_deletes_only_generated_state() { - let profile = tempfile::tempdir().unwrap(); - let managed = profile.path().join("agent_managed"); - std::fs::create_dir_all(&managed).unwrap(); - let snapshot = managed.join("memory_digest.json"); - let targets = managed.join("memory_digest_targets.json"); - let preserved = managed.join("operator.json"); - for path in [&snapshot, &targets, &preserved] { - std::fs::write(path, "generated").unwrap(); - } - - remove_state(profile.path()).unwrap(); - - assert!(!snapshot.exists()); - assert!(!targets.exists()); - assert!(preserved.exists()); - } -} diff --git a/crates/tracedecay-agent-hosts/src/agents/roo_code.rs b/crates/tracedecay-agent-hosts/src/agents/roo_code.rs index 3c1ac96774..801f93bb9f 100644 --- a/crates/tracedecay-agent-hosts/src/agents/roo_code.rs +++ b/crates/tracedecay-agent-hosts/src/agents/roo_code.rs @@ -13,9 +13,8 @@ use tracedecay_domain::errors::Result; use super::{ AgentIntegration, DoctorCounters, HealthcheckContext, InstallContext, JsonConfigDialect, - McpDoctorLabels, McpUninstallPolicy, config_backup_path, doctor_check_mcp_registration, - install_mcp_server_entry, load_json_file, mcp_servers_registration_state, - uninstall_mcp_server_entry, + McpDoctorLabels, McpUninstallPolicy, doctor_check_mcp_registration, install_mcp_server_entry, + load_json_file, mcp_servers_registration_state, uninstall_mcp_server_entry, }; pub struct RooCodeIntegration; @@ -67,8 +66,7 @@ impl AgentIntegration for RooCodeIntegration { home: &Path, ) -> Vec { if components == [super::host_bundle::HostComponentV1::ContextMcp] { - let path = roo_settings_path(home); - vec![path.clone(), config_backup_path(&path)] + vec![roo_settings_path(home)] } else { Vec::new() } diff --git a/crates/tracedecay-agent-hosts/src/agents/text_file_transaction.rs b/crates/tracedecay-agent-hosts/src/agents/text_file_transaction.rs index 68567e0ae2..1adcb5ac7b 100644 --- a/crates/tracedecay-agent-hosts/src/agents/text_file_transaction.rs +++ b/crates/tracedecay-agent-hosts/src/agents/text_file_transaction.rs @@ -140,28 +140,36 @@ pub(super) fn lock_host_file_write(path: &Path) -> Result { ), }); } - lock.lock().map_err(|error| TraceDecayError::Config { - message: format!("failed to lock host config {}: {error}", path.display()), - })?; let locked = Handle::from_file(lock).map_err(|error| TraceDecayError::Config { message: format!( "failed to identify host config lock {}: {error}", parent.join(&lock_name).display() ), })?; + locked + .as_file() + .lock() + .map_err(|error| TraceDecayError::Config { + message: format!("failed to lock host config {}: {error}", path.display()), + })?; // A prior holder may have unlinked the inode we waited on; the lock is // valid only while the directory entry still names the locked file. let current = match directory.open_with(&lock_name, &probe) { - Ok(file) => { - Handle::from_file(file.into_std()).map_err(|error| TraceDecayError::Config { - message: format!( - "failed to identify host config lock {}: {error}", - parent.join(&lock_name).display() - ), - })? + Ok(file) => Handle::from_file(file.into_std()).map(Some), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(error) => Err(error), + }; + match current { + Ok(Some(current)) if current == locked => { + return Ok(HostFileWriteLock { + directory, + lock_name, + handle: locked, + }); } - Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, + Ok(_) => release_abandoned_lock(&locked, &lock_name), Err(error) => { + release_abandoned_lock(&locked, &lock_name); return Err(TraceDecayError::Config { message: format!( "failed to inspect host config lock {}: {error}", @@ -169,13 +177,6 @@ pub(super) fn lock_host_file_write(path: &Path) -> Result { ), }); } - }; - if current == locked { - return Ok(HostFileWriteLock { - directory, - lock_name, - handle: locked, - }); } } Err(TraceDecayError::Config { @@ -186,6 +187,18 @@ pub(super) fn lock_host_file_write(path: &Path) -> Result { }) } +/// Closing does not release an `flock` while a forked child still shares the +/// descriptor, so a lock that is not kept must be unlocked explicitly. +fn release_abandoned_lock(locked: &Handle, lock_name: &str) { + if let Err(error) = locked.as_file().unlock() { + tracing::warn!( + lock_name = %lock_name, + error = %error, + "abandoned host config lock could not be released" + ); + } +} + #[cfg(unix)] #[derive(Clone, Debug, PartialEq, Eq)] struct HostFileObjectIdentity { @@ -379,14 +392,13 @@ fn verify_host_file_snapshot(path: &Path, expected: &HostFileSnapshot) -> std::i pub(super) fn write_bytes_file_locked( path: &Path, contents: &[u8], - backup: Option<&Path>, replacement_metadata: Option<&HostFileMetadataIdentityV1>, ) -> Result<()> { let _lock = lock_host_file_write(path)?; let observed = capture_host_file_snapshot(path).map_err(|error| TraceDecayError::Config { message: format!("failed to capture metadata for {}: {error}", path.display()), })?; - safe_write_bytes_file_from_snapshot(path, contents, backup, replacement_metadata, &observed) + safe_write_bytes_file_from_snapshot(path, contents, replacement_metadata, &observed) } pub(super) fn restore_bytes_file_if_unchanged( @@ -410,7 +422,7 @@ pub(super) fn restore_bytes_file_if_unchanged( ), }); } - safe_write_bytes_file_from_snapshot(path, original, None, Some(original_metadata), &observed) + safe_write_bytes_file_from_snapshot(path, original, Some(original_metadata), &observed) } pub(crate) enum TextFileMutation { @@ -419,34 +431,19 @@ pub(crate) enum TextFileMutation { Remove, } -/// Whether a mutating transaction leaves a `.bak` of the observed bytes. -#[derive(Clone, Copy)] -enum MutationBackup { - /// Prompt/rule files: publish without a recovery copy. - None, - /// Structured host configs: every rewrite or removal of an existing file - /// leaves a `.bak` the operator can restore (issue #63). - BackupExisting, -} - /// Run a strict UTF-8 read-transform-mutate while holding the host-file lock. pub(crate) fn update_text_file_transactionally( path: &Path, update: impl FnOnce(&str) -> Result<(T, TextFileMutation)>, ) -> Result { - update_file_transactionally(path, MutationBackup::None, update) -} - -/// [`update_text_file_transactionally`] for structured host configs -/// (JSON/JSONC/TOML): identical read-under-lock → transform → -/// publish-from-snapshot shape, except that rewriting or removing an existing -/// file first leaves a `.bak` recovery copy whose path is threaded into the -/// publish error hint. -pub(crate) fn update_config_file_transactionally( - path: &Path, - update: impl FnOnce(&str) -> Result<(T, TextFileMutation)>, -) -> Result { - update_file_transactionally(path, MutationBackup::BackupExisting, update) + let _lock = lock_host_file_write(path)?; + let observed = capture_host_file_snapshot(path).map_err(|error| TraceDecayError::Config { + message: format!("failed to read {}: {error}", path.display()), + })?; + let existing = snapshot_utf8(path, &observed)?; + let (output, mutation) = update(existing)?; + apply_file_mutation(path, &observed, &mutation)?; + Ok(output) } /// Mutate two structured host documents as one rollback boundary. @@ -486,21 +483,10 @@ pub(crate) fn update_two_config_files_transactionally( let first_existing = snapshot_utf8(first_path, &first_snapshot)?; let second_existing = snapshot_utf8(second_path, &second_snapshot)?; let (output, first_mutation, second_mutation) = update(first_existing, second_existing)?; - let first_backup = mutation_backup(first_path, &first_snapshot, &first_mutation)?; - let second_backup = mutation_backup(second_path, &second_snapshot, &second_mutation)?; let expected_first = mutation_result_bytes(&first_snapshot, &first_mutation); - apply_file_mutation( - first_path, - &first_snapshot, - &first_mutation, - first_backup.as_deref(), - )?; - if let Err(second_error) = apply_file_mutation( - second_path, - &second_snapshot, - &second_mutation, - second_backup.as_deref(), - ) { + apply_file_mutation(first_path, &first_snapshot, &first_mutation)?; + if let Err(second_error) = apply_file_mutation(second_path, &second_snapshot, &second_mutation) + { return match restore_group_snapshot(first_path, &first_snapshot, expected_first.as_deref()) { Ok(()) => Err(second_error), @@ -524,20 +510,6 @@ fn snapshot_utf8<'a>(path: &Path, snapshot: &'a HostFileSnapshot) -> Result<&'a } } -fn mutation_backup( - path: &Path, - snapshot: &HostFileSnapshot, - mutation: &TextFileMutation, -) -> Result> { - if matches!(mutation, TextFileMutation::Unchanged) - || matches!(snapshot, HostFileSnapshot::Missing) - { - Ok(None) - } else { - super::backup_config_file(path) - } -} - fn mutation_result_bytes( snapshot: &HostFileSnapshot, mutation: &TextFileMutation, @@ -553,17 +525,12 @@ fn apply_file_mutation( path: &Path, snapshot: &HostFileSnapshot, mutation: &TextFileMutation, - backup: Option<&Path>, ) -> Result<()> { match mutation { TextFileMutation::Unchanged => Ok(()), - TextFileMutation::Write(replacement) => safe_write_bytes_file_from_snapshot( - path, - replacement.as_bytes(), - backup, - None, - snapshot, - ), + TextFileMutation::Write(replacement) => { + safe_write_bytes_file_from_snapshot(path, replacement.as_bytes(), None, snapshot) + } TextFileMutation::Remove => remove_host_file_from_snapshot(path, snapshot), } } @@ -591,52 +558,8 @@ fn restore_group_snapshot( HostFileSnapshot::Missing => remove_host_file_from_snapshot(path, ¤t), HostFileSnapshot::Present { contents, metadata, .. - } => safe_write_bytes_file_from_snapshot(path, contents, None, Some(metadata), ¤t), - } -} - -fn update_file_transactionally( - path: &Path, - backup: MutationBackup, - update: impl FnOnce(&str) -> Result<(T, TextFileMutation)>, -) -> Result { - let _lock = lock_host_file_write(path)?; - let observed = capture_host_file_snapshot(path).map_err(|error| TraceDecayError::Config { - message: format!("failed to read {}: {error}", path.display()), - })?; - let existing = match observed.contents() { - Some(contents) => { - std::str::from_utf8(contents).map_err(|error| TraceDecayError::Config { - message: format!("failed to read {} as UTF-8: {error}", path.display()), - })? - } - None => "", - }; - let (output, mutation) = update(existing)?; - let backup = match (&mutation, backup, &observed) { - (TextFileMutation::Unchanged, _, _) - | (_, MutationBackup::None, _) - | (_, MutationBackup::BackupExisting, HostFileSnapshot::Missing) => None, - (_, MutationBackup::BackupExisting, HostFileSnapshot::Present { .. }) => { - super::backup_config_file(path)? - } - }; - match mutation { - TextFileMutation::Unchanged => {} - TextFileMutation::Write(replacement) => { - safe_write_bytes_file_from_snapshot( - path, - replacement.as_bytes(), - backup.as_deref(), - None, - &observed, - )?; - } - TextFileMutation::Remove => { - remove_host_file_from_snapshot(path, &observed)?; - } + } => safe_write_bytes_file_from_snapshot(path, contents, Some(metadata), ¤t), } - Ok(output) } fn remove_host_file_from_snapshot(path: &Path, observed: &HostFileSnapshot) -> Result<()> { @@ -667,7 +590,6 @@ fn remove_host_file_from_snapshot(path: &Path, observed: &HostFileSnapshot) -> R fn safe_write_bytes_file_from_snapshot( path: &Path, contents: &[u8], - backup: Option<&Path>, replacement_metadata: Option<&HostFileMetadataIdentityV1>, observed: &HostFileSnapshot, ) -> Result<()> { @@ -736,17 +658,11 @@ fn safe_write_bytes_file_from_snapshot( }, tracedecay_private_fs::framed_log::DirectorySyncPolicy::TolerateUnsupported, ) { - let hint = if let Some(b) = backup { - format!( - "\n Backup is at: {}\n \ - The original file was NOT modified.", - b.display() - ) - } else { - "\n The original file was NOT modified.".to_string() - }; return Err(TraceDecayError::Config { - message: format!("failed to atomically replace {}: {e}{hint}", path.display()), + message: format!( + "failed to atomically replace {}: {e}\n The original file was NOT modified.", + path.display() + ), }); } #[cfg(feature = "test-transport")] diff --git a/crates/tracedecay-agent-hosts/src/agents/vibe.rs b/crates/tracedecay-agent-hosts/src/agents/vibe.rs index d9011f2ae8..fa9a41b126 100644 --- a/crates/tracedecay-agent-hosts/src/agents/vibe.rs +++ b/crates/tracedecay-agent-hosts/src/agents/vibe.rs @@ -31,7 +31,7 @@ use super::host_bundle::{HostBundleRegistrationStateV1, HostComponentV1}; use super::prompt_rules::{PROMPT_RULE_MARKER, PromptRulesOptions}; use super::{ AgentIntegration, DoctorCounters, HealthcheckContext, InstallContext, TextFileMutation, - config_backup_path, update_config_file_transactionally, + update_text_file_transactionally, }; pub struct VibeIntegration; @@ -54,10 +54,6 @@ fn project_vibe_home(project: &Path) -> PathBuf { project.join(".vibe") } -fn original_config_path(config: &Path) -> PathBuf { - PathBuf::from(format!("{}.tracedecay-original", config.display())) -} - /// Whether one Vibe home (user-level or project-level) carries a live /// tracedecay registration in either of its two documents. fn vibe_home_has_tracedecay(vibe_home: &Path) -> bool { @@ -190,7 +186,6 @@ impl AgentIntegration for VibeIntegration { components, &vibe_config_path(&ctx.home), &vibe_prompt_path(&ctx.home), - &ctx.home, ) } @@ -203,29 +198,21 @@ impl AgentIntegration for VibeIntegration { let root = project_vibe_home(project_path); let config = root.join("config.toml"); let prompt = root.join("prompts/cli.md"); - let original = original_config_path(&config); - super::ensure_project_local_safe_paths( - project_path, - [config.as_path(), prompt.as_path(), original.as_path()], - )?; + super::ensure_project_local_safe_paths(project_path, [config.as_path(), prompt.as_path()])?; activate_components(components, &config, &prompt, ctx) } fn deactivate_project_host_component_registration( &self, components: &[HostComponentV1], - ctx: &InstallContext, + _ctx: &InstallContext, project_path: &Path, ) -> Result<()> { let root = project_vibe_home(project_path); let config = root.join("config.toml"); let prompt = root.join("prompts/cli.md"); - let original = original_config_path(&config); - super::ensure_project_local_safe_paths( - project_path, - [config.as_path(), prompt.as_path(), original.as_path()], - )?; - deactivate_components(components, &config, &prompt, &ctx.home) + super::ensure_project_local_safe_paths(project_path, [config.as_path(), prompt.as_path()])?; + deactivate_components(components, &config, &prompt) } fn reports_absence_to_doctor(&self) -> bool { @@ -279,8 +266,6 @@ fn registration_paths( let mut paths = Vec::new(); if components.contains(&HostComponentV1::ContextMcp) { paths.push(config.to_path_buf()); - paths.push(config_backup_path(config)); - paths.push(original_config_path(config)); } if components.contains(&HostComponentV1::Core) { paths.push(prompt.to_path_buf()); @@ -392,13 +377,8 @@ fn doctor_check_registration(dc: &mut DoctorCounters, config: &Path, prompt: &Pa } fn install_mcp(config: &Path, binary: &str) -> Result<()> { - let original = original_config_path(config); - update_config_file_transactionally(config, |existing| { + update_text_file_transactionally(config, |existing| { let mut document = parse_document(config, existing)?; - let had_registration = tracedecay_server(&document).is_some(); - if !had_registration && config.is_file() && !original.exists() { - super::safe_write_bytes_file(&original, existing.as_bytes(), None)?; - } let servers = document .entry("mcp_servers") .or_insert(Item::ArrayOfTables(ArrayOfTables::new())) @@ -429,63 +409,33 @@ fn install_mcp(config: &Path, binary: &str) -> Result<()> { }) } -#[derive(Clone, Copy)] -enum McpRemoval { - NoEntry, - RestoredOriginal, - RemovedFile, - Rewritten, -} - fn uninstall_mcp(config: &Path) -> Result<()> { if !config.exists() { return Ok(()); } - let original = original_config_path(config); - let outcome = update_config_file_transactionally(config, |existing| { + update_text_file_transactionally(config, |existing| { let mut document = parse_document(config, existing)?; let Some(servers) = document .get_mut("mcp_servers") .and_then(Item::as_array_of_tables_mut) else { - return Ok((McpRemoval::NoEntry, TextFileMutation::Unchanged)); + return Ok(((), TextFileMutation::Unchanged)); }; let Some(index) = servers .iter() .position(|server| server.get("name").and_then(Item::as_str) == Some("tracedecay")) else { - return Ok((McpRemoval::NoEntry, TextFileMutation::Unchanged)); + return Ok(((), TextFileMutation::Unchanged)); }; servers.remove(index); if servers.is_empty() { document.remove("mcp_servers"); } - if let Ok(bytes) = std::fs::read(&original) - && toml::from_slice::(&bytes).ok() - == toml::from_str::(&document.to_string()).ok() - { - let original = String::from_utf8(bytes).map_err(|error| TraceDecayError::Config { - message: format!("{} is not valid UTF-8: {error}", original.display()), - })?; - return Ok(( - McpRemoval::RestoredOriginal, - TextFileMutation::Write(original), - )); - } if document.is_empty() { - return Ok((McpRemoval::RemovedFile, TextFileMutation::Remove)); + return Ok(((), TextFileMutation::Remove)); } - Ok(( - McpRemoval::Rewritten, - TextFileMutation::Write(document.to_string()), - )) - })?; - if matches!(outcome, McpRemoval::RestoredOriginal) { - super::safe_remove_host_file(&original).map_err(|error| TraceDecayError::Config { - message: format!("failed to remove {}: {error}", original.display()), - })?; - } - Ok(()) + Ok(((), TextFileMutation::Write(document.to_string()))) + }) } fn install_prompt(prompt: &Path, profile_home: &Path) -> Result<()> { @@ -499,8 +449,8 @@ fn install_prompt(prompt: &Path, profile_home: &Path) -> Result<()> { super::install_managed_skill_prompt_index(profile_home, prompt, SkillInstallTarget::Agents) } -fn uninstall_prompt(prompt: &Path, profile_home: &Path) -> Result<()> { - super::remove_managed_skill_prompt_index(profile_home, prompt, SkillInstallTarget::Agents)?; +fn uninstall_prompt(prompt: &Path) -> Result<()> { + super::remove_managed_skill_prompt_index(prompt, SkillInstallTarget::Agents)?; super::prompt_rules::remove_standard_prompt_rules(prompt) } @@ -523,10 +473,9 @@ fn deactivate_components( components: &[HostComponentV1], config: &Path, prompt: &Path, - profile_home: &Path, ) -> Result<()> { if components.contains(&HostComponentV1::Core) { - uninstall_prompt(prompt, profile_home)?; + uninstall_prompt(prompt)?; } if components.contains(&HostComponentV1::ContextMcp) { uninstall_mcp(config)?; @@ -542,14 +491,13 @@ mod tests { InstallContext { home: home.to_path_buf(), tracedecay_bin: binary.to_string(), - tool_permissions: Vec::new(), project_root: None, dashboard: false, } } #[test] - fn vibe_mcp_lifecycle_preserves_foreign_servers_and_restores_original_bytes() { + fn vibe_mcp_lifecycle_preserves_foreign_servers_and_keeps_no_copy() { let home = tempfile::tempdir().unwrap(); let config = vibe_config_path(home.path()); std::fs::create_dir_all(config.parent().unwrap()).unwrap(); @@ -578,7 +526,15 @@ mod tests { ) .unwrap(); - assert_eq!(std::fs::read(&config).unwrap(), original); + assert_eq!( + toml::from_str::(&std::fs::read_to_string(&config).unwrap()).unwrap(), + toml::from_slice::(original).unwrap() + ); + let siblings: Vec<_> = std::fs::read_dir(config.parent().unwrap()) + .unwrap() + .map(|entry| entry.unwrap().file_name()) + .collect(); + assert_eq!(siblings, vec![std::ffi::OsString::from("config.toml")]); } #[test] @@ -595,7 +551,7 @@ mod tests { ); assert!(!config.exists()); - deactivate_components(&[HostComponentV1::Core], &config, &prompt, home.path()).unwrap(); + deactivate_components(&[HostComponentV1::Core], &config, &prompt).unwrap(); activate_components(&[HostComponentV1::ContextMcp], &config, &prompt, &install).unwrap(); assert_eq!( mcp_registration_state(&config, Some("/tmp/tracedecay")), @@ -613,7 +569,7 @@ mod tests { let components = [HostComponentV1::ContextMcp, HostComponentV1::Core]; activate_components(&components, &config, &prompt, &install).unwrap(); - deactivate_components(&components, &config, &prompt, home.path()).unwrap(); + deactivate_components(&components, &config, &prompt).unwrap(); assert!(!config.exists()); assert!(!prompt.exists()); diff --git a/crates/tracedecay-agent-hosts/src/agents/zed.rs b/crates/tracedecay-agent-hosts/src/agents/zed.rs index d040933853..a0becc9c03 100644 --- a/crates/tracedecay-agent-hosts/src/agents/zed.rs +++ b/crates/tracedecay-agent-hosts/src/agents/zed.rs @@ -9,9 +9,7 @@ //! that capability is an open feature request, not an implemented one, and //! extensions are installed through the Command Palette and the Agent Panel. //! There is nothing to drive, so the settings merge below is the only route. -//! Zed settings are JSONC, whose comments cannot survive a serde round-trip; -//! the byte-exact `.tracedecay-original` snapshot is therefore the peer-safety -//! authority and is restored when no later foreign edit prevents it. +//! Zed settings are JSONC, whose comments cannot survive a serde round-trip. //! See . use std::path::{Path, PathBuf}; @@ -23,8 +21,8 @@ use tracedecay_domain::errors::{Result, TraceDecayError}; use super::host_bundle::{HostBundleRegistrationStateV1, HostComponentV1}; use super::{ AgentIntegration, DoctorCounters, HealthcheckContext, InstallContext, JsonConfigDialect, - McpDoctorLabels, TextFileMutation, config_backup_path, load_jsonc_file, - report_mcp_registration, update_config_file_transactionally, + McpDoctorLabels, TextFileMutation, load_jsonc_file, report_mcp_registration, + update_text_file_transactionally, }; pub struct ZedIntegration; @@ -113,7 +111,7 @@ impl AgentIntegration for ZedIntegration { if components != [HostComponentV1::ContextMcp] { return Vec::new(); } - zed_registration_paths(&zed_settings_path(home)) + vec![zed_settings_path(home)] } fn project_host_component_registration_paths( @@ -125,9 +123,7 @@ impl AgentIntegration for ZedIntegration { if components != [HostComponentV1::ContextMcp] { return Ok(Vec::new()); } - Ok(zed_registration_paths(&zed_project_settings_path( - project_path, - ))) + Ok(vec![zed_project_settings_path(project_path)]) } #[hotpath::measure(label = "zed_mcp_install")] @@ -154,8 +150,7 @@ impl AgentIntegration for ZedIntegration { project_path: &Path, ) -> Result<()> { let path = zed_project_settings_path(project_path); - let original = zed_original_config_path(&path); - super::ensure_project_local_safe_paths(project_path, [path.as_path(), original.as_path()])?; + super::ensure_project_local_safe_path(project_path, &path)?; install_mcp_if_selected(components, &path, ctx) } @@ -166,8 +161,7 @@ impl AgentIntegration for ZedIntegration { project_path: &Path, ) -> Result<()> { let path = zed_project_settings_path(project_path); - let original = zed_original_config_path(&path); - super::ensure_project_local_safe_paths(project_path, [path.as_path(), original.as_path()])?; + super::ensure_project_local_safe_path(project_path, &path)?; uninstall_mcp_if_selected(components, &path) } @@ -192,18 +186,6 @@ fn zed_project_settings_path(project: &Path) -> PathBuf { project.join(".zed/settings.json") } -fn zed_original_config_path(config: &Path) -> PathBuf { - PathBuf::from(format!("{}.tracedecay-original", config.display())) -} - -fn zed_registration_paths(config: &Path) -> Vec { - vec![ - config.to_path_buf(), - config_backup_path(config), - zed_original_config_path(config), - ] -} - fn zed_mcp_registration_state( config: &Path, expected_binary: Option<&str>, @@ -275,8 +257,7 @@ fn install_mcp_if_selected( if !components.contains(&HostComponentV1::ContextMcp) { return Ok(()); } - let original = zed_original_config_path(config); - update_config_file_transactionally(config, |existing| { + update_text_file_transactionally(config, |existing| { let mut settings = JsonConfigDialect::Jsonc.parse_for_edit(config, existing)?; let root = settings .as_object_mut() @@ -290,9 +271,6 @@ fn install_mcp_if_selected( .ok_or_else(|| TraceDecayError::Config { message: format!("{}.context_servers must be a JSON object", config.display()), })?; - if !servers.contains_key("tracedecay") && config.is_file() && !original.exists() { - super::safe_write_bytes_file(&original, existing.as_bytes(), None)?; - } servers.insert( "tracedecay".to_string(), json!({ @@ -302,26 +280,19 @@ fn install_mcp_if_selected( ); Ok(( (), - TextFileMutation::Write(super::render_json_config(config, &settings)?), + TextFileMutation::Write( + JsonConfigDialect::Jsonc.render_edit(config, existing, &settings)?, + ), )) })?; Ok(()) } -#[derive(Clone, Copy)] -enum ZedMcpRemoval { - NoEntry, - RestoredOriginal, - RemovedFile, - Rewritten, -} - fn uninstall_mcp_if_selected(components: &[HostComponentV1], config: &Path) -> Result<()> { if !components.contains(&HostComponentV1::ContextMcp) || !config.exists() { return Ok(()); } - let original = zed_original_config_path(config); - let outcome = update_config_file_transactionally(config, |existing| { + update_text_file_transactionally(config, |existing| { let mut settings = JsonConfigDialect::Jsonc.parse_for_edit(config, existing)?; let Some(root) = settings.as_object_mut() else { return Err(TraceDecayError::Config { @@ -332,44 +303,24 @@ fn uninstall_mcp_if_selected(components: &[HostComponentV1], config: &Path) -> R .get_mut("context_servers") .and_then(serde_json::Value::as_object_mut) else { - return Ok((ZedMcpRemoval::NoEntry, TextFileMutation::Unchanged)); + return Ok(((), TextFileMutation::Unchanged)); }; if servers.remove("tracedecay").is_none() { - return Ok((ZedMcpRemoval::NoEntry, TextFileMutation::Unchanged)); + return Ok(((), TextFileMutation::Unchanged)); } if servers.is_empty() { root.remove("context_servers"); } - let root_is_empty = root.is_empty(); - if let Ok(bytes) = std::fs::read(&original) - && serde_json::from_slice::( - super::strip_jsonc_comments(&String::from_utf8_lossy(&bytes)).as_bytes(), - ) - .ok() - == Some(settings.clone()) - { - let bytes = String::from_utf8(bytes).map_err(|error| TraceDecayError::Config { - message: format!("{} is not valid UTF-8: {error}", original.display()), - })?; - return Ok(( - ZedMcpRemoval::RestoredOriginal, - TextFileMutation::Write(bytes), - )); - } - if root_is_empty { - return Ok((ZedMcpRemoval::RemovedFile, TextFileMutation::Remove)); + if root.is_empty() { + return Ok(((), TextFileMutation::Remove)); } Ok(( - ZedMcpRemoval::Rewritten, - TextFileMutation::Write(super::render_json_config(config, &settings)?), + (), + TextFileMutation::Write( + JsonConfigDialect::Jsonc.render_edit(config, existing, &settings)?, + ), )) - })?; - if matches!(outcome, ZedMcpRemoval::RestoredOriginal) { - super::safe_remove_host_file(&original).map_err(|error| TraceDecayError::Config { - message: format!("failed to remove {}: {error}", original.display()), - })?; - } - Ok(()) + }) } #[cfg(test)] @@ -381,23 +332,22 @@ mod tests { InstallContext { home: home.to_path_buf(), tracedecay_bin: binary.to_string(), - tool_permissions: Vec::new(), project_root: None, dashboard: false, } } #[test] - fn zed_lifecycle_preserves_jsonc_peers_and_restores_original_bytes() { + fn zed_lifecycle_restores_operator_bytes_and_keeps_no_copy() { let home = tempfile::tempdir().unwrap(); let config = zed_settings_path(home.path()); std::fs::create_dir_all(config.parent().unwrap()).unwrap(); let original = br#"{ - // operator comment cannot survive serde rendering - "context_servers": { - "foreign": {"command": "foreign-mcp"} - }, - "theme": "dark" + // operator comment + "theme": "dark", /* inline */ + "context_servers": { + "foreign": {"command": "foreign-mcp",}, + }, } "#; std::fs::write(&config, original).unwrap(); @@ -408,6 +358,12 @@ mod tests { .activate_deployed_host_component_registration(&components, &install) .unwrap(); + let installed_text = std::fs::read_to_string(&config).unwrap(); + assert!( + installed_text + .starts_with("{\n // operator comment\n \"theme\": \"dark\", /* inline */\n"), + "{installed_text}" + ); let installed = load_jsonc_file(&config); assert_eq!( installed["context_servers"]["foreign"]["command"], @@ -422,17 +378,17 @@ mod tests { zed_mcp_registration_state(&config, Some("/tmp/tracedecay")), HostBundleRegistrationStateV1::Current ); - assert_eq!( - std::fs::read(zed_original_config_path(&config)).unwrap(), - original - ); ZedIntegration .deactivate_deployed_host_component_registration(&components, &install) .unwrap(); assert_eq!(std::fs::read(&config).unwrap(), original); - assert!(!zed_original_config_path(&config).exists()); + let siblings: Vec<_> = std::fs::read_dir(config.parent().unwrap()) + .unwrap() + .map(|entry| entry.unwrap().file_name()) + .collect(); + assert_eq!(siblings, vec![std::ffi::OsString::from("settings.json")]); } #[test] @@ -452,7 +408,6 @@ mod tests { .unwrap(); assert!(!config.exists()); - assert!(!zed_original_config_path(&config).exists()); } #[test] diff --git a/crates/tracedecay-agent-hosts/src/hooks/analytics.rs b/crates/tracedecay-agent-hosts/src/hooks/analytics.rs index dc8a3f657a..46e63924ac 100644 --- a/crates/tracedecay-agent-hosts/src/hooks/analytics.rs +++ b/crates/tracedecay-agent-hosts/src/hooks/analytics.rs @@ -14,8 +14,9 @@ use tracedecay_sessions::admission::{ HostAdmissionTelemetryDisposition as HookDispositionTelemetry, }; -use super::tool_hints::{HintAgent, ToolHint}; +use super::tool_hints::ToolHint; use super::{HookWorkspaceStatus, claude, prompt_like_text}; +use tracedecay_domain::HostIntegrationIdV1; pub(crate) const HOOK_ANALYTICS_FILENAME: &str = "hook_analytics.jsonl"; @@ -111,7 +112,7 @@ impl HookTimingSpan { fn new( runtime: &HookRuntimeV1, root: Option<&Path>, - agent: HintAgent, + agent: HostIntegrationIdV1, hook_name: &str, prompt_category: Option<&'static str>, payload_bytes: Option, @@ -484,7 +485,7 @@ fn disposition_from_daemon_error(error: &TraceDecayError) -> HookDispositionTele /// Shared implementation for [`record_hook_invoked`] and /// [`record_other_hook_invoked`], which differ only in how the analytics -/// `agent` key is derived (a typed [`HintAgent`] vs. the literal `"other"`). +/// `agent` key is derived (a typed [`HostIntegrationIdV1`] vs. the literal `"other"`). fn record_hook_invoked_named( runtime: &HookRuntimeV1, root: Option<&Path>, @@ -522,7 +523,7 @@ fn record_hook_invoked_named( pub(crate) fn record_hook_invoked( runtime: &HookRuntimeV1, root: Option<&Path>, - agent: HintAgent, + agent: HostIntegrationIdV1, hook_name: &str, event_json: &str, ) -> HookTimingSpan { @@ -543,7 +544,7 @@ pub(crate) fn record_hook_invoked( pub(crate) fn record_hook_invoked_parsed( runtime: &HookRuntimeV1, root: Option<&Path>, - agent: HintAgent, + agent: HostIntegrationIdV1, hook_name: &str, event_json: &str, parsed: &Value, @@ -575,7 +576,7 @@ pub(super) fn mint_hint_id() -> String { pub(super) fn record_hint_analytics( root: Option<&Path>, event: &str, - agent: HintAgent, + agent: HostIntegrationIdV1, session_id: Option<&str>, hint_id: &str, hint: &ToolHint, @@ -601,7 +602,7 @@ pub(super) fn record_workspace_status_analytics( root, "workspace_status", serde_json::json!({ - "agent": HintAgent::Codex.as_key(), + "agent": HostIntegrationIdV1::Codex.as_key(), "session_id": session_id, "workspace_status": status.as_key(), }), @@ -610,7 +611,7 @@ pub(super) fn record_workspace_status_analytics( pub(super) fn record_hint_emitted( root: Option<&Path>, - agent: HintAgent, + agent: HostIntegrationIdV1, session_id: Option<&str>, hint_id: &str, hint: &ToolHint, diff --git a/crates/tracedecay-agent-hosts/src/hooks/analytics/tests.rs b/crates/tracedecay-agent-hosts/src/hooks/analytics/tests.rs index 7c12399f07..9377b38045 100644 --- a/crates/tracedecay-agent-hosts/src/hooks/analytics/tests.rs +++ b/crates/tracedecay-agent-hosts/src/hooks/analytics/tests.rs @@ -35,7 +35,7 @@ fn unbound_hook_analytics_do_not_create_a_missing_profile() { drop(record_hook_invoked_parsed( &crate::ports::hook_runtime::crate_test_runtime(), None, - HintAgent::Claude, + HostIntegrationIdV1::Claude, "Stop", event, &parsed, @@ -114,7 +114,7 @@ fn timing_span_defaults_to_recording_without_a_registered_authority() { let span = HookTimingSpan::new( &crate::ports::hook_runtime::crate_test_runtime(), Some(&project_root), - HintAgent::Claude, + HostIntegrationIdV1::Claude, "missingConfiguration", None, None, @@ -154,7 +154,7 @@ fn payload_bytes_are_length_only_and_omit_forbidden_content() { let span = record_hook_invoked( &crate::ports::hook_runtime::crate_test_runtime(), Some(&project_root), - HintAgent::Claude, + HostIntegrationIdV1::Claude, "Stop", &event, ); @@ -239,7 +239,7 @@ fn daemon_hook_action_records_completed_rtt_and_wire_length() { let span = record_hook_invoked( &crate::ports::hook_runtime::crate_test_runtime(), Some(&project_root), - HintAgent::Cursor, + HostIntegrationIdV1::Cursor, "daemonBoundary", r#"{"hook_event_name":"daemonBoundary"}"#, ); @@ -280,7 +280,7 @@ fn one_way_notification_does_not_claim_round_trip_time() { let span = record_hook_invoked( &crate::ports::hook_runtime::crate_test_runtime(), Some(&project_root), - HintAgent::Cursor, + HostIntegrationIdV1::Cursor, "notificationBoundary", r#"{"hook_event_name":"notificationBoundary"}"#, ); @@ -333,7 +333,7 @@ fn hook_disposition_aggregation_preserves_failures_and_sticky_timeout() { let span = record_hook_invoked( &crate::ports::hook_runtime::crate_test_runtime(), Some(&project_root), - HintAgent::Claude, + HostIntegrationIdV1::Claude, "failureThenSuccess", "{}", ); @@ -344,7 +344,7 @@ fn hook_disposition_aggregation_preserves_failures_and_sticky_timeout() { let span = record_hook_invoked( &crate::ports::hook_runtime::crate_test_runtime(), Some(&project_root), - HintAgent::Codex, + HostIntegrationIdV1::Codex, "successThenFailure", "{}", ); @@ -355,7 +355,7 @@ fn hook_disposition_aggregation_preserves_failures_and_sticky_timeout() { let span = record_hook_invoked( &crate::ports::hook_runtime::crate_test_runtime(), Some(&project_root), - HintAgent::Kiro, + HostIntegrationIdV1::Kiro, "backpressureThenSuccess", "{}", ); @@ -366,7 +366,7 @@ fn hook_disposition_aggregation_preserves_failures_and_sticky_timeout() { let span = record_hook_invoked( &crate::ports::hook_runtime::crate_test_runtime(), Some(&project_root), - HintAgent::Cursor, + HostIntegrationIdV1::Cursor, "stickyTimeout", "{}", ); @@ -435,7 +435,7 @@ fn hook_disposition_order_permutations_unknown_typed_timeout_cancel() { let span = record_hook_invoked( &crate::ports::hook_runtime::crate_test_runtime(), Some(&project_root), - HintAgent::Claude, + HostIntegrationIdV1::Claude, "unknownThenSuccess", "{}", ); @@ -446,7 +446,7 @@ fn hook_disposition_order_permutations_unknown_typed_timeout_cancel() { let span = record_hook_invoked( &crate::ports::hook_runtime::crate_test_runtime(), Some(&project_root), - HintAgent::Codex, + HostIntegrationIdV1::Codex, "unknownThenFailure", "{}", ); @@ -457,7 +457,7 @@ fn hook_disposition_order_permutations_unknown_typed_timeout_cancel() { let span = record_hook_invoked( &crate::ports::hook_runtime::crate_test_runtime(), Some(&project_root), - HintAgent::Kiro, + HostIntegrationIdV1::Kiro, "successThenUnknown", "{}", ); @@ -468,7 +468,7 @@ fn hook_disposition_order_permutations_unknown_typed_timeout_cancel() { let span = record_hook_invoked( &crate::ports::hook_runtime::crate_test_runtime(), Some(&project_root), - HintAgent::Cursor, + HostIntegrationIdV1::Cursor, "failureThenUnknown", "{}", ); @@ -479,7 +479,7 @@ fn hook_disposition_order_permutations_unknown_typed_timeout_cancel() { let span = record_hook_invoked( &crate::ports::hook_runtime::crate_test_runtime(), Some(&project_root), - HintAgent::Claude, + HostIntegrationIdV1::Claude, "unknownThenTimeout", "{}", ); @@ -490,7 +490,7 @@ fn hook_disposition_order_permutations_unknown_typed_timeout_cancel() { let span = record_hook_invoked( &crate::ports::hook_runtime::crate_test_runtime(), Some(&project_root), - HintAgent::Codex, + HostIntegrationIdV1::Codex, "timeoutThenUnknown", "{}", ); @@ -501,7 +501,7 @@ fn hook_disposition_order_permutations_unknown_typed_timeout_cancel() { let span = record_hook_invoked( &crate::ports::hook_runtime::crate_test_runtime(), Some(&project_root), - HintAgent::Kiro, + HostIntegrationIdV1::Kiro, "unknownThenCancel", "{}", ); @@ -512,7 +512,7 @@ fn hook_disposition_order_permutations_unknown_typed_timeout_cancel() { let span = record_hook_invoked( &crate::ports::hook_runtime::crate_test_runtime(), Some(&project_root), - HintAgent::Cursor, + HostIntegrationIdV1::Cursor, "cancelThenUnknown", "{}", ); @@ -591,14 +591,14 @@ fn concurrent_spans_keep_rtt_payload_and_disposition_isolated() { let first = record_hook_invoked( &crate::ports::hook_runtime::crate_test_runtime(), Some(&project_root), - HintAgent::Cursor, + HostIntegrationIdV1::Cursor, "firstHook", r#"{"hook_event_name":"firstHook"}"#, ); let second = record_hook_invoked( &crate::ports::hook_runtime::crate_test_runtime(), Some(&project_root), - HintAgent::Kiro, + HostIntegrationIdV1::Kiro, "secondHook", r#"{"hook_event_name":"secondHook"}"#, ); @@ -664,7 +664,7 @@ fn untyped_ok_daemon_output_emits_unknown_not_default_success() { let span = record_hook_invoked( &crate::ports::hook_runtime::crate_test_runtime(), Some(&project_root), - HintAgent::Claude, + HostIntegrationIdV1::Claude, "untypedOk", "{}", ); diff --git a/crates/tracedecay-agent-hosts/src/hooks/claude.rs b/crates/tracedecay-agent-hosts/src/hooks/claude.rs index 6b144df4a3..2737a633b9 100644 --- a/crates/tracedecay-agent-hosts/src/hooks/claude.rs +++ b/crates/tracedecay-agent-hosts/src/hooks/claude.rs @@ -10,19 +10,20 @@ use serde_json::Value; use crate::ports::hook_runtime::HookRuntimeV1; use super::post_tool_use::is_post_tool_use_failure_event; -use super::tool_hints::{HintAgent, ToolHintInput, decide_hint}; +use super::tool_hints::{ToolHintInput, decide_hint}; use super::{ additional_context_json, compact_daemon_args, event_project_root_with_identity, event_session_id, prompt_like_text, read_hook_event, record_hook_invoked_parsed, research_block_reason, }; +use tracedecay_domain::HostIntegrationIdV1; /// Pure decision logic for the `PreToolUse` hook. pub fn evaluate_hook_decision(tool_input: &str) -> String { let parsed: serde_json::Value = serde_json::from_str(tool_input).unwrap_or_else(|_| serde_json::json!({})); let hint = decide_hint(&ToolHintInput { - agent: HintAgent::Claude, + agent: HostIntegrationIdV1::Claude, session_id: event_session_id(&parsed), tool_name: Some("Agent".to_string()), command: None, @@ -104,7 +105,7 @@ pub async fn hook_claude_post_compact(runtime: &HookRuntimeV1) -> i32 { let hook_telemetry = record_hook_invoked_parsed( runtime, root.as_deref(), - HintAgent::Claude, + HostIntegrationIdV1::Claude, "PostCompact", &event, &parsed, @@ -117,7 +118,7 @@ pub async fn hook_claude_post_compact(runtime: &HookRuntimeV1) -> i32 { } if !super::write_hook_output( root.as_deref(), - tracedecay_hooks::HookHostV1::ClaudeCode, + tracedecay_domain::NativeHostIdentityV1::ClaudeCode, &event, &serde_json::json!({}).to_string(), ) @@ -137,7 +138,7 @@ pub async fn hook_claude_post_tool_use(runtime: &HookRuntimeV1) -> i32 { if let Some(response) = response && !super::write_hook_output( root.as_deref(), - tracedecay_hooks::HookHostV1::ClaudeCode, + tracedecay_domain::NativeHostIdentityV1::ClaudeCode, &event, &response, ) @@ -165,14 +166,14 @@ async fn claude_post_tool_use_response( let hook_telemetry = record_hook_invoked_parsed( runtime, root.as_deref(), - HintAgent::Claude, + HostIntegrationIdV1::Claude, hook_event_name, event, &parsed, ); let response = super::dispatch::dispatch_for_scope( runtime, - tracedecay_hooks::HookHostV1::ClaudeCode, + tracedecay_domain::NativeHostIdentityV1::ClaudeCode, event, root.as_deref(), Some(&hook_telemetry), @@ -201,14 +202,14 @@ async fn claude_guidance_hook(runtime: &HookRuntimeV1, hook_name: &'static str) let hook_telemetry = record_hook_invoked_parsed( runtime, root.as_deref(), - HintAgent::Claude, + HostIntegrationIdV1::Claude, hook_name, &event, &parsed, ); let output = super::dispatch::dispatch_for_scope( runtime, - tracedecay_hooks::HookHostV1::ClaudeCode, + tracedecay_domain::NativeHostIdentityV1::ClaudeCode, &event, root.as_deref(), Some(&hook_telemetry), @@ -223,7 +224,7 @@ async fn claude_guidance_hook(runtime: &HookRuntimeV1, hook_name: &'static str) ); if !super::write_hook_output( root.as_deref(), - tracedecay_hooks::HookHostV1::ClaudeCode, + tracedecay_domain::NativeHostIdentityV1::ClaudeCode, &event, &output, ) diff --git a/crates/tracedecay-agent-hosts/src/hooks/codex.rs b/crates/tracedecay-agent-hosts/src/hooks/codex.rs index 2e8154ca92..643187d1eb 100644 --- a/crates/tracedecay-agent-hosts/src/hooks/codex.rs +++ b/crates/tracedecay-agent-hosts/src/hooks/codex.rs @@ -12,7 +12,7 @@ use crate::ports::hook_runtime::HookRuntimeV1; use super::claude::is_code_research_prompt; use super::steering::{HookWorkspaceStatus, index_status_line}; -use super::tool_hints::{HintAgent, HintCategory, ToolHint, ToolHintInput, decide_hint}; +use super::tool_hints::{HintCategory, ToolHint, ToolHintInput, decide_hint}; use super::{ additional_context_json, append_tool_hint, compact_daemon_args, deduped_project_hint_with_id, event_cwd_from_parsed, event_project_root, event_project_root_from_json, @@ -21,6 +21,7 @@ use super::{ record_hint_analytics, record_hook_analytics, record_hook_invoked_parsed, record_workspace_status_analytics, rel_under_root, text_field, }; +use tracedecay_domain::HostIntegrationIdV1; const CODEX_SUBAGENT_START_CONTEXT: &str = "TraceDecay context for this new or code-research \ subagent: when the task needs unfamiliar code context, use tracedecay_context for concepts, \ @@ -45,14 +46,14 @@ pub async fn hook_codex_session_start(runtime: &HookRuntimeV1) -> i32 { let hook_telemetry = record_hook_invoked_parsed( runtime, root.as_deref(), - HintAgent::Codex, + HostIntegrationIdV1::Codex, "SessionStart", &event, &parsed, ); let guidance = super::dispatch::dispatch_for_scope( runtime, - tracedecay_hooks::HookHostV1::Codex, + tracedecay_domain::NativeHostIdentityV1::Codex, &event, root.as_deref(), Some(&hook_telemetry), @@ -67,7 +68,7 @@ pub async fn hook_codex_session_start(runtime: &HookRuntimeV1) -> i32 { ); if !super::write_hook_output( root.as_deref(), - tracedecay_hooks::HookHostV1::Codex, + tracedecay_domain::NativeHostIdentityV1::Codex, &event, &output, ) @@ -96,8 +97,13 @@ pub async fn hook_codex_user_prompt_submit(runtime: &HookRuntimeV1) -> i32 { match profile { Ok(None) => { return i32::from( - !super::write_hook_output(None, tracedecay_hooks::HookHostV1::Codex, &event, "{}") - .await, + !super::write_hook_output( + None, + tracedecay_domain::NativeHostIdentityV1::Codex, + &event, + "{}", + ) + .await, ); } Err(error) => { @@ -109,7 +115,7 @@ pub async fn hook_codex_user_prompt_submit(runtime: &HookRuntimeV1) -> i32 { let hook_telemetry = record_hook_invoked_parsed( runtime, root.as_deref(), - HintAgent::Codex, + HostIntegrationIdV1::Codex, "UserPromptSubmit", &event, &parsed, @@ -141,7 +147,7 @@ pub async fn hook_codex_user_prompt_submit(runtime: &HookRuntimeV1) -> i32 { }; if !super::write_hook_output( root.as_deref(), - tracedecay_hooks::HookHostV1::Codex, + tracedecay_domain::NativeHostIdentityV1::Codex, &event, &output, ) @@ -197,14 +203,14 @@ pub async fn hook_codex_post_tool_use(runtime: &HookRuntimeV1) -> i32 { let hook_telemetry = record_hook_invoked_parsed( runtime, root.as_deref(), - HintAgent::Codex, + HostIntegrationIdV1::Codex, "PostToolUse", &event, &parsed, ); let guidance = super::dispatch::dispatch_for_scope( runtime, - tracedecay_hooks::HookHostV1::Codex, + tracedecay_domain::NativeHostIdentityV1::Codex, &event, root.as_deref(), Some(&hook_telemetry), @@ -216,7 +222,7 @@ pub async fn hook_codex_post_tool_use(runtime: &HookRuntimeV1) -> i32 { if let Some(guidance) = guidance && !super::write_hook_output( root.as_deref(), - tracedecay_hooks::HookHostV1::Codex, + tracedecay_domain::NativeHostIdentityV1::Codex, &event, &additional_context_json("PostToolUse", &guidance), ) @@ -241,19 +247,21 @@ pub async fn hook_codex_post_compact(runtime: &HookRuntimeV1) -> i32 { let hook_telemetry = record_hook_invoked_parsed( runtime, root.as_deref(), - HintAgent::Codex, + HostIntegrationIdV1::Codex, "PostCompact", &event, &parsed, ); - if std::env::var_os(tracedecay_sessions::runtime::codex_app_server::CODEX_SUMMARY_CHILD_ENV) - .is_none() + if std::env::var_os( + tracedecay_sessions::runtime::hosts::codex_app_server::CODEX_SUMMARY_CHILD_ENV, + ) + .is_none() { codex_post_compact(runtime, &event, Some(&hook_telemetry)).await; } if !super::write_hook_output( root.as_deref(), - tracedecay_hooks::HookHostV1::Codex, + tracedecay_domain::NativeHostIdentityV1::Codex, &event, &serde_json::json!({}).to_string(), ) @@ -299,7 +307,7 @@ pub fn evaluate_codex_subagent_start(event_json: &str) -> Option { record_hint_analytics( root.as_deref(), "hint_candidate", - HintAgent::Codex, + HostIntegrationIdV1::Codex, event_session_id(&parsed).as_deref(), &hint_id, &hint, @@ -342,7 +350,7 @@ pub async fn record_codex_subagent_start(runtime: &HookRuntimeV1, event_json: &s Some(&root), "codex_subagent_start", serde_json::json!({ - "agent": HintAgent::Codex.as_key(), + "agent": HostIntegrationIdV1::Codex.as_key(), "session_id": analytics_session_id.as_deref(), "agent_type": agent_type, "count": next, @@ -534,7 +542,7 @@ async fn codex_post_compact( fn deduped_codex_hint(parsed: &Value, hint_id: &str, hint: ToolHint) -> Option { deduped_project_hint_with_id( event_project_root(parsed).as_deref(), - HintAgent::Codex, + HostIntegrationIdV1::Codex, event_session_id(parsed), hint_id, hint, @@ -543,7 +551,7 @@ fn deduped_codex_hint(parsed: &Value, hint_id: &str, hint: ToolHint) -> Option Option { let hint = decide_hint(&ToolHintInput { - agent: HintAgent::Codex, + agent: HostIntegrationIdV1::Codex, session_id: event_session_id(parsed), tool_name: None, command: None, @@ -560,7 +568,7 @@ fn codex_prompt_hint(parsed: &Value) -> Option { record_hint_analytics( root.as_deref(), "hint_candidate", - HintAgent::Codex, + HostIntegrationIdV1::Codex, event_session_id(parsed).as_deref(), &hint_id, &hint, diff --git a/crates/tracedecay-agent-hosts/src/hooks/cursor.rs b/crates/tracedecay-agent-hosts/src/hooks/cursor.rs index 0a5cdf9751..f5119e59b3 100644 --- a/crates/tracedecay-agent-hosts/src/hooks/cursor.rs +++ b/crates/tracedecay-agent-hosts/src/hooks/cursor.rs @@ -12,12 +12,13 @@ use serde_json::Value; use crate::ports::hook_runtime::HookRuntimeV1; use super::post_tool_use::{captured_tool_output, trusted_tool_failure}; -use super::tool_hints::{HintAgent, ToolHint, ToolHintInput, decide_hint}; +use super::tool_hints::{ToolHint, ToolHintInput, decide_hint}; use super::{ deduped_project_hint_with_id, event_session_id, format_tool_hint, mint_hint_id, nearest_project_like_root, read_hook_event, record_hint_analytics, record_hook_invoked_parsed, text_field, }; +use tracedecay_domain::HostIntegrationIdV1; /// Largest transcript tail a low-priority Cursor catch-up hook will read. /// Oversized backlogs stay queued instead of blocking hook execution. @@ -59,7 +60,7 @@ pub async fn hook_cursor_post_tool_use(runtime: &HookRuntimeV1) -> i32 { let _hook_telemetry = record_hook_invoked_parsed( runtime, root.as_deref(), - HintAgent::Cursor, + HostIntegrationIdV1::Cursor, "postToolUse", &event, &parsed, @@ -67,7 +68,7 @@ pub async fn hook_cursor_post_tool_use(runtime: &HookRuntimeV1) -> i32 { if let Some(decision) = cursor_post_tool_use_decision(runtime, &event) && !super::write_hook_output( root.as_deref(), - tracedecay_hooks::HookHostV1::CursorDesktop, + tracedecay_domain::NativeHostIdentityV1::CursorDesktop, &event, &decision, ) @@ -85,7 +86,7 @@ pub async fn hook_cursor_session_start(runtime: &HookRuntimeV1) -> i32 { let (root, output) = cursor_session_start_response(runtime, &event, started).await; if !super::write_hook_output( root.as_deref(), - tracedecay_hooks::HookHostV1::CursorDesktop, + tracedecay_domain::NativeHostIdentityV1::CursorDesktop, &event, &output, ) @@ -108,14 +109,14 @@ async fn cursor_session_start_response( let hook_telemetry = record_hook_invoked_parsed( runtime, root.as_deref(), - HintAgent::Cursor, + HostIntegrationIdV1::Cursor, "sessionStart", event, &parsed, ); let guidance = super::dispatch::dispatch_for_scope( runtime, - tracedecay_hooks::HookHostV1::CursorDesktop, + tracedecay_domain::NativeHostIdentityV1::CursorDesktop, event, root.as_deref(), Some(&hook_telemetry), @@ -157,7 +158,7 @@ fn prepare_cursor_post_tool_use_hint(event_json: &str) -> Option<(String, ToolHi record_hint_analytics( root.as_deref(), "hint_candidate", - HintAgent::Cursor, + HostIntegrationIdV1::Cursor, event_session_id(&parsed).as_deref(), &hint_id, &hint, @@ -191,7 +192,7 @@ fn cursor_hint_root( record_hint_analytics( None, "dropped_no_root", - HintAgent::Cursor, + HostIntegrationIdV1::Cursor, None, hint_id, hint, @@ -203,7 +204,7 @@ fn cursor_hint_root( record_hint_analytics( None, "dropped_no_root", - HintAgent::Cursor, + HostIntegrationIdV1::Cursor, session_id.as_deref(), hint_id, hint, @@ -224,14 +225,20 @@ fn deduped_cursor_hint( record_hint_analytics( Some(&root), "suppressed_uninitialized", - HintAgent::Cursor, + HostIntegrationIdV1::Cursor, session_id.as_deref(), hint_id, &hint, ); return None; } - deduped_project_hint_with_id(Some(&root), HintAgent::Cursor, session_id, hint_id, hint) + deduped_project_hint_with_id( + Some(&root), + HostIntegrationIdV1::Cursor, + session_id, + hint_id, + hint, + ) } pub fn cursor_project_root_from_event(event_json: &str) -> Option { @@ -357,7 +364,7 @@ fn cursor_tool_hint_input(parsed: &Value) -> ToolHintInput { .or_else(|| parsed.get("input")) .unwrap_or(&Value::Null); ToolHintInput { - agent: HintAgent::Cursor, + agent: HostIntegrationIdV1::Cursor, session_id: event_session_id(parsed), tool_name: text_field(parsed, &["tool_name", "toolName", "name"]), command: text_field(tool_input, &["command", "cmd"]) diff --git a/crates/tracedecay-agent-hosts/src/hooks/dispatch.rs b/crates/tracedecay-agent-hosts/src/hooks/dispatch.rs index 53ff3b73cd..df1b61a2ba 100644 --- a/crates/tracedecay-agent-hosts/src/hooks/dispatch.rs +++ b/crates/tracedecay-agent-hosts/src/hooks/dispatch.rs @@ -7,13 +7,14 @@ use tracedecay_contracts::ResolvedScope; use tracedecay_contracts::context_scout::{ ContextScoutAddressV1, ContextScoutDeliveryOutcomeV1, ContextScoutDeliveryReceiptV1, }; +use tracedecay_domain::NativeHostIdentityV1; use tracedecay_domain::{ProjectId, UtcMicros}; #[cfg(test)] use tracedecay_hooks::HookImmediateAdmissionStateV1; use tracedecay_hooks::{ AsyncHookFeedbackDeliveryPortV1, HookConfigurationFileReaderV1, HookConfigurationReadOutcomeV1, HookConfigurationSnapshotV1, HookConfigurationSubscriberV1, HookEventEnvelopeV2, - HookFeedbackDeliveryV1, HookFeedbackRollbackSwitchV1, HookGuidanceStateV1, HookHostV1, + HookFeedbackDeliveryV1, HookFeedbackRollbackSwitchV1, HookGuidanceStateV1, HookImmediateAdmissionV1, HookRuntimeControlV1, HookScopeBindingV1, HookSpoolConfigV1, HookSpoolError, HookSpoolV1, HookSynchronousDeadlineV1, HookTransportDispositionV1, NativeContextScoutLifecycleV1, NativeEnvelopeMaterialV1, NativeHookDecodeError, @@ -68,14 +69,14 @@ impl HookDispatch { } } -pub const NATIVE_HOOK_HOSTS: &[HookHostV1] = &[ - HookHostV1::ClaudeCode, - HookHostV1::Codex, - HookHostV1::CursorDesktop, - HookHostV1::Hermes, - HookHostV1::Kiro, - HookHostV1::KimiCode, - HookHostV1::OpenCode, +pub const NATIVE_HOOK_HOSTS: &[NativeHostIdentityV1] = &[ + NativeHostIdentityV1::ClaudeCode, + NativeHostIdentityV1::Codex, + NativeHostIdentityV1::CursorDesktop, + NativeHostIdentityV1::Hermes, + NativeHostIdentityV1::Kiro, + NativeHostIdentityV1::KimiCode, + NativeHostIdentityV1::OpenCode, ]; pub fn project_id_for_layout( @@ -402,15 +403,16 @@ impl NativeIdentityFields { } fn native_context_scout_lifecycle( - host: HookHostV1, + host: NativeHostIdentityV1, fields: &NativeIdentityFields, event_id: [u8; 16], ) -> Option { - matches!(host, HookHostV1::KimiCode | HookHostV1::OpenCode) - .then(|| { - NativeContextScoutLifecycleV1::new(fields.session_id()?, fields.call_id()?, event_id) - }) - .flatten() + matches!( + host, + NativeHostIdentityV1::KimiCode | NativeHostIdentityV1::OpenCode + ) + .then(|| NativeContextScoutLifecycleV1::new(fields.session_id()?, fields.call_id()?, event_id)) + .flatten() } const HOOK_ADMISSION_ACK_BUDGET_MICROS: u64 = 25_000; @@ -430,7 +432,7 @@ fn admission_window_after_elapsed(elapsed: u64) -> Option<(HookSynchronousDeadli #[hotpath::measure(future = true, label = "hosts.hooks.dispatch")] pub(crate) async fn dispatch( runtime: &HookRuntimeV1, - host: HookHostV1, + host: NativeHostIdentityV1, event_json: &str, project_root: &Path, telemetry: Option<&HookTimingSpan>, @@ -476,7 +478,7 @@ pub(crate) async fn dispatch( /// project identity. Both paths send only the closed event material. pub(crate) async fn dispatch_for_scope( runtime: &HookRuntimeV1, - host: HookHostV1, + host: NativeHostIdentityV1, event_json: &str, project_root: Option<&Path>, telemetry: Option<&HookTimingSpan>, @@ -492,7 +494,7 @@ pub(crate) async fn dispatch_for_scope( async fn dispatch_profile_scoped( runtime: &HookRuntimeV1, - host: HookHostV1, + host: NativeHostIdentityV1, event_json: &str, telemetry: Option<&HookTimingSpan>, started: Instant, @@ -578,7 +580,7 @@ pub(crate) async fn dispatch_opencode_tool_after( }; let Some(prepared) = prepare_bound_hook( runtime, - HookHostV1::OpenCode, + NativeHostIdentityV1::OpenCode, event_json, project_root, decoded, @@ -631,7 +633,7 @@ pub(crate) async fn dispatch_opencode_lsp_updated( } struct PreparedBoundHook { - host: HookHostV1, + host: NativeHostIdentityV1, layout: tracedecay_runtime_core::storage::StoreLayout, snapshot: HookConfigurationSnapshotV1, envelope: HookEventEnvelopeV2, @@ -642,7 +644,7 @@ struct PreparedBoundHook { fn prepare_bound_hook( runtime: &HookRuntimeV1, - host: HookHostV1, + host: NativeHostIdentityV1, event_json: &str, project_root: &Path, decoded: tracedecay_hooks::DecodedNativeHookEventV1, @@ -856,7 +858,7 @@ fn render_host_delivery( /// exit 0 and the event was never spooled. fn append_for_replay( data_root: &Path, - host: HookHostV1, + host: NativeHostIdentityV1, envelope: &HookEventEnvelopeV2, native_lifecycle: Option, binding: &HookScopeBindingV1, @@ -890,7 +892,7 @@ enum PendingEnvelopeV1 { fn replay_envelope_if_pending( data_root: &Path, - host: HookHostV1, + host: NativeHostIdentityV1, binding: &HookScopeBindingV1, retry: &HookEventEnvelopeV2, now: UtcMicros, diff --git a/crates/tracedecay-agent-hosts/src/hooks/dispatch/tests.rs b/crates/tracedecay-agent-hosts/src/hooks/dispatch/tests.rs index 16f5bba84a..c0438a5ca0 100644 --- a/crates/tracedecay-agent-hosts/src/hooks/dispatch/tests.rs +++ b/crates/tracedecay-agent-hosts/src/hooks/dispatch/tests.rs @@ -53,17 +53,17 @@ fn hook_binding_uses_exact_resolved_worktree_and_revision_epoch() { #[test] fn every_host_with_a_native_advisory_event_receives_a_daemon_binding() { let hosts = [ - HookHostV1::ClaudeCode, - HookHostV1::Codex, - HookHostV1::CursorDesktop, - HookHostV1::CursorCloud, - HookHostV1::Hermes, - HookHostV1::Kiro, - HookHostV1::KimiCode, - HookHostV1::OpenCode, - HookHostV1::Cline, - HookHostV1::RooCode, - HookHostV1::Kilo, + NativeHostIdentityV1::ClaudeCode, + NativeHostIdentityV1::Codex, + NativeHostIdentityV1::CursorDesktop, + NativeHostIdentityV1::CursorCloud, + NativeHostIdentityV1::Hermes, + NativeHostIdentityV1::Kiro, + NativeHostIdentityV1::KimiCode, + NativeHostIdentityV1::OpenCode, + NativeHostIdentityV1::Cline, + NativeHostIdentityV1::RooCode, + NativeHostIdentityV1::Kilo, ]; let families = [ tracedecay_hooks::HookEventFamily::SessionBoundary, @@ -162,7 +162,7 @@ fn daemon_feedback_notice_survives_into_host_delivery() { let current_envelope = HookEventEnvelopeV2 { schema_version: tracedecay_hooks::HOOK_EVENT_SCHEMA_VERSION, event_id: [1; 16], - producer: HookHostV1::ClaudeCode, + producer: NativeHostIdentityV1::ClaudeCode, protected_session_id: [2; 32], project_id: envelope_identity_hash16("project", notice.scope.project_id.as_str()), repository_id: envelope_identity_hash16("repository", notice.scope.repository_id.as_str()), @@ -312,7 +312,7 @@ fn sample_envelope( HookEventEnvelopeV2 { schema_version: tracedecay_hooks::HOOK_EVENT_SCHEMA_VERSION, event_id: [1; 16], - producer: HookHostV1::ClaudeCode, + producer: NativeHostIdentityV1::ClaudeCode, protected_session_id: [2; 32], project_id: envelope_identity_hash16("project", notice.scope.project_id.as_str()), repository_id: envelope_identity_hash16("repository", notice.scope.repository_id.as_str()), @@ -670,7 +670,7 @@ fn retry_identity_and_timestamp_reuse_are_stable() { assert_eq!(retry.event_id, first.event_id); let temporary = tempfile::tempdir().unwrap(); - let host = HookHostV1::ClaudeCode; + let host = NativeHostIdentityV1::ClaudeCode; let binding = spool_binding(host, [family]); let decoded = tracedecay_hooks::decode_native_hook_event( host, @@ -705,7 +705,7 @@ fn retry_identity_and_timestamp_reuse_are_stable() { } fn spool_binding( - host: HookHostV1, + host: NativeHostIdentityV1, families: impl IntoIterator, ) -> HookScopeBindingV1 { HookScopeBindingV1 { @@ -788,7 +788,8 @@ fn kimi_rendered_hook_fixture_queues_only_native_session_and_call_identity() { .unwrap(); let lifecycle = - native_context_scout_lifecycle(HookHostV1::KimiCode, &fields, material.event_id).unwrap(); + native_context_scout_lifecycle(NativeHostIdentityV1::KimiCode, &fields, material.event_id) + .unwrap(); assert_eq!(lifecycle.session_id.as_str(), "session.kimi.native"); assert_eq!(lifecycle.call_id.as_str(), "call.kimi.native"); @@ -855,7 +856,8 @@ fn opencode_rendered_plugin_queues_only_tool_after_lifecycle_identity() { ) .unwrap(); let lifecycle = - native_context_scout_lifecycle(HookHostV1::OpenCode, &fields, material.event_id).unwrap(); + native_context_scout_lifecycle(NativeHostIdentityV1::OpenCode, &fields, material.event_id) + .unwrap(); assert_eq!(lifecycle.session_id.as_str(), "session.opencode.native"); assert_eq!(lifecycle.call_id.as_str(), "call.opencode.native"); @@ -865,7 +867,7 @@ fn opencode_rendered_plugin_queues_only_tool_after_lifecycle_identity() { ) .unwrap(); let binding = spool_binding( - HookHostV1::OpenCode, + NativeHostIdentityV1::OpenCode, [tracedecay_hooks::HookEventFamily::SavedEdit], ); let envelope = decoded.into_envelope(&binding, material).unwrap(); @@ -873,7 +875,7 @@ fn opencode_rendered_plugin_queues_only_tool_after_lifecycle_identity() { assert_eq!( append_for_replay( temporary.path(), - HookHostV1::OpenCode, + NativeHostIdentityV1::OpenCode, &envelope, Some(lifecycle.clone()), &binding, @@ -884,10 +886,10 @@ fn opencode_rendered_plugin_queues_only_tool_after_lifecycle_identity() { let spool_root = temporary .path() .join("hook-v2-spool") - .join(HookHostV1::OpenCode.hook_key()); + .join(NativeHostIdentityV1::OpenCode.hook_key()); let (mut spool, _) = HookSpoolV1::open( spool_root, - HookSpoolConfigV1::stock(HookHostV1::OpenCode), + HookSpoolConfigV1::stock(NativeHostIdentityV1::OpenCode), UtcMicros(10), ) .unwrap(); @@ -902,5 +904,7 @@ fn opencode_rendered_plugin_queues_only_tool_after_lifecycle_identity() { .unwrap()["request"] .to_string(); let fields = serde_json::from_str::(&file_edit).unwrap(); - assert!(native_context_scout_lifecycle(HookHostV1::OpenCode, &fields, [1; 16]).is_none()); + assert!( + native_context_scout_lifecycle(NativeHostIdentityV1::OpenCode, &fields, [1; 16]).is_none() + ); } diff --git a/crates/tracedecay-agent-hosts/src/hooks/hint_analytics_tests.rs b/crates/tracedecay-agent-hosts/src/hooks/hint_analytics_tests.rs index 161ae9e359..95917504d9 100644 --- a/crates/tracedecay-agent-hosts/src/hooks/hint_analytics_tests.rs +++ b/crates/tracedecay-agent-hosts/src/hooks/hint_analytics_tests.rs @@ -1,6 +1,6 @@ use super::tool_hints::{HintCategory, MAX_HINTS_PER_SESSION}; use super::{ - EnvGuard, HintAgent, Path, PathBuf, ToolHint, Value, deduped_project_hint_with_id, + EnvGuard, HostIntegrationIdV1, Path, PathBuf, ToolHint, Value, deduped_project_hint_with_id, mint_hint_id, record_hint_emitted, record_hook_invoked, }; use tracedecay_runtime_core::config::USER_DATA_DIR_ENV; @@ -95,7 +95,7 @@ fn hook_invocation_rows_include_duration_telemetry() { let _hook_telemetry = record_hook_invoked( &crate::ports::hook_runtime::crate_test_runtime(), Some(&project_root), - HintAgent::Codex, + HostIntegrationIdV1::Codex, "PostToolUse", r#"{"session_id":"s1","tool_name":"Bash","cwd":"/tmp"}"#, ); @@ -137,7 +137,13 @@ fn record_hint_emitted_missing_session_is_single_terminal() { let hint = test_hint(); let id = mint_hint_id(); - record_hint_emitted(Some(&project_root), HintAgent::Cursor, None, &id, &hint); + record_hint_emitted( + Some(&project_root), + HostIntegrationIdV1::Cursor, + None, + &id, + &hint, + ); let rows = recorded_rows(&data_root, &profile_root); let seq: Vec<&str> = events_for(&rows, &id) @@ -168,7 +174,7 @@ fn every_hint_branch_yields_exactly_one_terminal_with_hint_id() { assert!( deduped_project_hint_with_id( Some(&project_root), - HintAgent::Cursor, + HostIntegrationIdV1::Cursor, Some("session-emit".to_string()), &emit_id, test_hint(), @@ -181,7 +187,7 @@ fn every_hint_branch_yields_exactly_one_terminal_with_hint_id() { assert!( deduped_project_hint_with_id( Some(&project_root), - HintAgent::Cursor, + HostIntegrationIdV1::Cursor, Some("session-emit".to_string()), &dup_id, test_hint(), @@ -194,7 +200,7 @@ fn every_hint_branch_yields_exactly_one_terminal_with_hint_id() { assert!( deduped_project_hint_with_id( Some(&project_root), - HintAgent::Cursor, + HostIntegrationIdV1::Cursor, None, &no_session_id, test_hint(), @@ -207,7 +213,7 @@ fn every_hint_branch_yields_exactly_one_terminal_with_hint_id() { assert!( deduped_project_hint_with_id( None, - HintAgent::Cursor, + HostIntegrationIdV1::Cursor, Some("session-noroot".to_string()), &no_root_id, test_hint(), @@ -273,7 +279,7 @@ fn hints_without_project_root_dedupe_in_the_user_profile() { assert!( deduped_project_hint_with_id( None, - HintAgent::Codex, + HostIntegrationIdV1::Codex, session.clone(), &mint_hint_id(), test_hint(), @@ -283,7 +289,7 @@ fn hints_without_project_root_dedupe_in_the_user_profile() { assert!( deduped_project_hint_with_id( None, - HintAgent::Codex, + HostIntegrationIdV1::Codex, session, &mint_hint_id(), test_hint(), @@ -323,7 +329,7 @@ fn budget_exhaustion_records_suppressed_budget_terminal() { assert!( deduped_project_hint_with_id( Some(&project_root), - HintAgent::Cursor, + HostIntegrationIdV1::Cursor, Some(session.clone()), &mint_hint_id(), hint, @@ -338,7 +344,7 @@ fn budget_exhaustion_records_suppressed_budget_terminal() { let over_id = mint_hint_id(); let over = deduped_project_hint_with_id( Some(&project_root), - HintAgent::Cursor, + HostIntegrationIdV1::Cursor, Some(session.clone()), &over_id, ToolHint { @@ -375,7 +381,7 @@ fn repeated_usage_records_hint_escalated_terminal() { let emit = |id: &str| { deduped_project_hint_with_id( Some(&project_root), - HintAgent::Cursor, + HostIntegrationIdV1::Cursor, Some(session.clone()), id, test_hint(), diff --git a/crates/tracedecay-agent-hosts/src/hooks/hint_outcomes.rs b/crates/tracedecay-agent-hosts/src/hooks/hint_outcomes.rs index b28560482d..70c2dbe8b6 100644 --- a/crates/tracedecay-agent-hosts/src/hooks/hint_outcomes.rs +++ b/crates/tracedecay-agent-hosts/src/hooks/hint_outcomes.rs @@ -5,7 +5,7 @@ //! time a soft hint surfaces. Whether the model *acted* on that hint is not //! known at emit time, it depends on which tools fire next. This module closes //! that loop after the fact: for each emitted hint that has not yet been -//! resolved, it inspects the session's ingested `session_messages` activity +//! resolved, it inspects the session's ingested session message activity //! *after* the hint timestamp and appends a new `hint_outcome` analytics event: //! //! * `acted` , a tracedecay tool matching the hint's category fired inside the diff --git a/crates/tracedecay-agent-hosts/src/hooks/hook_boundary_failure_matrix.rs b/crates/tracedecay-agent-hosts/src/hooks/hook_boundary_failure_matrix.rs index e498fcd629..32e9f6f23d 100644 --- a/crates/tracedecay-agent-hosts/src/hooks/hook_boundary_failure_matrix.rs +++ b/crates/tracedecay-agent-hosts/src/hooks/hook_boundary_failure_matrix.rs @@ -12,8 +12,8 @@ use std::path::{Path, PathBuf}; use serde_json::Value; use super::analytics::{HOOK_ANALYTICS_FILENAME, record_hook_invoked}; -use super::tool_hints::HintAgent; use super::{EnvGuard, TestDaemonHookActionGuard, daemon_hook_action, lock_test_env}; +use tracedecay_domain::HostIntegrationIdV1; use tracedecay_runtime_core::config::USER_DATA_DIR_ENV; fn enroll_project(project_root: &Path, project_id: &str) -> PathBuf { @@ -54,7 +54,7 @@ fn matrix_rejects_default_success_when_disposition_absent() { let _span = record_hook_invoked( &crate::ports::hook_runtime::crate_test_runtime(), Some(&project_root), - HintAgent::Claude, + HostIntegrationIdV1::Claude, "noDisposition", "{}", ); @@ -107,7 +107,7 @@ fn matrix_sticky_failure_survives_later_success_for_unavailable_cancel_backpress let span = record_hook_invoked( &crate::ports::hook_runtime::crate_test_runtime(), Some(&project_root), - HintAgent::Claude, + HostIntegrationIdV1::Claude, "unavailableThenSuccess", "{}", ); @@ -118,7 +118,7 @@ fn matrix_sticky_failure_survives_later_success_for_unavailable_cancel_backpress let span = record_hook_invoked( &crate::ports::hook_runtime::crate_test_runtime(), Some(&project_root), - HintAgent::Kiro, + HostIntegrationIdV1::Kiro, "cancelThenSuccess", "{}", ); @@ -129,7 +129,7 @@ fn matrix_sticky_failure_survives_later_success_for_unavailable_cancel_backpress let span = record_hook_invoked( &crate::ports::hook_runtime::crate_test_runtime(), Some(&project_root), - HintAgent::Codex, + HostIntegrationIdV1::Codex, "backpressureThenSuccess", "{}", ); @@ -140,7 +140,7 @@ fn matrix_sticky_failure_survives_later_success_for_unavailable_cancel_backpress let span = record_hook_invoked( &crate::ports::hook_runtime::crate_test_runtime(), Some(&project_root), - HintAgent::Cursor, + HostIntegrationIdV1::Cursor, "timeoutThenSuccess", "{}", ); @@ -189,7 +189,7 @@ fn matrix_daemon_unavailable_transport_does_not_invent_success() { let span = record_hook_invoked( &crate::ports::hook_runtime::crate_test_runtime(), Some(&project_root), - HintAgent::Cursor, + HostIntegrationIdV1::Cursor, "daemonDown", "{}", ); diff --git a/crates/tracedecay-agent-hosts/src/hooks/kiro.rs b/crates/tracedecay-agent-hosts/src/hooks/kiro.rs index e61d86e76b..3e348e16cd 100644 --- a/crates/tracedecay-agent-hosts/src/hooks/kiro.rs +++ b/crates/tracedecay-agent-hosts/src/hooks/kiro.rs @@ -12,11 +12,12 @@ use serde_json::Value; use crate::ports::hook_runtime::HookRuntimeV1; use super::claude::is_code_research_prompt; -use super::tool_hints::{HintAgent, ToolHintInput, decide_hint}; +use super::tool_hints::{ToolHintInput, decide_hint}; use super::{ event_cwd_from_parsed, event_project_root_or_process_cwd, event_session_id, read_hook_event, record_hook_invoked_parsed, rel_under_root, research_block_reason, }; +use tracedecay_domain::HostIntegrationIdV1; /// Largest transcript tail the Kiro `userPromptSubmit` hook will read per call. const KIRO_HOT_INGEST_MAX_BYTES: u64 = 256 * 1024; @@ -38,7 +39,7 @@ pub fn evaluate_kiro_pre_tool_use(event_json: &str) -> Option { let tool_input = parsed.get("tool_input").unwrap_or(&Value::Null); if let Some(prompt) = kiro_event_text(tool_input).filter(|text| is_code_research_prompt(text)) { let hint = decide_hint(&ToolHintInput { - agent: HintAgent::Kiro, + agent: HostIntegrationIdV1::Kiro, session_id: event_session_id(&parsed), tool_name: Some(tool_name.to_string()), command: None, @@ -130,8 +131,13 @@ pub async fn hook_kiro_prompt_submit(runtime: &HookRuntimeV1) -> i32 { match profile { Ok(None) => { return i32::from( - !super::write_hook_output(None, tracedecay_hooks::HookHostV1::Kiro, &event, "{}") - .await, + !super::write_hook_output( + None, + tracedecay_domain::NativeHostIdentityV1::Kiro, + &event, + "{}", + ) + .await, ); } Err(error) => { @@ -144,7 +150,7 @@ pub async fn hook_kiro_prompt_submit(runtime: &HookRuntimeV1) -> i32 { let hook_telemetry = record_hook_invoked_parsed( runtime, root.as_deref(), - HintAgent::Kiro, + HostIntegrationIdV1::Kiro, "userPromptSubmit", &event, &parsed, @@ -152,7 +158,7 @@ pub async fn hook_kiro_prompt_submit(runtime: &HookRuntimeV1) -> i32 { let dispatch_guidance = if let Some(root) = root.as_deref() { super::dispatch::dispatch( runtime, - tracedecay_hooks::HookHostV1::Kiro, + tracedecay_domain::NativeHostIdentityV1::Kiro, &event, root, Some(&hook_telemetry), @@ -187,7 +193,7 @@ pub async fn hook_kiro_prompt_submit(runtime: &HookRuntimeV1) -> i32 { .unwrap_or_else(|| serde_json::json!({}).to_string()); if !super::write_hook_output( root.as_deref(), - tracedecay_hooks::HookHostV1::Kiro, + tracedecay_domain::NativeHostIdentityV1::Kiro, &event, &output, ) diff --git a/crates/tracedecay-agent-hosts/src/hooks/mod.rs b/crates/tracedecay-agent-hosts/src/hooks/mod.rs index b2dd9bce1b..b489615d93 100644 --- a/crates/tracedecay-agent-hosts/src/hooks/mod.rs +++ b/crates/tracedecay-agent-hosts/src/hooks/mod.rs @@ -95,7 +95,7 @@ pub fn aggregate_hook_completed_readiness(rows: &[Value]) -> HookCompletedReadin pub fn record_native_capture_invoked( runtime: &HookRuntimeV1, project_root: Option<&Path>, - host: tracedecay_hooks::HookHostV1, + host: NativeHostIdentityV1, hook_name: Option<&str>, event_json: &str, ) { @@ -129,24 +129,25 @@ pub fn record_native_capture_invoked( /// Analytics agent key for a native host. Hosts outside the five typed /// integrations record under the shared `other` key, matching the OpenCode and /// Kimi dispatchers above. -const fn native_capture_agent(host: tracedecay_hooks::HookHostV1) -> Option { - use tracedecay_hooks::HookHostV1; - +const fn native_capture_agent(host: NativeHostIdentityV1) -> Option { match host { - HookHostV1::ClaudeCode => Some(HintAgent::Claude), - HookHostV1::Codex => Some(HintAgent::Codex), - HookHostV1::CursorDesktop | HookHostV1::CursorCloud => Some(HintAgent::Cursor), - HookHostV1::Hermes => Some(HintAgent::Hermes), - HookHostV1::Kiro => Some(HintAgent::Kiro), - HookHostV1::Cline - | HookHostV1::RooCode - | HookHostV1::Kilo - | HookHostV1::KimiCode - | HookHostV1::OpenCode => None, + NativeHostIdentityV1::ClaudeCode => Some(HostIntegrationIdV1::Claude), + NativeHostIdentityV1::Codex => Some(HostIntegrationIdV1::Codex), + NativeHostIdentityV1::CursorDesktop | NativeHostIdentityV1::CursorCloud => { + Some(HostIntegrationIdV1::Cursor) + } + NativeHostIdentityV1::Hermes => Some(HostIntegrationIdV1::Hermes), + NativeHostIdentityV1::Kiro => Some(HostIntegrationIdV1::Kiro), + NativeHostIdentityV1::Cline + | NativeHostIdentityV1::RooCode + | NativeHostIdentityV1::Kilo + | NativeHostIdentityV1::KimiCode + | NativeHostIdentityV1::OpenCode => None, } } -use tool_hints::{HintAgent, ToolHint}; +use tool_hints::ToolHint; +use tracedecay_domain::{HostIntegrationIdV1, NativeHostIdentityV1}; use tracedecay_policy::hint_delivery::HintDeliveryDecisionV1; #[hotpath::measure(future = true, label = "agent_hosts.hooks.dispatch_kimi_event")] @@ -160,7 +161,7 @@ pub async fn dispatch_kimi_event( record_other_hook_invoked(runtime, Some(project_root), "kimi_event", event_json); dispatch::dispatch( runtime, - tracedecay_hooks::HookHostV1::KimiCode, + NativeHostIdentityV1::KimiCode, event_json, project_root, Some(&telemetry), @@ -186,7 +187,7 @@ pub async fn dispatch_opencode_event( } else { dispatch::dispatch( runtime, - tracedecay_hooks::HookHostV1::OpenCode, + NativeHostIdentityV1::OpenCode, event_json, project_root, Some(&telemetry), @@ -237,7 +238,7 @@ pub async fn dispatch_opencode_tool_after( #[hotpath::measure(future = true, label = "hosts.hooks.write_output")] pub(crate) async fn write_hook_output( project_root: Option<&Path>, - host: tracedecay_hooks::HookHostV1, + host: NativeHostIdentityV1, event_json: &str, output: &str, ) -> bool { @@ -347,7 +348,7 @@ pub(crate) async fn write_hook_output( } fn hook_output_owner_event_id( - host: tracedecay_hooks::HookHostV1, + host: NativeHostIdentityV1, event_json: &str, output: &str, ) -> Option { @@ -378,7 +379,7 @@ pub(crate) use read_hook_event; /// project root, dispatch, and deliver any guidance for `host`. async fn hook_native_event( runtime: &HookRuntimeV1, - host: tracedecay_hooks::HookHostV1, + host: NativeHostIdentityV1, dispatch: impl AsyncFnOnce(&HookRuntimeV1, &str, &Path, Instant) -> Option, ) -> i32 { let started = Instant::now(); @@ -396,19 +397,14 @@ async fn hook_native_event( #[hotpath::measure(future = true, label = "hosts.hooks.kimi_event")] pub async fn hook_kimi_event(runtime: &HookRuntimeV1) -> i32 { - hook_native_event( - runtime, - tracedecay_hooks::HookHostV1::KimiCode, - dispatch_kimi_event, - ) - .await + hook_native_event(runtime, NativeHostIdentityV1::KimiCode, dispatch_kimi_event).await } #[hotpath::measure(future = true, label = "hosts.hooks.opencode_event")] pub async fn hook_opencode_event(runtime: &HookRuntimeV1) -> i32 { hook_native_event( runtime, - tracedecay_hooks::HookHostV1::OpenCode, + NativeHostIdentityV1::OpenCode, dispatch_opencode_event, ) .await @@ -418,7 +414,7 @@ pub async fn hook_opencode_event(runtime: &HookRuntimeV1) -> i32 { pub async fn hook_opencode_tool_after(runtime: &HookRuntimeV1) -> i32 { hook_native_event( runtime, - tracedecay_hooks::HookHostV1::OpenCode, + NativeHostIdentityV1::OpenCode, dispatch_opencode_tool_after, ) .await @@ -719,13 +715,13 @@ pub async fn hook_hermes_terminal_receipt(runtime: &HookRuntimeV1) -> i32 { let hook_telemetry = record_hook_invoked( runtime, project_root.as_deref(), - HintAgent::Hermes, + HostIntegrationIdV1::Hermes, hook_name, &event_json, ); let guidance = dispatch::dispatch_for_scope( runtime, - tracedecay_hooks::HookHostV1::Hermes, + NativeHostIdentityV1::Hermes, &event_json, project_root.as_deref(), Some(&hook_telemetry), @@ -762,7 +758,7 @@ pub async fn hook_hermes_terminal_receipt(runtime: &HookRuntimeV1) -> i32 { ); if !write_hook_output( project_root.as_deref(), - tracedecay_hooks::HookHostV1::Hermes, + NativeHostIdentityV1::Hermes, &event_json, &output, ) @@ -960,7 +956,7 @@ fn hook_route_session_id(parsed: &Value) -> Option { fn deduped_project_hint_with_id( root: Option<&Path>, - agent: HintAgent, + agent: HostIntegrationIdV1, session_id: Option, hint_id: &str, hint: ToolHint, diff --git a/crates/tracedecay-agent-hosts/src/hooks/tests.rs b/crates/tracedecay-agent-hosts/src/hooks/tests.rs index 7c4e5c3ecc..d251b4da15 100644 --- a/crates/tracedecay-agent-hosts/src/hooks/tests.rs +++ b/crates/tracedecay-agent-hosts/src/hooks/tests.rs @@ -146,7 +146,7 @@ fn marker_at_temp_root_does_not_make_descendant_project_like() { #[test] fn direct_hook_owner_identity_is_stable_across_retry_time() { - let host = tracedecay_hooks::HookHostV1::Codex; + let host = tracedecay_domain::NativeHostIdentityV1::Codex; let event = r#"{"session_id":"session-1","hook_event_name":"Stop"}"#; let output = r#"{"hookSpecificOutput":{"hookEventName":"Stop"}}"#; let first = hook_output_owner_event_id(host, event, output).expect("owner identity"); diff --git a/crates/tracedecay-agent-hosts/src/hooks/tool_hints.rs b/crates/tracedecay-agent-hosts/src/hooks/tool_hints.rs index 387294196b..3b160fc1b6 100644 --- a/crates/tracedecay-agent-hosts/src/hooks/tool_hints.rs +++ b/crates/tracedecay-agent-hosts/src/hooks/tool_hints.rs @@ -7,7 +7,7 @@ use std::collections::HashMap; use std::path::Path; -pub use tracedecay_domain::HostIntegrationIdV1 as HintAgent; +use tracedecay_domain::HostIntegrationIdV1; use tracedecay_policy::hint_delivery::{ HintDeliveryDecisionV1, HintDeliveryInputV1, decide_hint_delivery, }; @@ -193,12 +193,12 @@ const CATEGORY_SPECS: &[HintCategorySpec] = &[ key: "file_read", label: "file read", skill: "exploring-code", - message: "Before reading whole files, consider tracedecay_outline, tracedecay_body, or tracedecay_read.", - context: "tracedecay_outline gives a file's table of contents, tracedecay_body returns one symbol's source, and tracedecay_read (mode: \"lines\") slices a range, usually far cheaper than a full-file read. If you are opening the file only to find a string in it, tracedecay_grep locates the literal or regex match with its enclosing symbol instead.", + message: "Before reading whole files, consider tracedecay_source_outline or tracedecay_source_body.", + context: "tracedecay_source_outline gives a file's table of contents, and tracedecay_source_body returns one symbol's source for a node ID from tracedecay_find_exact_symbol or tracedecay_search, usually far cheaper than a full-file read. If you are opening the file only to find a string in it, tracedecay_grep locates the literal or regex match with its enclosing symbol instead.", expected_tools: &[ - "tracedecay_outline", - "tracedecay_body", - "tracedecay_read", + "tracedecay_source_outline", + "tracedecay_source_body", + "tracedecay_find_exact_symbol", "tracedecay_grep", ], nonblocking: true, @@ -327,11 +327,10 @@ const CATEGORY_SPECS: &[HintCategorySpec] = &[ label: "type orientation", skill: "exploring-code", message: "For type, constructor, field, trait, or duplicate-logic questions, use TraceDecay's AST orientation tools.", - context: "Use tracedecay_constructors for struct literal sites, tracedecay_field_sites for reads/writes, tracedecay_impls or tracedecay_implementations for trait methods, and tracedecay_type_hierarchy for a trait/interface/class's full recursive implementor/extender tree.", + context: "Use tracedecay_constructors for struct literal sites, tracedecay_field_sites for reads/writes, tracedecay_implementations for a trait's implementors or every body of a method name, and tracedecay_type_hierarchy for a trait/interface/class's full recursive implementor/extender tree.", expected_tools: &[ "tracedecay_constructors", "tracedecay_field_sites", - "tracedecay_impls", "tracedecay_implementations", "tracedecay_type_hierarchy", ], @@ -425,7 +424,7 @@ const CATEGORY_SPECS: &[HintCategorySpec] = &[ #[derive(Debug, Clone, PartialEq, Eq)] pub struct ToolHintInput { - pub agent: HintAgent, + pub agent: HostIntegrationIdV1, pub session_id: Option, pub tool_name: Option, pub command: Option, @@ -449,7 +448,7 @@ pub struct ToolHintInput { impl Default for ToolHintInput { fn default() -> Self { Self { - agent: HintAgent::Cursor, + agent: HostIntegrationIdV1::Cursor, session_id: None, tool_name: None, command: None, @@ -593,41 +592,9 @@ impl ToolHintDedupe { #[hotpath::measure(label = "hosts.hooks.tool_hints.load")] pub fn load(path: &Path) -> std::io::Result { let content = std::fs::read_to_string(path)?; - // v2: {"version":2, "sessions":[...], "categories":[...]}. v1: a bare - // array of {session_id, category}. Probe for the versioned object first, - // then fall back to the legacy array so old files load losslessly. - if let Ok(persisted) = serde_json::from_str::(&content) { - return Ok(Self::from_persisted(persisted)); - } - let entries: Vec = serde_json::from_str(&content).unwrap_or_default(); - Ok(Self::from_v1_entries(entries)) - } - - fn from_v1_entries(entries: Vec) -> Self { - // A v1 entry records that the category was hinted once; it carries no - // budget or escalation counters. Reconstruct `hinted` state and - // per-session emitted counts so a v1->v2 load preserves suppression. - let mut dedupe = Self::default(); - for entry in entries { - let Some(category) = HintCategory::from_key(&entry.category) else { - continue; - }; - let already = dedupe - .categories - .insert( - (entry.session_id.clone(), category), - CategoryState { - hinted: true, - triggers_after_hint: 0, - escalated: false, - }, - ) - .is_some(); - if !already { - *dedupe.emitted.entry(entry.session_id).or_default() += 1; - } - } - dedupe + let persisted = serde_json::from_str::(&content) + .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?; + Ok(Self::from_persisted(persisted)) } fn from_persisted(persisted: PersistedHints) -> Self { @@ -728,14 +695,6 @@ struct PersistedCategory { escalated: bool, } -/// Legacy v1 entry: a bare `{session_id, category}` pair meaning "this category -/// was hinted once this session". Still parsed so old stores migrate losslessly. -#[derive(serde::Serialize, serde::Deserialize)] -struct PersistedHintEntry { - session_id: String, - category: String, -} - #[hotpath::measure(label = "hosts.hooks.tool_hints.decide")] pub fn decide_hint(input: &ToolHintInput) -> Option { if !input.hints_enabled { diff --git a/crates/tracedecay-agent-hosts/src/hooks/tool_hints/evals/cases_dynamic.rs b/crates/tracedecay-agent-hosts/src/hooks/tool_hints/evals/cases_dynamic.rs index 740fb3e74b..c20629c253 100644 --- a/crates/tracedecay-agent-hosts/src/hooks/tool_hints/evals/cases_dynamic.rs +++ b/crates/tracedecay-agent-hosts/src/hooks/tool_hints/evals/cases_dynamic.rs @@ -87,7 +87,7 @@ pub(super) fn dynamic_action_context_cases() -> Vec { ..ToolHintInput::default() }, Some(HintCategory::FileRead), - &["tracedecay_outline", "tracedecay_body", "tracedecay_read"], + &["tracedecay_source_outline", "tracedecay_source_body"], ), input_eval( "shell-cat-config-read", @@ -98,7 +98,7 @@ pub(super) fn dynamic_action_context_cases() -> Vec { ..ToolHintInput::default() }, Some(HintCategory::FileRead), - &["tracedecay_outline"], + &["tracedecay_source_outline"], ), input_eval( "single-file-read-action", @@ -108,7 +108,7 @@ pub(super) fn dynamic_action_context_cases() -> Vec { ..ToolHintInput::default() }, Some(HintCategory::FileRead), - &["tracedecay_outline", "tracedecay_body", "tracedecay_read"], + &["tracedecay_source_outline", "tracedecay_source_body"], ), input_eval( "windows-tool-descriptor-read", @@ -233,7 +233,7 @@ pub(super) fn dynamic_action_context_cases() -> Vec { input_eval( "codex-apply-patch-nudges-redundancy", ToolHintInput { - agent: HintAgent::Codex, + agent: HostIntegrationIdV1::Codex, tool_name: Some("Edit".to_string()), file_path: Some("src/util.rs".to_string()), edit_text: Some( diff --git a/crates/tracedecay-agent-hosts/src/hooks/tool_hints/evals/cases_synthetic.rs b/crates/tracedecay-agent-hosts/src/hooks/tool_hints/evals/cases_synthetic.rs index 2c75eedfe8..54b7093da9 100644 --- a/crates/tracedecay-agent-hosts/src/hooks/tool_hints/evals/cases_synthetic.rs +++ b/crates/tracedecay-agent-hosts/src/hooks/tool_hints/evals/cases_synthetic.rs @@ -127,21 +127,21 @@ pub(super) fn synthetic_prompt_cases() -> Vec { "head -n 60 src/hooks/tool_hints.rs", "inspect top of hook hints file", Some(HintCategory::FileRead), - &["tracedecay_outline"], + &["tracedecay_source_outline"], ), shell_eval( "shell-tail-source-read", "tail -n 80 src/hooks/tool_hints/classifiers.rs", "inspect classifier bottom", Some(HintCategory::FileRead), - &["tracedecay_outline"], + &["tracedecay_source_outline"], ), shell_eval( "shell-nl-source-read", "nl -ba src/hooks/tool_hints/evals.rs", "read evals with line numbers", Some(HintCategory::FileRead), - &["tracedecay_outline"], + &["tracedecay_source_outline"], ), prompt_eval( "call-chain-question", diff --git a/crates/tracedecay-agent-hosts/src/hooks/tool_hints/evals/host_cases.rs b/crates/tracedecay-agent-hosts/src/hooks/tool_hints/evals/host_cases.rs index c06b5a19e4..a902c8b5c5 100644 --- a/crates/tracedecay-agent-hosts/src/hooks/tool_hints/evals/host_cases.rs +++ b/crates/tracedecay-agent-hosts/src/hooks/tool_hints/evals/host_cases.rs @@ -122,7 +122,7 @@ pub(super) fn expanded_transcript_host_evals() -> Vec { "prompt-type-orientation-impls", "find trait impls and field writes for ToolHintInput", Some(HintCategory::TypeOrientation), - &["tracedecay_field_sites", "tracedecay_impls"], + &["tracedecay_field_sites", "tracedecay_implementations"], ), input_eval( "subagent-context-handoff", @@ -199,14 +199,14 @@ pub(super) fn expanded_transcript_host_evals() -> Vec { "sed -n '1,120p' src/hooks/tool_hints.rs", "read this source range", Some(HintCategory::FileRead), - &["tracedecay_outline", "tracedecay_read"], + &["tracedecay_source_outline"], ), shell_eval( "cat-config-file-read", "cat Cargo.toml", "read config file", Some(HintCategory::FileRead), - &["tracedecay_outline", "tracedecay_read"], + &["tracedecay_source_outline"], ), shell_eval( "cargo-nextest-behavioral-failure-silent", @@ -318,14 +318,14 @@ pub(super) fn expanded_transcript_host_evals() -> Vec { "Read", Some("package.json"), Some(HintCategory::FileRead), - &["tracedecay_outline"], + &["tracedecay_source_outline"], ), tool_eval( "cursor-read-file-alias", "read_file", Some("src/hooks/cursor.rs"), Some(HintCategory::FileRead), - &["tracedecay_outline"], + &["tracedecay_source_outline"], ), tool_eval( "cursor-list-dir-alias", diff --git a/crates/tracedecay-agent-hosts/src/hooks/tool_hints/tests.rs b/crates/tracedecay-agent-hosts/src/hooks/tool_hints/tests.rs index be0e7253a6..26afada256 100644 --- a/crates/tracedecay-agent-hosts/src/hooks/tool_hints/tests.rs +++ b/crates/tracedecay-agent-hosts/src/hooks/tool_hints/tests.rs @@ -1,4 +1,5 @@ use super::*; +use crate::agents::plugin_bundle::claude_files; #[test] fn benign_git_narration_does_not_fire_the_unexpected_change_hint() { @@ -68,22 +69,29 @@ fn every_category_has_compact_skill_backed_rendering() { HintCategory::UnexpectedChanges, ]; + let shipped = claude_files(); for category in categories { let hint = hint_for_category(category); let visible = format!("{}\n{}", hint.message, hint.context); - assert_eq!(hint.category, category); - assert!(!hint.message.is_empty(), "{category:?}"); - assert!(!hint.context.is_empty(), "{category:?}"); assert!( visible.len() <= 850, "{category:?} hint is too verbose: {} chars\n{}", visible.len(), visible ); - let skill = category_skill(category); + let skill = visible + .split_once("Skill: tracedecay:") + .and_then(|(_, rest)| rest.split_once('.')) + .map(|(skill, _)| skill) + .unwrap_or_else(|| panic!("{category:?} hint names no skill:\n{visible}")); + let skill_path = format!("skills/{skill}/SKILL.md"); + let (_, skill_body) = shipped + .iter() + .find(|(relative, _)| *relative == skill_path) + .unwrap_or_else(|| panic!("{category:?} points at unshipped skill {skill_path}")); assert!( - visible.contains(&format!("Skill: tracedecay:{skill}.")), - "{category:?} missing skill trigger" + skill_body.starts_with(&format!("---\nname: {skill}\n")), + "{skill_path} frontmatter must name {skill}" ); } } @@ -225,52 +233,6 @@ fn save_writes_versioned_schema() { assert!(value["categories"].is_array()); } -#[test] -fn legacy_store_migrates_to_versioned_schema() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("tool_hints_seen.json"); - // Legacy v1 file: a bare array of {session_id, category}. - std::fs::write( - &path, - r#"[{"session_id":"s1","category":"search"},{"session_id":"s1","category":"file_read"}]"#, - ) - .unwrap(); - - let mut dedupe = ToolHintDedupe::load_or_default(&path); - // v1 categories load as already-hinted: they suppress, not re-emit. - assert_eq!( - dedupe.decide("s1", HintCategory::Search), - HintDeliveryDecisionV1::SuppressDuplicate - ); - assert_eq!( - dedupe.decide("s1", HintCategory::FileRead), - HintDeliveryDecisionV1::SuppressDuplicate - ); - // The two migrated hints already count against s1's budget, so only one - // more distinct category can emit before the cap. - assert_eq!( - dedupe.decide("s1", HintCategory::Impact), - HintDeliveryDecisionV1::Deliver - ); - assert_eq!( - dedupe.decide("s1", HintCategory::CallGraph), - HintDeliveryDecisionV1::SuppressBudget - ); - - // Persisting rewrites the file in v2 shape. - dedupe.save(&path).unwrap(); - let value: serde_json::Value = - serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap(); - assert_eq!(value["version"], 2); - - // Reload from v2 preserves the migrated suppression state. - let mut reloaded = ToolHintDedupe::load_or_default(&path); - assert_eq!( - reloaded.decide("s1", HintCategory::Search), - HintDeliveryDecisionV1::SuppressDuplicate - ); -} - #[test] fn oversized_store_resets() { let dir = tempfile::tempdir().unwrap(); @@ -306,6 +268,15 @@ fn dedupe_load_tolerates_missing_and_corrupt_files() { dedupe.decide("s1", HintCategory::Search), HintDeliveryDecisionV1::Deliver ); + + // A pre-versioned bare-array store is not migrated; it starts over. + let old_shape = dir.path().join("old-shape.json"); + std::fs::write(&old_shape, r#"[{"session_id":"s1","category":"search"}]"#).unwrap(); + let mut dedupe = ToolHintDedupe::load_or_default(&old_shape); + assert_eq!( + dedupe.decide("s1", HintCategory::Search), + HintDeliveryDecisionV1::Deliver + ); } fn shell_input(command: &str) -> ToolHintInput { diff --git a/crates/tracedecay-agent-hosts/src/ports.rs b/crates/tracedecay-agent-hosts/src/ports.rs index f41ca9b658..495710061d 100644 --- a/crates/tracedecay-agent-hosts/src/ports.rs +++ b/crates/tracedecay-agent-hosts/src/ports.rs @@ -13,11 +13,10 @@ //! installs it whole; a process that never did is a bootstrap failure that //! every reader reports as such, rather than nine slots each answering with //! a plausible production value. -//! - **Direct reads of a lower owner.** [`mcp_tools`] and [`pricing`] name the -//! crate that owns the data. Neither was ever a root-only capability once -//! the split settled, and inverting them cost real safety: an unregistered -//! tool catalog answered empty, which installers wrote as a permission -//! allowlist. +//! - **Direct reads of a lower owner.** [`mcp_tools`] names the crate that +//! owns the data. It was never a root-only capability once the split +//! settled, and inverting it cost real safety: an unregistered tool catalog +//! answered empty, which installers wrote as a permission allowlist. //! - **Boundary contracts.** Values that cross a remaining upward boundary are //! owned here only when no lower canonical crate owns their identity. //! @@ -26,4 +25,3 @@ pub mod hook_runtime; pub mod mcp_tools; -pub mod pricing; diff --git a/crates/tracedecay-agent-hosts/src/ports/hook_runtime.rs b/crates/tracedecay-agent-hosts/src/ports/hook_runtime.rs index 82a6d441b5..83ac344858 100644 --- a/crates/tracedecay-agent-hosts/src/ports/hook_runtime.rs +++ b/crates/tracedecay-agent-hosts/src/ports/hook_runtime.rs @@ -137,8 +137,7 @@ mod test_runtime { /// registered identity authority; this crate's tests seed the markers /// directly, so the fixture reads them. fn initialized(project_root: &Path) -> bool { - tracedecay_runtime_core::config::has_project_database(project_root) - || tracedecay_runtime_core::storage::has_repository_identity_marker(project_root) + tracedecay_runtime_core::storage::has_repository_identity_marker(project_root) } fn layout(_: &Path) -> Pin> + Send + '_>> { diff --git a/crates/tracedecay-agent-hosts/src/ports/pricing.rs b/crates/tracedecay-agent-hosts/src/ports/pricing.rs deleted file mode 100644 index f9ce92b59a..0000000000 --- a/crates/tracedecay-agent-hosts/src/ports/pricing.rs +++ /dev/null @@ -1,26 +0,0 @@ -//! Model turn pricing. -//! -//! Context Scout reads the same all-provider authority as costs, CLI, MCP, and -//! HTTP. Unknown models stay unavailable instead of becoming zero-dollar work. - -/// Dollar cost of one turn, or `None` when its provider/model is unpriced. -#[must_use] -pub fn cost_of_turn( - provider: &str, - model: &str, - input_tokens: u64, - output_tokens: u64, - cache_write_tokens: u64, - cache_read_tokens: u64, -) -> Option { - let table = tracedecay_session_memory::provider_pricing::load_table(); - tracedecay_session_memory::provider_pricing::cost_of_usage( - table, - provider, - model, - input_tokens, - output_tokens, - Some(cache_read_tokens), - Some(cache_write_tokens), - ) -} diff --git a/crates/tracedecay-agent-hosts/src/tool_name.rs b/crates/tracedecay-agent-hosts/src/tool_name.rs index 245fbb461b..9642c4c7c9 100644 --- a/crates/tracedecay-agent-hosts/src/tool_name.rs +++ b/crates/tracedecay-agent-hosts/src/tool_name.rs @@ -13,8 +13,3 @@ /// rather than `tracedecay` so the host UI renders `plugin tracedecay graph` /// instead of the redundant `plugin tracedecay tracedecay`. pub const PLUGIN_TOOL_PREFIX: &str = "mcp__plugin_tracedecay_graph__"; - -/// Legacy config-managed namespace. It does NOT match the plugin namespace, so -/// an install that wrote only these entries prompted interactively on every -/// plugin tool call; the installer now writes the plugin-namespace twins too. -pub const LEGACY_TOOL_PREFIX: &str = "mcp__tracedecay__"; diff --git a/crates/tracedecay-agent-hosts/tests/context_scout_evidence.rs b/crates/tracedecay-agent-hosts/tests/context_scout_evidence.rs index 8e9875b064..69a77cd967 100644 --- a/crates/tracedecay-agent-hosts/tests/context_scout_evidence.rs +++ b/crates/tracedecay-agent-hosts/tests/context_scout_evidence.rs @@ -1,6 +1,6 @@ use std::collections::BTreeSet; -use tracedecay_agent_hosts::agents::context_scout::ports::context_scout_candidates_from_publication; +use tracedecay_agent_hosts::agents::context_scout::address_registry::context_scout_candidates_from_publication; use tracedecay_agent_hosts::agents::context_scout::{ ContextScoutControlV1, ContextScoutDecisionV1, ContextScoutEvidenceEnvelopeExt, ContextScoutLimitsV1, ContextScoutRuntimeModeV1, ContextScoutSelectionInputV1, diff --git a/crates/tracedecay-agent-hosts/tests/hooks_boundary.rs b/crates/tracedecay-agent-hosts/tests/hooks_boundary.rs index 4626d6d8e4..76e47a377f 100644 --- a/crates/tracedecay-agent-hosts/tests/hooks_boundary.rs +++ b/crates/tracedecay-agent-hosts/tests/hooks_boundary.rs @@ -8,11 +8,10 @@ use tracedecay_agent_hosts::hooks::{ }; use tracedecay_agent_hosts::ports::hook_runtime; use tracedecay_contracts::ResolvedScope; +use tracedecay_domain::NativeHostIdentityV1; use tracedecay_domain::errors::TraceDecayError; use tracedecay_domain::{ProjectId, UtcMicros}; -use tracedecay_hooks::{ - DaemonHookEvent, HookHostV1, NativeHookCaptureSourceV1, NativeHookDecodeError, -}; +use tracedecay_hooks::{DaemonHookEvent, NativeHookCaptureSourceV1, NativeHookDecodeError}; use tracedecay_runtime_core::storage::StoreLayout; #[test] @@ -51,7 +50,7 @@ fn native_identity_ignores_provider_content_but_preserves_typed_ids() { "permission_mode":"default","stop_hook_active":false, "last_assistant_message":"secret two" }"#; - let source = NativeHookCaptureSourceV1::Host(HookHostV1::Codex); + let source = NativeHookCaptureSourceV1::Host(NativeHostIdentityV1::Codex); let first = native_capture_material(source, first, UtcMicros(42)).expect("first material"); let second = native_capture_material(source, second, UtcMicros(42)).expect("second material"); @@ -72,7 +71,7 @@ fn installed_but_unsupported_events_remain_successful_noop_candidates() { assert!(matches!( native_capture_material( - NativeHookCaptureSourceV1::Host(HookHostV1::Codex), + NativeHookCaptureSourceV1::Host(NativeHostIdentityV1::Codex), codex_subagent, UtcMicros(42), ), @@ -80,7 +79,7 @@ fn installed_but_unsupported_events_remain_successful_noop_candidates() { )); assert!(matches!( native_capture_material( - NativeHookCaptureSourceV1::Host(HookHostV1::CursorDesktop), + NativeHookCaptureSourceV1::Host(NativeHostIdentityV1::CursorDesktop), cursor_session_end, UtcMicros(42), ), diff --git a/crates/tracedecay-api/Cargo.toml b/crates/tracedecay-api/Cargo.toml index f5e3ca8ac2..d12de76a55 100644 --- a/crates/tracedecay-api/Cargo.toml +++ b/crates/tracedecay-api/Cargo.toml @@ -14,7 +14,7 @@ doctest = false axum = "0.8" futures-util = "0.3" hotpath.workspace = true -schemars = "1.2.1" +schemars.workspace = true serde = { version = "1", features = ["derive"] } serde_json = "1" thiserror = "2" diff --git a/crates/tracedecay-api/src/doctor.rs b/crates/tracedecay-api/src/doctor.rs index 3353f3ddab..416df25f50 100644 --- a/crates/tracedecay-api/src/doctor.rs +++ b/crates/tracedecay-api/src/doctor.rs @@ -12,9 +12,9 @@ use std::fmt; use serde::Deserialize; use tracedecay_contracts::doctor::{ - DoctorCoverageCompletenessV1, DoctorEvidenceStateV1, DoctorFamilyConsultationV1, - DoctorFamilyUnavailableReasonV1, DoctorFindingFamilyV1, DoctorReportCoverageV1, - DoctorReportEntryV1, DoctorReportV1, + DOCTOR_FINDING_FAMILIES, DoctorCoverageCompletenessV1, DoctorEvidenceStateV1, + DoctorFamilyConsultationV1, DoctorFamilyUnavailableReasonV1, DoctorFindingFamilyV1, + DoctorReportCoverageV1, DoctorReportEntryV1, DoctorReportV1, }; use crate::read_model::{ @@ -30,15 +30,9 @@ pub const DOCTOR_FINDINGS_REFRESH_OPERATION: &str = "use-case.dashboard.doctor.f pub const DOCTOR_REPORT_SOURCE_UNSUPPORTED_NOTE: &str = "no admitted Doctor report source is available for this dashboard scope"; -/// The closed Doctor finding-family vocabulary the read routes project. -pub use tracedecay_contracts::doctor::DOCTOR_FINDING_FAMILIES as KNOWN_DOCTOR_FINDING_FAMILIES; - /// Path of the Doctor finding read route, filtered by the caller's query. pub const DOCTOR_FINDINGS_ROUTE_PATH: &str = "/api/doctor/findings"; -/// Path of the storage-family compatibility projection of the same report. -pub const STORAGE_FINDINGS_ROUTE_PATH: &str = "/api/storage/findings"; - /// Query DTO for the Doctor findings read route. /// /// Unknown query parameters are ignored rather than rejected; only `family` is @@ -221,10 +215,9 @@ fn family_coverage( } match report.coverage().completeness() { - DoctorCoverageCompletenessV1::Complete => DashboardCoverageV1::complete( - KNOWN_DOCTOR_FINDING_FAMILIES.len() as u64, - "doctor_families", - ), + DoctorCoverageCompletenessV1::Complete => { + DashboardCoverageV1::complete(DOCTOR_FINDING_FAMILIES.len() as u64, "doctor_families") + } DoctorCoverageCompletenessV1::Partial => { let consulted = report .coverage() @@ -250,7 +243,7 @@ fn family_coverage( }) .collect(); DashboardCoverageV1::partial( - KNOWN_DOCTOR_FINDING_FAMILIES.len() as u64, + DOCTOR_FINDING_FAMILIES.len() as u64, consulted, "doctor_families", omissions, diff --git a/crates/tracedecay-api/src/http.rs b/crates/tracedecay-api/src/http.rs index d5ce08bdcc..77458de5cc 100644 --- a/crates/tracedecay-api/src/http.rs +++ b/crates/tracedecay-api/src/http.rs @@ -13,8 +13,8 @@ use serde_json::Value; use tracedecay_contracts::{ ApplicationContractError, ApplicationProblem, ApplicationProblemEnvelope, ApplicationProblemKind, CancellationSignal, Deadline, OpaqueCursor, PageRequest, - ProblemOwningLayer, RequestId, ResultContractRef, RetainedSurfaceOperation, RetryDirective, - SafeDiagnostic, application_operation_default_page_size, + ProblemOwningLayer, RequestId, ResultContractRef, RetryDirective, SafeDiagnostic, + application_operation_default_page_size, }; use tracedecay_tool_catalog::{ ApplicationSurfaceOperation, BindingSurface, CapabilityId, CatalogSnapshotV1, FeatureId, @@ -100,6 +100,7 @@ pub enum HttpApplicationOwnerKind { Observatory, Configuration, ContextScout, + Retained, } /// Whether the operation is addressed under `/code/{operation}`. @@ -183,18 +184,12 @@ pub fn http_route_documents( authorized_capabilities, available_scope, ) { - let path = - match ApplicationSurfaceOperation::from_catalog_name(binding.operation().as_str()) { - Some(operation) => http_application_full_route_path(operation), - None => { - let Some(operation) = - RetainedSurfaceOperation::from_operation_name(binding.operation().as_str()) - else { - continue; - }; - crate::retained::retained_application_route_path(operation) - } - }; + let Some(operation) = + ApplicationSurfaceOperation::from_catalog_name(binding.operation().as_str()) + else { + continue; + }; + let path = http_application_full_route_path(operation); documents.push(HttpRouteDocumentV1 { method: "POST", path, @@ -277,6 +272,8 @@ pub trait HttpApplicationOwners: Clone + Send + Sync + 'static { &self, request: HttpApplicationRequest, ) -> HttpApplicationInvocationFuture; + + fn invoke_retained(&self, request: HttpApplicationRequest) -> HttpApplicationInvocationFuture; } impl HttpApplicationOwners for F @@ -332,6 +329,10 @@ where ) -> HttpApplicationInvocationFuture { Box::pin((self)(request)) } + + fn invoke_retained(&self, request: HttpApplicationRequest) -> HttpApplicationInvocationFuture { + Box::pin((self)(request)) + } } pub fn application_problem_status(kind: ApplicationProblemKind) -> StatusCode { @@ -477,6 +478,7 @@ where "/native-integration/{operation}", post(native_integration_operation::), ) + .route("/retained/{operation}", post(retained_operation::)) .layer(DefaultBodyLimit::max(MAX_HTTP_APPLICATION_BODY_BYTES)) .with_state(owners) } @@ -518,7 +520,7 @@ where fn parse_git_read_operation(operation: &str) -> Option { ApplicationSurfaceOperation::from_catalog_name(&format!("git_{operation}")).filter( - |operation| http_application_owner_kind(*operation) == HttpApplicationOwnerKind::Git, + |operation| http_application_owner_kind(*operation) == Some(HttpApplicationOwnerKind::Git), ) } @@ -528,7 +530,9 @@ fn parse_feedback_read_operation(operation: &str) -> Option Option { ApplicationSurfaceOperation::from_catalog_name(&format!("feedback_{operation}")).filter( - |operation| http_application_owner_kind(*operation) == HttpApplicationOwnerKind::Feedback, + |operation| { + http_application_owner_kind(*operation) == Some(HttpApplicationOwnerKind::Feedback) + }, ) } @@ -549,7 +553,7 @@ constant_operation_handlers! { fn parse_primitive_read_operation(operation: &str) -> Option { ApplicationSurfaceOperation::from_catalog_name(operation).filter(|operation| { - http_application_owner_kind(*operation) == HttpApplicationOwnerKind::Primitive + http_application_owner_kind(*operation) == Some(HttpApplicationOwnerKind::Primitive) && *operation != ApplicationSurfaceOperation::TestResults && !is_callable_code_route(*operation) }) @@ -562,13 +566,13 @@ fn parse_callable_code_operation(operation: &str) -> Option Option { ApplicationSurfaceOperation::from_catalog_name(operation).filter(|operation| { - http_application_owner_kind(*operation) == HttpApplicationOwnerKind::Configuration + http_application_owner_kind(*operation) == Some(HttpApplicationOwnerKind::Configuration) }) } fn parse_context_scout_operation(operation: &str) -> Option { ApplicationSurfaceOperation::from_catalog_name(operation).filter(|operation| { - http_application_owner_kind(*operation) == HttpApplicationOwnerKind::ContextScout + http_application_owner_kind(*operation) == Some(HttpApplicationOwnerKind::ContextScout) }) } @@ -613,11 +617,18 @@ parsed_operation_handlers! { configuration_operation => parse_configuration_operation; context_scout_operation => parse_context_scout_operation; native_integration_operation => parse_native_integration_operation; + retained_operation => parse_retained_operation; +} + +fn parse_retained_operation(operation: &str) -> Option { + ApplicationSurfaceOperation::from_catalog_name(operation).filter(|operation| { + http_application_owner_kind(*operation) == Some(HttpApplicationOwnerKind::Retained) + }) } fn parse_native_integration_operation(operation: &str) -> Option { ApplicationSurfaceOperation::from_catalog_name(operation).filter(|operation| { - http_application_owner_kind(*operation) == HttpApplicationOwnerKind::NativeIntegration + http_application_owner_kind(*operation) == Some(HttpApplicationOwnerKind::NativeIntegration) }) } @@ -648,7 +659,12 @@ where Ok(request) => request, Err(response) => return *response, }; - let owner_kind = http_application_owner_kind(request.operation); + let Some(owner_kind) = http_application_owner_kind(request.operation) else { + return adapter_problem_response( + request.request_id, + ApplicationProblem::not_found_or_not_authorized(RetryDirective::Never), + ); + }; let invocation = match owner_kind { HttpApplicationOwnerKind::Git => owners.invoke_git(request), HttpApplicationOwnerKind::Feedback => owners.invoke_feedback(request), @@ -658,6 +674,7 @@ where HttpApplicationOwnerKind::Configuration => owners.invoke_configuration(request), HttpApplicationOwnerKind::ContextScout => owners.invoke_context_scout(request), HttpApplicationOwnerKind::NativeIntegration => owners.invoke_native_integration(request), + HttpApplicationOwnerKind::Retained => owners.invoke_retained(request), }; match hotpath::future!(invocation, label = "api.http.handler").await { Ok(result) => result.into_http_response(), diff --git a/crates/tracedecay-api/src/http/application_operation_owner.rs b/crates/tracedecay-api/src/http/application_operation_owner.rs index 80ad9467ce..2282f5f521 100644 --- a/crates/tracedecay-api/src/http/application_operation_owner.rs +++ b/crates/tracedecay-api/src/http/application_operation_owner.rs @@ -4,10 +4,11 @@ use tracedecay_tool_catalog::ApplicationSurfaceOperation; use super::HttpApplicationOwnerKind; +/// `None` for operations that have no HTTP binding. pub const fn http_application_owner_kind( operation: ApplicationSurfaceOperation, -) -> HttpApplicationOwnerKind { - match operation { +) -> Option { + Some(match operation { ApplicationSurfaceOperation::GitStatus | ApplicationSurfaceOperation::GitDiff | ApplicationSurfaceOperation::GitHistory @@ -90,5 +91,51 @@ pub const fn http_application_owner_kind( | ApplicationSurfaceOperation::ContextScoutFeedback => { HttpApplicationOwnerKind::ContextScout } - } + ApplicationSurfaceOperation::StrReplace + | ApplicationSurfaceOperation::MultiStrReplace + | ApplicationSurfaceOperation::InsertAt + | ApplicationSurfaceOperation::AstGrepRewrite + | ApplicationSurfaceOperation::ReplaceSymbol + | ApplicationSurfaceOperation::InsertAtSymbol + | ApplicationSurfaceOperation::MoveSymbol + | ApplicationSurfaceOperation::RenameSymbol + | ApplicationSurfaceOperation::SourceEditReconcile + | ApplicationSurfaceOperation::SourceEditRollback + | ApplicationSurfaceOperation::Context + | ApplicationSurfaceOperation::Node + | ApplicationSurfaceOperation::Impact + | ApplicationSurfaceOperation::Similar + | ApplicationSurfaceOperation::Redundancy + | ApplicationSurfaceOperation::RenamePreview + | ApplicationSurfaceOperation::PortStatus + | ApplicationSurfaceOperation::PortOrder + | ApplicationSurfaceOperation::Todos => return None, + ApplicationSurfaceOperation::FactStoreCurate + | ApplicationSurfaceOperation::FactStoreAdd + | ApplicationSurfaceOperation::FactStoreSearch + | ApplicationSurfaceOperation::FactStoreProbe + | ApplicationSurfaceOperation::FactStoreRelated + | ApplicationSurfaceOperation::FactStoreReason + | ApplicationSurfaceOperation::FactStoreContradict + | ApplicationSurfaceOperation::FactStoreGet + | ApplicationSurfaceOperation::FactStoreUpdate + | ApplicationSurfaceOperation::FactStoreRemove + | ApplicationSurfaceOperation::FactStoreSupersede + | ApplicationSurfaceOperation::FactStoreList + | ApplicationSurfaceOperation::FactFeedback + | ApplicationSurfaceOperation::MemoryStatus + | ApplicationSurfaceOperation::SessionRefreshStatus + | ApplicationSurfaceOperation::SessionRefreshCancel + | ApplicationSurfaceOperation::SessionRefreshBegin + | ApplicationSurfaceOperation::MessageSearch + | ApplicationSurfaceOperation::SessionsFor + | ApplicationSurfaceOperation::Workflows + | ApplicationSurfaceOperation::LcmStatus + | ApplicationSurfaceOperation::LcmDoctor + | ApplicationSurfaceOperation::LcmLoadSession + | ApplicationSurfaceOperation::LcmGrep + | ApplicationSurfaceOperation::LcmDescribe + | ApplicationSurfaceOperation::LcmExpand + | ApplicationSurfaceOperation::LcmExpandQuery => HttpApplicationOwnerKind::Retained, + }) } diff --git a/crates/tracedecay-api/src/http/tests.rs b/crates/tracedecay-api/src/http/tests.rs index ce175ad696..3c67f8ef65 100644 --- a/crates/tracedecay-api/src/http/tests.rs +++ b/crates/tracedecay-api/src/http/tests.rs @@ -74,7 +74,7 @@ fn git_read_operation_parser_is_exact_and_read_only() { assert_eq!(parse_git_read_operation(route), Some(operation)); assert_eq!( http_application_owner_kind(operation), - HttpApplicationOwnerKind::Git + Some(HttpApplicationOwnerKind::Git) ); assert_eq!(operation.as_str(), format!("git_{route}")); } @@ -101,7 +101,7 @@ fn feedback_read_operation_parser_is_exact_and_separately_owned() { assert_eq!(parse_feedback_read_operation(route), Some(operation)); assert_eq!( http_application_owner_kind(operation), - HttpApplicationOwnerKind::Feedback + Some(HttpApplicationOwnerKind::Feedback) ); assert_eq!(operation.as_str(), format!("feedback_{route}")); } @@ -181,7 +181,7 @@ fn callable_code_operation_parser_is_exact_and_separately_owned() { ] { assert_eq!(parse_callable_code_operation(name), Some(operation)); assert_eq!(operation.as_str(), name); - assert_eq!(http_application_owner_kind(operation), owner); + assert_eq!(http_application_owner_kind(operation), Some(owner)); } for rejected in [ "", @@ -253,7 +253,7 @@ fn configuration_operation_parser_is_exact_and_closed() { ); assert_eq!( http_application_owner_kind(operation), - super::HttpApplicationOwnerKind::Configuration + Some(super::HttpApplicationOwnerKind::Configuration) ); } for rejected in [ @@ -312,7 +312,7 @@ fn context_scout_operation_parser_is_exact_and_backend_only() { ); assert_eq!( http_application_owner_kind(operation), - HttpApplicationOwnerKind::ContextScout + Some(HttpApplicationOwnerKind::ContextScout) ); } assert_eq!(parse_context_scout_operation("context_scout"), None); @@ -348,15 +348,15 @@ fn canonical_operation_authority_covers_all_surface_names_and_git_mutations() { ); assert_eq!( http_application_owner_kind(ApplicationSurfaceOperation::ObservatoryRead), - HttpApplicationOwnerKind::Observatory + Some(HttpApplicationOwnerKind::Observatory) ); assert_eq!( http_application_owner_kind(ApplicationSurfaceOperation::GitPreview), - HttpApplicationOwnerKind::Git + Some(HttpApplicationOwnerKind::Git) ); assert_eq!( http_application_owner_kind(ApplicationSurfaceOperation::GitApply), - HttpApplicationOwnerKind::Git + Some(HttpApplicationOwnerKind::Git) ); assert!( is_http_application_operation_exposed(ApplicationSurfaceOperation::GitHubStackSignalExpand) diff --git a/crates/tracedecay-api/src/lib.rs b/crates/tracedecay-api/src/lib.rs index 99b56d1671..c8b20c1c9b 100644 --- a/crates/tracedecay-api/src/lib.rs +++ b/crates/tracedecay-api/src/lib.rs @@ -57,11 +57,7 @@ pub use multi_root::{ MultiRootApplicationOwner, MultiRootHttpOperation, MultiRootHttpRequest, MultiRootInvocationFuture, multi_root_application_router, }; -pub use retained::{ - RetainedApplicationOwner, RetainedHttpRequest, RetainedInvocationFuture, - retained_application_route_path, retained_application_router, - retained_invalid_request_response, retained_operation_id, retained_route_path, -}; +pub use retained::{retained_invalid_request_response, retained_route_path}; pub use sse::sse_response; pub use work::{ WorkApplicationOwner, WorkHttpRequest, WorkInvocationFuture, WorkOperation, @@ -319,7 +315,7 @@ mod tests { fn http_operations_dispatch_to_concrete_owner_families() { assert_eq!( http_application_owner_kind(ApplicationSurfaceOperation::DiagnosticsRead), - HttpApplicationOwnerKind::Primitive + Some(HttpApplicationOwnerKind::Primitive) ); for operation in [ "multi_root_scope_set_read", @@ -529,7 +525,8 @@ mod tests { for (index, operation) in ApplicationSurfaceOperation::ALL .into_iter() .filter(|operation| { - http_application_owner_kind(*operation) == HttpApplicationOwnerKind::Configuration + http_application_owner_kind(*operation) + == Some(HttpApplicationOwnerKind::Configuration) }) .enumerate() { diff --git a/crates/tracedecay-api/src/retained.rs b/crates/tracedecay-api/src/retained.rs index c852cff1cd..6408364570 100644 --- a/crates/tracedecay-api/src/retained.rs +++ b/crates/tracedecay-api/src/retained.rs @@ -1,120 +1,15 @@ -//! Canonical public HTTP adapter for retained application operations. +//! Public HTTP helpers shared by the retained application operations. -use std::future::Future; -use std::pin::Pin; - -use axum::extract::rejection::JsonRejection; -use axum::extract::{DefaultBodyLimit, Extension, State}; use axum::response::Response; -use axum::routing::post; -use axum::{Json, Router}; -use serde_json::Value; use tracedecay_contracts::RequestId; use tracedecay_contracts::retained_surfaces::RetainedSurfaceOperation; -use crate::http::{ - HttpApplicationControls, MAX_HTTP_APPLICATION_BODY_BYTES, invalid_request_response, -}; - -pub fn retained_operation_id(operation: RetainedSurfaceOperation) -> String { - format!("operation.application.{}", operation.as_str()) -} +use crate::http::invalid_request_response; pub fn retained_route_path(operation: RetainedSurfaceOperation) -> String { format!("/retained/{}", operation.as_str()) } -pub fn retained_application_route_path(operation: RetainedSurfaceOperation) -> String { - format!("/application{}", retained_route_path(operation)) -} - -#[derive(Clone, Debug)] -pub struct RetainedHttpRequest { - pub operation: RetainedSurfaceOperation, - pub request_id: RequestId, - pub controls: HttpApplicationControls, - pub body: Value, -} - -pub type RetainedInvocationFuture = Pin + Send>>; - -pub trait RetainedApplicationOwner: Clone + Send + Sync + 'static { - fn invoke_retained(&self, request: RetainedHttpRequest) -> RetainedInvocationFuture; -} - -impl RetainedApplicationOwner for F -where - F: Fn(RetainedHttpRequest) -> Fut + Clone + Send + Sync + 'static, - Fut: Future + Send + 'static, -{ - fn invoke_retained(&self, request: RetainedHttpRequest) -> RetainedInvocationFuture { - Box::pin((self)(request)) - } -} - -/// Registers one explicit `POST` route per callable retained operation, so -/// the routing table itself is the per-binding mount authority: a callable -/// operation's path answers method-mismatch (`405`) probes, and an unknown or -/// non-callable segment answers the router's own `404` instead of a handler's -/// concealed problem envelope. -pub fn retained_application_router(owner: O) -> Router -where - O: RetainedApplicationOwner, -{ - let mut router = Router::new(); - for operation in RetainedSurfaceOperation::CALLABLE { - router = router.route( - &retained_route_path(operation), - post( - move |State(owner): State, - Extension(request_id): Extension, - Extension(controls): Extension, - body: Result, JsonRejection>| { - invoke(operation, owner, request_id, controls, body) - }, - ), - ); - } - router - .layer(DefaultBodyLimit::max(MAX_HTTP_APPLICATION_BODY_BYTES)) - .with_state(owner) -} - -async fn invoke( - operation: RetainedSurfaceOperation, - owner: O, - request_id: RequestId, - controls: HttpApplicationControls, - body: Result, JsonRejection>, -) -> Response -where - O: RetainedApplicationOwner, -{ - let request = match hotpath::measure_block!("api.http.admission", { - match body { - Ok(Json(body)) => Ok(RetainedHttpRequest { - operation, - request_id, - controls, - body, - }), - Err(_) => Err(invalid_request_response( - request_id, - "retained.invalid_body", - "The retained application request body is invalid or exceeds the configured limit", - )), - } - }) { - Ok(request) => request, - Err(response) => return response, - }; - hotpath::future!( - async move { owner.invoke_retained(request).await }, - label = "api.http.handler" - ) - .await -} - pub fn retained_invalid_request_response(request_id: RequestId) -> Response { invalid_request_response( request_id, diff --git a/crates/tracedecay-api/src/work.rs b/crates/tracedecay-api/src/work.rs index 65b1aec8db..f7b359f3bc 100644 --- a/crates/tracedecay-api/src/work.rs +++ b/crates/tracedecay-api/src/work.rs @@ -557,40 +557,4 @@ mod tests { ); } } - - #[test] - fn execution_admission_publishes_the_snapshot_consumed_by_attempt_start() { - for operation in [ - WorkOperation::Create, - WorkOperation::ReviewProposal, - WorkOperation::AcceptProposal, - ] { - assert_eq!( - operation.result_schema_name(), - "WorkProductMutationReceiptV1", - "{}", - operation.operation_key() - ); - } - assert_eq!( - WorkOperation::Create.request_schema_name(), - "CreateWorkTaskRequestV1" - ); - assert_eq!( - WorkOperation::ReviewProposal.request_schema_name(), - "ReviewWorkProposalRequestV1" - ); - assert_eq!( - WorkOperation::AcceptProposal.request_schema_name(), - "AcceptWorkProposalRequestV1" - ); - assert_eq!( - WorkOperation::AdmitExecution.request_schema_name(), - "AdmitWorkExecutionRequestV1" - ); - assert_eq!( - WorkOperation::AdmitExecution.result_schema_name(), - "AdmittedWorkExecutionV1" - ); - } } diff --git a/crates/tracedecay-api/tests/handoff_routes.rs b/crates/tracedecay-api/tests/handoff_routes.rs index ca01ce2ee5..b59d29bb24 100644 --- a/crates/tracedecay-api/tests/handoff_routes.rs +++ b/crates/tracedecay-api/tests/handoff_routes.rs @@ -9,37 +9,6 @@ use tracedecay_api::{HandoffOperation, HttpApplicationControls, handoff_applicat use tracedecay_contracts::{CancellationSignal, Deadline, RequestId}; use tracedecay_domain::UtcMicros; -#[test] -fn descriptor_matches_the_typed_handoff_registry_routes() { - assert_eq!( - HandoffOperation::ALL - .into_iter() - .map(|operation| ( - operation.operation_id_str(), - operation.application_route_path() - )) - .collect::>(), - vec![ - ( - "operation.handoff.issue_task_handoff", - "/application/handoff/issue-task", - ), - ( - "operation.handoff.list_task_handoffs", - "/application/handoff/list-task", - ), - ( - "operation.handoff.open_investigation_handoff", - "/application/handoff/open-investigation", - ), - ( - "operation.handoff.open_task_handoff", - "/application/handoff/open-task", - ), - ] - ); -} - #[tokio::test] async fn router_dispatches_every_operation_to_one_application_owner() { let observed = Arc::new(Mutex::new(Vec::new())); diff --git a/crates/tracedecay-application/Cargo.toml b/crates/tracedecay-application/Cargo.toml index b90b82d8c9..584136db46 100644 --- a/crates/tracedecay-application/Cargo.toml +++ b/crates/tracedecay-application/Cargo.toml @@ -36,7 +36,7 @@ glob = "0.3" hex = "0.4" hotpath.workspace = true same-file = "1" -schemars = "1.2.1" +schemars.workspace = true serde = { version = "1", features = ["derive"] } serde_json = "1" sha2 = "0.11" diff --git a/crates/tracedecay-application/src/advisory/ci_runtime.rs b/crates/tracedecay-application/src/advisory/ci_runtime.rs index 00d3be333b..a67b05844d 100644 --- a/crates/tracedecay-application/src/advisory/ci_runtime.rs +++ b/crates/tracedecay-application/src/advisory/ci_runtime.rs @@ -138,17 +138,12 @@ const fn state_matches_coverage( ) } -pub type GitHubCiWorkflowRunV1 = GitHubActionsWorkflowRunV1; -pub type GitHubCiCheckRunV1 = GitHubActionsCheckRunV1; -pub type GitHubCiCheckAnnotationV1 = GitHubCheckAnnotationV1; -pub type GitHubCiAnnotationLevelV1 = GitHubCheckAnnotationLevelV1; - #[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct GitHubCiProviderRecordV1 { - pub workflow_run: GitHubCiWorkflowRunV1, + pub workflow_run: GitHubActionsWorkflowRunV1, pub workflow_job: GitHubActionsWorkflowJobV1, - pub check_run: GitHubCiCheckRunV1, - pub annotations: Vec, + pub check_run: GitHubActionsCheckRunV1, + pub annotations: Vec, } impl GitHubCiProviderRecordV1 { @@ -163,7 +158,7 @@ impl GitHubCiProviderRecordV1 { .min_by_key(|step| step.number) } - pub fn failed_annotation(&self) -> Option<&GitHubCiCheckAnnotationV1> { + pub fn failed_annotation(&self) -> Option<&GitHubCheckAnnotationV1> { self.annotations .iter() .filter(|annotation| { @@ -202,20 +197,20 @@ impl GitHubCiOfficialResponseDecoderV1 { annotations: &str, ) -> Result { Ok(GitHubCiProviderRecordV1 { - workflow_run: serde_json::from_str::>( - workflow_run, - )? + workflow_run: serde_json::from_str::< + GitHubRetainedResponseV1, + >(workflow_run)? .response, workflow_job: serde_json::from_str::< GitHubRetainedResponseV1, >(workflow_job)? .response, - check_run: serde_json::from_str::>( + check_run: serde_json::from_str::>( check_run, )? .response, annotations: serde_json::from_str::< - GitHubRetainedResponseV1>, + GitHubRetainedResponseV1>, >(annotations)? .response, }) diff --git a/crates/tracedecay-application/src/advisory/github_runtime/access.rs b/crates/tracedecay-application/src/advisory/github_runtime/access.rs index c023c0a0c0..30604a1385 100644 --- a/crates/tracedecay-application/src/advisory/github_runtime/access.rs +++ b/crates/tracedecay-application/src/advisory/github_runtime/access.rs @@ -2,7 +2,7 @@ use tracedecay_contracts::feedback::{ CI_FAILURE_LOCALIZE_CAPABILITY_ID_V1, FeedbackPortFuture, GITHUB_REVIEW_INGEST_CAPABILITY_ID_V1, GitHubReviewReadRequestV1, feedback_surface_operation, }; -use tracedecay_contracts::{AuthorizationPhase, AuthorizationRequest, ResolvedScope, now_micros}; +use tracedecay_contracts::{AuthorizationRequest, ResolvedScope, now_micros}; use tracedecay_domain::configuration::SourceKindV1; use tracedecay_domain::{LocatorDigest, canonical_sha256}; @@ -69,7 +69,6 @@ where let authorization = AuthorizationRequest { context, operation: &operation, - phase: AuthorizationPhase::Admission, observed_at, }; match project_source_access_snapshot_for_request( @@ -128,7 +127,6 @@ where let authorization = AuthorizationRequest { context, operation: &operation, - phase: AuthorizationPhase::Admission, observed_at, }; match project_source_access_snapshot_for_request( diff --git a/crates/tracedecay-application/src/advisory/github_runtime/network/stack_network.rs b/crates/tracedecay-application/src/advisory/github_runtime/network/stack_network.rs index e7abb94fb7..64f96ecb34 100644 --- a/crates/tracedecay-application/src/advisory/github_runtime/network/stack_network.rs +++ b/crates/tracedecay-application/src/advisory/github_runtime/network/stack_network.rs @@ -148,8 +148,7 @@ mod tests { use serde_json::json; use tracedecay_contracts::feedback::{FeedbackPortFuture, GitHubReviewReadRequestV1}; use tracedecay_contracts::retrieval::{ - GitTopologyAnchorAuthorityV2, GitTopologyAnchorResolutionOutcomeV2, - GitTopologyAnchorResolutionV2, + GitTopologyAnchorAuthority, GitTopologyAnchorResolution, GitTopologyAnchorResolutionOutcome, }; use tracedecay_contracts::{RequestContext, now_micros}; use tracedecay_domain::ObservationScopeV1; @@ -476,13 +475,13 @@ mod tests { lineage.extend_from_slice(snapshot.source_anchors()); for source in lineage { let store = - tracedecay_global_db::RegisteredGitTopologyAnchorAuthorityV2::new(database.clone()); + tracedecay_global_db::RegisteredGitTopologyAnchorAuthority::new(database.clone()); let owner = ObservationScopeV1::Project { project_id: scope.project_id.clone(), }; - let Ok(GitTopologyAnchorResolutionOutcomeV2::Resolved(source_record)) = store + let Ok(GitTopologyAnchorResolutionOutcome::Resolved(source_record)) = store .resolve( - GitTopologyAnchorResolutionV2::new(owner.clone(), source.anchor_id().clone()) + GitTopologyAnchorResolution::new(owner.clone(), source.anchor_id().clone()) .unwrap(), ) .await @@ -493,14 +492,14 @@ mod tests { assert!(matches!( store .resolve( - GitTopologyAnchorResolutionV2::new( + GitTopologyAnchorResolution::new( owner.clone(), nested.anchor_id().clone(), ) .unwrap(), ) .await, - Ok(GitTopologyAnchorResolutionOutcomeV2::Resolved(_)) + Ok(GitTopologyAnchorResolutionOutcome::Resolved(_)) )); } } diff --git a/crates/tracedecay-application/src/advisory/github_runtime/stack_anchors.rs b/crates/tracedecay-application/src/advisory/github_runtime/stack_anchors.rs index 8d69419bbb..356e66a8fe 100644 --- a/crates/tracedecay-application/src/advisory/github_runtime/stack_anchors.rs +++ b/crates/tracedecay-application/src/advisory/github_runtime/stack_anchors.rs @@ -10,23 +10,23 @@ use tracedecay_contracts::feedback::{ GitHubReviewReadRequestV1, }; use tracedecay_contracts::retrieval::{ - GitTopologyAnchorAuthorityErrorV2, GitTopologyAnchorAuthorityV2, - GitTopologyAnchorPublicationOutcomeV2, GitTopologyAnchorPublicationV2, - GitTopologyAnchorResolutionOutcomeV2, GitTopologyAnchorResolutionV2, + GitTopologyAnchorAuthority, GitTopologyAnchorAuthorityError, GitTopologyAnchorPublication, + GitTopologyAnchorPublicationOutcome, GitTopologyAnchorResolution, + GitTopologyAnchorResolutionOutcome, }; use tracedecay_domain::feedback::FeedbackScopeV1; use tracedecay_domain::{ - AccessPolicyDigest, AnchorDurabilityClass, AnchorLineageRefV2, AnchorOwnerBindingV1, - AnchorProvenanceRelationV2, AnchorSourceGenerationV2, CapabilityId, CoverageReportV1, + AccessPolicyDigest, AnchorDurabilityClass, AnchorLineageRef, AnchorOwnerBindingV1, + AnchorProvenanceRelation, AnchorSourceGeneration, CapabilityId, CoverageReportV1, EvidenceClass, GitHubStackCapabilityStateV1, GitTopologyAnchorTargetV1, GitTopologySourceRoleV1, ObservationScopeV1, OrderedGitTopologySourceV1, PayloadAccessState, PrivacyDomainBoundLocatorDigest, PrivacyDomainId, ProjectionGenerationId, ProviderId, PullRequestSnapshotAnchorRefV1, RepositoryId, ResolutionAuthorizationV1, RetentionClass, - RetrievalAnchorId, RetrievalAnchorRecordV2, RetrievalAnchorRecordV2Parts, - RetrievalAnchorTargetV2, ScopeResolutionId, UserProfileId, UtcMicros, VectorWatermark, - canonical_sha256, sha256_hex_suffix, + RetrievalAnchorId, RetrievalAnchorRecord, RetrievalAnchorRecordParts, RetrievalAnchorTarget, + ScopeResolutionId, UserProfileId, UtcMicros, VectorWatermark, canonical_sha256, + sha256_hex_suffix, }; -use tracedecay_global_db::{RegisteredGitTopologyAnchorAuthorityV2, RegisteredGlobalDbLeaseV1}; +use tracedecay_global_db::{RegisteredGitTopologyAnchorAuthority, RegisteredGlobalDbLeaseV1}; use tracedecay_tool_catalog::{CapabilityId as GrantCapabilityId, UseCaseId as GrantUseCaseId}; use super::stack::DecodedGitHubStackSnapshotV1; @@ -50,7 +50,7 @@ pub enum GitHubStackAnchorPublicationOutcomeV1 { #[derive(Clone, Debug, PartialEq, Eq)] pub enum GitHubStackAnchorReadOutcomeV1 { - Current(Box), + Current(Box), Denied, Stale, Unavailable, @@ -72,14 +72,14 @@ pub(super) trait GitHubStackReadAuthorityV1: Sync { #[serde(deny_unknown_fields)] pub struct GitHubStackDurableObservationV1 { pub observation: GitHubStackObservationV1, - pub capability_anchor: RetrievalAnchorRecordV2, - pub snapshot_anchor: Option, + pub capability_anchor: RetrievalAnchorRecord, + pub snapshot_anchor: Option, } #[derive(Clone)] pub struct ProjectGitHubStackAnchorAuthorityV1 { database: RegisteredGlobalDbLeaseV1, - anchors: Arc, + anchors: Arc, scope: FeedbackScopeV1, } @@ -87,9 +87,7 @@ impl ProjectGitHubStackAnchorAuthorityV1 { pub fn new(database: RegisteredGlobalDbLeaseV1, scope: FeedbackScopeV1) -> Option { scope.validate().ok()?; (database.binding().shard_id.scope.project_id() == Some(&scope.project_id)).then(|| Self { - anchors: Arc::new(RegisteredGitTopologyAnchorAuthorityV2::new( - database.clone(), - )), + anchors: Arc::new(RegisteredGitTopologyAnchorAuthority::new(database.clone())), database, scope, }) @@ -131,7 +129,7 @@ impl ProjectGitHubStackAnchorAuthorityV1 { .await?; if capability_anchor.owner() != &owner || capability_anchor.target() - != &RetrievalAnchorTargetV2::GitTopology(Box::new( + != &RetrievalAnchorTarget::GitTopology(Box::new( GitTopologyAnchorTargetV1::GitHubStackCapability( observation.capability.clone(), ), @@ -144,7 +142,7 @@ impl ProjectGitHubStackAnchorAuthorityV1 { let record = resolve_v2(self.anchors.as_ref(), &owner, anchor_id).await?; if record.owner() != &owner || record.target() - != &RetrievalAnchorTargetV2::GitTopology(Box::new( + != &RetrievalAnchorTarget::GitTopology(Box::new( GitTopologyAnchorTargetV1::GitHubStackSnapshot(snapshot.clone()), )) { @@ -245,14 +243,14 @@ impl ProjectGitHubStackAnchorAuthorityV1 { let owner = ObservationScopeV1::Project { project_id: self.scope.project_id.clone(), }; - let Ok(publication) = GitTopologyAnchorPublicationV2::new(owner, records) else { + let Ok(publication) = GitTopologyAnchorPublication::new(owner, records) else { return GitHubStackAnchorPublicationOutcomeV1::Unavailable; }; let publication = match self.anchors.publish(publication).await { - Ok(GitTopologyAnchorPublicationOutcomeV2::Published) => { + Ok(GitTopologyAnchorPublicationOutcome::Published) => { GitHubStackAnchorPublicationOutcomeV1::Published } - Ok(GitTopologyAnchorPublicationOutcomeV2::Replayed) => { + Ok(GitTopologyAnchorPublicationOutcome::Replayed) => { GitHubStackAnchorPublicationOutcomeV1::Replayed } Err(_) => GitHubStackAnchorPublicationOutcomeV1::Unavailable, @@ -296,17 +294,16 @@ impl ProjectGitHubStackAnchorAuthorityV1 { let owner = ObservationScopeV1::Project { project_id: self.scope.project_id.clone(), }; - let Ok(resolution) = - GitTopologyAnchorResolutionV2::new(owner.clone(), anchor_id.clone()) + let Ok(resolution) = GitTopologyAnchorResolution::new(owner.clone(), anchor_id.clone()) else { return GitHubStackAnchorReadOutcomeV1::Denied; }; let record = match self.anchors.resolve(resolution).await { - Ok(GitTopologyAnchorResolutionOutcomeV2::Resolved(record)) => record, - Ok(GitTopologyAnchorResolutionOutcomeV2::Unavailable) - | Err(GitTopologyAnchorAuthorityErrorV2::Unavailable) - | Err(GitTopologyAnchorAuthorityErrorV2::ResetRequired) - | Err(GitTopologyAnchorAuthorityErrorV2::Conflict) => { + Ok(GitTopologyAnchorResolutionOutcome::Resolved(record)) => record, + Ok(GitTopologyAnchorResolutionOutcome::Unavailable) + | Err(GitTopologyAnchorAuthorityError::Unavailable) + | Err(GitTopologyAnchorAuthorityError::ResetRequired) + | Err(GitTopologyAnchorAuthorityError::Conflict) => { return GitHubStackAnchorReadOutcomeV1::Unavailable; } }; @@ -453,14 +450,14 @@ fn canonical_stack_position(provider_position: u32) -> Option { } async fn resolve_v2( - authority: &dyn GitTopologyAnchorAuthorityV2, + authority: &dyn GitTopologyAnchorAuthority, owner: &ObservationScopeV1, anchor_id: &RetrievalAnchorId, -) -> Option { - let resolution = GitTopologyAnchorResolutionV2::new(owner.clone(), anchor_id.clone()).ok()?; +) -> Option { + let resolution = GitTopologyAnchorResolution::new(owner.clone(), anchor_id.clone()).ok()?; match authority.resolve(resolution).await.ok()? { - GitTopologyAnchorResolutionOutcomeV2::Resolved(record) => Some(*record), - GitTopologyAnchorResolutionOutcomeV2::Unavailable => None, + GitTopologyAnchorResolutionOutcome::Resolved(record) => Some(*record), + GitTopologyAnchorResolutionOutcome::Unavailable => None, } } @@ -470,7 +467,7 @@ fn exact_commit_source_record( repository_id: &RepositoryId, commit_id: &tracedecay_domain::CommitId, ingested_at: UtcMicros, -) -> Option { +) -> Option { let digest = canonical_sha256(&( "tracedecay.github-stack.provider-commit-source.v1", owner, @@ -482,8 +479,8 @@ fn exact_commit_source_record( let mut source_authorization = authorization.clone(); source_authorization.canonical_request_digest = PrivacyDomainBoundLocatorDigest::new(digest.as_str()).ok()?; - RetrievalAnchorRecordV2::new(RetrievalAnchorRecordV2Parts { - target: RetrievalAnchorTargetV2::ExactRepositoryCommit { + RetrievalAnchorRecord::new(RetrievalAnchorRecordParts { + target: RetrievalAnchorTarget::ExactRepositoryCommit { repository_id: repository_id.clone(), commit_id: commit_id.clone(), }, @@ -492,7 +489,7 @@ fn exact_commit_source_record( occurred_at: None, ingested_at, evidence_class: EvidenceClass::ProviderDeclared, - source_generation: AnchorSourceGenerationV2::Unknown, + source_generation: AnchorSourceGeneration::Unknown, projection_generation: ProjectionGenerationId::new(format!( "generation.github-stack-source.{suffix}" )) @@ -527,7 +524,7 @@ fn build_records( context: &RequestContext, request: &GitHubReviewReadRequestV1, observation: &GitHubStackObservationV1, -) -> Option> { +) -> Option> { let authorization = authorization(profile_id, context, request)?; let owner = ObservationScopeV1::Project { project_id: observation.scope.project_id.clone(), @@ -616,8 +613,8 @@ fn build_records( } fn insert_record( - records: &mut BTreeMap, - record: RetrievalAnchorRecordV2, + records: &mut BTreeMap, + record: RetrievalAnchorRecord, ) -> Option<()> { match records.get(record.anchor_id()) { Some(existing) if existing.is_semantic_replay_of(&record) => Some(()), @@ -634,24 +631,24 @@ fn retrieval_record( target: GitTopologyAnchorTargetV1, ingested_at: UtcMicros, authorization: ResolutionAuthorizationV1, -) -> Option { +) -> Option { let mut seen = BTreeSet::new(); let source_anchors = target .ordered_sources() .iter() .filter(|source| seen.insert(source.anchor_id.clone())) .map(|source| { - AnchorLineageRefV2::new( - AnchorProvenanceRelationV2::Observed, + AnchorLineageRef::new( + AnchorProvenanceRelation::Observed, source.anchor_id.clone(), owner.clone(), ) .ok() }) .collect::>>()?; - let source_generation = AnchorSourceGenerationV2::GitTopology(target.generation()); + let source_generation = AnchorSourceGeneration::GitTopology(target.generation()); let projection_generation = match &source_generation { - AnchorSourceGenerationV2::GitTopology( + AnchorSourceGeneration::GitTopology( tracedecay_domain::GitTopologyGenerationRefV1::GitHubStackCapability { generation_id, .. @@ -661,7 +658,7 @@ fn retrieval_record( .. }, ) => generation_id.clone(), - AnchorSourceGenerationV2::GitTopology( + AnchorSourceGeneration::GitTopology( tracedecay_domain::GitTopologyGenerationRefV1::ProviderCommit { source_anchor_id, commit_id, @@ -681,8 +678,8 @@ fn retrieval_record( } _ => return None, }; - RetrievalAnchorRecordV2::new(RetrievalAnchorRecordV2Parts { - target: RetrievalAnchorTargetV2::GitTopology(Box::new(target)), + RetrievalAnchorRecord::new(RetrievalAnchorRecordParts { + target: RetrievalAnchorTarget::GitTopology(Box::new(target)), owner, aliases: Vec::new(), occurred_at: None, @@ -749,8 +746,8 @@ fn observation_matches_scope( && observation.capability.worktree_id == scope.worktree_id } -fn record_matches_scope(record: &RetrievalAnchorRecordV2, scope: &FeedbackScopeV1) -> bool { - let RetrievalAnchorTargetV2::GitTopology(target) = record.target() else { +fn record_matches_scope(record: &RetrievalAnchorRecord, scope: &FeedbackScopeV1) -> bool { + let RetrievalAnchorTarget::GitTopology(target) = record.target() else { return false; }; target.project_id() == &scope.project_id && target.repository_id() == &scope.repository_id diff --git a/crates/tracedecay-application/src/advisory/host_delivery_consume_tests.rs b/crates/tracedecay-application/src/advisory/host_delivery_consume_tests.rs index dda1718895..f4edcdc943 100644 --- a/crates/tracedecay-application/src/advisory/host_delivery_consume_tests.rs +++ b/crates/tracedecay-application/src/advisory/host_delivery_consume_tests.rs @@ -330,19 +330,6 @@ impl FeedbackImpactPort for FixedImpact { } } -#[derive(Clone)] -struct Observations(Arc); - -impl tracedecay_contracts::feedback::FeedbackObservationPort for Observations { - fn observe( - &self, - input: &tracedecay_domain::feedback::FeedbackEvaluationInputV1, - observation: tracedecay_domain::feedback::FeedbackCycleObservationV1, - ) { - self.0.observe(input, observation); - } -} - struct NoopFeedbackCycle; impl FeedbackCycleRuntimePort for NoopFeedbackCycle { @@ -505,7 +492,7 @@ async fn consume_fixture() -> ConsumeFixture { }, FixedImpact(impact), runtime.publication_store(), - Observations(runtime.observation_port()), + runtime.observation_port(), runtime.route_authorization(), operation, ); diff --git a/crates/tracedecay-application/src/advisory/mod.rs b/crates/tracedecay-application/src/advisory/mod.rs index 973172a09d..355a8b34fe 100644 --- a/crates/tracedecay-application/src/advisory/mod.rs +++ b/crates/tracedecay-application/src/advisory/mod.rs @@ -30,9 +30,8 @@ pub use ci_runtime::{ CiRetainedProviderObservationAuthorityV1, CiRetainedProviderObservationV1, CiRetainedProviderRecordV1, CiSourceAccessAuthorityV1, CiSourceAccessOutcomeV1, ConcreteCiFailureLocalizationOwnerV1, DaemonCiReadOnlyEvidenceSourceV1, - GitHubCiAnnotationLevelV1, GitHubCiCheckAnnotationV1, GitHubCiCheckRunV1, - GitHubCiOfficialResponseDecoderV1, GitHubCiProviderRecordV1, GitHubCiWorkflowRunV1, - MAX_CI_RETAINED_ANNOTATIONS_V1, MAX_CI_RETAINED_CHECKS_V1, MAX_CI_RETAINED_FAILURES_V1, + GitHubCiOfficialResponseDecoderV1, GitHubCiProviderRecordV1, MAX_CI_RETAINED_ANNOTATIONS_V1, + MAX_CI_RETAINED_CHECKS_V1, MAX_CI_RETAINED_FAILURES_V1, MAX_CI_RETAINED_OBSERVATION_MANIFEST_ENTRIES_V1, ProductionCiArchiveHandleV1, ProductionCiExactEvidenceHandleV1, ProductionCiFailureDiscoveryOutcomeV1, ProductionCiProviderAuthoritiesV1, ProductionCiProviderConfigV1, @@ -53,22 +52,23 @@ pub use github::{ pub use github_runtime::{ GITHUB_REVIEW_THREADS_QUERY_V1, GitHubActionsConclusionV1, GitHubActionsStatusV1, GitHubCanonicalReviewAnchorAuthorityV1, GitHubCanonicalReviewAnchorsV1, - GitHubCiRepositoryTargetV1, GitHubCiTransportOutcomeV1, GitHubGraphQlReadRequestV1, - GitHubHttpReadConfigV1, GitHubOfficialResponseDecoderV1, GitHubProviderLifecycleV1, - GitHubReadCheckpointAuthorityV1, GitHubReadCheckpointLoadOutcomeV1, - GitHubReadNetworkMetadataV1, GitHubReadNetworkOutcomeV1, GitHubReadNetworkResponseV1, - GitHubReadNetworkStatusV1, GitHubReadOnlyClientV1, GitHubReadOnlyCredentialAuthorityOutcomeV1, - GitHubReadOnlyCredentialAuthorityV1, GitHubReadOnlyCredentialSecretV1, - GitHubReadOnlyCredentialV1, GitHubReadOnlyNetworkAuthorityV1, GitHubReadOnlyRuntimeTransportV1, - GitHubReadPermissionV1, GitHubReadResponseDecoderV1, GitHubReadResumeV1, GitHubReleaseAssetV1, - GitHubReleaseReadControlV1, GitHubReleaseTagV1, GitHubReleaseV1, GitHubRepositoryTargetV1, - GitHubRestReadRequestV1, GitHubReviewAnchorSeedV1, GitHubReviewAtomicRefreshStoreV1, - GitHubReviewBodyEvidenceAuthorityV1, GitHubReviewBodyEvidenceV1, GitHubReviewBodyReadOutcomeV1, - GitHubReviewCompleteGenerationV1, GitHubReviewProviderIdentityV1, - GitHubReviewRefreshCoordinatorV1, GitHubReviewRefreshOutcomeV1, GitHubReviewRefreshReceiptV1, - GitHubReviewRefreshStateV1, GitHubReviewRefreshStoreCommitOutcomeV1, - GitHubReviewRefreshStoreReadOutcomeV1, GitHubReviewRuntimeOwnerBuildErrorV1, - GitHubReviewRuntimeOwnerConfigV1, GitHubReviewRuntimeOwnerV1, GitHubReviewStoreManifestEntryV1, + GitHubCheckAnnotationLevelV1, GitHubCheckAnnotationV1, GitHubCiRepositoryTargetV1, + GitHubCiTransportOutcomeV1, GitHubGraphQlReadRequestV1, GitHubHttpReadConfigV1, + GitHubOfficialResponseDecoderV1, GitHubProviderLifecycleV1, GitHubReadCheckpointAuthorityV1, + GitHubReadCheckpointLoadOutcomeV1, GitHubReadNetworkMetadataV1, GitHubReadNetworkOutcomeV1, + GitHubReadNetworkResponseV1, GitHubReadNetworkStatusV1, GitHubReadOnlyClientV1, + GitHubReadOnlyCredentialAuthorityOutcomeV1, GitHubReadOnlyCredentialAuthorityV1, + GitHubReadOnlyCredentialSecretV1, GitHubReadOnlyCredentialV1, GitHubReadOnlyNetworkAuthorityV1, + GitHubReadOnlyRuntimeTransportV1, GitHubReadPermissionV1, GitHubReadResponseDecoderV1, + GitHubReadResumeV1, GitHubReleaseAssetV1, GitHubReleaseReadControlV1, GitHubReleaseTagV1, + GitHubReleaseV1, GitHubRepositoryTargetV1, GitHubRestReadRequestV1, GitHubReviewAnchorSeedV1, + GitHubReviewAtomicRefreshStoreV1, GitHubReviewBodyEvidenceAuthorityV1, + GitHubReviewBodyEvidenceV1, GitHubReviewBodyReadOutcomeV1, GitHubReviewCompleteGenerationV1, + GitHubReviewProviderIdentityV1, GitHubReviewRefreshCoordinatorV1, GitHubReviewRefreshOutcomeV1, + GitHubReviewRefreshReceiptV1, GitHubReviewRefreshStateV1, + GitHubReviewRefreshStoreCommitOutcomeV1, GitHubReviewRefreshStoreReadOutcomeV1, + GitHubReviewRuntimeOwnerBuildErrorV1, GitHubReviewRuntimeOwnerConfigV1, + GitHubReviewRuntimeOwnerV1, GitHubReviewStoreManifestEntryV1, GitHubReviewStoreManifestLoadOutcomeV1, GitHubReviewStoreManifestV1, GitHubStackObservabilityV1, MAX_GITHUB_READ_RESPONSE_BYTES_V1, MAX_GITHUB_REVIEW_STORE_MANIFEST_ENTRIES_V1, ProjectGitHubAnchorAuthorityV1, diff --git a/crates/tracedecay-application/src/advisory/proximity_runtime.rs b/crates/tracedecay-application/src/advisory/proximity_runtime.rs index 284e10802a..955f8bade7 100644 --- a/crates/tracedecay-application/src/advisory/proximity_runtime.rs +++ b/crates/tracedecay-application/src/advisory/proximity_runtime.rs @@ -22,7 +22,7 @@ use tracedecay_domain::feedback::{ FeedbackScopeV1, PROXIMITY_RISK_THRESHOLD_SETTING_KEY_V1, ProviderEvaluationStateV1, ProximityAddressV1, ProximityContributionIdV1, ProximityContributionV1, ProximityCoverageV1, ProximityInclusionV1, ProximityObservationIdV1, ProximityRelationPathV1, ProximityRiskInputsV1, - ProximityTierV1, ProximityWarningClassV1, ProximityWarningIdV1, + ProximityTierV1, ProximityWarningClassV1, }; use tracedecay_domain::{ CanonicalObservationEnvelopeV1, CodeGenerationId, ManifestDigest, RetrievalAnchorId, UtcMicros, @@ -645,9 +645,6 @@ fn build_proximity_contribution( let contribution = ProximityContributionV1 { contribution_id: ProximityContributionIdV1::new(format!("contribution.proximity.{suffix}")) .ok()?, - // Shared domain/publication compatibility still carries this alias. - // Runtime-local dedupe no longer stores or compares it. - warning_id: ProximityWarningIdV1::new(format!("warning.proximity.{suffix}")).ok()?, warning_class: evidence.warning_class, source_observation_ids, retrieval_anchor_ids: evidence.retrieval_anchor_ids, diff --git a/crates/tracedecay-application/src/advisory/runtime.rs b/crates/tracedecay-application/src/advisory/runtime.rs index 6dd4618625..653e1cb133 100644 --- a/crates/tracedecay-application/src/advisory/runtime.rs +++ b/crates/tracedecay-application/src/advisory/runtime.rs @@ -36,8 +36,8 @@ use tracedecay_contracts::feedback::observations::{ FeedbackProximityTransitionV1, FeedbackSourceEventV1, }; use tracedecay_global_db::configuration::contracts::ports::ConfigurationControlStore; +use tracedecay_runtime_core::cancellation::MonotonicDeadline; use tracedecay_runtime_core::db::Database; -use tracedecay_session_memory::context::MonotonicDeadline; use super::ci_runtime::{ CiExactEvidenceAuthorityV1, CiReadOnlyProviderArchiveV1, ConcreteCiFailureLocalizationOwnerV1, diff --git a/crates/tracedecay-application/src/code_index.rs b/crates/tracedecay-application/src/code_index.rs index 22f315534f..55d5a79de7 100644 --- a/crates/tracedecay-application/src/code_index.rs +++ b/crates/tracedecay-application/src/code_index.rs @@ -21,9 +21,8 @@ use tracedecay_code_index::{ }, projection::CodeChunkProjectionSink, }; -use tracedecay_session_memory::context::{ - CancellationToken, RequestInterruption, application_request_interruption, -}; +use tracedecay_runtime_core::cancellation::CancellationToken; +use tracedecay_session_memory::context::{RequestInterruption, application_request_interruption}; /// Production owner type exposed to daemon, CLI, MCP, and hook composition. pub type ProductionCodeIndexOwnerV1 = CodeIndexProductionOwnerV1; diff --git a/crates/tracedecay-application/src/delivery.rs b/crates/tracedecay-application/src/delivery.rs index a1157bbe81..f2e4944558 100644 --- a/crates/tracedecay-application/src/delivery.rs +++ b/crates/tracedecay-application/src/delivery.rs @@ -34,7 +34,7 @@ use tracedecay_tool_catalog::{CapabilityId, UseCaseId}; use crate::advisory::github_runtime::GitHubSourceAccessAuthorityV1; use crate::advisory::{ CiRetainedObservationManifestLoadOutcomeV1, GitHubActionsConclusionV1, GitHubActionsStatusV1, - GitHubCiAnnotationLevelV1, GitHubCiCheckAnnotationV1, GitHubCiRepositoryTargetV1, + GitHubCheckAnnotationLevelV1, GitHubCheckAnnotationV1, GitHubCiRepositoryTargetV1, GitHubHttpReadConfigV1, GitHubReleaseReadControlV1, GitHubReviewBodyEvidenceAuthorityV1, GitHubReviewBodyReadOutcomeV1, GitHubReviewStoreManifestLoadOutcomeV1, ProjectCiRetainedObservationStoreV1, ProjectGitHubReleaseAuthorityOpenOutcomeV1, @@ -2046,15 +2046,15 @@ fn delivery_ci_conclusion(conclusion: &GitHubActionsConclusionV1) -> ProjectDeli } } -fn delivery_ci_annotation(annotation: &GitHubCiCheckAnnotationV1) -> ProjectDeliveryCiAnnotationV1 { +fn delivery_ci_annotation(annotation: &GitHubCheckAnnotationV1) -> ProjectDeliveryCiAnnotationV1 { ProjectDeliveryCiAnnotationV1 { path: annotation.path.clone(), start_line: annotation.start_line, end_line: annotation.end_line, level: match annotation.annotation_level { - GitHubCiAnnotationLevelV1::Notice => ProjectDeliveryCiAnnotationLevelV1::Notice, - GitHubCiAnnotationLevelV1::Warning => ProjectDeliveryCiAnnotationLevelV1::Warning, - GitHubCiAnnotationLevelV1::Failure => ProjectDeliveryCiAnnotationLevelV1::Failure, + GitHubCheckAnnotationLevelV1::Notice => ProjectDeliveryCiAnnotationLevelV1::Notice, + GitHubCheckAnnotationLevelV1::Warning => ProjectDeliveryCiAnnotationLevelV1::Warning, + GitHubCheckAnnotationLevelV1::Failure => ProjectDeliveryCiAnnotationLevelV1::Failure, }, title: annotation.title.clone(), } @@ -2596,7 +2596,22 @@ mod tests { && item.coverage == ProjectDeliveryInboxCoverageV1::Unsupported }) .collect::>(); - assert!(!unsupported.is_empty()); + assert_eq!( + unsupported + .iter() + .map(|item| item.source) + .collect::>(), + [ + ProjectDeliveryAttentionSourceV1::Contradiction, + ProjectDeliveryAttentionSourceV1::UnsafePattern, + ProjectDeliveryAttentionSourceV1::TestRisk, + ProjectDeliveryAttentionSourceV1::UnreviewedChangedCode, + ProjectDeliveryAttentionSourceV1::WeakEvidence, + ProjectDeliveryAttentionSourceV1::OverlappingEdit, + ProjectDeliveryAttentionSourceV1::ConfirmedConflict, + ProjectDeliveryAttentionSourceV1::DivergentSharedImplementation, + ] + ); assert!(unsupported.iter().all(|item| { item.evidence.is_empty() && item.coverage == ProjectDeliveryInboxCoverageV1::Unsupported diff --git a/crates/tracedecay-application/src/diagnostics_publication.rs b/crates/tracedecay-application/src/diagnostics_publication.rs index 6ac67ed93a..cc13eeeb7c 100644 --- a/crates/tracedecay-application/src/diagnostics_publication.rs +++ b/crates/tracedecay-application/src/diagnostics_publication.rs @@ -1240,13 +1240,6 @@ mod tests { }; assert_eq!(report.publication_revision, 2); assert_eq!(report.inserted, 1); - assert!( - store - .records_for_publication(resolver.0.generation_id(), 1) - .await - .unwrap() - .is_empty() - ); let current = store .records_for_generation(resolver.0.generation_id()) .await @@ -1276,11 +1269,6 @@ mod tests { }; assert_eq!(report.publication_revision, 3); assert_eq!(report.inserted, 1); - let prior = store - .records_for_publication(resolver.0.generation_id(), 2) - .await - .unwrap(); - assert_eq!(prior, current, "revision 2 must remain immutable"); let latest = store .records_for_generation(resolver.0.generation_id()) .await diff --git a/crates/tracedecay-application/src/diagnostics_query.rs b/crates/tracedecay-application/src/diagnostics_query.rs index 8488c21f52..b869475cff 100644 --- a/crates/tracedecay-application/src/diagnostics_query.rs +++ b/crates/tracedecay-application/src/diagnostics_query.rs @@ -5,28 +5,21 @@ //! `Truncated`, or `StoreUnavailable`, so a partial or failed read is never //! presented as a clean result. All list lanes are bounded by a limit //! plus an opaque cursor and are deterministic: records page in ascending -//! anchor order, chains page in chain order. -//! -//! Supersession navigation mirrors the store's logical finding key exactly -//! (repository, producer, code, file occurrence, span, message digest): -//! forward walks follow `Superseded { successor_generation }` edges toward -//! newer records, backward walks invert those edges toward older records. +//! anchor order. //! //! The overlay merge composes a session-only [`DirtyDiagnosticOverlay`] with //! the durable current set into one deterministic view; the overlay wins on //! the same logical finding key and every entry is marked with its //! provenance (persisted vs overlay). Overlay state is never persisted. -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::BTreeMap; use std::fmt; use tracedecay_domain::{ - CodeGenerationId, DiagnosticRecordStateV1, FileOccurrenceId, GenerationDiagnosticV1, - RetrievalAnchorId, + CodeGenerationId, FileOccurrenceId, GenerationDiagnosticV1, RetrievalAnchorId, }; use crate::diagnostics_store::{DiagnosticsStore, DirtyDiagnosticOverlay}; -use tracedecay_domain::errors::Result as CrateResult; use tracedecay_runtime_core::db::Database; #[cfg(test)] use tracedecay_runtime_core::db::engine::Connection; @@ -139,37 +132,6 @@ pub struct CurrentDiagnosticGeneration { pub coverage: DiagnosticQueryCoverage, } -/// One persisted finding republished by a successor generation under a new -/// anchor: the prior record and its successor share one logical finding key. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct DiagnosticSupersessionPair { - pub prior: GenerationDiagnosticV1, - pub successor: GenerationDiagnosticV1, -} - -/// Generation-aware answer to "what changed for this file between -/// generations `from_generation` and `to_generation`", computed from the -/// store's records and supersession chains. Lanes are deterministic -/// (ascending anchor order; pairs ordered by successor anchor) and each lane -/// is capped at the request limit. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct GenerationDiagnosticDiff { - pub from_generation: CodeGenerationId, - pub to_generation: CodeGenerationId, - pub file_occurrence_id: FileOccurrenceId, - /// Findings present in `to_generation` with no same-key record in - /// `from_generation`. - pub introduced: Vec, - /// Findings present in both generations: the `from_generation` record - /// was superseded (or otherwise carried) into the `to_generation` record - /// for the same logical finding key. - pub superseded: Vec, - /// Findings present in `from_generation` with no same-key record in - /// `to_generation`. - pub cleared: Vec, - pub coverage: DiagnosticQueryCoverage, -} - /// Where one entry of the merged current view came from. The durable lane /// and the session-only overlay lane stay typed and separate even after /// merging; overlay findings are never published as durable LSP @@ -221,11 +183,8 @@ impl MergedDiagnosticView { /// type, they surface as [`DiagnosticQueryCoverage::StoreUnavailable`]. #[derive(Clone, Debug, PartialEq, Eq)] pub enum DiagnosticQueryError { - /// The cursor is malformed or does not name a record on the chain lane - /// it was minted from. + /// The cursor is malformed. InvalidCursor { cursor: String }, - /// A generation diff requires two distinct generations. - SameGeneration { generation: CodeGenerationId }, /// The overlay is bound to a different clean generation than the query. OverlayGenerationMismatch { overlay_generation: CodeGenerationId, @@ -239,10 +198,6 @@ impl fmt::Display for DiagnosticQueryError { Self::InvalidCursor { cursor } => { write!(formatter, "invalid diagnostic query cursor: {cursor}") } - Self::SameGeneration { generation } => write!( - formatter, - "a generation diagnostic diff requires two distinct generations, got {generation} twice" - ), Self::OverlayGenerationMismatch { overlay_generation, query_generation, @@ -256,10 +211,9 @@ impl fmt::Display for DiagnosticQueryError { impl std::error::Error for DiagnosticQueryError {} -/// The logical finding key, mirroring the store's supersession successor -/// match exactly: (repository, producer, code, file occurrence, span, -/// message digest). Records sharing a key are the same logical finding -/// republished across generations. +/// The logical finding key: (repository, producer, code, file occurrence, +/// span, message digest). A durable record and an overlay entry sharing a +/// key are the same logical finding. #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] struct LogicalFindingKey { repository: String, @@ -386,46 +340,6 @@ impl<'a> DiagnosticsQuery<'a> { } } - /// Stale (superseded or cleared) records bound to `generation`. Stale - /// findings remain queryable but never re-enter active publication. - #[hotpath::measure( - label = "usecases.diagnostics_query.stale_by_generation", - future = true - )] - pub async fn stale_by_generation( - &self, - generation: &CodeGenerationId, - request: &DiagnosticPageRequest, - ) -> Result { - let operation = "diagnostics query stale_by_generation"; - match self.store.stale_records(generation).await { - Ok(records) => Ok(paginate_sorted(records, request)), - Err(error) => Ok(DiagnosticPage::unavailable(operation, error)), - } - } - - /// Stale (superseded or cleared) records for one file occurrence inside - /// `generation`, paged in ascending anchor order. - #[hotpath::measure(label = "usecases.diagnostics_query.stale_by_file", future = true)] - pub async fn stale_by_file( - &self, - generation: &CodeGenerationId, - file_occurrence_id: &FileOccurrenceId, - request: &DiagnosticPageRequest, - ) -> Result { - let operation = "diagnostics query stale_by_file"; - match self.store.stale_records(generation).await { - Ok(records) => Ok(paginate_sorted( - records - .into_iter() - .filter(|record| record.file_occurrence_id == *file_occurrence_id) - .collect(), - request, - )), - Err(error) => Ok(DiagnosticPage::unavailable(operation, error)), - } - } - /// Fetches one record by its retrieval anchor. A miss is `Complete` with /// no record; a store failure is typed `StoreUnavailable`. #[hotpath::measure(label = "usecases.diagnostics_query.by_anchor", future = true)] @@ -452,218 +366,6 @@ impl<'a> DiagnosticsQuery<'a> { } } - /// Forward supersession navigation from `anchor`: the chain walks - /// `Superseded { successor_generation }` edges toward newer records and - /// is returned oldest-first including the starting record. The chain - /// ends at a current, cleared, or missing successor. - #[hotpath::measure( - label = "usecases.diagnostics_query.supersession_forward", - future = true - )] - pub async fn supersession_forward( - &self, - anchor: &RetrievalAnchorId, - request: &DiagnosticPageRequest, - ) -> Result { - let operation = "diagnostics query supersession_forward"; - match self.store.supersession_chain(anchor).await { - Ok(chain) => paginate_chain(chain, request), - Err(error) => Ok(DiagnosticPage::unavailable(operation, error)), - } - } - - /// Backward supersession navigation from `anchor`: the chain starts at - /// the named record and walks toward older records, newest-first, by - /// inverting the store's forward edges, one step back is the unique - /// same-key record whose `Superseded { successor_generation }` names the - /// current record's generation. The walk stops deterministically when - /// there is no unique predecessor. - #[hotpath::measure( - label = "usecases.diagnostics_query.supersession_backward", - future = true - )] - pub async fn supersession_backward( - &self, - anchor: &RetrievalAnchorId, - request: &DiagnosticPageRequest, - ) -> Result { - let operation = "diagnostics query supersession_backward"; - let start = match self.store.record_by_anchor(anchor).await { - Ok(Some(record)) => record, - Ok(None) => { - return Ok(DiagnosticPage { - records: Vec::new(), - total: 0, - coverage: DiagnosticQueryCoverage::Complete, - next_cursor: None, - }); - } - Err(error) => return Ok(DiagnosticPage::unavailable(operation, error)), - }; - let key = LogicalFindingKey::of(&start); - let generations = match self.list_generations().await { - Ok(generations) => generations, - Err(error) => return Ok(DiagnosticPage::unavailable(operation, error)), - }; - let mut same_key_by_generation: BTreeMap> = - BTreeMap::new(); - for generation in &generations { - let generation_id = match CodeGenerationId::try_from(generation.clone()) { - Ok(generation_id) => generation_id, - Err(error) => { - return Ok(DiagnosticPage::unavailable( - operation, - format!("stored generation id {generation}: {error}"), - )); - } - }; - let records = match self.store.records_for_generation(&generation_id).await { - Ok(records) => records, - Err(error) => return Ok(DiagnosticPage::unavailable(operation, error)), - }; - let matching: Vec = records - .into_iter() - .filter(|record| LogicalFindingKey::of(record) == key) - .collect(); - if !matching.is_empty() { - same_key_by_generation.insert(generation.clone(), matching); - } - } - - let mut chain = vec![start.clone()]; - let mut current = start; - let mut visited: BTreeSet = BTreeSet::new(); - visited.insert(current.diagnostic_anchor.as_str().to_owned()); - loop { - // Ambiguity guard: the current record must be the unique - // same-key record of its generation, mirroring the store's - // forward-walk ambiguity rule. - let unique_current = same_key_by_generation - .get(current.generation_id.as_str()) - .is_some_and(|records| records.len() == 1); - if !unique_current { - break; - } - let predecessors: Vec<&GenerationDiagnosticV1> = same_key_by_generation - .values() - .flatten() - .filter(|record| { - matches!( - &record.state, - DiagnosticRecordStateV1::Superseded { - successor_generation - } if *successor_generation == current.generation_id - ) - }) - .collect(); - if predecessors.len() != 1 { - break; - } - let predecessor = (*predecessors[0]).clone(); - if !visited.insert(predecessor.diagnostic_anchor.as_str().to_owned()) { - break; - } - chain.push(predecessor.clone()); - current = predecessor; - } - paginate_chain(chain, request) - } - - /// Answers "what changed for `file_occurrence_id` between - /// `from_generation` and `to_generation`" from the store's records and - /// chains: findings are keyed by the logical finding key, so a finding - /// republished under a new anchor in `to_generation` lands in the - /// `superseded` lane, a finding with no `from_generation` counterpart - /// lands in `introduced`, and a finding with no `to_generation` - /// counterpart lands in `cleared`. Each lane is capped at `limit`. - #[hotpath::measure( - label = "usecases.diagnostics_query.generation_file_diff", - future = true - )] - pub async fn generation_file_diff( - &self, - from_generation: &CodeGenerationId, - to_generation: &CodeGenerationId, - file_occurrence_id: &FileOccurrenceId, - limit: usize, - ) -> Result { - let operation = "diagnostics query generation_file_diff"; - if from_generation == to_generation { - return Err(DiagnosticQueryError::SameGeneration { - generation: from_generation.clone(), - }); - } - let limit = normalize_limit(limit); - let from_records = match self.store.records_for_generation(from_generation).await { - Ok(records) => records, - Err(error) => { - return Ok(diff_store_unavailable( - operation, - error, - from_generation, - to_generation, - file_occurrence_id, - )); - } - }; - let to_records = match self.store.records_for_generation(to_generation).await { - Ok(records) => records, - Err(error) => { - return Ok(diff_store_unavailable( - operation, - error, - from_generation, - to_generation, - file_occurrence_id, - )); - } - }; - // First same-key record wins; store reads are anchor-ordered so this - // is deterministic. - let from_by_key = key_file_records(from_records, file_occurrence_id); - let to_by_key = key_file_records(to_records, file_occurrence_id); - - let mut introduced: Vec = Vec::new(); - let mut superseded: Vec = Vec::new(); - let mut cleared: Vec = Vec::new(); - for (key, successor) in &to_by_key { - match from_by_key.get(key) { - Some(prior) => superseded.push(DiagnosticSupersessionPair { - prior: prior.clone(), - successor: successor.clone(), - }), - None => introduced.push(successor.clone()), - } - } - for (key, prior) in &from_by_key { - if !to_by_key.contains_key(key) { - cleared.push(prior.clone()); - } - } - introduced.sort_by(anchor_cmp); - superseded.sort_by(|left, right| anchor_cmp(&left.successor, &right.successor)); - cleared.sort_by(anchor_cmp); - - let mut coverage = DiagnosticQueryCoverage::Complete; - for lane_len in [introduced.len(), superseded.len(), cleared.len()] { - if lane_len > limit { - coverage = DiagnosticQueryCoverage::Truncated; - } - } - introduced.truncate(limit); - superseded.truncate(limit); - cleared.truncate(limit); - Ok(GenerationDiagnosticDiff { - from_generation: from_generation.clone(), - to_generation: to_generation.clone(), - file_occurrence_id: file_occurrence_id.clone(), - introduced, - superseded, - cleared, - coverage, - }) - } - /// Composes the durable current set for `generation` with a dirty /// overlay into one deterministic merged view. On the same logical /// finding key the overlay entry wins; every entry carries typed @@ -735,33 +437,6 @@ impl<'a> DiagnosticsQuery<'a> { next_cursor, }) } - - /// Every published generation id, ascending. Read-only probe over the - /// store's publication ledger used to scope backward chain walks. - async fn list_generations(&self) -> CrateResult> { - self.store.published_generation_ids().await - } -} - -fn diff_store_unavailable( - operation: &'static str, - error: impl fmt::Display, - from_generation: &CodeGenerationId, - to_generation: &CodeGenerationId, - file_occurrence_id: &FileOccurrenceId, -) -> GenerationDiagnosticDiff { - GenerationDiagnosticDiff { - from_generation: from_generation.clone(), - to_generation: to_generation.clone(), - file_occurrence_id: file_occurrence_id.clone(), - introduced: Vec::new(), - superseded: Vec::new(), - cleared: Vec::new(), - coverage: DiagnosticQueryCoverage::StoreUnavailable { - operation, - reason: error.to_string(), - }, - } } fn normalize_limit(limit: usize) -> usize { @@ -778,22 +453,6 @@ fn anchor_cmp(left: &GenerationDiagnosticV1, right: &GenerationDiagnosticV1) -> .cmp(right.diagnostic_anchor.as_str()) } -fn key_file_records( - records: Vec, - file_occurrence_id: &FileOccurrenceId, -) -> BTreeMap { - let mut by_key = BTreeMap::new(); - for record in records { - if record.file_occurrence_id != *file_occurrence_id { - continue; - } - by_key - .entry(LogicalFindingKey::of(&record)) - .or_insert(record); - } - by_key -} - /// Pages a set of items ordered by ascending anchor. The cursor resumes /// strictly after the anchor it encodes; an anchor that no longer exists /// still resumes at the first greater anchor, so sorted lanes are total. @@ -824,22 +483,6 @@ fn paginate_items( (page, coverage, next_cursor) } -fn paginate_sorted( - records: Vec, - request: &DiagnosticPageRequest, -) -> DiagnosticPage { - let total = records.len(); - let (records, coverage, next_cursor) = - paginate_items(records, |record| record.diagnostic_anchor.as_str(), request); - crate::hotpath_observe::diagnostics_query(records.len(), total); - DiagnosticPage { - records, - total, - coverage, - next_cursor, - } -} - fn page_from_bounded_records( records: Vec, total: usize, @@ -865,48 +508,12 @@ fn page_from_bounded_records( } } -/// Pages a supersession chain in chain order (not anchor order). The cursor -/// resumes strictly after the chain position it encodes; a cursor whose -/// anchor is not on the chain is a caller error. -fn paginate_chain( - chain: Vec, - request: &DiagnosticPageRequest, -) -> Result { - let limit = normalize_limit(request.limit); - let start = match &request.cursor { - Some(cursor) => chain - .iter() - .position(|record| record.diagnostic_anchor.as_str() == cursor.anchor()) - .map(|position| position + 1) - .ok_or_else(|| DiagnosticQueryError::InvalidCursor { - cursor: cursor.encode().to_owned(), - })?, - None => 0, - }; - let end = (start + limit).min(chain.len()); - let page: Vec = chain[start..end].to_vec(); - let (coverage, next_cursor) = if end < chain.len() { - let cursor = page - .last() - .map(|record| DiagnosticQueryCursor::after_anchor(&record.diagnostic_anchor)); - (DiagnosticQueryCoverage::Truncated, cursor) - } else { - (DiagnosticQueryCoverage::Complete, None) - }; - Ok(DiagnosticPage { - records: page, - total: chain.len(), - coverage, - next_cursor, - }) -} - #[cfg(test)] mod tests { use super::*; use tracedecay_domain::{ DiagnosticEvidenceClassV1, DiagnosticProducerKindV1, DiagnosticProvenanceV1, - DiagnosticSeverityV1, SourceSpan, UtcMicros, + DiagnosticRecordStateV1, DiagnosticSeverityV1, SourceSpan, UtcMicros, }; use tracedecay_runtime_core::db::engine::TestConnection; @@ -979,9 +586,9 @@ mod tests { const GEN2: &str = "generation.clean.2"; /// Seeds two generations: gen1 publishes A1 (anchor.1, E0308) and B1 - /// (anchor.2, `dead_code`); gen1 is superseded by gen2; gen2 republishes - /// A1's logical finding as A2 (anchor.3) and adds the new finding C2 - /// (anchor.4, `unused_variables`). B1 has no gen2 successor. + /// (anchor.2, `dead_code`); gen2 republishes A1's logical finding as A2 + /// (anchor.3) and adds the new finding C2 (anchor.4, `unused_variables`). + /// Publishing gen2 clears both gen1 records. async fn seed_two_generations(conn: &Connection) { let store = DiagnosticsStore::new_runtime(conn); store @@ -998,10 +605,6 @@ mod tests { ) .await .expect("publish gen1"); - store - .supersede_generation(&id(GEN1), &id(GEN2)) - .await - .expect("supersede gen1"); store .publish_clean_generation( &id(GEN2), @@ -1102,7 +705,7 @@ mod tests { } #[tokio::test] - async fn current_and_stale_file_lanes_filter_by_file_and_state() { + async fn current_file_lane_filters_by_file() { let temp = tempfile::tempdir().unwrap(); let conn = open_store(&temp.path().join("diagnostics.db")).await; seed_two_generations(&conn).await; @@ -1138,40 +741,6 @@ mod tests { .unwrap(); assert!(other_file.records.is_empty()); assert_eq!(other_file.coverage, DiagnosticQueryCoverage::Complete); - - let stale = query - .stale_by_generation(&id(GEN1), &DiagnosticPageRequest::default()) - .await - .unwrap(); - assert_eq!( - anchors(&stale), - vec!["anchor.diagnostic.1", "anchor.diagnostic.2"] - ); - assert!( - stale - .records - .iter() - .all(|record| !record.state.is_current()) - ); - - let stale_file = query - .stale_by_file( - &id(GEN1), - &id("file.occurrence.1"), - &DiagnosticPageRequest::default(), - ) - .await - .unwrap(); - assert_eq!(anchors(&stale_file), anchors(&stale)); - let stale_other = query - .stale_by_file( - &id(GEN1), - &id("file.occurrence.other"), - &DiagnosticPageRequest::default(), - ) - .await - .unwrap(); - assert!(stale_other.records.is_empty()); } #[tokio::test] @@ -1186,9 +755,9 @@ mod tests { let record = hit.record.expect("anchor.1 is persisted"); assert!(matches!( &record.state, - DiagnosticRecordStateV1::Superseded { - successor_generation - } if successor_generation.as_str() == GEN2 + DiagnosticRecordStateV1::Cleared { + cleared_in_generation + } if cleared_in_generation.as_str() == GEN2 )); let miss = query @@ -1199,202 +768,6 @@ mod tests { assert!(miss.record.is_none()); } - #[tokio::test] - async fn supersession_navigation_walks_forward_and_backward() { - let temp = tempfile::tempdir().unwrap(); - let conn = open_store(&temp.path().join("diagnostics.db")).await; - seed_two_generations(&conn).await; - let query = DiagnosticsQuery::new_runtime(&conn); - - // Forward from the gen1 record crosses into its gen2 successor. - let forward = query - .supersession_forward( - &id("anchor.diagnostic.1"), - &DiagnosticPageRequest::default(), - ) - .await - .unwrap(); - assert_eq!( - anchors(&forward), - vec!["anchor.diagnostic.1", "anchor.diagnostic.3"] - ); - assert_eq!(forward.coverage, DiagnosticQueryCoverage::Complete); - - // Backward from the gen2 successor reaches the gen1 record. - let backward = query - .supersession_backward( - &id("anchor.diagnostic.3"), - &DiagnosticPageRequest::default(), - ) - .await - .unwrap(); - assert_eq!( - anchors(&backward), - vec!["anchor.diagnostic.3", "anchor.diagnostic.1"] - ); - assert_eq!(backward.coverage, DiagnosticQueryCoverage::Complete); - - // A finding without a successor (or predecessor) is a one-record chain. - let forward_dead_end = query - .supersession_forward( - &id("anchor.diagnostic.2"), - &DiagnosticPageRequest::default(), - ) - .await - .unwrap(); - assert_eq!(anchors(&forward_dead_end), vec!["anchor.diagnostic.2"]); - let backward_dead_end = query - .supersession_backward( - &id("anchor.diagnostic.4"), - &DiagnosticPageRequest::default(), - ) - .await - .unwrap(); - assert_eq!(anchors(&backward_dead_end), vec!["anchor.diagnostic.4"]); - - // Chain lanes paginate in chain order with cursor resumption. - let page = query - .supersession_forward( - &id("anchor.diagnostic.1"), - &DiagnosticPageRequest::new(1, None), - ) - .await - .unwrap(); - assert_eq!(anchors(&page), vec!["anchor.diagnostic.1"]); - assert_eq!(page.coverage, DiagnosticQueryCoverage::Truncated); - let rest = query - .supersession_forward( - &id("anchor.diagnostic.1"), - &DiagnosticPageRequest::new(1, page.next_cursor.clone()), - ) - .await - .unwrap(); - assert_eq!(anchors(&rest), vec!["anchor.diagnostic.3"]); - assert_eq!(rest.coverage, DiagnosticQueryCoverage::Complete); - - // A cursor from a sorted lane is not valid on a chain lane. - let sorted_cursor = DiagnosticQueryCursor::decode("dq1:anchor.diagnostic.9").unwrap(); - assert!(matches!( - query - .supersession_forward( - &id("anchor.diagnostic.1"), - &DiagnosticPageRequest::new(1, Some(sorted_cursor)), - ) - .await, - Err(DiagnosticQueryError::InvalidCursor { .. }) - )); - } - - #[tokio::test] - async fn generation_diff_reports_introduced_superseded_cleared_lanes() { - let temp = tempfile::tempdir().unwrap(); - let conn = open_store(&temp.path().join("diagnostics.db")).await; - seed_two_generations(&conn).await; - let query = DiagnosticsQuery::new_runtime(&conn); - - let diff = query - .generation_file_diff(&id(GEN1), &id(GEN2), &id("file.occurrence.1"), 0) - .await - .unwrap(); - assert_eq!(diff.coverage, DiagnosticQueryCoverage::Complete); - assert_eq!(diff.introduced.len(), 1); - assert_eq!( - diff.introduced[0].diagnostic_anchor.as_str(), - "anchor.diagnostic.4" - ); - assert_eq!(diff.superseded.len(), 1); - assert_eq!( - diff.superseded[0].prior.diagnostic_anchor.as_str(), - "anchor.diagnostic.1" - ); - assert_eq!( - diff.superseded[0].successor.diagnostic_anchor.as_str(), - "anchor.diagnostic.3" - ); - assert_eq!(diff.cleared.len(), 1); - assert_eq!( - diff.cleared[0].diagnostic_anchor.as_str(), - "anchor.diagnostic.2" - ); - - // Reversing the direction swaps the introduced and cleared lanes. - let reverse = query - .generation_file_diff(&id(GEN2), &id(GEN1), &id("file.occurrence.1"), 0) - .await - .unwrap(); - assert_eq!( - reverse.introduced[0].diagnostic_anchor.as_str(), - "anchor.diagnostic.2" - ); - assert_eq!( - reverse.cleared[0].diagnostic_anchor.as_str(), - "anchor.diagnostic.4" - ); - assert_eq!(reverse.superseded.len(), 1); - - // A diff needs two distinct generations. - assert!(matches!( - query - .generation_file_diff(&id(GEN1), &id(GEN1), &id("file.occurrence.1"), 0) - .await, - Err(DiagnosticQueryError::SameGeneration { .. }) - )); - } - - #[tokio::test] - async fn generation_diff_lane_limit_truncates() { - let temp = tempfile::tempdir().unwrap(); - let conn = open_store(&temp.path().join("diagnostics.db")).await; - seed_two_generations(&conn).await; - let query = DiagnosticsQuery::new_runtime(&conn); - - // Introduce a second new finding in gen2 so the introduced lane - // exceeds a lane limit of 1. - let store = DiagnosticsStore::new_runtime(&conn); - let extra = with_message( - fixture_record(GEN2, "anchor.diagnostic.5"), - "unused_imports", - "unused import", - ); - store - .publish_clean_generation( - &id("generation.clean.3"), - &[ - fixture_record("generation.clean.3", "anchor.diagnostic.6"), - with_message( - fixture_record("generation.clean.3", "anchor.diagnostic.7"), - "unused_variables", - "unused variable: `tmp`", - ), - GenerationDiagnosticV1 { - generation_id: id("generation.clean.3"), - diagnostic_anchor: id("anchor.diagnostic.8"), - ..extra - }, - ], - ) - .await - .expect("publish gen3"); - - let diff = query - .generation_file_diff( - &id(GEN1), - &id("generation.clean.3"), - &id("file.occurrence.1"), - 1, - ) - .await - .unwrap(); - assert_eq!(diff.coverage, DiagnosticQueryCoverage::Truncated); - assert_eq!(diff.introduced.len(), 1); - assert_eq!(diff.superseded.len(), 1); - assert_eq!(diff.cleared.len(), 1); - assert_eq!( - diff.introduced[0].diagnostic_anchor.as_str(), - "anchor.diagnostic.7" - ); - } - #[tokio::test] async fn overlay_merge_prefers_overlay_and_marks_provenance() { let temp = tempfile::tempdir().unwrap(); @@ -1529,50 +902,10 @@ mod tests { .unwrap(); assert!(is_unavailable(&page.coverage)); - let page = query - .stale_by_generation(&id(GEN1), &DiagnosticPageRequest::default()) - .await - .unwrap(); - assert!(is_unavailable(&page.coverage)); - - let page = query - .stale_by_file( - &id(GEN1), - &id("file.occurrence.1"), - &DiagnosticPageRequest::default(), - ) - .await - .unwrap(); - assert!(is_unavailable(&page.coverage)); - let lookup = query.by_anchor(&id("anchor.diagnostic.1")).await.unwrap(); assert!(lookup.record.is_none()); assert!(is_unavailable(&lookup.coverage)); - let forward = query - .supersession_forward( - &id("anchor.diagnostic.1"), - &DiagnosticPageRequest::default(), - ) - .await - .unwrap(); - assert!(is_unavailable(&forward.coverage)); - - let backward = query - .supersession_backward( - &id("anchor.diagnostic.3"), - &DiagnosticPageRequest::default(), - ) - .await - .unwrap(); - assert!(is_unavailable(&backward.coverage)); - - let diff = query - .generation_file_diff(&id(GEN1), &id(GEN2), &id("file.occurrence.1"), 0) - .await - .unwrap(); - assert!(is_unavailable(&diff.coverage)); - let overlay = DirtyDiagnosticOverlay::new(id(GEN2)); let merged = query .merged_current_with_overlay(&id(GEN2), &overlay, &DiagnosticPageRequest::default()) diff --git a/crates/tracedecay-application/src/diagnostics_store.rs b/crates/tracedecay-application/src/diagnostics_store.rs index 71785d5391..2a4e62a4f8 100644 --- a/crates/tracedecay-application/src/diagnostics_store.rs +++ b/crates/tracedecay-application/src/diagnostics_store.rs @@ -4,7 +4,7 @@ //! //! Durable `GenerationDiagnosticV1` records persist in the project store and //! survive restarts. Publication is version-monotone: a newer clean -//! generation clears or supersedes prior current records deterministically, +//! generation clears prior current records deterministically, //! stale findings never cross snapshots, and dirty editor overlays live only //! in memory, they are never sealed into the durable store. @@ -19,10 +19,8 @@ use tracedecay_domain::{ RetrievalAnchorId, SourceSpan, UtcMicros, }; use tracedecay_store::{ - DIAGNOSTIC_STATE_CLEARED, DIAGNOSTIC_STATE_CURRENT, DIAGNOSTIC_STATE_SUPERSEDED, - DiagnosticPublicationDispositionV1, DiagnosticPublicationReceiptV1, - DiagnosticRecordStateKindV1, DiagnosticStore as DiagnosticStorePort, DiagnosticStoreError, - DiagnosticStoreResult, SanitizedCleanDiagnosticSnapshotV1, diagnostic_evidence_class_name, + DIAGNOSTIC_STATE_CLEARED, DIAGNOSTIC_STATE_CURRENT, DiagnosticRecordStateKindV1, + DiagnosticStore, DiagnosticStoreError, DiagnosticStoreResult, diagnostic_evidence_class_name, diagnostic_producer_kind_name, diagnostic_severity_name, diagnostic_snapshot_observation_eq, diagnostic_state_columns, parse_diagnostic_evidence_class, parse_diagnostic_producer_kind, parse_diagnostic_severity, @@ -48,7 +46,6 @@ pub const SCHEMA: &str = tracedecay_store::GENERATION_DIAGNOSTICS_SCHEMA_DDL; // this engine and the rusqlite-runtime `DiagnosticExecutor` cannot drift apart // across a cutover. These aliases keep the SQL below readable. const STATE_CURRENT: &str = DIAGNOSTIC_STATE_CURRENT; -const STATE_SUPERSEDED: &str = DIAGNOSTIC_STATE_SUPERSEDED; const STATE_CLEARED: &str = DIAGNOSTIC_STATE_CLEARED; /// SQLite-backed store for durable generation-bound diagnostics. @@ -296,40 +293,6 @@ impl<'a> DiagnosticsStore<'a> { .await } - /// Every published generation id in deterministic order. This stays on - /// the guarded diagnostics store so read-only query clients never need a - /// raw engine connection. - pub(crate) async fn published_generation_ids(&self) -> Result> { - hotpath::future!( - async { - let operation = "diagnostics published_generation_ids"; - let mut rows = self - .conn - .query( - "SELECT DISTINCT generation_id FROM diagnostic_generation_publications \ - ORDER BY generation_id", - params![], - ) - .await - .map_err(|error| db_error(operation, error))?; - let mut generations = Vec::new(); - while let Some(row) = rows - .next() - .await - .map_err(|error| db_error(operation, error))? - { - generations.push( - row.get::(0) - .map_err(|error| db_error(operation, error))?, - ); - } - Ok(generations) - }, - label = "usecases.diagnostics_store.published_generation_ids" - ) - .await - } - /// Runs `work` inside an immediate transaction, committing on success and /// rolling back on error or cancellation. The transactional store routes /// every statement through that exact transaction. @@ -579,117 +542,6 @@ impl<'a> DiagnosticsStore<'a> { .await } - /// Marks every `Current` record of `prior_generation` as superseded by - /// `successor_generation`. A generation can never supersede itself. - /// Returns the number of rows transitioned. - pub async fn supersede_generation( - &self, - prior_generation: &CodeGenerationId, - successor_generation: &CodeGenerationId, - ) -> Result { - let operation = "diagnostics supersede_generation"; - if prior_generation == successor_generation { - return Err(db_message( - operation, - "a generation cannot supersede itself", - )); - } - let prior_generation = prior_generation.clone(); - let successor_generation = successor_generation.clone(); - hotpath::future!( - self.with_immediate_tx(operation, move |store| { - Box::pin(async move { - let transitioned = store - .conn - .execute( - "UPDATE generation_diagnostics - SET record_state = ?1, state_generation = ?2 - WHERE record_state = ?3 AND generation_id = ?4 - AND publication_revision = ( - SELECT publication_revision FROM diagnostic_generation_publications - WHERE generation_id = ?4 AND record_state = ?3 - )", - params![ - STATE_SUPERSEDED, - successor_generation.as_str(), - STATE_CURRENT, - prior_generation.as_str() - ], - ) - .await - .map_err(|e| db_error(operation, e))?; - store - .conn - .execute( - "UPDATE diagnostic_generation_publications - SET record_state = ?1, state_generation = ?2 - WHERE record_state = ?3 AND generation_id = ?4", - params![ - STATE_SUPERSEDED, - successor_generation.as_str(), - STATE_CURRENT, - prior_generation.as_str() - ], - ) - .await - .map_err(|e| db_error(operation, e))?; - Ok(transitioned) - }) - }), - label = "usecases.diagnostics_store.supersede" - ) - .await - } - - /// Walks the supersession chain starting at `anchor`. Each step follows - /// the record's `Superseded { successor_generation }` edge to the current - /// record in the successor generation with the same logical finding key - /// (repository, producer, code, file occurrence, span, message digest). - /// The chain ends at a current, cleared, or missing successor and is - /// returned oldest-first including the starting record. - pub async fn supersession_chain( - &self, - anchor: &RetrievalAnchorId, - ) -> Result> { - hotpath::future!( - async { - let mut chain = Vec::new(); - let Some(start) = self.record_by_anchor(anchor).await? else { - return Ok(chain); - }; - chain.push(start); - loop { - let Some(last) = chain.last() else { - // Unreachable: `chain` is seeded before the loop and only - // grows; bail out with what we have rather than panic. - return Ok(chain); - }; - let DiagnosticRecordStateV1::Superseded { - successor_generation, - } = &last.state - else { - return Ok(chain); - }; - let successor = self - .find_logical_successor(last, successor_generation) - .await?; - match successor { - Some(record) - if !chain - .iter() - .any(|seen| seen.diagnostic_anchor == record.diagnostic_anchor) => - { - chain.push(record); - } - _ => return Ok(chain), - } - } - }, - label = "usecases.diagnostics_store.supersession_chain" - ) - .await - } - /// Records in the latest immutable publication for `generation`, ordered by anchor. pub async fn records_for_generation( &self, @@ -702,39 +554,6 @@ impl<'a> DiagnosticsStore<'a> { .await } - /// Records in one exact immutable publication of `generation`. - pub async fn records_for_publication( - &self, - generation: &CodeGenerationId, - publication_revision: u64, - ) -> Result> { - let operation = "diagnostics records_for_publication"; - if publication_revision == 0 { - return Err(db_message( - operation, - "diagnostic publication revision must be positive", - )); - } - let revision = i64::try_from(publication_revision).map_err(|_| { - db_message( - operation, - "diagnostic publication revision exceeds SQLite range", - ) - })?; - let mut rows = self - .conn - .query( - &format!( - "{SELECT_RECORDS} WHERE generation_id = ?1 AND publication_revision = ?2 \ - ORDER BY diagnostic_anchor" - ), - params![generation.as_str(), revision], - ) - .await - .map_err(|error| db_error(operation, error))?; - collect_rows(&mut rows, operation).await - } - /// Current records bound to `generation`, the only set eligible for /// active publication. pub async fn current_records( @@ -901,35 +720,6 @@ impl<'a> DiagnosticsStore<'a> { .await } - /// Stale (superseded or cleared) records bound to `generation`. Stale - /// findings remain queryable but never re-enter active publication. - pub async fn stale_records( - &self, - generation: &CodeGenerationId, - ) -> Result> { - hotpath::future!( - async { - let operation = "diagnostics stale_records"; - let mut rows = self - .conn - .query( - &format!( - "{SELECT_RECORDS} WHERE generation_id = ?1 AND record_state != ?2 \ - AND publication_revision = (SELECT MAX(publication_revision) FROM \ - diagnostic_generation_publications WHERE generation_id = ?1) \ - ORDER BY diagnostic_anchor" - ), - params![generation.as_str(), STATE_CURRENT], - ) - .await - .map_err(|e| db_error(operation, e))?; - collect_rows(&mut rows, operation).await - }, - label = "usecases.diagnostics_store.stale_records" - ) - .await - } - /// Fetches one record by its Plan 13 anchor. pub async fn record_by_anchor( &self, @@ -1134,76 +924,9 @@ impl<'a> DiagnosticsStore<'a> { .map_err(|e| db_error(operation, e))?; collect_rows(&mut rows, operation).await } - - #[hotpath::measure(label = "usecases.diagnostics_store.find_successor", future = true)] - async fn find_logical_successor( - &self, - prior: &GenerationDiagnosticV1, - successor_generation: &CodeGenerationId, - ) -> Result> { - let operation = "diagnostics find_logical_successor"; - let mut rows = self - .conn - .query( - &format!( - "{SELECT_RECORDS} WHERE generation_id = ?1 AND publication_revision = (\ - SELECT MAX(publication_revision) FROM diagnostic_generation_publications \ - WHERE generation_id = ?1) AND repository = ?2 \ - AND producer = ?3 AND code = ?4 AND file_occurrence_id = ?5 \ - AND span_start = ?6 AND span_end = ?7 AND message_digest = ?8 \ - ORDER BY diagnostic_anchor" - ), - params![ - successor_generation.as_str(), - prior.repository.as_str(), - prior.provenance.producer.as_str(), - prior.code.as_str(), - prior.file_occurrence_id.as_str(), - prior.span.start_byte as i64, - prior.span.end_byte as i64, - prior.message_digest.as_str(), - ], - ) - .await - .map_err(|e| db_error(operation, e))?; - let mut records = collect_rows(&mut rows, operation).await?; - if records.len() > 1 { - return Err(db_message( - operation, - format!( - "ambiguous logical successor for {} in {}", - prior.diagnostic_anchor, successor_generation - ), - )); - } - Ok(records.pop()) - } } -impl DiagnosticStorePort for DiagnosticsStore<'_> { - async fn publish_clean_diagnostics( - &self, - snapshot: SanitizedCleanDiagnosticSnapshotV1, - ) -> DiagnosticStoreResult { - let (generation, records) = snapshot.into_parts(); - let (inserted, cleared, exact_replay, publication_revision) = self - .publish_clean_generation_with_disposition(&generation, &records) - .await - .map_err(|error| port_error("publish_clean_diagnostics", error))?; - let disposition = if exact_replay { - DiagnosticPublicationDispositionV1::ExactReplay - } else { - DiagnosticPublicationDispositionV1::Committed - }; - Ok(DiagnosticPublicationReceiptV1::new( - generation, - publication_revision, - inserted, - cleared, - disposition, - )) - } - +impl DiagnosticStore for DiagnosticsStore<'_> { async fn current_diagnostic_generation( &self, ) -> DiagnosticStoreResult> { @@ -1221,16 +944,6 @@ impl DiagnosticStorePort for DiagnosticsStore<'_> { .map_err(|error| port_error("diagnostics_for_generation", error)) } - async fn diagnostics_for_publication( - &self, - generation: &CodeGenerationId, - publication_revision: u64, - ) -> DiagnosticStoreResult> { - self.records_for_publication(generation, publication_revision) - .await - .map_err(|error| port_error("diagnostics_for_publication", error)) - } - async fn current_diagnostics( &self, generation: &CodeGenerationId, @@ -1250,15 +963,6 @@ impl DiagnosticStorePort for DiagnosticsStore<'_> { .map_err(|error| port_error("current_diagnostics_for_file", error)) } - async fn stale_diagnostics( - &self, - generation: &CodeGenerationId, - ) -> DiagnosticStoreResult> { - self.stale_records(generation) - .await - .map_err(|error| port_error("stale_diagnostics", error)) - } - async fn diagnostic_by_anchor( &self, anchor: &RetrievalAnchorId, @@ -1267,25 +971,6 @@ impl DiagnosticStorePort for DiagnosticsStore<'_> { .await .map_err(|error| port_error("diagnostic_by_anchor", error)) } - - async fn diagnostic_supersession_chain( - &self, - anchor: &RetrievalAnchorId, - ) -> DiagnosticStoreResult> { - self.supersession_chain(anchor) - .await - .map_err(|error| port_error("diagnostic_supersession_chain", error)) - } - - async fn supersede_diagnostic_generation( - &self, - prior_generation: &CodeGenerationId, - successor_generation: &CodeGenerationId, - ) -> DiagnosticStoreResult { - self.supersede_generation(prior_generation, successor_generation) - .await - .map_err(|error| port_error("supersede_diagnostic_generation", error)) - } } /// One overlay entry: diagnostics computed against unsaved editor content for @@ -1477,17 +1162,8 @@ fn record_from_row(row: &Row, operation: &str) -> Result })?; let state_generation = match kind.state_generation_field() { Some(field) => Some(stored_id( - optional_text(23)?.ok_or_else(|| { - db_message( - operation, - match kind { - DiagnosticRecordStateKindV1::Cleared => { - "cleared record missing state_generation" - } - _ => "superseded record missing state_generation", - }, - ) - })?, + optional_text(23)? + .ok_or_else(|| db_message(operation, "cleared record missing state_generation"))?, operation, field, )?), @@ -1745,76 +1421,15 @@ mod tests { assert_eq!((inserted, cleared), (0, 2)); assert!(store.current_records(&id(gen1)).await.unwrap().is_empty()); - let stale = store.stale_records(&id(gen1)).await.unwrap(); - assert_eq!(stale.len(), 2); - assert!(stale.iter().all(|record| matches!( + // History stays queryable after clearing. + let history = store.records_for_generation(&id(gen1)).await.unwrap(); + assert_eq!(history.len(), 2); + assert!(history.iter().all(|record| matches!( &record.state, DiagnosticRecordStateV1::Cleared { cleared_in_generation } if cleared_in_generation.as_str() == gen2 ))); - // History stays queryable after clearing. - assert_eq!( - store.records_for_generation(&id(gen1)).await.unwrap().len(), - 2 - ); - } - - #[tokio::test] - async fn supersession_marks_old_records_and_chains() { - let temp = tempfile::tempdir().unwrap(); - let path = temp.path().join("diagnostics.db"); - let conn = open_store(&path).await; - let store = DiagnosticsStore::new_runtime(&conn); - let gen1 = "generation.clean.1"; - let gen2 = "generation.clean.2"; - - let prior = fixture_record(gen1, "anchor.diagnostic.1"); - store - .publish_clean_generation(&id(gen1), std::slice::from_ref(&prior)) - .await - .unwrap(); - - assert!( - store - .supersede_generation(&id(gen1), &id(gen1)) - .await - .is_err(), - "a generation cannot supersede itself" - ); - - assert_eq!( - store - .supersede_generation(&id(gen1), &id(gen2)) - .await - .unwrap(), - 1 - ); - let marked = store - .record_by_anchor(&id("anchor.diagnostic.1")) - .await - .unwrap() - .unwrap(); - assert!(matches!( - &marked.state, - DiagnosticRecordStateV1::Superseded { - successor_generation - } if successor_generation.as_str() == gen2 - )); - assert!(!marked.is_current()); - - // The successor publication republishes the same logical finding - // under a new anchor; the chain walks old -> new. - let successor = fixture_record(gen2, "anchor.diagnostic.2"); - store - .publish_clean_generation(&id(gen2), std::slice::from_ref(&successor)) - .await - .unwrap(); - let chain = store - .supersession_chain(&id("anchor.diagnostic.1")) - .await - .unwrap(); - assert_eq!(chain, vec![prior.supersede(id(gen2)).unwrap(), successor]); } #[tokio::test] @@ -1954,7 +1569,7 @@ mod tests { // Stale findings cannot cross snapshots: publishing a non-current // record is rejected. let stale = fixture_record(gen1, "anchor.diagnostic.1") - .supersede(id(gen2)) + .clear(id(gen2)) .unwrap(); assert!( store @@ -2084,12 +1699,6 @@ mod tests { Some(second.clone()), "anchor lookup must follow the current publication header" ); - assert_eq!( - store.records_for_publication(&generation, 1).await.unwrap(), - vec![first], - "the prior immutable publication must remain readable" - ); - assert!(store.records_for_publication(&generation, 0).await.is_err()); } #[tokio::test] @@ -2129,14 +1738,13 @@ mod tests { } #[tokio::test] - async fn cleared_or_superseded_generation_cannot_be_reactivated() { + async fn cleared_generation_cannot_be_reactivated() { let temp = tempfile::tempdir().unwrap(); let path = temp.path().join("diagnostics.db"); let conn = open_store(&path).await; let store = DiagnosticsStore::new_runtime(&conn); let gen1 = "generation.clean.1"; let gen2 = "generation.clean.2"; - let gen3 = "generation.clean.3"; let cleared = fixture_record(gen1, "anchor.diagnostic.cleared"); store @@ -2154,23 +1762,6 @@ mod tests { .is_err(), "a cleared generation must stay historical" ); - - let superseded = fixture_record(gen3, "anchor.diagnostic.superseded"); - store - .publish_clean_generation(&id(gen3), std::slice::from_ref(&superseded)) - .await - .unwrap(); - store - .supersede_generation(&id(gen3), &id("generation.clean.4")) - .await - .unwrap(); - assert!( - store - .publish_clean_generation(&id(gen3), std::slice::from_ref(&superseded)) - .await - .is_err(), - "a superseded generation must stay historical" - ); } #[test] @@ -2220,36 +1811,4 @@ mod tests { "an empty newer overlay snapshot must still fence stale updates" ); } - - #[tokio::test] - async fn root_port_reports_commit_and_exact_replay() { - let temp = tempfile::tempdir().unwrap(); - let path = temp.path().join("diagnostics.db"); - let conn = open_store(&path).await; - let store = DiagnosticsStore::new_runtime(&conn); - let generation = id("generation.clean.1"); - let snapshot = SanitizedCleanDiagnosticSnapshotV1::new( - generation, - vec![fixture_record("generation.clean.1", "anchor.diagnostic.1")], - ) - .unwrap(); - - let committed = store - .publish_clean_diagnostics(snapshot.clone()) - .await - .unwrap(); - assert_eq!( - committed.disposition(), - DiagnosticPublicationDispositionV1::Committed - ); - assert_eq!(committed.inserted_records(), 1); - - let replayed = store.publish_clean_diagnostics(snapshot).await.unwrap(); - assert_eq!( - replayed.disposition(), - DiagnosticPublicationDispositionV1::ExactReplay - ); - assert_eq!(replayed.inserted_records(), 0); - assert_eq!(replayed.cleared_records(), 0); - } } diff --git a/crates/tracedecay-application/src/feedback/concrete.rs b/crates/tracedecay-application/src/feedback/concrete.rs index 630a3b0e44..f9c6ac4931 100644 --- a/crates/tracedecay-application/src/feedback/concrete.rs +++ b/crates/tracedecay-application/src/feedback/concrete.rs @@ -30,7 +30,7 @@ use tracedecay_contracts::feedback::{ FeedbackListRequestV1, FeedbackListResultV1, FeedbackObservationPort, FeedbackPortFuture, FeedbackPublicationReadPort, FeedbackPublicationRecordState, FeedbackPublicationV1, FeedbackReadPort, FeedbackReadPortContext, FeedbackReadPortFuture, FeedbackReadService, - FeedbackRouteAdmission, FeedbackRouteAuthorizationPort, feedback_surface_operation, + FeedbackRouteAuthorizationPort, feedback_surface_operation, }; use tracedecay_contracts::{ ApplicationContractError, ApplicationOperation, ApplicationProblem, AuthorityReceipt, @@ -782,7 +782,7 @@ impl FeedbackRouteAuthorizationPort for ProjectFeedbackRouteAuthorization { context: &RequestContext, operation: &ApplicationOperation, observed_at: UtcMicros, - ) -> Result { + ) -> Result { match context.admission_at(observed_at) { RequestAdmission::Cancelled => { return Err(ApplicationProblem::cancelled_before_admission()); @@ -810,7 +810,6 @@ impl FeedbackRouteAuthorizationPort for ProjectFeedbackRouteAuthorization { .map_err(|_| ApplicationProblem::not_found_or_not_authorized(RetryDirective::Never))?, observed_at, ) - .map(FeedbackRouteAdmission::Routed) .map_err(|_| ApplicationProblem::not_found_or_not_authorized(RetryDirective::Never)) } @@ -818,20 +817,10 @@ impl FeedbackRouteAuthorizationPort for ProjectFeedbackRouteAuthorization { &self, context: &RequestContext, operation: &ApplicationOperation, - admission: &FeedbackRouteAdmission, + admission: &AuthorityReceipt, observed_at: UtcMicros, ) -> Result { let current = self.admit(context, operation, observed_at)?; - let FeedbackRouteAdmission::Routed(admission) = admission else { - return Err(ApplicationProblem::not_found_or_not_authorized( - RetryDirective::Never, - )); - }; - let FeedbackRouteAdmission::Routed(current) = current else { - return Err(ApplicationProblem::not_found_or_not_authorized( - RetryDirective::Never, - )); - }; if admission.grant_id != current.grant_id || admission.grant_revision != current.grant_revision || admission.grant_digest != current.grant_digest diff --git a/crates/tracedecay-application/src/feedback/cycle_runtime.rs b/crates/tracedecay-application/src/feedback/cycle_runtime.rs index 5a94bd0a94..bfe70f9bdb 100644 --- a/crates/tracedecay-application/src/feedback/cycle_runtime.rs +++ b/crates/tracedecay-application/src/feedback/cycle_runtime.rs @@ -20,8 +20,7 @@ use tracedecay_contracts::feedback::{ FeedbackCycleAdvisoryV1, FeedbackCycleExecutionRequest, FeedbackCycleExecutionResult, FeedbackCycleService, FeedbackDiagnosticsReadRequestV1, FeedbackExpandRequestV1, FeedbackImpactPort, FeedbackImpactPortOutcome, FeedbackImpactRequest, FeedbackObservationPort, - FeedbackPortFuture, FeedbackRuntimeStatePort, FeedbackRuntimeStateV1, - GenerationBoundFeedbackDiagnosticsAdapter, + FeedbackPortFuture, FeedbackRuntimeStatePort, GenerationBoundFeedbackDiagnosticsAdapter, }; use tracedecay_contracts::retrieval::{ AffectedTestsRequest, AffectedTestsResult, AffectedTestsRetrievalPort, AnchorExpandRequest, @@ -237,13 +236,13 @@ impl FeedbackCycleRuntimeError { } type ProductionFeedbackCycleService = FeedbackCycleService< - SharedFeedbackRuntimeState, + Arc, GenerationBoundFeedbackDiagnosticsAdapter< DiagnosticStoreFeedbackProvider, >, DirectFeedbackImpactAdapter, ProjectFeedbackStore, - SharedFeedbackObservations, + Arc, ProjectFeedbackRouteAuthorization, >; @@ -319,18 +318,18 @@ pub fn open_feedback_cycle_runtime( let impact = DirectFeedbackImpactAdapter::new( project_root, code_graph, - SharedAffectedTests(affected_tests), + affected_tests, route_authorization.clone(), graph_operation, tests_operation, code_index_identity, ); let service = FeedbackCycleService::new( - SharedFeedbackRuntimeState(runtime_state), + runtime_state, diagnostics, impact, publications.clone(), - SharedFeedbackObservations(observations), + observations, route_authorization, operation, ); @@ -709,22 +708,10 @@ impl FeedbackCycleRuntimePort for FeedbackCycleRuntime { } } -struct SharedFeedbackRuntimeState(Arc); - -impl FeedbackRuntimeStatePort for SharedFeedbackRuntimeState { - fn resolve<'a>( - &'a self, - context: &'a RequestContext, - input: &'a tracedecay_domain::feedback::FeedbackEvaluationInputV1, - ) -> FeedbackPortFuture<'a, Option> { - self.0.resolve(context, input) - } -} - struct DirectFeedbackImpactAdapter { project_root: PathBuf, code_graph: Arc, - tests: SharedAffectedTests, + tests: Arc, authorization: ProjectFeedbackRouteAuthorization, graph_operation: ApplicationOperation, tests_operation: ApplicationOperation, @@ -739,7 +726,7 @@ impl DirectFeedbackImpactAdapter { fn new( project_root: PathBuf, code_graph: Arc, - tests: SharedAffectedTests, + tests: Arc, authorization: ProjectFeedbackRouteAuthorization, graph_operation: ApplicationOperation, tests_operation: ApplicationOperation, @@ -1138,18 +1125,6 @@ impl FeedbackImpactPort for DirectFeedbackImpactAdapter { } } -struct SharedAffectedTests(Arc); - -impl AffectedTestsRetrievalPort for SharedAffectedTests { - fn affected_tests( - &self, - context: &RetrievalPortContext<'_>, - request: &AffectedTestsRequest, - ) -> RetrievalPortOutcome { - self.0.affected_tests(context, request) - } -} - enum DirectAffectedTestsOutcome { Evidence { tests: Vec, @@ -1180,7 +1155,7 @@ fn affected_tests_outcome( DirectAffectedTestsOutcome::Evidence { tests: evidence .payload - .map_or_else(Vec::new, |result| result.tests), + .map_or_else(Vec::new, |result| result.current_tests()), state, } } @@ -1191,7 +1166,7 @@ fn affected_tests_outcome( DirectAffectedTestsOutcome::Evidence { tests: evidence .payload - .map_or_else(Vec::new, |result| result.tests), + .map_or_else(Vec::new, |result| result.current_tests()), state: FeedbackImpactStateV1::Partial, } } @@ -1206,18 +1181,6 @@ fn affected_tests_outcome( } } -struct SharedFeedbackObservations(Arc); - -impl FeedbackObservationPort for SharedFeedbackObservations { - fn observe( - &self, - input: &tracedecay_domain::feedback::FeedbackEvaluationInputV1, - observation: tracedecay_domain::feedback::FeedbackCycleObservationV1, - ) { - self.0.observe(input, observation); - } -} - fn lsp_trigger_matches_invocation( trigger: DiagnosticTrigger, invocation: &FeedbackCycleInvocation, diff --git a/crates/tracedecay-application/src/feedback/diagnostics.rs b/crates/tracedecay-application/src/feedback/diagnostics.rs index a73c6fd70d..ebca7b68bd 100644 --- a/crates/tracedecay-application/src/feedback/diagnostics.rs +++ b/crates/tracedecay-application/src/feedback/diagnostics.rs @@ -11,10 +11,7 @@ use tracedecay_contracts::diagnostics::{ ProviderSourceIdentity, }; use tracedecay_domain::{CodeGenerationId, GenerationDiagnosticV1, RetrievalAnchorId}; -use tracedecay_store::{ - DiagnosticPublicationReceiptV1, DiagnosticStore, DiagnosticStoreResult, - SanitizedCleanDiagnosticSnapshotV1, -}; +use tracedecay_store::{DiagnosticStore, DiagnosticStoreResult}; use crate::diagnostics_store::DiagnosticsStore; use crate::lsp_runtime::LspFeedbackDiagnosticRecordPort; @@ -53,15 +50,6 @@ impl LspFeedbackDiagnosticRecordPort for DatabaseDiagnosticStore { } impl DiagnosticStore for DatabaseDiagnosticStore { - async fn publish_clean_diagnostics( - &self, - snapshot: SanitizedCleanDiagnosticSnapshotV1, - ) -> DiagnosticStoreResult { - DiagnosticsStore::new(self.database.clone()) - .publish_clean_diagnostics(snapshot) - .await - } - async fn current_diagnostic_generation( &self, ) -> DiagnosticStoreResult> { @@ -82,16 +70,6 @@ impl DiagnosticStore for DatabaseDiagnosticStore { Ok(records) } - async fn diagnostics_for_publication( - &self, - generation: &CodeGenerationId, - publication_revision: u64, - ) -> DiagnosticStoreResult> { - DiagnosticsStore::new(self.database.clone()) - .diagnostics_for_publication(generation, publication_revision) - .await - } - #[hotpath::measure(label = "usecases.diagnostics.current", future = true)] async fn current_diagnostics( &self, @@ -115,15 +93,6 @@ impl DiagnosticStore for DatabaseDiagnosticStore { .await } - async fn stale_diagnostics( - &self, - generation: &CodeGenerationId, - ) -> DiagnosticStoreResult> { - DiagnosticsStore::new(self.database.clone()) - .stale_diagnostics(generation) - .await - } - #[hotpath::measure(label = "usecases.diagnostics.by_anchor", future = true)] async fn diagnostic_by_anchor( &self, @@ -133,25 +102,6 @@ impl DiagnosticStore for DatabaseDiagnosticStore { .diagnostic_by_anchor(anchor) .await } - - async fn diagnostic_supersession_chain( - &self, - anchor: &RetrievalAnchorId, - ) -> DiagnosticStoreResult> { - DiagnosticsStore::new(self.database.clone()) - .diagnostic_supersession_chain(anchor) - .await - } - - async fn supersede_diagnostic_generation( - &self, - prior_generation: &CodeGenerationId, - successor_generation: &CodeGenerationId, - ) -> DiagnosticStoreResult { - DiagnosticsStore::new(self.database.clone()) - .supersede_diagnostic_generation(prior_generation, successor_generation) - .await - } } /// Concrete adapter over the existing diagnostic-store read port. It is kept diff --git a/crates/tracedecay-application/src/feedback/owner.rs b/crates/tracedecay-application/src/feedback/owner.rs index a16865fcfa..487f0ae60b 100644 --- a/crates/tracedecay-application/src/feedback/owner.rs +++ b/crates/tracedecay-application/src/feedback/owner.rs @@ -342,6 +342,7 @@ fn project_feedback_evidence( request_id, scope, outcome, + .. } = envelope; let ApplicationOutcome::Evidence(packet) = outcome else { unreachable!("feedback reads return evidence outcomes"); diff --git a/crates/tracedecay-application/src/lsp_runtime/advisory_source_tests.rs b/crates/tracedecay-application/src/lsp_runtime/advisory_source_tests.rs index e49137b363..518e38f578 100644 --- a/crates/tracedecay-application/src/lsp_runtime/advisory_source_tests.rs +++ b/crates/tracedecay-application/src/lsp_runtime/advisory_source_tests.rs @@ -378,19 +378,6 @@ impl FeedbackImpactPort for FixedImpact { } } -#[derive(Clone)] -struct Observations(Arc); - -impl FeedbackObservationPort for Observations { - fn observe( - &self, - input: &tracedecay_domain::feedback::FeedbackEvaluationInputV1, - observation: tracedecay_domain::feedback::FeedbackCycleObservationV1, - ) { - self.0.observe(input, observation); - } -} - fn feedback_service( runtime: Arc, request: &FeedbackCycleExecutionRequest, @@ -399,7 +386,7 @@ fn feedback_service( SavedDiagnostics, FixedImpact, crate::feedback::concrete::ProjectFeedbackStore, - Observations, + Arc, crate::feedback::concrete::ProjectFeedbackRouteAuthorization, > { let provider = request.providers.first().expect("saved provider").clone(); @@ -446,7 +433,7 @@ fn feedback_service( }, FixedImpact(impact), runtime.publication_store(), - Observations(runtime.observation_port()), + runtime.observation_port(), runtime.route_authorization(), operation(), ) diff --git a/crates/tracedecay-application/src/observability/delivery_recorder.rs b/crates/tracedecay-application/src/observability/delivery_recorder.rs index 8804aa7b8d..a087d5c146 100644 --- a/crates/tracedecay-application/src/observability/delivery_recorder.rs +++ b/crates/tracedecay-application/src/observability/delivery_recorder.rs @@ -290,11 +290,9 @@ async fn drain_once( let mut acknowledged = 0_usize; for receipt in receipts { let result = async { - let receipt_authority = match receipt.emission_identity.as_ref() { - Some(identity) => authority.alias_for_durable_replay(identity), - None => authority.alias_with_policy_identity(authority.identity().clone()), - } - .map_err(|error| ApplicationContractError::Domain(error.to_owned()))?; + let receipt_authority = authority + .alias_for_durable_replay(&receipt.emission_identity) + .map_err(|error| ApplicationContractError::Domain(error.to_owned()))?; receipt_authority.begin(&receipt.settlement.attempt).await?; receipt_authority.settle(&receipt.settlement).await?; spool @@ -348,97 +346,3 @@ const fn map_spool_admission_error(error: DeliveryRecorderSpoolError) -> &'stati } } } - -#[cfg(test)] -mod tests { - use tracedecay_domain::{ - DeliveryChannelIdentityV1, DeliveryEventClassV1, DeliverySettlementAttemptV1, - DeliverySettlementOutcomeV1, DeliverySurfaceFamilyV1, ProjectId, UtcMicros, - canonical_sha256, sha256_hex_suffix, - }; - - use super::super::BoundedObservabilityProducerV1; - use super::*; - - fn legacy_receipt_id(settlement: &DeliverySettlementV1) -> [u8; 16] { - let digest = - canonical_sha256(&("tracedecay.delivery-recorder-source-receipt.v1", settlement)) - .expect("legacy receipt digest"); - let hex = sha256_hex_suffix(digest.as_str()).expect("canonical digest prefix"); - let mut receipt_id = [0_u8; 16]; - for (index, slot) in receipt_id.iter_mut().enumerate() { - let offset = index * 2; - *slot = u8::from_str_radix(&hex[offset..offset + 2], 16).expect("canonical digest hex"); - } - receipt_id - } - - #[tokio::test] - async fn legacy_v1_receipt_replays_through_current_process_identity() { - let _pin = tracedecay_runtime_core::config::PinnedUserDataDir::new(); - let project = tempfile::tempdir().expect("project"); - let project_id = ProjectId::new("project.delivery.legacy-replay").expect("project id"); - let runtime = tracedecay_global_db::tests::harness::RegisteredGlobalDbTestRuntime::project( - tracedecay_runtime_core::storage::default_profile_root().expect("profile root"), - project.path(), - project_id.clone(), - ) - .await - .expect("registered runtime"); - let db = runtime.project_database_arc().expect("project database"); - let identity = ObservabilityProducerIdentityV1 { - authorized_scope_ref: project_id.as_str().to_owned(), - process_boot_id: "boot:delivery-legacy-replay".to_owned(), - producer_revision: "delivery-legacy-replay-producer.v1".to_owned(), - configuration_revision: "delivery-legacy-replay-config.v1".to_owned(), - policy_revision: "delivery-legacy-replay-policy.v1".to_owned(), - }; - let producer = Arc::new( - BoundedObservabilityProducerV1::start(db.clone(), identity.clone(), 8) - .expect("producer"), - ); - let authority = Arc::new( - DeliverySettlementAuthorityV1::new(db, Arc::clone(&producer), identity) - .expect("settlement authority"), - ); - let settlement = DeliverySettlementV1 { - attempt: DeliverySettlementAttemptV1 { - owner_event_id: "work:delivery-legacy-replay".to_owned(), - event_class: DeliveryEventClassV1::OperationTerminal, - channel: DeliveryChannelIdentityV1 { - surface: DeliverySurfaceFamilyV1::Mcp, - channel_ref: "mcp:delivery-legacy-replay".to_owned(), - }, - work_attempt: None, - eligible: 1, - valid_at: UtcMicros(100), - attempted_at: UtcMicros(110), - }, - outcome: DeliverySettlementOutcomeV1::Delivered, - settled_at: UtcMicros(120), - drop_reason: None, - }; - let spool = DeliveryRecorderSpoolV1::open(authority.spool_root()).expect("spool"); - spool - .append(&DeliveryRecorderSourceReceiptV1 { - receipt_id: legacy_receipt_id(&settlement), - settlement, - emission_identity: None, - }) - .expect("append legacy receipt"); - let mut summary = DeliverySettlementRecorderSummaryV1::default(); - - assert_eq!( - drain_once(authority.as_ref(), &spool, &mut summary).await, - 1 - ); - assert_eq!(summary.settled, 1); - assert_eq!(spool.len().expect("pending receipts"), 0); - - drop(spool); - drop(authority); - let producer = Arc::try_unwrap(producer) - .unwrap_or_else(|_| panic!("authority must release producer after replay")); - producer.shutdown().await.expect("flush producer"); - } -} diff --git a/crates/tracedecay-application/src/observability/delivery_spool.rs b/crates/tracedecay-application/src/observability/delivery_spool.rs index c68419ec8e..8cd9f61610 100644 --- a/crates/tracedecay-application/src/observability/delivery_spool.rs +++ b/crates/tracedecay-application/src/observability/delivery_spool.rs @@ -1,6 +1,6 @@ use std::collections::BTreeSet; use std::ffi::OsString; -use std::fs::{self, File, OpenOptions}; +use std::fs::{self, OpenOptions}; #[cfg(unix)] use std::os::unix::fs::{OpenOptionsExt, PermissionsExt}; use std::path::{Path, PathBuf}; @@ -12,6 +12,7 @@ use tracedecay_domain::canonical_text::is_lowercase_hex; use tracedecay_domain::{ DeliverySettlementV1, canonical_json_bytes, canonical_sha256, sha256_hex_suffix, }; +use tracedecay_private_fs::FileLease; use tracedecay_private_fs::framed_log::{ DirectorySyncPolicy, atomic_write, is_owned_temporary_name, read_bounded, remove_abandoned_temporaries, sync_directory, validate_regular_or_missing, @@ -30,11 +31,8 @@ const DIRECTORY_POLICY: DirectorySyncPolicy = DirectorySyncPolicy::Strict; pub(super) struct DeliveryRecorderSourceReceiptV1 { pub receipt_id: [u8; 16], pub settlement: DeliverySettlementV1, - /// Exact linked-root emission identity selected at admission. Receipts - /// written before policy-specific frontends omit this and replay through - /// the recorder's store-core identity. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub emission_identity: Option, + /// Exact linked-root emission identity selected at admission. + pub emission_identity: ObservabilityProducerIdentityV1, } impl DeliveryRecorderSourceReceiptV1 { @@ -61,7 +59,7 @@ impl DeliveryRecorderSourceReceiptV1 { Ok(Self { receipt_id, settlement, - emission_identity: Some(emission_identity), + emission_identity, }) } @@ -69,33 +67,14 @@ impl DeliveryRecorderSourceReceiptV1 { if self.receipt_id == [0; 16] { return Err(DeliveryRecorderSpoolError::InvalidReceipt); } - self.settlement - .validate() - .map_err(|_| DeliveryRecorderSpoolError::InvalidReceipt)?; - let expected_receipt_id = if let Some(emission_identity) = &self.emission_identity { - Self::new(self.settlement.clone(), emission_identity.clone())?.receipt_id - } else { - legacy_receipt_id(&self.settlement)? - }; - if expected_receipt_id != self.receipt_id { + let expected = Self::new(self.settlement.clone(), self.emission_identity.clone())?; + if expected.receipt_id != self.receipt_id { return Err(DeliveryRecorderSpoolError::InvalidReceipt); } Ok(()) } } -fn legacy_receipt_id( - settlement: &DeliverySettlementV1, -) -> Result<[u8; 16], DeliveryRecorderSpoolError> { - let digest = canonical_sha256(&("tracedecay.delivery-recorder-source-receipt.v1", settlement)) - .map_err(|_| DeliveryRecorderSpoolError::InvalidReceipt)?; - let hex = - sha256_hex_suffix(digest.as_str()).ok_or(DeliveryRecorderSpoolError::InvalidReceipt)?; - let mut receipt_id = [0_u8; 16]; - decode_hex_prefix(hex, &mut receipt_id)?; - Ok(receipt_id) -} - #[derive(Debug, Error, Eq, PartialEq)] pub(super) enum DeliveryRecorderSpoolError { #[error("delivery recorder receipt is invalid")] @@ -117,7 +96,7 @@ pub(super) enum DeliveryRecorderSpoolError { /// One process lease over the durable, bounded source receipts for a project. pub(super) struct DeliveryRecorderSpoolV1 { root: PathBuf, - _lease: File, + _lease: FileLease, state: Mutex, } @@ -149,6 +128,7 @@ impl DeliveryRecorderSpoolV1 { std::fs::TryLockError::WouldBlock => DeliveryRecorderSpoolError::Busy, std::fs::TryLockError::Error(_) => DeliveryRecorderSpoolError::Io, })?; + let lease = FileLease::held(lease, "delivery_recorder.spool"); // The lease is exclusive now, so every staging temporary still in the // root was abandoned by a killed publisher. Sweeping it here is what // makes a crashed daemon's project reopenable. @@ -420,11 +400,7 @@ mod tests { fn v2_receipt_refuses_tampered_emission_identity() { let mut receipt = DeliveryRecorderSourceReceiptV1::new(settlement(), identity()).expect("v2 receipt"); - receipt - .emission_identity - .as_mut() - .expect("v2 emission identity") - .policy_revision = "delivery-spool-tampered-policy.v1".to_owned(); + receipt.emission_identity.policy_revision = "delivery-spool-tampered-policy.v1".to_owned(); assert_eq!( receipt.validate(), diff --git a/crates/tracedecay-application/src/observability/product_view_emit.rs b/crates/tracedecay-application/src/observability/product_view_emit.rs index c8154f3910..bff59e5960 100644 --- a/crates/tracedecay-application/src/observability/product_view_emit.rs +++ b/crates/tracedecay-application/src/observability/product_view_emit.rs @@ -1,6 +1,6 @@ //! Product-view observations emitted only from exact Work owner results. -use tracedecay_contracts::{GeneratedWorkProposal, ReviewProposalDispositionV1}; +use tracedecay_contracts::{GeneratedWorkProposal, ReviewWorkProposalDispositionV1}; use tracedecay_domain::{ AppropriateRelianceObservedV1, AutomationFunnelObservedV1, AutomationTerminalV1, CoverageStateV1, ObservabilityPayloadV1, ObservabilityTerminalResultV1, ObservedTernaryV1, @@ -103,7 +103,7 @@ pub fn record_reliance_decision( producer: Option<&BoundedObservabilityProducerV1>, proposal_ref: &str, command_ref: &str, - disposition: Option, + disposition: Option, observed_at: UtcMicros, ) -> WorkOwnerObservationResultV1 { let Some(producer) = producer else { @@ -111,10 +111,10 @@ pub fn record_reliance_decision( }; let decision = match disposition { None => RelianceDecisionV1::Accepted, - Some(ReviewProposalDispositionV1::Rejected) => RelianceDecisionV1::Rejected, + Some(ReviewWorkProposalDispositionV1::Rejected) => RelianceDecisionV1::Rejected, // Superseding does not carry the rationale required to classify an // override, so it is deliberately not projected as reliance. - Some(ReviewProposalDispositionV1::Superseded) => { + Some(ReviewWorkProposalDispositionV1::Superseded) => { return WorkOwnerObservationResultV1::Unavailable; } }; diff --git a/crates/tracedecay-application/src/operation_stream.rs b/crates/tracedecay-application/src/operation_stream.rs index 6ebccf8521..69d765102d 100644 --- a/crates/tracedecay-application/src/operation_stream.rs +++ b/crates/tracedecay-application/src/operation_stream.rs @@ -34,9 +34,10 @@ use tracedecay_domain::{ use tracedecay_tool_catalog::{CapabilityId, SchemaId, UseCaseId}; use tracedecay_temporal_query::cursor::{CursorError, StableSortKey, encode_cursor, verify_cursor}; -use tracedecay_temporal_query::ports::{ - BindingDigest, InMemoryCursorAuthenticator, KernelVersions, TemporalExecutionSnapshot, - TemporalSnapshotRequest, TemporalWatermarks, +use tracedecay_temporal_query::execution::BindingDigest; +use tracedecay_temporal_query::ports::{InMemoryCursorAuthenticator, TemporalSnapshotRequest}; +use tracedecay_temporal_query::snapshot::{ + KernelVersions, TemporalExecutionSnapshot, TemporalWatermarks, }; use tracedecay_temporal_query::resolution::ValidatedAuthorization; @@ -1184,13 +1185,26 @@ impl OperationEventAuthority { }) } - /// Drops all memory-retained frontiers. Existing streams close; reconnects - /// receive `FrontierExpired` rather than a fabricated snapshot. - #[hotpath::measure(label = "usecases.operation.expire_all", future = true)] - pub async fn expire_all(&self) { + /// Drops every memory-retained frontier that no live producer still + /// writes. Streams on those operations close; reconnects receive + /// `FrontierExpired` rather than a fabricated snapshot. + /// + /// A record whose `OperationEmitter` is still alive is kept. The + /// process-global authority is shared by every daemon composition in the + /// process and by the project workflow handlers that begin managed test + /// runs outside any composition, so one composition's shutdown must not + /// truncate a stream another producer is between admission and its first + /// result on. The emitter is the only holder of the cancellation + /// receiver, so its receiver count is the producer liveness signal. + #[hotpath::measure(label = "usecases.operation.expire_idle", future = true)] + pub async fn expire_idle(&self) { let mut state = self.inner.state.lock().await; - state.operations.clear(); - state.insertion_order.clear(); + let AuthorityState { + operations, + insertion_order, + } = &mut *state; + operations.retain(|_, record| record.cancellation.receiver_count() > 0); + insertion_order.retain(|operation_id| operations.contains_key(operation_id)); } #[hotpath::measure(label = "usecases.operation.emit_progress", future = true)] @@ -1787,7 +1801,8 @@ mod tests { use super::{ CanonicalManagedTestRunReader, ManagedTestRunCurrentScope, ManagedTestRunReadOutcome, ManagedTestRunStaleReason, ManagedTestRunUnavailableReason, OperationCancelOutcome, - OperationEventAuthority, OperationEventError, OperationId, + OperationEventAuthority, OperationEventError, OperationEventItem, OperationId, + StreamEventKind, }; #[test] @@ -1870,6 +1885,76 @@ mod tests { assert_eq!(snapshot.receipt, Some(receipt)); } + /// A daemon composition shutting down in the same process (the harness + /// runs many) expires the shared authority between a managed run's + /// admission and its first result. The accepted record must survive + /// while its producer is alive; only producer-less frontiers expire. + #[tokio::test] + async fn operation_history_keeps_the_accepted_record_for_a_live_producer_across_expiry() { + let authority = OperationEventAuthority::default(); + let live_request = RequestId::new("request.test-run.live-producer").expect("request id"); + let live = authority + .begin_managed_test_run( + "file:///workspace/live".to_owned(), + live_request.clone(), + None, + None, + BTreeMap::new(), + Deadline::new(UtcMicros(10_000)).expect("deadline"), + ) + .await + .expect("live managed test run"); + drop( + authority + .begin_managed_test_run( + "file:///workspace/abandoned".to_owned(), + RequestId::new("request.test-run.abandoned-producer").expect("request id"), + None, + None, + BTreeMap::new(), + Deadline::new(UtcMicros(10_000)).expect("deadline"), + ) + .await + .expect("abandoned managed test run"), + ); + + authority.expire_idle().await; + + let first_result = live + .test_result("suite::first".to_owned(), true) + .await + .expect("first result after a peer composition expired idle frontiers"); + assert_eq!(first_result.sequence, 1); + let state = authority.inner.state.lock().await; + let history = &state.operations[&OperationId::from_request(live_request)].history; + assert!( + matches!( + history.front().map(|event| (event.sequence, &event.kind)), + Some(( + 0, + StreamEventKind::Item(OperationEventItem::Accepted { .. }) + )) + ), + "the accepted record must precede the first result: {history:?}" + ); + assert_eq!(history.len(), 2); + assert!( + !state.operations.contains_key(&OperationId::from_request( + RequestId::new("request.test-run.abandoned-producer").expect("request id"), + )), + "a frontier without a live producer expires" + ); + assert_eq!(state.insertion_order.len(), 1); + drop(state); + assert_eq!( + authority + .latest_managed_test_run("file:///workspace/abandoned") + .await + .err(), + Some(OperationEventError::FrontierExpired) + ); + } + #[tokio::test] async fn managed_test_authority_cancellation_reaches_the_emitter() { let authority = OperationEventAuthority::default(); diff --git a/crates/tracedecay-application/src/pr_tracking.rs b/crates/tracedecay-application/src/pr_tracking.rs index ece4e6bc1e..c20dea0acf 100644 --- a/crates/tracedecay-application/src/pr_tracking.rs +++ b/crates/tracedecay-application/src/pr_tracking.rs @@ -102,7 +102,6 @@ pub struct PrDiscovery { pub struct ManagedPr { pub pr: u64, pub head_branch: String, - #[serde(default)] pub head_sha: String, pub worktree: PathBuf, pub tracking_ref: String, @@ -585,19 +584,4 @@ mod tests { assert!(parse_gh_pr_list(json, 2).expect("partial list").partial); assert!(!parse_gh_pr_list(json, 3).expect("complete list").partial); } - - #[test] - fn legacy_state_without_head_sha_remains_refreshable() { - let store = tempfile::tempdir().expect("store root"); - std::fs::write( - state_path(store.path()), - r#"{"managed":{"pr/8":{"pr":8,"head_branch":"legacy","worktree":"pr-worktrees/pr-8","tracking_ref":"refs/tracedecay/pr/8"}}}"#, - ) - .expect("legacy state"); - - assert_eq!( - load_state(store.path()).expect("load legacy state").managed["pr/8"].head_sha, - "" - ); - } } diff --git a/crates/tracedecay-application/src/pr_tracking/worktrees.rs b/crates/tracedecay-application/src/pr_tracking/worktrees.rs index d671021392..e34eb4696f 100644 --- a/crates/tracedecay-application/src/pr_tracking/worktrees.rs +++ b/crates/tracedecay-application/src/pr_tracking/worktrees.rs @@ -3,6 +3,7 @@ use std::path::{Path, PathBuf}; use tracedecay_domain::canonical_text::sha256_hex; +use tracedecay_private_fs::FileLease; use tracedecay_runtime_core::branch::BranchAddOutcome; use super::{ @@ -94,7 +95,7 @@ impl ManualBranchArtifactsV1 { /// partially replaced branch route. pub struct ManualBranchLifecycleLeaseV1 { branch: String, - _lock: std::fs::File, + _lock: FileLease, } impl ManualBranchLifecycleLeaseV1 { @@ -141,7 +142,7 @@ pub fn try_acquire_manual_branch_lifecycle( })?; Ok(ManualBranchLifecycleLeaseV1 { branch: branch.to_owned(), - _lock: lock, + _lock: FileLease::held(lock, "pr_tracking.manual_branch_lifecycle"), }) } @@ -777,7 +778,6 @@ pub fn cleanup_pr_worktree( data_root: &Path, pr: u64, expected_head: &str, - remove_synthetic_branch: bool, command_control: &PrCommandControlV1, ) -> std::result::Result { let worktree = data_root.join("pr-worktrees").join(format!("pr-{pr}")); @@ -785,14 +785,11 @@ pub fn cleanup_pr_worktree( let label = pr_label(pr); let branch_ref = format!("refs/heads/{label}"); let artifacts = || { - let mut artifacts = vec![ + vec![ PrCleanupArtifact::Worktree(worktree.clone()), PrCleanupArtifact::TrackingRef(tracking_ref.clone()), - ]; - if remove_synthetic_branch { - artifacts.push(PrCleanupArtifact::Branch(branch_ref.clone())); - } - artifacts + PrCleanupArtifact::Branch(branch_ref.clone()), + ] }; if command_control.is_cancelled() { return Err(PrCleanupError::Remaining(artifacts())); @@ -814,9 +811,7 @@ pub fn cleanup_pr_worktree( } })?; if let Some(owned_head) = owned_head { - if remove_synthetic_branch - && ref_points_to(repo_root, &branch_ref, &owned_head, command_control)? - { + if ref_points_to(repo_root, &branch_ref, &owned_head, command_control)? { successful_git_with_control(repo_root, &["branch", "-D", &label], command_control) .map_err(|source| PrCleanupError::Command { artifact: PrCleanupArtifact::Branch(branch_ref.clone()), @@ -839,7 +834,7 @@ pub fn cleanup_pr_worktree( let remaining = remaining_pr_artifacts( repo_root, &worktree, - remove_synthetic_branch.then_some(branch_ref.as_str()), + &branch_ref, &tracking_ref, &verification_control, )?; @@ -897,7 +892,7 @@ fn cleanup_artifact_for_ref(reference: &str) -> PrCleanupArtifact { fn remaining_pr_artifacts( repo_root: &Path, worktree: &Path, - branch_ref: Option<&str>, + branch_ref: &str, tracking_ref: &str, command_control: &PrCommandControlV1, ) -> std::result::Result, PrCleanupError> { @@ -926,9 +921,7 @@ fn remaining_pr_artifacts( { remaining.push(PrCleanupArtifact::Worktree(worktree.to_owned())); } - if let Some(branch_ref) = branch_ref - && ref_sha(repo_root, branch_ref, command_control)?.is_some() - { + if ref_sha(repo_root, branch_ref, command_control)?.is_some() { remaining.push(PrCleanupArtifact::Branch(branch_ref.to_owned())); } if ref_sha(repo_root, tracking_ref, command_control)?.is_some() { @@ -1053,21 +1046,13 @@ pub async fn cleanup_pr_worktree_off_runtime( data_root: &Path, pr: u64, expected_head: &str, - remove_synthetic_branch: bool, command_control: PrCommandControlV1, ) -> std::result::Result { let repo_root = repo_root.to_path_buf(); let data_root = data_root.to_path_buf(); let expected_head = expected_head.to_owned(); tokio::task::spawn_blocking(move || { - cleanup_pr_worktree( - &repo_root, - &data_root, - pr, - &expected_head, - remove_synthetic_branch, - &command_control, - ) + cleanup_pr_worktree(&repo_root, &data_root, pr, &expected_head, &command_control) }) .await .map_err(|error| PrCleanupError::Join(error.to_string()))? diff --git a/crates/tracedecay-application/src/primitives/concrete.rs b/crates/tracedecay-application/src/primitives/concrete.rs index 3e0f5e8b8f..6bb2755705 100644 --- a/crates/tracedecay-application/src/primitives/concrete.rs +++ b/crates/tracedecay-application/src/primitives/concrete.rs @@ -14,13 +14,15 @@ use tracedecay_contracts::{ }; use tracedecay_domain::{CodeGenerationId, UtcMicros}; use tracedecay_runtime_core::db::Database; +use tracedecay_runtime_core::path_safety::plain_host_path; use super::symbol_graph::{SymbolGraphCursorFuture, SymbolGraphCursorPort, SymbolGraphPageClaim}; use tracedecay_graph_query::SourceReadContext; use tracedecay_graph_query::context::read_modes::{LineRange, ReadMode}; use tracedecay_graph_query::context::source_read::{SourceReadRequest, read_source}; use tracedecay_temporal_query::cursor::{CursorError, StableSortKey, encode_cursor, verify_cursor}; -use tracedecay_temporal_query::ports::{SessionCursorAuthenticator, TemporalExecutionSnapshot}; +use tracedecay_temporal_query::ports::SessionCursorAuthenticator; +use tracedecay_temporal_query::snapshot::TemporalExecutionSnapshot; /// Production source-read adapter bound to one admitted project root. /// @@ -42,7 +44,7 @@ impl SourceReadAdapter { code_graph: Arc, scope: ResolvedScope, ) -> Result { - let admitted_project_root = source_runtime.project_root().to_path_buf(); + let admitted_project_root = plain_host_path(source_runtime.project_root()); Self::new_bound(source_runtime, code_graph, scope, &admitted_project_root) } @@ -57,7 +59,10 @@ impl SourceReadAdapter { return Err(source_binding_error()); } let project_root = source_runtime.project_root(); - if project_root != admitted_project_root { + // The admitted root arrives through a file URL, which cannot carry the + // Windows `\\?\` verbatim prefix `canonicalize` gives the runtime root, + // so the runtime root is compared in the spelling a URL can publish. + if plain_host_path(project_root) != admitted_project_root { return Err(source_binding_error()); } Ok(Self { @@ -454,6 +459,7 @@ fn primitive_failure( #[cfg(test)] mod tests { use std::collections::BTreeSet; + use std::path::{Path, PathBuf}; use std::sync::Arc; use tracedecay_contracts::{ @@ -473,15 +479,16 @@ mod tests { use super::{ AuthenticatedSymbolGraphCursorAdapter, SourceReadAdapter, SymbolGraphCursorSnapshot, - SymbolGraphCursorSnapshotAuthority, + SymbolGraphCursorSnapshotAuthority, source_binding_error, }; use crate::primitives::SymbolGraphCursorPort; use tracedecay_contracts::retrieval::PrimitiveFailureKind; - use tracedecay_temporal_query::ports::{ - BindingDigest, InMemoryCursorAuthenticator, KernelVersions, TemporalExecutionSnapshot, - TemporalSnapshotRequest, TemporalWatermarks, - }; + use tracedecay_temporal_query::execution::BindingDigest; + use tracedecay_temporal_query::ports::{InMemoryCursorAuthenticator, TemporalSnapshotRequest}; use tracedecay_temporal_query::resolution::ValidatedAuthorization; + use tracedecay_temporal_query::snapshot::{ + KernelVersions, TemporalExecutionSnapshot, TemporalWatermarks, + }; const NOW: UtcMicros = UtcMicros(1_000); @@ -546,6 +553,35 @@ mod tests { .is_err() ); + // A Windows runtime root is spelled `\\?\D:\...` by `canonicalize`, + // while the root admitted through its file URL reads `D:\...`. + let verbatim_root = |root: &str| { + Arc::new(SourceReadContext::new( + PathBuf::from(format!(r"\\?\{root}")), + database.clone(), + true, + scope.project_id.as_str().to_owned(), + )) + }; + let verbatim = SourceReadAdapter::new_bound( + verbatim_root(r"D:\repo"), + Arc::clone(&projection), + scope.clone(), + Path::new(r"D:\repo"), + ) + .expect("a verbatim runtime root binds to its URL spelling"); + assert_eq!(verbatim.project_root, PathBuf::from(r"\\?\D:\repo")); + assert_eq!( + SourceReadAdapter::new_bound( + verbatim_root(r"D:\foreign"), + Arc::clone(&projection), + scope.clone(), + Path::new(r"D:\repo"), + ) + .err(), + Some(source_binding_error()) + ); + let matching = Arc::new(SourceReadContext::new( admitted_root.clone(), database, diff --git a/crates/tracedecay-application/src/primitives/production.rs b/crates/tracedecay-application/src/primitives/production.rs index e13f1498b2..2d5022e155 100644 --- a/crates/tracedecay-application/src/primitives/production.rs +++ b/crates/tracedecay-application/src/primitives/production.rs @@ -42,15 +42,16 @@ use tracedecay_graph_query::queries::{GraphQueryManager, is_test_marker}; use tracedecay_graph_query::{ CodeGraphProjectionReadPort, CodeGraphReadError, CodeGraphReadRequest, }; -use tracedecay_session_temporal_store::SessionTemporalCursorKeyProvider; +use tracedecay_session_temporal_store::{SessionTemporalAccess, SessionTemporalCursorKeyProvider}; use tracedecay_temporal_query::cursor::{ CURSOR_LIFETIME_MICROS, StableSortKey, encode_cursor, verify_cursor, }; -use tracedecay_temporal_query::ports::{ - BindingDigest, KernelVersions, SessionCursorAuthenticator, TemporalExecutionSnapshot, - TemporalSnapshotRequest, TemporalWatermarks, -}; +use tracedecay_temporal_query::execution::BindingDigest; +use tracedecay_temporal_query::ports::{SessionCursorAuthenticator, TemporalSnapshotRequest}; use tracedecay_temporal_query::resolution::ValidatedAuthorization; +use tracedecay_temporal_query::snapshot::{ + KernelVersions, TemporalExecutionSnapshot, TemporalWatermarks, +}; mod affected_tests; #[cfg(test)] @@ -628,8 +629,7 @@ pub async fn open_production_primitive_runtime( let project_root = source_runtime.project_root().to_path_buf(); let scope = access.scope.clone(); let configuration_digest = access.configuration_digest.clone(); - let key = session_db - .as_ref() + let key = SessionTemporalAccess::new(session_db.as_ref()) .ensure_active_session_cursor_key_result() .await .map_err(|_| ApplicationContractError::Inconsistent { diff --git a/crates/tracedecay-application/src/primitives/production/affected_tests.rs b/crates/tracedecay-application/src/primitives/production/affected_tests.rs index 81d2931ba1..046bb336ae 100644 --- a/crates/tracedecay-application/src/primitives/production/affected_tests.rs +++ b/crates/tracedecay-application/src/primitives/production/affected_tests.rs @@ -172,7 +172,6 @@ pub(super) fn attributed_tests_outcome( ); } - let mut tests = Vec::new(); let mut attributions = Vec::new(); let mut matching_incomplete = false; for record in &join.records { @@ -200,7 +199,13 @@ pub(super) fn attributed_tests_outcome( FreshnessState::Unknown, ); }; - if test_occurrence.occurrence_id != record.attribution.test_occurrence { + let attribution = AffectedTestAttributionV1 { + test: test_occurrence.occurrence_id.clone(), + evidence_class: *evidence_class, + }; + if attribution.test != record.attribution.test_occurrence + || !attribution.is_current_candidate() + { return affected_tests_unavailable( request, finished_at, @@ -208,11 +213,7 @@ pub(super) fn attributed_tests_outcome( FreshnessState::Unknown, ); } - tests.push(test_occurrence.occurrence_id.clone()); - attributions.push(AffectedTestAttributionV1 { - test: test_occurrence.occurrence_id.clone(), - evidence_class: *evidence_class, - }); + attributions.push(attribution); } else { matching_incomplete = true; if matches!( @@ -229,19 +230,18 @@ pub(super) fn attributed_tests_outcome( } } } - tests.sort(); - tests.dedup(); attributions.sort_by(|left, right| { (&left.test, left.evidence_class).cmp(&(&right.test, right.evidence_class)) }); attributions.dedup(); + let result = AffectedTestsResult { attributions }; let complete = read.provider_state == ProviderEvaluationStateV1::SupportedCompletedComplete && read.coverage.is_complete() && matches!(join.coverage, GenerationTestJoinCoverageV1::Complete) && !matching_incomplete; let (visited, eligible) = affected_tests_provider_counts(&read.coverage); - if eligible.is_some_and(|eligible| tests.len() as u64 > eligible) { + if eligible.is_some_and(|eligible| result.current_tests().len() as u64 > eligible) { return affected_tests_unavailable( request, finished_at, @@ -251,10 +251,7 @@ pub(super) fn attributed_tests_outcome( } let evidence = affected_tests_evidence( request, - Some(AffectedTestsResult { - tests, - attributions, - }), + Some(result), finished_at, if complete { CoverageCompleteness::Complete @@ -351,7 +348,7 @@ pub(super) fn affected_tests_evidence( ) -> RetrievalEvidence { let returned = payload .as_ref() - .map_or(0, |result| result.tests.len() as u64); + .map_or(0, |result| result.current_tests().len() as u64); RetrievalEvidence { payload, temporal: TemporalState { diff --git a/crates/tracedecay-application/src/primitives/production/affected_tests_tests.rs b/crates/tracedecay-application/src/primitives/production/affected_tests_tests.rs index d7e07f143c..4949a923c0 100644 --- a/crates/tracedecay-application/src/primitives/production/affected_tests_tests.rs +++ b/crates/tracedecay-application/src/primitives/production/affected_tests_tests.rs @@ -721,7 +721,7 @@ fn exact_project_and_generation_route_canonical_attribution() { ); let payload = evidence.payload.expect("payload"); assert_eq!( - payload.tests, + payload.current_tests(), vec![SymbolOccurrenceId::new("symbol.test").expect("test")] ); assert_eq!( @@ -812,13 +812,48 @@ fn unknown_attribution_remains_typed_partial() { panic!("unknown attribution must stay partial"); }; let payload = evidence.payload.expect("payload"); - assert!(payload.tests.is_empty()); + assert!(payload.current_tests().is_empty()); assert_eq!( payload.attributions[0].evidence_class, TestAttributionEvidenceClassV1::UnknownUnsupported ); } +#[test] +fn current_disposition_with_non_candidate_class_fails_closed() { + for evidence_class in [ + TestAttributionEvidenceClassV1::StaleEvidence, + TestAttributionEvidenceClassV1::UnknownUnsupported, + ] { + let project_id = ProjectId::new("project.affected-tests").expect("project"); + let generation = generation("generation.affected-tests.1"); + let mut read = complete_read(generation.clone()); + read.evidence.as_mut().expect("join").records[0].disposition = + GenerationTestJoinDispositionV1::Current { evidence_class }; + let port = TraceDecayAffectedTestsPortV1::from_binding( + Some(project_id.clone()), + generation.clone(), + Some(Arc::new(AttributionFixture { + calls: AtomicUsize::new(0), + read, + })), + ); + let (context, operation, _) = context(project_id); + + let RetrievalPortOutcome::Unavailable(evidence) = port.affected_tests( + &RetrievalPortContext { + request: &context, + operation: &operation, + }, + &request(generation), + ) else { + panic!("a current disposition must carry a candidate class"); + }; + assert!(evidence.payload.is_none()); + assert_eq!(evidence.omissions[0].reason, OmissionReason::Failed); + } +} + #[test] fn absent_or_mismatched_authority_never_fabricates_complete_empty() { let project_id = ProjectId::new("project.affected-tests").expect("project"); diff --git a/crates/tracedecay-application/src/primitives/production/extended_primitive.rs b/crates/tracedecay-application/src/primitives/production/extended_primitive.rs index 64c3559db3..aab10998b0 100644 --- a/crates/tracedecay-application/src/primitives/production/extended_primitive.rs +++ b/crates/tracedecay-application/src/primitives/production/extended_primitive.rs @@ -28,7 +28,7 @@ use super::super::runtime::{ SourceOutlinePrimitiveRequest, SourceOutlinePrimitiveResult, StorageStatusHistoryPointV1, StorageStatusPrimitiveRequest, StorageStatusPrimitiveResult, }; -use super::super::symbol_graph::symbol_record; +use super::super::symbol_graph::{read_symbol_source_body, symbol_record}; use super::{ AuthenticatedDiagnosticCursorAuthorityV1, DIAGNOSTIC_CURSOR_LANE_WORKSPACE, all_code_graph_symbols, completed, diagnostics_result, diagnostics_unavailable, @@ -571,18 +571,16 @@ impl ExtendedPrimitivePort for TraceDecayExtendedPrimitivePortV1 { let Some(end_line) = metadata.start_line.checked_add(line_span) else { return failed(EvidenceDomain::Source, now_observed()); }; - let path = self.source_runtime.project_root().join(&file); - let Ok(content) = tokio::fs::read_to_string(&path).await else { + let Ok(body) = read_symbol_source_body( + self.source_runtime.project_root(), + &file, + metadata.start_line, + end_line, + ) + .await + else { return failed(EvidenceDomain::Source, now_observed()); }; - let start = metadata.start_line as usize; - let end = end_line as usize; - let body = content - .lines() - .skip(start) - .take(end.saturating_sub(start).saturating_add(1)) - .collect::>() - .join("\n"); completed( SourceBodyPrimitiveResult { node_id: occurrence.as_str().to_owned(), diff --git a/crates/tracedecay-application/src/primitives/production/symbol_graph_snapshot.rs b/crates/tracedecay-application/src/primitives/production/symbol_graph_snapshot.rs index 05b504e043..49963de532 100644 --- a/crates/tracedecay-application/src/primitives/production/symbol_graph_snapshot.rs +++ b/crates/tracedecay-application/src/primitives/production/symbol_graph_snapshot.rs @@ -8,11 +8,12 @@ use tracedecay_domain::{ CommitId, ManifestDigest, RetrievalGrainV1, SessionId, SignedCursorKeyRefV1, TemporalModeV1, UtcMicros, canonical_sha256, }; -use tracedecay_temporal_query::ports::{ - BindingDigest, KernelVersions, TemporalExecutionSnapshot, TemporalSnapshotRequest, - TemporalWatermarks, -}; +use tracedecay_temporal_query::execution::BindingDigest; +use tracedecay_temporal_query::ports::TemporalSnapshotRequest; use tracedecay_temporal_query::resolution::ValidatedAuthorization; +use tracedecay_temporal_query::snapshot::{ + KernelVersions, TemporalExecutionSnapshot, TemporalWatermarks, +}; use super::super::concrete::{SymbolGraphCursorSnapshot, SymbolGraphCursorSnapshotAuthority}; use crate::lsp_runtime::LspCodeIndexProjectionIdentityPort; diff --git a/crates/tracedecay-application/src/primitives/runtime.rs b/crates/tracedecay-application/src/primitives/runtime.rs index 26e30681f2..1d155e36e4 100644 --- a/crates/tracedecay-application/src/primitives/runtime.rs +++ b/crates/tracedecay-application/src/primitives/runtime.rs @@ -522,6 +522,7 @@ pub fn open_primitive_project_runtime( let symbol_graph: Arc = Arc::new(CanonicalSymbolGraphAdapter::new( Arc::clone(&code_graph), + source_runtime.project_root().to_path_buf(), symbol_graph_cursors, ignored_dependency_admission, )); diff --git a/crates/tracedecay-application/src/primitives/symbol_graph.rs b/crates/tracedecay-application/src/primitives/symbol_graph.rs index 6b65251ca0..d3cb73af80 100644 --- a/crates/tracedecay-application/src/primitives/symbol_graph.rs +++ b/crates/tracedecay-application/src/primitives/symbol_graph.rs @@ -1,5 +1,6 @@ use std::collections::HashSet; use std::future::Future; +use std::path::{Path, PathBuf}; use std::pin::Pin; use std::sync::Arc; @@ -8,11 +9,12 @@ use tracedecay_code_index::graph_projection::{ }; use tracedecay_code_index::lineage::LineageSymbolRecordV1; use tracedecay_contracts::retrieval::{ - ExactSymbolRequest, GraphImpactPrimitiveRequest, GraphRelationRequest, ImplementationSelector, - ImplementationsRequest, PrimitiveFailure, PrimitiveFailureKind, PrimitiveSupportGap, - SignatureSearchRequest, SymbolGraphPage, SymbolGraphPortContext, SymbolGraphPortFuture, - SymbolGraphPortOutcome, SymbolGraphPrimitivePort, SymbolGraphScope, SymbolPrimitiveRecord, - SymbolRelationRecord, SymbolSearchPrimitiveRequest, TypeHierarchyRecord, TypeHierarchyRequest, + ExactSymbolRequest, GraphImpactPrimitiveRequest, GraphRelationRequest, ImplementationRecord, + ImplementationSelector, ImplementationsRequest, PrimitiveFailure, PrimitiveFailureKind, + PrimitiveSupportGap, SignatureSearchRequest, SymbolGraphPage, SymbolGraphPortContext, + SymbolGraphPortFuture, SymbolGraphPortOutcome, SymbolGraphPrimitivePort, SymbolGraphScope, + SymbolPrimitiveRecord, SymbolRelationRecord, SymbolSearchPrimitiveRequest, TypeHierarchyRecord, + TypeHierarchyRequest, }; use tracedecay_contracts::{OpaqueCursor, OperationBudgetUsage, PageRequest, RequestContext}; use tracedecay_domain::code_intelligence::NodeKind; @@ -118,6 +120,8 @@ where /// to the admitted, generation-pinned graph projection. pub struct CanonicalSymbolGraphAdapter { code_graph: Arc, + /// Admitted project root implementation bodies are read from. + source_root: PathBuf, cursors: C, ignored_dependency_admission: Option>, } @@ -365,7 +369,7 @@ where &'a self, context: SymbolGraphPortContext<'a>, request: &'a ImplementationsRequest, - ) -> SymbolGraphPortFuture<'a, SymbolRelationRecord> { + ) -> SymbolGraphPortFuture<'a, ImplementationRecord> { Box::pin(hotpath::future!( async move { let claim = match claim_generation( @@ -433,7 +437,7 @@ where records } }; - complete_or_failed( + let outcome = complete_or_failed( &self.cursors, context, &request.meta.page, @@ -444,7 +448,8 @@ where Vec::new(), None, ) - .await + .await; + with_implementation_bodies(&self.source_root, context, outcome).await }, label = "usecases.primitives.implementations" )) @@ -475,7 +480,17 @@ where .symbol_summary(&root_id, Arc::clone(&graph.cancellation)) { Ok(Some(node)) if in_scope(&node, &request.scope) => node, - Ok(_) => { + Ok(None) => { + return failed_with( + context, + primitive_failure( + PrimitiveFailureKind::NotFoundOrNotAuthorized, + "application.symbol-graph.node-not-found", + "type hierarchy root is not in the admitted graph", + ), + ); + } + Ok(Some(_)) => { return complete_or_failed( &self.cursors, context, @@ -929,17 +944,33 @@ fn signature_metadata_matches( .all(|param| parameters.contains(param)) } +/// Byte offsets of the parameter list's balanced parentheses, so tuple +/// parameters and parenthesized return types stay in their own regions. +fn parameter_parens(signature: &str) -> Option<(usize, usize)> { + let open = signature.find('(')?; + let mut depth = 0_usize; + for (index, byte) in signature.bytes().enumerate().skip(open) { + match byte { + b'(' => depth += 1, + b')' => { + depth -= 1; + if depth == 0 { + return Some((open, index)); + } + } + _ => {} + } + } + None +} + fn parameter_region(signature: &str) -> &str { - let Some(start) = signature.find('(') else { - return ""; - }; - let end = signature.rfind(')').unwrap_or(signature.len()); - signature.get(start + 1..end).unwrap_or("") + parameter_parens(signature).map_or("", |(open, close)| &signature[open + 1..close]) } fn return_region(signature: &str) -> &str { - signature - .split_once("->") + let tail = parameter_parens(signature).map_or(signature, |(_, close)| &signature[close + 1..]); + tail.split_once("->") .map_or("", |(_, returns)| returns.trim()) } @@ -1106,6 +1137,109 @@ pub(crate) fn symbol_record( }) } +/// Exact zero-based, inclusive `start_line..=end_line` source of one indexed +/// symbol, the slice every symbol-body read serves. +pub(crate) async fn read_symbol_source_body( + project_root: &Path, + file: &str, + start_line: u32, + end_line: u32, +) -> std::io::Result { + let content = tokio::fs::read_to_string(project_root.join(file)).await?; + let start = start_line as usize; + let end = end_line as usize; + Ok(content + .lines() + .skip(start) + .take(end.saturating_sub(start).saturating_add(1)) + .collect::>() + .join("\n")) +} + +/// Hydrates only the served page, so a large implementation set reads the +/// source of at most one page of matches. +async fn with_implementation_bodies( + source_root: &Path, + context: SymbolGraphPortContext<'_>, + outcome: SymbolGraphPortOutcome, +) -> SymbolGraphPortOutcome { + let (page, partial, finished_at, budget) = match outcome { + SymbolGraphPortOutcome::Completed { + page, + finished_at, + budget, + } => (page, false, finished_at, budget), + SymbolGraphPortOutcome::Partial { + page, + finished_at, + budget, + } => (page, true, finished_at, budget), + SymbolGraphPortOutcome::Failed { + failure, + finished_at, + budget, + } => { + return SymbolGraphPortOutcome::Failed { + failure, + finished_at, + budget, + }; + } + }; + let SymbolGraphPage { + generation, + freshness, + items, + total, + next_cursor, + truncated, + related_edge_count, + support_gaps, + } = page; + let mut hydrated = Vec::with_capacity(items.len()); + for record in items { + let Ok(body) = read_symbol_source_body( + source_root, + &record.symbol.file, + record.symbol.line.saturating_sub(1), + record.symbol.end_line.saturating_sub(1), + ) + .await + else { + return failed(context, "implementation source body was unavailable"); + }; + hydrated.push(ImplementationRecord { + symbol: record.symbol, + edge_kind: record.edge_kind, + dispatch_from: record.dispatch_from, + body, + }); + } + let page = SymbolGraphPage { + generation, + freshness, + items: hydrated, + total, + next_cursor, + truncated, + related_edge_count, + support_gaps, + }; + if partial { + SymbolGraphPortOutcome::Partial { + page, + finished_at, + budget, + } + } else { + SymbolGraphPortOutcome::Completed { + page, + finished_at, + budget, + } + } +} + fn in_scope(node: &CodeGraphSymbolSummaryV1, scope: &SymbolGraphScope) -> bool { in_scope_parts(node.binding.as_ref(), scope) } diff --git a/crates/tracedecay-application/src/primitives/symbol_graph/ignored_dependency.rs b/crates/tracedecay-application/src/primitives/symbol_graph/ignored_dependency.rs index dd0c00c072..4ad4441728 100644 --- a/crates/tracedecay-application/src/primitives/symbol_graph/ignored_dependency.rs +++ b/crates/tracedecay-application/src/primitives/symbol_graph/ignored_dependency.rs @@ -1,3 +1,4 @@ +use std::path::PathBuf; use std::sync::Arc; use crate::code_index::{ @@ -10,7 +11,7 @@ use tracedecay_code_index::graph_projection::{ use tracedecay_contracts::retrieval::{ PrimitiveFailure, PrimitiveFailureKind, SymbolGraphPortContext, SymbolGraphScope, }; -use tracedecay_temporal_query::ports::TemporalExecutionSnapshot; +use tracedecay_temporal_query::snapshot::TemporalExecutionSnapshot; use super::{ CanonicalSymbolGraphAdapter, MAX_COMPATIBILITY_RESULTS, OpenSymbolGraph, SymbolGraphCursorPort, @@ -20,11 +21,13 @@ use super::{ impl CanonicalSymbolGraphAdapter { pub fn new( code_graph: Arc, + source_root: PathBuf, cursors: C, ignored_dependency_admission: Option>, ) -> Self { Self { code_graph, + source_root, cursors, ignored_dependency_admission, } diff --git a/crates/tracedecay-application/src/primitives/symbol_graph_ignored_dependency_tests.rs b/crates/tracedecay-application/src/primitives/symbol_graph_ignored_dependency_tests.rs index 900558d070..f0d64ed2fa 100644 --- a/crates/tracedecay-application/src/primitives/symbol_graph_ignored_dependency_tests.rs +++ b/crates/tracedecay-application/src/primitives/symbol_graph_ignored_dependency_tests.rs @@ -1,4 +1,5 @@ use std::collections::{BTreeMap, BTreeSet}; +use std::path::PathBuf; use std::sync::{Arc, Mutex}; use serde::Serialize; @@ -31,11 +32,12 @@ use tracedecay_graph_db::{ GraphLabel, GraphNamespace, GraphProjectorRevision, GraphProperty, GraphPropertyName, GraphRelationId, GraphRelationKind, NeverCancelled, VerifiedGraphSnapshot, }; -use tracedecay_temporal_query::ports::{ - BindingDigest, KernelVersions, TemporalExecutionSnapshot, TemporalSnapshotRequest, - TemporalWatermarks, -}; +use tracedecay_temporal_query::execution::BindingDigest; +use tracedecay_temporal_query::ports::TemporalSnapshotRequest; use tracedecay_temporal_query::resolution::ValidatedAuthorization; +use tracedecay_temporal_query::snapshot::{ + KernelVersions, TemporalExecutionSnapshot, TemporalWatermarks, +}; use tracedecay_tool_catalog::{CapabilityId, SchemaId, UseCaseId}; use super::symbol_graph::{ @@ -317,7 +319,12 @@ fn adapter( let port: Arc = scheduler; port }); - CanonicalSymbolGraphAdapter::new(fixture.graph.clone(), fixture.cursor.clone(), scheduler) + CanonicalSymbolGraphAdapter::new( + fixture.graph.clone(), + PathBuf::new(), + fixture.cursor.clone(), + scheduler, + ) } fn port_context(fixture: &Fixture) -> SymbolGraphPortContext<'_> { diff --git a/crates/tracedecay-application/src/primitives/symbol_graph_ignored_dependency_tests/edge_cases.rs b/crates/tracedecay-application/src/primitives/symbol_graph_ignored_dependency_tests/edge_cases.rs index 7800cc3b7a..d494bb56b7 100644 --- a/crates/tracedecay-application/src/primitives/symbol_graph_ignored_dependency_tests/edge_cases.rs +++ b/crates/tracedecay-application/src/primitives/symbol_graph_ignored_dependency_tests/edge_cases.rs @@ -1,3 +1,4 @@ +use std::path::PathBuf; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -235,7 +236,7 @@ fn candidate_read_failures_preserve_stable_typed_semantics() { #[derive(Clone)] struct MismatchedClaimCursor { - snapshot: tracedecay_temporal_query::ports::TemporalExecutionSnapshot, + snapshot: tracedecay_temporal_query::snapshot::TemporalExecutionSnapshot, source_generation: tracedecay_domain::CodeGenerationId, finishes: Arc, } @@ -295,7 +296,8 @@ async fn mismatched_claim_and_reader_generation_fails_stale_before_query_or_admi next_generation(), ))); let scheduler_port: Arc = scheduler.clone(); - let adapter = CanonicalSymbolGraphAdapter::new(graph, cursor, Some(scheduler_port)); + let adapter = + CanonicalSymbolGraphAdapter::new(graph, PathBuf::new(), cursor, Some(scheduler_port)); let outcome = adapter .exact_symbol( @@ -316,7 +318,7 @@ async fn mismatched_claim_and_reader_generation_fails_stale_before_query_or_admi #[derive(Clone)] struct AdvancingFinishCursor { - snapshot: tracedecay_temporal_query::ports::TemporalExecutionSnapshot, + snapshot: tracedecay_temporal_query::snapshot::TemporalExecutionSnapshot, source_generation: tracedecay_domain::CodeGenerationId, finishes: Arc, } @@ -373,8 +375,12 @@ async fn stale_claim_finish_prevents_lazy_scheduler_mutation() { next_generation(), ))); let scheduler_port: Arc = scheduler.clone(); - let adapter = - CanonicalSymbolGraphAdapter::new(fixture.graph.clone(), cursor, Some(scheduler_port)); + let adapter = CanonicalSymbolGraphAdapter::new( + fixture.graph.clone(), + PathBuf::new(), + cursor, + Some(scheduler_port), + ); let outcome = adapter .exact_symbol( diff --git a/crates/tracedecay-application/src/project_adoption.rs b/crates/tracedecay-application/src/project_adoption.rs index c10aafd7c7..f585d43ecc 100644 --- a/crates/tracedecay-application/src/project_adoption.rs +++ b/crates/tracedecay-application/src/project_adoption.rs @@ -256,25 +256,6 @@ fn moved_store_evidence( return Ok(MovedStoreEvidence::RecordsPreviousRoot); } } - if layout.config_path.is_file() { - let config = - tracedecay_configuration::load_config_from_path(previous_root, &layout.config_path) - .map_err(|error| TraceDecayError::Config { - message: format!( - "cannot evaluate moved-store adoption evidence from '{}': {error}; \ - repair or remove the store config, or re-run `tracedecay init` \ - with --fresh to mint a new identity without adoption", - layout.config_path.display() - ), - })?; - let recorded = PathBuf::from(&config.root_dir); - if paths_record_same_root(&recorded, new_root) { - return Ok(MovedStoreEvidence::RecordsNewRoot); - } - if paths_record_same_root(&recorded, previous_root) { - return Ok(MovedStoreEvidence::RecordsPreviousRoot); - } - } Ok(MovedStoreEvidence::NoMatch) } @@ -290,7 +271,7 @@ fn paths_record_same_root(recorded: &Path, previous_root: &Path) -> bool { /// Rebinds `candidate` onto `new_root` as a journaled sequence. /// -/// Store-side evidence (shard manifest, then config) is written first: a +/// Store-side evidence (the shard manifest) is written first: a /// manifest recording the new root is the journal record an interrupted remap /// resumes from, because it is positive linkage between this store and the /// root. The registry upsert commits last, it is what makes the root resolve, @@ -312,22 +293,6 @@ async fn remap_moved_nongit_project( }, )?; storage::write_store_manifest(&layout)?; - if layout.config_path.is_file() { - let root_dir = new_root - .to_str() - .ok_or_else(|| TraceDecayError::Config { - message: format!( - "moved-project root '{}' is not valid UTF-8 and cannot be recorded \ - in the store config", - new_root.display() - ), - })? - .to_owned(); - let mut config = - tracedecay_configuration::load_config_from_path(new_root, &layout.config_path)?; - config.root_dir = root_dir; - tracedecay_configuration::save_config_to_path(&layout.config_path, &config)?; - } registry .upsert_code_project(&candidate.project_id, new_root, None, None, None) .await?; diff --git a/crates/tracedecay-application/src/work/work_evidence_retrieval.rs b/crates/tracedecay-application/src/work/work_evidence_retrieval.rs index 8f7bc98d9d..15686cc849 100644 --- a/crates/tracedecay-application/src/work/work_evidence_retrieval.rs +++ b/crates/tracedecay-application/src/work/work_evidence_retrieval.rs @@ -39,7 +39,7 @@ use tracedecay_session_temporal_store::execution::{ TaskSessionReauthorizationStageV1, TaskSessionSelectionCallbackErrorV1, }; use tracedecay_temporal_query::context::ContextBudget; -use tracedecay_temporal_query::ports::ExecutionLimits; +use tracedecay_temporal_query::execution::ExecutionLimits; use tracedecay_temporal_query::ranking::DiversityLimits; const WORK_EVIDENCE_CONTEXT_BYTES: u64 = 64 * 1024; @@ -682,7 +682,7 @@ const fn work_hydration_state(state: HydrationStateV1) -> WorkTaskSessionHydrati HydrationStateV1::RetentionExpired => WorkTaskSessionHydrationStateV1::RetentionExpired, HydrationStateV1::Unauthorized => WorkTaskSessionHydrationStateV1::Unauthorized, HydrationStateV1::Locked => WorkTaskSessionHydrationStateV1::Locked, - HydrationStateV1::UnverifiableLegacy => WorkTaskSessionHydrationStateV1::UnverifiableLegacy, + HydrationStateV1::Unverifiable => WorkTaskSessionHydrationStateV1::Unverifiable, } } diff --git a/crates/tracedecay-application/tests/application_suite/github_stack_anchor_authority.rs b/crates/tracedecay-application/tests/application_suite/github_stack_anchor_authority.rs index 0f435c98c0..c883032555 100644 --- a/crates/tracedecay-application/tests/application_suite/github_stack_anchor_authority.rs +++ b/crates/tracedecay-application/tests/application_suite/github_stack_anchor_authority.rs @@ -1,19 +1,18 @@ use tracedecay_contracts::retrieval::{ - GitTopologyAnchorAuthorityV2, GitTopologyAnchorPublicationOutcomeV2, - GitTopologyAnchorPublicationV2, GitTopologyAnchorResolutionOutcomeV2, - GitTopologyAnchorResolutionV2, + GitTopologyAnchorAuthority, GitTopologyAnchorPublication, GitTopologyAnchorPublicationOutcome, + GitTopologyAnchorResolution, GitTopologyAnchorResolutionOutcome, }; use tracedecay_domain::{ - AccessPolicyDigest, AnchorDurabilityClass, AnchorLineageRefV2, AnchorProvenanceRelationV2, - AnchorSourceGenerationV2, CapabilityId, CommitId, CoverageReportV1, EvidenceClass, + AccessPolicyDigest, AnchorDurabilityClass, AnchorLineageRef, AnchorProvenanceRelation, + AnchorSourceGeneration, CapabilityId, CommitId, CoverageReportV1, EvidenceClass, GitHubStackCapabilitySnapshotV1, GitHubStackCapabilityStateV1, GitTopologyAnchorTargetV1, ObservationScopeV1, PayloadAccessState, PrivacyDomainBoundLocatorDigest, PrivacyDomainId, ProjectId, ProjectionGenerationId, ProviderId, RepositoryId, ResolutionAuthorizationV1, - RetentionClass, RetrievalAnchorRecordV2, RetrievalAnchorRecordV2Parts, RetrievalAnchorTargetV2, + RetentionClass, RetrievalAnchorRecord, RetrievalAnchorRecordParts, RetrievalAnchorTarget, ScopeResolutionId, UtcMicros, VectorWatermark, WorktreeId, }; use tracedecay_global_db::{ - RegisteredGitTopologyAnchorAuthorityV2, tests::harness::RegisteredGlobalDbTestRuntime, + RegisteredGitTopologyAnchorAuthority, tests::harness::RegisteredGlobalDbTestRuntime, }; const SHA: &str = "sha256:0000000000000000000000000000000000000000000000000000000000000000"; @@ -30,12 +29,12 @@ fn authorization() -> ResolutionAuthorizationV1 { fn record( owner: ObservationScopeV1, - target: RetrievalAnchorTargetV2, - source_generation: AnchorSourceGenerationV2, + target: RetrievalAnchorTarget, + source_generation: AnchorSourceGeneration, projection_generation: ProjectionGenerationId, - source_anchors: Vec, -) -> RetrievalAnchorRecordV2 { - RetrievalAnchorRecordV2::new(RetrievalAnchorRecordV2Parts { + source_anchors: Vec, +) -> RetrievalAnchorRecord { + RetrievalAnchorRecord::new(RetrievalAnchorRecordParts { target, owner, aliases: Vec::new(), @@ -58,7 +57,7 @@ fn record( } #[tokio::test] -async fn degraded_capability_persists_through_the_v2_git_topology_authority() { +async fn degraded_capability_persists_through_the_git_topology_authority() { let _pin = tracedecay_runtime_core::config::PinnedUserDataDir::new(); let profile = tempfile::tempdir().unwrap(); let project = tempfile::tempdir().unwrap(); @@ -70,11 +69,11 @@ async fn degraded_capability_persists_through_the_v2_git_topology_authority() { let generation_id = ProjectionGenerationId::new("generation.github-stack.degraded").unwrap(); let source = record( owner.clone(), - RetrievalAnchorTargetV2::ExactRepositoryCommit { + RetrievalAnchorTarget::ExactRepositoryCommit { repository_id: repository_id.clone(), commit_id: CommitId::new("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").unwrap(), }, - AnchorSourceGenerationV2::Unknown, + AnchorSourceGeneration::Unknown, ProjectionGenerationId::new("generation.github-stack.source.degraded").unwrap(), Vec::new(), ); @@ -91,14 +90,14 @@ async fn degraded_capability_persists_through_the_v2_git_topology_authority() { let capability_generation = capability.generation(); let capability_record = record( owner.clone(), - RetrievalAnchorTargetV2::GitTopology(Box::new( + RetrievalAnchorTarget::GitTopology(Box::new( GitTopologyAnchorTargetV1::GitHubStackCapability(capability), )), - AnchorSourceGenerationV2::GitTopology(capability_generation), + AnchorSourceGeneration::GitTopology(capability_generation), generation_id, vec![ - AnchorLineageRefV2::new( - AnchorProvenanceRelationV2::Observed, + AnchorLineageRef::new( + AnchorProvenanceRelation::Observed, source.anchor_id().clone(), owner.clone(), ) @@ -111,18 +110,15 @@ async fn degraded_capability_persists_through_the_v2_git_topology_authority() { .await .unwrap(); let database = runtime.project_database_arc().unwrap(); - let authority = RegisteredGitTopologyAnchorAuthorityV2::new(database.clone()); + let authority = RegisteredGitTopologyAnchorAuthority::new(database.clone()); assert_eq!( authority .publish( - GitTopologyAnchorPublicationV2::new( - owner.clone(), - vec![source, capability_record], - ) - .unwrap(), + GitTopologyAnchorPublication::new(owner.clone(), vec![source, capability_record],) + .unwrap(), ) .await, - Ok(GitTopologyAnchorPublicationOutcomeV2::Published) + Ok(GitTopologyAnchorPublicationOutcome::Published) ); drop(authority); drop(database); @@ -133,14 +129,14 @@ async fn degraded_capability_persists_through_the_v2_git_topology_authority() { .await .unwrap(); let authority = - RegisteredGitTopologyAnchorAuthorityV2::new(restarted.project_database_arc().unwrap()); + RegisteredGitTopologyAnchorAuthority::new(restarted.project_database_arc().unwrap()); let resolved = authority - .resolve(GitTopologyAnchorResolutionV2::new(owner, capability_anchor_id).unwrap()) + .resolve(GitTopologyAnchorResolution::new(owner, capability_anchor_id).unwrap()) .await; assert!(matches!( resolved, - Ok(GitTopologyAnchorResolutionOutcomeV2::Resolved(record)) - if matches!(record.target(), RetrievalAnchorTargetV2::GitTopology(target) + Ok(GitTopologyAnchorResolutionOutcome::Resolved(record)) + if matches!(record.target(), RetrievalAnchorTarget::GitTopology(target) if matches!(target.as_ref(), GitTopologyAnchorTargetV1::GitHubStackCapability(capability) if capability.state == GitHubStackCapabilityStateV1::Degraded)) )); diff --git a/crates/tracedecay-application/tests/application_suite/github_stack_coordinator.rs b/crates/tracedecay-application/tests/application_suite/github_stack_coordinator.rs index 73b8d29b2a..6e61561cc1 100644 --- a/crates/tracedecay-application/tests/application_suite/github_stack_coordinator.rs +++ b/crates/tracedecay-application/tests/application_suite/github_stack_coordinator.rs @@ -326,7 +326,15 @@ fn production_coordinator_materializes_all_four_states_and_exact_stack_anchors() snapshot.layers[0].head_ref_id.as_str(), "refs/heads/feature" ); - assert!(enabled.snapshot_anchor_id.is_some()); + assert_eq!( + enabled + .snapshot_anchor_id + .as_ref() + .map(RetrievalAnchorId::as_str), + Some( + "retrieval.v3.sha256:331a85c93b00160876582f934a2a61231d3849efdba0eb61baa4aa8746232ae0" + ) + ); assert_eq!( coordinator .observe_policy( @@ -770,5 +778,14 @@ fn daemon_restart_returns_unavailable_until_a_delayed_exact_probe_upgrades_state upgraded.capability.state, tracedecay_domain::GitHubStackCapabilityStateV1::Enabled ); - assert!(upgraded.snapshot.is_some()); + let snapshot = upgraded.snapshot.as_ref().expect("upgraded snapshot"); + assert_eq!(snapshot.final_target_ref_id.as_str(), "refs/heads/main"); + assert_eq!( + snapshot.layers[0].pull_request.pull_request_id.as_str(), + "41" + ); + assert_eq!( + snapshot.layers[0].head_ref_id.as_str(), + "refs/heads/feature" + ); } diff --git a/crates/tracedecay-automation-runtime/Cargo.toml b/crates/tracedecay-automation-runtime/Cargo.toml index 47e268993f..c8b0bbeb5e 100644 --- a/crates/tracedecay-automation-runtime/Cargo.toml +++ b/crates/tracedecay-automation-runtime/Cargo.toml @@ -36,6 +36,7 @@ rustls = { version = "0.23", default-features = false, features = [ "std", "tls12", ] } +schemars.workspace = true serde = { version = "1", features = ["derive"] } serde_json = "1" sha2 = "0.11" diff --git a/crates/tracedecay-automation-runtime/src/automation/agent_targets.rs b/crates/tracedecay-automation-runtime/src/automation/agent_targets.rs index 514666aec8..d95b7a400f 100644 --- a/crates/tracedecay-automation-runtime/src/automation/agent_targets.rs +++ b/crates/tracedecay-automation-runtime/src/automation/agent_targets.rs @@ -49,7 +49,7 @@ pub fn install_codex_managed_agents( for agent in agents { let id = generated_agent_id(agent.relative); let path = agents_dir.join(agent.relative); - host_io.safe_write_text_file(&path, agent.contents, None)?; + host_io.safe_write_text_file(&path, agent.contents)?; exported.push(ManagedAgentExportEntry { id: id.to_string(), path, @@ -63,7 +63,6 @@ pub fn install_codex_managed_agents( host_io.safe_write_text_file( &agents_dir.join(MANIFEST_FILE), &format!("{}\n", serde_json::to_string_pretty(&manifest)?), - None, )?; Ok(ManagedAgentInstallSummary { diff --git a/crates/tracedecay-automation-runtime/src/automation/artifact_payloads.rs b/crates/tracedecay-automation-runtime/src/automation/artifact_payloads.rs index b92b8f6423..1f341f1ecd 100644 --- a/crates/tracedecay-automation-runtime/src/automation/artifact_payloads.rs +++ b/crates/tracedecay-automation-runtime/src/automation/artifact_payloads.rs @@ -591,6 +591,7 @@ mod tests { rejected_count: 0, skipped_count: 0, fallback_status: None, + session_evidence_budget_stage: None, error: None, error_classification: None, error_retryable: None, diff --git a/crates/tracedecay-automation-runtime/src/automation/artifacts.rs b/crates/tracedecay-automation-runtime/src/automation/artifacts.rs index 0b7d432529..60bc3cf50b 100644 --- a/crates/tracedecay-automation-runtime/src/automation/artifacts.rs +++ b/crates/tracedecay-automation-runtime/src/automation/artifacts.rs @@ -230,6 +230,7 @@ mod tests { backend_attempt_count: 0, backend_attempts: Vec::new(), fallback_status: None, + session_evidence_budget_stage: None, report_ref: None, artifacts: Vec::new(), started_at: "0".to_string(), diff --git a/crates/tracedecay-automation-runtime/src/automation/automatic_facts.rs b/crates/tracedecay-automation-runtime/src/automation/automatic_facts.rs index f3d025b2cf..f0d9ce6d67 100644 --- a/crates/tracedecay-automation-runtime/src/automation/automatic_facts.rs +++ b/crates/tracedecay-automation-runtime/src/automation/automatic_facts.rs @@ -2,27 +2,14 @@ //! //! Candidate discovery and validation belong to the automation run receipt. //! This module records and reads only terminal applied or quarantined effects. -//! It also recognizes the independently shipped v1 proposal sidecar for a -//! explicit retirement boundary. This crate only classifies the exact shipped -//! bytes. The daemon journals terminal-history retirement before archive and -//! removal; unresolved records are never approved or imported. use std::collections::HashSet; -use std::io::Read; -use std::path::{Path, PathBuf}; - -#[cfg(unix)] -use cap_fs_ext::{FollowSymlinks, OpenOptionsFollowExt}; -#[cfg(unix)] -use cap_std::{ - ambient_authority, - fs::{Dir, MetadataExt, OpenOptions as CapOpenOptions}, -}; + +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use serde_json::Value; use sha2::{Digest, Sha256}; -use tracedecay_domain::canonical_text::encode_tagged_lowercase_hex; -use tracedecay_domain::{ActorId, Confidence, FactCategoryV1, ProvenanceId, RunId}; +use tracedecay_domain::{ActorId, ProvenanceId, RunId}; use tracedecay_store::{ FactReadControl, MAX_PROJECT_MEMORY_AUTOMATIC_FACT_RECEIPTS, ProjectMemoryAutomaticFactApplyResultV1, ProjectMemoryAutomaticFactEvidenceV1, @@ -37,76 +24,7 @@ use tracedecay_session_memory::memory::{ }; use tracedecay_session_memory::memory::{MemoryMutationError, ProjectMemoryFactAddRequest}; -const SHIPPED_FACT_PROPOSALS_FILENAME: &str = "fact_proposals.json"; - -/// The shipped v1 store is one JSON document that retirement copies byte-exact -/// into one archive. A 16 MiB whole-record ceiling preserves generously sized -/// historical display metadata while bounding both parse allocation and copy. -pub const MAX_SHIPPED_FACT_PROPOSAL_BYTES: usize = 16 * 1024 * 1024; - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] -#[serde(rename_all = "snake_case")] -enum ShippedFactProposalStateV1 { - PendingApproval, - Applied, - Rejected, -} - -#[derive(Debug, Clone, PartialEq, Deserialize)] -#[serde(deny_unknown_fields)] -struct ShippedAddFactRequestV1 { - content: String, - category: FactCategoryV1, - #[serde(rename = "source", alias = "source_label")] - source_label: Option, - tags: Vec, - entities: Vec, - trust: Option, - metadata: Value, -} - -#[derive(Debug, Clone, PartialEq, Deserialize)] -#[serde(deny_unknown_fields)] -struct ShippedFactProposalRecordV1 { - schema_version: u32, - proposal_id: String, - run_id: String, - #[serde(default)] - evidence_hash: Option, - state: ShippedFactProposalStateV1, - #[serde(default)] - add_fact_request: Option, - #[serde(default)] - proposal: Option, - #[serde(default)] - validation_reason: Option, - #[serde(default)] - validation: Option, - #[serde(default)] - reviewer: Option, - #[serde(default)] - applied_fact_id: Option, - #[serde(default)] - apply_outcome: Option, - created_at: i64, - updated_at: i64, - #[serde(default)] - duplicate_count: u32, - #[serde(default)] - last_duplicate_run_id: Option, - #[serde(default)] - folded_contents: Vec, -} - -#[derive(Debug, Clone, PartialEq, Deserialize)] -#[serde(deny_unknown_fields)] -struct ShippedFactProposalStoreV1 { - schema_version: u32, - #[serde(default)] - proposals: Vec, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "snake_case")] pub enum AutomaticFactState { Applied, @@ -125,7 +43,7 @@ impl AutomaticFactState { } } -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] pub struct AutomaticFactReceipt { pub schema_version: u32, @@ -269,238 +187,6 @@ pub async fn record_session_automatic_facts( }) } -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum ShippedFactProposalDisposition { - Absent, - TerminalHistory { - source_path: PathBuf, - source_digest: String, - source_bytes: Vec, - }, - ResetRequired { - source_path: PathBuf, - source_digest: String, - reason: String, - }, -} - -pub async fn inspect_shipped_fact_proposals( - dashboard_root: &Path, -) -> Result { - let source_path = dashboard_root.join(SHIPPED_FACT_PROPOSALS_FILENAME); - let bytes = match read_shipped_fact_proposal_bytes(&source_path)? { - Some(bytes) => bytes, - None => { - return Ok(ShippedFactProposalDisposition::Absent); - } - }; - let source_digest = encode_tagged_lowercase_hex("sha256:", &Sha256::digest(&bytes)); - let store = match serde_json::from_slice::(&bytes) { - Ok(store) => store, - Err(error) => { - return Ok(shipped_fact_proposal_reset_required( - source_path, - source_digest, - format!("the shipped v1 JSON is malformed: {error}"), - )); - } - }; - if store.schema_version != 1 { - return Ok(shipped_fact_proposal_reset_required( - source_path, - source_digest, - format!( - "root schema version {} is not the shipped version 1", - store.schema_version - ), - )); - } - if let Some(record) = store - .proposals - .iter() - .find(|record| record.schema_version != 1) - { - return Ok(shipped_fact_proposal_reset_required( - source_path, - source_digest, - format!( - "proposal '{}' has unsupported schema version {}", - record.proposal_id, record.schema_version - ), - )); - } - - let mut proposal_ids = HashSet::new(); - for record in &store.proposals { - if !proposal_ids.insert(record.proposal_id.as_str()) { - return Ok(shipped_fact_proposal_reset_required( - source_path, - source_digest, - format!( - "proposal identity '{}' occurs more than once", - record.proposal_id - ), - )); - } - if record.state == ShippedFactProposalStateV1::PendingApproval { - return Ok(shipped_fact_proposal_reset_required( - source_path, - source_digest, - format!( - "unresolved proposal '{}' cannot be imported because final-V2 has no fact approval authority", - record.proposal_id - ), - )); - } - } - Ok(ShippedFactProposalDisposition::TerminalHistory { - source_path, - source_digest, - source_bytes: bytes, - }) -} - -/// Reads an exact shipped proposal source or archive through a no-follow, -/// owner-private handle and rejects any file that changes length while read. -/// -/// `None` means the exact leaf was absent. Every other namespace, privacy, or -/// byte-bound failure remains typed so retirement cannot digest, archive, or -/// delete bytes that were not read from the admitted regular file. -#[hotpath::measure(label = "automation.automatic_facts.read_proposal")] -pub fn read_shipped_fact_proposal_bytes(path: &Path) -> Result>> { - let file = match open_shipped_fact_proposal_file(path) { - Ok(file) => file, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), - Err(error) => { - return Err(config_error(format!( - "failed to open shipped fact proposal file '{}': {error}", - path.display() - ))); - } - }; - let initial = file.metadata().map_err(|error| { - config_error(format!( - "failed to inspect shipped fact proposal file '{}': {error}", - path.display() - )) - })?; - if !initial.is_file() || initial.len() > MAX_SHIPPED_FACT_PROPOSAL_BYTES as u64 { - return Err(config_error(format!( - "shipped fact proposal file '{}' is not a regular file within the {}-byte limit", - path.display(), - MAX_SHIPPED_FACT_PROPOSAL_BYTES - ))); - } - - read_opened_shipped_fact_proposal_bytes(path, file, initial).map(Some) -} - -fn read_opened_shipped_fact_proposal_bytes( - path: &Path, - mut file: std::fs::File, - initial: std::fs::Metadata, -) -> Result> { - let mut bytes = Vec::with_capacity(initial.len() as usize); - (&mut file) - .take(MAX_SHIPPED_FACT_PROPOSAL_BYTES as u64 + 1) - .read_to_end(&mut bytes) - .map_err(|error| { - config_error(format!( - "failed to read shipped fact proposal file '{}': {error}", - path.display() - )) - })?; - if bytes.len() > MAX_SHIPPED_FACT_PROPOSAL_BYTES { - return Err(config_error(format!( - "shipped fact proposal file '{}' grew beyond the {}-byte limit", - path.display(), - MAX_SHIPPED_FACT_PROPOSAL_BYTES - ))); - } - let final_metadata = file.metadata().map_err(|error| { - config_error(format!( - "failed to reinspect shipped fact proposal file '{}': {error}", - path.display() - )) - })?; - if !final_metadata.is_file() - || final_metadata.len() > MAX_SHIPPED_FACT_PROPOSAL_BYTES as u64 - || final_metadata.len() != initial.len() - || bytes.len() as u64 != final_metadata.len() - { - return Err(config_error(format!( - "shipped fact proposal file '{}' changed length while being read", - path.display() - ))); - } - Ok(bytes) -} - -#[cfg(unix)] -fn open_shipped_fact_proposal_file(path: &Path) -> std::io::Result { - tracedecay_runtime_core::storage::reject_symlink_components( - path, - "shipped fact proposal file", - )?; - let parent = path.parent().ok_or_else(|| { - std::io::Error::new( - std::io::ErrorKind::InvalidInput, - "shipped fact proposal file has no parent directory", - ) - })?; - let name = path.file_name().ok_or_else(|| { - std::io::Error::new( - std::io::ErrorKind::InvalidInput, - "shipped fact proposal file has no filename", - ) - })?; - let directory = Dir::open_ambient_dir(parent, ambient_authority())?; - let directory_metadata = directory.dir_metadata()?; - let mut options = CapOpenOptions::new(); - options.read(true).follow(FollowSymlinks::No); - let file = directory.open_with(name, &options)?; - let metadata = file.metadata()?; - if !metadata.is_file() - || metadata.mode() & 0o777 != 0o600 - || metadata.uid() != directory_metadata.uid() - { - return Err(std::io::Error::new( - std::io::ErrorKind::PermissionDenied, - "shipped fact proposal file is not private to its directory owner", - )); - } - Ok(file.into_std()) -} - -#[cfg(windows)] -fn open_shipped_fact_proposal_file(path: &Path) -> std::io::Result { - tracedecay_runtime_core::storage::reject_symlink_components( - path, - "shipped fact proposal file", - )?; - tracedecay_runtime_core::windows_security::open_private_file(path) -} - -#[cfg(not(any(unix, windows)))] -fn open_shipped_fact_proposal_file(_path: &Path) -> std::io::Result { - Err(std::io::Error::new( - std::io::ErrorKind::Unsupported, - "bounded no-follow shipped proposal reads are unavailable on this platform", - )) -} - -fn shipped_fact_proposal_reset_required( - source_path: PathBuf, - source_digest: String, - reason: impl Into, -) -> ShippedFactProposalDisposition { - ShippedFactProposalDisposition::ResetRequired { - source_path, - source_digest, - reason: reason.into(), - } -} - pub async fn list_automatic_fact_receipts( memory: &MemoryApplication, state: Option, diff --git a/crates/tracedecay-automation-runtime/src/automation/automatic_facts_test.rs b/crates/tracedecay-automation-runtime/src/automation/automatic_facts_test.rs index dc903bd9ff..32ddd99225 100644 --- a/crates/tracedecay-automation-runtime/src/automation/automatic_facts_test.rs +++ b/crates/tracedecay-automation-runtime/src/automation/automatic_facts_test.rs @@ -186,323 +186,6 @@ async fn canonical_fact_count( .count() } -fn shipped_sidecar(pending_request: Option) -> serde_json::Value { - let pending_request = pending_request.map(|request| { - let mut request = serde_json::to_value(request).unwrap(); - let source_label = request - .as_object_mut() - .and_then(|request| request.remove("source_label")); - if let (Some(source_label), Some(request)) = (source_label, request.as_object_mut()) { - request.insert("source".to_string(), source_label); - } - request - }); - serde_json::json!({ - "schema_version": 1, - "proposals": [ - { - "schema_version": 1, - "proposal_id": "fact_0123456789abcdef", - "run_id": "run-shipped-sidecar", - "evidence_hash": "shipped-evidence-hash", - "state": "pending_approval", - "add_fact_request": pending_request, - "proposal": { - "content": "Preserve shipped proposal provenance", - "source_span": {"message_id": "msg-shipped"} - }, - "validation": {"status": "accepted"}, - "created_at": 1_700_000_000, - "updated_at": 1_700_000_001, - "duplicate_count": 2, - "last_duplicate_run_id": "run-shipped-duplicate", - "folded_contents": ["Earlier wording"] - }, - { - "schema_version": 1, - "proposal_id": "fact_fedcba9876543210", - "run_id": "run-shipped-sidecar", - "state": "rejected", - "proposal": {"content": "Transient rejected item"}, - "validation_reason": "not durable", - "reviewer": "validator", - "created_at": 1_700_000_002, - "updated_at": 1_700_000_003 - } - ] - }) -} - -fn terminal_shipped_sidecar() -> serde_json::Value { - let mut sidecar = shipped_sidecar(None); - let record = sidecar["proposals"][0] - .as_object_mut() - .expect("fixture proposal must remain an object"); - record.insert("state".to_string(), serde_json::json!("applied")); - record.insert("applied_fact_id".to_string(), serde_json::json!(42)); - record.insert( - "apply_outcome".to_string(), - serde_json::json!({"state": "applied", "fact_id": 42}), - ); - sidecar -} - -fn write_private_file(path: &Path, bytes: &[u8]) { - std::fs::write(path, bytes).unwrap(); - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - - std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)).unwrap(); - } - #[cfg(windows)] - drop(tracedecay_runtime_core::windows_security::make_private_file(path).unwrap()); -} - -#[tokio::test] -async fn shipped_pending_proposals_require_reset_without_mutation_or_archive() { - let temp = tempfile::tempdir().unwrap(); - let dashboard_root = temp.path().join("dashboard"); - tokio::fs::create_dir_all(&dashboard_root).await.unwrap(); - let source_path = dashboard_root.join("fact_proposals.json"); - let source_bytes = serde_json::to_vec_pretty(&shipped_sidecar(Some(request( - "Apply this shipped pending proposal through canonical memory", - )))) - .unwrap(); - let shipped_request = serde_json::from_slice::(&source_bytes) - .unwrap()["proposals"][0]["add_fact_request"] - .clone(); - assert_eq!( - shipped_request["source"], - serde_json::json!("automatic-fact-test") - ); - assert!(shipped_request.get("source_label").is_none()); - write_private_file(&source_path, &source_bytes); - let db = database( - &temp.path().join("memory.db"), - TestDatabaseRuntimeMode::Initialize, - ) - .await; - let memory = MemoryApplication::new(FactOwnerV1::Profile, DatabaseFactStore::new(&db)).unwrap(); - let (run_control, _) = test_run_control(false); - - let disposition = inspect_shipped_fact_proposals(&dashboard_root) - .await - .unwrap(); - - assert!(matches!( - disposition, - ShippedFactProposalDisposition::ResetRequired { .. } - )); - assert_eq!(tokio::fs::read(&source_path).await.unwrap(), source_bytes); - assert!(!dashboard_root.join("fact_proposals.archive").exists()); - assert_eq!(canonical_fact_count(&memory, &run_control).await, 0); - assert!( - load_automatic_fact_receipt(&memory, "fact_0123456789abcdef", run_control.read_control(),) - .await - .unwrap() - .is_none() - ); -} - -#[tokio::test] -async fn shipped_pending_record_without_request_still_requires_reset_without_effects() { - let temp = tempfile::tempdir().unwrap(); - let dashboard_root = temp.path().join("dashboard"); - tokio::fs::create_dir_all(&dashboard_root).await.unwrap(); - let source_path = dashboard_root.join("fact_proposals.json"); - let source_bytes = serde_json::to_vec(&shipped_sidecar(None)).unwrap(); - write_private_file(&source_path, &source_bytes); - let db = database( - &temp.path().join("memory.db"), - TestDatabaseRuntimeMode::Initialize, - ) - .await; - let memory = MemoryApplication::new(FactOwnerV1::Profile, DatabaseFactStore::new(&db)).unwrap(); - let (run_control, _) = test_run_control(false); - - let disposition = inspect_shipped_fact_proposals(&dashboard_root) - .await - .unwrap(); - - assert!(matches!( - disposition, - ShippedFactProposalDisposition::ResetRequired { .. } - )); - assert_eq!(tokio::fs::read(&source_path).await.unwrap(), source_bytes); - assert_eq!(canonical_fact_count(&memory, &run_control).await, 0); - assert!(!dashboard_root.join("fact_proposals.archive").exists()); -} - -#[tokio::test] -async fn terminal_shipped_records_are_classified_without_mutation() { - let temp = tempfile::tempdir().unwrap(); - let dashboard_root = temp.path().join("dashboard"); - tokio::fs::create_dir_all(&dashboard_root).await.unwrap(); - let source_path = dashboard_root.join("fact_proposals.json"); - let source_bytes = serde_json::to_vec_pretty(&terminal_shipped_sidecar()).unwrap(); - write_private_file(&source_path, &source_bytes); - let db = database( - &temp.path().join("memory.db"), - TestDatabaseRuntimeMode::Initialize, - ) - .await; - let memory = MemoryApplication::new(FactOwnerV1::Profile, DatabaseFactStore::new(&db)).unwrap(); - let (run_control, _) = test_run_control(false); - - let disposition = inspect_shipped_fact_proposals(&dashboard_root) - .await - .unwrap(); - assert!(matches!( - disposition, - ShippedFactProposalDisposition::TerminalHistory { .. } - )); - assert_eq!(tokio::fs::read(&source_path).await.unwrap(), source_bytes); - assert_eq!(canonical_fact_count(&memory, &run_control).await, 0); - assert!(!dashboard_root.join("fact_proposals.archive").exists()); -} - -#[cfg(unix)] -#[tokio::test] -async fn shipped_sidecar_symlink_is_rejected_without_reading_its_target() { - use std::os::unix::fs::symlink; - - let temp = tempfile::tempdir().unwrap(); - let dashboard_root = temp.path().join("dashboard"); - std::fs::create_dir_all(&dashboard_root).unwrap(); - let target_path = temp.path().join("private-terminal-target.json"); - let target_bytes = serde_json::to_vec(&terminal_shipped_sidecar()).unwrap(); - write_private_file(&target_path, &target_bytes); - let source_path = dashboard_root.join("fact_proposals.json"); - symlink(&target_path, &source_path).unwrap(); - - let error = inspect_shipped_fact_proposals(&dashboard_root) - .await - .expect_err("the shipped source must never follow a symbolic link"); - - assert!(error.to_string().contains("failed to open")); - assert_eq!(std::fs::read(&target_path).unwrap(), target_bytes); - assert!( - std::fs::symlink_metadata(&source_path) - .unwrap() - .file_type() - .is_symlink() - ); -} - -#[tokio::test] -async fn nonregular_shipped_sidecar_is_rejected_before_read() { - let temp = tempfile::tempdir().unwrap(); - let dashboard_root = temp.path().join("dashboard"); - std::fs::create_dir_all(&dashboard_root).unwrap(); - let source_path = dashboard_root.join("fact_proposals.json"); - std::fs::create_dir(&source_path).unwrap(); - - let error = inspect_shipped_fact_proposals(&dashboard_root) - .await - .expect_err("a directory cannot become shipped proposal bytes"); - - assert!(error.to_string().contains("failed to open")); - assert!(std::fs::metadata(source_path).unwrap().is_dir()); -} - -#[cfg(unix)] -#[tokio::test] -async fn shipped_sidecar_requires_exact_unix_private_mode() { - use std::os::unix::fs::PermissionsExt; - - let temp = tempfile::tempdir().unwrap(); - let dashboard_root = temp.path().join("dashboard"); - std::fs::create_dir_all(&dashboard_root).unwrap(); - let source_path = dashboard_root.join("fact_proposals.json"); - let source_bytes = serde_json::to_vec(&terminal_shipped_sidecar()).unwrap(); - write_private_file(&source_path, &source_bytes); - std::fs::set_permissions(&source_path, std::fs::Permissions::from_mode(0o640)).unwrap(); - - let error = inspect_shipped_fact_proposals(&dashboard_root) - .await - .expect_err("a group-readable shipped sidecar must fail closed"); - - assert!(error.to_string().contains("not private")); - assert_eq!(std::fs::read(source_path).unwrap(), source_bytes); -} - -#[tokio::test] -async fn oversized_sparse_shipped_sidecar_is_rejected_before_allocation() { - let temp = tempfile::tempdir().unwrap(); - let dashboard_root = temp.path().join("dashboard"); - std::fs::create_dir_all(&dashboard_root).unwrap(); - let source_path = dashboard_root.join("fact_proposals.json"); - write_private_file(&source_path, b""); - std::fs::OpenOptions::new() - .write(true) - .open(&source_path) - .unwrap() - .set_len(MAX_SHIPPED_FACT_PROPOSAL_BYTES as u64 + 1) - .unwrap(); - - let error = inspect_shipped_fact_proposals(&dashboard_root) - .await - .expect_err("an oversized sparse legacy sidecar must fail closed"); - - assert!(error.to_string().contains("byte limit")); - assert_eq!( - std::fs::metadata(source_path).unwrap().len(), - MAX_SHIPPED_FACT_PROPOSAL_BYTES as u64 + 1 - ); -} - -#[test] -fn shipped_sidecar_growth_after_metadata_is_rejected_by_bounded_read() { - let temp = tempfile::tempdir().unwrap(); - let source_path = temp.path().join("fact_proposals.json"); - write_private_file(&source_path, b"bounded"); - let file = open_shipped_fact_proposal_file(&source_path).unwrap(); - let initial = file.metadata().unwrap(); - std::fs::OpenOptions::new() - .write(true) - .open(&source_path) - .unwrap() - .set_len(MAX_SHIPPED_FACT_PROPOSAL_BYTES as u64 + 1) - .unwrap(); - - let error = read_opened_shipped_fact_proposal_bytes(&source_path, file, initial) - .expect_err("growth after the initial metadata check must hit the bounded sentinel"); - - assert!(error.to_string().contains("grew beyond")); -} - -#[tokio::test] -async fn exact_maximum_shipped_sidecar_remains_a_valid_terminal_history() { - let temp = tempfile::tempdir().unwrap(); - let dashboard_root = temp.path().join("dashboard"); - std::fs::create_dir_all(&dashboard_root).unwrap(); - let source_path = dashboard_root.join("fact_proposals.json"); - let mut sidecar = terminal_shipped_sidecar(); - sidecar["proposals"][0]["proposal"]["bounded_padding"] = serde_json::json!(""); - let base = serde_json::to_vec(&sidecar).unwrap(); - assert!(base.len() < MAX_SHIPPED_FACT_PROPOSAL_BYTES); - sidecar["proposals"][0]["proposal"]["bounded_padding"] = - serde_json::json!("x".repeat(MAX_SHIPPED_FACT_PROPOSAL_BYTES - base.len())); - let source_bytes = serde_json::to_vec(&sidecar).unwrap(); - assert_eq!(source_bytes.len(), MAX_SHIPPED_FACT_PROPOSAL_BYTES); - write_private_file(&source_path, &source_bytes); - - let disposition = inspect_shipped_fact_proposals(&dashboard_root) - .await - .expect("the exact byte ceiling remains admitted"); - - let ShippedFactProposalDisposition::TerminalHistory { - source_bytes: observed, - .. - } = disposition - else { - panic!("the bounded terminal sidecar must remain terminal history") - }; - assert_eq!(observed.len(), MAX_SHIPPED_FACT_PROPOSAL_BYTES); - assert_eq!(observed, source_bytes); -} - #[tokio::test] async fn automatic_apply_commits_a_terminal_receipt_with_canonical_evidence() { let temp = tempfile::tempdir().unwrap(); @@ -538,11 +221,36 @@ async fn automatic_apply_commits_a_terminal_receipt_with_canonical_evidence() { receipt.validation, Some(serde_json::json!({"dedupe": {"source_index": 3}})) ); - assert!(receipt.applied_fact_id.is_some()); assert_eq!( receipt.add_fact_request.source_label.as_deref(), Some("automatic-fact-test") ); + let stored = memory + .list_project_memory_facts( + ProjectMemoryFactListQueryV1::new(memory.owner().clone(), None, None, None, 10) + .unwrap(), + run_control.read_control(), + ) + .await + .unwrap() + .facts() + .iter() + .filter_map(|projection| match projection { + ProjectMemoryFactProjectionV1::Available(fact) => Some(( + Some(fact.fact_id().as_str().to_owned()), + fact.content().to_owned(), + )), + ProjectMemoryFactProjectionV1::Unavailable(_) => None, + }) + .collect::>(); + assert_eq!( + stored, + [( + receipt.applied_fact_id.clone(), + "Keep automatic fact effects in the canonical memory authority".to_owned() + )], + "the receipt names the one canonical fact it applied" + ); let loaded = load_automatic_fact_receipt(&memory, &receipt.apply_id, run_control.read_control()) @@ -862,43 +570,6 @@ fn unprojectable_invalid_authority_fact_retains_raw_receipt_with_null_run_id() { assert_eq!(ledger["disposition"], serde_json::json!("quarantined")); } -#[tokio::test] -async fn terminal_sidecar_never_touches_an_existing_archive_path() { - let temp = tempfile::tempdir().unwrap(); - let dashboard_root = temp.path().join("dashboard"); - tokio::fs::create_dir_all(&dashboard_root).await.unwrap(); - let source_path = dashboard_root.join("fact_proposals.json"); - let source_bytes = serde_json::to_vec(&terminal_shipped_sidecar()).unwrap(); - write_private_file(&source_path, &source_bytes); - let archive_blocker = dashboard_root.join("fact_proposals.archive"); - tokio::fs::write(&archive_blocker, b"not a directory") - .await - .unwrap(); - let db = database( - &temp.path().join("memory.db"), - TestDatabaseRuntimeMode::Initialize, - ) - .await; - let memory = MemoryApplication::new(FactOwnerV1::Profile, DatabaseFactStore::new(&db)).unwrap(); - let (run_control, _) = test_run_control(false); - - let disposition = inspect_shipped_fact_proposals(&dashboard_root) - .await - .unwrap(); - assert!(matches!( - disposition, - ShippedFactProposalDisposition::TerminalHistory { .. } - )); - assert_eq!(tokio::fs::read(&source_path).await.unwrap(), source_bytes); - assert!( - tokio::fs::metadata(&archive_blocker) - .await - .unwrap() - .is_file() - ); - assert_eq!(canonical_fact_count(&memory, &run_control).await, 0); -} - #[test] fn automatic_fact_state_serializes_only_terminal_values() { for (state, wire) in [ diff --git a/crates/tracedecay-automation-runtime/src/automation/backend.rs b/crates/tracedecay-automation-runtime/src/automation/backend.rs index ccdcf6d93c..b49298da51 100644 --- a/crates/tracedecay-automation-runtime/src/automation/backend.rs +++ b/crates/tracedecay-automation-runtime/src/automation/backend.rs @@ -1,5 +1,6 @@ //! Runtime adapters for leaf-owned automation backend contracts and policies. +use schemars::JsonSchema; use std::path::Path; use std::time::{Duration, Instant}; @@ -12,6 +13,7 @@ pub use tracedecay_automation::backend::{ AgentTaskResponse, agent_task_contract, agent_task_failure_disposition, classify_agent_task_error_message, prompt_version, task_key, }; +use tracedecay_domain::configuration::LcmSummarizerExecutableV1; use tracedecay_domain::errors::Result; use super::config::{AutomationBackend, AutomationConfig}; @@ -59,7 +61,7 @@ impl BackendRetryPolicy { } } -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize, JsonSchema)] pub struct AgentTaskRetryAttempt { pub attempt: u32, pub succeeded: bool, @@ -87,7 +89,17 @@ impl AgentTaskRetryReport { } } -pub fn backend_availability(config: &AutomationConfig) -> AgentBackendAvailability { +/// Reason reported while `lcm.summarizer_executables.v1` binds no `codex` +/// executable for the project. +pub const CODEX_EXECUTABLE_UNCONFIGURED: &str = + "codex app-server backend executable is not configured (lcm.summarizer_executables.v1)"; + +/// Whether the configured backend can run, given the `codex` executable the +/// project's configuration snapshot binds. Nothing is resolved from `PATH`. +pub fn backend_availability( + config: &AutomationConfig, + codex: &LcmSummarizerExecutableV1, +) -> AgentBackendAvailability { match config.backend { AutomationBackend::Disabled => AgentBackendAvailability { backend: AutomationBackend::Disabled, @@ -95,44 +107,35 @@ pub fn backend_availability(config: &AutomationConfig) -> AgentBackendAvailabili executable: None, reason: Some("automation backend is disabled".to_string()), }, - AutomationBackend::CodexAppServer => { - let summary_config = CodexAppServerSummaryConfig::from_env(); - let executable = summary_config.codex_bin.clone(); - match executable_resolution(&executable) { - Ok(true) => AgentBackendAvailability { - backend: AutomationBackend::CodexAppServer, - available: true, - executable: Some(executable), - reason: None, - }, - Ok(false) => AgentBackendAvailability { - backend: AutomationBackend::CodexAppServer, - available: false, - executable: Some(executable.clone()), - reason: Some(format!( - "codex app-server backend executable '{executable}' was not found" - )), - }, - Err(error) => AgentBackendAvailability { - backend: AutomationBackend::CodexAppServer, - available: false, - executable: Some(executable), - reason: Some(error.to_string()), - }, + AutomationBackend::CodexAppServer => match codex.canonical_path() { + None => AgentBackendAvailability { + backend: AutomationBackend::CodexAppServer, + available: false, + executable: None, + reason: Some(CODEX_EXECUTABLE_UNCONFIGURED.to_string()), + }, + Some(path) => { + let executable = path.to_string_lossy().into_owned(); + if path.is_file() { + AgentBackendAvailability { + backend: AutomationBackend::CodexAppServer, + available: true, + executable: Some(executable), + reason: None, + } + } else { + AgentBackendAvailability { + backend: AutomationBackend::CodexAppServer, + available: false, + reason: Some(format!( + "codex app-server backend executable '{executable}' was not found" + )), + executable: Some(executable), + } + } } - } - } -} - -fn executable_resolution(bin: &str) -> Result { - let path = Path::new(bin); - if path.components().count() > 1 { - return Ok(path.is_file()); + }, } - Ok( - super::executable_lookup::resolve_on_path(bin, std::env::var_os("PATH").as_deref())? - .is_some(), - ) } pub async fn run_agent_task_with_retry( @@ -203,27 +206,43 @@ pub fn extract_json_object_prefix(text: &str) -> Result { leaf_backend::extract_json_object_prefix(text).map_err(Into::into) } +/// The Codex app-server backend bound to the project's configured executable. +/// +/// `config` is `None` while `lcm.summarizer_executables.v1` binds no `codex` +/// executable: every task then settles as `Unavailable` and nothing is spawned. #[derive(Debug, Clone)] pub struct CodexAppServerBackend { - config: CodexAppServerSummaryConfig, + config: Option, } impl CodexAppServerBackend { - pub fn from_automation_config(config: &AutomationConfig) -> Self { - Self::new(config.model_id.clone(), config.timeout_secs) + pub fn from_automation_config( + config: &AutomationConfig, + codex: &LcmSummarizerExecutableV1, + ) -> Self { + Self::new(config.model_id.clone(), config.timeout_secs, codex) } - pub fn new(model: Option, timeout_secs: u64) -> Self { - let mut config = CodexAppServerSummaryConfig::from_env(); - if let Some(model) = model.filter(|model| !model.trim().is_empty()) { - config.model = Some(model); - } - config.timeout = Duration::from_secs(timeout_secs.clamp(5, 300)); + pub fn new( + model: Option, + timeout_secs: u64, + codex: &LcmSummarizerExecutableV1, + ) -> Self { + let config = codex.canonical_path().map(|codex_bin| { + let mut config = CodexAppServerSummaryConfig::for_executable(codex_bin); + if let Some(model) = model.filter(|model| !model.trim().is_empty()) { + config.model = Some(model); + } + config.timeout = Duration::from_secs(timeout_secs.clamp(5, 300)); + config + }); Self { config } } pub fn from_config(config: CodexAppServerSummaryConfig) -> Self { - Self { config } + Self { + config: Some(config), + } } } @@ -238,6 +257,11 @@ impl AgentTaskBackend for CodexAppServerBackend { &self, request: &AgentTaskRequest, ) -> std::result::Result { + let Some(config) = self.config.as_ref() else { + return Err(AgentTaskError::Unavailable { + reason: CODEX_EXECUTABLE_UNCONFIGURED.to_string(), + }); + }; let backend_message = request .backend_message() @@ -248,7 +272,7 @@ impl AgentTaskBackend for CodexAppServerBackend { // taxonomy admits that string exactly once, at this boundary. let summary = run_prompt_with_codex_app_server( &backend_message, - &self.config, + config, "tracedecay_automation", matches!( request.task, @@ -270,12 +294,18 @@ impl AgentTaskBackend for CodexAppServerBackend { task: request.task, output_json, output_text: summary.text, - model: summary.model.or_else(|| self.config.model.clone()), + model: summary.model.or_else(|| config.model.clone()), provider: Some("codex".to_owned()), input_tokens: None, output_tokens: None, }) } + + fn executable(&self) -> Option<&Path> { + self.config + .as_ref() + .map(|config| config.codex_bin.as_path()) + } } #[cfg(test)] @@ -323,6 +353,10 @@ mod tests { output_tokens: None, }) } + + fn executable(&self) -> Option<&Path> { + None + } } fn request() -> AgentTaskRequest { diff --git a/crates/tracedecay-automation-runtime/src/automation/backend_identity.rs b/crates/tracedecay-automation-runtime/src/automation/backend_identity.rs index 5256abfca9..01e802abc8 100644 --- a/crates/tracedecay-automation-runtime/src/automation/backend_identity.rs +++ b/crates/tracedecay-automation-runtime/src/automation/backend_identity.rs @@ -17,12 +17,14 @@ //! rewrite that leaves the settings identical. Backend kind, host mode, //! model, timeout, and every per-task setting are inside it. //! * the **backend executable identity**, the opened `codex` binary the port -//! will actually spawn. The component carries stable opened-file identity -//! (Unix `dev`/`ino`, Windows volume serial / file index / link count) plus -//! revision evidence (length, mtime, and a `sha256:` content digest). A -//! missing or unreadable executable is a typed `spec` + `unreadable` -//! component, not a hasher error. Pointing the backend at a different or -//! upgraded executable is a backend change even when no setting moved. +//! will actually spawn, as the configuration authority bound it. The +//! component carries stable opened-file identity (Unix `dev`/`ino`, Windows +//! volume serial / file index / link count) plus revision evidence (length, +//! mtime, and a `sha256:` content digest). A missing or unreadable +//! executable is a typed `path` + `unreadable` component, and an +//! unconfigured one is `null`, never a hasher error. Pointing the backend at +//! a different or upgraded executable is a backend change even when no +//! automation setting moved. //! * the **protocol revision**, `AGENT_BACKEND_PROTOCOL_REVISION`, our own //! side of the transport contract. The app-server handshake, framing, and //! process lifetime are ours, so shipping a transport fix is a backend @@ -45,7 +47,6 @@ use tracedecay_domain::canonical_sha256; use super::backend::AgentTaskFailureClass; use super::config::{AutomationBackend, AutomationConfig}; use super::config_error; -use crate::ports::codex_app_server::SummaryConfig as CodexAppServerSummaryConfig; use tracedecay_domain::errors::Result; /// Skip reason published when a settled deterministic backend failure @@ -90,13 +91,15 @@ fn executable_digest_cache() -> &'static Mutex Result { +/// `executable` is the configured path the port itself would spawn, so the +/// identity tracks the binary that would actually run; `None` records that +/// the backend spawns nothing (disabled, in-process, or unconfigured). +/// Envelope hashing stays [`canonical_sha256`]; hasher failures are +/// `config_error` results. +pub fn backend_identity(config: &AutomationConfig, executable: Option<&Path>) -> Result { // `canonical_sha256` is the crate's one identity primitive: key-ordered, // whitespace-free, and already used to derive the configuration identity // that curation decisions are bound to. @@ -107,7 +110,7 @@ pub fn backend_identity(config: &AutomationConfig) -> Result { "kind": "automation.backend_identity.v1", "configuration_revision": configuration_revision.as_str(), "backend": config.backend.as_str(), - "executable": backend_executable_identity(config), + "executable": backend_executable_identity(config, executable), "protocol_revision": AGENT_BACKEND_PROTOCOL_REVISION, })) .map(|digest| digest.as_str().to_owned()) @@ -117,62 +120,33 @@ pub fn backend_identity(config: &AutomationConfig) -> Result { /// The executable the configured backend would spawn, or `None` for a backend /// that spawns nothing. /// -/// The component is the opened file, not the resolved path alone: stable +/// The component is the opened file, not the configured path alone: stable /// device/index identity plus length, mtime, and a receipt-tagged content /// digest. A missing or unreadable executable stays a typed -/// `spec` + `unreadable` component so identity remains computable. -fn backend_executable_identity(config: &AutomationConfig) -> Option { +/// `path` + `unreadable` component so identity remains computable. +fn backend_executable_identity( + config: &AutomationConfig, + executable: Option<&Path>, +) -> Option { match config.backend { AutomationBackend::Disabled => None, - AutomationBackend::CodexAppServer => { - let spec = CodexAppServerSummaryConfig::from_env().codex_bin; - Some(codex_executable_identity(&spec)) - } - } -} - -fn codex_executable_identity(spec: &str) -> Value { - match locate_backend_executable(spec) { - Ok(Some(path)) => opened_executable_identity(spec, &path) - .unwrap_or_else(|| unreadable_executable_identity(spec, Some(path.as_path()))), - Ok(None) => unreadable_executable_identity(spec, None), - Err(error) => json!({ - "spec": spec, - "state": "host_io_unavailable", - "error": error.to_string(), + AutomationBackend::CodexAppServer => executable.map(|path| { + opened_executable_identity(path).unwrap_or_else(|| { + json!({ + "path": path.to_string_lossy(), + "state": "unreadable", + }) + }) }), } } -fn locate_backend_executable(spec: &str) -> Result> { - let spec_path = Path::new(spec); - if spec_path.is_absolute() || spec.contains(std::path::MAIN_SEPARATOR) { - return Ok(spec_path.is_file().then(|| spec_path.to_path_buf())); - } - super::executable_lookup::resolve_on_path(spec, std::env::var_os("PATH").as_deref()) -} - -fn unreadable_executable_identity(spec: &str, path: Option<&Path>) -> Value { - match path { - Some(path) => json!({ - "spec": spec, - "path": path.to_string_lossy(), - "state": "unreadable", - }), - None => json!({ - "spec": spec, - "state": "unreadable", - }), - } -} - -fn opened_executable_identity(spec: &str, path: &Path) -> Option { +fn opened_executable_identity(path: &Path) -> Option { let mut file = File::open(path).ok()?; let metadata = file.metadata().ok()?; let revision = opened_file_revision(&file, &metadata)?; let content = cached_executable_content_digest(&revision, path, &mut file)?; let mut identity = revision.fields; - identity["spec"] = json!(spec); identity["path"] = json!(path.to_string_lossy()); identity["content"] = json!(content); Some(identity) @@ -296,42 +270,6 @@ pub fn is_deterministic_failure_class(class: AgentTaskFailureClass) -> bool { matches!(class, AgentTaskFailureClass::Permanent) } -#[cfg(test)] -pub(crate) struct CodexBinEnvGuard { - previous: Option, - _lock: std::sync::MutexGuard<'static, ()>, -} - -#[cfg(test)] -impl CodexBinEnvGuard { - pub(crate) fn set(path: &std::path::Path) -> Self { - let lock = tracedecay_runtime_core::config::lock_user_data_dir_test_env(); - let previous = std::env::var_os("TRACEDECAY_CODEX_BIN"); - // SAFETY: the shared user-data-dir lock is held for the guard - // lifetime, so sibling env tests cannot observe this override. - unsafe { - std::env::set_var("TRACEDECAY_CODEX_BIN", path); - } - Self { - previous, - _lock: lock, - } - } -} - -#[cfg(test)] -impl Drop for CodexBinEnvGuard { - fn drop(&mut self) { - // SAFETY: see `CodexBinEnvGuard::set`. - unsafe { - match self.previous.take() { - Some(previous) => std::env::set_var("TRACEDECAY_CODEX_BIN", previous), - None => std::env::remove_var("TRACEDECAY_CODEX_BIN"), - } - } - } -} - #[cfg(test)] #[allow(clippy::unwrap_used)] mod tests { diff --git a/crates/tracedecay-automation-runtime/src/automation/config.rs b/crates/tracedecay-automation-runtime/src/automation/config.rs index bdad0a917e..ce07ee0b5e 100644 --- a/crates/tracedecay-automation-runtime/src/automation/config.rs +++ b/crates/tracedecay-automation-runtime/src/automation/config.rs @@ -3,7 +3,6 @@ pub use tracedecay_automation::config::{ AutomationBackend, AutomationConfig, AutomationConfigPatch, AutomationHostMode, AutomationSchedule, AutomationTaskConfig, AutomationTaskPatch, AutomationTaskSet, CronSchedule, - DEFAULT_ANALYTICS_EVENTS_RETENTION_DAYS, DEFAULT_LEGACY_SESSION_RETENTION_DAYS, DEFAULT_SCHEDULER_TICK_SECS, RetentionConfig, parse_schedule, }; diff --git a/crates/tracedecay-automation-runtime/src/automation/effect_runtime.rs b/crates/tracedecay-automation-runtime/src/automation/effect_runtime.rs index 7f9ebf71e3..6a89be6637 100644 --- a/crates/tracedecay-automation-runtime/src/automation/effect_runtime.rs +++ b/crates/tracedecay-automation-runtime/src/automation/effect_runtime.rs @@ -10,7 +10,6 @@ pub mod projection; pub mod recovery_index; #[cfg(not(any(test, feature = "test-helpers")))] mod recovery_index; -pub mod retirement; pub mod settlement; pub mod terminal; diff --git a/crates/tracedecay-automation-runtime/src/automation/effect_runtime/authority.rs b/crates/tracedecay-automation-runtime/src/automation/effect_runtime/authority.rs index 4f910c08fc..354b1c3ded 100644 --- a/crates/tracedecay-automation-runtime/src/automation/effect_runtime/authority.rs +++ b/crates/tracedecay-automation-runtime/src/automation/effect_runtime/authority.rs @@ -1,108 +1,20 @@ -//! Blocking journal cleanup and retirement finalization. +//! Blocking journal cleanup after a durable terminal. use std::path::Path; -use super::journal::DurableAutomationAdmission; -use super::{AutomationSettledTerminal, contract_error, recovery_index, retirement}; +use super::{contract_error, recovery_index}; use tracedecay_domain::errors::Result; -pub async fn finalize_terminal_housekeeping( - dashboard_root: &Path, - journal_path: &Path, - admission: &DurableAutomationAdmission, - terminal: &AutomationSettledTerminal, - live_retirement: Option, -) -> Result<()> { - let retirement_binding = match ( - admission.retirement().cloned(), - terminal.is_retirement_terminal(), - ) { - (Some(binding), true) => Some(binding), - (Some(_), false) if terminal.problem().is_some() => None, - (Some(_), false) => { - return Err(contract_error( - "retirement-bound automation terminal is neither its exact zero-effect retirement nor a typed problem", - )); - } - (None, true) => { - return Err(contract_error( - "automation retirement terminal has no admitted source binding", - )); - } - (None, false) if live_retirement.is_some() => { - return Err(contract_error( - "live retirement plan has no durable admission binding", - )); - } - (None, false) => None, - }; - finalize_terminal_housekeeping_owned( - dashboard_root, - journal_path, - admission.clone(), - retirement_binding, - live_retirement, - ) - .await -} - #[hotpath::measure(label = "daemon.automation.effect.housekeeping", future = true)] -async fn finalize_terminal_housekeeping_owned( +pub async fn finalize_terminal_housekeeping( dashboard_root: &Path, journal_path: &Path, - admission: DurableAutomationAdmission, - retirement_binding: Option, - live_plan: Option, ) -> Result<()> { let dashboard_root = dashboard_root.to_path_buf(); let journal_path = journal_path.to_path_buf(); - spawn_terminal_housekeeping( - { - let dashboard_root = dashboard_root.clone(); - move || { - retirement_binding - .map(|binding| { - retirement::finalize_after_terminal( - &dashboard_root, - &binding, - live_plan.as_ref(), - ) - }) - .transpose() - } - }, - { - let dashboard_root = dashboard_root.clone(); - let journal_path = journal_path.clone(); - let admission = admission.clone(); - move |closure| { - if let Some(closure) = closure { - recovery_index::remove_pending_for_retirement_blocking( - &dashboard_root, - &journal_path, - &admission, - closure, - ) - } else { - recovery_index::remove_pending_blocking(&dashboard_root, &journal_path) - } - } - }, - { - let dashboard_root = dashboard_root.clone(); - let journal_path = journal_path.clone(); - let admission = admission.clone(); - move |closure| { - retirement::complete_after_pending_removal(&closure)?; - recovery_index::finish_retirement_transition_blocking( - &dashboard_root, - &journal_path, - &admission, - &closure, - ) - } - }, - ) + tokio::task::spawn_blocking(move || { + recovery_index::remove_pending_blocking(&dashboard_root, &journal_path) + }) .await .map_err(|error| { contract_error(format!( @@ -110,99 +22,3 @@ async fn finalize_terminal_housekeeping_owned( )) })? } - -fn spawn_terminal_housekeeping( - finalize: impl FnOnce() -> Result> + Send + 'static, - remove_pending: impl FnOnce(Option<&T>) -> Result<()> + Send + 'static, - complete: impl FnOnce(T) -> Result<()> + Send + 'static, -) -> tokio::task::JoinHandle> { - tokio::task::spawn_blocking(move || { - run_terminal_housekeeping(finalize, remove_pending, complete) - }) -} - -fn run_terminal_housekeeping( - finalize: impl FnOnce() -> Result>, - remove_pending: impl FnOnce(Option<&T>) -> Result<()>, - complete: impl FnOnce(T) -> Result<()>, -) -> Result<()> { - let closure = finalize()?; - remove_pending(closure.as_ref())?; - if let Some(closure) = closure { - complete(closure)?; - } - Ok(()) -} - -#[cfg(test)] -#[allow(clippy::unwrap_used, clippy::expect_used)] -mod tests { - use super::*; - - async fn receive_boundary(receiver: std::sync::mpsc::Receiver<()>, label: &'static str) { - tokio::task::spawn_blocking(move || receiver.recv().expect(label)) - .await - .expect(label); - } - - #[tokio::test] - async fn abort_after_pending_removal_cannot_skip_witness_completion() { - let (removed_sender, removed_receiver) = std::sync::mpsc::channel(); - let (release_sender, release_receiver) = std::sync::mpsc::channel(); - let (completed_sender, completed_receiver) = std::sync::mpsc::channel(); - let caller = tokio::spawn(async move { - spawn_terminal_housekeeping( - || Ok(Some(())), - move |_| { - removed_sender.send(()).expect("removed signal"); - release_receiver.recv().expect("remove release"); - Ok(()) - }, - move |()| { - completed_sender.send(()).expect("completion signal"); - Ok(()) - }, - ) - .await - .expect("blocking owner join") - }); - - receive_boundary(removed_receiver, "pending removal boundary").await; - caller.abort(); - release_sender.send(()).expect("release pending removal"); - receive_boundary(completed_receiver, "witness completion").await; - - assert!(caller.await.expect_err("caller was aborted").is_cancelled()); - } - - #[tokio::test] - async fn abort_during_failed_witness_completion_leaves_transition_authority_untouched() { - let (completion_sender, completion_receiver) = std::sync::mpsc::channel(); - let (release_sender, release_receiver) = std::sync::mpsc::channel(); - let transition = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); - let transition_for_owner = std::sync::Arc::clone(&transition); - let caller = tokio::spawn(async move { - spawn_terminal_housekeeping( - || Ok(Some(())), - move |_| { - transition_for_owner.store(true, std::sync::atomic::Ordering::SeqCst); - Ok(()) - }, - move |()| { - completion_sender.send(()).expect("completion boundary"); - release_receiver.recv().expect("completion release"); - Err(contract_error("injected witness completion failure")) - }, - ) - .await - .expect("blocking owner join") - }); - - receive_boundary(completion_receiver, "witness completion boundary").await; - caller.abort(); - release_sender.send(()).expect("release witness completion"); - - assert!(caller.await.expect_err("caller was aborted").is_cancelled()); - assert!(transition.load(std::sync::atomic::Ordering::SeqCst)); - } -} diff --git a/crates/tracedecay-automation-runtime/src/automation/effect_runtime/journal.rs b/crates/tracedecay-automation-runtime/src/automation/effect_runtime/journal.rs index 65eeae011b..0f55fc3d4b 100644 --- a/crates/tracedecay-automation-runtime/src/automation/effect_runtime/journal.rs +++ b/crates/tracedecay-automation-runtime/src/automation/effect_runtime/journal.rs @@ -15,13 +15,12 @@ use tracedecay_contracts::{ ResolvedScope, retained_surfaces::{AutomationRunRequestV1, AutomationTaskV1}, }; -use tracedecay_domain::{ActorId, FactOwnerV1, ManifestDigest, sha256_hex_suffix}; +use tracedecay_domain::{ActorId, FactOwnerV1, ManifestDigest}; use tracedecay_private_fs::framed_log::{ DirectorySyncPolicy, sync_parent_directory, with_owned_temp_publish, }; use super::contract::contract_error; -use super::retirement::RetirementBinding; use super::terminal::{AutomationSettledProblem, AutomationSettledTerminal}; use tracedecay_domain::errors::Result; @@ -104,8 +103,6 @@ pub enum AutomationRecoveryBinding { Memory { owner: FactOwnerV1, recovery_problem: AutomationSettledProblem, - retirement: Option, - reset_source_digest: Option, }, /// External effects have no canonical destination-side receipt store. /// Restart recovery must close them with this typed indeterminate problem @@ -132,23 +129,6 @@ impl DurableAutomationAdmission { } } - pub fn retirement(&self) -> Option<&RetirementBinding> { - match &self.recovery { - AutomationRecoveryBinding::Memory { retirement, .. } => retirement.as_ref(), - AutomationRecoveryBinding::External { .. } => None, - } - } - - pub fn reset_source_digest(&self) -> Option<&str> { - match &self.recovery { - AutomationRecoveryBinding::Memory { - reset_source_digest, - .. - } => reset_source_digest.as_deref(), - AutomationRecoveryBinding::External { .. } => None, - } - } - pub fn is_external(&self) -> bool { matches!(self.recovery, AutomationRecoveryBinding::External { .. }) } @@ -199,7 +179,7 @@ impl DurableAutomationTerminalBinding { } } -#[derive(Clone, Debug, Serialize, PartialEq)] +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] #[serde(deny_unknown_fields)] pub struct DurableAutomationRecord { #[cfg(any(test, feature = "test-helpers"))] @@ -210,74 +190,6 @@ pub struct DurableAutomationRecord { pub state: DurableAutomationState, #[cfg(not(any(test, feature = "test-helpers")))] state: DurableAutomationState, - #[serde(skip)] - #[cfg(any(test, feature = "test-helpers"))] - pub legacy_terminal: Option, - #[serde(skip)] - #[cfg(not(any(test, feature = "test-helpers")))] - legacy_terminal: Option, -} - -#[derive(Deserialize)] -#[serde(deny_unknown_fields)] -struct CurrentDurableAutomationRecord { - admission: DurableAutomationAdmission, - state: DurableAutomationState, -} - -#[derive(Deserialize)] -#[serde(deny_unknown_fields)] -struct LegacyDurableAutomationRecord { - admission: DurableAutomationAdmission, - state: LegacyDurableAutomationState, -} - -#[derive(Deserialize)] -#[serde( - tag = "state", - content = "terminal", - rename_all = "snake_case", - deny_unknown_fields -)] -enum LegacyDurableAutomationState { - Reserved, - Terminal(Box), -} - -#[derive(Deserialize)] -#[serde(untagged)] -enum DurableAutomationRecordWire { - Current(CurrentDurableAutomationRecord), - Legacy(LegacyDurableAutomationRecord), -} - -impl<'de> Deserialize<'de> for DurableAutomationRecord { - fn deserialize( - deserializer: Deserializer, - ) -> std::result::Result - where - Deserializer: serde::Deserializer<'de>, - { - match DurableAutomationRecordWire::deserialize(deserializer)? { - DurableAutomationRecordWire::Current(record) => Ok(Self { - admission: record.admission, - state: record.state, - legacy_terminal: None, - }), - DurableAutomationRecordWire::Legacy(record) => match record.state { - LegacyDurableAutomationState::Reserved => Ok(Self { - admission: record.admission, - state: DurableAutomationState::Reserved, - legacy_terminal: None, - }), - LegacyDurableAutomationState::Terminal(terminal) => Ok(Self { - admission: record.admission, - state: DurableAutomationState::Reserved, - legacy_terminal: Some(*terminal), - }), - }, - } - } } impl DurableAutomationRecord { @@ -308,30 +220,27 @@ impl DurableAutomationRecord { pub enum ReservationResult { Execute { claim: AutomationReservationClaim, - retirement: Option, }, Replay { terminal: AutomationSettledTerminal, publication: Option, - retirement: Option, }, /// A prior process durably reserved this exact admission but did not /// publish its outer terminal. The caller must reconcile against the /// canonical memory receipt authority before this reservation can close. - Recover { - retirement: Option, - }, + Recover, /// A terminal was accepted and its exact ledger row was durably staged, /// but publication did not finish before the prior process stopped. RecoverPrepared { terminal: AutomationSettledTerminal, publication: ExactRunPublication, - retirement: Option, }, /// The run identity already has a valid durable record, but the newly /// prepared admission does not match the authority bound to that record. /// This is an idempotency conflict, not a journal I/O or shape failure. - Conflict { terminal: bool }, + Conflict { + terminal: bool, + }, } /// Process-local proof that an `Execute` admission still has a live owner. @@ -397,21 +306,6 @@ fn reservation_claim_is_live(path: &Path) -> bool { .is_some() } -pub fn retained_source_bindings( - path: &Path, -) -> Result<(Option, Option)> { - with_journal_lock(path, || { - Ok(read_stabilized_record(path)? - .map(|record| { - ( - record.admission.retirement().cloned(), - record.admission.reset_source_digest().map(str::to_owned), - ) - }) - .unwrap_or_default()) - }) -} - #[cfg(any(test, feature = "test-helpers"))] pub fn reserve_or_replay_blocking( path: &Path, @@ -454,16 +348,14 @@ fn reserve_or_replay_with_index_and_writer( None => { let claim = acquire_reservation_claim(path)?; ensure_pending()?; - let retirement = requested.retirement().cloned(); write_fresh( path, &DurableAutomationRecord { admission: requested, state: DurableAutomationState::Reserved, - legacy_terminal: None, }, )?; - Ok(ReservationResult::Execute { claim, retirement }) + Ok(ReservationResult::Execute { claim }) } Some(record) => { if !stable_admission_matches(&record.admission, &requested) { @@ -481,7 +373,6 @@ fn reserve_or_replay_with_index_and_writer( } => Ok(ReservationResult::Replay { terminal: read_terminal_sidecar(path, &binding)?, publication, - retirement: record.admission.retirement().cloned(), }), DurableAutomationState::Prepared { terminal: binding, @@ -489,14 +380,11 @@ fn reserve_or_replay_with_index_and_writer( } => Ok(ReservationResult::RecoverPrepared { terminal: read_terminal_sidecar(path, &binding)?, publication, - retirement: record.admission.retirement().cloned(), }), DurableAutomationState::Reserved if reservation_claim_is_live(path) => Err( contract_error("an identical memory automation run is already in flight"), ), - DurableAutomationState::Reserved => Ok(ReservationResult::Recover { - retirement: record.admission.retirement().cloned(), - }), + DurableAutomationState::Reserved => Ok(ReservationResult::Recover), } } } @@ -1148,22 +1036,9 @@ fn read_record(path: &Path) -> Result> { )); } { - let mut record = + let record = serde_json::from_slice::(&bytes).map_err(contract_error)?; validate_admission_shape(&record.admission)?; - if let Some(legacy_terminal) = record.legacy_terminal.take() { - if !legacy_terminal.matches_admission(&record.admission) { - return Err(contract_error( - "legacy automation terminal is inconsistent with its admission", - )); - } - let binding = write_terminal_sidecar(path, &legacy_terminal)?; - record.state = DurableAutomationState::Terminal { - terminal: binding, - publication: None, - }; - write_record(path, &record)?; - } match &record.state { DurableAutomationState::Reserved => { // A terminal sidecar without Prepared is the crash residue of @@ -1613,33 +1488,6 @@ fn validate_admission_shape(admission: &DurableAutomationAdmission) -> Result<() "automation recovery binding does not match the admitted task", )); } - if let AutomationRecoveryBinding::Memory { - retirement, - reset_source_digest, - .. - } = &admission.recovery - { - if retirement.is_some() && reset_source_digest.is_some() { - return Err(contract_error( - "automation recovery cannot retire and reset the same source", - )); - } - if let Some(retirement) = retirement { - validate_sha256_text(&retirement.source_digest)?; - let expected = format!( - "fact_proposals.{}.json", - retirement.source_digest.trim_start_matches("sha256:") - ); - if retirement.archive_name != expected { - return Err(contract_error( - "automation retirement archive identity is inconsistent", - )); - } - } - if let Some(source_digest) = reset_source_digest { - validate_sha256_text(source_digest)?; - } - } admission.scope.validate().map_err(contract_error)?; admission.input_digest.validate().map_err(contract_error)?; admission @@ -1689,24 +1537,6 @@ fn validate_admission_shape(admission: &DurableAutomationAdmission) -> Result<() Ok(()) } -fn validate_sha256_text(digest: &str) -> Result<()> { - let Some(raw) = sha256_hex_suffix(digest) else { - return Err(contract_error( - "automation recovery source digest is not canonical SHA-256", - )); - }; - if raw.len() != 64 - || !raw - .bytes() - .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) - { - return Err(contract_error( - "automation recovery source digest is not canonical SHA-256", - )); - } - Ok(()) -} - fn stable_admission_matches( stored: &DurableAutomationAdmission, requested: &DurableAutomationAdmission, diff --git a/crates/tracedecay-automation-runtime/src/automation/effect_runtime/problem.rs b/crates/tracedecay-automation-runtime/src/automation/effect_runtime/problem.rs index f57b4a82e3..756c4444bf 100644 --- a/crates/tracedecay-automation-runtime/src/automation/effect_runtime/problem.rs +++ b/crates/tracedecay-automation-runtime/src/automation/effect_runtime/problem.rs @@ -56,27 +56,6 @@ pub fn indeterminate_external_effect_problem( ) } -pub fn shipped_proposal_reset_required_problem( - operation: &tracedecay_contracts::ApplicationOperation, - context: &RequestContext, - request: &AutomationRunRequestV1, -) -> Result { - zero_effect_terminal( - operation, - context, - request, - ApplicationProblem::ResetRequired { - diagnostic: SafeDiagnostic::new( - "application.memory-automation-run.shipped-proposals-reset-required", - "Unresolved shipped fact-proposal state cannot be imported because final-V2 has no approval authority; preserve it and explicitly reset its exact file.", - ) - .map_err(contract_error)?, - retry: RetryDirective::Never, - legal_actions: vec![LegalAction::Reset], - }, - ) -} - pub fn failed_ledger_problem( context: &RequestContext, cancellation: &CancellationSignal, @@ -227,7 +206,7 @@ mod tests { fn failed_ledger() -> AutomationRunLedgerRecord { serde_json::from_value(json!({ - "schema_version": 1, + "schema_version": 2, "run_id": "run.failed-ledger-problem", "trigger": "scheduler", "task": "memory_curator", @@ -237,8 +216,8 @@ mod tests { "rejected_count": 0, "error": "backend failed", "error_classification": "permanent", - "started_at": "2026-08-12T00:00:00Z", - "completed_at": "2026-08-12T00:00:01Z" + "started_at": "1786492800", + "completed_at": "1786492801" })) .expect("failed ledger") } diff --git a/crates/tracedecay-automation-runtime/src/automation/effect_runtime/recovery_index.rs b/crates/tracedecay-automation-runtime/src/automation/effect_runtime/recovery_index.rs index b319176577..922be7b527 100644 --- a/crates/tracedecay-automation-runtime/src/automation/effect_runtime/recovery_index.rs +++ b/crates/tracedecay-automation-runtime/src/automation/effect_runtime/recovery_index.rs @@ -17,13 +17,13 @@ use tracedecay_contracts::{ DisclosureClass, EffectReceipt, ProblemOwningLayer, RequestId, ResolvedScope, retained_surface_application_operation, retained_surface_execution_problem, }; -use tracedecay_domain::{ActorId, ManifestDigest, ProjectId, RunId, sha256_hex_suffix}; +use tracedecay_domain::{ActorId, ManifestDigest, ProjectId, RunId}; use tracedecay_private_fs::framed_log::{DirectorySyncPolicy, with_owned_temp_publish}; use tracedecay_store::{FactReadControl, ProjectMemoryAutomationRunReceiptsV1}; use super::journal::{self, DurableAutomationAdmission}; use super::projection::project_recovered_committed_receipts; -use super::{AutomationSettledTerminal, contract_error, digest, retirement}; +use super::{AutomationSettledTerminal, contract_error, digest}; use crate::automation::run_ledger::{self, ExactRunPublishOutcome, ExactRunUnboundDiscardOutcome}; use tracedecay_domain::errors::Result; @@ -49,13 +49,11 @@ pub enum AutomationEffectRecoveryPreparation { pub struct PreparedAutomationEffectRecovery { dashboard_root: PathBuf, - transitions: Vec, } #[hotpath::measure(label = "daemon.automation.effect.prepare_recovery", future = true)] pub async fn prepare_reserved_automation_effect_recovery( dashboard_root: &Path, - cancellation: &CancellationSignal, ) -> Result { let repair_root = dashboard_root.to_path_buf(); tokio::task::spawn_blocking(move || { @@ -68,25 +66,12 @@ pub async fn prepare_reserved_automation_effect_recovery( )) })??; let recovery_root = dashboard_root.to_path_buf(); - let (transitions, indexed) = - tokio::task::spawn_blocking(move || indexed_recovery_blocking(&recovery_root)) - .await - .map_err(|error| { - contract_error(format!("automation recovery index reader failed: {error}")) - })??; - if transitions.is_empty() && indexed.is_empty() { - if !cancellation.is_cancelled() { - let retirement_root = dashboard_root.to_path_buf(); - tokio::task::spawn_blocking(move || { - reject_unbound_retirement_witness_if_index_empty(&retirement_root) - }) - .await - .map_err(|error| { - contract_error(format!( - "automation retirement witness audit failed to join: {error}" - )) - })??; - } + let indexed = tokio::task::spawn_blocking(move || indexed_recovery_blocking(&recovery_root)) + .await + .map_err(|error| { + contract_error(format!("automation recovery index reader failed: {error}")) + })??; + if indexed.is_empty() { let report = AutomationEffectRecoveryReport::default(); observe_recovery_report(&report); return Ok(AutomationEffectRecoveryPreparation::Complete(report)); @@ -94,7 +79,6 @@ pub async fn prepare_reserved_automation_effect_recovery( Ok(AutomationEffectRecoveryPreparation::Pending( PreparedAutomationEffectRecovery { dashboard_root: dashboard_root.to_path_buf(), - transitions, }, )) } @@ -113,10 +97,7 @@ where F: Fn(RunId, FactReadControl) -> Fut + Sync, Fut: Future> + Send, { - let PreparedAutomationEffectRecovery { - dashboard_root, - transitions, - } = preparation; + let PreparedAutomationEffectRecovery { dashboard_root } = preparation; let tracedecay_domain::FactOwnerV1::Project { project_id } = owner else { return Err(contract_error( "automation recovery requires a project owner", @@ -132,31 +113,6 @@ where retained_surface_application_operation(RetainedSurfaceOperation::FactStoreCurate) .map_err(contract_error)?; let mut report = AutomationEffectRecoveryReport::default(); - for transition in transitions { - if cancellation.is_cancelled() { - break; - } - report.inspected += 1; - match reconcile_indexed_retirement_transition( - &dashboard_root, - owner, - &scope, - &operation, - &transition, - ) - .await - { - Ok(()) => report.already_terminal += 1, - Err(error) => { - tracing::warn!( - event = "automation_retirement_transition_deferred", - journal = %transition.path.display(), - error = %error, - ); - report.deferred += 1; - } - } - } let indexed_root = dashboard_root.clone(); let indexed_scope = scope.clone(); let indexed = tokio::task::spawn_blocking(move || { @@ -186,9 +142,7 @@ where Ok(EntryRecoveryOutcome::ResetRequired) => report.reset_required += 1, Ok(EntryRecoveryOutcome::Indeterminate) => report.indeterminate += 1, Ok(EntryRecoveryOutcome::AlreadyTerminal) => report.already_terminal += 1, - Ok(EntryRecoveryOutcome::Deferred | EntryRecoveryOutcome::Dormant) => { - report.deferred += 1; - } + Ok(EntryRecoveryOutcome::Deferred) => report.deferred += 1, Ok(EntryRecoveryOutcome::Cancelled) => break, Err(error) => { tracing::warn!( @@ -200,18 +154,6 @@ where } } } - if !cancellation.is_cancelled() { - let retirement_root = dashboard_root; - tokio::task::spawn_blocking(move || { - reject_unbound_retirement_witness_if_index_empty(&retirement_root) - }) - .await - .map_err(|error| { - contract_error(format!( - "automation retirement witness audit failed to join: {error}" - )) - })??; - } observe_recovery_report(&report); Ok(report) } @@ -227,17 +169,6 @@ fn observe_recovery_report(report: &AutomationEffectRecoveryReport) { hotpath::gauge!("daemon.automation.effect.reconcile.deferred_total").inc(report.deferred); } -pub fn reject_unbound_retirement_witness_if_index_empty(dashboard_root: &Path) -> Result<()> { - let path = index_path(dashboard_root); - with_index_lock(&path, || { - let index = read_index(&path)?; - if index.entries.is_empty() && index.retirement_transitions.is_empty() { - retirement::reject_unbound_retirement_witness(dashboard_root)?; - } - Ok(()) - }) -} - #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum EntryRecoveryOutcome { PartialEffect, @@ -245,89 +176,9 @@ enum EntryRecoveryOutcome { Indeterminate, AlreadyTerminal, Deferred, - Dormant, Cancelled, } -#[hotpath::measure(label = "daemon.automation.effect.retire.reconcile", future = true)] -async fn reconcile_indexed_retirement_transition( - dashboard_root: &Path, - owner: &tracedecay_domain::FactOwnerV1, - scope: &ResolvedScope, - operation: &tracedecay_contracts::ApplicationOperation, - indexed: &IndexedRetirementTransition, -) -> Result<()> { - if indexed.project_id != scope.project_id || indexed.scope_digest != scope.scope_digest { - return Err(contract_error( - "automation retirement transition escaped its exact project scope", - )); - } - let path = indexed.path.clone(); - let record = tokio::task::spawn_blocking(move || journal::read_indexed_record_blocking(&path)) - .await - .map_err(|error| { - contract_error(format!( - "automation retirement transition journal reader failed: {error}" - )) - })?? - .ok_or_else(|| { - contract_error("automation retirement transition lost its exact durable journal") - })?; - let admission = record.admission().clone(); - let binding = admission.retirement().ok_or_else(|| { - contract_error("automation retirement transition journal has no source binding") - })?; - if !record.is_terminal() - || record.publication().is_some() - || admission.schema_version != INDEX_SCHEMA_VERSION - || !admission.request.validate() - || admission.memory_owner() != Some(owner) - || admission.scope != *scope - || indexed.path.file_name().and_then(|name| name.to_str()) - != Some(&automation_journal_filename(&admission.request.run_id)?) - || binding.source_digest != indexed.source_digest - || !admission_has_exact_authority(&admission, operation)? - { - return Err(contract_error( - "automation retirement transition conflicts with its exact journal authority", - )); - } - let binding = binding.clone(); - let terminal_path = indexed.path.clone(); - let terminal = tokio::task::spawn_blocking(move || { - journal::read_indexed_terminal_blocking(&terminal_path) - }) - .await - .map_err(|error| { - contract_error(format!( - "automation retirement transition terminal reader failed: {error}" - )) - })?? - .ok_or_else(|| contract_error("automation retirement transition lost its terminal sidecar"))?; - if !terminal.is_retirement_terminal() { - return Err(contract_error( - "automation retirement transition journal is not its exact retirement terminal", - )); - } - - let root = dashboard_root.to_path_buf(); - let path = indexed.path.clone(); - let capture_expected = indexed.capture_expected; - tokio::task::spawn_blocking(move || { - let closure = - retirement::closure_for_durable_transition(&root, &binding, capture_expected)?; - remove_pending_for_retirement_blocking(&root, &path, &admission, &closure)?; - retirement::complete_after_pending_removal(&closure)?; - finish_retirement_transition_blocking(&root, &path, &admission, &closure) - }) - .await - .map_err(|error| { - contract_error(format!( - "automation retirement transition settlement failed to join: {error}" - )) - })? -} - async fn reconcile_indexed_automation_effect( read_receipts: &F, dashboard_root: &Path, @@ -403,14 +254,7 @@ where ) .await?; } - super::finalize_terminal_housekeeping( - dashboard_root, - &indexed.path, - &admission, - &terminal, - None, - ) - .await?; + super::finalize_terminal_housekeeping(dashboard_root, &indexed.path).await?; return Ok(EntryRecoveryOutcome::AlreadyTerminal); } if record.prepared().is_some() { @@ -429,9 +273,6 @@ where .await .map_err(|error| contract_error(format!("prepared terminal reader failed: {error}")))??; let (terminal, publication) = terminal; - if admission.retirement().is_some() || terminal.is_retirement_terminal() { - return Ok(EntryRecoveryOutcome::Deferred); - } let published = run_ledger::publish_staged_run_record_exact( dashboard_root, admission.request.run_id.as_str(), @@ -509,24 +350,6 @@ where return Ok(EntryRecoveryOutcome::Cancelled); } let committed = project_recovered_committed_receipts(&admission.request, &recovered)?; - if let Some(reason) = special_recovery_defer_reason(&admission, committed.is_empty()) { - tracing::warn!( - event = "automation_effect_recovery_dormant", - journal = %indexed.path.display(), - reason, - ); - // The exact run-id admission remains on disk and a direct retry will - // re-index it before exact finalization. Removing only the pending - // index entry releases bounded recovery capacity without fabricating a - // retirement/reset terminal or repeating a possibly executed effect. - remove_pending_async(dashboard_root, &indexed.path).await?; - return Ok(EntryRecoveryOutcome::Dormant); - } - if admission.retirement().is_some() && !committed.is_empty() { - return Err(contract_error( - "proposal retirement recovery found unrelated canonical memory commits", - )); - } let outcome = if recovered.is_empty() { EntryRecoveryOutcome::ResetRequired } else { @@ -582,19 +405,6 @@ async fn persist_reserved_recovery( }) } -pub fn special_recovery_defer_reason( - admission: &DurableAutomationAdmission, - committed_receipts_empty: bool, -) -> Option<&'static str> { - if committed_receipts_empty && admission.retirement().is_some() { - Some("retirement_requires_exact_finalization") - } else if committed_receipts_empty && admission.reset_source_digest().is_some() { - Some("shipped_proposals_require_exact_reset_diagnostic") - } else { - None - } -} - pub fn admission_has_exact_authority( admission: &DurableAutomationAdmission, operation: &tracedecay_contracts::ApplicationOperation, @@ -741,23 +551,11 @@ struct PendingIndexEntry { scope_digest: ManifestDigest, } -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -struct PendingRetirementTransition { - journal_file: String, - project_id: ProjectId, - scope_digest: ManifestDigest, - source_digest: String, - capture_expected: bool, -} - #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] struct PendingIndex { schema_version: u32, entries: Vec, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - retirement_transitions: Vec, } pub struct IndexedJournal { @@ -766,15 +564,6 @@ pub struct IndexedJournal { pub scope_digest: ManifestDigest, } -#[derive(Clone)] -struct IndexedRetirementTransition { - path: PathBuf, - project_id: ProjectId, - scope_digest: ManifestDigest, - source_digest: String, - capture_expected: bool, -} - pub fn add_pending_blocking( dashboard_root: &Path, journal_path: &Path, @@ -817,231 +606,6 @@ pub fn remove_pending_blocking(dashboard_root: &Path, journal_path: &Path) -> Re }) } -pub fn remove_pending_for_retirement_blocking( - dashboard_root: &Path, - journal_path: &Path, - admission: &DurableAutomationAdmission, - closure: &retirement::RetirementClosure, -) -> Result<()> { - remove_pending_for_retirement_with_writer( - dashboard_root, - journal_path, - admission, - closure, - write_pending_index, - ) -} - -fn remove_pending_for_retirement_with_writer( - dashboard_root: &Path, - journal_path: &Path, - admission: &DurableAutomationAdmission, - closure: &retirement::RetirementClosure, - mut write_index: impl FnMut(&Path, &[u8]) -> Result<()>, -) -> Result<()> { - let expected = entry_for(journal_path, &admission.scope)?; - let transition = retirement_transition_for(&expected, admission, closure)?; - let path = index_path(dashboard_root); - with_index_lock(&path, || { - let original = read_index(&path)?; - if original.entries.iter().any(|candidate| { - candidate.journal_file == expected.journal_file && candidate != &expected - }) { - return Err(contract_error( - "automation pending index journal identity conflicts with its project binding", - )); - } - if original.retirement_transitions.iter().any(|candidate| { - candidate.journal_file == transition.journal_file && candidate != &transition - }) { - return Err(contract_error( - "automation retirement transition conflicts with its durable journal binding", - )); - } - if !original.entries.contains(&expected) - && !original.retirement_transitions.contains(&transition) - { - return Ok(()); - } - let protected = publish_retirement_transition_with_writer( - &path, - &original, - &transition, - &mut write_index, - )?; - if !protected.entries.contains(&expected) { - return Ok(()); - } - remove_pending_after_transition_with_writer(&path, &protected, &expected, &mut write_index) - }) -} - -fn publish_retirement_transition_with_writer( - path: &Path, - original: &PendingIndex, - transition: &PendingRetirementTransition, - write_index: &mut impl FnMut(&Path, &[u8]) -> Result<()>, -) -> Result { - if original.retirement_transitions.iter().any(|candidate| { - candidate.journal_file == transition.journal_file && candidate != transition - }) { - return Err(contract_error( - "automation retirement transition conflicts with its durable journal binding", - )); - } - if original.retirement_transitions.contains(transition) { - return Ok(original.clone()); - } - if original.retirement_transitions.len() >= MAX_PENDING_AUTOMATION_EFFECTS { - return Err(contract_error( - "automation retirement transition index reached its bounded capacity", - )); - } - let mut protected = original.clone(); - protected.retirement_transitions.push(transition.clone()); - protected - .retirement_transitions - .sort_by(|left, right| left.journal_file.cmp(&right.journal_file)); - if let Err(error) = publish_index_state(path, &protected, write_index) { - match read_index(path) { - Ok(visible) if visible == protected => {} - Ok(visible) if visible == *original => { - return Err(contract_error(format!( - "automation retirement transition publication failed before pending removal: {error}" - ))); - } - Ok(_) => { - return Err(contract_error(format!( - "automation retirement transition publication left an unrecognized durable index state: {error}" - ))); - } - Err(read_error) => { - return Err(contract_error(format!( - "automation retirement transition publication is uncertain: {error}; readback failed: {read_error}" - ))); - } - } - } - Ok(protected) -} - -fn remove_pending_after_transition_with_writer( - path: &Path, - protected: &PendingIndex, - expected: &PendingIndexEntry, - write_index: &mut impl FnMut(&Path, &[u8]) -> Result<()>, -) -> Result<()> { - let mut removed = protected.clone(); - removed.entries.retain(|candidate| candidate != expected); - let removed_bytes = encode_pending_index(&removed)?; - match write_index(path, &removed_bytes) { - Ok(()) => require_index_state(path, &removed), - Err(removal_error) => match read_index(path) { - Ok(visible) if visible == removed => Ok(()), - Ok(visible) if visible == *protected => Err(contract_error(format!( - "automation pending removal retained its marker-protected prior state: {removal_error}" - ))), - Ok(_) => Err(contract_error(format!( - "automation pending removal left an unrecognized marker-protected index state: {removal_error}" - ))), - Err(read_error) => Err(contract_error(format!( - "automation pending removal is uncertain while its transition marker remains required: {removal_error}; readback failed: {read_error}" - ))), - }, - } -} - -#[hotpath::measure(label = "daemon.automation.effect.retire.finish")] -pub fn finish_retirement_transition_blocking( - dashboard_root: &Path, - journal_path: &Path, - admission: &DurableAutomationAdmission, - closure: &retirement::RetirementClosure, -) -> Result<()> { - let entry = entry_for(journal_path, &admission.scope)?; - let transition = retirement_transition_for(&entry, admission, closure)?; - let path = index_path(dashboard_root); - with_index_lock(&path, || { - let original = read_index(&path)?; - finish_retirement_transition_with_writer( - &path, - &original, - &transition, - &mut write_pending_index, - ) - }) -} - -fn finish_retirement_transition_with_writer( - path: &Path, - original: &PendingIndex, - transition: &PendingRetirementTransition, - write_index: &mut impl FnMut(&Path, &[u8]) -> Result<()>, -) -> Result<()> { - if !original.retirement_transitions.contains(transition) { - return Ok(()); - } - let mut completed = original.clone(); - completed - .retirement_transitions - .retain(|candidate| candidate != transition); - match publish_index_state(path, &completed, write_index) { - Ok(()) => Ok(()), - Err(error) => match read_index(path) { - Ok(visible) if visible == completed => Ok(()), - Ok(visible) if visible == *original => Err(error), - Ok(_) => Err(contract_error(format!( - "automation retirement transition removal left an unrecognized durable index state: {error}" - ))), - Err(read_error) => Err(contract_error(format!( - "automation retirement transition removal is uncertain: {error}; readback failed: {read_error}" - ))), - }, - } -} - -fn retirement_transition_for( - entry: &PendingIndexEntry, - admission: &DurableAutomationAdmission, - closure: &retirement::RetirementClosure, -) -> Result { - let binding = admission.retirement().ok_or_else(|| { - contract_error("automation retirement transition has no durable admission binding") - })?; - if binding.source_digest != closure.source_digest() { - return Err(contract_error( - "automation retirement transition conflicts with its source digest", - )); - } - Ok(PendingRetirementTransition { - journal_file: entry.journal_file.clone(), - project_id: entry.project_id.clone(), - scope_digest: entry.scope_digest.clone(), - source_digest: binding.source_digest.clone(), - capture_expected: closure.capture_expected(), - }) -} - -fn publish_index_state( - path: &Path, - expected: &PendingIndex, - write_index: &mut impl FnMut(&Path, &[u8]) -> Result<()>, -) -> Result<()> { - let bytes = encode_pending_index(expected)?; - write_index(path, &bytes)?; - require_index_state(path, expected) -} - -fn require_index_state(path: &Path, expected: &PendingIndex) -> Result<()> { - if read_index(path)? == *expected { - Ok(()) - } else { - Err(contract_error( - "automation pending recovery index did not replay its exact expected state", - )) - } -} - fn encode_pending_index(index: &PendingIndex) -> Result> { let bytes = serde_json::to_vec_pretty(index).map_err(contract_error)?; if bytes.len() > MAX_INDEX_BYTES as usize { @@ -1057,7 +621,6 @@ pub fn indexed_journals_blocking( scope: &ResolvedScope, ) -> Result> { Ok(indexed_recovery_blocking(dashboard_root)? - .1 .into_iter() .filter(|entry| { entry.project_id == scope.project_id && entry.scope_digest == scope.scope_digest @@ -1065,32 +628,12 @@ pub fn indexed_journals_blocking( .collect()) } -#[cfg(test)] -fn indexed_retirement_transitions_blocking( - dashboard_root: &Path, -) -> Result> { - Ok(indexed_recovery_blocking(dashboard_root)?.0) -} - -fn indexed_recovery_blocking( - dashboard_root: &Path, -) -> Result<(Vec, Vec)> { +fn indexed_recovery_blocking(dashboard_root: &Path) -> Result> { let index_path = index_path(dashboard_root); with_index_lock(&index_path, || { let index = read_index(&index_path)?; let automation_root = automation_root(dashboard_root); - let transitions = index - .retirement_transitions - .into_iter() - .map(|transition| IndexedRetirementTransition { - path: automation_root.join(&transition.journal_file), - project_id: transition.project_id, - scope_digest: transition.scope_digest, - source_digest: transition.source_digest, - capture_expected: transition.capture_expected, - }) - .collect(); - let indexed = index + Ok(index .entries .into_iter() .map(|entry| IndexedJournal { @@ -1098,8 +641,7 @@ fn indexed_recovery_blocking( project_id: entry.project_id, scope_digest: entry.scope_digest, }) - .collect(); - Ok((transitions, indexed)) + .collect()) }) } @@ -1178,7 +720,6 @@ fn read_index(path: &Path) -> Result { return Ok(PendingIndex { schema_version: INDEX_SCHEMA_VERSION, entries: Vec::new(), - retirement_transitions: Vec::new(), }); } Err(error) => { @@ -1212,7 +753,6 @@ fn read_index(path: &Path) -> Result { let index: PendingIndex = serde_json::from_slice(&bytes).map_err(contract_error)?; if index.schema_version != INDEX_SCHEMA_VERSION || index.entries.len() > MAX_PENDING_AUTOMATION_EFFECTS - || index.retirement_transitions.len() > MAX_PENDING_AUTOMATION_EFFECTS { return Err(contract_error( "automation pending index has an unsupported or unbounded shape", @@ -1223,33 +763,9 @@ fn read_index(path: &Path) -> Result { entry.project_id.validate().map_err(contract_error)?; entry.scope_digest.validate().map_err(contract_error)?; } - for transition in &index.retirement_transitions { - validate_journal_filename(&transition.journal_file)?; - transition.project_id.validate().map_err(contract_error)?; - transition.scope_digest.validate().map_err(contract_error)?; - validate_sha256_digest(&transition.source_digest)?; - } Ok(index) } -fn validate_sha256_digest(digest: &str) -> Result<()> { - let Some(body) = sha256_hex_suffix(digest) else { - return Err(contract_error( - "automation retirement transition digest prefix is invalid", - )); - }; - if body.len() != 64 - || !body - .bytes() - .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) - { - return Err(contract_error( - "automation retirement transition digest is invalid", - )); - } - Ok(()) -} - fn entry_for(path: &Path, scope: &ResolvedScope) -> Result { Ok(PendingIndexEntry { journal_file: journal_filename(path)?, @@ -1384,7 +900,6 @@ mod tests { fn prepared_recovery(dashboard_root: &Path) -> PreparedAutomationEffectRecovery { PreparedAutomationEffectRecovery { dashboard_root: dashboard_root.to_path_buf(), - transitions: Vec::new(), } } @@ -1578,164 +1093,4 @@ mod tests { .is_empty() ); } - - fn pending_entry(label: char) -> PendingIndexEntry { - let scope_label = match label { - 'a' => 'e', - 'b' => 'f', - _ => label, - }; - PendingIndexEntry { - journal_file: format!("{}.json", label.to_string().repeat(64)), - project_id: ProjectId::new(format!("project.retirement-{label}")).expect("project"), - scope_digest: ManifestDigest::new(format!( - "sha256:{}", - scope_label.to_string().repeat(64) - )) - .expect("scope digest"), - } - } - - fn retirement_transition( - entry: &PendingIndexEntry, - digest_label: char, - ) -> PendingRetirementTransition { - PendingRetirementTransition { - journal_file: entry.journal_file.clone(), - project_id: entry.project_id.clone(), - scope_digest: entry.scope_digest.clone(), - source_digest: format!("sha256:{}", digest_label.to_string().repeat(64)), - capture_expected: true, - } - } - - fn pending_index( - entries: Vec, - retirement_transitions: Vec, - ) -> PendingIndex { - PendingIndex { - schema_version: INDEX_SCHEMA_VERSION, - entries, - retirement_transitions, - } - } - - #[test] - fn retirement_handoff_resolves_visible_write_uncertainty_and_preserves_other_transitions() { - let temp = tempfile::tempdir().expect("tempdir"); - let path = index_path(temp.path()); - std::fs::create_dir_all(path.parent().expect("pending index parent")) - .expect("create automation root"); - let entry_a = pending_entry('a'); - let entry_b = pending_entry('b'); - let transition_a = retirement_transition(&entry_a, 'c'); - let transition_b = retirement_transition(&entry_b, 'd'); - let original = pending_index( - vec![entry_a.clone(), entry_b.clone()], - vec![transition_b.clone()], - ); - write_pending_index( - &path, - &encode_pending_index(&original).expect("original bytes"), - ) - .expect("original index"); - - let protected = publish_retirement_transition_with_writer( - &path, - &original, - &transition_a, - &mut |path, bytes| { - write_pending_index(path, bytes)?; - Err(contract_error( - "injected error after visible transition publication", - )) - }, - ) - .expect("visible marker publication is resolved by exact readback"); - assert_eq!( - protected.retirement_transitions, - vec![transition_a.clone(), transition_b.clone()] - ); - assert_eq!( - indexed_retirement_transitions_blocking(temp.path()) - .expect("bounded transition inventory") - .len(), - 2 - ); - - remove_pending_after_transition_with_writer( - &path, - &protected, - &entry_a, - &mut |path, bytes| { - write_pending_index(path, bytes)?; - Err(contract_error( - "injected error after visible pending removal", - )) - }, - ) - .expect("visible pending removal is resolved by exact readback"); - let marker_only = read_index(&path).expect("marker-only index"); - assert_eq!(marker_only.entries, vec![entry_b]); - assert_eq!( - marker_only.retirement_transitions, - vec![transition_a.clone(), transition_b.clone()] - ); - - finish_retirement_transition_with_writer( - &path, - &marker_only, - &transition_a, - &mut |path, bytes| { - write_pending_index(path, bytes)?; - Err(contract_error( - "injected error after visible transition removal", - )) - }, - ) - .expect("visible marker removal is resolved by exact readback"); - let completed = read_index(&path).expect("completed index"); - assert_eq!(completed.retirement_transitions, vec![transition_b]); - } - - #[test] - fn retirement_handoff_preserves_prepublication_state_and_rejects_mismatched_marker() { - let temp = tempfile::tempdir().expect("tempdir"); - let path = index_path(temp.path()); - std::fs::create_dir_all(path.parent().expect("pending index parent")) - .expect("create automation root"); - let entry = pending_entry('a'); - let transition = retirement_transition(&entry, 'c'); - let original = pending_index(vec![entry.clone()], Vec::new()); - write_pending_index( - &path, - &encode_pending_index(&original).expect("original bytes"), - ) - .expect("original index"); - - let error = publish_retirement_transition_with_writer( - &path, - &original, - &transition, - &mut |_path, _bytes| Err(contract_error("injected publication failure")), - ) - .expect_err("an invisible marker publication must not authorize pending removal"); - assert!(error.to_string().contains("before pending removal")); - assert_eq!(read_index(&path).expect("unchanged index"), original); - - let conflicting = retirement_transition(&entry, 'd'); - let mismatched = pending_index(vec![entry], vec![conflicting]); - let error = publish_retirement_transition_with_writer( - &path, - &mismatched, - &transition, - &mut |_path, _bytes| panic!("mismatched marker must not be overwritten"), - ) - .expect_err("a marker with a foreign source digest must fail closed"); - assert!( - error - .to_string() - .contains("conflicts with its durable journal binding") - ); - } } diff --git a/crates/tracedecay-automation-runtime/src/automation/effect_runtime/retirement.rs b/crates/tracedecay-automation-runtime/src/automation/effect_runtime/retirement.rs deleted file mode 100644 index dd40a03510..0000000000 --- a/crates/tracedecay-automation-runtime/src/automation/effect_runtime/retirement.rs +++ /dev/null @@ -1,1609 +0,0 @@ -//! Finalization of shipped proposal history after the main typed terminal. - -use std::ffi::OsStr; -use std::io::Write; -use std::path::{Path, PathBuf}; - -use crate::automation::automatic_facts::{ - MAX_SHIPPED_FACT_PROPOSAL_BYTES, ShippedFactProposalDisposition, - read_shipped_fact_proposal_bytes, -}; -use cap_std::{ambient_authority, fs::Dir}; -use serde::{Deserialize, Serialize}; -use sha2::{Digest, Sha256}; -use tracedecay_contracts::retained_surfaces::AutomationTaskV1; -use tracedecay_domain::canonical_text::{encode_tagged_lowercase_hex, is_tagged_lowercase_hex}; -#[cfg(test)] -use tracedecay_domain::sha256_hex_suffix; -use tracedecay_private_fs::capability_dir::rename_noreplace; -use tracedecay_private_fs::framed_log::{ - DirectorySyncPolicy, sync_parent_directory, with_owned_temp_publish, -}; - -use tracedecay_domain::errors::{Result, TraceDecayError}; - -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct RetirementBinding { - pub source_digest: String, - pub archive_name: String, -} - -pub struct RetirementPlan { - pub binding: RetirementBinding, - source_path: PathBuf, - source_bytes: Vec, -} - -#[derive(Debug)] -pub struct RetirementClosure { - source_path: PathBuf, - source_digest: String, - capture_path: Option, -} - -impl RetirementClosure { - pub fn source_digest(&self) -> &str { - &self.source_digest - } - - pub fn capture_expected(&self) -> bool { - self.capture_path.is_some() - } -} - -pub fn classify(disposition: ShippedFactProposalDisposition) -> Result { - match disposition { - ShippedFactProposalDisposition::Absent => Ok(RetirementClassification::Absent), - ShippedFactProposalDisposition::ResetRequired { - source_path, - source_digest, - reason, - } => { - validate_digest(&source_digest)?; - Ok(RetirementClassification::ResetRequired { - source_digest, - detail: format!( - "{reason} at '{}'; final-V2 will not approve, import, archive, or delete unresolved shipped proposal state", - source_path.display() - ), - }) - } - ShippedFactProposalDisposition::TerminalHistory { - source_path, - source_digest, - source_bytes, - } => { - if source_bytes.len() > MAX_SHIPPED_FACT_PROPOSAL_BYTES { - return Err(contract_error( - "live shipped proposal history exceeds its retirement byte bound", - )); - } - require_digest(&source_bytes, &source_digest)?; - let archive_name = format!( - "fact_proposals.{}.json", - canonical_digest_body(&source_digest)? - ); - Ok(RetirementClassification::Terminal(RetirementPlan { - binding: RetirementBinding { - source_digest, - archive_name, - }, - source_path, - source_bytes, - })) - } - } -} - -#[hotpath::measure(label = "daemon.automation_retirement.classify", future = true)] -pub async fn classify_for_task( - task: AutomationTaskV1, - dashboard_root: &Path, -) -> Result { - if task != AutomationTaskV1::SessionReflector { - return Ok(RetirementClassification::Absent); - } - classify( - crate::automation::automatic_facts::inspect_shipped_fact_proposals(dashboard_root).await?, - ) -} - -pub enum RetirementClassification { - Absent, - ResetRequired { - source_digest: String, - detail: String, - }, - Terminal(RetirementPlan), -} - -pub fn verify_plan_matches_binding( - plan: &RetirementPlan, - binding: &RetirementBinding, -) -> Result<()> { - if plan.binding == *binding { - Ok(()) - } else { - Err(contract_error( - "live shipped proposal history conflicts with its admitted retirement", - )) - } -} - -/// Completes only after the caller has durably persisted the main typed -/// zero-effect `AutomationRun` terminal that contains this binding in -/// its admitted input digest. -#[hotpath::measure(label = "daemon.automation_retirement.finalize")] -pub fn finalize_after_terminal( - dashboard_root: &Path, - binding: &RetirementBinding, - live_plan: Option<&RetirementPlan>, -) -> Result { - finalize_after_terminal_impl( - dashboard_root, - binding, - live_plan, - publish_archive_bytes, - capture_exact_source, - ) -} - -fn finalize_after_terminal_impl( - dashboard_root: &Path, - binding: &RetirementBinding, - live_plan: Option<&RetirementPlan>, - publish: impl FnOnce(&Path, &[u8]) -> Result<()>, - capture_source: impl FnOnce(&Path, &[u8]) -> Result>, -) -> Result { - validate_binding(binding)?; - let source_path = dashboard_root.join("fact_proposals.json"); - let archive_path = archive_path(dashboard_root, binding)?; - if let Some(captured) = recover_retirement_capture(&source_path, &binding.source_digest)? { - let archived = read_retirement_bytes(&archive_path, "archive")?.ok_or_else(|| { - contract_error( - "captured proposal history cannot retire without its exact durable archive", - ) - })?; - require_digest(&archived, &binding.source_digest)?; - if archived != captured { - return Err(contract_error( - "captured proposal history conflicts with its durable archive", - )); - } - let capture_path = capture_path_for_digest( - source_path - .parent() - .ok_or_else(|| contract_error("shipped proposal source has no parent directory"))?, - &hex::decode(canonical_digest_body(&binding.source_digest)?).map_err(|error| { - contract_error(format!("retirement digest decode failed: {error}")) - })?, - ); - return Ok(RetirementClosure { - source_path, - source_digest: binding.source_digest.clone(), - capture_path: Some(capture_path), - }); - } - let (bytes, capture_live_source) = match live_plan { - Some(plan) => { - verify_plan_matches_binding(plan, binding)?; - if plan.source_path != source_path { - return Err(contract_error( - "shipped proposal retirement source escaped dashboard authority", - )); - } - match read_retirement_bytes(&source_path, "source")? { - Some(bytes) => { - require_digest(&bytes, &binding.source_digest)?; - if bytes != plan.source_bytes { - return Err(contract_error( - "live shipped proposal source changed after retirement admission", - )); - } - (bytes, true) - } - None => { - let bytes = read_retirement_bytes(&archive_path, "archive")?.ok_or_else(|| { - contract_error( - "admitted proposal history is absent from both live and archive paths", - ) - })?; - require_digest(&bytes, &binding.source_digest)?; - if bytes != plan.source_bytes { - return Err(contract_error( - "retirement archive conflicts with the admitted live plan", - )); - } - (bytes, false) - } - } - } - None => match read_retirement_bytes(&archive_path, "archive")? { - Some(bytes) => { - require_digest(&bytes, &binding.source_digest)?; - let capture_live_source = match read_retirement_bytes(&source_path, "source")? { - Some(source) => source == bytes, - None => false, - }; - (bytes, capture_live_source) - } - None => { - let bytes = read_retirement_bytes(&source_path, "source")?.ok_or_else(|| { - contract_error( - "retired proposal history is absent from both live and archive paths", - ) - })?; - require_digest(&bytes, &binding.source_digest)?; - (bytes, true) - } - }, - }; - publish_archive_with(&archive_path, &bytes, publish)?; - let capture_path = if capture_live_source { - capture_source(&source_path, &bytes)? - } else { - None - }; - Ok(RetirementClosure { - source_path, - source_digest: binding.source_digest.clone(), - capture_path, - }) -} - -#[hotpath::measure(label = "daemon.automation_retirement.complete")] -pub fn complete_after_pending_removal(closure: &RetirementClosure) -> Result<()> { - complete_after_pending_removal_with(closure, |_| Ok(())) -} - -pub fn closure_for_durable_transition( - dashboard_root: &Path, - binding: &RetirementBinding, - capture_expected: bool, -) -> Result { - validate_binding(binding)?; - let digest = canonical_digest_body(&binding.source_digest)?; - let source_path = dashboard_root.join("fact_proposals.json"); - let archive_path = archive_path(dashboard_root, binding)?; - let archived = read_retirement_bytes(&archive_path, "transition archive")? - .ok_or_else(|| contract_error("retirement transition has no exact durable archive"))?; - require_digest(&archived, &binding.source_digest)?; - - let digest_bytes = hex::decode(digest) - .map_err(|error| contract_error(format!("retirement digest decode failed: {error}")))?; - let capture_path = capture_path_for_digest(dashboard_root, &digest_bytes); - let retired_path = retired_path_for_digest(dashboard_root, digest); - let captured = read_retirement_bytes(&capture_path, "transition captured source")?; - let retired = read_retirement_bytes(&retired_path, "transition retired source")?; - if captured.is_some() && retired.is_some() { - return Err(contract_error( - "retirement transition has both captured and retired witnesses", - )); - } - for witness in captured.iter().chain(retired.iter()) { - require_digest(witness, &binding.source_digest)?; - if witness != &archived { - return Err(contract_error( - "retirement transition witness conflicts with its durable archive", - )); - } - } - if !capture_expected && (captured.is_some() || retired.is_some()) { - return Err(contract_error( - "retirement transition unexpectedly owns a source witness", - )); - } - Ok(RetirementClosure { - source_path, - source_digest: binding.source_digest.clone(), - capture_path: capture_expected.then_some(capture_path), - }) -} - -fn complete_after_pending_removal_with( - closure: &RetirementClosure, - after_retirement: impl FnOnce(&Path) -> Result<()>, -) -> Result<()> { - let Some(capture_path) = closure.capture_path.as_ref() else { - return Ok(()); - }; - let lock_path = tracedecay_runtime_core::storage::append_lock_path(&closure.source_path); - tracedecay_runtime_core::storage::reject_symlink_components( - &lock_path, - "shipped proposal retirement lock", - ) - .map_err(contract_error)?; - let lock = tracedecay_runtime_core::storage::acquire_sidecar_lock_blocking(&lock_path) - .map_err(|error| { - contract_error(format!("shipped proposal retirement lock failed: {error}")) - })?; - let result = (|| { - let parent_path = closure.source_path.parent().ok_or_else(|| { - contract_error("shipped proposal retirement source has no parent directory") - })?; - let parent = Dir::open_ambient_dir(parent_path, ambient_authority()).map_err(|error| { - contract_error(format!( - "shipped proposal retirement parent open failed: {error}" - )) - })?; - let digest = canonical_digest_body(&closure.source_digest)?; - let retired_path = retired_path_for_digest(parent_path, digest); - let captured = read_retirement_bytes(capture_path, "captured source")?; - let retired = read_retirement_bytes(&retired_path, "retired source witness")?; - match (captured, retired) { - (Some(captured), None) => { - require_digest(&captured, &closure.source_digest)?; - let captured_name = capture_path.file_name().ok_or_else(|| { - contract_error("captured shipped proposal source has no filename") - })?; - let retired_name = retired_path.file_name().ok_or_else(|| { - contract_error("retired shipped proposal witness has no filename") - })?; - tracedecay_runtime_core::storage::retry_transient_file_op(|| { - rename_noreplace(&parent, captured_name, &parent, retired_name) - }) - .map_err(|error| { - contract_error(format!( - "captured shipped proposal source retirement failed: {error}" - )) - })?; - sync_parent_directory(&closure.source_path, DirectorySyncPolicy::Strict) - .map_err(contract_error)?; - after_retirement(&retired_path)?; - let retired = read_retirement_bytes(&retired_path, "retired source witness")? - .ok_or_else(|| { - contract_error("retired shipped proposal witness disappeared") - })?; - require_digest(&retired, &closure.source_digest)?; - if read_retirement_bytes(capture_path, "captured source")?.is_some() { - return Err(contract_error( - "captured shipped proposal source remained after retirement", - )); - } - remove_retired_witness(&retired_path, &closure.source_path) - } - (None, Some(retired)) => { - require_digest(&retired, &closure.source_digest)?; - after_retirement(&retired_path)?; - remove_retired_witness(&retired_path, &closure.source_path) - } - (None, None) => Ok(()), - (Some(_), Some(_)) => Err(contract_error( - "captured and retired shipped proposal witnesses both exist", - )), - } - })(); - let unlock = lock.unlock().map_err(|error| { - contract_error(format!( - "shipped proposal retirement unlock failed: {error}" - )) - }); - match (result, unlock) { - (Err(error), _) => Err(error), - (Ok(()), Err(error)) => Err(error), - (Ok(()), Ok(())) => Ok(()), - } -} - -pub fn reject_unbound_retirement_witness(dashboard_root: &Path) -> Result<()> { - if let Some((witness_path, _)) = orphaned_retirement_witness(dashboard_root)? { - return Err(contract_error(format!( - "retirement witness '{}' has no durable journal transition authority", - witness_path.display() - ))); - } - Ok(()) -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum RetirementWitnessKind { - Captured, - Retired, -} - -fn orphaned_retirement_witness( - dashboard_root: &Path, -) -> Result> { - const MAX_RETIREMENT_ROOT_ENTRIES: usize = 1_024; - let entries = match std::fs::read_dir(dashboard_root) { - Ok(entries) => entries, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), - Err(error) => { - return Err(contract_error(format!( - "retirement capture inventory failed: {error}" - ))); - } - }; - let mut witness = None; - for (index, entry) in entries.enumerate() { - if index >= MAX_RETIREMENT_ROOT_ENTRIES { - return Err(contract_error(format!( - "retirement capture inventory exceeds its {MAX_RETIREMENT_ROOT_ENTRIES}-entry bound" - ))); - } - let entry = entry - .map_err(|error| contract_error(format!("retirement capture entry failed: {error}")))?; - let name = entry.file_name(); - let Some(name) = name.to_str() else { - continue; - }; - if !name.starts_with(".fact_proposals.retirement-") { - continue; - } - let (_, kind) = retirement_witness_digest_name(name)?; - if witness.replace((entry.path(), kind)).is_some() { - return Err(contract_error( - "multiple orphaned retirement witnesses require operator repair", - )); - } - } - Ok(witness) -} - -fn retirement_witness_digest_name(name: &str) -> Result<(&str, RetirementWitnessKind)> { - let value = name - .strip_prefix(".fact_proposals.retirement-") - .ok_or_else(|| contract_error("retirement witness name is not canonical"))?; - let (digest, kind) = if let Some(digest) = value.strip_suffix(".captured") { - (digest, RetirementWitnessKind::Captured) - } else if let Some(digest) = value.strip_suffix(".retired") { - (digest, RetirementWitnessKind::Retired) - } else { - return Err(contract_error("retirement witness name is not canonical")); - }; - if digest.len() == 64 - && digest - .bytes() - .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) - { - Ok((digest, kind)) - } else { - Err(contract_error( - "retirement capture digest is not canonical SHA-256", - )) - } -} - -fn remove_retired_witness(retired_path: &Path, source_path: &Path) -> Result<()> { - let removal = std::fs::remove_file(retired_path); - let sync = - sync_parent_directory(source_path, DirectorySyncPolicy::Strict).map_err(contract_error); - let absent = read_retirement_bytes(retired_path, "retired source witness")?.is_none(); - if absent && sync.is_ok() { - return Ok(()); - } - match (removal, sync, absent) { - (Err(error), _, false) => Err(contract_error(format!( - "retired shipped proposal witness removal failed: {error}" - ))), - (_, Err(error), _) => Err(error), - (_, _, false) => Err(contract_error( - "retired shipped proposal witness remained after removal", - )), - _ => Ok(()), - } -} - -#[cfg(test)] -fn finalize_after_terminal_with_io( - dashboard_root: &Path, - binding: &RetirementBinding, - live_plan: Option<&RetirementPlan>, - publish: impl FnOnce(&Path, &[u8]) -> Result<()>, - remove_source: impl FnOnce(&Path, &[u8]) -> Result<()>, -) -> Result<()> { - let closure = finalize_after_terminal_impl( - dashboard_root, - binding, - live_plan, - publish, - |path, bytes| { - remove_source(path, bytes)?; - Ok(None) - }, - )?; - complete_after_pending_removal(&closure) -} - -fn publish_archive_with( - path: &Path, - bytes: &[u8], - publish: impl FnOnce(&Path, &[u8]) -> Result<()>, -) -> Result<()> { - match read_retirement_bytes(path, "archive")? { - Some(existing) if existing == bytes => {} - Some(_) => { - return Err(contract_error( - "shipped proposal retirement archive conflicts with admitted bytes", - )); - } - None => {} - } - - // An exact visible archive may be the commit-uncertain residue of a - // Windows replacement that surfaced an error after changing the namespace. - // Republish the admitted bytes through the write-through authority on every - // retry, then require exact readback before source retirement. - publish(path, bytes)?; - match read_retirement_bytes(path, "archive")? { - Some(existing) if existing == bytes => Ok(()), - Some(_) => Err(contract_error( - "shipped proposal retirement archive changed after write-through publication", - )), - None => Err(contract_error( - "shipped proposal retirement archive disappeared after write-through publication", - )), - } -} - -fn publish_archive_bytes(path: &Path, bytes: &[u8]) -> Result<()> { - publish_archive_bytes_with(path, bytes, replace_archive_file) -} - -fn publish_archive_bytes_with( - path: &Path, - bytes: &[u8], - publish: impl FnOnce(&Path, &Path) -> std::io::Result<()>, -) -> Result<()> { - let parent = path - .parent() - .ok_or_else(|| contract_error("shipped proposal archive has no parent directory"))?; - tracedecay_runtime_core::storage::PrivateStoreIo::create_dir_all_durable(parent).map_err( - |error| { - contract_error(format!( - "shipped proposal archive directory creation failed: {error}" - )) - }, - )?; - with_owned_temp_publish( - path, - "shipped-proposal-retirement-archive", - publish, - |output| output.write_all(bytes), - DirectorySyncPolicy::Strict, - ) - .map_err(contract_error) -} - -fn replace_archive_file(temporary: &Path, destination: &Path) -> std::io::Result<()> { - #[cfg(windows)] - { - let temporary_file = - tracedecay_runtime_core::windows_security::make_private_file(temporary)?; - temporary_file.sync_all()?; - drop(temporary_file); - } - - tracedecay_runtime_core::db::DatabaseAuthority::replace_file_atomically( - temporary, - destination, - "shipped proposal retirement archive", - ) - .map_err(std::io::Error::other)?; - - #[cfg(windows)] - tracedecay_runtime_core::windows_security::validate_private_file(destination)?; - - Ok(()) -} - -fn capture_exact_source(path: &Path, expected: &[u8]) -> Result> { - capture_exact_source_with_capture(path, expected, |_| Ok(())) -} - -#[cfg(test)] -fn remove_exact_source(path: &Path, expected: &[u8]) -> Result<()> { - complete_after_pending_removal(&RetirementClosure { - source_path: path.to_path_buf(), - source_digest: encode_tagged_lowercase_hex("sha256:", &Sha256::digest(expected)), - capture_path: capture_exact_source(path, expected)?, - }) -} - -#[cfg(test)] -fn remove_exact_source_with_capture( - path: &Path, - expected: &[u8], - after_capture: impl FnOnce(&Path) -> Result<()>, -) -> Result<()> { - complete_after_pending_removal(&RetirementClosure { - source_path: path.to_path_buf(), - source_digest: encode_tagged_lowercase_hex("sha256:", &Sha256::digest(expected)), - capture_path: capture_exact_source_with_capture(path, expected, after_capture)?, - }) -} - -fn recover_retirement_capture(path: &Path, expected_digest: &str) -> Result>> { - let parent_path = path - .parent() - .ok_or_else(|| contract_error("shipped proposal source has no parent directory"))?; - let expected_raw = hex::decode(canonical_digest_body(expected_digest)?) - .map_err(|error| contract_error(format!("retirement digest decode failed: {error}")))?; - let tombstone_path = capture_path_for_digest(parent_path, &expected_raw); - let Some(captured) = read_retirement_bytes(&tombstone_path, "captured source")? else { - return Ok(None); - }; - if require_digest(&captured, expected_digest).is_err() { - restore_existing_capture(path, &tombstone_path)?; - return Err(contract_error( - "captured shipped proposal source changed before retirement replay", - )); - } - Ok(Some(captured)) -} - -fn restore_existing_capture(source_path: &Path, captured_path: &Path) -> Result<()> { - let parent_path = source_path - .parent() - .ok_or_else(|| contract_error("shipped proposal source has no parent directory"))?; - let source_name = source_path - .file_name() - .ok_or_else(|| contract_error("shipped proposal source has no filename"))?; - let captured_name = captured_path - .file_name() - .ok_or_else(|| contract_error("captured shipped proposal source has no filename"))?; - let lock_path = tracedecay_runtime_core::storage::append_lock_path(source_path); - tracedecay_runtime_core::storage::reject_symlink_components( - &lock_path, - "shipped proposal retirement lock", - ) - .map_err(contract_error)?; - let lock = tracedecay_runtime_core::storage::acquire_sidecar_lock_blocking(&lock_path) - .map_err(|error| { - contract_error(format!("shipped proposal retirement lock failed: {error}")) - })?; - let result = (|| { - let parent = Dir::open_ambient_dir(parent_path, ambient_authority()).map_err(|error| { - contract_error(format!( - "shipped proposal retirement parent open failed: {error}" - )) - })?; - restore_captured_source(&parent, captured_name, source_name, source_path) - })(); - let unlock = lock.unlock().map_err(|error| { - contract_error(format!( - "shipped proposal retirement unlock failed: {error}" - )) - }); - match (result, unlock) { - (Err(error), _) => Err(error), - (Ok(()), Err(error)) => Err(error), - (Ok(()), Ok(())) => Ok(()), - } -} - -fn capture_exact_source_with_capture( - path: &Path, - expected: &[u8], - after_capture: impl FnOnce(&Path) -> Result<()>, -) -> Result> { - let parent_path = path - .parent() - .ok_or_else(|| contract_error("shipped proposal source has no parent directory"))?; - let source_name = path - .file_name() - .ok_or_else(|| contract_error("shipped proposal source has no filename"))?; - let lock_path = tracedecay_runtime_core::storage::append_lock_path(path); - tracedecay_runtime_core::storage::reject_symlink_components( - &lock_path, - "shipped proposal retirement lock", - ) - .map_err(contract_error)?; - let lock = tracedecay_runtime_core::storage::acquire_sidecar_lock_blocking(&lock_path) - .map_err(|error| { - contract_error(format!("shipped proposal retirement lock failed: {error}")) - })?; - let result = (|| { - let parent = Dir::open_ambient_dir(parent_path, ambient_authority()).map_err(|error| { - contract_error(format!( - "shipped proposal retirement parent open failed: {error}" - )) - })?; - let (tombstone_name, tombstone_path) = capture_path(parent_path, expected); - if let Some(captured) = read_retirement_bytes(&tombstone_path, "captured source")? { - if captured == expected { - return Ok(Some(tombstone_path)); - } - restore_captured_source(&parent, &tombstone_name, source_name, path)?; - return Err(contract_error( - "captured shipped proposal source changed before typed retirement capture", - )); - } - match tracedecay_runtime_core::storage::retry_transient_file_op(|| { - rename_noreplace(&parent, source_name, &parent, &tombstone_name) - }) { - Ok(()) => {} - Err(error) if error.kind() == std::io::ErrorKind::NotFound => { - sync_parent_directory(path, DirectorySyncPolicy::Strict).map_err(contract_error)?; - return Ok(None); - } - Err(error) => { - return Err(contract_error(format!( - "shipped proposal source capture failed: {error}" - ))); - } - } - sync_parent_directory(path, DirectorySyncPolicy::Strict).map_err(contract_error)?; - after_capture(&tombstone_path)?; - let captured = match read_retirement_bytes(&tombstone_path, "captured source") { - Ok(Some(captured)) => captured, - Ok(None) => { - return Err(contract_error( - "captured shipped proposal source disappeared", - )); - } - Err(error) => { - restore_captured_source(&parent, &tombstone_name, source_name, path)?; - return Err(error); - } - }; - if captured != expected { - restore_captured_source(&parent, &tombstone_name, source_name, path)?; - return Err(contract_error( - "captured shipped proposal source changed after typed retirement terminal", - )); - } - Ok(Some(tombstone_path)) - })(); - let unlock = lock.unlock().map_err(|error| { - contract_error(format!( - "shipped proposal retirement unlock failed: {error}" - )) - }); - match (result, unlock) { - (Err(error), _) => Err(error), - (Ok(_), Err(error)) => Err(error), - (Ok(value), Ok(())) => Ok(value), - } -} - -fn capture_path(parent: &Path, expected: &[u8]) -> (std::ffi::OsString, PathBuf) { - let digest = Sha256::digest(expected); - let path = capture_path_for_digest(parent, &digest); - let name = std::ffi::OsString::from(format!( - ".fact_proposals.retirement-{}.captured", - hex::encode(digest) - )); - (name, path) -} - -fn capture_path_for_digest(parent: &Path, digest: &[u8]) -> PathBuf { - parent.join(format!( - ".fact_proposals.retirement-{}.captured", - hex::encode(digest) - )) -} - -fn retired_path_for_digest(parent: &Path, digest: &str) -> PathBuf { - parent.join(format!(".fact_proposals.retirement-{digest}.retired")) -} - -fn restore_captured_source( - parent: &Dir, - captured: &OsStr, - source: &OsStr, - source_path: &Path, -) -> Result<()> { - tracedecay_runtime_core::storage::retry_transient_file_op(|| rename_noreplace(parent, captured, parent, source)) - .map_err(|error| { - contract_error(format!( - "captured shipped proposal source conflicts with a replacement and could not be restored: {error}" - )) - })?; - sync_parent_directory(source_path, DirectorySyncPolicy::Strict).map_err(contract_error) -} - -fn read_retirement_bytes(path: &Path, label: &str) -> Result>> { - read_shipped_fact_proposal_bytes(path).map_err(|error| { - contract_error(format!( - "shipped proposal {label} bounded private read failed: {error}" - )) - }) -} - -fn archive_path(dashboard_root: &Path, binding: &RetirementBinding) -> Result { - if binding.archive_name.contains('/') || binding.archive_name.contains('\\') { - return Err(contract_error("retirement archive name is not a basename")); - } - Ok(dashboard_root - .join("fact_proposals.archive") - .join(&binding.archive_name)) -} - -fn validate_binding(binding: &RetirementBinding) -> Result<()> { - let raw = canonical_digest_body(&binding.source_digest)?; - if binding.archive_name != format!("fact_proposals.{raw}.json") { - return Err(contract_error( - "retirement archive basename is not digest-derived", - )); - } - Ok(()) -} - -fn validate_digest(digest: &str) -> Result<()> { - canonical_digest_body(digest).map(|_| ()) -} - -fn canonical_digest_body(digest: &str) -> Result<&str> { - if is_tagged_lowercase_hex(digest, "sha256:", 64) { - Ok(&digest["sha256:".len()..]) - } else { - Err(contract_error("retirement digest is not canonical SHA-256")) - } -} - -fn require_digest(bytes: &[u8], expected: &str) -> Result<()> { - let actual = encode_tagged_lowercase_hex("sha256:", &Sha256::digest(bytes)); - if actual == expected { - Ok(()) - } else { - Err(contract_error( - "shipped proposal bytes do not match admitted retirement digest", - )) - } -} - -fn contract_error(error: impl std::fmt::Display) -> TraceDecayError { - TraceDecayError::Config { - message: format!("shipped proposal retirement is invalid: {error}"), - } -} - -#[cfg(test)] -#[allow(clippy::unwrap_used, clippy::expect_used)] -mod tests { - use super::*; - - fn terminal_shipped_sidecar() -> serde_json::Value { - serde_json::json!({ - "schema_version": 1, - "proposals": [ - { - "schema_version": 1, - "proposal_id": "fact_0123456789abcdef", - "run_id": "run-shipped-sidecar", - "evidence_hash": "shipped-evidence-hash", - "state": "applied", - "proposal": { - "content": "Preserve shipped proposal provenance", - "source_span": {"message_id": "msg-shipped"} - }, - "validation": {"status": "accepted"}, - "applied_fact_id": 42, - "apply_outcome": {"state": "applied", "fact_id": 42}, - "created_at": 1_700_000_000, - "updated_at": 1_700_000_001, - "duplicate_count": 2, - "last_duplicate_run_id": "run-shipped-duplicate", - "folded_contents": ["Earlier wording"] - }, - { - "schema_version": 1, - "proposal_id": "fact_fedcba9876543210", - "run_id": "run-shipped-sidecar", - "state": "rejected", - "proposal": {"content": "Transient rejected item"}, - "validation_reason": "not durable", - "reviewer": "validator", - "created_at": 1_700_000_002, - "updated_at": 1_700_000_003 - } - ] - }) - } - - fn plan(root: &Path, bytes: &[u8]) -> RetirementPlan { - let digest = encode_tagged_lowercase_hex("sha256:", &Sha256::digest(bytes)); - RetirementPlan { - binding: RetirementBinding { - source_digest: digest.clone(), - archive_name: format!( - "fact_proposals.{}.json", - sha256_hex_suffix(&digest).unwrap() - ), - }, - source_path: root.join("fact_proposals.json"), - source_bytes: bytes.to_vec(), - } - } - - fn write_private_file(path: &Path, bytes: &[u8]) { - std::fs::write(path, bytes).unwrap(); - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - - std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)).unwrap(); - } - #[cfg(windows)] - drop(tracedecay_runtime_core::windows_security::make_private_file(path).unwrap()); - } - - #[test] - fn retirement_plan_rejects_unbounded_terminal_bytes() { - let root = tempfile::tempdir().unwrap(); - let source_bytes = vec![b'x'; MAX_SHIPPED_FACT_PROPOSAL_BYTES + 1]; - let source_digest = encode_tagged_lowercase_hex("sha256:", &Sha256::digest(&source_bytes)); - - let error = match classify(ShippedFactProposalDisposition::TerminalHistory { - source_path: root.path().join("fact_proposals.json"), - source_digest, - source_bytes, - }) { - Err(error) => error, - Ok(_) => panic!("an unbounded disposition must not become a live retirement plan"), - }; - - assert!(error.to_string().contains("retirement byte bound")); - } - - #[test] - fn retirement_binding_rejects_repeated_sha256_prefix() { - let raw = "a".repeat(64); - let binding = RetirementBinding { - source_digest: format!("sha256:sha256:{raw}"), - archive_name: format!("fact_proposals.{raw}.json"), - }; - - let error = validate_binding(&binding) - .expect_err("a canonical retirement digest has exactly one algorithm prefix"); - - assert!(error.to_string().contains("canonical SHA-256")); - } - - #[test] - fn typed_terminal_finalizer_archives_then_removes_exact_bytes() { - let root = tempfile::tempdir().unwrap(); - let bytes = br#"{"schema_version":1,"proposals":[]}"#; - let plan = plan(root.path(), bytes); - write_private_file(&plan.source_path, bytes); - - complete_after_pending_removal( - &finalize_after_terminal(root.path(), &plan.binding, Some(&plan)).unwrap(), - ) - .unwrap(); - - assert!(!plan.source_path.exists()); - assert_eq!( - std::fs::read(archive_path(root.path(), &plan.binding).unwrap()).unwrap(), - bytes - ); - } - - #[test] - fn archive_uncertainty_retains_source_until_exact_republication_readback() { - let root = tempfile::tempdir().unwrap(); - let bytes = br#"{"schema_version":1,"proposals":[]}"#; - let plan = plan(root.path(), bytes); - let archive = archive_path(root.path(), &plan.binding).unwrap(); - write_private_file(&plan.source_path, bytes); - - let visible_error = finalize_after_terminal_with_io( - root.path(), - &plan.binding, - Some(&plan), - |path, payload| { - publish_archive_bytes(path, payload)?; - Err(contract_error( - "injected uncertainty after visible archive publication", - )) - }, - |_, _| panic!("uncertain archive publication must retain the source"), - ) - .expect_err("visible archive publication uncertainty must surface"); - assert!(visible_error.to_string().contains("injected uncertainty")); - assert_eq!(std::fs::read(&plan.source_path).unwrap(), bytes); - assert_eq!(std::fs::read(&archive).unwrap(), bytes); - - let redurability_attempted = std::cell::Cell::new(false); - let redurability_error = finalize_after_terminal_with_io( - root.path(), - &plan.binding, - Some(&plan), - |path, payload| { - redurability_attempted.set(true); - assert_eq!(std::fs::read(path).unwrap(), payload); - Err(contract_error( - "injected exact archive redurability failure", - )) - }, - |_, _| panic!("failed exact redurability must retain the source"), - ) - .expect_err("an exact visible archive still requires write-through republication"); - assert!( - redurability_error - .to_string() - .contains("redurability failure") - ); - assert!(redurability_attempted.get()); - assert_eq!(std::fs::read(&plan.source_path).unwrap(), bytes); - - let removal_attempted = std::cell::Cell::new(false); - let readback_error = finalize_after_terminal_with_io( - root.path(), - &plan.binding, - Some(&plan), - |path, payload| { - publish_archive_bytes(path, payload)?; - tracedecay_runtime_core::storage::PrivateStoreIo::remove_file_durable(path) - .map(|_| ()) - .map_err(contract_error) - }, - |_, _| { - removal_attempted.set(true); - Ok(()) - }, - ) - .expect_err("archive disappearance after publication must fail exact readback"); - assert!(readback_error.to_string().contains("disappeared")); - assert!(!removal_attempted.get()); - assert_eq!(std::fs::read(&plan.source_path).unwrap(), bytes); - - finalize_after_terminal_with_io( - root.path(), - &plan.binding, - Some(&plan), - publish_archive_bytes, - |path, expected| { - assert_eq!(std::fs::read(&archive).unwrap(), bytes); - remove_exact_source(path, expected) - }, - ) - .unwrap(); - assert!(!plan.source_path.exists()); - assert_eq!(std::fs::read(&archive).unwrap(), bytes); - } - - #[cfg(windows)] - #[test] - fn production_archive_publisher_creates_a_private_windows_file() { - let root = tempfile::tempdir().unwrap(); - let bytes = br#"{"schema_version":1,"proposals":[]}"#; - let plan = plan(root.path(), bytes); - let archive = archive_path(root.path(), &plan.binding).unwrap(); - - publish_archive_bytes(&archive, bytes).unwrap(); - - tracedecay_runtime_core::windows_security::validate_private_file(&archive).unwrap(); - assert_eq!(std::fs::read(archive).unwrap(), bytes); - } - - #[cfg(unix)] - #[test] - fn archive_publisher_stages_a_mode_0600_unix_temp_file() { - use std::os::unix::fs::PermissionsExt; - - let root = tempfile::tempdir().unwrap(); - let bytes = br#"{"schema_version":1,"proposals":[]}"#; - let plan = plan(root.path(), bytes); - let archive = archive_path(root.path(), &plan.binding).unwrap(); - let observed_mode = std::cell::Cell::new(None); - - publish_archive_bytes_with(&archive, bytes, |temporary, destination| { - observed_mode.set(Some( - std::fs::metadata(temporary)?.permissions().mode() & 0o777, - )); - replace_archive_file(temporary, destination) - }) - .unwrap(); - - assert_eq!(observed_mode.get(), Some(0o600)); - assert_eq!(std::fs::read(archive).unwrap(), bytes); - } - - #[test] - fn replay_recovers_when_source_was_removed_after_archive() { - let root = tempfile::tempdir().unwrap(); - let bytes = br#"{"schema_version":1,"proposals":[]}"#; - let plan = plan(root.path(), bytes); - let archive = archive_path(root.path(), &plan.binding).unwrap(); - publish_archive_bytes(&archive, bytes).unwrap(); - - complete_after_pending_removal( - &finalize_after_terminal(root.path(), &plan.binding, None).unwrap(), - ) - .unwrap(); - - assert!(!plan.source_path.exists()); - assert_eq!(std::fs::read(&archive).unwrap(), bytes); - } - - #[test] - fn atomic_capture_never_deletes_a_post_capture_replacement() { - let root = tempfile::tempdir().unwrap(); - let admitted = br#"{"schema_version":1,"proposals":[]}"#; - let replacement = br#"{"schema_version":1,"proposals":[{"state":"pending_approval"}]}"#; - let plan = plan(root.path(), admitted); - write_private_file(&plan.source_path, admitted); - - remove_exact_source_with_capture(&plan.source_path, admitted, |captured| { - assert_eq!( - read_retirement_bytes(captured, "test capture")?, - Some(admitted.to_vec()) - ); - write_private_file(&plan.source_path, replacement); - Ok(()) - }) - .unwrap(); - - assert_eq!( - read_retirement_bytes(&plan.source_path, "replacement").unwrap(), - Some(replacement.to_vec()) - ); - let captured = std::fs::read_dir(root.path()) - .unwrap() - .filter_map(std::result::Result::ok) - .filter(|entry| { - entry - .file_name() - .to_string_lossy() - .starts_with(".fact_proposals.retirement-") - }) - .count(); - assert_eq!(captured, 0); - } - - #[test] - fn exact_captured_source_is_recovered_after_commit_uncertainty() { - let root = tempfile::tempdir().unwrap(); - let admitted = br#"{"schema_version":1,"proposals":[]}"#; - let plan = plan(root.path(), admitted); - write_private_file(&plan.source_path, admitted); - - let error = remove_exact_source_with_capture(&plan.source_path, admitted, |_| { - Err(contract_error( - "injected uncertainty after durable source capture", - )) - }) - .expect_err("capture uncertainty must remain visible"); - assert!(error.to_string().contains("injected uncertainty")); - assert!(!plan.source_path.exists()); - - remove_exact_source(&plan.source_path, admitted).unwrap(); - - assert!(!plan.source_path.exists()); - assert_eq!( - std::fs::read_dir(root.path()) - .unwrap() - .filter_map(std::result::Result::ok) - .filter(|entry| { - entry - .file_name() - .to_string_lossy() - .starts_with(".fact_proposals.retirement-") - }) - .count(), - 0 - ); - } - - #[test] - fn exact_capture_recovery_preserves_a_post_capture_replacement() { - let root = tempfile::tempdir().unwrap(); - let admitted = br#"{"schema_version":1,"proposals":[]}"#; - let replacement = br#"{"schema_version":1,"proposals":[{"state":"pending_approval"}]}"#; - let plan = plan(root.path(), admitted); - write_private_file(&plan.source_path, admitted); - - remove_exact_source_with_capture(&plan.source_path, admitted, |_| { - write_private_file(&plan.source_path, replacement); - Err(contract_error( - "injected crash after replacement followed durable capture", - )) - }) - .expect_err("the simulated crash leaves exact captured bytes for retry"); - - publish_archive_bytes(&archive_path(root.path(), &plan.binding).unwrap(), admitted) - .unwrap(); - complete_after_pending_removal( - &finalize_after_terminal(root.path(), &plan.binding, Some(&plan)).unwrap(), - ) - .unwrap(); - - assert_eq!( - read_retirement_bytes(&plan.source_path, "replacement").unwrap(), - Some(replacement.to_vec()) - ); - assert_eq!(retirement_capture_count(root.path()), 0); - } - - #[test] - fn pending_cleanup_boundary_retains_witness_and_retry_preserves_replacement() { - let root = tempfile::tempdir().unwrap(); - let admitted = br#"{"schema_version":1,"proposals":[]}"#; - let replacement = br#"{"schema_version":1,"proposals":[{"state":"pending_approval"}]}"#; - let plan = plan(root.path(), admitted); - write_private_file(&plan.source_path, admitted); - - let uncompleted = finalize_after_terminal(root.path(), &plan.binding, Some(&plan)).unwrap(); - assert!(!plan.source_path.exists()); - assert_eq!(retirement_capture_count(root.path()), 1); - - write_private_file(&plan.source_path, replacement); - let recovered = finalize_after_terminal(root.path(), &plan.binding, None).unwrap(); - assert_eq!( - read_retirement_bytes(&plan.source_path, "replacement").unwrap(), - Some(replacement.to_vec()) - ); - assert_eq!(retirement_capture_count(root.path()), 1); - - drop(uncompleted); - complete_after_pending_removal(&recovered).unwrap(); - assert_eq!(retirement_capture_count(root.path()), 0); - assert_eq!( - read_retirement_bytes(&plan.source_path, "replacement").unwrap(), - Some(replacement.to_vec()) - ); - } - - #[test] - fn durable_archive_replay_still_captures_the_exact_live_source() { - let root = tempfile::tempdir().unwrap(); - let admitted = br#"{"schema_version":1,"proposals":[]}"#; - let plan = plan(root.path(), admitted); - write_private_file(&plan.source_path, admitted); - publish_archive_bytes(&archive_path(root.path(), &plan.binding).unwrap(), admitted) - .unwrap(); - - let closure = finalize_after_terminal(root.path(), &plan.binding, None).unwrap(); - - assert!(!plan.source_path.exists()); - assert_eq!(retirement_capture_count(root.path()), 1); - complete_after_pending_removal(&closure).unwrap(); - assert_eq!(retirement_capture_count(root.path()), 0); - assert!(!plan.source_path.exists()); - } - - #[test] - fn unbound_capture_is_never_retired_even_with_an_exact_archive() { - let root = tempfile::tempdir().unwrap(); - let admitted = br#"{"schema_version":1,"proposals":[]}"#; - let replacement = br#"{"schema_version":1,"proposals":[{"state":"pending_approval"}]}"#; - let plan = plan(root.path(), admitted); - let (_, captured_path) = capture_path(root.path(), admitted); - let archive = archive_path(root.path(), &plan.binding).unwrap(); - write_private_file(&captured_path, admitted); - write_private_file(&plan.source_path, replacement); - - let missing = reject_unbound_retirement_witness(root.path()) - .expect_err("an unbound capture without its journal marker must fail closed"); - assert!( - missing - .to_string() - .contains("no durable journal transition") - ); - assert_eq!( - read_retirement_bytes(&captured_path, "capture").unwrap(), - Some(admitted.to_vec()) - ); - assert_eq!( - read_retirement_bytes(&plan.source_path, "replacement").unwrap(), - Some(replacement.to_vec()) - ); - - std::fs::create_dir_all(archive.parent().unwrap()).unwrap(); - write_private_file(&archive, replacement); - let mismatch = reject_unbound_retirement_witness(root.path()) - .expect_err("an unbound capture with a mismatched archive must remain untouched"); - assert!( - mismatch - .to_string() - .contains("no durable journal transition") - ); - assert_eq!( - read_retirement_bytes(&captured_path, "capture").unwrap(), - Some(admitted.to_vec()) - ); - assert_eq!( - read_retirement_bytes(&plan.source_path, "replacement").unwrap(), - Some(replacement.to_vec()) - ); - - write_private_file(&archive, admitted); - reject_unbound_retirement_witness(root.path()) - .expect_err("even exact bytes require journal-bound transition authority"); - assert!(captured_path.exists()); - assert_eq!( - read_retirement_bytes(&plan.source_path, "replacement").unwrap(), - Some(replacement.to_vec()) - ); - } - - #[test] - fn malformed_orphaned_capture_name_is_never_removed() { - let root = tempfile::tempdir().unwrap(); - let malformed = root - .path() - .join(".fact_proposals.retirement-not-a-digest.captured"); - write_private_file(&malformed, b"captured"); - - let error = reject_unbound_retirement_witness(root.path()) - .expect_err("a malformed capture name must fail closed"); - - assert!(error.to_string().contains("canonical SHA-256")); - assert_eq!(std::fs::read(&malformed).unwrap(), b"captured"); - } - - #[test] - fn retired_witness_uncertainty_replays_to_durable_absence_and_preserves_replacement() { - let root = tempfile::tempdir().unwrap(); - let admitted = br#"{"schema_version":1,"proposals":[]}"#; - let replacement = br#"{"schema_version":1,"proposals":[{"state":"pending_approval"}]}"#; - let plan = plan(root.path(), admitted); - write_private_file(&plan.source_path, admitted); - let closure = finalize_after_terminal(root.path(), &plan.binding, Some(&plan)).unwrap(); - write_private_file(&plan.source_path, replacement); - - let error = complete_after_pending_removal_with(&closure, |retired| { - assert_eq!( - read_retirement_bytes(retired, "retired uncertainty witness")?, - Some(admitted.to_vec()) - ); - Err(contract_error( - "injected retired witness deletion uncertainty", - )) - }) - .expect_err("retired witness uncertainty must remain recoverable"); - assert!(error.to_string().contains("deletion uncertainty")); - assert_eq!(retirement_capture_count(root.path()), 1); - - let recovered = closure_for_durable_transition(root.path(), &plan.binding, true).unwrap(); - complete_after_pending_removal(&recovered).unwrap(); - - assert_eq!(retirement_capture_count(root.path()), 0); - assert_eq!( - read_retirement_bytes(&plan.source_path, "replacement").unwrap(), - Some(replacement.to_vec()) - ); - } - - #[test] - fn mismatched_capture_recovery_restores_the_captured_replacement() { - let root = tempfile::tempdir().unwrap(); - let admitted = br#"{"schema_version":1,"proposals":[]}"#; - let replacement = br#"{"schema_version":1,"proposals":[{"state":"pending_approval"}]}"#; - let plan = plan(root.path(), admitted); - let (_, captured_path) = capture_path(root.path(), admitted); - write_private_file(&captured_path, replacement); - - let error = remove_exact_source(&plan.source_path, admitted) - .expect_err("a captured replacement must be restored, never deleted"); - - assert!( - error - .to_string() - .contains("changed before typed retirement capture") - ); - assert_eq!( - read_retirement_bytes(&plan.source_path, "restored replacement").unwrap(), - Some(replacement.to_vec()) - ); - assert_eq!(retirement_capture_count(root.path()), 0); - } - - #[test] - fn exact_capture_without_archive_stays_recoverable() { - let root = tempfile::tempdir().unwrap(); - let admitted = br#"{"schema_version":1,"proposals":[]}"#; - let plan = plan(root.path(), admitted); - let (_, captured_path) = capture_path(root.path(), admitted); - write_private_file(&captured_path, admitted); - - let error = finalize_after_terminal(root.path(), &plan.binding, Some(&plan)) - .expect_err("captured bytes alone cannot prove an exact durable archive"); - - assert!(error.to_string().contains("exact durable archive")); - assert_eq!(retirement_capture_count(root.path()), 1); - assert_eq!( - read_retirement_bytes(&captured_path, "retained capture").unwrap(), - Some(admitted.to_vec()) - ); - assert!(!plan.source_path.exists()); - } - - #[test] - fn finalizer_replay_restores_a_mismatched_capture_to_an_absent_source() { - let root = tempfile::tempdir().unwrap(); - let admitted = br#"{"schema_version":1,"proposals":[]}"#; - let replacement = br#"{"schema_version":1,"proposals":[{"state":"pending_approval"}]}"#; - let plan = plan(root.path(), admitted); - let (_, captured_path) = capture_path(root.path(), admitted); - write_private_file(&captured_path, replacement); - - let error = finalize_after_terminal(root.path(), &plan.binding, Some(&plan)) - .expect_err("replay must restore a mismatched capture before reporting conflict"); - - assert!( - error - .to_string() - .contains("changed before retirement replay") - ); - assert_eq!( - read_retirement_bytes(&plan.source_path, "restored replacement").unwrap(), - Some(replacement.to_vec()) - ); - assert_eq!(retirement_capture_count(root.path()), 0); - } - - #[test] - fn finalizer_replay_never_replaces_a_recreated_source() { - let root = tempfile::tempdir().unwrap(); - let admitted = br#"{"schema_version":1,"proposals":[]}"#; - let captured_replacement = - br#"{"schema_version":1,"proposals":[{"state":"pending_approval"}]}"#; - let recreated_source = br#"{"schema_version":1,"proposals":[{"state":"applied"}]}"#; - let plan = plan(root.path(), admitted); - let (_, captured_path) = capture_path(root.path(), admitted); - write_private_file(&captured_path, captured_replacement); - write_private_file(&plan.source_path, recreated_source); - - let error = finalize_after_terminal(root.path(), &plan.binding, Some(&plan)) - .expect_err("replay must not replace an occupied source destination"); - - assert!(error.to_string().contains("conflicts with a replacement")); - assert_eq!( - read_retirement_bytes(&plan.source_path, "recreated source").unwrap(), - Some(recreated_source.to_vec()) - ); - assert_eq!( - read_retirement_bytes(&captured_path, "retained captured replacement").unwrap(), - Some(captured_replacement.to_vec()) - ); - } - - #[test] - fn occupied_digest_tombstone_blocks_capture_without_mutation() { - let root = tempfile::tempdir().unwrap(); - let admitted = br#"{"schema_version":1,"proposals":[]}"#; - let tombstone_blocker = - br#"{"schema_version":1,"proposals":[{"state":"pending_approval"}]}"#; - let plan = plan(root.path(), admitted); - let (_, captured_path) = capture_path(root.path(), admitted); - write_private_file(&plan.source_path, admitted); - write_private_file(&captured_path, tombstone_blocker); - - let error = remove_exact_source(&plan.source_path, admitted) - .expect_err("an occupied digest tombstone must never be replaced"); - - assert!(error.to_string().contains("conflicts with a replacement")); - assert_eq!( - read_retirement_bytes(&plan.source_path, "admitted source").unwrap(), - Some(admitted.to_vec()) - ); - assert_eq!( - read_retirement_bytes(&captured_path, "tombstone blocker").unwrap(), - Some(tombstone_blocker.to_vec()) - ); - } - - fn retirement_capture_count(root: &Path) -> usize { - std::fs::read_dir(root) - .unwrap() - .filter_map(std::result::Result::ok) - .filter(|entry| { - entry - .file_name() - .to_string_lossy() - .starts_with(".fact_proposals.retirement-") - }) - .count() - } - - #[cfg(unix)] - #[test] - fn live_retirement_rejects_a_symlink_source_before_archive_publication() { - use std::os::unix::fs::symlink; - - let root = tempfile::tempdir().unwrap(); - let bytes = br#"{"schema_version":1,"proposals":[]}"#; - let plan = plan(root.path(), bytes); - let target = root.path().join("outside-source.json"); - write_private_file(&target, bytes); - symlink(&target, &plan.source_path).unwrap(); - let archive = archive_path(root.path(), &plan.binding).unwrap(); - - let error = finalize_after_terminal(root.path(), &plan.binding, Some(&plan)) - .expect_err("a changed source symlink must fail before retirement publication"); - - assert!(error.to_string().contains("bounded private read failed")); - assert!(!archive.exists()); - assert_eq!(std::fs::read(target).unwrap(), bytes); - assert!( - std::fs::symlink_metadata(&plan.source_path) - .unwrap() - .file_type() - .is_symlink() - ); - } - - #[cfg(unix)] - #[test] - fn retirement_replay_rejects_a_symlink_archive_without_reading_target() { - use std::os::unix::fs::symlink; - - let root = tempfile::tempdir().unwrap(); - let bytes = br#"{"schema_version":1,"proposals":[]}"#; - let plan = plan(root.path(), bytes); - let archive = archive_path(root.path(), &plan.binding).unwrap(); - std::fs::create_dir_all(archive.parent().unwrap()).unwrap(); - let target = root.path().join("outside-archive.json"); - write_private_file(&target, bytes); - symlink(&target, &archive).unwrap(); - - let error = finalize_after_terminal(root.path(), &plan.binding, None) - .expect_err("retirement replay must never follow an archive symlink"); - - assert!(error.to_string().contains("bounded private read failed")); - assert_eq!(std::fs::read(target).unwrap(), bytes); - assert!( - std::fs::symlink_metadata(archive) - .unwrap() - .file_type() - .is_symlink() - ); - } - - #[test] - fn retirement_replay_rejects_an_oversized_sparse_archive_before_digest() { - let root = tempfile::tempdir().unwrap(); - let bytes = br#"{"schema_version":1,"proposals":[]}"#; - let plan = plan(root.path(), bytes); - let archive = archive_path(root.path(), &plan.binding).unwrap(); - std::fs::create_dir_all(archive.parent().unwrap()).unwrap(); - write_private_file(&archive, b""); - std::fs::OpenOptions::new() - .write(true) - .open(&archive) - .unwrap() - .set_len(MAX_SHIPPED_FACT_PROPOSAL_BYTES as u64 + 1) - .unwrap(); - - let error = finalize_after_terminal(root.path(), &plan.binding, None) - .expect_err("an oversized replay archive must fail before digest or publication"); - - assert!(error.to_string().contains("byte limit")); - assert_eq!( - std::fs::metadata(archive).unwrap().len(), - MAX_SHIPPED_FACT_PROPOSAL_BYTES as u64 + 1 - ); - } - - #[test] - fn changed_source_is_never_removed() { - let root = tempfile::tempdir().unwrap(); - let original = br#"{"schema_version":1,"proposals":[]}"#; - let changed = br#"{"schema_version":1,"proposals":[{}]}"#; - let plan = plan(root.path(), original); - write_private_file(&plan.source_path, changed); - - assert!(finalize_after_terminal(root.path(), &plan.binding, None).is_err()); - assert_eq!(std::fs::read(&plan.source_path).unwrap(), changed); - } - - #[tokio::test] - async fn memory_curator_leaves_terminal_sidecar_for_session_reflector_retirement() { - let root = tempfile::tempdir().unwrap(); - let source_path = root.path().join("fact_proposals.json"); - let source_bytes = serde_json::to_vec_pretty(&terminal_shipped_sidecar()).unwrap(); - write_private_file(&source_path, &source_bytes); - - let curator = classify_for_task( - tracedecay_contracts::retained_surfaces::AutomationTaskV1::MemoryCurator, - root.path(), - ) - .await - .unwrap(); - assert!(matches!(curator, RetirementClassification::Absent)); - assert_eq!(tokio::fs::read(&source_path).await.unwrap(), source_bytes); - - let reflector = classify_for_task( - tracedecay_contracts::retained_surfaces::AutomationTaskV1::SessionReflector, - root.path(), - ) - .await - .unwrap(); - assert!(matches!(reflector, RetirementClassification::Terminal(_))); - assert_eq!(tokio::fs::read(&source_path).await.unwrap(), source_bytes); - } -} diff --git a/crates/tracedecay-automation-runtime/src/automation/effect_runtime/settlement.rs b/crates/tracedecay-automation-runtime/src/automation/effect_runtime/settlement.rs index ce9b0eb478..2a7735cfc8 100644 --- a/crates/tracedecay-automation-runtime/src/automation/effect_runtime/settlement.rs +++ b/crates/tracedecay-automation-runtime/src/automation/effect_runtime/settlement.rs @@ -36,11 +36,11 @@ use crate::automation::effect_runtime::journal::{ classify_durable_settlement_blocking, persist_prepared_terminal_blocking, persist_recovered_terminal_blocking, persist_terminal_blocking, promote_prepared_terminal_blocking, replay_exact_binding_after_error_blocking, - reserve_or_replay_indexed_blocking, retained_source_bindings, + reserve_or_replay_indexed_blocking, }; use crate::automation::effect_runtime::problem::{ failed_ledger_problem, indeterminate_external_effect_problem, reset_required_problem, - runtime_problem, shipped_proposal_reset_required_problem, + runtime_problem, }; use crate::automation::effect_runtime::projection::{ project_committed_receipts, project_recovered_committed_receipts, project_run_summary, @@ -50,7 +50,6 @@ use crate::automation::effect_runtime::{ AutomationSettledProblem, AutomationSettledTerminal, add_pending_blocking, contract_error, digest, effect_authority_digest as calculate_effect_authority_digest, finalize_terminal_housekeeping, journal, recovered_partial_terminal, remove_pending_blocking, - retirement, }; use crate::automation::run_ledger::{ self, AutomationRunLedgerRecord, AutomationRunStatus, AutomationTrigger, ExactRunPublication, @@ -453,7 +452,6 @@ fn ledger_record_matches_result( } AutomationRunTerminalV1::Skipped { reason, .. } => { record.status == AutomationRunStatus::Skipped - && record.error == record.fallback_status && record .error .as_deref() @@ -1065,50 +1063,6 @@ impl AutomationEffectAuthority { journal_key.as_str().trim_start_matches("sha256:") )); let task = request.task_kind(); - let (retained_binding, retained_reset_digest) = - if task == AutomationTaskV1::SessionReflector { - let binding_path = journal_path.clone(); - tokio::task::spawn_blocking(move || retained_source_bindings(&binding_path)) - .await - .map_err(|error| { - contract_error(format!( - "automation retirement binding reader failed: {error}" - )) - })?? - } else { - (None, None) - }; - let classification = if retained_binding.is_some() { - // A durable retirement binding owns the exact admitted source. - // Current shipped bytes may be a later replacement and must not - // override replay/recovery of that retained authority. - retirement::RetirementClassification::Absent - } else { - retirement::classify_for_task(task, &dashboard_root).await? - }; - let (live_retirement, shipped_reset) = match classification { - retirement::RetirementClassification::Absent => (None, None), - retirement::RetirementClassification::ResetRequired { - source_digest, - detail, - } => (None, Some((source_digest, detail))), - retirement::RetirementClassification::Terminal(plan) => (Some(plan), None), - }; - if let (Some(plan), Some(binding)) = (&live_retirement, &retained_binding) { - retirement::verify_plan_matches_binding(plan, binding)?; - } - let retirement_binding = - retained_binding.or_else(|| live_retirement.as_ref().map(|plan| plan.binding.clone())); - let reset_source_digest = match (retained_reset_digest, shipped_reset.as_ref()) { - (Some(stored), Some((live, _))) if stored != *live => { - return Err(contract_error( - "unresolved shipped proposal bytes changed after durable admission", - )); - } - (Some(stored), _) => Some(stored), - (None, Some((live, _))) => Some(live.clone()), - (None, None) => None, - }; let retained_operation = RetainedSurfaceOperation::FactStoreCurate; let operation = retained_surface_application_operation(retained_operation).map_err(contract_error)?; @@ -1119,8 +1073,6 @@ impl AutomationEffectAuthority { context.scope(), &configuration_digest, &request, - &retirement_binding, - &reset_source_digest, ))?; let execution_cancellation = cancellation.clone(); let execution = RetainedSurfaceExecutionContextV1 { @@ -1133,7 +1085,7 @@ impl AutomationEffectAuthority { &execution, retained_operation, &configuration_digest, - &(&request, &retirement_binding, &reset_source_digest), + &request, request.run_id.as_str(), ) .map_err(|_| contract_error("canonical memory automation effect preparation failed"))?; @@ -1167,15 +1119,8 @@ impl AutomationEffectAuthority { AutomationRecoveryBinding::Memory { owner: memory_owner()?, recovery_problem, - retirement: retirement_binding, - reset_source_digest, } } else { - if retirement_binding.is_some() || reset_source_digest.is_some() { - return Err(contract_error( - "external automation admission carried memory retirement state", - )); - } AutomationRecoveryBinding::External { recovery_problem } }; let effect_authority_digest = calculate_effect_authority_digest( @@ -1261,9 +1206,7 @@ impl AutomationEffectAuthority { ReservationResult::Replay { terminal, publication, - retirement, } => { - validate_retirement_binding(&admission, retirement.as_ref())?; if let Some(publication) = publication.as_ref() { let published = run_ledger::publish_staged_run_record_exact( &dashboard_root, @@ -1296,18 +1239,10 @@ impl AutomationEffectAuthority { ) .await?; } - finalize_terminal_housekeeping( - &dashboard_root, - &journal_path, - &admission, - &terminal, - live_retirement, - ) - .await?; + finalize_terminal_housekeeping(&dashboard_root, &journal_path).await?; Ok(AutomationEffectAdmission::Replay(Box::new(terminal))) } - ReservationResult::Execute { claim, retirement } => { - validate_retirement_binding(&admission, retirement.as_ref())?; + ReservationResult::Execute { claim } => { let authority = Self { context, cancellation: cancellation.clone(), @@ -1318,40 +1253,9 @@ impl AutomationEffectAuthority { dashboard_root: dashboard_root.clone(), _reservation_claim: Some(claim), }; - if let Some((_digest, _detail)) = shipped_reset { - let problem = shipped_proposal_reset_required_problem( - &authority.operation, - &authority.context, - &authority.admission.request, - )?; - let terminal = authority - .persist_terminal(AutomationSettledTerminal::Problem(problem)) - .await?; - finalize_terminal_housekeeping( - &dashboard_root, - &authority.journal_path, - &authority.admission, - &terminal, - None, - ) - .await?; - Ok(AutomationEffectAdmission::Replay(Box::new(terminal))) - } else if retirement.is_some() { - let terminal = authority.settle_retirement().await?; - finalize_terminal_housekeeping( - &dashboard_root, - &authority.journal_path, - &authority.admission, - &terminal, - live_retirement, - ) - .await?; - Ok(AutomationEffectAdmission::Replay(Box::new(terminal))) - } else { - Ok(AutomationEffectAdmission::Execute(Box::new(authority))) - } + Ok(AutomationEffectAdmission::Execute(Box::new(authority))) } - ReservationResult::Recover { retirement } => { + ReservationResult::Recover => { discard_direct_recovery_unbound_spools(&dashboard_root, &journal_path, &admission) .await?; let authority = Self { @@ -1370,14 +1274,8 @@ impl AutomationEffectAuthority { authority.admission.recovery_problem().clone(), )) .await?; - finalize_terminal_housekeeping( - &dashboard_root, - &authority.journal_path, - &authority.admission, - &terminal, - None, - ) - .await?; + finalize_terminal_housekeeping(&dashboard_root, &authority.journal_path) + .await?; let admission = AutomationEffectAdmission::Replay(Box::new(terminal)); observe_admission_decision(&admission); return Ok(admission); @@ -1395,14 +1293,7 @@ impl AutomationEffectAuthority { })?; let committed_receipts = project_recovered_committed_receipts(&authority.admission.request, &recovered)?; - if retirement.is_some() && !committed_receipts.is_empty() { - return Err(contract_error( - "proposal retirement recovery found unrelated canonical memory commits", - )); - } - let terminal = if retirement.is_some() { - authority.settle_recovered_retirement().await? - } else if !committed_receipts.is_empty() { + let terminal = if !committed_receipts.is_empty() { authority .persist_recovered_terminal(recovered_partial_terminal( &authority.admission, @@ -1410,42 +1301,19 @@ impl AutomationEffectAuthority { &authority.operation, )?) .await? - } else if authority.admission.reset_source_digest().is_some() { - let problem = shipped_proposal_reset_required_problem( - &authority.operation, - &authority.context, - &authority.admission.request, - )?; - authority - .persist_recovered_terminal(AutomationSettledTerminal::Problem(problem)) - .await? } else { let terminal = AutomationSettledTerminal::Problem( authority.admission.recovery_problem().clone(), ); authority.persist_recovered_terminal(terminal).await? }; - validate_retirement_binding(&authority.admission, retirement.as_ref())?; - finalize_terminal_housekeeping( - &dashboard_root, - &authority.journal_path, - &authority.admission, - &terminal, - live_retirement, - ) - .await?; + finalize_terminal_housekeeping(&dashboard_root, &authority.journal_path).await?; Ok(AutomationEffectAdmission::Replay(Box::new(terminal))) } ReservationResult::RecoverPrepared { terminal, publication, - retirement, } => { - if retirement.is_some() || terminal.is_retirement_terminal() { - return Err(contract_error( - "proposal retirement cannot carry a prepared run publication", - )); - } let authority = Self { context, cancellation: cancellation.clone(), @@ -1515,15 +1383,18 @@ impl AutomationEffectAuthority { )?, }, AutomationRunStatus::Skipped => { - if ledger.error != ledger.fallback_status { + let reason = project_skip_reason(ledger.error.as_deref().ok_or_else(|| { + contract_error("skipped automation terminal has no exact reason") + })?)?; + if (reason == AutomationSkipReasonV1::SessionEvidenceBudgetExhausted) + != ledger.session_evidence_budget_stage.is_some() + { return Err(contract_error( - "skipped automation ledger reason disagrees with its fallback status", + "a budget-exhausted skip must carry exactly its exhausted stage", )); } AutomationRunTerminalV1::Skipped { - reason: project_skip_reason(ledger.error.as_deref().ok_or_else(|| { - contract_error("skipped automation terminal has no exact reason") - })?)?, + reason, summary: AutomationRunSummaryV1 { reviewed_count: 0, accepted_count: 0, @@ -1564,53 +1435,6 @@ impl AutomationEffectAuthority { self.success_terminal(result) } - #[hotpath::skip] - async fn settle_retirement(&self) -> Result { - self.persist_success_result(self.retirement_result()?).await - } - - #[hotpath::skip] - async fn settle_recovered_retirement(&self) -> Result { - let terminal = self.success_terminal(self.retirement_result()?)?; - self.persist_recovered_terminal(terminal).await - } - - fn retirement_result(&self) -> Result { - let reason = - AutomationSkipReasonV1::from_ledger_reason("shipped_fact_proposal_history_retired") - .ok_or_else(|| { - contract_error("shipped proposal retirement reason is not registered") - })?; - Ok(AutomationRunResultV1 { - run_id: self.admission.request.run_id.clone(), - task: self.admission.request.task_kind(), - request_digest: self - .admission - .request - .input_digest() - .map_err(contract_error)?, - terminal: AutomationRunTerminalV1::Skipped { - reason, - summary: AutomationRunSummaryV1 { - reviewed_count: 0, - accepted_count: 0, - rejected_count: 0, - skipped_count: 1, - }, - }, - committed_receipts: Vec::new(), - }) - } - - #[hotpath::skip] - async fn persist_success_result( - &self, - result: AutomationRunResultV1, - ) -> Result { - let terminal = self.success_terminal(result)?; - self.persist_terminal(terminal).await - } - fn success_terminal(&self, result: AutomationRunResultV1) -> Result { if !result.matches_terminal() { return Err(contract_error( @@ -1743,7 +1567,6 @@ impl AutomationEffectAuthority { || prior_task_key != reused.task_key || reused.prior_record.trigger != AutomationTrigger::Scheduler || reused.prior_record.status != AutomationRunStatus::Skipped - || reused.prior_record.error != reused.prior_record.fallback_status || reused.prior_record.error.as_deref() != Some(reused.reason.as_str()) || reused.reason.is_empty() { @@ -1833,20 +1656,6 @@ impl AutomationEffectAuthority { .map_err(contract_error) } - #[hotpath::skip] - async fn persist_terminal( - &self, - terminal: AutomationSettledTerminal, - ) -> Result { - let path = self.journal_path.clone(); - let admission = self.admission.clone(); - tokio::task::spawn_blocking(move || persist_terminal_blocking(&path, &admission, terminal)) - .await - .map_err(|error| { - contract_error(format!("automation terminal writer failed: {error}")) - })? - } - #[hotpath::skip] async fn promote_prepared_terminal( &self, @@ -1882,14 +1691,7 @@ impl AutomationEffectAuthority { &publication, ) .await?; - finalize_terminal_housekeeping( - &self.dashboard_root, - &self.journal_path, - &self.admission, - &terminal, - None, - ) - .await?; + finalize_terminal_housekeeping(&self.dashboard_root, &self.journal_path).await?; Ok(terminal) } @@ -2382,21 +2184,8 @@ fn reservation_conflict_admission( _journal_path: &Path, _terminal: bool, ) -> AutomationEffectAdmission { - // A conflicting caller does not own the existing terminal's retirement or - // staged-publication cleanup proof. Project recovery may retire an exact - // stale index entry, but this admission must not erase that authority. + // A conflicting caller does not own the existing terminal's staged- + // publication cleanup proof. Project recovery may retire an exact stale + // index entry, but this admission must not erase that authority. AutomationEffectAdmission::Conflict } - -fn validate_retirement_binding( - admission: &DurableAutomationAdmission, - classified: Option<&retirement::RetirementBinding>, -) -> Result<()> { - if admission.retirement() == classified { - Ok(()) - } else { - Err(contract_error( - "automation retirement classification changed after durable admission", - )) - } -} diff --git a/crates/tracedecay-automation-runtime/src/automation/effect_runtime/settlement/tests.rs b/crates/tracedecay-automation-runtime/src/automation/effect_runtime/settlement/tests.rs index 946a41543e..14f4af0c9d 100644 --- a/crates/tracedecay-automation-runtime/src/automation/effect_runtime/settlement/tests.rs +++ b/crates/tracedecay-automation-runtime/src/automation/effect_runtime/settlement/tests.rs @@ -39,6 +39,10 @@ impl crate::automation::backend::AgentTaskBackend for NeverAutomationBackend { > { panic!("disabled retained automation must not invoke its backend") } + + fn executable(&self) -> Option<&std::path::Path> { + None + } } use tracedecay_domain::test_fixtures::digest; @@ -141,8 +145,6 @@ fn admission(run_id: &str, request_id: &str) -> DurableAutomationAdmission { project_id: scope.project_id.clone(), }, recovery_problem: reset_problem(&request_id, &scope, &request), - retirement: None, - reset_source_digest: None, }, }) } @@ -195,8 +197,6 @@ fn session_reflector_admission(run_id: &str, request_id: &str) -> DurableAutomat project_id: admission.scope.project_id.clone(), }, recovery_problem: reset_problem(&admission.request_id, &admission.scope, &request), - retirement: None, - reset_source_digest: None, }; admission.request = request; seal_effect_authority(admission) @@ -683,7 +683,6 @@ fn session_evidence_timeout_ledger_projects_a_typed_terminal() { "rejected_count": 0, "skipped_count": 1, "error": "session_evidence_timed_out", - "fallback_status": "session_evidence_timed_out", "started_at": "100", "completed_at": "100", "completed_at_micros": 100_000_000, @@ -732,39 +731,48 @@ fn durable_admission_accepts_distinct_run_and_retained_effect_input_digests() { } #[test] -fn legacy_terminal_wire_shape_migrates_without_losing_replay() { +fn inline_terminal_wire_shape_is_rejected_without_rewrite() { let temp = tempfile::tempdir().expect("tempdir"); - let path = temp.path().join("legacy-terminal.json"); - let admitted = admission("run.legacy-journal", "request.legacy-journal"); - let terminal = success_terminal(&admitted, "run.legacy-journal"); - let legacy = json!({ - "admission": admitted, - "state": { - "state": "terminal", - "terminal": terminal, - }, - }); - std::fs::write( - &path, - serde_json::to_vec_pretty(&legacy).expect("legacy bytes"), - ) - .expect("legacy journal"); - - let requested = admission("run.legacy-journal", "request.legacy-journal"); + let current = temp.path().join("current-terminal.json"); + let admitted = admission("run.inline-journal", "request.inline-journal"); + let terminal = success_terminal(&admitted, "run.inline-journal"); + assert!(matches!( + reserve_or_replay_blocking(¤t, admitted.clone()).expect("reserve"), + ReservationResult::Execute { .. } + )); + persist_terminal_blocking(¤t, &admitted, terminal.clone()).expect("persist terminal"); + let mut journal: serde_json::Value = + serde_json::from_slice(&std::fs::read(¤t).expect("current journal")) + .expect("current journal json"); + assert_eq!(journal["state"]["state"], "terminal"); + assert_eq!(journal["state"]["value"]["terminal"]["schema_version"], 1); let ReservationResult::Replay { - terminal: replayed, .. - } = reserve_or_replay_blocking(&path, requested).expect("legacy replay") + terminal: replay, .. + } = reserve_or_replay_blocking(¤t, admitted.clone()).expect("current replay") else { - panic!("legacy terminal must replay") + panic!("current nested terminal must replay") }; - assert_eq!(replayed, terminal); - let migrated: serde_json::Value = - serde_json::from_slice(&std::fs::read(&path).expect("migrated bytes")) - .expect("migrated journal"); - assert_eq!(migrated["state"]["state"], "terminal"); - assert!(migrated["state"].get("terminal").is_none()); - assert_eq!(migrated["state"]["value"]["terminal"]["schema_version"], 1); - assert!(terminal_sidecar_path(&path).expect("sidecar").exists()); + assert_eq!(replay, terminal); + + let path = temp.path().join("inline-terminal.json"); + journal["state"] = json!({ + "state": "terminal", + "terminal": terminal, + }); + let inline = serde_json::to_vec_pretty(&journal).expect("inline bytes"); + std::fs::write(&path, &inline).expect("inline journal"); + + let Err(error) = reserve_or_replay_blocking(&path, admitted) else { + panic!("inline terminal journal must be rejected") + }; + assert!( + error + .to_string() + .contains(r#"string "terminal", expected "state" or "value""#), + "unexpected rejection: {error}" + ); + assert_eq!(std::fs::read(&path).expect("preserved bytes"), inline); + assert!(!terminal_sidecar_path(&path).expect("sidecar").exists()); } #[test] @@ -830,7 +838,7 @@ fn reserved_read_removes_an_orphan_terminal_sidecar() { // replaying or re-executing. assert!(matches!( reserve_or_replay_blocking(&path, admitted).expect("orphan cleanup enters recovery"), - ReservationResult::Recover { .. } + ReservationResult::Recover )); assert!(!sidecar.exists()); } @@ -1043,7 +1051,6 @@ fn journal_sidecar_and_index_publishers_stage_mode_0600() { let record = DurableAutomationRecord { admission: admission.clone(), state: DurableAutomationState::Reserved, - legacy_terminal: None, }; write_record_with_publisher(&journal_path, &record, |temporary, destination| { assert_private_unix_stage_and_replace(temporary, destination, "automation terminal journal") @@ -1396,7 +1403,6 @@ fn bound_state_stabilization_rereads_after_parent_sync() { let replacement = DurableAutomationRecord { admission, state: DurableAutomationState::Reserved, - legacy_terminal: None, }; let error = stabilize_bound_record_after_visibility_with(&path, &visible, |path, _| { @@ -1422,7 +1428,6 @@ fn oversized_journal_prewrite_preserves_the_valid_reservation() { .expect("binding"), publication: None, }, - legacy_terminal: None, }; assert!(write_record(&path, &oversized).is_err()); @@ -1448,7 +1453,7 @@ fn foreign_external_reservation_closes_indeterminate_without_a_second_execution( let mut reopened = original.clone(); reopened.process_run_id = "process.external-journal.reopened".to_owned(); - let ReservationResult::Recover { .. } = + let ReservationResult::Recover = reserve_or_replay_blocking(&path, reopened.clone()).expect("recover external reservation") else { panic!("foreign external reservation must recover") @@ -1609,27 +1614,6 @@ fn recovery_authority_digest_rejects_every_mutable_recovery_and_digest_domain() *recovery_problem = changed_problem; mutations.push(("memory recovery problem", changed)); - let mut changed = original.clone(); - let AutomationRecoveryBinding::Memory { retirement, .. } = &mut changed.recovery else { - panic!("memory admission must carry memory recovery") - }; - *retirement = Some(super::retirement::RetirementBinding { - source_digest: format!("sha256:{}", "d".repeat(64)), - archive_name: format!("fact_proposals.{}.json", "d".repeat(64)), - }); - mutations.push(("memory retirement", changed)); - - let mut changed = original.clone(); - let AutomationRecoveryBinding::Memory { - reset_source_digest, - .. - } = &mut changed.recovery - else { - panic!("memory admission must carry memory recovery") - }; - *reset_source_digest = Some(format!("sha256:{}", "e".repeat(64))); - mutations.push(("memory reset source", changed)); - let external = external_admission_for_job( "run.external-authority-binding", "request.external-authority-binding", @@ -1698,7 +1682,7 @@ fn abandoned_same_process_reservation_enters_recovery_without_reexecution() { assert!(matches!( reserve_or_replay_blocking(&path, original).expect("recover dropped authority"), - ReservationResult::Recover { .. } + ReservationResult::Recover )); } @@ -1718,7 +1702,6 @@ async fn direct_recover_retires_spool_staged_before_prepared_binding() { "accepted_count": 0, "rejected_count": 0, "error": "no_memory_curator_evidence", - "fallback_status": "no_memory_curator_evidence", "started_at": "1", "completed_at": "2" })) @@ -1737,7 +1720,7 @@ async fn direct_recover_retires_spool_staged_before_prepared_binding() { drop(claim); assert!(matches!( reserve_or_replay_blocking(&path, original.clone()).expect("direct recover"), - ReservationResult::Recover { .. } + ReservationResult::Recover )); super::discard_direct_recovery_unbound_spools(temp.path(), &path, &original) @@ -1817,87 +1800,6 @@ fn physical_reopen_retains_original_grant_when_current_registration_rotates() { ); } -#[test] -fn project_open_crash_recovery_defers_retirement_until_exact_finalization() { - let temp = tempfile::tempdir().expect("tempdir"); - let path = temp.path().join("terminal.json"); - let mut original = admission("run.memory-retirement", "request.memory-retirement"); - let binding = super::retirement::RetirementBinding { - source_digest: format!("sha256:{}", "a".repeat(64)), - archive_name: format!("fact_proposals.{}.json", "a".repeat(64)), - }; - let AutomationRecoveryBinding::Memory { retirement, .. } = &mut original.recovery else { - panic!("memory admission must carry memory recovery") - }; - *retirement = Some(binding.clone()); - reserve_or_replay_blocking(&path, original.clone()).expect("reserve retirement"); - let mut reopened = original.clone(); - reopened.process_run_id = "process.memory-journal.reopened".to_owned(); - let ReservationResult::Recover { retirement } = - reserve_or_replay_blocking(&path, reopened.clone()).expect("recover crashed reservation") - else { - panic!("crashed retirement must require canonical receipt recovery") - }; - assert_eq!(retirement, Some(binding.clone())); - let reopened_record = read_indexed_record_blocking(&path) - .expect("physical reopen") - .expect("reserved retirement"); - assert_eq!(reopened_record.admission().retirement(), Some(&binding)); - assert_eq!( - recovery_index::special_recovery_defer_reason(reopened_record.admission(), true,), - Some("retirement_requires_exact_finalization") - ); - assert_eq!( - recovery_index::special_recovery_defer_reason(reopened_record.admission(), false,), - None - ); - assert!(!reopened_record.is_terminal()); - assert!(matches!( - reserve_or_replay_blocking(&path, reopened) - .expect("deferred retirement remains recoverable"), - ReservationResult::Recover { .. } - )); -} - -#[test] -fn project_open_crash_recovery_preserves_shipped_reset_digest_until_exact_diagnostic() { - let temp = tempfile::tempdir().expect("tempdir"); - let path = temp.path().join("terminal.json"); - let mut original = admission("run.memory-reset", "request.memory-reset"); - let reset_digest = format!("sha256:{}", "b".repeat(64)); - let AutomationRecoveryBinding::Memory { - reset_source_digest, - .. - } = &mut original.recovery - else { - panic!("memory admission must carry memory recovery") - }; - *reset_source_digest = Some(reset_digest.clone()); - reserve_or_replay_blocking(&path, original.clone()).expect("reserve shipped reset"); - let mut reopened = original.clone(); - reopened.process_run_id = "process.memory-journal.reopened".to_owned(); - assert!(matches!( - reserve_or_replay_blocking(&path, reopened.clone()).expect("recover crashed reset"), - ReservationResult::Recover { .. } - )); - let reopened_record = read_indexed_record_blocking(&path) - .expect("physical reopen") - .expect("reserved shipped reset"); - assert_eq!( - reopened_record.admission().reset_source_digest(), - Some(reset_digest.as_str()) - ); - assert_eq!( - recovery_index::special_recovery_defer_reason(reopened_record.admission(), true,), - Some("shipped_proposals_require_exact_reset_diagnostic") - ); - assert!(!reopened_record.is_terminal()); - assert!(matches!( - reserve_or_replay_blocking(&path, reopened).expect("deferred reset remains recoverable"), - ReservationResult::Recover { .. } - )); -} - #[test] fn foreign_reservation_recovery_persists_exact_partial_terminal() { let temp = tempfile::tempdir().expect("tempdir"); @@ -1908,7 +1810,7 @@ fn foreign_reservation_recovery_persists_exact_partial_terminal() { reopened.process_run_id = "process.memory-journal.reopened".to_owned(); assert!(matches!( reserve_or_replay_blocking(&path, reopened.clone()).expect("recover"), - ReservationResult::Recover { .. } + ReservationResult::Recover )); let partial = partial_terminal(&original); let stored = persist_recovered_terminal_blocking(&path, &reopened, partial.clone(), None) @@ -2017,7 +1919,6 @@ fn physical_reopen_rejects_a_corrupt_swapped_terminal() { .expect("swapped sidecar"), publication: None, }, - legacy_terminal: None, }; write_private_test_file( &path, @@ -3120,7 +3021,7 @@ async fn retained_user_job_rebinds_and_recovery_retires_only_terminal_corrupt_sp let cancellation = CancellationSignal::active("cancellation.corrupt-spool-recovery") .expect("recovery cancellation"); let recovery_index::AutomationEffectRecoveryPreparation::Pending(preparation) = - recovery_index::prepare_reserved_automation_effect_recovery(dashboard_root, &cancellation) + recovery_index::prepare_reserved_automation_effect_recovery(dashboard_root) .await .expect("prepare canonical recovery") else { diff --git a/crates/tracedecay-automation-runtime/src/automation/effect_runtime/terminal.rs b/crates/tracedecay-automation-runtime/src/automation/effect_runtime/terminal.rs index 130e0d4c63..4629590c6a 100644 --- a/crates/tracedecay-automation-runtime/src/automation/effect_runtime/terminal.rs +++ b/crates/tracedecay-automation-runtime/src/automation/effect_runtime/terminal.rs @@ -2,7 +2,7 @@ use serde::{Deserialize, Serialize}; use tracedecay_contracts::retained_surfaces::{ - AutomationRunProblemV1, AutomationRunResultV1, AutomationRunTerminalV1, AutomationSkipReasonV1, + AutomationRunProblemV1, AutomationRunResultV1, AutomationRunTerminalV1, RetainedSurfaceOperation, RetainedSurfaceResultV1, }; use tracedecay_contracts::{ @@ -104,24 +104,4 @@ impl AutomationSettledTerminal { }; matches!(result.terminal, AutomationRunTerminalV1::Completed { .. }) } - - pub fn is_retirement_terminal(&self) -> bool { - let Self::Outcome { outcome, .. } = self else { - return false; - }; - let ApplicationOutcome::Effect(effect) = outcome.as_ref() else { - return false; - }; - let Some(RetainedSurfaceResultV1::FactStoreCurate(result)) = effect.payload.as_ref() else { - return false; - }; - matches!( - &result.terminal, - AutomationRunTerminalV1::Skipped { reason, .. } - if Some(*reason) - == AutomationSkipReasonV1::from_ledger_reason( - "shipped_fact_proposal_history_retired" - ) - ) && result.committed_receipts.is_empty() - } } diff --git a/crates/tracedecay-automation-runtime/src/automation/host_io.rs b/crates/tracedecay-automation-runtime/src/automation/host_io.rs index 9006aae9e4..fcc3eb96f9 100644 --- a/crates/tracedecay-automation-runtime/src/automation/host_io.rs +++ b/crates/tracedecay-automation-runtime/src/automation/host_io.rs @@ -15,10 +15,6 @@ use serde_json::Value; use super::skill_targets::SkillInstallSummary; use tracedecay_domain::errors::Result; -/// The unslugged managed-skill start marker. Same literal the agent-hosts -/// prompt-rules block-splicer stops at. -pub const SKILL_INDEX_START: &str = ""; - /// Per-agent outcome of a managed-skill export refresh. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct ManagedSkillExportReport { @@ -37,8 +33,8 @@ pub struct PluginFile { pub type ExportToAgents = fn(&Path, &Path) -> Vec; pub type ExportToAgentHosts = fn(&Path, &Path, &Path) -> Vec; -pub type WriteText = fn(&Path, &str, Option<&Path>) -> Result<()>; -pub type WriteJson = fn(&Path, &Value, Option<&Path>) -> Result<()>; +pub type WriteText = fn(&Path, &str) -> Result<()>; +pub type WriteJson = fn(&Path, &Value) -> Result<()>; pub type RemoveHostFile = fn(&Path) -> std::io::Result<()>; pub type CodexAgentFiles = fn() -> &'static [PluginFile]; @@ -81,22 +77,12 @@ impl HostIo { (self.export_to_agent_hosts)(home, project_root, profile_root) } - pub fn safe_write_text_file( - &self, - path: &Path, - contents: &str, - backup: Option<&Path>, - ) -> Result<()> { - (self.write_text)(path, contents, backup) + pub fn safe_write_text_file(&self, path: &Path, contents: &str) -> Result<()> { + (self.write_text)(path, contents) } - pub fn safe_write_json_file( - &self, - path: &Path, - value: &Value, - backup: Option<&Path>, - ) -> Result<()> { - (self.write_json)(path, value, backup) + pub fn safe_write_json_file(&self, path: &Path, value: &Value) -> Result<()> { + (self.write_json)(path, value) } pub fn safe_remove_host_file(&self, path: &Path) -> std::io::Result<()> { diff --git a/crates/tracedecay-automation-runtime/src/automation/host_receipts.rs b/crates/tracedecay-automation-runtime/src/automation/host_receipts.rs index 37e02145a4..98fd13a242 100644 --- a/crates/tracedecay-automation-runtime/src/automation/host_receipts.rs +++ b/crates/tracedecay-automation-runtime/src/automation/host_receipts.rs @@ -8,6 +8,7 @@ use serde::{Deserialize, Serialize}; use super::config_error; use tracedecay_domain::errors::Result; use tracedecay_hooks::{HookRouteMetadata, HookTerminalReceipt}; +use tracedecay_private_fs::FileLease; use tracedecay_runtime_core::storage::PrivateStoreIo; use tracedecay_runtime_core::tracedecay::current_timestamp; @@ -106,7 +107,7 @@ fn with_locked_state( config_error(format!("failed to create host receipt directory: {error}")) })?; let (state_path, temp_path, lock_path) = paths(dashboard_root); - let mut lock = OpenOptions::new() + let lock = OpenOptions::new() .create(true) .read(true) .write(true) @@ -115,6 +116,7 @@ fn with_locked_state( .map_err(|error| config_error(format!("failed to open host receipt lock: {error}")))?; lock.lock() .map_err(|error| config_error(format!("failed to lock host receipts: {error}")))?; + let mut lock = FileLease::held(lock, "automation.host_receipts"); let mut state = std::fs::read(&state_path) .ok() .and_then(|bytes| serde_json::from_slice(&bytes).ok()) @@ -124,7 +126,7 @@ fn with_locked_state( PrivateStoreIo::write_file_atomically(&state_path, &temp_path, &bytes) .map_err(|error| config_error(format!("failed to persist host receipts: {error}")))?; let _ = lock.flush(); - let _ = lock.unlock(); + drop(lock); Ok(output) } diff --git a/crates/tracedecay-automation-runtime/src/automation/jobs.rs b/crates/tracedecay-automation-runtime/src/automation/jobs.rs index ee3d208cb7..7486264257 100644 --- a/crates/tracedecay-automation-runtime/src/automation/jobs.rs +++ b/crates/tracedecay-automation-runtime/src/automation/jobs.rs @@ -2,6 +2,7 @@ use std::collections::BTreeMap; use std::path::{Component, Path, PathBuf}; use std::time::Duration; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use tracedecay_contracts::retained_surfaces::AutomationSkipReasonV1; @@ -43,7 +44,7 @@ const DEFAULT_JOB_FAILURE_COOLDOWN_SECS: u64 = 300; const DEFAULT_JOB_STALE_LOCK_SECS: u64 = 6 * 60 * 60; const WEBHOOK_TIMEOUT_SECS: u64 = 10; -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(tag = "mode", rename_all = "snake_case")] pub enum JobDelivery { File { @@ -61,7 +62,7 @@ impl Default for JobDelivery { } } -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct AutomationJob { pub id: String, pub name: String, @@ -519,6 +520,7 @@ async fn run_user_job_with_backend_publication( let ctx = JobRunContext { dashboard_root, config, + executable: backend.executable(), job, run_id: &run_id, trigger, @@ -546,6 +548,7 @@ async fn run_user_job_with_backend_publication( return scheduler_gate::record_scheduler_lock_skip( dashboard_root, config, + backend.executable(), job, &run_id, &started_at, @@ -750,6 +753,8 @@ fn config_skip_reason(config: &AutomationConfig) -> Option { dashboard_root: &'a Path, config: &'a AutomationConfig, + /// The executable the job's backend spawns (`AgentTaskBackend::executable`). + executable: Option<&'a Path>, job: &'a AutomationJob, run_id: &'a str, trigger: AutomationTrigger, @@ -786,7 +791,11 @@ impl JobRunContext<'_> { task: AgentTaskKind::UserJob, task_key: Some(job_task_key(&self.job.id)), backend: self.config.backend.as_str().to_string(), - backend_identity: super::backend_identity::backend_identity(self.config).ok(), + backend_identity: super::backend_identity::backend_identity( + self.config, + self.executable, + ) + .ok(), host_mode: Some(self.config.host_mode.as_str().to_string()), prompt_version: Some( super::backend::prompt_version(AgentTaskKind::UserJob).to_string(), @@ -806,12 +815,9 @@ impl JobRunContext<'_> { accepted_count: 0, rejected_count: 0, skipped_count: usize::from(status == AutomationRunStatus::Skipped), - fallback_status: if status == AutomationRunStatus::Skipped { - error.clone() - } else { - None - }, + fallback_status: None, error, + session_evidence_budget_stage: None, error_classification, error_retryable: error_classification .map(super::backend::AgentTaskFailureClass::is_retryable), diff --git a/crates/tracedecay-automation-runtime/src/automation/jobs/scheduler_config_tests.rs b/crates/tracedecay-automation-runtime/src/automation/jobs/scheduler_config_tests.rs index 3ff9f4b164..0e6c2e8fae 100644 --- a/crates/tracedecay-automation-runtime/src/automation/jobs/scheduler_config_tests.rs +++ b/crates/tracedecay-automation-runtime/src/automation/jobs/scheduler_config_tests.rs @@ -150,6 +150,7 @@ async fn persisted_and_deduplicated_skips_do_not_block_later_due_execution() { let first = super::scheduler_gate::evaluate_and_record_scheduler_skip_at( temp.path(), &config, + None, &job, "next-occurrence", 101, @@ -166,6 +167,7 @@ async fn persisted_and_deduplicated_skips_do_not_block_later_due_execution() { let repeated = super::scheduler_gate::evaluate_and_record_scheduler_skip_at( temp.path(), &config, + None, &job, "next-occurrence", 102, @@ -194,6 +196,7 @@ async fn persisted_and_deduplicated_skips_do_not_block_later_due_execution() { let due = super::scheduler_gate::evaluate_and_record_scheduler_skip_at( temp.path(), &config, + None, &job, "next-occurrence", 160, @@ -220,6 +223,7 @@ async fn persisted_and_deduplicated_skips_do_not_block_later_due_execution() { let next_skip = super::scheduler_gate::evaluate_and_record_scheduler_skip_at( temp.path(), &config, + None, &job, "following-occurrence", 161, @@ -232,6 +236,7 @@ async fn persisted_and_deduplicated_skips_do_not_block_later_due_execution() { let repeated_next = super::scheduler_gate::evaluate_and_record_scheduler_skip_at( temp.path(), &config, + None, &job, "following-occurrence", 162, @@ -261,6 +266,7 @@ async fn persisted_and_deduplicated_skips_do_not_block_later_due_execution() { let after_noise = super::scheduler_gate::evaluate_and_record_scheduler_skip_at( temp.path(), &config, + None, &job, "following-occurrence", 163, @@ -290,6 +296,7 @@ async fn scheduler_lock_skip_uses_a_diagnostic_identity_outside_the_effect_occur let skip = super::scheduler_gate::record_scheduler_lock_skip( temp.path(), &config, + None, &job, occurrence, &started_at, diff --git a/crates/tracedecay-automation-runtime/src/automation/jobs/scheduler_gate.rs b/crates/tracedecay-automation-runtime/src/automation/jobs/scheduler_gate.rs index d8cc1ed094..a621cd92c0 100644 --- a/crates/tracedecay-automation-runtime/src/automation/jobs/scheduler_gate.rs +++ b/crates/tracedecay-automation-runtime/src/automation/jobs/scheduler_gate.rs @@ -36,6 +36,7 @@ use super::{ pub async fn evaluate_and_record_scheduler_skip( dashboard_root: &Path, config: &AutomationConfig, + executable: Option<&Path>, job: &AutomationJob, run_id: &str, occurrence_anchor_run_id: Option<&str>, @@ -45,6 +46,7 @@ pub async fn evaluate_and_record_scheduler_skip( return record_scheduler_diagnostic( dashboard_root, config, + executable, job, run_id, reason, @@ -64,6 +66,7 @@ pub async fn evaluate_and_record_scheduler_skip( return record_scheduler_lock_skip( dashboard_root, config, + executable, job, run_id, ¤t_timestamp().to_string(), @@ -83,6 +86,7 @@ pub async fn evaluate_and_record_scheduler_skip( record_scheduler_diagnostic( dashboard_root, config, + executable, job, run_id, reason, @@ -97,6 +101,7 @@ pub async fn evaluate_and_record_scheduler_skip( pub(super) async fn evaluate_and_record_scheduler_skip_at( dashboard_root: &Path, config: &AutomationConfig, + executable: Option<&Path>, job: &AutomationJob, run_id: &str, now_secs: i64, @@ -107,6 +112,7 @@ pub(super) async fn evaluate_and_record_scheduler_skip_at( return record_scheduler_diagnostic( dashboard_root, config, + executable, job, run_id, reason, @@ -122,6 +128,7 @@ pub(super) async fn evaluate_and_record_scheduler_skip_at( return record_scheduler_lock_skip( dashboard_root, config, + executable, job, run_id, &now_secs.to_string(), @@ -138,6 +145,7 @@ pub(super) async fn evaluate_and_record_scheduler_skip_at( record_scheduler_diagnostic( dashboard_root, config, + executable, job, run_id, reason, @@ -167,6 +175,7 @@ async fn load_scheduler_summary( async fn record_scheduler_diagnostic( dashboard_root: &Path, config: &AutomationConfig, + executable: Option<&Path>, job: &AutomationJob, occurrence_run_id: &str, reason: AutomationSkipReasonV1, @@ -177,6 +186,7 @@ async fn record_scheduler_diagnostic( JobRunContext { dashboard_root, config, + executable, job, run_id: &diagnostic_run_id, trigger: AutomationTrigger::Scheduler, @@ -193,6 +203,7 @@ async fn record_scheduler_diagnostic( pub(super) async fn record_scheduler_lock_skip( dashboard_root: &Path, config: &AutomationConfig, + executable: Option<&Path>, job: &AutomationJob, occurrence_run_id: &str, started_at: &str, @@ -201,6 +212,7 @@ pub(super) async fn record_scheduler_lock_skip( record_scheduler_diagnostic( dashboard_root, config, + executable, job, occurrence_run_id, AutomationSkipReasonV1::SchedulerLockActive, diff --git a/crates/tracedecay-automation-runtime/src/automation/lifecycle.rs b/crates/tracedecay-automation-runtime/src/automation/lifecycle.rs index 262a419e8a..4d8d70d321 100644 --- a/crates/tracedecay-automation-runtime/src/automation/lifecycle.rs +++ b/crates/tracedecay-automation-runtime/src/automation/lifecycle.rs @@ -321,6 +321,9 @@ pub(crate) struct AgentTaskRunContext<'a> { /// the scheduler activity signal. sessions_db: RegisteredGlobalDbLeaseV1, config: &'a AutomationConfig, + /// The executable the run's backend spawns (`AgentTaskBackend::executable`), + /// stamped into every terminal record's backend identity. + executable: Option, task: AgentTaskKind, started_at: String, ledger_publication: AutomationRunLedgerPublication, @@ -339,6 +342,7 @@ impl<'a> AgentTaskRunContext<'a> { run_id_prefix: &'static str, trigger: AutomationTrigger, config: &'a AutomationConfig, + executable: Option<&Path>, task: AgentTaskKind, ) -> Self { Self { @@ -347,6 +351,7 @@ impl<'a> AgentTaskRunContext<'a> { dashboard_root, sessions_db, config, + executable: executable.map(Path::to_path_buf), task, started_at: current_timestamp().to_string(), ledger_publication: AutomationRunLedgerPublication::Immediate, @@ -377,9 +382,14 @@ impl<'a> AgentTaskRunContext<'a> { &self.started_at } + pub(crate) fn executable(&self) -> Option<&Path> { + self.executable.as_deref() + } + pub(crate) async fn gate(&mut self) -> Result { let (gate, summary) = task_run_gate_with_lock_retention( self.config, + self.executable(), &self.dashboard_root, self.sessions_db.as_ref(), self.task, @@ -458,6 +468,7 @@ impl<'a> AgentTaskRunContext<'a> { &self.run_id, self.trigger, self.config, + self.executable(), self.task, self.started_at(), input_hash, @@ -486,6 +497,7 @@ pub(crate) fn task_skip_reason( #[hotpath::measure(label = "automation.scheduler.gate", future = true)] async fn scheduler_gate_with_lock_retention( config: &AutomationConfig, + executable: Option<&Path>, dashboard_root: &Path, sessions_db: &RegisteredGlobalDb, task: AgentTaskKind, @@ -528,13 +540,21 @@ async fn scheduler_gate_with_lock_retention( let decision = if trigger == AutomationTrigger::HostReceipt { super::scheduler::host_receipt_decision( config, + executable, task, summary.records(), activity, decision_now_secs, ) } else { - schedule_decision(config, task, summary.records(), activity, decision_now_secs) + schedule_decision( + config, + executable, + task, + summary.records(), + activity, + decision_now_secs, + ) }; if let Some(reason) = decision.skip_reason() { super::scheduler_metrics::observe_skip_reason(reason); @@ -547,17 +567,27 @@ async fn scheduler_gate_with_lock_retention( pub(crate) async fn task_run_gate( config: &AutomationConfig, + executable: Option<&Path>, dashboard_root: &Path, sessions_db: &RegisteredGlobalDb, task: AgentTaskKind, trigger: AutomationTrigger, ) -> Result<(SchedulerGate, Option)> { - task_run_gate_with_lock_retention(config, dashboard_root, sessions_db, task, trigger, None) - .await + task_run_gate_with_lock_retention( + config, + executable, + dashboard_root, + sessions_db, + task, + trigger, + None, + ) + .await } pub(crate) async fn task_run_gate_for_retained_settlement( config: &AutomationConfig, + executable: Option<&Path>, dashboard_root: &Path, sessions_db: &RegisteredGlobalDb, task: AgentTaskKind, @@ -567,6 +597,7 @@ pub(crate) async fn task_run_gate_for_retained_settlement( let retention = settlement_guard.retention(); task_run_gate_with_lock_retention( config, + executable, dashboard_root, sessions_db, task, @@ -578,6 +609,7 @@ pub(crate) async fn task_run_gate_for_retained_settlement( async fn task_run_gate_with_lock_retention( config: &AutomationConfig, + executable: Option<&Path>, dashboard_root: &Path, sessions_db: &RegisteredGlobalDb, task: AgentTaskKind, @@ -586,6 +618,7 @@ async fn task_run_gate_with_lock_retention( ) -> Result<(SchedulerGate, Option)> { let (gate, records) = scheduler_gate_with_lock_retention( config, + executable, dashboard_root, sessions_db, task, @@ -789,6 +822,7 @@ pub(crate) struct AgentRunFinalizer<'a> { run_id: &'a str, trigger: AutomationTrigger, config: &'a AutomationConfig, + executable: Option<&'a Path>, task: AgentTaskKind, started_at: &'a str, input_hash: Option, @@ -807,6 +841,7 @@ impl<'a> AgentRunFinalizer<'a> { run_id: &'a str, trigger: AutomationTrigger, config: &'a AutomationConfig, + executable: Option<&'a Path>, task: AgentTaskKind, started_at: &'a str, input_hash: Option, @@ -816,6 +851,7 @@ impl<'a> AgentRunFinalizer<'a> { run_id, trigger, config, + executable, task, started_at, input_hash, @@ -829,6 +865,7 @@ impl<'a> AgentRunFinalizer<'a> { run_id: &'a str, trigger: AutomationTrigger, config: &'a AutomationConfig, + executable: Option<&'a Path>, task: AgentTaskKind, started_at: &'a str, input_hash: Option, @@ -840,6 +877,7 @@ impl<'a> AgentRunFinalizer<'a> { run_id, trigger, config, + executable, task, started_at, input_hash, @@ -1175,7 +1213,11 @@ impl<'a> AgentRunFinalizer<'a> { // configuration it failed under changes. A digest that cannot be // computed is left absent rather than guessed: an unidentified // failure must not suppress anything. - backend_identity: super::backend_identity::backend_identity(self.config).ok(), + backend_identity: super::backend_identity::backend_identity( + self.config, + self.executable, + ) + .ok(), host_mode: Some(self.config.host_mode.as_str().to_string()), prompt_version: Some(prompt_version(self.task).to_string()), response_schema: Some(contract.response_schema), @@ -1193,10 +1235,9 @@ impl<'a> AgentRunFinalizer<'a> { accepted_count: outcome.accepted_count, rejected_count: outcome.rejected_count, skipped_count: usize::from(outcome.status == AutomationRunStatus::Skipped), - fallback_status: (outcome.status == AutomationRunStatus::Skipped) - .then(|| outcome.error.clone()) - .flatten(), + fallback_status: None, error: outcome.error, + session_evidence_budget_stage: None, error_classification, error_retryable: error_classification .map(super::backend::AgentTaskFailureClass::is_retryable), @@ -1295,6 +1336,7 @@ mod recorded_failure_tests { "prior_scheduler_skip", AutomationTrigger::Scheduler, &config, + None, AgentTaskKind::SessionReflector, "0", None, diff --git a/crates/tracedecay-automation-runtime/src/automation/lifecycle/tests.rs b/crates/tracedecay-automation-runtime/src/automation/lifecycle/tests.rs index b516ad0d0a..3849a6eb61 100644 --- a/crates/tracedecay-automation-runtime/src/automation/lifecycle/tests.rs +++ b/crates/tracedecay-automation-runtime/src/automation/lifecycle/tests.rs @@ -64,6 +64,7 @@ fn pre_epoch_completion_clock_fails_before_ledger_mutation() { "run.pre-epoch", AutomationTrigger::ManualCli, &config, + None, AgentTaskKind::SessionReflector, "0", None, @@ -87,6 +88,7 @@ fn terminal_clock_is_fresh_and_posteffect_failure_stays_typed() { "run.clock-progression", AutomationTrigger::ManualCli, &config, + None, AgentTaskKind::SessionReflector, "1", None, @@ -219,6 +221,7 @@ async fn append_skip( "test", trigger, &config, + None, task, ); run.gate().await.expect("gate"); @@ -333,6 +336,7 @@ async fn on_demand_triggers_bypass_only_scheduler_enablement() { ] { let (gate, _) = task_run_gate( &disabled, + None, temp.path(), sessions.db.as_ref(), AgentTaskKind::MemoryCurator, @@ -351,6 +355,7 @@ async fn concurrent_on_demand_runs_share_the_canonical_task_lock() { let config = scheduler_enabled_config(); let (first, _) = task_run_gate( &config, + None, temp.path(), sessions.db.as_ref(), AgentTaskKind::MemoryCurator, @@ -364,6 +369,7 @@ async fn concurrent_on_demand_runs_share_the_canonical_task_lock() { let (concurrent, _) = task_run_gate( &config, + None, temp.path(), sessions.db.as_ref(), AgentTaskKind::MemoryCurator, @@ -379,6 +385,7 @@ async fn concurrent_on_demand_runs_share_the_canonical_task_lock() { drop(first_lock); let (next, _) = task_run_gate( &config, + None, temp.path(), sessions.db.as_ref(), AgentTaskKind::MemoryCurator, @@ -402,6 +409,7 @@ async fn scheduler_trigger_still_obeys_global_enablement() { let (gate, _) = task_run_gate( &disabled, + None, temp.path(), sessions.db.as_ref(), AgentTaskKind::MemoryCurator, @@ -427,6 +435,7 @@ async fn on_demand_trigger_does_not_bypass_backend_or_host_admission() { }; let (gate, _) = task_run_gate( &unavailable, + None, temp.path(), sessions.db.as_ref(), AgentTaskKind::MemoryCurator, @@ -446,6 +455,7 @@ async fn on_demand_trigger_does_not_bypass_backend_or_host_admission() { }; let (gate, _) = task_run_gate( &delegated, + None, temp.path(), sessions.db.as_ref(), AgentTaskKind::MemoryCurator, @@ -486,6 +496,7 @@ async fn post_gate_scheduler_skip(dashboard_root: &Path, run_id: &str, reason: & "test", AutomationTrigger::Scheduler, &config, + None, AgentTaskKind::MemoryCurator, ); let gate = run.gate().await.expect("gate"); @@ -544,6 +555,7 @@ async fn page_cursor_transitions_bypass_reason_only_skip_deduplication() { "test", AutomationTrigger::Scheduler, &config, + None, AgentTaskKind::MemoryCurator, ); let SchedulerGate::Proceed(lock) = run.gate().await.expect("gate") else { @@ -595,6 +607,7 @@ async fn append_path_relies_solely_on_caller_computed_repeat_flag() { "memory_curator", AutomationTrigger::Scheduler, &config, + None, task, ); append_skipped_record(&run, None, "scheduler_interval_not_elapsed", false) @@ -615,6 +628,7 @@ async fn append_path_relies_solely_on_caller_computed_repeat_flag() { "memory_curator", AutomationTrigger::Scheduler, &config, + None, task, ); append_skipped_record(&run, None, "scheduler_interval_not_elapsed", true) diff --git a/crates/tracedecay-automation-runtime/src/automation/managed_skills.rs b/crates/tracedecay-automation-runtime/src/automation/managed_skills.rs index 1bf299607c..58d17c7075 100644 --- a/crates/tracedecay-automation-runtime/src/automation/managed_skills.rs +++ b/crates/tracedecay-automation-runtime/src/automation/managed_skills.rs @@ -7,6 +7,7 @@ use super::config_error; use serde::{Deserialize, Serialize}; use tracedecay_automation::run_labels::SKILL_OVERLAP_REMOVAL_TOMBSTONE; use tracedecay_domain::errors::Result; +use tracedecay_private_fs::FileLease; use tracedecay_private_fs::framed_log::DirectorySyncPolicy; pub use tracedecay_automation::managed_skills::validate_managed_support_files; @@ -20,60 +21,6 @@ use tracedecay_automation::managed_skills::{ validate_managed_skill, validate_managed_skill_update, validate_skill_id, }; -/// Decode the retained summary-only format without mutating inspection state. -fn decode_retained_skill(bytes: &[u8]) -> Result<(ManagedSkill, bool)> { - let mut value: serde_json::Value = serde_json::from_slice(bytes)?; - let metadata = value - .get_mut("metadata") - .and_then(serde_json::Value::as_object_mut) - .ok_or_else(|| config_error("managed skill metadata must be an object"))?; - let legacy = !metadata.contains_key("routing_description"); - if legacy { - let summary = metadata - .get("summary") - .and_then(serde_json::Value::as_str) - .ok_or_else(|| config_error("retained managed skill summary is required"))?; - let routing = - tracedecay_automation::managed_skills::legacy_managed_skill_routing_description( - summary, - ); - metadata.insert( - "routing_description".to_owned(), - serde_json::Value::String(routing), - ); - } - let skill: ManagedSkill = serde_json::from_value(value)?; - validate_managed_skill(&skill)?; - Ok((skill, legacy)) -} - -/// Upgrade retained routing metadata before authoring evidence is captured. -/// Inspection reads use the decoder only; this explicit mutation uses the same -/// journal and lock as skill edits, without changing authored timestamps or provenance. -pub async fn migrate_managed_skill_routing(profile_root: &Path) -> Result<()> { - if !managed_skill_root(profile_root).exists() { - return Ok(()); - } - let _lock = lock_skill_store_async(profile_root).await?; - let root = managed_skill_root(profile_root); - let mut migrated = Vec::new(); - for entry in std::fs::read_dir(&root)? { - let path = entry?.path().join("skill.json"); - if !path.is_file() { - continue; - } - let (mut skill, legacy) = decode_retained_skill(&std::fs::read(path)?)?; - if legacy { - skill.refresh_checksum(); - migrated.push(skill); - } - } - if !migrated.is_empty() { - persist_skill_transaction_unlocked(profile_root, &migrated.iter().collect::>())?; - } - Ok(()) -} - pub fn managed_skill_root(profile_root: &Path) -> PathBuf { profile_root.join("agent_managed").join("skills") } @@ -102,15 +49,7 @@ impl SkillConsolidationKind<'_> { } } -struct SkillStoreLock(File); - -impl Drop for SkillStoreLock { - fn drop(&mut self) { - let _ = self.0.unlock(); - } -} - -fn lock_skill_store(profile_root: &Path) -> Result { +fn lock_skill_store(profile_root: &Path) -> Result { let root = managed_skill_root(profile_root); std::fs::create_dir_all(&root).map_err(|e| { config_error(format!( @@ -137,11 +76,12 @@ fn lock_skill_store(profile_root: &Path) -> Result { path.display() )) })?; + let lock = FileLease::held(file, "automation.managed_skills.store"); recover_skill_transaction(&root)?; - Ok(SkillStoreLock(file)) + Ok(lock) } -async fn lock_skill_store_async(profile_root: &Path) -> Result { +async fn lock_skill_store_async(profile_root: &Path) -> Result { let profile_root = profile_root.to_path_buf(); tokio::task::spawn_blocking(move || lock_skill_store(&profile_root)) .await @@ -605,16 +545,20 @@ fn load_managed_skill_unlocked(profile_root: &Path, id: &str) -> Result Result { + let invalid = |e: &dyn std::fmt::Display| { + config_error(format!( + "invalid managed skill record '{}': {e}", + path.display() + )) + }; + let skill: ManagedSkill = serde_json::from_slice(bytes).map_err(|e| invalid(&e))?; + validate_managed_skill(&skill).map_err(|e| invalid(&e))?; Ok(skill) } @@ -658,17 +602,7 @@ fn list_managed_skills_unlocked(profile_root: &Path) -> Result path.display() )) })?; - let mut skill = decode_retained_skill(&bytes) - .map(|(skill, _)| skill) - .map_err(|e| { - config_error(format!( - "failed to parse managed skill record '{}': {e}", - path.display() - )) - })?; - skill.normalize_timestamps(); - validate_managed_skill(&skill)?; - skills.push(skill); + skills.push(decode_managed_skill_record(&path, &bytes)?); } skills.sort_by(|a, b| a.metadata.id.cmp(&b.metadata.id)); Ok(skills) diff --git a/crates/tracedecay-automation-runtime/src/automation/managed_skills/routing_tests.rs b/crates/tracedecay-automation-runtime/src/automation/managed_skills/routing_tests.rs index 8b0c4492e9..2cbb43f161 100644 --- a/crates/tracedecay-automation-runtime/src/automation/managed_skills/routing_tests.rs +++ b/crates/tracedecay-automation-runtime/src/automation/managed_skills/routing_tests.rs @@ -1,24 +1,17 @@ +use tracedecay_domain::errors::TraceDecayError; + use super::{ - ManagedSkillDraft, ManagedSkillProvenance, ManagedSkillSource, ManagedSkillState, - ManagedSkillUpdate, ManagedSupportFile, SkillInstallTarget, apply_managed_skill_update, - create_managed_skill, decode_retained_skill, list_managed_skills, load_managed_skill, - managed_skill_dir, migrate_managed_skill_routing, set_managed_skill_pinned, - set_managed_skill_state, + ManagedSkillDraft, ManagedSkillProvenance, ManagedSkillSource, ManagedSupportFile, + SkillInstallTarget, create_managed_skill, list_managed_skills, load_managed_skill, + managed_skill_dir, }; -use tracedecay_automation::managed_skills::legacy_managed_skill_routing_description; - -// SHA-256 of this fixture's historical content encoding: id, title, summary, -// category, target tags, body, and support file path/bytes, without routing text. -const HISTORICAL_CHECKSUM: &str = - "sha256:566d3d67af3e00c94b623ede09c424261f796988ae458803cc09171143572e6c"; fn routing_draft() -> ManagedSkillDraft { - let summary = r#"Diagnose "quoted" paths"#; ManagedSkillDraft { - id: "legacy-routing".to_string(), - title: "Legacy routing".to_string(), - summary: summary.to_string(), - routing_description: legacy_managed_skill_routing_description(summary), + id: "routing".to_string(), + title: "Routing".to_string(), + summary: r#"Diagnose "quoted" paths"#.to_string(), + routing_description: r#"Use when diagnosing "quoted" paths"#.to_string(), category: "testing".to_string(), targets: vec![SkillInstallTarget::Codex, SkillInstallTarget::Claude], body_markdown: "# Routing\n\nInspect the failing path.\n".to_string(), @@ -28,224 +21,64 @@ fn routing_draft() -> ManagedSkillDraft { ], provenance: ManagedSkillProvenance { source: ManagedSkillSource::AutomationRun, - actor: "routing-migration-test".to_string(), - run_id: Some("retained-run".to_string()), + actor: "routing-test".to_string(), + run_id: Some("run".to_string()), }, } } #[tokio::test] -async fn retained_routing_migration_preserves_exports_and_fences_old_checksums() { - let profile = tempfile::TempDir::new().unwrap(); - let created = create_managed_skill(profile.path(), routing_draft()) - .await - .unwrap(); - let id = &created.metadata.id; - set_managed_skill_state(profile.path(), id, ManagedSkillState::Disabled) - .await - .unwrap(); - let current = set_managed_skill_pinned(profile.path(), id, true) - .await - .unwrap(); - let expected_native = current.render_native_skill_markdown().unwrap(); - let expected_materialized = current.render_materialized_skill_markdown().unwrap(); - let dir = managed_skill_dir(profile.path(), id).unwrap(); - let record = dir.join("skill.json"); - let mut legacy_value = serde_json::to_value(¤t).unwrap(); - legacy_value["metadata"] - .as_object_mut() - .unwrap() - .remove("routing_description"); - legacy_value["metadata"]["checksum"] = HISTORICAL_CHECKSUM.into(); - let legacy_bytes = serde_json::to_vec_pretty(&legacy_value).unwrap(); - std::fs::write(&record, &legacy_bytes).unwrap(); - let stored_markdown = std::fs::read(dir.join("SKILL.md")).unwrap(); - - let mut expected_legacy = current.clone(); - expected_legacy.metadata.checksum = HISTORICAL_CHECKSUM.to_string(); - assert_eq!( - load_managed_skill(profile.path(), id).await.unwrap(), - expected_legacy - ); - assert_eq!( - list_managed_skills(profile.path()).await.unwrap(), - vec![expected_legacy] - ); - assert_eq!(std::fs::read(&record).unwrap(), legacy_bytes); - assert_eq!( - std::fs::read(dir.join("SKILL.md")).unwrap(), - stored_markdown - ); - - migrate_managed_skill_routing(profile.path()).await.unwrap(); - let migrated = load_managed_skill(profile.path(), id).await.unwrap(); - // Full equality covers state, pin, timestamps, provenance, body, and support. - assert_eq!(migrated, current); - assert_eq!( - migrated.render_native_skill_markdown().unwrap(), - expected_native - ); - assert_eq!( - migrated.render_materialized_skill_markdown().unwrap(), - expected_materialized - ); - assert_eq!( - std::fs::read(dir.join("references/paths.md")).unwrap(), - b"Preserve path identity.\n" - ); - let migrated_bytes = std::fs::read(&record).unwrap(); - assert_ne!(migrated_bytes, legacy_bytes); - let (decoded, needs_migration) = decode_retained_skill(&migrated_bytes).unwrap(); - assert!(!needs_migration); - assert_eq!(decoded, migrated); - migrate_managed_skill_routing(profile.path()).await.unwrap(); - assert_eq!(std::fs::read(&record).unwrap(), migrated_bytes); - assert_eq!( - load_managed_skill(profile.path(), id).await.unwrap(), - migrated - ); - - let update = ManagedSkillUpdate { - routing_description: Some("Diagnose Windows path quoting failures.".to_string()), - ..Default::default() - }; - let error = apply_managed_skill_update(profile.path(), id, HISTORICAL_CHECKSUM, update.clone()) - .await - .unwrap_err(); - assert!(error.to_string().contains("base_checksum")); - assert!(error.to_string().contains("stale")); - assert_eq!(std::fs::read(&record).unwrap(), migrated_bytes); - let updated = - apply_managed_skill_update(profile.path(), id, &migrated.metadata.checksum, update) - .await - .unwrap(); - assert_eq!( - updated.metadata.routing_description, - "Diagnose Windows path quoting failures." - ); - assert_ne!(updated.metadata.checksum, migrated.metadata.checksum); - assert_eq!( - load_managed_skill(profile.path(), id).await.unwrap(), - updated - ); -} - -#[tokio::test] -async fn present_invalid_routing_is_rejected_without_legacy_fallback() { - for invalid in [ - serde_json::Value::Null, - serde_json::Value::String(String::new()), +async fn missing_or_invalid_routing_is_rejected_without_rewriting_records() { + for (invalid, expected_error) in [ + (None, "missing field `routing_description`"), + ( + Some(serde_json::Value::Null), + "invalid type: null, expected a string", + ), + ( + Some(serde_json::Value::String(String::new())), + "native description cannot be empty", + ), ] { let profile = tempfile::TempDir::new().unwrap(); let skill = create_managed_skill(profile.path(), routing_draft()) .await .unwrap(); + let loaded = load_managed_skill(profile.path(), &skill.metadata.id) + .await + .unwrap(); + assert_eq!( + loaded.metadata.routing_description, + r#"Use when diagnosing "quoted" paths"# + ); let dir = managed_skill_dir(profile.path(), &skill.metadata.id).unwrap(); let record = dir.join("skill.json"); let mut value = serde_json::to_value(&skill).unwrap(); - value["metadata"]["routing_description"] = invalid; + let metadata = value["metadata"].as_object_mut().unwrap(); + match invalid { + Some(invalid) => { + metadata.insert("routing_description".to_string(), invalid); + } + None => { + metadata.remove("routing_description"); + } + } let bytes = serde_json::to_vec_pretty(&value).unwrap(); std::fs::write(&record, &bytes).unwrap(); let markdown = std::fs::read(dir.join("SKILL.md")).unwrap(); - assert!(decode_retained_skill(&bytes).is_err()); - assert!( + for error in [ load_managed_skill(profile.path(), &skill.metadata.id) .await - .is_err() - ); - assert!(list_managed_skills(profile.path()).await.is_err()); - assert!(migrate_managed_skill_routing(profile.path()).await.is_err()); - assert_eq!(std::fs::read(&record).unwrap(), bytes); - assert_eq!(std::fs::read(dir.join("SKILL.md")).unwrap(), markdown); - } -} - -#[tokio::test] -async fn legacy_lifecycle_mutations_complete_routing_checksum_cutover() { - for operation in ["pin", "state", "pinned-only update"] { - let profile = tempfile::TempDir::new().unwrap(); - let current = create_managed_skill(profile.path(), routing_draft()) - .await - .unwrap(); - let id = ¤t.metadata.id; - let record = managed_skill_dir(profile.path(), id) - .unwrap() - .join("skill.json"); - let mut legacy = serde_json::to_value(¤t).unwrap(); - legacy["metadata"] - .as_object_mut() - .unwrap() - .remove("routing_description"); - legacy["metadata"]["checksum"] = HISTORICAL_CHECKSUM.into(); - std::fs::write(&record, serde_json::to_vec_pretty(&legacy).unwrap()).unwrap(); - - let updated = match operation { - "pin" => set_managed_skill_pinned(profile.path(), id, true) - .await - .unwrap(), - "state" => set_managed_skill_state(profile.path(), id, ManagedSkillState::Disabled) - .await - .unwrap(), - _ => apply_managed_skill_update( - profile.path(), - id, - HISTORICAL_CHECKSUM, - ManagedSkillUpdate { - pinned: Some(true), - ..Default::default() - }, - ) - .await - .unwrap(), - }; - assert_eq!( - updated.metadata.checksum, current.metadata.checksum, - "{operation}" - ); - assert_ne!( - updated.metadata.checksum, HISTORICAL_CHECKSUM, - "{operation}" - ); - assert_eq!( - updated.metadata.routing_description, - current.metadata.routing_description - ); - assert_eq!(updated.metadata.provenance, current.metadata.provenance); - assert_eq!(updated.body_markdown, current.body_markdown); - assert_eq!(updated.support_files, current.support_files); - if operation == "state" { - assert_eq!(updated.metadata.state, ManagedSkillState::Disabled); - } else { - assert!(updated.metadata.pinned); + .unwrap_err(), + list_managed_skills(profile.path()).await.unwrap_err(), + ] { + assert!( + matches!(&error, TraceDecayError::Config { message } if message.contains(expected_error)), + "unexpected rejection: {error:?}" + ); } - assert_eq!( - load_managed_skill(profile.path(), id).await.unwrap(), - updated - ); - let bytes = std::fs::read(&record).unwrap(); - let (decoded, needs_migration) = decode_retained_skill(&bytes).unwrap(); - assert!(!needs_migration); - assert_eq!(decoded, updated); - migrate_managed_skill_routing(profile.path()).await.unwrap(); - assert_eq!(std::fs::read(&record).unwrap(), bytes); - assert_eq!( - load_managed_skill(profile.path(), id).await.unwrap(), - updated - ); - - let error = apply_managed_skill_update( - profile.path(), - id, - HISTORICAL_CHECKSUM, - ManagedSkillUpdate { - routing_description: Some("Diagnose path escaping failures.".to_string()), - ..Default::default() - }, - ) - .await - .unwrap_err(); - assert!(error.to_string().contains("stale"), "{operation}: {error}"); assert_eq!(std::fs::read(&record).unwrap(), bytes); + assert_eq!(std::fs::read(dir.join("SKILL.md")).unwrap(), markdown); } } diff --git a/crates/tracedecay-automation-runtime/src/automation/memory_curator.rs b/crates/tracedecay-automation-runtime/src/automation/memory_curator.rs index 31ecc29e17..7e500ff50b 100644 --- a/crates/tracedecay-automation-runtime/src/automation/memory_curator.rs +++ b/crates/tracedecay-automation-runtime/src/automation/memory_curator.rs @@ -192,6 +192,7 @@ async fn run_memory_curator_for_store_with_publication( "memory_curator", options.trigger, config, + backend.executable(), AgentTaskKind::MemoryCurator, ) .with_ledger_publication(ledger_publication) diff --git a/crates/tracedecay-automation-runtime/src/automation/observation.rs b/crates/tracedecay-automation-runtime/src/automation/observation.rs index 4fbf4985d6..5471a0b941 100644 --- a/crates/tracedecay-automation-runtime/src/automation/observation.rs +++ b/crates/tracedecay-automation-runtime/src/automation/observation.rs @@ -108,6 +108,7 @@ mod tests { backend_attempt_count: 0, backend_attempts: Vec::new(), fallback_status: None, + session_evidence_budget_stage: None, report_ref: Some(json!({"run_id": "run-42"})), artifacts: Vec::new(), started_at: "1700000000".to_owned(), @@ -193,24 +194,4 @@ mod tests { Err("invalid_completed_at") ); } - - #[test] - fn legacy_reused_scheduler_skip_keeps_its_exact_rfc3339_observation_time() { - let mut record = ledger_record(AutomationRunStatus::Skipped); - record.schema_version = 1; - record.run_id = "legacy-reused-scheduler-skip".to_owned(); - record.started_at = "1970-01-01T00:00:00Z".to_owned(); - record.completed_at = "1970-01-01T00:00:01.123456Z".to_owned(); - record.completed_at_micros = None; - record.error = Some("scheduler_interval_not_elapsed".to_owned()); - - // Reused scheduler skips pass their exact prior row through this same - // mapper after durable abandonment; no second timestamp path exists. - let (observation, observed_at) = - automation_funnel_observation_from_record(&record).expect("valid legacy exact row"); - - assert_eq!(observed_at, UtcMicros(1_123_456)); - assert_eq!(observation.run_ref, "legacy-reused-scheduler-skip"); - assert_eq!(observation.terminal, AutomationTerminalV1::Skipped); - } } diff --git a/crates/tracedecay-automation-runtime/src/automation/outcomes.rs b/crates/tracedecay-automation-runtime/src/automation/outcomes.rs index bc53223803..75110c616d 100644 --- a/crates/tracedecay-automation-runtime/src/automation/outcomes.rs +++ b/crates/tracedecay-automation-runtime/src/automation/outcomes.rs @@ -20,6 +20,7 @@ use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, LazyLock, Mutex, Weak}; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use tracedecay_domain::PayloadAccessState; @@ -50,7 +51,7 @@ pub const SKILL_ACTIVATION_WINDOW_SECS: i64 = 7 * 24 * 60 * 60; const SECS_PER_DAY: i64 = 24 * 60 * 60; -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "snake_case")] pub enum SkillOutcomeVerdict { Adopted, @@ -68,7 +69,7 @@ impl SkillOutcomeVerdict { } } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "snake_case")] pub enum FactOutcomeVerdict { RecalledAndHelpful, @@ -92,7 +93,7 @@ impl FactOutcomeVerdict { } } -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct SkillOutcomeRecord { pub skill_id: String, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -104,7 +105,7 @@ pub struct SkillOutcomeRecord { pub verdict: SkillOutcomeVerdict, } -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct FactOutcomeRecord { /// Immutable identity of the terminal automatic-fact receipt. pub apply_id: String, diff --git a/crates/tracedecay-automation-runtime/src/automation/run_ledger.rs b/crates/tracedecay-automation-runtime/src/automation/run_ledger.rs index 162621dd72..92976a48b9 100644 --- a/crates/tracedecay-automation-runtime/src/automation/run_ledger.rs +++ b/crates/tracedecay-automation-runtime/src/automation/run_ledger.rs @@ -1,9 +1,11 @@ use std::path::{Path, PathBuf}; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use serde_json::Value; use tracedecay_automation::evidence_budget::SESSION_EVIDENCE_BUDGET_EXHAUSTED; +use tracedecay_contracts::retrieval::SessionRetrievalBudgetStageV1; use super::backend::{ AgentTaskFailureClass, AgentTaskKind, AgentTaskRetryAttempt, task_key as canonical_task_key, @@ -38,7 +40,7 @@ const RUN_ARTIFACTS_DIR: &str = "automation_artifacts"; /// this window or the complete append-only ledger. const RUN_LEDGER_TAIL_CHUNK_BYTES: u64 = 256 * 1024; -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema, Default)] #[serde(rename_all = "snake_case")] pub enum AutomationTrigger { #[default] @@ -62,7 +64,7 @@ impl AutomationTrigger { } } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "snake_case")] pub enum AutomationRunStatus { Queued, @@ -88,7 +90,7 @@ impl AutomationRunStatus { } } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "snake_case")] pub enum AutomationRunArtifactKind { Traces, @@ -124,7 +126,7 @@ impl AutomationRunArtifactKind { } } -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct AutomationRunArtifact { pub schema_version: u32, pub kind: String, @@ -135,7 +137,7 @@ pub struct AutomationRunArtifact { pub created_at: String, } -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct AutomationRunLedgerRecord { pub schema_version: u32, pub run_id: String, @@ -187,6 +189,10 @@ pub struct AutomationRunLedgerRecord { pub skipped_count: usize, #[serde(default, skip_serializing_if = "Option::is_none")] pub error: Option, + /// Exhausted retrieval boundary of a `session_evidence_budget_exhausted` + /// skip; present exactly on those skips. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub session_evidence_budget_stage: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub error_classification: Option, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -499,34 +505,18 @@ pub(super) fn canonical_completion_parts( completed_at: &str, completed_at_micros: Option, ) -> Result<(i64, i64)> { - let (completed_at, canonical_micros) = match schema_version { - 1 => parse_schema_v1_rfc3339_micros(completed_at, "completion timestamp")?, - 2 => { - let completed_at = - parse_nonnegative_unix_integer(completed_at, "schema-v2 completion timestamp")?; - let canonical_micros = completed_at.checked_mul(1_000_000).ok_or_else(|| { - config_error("automation completion timestamp overflows signed microseconds") - })?; - (completed_at, canonical_micros) - } - _ => { - return Err(config_error(format!( - "automation run ledger schema version {schema_version} is unsupported" - ))); - } - }; + require_supported_schema(schema_version)?; + let completed_at = parse_nonnegative_unix_integer(completed_at, "completion timestamp")?; + let canonical_micros = completed_at.checked_mul(1_000_000).ok_or_else(|| { + config_error("automation completion timestamp overflows signed microseconds") + })?; let completed_at_micros = completed_at_micros.unwrap_or(canonical_micros); if completed_at_micros < 0 { return Err(config_error( "automation completion timestamp predates the UNIX epoch", )); } - let consistent = match schema_version { - 1 => completed_at_micros == canonical_micros, - 2 => completed_at_micros.div_euclid(1_000_000) == completed_at, - _ => false, - }; - if !consistent { + if completed_at_micros.div_euclid(1_000_000) != completed_at { return Err(config_error( "automation completion timestamp seconds and microseconds disagree", )); @@ -545,11 +535,8 @@ pub fn canonical_record_completion_micros(record: &AutomationRunLedgerRecord) -> .map(|(_, completed_at_micros)| completed_at_micros) } -/// Schema-aware start instant in Unix seconds. -/// -/// Schema v1 rows store RFC3339. Schema v2 rows store nonnegative Unix -/// seconds. Callers that window the ledger, including analytics, must use -/// this instead of assuming one textual form. +/// Validated start instant in Unix seconds. Callers that window the ledger, +/// including analytics, must use this instead of parsing `started_at`. pub fn canonical_record_started_at_seconds( record: &AutomationRunLedgerRecord, label: &str, @@ -562,18 +549,17 @@ pub(super) fn canonical_started_at_seconds( started_at: &str, label: &str, ) -> Result { - match schema_version { - 1 => tracedecay_runtime_core::timeutil::parse_rfc3339_timestamp(started_at).ok_or_else( - || { - config_error(format!( - "automation schema-v1 {label} '{started_at}' is not valid RFC3339" - )) - }, - ), - 2 => parse_nonnegative_unix_integer(started_at, label), - schema_version => Err(config_error(format!( + require_supported_schema(schema_version)?; + parse_nonnegative_unix_integer(started_at, label) +} + +fn require_supported_schema(schema_version: u32) -> Result<()> { + if schema_version == 2 { + Ok(()) + } else { + Err(config_error(format!( "automation run ledger schema version {schema_version} is unsupported" - ))), + ))) } } @@ -589,55 +575,6 @@ pub(super) fn validate_run_ledger_record_semantics( .map(|_| ()) } -fn parse_schema_v1_rfc3339_micros(value: &str, label: &str) -> Result<(i64, i64)> { - let seconds = - tracedecay_runtime_core::timeutil::parse_rfc3339_timestamp(value).ok_or_else(|| { - config_error(format!( - "automation schema-v1 {label} '{value}' is not valid RFC3339" - )) - })?; - let fraction_micros = rfc3339_fraction_micros(value, label)?; - let micros = seconds - .checked_mul(1_000_000) - .and_then(|whole| whole.checked_add(fraction_micros)) - .ok_or_else(|| { - config_error(format!( - "automation schema-v1 {label} overflows signed microseconds" - )) - })?; - Ok((seconds, micros)) -} - -fn rfc3339_fraction_micros(value: &str, label: &str) -> Result { - let Some(dot) = value.find('.') else { - return Ok(0); - }; - let digits = value.as_bytes()[dot + 1..] - .iter() - .take_while(|byte| byte.is_ascii_digit()); - let mut micros = 0_i64; - let mut count = 0_usize; - for digit in digits { - count += 1; - if count <= 6 { - micros = micros * 10 + i64::from(*digit - b'0'); - } else if *digit != b'0' { - return Err(config_error(format!( - "automation schema-v1 {label} has precision finer than exact microseconds" - ))); - } - } - if count == 0 { - return Err(config_error(format!( - "automation schema-v1 {label} has an empty fractional component" - ))); - } - for _ in count..6 { - micros *= 10; - } - Ok(micros) -} - fn parse_nonnegative_unix_integer(value: &str, label: &str) -> Result { match value.parse::() { Ok(seconds) if seconds >= 0 => Ok(seconds), @@ -1287,16 +1224,8 @@ fn is_session_evidence_budget_exhausted_skip( status == AutomationRunStatus::Skipped && session_evidence_budget_exhausted_error } -/// Classifies the legacy exhaustion anchor and its bounded stage-specific -/// successors. Scheduler backoff and ledger projection use this same rule. pub(super) fn is_session_evidence_budget_exhausted_reason(reason: Option<&str>) -> bool { - let Some(reason) = reason else { - return false; - }; - reason == SESSION_EVIDENCE_BUDGET_EXHAUSTED - || reason - .strip_prefix(SESSION_EVIDENCE_BUDGET_EXHAUSTED) - .is_some_and(|suffix| suffix.starts_with('_')) + reason == Some(SESSION_EVIDENCE_BUDGET_EXHAUSTED) } #[hotpath::measure(label = "automation_runtime.run_ledger.scan_task_summary")] @@ -1843,8 +1772,8 @@ mod tests { fn task_summary_keeps_the_budget_exhausted_anchor_visible_past_newer_skips() { let lines = vec![ skipped_session_reflector_line( - "run-budget-stage", - "session_evidence_budget_exhausted_request_candidate_bytes", + "run-budget-exhausted", + SESSION_EVIDENCE_BUDGET_EXHAUSTED, 100, ), skipped_session_reflector_line( @@ -1867,51 +1796,30 @@ mod tests { "run-suppressed" ); let anchor = summary.latest_session_evidence_budget_exhausted().unwrap(); - assert_eq!(anchor.run_id, "run-budget-stage"); - assert_eq!( - anchor.error.as_deref(), - Some("session_evidence_budget_exhausted_request_candidate_bytes") - ); + assert_eq!(anchor.run_id, "run-budget-exhausted"); assert!( summary .records() .iter() - .any(|record| record.run_id == "run-budget-stage"), - "stage-specific budget anchors reach schedule decisions through records()" - ); - } - - #[test] - fn task_summary_reads_legacy_budget_exhaustion_anchors() { - let lines = vec![skipped_session_reflector_line( - "run-budget-legacy", - SESSION_EVIDENCE_BUDGET_EXHAUSTED, - 100, - )]; - let (_temp, path) = write_ledger(&lines); - - let summary = read_run_ledger_task_summary( - &path, - AgentTaskKind::SessionReflector, - "session_reflector", - ) - .unwrap(); - - let anchor = summary.latest_session_evidence_budget_exhausted().unwrap(); - assert_eq!(anchor.run_id, "run-budget-legacy"); - assert_eq!( - anchor.error.as_deref(), - Some(SESSION_EVIDENCE_BUDGET_EXHAUSTED) + .any(|record| record.run_id == "run-budget-exhausted"), + "budget anchors reach schedule decisions through records()" ); } #[test] - fn task_summary_does_not_select_near_prefix_budget_errors() { - let lines = vec![skipped_session_reflector_line( - "run-budget-near-prefix", - "session_evidence_budget_exhaustedX", - 100, - )]; + fn task_summary_selects_only_the_exact_budget_exhausted_token() { + let lines = vec![ + skipped_session_reflector_line( + "run-budget-near-prefix", + "session_evidence_budget_exhaustedX", + 100, + ), + skipped_session_reflector_line( + "run-budget-stage-suffix", + "session_evidence_budget_exhausted_request_candidate_bytes", + 200, + ), + ]; let (_temp, path) = write_ledger(&lines); let summary = read_run_ledger_task_summary( @@ -1919,7 +1827,7 @@ mod tests { AgentTaskKind::SessionReflector, "session_reflector", ) - .expect("near-prefix error must not create a projection/decode mismatch"); + .expect("non-canonical errors must not create a projection/decode mismatch"); assert!(summary.latest_session_evidence_budget_exhausted().is_none()); } diff --git a/crates/tracedecay-automation-runtime/src/automation/run_ledger/exact_lookup.rs b/crates/tracedecay-automation-runtime/src/automation/run_ledger/exact_lookup.rs index 2c2c077394..06fd89f277 100644 --- a/crates/tracedecay-automation-runtime/src/automation/run_ledger/exact_lookup.rs +++ b/crates/tracedecay-automation-runtime/src/automation/run_ledger/exact_lookup.rs @@ -5,6 +5,7 @@ use std::path::Path; use serde::de::DeserializeOwned; use serde_json::Value; use sha2::{Digest, Sha256}; +use tracedecay_contracts::retrieval::SessionRetrievalBudgetStageV1; use tracedecay_domain::ManifestDigest; use tracedecay_domain::canonical_text::encode_tagged_lowercase_hex; @@ -61,6 +62,7 @@ const RECORD_KEYS: &[&str] = &[ "rejected_count", "skipped_count", "error", + "session_evidence_budget_stage", "error_classification", "error_retryable", "backend_attempt_count", @@ -96,8 +98,8 @@ pub(super) struct RunLedgerRowProjection { pub(super) trigger: AutomationTrigger, pub(super) task: AgentTaskKind, pub(super) task_key: Option, - /// True when the row's `error` field is an exact or stage-specific - /// session-evidence budget-exhausted label. The projection compares while + /// True when the row's `error` field is exactly the session-evidence + /// budget-exhausted token. The projection compares while /// streaming instead of capturing the field: failed runs carry /// arbitrarily long backend error messages that must not be bounded or /// allocated here. @@ -1029,6 +1031,7 @@ struct RecordFields { rejected_count: bool, skipped_count: bool, error: bool, + session_evidence_budget_stage: bool, error_classification: bool, error_retryable: bool, backend_attempt_count: bool, @@ -1068,6 +1071,7 @@ impl RecordFields { "rejected_count" => &mut self.rejected_count, "skipped_count" => &mut self.skipped_count, "error" => &mut self.error, + "session_evidence_budget_stage" => &mut self.session_evidence_budget_stage, "error_classification" => &mut self.error_classification, "error_retryable" => &mut self.error_retryable, "backend_attempt_count" => &mut self.backend_attempt_count, @@ -1150,36 +1154,15 @@ struct ArtifactFields { created_at: bool, } -#[derive(Clone, Copy)] -enum StringMatchMode { - Exact, - ExactOrUnderscoreSuffix, -} - -fn compare_decoded_bytes_with_match_mode( +fn compare_decoded_bytes( expected: &[u8], compared: &mut usize, matches: &mut bool, decoded: &[u8], - mode: StringMatchMode, ) { - if *matches && *compared < expected.len() { - let end = compared.saturating_add(decoded.len()); - let matched_len = expected.len().saturating_sub(*compared).min(decoded.len()); - *matches = expected.get(*compared..(*compared + matched_len)) == decoded.get(..matched_len); - if *matches && end > expected.len() { - *matches = match mode { - StringMatchMode::Exact => false, - StringMatchMode::ExactOrUnderscoreSuffix => decoded.get(matched_len) == Some(&b'_'), - }; - } - } else if *matches && *compared == expected.len() && !decoded.is_empty() { - *matches = match mode { - StringMatchMode::Exact => false, - StringMatchMode::ExactOrUnderscoreSuffix => decoded.first() == Some(&b'_'), - }; - } - *compared = compared.saturating_add(decoded.len()); + let end = compared.saturating_add(decoded.len()); + *matches = *matches && expected.get(*compared..end) == Some(decoded); + *compared = end; } impl ArtifactFields { @@ -1320,7 +1303,7 @@ impl<'a> JsonRangeReader<'a> { self.skip_literal(b"null")?; false } else { - self.read_string_equals_or_has_underscore_suffix( + self.read_string_equals( tracedecay_automation::evidence_budget::SESSION_EVIDENCE_BUDGET_EXHAUSTED, )? }; @@ -1433,6 +1416,8 @@ impl<'a> JsonRangeReader<'a> { "automation failure classification", parse_failure_class, ), + "session_evidence_budget_stage" => reader + .validate_optional_enum("session evidence budget stage", parse_budget_stage), "backend_attempts" => reader.validate_retry_attempts(2), "artifacts" => reader.validate_artifacts(2), "completed_at_micros" => reader.validate_optional_i64("completion timestamp"), @@ -2007,14 +1992,6 @@ impl<'a> JsonRangeReader<'a> { } fn read_string_equals(&mut self, expected: &str) -> Result { - self.read_string_matches(expected, StringMatchMode::Exact) - } - - fn read_string_equals_or_has_underscore_suffix(&mut self, expected: &str) -> Result { - self.read_string_matches(expected, StringMatchMode::ExactOrUnderscoreSuffix) - } - - fn read_string_matches(&mut self, expected: &str, mode: StringMatchMode) -> Result { self.expect_byte(b'"', "expected JSON string")?; let expected = expected.as_bytes(); let mut compared = 0_usize; @@ -2025,10 +2002,7 @@ impl<'a> JsonRangeReader<'a> { .ok_or_else(|| config_error("unexpected EOF in JSON string"))?; match byte { b'"' => { - return Ok(matches - && compared >= expected.len() - && (!matches!(mode, StringMatchMode::Exact) - || compared == expected.len())); + return Ok(matches && compared == expected.len()); } b'\\' => { let escaped = self @@ -2036,67 +2010,35 @@ impl<'a> JsonRangeReader<'a> { .ok_or_else(|| config_error("unexpected EOF in JSON escape"))?; match escaped { b'"' | b'\\' | b'/' => { - compare_decoded_bytes_with_match_mode( + compare_decoded_bytes( expected, &mut compared, &mut matches, &[escaped], - mode, ); } - b'b' => compare_decoded_bytes_with_match_mode( - expected, - &mut compared, - &mut matches, - &[8], - mode, - ), + b'b' => compare_decoded_bytes(expected, &mut compared, &mut matches, &[8]), b'f' => { - compare_decoded_bytes_with_match_mode( - expected, - &mut compared, - &mut matches, - &[12], - mode, - ); + compare_decoded_bytes(expected, &mut compared, &mut matches, &[12]); } b'n' => { - compare_decoded_bytes_with_match_mode( - expected, - &mut compared, - &mut matches, - b"\n", - mode, - ); + compare_decoded_bytes(expected, &mut compared, &mut matches, b"\n"); } b'r' => { - compare_decoded_bytes_with_match_mode( - expected, - &mut compared, - &mut matches, - b"\r", - mode, - ); + compare_decoded_bytes(expected, &mut compared, &mut matches, b"\r"); } b't' => { - compare_decoded_bytes_with_match_mode( - expected, - &mut compared, - &mut matches, - b"\t", - mode, - ); + compare_decoded_bytes(expected, &mut compared, &mut matches, b"\t"); } b'u' => { let scalar = self.read_unicode_escape()?; let mut encoded = [0_u8; 4]; let encoded = scalar.encode_utf8(&mut encoded); - compare_decoded_bytes_with_match_mode( + compare_decoded_bytes( expected, &mut compared, &mut matches, encoded.as_bytes(), - mode, ); } _ => return self.fail("invalid JSON string escape"), @@ -2104,24 +2046,17 @@ impl<'a> JsonRangeReader<'a> { } 0x00..=0x1f => return self.fail("unescaped control byte in JSON string"), 0x20..=0x7f => { - compare_decoded_bytes_with_match_mode( - expected, - &mut compared, - &mut matches, - &[byte], - mode, - ); + compare_decoded_bytes(expected, &mut compared, &mut matches, &[byte]); } _ => { let scalar = self.read_utf8_scalar(byte)?; let mut encoded = [0_u8; 4]; let encoded = scalar.encode_utf8(&mut encoded); - compare_decoded_bytes_with_match_mode( + compare_decoded_bytes( expected, &mut compared, &mut matches, encoded.as_bytes(), - mode, ); } } @@ -2433,6 +2368,13 @@ fn parse_task(value: &str) -> Option { } } +fn parse_budget_stage(value: &str) -> Option { + serde::Deserialize::deserialize( + serde::de::value::StrDeserializer::::new(value), + ) + .ok() +} + fn parse_failure_class(value: &str) -> Option<()> { matches!( value, @@ -2522,18 +2464,14 @@ mod tests { } #[test] - fn reads_legacy_row_without_fabricating_completion_precision() { + fn rejects_schema_v1_rfc3339_row() { let line = "{\"schema_version\":1,\"run_id\":\"target\",\"trigger\":\"manual_cli\",\ \"task\":\"memory_curator\",\"backend\":\"codex_app_server\",\"status\":\"succeeded\",\ \"accepted_count\":0,\"rejected_count\":0,\"started_at\":\"1970-01-01T00:00:01Z\",\ \"completed_at\":\"1970-01-01T00:00:02Z\"}"; let (_temp, path) = write_ledger(&[line.to_owned()]); - let record = read_exact_run_record_bounded(&path, "target") - .expect("bounded read") - .expect("legacy record"); - - assert_eq!(record.completed_at_micros, None); + assert!(read_exact_run_record_bounded(&path, "target").is_err()); } #[test] diff --git a/crates/tracedecay-automation-runtime/src/automation/run_ledger/exact_publication.rs b/crates/tracedecay-automation-runtime/src/automation/run_ledger/exact_publication.rs index 25922ca12b..853b63f724 100644 --- a/crates/tracedecay-automation-runtime/src/automation/run_ledger/exact_publication.rs +++ b/crates/tracedecay-automation-runtime/src/automation/run_ledger/exact_publication.rs @@ -1813,8 +1813,7 @@ mod tests { "status": AutomationRunStatus::Skipped, "accepted_count": 0, "rejected_count": 0, - "error": "no_skill_writer_evidence", - "fallback_status": "no_skill_writer_evidence", + "error": "no_session_evidence", "started_at": "1", "completed_at": "2" })) diff --git a/crates/tracedecay-automation-runtime/src/automation/runner.rs b/crates/tracedecay-automation-runtime/src/automation/runner.rs index 3a34654a92..1ec6ff4db6 100644 --- a/crates/tracedecay-automation-runtime/src/automation/runner.rs +++ b/crates/tracedecay-automation-runtime/src/automation/runner.rs @@ -26,7 +26,6 @@ use super::skill_writer::{ validate_skill_proposals, }; use crate::ports::project_runtime::AutomationProjectContext; -use crate::ports::session_store::AutomationSessionStore; use tracedecay_domain::errors::{Result, TraceDecayError}; use tracedecay_global_db::{RegisteredGlobalDb, RegisteredGlobalDbLeaseV1}; use tracedecay_policy::CurationApplyAuthorityV1; @@ -383,6 +382,7 @@ pub async fn run_combined_review_with_backend_and_retrieval_for_retained_settlem /// `Ok(Err(dispatch))` is the not-due answer the caller returns verbatim. async fn acquire_combined_task_lock( config: &AutomationConfig, + executable: Option<&Path>, dashboard_root: &Path, sessions_db: &RegisteredGlobalDb, task: AgentTaskKind, @@ -394,6 +394,7 @@ async fn acquire_combined_task_lock( Some(guard) => { task_run_gate_for_retained_settlement( config, + executable, dashboard_root, sessions_db, task, @@ -402,7 +403,17 @@ async fn acquire_combined_task_lock( ) .await? } - None => task_run_gate(config, dashboard_root, sessions_db, task, trigger).await?, + None => { + task_run_gate( + config, + executable, + dashboard_root, + sessions_db, + task, + trigger, + ) + .await? + } }; Ok(match gate { SchedulerGate::Proceed(lock) => Ok(lock), @@ -495,6 +506,7 @@ fn run_combined_review_for_retrieval_inner<'a>( let sessions_db = project_automation_sessions(cg); let _reflector_lock = match acquire_combined_task_lock( config, + backend.executable(), &dashboard_root, sessions_db.as_ref(), AgentTaskKind::SessionReflector, @@ -509,6 +521,7 @@ fn run_combined_review_for_retrieval_inner<'a>( }; let _skill_lock = match acquire_combined_task_lock( config, + backend.executable(), &dashboard_root, sessions_db.as_ref(), AgentTaskKind::SkillWriter, @@ -599,6 +612,7 @@ fn run_combined_review_for_retrieval_inner<'a>( &reflector_run_id, options.trigger, config, + backend.executable(), AgentTaskKind::SessionReflector, &started_at, input_hash.clone(), @@ -610,6 +624,7 @@ fn run_combined_review_for_retrieval_inner<'a>( &skill_run_id, options.trigger, config, + backend.executable(), AgentTaskKind::SkillWriter, &started_at, input_hash, diff --git a/crates/tracedecay-automation-runtime/src/automation/runner/curation.rs b/crates/tracedecay-automation-runtime/src/automation/runner/curation.rs index eebe36f113..693698f7fe 100644 --- a/crates/tracedecay-automation-runtime/src/automation/runner/curation.rs +++ b/crates/tracedecay-automation-runtime/src/automation/runner/curation.rs @@ -1,4 +1,5 @@ use serde_json::{Value, json}; +use tracedecay_contracts::retrieval::SessionRetrievalBudgetStageV1; use tracedecay_domain::{ManifestDigest, canonical_sha256}; use tracedecay_policy::{ CurationApplyAuthorityV1, CurationApplyDecisionV1, CurationApplyPolicyInputV1, @@ -114,6 +115,7 @@ pub(super) fn unpersisted_rejected_parts( config: &AutomationConfig, task: AgentTaskKind, reason: &str, + budget_stage: Option, evidence_hash: Option, report_task: &'static str, ) -> (Value, AutomationRunLedgerRecord) { @@ -132,7 +134,11 @@ pub(super) fn unpersisted_rejected_parts( task, task_key: Some(task_key(task).to_string()), backend: config.backend.as_str().to_string(), - backend_identity: crate::automation::backend_identity::backend_identity(config).ok(), + backend_identity: crate::automation::backend_identity::backend_identity( + config, + run.executable(), + ) + .ok(), host_mode: Some(config.host_mode.as_str().to_string()), prompt_version: Some(prompt_version(task).to_string()), response_schema: Some(contract.response_schema), @@ -151,11 +157,12 @@ pub(super) fn unpersisted_rejected_parts( rejected_count: 0, skipped_count: 1, error: Some(reason.to_string()), + session_evidence_budget_stage: budget_stage, error_classification: None, error_retryable: None, backend_attempt_count: 0, backend_attempts: Vec::new(), - fallback_status: Some(reason.to_string()), + fallback_status: None, report_ref: Some(json!({ "dashboard_runs": "/api/automation/runs", "run_id": run.run_id, diff --git a/crates/tracedecay-automation-runtime/src/automation/runner/evidence.rs b/crates/tracedecay-automation-runtime/src/automation/runner/evidence.rs index 19ae3d5f58..866ea1b49a 100644 --- a/crates/tracedecay-automation-runtime/src/automation/runner/evidence.rs +++ b/crates/tracedecay-automation-runtime/src/automation/runner/evidence.rs @@ -1,5 +1,7 @@ use serde::Serialize; use serde_json::{Value, json}; +use tracedecay_contracts::retained_surfaces::AutomationSkipReasonV1; +use tracedecay_contracts::retrieval::SessionRetrievalBudgetStageV1; use tracedecay_domain::TemporalCoverageCountsV1; use tracedecay_lcm::{LcmGrepHit, LcmGrepSort, LcmScope}; @@ -14,16 +16,16 @@ use crate::automation::skill_usage::{ use crate::automation::skill_writer::{ skill_improvement_recommendations, support_file_evidence as skill_writer_support_file_evidence, }; -use crate::ports::session_store::AutomationSessionStore; use std::collections::{BTreeMap, BTreeSet}; use std::path::PathBuf; use tracedecay_automation::analytics::{ToolUsageObservation, underused_tool_family_signals}; use tracedecay_automation::text::truncate_chars_for_prompt; use tracedecay_domain::errors::Result; +use tracedecay_global_db::RegisteredGlobalDb; use tracedecay_runtime_core::tracedecay::current_timestamp; use super::retrieval::{ - AutomationSessionRetrieval, AutomationTemporalRetrieval, automation_structural_refusal_reason, + AutomationSessionRetrieval, AutomationTemporalRetrieval, automation_structural_refusal_skip, retrieve_automation_session_evidence, }; use super::session_reflector::{ @@ -140,6 +142,7 @@ pub(super) enum SkillWriterEvidenceOutcome { Ready(SkillWriterEvidenceBundle), Skipped { reason: &'static str, + budget_stage: Option, evidence_hash: Option, }, } @@ -153,6 +156,7 @@ pub(super) enum SessionReflectorEvidenceOutcome { Ready(SessionReflectorEvidenceBundle), Skipped { reason: &'static str, + budget_stage: Option, evidence_hash: Option, }, } @@ -637,6 +641,7 @@ pub(super) async fn build_session_reflector_evidence( { return Ok(SessionReflectorEvidenceOutcome::Skipped { reason: "session_evidence_filter_unavailable", + budget_stage: None, evidence_hash: None, }); } @@ -670,6 +675,7 @@ pub(super) async fn build_session_reflector_evidence( Err(reason) => { return Ok(SessionReflectorEvidenceOutcome::Skipped { reason, + budget_stage: None, evidence_hash: None, }); } @@ -685,12 +691,15 @@ pub(super) async fn build_session_reflector_evidence( AutomationTemporalRetrieval::Rejected(reason) => { return Ok(SessionReflectorEvidenceOutcome::Skipped { reason, + budget_stage: None, evidence_hash: None, }); } AutomationTemporalRetrieval::StructuralRefusal(refusal) => { + let (reason, budget_stage) = automation_structural_refusal_skip(refusal); return Ok(SessionReflectorEvidenceOutcome::Skipped { - reason: automation_structural_refusal_reason(refusal), + reason, + budget_stage, evidence_hash: None, }); } @@ -730,6 +739,7 @@ pub(super) async fn build_session_reflector_evidence( if !has_grep_hits && !has_replay_sessions { return Ok(SessionReflectorEvidenceOutcome::Skipped { reason: "no_session_evidence", + budget_stage: None, evidence_hash, }); } @@ -745,7 +755,7 @@ pub(super) async fn build_session_reflector_evidence( pub(super) async fn build_skill_writer_evidence( retrieval: &dyn AutomationSessionRetrieval, analytics_project_root: Option<&std::path::Path>, - analytics_db: Option<&dyn AutomationSessionStore>, + analytics_db: Option<&RegisteredGlobalDb>, options: SkillWriterAutomationOptions, ) -> Result { let profile_root = match options.profile_root { @@ -778,6 +788,7 @@ pub(super) async fn build_skill_writer_evidence( Err(reason) => { return Ok(SkillWriterEvidenceOutcome::Skipped { reason, + budget_stage: None, evidence_hash: None, }); } @@ -793,12 +804,15 @@ pub(super) async fn build_skill_writer_evidence( AutomationTemporalRetrieval::Rejected(reason) => { return Ok(SkillWriterEvidenceOutcome::Skipped { reason, + budget_stage: None, evidence_hash: None, }); } AutomationTemporalRetrieval::StructuralRefusal(refusal) => { + let (reason, budget_stage) = automation_structural_refusal_skip(refusal); return Ok(SkillWriterEvidenceOutcome::Skipped { - reason: automation_structural_refusal_reason(refusal), + reason, + budget_stage, evidence_hash: None, }); } @@ -819,7 +833,8 @@ pub(super) async fn build_skill_writer_evidence( .is_none_or(std::vec::Vec::is_empty) { return Ok(SkillWriterEvidenceOutcome::Skipped { - reason: "no_skill_writer_evidence", + reason: AutomationSkipReasonV1::NoSessionEvidence.as_str(), + budget_stage: None, evidence_hash: Some(canonical_evidence_hash(&json!({ "evidence_mode": evidence_mode_label(recent_session_slices.is_some()), "temporal_mode": "forensic", @@ -831,7 +846,6 @@ pub(super) async fn build_skill_writer_evidence( }))?), }); } - crate::automation::managed_skills::migrate_managed_skill_routing(&profile_root).await?; let existing_skills = list_managed_skills(&profile_root).await?; if let (Some(project_root), Some(analytics_db)) = (analytics_project_root, analytics_db) { ingest_project_analytics_events( @@ -911,7 +925,8 @@ pub(super) async fn build_skill_writer_evidence( .is_some_and(|sessions| !sessions.is_empty()); if !has_grep_hits && !has_replay_sessions { return Ok(SkillWriterEvidenceOutcome::Skipped { - reason: "no_skill_writer_evidence", + reason: AutomationSkipReasonV1::NoSessionEvidence.as_str(), + budget_stage: None, evidence_hash, }); } diff --git a/crates/tracedecay-automation-runtime/src/automation/runner/retrieval.rs b/crates/tracedecay-automation-runtime/src/automation/runner/retrieval.rs index 3877f5bc39..f28575d8ac 100644 --- a/crates/tracedecay-automation-runtime/src/automation/runner/retrieval.rs +++ b/crates/tracedecay-automation-runtime/src/automation/runner/retrieval.rs @@ -13,7 +13,10 @@ use std::time::Duration; use serde_json::Value; use sha2::{Digest, Sha256}; -use tracedecay_contracts::retrieval::SessionRetrievalStructuralRefusalV1; +use tracedecay_contracts::retained_surfaces::AutomationSkipReasonV1; +use tracedecay_contracts::retrieval::{ + SessionRetrievalBudgetStageV1, SessionRetrievalStructuralRefusalV1, +}; use tracedecay_contracts::{ CancellationContext, CapabilityGrantId, CapabilityGrantSnapshot, Deadline, DisclosureClass, ProfileIdentityReadPort, RequestContext, RequestId, @@ -29,9 +32,10 @@ use tracedecay_contracts::request_identity::{GlobalRequestSurface, mint_global_r use tracedecay_domain::errors::{Result, TraceDecayError}; use tracedecay_global_db::{RegisteredGlobalDb, RegisteredGlobalDbLeaseV1}; use tracedecay_lcm::LcmScope; +use tracedecay_runtime_core::cancellation::CancellationToken; use tracedecay_session_memory::context::{ - BranchId, CancellationToken, CapabilityDigest, ConfigurationDigest, PolicyDigest, ProfileId, - RequestBudgets, ResolvedGitRoute, ResolvedSessionIdentity, SessionRootId, SessionStoreId, + BranchId, CapabilityDigest, ConfigurationDigest, PolicyDigest, ProfileId, RequestBudgets, + ResolvedGitRoute, ResolvedSessionIdentity, SessionRootId, SessionStoreId, application_observed_at, session_application_grant_digest, }; use tracedecay_session_memory::session::{ @@ -44,7 +48,7 @@ use tracedecay_session_memory::session::{ use tracedecay_session_temporal_store::RegisteredGlobalDbSessionTemporalExecution; use tracedecay_temporal_query::TemporalKernelResult; use tracedecay_temporal_query::context::{ContextBudget, TokenPolicy, VersionedTokenEstimator}; -use tracedecay_temporal_query::ports::ExecutionLimits; +use tracedecay_temporal_query::execution::ExecutionLimits; use tracedecay_temporal_query::ranking::{DiversityLimits, RankedCandidate}; pub(super) const AUTOMATION_SESSION_MAX_BYTES: u64 = 2 * 1024 * 1024; @@ -566,81 +570,18 @@ pub(super) fn ranked_evidence_owner(ranked: &RankedCandidate) -> Option<(&str, & )) } -pub(super) const fn automation_structural_refusal_reason( +pub(super) const fn automation_structural_refusal_skip( refusal: SessionRetrievalStructuralRefusalV1, -) -> &'static str { +) -> (&'static str, Option) { match refusal { - SessionRetrievalStructuralRefusalV1::CursorManifestLimitExceeded { - kind: tracedecay_domain::CursorManifestLimitKindV1::Participants, - .. - } => "session_cursor_manifest_participants_limit_exceeded", - SessionRetrievalStructuralRefusalV1::CursorManifestLimitExceeded { - kind: tracedecay_domain::CursorManifestLimitKindV1::CanonicalBytes, - .. - } => "session_cursor_manifest_canonical_bytes_limit_exceeded", - SessionRetrievalStructuralRefusalV1::BudgetExhausted { stage, .. } => { - automation_budget_refusal_reason(stage) - } - } -} - -const fn automation_budget_refusal_reason( - stage: tracedecay_contracts::retrieval::SessionRetrievalBudgetStageV1, -) -> &'static str { - use tracedecay_contracts::retrieval::SessionRetrievalBudgetStageV1; - - match stage { - SessionRetrievalBudgetStageV1::RequestResultLimit => { - "session_evidence_budget_exhausted_request_result_limit" - } - SessionRetrievalBudgetStageV1::RequestHydrationLimit => { - "session_evidence_budget_exhausted_request_hydration_limit" - } - SessionRetrievalBudgetStageV1::RequestContextBytes => { - "session_evidence_budget_exhausted_request_context_bytes" - } - SessionRetrievalBudgetStageV1::RequestCandidateBytes => { - "session_evidence_budget_exhausted_request_candidate_bytes" - } - SessionRetrievalBudgetStageV1::RequestRecordBytes => { - "session_evidence_budget_exhausted_request_record_bytes" - } - SessionRetrievalBudgetStageV1::RequestHydrationBytes => { - "session_evidence_budget_exhausted_request_hydration_bytes" - } - SessionRetrievalBudgetStageV1::EstimatorVersionMismatch => { - "session_evidence_budget_exhausted_estimator_version_mismatch" - } - SessionRetrievalBudgetStageV1::ExecutionWorkExhausted => { - "session_evidence_budget_exhausted_execution_work_exhausted" - } - SessionRetrievalBudgetStageV1::CandidateReadExhausted => { - "session_evidence_budget_exhausted_candidate_read_exhausted" - } - SessionRetrievalBudgetStageV1::RecordReadExhausted => { - "session_evidence_budget_exhausted_record_read_exhausted" - } - SessionRetrievalBudgetStageV1::KernelResultLimit => { - "session_evidence_budget_exhausted_kernel_result_limit" - } - SessionRetrievalBudgetStageV1::CursorManifestLimit => { - "session_evidence_budget_exhausted_cursor_manifest_limit" - } - SessionRetrievalBudgetStageV1::ParticipantManifestParticipants => { - "session_evidence_budget_exhausted_participant_manifest_participants" - } - SessionRetrievalBudgetStageV1::ParticipantManifestCanonicalBytes => { - "session_evidence_budget_exhausted_participant_manifest_canonical_bytes" - } - SessionRetrievalBudgetStageV1::HydrationBytes => { - "session_evidence_budget_exhausted_hydration_bytes" - } - SessionRetrievalBudgetStageV1::ContextBytes => { - "session_evidence_budget_exhausted_context_bytes" - } - SessionRetrievalBudgetStageV1::ContextTokens => { - "session_evidence_budget_exhausted_context_tokens" - } + SessionRetrievalStructuralRefusalV1::CursorManifestLimitExceeded { .. } => ( + AutomationSkipReasonV1::SessionCursorManifestLimitExceeded.as_str(), + None, + ), + SessionRetrievalStructuralRefusalV1::BudgetExhausted { stage, .. } => ( + AutomationSkipReasonV1::SessionEvidenceBudgetExhausted.as_str(), + Some(stage), + ), } } diff --git a/crates/tracedecay-automation-runtime/src/automation/runner/session_reflector.rs b/crates/tracedecay-automation-runtime/src/automation/runner/session_reflector.rs index 262ae16e3f..d69d8ef96d 100644 --- a/crates/tracedecay-automation-runtime/src/automation/runner/session_reflector.rs +++ b/crates/tracedecay-automation-runtime/src/automation/runner/session_reflector.rs @@ -2,6 +2,7 @@ use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use std::path::PathBuf; +use tracedecay_contracts::retrieval::SessionRetrievalBudgetStageV1; use tracedecay_store::ProjectMemoryFactStore; use crate::automation::automatic_facts::{ @@ -335,6 +336,7 @@ pub(super) fn rejected_session_reflector_run( run: &AgentTaskRunContext<'_>, config: &AutomationConfig, reason: &str, + budget_stage: Option, evidence_hash: Option, ) -> SessionReflectorAutomationRun { let (report, record) = unpersisted_rejected_parts( @@ -342,6 +344,7 @@ pub(super) fn rejected_session_reflector_run( config, AgentTaskKind::SessionReflector, reason, + budget_stage, evidence_hash, "session_reflector", ); @@ -682,6 +685,7 @@ fn run_session_reflector_for_store_with_publication_inner<'a, A: ProjectMemoryFa "session_reflector", options.trigger, config, + backend.executable(), AgentTaskKind::SessionReflector, ) .with_ledger_publication(ledger_publication) @@ -700,12 +704,14 @@ fn run_session_reflector_for_store_with_publication_inner<'a, A: ProjectMemoryFa SessionReflectorEvidenceOutcome::Ready(bundle) => bundle, SessionReflectorEvidenceOutcome::Skipped { reason, + budget_stage, evidence_hash, } => { return Ok(rejected_session_reflector_run( &run, config, reason, + budget_stage, evidence_hash, )); } diff --git a/crates/tracedecay-automation-runtime/src/automation/runner/skill_writer.rs b/crates/tracedecay-automation-runtime/src/automation/runner/skill_writer.rs index 8a989f1ce6..8e1409fbaa 100644 --- a/crates/tracedecay-automation-runtime/src/automation/runner/skill_writer.rs +++ b/crates/tracedecay-automation-runtime/src/automation/runner/skill_writer.rs @@ -4,7 +4,7 @@ use serde::{Deserialize, Serialize}; use serde_json::Value; use crate::automation::artifacts::sha256_json; -use crate::automation::backend::{AgentTaskKind, AgentTaskResponse}; +use crate::automation::backend::{AgentTaskBackend, AgentTaskKind, AgentTaskResponse}; use crate::automation::config::AutomationConfig; use crate::automation::host_io::HostIo; use crate::automation::lifecycle::{ @@ -12,6 +12,7 @@ use crate::automation::lifecycle::{ AutomationRunSettlementGuard, RetainedAutomationRun, }; use crate::automation::run_ledger::{AutomationRunLedgerRecord, AutomationTrigger}; +use tracedecay_contracts::retrieval::SessionRetrievalBudgetStageV1; use tracedecay_domain::errors::{Result, TraceDecayError}; use super::curation::unpersisted_rejected_parts; @@ -288,6 +289,7 @@ fn run_skill_writer_for_store_with_publication_inner<'a>( "skill_writer", options.trigger, config, + backend.executable(), AgentTaskKind::SkillWriter, ) .with_ledger_publication(ledger_publication) @@ -305,7 +307,7 @@ fn run_skill_writer_for_store_with_publication_inner<'a>( None => match build_skill_writer_evidence( retrieval, analytics_project_root, - analytics_db.map(|database| database as &dyn AutomationSessionStore), + analytics_db, options, ) .await? @@ -313,12 +315,14 @@ fn run_skill_writer_for_store_with_publication_inner<'a>( SkillWriterEvidenceOutcome::Ready(bundle) => bundle, SkillWriterEvidenceOutcome::Skipped { reason, + budget_stage, evidence_hash, } => { return Ok(rejected_skill_writer_run( &run, config, reason, + budget_stage, evidence_hash, )); } @@ -894,6 +898,7 @@ pub(super) fn rejected_skill_writer_run( run: &AgentTaskRunContext<'_>, config: &AutomationConfig, reason: &str, + budget_stage: Option, evidence_hash: Option, ) -> SkillWriterAutomationRun { let (report, record) = unpersisted_rejected_parts( @@ -901,6 +906,7 @@ pub(super) fn rejected_skill_writer_run( config, AgentTaskKind::SkillWriter, reason, + budget_stage, evidence_hash, "skill_writer", ); diff --git a/crates/tracedecay-automation-runtime/src/automation/runner/tests.rs b/crates/tracedecay-automation-runtime/src/automation/runner/tests.rs index 1d4306f203..1dbc815c25 100644 --- a/crates/tracedecay-automation-runtime/src/automation/runner/tests.rs +++ b/crates/tracedecay-automation-runtime/src/automation/runner/tests.rs @@ -12,9 +12,10 @@ use tracedecay_domain::{ ActorId, FactOwnerV1, ProjectId, RepositoryId, RetrievalAnchorId, RetrievalGrainV1, SessionId, TemporalCoverageCountsV1, UtcMicros, WorktreeId, }; +use tracedecay_runtime_core::cancellation::CancellationToken; use tracedecay_session_memory::context::{ - BranchId, CancellationToken, CapabilityDigest, ConfigurationDigest, PolicyDigest, ProfileId, - RequestBudgets, ResolvedGitRoute, ResolvedSessionIdentity, SessionRootId, SessionStoreId, + BranchId, CapabilityDigest, ConfigurationDigest, PolicyDigest, ProfileId, RequestBudgets, + ResolvedGitRoute, ResolvedSessionIdentity, SessionRootId, SessionStoreId, session_application_grant_digest, }; use tracedecay_session_memory::memory::MemoryApplication; @@ -27,7 +28,7 @@ use tracedecay_session_memory::session::{ }; use tracedecay_temporal_query::TemporalKernelResult; use tracedecay_temporal_query::context::VersionedTokenEstimator; -use tracedecay_temporal_query::ports::ExecutionLimits; +use tracedecay_temporal_query::execution::ExecutionLimits; use tracedecay_temporal_query::ranking::RankedCandidate; use tracedecay_tool_catalog::{CapabilityId, UseCaseId}; @@ -40,7 +41,7 @@ use super::evidence::{ }; use super::retrieval::{ AUTOMATION_SESSION_MAX_BYTES, AutomationWordEstimator, accept_automation_temporal_outcome, - automation_structural_refusal_reason, ranked_evidence_owner, + automation_structural_refusal_skip, ranked_evidence_owner, retrieve_automation_session_evidence, }; use super::{ @@ -112,7 +113,11 @@ fn asymmetric_combined_failure_preserves_the_successful_sibling_record() { assert_eq!(failure.reflector_record, Some(record)); assert!(failure.reflector_error.is_none()); assert!(failure.skill_writer_record.is_none()); - assert!(failure.skill_writer_error.is_some()); + assert!(matches!( + failure.skill_writer_error, + Some(tracedecay_domain::errors::TraceDecayError::Config { ref message }) + if message == "skill terminal construction failed" + )); } #[test] @@ -521,18 +526,16 @@ fn temporal_automation_evidence_fails_closed_for_non_complete_outcomes() { #[test] fn temporal_automation_evidence_preserves_cursor_manifest_refusal() { - for (kind, observed, maximum, expected_reason) in [ + for (kind, observed, maximum) in [ ( tracedecay_domain::CursorManifestLimitKindV1::Participants, 257, 256, - "session_cursor_manifest_participants_limit_exceeded", ), ( tracedecay_domain::CursorManifestLimitKindV1::CanonicalBytes, 65_537, 65_536, - "session_cursor_manifest_canonical_bytes_limit_exceeded", ), ] { let actual = accept_automation_temporal_outcome(SessionRetrievalOutcome::< @@ -554,8 +557,8 @@ fn temporal_automation_evidence_preserves_cursor_manifest_refusal() { } ); assert_eq!( - automation_structural_refusal_reason(refusal), - expected_reason + automation_structural_refusal_skip(refusal), + ("session_cursor_manifest_limit_exceeded", None) ); } } @@ -627,6 +630,16 @@ async fn combined_reflector_first_preserves_budget_stage_for_sequential_fallback ) .await .expect("structural refusal is a terminal skip"); + assert!(matches!( + evidence, + super::evidence::SessionReflectorEvidenceOutcome::Skipped { + reason: "session_evidence_budget_exhausted", + budget_stage: Some( + tracedecay_contracts::retrieval::SessionRetrievalBudgetStageV1::RequestCandidateBytes + ), + .. + } + )); let dispatch = match combined_reflector_evidence_or_not_combined(evidence) { Ok(_) => panic!("reflector refusal must fall back to the per-task runner"), @@ -636,7 +649,7 @@ async fn combined_reflector_first_preserves_budget_stage_for_sequential_fallback assert!(matches!( dispatch, CombinedReviewDispatch::NotCombined { - reason: "session_evidence_budget_exhausted_request_candidate_bytes", + reason: "session_evidence_budget_exhausted", } )); assert_eq!(retrieval.calls.load(Ordering::SeqCst), 1); @@ -665,6 +678,16 @@ async fn combined_skill_second_preserves_distinct_budget_stage_for_sequential_fa ) .await .expect("structural refusal is a terminal skip"); + assert!(matches!( + evidence, + super::evidence::SkillWriterEvidenceOutcome::Skipped { + reason: "session_evidence_budget_exhausted", + budget_stage: Some( + tracedecay_contracts::retrieval::SessionRetrievalBudgetStageV1::ExecutionWorkExhausted + ), + .. + } + )); let dispatch = match combined_skill_writer_evidence_or_not_combined(evidence) { Ok(_) => panic!("skill refusal must fall back to the per-task runner"), @@ -674,7 +697,7 @@ async fn combined_skill_second_preserves_distinct_budget_stage_for_sequential_fa assert!(matches!( dispatch, CombinedReviewDispatch::NotCombined { - reason: "session_evidence_budget_exhausted_execution_work_exhausted", + reason: "session_evidence_budget_exhausted", } )); assert_eq!(retrieval.calls.load(Ordering::SeqCst), 1); diff --git a/crates/tracedecay-automation-runtime/src/automation/runner/tests/early_gate.rs b/crates/tracedecay-automation-runtime/src/automation/runner/tests/early_gate.rs index ec7b66ba42..7105e30f82 100644 --- a/crates/tracedecay-automation-runtime/src/automation/runner/tests/early_gate.rs +++ b/crates/tracedecay-automation-runtime/src/automation/runner/tests/early_gate.rs @@ -64,6 +64,10 @@ impl AgentTaskBackend for RecordingBackend { self.calls.fetch_add(1, Ordering::SeqCst); panic!("scheduled-disabled automation must not invoke its backend") } + + fn executable(&self) -> Option<&std::path::Path> { + None + } } fn scheduled_disabled_config() -> AutomationConfig { diff --git a/crates/tracedecay-automation-runtime/src/automation/scheduler.rs b/crates/tracedecay-automation-runtime/src/automation/scheduler.rs index 477990f7db..af528e35b9 100644 --- a/crates/tracedecay-automation-runtime/src/automation/scheduler.rs +++ b/crates/tracedecay-automation-runtime/src/automation/scheduler.rs @@ -26,9 +26,10 @@ use super::run_ledger::{ canonical_record_started_at_seconds, is_session_evidence_budget_exhausted_reason, latest_record_by_canonical_completion, latest_record_by_canonical_completion_key, }; -use crate::ports::session_store::AutomationSessionStore; use tracedecay_contracts::retained_surfaces::AutomationSkipReasonV1; use tracedecay_domain::errors::{Result, TraceDecayError}; +use tracedecay_global_db::RegisteredGlobalDb; +use tracedecay_private_fs::FileLease; const DEFAULT_FAILURE_COOLDOWN_SECS: u64 = 300; const DEFAULT_STALE_LOCK_SECS: u64 = 6 * 60 * 60; @@ -69,12 +70,23 @@ impl SessionActivity { /// /// This reads from the read-only store using bounded indexed timestamp lookups, /// so it is cheap and race-safe to call from every scheduler tick; concurrent -/// ingest writers only ever move the value forward. +/// ingest writers only ever move the value forward. A failed read is logged and +/// reported as no activity: a store the scheduler cannot read has no observable +/// new activity, so automation stays idle instead of running against it. #[hotpath::measure(label = "automation.run.load_session_activity", future = true)] -pub async fn load_session_activity(sessions_db: &dyn AutomationSessionStore) -> SessionActivity { - SessionActivity { - last_activity_secs: sessions_db.latest_session_activity_secs().await, - } +pub async fn load_session_activity(sessions_db: &RegisteredGlobalDb) -> SessionActivity { + let last_activity_secs = match sessions_db.latest_session_activity_secs().await { + Ok(latest) => latest, + Err(error) => { + tracing::warn!( + database = %sessions_db.db_path().display(), + %error, + "session-activity read failed; scheduler observes no new activity" + ); + None + } + }; + SessionActivity { last_activity_secs } } /// Consecutive project-open failures after which one scheduler loop exits. @@ -349,36 +361,52 @@ where } #[hotpath::measure(label = "automation.scheduler.decision")] +/// Decides whether `task` is due under `config`, given the ledger `records` +/// and the executable the backend now in force would spawn +/// (`AgentTaskBackend::executable`); the latter is part of the backend +/// identity a settled deterministic failure is judged against. pub fn schedule_decision( config: &AutomationConfig, + executable: Option<&Path>, task: AgentTaskKind, records: &[AutomationRunLedgerRecord], activity: SessionActivity, now_secs: i64, ) -> AutomationScheduleDecision { - schedule_decision_or_history_denial(config, task, records, activity, now_secs, true) + schedule_decision_or_history_denial(config, executable, task, records, activity, now_secs, true) } pub fn host_receipt_decision( config: &AutomationConfig, + executable: Option<&Path>, task: AgentTaskKind, records: &[AutomationRunLedgerRecord], activity: SessionActivity, now_secs: i64, ) -> AutomationScheduleDecision { - schedule_decision_or_history_denial(config, task, records, activity, now_secs, false) + schedule_decision_or_history_denial( + config, executable, task, records, activity, now_secs, false, + ) } fn schedule_decision_or_history_denial( config: &AutomationConfig, + executable: Option<&Path>, task: AgentTaskKind, records: &[AutomationRunLedgerRecord], activity: SessionActivity, now_secs: i64, enforce_schedule: bool, ) -> AutomationScheduleDecision { - match schedule_decision_for_trigger(config, task, records, activity, now_secs, enforce_schedule) - { + match schedule_decision_for_trigger( + config, + executable, + task, + records, + activity, + now_secs, + enforce_schedule, + ) { Ok(decision) => decision, Err(_) => { AutomationScheduleDecision::skipped(AutomationSkipReasonV1::SchedulerHistoryInvalid) @@ -388,6 +416,7 @@ fn schedule_decision_or_history_denial( fn schedule_decision_for_trigger( config: &AutomationConfig, + executable: Option<&Path>, task: AgentTaskKind, records: &[AutomationRunLedgerRecord], activity: SessionActivity, @@ -503,7 +532,7 @@ fn schedule_decision_for_trigger( ); // Identity-stand first: a deterministic failure stamped under the // current backend stays suppressed until that identity changes. - match deterministic_backend_failure_standing(record, config) { + match deterministic_backend_failure_standing(record, config, executable) { Ok(BackendFailureStanding::Stands) => { return Ok(AutomationScheduleDecision::skipped( AutomationSkipReasonV1::BackendIdentitySuppressed, @@ -647,6 +676,7 @@ enum BackendFailureStanding { fn deterministic_backend_failure_standing( record: &AutomationRunLedgerRecord, config: &AutomationConfig, + executable: Option<&Path>, ) -> Result { let Some(classification) = record.error_classification else { return Ok(BackendFailureStanding::NotSettled); @@ -657,7 +687,7 @@ fn deterministic_backend_failure_standing( let Some(recorded_identity) = record.backend_identity.as_deref() else { return Ok(BackendFailureStanding::NotSettled); }; - if backend_identity(config)? != recorded_identity { + if backend_identity(config, executable)? != recorded_identity { return Ok(BackendFailureStanding::IdentityChanged); } let ladder_reproduced_the_class = !record.backend_attempts.is_empty() @@ -1204,7 +1234,7 @@ fn remove_owned_task_lock_blocking(path: &Path, ownership_token: &str) -> std::i tracedecay_runtime_core::storage::PrivateStoreIo::remove_file_durable(path).map(|_| ()) } -fn acquire_task_lock_coordination(path: &Path) -> std::io::Result { +fn acquire_task_lock_coordination(path: &Path) -> std::io::Result { let path = resolve_task_lock_parent(path)?; let coordination_path = tracedecay_runtime_core::storage::append_lock_path(&path); tracedecay_runtime_core::storage::reject_symlink_components( @@ -1252,6 +1282,7 @@ fn acquire_task_lock_coordination(path: &Path) -> std::io::Result file }; file.lock()?; + let file = FileLease::held(file, "automation.task_lock.coordination"); file.sync_all()?; tracedecay_private_fs::framed_log::sync_parent_directory( &coordination_path, @@ -1617,6 +1648,7 @@ mod tests { backend_attempt_count: 0, backend_attempts: Vec::new(), fallback_status: None, + session_evidence_budget_stage: None, report_ref: None, artifacts: Vec::new(), started_at: completed_at.to_string(), @@ -1649,6 +1681,7 @@ mod tests { /// every attempt, stamped with `identity`. fn settled_backend_failure( config: &AutomationConfig, + executable: Option<&Path>, error: &str, classification: AgentTaskFailureClass, attempts: u32, @@ -1673,7 +1706,7 @@ mod tests { backoff_millis: 0, }) .collect(); - record.backend_identity = identity.or_else(|| backend_identity(config).ok()); + record.backend_identity = identity.or_else(|| backend_identity(config, executable).ok()); record } @@ -1683,13 +1716,10 @@ disconnected: config error: codex app-server closed stdout before completing"; #[test] fn deterministic_backend_failure_settles_once_and_never_relaunches() { - // The executable override is process-global. Keep the stamped - // identity and every later suppression read under one environment - // lock so the same-path replacement test cannot interleave them. - let _env_lock = tracedecay_runtime_core::config::lock_user_data_dir_test_env(); let config = curator_config(); let records = vec![settled_backend_failure( &config, + None, PERMANENT_PROTOCOL_ERROR, AgentTaskFailureClass::Permanent, 3, @@ -1709,6 +1739,7 @@ disconnected: config error: codex app-server closed stdout before completing"; assert_eq!( schedule_decision( &config, + None, AgentTaskKind::MemoryCurator, &records, SessionActivity::none(), @@ -1727,6 +1758,7 @@ disconnected: config error: codex app-server closed stdout before completing"; let config = curator_config(); let mut record = settled_backend_failure( &config, + None, DISCONNECT_ERROR, AgentTaskFailureClass::Disconnected, 3, @@ -1739,6 +1771,7 @@ disconnected: config error: codex app-server closed stdout before completing"; assert!( schedule_decision( &config, + None, AgentTaskKind::MemoryCurator, &records, SessionActivity::none(), @@ -1758,10 +1791,11 @@ evidence about it", let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("codex-backend"); std::fs::write(&path, b"backend-revision-one").unwrap(); - let _env = super::super::backend_identity::CodexBinEnvGuard::set(&path); + let executable = Some(path.as_path()); let config = curator_config(); let records = vec![settled_backend_failure( &config, + executable, PERMANENT_PROTOCOL_ERROR, AgentTaskFailureClass::Permanent, 3, @@ -1772,6 +1806,7 @@ evidence about it", assert_eq!( schedule_decision( &config, + executable, AgentTaskKind::MemoryCurator, &records, SessionActivity::none(), @@ -1785,6 +1820,7 @@ evidence about it", assert!( schedule_decision( &config, + executable, AgentTaskKind::MemoryCurator, &records, SessionActivity::none(), @@ -1818,6 +1854,7 @@ evidence about it", ] { let records = vec![settled_backend_failure( &config, + None, error, classification, 3, @@ -1829,6 +1866,7 @@ evidence about it", assert_eq!( schedule_decision( &config, + None, AgentTaskKind::MemoryCurator, &records, SessionActivity::none(), @@ -1843,6 +1881,7 @@ evidence about it", assert!( schedule_decision( &config, + None, AgentTaskKind::MemoryCurator, &records, SessionActivity::none(), @@ -1859,6 +1898,7 @@ evidence about it", let config = curator_config(); let mut record = settled_backend_failure( &config, + None, DISCONNECT_ERROR, AgentTaskFailureClass::Disconnected, 3, @@ -1873,6 +1913,7 @@ evidence about it", assert!( schedule_decision( &config, + None, AgentTaskKind::MemoryCurator, &records, SessionActivity::none(), @@ -1911,59 +1952,41 @@ evidence about it", } #[test] - fn legacy_and_stage_specific_budget_reasons_share_the_same_backoff() { - const EXHAUSTION_REASONS: &[&str] = &[ - SESSION_EVIDENCE_BUDGET_EXHAUSTED, - "session_evidence_budget_exhausted_request_result_limit", - "session_evidence_budget_exhausted_request_hydration_limit", - "session_evidence_budget_exhausted_request_context_bytes", - "session_evidence_budget_exhausted_request_candidate_bytes", - "session_evidence_budget_exhausted_request_record_bytes", - "session_evidence_budget_exhausted_request_hydration_bytes", - "session_evidence_budget_exhausted_estimator_version_mismatch", - "session_evidence_budget_exhausted_execution_work_exhausted", - "session_evidence_budget_exhausted_kernel_result_limit", - "session_evidence_budget_exhausted_participant_manifest_participants", - "session_evidence_budget_exhausted_participant_manifest_canonical_bytes", - "session_evidence_budget_exhausted_hydration_bytes", - "session_evidence_budget_exhausted_context_bytes", - "session_evidence_budget_exhausted_context_tokens", - ]; - + fn only_the_canonical_budget_token_activates_the_backoff() { let config = session_evidence_config(); - for reason in EXHAUSTION_REASONS { - let records = vec![budget_exhausted_skip_with_reason( - "run-exhausted", + let decision_at = |reason: &'static str, now: i64| { + schedule_decision( + &config, + None, AgentTaskKind::SessionReflector, - reason, - 2_000, - )]; - - assert_eq!( - schedule_decision( - &config, + &[budget_exhausted_skip_with_reason( + "run-exhausted", AgentTaskKind::SessionReflector, - &records, - SessionActivity::at(2_500), - 2_060, - ) + reason, + 2_000, + )], + SessionActivity::at(2_500), + now, + ) + }; + + assert_eq!( + decision_at(SESSION_EVIDENCE_BUDGET_EXHAUSTED, 2_060) .skip_reason() .map(AutomationSkipReasonV1::as_str), - Some(SESSION_EVIDENCE_BUDGET_SUPPRESSED), - "{reason} must activate the same backoff", - ); - assert!( - schedule_decision( - &config, - AgentTaskKind::SessionReflector, - &records, - SessionActivity::at(2_500), - 2_000 + 3_600, - ) - .is_due(), - "{reason} must release at the same boundary", - ); - } + Some(SESSION_EVIDENCE_BUDGET_SUPPRESSED), + ); + assert!(decision_at(SESSION_EVIDENCE_BUDGET_EXHAUSTED, 2_000 + 3_600).is_due()); + assert_ne!( + decision_at( + "session_evidence_budget_exhausted_request_candidate_bytes", + 2_060 + ) + .skip_reason() + .map(AutomationSkipReasonV1::as_str), + Some(SESSION_EVIDENCE_BUDGET_SUPPRESSED), + "stage-suffixed reasons are not a budget-exhaustion wire form", + ); } #[test] @@ -1985,6 +2008,7 @@ evidence about it", assert!( schedule_decision( &config, + None, AgentTaskKind::SessionReflector, &records, SessionActivity::at(2_500), @@ -2012,6 +2036,7 @@ evidence about it", assert_eq!( schedule_decision( &config, + None, AgentTaskKind::SessionReflector, &[exhausted, earlier_success], SessionActivity::at(2_500), @@ -2042,6 +2067,7 @@ evidence about it", assert_eq!( schedule_decision( &config, + None, AgentTaskKind::SkillWriter, &records, SessionActivity::at(2_500), @@ -2054,6 +2080,7 @@ evidence about it", assert!( schedule_decision( &config, + None, AgentTaskKind::SkillWriter, &records, SessionActivity::at(2_500), @@ -2080,6 +2107,7 @@ evidence about it", assert!( schedule_decision( &config, + None, AgentTaskKind::SkillWriter, &records, SessionActivity::at(2_500), @@ -2103,6 +2131,7 @@ evidence about it", assert!( schedule_decision( &config, + None, AgentTaskKind::SessionReflector, &records, SessionActivity::at(2_500), @@ -2136,6 +2165,7 @@ evidence about it", assert_eq!( schedule_decision( &config, + None, AgentTaskKind::SessionReflector, std::slice::from_ref(&timeout), SessionActivity::at(1_500), @@ -2148,6 +2178,7 @@ evidence about it", assert!( schedule_decision( &config, + None, AgentTaskKind::SessionReflector, std::slice::from_ref(&timeout), SessionActivity::at(1_500), @@ -2172,6 +2203,7 @@ evidence about it", assert_eq!( schedule_decision( &config, + None, AgentTaskKind::SessionReflector, &[cancelled], SessionActivity::at(1_500), diff --git a/crates/tracedecay-automation-runtime/src/automation/scheduler_metrics.rs b/crates/tracedecay-automation-runtime/src/automation/scheduler_metrics.rs index e013f83254..8d62f20a4b 100644 --- a/crates/tracedecay-automation-runtime/src/automation/scheduler_metrics.rs +++ b/crates/tracedecay-automation-runtime/src/automation/scheduler_metrics.rs @@ -208,8 +208,7 @@ fn count_skip_reason(reason: AutomationSkipReasonV1) { | AutomationSkipReasonV1::SessionEvidenceBudgetExhausted | AutomationSkipReasonV1::SessionEvidenceTimedOut | AutomationSkipReasonV1::SessionEvidenceCancelled - | AutomationSkipReasonV1::NoSessionEvidence - | AutomationSkipReasonV1::ShippedFactProposalHistoryRetired => { + | AutomationSkipReasonV1::NoSessionEvidence => { hotpath::gauge!("automation.skips.other_total").inc(1_u64); } } diff --git a/crates/tracedecay-automation-runtime/src/automation/skill_materialization.rs b/crates/tracedecay-automation-runtime/src/automation/skill_materialization.rs index 038ecde07b..bcf3fc84c8 100644 --- a/crates/tracedecay-automation-runtime/src/automation/skill_materialization.rs +++ b/crates/tracedecay-automation-runtime/src/automation/skill_materialization.rs @@ -41,6 +41,7 @@ pub use crate::automation::managed_skills::managed_skill_root; use crate::automation::managed_skills::{ManagedSkill, ManagedSkillState}; use tracedecay_automation::skill_frontmatter::{SkillFrontmatterValue, parse_skill_frontmatter}; use tracedecay_domain::errors::Result; +use tracedecay_private_fs::FileLease; pub use tracedecay_automation::managed_skills::MATERIALIZED_SKILL_MANAGED_BY; @@ -216,13 +217,11 @@ impl ReconcileReport { // Provenance parsing / fork detection // --------------------------------------------------------------------------- -/// The provenance a materialized file carries, plus the body markdown as it -/// currently sits on disk (for fork detection). +/// The provenance a materialized file carries in its frontmatter. struct FileProvenance { managed_by: Option, skill_id: Option, content_hash: Option, - body_hash: Option, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -273,19 +272,6 @@ impl FileProvenance { fn is_managed(&self) -> bool { self.managed_by.as_deref() == Some(MATERIALIZED_SKILL_MANAGED_BY) } - - /// Legacy (pre-package-hash) fork check: the recorded `content-hash` was - /// the body-only hash, so a file is a fork when the body on disk no longer - /// hashes to it. Only meaningful for the body-hash domain; callers first - /// try [`recompute_on_disk_package`] for the package-hash domain. A managed - /// file missing a content-hash is treated as forked so we never silently - /// overwrite something we cannot verify. - fn is_legacy_forked(&self) -> bool { - match (&self.content_hash, &self.body_hash) { - (Some(recorded), Some(actual)) => recorded != actual, - _ => true, - } - } } /// Reserved package-support paths that are never user support files. @@ -410,18 +396,6 @@ fn frontmatter_scalar<'a>( fields.get(key).and_then(SkillFrontmatterValue::as_scalar) } -/// Extracts the raw body region after the leading frontmatter block, then -/// strips exactly one leading and one trailing newline to recover the original -/// `body_markdown` we wrote. Returns `None` when the file has no frontmatter. -fn on_disk_body_markdown(contents: &str) -> Option { - let after_open = contents.strip_prefix("---\n")?; - let close_at = after_open.find("\n---\n")?; - let region = &after_open[close_at + "\n---\n".len()..]; - let region = region.strip_prefix('\n').unwrap_or(region); - let region = region.strip_suffix('\n').unwrap_or(region); - Some(region.to_string()) -} - const INSTALLATION_ID_FILE: &str = ".materialization-installation-id"; /// Returns a stable id for the local profile/installation, persisting a random @@ -475,15 +449,18 @@ fn read_file_provenance(path: &Path) -> Result> { ), None => (None, None, None), }; - let body_hash = on_disk_body_markdown(&contents).map(|body| sha256_bytes(body.as_bytes())); Ok(Some(FileProvenance { managed_by, skill_id, content_hash, - body_hash, })) } +fn package_lock_path(package_dir: &Path) -> PathBuf { + let key = sha256_hex(package_dir.to_string_lossy().as_bytes()); + std::env::temp_dir().join(format!("tracedecay-materialization-{key}.lock")) +} + /// Inter-process lock held for the duration of a single package's /// materialize/remove transaction. Serializes concurrent `tracedecay update` / /// automatic validated activation runs (and multiple worktrees @@ -492,21 +469,8 @@ fn read_file_provenance(path: &Path) -> Result> { /// the tracked skills tree (OS temp dir, keyed by the package path) so it never /// pollutes a repo; flock semantics only need to hold within one machine, which /// is exactly where the race occurs. -struct PackageLock(fs::File); - -impl Drop for PackageLock { - fn drop(&mut self) { - let _ = self.0.unlock(); - } -} - -fn package_lock_path(package_dir: &Path) -> PathBuf { - let key = sha256_hex(package_dir.to_string_lossy().as_bytes()); - std::env::temp_dir().join(format!("tracedecay-materialization-{key}.lock")) -} - #[hotpath::measure(label = "hosts.automation.skill_materialization.lock")] -fn lock_package(package_dir: &Path) -> Result { +fn lock_package(package_dir: &Path) -> Result { let path = package_lock_path(package_dir); let file = fs::OpenOptions::new() .read(true) @@ -515,7 +479,10 @@ fn lock_package(package_dir: &Path) -> Result { .truncate(false) .open(&path)?; tracedecay_runtime_core::storage::retry_transient_file_op(|| file.lock())?; - Ok(PackageLock(file)) + Ok(FileLease::held( + file, + "automation.skill_materialization.package", + )) } fn relative_artifact_path(relative: &str) -> Result<&Path> { @@ -731,7 +698,7 @@ fn write_materialization_manifest( })?; let path = checked_descendant_path(dir, Path::new(MATERIALIZATION_MANIFEST_FILE))?; ensure_not_symlink(&PathBuf::from(format!("{}.new", path.display())))?; - host_io.safe_write_json_file(&path, &value, None) + host_io.safe_write_json_file(&path, &value) } fn write_pending_materialization( @@ -746,7 +713,7 @@ fn write_pending_materialization( })?; let path = checked_descendant_path(dir, Path::new(MATERIALIZATION_PENDING_FILE))?; ensure_not_symlink(&PathBuf::from(format!("{}.new", path.display())))?; - host_io.safe_write_json_file(&path, &value, None) + host_io.safe_write_json_file(&path, &value) } fn decode_pending_artifacts(pending: &PendingMaterialization) -> Result>> { @@ -1016,26 +983,6 @@ fn reconcile_owned_package( ) } -fn legacy_support_files_are_forked( - dir: &Path, - artifacts: &BTreeMap>, -) -> Result { - for (relative, desired) in artifacts { - if relative == SKILL_FILE { - continue; - } - let path = artifact_path(dir, relative)?; - if !path_exists_without_following_links(&path)? { - continue; - } - let desired_hash = sha256_bytes(desired); - if current_artifact_hash(&path)?.as_deref() != Some(desired_hash.as_str()) { - return Ok(true); - } - } - Ok(false) -} - fn initial_support_path_conflicts( dir: &Path, artifacts: &BTreeMap>, @@ -1082,7 +1029,7 @@ fn materialize_skill_into( let package_hash = skill.materialized_package_hash()?; let artifacts = desired_artifacts(skill)?; // Hold the per-package lock across the whole read-decide-commit window so a - // concurrent transaction cannot interleave (see [`PackageLock`]). + // concurrent transaction cannot interleave (see [`lock_package`]). let _lock = lock_package(&dir)?; if let Some(action @ (MaterializeAction::SkippedForeign | MaterializeAction::SkippedForked)) = recover_pending_materialization(host_io, &dir, Some(&skill.metadata.id))? @@ -1115,9 +1062,8 @@ fn materialize_skill_into( (Some(existing), ManifestState::Missing) => { // Manifest lost (e.g. gitignored/uncommitted sidecar on a fresh // clone) but a managed file is on disk. Re-derive from disk: a - // pristine package-hash (#366+) package is treated as owned and - // reconciled (re-writing the manifest); otherwise fall back to the - // legacy body-hash (#362) domain before declaring a user fork. + // pristine package is treated as owned and reconciled (re-writing + // the manifest); anything else is a user fork. if let Some(rederived) = recompute_on_disk_package(&dir, existing)? { fs::create_dir_all(&dir)?; reconcile_owned_package( @@ -1129,23 +1075,8 @@ fn materialize_skill_into( installation_id, &artifacts, )? - } else if existing.is_legacy_forked() - || legacy_support_files_are_forked(&dir, &artifacts)? - { - MaterializeAction::SkippedForked } else { - fs::create_dir_all(&dir)?; - let previous_files = current_artifact_hashes(&dir, &artifacts)?; - commit_materialization_transaction( - host_io, - &dir, - skill, - package_hash, - installation_id, - &artifacts, - previous_files, - BTreeMap::new(), - )? + MaterializeAction::SkippedForked } } (None, ManifestState::Missing) if initial_support_conflict => { @@ -1258,31 +1189,11 @@ pub fn remove_materialized_skill( match read_materialization_manifest(&dir, existing.skill_id.as_deref().unwrap_or(slug))? { ManifestState::Foreign => return Ok(RemoveAction::SkippedForked), ManifestState::Owned(manifest) => manifest, - ManifestState::Missing => { - if let Some(rederived) = recompute_on_disk_package(&dir, &existing)? { - // Pristine package-hash (#366+) package with a lost manifest. - rederived - } else if existing.is_legacy_forked() { - return Ok(RemoveAction::SkippedForked); - } else { - // Pristine legacy (#362) single-file package: synthesize a - // manifest so the profile gate and owned-removal path apply. - let mut files = BTreeMap::new(); - if let Some(hash) = current_artifact_hash(&path)? { - files.insert(SKILL_FILE.to_string(), hash); - } - MaterializationManifest { - managed_by: MATERIALIZED_SKILL_MANAGED_BY.to_string(), - skill_id: existing - .skill_id - .clone() - .unwrap_or_else(|| slug.to_string()), - package_hash: existing.content_hash.clone().unwrap_or_default(), - materialized_by: None, - files, - } - } - } + ManifestState::Missing => match recompute_on_disk_package(&dir, &existing)? { + // Pristine package with a lost manifest. + Some(rederived) => rederived, + None => return Ok(RemoveAction::SkippedForked), + }, }; if !may_remove_owned(scope, &manifest, installation_id) { diff --git a/crates/tracedecay-automation-runtime/src/automation/skill_targets.rs b/crates/tracedecay-automation-runtime/src/automation/skill_targets.rs index cbc39762e1..911222a1ed 100644 --- a/crates/tracedecay-automation-runtime/src/automation/skill_targets.rs +++ b/crates/tracedecay-automation-runtime/src/automation/skill_targets.rs @@ -15,11 +15,6 @@ use tracedecay_runtime_core::config::{TRACEDECAY_DIR, USER_DATA_DIR_ENV}; const NATIVE_NAMESPACE_DIR: &str = "agent-managed"; const NATIVE_MANIFEST_FILE: &str = ".tracedecay-managed-skills.json"; -/// The unslugged legacy managed-skill start marker. Reuses the same literal the -/// prompt-rules block-splicer stops at, keeping the two in sync. -const PROMPT_INDEX_START: &str = super::host_io::SKILL_INDEX_START; -const PROMPT_INDEX_END: &str = ""; - const ALL_SKILL_INSTALL_TARGETS: [SkillInstallTarget; 8] = [ SkillInstallTarget::Cursor, SkillInstallTarget::Codex, @@ -235,7 +230,7 @@ pub fn export_prompt_skill_index( if let Some(parent) = prompt_path.parent() { fs::create_dir_all(parent)?; } - host_io.safe_write_text_file(prompt_path, &updated, None)?; + host_io.safe_write_text_file(prompt_path, &updated)?; } let exported = skills @@ -288,7 +283,7 @@ fn remove_prompt_skill_indexes( if updated.trim().is_empty() { fs::remove_file(prompt_path)?; } else { - host_io.safe_write_text_file(prompt_path, &updated, None)?; + host_io.safe_write_text_file(prompt_path, &updated)?; } Ok(()) } @@ -324,16 +319,8 @@ pub fn stale_prompt_index_ids( Err(err) => return Err(err.into()), }; let (start_marker, end_marker) = prompt_index_markers(target); - let range = match managed_block_range(&existing, target, &start_marker, &end_marker)? { - Some(range) => Some(range), - // Mirror the removal path's legacy fallback: an unslugged block is this - // target's only when no other target has claimed the file with a slug. - None if !has_other_slugged_block(&existing, target) => { - managed_block_range(&existing, target, PROMPT_INDEX_START, PROMPT_INDEX_END)? - } - None => None, - }; - let Some((start, end)) = range else { + let Some((start, end)) = managed_block_range(&existing, target, &start_marker, &end_marker)? + else { return Ok(Vec::new()); }; let active = load_active_managed_skills_for_target(profile_root, target)? @@ -382,12 +369,7 @@ fn replace_or_append_marked_block( block: &str, ) -> Result { let (start_marker, end_marker) = prompt_index_markers(target); - // Prefer this target's slugged block; fall back to the legacy unslugged one. - let existing_range = match managed_block_range(existing, target, &start_marker, &end_marker)? { - Some(range) => Some(range), - None => managed_block_range(existing, target, PROMPT_INDEX_START, PROMPT_INDEX_END)?, - }; - if let Some((start, end)) = existing_range { + if let Some((start, end)) = managed_block_range(existing, target, &start_marker, &end_marker)? { Ok(splice_range(existing, start, end, block)) } else { let mut updated = String::new(); @@ -420,30 +402,9 @@ fn remove_marked_block_for_target(existing: &str, target: SkillInstallTarget) -> if let Some((start, end)) = managed_block_range(existing, target, &start_marker, &end_marker)? { return Ok(remove_range(existing, start, end)); } - // Legacy fallback: older installs wrote an unslugged block. Only claim it as - // this target's when NO other target's slugged block is present. On a shared - // file mid-migration (one host slugged, another still legacy-unslugged), - // removing the legacy block here would delete the other host's block, so - // leave it untouched and let the remove-all path handle it instead. - if !has_other_slugged_block(existing, target) - && let Some((start, end)) = - managed_block_range(existing, target, PROMPT_INDEX_START, PROMPT_INDEX_END)? - { - return Ok(remove_range(existing, start, end)); - } Ok(existing.to_string()) } -/// True when the file contains a slugged managed-skill block belonging to a -/// target other than `target`. -fn has_other_slugged_block(existing: &str, target: SkillInstallTarget) -> bool { - ALL_SKILL_INSTALL_TARGETS - .iter() - .copied() - .filter(|candidate| *candidate != target) - .any(|candidate| existing.contains(&prompt_index_markers(candidate).0)) -} - fn remove_all_marked_blocks(existing: &str) -> Result { let mut updated = existing.to_string(); for target in ALL_SKILL_INSTALL_TARGETS @@ -457,35 +418,9 @@ fn remove_all_marked_blocks(existing: &str) -> Result { updated = remove_range(&updated, start, end); } } - if let Some((start, end)) = legacy_managed_block_range(&updated)? { - updated = remove_range(&updated, start, end); - } Ok(updated) } -fn marked_block_range( - existing: &str, - start_marker: &str, - end_marker: &str, -) -> Result> { - match (existing.find(start_marker), existing.find(end_marker)) { - (Some(start), Some(end)) if start <= end => { - if existing.match_indices(start_marker).count() != 1 - || existing.match_indices(end_marker).count() != 1 - { - return Err(config_error( - "managed skill prompt index markers are ambiguous".to_string(), - )); - } - Ok(Some((start, end + end_marker.len()))) - } - (None, None) => Ok(None), - _ => Err(config_error( - "managed skill prompt index markers are unbalanced".to_string(), - )), - } -} - /// Finds a normal marker-delimited block, or a generated block whose start /// marker was lost while its exact preamble and end marker remain. Recovery /// begins at the preamble, so preceding user-authored text is never claimed. @@ -514,49 +449,6 @@ fn managed_block_range( } } -fn legacy_managed_block_range(existing: &str) -> Result> { - match ( - existing.find(PROMPT_INDEX_START), - existing.find(PROMPT_INDEX_END), - ) { - (Some(_), Some(_)) => marked_block_range(existing, PROMPT_INDEX_START, PROMPT_INDEX_END), - (None, None) => Ok(None), - (None, Some(end)) => { - if existing.match_indices(PROMPT_INDEX_END).count() != 1 { - return Err(config_error( - "managed skill prompt index markers are ambiguous".to_string(), - )); - } - let mut starts = Vec::new(); - for target in ALL_SKILL_INSTALL_TARGETS - .into_iter() - .filter(|target| target.writes_prompt_index()) - { - let preamble = prompt_index_preamble(target); - starts.extend( - existing[..end] - .match_indices(&preamble) - .map(|(start, _)| start), - ); - } - let Some(start) = starts.first().copied() else { - return Err(config_error( - "managed skill prompt index markers are unbalanced".to_string(), - )); - }; - if starts.len() != 1 { - return Err(config_error( - "managed skill prompt index markers are ambiguous".to_string(), - )); - } - Ok(Some((start, end + PROMPT_INDEX_END.len()))) - } - _ => Err(config_error( - "managed skill prompt index markers are unbalanced".to_string(), - )), - } -} - fn orphaned_generated_block_range( existing: &str, target: SkillInstallTarget, diff --git a/crates/tracedecay-automation-runtime/src/automation/skill_usage.rs b/crates/tracedecay-automation-runtime/src/automation/skill_usage.rs index d45d0f3a74..e63b8360d9 100644 --- a/crates/tracedecay-automation-runtime/src/automation/skill_usage.rs +++ b/crates/tracedecay-automation-runtime/src/automation/skill_usage.rs @@ -13,7 +13,6 @@ mod overlap; mod recommendations; mod store; -pub use crate::ports::session_store::AnalyticsEventRecord; pub use analytics::analytics_import_key_for_request; pub use analytics::ingest_analytics_events; pub use analytics::ingest_project_analytics_events; @@ -24,8 +23,6 @@ pub use overlap::{ }; pub use recommendations::{skill_improvement_recommendations, stale_skill_recommendations}; -const SKILL_USAGE_LEDGER_FILENAME: &str = "skill_usage.json"; - #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum SkillUsageAction { @@ -189,12 +186,6 @@ impl SkillUsageRecord { } } -pub fn skill_usage_ledger_path(profile_root: &Path) -> PathBuf { - profile_root - .join("agent_managed") - .join(SKILL_USAGE_LEDGER_FILENAME) -} - pub fn skill_usage_record_path(profile_root: &Path, skill_id: &str) -> PathBuf { store::skill_usage_record_path(profile_root, skill_id) } @@ -425,9 +416,8 @@ mod tests { after.records["skill-b"].view_count, 1, "skill B's last-view must survive skill A's write" ); - assert!( - !super::skill_usage_ledger_path(root.path()).exists(), - "independent skills must not share skill_usage.json" - ); + for skill in ["skill-a", "skill-b"] { + assert!(super::skill_usage_record_path(root.path(), skill).is_file()); + } } } diff --git a/crates/tracedecay-automation-runtime/src/automation/skill_usage/analytics.rs b/crates/tracedecay-automation-runtime/src/automation/skill_usage/analytics.rs index 910b2650b8..f9895295e7 100644 --- a/crates/tracedecay-automation-runtime/src/automation/skill_usage/analytics.rs +++ b/crates/tracedecay-automation-runtime/src/automation/skill_usage/analytics.rs @@ -1,11 +1,9 @@ use std::collections::BTreeSet; use std::path::Path; -use crate::ports::session_store::{ - AnalyticsEventQuery, AnalyticsEventRecord, AutomationSessionStore, canonical_project_key, -}; use tracedecay_automation::analytics::{UsageKind, infer_usage_events}; use tracedecay_domain::errors::Result; +use tracedecay_global_db::{AnalyticsEventQuery, AnalyticsEventRecord, RegisteredGlobalDb}; use super::store; use super::{SkillUsageAction, SkillUsageEvent, SkillUsageRecord, config_error, ledger_skill_id}; @@ -61,7 +59,7 @@ pub async fn ingest_analytics_events( pub async fn ingest_project_analytics_events( profile_root: &Path, project_root: &Path, - global_db: Option<&dyn AutomationSessionStore>, + global_db: Option<&RegisteredGlobalDb>, limit: usize, ) -> Result> { let Some(global_db) = global_db else { @@ -70,7 +68,7 @@ pub async fn ingest_project_analytics_events( let events = global_db .query_analytics_events(&AnalyticsEventQuery { provider: None, - project_id: Some(canonical_project_key(project_root)), + project_id: Some(RegisteredGlobalDb::canonical_project_key(project_root)), session_id: None, event_kind: None, since: None, diff --git a/crates/tracedecay-automation-runtime/src/automation/skill_usage/store.rs b/crates/tracedecay-automation-runtime/src/automation/skill_usage/store.rs index 4bd7cade39..f41ef37d16 100644 --- a/crates/tracedecay-automation-runtime/src/automation/skill_usage/store.rs +++ b/crates/tracedecay-automation-runtime/src/automation/skill_usage/store.rs @@ -1,23 +1,18 @@ -//! Per-skill usage files. Independent skills must not share a write target. -//! -//! The legacy `skill_usage.json` map is read once and split. After that, each -//! skill owns `skill_usage/.json`. Import-dedupe keys that -//! already name a skill live on that skill's record. The leftover legacy key -//! set is drained to a read-only file and never written by a new event. +//! Per-skill usage files. Independent skills must not share a write target: +//! each skill owns `skill_usage/.json`, and its import-dedupe +//! keys live on that record. -use std::collections::BTreeSet; use std::fs::{self, File, OpenOptions}; use std::io::{self, Write}; use std::path::{Path, PathBuf}; use serde::Serialize; -use super::{SkillUsageLedger, SkillUsageRecord, config_error, skill_usage_ledger_path}; +use super::{SkillUsageLedger, SkillUsageRecord, config_error}; use tracedecay_domain::errors::{Result, TraceDecayError}; +use tracedecay_private_fs::FileLease; const SKILL_USAGE_DIR: &str = "skill_usage"; -const LEGACY_IMPORTS_FILE: &str = "legacy-imported-events.json"; -const MIGRATE_LOCK_FILE: &str = ".migrate.lock"; pub(super) fn skill_usage_dir(profile_root: &Path) -> PathBuf { profile_root.join("agent_managed").join(SKILL_USAGE_DIR) @@ -43,7 +38,6 @@ pub(super) async fn update_record( let root = profile_root.to_path_buf(); let skill_id = skill_id.to_string(); tokio::task::spawn_blocking(move || { - migrate_legacy(&root)?; with_skill_lock( &root, &skill_id, @@ -69,16 +63,12 @@ pub(super) async fn record_imported_event( let root = profile_root.to_path_buf(); let skill_id = skill_id.to_string(); tokio::task::spawn_blocking(move || { - migrate_legacy(&root)?; - let legacy = read_legacy_imports(&root)?; let mut applied = false; let record = with_skill_lock( &root, &skill_id, |record| { - if record.imported_analytics_events.contains(&import_key) - || legacy.contains(&import_key) - { + if record.imported_analytics_events.contains(&import_key) { return Ok(false); } record.imported_analytics_events.insert(import_key.clone()); @@ -95,15 +85,11 @@ pub(super) async fn record_imported_event( } fn load_ledger_sync(profile_root: &Path) -> Result { - migrate_legacy(profile_root)?; let mut ledger = SkillUsageLedger::default(); let directory = skill_usage_dir(profile_root); let entries = match fs::read_dir(&directory) { Ok(entries) => entries, - Err(error) if error.kind() == io::ErrorKind::NotFound => { - ledger.imported_analytics_events = read_legacy_imports(profile_root)?; - return Ok(ledger); - } + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(ledger), Err(error) => { return Err(config_error(format!( "failed to read skill usage directory '{}': {error}", @@ -131,9 +117,6 @@ fn load_ledger_sync(profile_root: &Path) -> Result { .extend(record.imported_analytics_events.iter().cloned()); ledger.records.insert(record.skill_id.clone(), record); } - ledger - .imported_analytics_events - .extend(read_legacy_imports(profile_root)?); Ok(ledger) } @@ -168,6 +151,7 @@ fn with_skill_lock( "failed to lock skill usage record '{skill_id}': {error}" )) })?; + let lock = FileLease::held(lock, "automation.skill_usage.record"); let path = skill_usage_record_path(profile_root, skill_id); let existing = read_record_if_present(&path)?; let mut record = existing @@ -177,7 +161,7 @@ fn with_skill_lock( if mutate(&mut record)? { write_json(&path, &record)?; } - let _ = lock.unlock(); + drop(lock); Ok(if path.exists() { read_record(&path)? } else { @@ -185,99 +169,6 @@ fn with_skill_lock( }) } -fn migrate_legacy(profile_root: &Path) -> Result<()> { - let legacy_path = skill_usage_ledger_path(profile_root); - if !legacy_path.exists() { - return Ok(()); - } - let directory = skill_usage_dir(profile_root); - fs::create_dir_all(&directory).map_err(|error| { - config_error(format!( - "failed to create skill usage directory '{}': {error}", - directory.display() - )) - })?; - let lock_path = directory.join(MIGRATE_LOCK_FILE); - let lock = OpenOptions::new() - .read(true) - .write(true) - .create(true) - .truncate(false) - .open(&lock_path) - .map_err(|error| { - config_error(format!( - "failed to open skill usage migration lock '{}': {error}", - lock_path.display() - )) - })?; - lock.lock() - .map_err(|error| config_error(format!("failed to lock skill usage migration: {error}")))?; - let result = migrate_legacy_locked(profile_root, &legacy_path); - let _ = lock.unlock(); - result -} - -fn migrate_legacy_locked(profile_root: &Path, legacy_path: &Path) -> Result<()> { - if !legacy_path.exists() { - return Ok(()); - } - let ledger = read_legacy_ledger(legacy_path)?; - for (skill_id, record) in ledger.records { - let path = skill_usage_record_path(profile_root, &skill_id); - if path.exists() { - continue; - } - write_json(&path, &record)?; - } - let imports_path = skill_usage_dir(profile_root).join(LEGACY_IMPORTS_FILE); - if !imports_path.exists() && !ledger.imported_analytics_events.is_empty() { - write_json(&imports_path, &ledger.imported_analytics_events)?; - } - match fs::remove_file(legacy_path) { - Ok(()) => Ok(()), - Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), - Err(error) => Err(config_error(format!( - "failed to retire skill usage ledger '{}': {error}", - legacy_path.display() - ))), - } -} - -fn read_legacy_ledger(path: &Path) -> Result { - let bytes = fs::read(path).map_err(|error| { - config_error(format!( - "failed to read skill usage ledger '{}': {error}", - path.display() - )) - })?; - serde_json::from_slice(&bytes).map_err(|error| { - config_error(format!( - "failed to parse skill usage ledger '{}': {error}", - path.display() - )) - }) -} - -fn read_legacy_imports(profile_root: &Path) -> Result> { - let path = skill_usage_dir(profile_root).join(LEGACY_IMPORTS_FILE); - let bytes = match fs::read(&path) { - Ok(bytes) => bytes, - Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(BTreeSet::new()), - Err(error) => { - return Err(config_error(format!( - "failed to read legacy skill usage imports '{}': {error}", - path.display() - ))); - } - }; - serde_json::from_slice(&bytes).map_err(|error| { - config_error(format!( - "failed to parse legacy skill usage imports '{}': {error}", - path.display() - )) - }) -} - fn read_record_if_present(path: &Path) -> Result> { match fs::read(path) { Ok(bytes) => serde_json::from_slice(&bytes).map(Some).map_err(|error| { diff --git a/crates/tracedecay-automation-runtime/src/automation/skill_writer.rs b/crates/tracedecay-automation-runtime/src/automation/skill_writer.rs index 1f8eec9528..31d41d9933 100644 --- a/crates/tracedecay-automation-runtime/src/automation/skill_writer.rs +++ b/crates/tracedecay-automation-runtime/src/automation/skill_writer.rs @@ -593,9 +593,7 @@ fn accepted_skill_proposal_record( if let Some(object) = record.as_object_mut() { let reason = proposal.get("reason").cloned().unwrap_or(Value::Null); object.insert("action".to_string(), json!(action.as_str())); - object.insert("proposal_action".to_string(), json!(action.as_str())); - object.insert("reason".to_string(), reason.clone()); - object.insert("proposal_reason".to_string(), reason); + object.insert("reason".to_string(), reason); object.insert("target_skill_id".to_string(), json!(skill.metadata.id)); object.insert( "target_checksum".to_string(), @@ -695,11 +693,7 @@ fn skill_draft_from_proposal( let routing_description = required_routing_description(object.get("routing_description"))?; let category = required_proposal_string(object.get("category"), "category")?; let targets = proposal_targets_or_default(object.get("targets"))?; - let body_markdown = object - .get("body_markdown") - .or_else(|| object.get("body")) - .ok_or_else(|| "body_markdown is required".to_string()) - .and_then(|value| required_proposal_string(Some(value), "body_markdown"))?; + let body_markdown = required_proposal_string(object.get("body_markdown"), "body_markdown")?; let support_files = support_files_from_proposal(object.get("support_files"))?; let draft = ManagedSkillDraft { id, @@ -744,13 +738,13 @@ fn skill_proposal_action(proposal: &Value) -> std::result::Result Ok(SkillProposalAction::Create), - "update" | "patch" => Ok(SkillProposalAction::Update), - "merge" | "consolidate" => Ok(SkillProposalAction::Merge), + "update" => Ok(SkillProposalAction::Update), + "merge" => Ok(SkillProposalAction::Merge), "archive" => Ok(SkillProposalAction::Archive), other => Err(format!("unsupported skill proposal action '{other}'")), } @@ -793,9 +787,7 @@ fn skill_update_from_proposal( .transpose()?, category: optional_proposal_string(object.get("category"))?, targets: optional_proposal_targets(object.get("targets"))?, - body_markdown: optional_proposal_string( - object.get("body_markdown").or_else(|| object.get("body")), - )?, + body_markdown: optional_proposal_string(object.get("body_markdown"))?, support_files: if object .get("support_files") .is_some_and(|value| !value.is_null()) @@ -976,6 +968,39 @@ mod tests { } } + #[test] + fn proposal_action_accepts_only_the_prompt_schema_verbs() { + assert_eq!( + skill_proposal_action(&json!({"action": "update"})), + Ok(SkillProposalAction::Update) + ); + assert_eq!( + skill_proposal_action(&json!({"action": "merge"})), + Ok(SkillProposalAction::Merge) + ); + for verb in ["patch", "consolidate"] { + assert_err_eq( + skill_proposal_action(&json!({"action": verb})), + &format!("unsupported skill proposal action '{verb}'"), + ); + } + assert_eq!( + skill_proposal_action(&json!({"operation": "archive"})), + Ok(SkillProposalAction::Create) + ); + assert_err_eq( + skill_draft_from_proposal( + &json!({ + "id": "x", "title": "t", "summary": "s", "routing_description": "r", + "category": "c", "body": "b" + }), + "run-1", + &BTreeSet::new(), + ), + "body_markdown is required", + ); + } + #[test] fn proposal_targets_reject_unknown_or_malformed_values() { assert_err_eq( diff --git a/crates/tracedecay-automation-runtime/src/automation/skill_writer/consolidation.rs b/crates/tracedecay-automation-runtime/src/automation/skill_writer/consolidation.rs index d2f8f4984a..2a209f2d3c 100644 --- a/crates/tracedecay-automation-runtime/src/automation/skill_writer/consolidation.rs +++ b/crates/tracedecay-automation-runtime/src/automation/skill_writer/consolidation.rs @@ -143,9 +143,7 @@ pub(super) fn skill_merge_from_proposal( .transpose()?, category: optional_proposal_string(object.get("category"))?, targets: optional_proposal_targets(object.get("targets"))?, - body_markdown: optional_proposal_string( - object.get("body_markdown").or_else(|| object.get("body")), - )?, + body_markdown: optional_proposal_string(object.get("body_markdown"))?, support_files: if object .get("support_files") .is_some_and(|value| !value.is_null()) @@ -229,9 +227,7 @@ pub(super) fn applied_consolidation_record( let reason = proposal.get("reason").cloned().unwrap_or(Value::Null); let mut record = json!({ "action": action.as_str(), - "proposal_action": action.as_str(), - "reason": reason.clone(), - "proposal_reason": reason, + "reason": reason, "application_status": "applied", "resulting_state": "archived", "archived_skill_id": applied_source.metadata.id, @@ -270,7 +266,7 @@ mod tests { create_managed_skill, default_managed_skill_targets, load_managed_skill, }; #[cfg(unix)] - use super::super::super::skill_usage::skill_usage_ledger_path; + use super::super::super::skill_usage::skill_usage_record_path; use super::super::super::skill_usage::{DEFAULT_SKILL_OVERLAP_LIMIT, skill_overlap_candidates}; use super::*; @@ -459,8 +455,10 @@ mod tests { #[cfg(unix)] fn replace_usage_ledger_with_blocking_fifo( profile_root: &Path, + first_synced_skill_id: &str, ) -> (std::thread::JoinHandle, std::path::PathBuf) { - let ledger_path = skill_usage_ledger_path(profile_root); + let ledger_path = skill_usage_record_path(profile_root, first_synced_skill_id); + std::fs::create_dir_all(ledger_path.parent().unwrap()).unwrap(); match std::fs::remove_file(&ledger_path) { Ok(()) => {} Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} @@ -643,7 +641,13 @@ mod tests { )); assert_eq!(merge.target_skill_id, "workflow-a"); assert_eq!(merge.source_skill_id, "workflow-b"); - assert!(merge.update.is_some()); + assert_eq!( + merge + .update + .as_ref() + .and_then(|update| update.body_markdown.as_deref()), + Some("Merged workflow guidance covering both variants.") + ); let archive_only = assert_ok(skill_merge_from_proposal( &json!({ @@ -1080,7 +1084,8 @@ mod tests { "reason": "duplicate guidance" }); let archive = skill_archive_from_proposal(&proposal, &skills).unwrap(); - let (fifo_writer, ledger_path) = replace_usage_ledger_with_blocking_fifo(profile.path()); + let (fifo_writer, ledger_path) = + replace_usage_ledger_with_blocking_fifo(profile.path(), "workflow-b"); let profile_root = profile.path().to_path_buf(); let archive_for_task = archive.clone(); let mut task = @@ -1153,7 +1158,8 @@ mod tests { "reason": "duplicate guidance" }); let merge = skill_merge_from_proposal(&proposal, &skills).unwrap(); - let (fifo_writer, ledger_path) = replace_usage_ledger_with_blocking_fifo(profile.path()); + let (fifo_writer, ledger_path) = + replace_usage_ledger_with_blocking_fifo(profile.path(), "workflow-a"); let profile_root = profile.path().to_path_buf(); let merge_for_task = merge.clone(); let mut task = diff --git a/crates/tracedecay-automation-runtime/src/ports.rs b/crates/tracedecay-automation-runtime/src/ports.rs index 8f2a2d0c2d..0d949fb930 100644 --- a/crates/tracedecay-automation-runtime/src/ports.rs +++ b/crates/tracedecay-automation-runtime/src/ports.rs @@ -10,4 +10,3 @@ pub mod codex_app_server; pub mod project_runtime; -pub mod session_store; diff --git a/crates/tracedecay-automation-runtime/src/ports/codex_app_server.rs b/crates/tracedecay-automation-runtime/src/ports/codex_app_server.rs index 4322952d0e..a4edcce1c6 100644 --- a/crates/tracedecay-automation-runtime/src/ports/codex_app_server.rs +++ b/crates/tracedecay-automation-runtime/src/ports/codex_app_server.rs @@ -12,44 +12,40 @@ //! [`SummaryConfig`] to the session runtime's own config type. //! //! Unregistered, every run reports the backend as unavailable. That is the -//! same class of failure the backend already handles when the `codex` binary -//! is missing, so an unwired build degrades to "backend unavailable" instead -//! of panicking or silently producing an empty summary. +//! same class of failure the backend already handles when the `codex` +//! executable is unconfigured, so an unwired build degrades to "backend +//! unavailable" instead of panicking or silently producing an empty summary. +use std::path::{Path, PathBuf}; use std::sync::OnceLock; use std::time::Duration; use serde_json::Value; /// How to invoke `codex app-server` for one prompt. +/// +/// The executable is the exact path the configuration authority bound +/// (`lcm.summarizer_executables.v1`); this port never resolves `codex` from +/// `PATH` or the environment. #[derive(Debug, Clone, PartialEq, Eq)] pub struct SummaryConfig { - /// The `codex` executable to spawn. - pub codex_bin: String, + /// The configured `codex` executable to spawn. + pub codex_bin: PathBuf, /// Model selected for TraceDecay-owned turns. pub model: Option, /// Hard wall-clock budget for the run. pub timeout: Duration, } -impl Default for SummaryConfig { - fn default() -> Self { - Self { - codex_bin: "codex".to_string(), - model: Some("gpt-5.6-sol".to_owned()), - timeout: Duration::from_mins(2), - } - } -} - impl SummaryConfig { - /// Reads the operator overrides from the environment. + /// Tuning for an executable the caller resolved through configuration. + /// Only the model and timeout knobs come from the environment. /// /// The timeout is clamped to 5..=300 seconds: below that a real model turn /// cannot finish, and above it a stuck backend would outlive the /// automation run that is waiting on it. #[must_use] - pub fn from_env() -> Self { + pub fn for_executable(codex_bin: &Path) -> Self { fn non_empty_env(key: &str) -> Option { std::env::var(key) .ok() @@ -57,10 +53,11 @@ impl SummaryConfig { .filter(|value| !value.is_empty()) } - let mut config = Self::default(); - if let Some(bin) = non_empty_env("TRACEDECAY_CODEX_BIN") { - config.codex_bin = bin; - } + let mut config = Self { + codex_bin: codex_bin.to_path_buf(), + model: Some("gpt-5.6-sol".to_owned()), + timeout: Duration::from_mins(2), + }; if let Some(model) = non_empty_env("TRACEDECAY_CODEX_SUMMARY_MODEL") { config.model = Some(model); } diff --git a/crates/tracedecay-automation-runtime/src/ports/project_runtime.rs b/crates/tracedecay-automation-runtime/src/ports/project_runtime.rs index a50e42e339..a5e8395711 100644 --- a/crates/tracedecay-automation-runtime/src/ports/project_runtime.rs +++ b/crates/tracedecay-automation-runtime/src/ports/project_runtime.rs @@ -1,18 +1,13 @@ //! Runtime authorities supplied by the root composition layer. -use std::future::Future; use std::path::{Path, PathBuf}; -use std::pin::Pin; -use tracedecay_domain::errors::Result; use tracedecay_domain::{ProjectId, UserProfileId}; use tracedecay_global_db::RegisteredGlobalDbLeaseV1; use tracedecay_runtime_core::db::Database; use crate::automation::host_io::HostIo; -pub type RuntimeFuture<'a, T> = Pin> + Send + 'a>>; - /// Immutable project values captured by the composition root for one /// automation run. pub struct AutomationProjectContext { @@ -37,10 +32,3 @@ impl AutomationProjectContext { &self.project_root } } - -/// Profile runtime needed by projectless automation. -pub trait ProfileRuntime: Send + Sync { - fn profile_id(&self) -> &UserProfileId; - fn profile_sessions(&self) -> RuntimeFuture<'_, RegisteredGlobalDbLeaseV1>; - fn open_user_memory_db(&self) -> RuntimeFuture<'_, Database>; -} diff --git a/crates/tracedecay-automation-runtime/src/ports/session_store.rs b/crates/tracedecay-automation-runtime/src/ports/session_store.rs deleted file mode 100644 index 1c07c2c1c3..0000000000 --- a/crates/tracedecay-automation-runtime/src/ports/session_store.rs +++ /dev/null @@ -1,126 +0,0 @@ -//! The registered profile database, as automation reads it. -//! -//! A **port**. `global_db::RegisteredGlobalDb` is the user-level store holding -//! every project's sessions and analytics. It lives in `tracedecay-global-db`, -//! which sits beside this crate rather than beneath it, so automation names -//! the handful of reads it performs instead of the concrete handle. -//! -//! The scheduler asks when sessions were last active, skill-usage ingest -//! replays analytics rows, and evidence retrieval reads a snapshot and the -//! store's shard binding. Nothing here writes: automation's writes go through -//! the project store, not the profile database. -//! -//! Root wiring: the root implements [`AutomationSessionStore`] for -//! `RegisteredGlobalDb` (each method forwards to the identically named -//! inherent method) and registers [`register_canonical_project_key`] with -//! `RegisteredGlobalDb::canonical_project_key`. - -use std::path::Path; -use std::pin::Pin; -use std::sync::OnceLock; - -use tracedecay_runtime_core::db::DatabaseEngineReadSnapshot; -use tracedecay_store::StoreRuntimeBindingV1; - -pub use tracedecay_global_db::{AnalyticsEventQuery, AnalyticsEventRecord}; - -/// Boxed future returned by the port's asynchronous reads. -pub type StoreFuture<'a, T> = Pin + Send + 'a>>; - -/// The reads automation performs against the registered profile database. -pub trait AutomationSessionStore: Send + Sync { - /// Canonical path of the attached database file. - fn database_path(&self) -> &Path; - - /// Typed shard binding this attachment serves. - /// - /// Evidence retrieval checks the binding's shard against the active - /// profile identity before trusting a stored session as in-scope. - fn binding(&self) -> &StoreRuntimeBindingV1; - - /// Unix seconds of the most recent session activity, or `None` when the - /// store holds no timestamped messages. - /// - /// The scheduler uses this as a gate: no observed activity means nothing - /// new to run against. The registered adapter maps a failed read to - /// `None` with a logged warning, a store the scheduler cannot read has - /// no observable new activity, so automation stays idle instead of - /// running against a broken store. - fn latest_session_activity_secs(&self) -> StoreFuture<'_, Option>; - - /// Opens a read snapshot for a bounded direct query. - fn read_snapshot(&self) -> StoreFuture<'_, Result>; - - /// Runs one bounded analytics-event scan. - fn query_analytics_events<'a>( - &'a self, - query: &'a AnalyticsEventQuery, - ) -> StoreFuture<'a, Result, String>>; -} - -impl AutomationSessionStore for tracedecay_global_db::RegisteredGlobalDb { - fn database_path(&self) -> &Path { - self.db_path() - } - - fn binding(&self) -> &StoreRuntimeBindingV1 { - self.binding() - } - - fn latest_session_activity_secs(&self) -> StoreFuture<'_, Option> { - Box::pin(async move { - match self.latest_session_activity_secs().await { - Ok(latest) => latest, - Err(error) => { - tracing::warn!( - database = %self.db_path().display(), - %error, - "session-activity read failed; scheduler observes no new activity" - ); - None - } - } - }) - } - - fn read_snapshot(&self) -> StoreFuture<'_, Result> { - Box::pin(async move { - self.read_snapshot() - .await - .map_err(|error| error.to_string()) - }) - } - - fn query_analytics_events<'a>( - &'a self, - query: &'a AnalyticsEventQuery, - ) -> StoreFuture<'a, Result, String>> { - Box::pin(self.query_analytics_events(query)) - } -} - -/// Derives the profile database's project key from a project root. -pub type CanonicalProjectKey = fn(&Path) -> String; - -static CANONICAL_PROJECT_KEY: OnceLock = OnceLock::new(); - -/// Registers the root crate's project-key derivation. -/// -/// Idempotent: the first registration wins. -pub fn register_canonical_project_key(canonical_project_key: CanonicalProjectKey) { - let _ = CANONICAL_PROJECT_KEY.set(canonical_project_key); -} - -/// The profile database's key for `project_root`. -/// -/// Falls back to the lossy path string when the root never registered. That -/// matches the registered derivation for an already-canonical root, so an -/// unwired build still scopes its analytics query to one project rather than -/// silently querying every project's rows. -#[must_use] -pub fn canonical_project_key(project_root: &Path) -> String { - CANONICAL_PROJECT_KEY.get().map_or_else( - || project_root.to_string_lossy().into_owned(), - |canonical| canonical(project_root), - ) -} diff --git a/crates/tracedecay-automation/Cargo.toml b/crates/tracedecay-automation/Cargo.toml index c230ec7f95..0b7250d1d0 100644 --- a/crates/tracedecay-automation/Cargo.toml +++ b/crates/tracedecay-automation/Cargo.toml @@ -9,6 +9,7 @@ repository = "https://github.com/ScriptedAlchemy/tracedecay" [dependencies] hotpath.workspace = true +schemars.workspace = true serde = { version = "1", features = ["derive"] } serde_json = "1" sha2 = "0.11" diff --git a/crates/tracedecay-automation/src/analytics.rs b/crates/tracedecay-automation/src/analytics.rs index 86da6dc776..04722ca348 100644 --- a/crates/tracedecay-automation/src/analytics.rs +++ b/crates/tracedecay-automation/src/analytics.rs @@ -87,8 +87,7 @@ pub fn normalize_tool_name(raw: &str) -> String { /// Same host MCP namespaces as `tracedecay_agent_hosts::tool_name`. fn strip_host_tool_prefix(name: &str) -> &str { - const PREFIXES: [&str; 4] = [ - "mcp__plugin_tracedecay_tracedecay__", + const PREFIXES: [&str; 3] = [ "mcp__plugin_tracedecay_graph__", "mcp__tracedecay__", "mcp_tracedecay_", diff --git a/crates/tracedecay-automation/src/backend.rs b/crates/tracedecay-automation/src/backend.rs index f59dfe6da3..112e5da3c7 100644 --- a/crates/tracedecay-automation/src/backend.rs +++ b/crates/tracedecay-automation/src/backend.rs @@ -1,3 +1,4 @@ +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use serde_json::Value; use sha2::{Digest, Sha256}; @@ -7,7 +8,7 @@ use tracedecay_domain::canonical_text::encode_tagged_lowercase_hex; use crate::config::AutomationBackend; use crate::{AutomationError, Result, config_error}; -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "snake_case")] pub enum AgentTaskKind { MemoryCurator, @@ -125,7 +126,7 @@ pub struct AgentTaskResponse { pub output_tokens: Option, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "snake_case")] pub enum AgentTaskFailureClass { Retryable, @@ -427,7 +428,7 @@ fn skill_update_schema() -> Value { "routing_validation", "reason" ], "properties": { - "action": { "type": "string", "enum": ["update", "patch"] }, + "action": { "type": "string", "enum": ["update"] }, "id": { "type": "string" }, "base_checksum": { "type": "string" }, "title": nullable_string_schema(), @@ -668,6 +669,12 @@ pub trait AgentTaskBackend: Send + Sync { &self, request: &AgentTaskRequest, ) -> std::result::Result; + + /// The host executable this backend spawns for a task, or `None` when it + /// runs in-process or its executable is not configured. The durable + /// backend identity stamps the opened file behind this path so replacing + /// the binary in place re-admits a settled deterministic failure. + fn executable(&self) -> Option<&std::path::Path>; } /// Availability state returned by runtime adapters. This crate does not probe diff --git a/crates/tracedecay-automation/src/config.rs b/crates/tracedecay-automation/src/config.rs index 4d0c49c35f..ab04cf1803 100644 --- a/crates/tracedecay-automation/src/config.rs +++ b/crates/tracedecay-automation/src/config.rs @@ -8,16 +8,14 @@ pub use tracedecay_domain::configuration::{ use crate::{AutomationError, Result, config_error}; pub const DEFAULT_SCHEDULER_TICK_SECS: u64 = 60; -pub const DEFAULT_ANALYTICS_EVENTS_RETENTION_DAYS: u32 = 180; -pub const DEFAULT_LEGACY_SESSION_RETENTION_DAYS: u32 = 180; +const DEFAULT_ANALYTICS_EVENTS_RETENTION_DAYS: u32 = 180; +const DEFAULT_SESSION_MESSAGE_RETENTION_DAYS: u32 = 180; #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub struct RetentionConfig { #[serde(default = "default_analytics_events_days")] pub analytics_events_days: Option, - #[serde(default = "default_legacy_session_days")] - pub session_messages_days: Option, - #[serde(default = "default_legacy_session_days")] + #[serde(default = "default_session_message_days")] pub lcm_raw_messages_days: Option, } @@ -27,16 +25,15 @@ fn default_analytics_events_days() -> Option { } #[allow(clippy::unnecessary_wraps)] -fn default_legacy_session_days() -> Option { - Some(DEFAULT_LEGACY_SESSION_RETENTION_DAYS) +fn default_session_message_days() -> Option { + Some(DEFAULT_SESSION_MESSAGE_RETENTION_DAYS) } impl Default for RetentionConfig { fn default() -> Self { Self { analytics_events_days: default_analytics_events_days(), - session_messages_days: default_legacy_session_days(), - lcm_raw_messages_days: default_legacy_session_days(), + lcm_raw_messages_days: default_session_message_days(), } } } diff --git a/crates/tracedecay-automation/src/lib.rs b/crates/tracedecay-automation/src/lib.rs index f5a15b206b..63c83003af 100644 --- a/crates/tracedecay-automation/src/lib.rs +++ b/crates/tracedecay-automation/src/lib.rs @@ -29,7 +29,6 @@ pub mod managed_skills { ManagedSkillMaterializationScope, ManagedSkillMetadata, ManagedSkillProvenance, ManagedSkillSource, ManagedSkillState, ManagedSkillUpdate, ManagedSupportFile, SkillInstallTarget, current_metadata_timestamp, default_managed_skill_targets, - legacy_managed_skill_routing_description, }; pub use crate::managed_skill_validation::{ validate_managed_skill, validate_managed_skill_update, validate_managed_support_files, diff --git a/crates/tracedecay-automation/src/managed_skill_model.rs b/crates/tracedecay-automation/src/managed_skill_model.rs index 8efa7054db..d8af02a2db 100644 --- a/crates/tracedecay-automation/src/managed_skill_model.rs +++ b/crates/tracedecay-automation/src/managed_skill_model.rs @@ -2,6 +2,7 @@ use std::fmt::Write as _; use std::path::{Path, PathBuf}; use std::time::{SystemTime, UNIX_EPOCH}; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use tracedecay_domain::canonical_text::encode_tagged_lowercase_hex; @@ -9,8 +10,8 @@ use tracedecay_domain::canonical_text::encode_tagged_lowercase_hex; use crate::Result; use crate::managed_skill_format::{frontmatter_string, source_key, state_key, target_key}; use crate::managed_skill_validation::{ - MAX_NATIVE_SKILL_DESCRIPTION_CHARS, MAX_NATIVE_SKILL_NAME_CHARS, validate_managed_skill, - validate_native_skill_markdown, validate_support_file, + MAX_NATIVE_SKILL_NAME_CHARS, validate_managed_skill, validate_native_skill_markdown, + validate_support_file, }; pub const MAX_MANAGED_SUPPORT_FILES: usize = 20; @@ -21,7 +22,9 @@ pub const MAX_MANAGED_SKILL_BODY_BYTES: usize = 256 * 1024; /// by TraceDecay automation. pub const MATERIALIZED_SKILL_MANAGED_BY: &str = "tracedecay-automation"; -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[derive( + Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, JsonSchema, +)] #[serde(rename_all = "snake_case")] pub enum SkillInstallTarget { Cursor, @@ -71,23 +74,6 @@ pub fn default_managed_skill_targets() -> Vec { ] } -/// Preserve the discovery text exported by retained summary-only skill records. -pub fn legacy_managed_skill_routing_description(summary: &str) -> String { - let trimmed = summary.trim(); - let description = if trimmed - .get(..8) - .is_some_and(|prefix| prefix.eq_ignore_ascii_case("use when")) - || trimmed - .get(..19) - .is_some_and(|prefix| prefix.eq_ignore_ascii_case("use this skill when")) - { - trimmed.to_string() - } else { - format!("Use when {trimmed}") - }; - truncate_frontmatter_chars(&description, MAX_NATIVE_SKILL_DESCRIPTION_CHARS) -} - fn native_skill_name(id: &str) -> String { let mut normalized = String::with_capacity(id.len().min(MAX_NATIVE_SKILL_NAME_CHARS)); for byte in id.bytes() { @@ -121,7 +107,7 @@ fn truncate_frontmatter_chars(value: &str, max_chars: usize) -> String { .to_string() } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "snake_case")] pub enum ManagedSkillSource { AutomationRun, @@ -129,7 +115,7 @@ pub enum ManagedSkillSource { Import, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "snake_case")] pub enum ManagedSkillState { Active, @@ -137,7 +123,7 @@ pub enum ManagedSkillState { Archived, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "snake_case")] pub enum ManagedSkillMaterializationScope { #[default] @@ -156,14 +142,14 @@ impl ManagedSkillMaterializationScope { } } -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] pub struct ManagedSkillProvenance { pub source: ManagedSkillSource, pub actor: String, pub run_id: Option, } -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] pub struct ManagedSupportFile { pub path: PathBuf, pub bytes: Vec, @@ -226,7 +212,7 @@ impl ManagedSkillDraft { } } -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] pub struct ManagedSkillMetadata { pub id: String, pub title: String, @@ -243,9 +229,7 @@ pub struct ManagedSkillMetadata { pub materialization_scope: ManagedSkillMaterializationScope, pub pinned: bool, pub checksum: String, - #[serde(default)] pub created_at: i64, - #[serde(default)] pub updated_at: i64, #[serde(default, skip_serializing_if = "Option::is_none")] pub activated_at: Option, @@ -256,7 +240,7 @@ pub struct ManagedSkillMetadata { pub provenance: ManagedSkillProvenance, } -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] pub struct ManagedSkill { pub metadata: ManagedSkillMetadata, pub body_markdown: String, @@ -297,26 +281,6 @@ impl ManagedSkill { self.metadata.updated_at = current_metadata_timestamp(); } - pub fn normalize_timestamps(&mut self) { - let now = current_metadata_timestamp(); - match (self.metadata.created_at, self.metadata.updated_at) { - (0, 0) => { - self.metadata.created_at = now; - self.metadata.updated_at = now; - } - (0, updated_at) => { - self.metadata.created_at = updated_at; - } - (created_at, 0) => { - self.metadata.updated_at = created_at; - } - (created_at, updated_at) if updated_at < created_at => { - self.metadata.updated_at = created_at; - } - _ => {} - } - } - pub fn refresh_checksum(&mut self) { self.metadata.checksum = self.content_checksum(); } @@ -538,52 +502,4 @@ mod tests { skill.materialized_package_hash().unwrap() ); } - - #[test] - fn retained_summary_conversion_preserves_previous_export() { - assert_eq!( - legacy_managed_skill_routing_description(" Diagnose indexing "), - "Use when Diagnose indexing" - ); - for description in ["Use when indexing", "uSe ThIs SkIlL wHeN indexing"] { - assert_eq!( - legacy_managed_skill_routing_description(description), - description - ); - } - let summary = format!("{} tail", "é".repeat(1014)); - assert_eq!( - legacy_managed_skill_routing_description(&summary), - format!("Use when {}", "é".repeat(1014)) - ); - } - - #[test] - fn legacy_skill_without_consolidation_metadata_deserializes() { - let skill = ManagedSkillDraft { - id: "legacy-skill".to_string(), - title: "Legacy skill".to_string(), - summary: "Read records written before consolidation metadata.".to_string(), - routing_description: "Inspect retained skill consolidation records.".to_string(), - category: "testing".to_string(), - targets: vec![SkillInstallTarget::Codex], - body_markdown: "# Legacy\n".to_string(), - support_files: Vec::new(), - provenance: ManagedSkillProvenance { - source: ManagedSkillSource::AutomationRun, - actor: "legacy".to_string(), - run_id: None, - }, - } - .materialize() - .unwrap(); - let mut value = serde_json::to_value(skill).unwrap(); - let metadata = value["metadata"].as_object_mut().unwrap(); - metadata.remove("absorbed_into"); - metadata.remove("archived_reason"); - - let decoded: ManagedSkill = serde_json::from_value(value).unwrap(); - assert_eq!(decoded.metadata.absorbed_into, None); - assert_eq!(decoded.metadata.archived_reason, None); - } } diff --git a/crates/tracedecay-capture/src/claude/canonical.rs b/crates/tracedecay-capture/src/claude/canonical.rs index aa2e43c59d..e4ba441108 100644 --- a/crates/tracedecay-capture/src/claude/canonical.rs +++ b/crates/tracedecay-capture/src/claude/canonical.rs @@ -9,7 +9,9 @@ use tracedecay_domain::{ ProviderUsageScopeV1, SessionId, }; -use crate::{ObservationRecordParseErrorV1, parse_rfc3339_timestamp}; +use crate::{ + ObservationRecordParseErrorV1, parse_rfc3339_timestamp, parse_rfc3339_timestamp_micros, +}; const PROVIDER: &str = "claude"; @@ -27,16 +29,37 @@ pub fn stable_record_id( provider_observation_id(&candidate).ok_or(ObservationRecordParseErrorV1::NormalizationFailed) } +/// The spawn a Claude subagent transcript records about itself: the session +/// that owns its `subagents/` directory (or the subagent named by the +/// sidecar's `parentAgentId`) and the sidecar's `toolUseId`, the parent's +/// `tool_use` block id. Absent fields stay absent. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ClaudeSpawnParent<'a> { + pub session_id: &'a str, + pub tool_use_id: Option<&'a str>, +} + pub fn normalize( native: &Value, session_id: &str, stable_record_id: ObservationId, range: ObservationSourceRangeV1, +) -> Result { + normalize_spawned(native, session_id, None, stable_record_id, range) +} + +/// [`normalize`] for a record of a subagent transcript spawned by `parent`. +pub fn normalize_spawned( + native: &Value, + session_id: &str, + parent: Option>, + stable_record_id: ObservationId, + range: ObservationSourceRangeV1, ) -> Result { // Claude records order by file bytes, so the range length is the source // record's byte length. Failed normalizations are counted, never hidden. hotpath::gauge!("capture.claude.record_bytes").inc(range.end() - range.start()); - let envelope = normalize_record(native, session_id, stable_record_id, range); + let envelope = normalize_record(native, session_id, parent, stable_record_id, range); if envelope.is_err() { hotpath::gauge!("capture.claude.normalize_failures").inc(1u64); } @@ -48,6 +71,7 @@ pub fn normalize( fn normalize_record( native: &Value, session_id: &str, + parent: Option>, stable_record_id: ObservationId, range: ObservationSourceRangeV1, ) -> Result { @@ -136,6 +160,13 @@ fn normalize_record( if let Some(agent) = optional_id(native, &["agentId", "agent_id"]) { relations = relations.with_agent_id(agent); } + if let Some(parent) = parent { + relations = relations + .with_parent_session_id(SessionId::new(parent.session_id).map_err(|_| invalid())?); + if let Some(tool_use_id) = parent.tool_use_id.and_then(provider_observation_id) { + relations = relations.with_parent_tool_use_id(tool_use_id); + } + } let mut evidence = CanonicalObservationEvidenceV1::new(ObservationOrderingDomainV1::FileBytes, range); @@ -300,18 +331,38 @@ fn append_tool_use_result_facts( if let Some(branch) = native.get("gitBranch").cloned() { content.insert("gitBranch".to_owned(), branch); } - if tool_use_result + if let Some(path) = tool_use_result .get("filePath") .and_then(Value::as_str) - .is_some_and(|path| !path.is_empty()) + .filter(|path| !path.is_empty()) { + // The edit rollup reads these keys from the fact content; each is + // present only when Claude recorded it (`Write` reports `type`, + // `Edit`/`Write`/`MultiEdit` report `structuredPatch` hunks). + let mut edit = content.clone(); + if let Some(edited_at_micros) = native + .get("timestamp") + .and_then(Value::as_str) + .and_then(parse_rfc3339_timestamp_micros) + { + edit.insert("edited_at_micros".to_owned(), Value::from(edited_at_micros)); + } + if let Some(change_type) = tool_use_result.get("type").and_then(Value::as_str) { + edit.insert( + "change_type".to_owned(), + Value::String(change_type.to_owned()), + ); + } + if let Some(hunks) = tool_use_result + .get("structuredPatch") + .and_then(Value::as_array) + { + edit.insert("hunks".to_owned(), Value::from(hunks.len())); + } facts.push(CanonicalObservationFactV1::Git { evidence_kind: CanonicalGitEvidenceKindV1::FileEdit, - reference: tool_use_result - .get("filePath") - .and_then(Value::as_str) - .map(str::to_owned), - content: Some(Value::Object(content.clone())), + reference: Some(path.to_owned()), + content: Some(Value::Object(edit)), }); } if tool_use_result diff --git a/crates/tracedecay-capture/src/claude/mod.rs b/crates/tracedecay-capture/src/claude/mod.rs index c6f7b04442..c1dfa53a88 100644 --- a/crates/tracedecay-capture/src/claude/mod.rs +++ b/crates/tracedecay-capture/src/claude/mod.rs @@ -2,7 +2,7 @@ mod canonical; use tracedecay_domain::canonical_text::canonical_framed_sha256; -pub use canonical::{normalize, stable_record_id}; +pub use canonical::{ClaudeSpawnParent, normalize, normalize_spawned, stable_record_id}; const CURSOR_KEY_PREFIX: &str = "tracedecay-claude-cursor-v1"; const SOURCE_ID_PREFIX: &str = "tracedecay-claude-source-v1"; diff --git a/crates/tracedecay-capture/src/codex.rs b/crates/tracedecay-capture/src/codex.rs index 9cbabebc3c..8d22000481 100644 --- a/crates/tracedecay-capture/src/codex.rs +++ b/crates/tracedecay-capture/src/codex.rs @@ -334,7 +334,8 @@ fn append_codex_session_meta_agent_relations( native_thread_id: Option<&str>, ) -> CanonicalObservationRelationsV1 { let parent_session_id = string_field(payload, "forked_from_id") - .or_else(|| nested_string_field(payload, "/source/subagent/thread_spawn/parent_thread_id")); + .or_else(|| nested_string_field(payload, "/source/subagent/thread_spawn/parent_thread_id")) + .or_else(|| string_field(payload, "parent_thread_id")); let thread_source = string_field(payload, "thread_source"); let is_subagent = thread_source.as_deref() == Some("subagent") || parent_session_id.is_some() @@ -348,9 +349,16 @@ fn append_codex_session_meta_agent_relations( if let Some(agent_id) = native_thread_id.and_then(observation_id_from_native) { relations = relations.with_agent_id(agent_id); } - if let Some(parent_agent_id) = parent_session_id.and_then(|id| observation_id_from_native(&id)) - { - relations = relations.with_parent_agent_id(parent_agent_id); + // The spawning `spawn_agent` call id is recorded only in the parent's + // rollout (`SubAgentActivity` `started` item), never in the child's, so + // the child carries no parent tool-use id. + if let Some(parent) = parent_session_id { + if let Some(parent_agent_id) = observation_id_from_native(&parent) { + relations = relations.with_parent_agent_id(parent_agent_id); + } + if let Ok(parent_session_id) = SessionId::new(parent) { + relations = relations.with_parent_session_id(parent_session_id); + } } relations } @@ -387,6 +395,10 @@ fn append_codex_event_facts( }); return; }; + if item.get("type").and_then(Value::as_str) == Some("FileChange") { + append_codex_file_change_facts(payload, item, timestamp, facts); + return; + } if item.get("type").and_then(Value::as_str) != Some("UserMessage") { facts.push(CanonicalObservationFactV1::Unknown { native_kind: "item_completed".to_string(), @@ -458,6 +470,67 @@ fn append_codex_event_facts( } } +/// One `item_completed` `FileChange` item: Codex's record of an applied patch. +/// `changes` maps each path to `{type: add|update|delete, ...}`; `status` +/// reports whether the apply succeeded. Only a completed apply becomes +/// file-edit evidence, one fact per path, timed by the item's own +/// `completed_at_ms` when present and the record timestamp otherwise. Diff +/// bodies and file contents never leave the native record. +fn append_codex_file_change_facts( + payload: &Value, + item: &Value, + timestamp: Option, + facts: &mut Vec, +) { + let Some(changes) = item.get("changes").and_then(Value::as_object) else { + facts.push(CanonicalObservationFactV1::Unknown { + native_kind: "item_completed.FileChange".to_string(), + state: CanonicalUnknownStateV1::Malformed, + }); + return; + }; + if item.get("status").and_then(Value::as_str) != Some("completed") { + facts.push(CanonicalObservationFactV1::Unknown { + native_kind: "item_completed.FileChange".to_string(), + state: CanonicalUnknownStateV1::Unsupported, + }); + return; + } + let edited_at_micros = payload + .get("completed_at_ms") + .and_then(Value::as_i64) + .and_then(|millis| millis.checked_mul(1_000)) + .or_else(|| timestamp.and_then(|secs| secs.checked_mul(1_000_000))); + for (path, change) in changes.iter().filter(|(path, _)| !path.is_empty()) { + let mut content = serde_json::Map::new(); + content.insert("type".to_string(), Value::String("FileChange".to_string())); + if let Some(id) = item.get("id").filter(|id| id.is_string()) { + content.insert("id".to_string(), id.clone()); + } + if let Some(edited_at_micros) = edited_at_micros { + content.insert( + "edited_at_micros".to_string(), + Value::from(edited_at_micros), + ); + } + if let Some(change_type) = change.get("type").and_then(Value::as_str) { + content.insert( + "change_type".to_string(), + Value::String(change_type.to_string()), + ); + } + if let Some(diff) = change.get("unified_diff").and_then(Value::as_str) { + let hunks = diff.lines().filter(|line| line.starts_with("@@")).count(); + content.insert("hunks".to_string(), Value::from(hunks)); + } + facts.push(CanonicalObservationFactV1::Git { + evidence_kind: CanonicalGitEvidenceKindV1::FileEdit, + reference: Some(path.clone()), + content: Some(Value::Object(content)), + }); + } +} + fn codex_usage_counters(usage: &Value) -> [Option; 6] { [ canonical_u64(usage.get("input_tokens")), diff --git a/crates/tracedecay-capture/src/cursor_composer.rs b/crates/tracedecay-capture/src/cursor_composer.rs index 4eccab57a5..69e9597183 100644 --- a/crates/tracedecay-capture/src/cursor_composer.rs +++ b/crates/tracedecay-capture/src/cursor_composer.rs @@ -11,7 +11,10 @@ use tracedecay_domain::{ use tracedecay_store::cursor_dispatch::cursor_model_string; use crate::git_facts::append_diff_and_pull_request_facts; -use crate::{ObservationRecordParseErrorV1, parse::canonical_u64_i64, parse::sha256_hex}; +use crate::{ + ObservationRecordParseErrorV1, parse::canonical_u64_i64, parse::sha256_hex, + parse_rfc3339_timestamp_micros, +}; const PROVIDER: &str = "cursor"; @@ -67,7 +70,7 @@ fn normalize_composer_bubble_record( range: tracedecay_domain::ObservationSourceRangeV1, position: u64, ) -> Result { - let timestamp = epoch_ms_to_secs(native.get("createdAt").and_then(Value::as_i64)); + let timestamp = composer_created_at_secs(native.get("createdAt")); let mut relations = CanonicalObservationRelationsV1::new( SessionId::new(composer_id) .map_err(|_| ObservationRecordParseErrorV1::NormalizationFailed)?, @@ -77,6 +80,20 @@ fn normalize_composer_bubble_record( if let Ok(thread_id) = ObservationId::new(composer_id) { relations = relations.with_thread_id(thread_id); } + if let Some(parent) = native + .get("tracedecayParentComposerId") + .and_then(Value::as_str) + .and_then(|parent| SessionId::new(parent).ok()) + { + relations = relations.with_parent_session_id(parent); + if let Some(tool_call_id) = native + .get("tracedecayParentToolCallId") + .and_then(Value::as_str) + .and_then(|id| ObservationId::new(id).ok()) + { + relations = relations.with_parent_tool_use_id(tool_call_id); + } + } let mut facts = Vec::new(); if let Some(project_path) = native .get("tracedecayProjectPath") @@ -160,6 +177,27 @@ fn normalize_composer_bubble_record( .and_then(composer_tool_result_success), }); } + if let Some(path) = composer_edit_tool_path(tool) { + let mut content = serde_json::Map::new(); + content.insert( + "type".to_string(), + Value::String("toolFormerData".to_string()), + ); + if let Some(name) = tool.get("name").filter(|name| name.is_string()) { + content.insert("name".to_string(), name.clone()); + } + if let Some(edited_at_micros) = composer_created_at_micros(native.get("createdAt")) { + content.insert( + "edited_at_micros".to_string(), + Value::from(edited_at_micros), + ); + } + facts.push(CanonicalObservationFactV1::Git { + evidence_kind: CanonicalGitEvidenceKindV1::FileEdit, + reference: Some(path), + content: Some(Value::Object(content)), + }); + } } if let Some(thinking) = native @@ -384,7 +422,7 @@ fn normalize_composer_envelope_record( range: tracedecay_domain::ObservationSourceRangeV1, position: u64, ) -> Result { - let timestamp = epoch_ms_to_secs(native.get("createdAt").and_then(Value::as_i64)); + let timestamp = composer_created_at_secs(native.get("createdAt")); let mut relations = CanonicalObservationRelationsV1::new( SessionId::new(composer_id) .map_err(|_| ObservationRecordParseErrorV1::NormalizationFailed)?, @@ -408,8 +446,8 @@ fn normalize_composer_envelope_record( .get("name") .and_then(Value::as_str) .map(str::to_string), - started_at: epoch_ms_to_secs(native.get("createdAt").and_then(Value::as_i64)), - ended_at: epoch_ms_to_secs(native.get("lastUpdatedAt").and_then(Value::as_i64)), + started_at: composer_created_at_secs(native.get("createdAt")), + ended_at: composer_created_at_secs(native.get("lastUpdatedAt")), source: Some("cursor_composer".to_string()), native_source: Some("cursor".to_string()), profile: None, @@ -477,6 +515,23 @@ pub fn composer_observation_with_session( object.insert(key.to_string(), value.clone()); } } + // A subagent composer's `subagentInfo` names the composer that + // spawned it and the spawning `task_v2` calls in order; the first + // is the spawn, later ones resumed it. + if let Some(spawn) = envelope.get("subagentInfo") + && let Some(parent) = spawn + .get("parentComposerId") + .filter(|parent| parent.as_str().is_some_and(|parent| !parent.is_empty())) + { + object.insert("tracedecayParentComposerId".to_string(), parent.clone()); + if let Some(call) = spawn + .pointer("/toolCallIdHistory/0") + .or_else(|| spawn.get("toolCallId")) + .filter(|call| call.is_string()) + { + object.insert("tracedecayParentToolCallId".to_string(), call.clone()); + } + } } } native @@ -579,3 +634,47 @@ pub fn cursor_composer_envelope_native_record_id( fn epoch_ms_to_secs(ms: Option) -> Option { ms.filter(|value| *value > 0).map(|value| value / 1_000) } + +/// Bubble and envelope `createdAt` / `lastUpdatedAt`: epoch milliseconds in +/// older snapshots, RFC3339 text in current ones. +fn composer_created_at_micros(value: Option<&Value>) -> Option { + match value? { + Value::Number(millis) => millis + .as_i64() + .filter(|millis| *millis > 0)? + .checked_mul(1_000), + Value::String(text) => parse_rfc3339_timestamp_micros(text), + _ => None, + } +} + +fn composer_created_at_secs(value: Option<&Value>) -> Option { + composer_created_at_micros(value).map(|micros| micros / 1_000_000) +} + +/// Composer tools whose completed `toolFormerData.params.relativeWorkspacePath` +/// names the file they changed (`edit_file_v2`, `delete_file`, and the other +/// edit/write/patch/replace tools). Read-only tools name their target under +/// other keys and never match. +fn composer_edit_tool_path(tool: &Value) -> Option { + let name = tool + .get("name") + .and_then(Value::as_str)? + .to_ascii_lowercase(); + if !["edit", "write", "patch", "replace", "delete"] + .iter() + .any(|needle| name.contains(needle)) + || tool.get("status").and_then(Value::as_str) != Some("completed") + { + return None; + } + let params = match tool.get("params")? { + Value::String(raw) => serde_json::from_str::(raw).ok()?, + params => params.clone(), + }; + params + .get("relativeWorkspacePath") + .and_then(Value::as_str) + .filter(|path| !path.trim().is_empty()) + .map(str::to_string) +} diff --git a/crates/tracedecay-capture/src/lib.rs b/crates/tracedecay-capture/src/lib.rs index 7027a2bae1..3524b5ff7f 100644 --- a/crates/tracedecay-capture/src/lib.rs +++ b/crates/tracedecay-capture/src/lib.rs @@ -14,13 +14,12 @@ mod timestamp; pub mod vibe; pub use parse::{ - ClaudeRecordParseErrorV1, MAX_OBSERVATION_RECORD_BYTES, ObservationRecordParseErrorV1, - ParseLimits, ParsedClaudeRecordV1, ParsedObservationRecordV1, ParsedPolicyLimitViolation, - PreparedObservationRecordV1, normalize_prepared_observation_record_v1, parse_claude_record_v1, - parse_normalized_observation_record_v1, parse_observation_record_v1, - prepare_observation_record_v1, + MAX_OBSERVATION_RECORD_BYTES, ObservationRecordParseErrorV1, ParseLimits, + ParsedObservationRecordV1, ParsedPolicyLimitViolation, PreparedObservationRecordV1, + normalize_prepared_observation_record_v1, parse_normalized_observation_record_v1, + parse_observation_record_v1, prepare_observation_record_v1, }; pub use timestamp::{ normalize_timestamp_secs, parse_cursor_human_timestamp, parse_rfc3339_timestamp, - parse_yyyy_mm_dd_utc_start, + parse_rfc3339_timestamp_micros, parse_yyyy_mm_dd_utc_start, }; diff --git a/crates/tracedecay-capture/src/parse.rs b/crates/tracedecay-capture/src/parse.rs index 732d9ed619..ab2b74a8e9 100644 --- a/crates/tracedecay-capture/src/parse.rs +++ b/crates/tracedecay-capture/src/parse.rs @@ -10,7 +10,7 @@ use tracedecay_domain::{ }; #[derive(Clone, Copy, Debug, Error, PartialEq, Eq)] -pub enum ClaudeRecordParseErrorV1 { +pub enum ObservationRecordParseErrorV1 { #[error("Claude record is empty")] Empty, #[error("Claude record exceeds the byte limit")] @@ -59,10 +59,10 @@ impl ParseLimits { /// Parsed and structurally bounded evidence for one complete Claude JSONL record. /// -/// Construction is intentionally restricted to [`parse_claude_record_v1`]. +/// Construction is intentionally restricted to [`parse_observation_record_v1`]. /// Callers may inspect the parsed object to resolve scope, then move the token /// into the sanitizer without serializing or parsing it again. -pub struct ParsedClaudeRecordV1 { +pub struct ParsedObservationRecordV1 { value: Value, source_range: ObservationSourceRangeV1, ordering_domain: ObservationOrderingDomainV1, @@ -73,7 +73,7 @@ pub struct ParsedClaudeRecordV1 { canonical_provider: Option, } -impl ParsedClaudeRecordV1 { +impl ParsedObservationRecordV1 { pub fn value(&self) -> &Value { &self.value } @@ -116,9 +116,6 @@ impl ParsedClaudeRecordV1 { } } -pub type ParsedObservationRecordV1 = ParsedClaudeRecordV1; -pub type ObservationRecordParseErrorV1 = ClaudeRecordParseErrorV1; - /// Structurally validated native JSON that may be normalized independently /// for several observation scopes without decoding or hashing the source bytes /// again. It is process-retained evidence only; durable replay remains the @@ -174,13 +171,6 @@ fn decoded_value_retained_bytes(value: &Value) -> u64 { .saturating_add(payload(value)) } -pub fn parse_claude_record_v1( - record: &[u8], - source_range: ObservationSourceRangeV1, -) -> Result { - parse_observation_record_v1(record, source_range, ObservationOrderingDomainV1::FileBytes) -} - pub fn parse_observation_record_v1( record: &[u8], source_range: ObservationSourceRangeV1, @@ -248,10 +238,10 @@ fn prepare_observation_record( ) -> Result { let limits = ParseLimits::default_policy(); validate_record_frame(record, source_range, ordering_domain, limits)?; - let native = - serde_json::from_slice::(record).map_err(|_| ClaudeRecordParseErrorV1::Malformed)?; + let native = serde_json::from_slice::(record) + .map_err(|_| ObservationRecordParseErrorV1::Malformed)?; if !native.is_object() { - return Err(ClaudeRecordParseErrorV1::NonObject); + return Err(ObservationRecordParseErrorV1::NonObject); } validate_structure(&native, limits)?; let retained_bytes = decoded_value_retained_bytes(&native); @@ -301,15 +291,15 @@ fn finish_canonical_envelope( ) -> Result { envelope .validate() - .map_err(|_| ClaudeRecordParseErrorV1::InvalidCanonicalEnvelope)?; + .map_err(|_| ObservationRecordParseErrorV1::InvalidCanonicalEnvelope)?; if envelope.evidence().ordering_domain() != ordering_domain || envelope.evidence().range() != source_range { - return Err(ClaudeRecordParseErrorV1::InvalidCanonicalEnvelope); + return Err(ObservationRecordParseErrorV1::InvalidCanonicalEnvelope); } let canonical_provider = envelope.provider().clone(); let value = serde_json::to_value(&envelope) - .map_err(|_| ClaudeRecordParseErrorV1::InvalidCanonicalEnvelope)?; + .map_err(|_| ObservationRecordParseErrorV1::InvalidCanonicalEnvelope)?; let structure = validate_structure(&value, ParseLimits::default_policy())?; Ok(ParsedObservationRecordV1 { value, @@ -331,10 +321,10 @@ fn parse_observation_record( limits: ParseLimits, ) -> Result { validate_record_frame(record, source_range, ordering_domain, limits)?; - let value = - serde_json::from_slice::(record).map_err(|_| ClaudeRecordParseErrorV1::Malformed)?; + let value = serde_json::from_slice::(record) + .map_err(|_| ObservationRecordParseErrorV1::Malformed)?; if !value.is_object() { - return Err(ClaudeRecordParseErrorV1::NonObject); + return Err(ObservationRecordParseErrorV1::NonObject); } let structure = validate_structure(&value, limits)?; Ok(ParsedObservationRecordV1 { @@ -392,17 +382,17 @@ fn validate_record_frame( source_range: ObservationSourceRangeV1, ordering_domain: ObservationOrderingDomainV1, limits: ParseLimits, -) -> Result<(), ClaudeRecordParseErrorV1> { +) -> Result<(), ObservationRecordParseErrorV1> { if record.is_empty() { - return Err(ClaudeRecordParseErrorV1::Empty); + return Err(ObservationRecordParseErrorV1::Empty); } if record.len() > limits.record_bytes { - return Err(ClaudeRecordParseErrorV1::TooLarge); + return Err(ObservationRecordParseErrorV1::TooLarge); } if ordering_domain == ObservationOrderingDomainV1::FileBytes { let range_len = source_range.end() - source_range.start(); if u64::try_from(record.len()).ok() != Some(range_len) { - return Err(ClaudeRecordParseErrorV1::RangeLengthMismatch); + return Err(ObservationRecordParseErrorV1::RangeLengthMismatch); } } Ok(()) @@ -417,7 +407,7 @@ struct StructureMetrics { fn validate_structure( value: &Value, limits: ParseLimits, -) -> Result { +) -> Result { let mut stack = vec![(value, 1usize)]; let mut values = 0usize; let mut max_depth = 0usize; @@ -425,10 +415,10 @@ fn validate_structure( values = values.saturating_add(1); max_depth = max_depth.max(depth); if values > limits.values { - return Err(ClaudeRecordParseErrorV1::TooManyValues); + return Err(ObservationRecordParseErrorV1::TooManyValues); } if depth > limits.depth { - return Err(ClaudeRecordParseErrorV1::TooDeep); + return Err(ObservationRecordParseErrorV1::TooDeep); } match current { Value::Object(fields) => { @@ -464,7 +454,7 @@ mod canonical_envelope_tests { fn message_envelope( content: Value, range: ObservationSourceRangeV1, - ) -> Result { + ) -> Result { CanonicalObservationEnvelopeV1::new( ProviderId::new("codex").unwrap(), "message", @@ -480,7 +470,7 @@ mod canonical_envelope_tests { }], CanonicalObservationEvidenceV1::new(ObservationOrderingDomainV1::FileBytes, range), ) - .map_err(|_| ClaudeRecordParseErrorV1::NormalizationFailed) + .map_err(|_| ObservationRecordParseErrorV1::NormalizationFailed) } fn native_record(content: &Value) -> (Vec, ObservationSourceRangeV1) { @@ -688,7 +678,7 @@ mod canonical_envelope_tests { (at_limit, Ok(())), ( at_limit + 1, - Err(ClaudeRecordParseErrorV1::InvalidCanonicalEnvelope), + Err(ObservationRecordParseErrorV1::InvalidCanonicalEnvelope), ), ] { let content = Value::String("a".repeat(content_len)); @@ -725,7 +715,7 @@ mod canonical_envelope_tests { |native| message_envelope(native["content"].clone(), other), ) .err(), - Some(ClaudeRecordParseErrorV1::InvalidCanonicalEnvelope) + Some(ObservationRecordParseErrorV1::InvalidCanonicalEnvelope) ); assert_eq!( parse_normalized_observation_record_v1( @@ -735,7 +725,7 @@ mod canonical_envelope_tests { |native| message_envelope(native["content"].clone(), range), ) .err(), - Some(ClaudeRecordParseErrorV1::InvalidCanonicalEnvelope) + Some(ObservationRecordParseErrorV1::InvalidCanonicalEnvelope) ); } } diff --git a/crates/tracedecay-capture/src/timestamp.rs b/crates/tracedecay-capture/src/timestamp.rs index 3485fde232..3ed6ac4745 100644 --- a/crates/tracedecay-capture/src/timestamp.rs +++ b/crates/tracedecay-capture/src/timestamp.rs @@ -29,12 +29,22 @@ pub fn timestamp_secs(value: &Value) -> Option { /// Chrono deliberately accepts RFC3339's space separator and mixed-case /// literals, matching the provider timestamp forms capture accepts. pub fn parse_rfc3339_timestamp(value: &str) -> Option { + parse_rfc3339(value).map(|timestamp| timestamp.timestamp()) +} + +/// Parses RFC3339 timestamps into Unix microseconds, keeping the sub-second +/// precision hosts record on edit events. +pub fn parse_rfc3339_timestamp_micros(value: &str) -> Option { + parse_rfc3339(value).map(|timestamp| timestamp.timestamp_micros()) +} + +fn parse_rfc3339(value: &str) -> Option> { let bytes = value.as_bytes(); if bytes.get(17) == Some(&b'6') && bytes.get(18) == Some(&b'0') { return None; } - let timestamp = DateTime::parse_from_rfc3339(value).ok()?.timestamp(); - (timestamp >= 0).then_some(timestamp) + let timestamp = DateTime::parse_from_rfc3339(value).ok()?; + (timestamp.timestamp() >= 0).then_some(timestamp) } /// Parses Cursor's human-readable timestamp format into Unix seconds. diff --git a/crates/tracedecay-capture/tests/capture_suite/fixture_provenance.rs b/crates/tracedecay-capture/tests/capture_suite/fixture_provenance.rs deleted file mode 100644 index dfc9c5825d..0000000000 --- a/crates/tracedecay-capture/tests/capture_suite/fixture_provenance.rs +++ /dev/null @@ -1,165 +0,0 @@ -use std::collections::BTreeSet; -use std::fs; -use std::path::{Path, PathBuf}; - -use serde_json::Value; -use sha2::{Digest as _, Sha256}; - -const NORMALIZATION_PROVIDERS: [&str; 7] = [ - "claude", - "codex", - "cursor", - "cursor_composer", - "hermes", - "kiro", - "vibe", -]; -const CLINE_FAMILY_PROVIDERS: [&str; 3] = ["cline", "roo-code", "kilo"]; - -fn fixture_root(relative: &str) -> PathBuf { - PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("../../tests/fixtures") - .join(relative) -} - -fn read_json(path: &Path) -> Value { - serde_json::from_slice(&fs::read(path).unwrap()).unwrap() -} - -fn sha256(path: &Path) -> String { - hex::encode(Sha256::digest(fs::read(path).unwrap())) -} - -fn input_paths(root: &Path) -> BTreeSet { - let mut paths = BTreeSet::new(); - for provider in fs::read_dir(root).unwrap() { - let provider = provider.unwrap(); - if !provider.file_type().unwrap().is_dir() { - continue; - } - let provider_name = provider.file_name(); - for entry in fs::read_dir(provider.path()).unwrap() { - let entry = entry.unwrap(); - let name = entry.file_name(); - let name = name.to_string_lossy(); - if entry.file_type().unwrap().is_file() && name.ends_with(".input.json") { - paths.insert(format!("{}/{}", provider_name.to_string_lossy(), name)); - } - } - } - paths -} - -#[test] -fn provider_fixture_manifest_covers_every_accepted_native_input() { - let root = fixture_root("provider_normalization"); - let manifest = read_json(&root.join("manifest.json")); - assert_eq!(manifest["schema_version"], 1); - assert_eq!( - manifest["supported_providers"] - .as_array() - .unwrap() - .iter() - .map(|provider| provider.as_str().unwrap()) - .collect::>(), - NORMALIZATION_PROVIDERS.into_iter().collect() - ); - - let mut manifested_paths = BTreeSet::new(); - let mut manifested_providers = BTreeSet::new(); - let mut authoritative_providers = BTreeSet::new(); - for fixture in manifest["fixtures"].as_array().unwrap() { - let provider = fixture["provider"].as_str().unwrap(); - let path = fixture["path"].as_str().unwrap(); - let origin = fixture["origin"].as_str().unwrap(); - assert!( - matches!( - origin, - "redacted_native_capture" | "synthetic_value_contract" - ), - "{path}: unknown fixture origin {origin}" - ); - assert_eq!( - fixture["provider_version"].as_str(), - Some("unversioned"), - "{path}: provider version must be explicit without inventing a version" - ); - assert!( - !fixture["origin_evidence"] - .as_str() - .unwrap_or_default() - .is_empty(), - "{path}: missing origin evidence" - ); - assert_eq!( - fixture["sha256"].as_str().unwrap(), - sha256(&root.join(path)), - "{path}: payload bytes changed without provenance update" - ); - assert!( - manifested_paths.insert(path.to_owned()), - "{path}: duplicate" - ); - manifested_providers.insert(provider); - if origin == "redacted_native_capture" { - authoritative_providers.insert(provider); - } - } - - assert_eq!(manifested_paths, input_paths(&root)); - assert_eq!( - manifested_providers, - NORMALIZATION_PROVIDERS.into_iter().collect() - ); - assert_eq!( - authoritative_providers, - ["claude", "codex", "hermes"].into_iter().collect() - ); -} - -#[test] -fn cline_family_manifest_covers_every_snapshot_input() { - let root = fixture_root("transcript_golden/cline_like"); - let manifest = read_json(&root.join("manifest.json")); - assert_eq!( - manifest["providers"] - .as_array() - .unwrap() - .iter() - .map(|provider| provider["provider"].as_str().unwrap()) - .collect::>(), - CLINE_FAMILY_PROVIDERS.into_iter().collect() - ); - let provenance = &manifest["fixture_provenance"]; - assert_eq!(provenance["origin"], "synthetic_value_contract"); - assert_eq!(provenance["provider_version"], "unversioned"); - assert!( - !provenance["origin_evidence"] - .as_str() - .unwrap_or_default() - .is_empty() - ); - - let manifested = provenance["inputs"] - .as_array() - .unwrap() - .iter() - .map(|input| { - let path = input["path"].as_str().unwrap(); - assert_eq!( - input["sha256"].as_str().unwrap(), - sha256(&root.join(path)), - "{path}: payload bytes changed without provenance update" - ); - path.to_owned() - }) - .collect::>(); - let accepted = fs::read_dir(root.join("input")) - .unwrap() - .map(|entry| { - let entry = entry.unwrap(); - format!("input/{}", entry.file_name().to_string_lossy()) - }) - .collect::>(); - assert_eq!(manifested, accepted); -} diff --git a/crates/tracedecay-capture/tests/capture_suite/main.rs b/crates/tracedecay-capture/tests/capture_suite/main.rs index 555294dc8d..3709817fa2 100644 --- a/crates/tracedecay-capture/tests/capture_suite/main.rs +++ b/crates/tracedecay-capture/tests/capture_suite/main.rs @@ -7,7 +7,6 @@ //! allocator) and `hotpath_coverage` (sets `HOTPATH_*` process environment //! variables) stay their own binaries. -mod fixture_provenance; mod kiro; mod provider_identity; mod provider_usage_capture; diff --git a/crates/tracedecay-capture/tests/hotpath_coverage.rs b/crates/tracedecay-capture/tests/hotpath_coverage.rs index 781c566756..4d33cc9f93 100644 --- a/crates/tracedecay-capture/tests/hotpath_coverage.rs +++ b/crates/tracedecay-capture/tests/hotpath_coverage.rs @@ -10,8 +10,8 @@ //! the instrumentation is real rather than dead configuration. use serde_json::json; -use tracedecay_capture::parse_claude_record_v1; -use tracedecay_domain::ObservationSourceRangeV1; +use tracedecay_capture::parse_observation_record_v1; +use tracedecay_domain::{ObservationOrderingDomainV1, ObservationSourceRangeV1}; /// Deterministic, daemon-free workload that reaches this crate's measured /// parse path (`capture.parse.record` and `capture.parse.record_digest`). @@ -23,7 +23,9 @@ fn run_capture_parse_workload() -> usize { .expect("serialize claude record fixture"); let range = ObservationSourceRangeV1::new(0, record.len() as u64).expect("valid fixture byte range"); - let parsed = parse_claude_record_v1(&record, range).expect("parse claude record fixture"); + let parsed = + parse_observation_record_v1(&record, range, ObservationOrderingDomainV1::FileBytes) + .expect("parse claude record fixture"); assert_eq!(parsed.encoded_len(), record.len()); assert_eq!( parsed.value()["message"]["content"], diff --git a/crates/tracedecay-cli/Cargo.toml b/crates/tracedecay-cli/Cargo.toml index 74b459bdcd..0ebce3454d 100644 --- a/crates/tracedecay-cli/Cargo.toml +++ b/crates/tracedecay-cli/Cargo.toml @@ -185,6 +185,7 @@ tracedecay-lsp = { path = "../tracedecay-lsp", version = "0.1.0" } tracedecay-maintenance = { path = "../tracedecay-maintenance", version = "0.1.0" } tracedecay-mcp = { path = "../tracedecay-mcp", version = "0.1.0" } tracedecay-private-fs = { path = "../tracedecay-private-fs", version = "0.1.0" } +tracedecay-project = { path = "../tracedecay-project", version = "0.1.0-beta.37" } tracedecay-runtime-core = { path = "../tracedecay-runtime-core", version = "0.1.0" } tracedecay-sdk = { path = "../tracedecay-sdk", version = "0.1.0" } tracedecay-session-memory = { path = "../tracedecay-session-memory", version = "0.1.0" } @@ -218,6 +219,7 @@ tracedecay-global-db = { path = "../tracedecay-global-db", version = "0.1.0", fe # contracts directly since the lcm extraction; CLI test targets compile that # fixture via `#[path]` and need the extern. tracedecay-lcm = { path = "../tracedecay-lcm", version = "0.1.0" } +tracedecay-project = { path = "../tracedecay-project", version = "0.1.0-beta.37", features = ["test-helpers"] } tracedecay-runtime-core = { path = "../tracedecay-runtime-core", version = "0.1.0", features = ["test-helpers"] } tracedecay-session-temporal-store = { path = "../tracedecay-session-temporal-store", version = "0.1.0" } diff --git a/crates/tracedecay-cli/src/agent_cmd.rs b/crates/tracedecay-cli/src/agent_cmd.rs index 63555b9ad7..4432fb4269 100644 --- a/crates/tracedecay-cli/src/agent_cmd.rs +++ b/crates/tracedecay-cli/src/agent_cmd.rs @@ -34,36 +34,17 @@ pub(crate) enum AgentReinstallOutcome { Installed, } -/// Stage only the host-native source required for an operator activation. -/// -/// A ready host skips this path entirely and enters the catalog component -/// transaction without an out-of-band artifact write. A deferred host receives -/// its verified source and a truthful error, but no lifecycle receipt. -fn prepare_native_activation_if_needed( - integration: &dyn tracedecay_agent_hosts::agents::AgentIntegration, - context: &tracedecay_agent_hosts::agents::InstallContext, -) -> tracedecay_domain::errors::Result<()> { - if matches!( - integration.preflight_non_interactive_install(context)?, - tracedecay_agent_hosts::agents::NonInteractiveInstallOutcome::Ready - ) { - return Ok(()); - } - match integration.prepare_non_interactive_install(context)? { - tracedecay_agent_hosts::agents::NonInteractiveInstallOutcome::Ready => Ok(()), - tracedecay_agent_hosts::agents::NonInteractiveInstallOutcome::DeferredUserAction( - deferred, - ) => Err(tracedecay_domain::errors::TraceDecayError::Config { - message: deferred.remediation, - }), - } -} - -pub(crate) async fn handle_host_bundle_component_command( +/// The one host lifecycle entry point behind `install`, `update-plugin`, +/// `reinstall`, and `uninstall`. Without `--component` it runs every +/// component of each host's canonical set; with it, only the named one. +pub(crate) async fn handle_host_lifecycle_command( agent: Option, operation: HostBundleCliOperation, options: crate::cli::HostBundleCliOptions, + no_dashboard: bool, + automation: Option, ) -> tracedecay_domain::errors::Result<()> { + validate_codex_automation_flags(agent.as_deref(), automation)?; if component_mutation_still_requires_yes( operation, options.component.is_some(), @@ -82,94 +63,258 @@ pub(crate) async fn handle_host_bundle_component_command( message: "could not determine home directory".to_string(), } })?; - let lifecycle_root = - tracedecay_agent_hosts::agents::host_bundle::resolved_host_bundle_lifecycle_root() - .map_err(|error| tracedecay_domain::errors::TraceDecayError::Config { - message: format!("could not resolve host lifecycle root: {error}"), - })?; - let mut user_config = tracedecay_session_memory::user_config::UserConfig::load(); + let lifecycle_root = resolved_lifecycle_root()?; + let mut user_config = load_host_lifecycle_user_config()?; let explicitly_scoped = agent.is_some(); let agent_ids = match agent { Some(agent) => vec![agent], + // Nothing to configure is an ordinary first-run state: this is the + // command a user runs before any agent exists. None if operation == HostBundleCliOperation::Install => { - return Err(tracedecay_domain::errors::TraceDecayError::Config { - message: "component install requires --agent".to_string(), - }); + match tracedecay_agent_hosts::agents::select_detected_integrations( + &home, + &user_config.installed_agents, + ) { + Some(ids) => ids, + None => { + eprintln!(); + eprintln!( + "{}", + tracedecay_agent_hosts::agents::no_detected_integrations_notice(&home) + ); + return Ok(()); + } + } + } + None => { + // A tracked id that no longer resolves (a release renamed or + // removed it) would otherwise be retried forever. + let before = user_config.installed_agents.len(); + user_config + .installed_agents + .retain(|id| tracedecay_agent_hosts::agents::get_integration(id).is_ok()); + if user_config.installed_agents.len() != before + && let Err(err) = user_config.save() + { + eprintln!("warning: could not save tracedecay config: {err}"); + } + if user_config.installed_agents.is_empty() { + eprintln!("No installed agents found. Run `tracedecay install` first."); + return Ok(()); + } + user_config.installed_agents.clone() } - None => user_config.installed_agents.clone(), }; if agent_ids.is_empty() { - return Err(tracedecay_domain::errors::TraceDecayError::Config { - message: "no installed agents are tracked for component lifecycle".to_string(), - }); + eprintln!("No changes."); } - let now_unix = SystemTime::now() - .duration_since(UNIX_EPOCH) - .map_err(|_| tracedecay_domain::errors::TraceDecayError::Config { - message: "system clock is before the Unix epoch".to_string(), - })? - .as_secs(); + + // Each agent runs independently: one host whose CLI is missing or whose + // files conflict must not strand the others. + let mut failures: Vec = Vec::new(); + let mut refreshed: std::collections::BTreeSet = std::collections::BTreeSet::new(); for agent_id in &agent_ids { - if !explicitly_scoped - && let Some(component) = options.component - && component_is_not_applicable(agent_id, component) - { - eprintln!( - "not applicable: agent {agent_id:?} does not support the requested {component:?} component; continuing sweep" - ); - continue; - } - let component_set = canonical_host_component_set(agent_id, options.component, now_unix)?; - let Some(component_set) = component_set else { - if explicitly_scoped { - return Err(tracedecay_domain::errors::TraceDecayError::Config { - message: unsupported_host_component_set_message(agent_id), - }); + if !explicitly_scoped { + if let Some(component) = options.component + && component_is_not_applicable(agent_id, component) + { + eprintln!( + "not applicable: agent {agent_id:?} does not support the requested {component:?} component; continuing sweep" + ); + continue; } // A skipped host is a reported unavailable result, never a silent // success for the sweep that contained it. - eprintln!( - "skipped: {}", - unsupported_host_component_set_message(agent_id) - ); - continue; + if host_kind_for_agent(agent_id).is_err() { + eprintln!( + "skipped: {}", + unsupported_host_component_set_message(agent_id) + ); + continue; + } + } + let dashboard = match operation { + HostBundleCliOperation::Install => !no_dashboard, + // Removal must cover everything an install could have written. + HostBundleCliOperation::Uninstall => true, + HostBundleCliOperation::Update | HostBundleCliOperation::Repair => { + user_config.dashboard_enabled_for_agent(agent_id) + } }; + let mut result = run_host_component_lifecycle( + agent_id, + operation, + &options, + &home, + &lifecycle_root, + &ComponentSetApplyContext::resolved_with_dashboard(dashboard), + ); + if result.is_ok() + && !options.dry_run + && agent_id == "codex" + && let Some(automation) = automation + { + result = match validate_codex_automation_project_path() { + Ok(project_path) => { + hotpath::future!( + install_codex_daemon_automation(&project_path, &home, automation), + label = "cli.agent.automation" + ) + .await + } + Err(error) => Err(error), + }; + } + if let Err(error) = result { + failures.push(if explicitly_scoped { + error.to_string() + } else { + format!("{agent_id}: {error}") + }); + continue; + } if options.dry_run { - dry_run_canonical_component_set( - agent_id, - operation, - &component_set, - &options, - &home, - &lifecycle_root, - )?; - } else { - apply_canonical_component_set( - agent_id, - operation, - &component_set, - &options, - &home, - &lifecycle_root, - &ComponentSetApplyContext::resolved(), - )?; + continue; } - } - if operation == HostBundleCliOperation::Install && !options.dry_run { - for agent_id in agent_ids { - if !user_config.installed_agents.contains(&agent_id) { - user_config.installed_agents.push(agent_id); + refreshed.insert(agent_id.clone()); + match operation { + HostBundleCliOperation::Install => { + if !user_config.installed_agents.contains(agent_id) { + user_config.installed_agents.push(agent_id.clone()); + } + user_config + .agent_dashboard_enabled + .insert(agent_id.clone(), !no_dashboard); } + // Removing one component leaves the rest of the host tracked. + HostBundleCliOperation::Uninstall if options.component.is_none() => { + user_config.installed_agents.retain(|id| id != agent_id); + user_config.agent_dashboard_enabled.remove(agent_id); + } + HostBundleCliOperation::Uninstall + | HostBundleCliOperation::Update + | HostBundleCliOperation::Repair => {} } + } + if !options.dry_run { user_config .save() - .map_err(|error| tracedecay_domain::errors::TraceDecayError::Config { - message: format!("failed to save user config: {error}"), + .map_err(|err| tracedecay_domain::errors::TraceDecayError::Config { + message: format!("failed to save user config: {err}"), + })?; + } + if !failures.is_empty() { + let message = if explicitly_scoped { + failures.join("; ") + } else { + format!( + "agent {} failed for: {}", + operation_verb(operation), + failures.join("; ") + ) + }; + return Err(tracedecay_domain::errors::TraceDecayError::Config { message }); + } + if options.dry_run { + return Ok(()); + } + if options.component.is_none() + && matches!( + operation, + HostBundleCliOperation::Install | HostBundleCliOperation::Repair + ) + { + // A pass may disarm the startup silent reinstall (`previous_version`) + // only when every agent still tracked was refreshed by this very + // pass. Anything less advances `last_installed_version` alone: after + // an upgrade, the untouched agents still need the silent refresh. + if crate::update_cmd::install_pass_covers_tracked_agents( + &user_config.installed_agents, + &refreshed, + ) { + crate::update_cmd::record_completed_reinstall_pass(&mut user_config)?; + } else if operation == HostBundleCliOperation::Install { + user_config.last_installed_version = env!("CARGO_PKG_VERSION").to_string(); + user_config.save().map_err(|err| { + tracedecay_domain::errors::TraceDecayError::Config { + message: format!("failed to save user config: {err}"), + } })?; + } + } + // An install pass converges the managed-skill exports against the store, + // so a host never keeps advertising a skill the store no longer holds. + if operation == HostBundleCliOperation::Install { + crate::update_cmd::deploy_managed_skills_after_lifecycle(); } Ok(()) } +fn operation_verb(operation: HostBundleCliOperation) -> &'static str { + match operation { + HostBundleCliOperation::Install => "install", + HostBundleCliOperation::Update => "update", + HostBundleCliOperation::Repair => "reinstall", + HostBundleCliOperation::Uninstall => "uninstall", + } +} + +fn resolved_lifecycle_root() -> tracedecay_domain::errors::Result { + tracedecay_agent_hosts::agents::host_bundle::resolved_host_bundle_lifecycle_root().map_err( + |error| tracedecay_domain::errors::TraceDecayError::Config { + message: format!("could not resolve host lifecycle root: {error}"), + }, + ) +} + +/// Runs one agent's canonical component set, or the single `--component` +/// the options name, as one receipt-backed transaction (or its dry run). +fn run_host_component_lifecycle( + agent_id: &str, + operation: HostBundleCliOperation, + options: &crate::cli::HostBundleCliOptions, + home: &Path, + lifecycle_root: &Path, + context: &ComponentSetApplyContext, +) -> tracedecay_domain::errors::Result<()> { + let now_unix = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|_| tracedecay_domain::errors::TraceDecayError::Config { + message: "system clock is before the Unix epoch".to_string(), + })? + .as_secs(); + let component_set = canonical_host_component_set_with_tracedecay_bin( + agent_id, + options.component, + now_unix, + &context.tracedecay_bin, + )? + .ok_or_else(|| tracedecay_domain::errors::TraceDecayError::Config { + message: unsupported_host_component_set_message(agent_id), + })?; + if options.dry_run { + dry_run_canonical_component_set( + agent_id, + operation, + &component_set, + options, + home, + lifecycle_root, + context, + ) + } else { + apply_canonical_component_set( + agent_id, + operation, + &component_set, + options, + home, + lifecycle_root, + context, + ) + } +} + /// Truthful reason a host component set is unavailable, so a skipped or /// refused agent never reads as an empty success. fn unsupported_host_component_set_message(agent: &str) -> String { @@ -183,6 +328,7 @@ fn unsupported_host_component_set_message(agent: &str) -> String { } } +#[cfg(test)] fn canonical_host_component_set( agent: &str, component: Option, @@ -240,174 +386,6 @@ fn canonical_host_component_set_with_tracedecay_bin( }) } -fn ensure_artifact_only_restore_boundary( - agent_id: &str, - component_set: &tracedecay_agent_hosts::agents::host_bundle_registry::VerifiedEmbeddedHostComponentSetV1, - home: &Path, - lifecycle_root: &Path, -) -> tracedecay_domain::errors::Result<()> { - let registration = CatalogHostComponentRegistrationAuthority::new( - agent_id, - home, - lifecycle_root, - tracedecay_agent_hosts::agents::host_bundle::HostBundleLifecycleOpV1::Repair, - )?; - if registration.supports_artifact_only_backup_restore(&component_set.component_set) { - return Ok(()); - } - Err(tracedecay_domain::errors::TraceDecayError::Config { - message: format!( - "artifact backup/restore for agent {agent_id:?} is unavailable because this component \ - uses host registration state; this command does not manage registration state" - ), - }) -} - -#[hotpath::measure(label = "cli.agent.artifact")] -fn apply_host_bundle_artifact_action_at( - action: crate::cli::HostBundleAction, - options: crate::cli::HostBundleCliOptions, - home: &Path, - lifecycle_root: &Path, - now_unix: u64, -) -> tracedecay_domain::errors::Result<[u8; 16]> { - if options.dry_run { - return Err(tracedecay_domain::errors::TraceDecayError::Config { - message: "artifact backup/restore has no dry-run mode".to_string(), - }); - } - // The writer refuses either operation without explicit confirmation. - // Restore overwrites deployed bytes; backup publishes a receipt. The shell - // must not advertise a weaker policy than that contract. - if !options.yes { - let message = match &action { - crate::cli::HostBundleAction::ArtifactBackup { .. } => "artifact backup requires --yes", - crate::cli::HostBundleAction::ArtifactRestore { .. } => { - "artifact restore requires --yes" - } - crate::cli::HostBundleAction::Status | crate::cli::HostBundleAction::Recover { .. } => { - "status and recovery are not artifact backup/restore operations" - } - }; - return Err(tracedecay_domain::errors::TraceDecayError::Config { - message: message.to_string(), - }); - } - let component = - options - .component - .ok_or_else(|| tracedecay_domain::errors::TraceDecayError::Config { - message: "artifact backup/restore requires --component".to_string(), - })?; - let (agent_id, backup_operation_id) = match &action { - crate::cli::HostBundleAction::ArtifactBackup { agent } => (agent.as_str(), None), - crate::cli::HostBundleAction::ArtifactRestore { agent, backup_id } => { - let mut decoded = [0_u8; 16]; - hex::decode_to_slice(backup_id, &mut decoded).map_err(|_| { - tracedecay_domain::errors::TraceDecayError::Config { - message: - "artifact restore --backup-id must be 32 lowercase hexadecimal characters" - .to_string(), - } - })?; - if hex::encode(decoded) != *backup_id { - return Err(tracedecay_domain::errors::TraceDecayError::Config { - message: - "artifact restore --backup-id must be 32 lowercase hexadecimal characters" - .to_string(), - }); - } - (agent.as_str(), Some(decoded)) - } - crate::cli::HostBundleAction::Status | crate::cli::HostBundleAction::Recover { .. } => { - return Err(tracedecay_domain::errors::TraceDecayError::Config { - message: "status and recovery are not artifact backup/restore operations" - .to_string(), - }); - } - }; - let component_set = canonical_host_component_set(agent_id, Some(component), now_unix)? - .ok_or_else(|| tracedecay_domain::errors::TraceDecayError::Config { - message: unsupported_host_component_set_message(agent_id), - })?; - ensure_artifact_only_restore_boundary(agent_id, &component_set, home, lifecycle_root)?; - let [entry] = component_set.component_set.components.as_slice() else { - return Err(tracedecay_domain::errors::TraceDecayError::Config { - message: "artifact backup/restore requires exactly one canonical component".to_string(), - }); - }; - let operation_id = tracedecay_contracts::request_identity::mint_global_operation_id( - tracedecay_contracts::request_identity::GlobalOperationIdentityKind::HostArtifact, - ) - .map_err(|error| tracedecay_domain::errors::TraceDecayError::Config { - message: format!("could not generate host artifact operation id: {error}"), - })?; - let mut writer = - tracedecay_agent_hosts::agents::host_bundle::HostBundleWriterV1::open_with_lifecycle_root( - home, - lifecycle_root, - ) - .map_err(host_bundle_error)?; - match backup_operation_id { - None => { - writer - .backup_component(&entry.manifest, operation_id, options.yes, &component_set) - .map_err(host_bundle_error)?; - } - Some(backup_operation_id) => { - writer - .restore_component_backup( - backup_operation_id, - operation_id, - options.yes, - &component_set, - ) - .map_err(host_bundle_error)?; - } - } - Ok(operation_id) -} - -pub(crate) async fn handle_host_bundle_artifact_command( - action: crate::cli::HostBundleAction, - options: crate::cli::HostBundleCliOptions, -) -> tracedecay_domain::errors::Result<()> { - let home = tracedecay_agent_hosts::agents::home_dir().ok_or_else(|| { - tracedecay_domain::errors::TraceDecayError::Config { - message: "could not determine home directory".to_string(), - } - })?; - let lifecycle_root = - tracedecay_agent_hosts::agents::host_bundle::resolved_host_bundle_lifecycle_root() - .map_err(|error| tracedecay_domain::errors::TraceDecayError::Config { - message: format!("could not resolve host lifecycle root: {error}"), - })?; - let now_unix = SystemTime::now() - .duration_since(UNIX_EPOCH) - .map_err(|_| tracedecay_domain::errors::TraceDecayError::Config { - message: "system clock is before the Unix epoch".to_string(), - })? - .as_secs(); - let is_restore = matches!( - &action, - crate::cli::HostBundleAction::ArtifactRestore { .. } - ); - let operation_id = - apply_host_bundle_artifact_action_at(action, options, &home, &lifecycle_root, now_unix)?; - if is_restore { - eprintln!( - "\x1b[32m✔\x1b[0m managed artifact files restored; host registration was not changed; receipt {}", - hex::encode(operation_id) - ); - } else { - eprintln!( - "\x1b[32m✔\x1b[0m managed artifact files backed up; host registration was not captured; backup id {}", - hex::encode(operation_id) - ); - } - Ok(()) -} - /// Uninstall removes host registration. That stays behind `--yes`. /// Install, update, and repair are the command the operator already ran. pub(crate) fn component_mutation_still_requires_yes( @@ -493,6 +471,7 @@ fn dry_run_canonical_component_set( options: &crate::cli::HostBundleCliOptions, home: &Path, lifecycle_root: &Path, + context: &ComponentSetApplyContext, ) -> tracedecay_domain::errors::Result<()> { let preview = preview_canonical_component_set( agent_id, @@ -501,7 +480,7 @@ fn dry_run_canonical_component_set( options, home, lifecycle_root, - None, + Some(context), )?; eprintln!( "{} {:?}: plan={}, registration_base={}, registration_current={}, artifacts={}, confirmation={}", @@ -521,8 +500,6 @@ fn dry_run_canonical_component_set( hex::encode(claim.evidence_digest) ); } - let backup_root = - tracedecay_agent_hosts::agents::host_bundle::host_bundle_backup_root(lifecycle_root); for plan in &preview.component_plans { eprintln!( " {:?}: {} mutation(s), rollback={}", @@ -543,9 +520,6 @@ fn dry_run_canonical_component_set( artifact_disposition(&mutation.action, &owned, &mutation.relative_path) ); } - if plan.rollback_required { - eprintln!(" backups: {}/", backup_root.display()); - } } Ok(()) } @@ -589,9 +563,9 @@ fn artifact_disposition( Action::Noop if receipt_owned.contains(relative_path) => "unchanged", Action::Noop => "adopt", Action::WriteNew => "write-new", - Action::BackupThenRemove => "backup-then-remove", - Action::BackupThenReplace if receipt_owned.contains(relative_path) => "backup-then-replace", - Action::BackupThenReplace => "adopt", + Action::Remove => "remove", + Action::Replace if receipt_owned.contains(relative_path) => "replace", + Action::Replace => "adopt", } } @@ -602,26 +576,24 @@ fn preview_canonical_component_set( options: &crate::cli::HostBundleCliOptions, home: &Path, lifecycle_root: &Path, - install_context: Option<&tracedecay_agent_hosts::agents::InstallContext>, + context: Option<&ComponentSetApplyContext>, ) -> tracedecay_domain::errors::Result< tracedecay_agent_hosts::agents::host_bundle::HostComponentSetLifecyclePreviewV1, > { let request = component_set_request(component_set, operation, options.yes, options.adopt)?; - let mut registration = match install_context { - Some(install) => { + let mut registration = match context { + Some(context) => { CatalogHostComponentRegistrationAuthority::new_with_tracedecay_bin_and_dashboard( agent_id, home, - lifecycle_root, request.lifecycle.operation, - install.tracedecay_bin.clone(), - install.dashboard, + context.tracedecay_bin.clone(), + context.dashboard, )? } None => CatalogHostComponentRegistrationAuthority::new( agent_id, home, - lifecycle_root, request.lifecycle.operation, )?, }; @@ -636,50 +608,9 @@ fn preview_canonical_component_set( .map_err(|error| host_bundle_error_for_agent(agent_id, error)) } -/// Recover this host's outstanding component-set journal before a confirmed -/// apply mutates anything. -/// -/// `HostComponentSetTransactionV1::execute` already recovers first, but the -/// preview/`execute_confirmed` pair used by every non-interactive refresh did -/// not: a completed rollback intentionally leaves its journal behind as an -/// explicit reconciliation boundary, and the very next preview then refuses -/// with `RecoveryRequired`. Without this, a single failed apply wedges the -/// refresh loop until an operator runs `tracedecay host-bundle recover` by -/// hand. Recovery is bound to the *journal's* own operation, exactly like the -/// recover command, because the registration backup only validates against the -/// operation that wrote it. -fn recover_pending_component_set_journal( - agent_id: &str, - host: tracedecay_agent_hosts::agents::host_bundle::HostKindV1, - writer: &mut tracedecay_agent_hosts::agents::host_bundle::HostBundleWriterV1, - build_registration: impl FnOnce( - tracedecay_agent_hosts::agents::host_bundle::HostBundleLifecycleOpV1, - ) -> tracedecay_domain::errors::Result< - CatalogHostComponentRegistrationAuthority, - >, -) -> tracedecay_domain::errors::Result<()> { - let Some(operation) = writer - .pending_component_set_journal_operation(host) - .map_err(host_bundle_error)? - else { - return Ok(()); - }; - let mut registration = build_registration(operation)?; - tracedecay_agent_hosts::agents::host_bundle::HostComponentSetTransactionV1::new(writer) - .recover_host(host, &mut registration) - .map_err(|error| host_bundle_error_for_agent(agent_id, error)) -} - -/// How one component-set apply reaches the outside world: which `tracedecay` -/// binary the written registrations invoke, and whether the dashboard component -/// is registered alongside them. -/// -/// Both knobs were previously spelled as a telescoping chain of forwarding -/// overloads (`_with_dashboard`, `_with_tracedecay_bin`, -/// `_with_tracedecay_bin_and_dashboard`), so every caller had to know which rung -/// defaulted which knob. Naming them once keeps -/// [`apply_canonical_component_set`] a single entry point whose defaults are -/// chosen by the constructor the caller names. +/// How one component-set lifecycle reaches the outside world: which +/// `tracedecay` binary the written registrations invoke, and whether the +/// dashboard component is registered alongside them. #[derive(Clone, Debug)] struct ComponentSetApplyContext { tracedecay_bin: String, @@ -688,6 +619,7 @@ struct ComponentSetApplyContext { impl ComponentSetApplyContext { /// The production context: the resolved installed binary, dashboard on. + #[cfg(test)] fn resolved() -> Self { Self::resolved_with_dashboard(true) } @@ -740,21 +672,6 @@ fn apply_canonical_component_set( lifecycle_root, ) .map_err(|error| host_bundle_error_for_agent(agent_id, error))?; - recover_pending_component_set_journal( - agent_id, - component_set.component_set.host, - &mut writer, - |operation| { - CatalogHostComponentRegistrationAuthority::new_with_tracedecay_bin_and_dashboard( - agent_id, - home, - lifecycle_root, - operation, - tracedecay_bin.to_string(), - dashboard, - ) - }, - )?; let mut transaction = tracedecay_agent_hosts::agents::host_bundle::HostComponentSetTransactionV1::new( &mut writer, @@ -763,19 +680,10 @@ fn apply_canonical_component_set( CatalogHostComponentRegistrationAuthority::new_with_tracedecay_bin_and_dashboard( agent_id, home, - lifecycle_root, request.lifecycle.operation, tracedecay_bin.to_string(), dashboard, )?; - // Recover this host's own outstanding journal before previewing, exactly as - // `HostComponentSetTransactionV1::execute` does. Without this the residue of - // any earlier failure, including one that has since been fixed, makes - // every later run refuse with `RecoveryRequired` until somebody runs - // `host-bundle recover` by hand, so a transient fault becomes permanent. - transaction - .recover_host(component_set.component_set.host, &mut registration) - .map_err(|error| host_bundle_error_for_agent(agent_id, error))?; let preview = transaction .preview( &component_set.component_set, @@ -786,14 +694,15 @@ fn apply_canonical_component_set( .map_err(|error| host_bundle_error_for_agent(agent_id, error))?; // Receiptless-adoption authority is enforced inside the planner: without // `--adopt`, a receiptless file at a cataloged path is adopted only when - // the host adapter recognizes legacy first-party provenance in it, and - // anything else is refused as a typed ownership conflict naming the - // `--yes --adopt` remedy. Reaching this point means every planned + // it matches the staged bytes, and anything else is refused as a typed + // ownership conflict naming the `--yes --adopt` remedy. Reaching this point means every planned // adoption was authorized, so no separate CLI gate re-litigates it. - // A full default install is otherwise treated as confirmed. A competing - // third-party claim is exactly the ambiguity that must not be resolved on - // the operator's behalf, so it demands an explicit `--yes`. - if !preview.competing_extension_claims.is_empty() && !options.yes { + // A full canonical set registers alongside competing third-party claims + // (OpenCode's analyzer ownership projection accounts for them). A single + // `--component` names one surface, so a claim on it demands an explicit + // `--yes` rather than being resolved on the operator's behalf. + if !preview.competing_extension_claims.is_empty() && options.component.is_some() && !options.yes + { return Err(tracedecay_domain::errors::TraceDecayError::Config { message: format!( "agent {agent_id:?} already has {} third-party extension claim(s) on a surface \ @@ -825,11 +734,11 @@ fn apply_canonical_component_set( receipt.component_receipts.len(), hex::encode(receipt.operation_id) ); - if agent_id == "cursor" - && request.lifecycle.operation - != tracedecay_agent_hosts::agents::host_bundle::HostBundleLifecycleOpV1::Uninstall - { - tracedecay_agent_hosts::agents::cursor::sweep_retired_cursor_plugin_artifacts(home)?; + // The receipt owns the staged source; the host still has to activate it. + if let Some(remediation) = registration.deferred_activation() { + return Err(tracedecay_domain::errors::TraceDecayError::Config { + message: remediation.to_string(), + }); } // Hook trust is the one Codex activation step that stays host-owned, so a // successful (re)install finishes with the exact remaining action. @@ -844,50 +753,6 @@ fn apply_canonical_component_set( Ok(()) } -/// Apply the agent's default component set. `dashboard` decides whether the -/// dashboard component is registered with it; uninstall paths pass `true` -/// because removal must cover everything an install could have written. -fn apply_default_canonical_component_set( - agent_id: &str, - operation: HostBundleCliOperation, - home: &Path, - dashboard: bool, - adopt: bool, -) -> tracedecay_domain::errors::Result<()> { - let now_unix = SystemTime::now() - .duration_since(UNIX_EPOCH) - .map_err(|_| tracedecay_domain::errors::TraceDecayError::Config { - message: "system clock is before the Unix epoch".to_string(), - })? - .as_secs(); - let component_set = - canonical_host_component_set(agent_id, None, now_unix)?.ok_or_else(|| { - tracedecay_domain::errors::TraceDecayError::Config { - message: unsupported_host_component_set_message(agent_id), - } - })?; - let lifecycle_root = - tracedecay_agent_hosts::agents::host_bundle::resolved_host_bundle_lifecycle_root() - .map_err(|error| tracedecay_domain::errors::TraceDecayError::Config { - message: format!("could not resolve host lifecycle root: {error}"), - })?; - apply_canonical_component_set( - agent_id, - operation, - &component_set, - &crate::cli::HostBundleCliOptions { - component: None, - dry_run: false, - yes: true, - adopt, - }, - home, - &lifecycle_root, - &ComponentSetApplyContext::resolved_with_dashboard(dashboard), - )?; - Ok(()) -} - fn load_host_lifecycle_user_config() -> tracedecay_domain::errors::Result { UserConfig::load_strict().map_err(|error| tracedecay_domain::errors::TraceDecayError::Config { message: format!("failed to load host lifecycle policy: {error}"), @@ -925,7 +790,6 @@ pub(crate) async fn handle_project_local_lifecycle_command( let context = tracedecay_agent_hosts::agents::InstallContext { home: home.clone(), tracedecay_bin, - tool_permissions: tracedecay_agent_hosts::agents::expected_tool_perms()?, project_root: Some(project_path.clone()), dashboard: false, }; @@ -941,7 +805,6 @@ pub(crate) async fn handle_project_local_lifecycle_command( HostBundleCliOperation::Install | HostBundleCliOperation::Update | HostBundleCliOperation::Repair => { - prepare_native_activation_if_needed(integration.as_ref(), &context)?; integration.activate_project_host_component_registration( &components, &context, @@ -993,134 +856,6 @@ fn host_bundle_component( } } -/// Inspect or recover an interrupted first-party host component transaction. -/// -/// This is the supported replacement for hand-deleting -/// `~/.tracedecay/host-components/.tracedecay-host-bundle-v1/component-set-journal.*.json`, -/// which used to be the only way out of a wedged host lifecycle. -#[hotpath::measure(future = true, label = "cli.agent.recovery")] -pub(crate) async fn handle_host_bundle_recovery_command( - action: crate::cli::HostBundleAction, - dry_run: bool, - yes: bool, -) -> tracedecay_domain::errors::Result<()> { - handle_host_bundle_recovery_command_inner(action, dry_run, yes).await -} - -fn handle_host_bundle_recovery_command_inner( - action: crate::cli::HostBundleAction, - dry_run: bool, - yes: bool, -) -> std::pin::Pin< - Box> + Send + 'static>, -> { - // Erase the deeply nested host-bundle-recovery future before it reaches - // the measured wrapper so every profiling feature can compute its layout. - Box::pin(async move { - let home = tracedecay_agent_hosts::agents::home_dir().ok_or_else(|| { - tracedecay_domain::errors::TraceDecayError::Config { - message: "could not determine home directory".to_string(), - } - })?; - let lifecycle_root = - tracedecay_agent_hosts::agents::host_bundle::resolved_host_bundle_lifecycle_root() - .map_err(|error| tracedecay_domain::errors::TraceDecayError::Config { - message: format!("could not resolve host lifecycle root: {error}"), - })?; - let mut writer = - tracedecay_agent_hosts::agents::host_bundle::HostBundleWriterV1::open_with_lifecycle_root( - &home, - &lifecycle_root, - ) - .map_err(host_bundle_error)?; - - let (selected_agent, quarantine, status_only) = match action { - crate::cli::HostBundleAction::Status => (None, false, true), - crate::cli::HostBundleAction::Recover { agent, quarantine } => { - (agent, quarantine, dry_run) - } - crate::cli::HostBundleAction::ArtifactBackup { .. } - | crate::cli::HostBundleAction::ArtifactRestore { .. } => { - return Err(tracedecay_domain::errors::TraceDecayError::Config { - message: "artifact backup and restore are not recovery operations".to_string(), - }); - } - }; - - let mut pending = writer - .pending_component_set_journal_hosts() - .map_err(host_bundle_error)?; - if let Some(agent) = selected_agent.as_deref() { - let host = host_kind_for_agent(agent)?; - pending.retain(|pending_host| *pending_host == host); - } - if pending.is_empty() { - eprintln!("\x1b[32m✔\x1b[0m no host component lifecycle journal is awaiting recovery"); - return Ok(()); - } - for host in &pending { - eprintln!( - " pending: {} ({:?})", - tracedecay_agent_hosts::agents::integration_id_for_host(*host), - host - ); - } - if status_only { - return Ok(()); - } - if !yes { - return Err(tracedecay_domain::errors::TraceDecayError::Config { - message: "host component recovery mutates deployed files; re-run with --yes" - .to_string(), - }); - } - - let now_unix = SystemTime::now() - .duration_since(UNIX_EPOCH) - .map_err(|_| tracedecay_domain::errors::TraceDecayError::Config { - message: "system clock is before the Unix epoch".to_string(), - })? - .as_secs(); - for host in pending { - let agent_id = tracedecay_agent_hosts::agents::integration_id_for_host(host); - let operation = writer - .pending_component_set_journal_operation(host) - .map_err(host_bundle_error)? - .ok_or_else(|| tracedecay_domain::errors::TraceDecayError::Config { - message: format!("{agent_id}: pending lifecycle journal disappeared"), - })?; - let mut registration = CatalogHostComponentRegistrationAuthority::new( - agent_id, - &home, - &lifecycle_root, - operation, - )?; - let outcome = - tracedecay_agent_hosts::agents::host_bundle::HostComponentSetTransactionV1::new( - &mut writer, - ) - .recover_host(host, &mut registration); - match outcome { - Ok(()) => eprintln!("\x1b[32m✔\x1b[0m {agent_id}: lifecycle journal recovered"), - Err(error) if quarantine => { - let quarantined = writer - .quarantine_component_set_journal(host, now_unix) - .map_err(host_bundle_error)?; - match quarantined { - Some(path) => eprintln!( - "\x1b[33m!\x1b[0m {agent_id}: {error}; journal quarantined at {} (rollback backups preserved)", - path.display() - ), - None => eprintln!("\x1b[32m✔\x1b[0m {agent_id}: lifecycle journal cleared"), - } - } - Err(error) => return Err(host_bundle_error(error)), - } - } - Ok(()) - }) -} - /// Inverse of `integration_id_for_host`, derived from the stock host list so /// the two cannot drift apart. /// @@ -1178,10 +913,10 @@ fn host_bundle_error_for_agent( == tracedecay_agent_hosts::agents::host_bundle::HostBundleError::UnsupportedCapability { return tracedecay_domain::errors::TraceDecayError::Config { - message: "Codex activates plugins through its native cache. Run `codex plugin add \ - tracedecay@personal` after TraceDecay stages the source package. Confirm \ - the `codex` CLI is on PATH and retry; hook trust still requires `/hooks` \ - inside Codex after a successful add." + message: "Codex activates plugins through its native cache, which TraceDecay drives \ + with `codex plugin add tracedecay@personal` after deploying the source \ + package. Confirm the `codex` CLI is on PATH and retry; hook trust still \ + requires `/hooks` inside Codex after a successful add." .to_string(), }; } @@ -1191,186 +926,15 @@ fn host_bundle_error_for_agent( ) && agent_id == "codex" { return tracedecay_domain::errors::TraceDecayError::Config { - message: "Codex activates plugins through its native cache. Run `codex plugin add \ - tracedecay@personal` after TraceDecay stages the source package. Install \ - the `codex` CLI or add it to PATH, then retry." + message: "Codex activates plugins through its native cache, which TraceDecay drives \ + with `codex plugin add tracedecay@personal` after deploying the source \ + package. Install the `codex` CLI or add it to PATH, then retry." .to_string(), }; } host_bundle_error(error) } -pub(crate) async fn handle_install_command( - agent: Option, - local: bool, - no_dashboard: bool, - automation: Option, - adopt: bool, - git_hook: bool, -) -> tracedecay_domain::errors::Result<()> { - validate_codex_automation_flags(agent.as_deref(), automation)?; - if local { - let agent_id = agent.ok_or_else(|| tracedecay_domain::errors::TraceDecayError::Config { - message: "`tracedecay install --local` requires a project-capable `--agent`" - .to_string(), - })?; - handle_project_local_lifecycle_command(agent_id, HostBundleCliOperation::Install).await?; - if git_hook { - install_requested_git_hook()?; - } - return Ok(()); - } - let home = tracedecay_agent_hosts::agents::home_dir().ok_or_else(|| { - tracedecay_domain::errors::TraceDecayError::Config { - message: "could not determine home directory".to_string(), - } - })?; - let tracedecay_bin = tracedecay_agent_hosts::agents::which_tracedecay().ok_or_else(|| { - tracedecay_domain::errors::TraceDecayError::Config { - message: "tracedecay not found on PATH. Install the checksummed GitHub release:\n \ - https://github.com/ScriptedAlchemy/tracedecay/releases/latest" - .to_string(), - } - })?; - let mut user_cfg = load_host_lifecycle_user_config()?; - - let mut installed_names: Vec = Vec::new(); - // Ids this pass actually (re)installed at the current binary version. The - // tail uses this to decide whether the pass covered every tracked agent - // and may disarm the startup silent reinstall. - let mut refreshed_ids: std::collections::BTreeSet = std::collections::BTreeSet::new(); - if let Some(id) = agent { - let ag = tracedecay_agent_hosts::agents::get_integration(&id)?; - let name = ag.name().to_string(); - let context = tracedecay_agent_hosts::agents::InstallContext { - home: home.clone(), - tracedecay_bin: tracedecay_bin.clone(), - tool_permissions: tracedecay_agent_hosts::agents::expected_tool_perms()?, - project_root: None, - dashboard: !no_dashboard, - }; - prepare_native_activation_if_needed(ag.as_ref(), &context)?; - apply_default_canonical_component_set( - &id, - HostBundleCliOperation::Install, - &home, - !no_dashboard, - adopt, - )?; - refreshed_ids.insert(id.clone()); - if let Some(options) = automation.filter(|_| id == "codex") { - let scoped_project_path = validate_codex_automation_project_path()?; - hotpath::future!( - install_codex_daemon_automation(&scoped_project_path, &home, options), - label = "cli.agent.automation" - ) - .await?; - } - user_cfg - .agent_dashboard_enabled - .insert(id.clone(), !no_dashboard); - if !user_cfg.installed_agents.contains(&id) { - user_cfg.installed_agents.push(id); - installed_names.push(name); - } - user_cfg - .save() - .map_err(|err| tracedecay_domain::errors::TraceDecayError::Config { - message: format!("failed to save user config: {err}"), - })?; - } else { - // Nothing to configure is an ordinary first-run state, not a failure: - // this is the command a user runs before any agent exists. An explicit - // `--agent` that cannot be satisfied still fails, above. - let Some(to_install) = tracedecay_agent_hosts::agents::select_detected_integrations( - &home, - &user_cfg.installed_agents, - ) else { - eprintln!(); - eprintln!( - "{}", - tracedecay_agent_hosts::agents::no_detected_integrations_notice(&home) - ); - if git_hook { - install_requested_git_hook()?; - } - return Ok(()); - }; - - for id in &to_install { - let ag = tracedecay_agent_hosts::agents::get_integration(id)?; - let context = tracedecay_agent_hosts::agents::InstallContext { - home: home.clone(), - tracedecay_bin: tracedecay_bin.clone(), - tool_permissions: tracedecay_agent_hosts::agents::expected_tool_perms()?, - project_root: None, - dashboard: !no_dashboard, - }; - prepare_native_activation_if_needed(ag.as_ref(), &context)?; - apply_default_canonical_component_set( - id, - HostBundleCliOperation::Install, - &home, - !no_dashboard, - adopt, - )?; - refreshed_ids.insert(id.clone()); - installed_names.push(ag.name().to_string()); - if !user_cfg.installed_agents.contains(id) { - user_cfg.installed_agents.push(id.clone()); - } - user_cfg - .agent_dashboard_enabled - .insert(id.clone(), !no_dashboard); - } - user_cfg - .save() - .map_err(|err| tracedecay_domain::errors::TraceDecayError::Config { - message: format!("failed to save user config: {err}"), - })?; - } - - eprintln!(); - if installed_names.is_empty() { - eprintln!("No changes."); - } else { - for name in &installed_names { - eprintln!("\x1b[32m+\x1b[0m {name}"); - } - } - - // An explicit install pass only refreshes its selection delta, so it may - // disarm the startup silent reinstall (`previous_version`) only when every - // agent still tracked was (re)installed by this very pass. Anything less - // advances `last_installed_version` alone and leaves the arming intact: - // after an upgrade, the untouched agents still need the silent refresh. - if crate::update_cmd::install_pass_covers_tracked_agents( - &user_cfg.installed_agents, - &refreshed_ids, - ) { - crate::update_cmd::record_completed_reinstall_pass(&mut user_cfg)?; - } else { - user_cfg.last_installed_version = env!("CARGO_PKG_VERSION").to_string(); - user_cfg - .save() - .map_err(|err| tracedecay_domain::errors::TraceDecayError::Config { - message: format!("failed to save user config: {err}"), - })?; - } - - // An install pass is a lifecycle pass: converge the managed-skill exports - // (prompt indexes and materialized files) against the store, so a host is - // never left advertising a skill the store no longer holds. - crate::update_cmd::deploy_managed_skills_after_lifecycle(); - - if git_hook { - install_requested_git_hook()?; - } else { - tracedecay_agent_hosts::agents::report_git_post_commit_hook_status(); - } - Ok(()) -} - pub(crate) fn install_requested_git_hook() -> tracedecay_domain::errors::Result<()> { let tracedecay_bin = tracedecay_agent_hosts::agents::which_tracedecay().ok_or_else(|| { tracedecay_domain::errors::TraceDecayError::Config { @@ -1381,279 +945,6 @@ pub(crate) fn install_requested_git_hook() -> tracedecay_domain::errors::Result< .map_err(|message| tracedecay_domain::errors::TraceDecayError::Config { message }) } -pub(crate) async fn handle_reinstall_command(adopt: bool) -> tracedecay_domain::errors::Result<()> { - let home = tracedecay_agent_hosts::agents::home_dir().ok_or_else(|| { - tracedecay_domain::errors::TraceDecayError::Config { - message: "could not determine home directory".to_string(), - } - })?; - let tracedecay_bin = tracedecay_agent_hosts::agents::which_tracedecay().ok_or_else(|| { - tracedecay_domain::errors::TraceDecayError::Config { - message: "tracedecay not found on PATH".to_string(), - } - })?; - let mut user_cfg = load_host_lifecycle_user_config()?; - - if user_cfg.installed_agents.is_empty() { - eprintln!("No installed agents found. Run `tracedecay install` first."); - } else { - // Drop tracked ids that no longer resolve to an integration (a release - // renamed or removed one, or a typo landed in `installed_agents`). - // Without this the stale id is retried forever. Mirrors - // `run_post_update_mutations`. - let before = user_cfg.installed_agents.len(); - user_cfg - .installed_agents - .retain(|id| tracedecay_agent_hosts::agents::get_integration(id).is_ok()); - if user_cfg.installed_agents.len() != before - && let Err(err) = user_cfg.save() - { - eprintln!("warning: could not save tracedecay config: {err}"); - } - let agents = user_cfg.installed_agents.clone(); - eprintln!( - "Reinstalling {} agent(s): {}", - agents.len(), - agents.join(", ") - ); - let results = reinstall_agent_integrations_with_dashboard_policies( - &agents, - &home, - &tracedecay_bin, - &user_cfg.agent_dashboard_enabled, - adopt, - ) - .await; - // Reporting lives in `partition_reinstall_results`, which every - // reinstall pass shares. Keep the reason with the name, a bare id list - // left "failed for: claude, cursor, hermes, kimi" undiagnosable. - match crate::update_cmd::partition_reinstall_results(results) { - crate::update_cmd::ReinstallOutcome::AllOk => { - eprintln!("\x1b[32m✔\x1b[0m All agents reinstalled"); - } - crate::update_cmd::ReinstallOutcome::PartialFailure { failed } => { - return Err(tracedecay_domain::errors::TraceDecayError::Config { - message: format!("failed to reinstall agent(s): {}", failed.join("; ")), - }); - } - } - // Advance BOTH markers: `previous_version` is what arms the startup - // silent reinstall, so recording only `last_installed_version` here - // left this explicit pass re-running on the next ordinary command. - crate::update_cmd::record_completed_reinstall_pass(&mut user_cfg)?; - } - Ok(()) -} - -pub(crate) async fn handle_update_plugin_command( - adopt: bool, -) -> tracedecay_domain::errors::Result<()> { - let home = tracedecay_agent_hosts::agents::home_dir().ok_or_else(|| { - tracedecay_domain::errors::TraceDecayError::Config { - message: "could not determine home directory".to_string(), - } - })?; - let tracedecay_bin = tracedecay_agent_hosts::agents::which_tracedecay().ok_or_else(|| { - tracedecay_domain::errors::TraceDecayError::Config { - message: "tracedecay not found on PATH".to_string(), - } - })?; - let user_cfg = load_host_lifecycle_user_config()?; - - for id in &user_cfg.installed_agents { - let integration = tracedecay_agent_hosts::agents::get_integration(id)?; - let dashboard = user_cfg.dashboard_enabled_for_agent(id); - let context = tracedecay_agent_hosts::agents::InstallContext { - home: home.clone(), - tracedecay_bin: tracedecay_bin.clone(), - tool_permissions: tracedecay_agent_hosts::agents::expected_tool_perms()?, - project_root: None, - dashboard, - }; - prepare_native_activation_if_needed(integration.as_ref(), &context)?; - apply_default_canonical_component_set( - id, - HostBundleCliOperation::Update, - &home, - dashboard, - adopt, - )?; - } - Ok(()) -} - -#[hotpath::measure(label = "cli.agent.preflight")] -pub(crate) fn handle_reinstall_preflight_command() -> tracedecay_domain::errors::Result<()> { - let home = tracedecay_agent_hosts::agents::home_dir().ok_or_else(|| { - tracedecay_domain::errors::TraceDecayError::Config { - message: "could not determine home directory".to_string(), - } - })?; - let tracedecay_bin = tracedecay_agent_hosts::agents::which_tracedecay().ok_or_else(|| { - tracedecay_domain::errors::TraceDecayError::Config { - message: "could not resolve the canonical preflight binary".to_string(), - } - })?; - let user_config = load_host_lifecycle_user_config()?; - let agent_ids = user_config.installed_agents.clone(); - let project_path = std::env::current_dir().unwrap_or_else(|_| PathBuf::from(".")); - let health_context = tracedecay_agent_hosts::agents::HealthcheckContext { - home: home.clone(), - project_path, - }; - let mut failures = Vec::new(); - let mut checked = 0_usize; - let mut deferred = 0_usize; - - eprintln!( - "Read-only integration refresh preflight ({} tracked).", - agent_ids.len() - ); - for id in &agent_ids { - let integration = match tracedecay_agent_hosts::agents::get_integration(id) { - Ok(integration) => integration, - Err(_) => { - eprintln!(" \x1b[33m!\x1b[0m {id}: unknown tracked id; refresh will skip it"); - continue; - } - }; - checked += 1; - let dashboard = user_config.dashboard_enabled_for_agent(id); - let install_context = tracedecay_agent_hosts::agents::InstallContext { - home: home.clone(), - tracedecay_bin: tracedecay_bin.clone(), - tool_permissions: tracedecay_agent_hosts::agents::expected_tool_perms()?, - project_root: None, - dashboard, - }; - match integration.preflight_non_interactive_install(&install_context) { - Ok(tracedecay_agent_hosts::agents::NonInteractiveInstallOutcome::Ready) => {} - Ok( - tracedecay_agent_hosts::agents::NonInteractiveInstallOutcome::DeferredUserAction( - action, - ), - ) => { - deferred += 1; - eprintln!( - " \x1b[32m✔\x1b[0m {id}: deferred manual activation is accepted ({})", - action.remediation - ); - continue; - } - Err(error) => { - eprintln!(" \x1b[31m✘\x1b[0m {id}: {error}"); - failures.push(format!("{id}: {error}")); - continue; - } - } - - match preflight_agent_integration( - id, - integration.as_ref(), - &home, - &health_context, - &install_context, - ) { - Ok(summary) => eprintln!(" \x1b[32m✔\x1b[0m {id}: {summary}"), - Err(error) => { - eprintln!(" \x1b[31m✘\x1b[0m {id}: {error}"); - failures.push(format!("{id}: {error}")); - } - } - } - - if failures.is_empty() { - eprintln!("Integration refresh preflight passed: {checked} checked, {deferred} deferred."); - return Ok(()); - } - Err(tracedecay_domain::errors::TraceDecayError::Config { - message: format!( - "integration refresh preflight failed for: {}", - failures.join(", ") - ), - }) -} - -fn preflight_agent_integration( - agent_id: &str, - integration: &dyn tracedecay_agent_hosts::agents::AgentIntegration, - home: &Path, - health_context: &tracedecay_agent_hosts::agents::HealthcheckContext, - install_context: &tracedecay_agent_hosts::agents::InstallContext, -) -> tracedecay_domain::errors::Result { - use tracedecay_agent_hosts::agents::host_bundle::HostBundleRegistrationStateV1; - - let now_unix = SystemTime::now() - .duration_since(UNIX_EPOCH) - .map_err(|_| tracedecay_domain::errors::TraceDecayError::Config { - message: "system clock is before the Unix epoch".to_string(), - })? - .as_secs(); - let Some(component_set) = canonical_host_component_set(agent_id, None, now_unix)? else { - return Err(tracedecay_domain::errors::TraceDecayError::Config { - message: unsupported_host_component_set_message(agent_id), - }); - }; - - let mut registration_states = Vec::new(); - for component in &component_set.component_set.components { - let state = integration.host_component_registration_for_lifecycle( - component.manifest.component, - health_context, - install_context, - ); - if state == HostBundleRegistrationStateV1::Corrupt { - return Err(tracedecay_domain::errors::TraceDecayError::Config { - message: format!( - "{:?} registration config is corrupt", - component.manifest.component - ), - }); - } - registration_states.push(format!( - "{:?}={}", - component.manifest.component, - registration_state_label(state) - )); - } - let lifecycle_root = - tracedecay_agent_hosts::agents::host_bundle::resolved_host_bundle_lifecycle_root() - .map_err(|error| tracedecay_domain::errors::TraceDecayError::Config { - message: format!("could not resolve host lifecycle root: {error}"), - })?; - preview_canonical_component_set( - agent_id, - HostBundleCliOperation::Repair, - &component_set, - &crate::cli::HostBundleCliOptions { - component: None, - dry_run: true, - yes: true, - adopt: false, - }, - home, - &lifecycle_root, - Some(install_context), - )?; - Ok(format!( - "signed repair plan valid; registration {}", - registration_states.join(", ") - )) -} - -fn registration_state_label( - state: tracedecay_agent_hosts::agents::host_bundle::HostBundleRegistrationStateV1, -) -> &'static str { - use tracedecay_agent_hosts::agents::host_bundle::HostBundleRegistrationStateV1; - - match state { - HostBundleRegistrationStateV1::Current => "current", - HostBundleRegistrationStateV1::Repairable => "repairable", - HostBundleRegistrationStateV1::Missing => "missing", - HostBundleRegistrationStateV1::Corrupt => "corrupt", - } -} - /// Reinstalls tracked integrations while reusing lifecycle authority already /// held by post-update maintenance. pub(crate) async fn reinstall_agent_integrations_under_lease( @@ -1670,6 +961,8 @@ pub(crate) async fn reinstall_agent_integrations_under_lease( .await } +/// The tracked-agent repair pass `reinstall` runs, one result per agent so +/// maintenance can continue past a failing host. async fn reinstall_agent_integrations_with_persisted_dashboard_policies( agent_ids: &[String], home: &Path, @@ -1678,8 +971,10 @@ async fn reinstall_agent_integrations_with_persisted_dashboard_policies( String, tracedecay_domain::errors::Result, )> { - let user_config = match load_host_lifecycle_user_config() { - Ok(config) => config, + let environment = load_host_lifecycle_user_config() + .and_then(|config| resolved_lifecycle_root().map(|root| (config, root))); + let (user_config, lifecycle_root) = match environment { + Ok(environment) => environment, Err(error) => { let message = error.to_string(); return agent_ids @@ -1695,126 +990,39 @@ async fn reinstall_agent_integrations_with_persisted_dashboard_policies( .collect(); } }; - reinstall_agent_integrations_with_dashboard_policies( - agent_ids, - home, - tracedecay_bin, - &user_config.agent_dashboard_enabled, - false, - ) - .await -} - -async fn reinstall_agent_integrations_with_dashboard_policies( - agent_ids: &[String], - home: &Path, - tracedecay_bin: &str, - dashboard_policies: &std::collections::BTreeMap, - adopt: bool, -) -> Vec<( - String, - tracedecay_domain::errors::Result, -)> { + let options = crate::cli::HostBundleCliOptions { + component: None, + dry_run: false, + yes: false, + adopt: false, + }; let mut results = Vec::new(); for id in agent_ids { - let ag = match tracedecay_agent_hosts::agents::get_integration(id) { - Ok(ag) => ag, - Err(_) => { - tracing::warn!( - agent_id = id, - "skipping unknown tracked agent id; it will not gate the version-marker refresh" - ); - continue; - } - }; - let dashboard = dashboard_policies.get(id).copied().unwrap_or(true); - // An unreadable tool catalog is this agent's reinstall failure, not a - // silent install with an empty permission allowlist. - let tool_permissions = match tracedecay_agent_hosts::agents::expected_tool_perms() { - Ok(tool_permissions) => tool_permissions, - Err(error) => { - results.push((id.clone(), Err(error))); - continue; - } - }; - let context = tracedecay_agent_hosts::agents::InstallContext { - home: home.to_path_buf(), - tracedecay_bin: tracedecay_bin.to_string(), - tool_permissions, - project_root: None, - dashboard, - }; - if let Err(error) = prepare_native_activation_if_needed(ag.as_ref(), &context) { - results.push((id.clone(), Err(error))); + if tracedecay_agent_hosts::agents::get_integration(id).is_err() { + tracing::warn!( + agent_id = id, + "skipping unknown tracked agent id; it will not gate the version-marker refresh" + ); continue; } - match apply_default_canonical_component_set( + let context = ComponentSetApplyContext { + tracedecay_bin: tracedecay_bin.to_string(), + dashboard: user_config.dashboard_enabled_for_agent(id), + }; + let result = run_host_component_lifecycle( id, HostBundleCliOperation::Repair, + &options, home, - dashboard, - adopt, - ) { - Ok(()) => { - results.push((id.clone(), Ok(AgentReinstallOutcome::Installed))); - continue; - } - Err(error) => { - results.push((id.clone(), Err(error))); - continue; - } - } + &lifecycle_root, + &context, + ) + .map(|()| AgentReinstallOutcome::Installed); + results.push((id.clone(), result)); } results } -pub(crate) async fn handle_uninstall_command( - agent: Option, -) -> tracedecay_domain::errors::Result<()> { - let home = tracedecay_agent_hosts::agents::home_dir().ok_or_else(|| { - tracedecay_domain::errors::TraceDecayError::Config { - message: "could not determine home directory".to_string(), - } - })?; - let mut user_cfg = load_host_lifecycle_user_config()?; - - if let Some(id) = agent { - apply_default_canonical_component_set( - &id, - HostBundleCliOperation::Uninstall, - &home, - true, - false, - )?; - user_cfg.installed_agents.retain(|a| a != &id); - user_cfg.agent_dashboard_enabled.remove(&id); - user_cfg - .save() - .map_err(|err| tracedecay_domain::errors::TraceDecayError::Config { - message: format!("failed to save user config: {err}"), - })?; - } else { - for id in user_cfg.installed_agents.clone() { - apply_default_canonical_component_set( - &id, - HostBundleCliOperation::Uninstall, - &home, - true, - false, - )?; - } - user_cfg.installed_agents.clear(); - user_cfg.agent_dashboard_enabled.clear(); - user_cfg - .save() - .map_err(|err| tracedecay_domain::errors::TraceDecayError::Config { - message: format!("failed to save user config: {err}"), - })?; - eprintln!("All agent integrations removed."); - } - Ok(()) -} - #[cfg(test)] mod tests { use std::path::{Path, PathBuf}; @@ -1826,7 +1034,6 @@ mod tests { use super::{ AgentReinstallOutcome, CatalogHostComponentRegistrationAuthority, ComponentSetApplyContext, HostBundleCliOperation, apply_canonical_component_set, - apply_default_canonical_component_set, apply_host_bundle_artifact_action_at, broker_codex_daemon_automation_project, canonical_host_component_set, canonical_host_component_set_with_tracedecay_bin, component_is_not_applicable, component_mutation_still_requires_yes, component_set_request, @@ -1893,6 +1100,29 @@ mod tests { tracedecay_runtime_core::config::PinnedUserDataDir::new() } + /// One agent's whole canonical component set, as `install --agent ` + /// (with `--yes --adopt` when `adopt`) runs it. + fn run_default_component_set( + agent_id: &str, + operation: HostBundleCliOperation, + home: &Path, + adopt: bool, + ) -> tracedecay_domain::errors::Result<()> { + super::run_host_component_lifecycle( + agent_id, + operation, + &crate::cli::HostBundleCliOptions { + component: None, + dry_run: false, + yes: adopt, + adopt, + }, + home, + &super::resolved_lifecycle_root()?, + &ComponentSetApplyContext::resolved_with_dashboard(true), + ) + } + #[test] fn unscoped_component_sweep_distinguishes_unsupported_from_optional() { assert!(component_is_not_applicable( @@ -1947,6 +1177,9 @@ mod tests { .join("host-cli-test-homes"); std::fs::create_dir_all(&root) .unwrap_or_else(|error| panic!("failed to create {}: {error}", root.display())); + // Spelled without the `..` hops: Windows private-file writes beneath a + // long home refuse any path that is not exactly absolute. + let root = tracedecay_runtime_core::path_safety::canonical_root_identity(&root); tempfile::Builder::new() .prefix(".tmp") .tempdir_in(&root) @@ -1980,15 +1213,11 @@ mod tests { .collect::>(); assert_eq!( - super::artifact_disposition( - &Action::BackupThenReplace, - &owned, - "plugins/tracedecay.json" - ), - "backup-then-replace" + super::artifact_disposition(&Action::Replace, &owned, "plugins/tracedecay.json"), + "replace" ); assert_eq!( - super::artifact_disposition(&Action::BackupThenReplace, &owned, "plugins/unowned.json"), + super::artifact_disposition(&Action::Replace, &owned, "plugins/unowned.json"), "adopt" ); assert_eq!( @@ -1996,12 +1225,8 @@ mod tests { "write-new" ); assert_eq!( - super::artifact_disposition( - &Action::BackupThenRemove, - &owned, - "plugins/tracedecay.json" - ), - "backup-then-remove" + super::artifact_disposition(&Action::Remove, &owned, "plugins/tracedecay.json"), + "remove" ); assert_eq!( super::artifact_disposition(&Action::Noop, &owned, "plugins/tracedecay.json"), @@ -2013,8 +1238,8 @@ mod tests { ); } - /// An explicit component repair that would claim an unrecorded file with - /// no recognizable legacy provenance is refused without `--adopt`, even + /// An explicit component repair that would claim an unrecorded file is + /// refused without `--adopt`, even /// at preview time, which stays read-only, and the refusal names the /// contested path plus the explicit adoption remedy. #[tokio::test] @@ -2029,9 +1254,8 @@ mod tests { ) .unwrap() .unwrap(); - // A receiptless deployment carrying no recognizable provenance: the - // cataloged path exists on disk with foreign bytes and no receipt - // records it. + // A receiptless deployment: the cataloged path exists on disk with + // foreign bytes and no receipt records it. let adopted = &component_set.component_set.components[0].manifest.artifacts[0].relative_path; let deployed = home.path().join(adopted); @@ -2139,11 +1363,10 @@ mod tests { std::fs::create_dir_all(deployed.parent().unwrap()).unwrap(); std::fs::write(&deployed, b"pre-receipt").unwrap(); - let error = apply_default_canonical_component_set( + let error = run_default_component_set( "cursor", HostBundleCliOperation::Install, home.path(), - true, false, ) .unwrap_err() @@ -2151,95 +1374,11 @@ mod tests { assert!(error.contains("--adopt"), "{error}"); assert_eq!(std::fs::read(&deployed).unwrap(), b"pre-receipt"); - apply_default_canonical_component_set( - "cursor", - HostBundleCliOperation::Install, - home.path(), - true, - true, - ) - .unwrap(); + run_default_component_set("cursor", HostBundleCliOperation::Install, home.path(), true) + .unwrap(); assert_ne!(std::fs::read(deployed).unwrap(), b"pre-receipt"); } - #[test] - fn cursor_adoption_sweeps_retired_artifacts_and_preserves_user_files() { - let _profile = pinned_host_profile(); - let home = tempfile::tempdir().unwrap(); - let lifecycle = tempfile::tempdir().unwrap(); - let component_set = - canonical_host_component_set_with_tracedecay_bin("cursor", None, 0, KIRO_FIXTURE_BIN) - .unwrap() - .unwrap(); - let relative = - &component_set.component_set.components[0].manifest.artifacts[0].relative_path; - let deployed = home.path().join(relative); - std::fs::create_dir_all(deployed.parent().unwrap()).unwrap(); - std::fs::write(&deployed, b"pre-receipt").unwrap(); - let plugin_dir = - tracedecay_agent_hosts::agents::cursor::cursor_plugin_install_dir(home.path()); - let retired = plugin_dir.join("rules/tracedecay-memory.mdc"); - std::fs::create_dir_all(retired.parent().unwrap()).unwrap(); - std::fs::write( - &retired, - "", - ) - .unwrap(); - let user_file = plugin_dir.join("operator-notes.md"); - std::fs::write(&user_file, b"keep me").unwrap(); - - super::apply_canonical_component_set( - "cursor", - HostBundleCliOperation::Install, - &component_set, - &crate::cli::HostBundleCliOptions { - component: None, - dry_run: false, - yes: true, - adopt: true, - }, - home.path(), - lifecycle.path(), - &ComponentSetApplyContext::with_tracedecay_bin(KIRO_FIXTURE_BIN), - ) - .unwrap(); - - assert!( - !retired.exists(), - "a known retired Cursor artifact must not survive adoption" - ); - assert_eq!(std::fs::read(&user_file).unwrap(), b"keep me"); - - // A retired rule resurfacing later (e.g. an older release ran again) - // is swept by the plain `update-plugin` journey too: the now-current - // bundle carries recognizable provenance, so no `--adopt` is needed. - std::fs::write( - &retired, - "", - ) - .unwrap(); - super::apply_canonical_component_set( - "cursor", - HostBundleCliOperation::Update, - &component_set, - &crate::cli::HostBundleCliOptions { - component: None, - dry_run: false, - yes: true, - adopt: false, - }, - home.path(), - lifecycle.path(), - &ComponentSetApplyContext::with_tracedecay_bin(KIRO_FIXTURE_BIN), - ) - .unwrap(); - assert!( - !retired.exists(), - "update-plugin must sweep the retired rule so uninstall can see a clean bundle" - ); - assert_eq!(std::fs::read(&user_file).unwrap(), b"keep me"); - } - struct VerifyFailureRegistration { inner: CatalogHostComponentRegistrationAuthority, stale_export_path: PathBuf, @@ -2341,99 +1480,6 @@ mod tests { /// mid-test. const KIRO_FIXTURE_BIN: &str = "/usr/local/bin/tracedecay"; - /// Forwards the whole lifecycle to the real authority but always fails - /// `verify`, which interrupts the transaction after its artifacts are on - /// disk, the state that leaves a recovery journal behind. - struct AlwaysFailVerifyRegistration { - inner: CatalogHostComponentRegistrationAuthority, - } - - impl HostComponentSetRegistrationV1 for AlwaysFailVerifyRegistration { - fn current_revision( - &self, - component_set: &HostComponentSetV1, - request: &HostComponentSetExecutionRequestV1, - ) -> Result<[u8; 32], HostBundleError> { - self.inner.current_revision(component_set, request) - } - - fn discover_competing_extension_claims( - &self, - component_set: &HostComponentSetV1, - request: &HostComponentSetExecutionRequestV1, - ) -> Result, HostBundleError> { - self.inner - .discover_competing_extension_claims(component_set, request) - } - - fn confirm_preview( - &mut self, - component_set: &HostComponentSetV1, - request: &HostComponentSetExecutionRequestV1, - preview: &HostComponentSetLifecyclePreviewV1, - ) -> Result<(), HostBundleError> { - self.inner.confirm_preview(component_set, request, preview) - } - - fn declare_artifact_writes( - &mut self, - component_set: &HostComponentSetV1, - request: &HostComponentSetExecutionRequestV1, - paths: &[PathBuf], - ) -> Result<(), HostBundleError> { - self.inner - .declare_artifact_writes(component_set, request, paths) - } - - fn preflight( - &mut self, - component_set: &HostComponentSetV1, - request: &HostComponentSetExecutionRequestV1, - ) -> Result<(), HostBundleError> { - self.inner.preflight(component_set, request) - } - - fn stage( - &mut self, - component_set: &HostComponentSetV1, - request: &HostComponentSetExecutionRequestV1, - ) -> Result<(), HostBundleError> { - self.inner.stage(component_set, request) - } - - fn apply( - &mut self, - component_set: &HostComponentSetV1, - request: &HostComponentSetExecutionRequestV1, - ) -> Result<(), HostBundleError> { - self.inner.apply(component_set, request) - } - - fn verify( - &mut self, - _component_set: &HostComponentSetV1, - _request: &HostComponentSetExecutionRequestV1, - ) -> Result<(), HostBundleError> { - Err(tracedecay_host_integration::host_bundle_storage_failure!()) - } - - fn commit( - &mut self, - component_set: &HostComponentSetV1, - request: &HostComponentSetExecutionRequestV1, - ) -> Result<(), HostBundleError> { - self.inner.commit(component_set, request) - } - - fn rollback( - &mut self, - component_set: &HostComponentSetV1, - request: &HostComponentSetExecutionRequestV1, - ) -> Result<(), HostBundleError> { - self.inner.rollback(component_set, request) - } - } - use super::isolated_profile::EnvVarGuard; /// Keep Kiro lifecycle tests on the native `kiro-cli` route. The compiled @@ -2632,127 +1678,6 @@ mod tests { ); } - #[test] - fn artifact_restore_refuses_components_with_registration_state() { - let _profile = pinned_host_profile(); - let home = tempfile::tempdir().unwrap(); - let lifecycle = tempfile::tempdir().unwrap(); - let core = canonical_host_component_set( - "codex", - Some(crate::cli::HostBundleComponentArg::Core), - 0, - ) - .unwrap() - .unwrap(); - let error = super::ensure_artifact_only_restore_boundary( - "codex", - &core, - home.path(), - lifecycle.path(), - ) - .unwrap_err(); - assert!( - error - .to_string() - .contains("does not manage registration state") - ); - - let artifact_only = canonical_host_component_set( - "opencode", - Some(crate::cli::HostBundleComponentArg::Agent), - 0, - ) - .unwrap() - .unwrap(); - assert!( - super::ensure_artifact_only_restore_boundary( - "opencode", - &artifact_only, - home.path(), - lifecycle.path(), - ) - .is_ok() - ); - } - - #[test] - fn artifact_backup_requires_the_writer_confirmation() { - let error = apply_host_bundle_artifact_action_at( - crate::cli::HostBundleAction::ArtifactBackup { - agent: "opencode".to_string(), - }, - crate::cli::HostBundleCliOptions::default(), - Path::new("/tmp"), - Path::new("/tmp"), - 0, - ) - .unwrap_err(); - assert!( - error.to_string().contains("artifact backup requires --yes"), - "{error}" - ); - } - - #[test] - fn artifact_command_route_backs_up_and_restores_managed_files() { - let _profile = pinned_host_profile(); - let home = tempfile::tempdir().unwrap(); - let lifecycle = tempfile::tempdir().unwrap(); - let component_set = canonical_host_component_set( - "opencode", - Some(crate::cli::HostBundleComponentArg::Agent), - 0, - ) - .unwrap() - .unwrap(); - let options = crate::cli::HostBundleCliOptions { - component: Some(crate::cli::HostBundleComponentArg::Agent), - dry_run: false, - yes: true, - adopt: false, - }; - apply_canonical_component_set( - "opencode", - HostBundleCliOperation::Install, - &component_set, - &options, - home.path(), - lifecycle.path(), - &ComponentSetApplyContext::resolved(), - ) - .unwrap(); - - let backup_id = super::apply_host_bundle_artifact_action_at( - crate::cli::HostBundleAction::ArtifactBackup { - agent: "opencode".to_string(), - }, - options, - home.path(), - lifecycle.path(), - 0, - ) - .unwrap(); - let content = &component_set.component_set.components[0].contents[0]; - std::fs::write(home.path().join(&content.relative_path), b"diverged").unwrap(); - - let restore_id = super::apply_host_bundle_artifact_action_at( - crate::cli::HostBundleAction::ArtifactRestore { - agent: "opencode".to_string(), - backup_id: hex::encode(backup_id), - }, - options, - home.path(), - lifecycle.path(), - 0, - ) - .unwrap(); - assert_ne!(restore_id, backup_id); - assert_eq!( - std::fs::read(home.path().join(&content.relative_path)).unwrap(), - content.bytes - ); - } - #[test] fn explicit_context_component_lifecycle_preserves_other_opencode_state() { let _profile = pinned_host_profile(); @@ -2868,32 +1793,11 @@ mod tests { let kiro_cli_dir = tempfile::tempdir().unwrap(); write_fake_kiro_cli(&kiro_cli_dir.path().join("kiro-cli")); let _kiro_path = - tracedecay_runtime_core::config::HostProgramSearchPathGuard::set(kiro_cli_dir.path()); - let home = tempfile::tempdir().unwrap(); - let project = tempfile::tempdir().unwrap(); - let lifecycle = tempfile::tempdir().unwrap(); - std::fs::create_dir_all(home.path().join(".kiro/steering")).unwrap(); - std::fs::create_dir_all(home.path().join(".kiro/agents")).unwrap(); - let legacy_steering = home.path().join(".kiro/steering/tracedecay.md"); - let legacy_agent = home.path().join(".kiro/agents/tracedecay.json"); - std::fs::write( - &legacy_steering, - "## Prefer tracedecay MCP tools\nold rules\n\n", - ) - .unwrap(); - std::fs::write( - &legacy_agent, - serde_json::to_vec(&serde_json::json!({ - "name": "tracedecay", - "description": "Default Kiro agent with tracedecay MCP tools and code-research guardrails.", - "hooks": {"userPromptSubmit": [{ - "command": "tracedecay hook-kiro-prompt-submit", - "timeout_ms": 5_000 - }]} - })) - .unwrap(), - ) - .unwrap(); + tracedecay_runtime_core::config::HostProgramSearchPathGuard::set(kiro_cli_dir.path()); + let home = tempfile::tempdir().unwrap(); + let project = tempfile::tempdir().unwrap(); + let lifecycle = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(home.path().join(".kiro")).unwrap(); let component_set = canonical_host_component_set_with_tracedecay_bin("kiro", None, 0, KIRO_FIXTURE_BIN) @@ -2915,27 +1819,16 @@ mod tests { ) .unwrap(); - // Global install is MCP-only, so both leftovers are retired by - // definition and the install sweeps them. Refreshing the steering block - // instead left doctor advising a remedy no install could satisfy. - assert!( - !legacy_steering.exists() - || !std::fs::read_to_string(&legacy_steering) - .unwrap() - .contains("old rules"), - "global install must not leave a retired steering block behind" - ); - assert!( - !legacy_agent.exists(), - "global install must sweep the retired managed agent" - ); + // Global install is MCP-only: no steering or managed agent is written. + assert!(!home.path().join(".kiro/steering/tracedecay.md").exists()); + assert!(!home.path().join(".kiro/agents/tracedecay.json").exists()); let registered: serde_json::Value = serde_json::from_slice( &std::fs::read(home.path().join(".kiro/settings/mcp.json")).unwrap(), ) .unwrap(); assert!( registered["mcpServers"]["tracedecay"].is_object(), - "sweeping retired artifacts must not disturb the MCP registration: {registered}" + "global install must register the MCP server: {registered}" ); let mut counters = DoctorCounters::new(); @@ -2947,102 +1840,7 @@ mod tests { }, ); assert_eq!(counters.issues, 0); - assert_eq!( - counters.warnings, 0, - "the install the advisories name must clear every one of them" - ); - } - - /// A transaction interrupted after it staged registration leaves a journal - /// behind. The non-interactive path must recover that journal itself, or a - /// single transient fault wedges every later run behind a manual - /// `host-bundle recover`. - #[cfg(unix)] - #[test] - fn interrupted_component_set_journal_recovers_on_next_non_interactive_apply() { - let _profile = pinned_host_profile(); - #[cfg(unix)] - let kiro_cli_dir = tempfile::tempdir().unwrap(); - #[cfg(unix)] - let kiro_cli_path = kiro_cli_dir.path().join("kiro-cli"); - #[cfg(unix)] - write_fake_kiro_cli(&kiro_cli_path); - #[cfg(unix)] - let _kiro_path = - tracedecay_runtime_core::config::HostProgramSearchPathGuard::set(kiro_cli_dir.path()); - let home = tempfile::tempdir().unwrap(); - let lifecycle = tempfile::tempdir().unwrap(); - std::fs::create_dir_all(home.path().join(".kiro")).unwrap(); - let component_set = - canonical_host_component_set_with_tracedecay_bin("kiro", None, 0, KIRO_FIXTURE_BIN) - .unwrap() - .unwrap(); - let options = crate::cli::HostBundleCliOptions { - component: None, - dry_run: false, - yes: true, - adopt: false, - }; - - // Interrupt a real transaction the way a crash would: run it through a - // registration authority that fails `verify` after the artifacts are - // already on disk, which is exactly the state that leaves a journal. - let request = - component_set_request(&component_set, HostBundleCliOperation::Install, true, false) - .unwrap(); - let mut writer = - tracedecay_agent_hosts::agents::host_bundle::HostBundleWriterV1::open_with_lifecycle_root( - home.path(), - lifecycle.path(), - ) - .unwrap(); - let mut transaction = - tracedecay_agent_hosts::agents::host_bundle::HostComponentSetTransactionV1::new( - &mut writer, - ); - let mut interrupted_registration = AlwaysFailVerifyRegistration { - inner: CatalogHostComponentRegistrationAuthority::new( - "kiro", - home.path(), - lifecycle.path(), - request.lifecycle.operation, - ) - .unwrap(), - }; - transaction - .execute( - &component_set.component_set, - &request, - &component_set, - &mut interrupted_registration, - ) - .expect_err("the injected verify failure must interrupt this transaction"); - drop(writer); - let journal_path = lifecycle - .path() - .join(".tracedecay-host-bundle-v1/component-set-journal.kiro.v1.json"); - assert!( - journal_path.is_file(), - "the interrupted transaction must leave a recovery journal behind" - ); - - // The next non-interactive run must clear the residue by itself. - apply_canonical_component_set( - "kiro", - HostBundleCliOperation::Repair, - &component_set, - &options, - home.path(), - lifecycle.path(), - &ComponentSetApplyContext::with_tracedecay_bin(KIRO_FIXTURE_BIN), - ) - .expect("a leftover journal must be recovered, not turned into a permanent refusal"); - - let registered: serde_json::Value = serde_json::from_slice( - &std::fs::read(home.path().join(".kiro/settings/mcp.json")).unwrap(), - ) - .unwrap(); - assert!(registered["mcpServers"]["tracedecay"].is_object()); + assert_eq!(counters.warnings, 0); } /// A standing refusal such as an ownership conflict must reach the operator @@ -3102,7 +1900,6 @@ mod tests { let mut registration = CatalogHostComponentRegistrationAuthority::new( "kiro", home.path(), - lifecycle.path(), request.lifecycle.operation, ) .unwrap(); @@ -3237,7 +2034,6 @@ mod tests { let _profile = pinned_host_profile(); let home = tempfile::tempdir().unwrap(); - let lifecycle = tempfile::tempdir().unwrap(); let component_set = canonical_host_component_set("opencode", None, 0) .unwrap() .unwrap(); @@ -3247,7 +2043,6 @@ mod tests { let mut registration = CatalogHostComponentRegistrationAuthority::new( "opencode", home.path(), - lifecycle.path(), request.lifecycle.operation, ) .unwrap(); @@ -3313,7 +2108,6 @@ mod tests { let drive = |declare_the_write: bool| { let _profile = pinned_host_profile(); let home = tempfile::tempdir().unwrap(); - let lifecycle = tempfile::tempdir().unwrap(); let registration_path = home.path().join(".config/opencode/opencode.json"); // Create the directory up front: a registration directory that is // absent at stage and present at apply is its own (unrelated) @@ -3325,7 +2119,6 @@ mod tests { let mut registration = CatalogHostComponentRegistrationAuthority::new( "opencode", home.path(), - lifecycle.path(), request.lifecycle.operation, ) .unwrap(); @@ -3380,7 +2173,6 @@ mod tests { let _profile = pinned_host_profile(); let home = tempfile::tempdir().unwrap(); - let lifecycle = tempfile::tempdir().unwrap(); let preserved = seed_opencode_non_context_state(home.path()); let component_set = canonical_host_component_set( "opencode", @@ -3395,7 +2187,6 @@ mod tests { let mut registration = CatalogHostComponentRegistrationAuthority::new( "opencode", home.path(), - lifecycle.path(), request.lifecycle.operation, ) .unwrap(); @@ -3433,7 +2224,6 @@ mod tests { let context = tracedecay_agent_hosts::agents::InstallContext { home: home.path().to_path_buf(), tracedecay_bin: "tracedecay".to_string(), - tool_permissions: Vec::new(), project_root: None, dashboard: true, }; @@ -3463,7 +2253,6 @@ mod tests { let _profile = pinned_host_profile(); let home = tempfile::tempdir().unwrap(); - let lifecycle = tempfile::tempdir().unwrap(); let config_path = home.path().join(".config/opencode/opencode.json"); std::fs::create_dir_all(config_path.parent().unwrap()).unwrap(); std::fs::write(&config_path, OPENCODE_CONTEXT_CONFIG).unwrap(); @@ -3480,7 +2269,6 @@ mod tests { let mut registration = CatalogHostComponentRegistrationAuthority::new( "opencode", home.path(), - lifecycle.path(), request.lifecycle.operation, ) .unwrap(); @@ -3510,7 +2298,6 @@ mod tests { let _profile = pinned_host_profile(); let home = tempfile::tempdir().unwrap(); - let lifecycle = tempfile::tempdir().unwrap(); let config_path = home.path().join(".config/opencode/opencode.json"); let prompt_path = home.path().join(".config/opencode/AGENTS.md"); for path in [&config_path, &prompt_path] { @@ -3533,7 +2320,6 @@ mod tests { let mut registration = CatalogHostComponentRegistrationAuthority::new( "opencode", home.path(), - lifecycle.path(), request.lifecycle.operation, ) .unwrap(); @@ -3555,9 +2341,12 @@ mod tests { assert_eq!(std::fs::read(&prompt_path).unwrap(), original_prompt); } + /// The canonical Codex set (Core + ContextMcp) is the whole rendered + /// bundle Codex's activation probe compares against its cache; Core alone + /// omits `.mcp.json` and can never verify as `Current`. #[cfg(unix)] #[test] - fn codex_core_rollback_restores_generated_agent_exports_byte_for_byte() { + fn codex_canonical_rollback_restores_generated_agent_exports_byte_for_byte() { let _profile = pinned_host_profile(); // Core `apply` drives Codex's own `codex plugin add`, which is a hard // requirement of that path. Supply the host CLI rather than depending @@ -3592,29 +2381,10 @@ mod tests { std::fs::write(&user_path, user_bytes).unwrap(); std::fs::write(&manifest_path, manifest_bytes.as_bytes()).unwrap(); - // Core-component `apply` now drives Codex's own `codex plugin add`, - // which requires the plugin to already be registered in Codex's - // marketplace. Stage it exactly as the ordinary install path does. - let integration = tracedecay_agent_hosts::agents::get_integration("codex").unwrap(); - integration - .prepare_non_interactive_install(&tracedecay_agent_hosts::agents::InstallContext { - home: home.path().to_path_buf(), - tracedecay_bin: tracedecay_bin.clone(), - tool_permissions: tracedecay_agent_hosts::agents::expected_tool_perms() - .expect("tool catalog"), - project_root: None, - dashboard: true, - }) - .unwrap(); - - let component_set = canonical_host_component_set_with_tracedecay_bin( - "codex", - Some(crate::cli::HostBundleComponentArg::Core), - 0, - &tracedecay_bin, - ) - .unwrap() - .unwrap(); + let component_set = + canonical_host_component_set_with_tracedecay_bin("codex", None, 0, &tracedecay_bin) + .unwrap() + .unwrap(); let request = component_set_request(&component_set, HostBundleCliOperation::Repair, true, false) .unwrap(); @@ -3622,7 +2392,6 @@ mod tests { inner: CatalogHostComponentRegistrationAuthority::new_with_tracedecay_bin( "codex", home.path(), - lifecycle.path(), request.lifecycle.operation, tracedecay_bin, ) @@ -3869,19 +2638,6 @@ mod tests { if existing.is_some() { assert_eq!(config["mcpServers"]["other"]["command"], "other"); } - // No lifecycle journal may be left behind by a converged apply. - assert!( - tracedecay_agent_hosts::agents::host_bundle::HostBundleWriterV1::open_with_lifecycle_root( - home.path(), - lifecycle.path(), - ) - .unwrap() - .pending_component_set_journal_operation( - component_set.component_set.host - ) - .unwrap() - .is_none() - ); } apply_canonical_component_set( @@ -3928,7 +2684,6 @@ mod tests { ] { let _profile = pinned_host_profile(); let home = tempfile::tempdir().unwrap(); - let lifecycle = tempfile::tempdir().unwrap(); let component_set = canonical_host_component_set(agent, None, 0) .unwrap() .unwrap_or_else(|| panic!("{agent} must ship a canonical component set")); @@ -3938,7 +2693,6 @@ mod tests { let registration = CatalogHostComponentRegistrationAuthority::new( agent, home.path(), - lifecycle.path(), request.lifecycle.operation, ) .unwrap(); @@ -3970,95 +2724,6 @@ mod tests { ); } - /// A rolled-back apply intentionally leaves its journal behind as an - /// explicit reconciliation boundary. The non-interactive refresh must - /// recover it before its next attempt instead of wedging until an operator - /// runs `tracedecay host-bundle recover` by hand. - #[cfg(all(unix, feature = "test-transport"))] - #[tokio::test] - async fn a_wedged_kiro_journal_is_recovered_by_the_next_apply() { - let _profile = pinned_host_profile(); - #[cfg(unix)] - let kiro_cli_dir = tempfile::tempdir().unwrap(); - #[cfg(unix)] - let kiro_cli_path = kiro_cli_dir.path().join("kiro-cli"); - #[cfg(unix)] - write_fake_kiro_cli(&kiro_cli_path); - #[cfg(unix)] - let _kiro_path = - tracedecay_runtime_core::config::HostProgramSearchPathGuard::set(kiro_cli_dir.path()); - let tracedecay_bin = std::env::current_exe() - .unwrap() - .to_string_lossy() - .replace('\\', "/"); - let home = tempfile::tempdir().unwrap(); - let lifecycle = tempfile::tempdir().unwrap(); - let mcp_path = home.path().join(".kiro/settings/mcp.json"); - std::fs::create_dir_all(mcp_path.parent().unwrap()).unwrap(); - std::fs::write( - &mcp_path, - br#"{"mcpServers":{"other":{"command":"other"}}}"#, - ) - .unwrap(); - let component_set = - canonical_host_component_set_with_tracedecay_bin("kiro", None, 0, &tracedecay_bin) - .unwrap() - .unwrap(); - let options = crate::cli::HostBundleCliOptions { - component: None, - dry_run: false, - yes: true, - adopt: false, - }; - let pending_journal = || { - tracedecay_agent_hosts::agents::host_bundle::HostBundleWriterV1::open_with_lifecycle_root( - home.path(), - lifecycle.path(), - ) - .unwrap() - .pending_component_set_journal_operation(component_set.component_set.host) - .unwrap() - }; - - // Fail the transaction after registration has already been applied, so - // it rolls back and leaves the journal exactly as the live defect did. - let failure = EnvVarGuard::set("TRACEDECAY_TEST_FAIL_HOST_REGISTRATION_VERIFY", "1"); - apply_canonical_component_set( - "kiro", - HostBundleCliOperation::Install, - &component_set, - &options, - home.path(), - lifecycle.path(), - &ComponentSetApplyContext::with_tracedecay_bin(&tracedecay_bin), - ) - .unwrap_err(); - drop(failure); - assert!( - pending_journal().is_some(), - "the failed apply must leave its reconciliation boundary behind" - ); - - apply_canonical_component_set( - "kiro", - HostBundleCliOperation::Install, - &component_set, - &options, - home.path(), - lifecycle.path(), - &ComponentSetApplyContext::with_tracedecay_bin(&tracedecay_bin), - ) - .expect("the next apply must recover the wedged journal before mutating"); - assert!(pending_journal().is_none()); - let config: serde_json::Value = - serde_json::from_slice(&std::fs::read(&mcp_path).unwrap()).unwrap(); - assert_eq!( - config["mcpServers"]["tracedecay"]["command"], - tracedecay_bin - ); - assert_eq!(config["mcpServers"]["other"]["command"], "other"); - } - #[test] fn opencode_core_refuses_a_competing_analyzer_without_mutation() { let _profile = pinned_host_profile(); @@ -4107,8 +2772,17 @@ mod tests { } } + /// Kimi activates only through its interactive `/plugins install`, which + /// consumes the staged source. The transaction therefore commits that + /// source under a receipt and reports the host action, while Kimi's own + /// registry stays byte-for-byte untouched. #[tokio::test] - async fn kimi_tracked_reinstall_refuses_before_staging_without_native_activation() { + async fn kimi_tracked_reinstall_commits_staged_source_and_defers_native_activation() { + use tracedecay_agent_hosts::agents::host_bundle::{ + HostComponentV1, HostKindV1, latest_host_component_receipt_at, + resolved_host_bundle_lifecycle_root, + }; + let _profile = pinned_host_profile(); let home = tempfile::tempdir().unwrap(); let code_home = home.path().join(".kimi-code"); @@ -4130,16 +2804,30 @@ mod tests { .await; let [(id, Err(error))] = results.as_slice() else { - panic!("tracked Kimi reinstall should return one typed refusal"); + panic!("tracked Kimi reinstall should return one typed deferral"); }; assert_eq!(id, "kimi"); - assert!(error.to_string().contains("/plugins install")); + let staged = home + .path() + .join(".tracedecay/host-bundle-stage/kimi/tracedecay"); + assert!( + error + .to_string() + .contains(&format!("/plugins install {}", staged.display())), + "{error}" + ); assert_eq!(std::fs::read(&installed_path).unwrap(), original); assert!(!code_home.join("plugins/managed/tracedecay").exists()); + assert!(staged.join(".kimi-plugin/plugin.json").is_file()); assert!( - home.path() - .join(".tracedecay/host-bundle-stage/kimi/tracedecay/.kimi-plugin/plugin.json") - .is_file() + latest_host_component_receipt_at( + &resolved_host_bundle_lifecycle_root().unwrap(), + HostKindV1::KimiCode, + HostComponentV1::Core, + ) + .unwrap() + .is_some(), + "the staged source is receipt-owned, not an out-of-band write" ); } @@ -4159,19 +2847,16 @@ mod tests { ); let tracedecay_bin = tracedecay_agent_hosts::agents::which_tracedecay() .unwrap_or_else(|| "tracedecay".to_string()); - let integration = tracedecay_agent_hosts::agents::get_integration("kimi").unwrap(); - let ctx = tracedecay_agent_hosts::agents::InstallContext { - home: home.path().to_path_buf(), - tracedecay_bin: tracedecay_bin.clone(), - tool_permissions: tracedecay_agent_hosts::agents::expected_tool_perms() - .expect("tool catalog"), - project_root: None, - dashboard: true, - }; - assert!(matches!( - integration.prepare_non_interactive_install(&ctx).unwrap(), - tracedecay_agent_hosts::agents::NonInteractiveInstallOutcome::DeferredUserAction(_) - )); + let deferred = reinstall_agent_integrations_with_persisted_dashboard_policies( + &["kimi".to_string()], + home.path(), + &tracedecay_bin, + ) + .await; + assert!( + matches!(deferred.as_slice(), [(id, Err(_))] if id == "kimi"), + "{deferred:?}" + ); let staged = home .path() .join(".tracedecay/host-bundle-stage/kimi/tracedecay") @@ -4221,7 +2906,7 @@ mod tests { #[cfg(unix)] #[tokio::test] - async fn codex_native_activated_retry_tracks_component_set() { + async fn codex_repair_activates_through_host_cli_and_converges_stale_cache() { use tracedecay_agent_hosts::agents::host_bundle::{ HostComponentV1, HostKindV1, latest_host_component_receipt_at, resolved_host_bundle_lifecycle_root, @@ -4241,40 +2926,16 @@ mod tests { tracedecay_runtime_core::config::USER_DATA_DIR_ENV, &data_dir, ); - // The reinstall path renders the canonical Codex component set with - // the PATH-resolved binary, and the host-native activation probe - // compares the staged source byte-for-byte against that rendering. - // Stage with the same identity or the probe reports a stale cache. let tracedecay_bin = tracedecay_agent_hosts::agents::which_tracedecay() .unwrap_or_else(|| "tracedecay".to_string()); - let integration = tracedecay_agent_hosts::agents::get_integration("codex").unwrap(); - let ctx = tracedecay_agent_hosts::agents::InstallContext { - home: home.path().to_path_buf(), - tracedecay_bin: tracedecay_bin.clone(), - tool_permissions: tracedecay_agent_hosts::agents::expected_tool_perms() - .expect("tool catalog"), - project_root: None, - dashboard: true, - }; - assert!(matches!( - integration.prepare_non_interactive_install(&ctx).unwrap(), - tracedecay_agent_hosts::agents::NonInteractiveInstallOutcome::Ready - )); - let config_path = home.path().join(".codex/config.toml"); - std::fs::create_dir_all(config_path.parent().unwrap()).unwrap(); - std::fs::write( - &config_path, - "[plugins.\"tracedecay@personal\"]\nenabled = true\n", - ) - .unwrap(); - let cache_root = home + let cache_manifest = home .path() .join(".codex/plugins/cache/personal/tracedecay") - .join(tracedecay_agent_hosts::PRODUCT_VERSION); - let cache_manifest = cache_root.join(".codex-plugin/plugin.json"); - std::fs::create_dir_all(&cache_root).unwrap(); - copy_test_bundle(&home.path().join(".codex/plugins/tracedecay"), &cache_root); + .join(tracedecay_agent_hosts::PRODUCT_VERSION) + .join(".codex-plugin/plugin.json"); + // A fresh home: the transaction deploys the source, registers the + // personal marketplace, and drives `codex plugin add` in one pass. let results = reinstall_agent_integrations_with_persisted_dashboard_policies( &["codex".to_string()], home.path(), @@ -4435,8 +3096,13 @@ mod tests { ); } + /// Kimi exposes plugin install only through its interactive `/plugins` + /// host API. Every lifecycle operation therefore commits the staged + /// source under a receipt and reports that remaining host action, while + /// Kimi's own registry and managed plugin root stay byte-for-byte + /// untouched. #[tokio::test] - async fn kimi_canonical_component_set_fails_before_direct_host_mutation() { + async fn kimi_canonical_component_set_defers_activation_without_touching_host_registry() { let _profile = pinned_host_profile(); let home = tempfile::tempdir().unwrap(); let lifecycle = tempfile::tempdir().unwrap(); @@ -4450,12 +3116,10 @@ mod tests { tracedecay_runtime_core::config::HostProgramSearchPathGuard::set(empty_path.path()); let installed_path = code_home.join("plugins/installed.json"); std::fs::create_dir_all(installed_path.parent().unwrap()).unwrap(); - std::fs::write( - &installed_path, + let original = br#"{"version":1,"plugins":[{"id":"foreign","enabled":true}],"unrelated":"keep"} -"#, - ) - .unwrap(); +"#; + std::fs::write(&installed_path, original).unwrap(); let component_set = canonical_host_component_set("kimi", None, 0) .unwrap() .unwrap(); @@ -4465,6 +3129,9 @@ mod tests { yes: true, adopt: false, }; + let staged = home + .path() + .join(".tracedecay/host-bundle-stage/kimi/tracedecay"); for operation in [ HostBundleCliOperation::Install, @@ -4482,39 +3149,36 @@ mod tests { ) .unwrap_err() .to_string(); - let expected = match operation { - HostBundleCliOperation::Install => "host capability is unsupported", - HostBundleCliOperation::Update | HostBundleCliOperation::Repair => "cache is stale", - HostBundleCliOperation::Uninstall => unreachable!("not exercised by this loop"), - }; - assert!(error.contains(expected), "{operation:?}: {error}"); + assert!( + error.contains(&format!("/plugins install {}", staged.display())), + "{operation:?}: {error}" + ); } - assert_eq!( - std::fs::read(&installed_path).unwrap(), - br#"{"version":1,"plugins":[{"id":"foreign","enabled":true}],"unrelated":"keep"} -"# - ); + assert_eq!(std::fs::read(&installed_path).unwrap(), original); assert!( !code_home.join("plugins/managed/tracedecay").exists(), - "preflight must fail before deploying managed plugin bytes" + "TraceDecay never writes Kimi's managed plugin root" ); for artifact in &component_set.component_set.components[0].manifest.artifacts { assert!( - !home.path().join(&artifact.relative_path).exists(), - "failed Kimi preflight must not create artifact {}", + home.path().join(&artifact.relative_path).is_file(), + "the receipt-owned staged source must hold {}", artifact.relative_path ); } } + /// Without Kimi's native activation the registration preflight succeeds + /// with a deferred host action instead of refusing: the transaction goes + /// on to commit the staged source, and only Kimi's own registry stays + /// unwritten. #[tokio::test] - async fn kimi_registration_preflight_creates_no_backup_for_unavailable_api() { + async fn kimi_registration_preflight_defers_activation_for_unavailable_api() { use tracedecay_agent_hosts::agents::host_bundle::HostComponentSetRegistrationV1; let _profile = pinned_host_profile(); let home = tempfile::tempdir().unwrap(); - let lifecycle = tempfile::tempdir().unwrap(); let empty_path = tempfile::tempdir().unwrap(); let code_home = home.path().join(".kimi-code"); let _kimi_home = EnvVarGuard::set( @@ -4538,23 +3202,26 @@ mod tests { let mut registration = CatalogHostComponentRegistrationAuthority::new( "kimi", home.path(), - lifecycle.path(), request.lifecycle.operation, ) .unwrap(); assert_eq!( registration.preflight(&component_set.component_set, &request), - Err( - tracedecay_agent_hosts::agents::host_bundle::HostBundleError::UnsupportedCapability - ) + Ok(()) ); - assert_eq!(std::fs::read(installed_path).unwrap(), original); + let remediation = registration + .deferred_activation() + .expect("an inactive Kimi plugin defers activation to the host"); assert!( - !lifecycle - .path() - .join(".tracedecay-host-registration-v1") - .exists() + remediation.contains(&format!( + "/plugins install {}", + home.path() + .join(".tracedecay/host-bundle-stage/kimi/tracedecay") + .display() + )), + "{remediation}" ); + assert_eq!(std::fs::read(installed_path).unwrap(), original); } /// Kiro's supported route is its MCP registration alone. Core carries the @@ -4591,7 +3258,6 @@ mod tests { let _profile = pinned_host_profile(); let home = tempfile::tempdir().unwrap(); - let lifecycle = tempfile::tempdir().unwrap(); let component_set = canonical_host_component_set("opencode", None, 0) .unwrap() .unwrap(); @@ -4601,7 +3267,6 @@ mod tests { let mut registration = CatalogHostComponentRegistrationAuthority::new( "opencode", home.path(), - lifecycle.path(), request.lifecycle.operation, ) .unwrap(); @@ -4652,11 +3317,10 @@ mod tests { ); std::fs::create_dir_all(home.path().join(".hermes/profiles/work")).unwrap(); - apply_default_canonical_component_set( + run_default_component_set( "hermes", HostBundleCliOperation::Install, home.path(), - true, false, ) .unwrap(); @@ -4675,13 +3339,7 @@ mod tests { .unwrap(); } - apply_default_canonical_component_set( - "hermes", - HostBundleCliOperation::Install, - home.path(), - true, - true, - ) - .expect("a receiptless generated Hermes plugin must be adoptable"); + run_default_component_set("hermes", HostBundleCliOperation::Install, home.path(), true) + .expect("a receiptless generated Hermes plugin must be adoptable"); } } diff --git a/crates/tracedecay-cli/src/agent_cmd/feedback_rollback.rs b/crates/tracedecay-cli/src/agent_cmd/feedback_rollback.rs index 8340ace58e..ba8c91ae76 100644 --- a/crates/tracedecay-cli/src/agent_cmd/feedback_rollback.rs +++ b/crates/tracedecay-cli/src/agent_cmd/feedback_rollback.rs @@ -21,7 +21,6 @@ use super::{host_bundle_error, host_kind_for_agent, load_host_lifecycle_user_con enum FeedbackRollbackCliStatus { Prepared, Applied, - Restored, } const FEEDBACK_ROLLBACK_STATE_SCHEMA_VERSION: u16 = 6; @@ -171,12 +170,6 @@ struct FeedbackPreviewStorage; impl tracedecay_agent_hosts::agents::host_bundle::HostBundleLifecycleStorageV1 for FeedbackPreviewStorage { - fn recover_lifecycle( - &mut self, - ) -> Result<(), tracedecay_agent_hosts::agents::host_bundle::HostBundleError> { - Ok(()) - } - fn execute_lifecycle< V: tracedecay_agent_hosts::agents::host_bundle::HostBundleVerificationAdapterV1, >( @@ -742,7 +735,6 @@ fn restore_feedback_registration( tracedecay_agent_hosts::agents::safe_write_bytes_file_with_metadata( path, contents, - None, file.metadata.as_ref(), )?; if file.metadata.is_none() @@ -1235,7 +1227,6 @@ fn feedback_rollback_apply( home: home.clone(), tracedecay_bin: tracedecay_agent_hosts::agents::which_tracedecay() .unwrap_or_else(|| "tracedecay".to_string()), - tool_permissions: tracedecay_agent_hosts::agents::expected_tool_perms()?, project_root: None, dashboard: state.dashboard_enabled, }; @@ -1378,9 +1369,7 @@ fn feedback_rollback_restore(state_path: &Path) -> tracedecay_domain::errors::Re Some(&state.registration_intent_root), )?; read_feedback_contents(&home, &state.previous_manifest)?; - state.status = FeedbackRollbackCliStatus::Restored; - persist_feedback_state(state_path, &lifecycle_root, &state)?; - return Ok(()); + return retire_feedback_state(state_path, &lifecycle_root, &state); } if !state.compensation_preserves_registration { validate_feedback_registration_restore( @@ -1522,9 +1511,7 @@ fn feedback_rollback_restore(state_path: &Path) -> tracedecay_domain::errors::Re )?; read_feedback_contents(&home, &state.previous_manifest)?; restore_feedback_artifact_permissions(&home, &state.artifact_permissions)?; - state.status = FeedbackRollbackCliStatus::Restored; - persist_feedback_state(state_path, &lifecycle_root, &state)?; - return Ok(()); + return retire_feedback_state(state_path, &lifecycle_root, &state); } }; let verifier = feedback_pair_verifier(&state.previous_manifest, &state.target_manifest)?; @@ -1590,7 +1577,6 @@ fn feedback_rollback_restore(state_path: &Path) -> tracedecay_domain::errors::Re home, tracedecay_bin: tracedecay_agent_hosts::agents::which_tracedecay() .unwrap_or_else(|| "tracedecay".to_string()), - tool_permissions: tracedecay_agent_hosts::agents::expected_tool_perms()?, project_root: None, dashboard: state.dashboard_enabled, }; @@ -1616,12 +1602,47 @@ fn feedback_rollback_restore(state_path: &Path) -> tracedecay_domain::errors::Re writer .publish_feedback_component_set_receipt(&state.previous_manifest, &restore.restore_receipt) .map_err(host_bundle_error)?; - state.status = FeedbackRollbackCliStatus::Restored; - persist_feedback_state(state_path, &lifecycle_root, &state)?; + retire_feedback_state(state_path, &lifecycle_root, &state)?; println!( - "\x1b[32m✔\x1b[0m {} feedback route restored; state {}", - state.agent_id, - state_path.display() + "\x1b[32m✔\x1b[0m {} feedback route restored", + state.agent_id ); Ok(()) } + +/// A completed restore leaves nothing behind: the state file holds copies of +/// the prior route and host registration bytes, which serve only an +/// unfinished restore. +fn retire_feedback_state( + state_path: &Path, + lifecycle_root: &Path, + state: &FeedbackRollbackCliState, +) -> tracedecay_domain::errors::Result<()> { + for path in [ + state_path.to_path_buf(), + feedback_doctor_state_path(lifecycle_root, &state.agent_id), + ] { + match fs::remove_file(&path) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + return Err(tracedecay_domain::errors::TraceDecayError::Config { + message: format!( + "could not remove feedback rollback state {}: {error}", + path.display() + ), + }); + } + } + } + match fs::remove_dir_all(&state.registration_intent_root) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(tracedecay_domain::errors::TraceDecayError::Config { + message: format!( + "could not remove feedback registration intents {}: {error}", + state.registration_intent_root.display() + ), + }), + } +} diff --git a/crates/tracedecay-cli/src/automation_cli/config.rs b/crates/tracedecay-cli/src/automation_cli/config.rs index de2d1dc115..142c666f75 100644 --- a/crates/tracedecay-cli/src/automation_cli/config.rs +++ b/crates/tracedecay-cli/src/automation_cli/config.rs @@ -1,4 +1,4 @@ -use crate::cli::{AutomationConfigAction, AutomationConfigScope}; +use crate::cli::AutomationConfigAction; use crate::resolve_cli_project_root; pub(crate) fn project_automation_reconcile_args() -> serde_json::Value { @@ -34,29 +34,17 @@ pub(super) async fn handle_automation_config_command( | AutomationConfigAction::Disable { path, .. } | AutomationConfigAction::Set { path, .. } => path.clone(), }; - let scope = match &action { - AutomationConfigAction::Get { scope, .. } - | AutomationConfigAction::Explain { scope, .. } - | AutomationConfigAction::Enable { scope, .. } - | AutomationConfigAction::Disable { scope, .. } - | AutomationConfigAction::Set { scope, .. } => *scope, - }; - if scope != AutomationConfigScope::Project { - return Err(config_error( - "automation settings are project-scoped in the V2 configuration control plane; use --scope project", - )); - } - let requested = resolve_cli_project_root(path, None, None).await?; let resolved = crate::commands::resolve_project_scope(requested).await?; let current = load_canonical_automation_config(&resolved.project_path).await?; + let codex = load_canonical_codex_executable(&resolved.project_path).await?; let patch = match action { AutomationConfigAction::Get { json, .. } => { - print_automation_config(¤t, json, false)?; + print_automation_config(¤t, &codex, json, false)?; return Ok(()); } AutomationConfigAction::Explain { json, .. } => { - print_automation_config(¤t, json, true)?; + print_automation_config(¤t, &codex, json, true)?; return Ok(()); } AutomationConfigAction::Enable { .. } => AutomationConfigPatch { @@ -138,7 +126,7 @@ pub(super) async fn handle_automation_config_command( }; let effective = apply_project_automation_patch(&resolved.project_path, patch).await?; - print_automation_config(&effective, true, false) + print_automation_config(&effective, &codex, true, false) } pub(crate) async fn load_canonical_automation_config( @@ -162,6 +150,27 @@ pub(crate) async fn load_canonical_automation_config( } } +/// The `codex` executable the project's `lcm.summarizer_executables.v1` +/// setting binds; the automation backend spawns only this path. +async fn load_canonical_codex_executable( + project_path: &std::path::Path, +) -> tracedecay_domain::errors::Result +{ + match crate::commands::current_project_setting( + project_path, + tracedecay_domain::configuration::LCM_SUMMARIZER_EXECUTABLES_SETTING_KEY, + ) + .await? + { + tracedecay_domain::configuration::ConfigurationValueV1::LcmSummarizerExecutables( + executables, + ) => Ok(executables.codex), + _ => Err(config_error( + "lcm summarizer executables setting has the wrong canonical value kind", + )), + } +} + pub(crate) async fn apply_project_automation_patch( project_path: &std::path::Path, patch: tracedecay_automation_runtime::automation::config::AutomationConfigPatch, @@ -257,11 +266,12 @@ fn parse_optional_u64( fn print_automation_config( effective: &tracedecay_automation_runtime::automation::config::AutomationConfig, + codex: &tracedecay_domain::configuration::LcmSummarizerExecutableV1, json: bool, explain: bool, ) -> tracedecay_domain::errors::Result<()> { let availability = - tracedecay_automation_runtime::automation::backend::backend_availability(effective); + tracedecay_automation_runtime::automation::backend::backend_availability(effective, codex); let trace_decay_backend_calls = effective.enabled && effective.backend == tracedecay_automation_runtime::automation::config::AutomationBackend::CodexAppServer @@ -317,7 +327,7 @@ fn parse_automation_backend( use tracedecay_automation_runtime::automation::config::AutomationBackend; match value { "disabled" => Ok(AutomationBackend::Disabled), - "codex-app-server" | "codex_app_server" => Ok(AutomationBackend::CodexAppServer), + "codex-app-server" => Ok(AutomationBackend::CodexAppServer), _ => Err(config_error(format!( "unknown automation backend '{value}' (expected disabled, codex-app-server)" ))), @@ -332,7 +342,7 @@ fn parse_automation_host_mode( use tracedecay_automation_runtime::automation::config::AutomationHostMode; match value { "standalone" => Ok(AutomationHostMode::Standalone), - "delegated-host" | "delegated_host" => Ok(AutomationHostMode::DelegatedHost), + "delegated-host" => Ok(AutomationHostMode::DelegatedHost), _ => Err(config_error(format!( "unknown automation host mode '{value}' (expected standalone, delegated-host)" ))), diff --git a/crates/tracedecay-cli/src/cli.rs b/crates/tracedecay-cli/src/cli.rs index 31d44b9e77..fa13b9fdba 100644 --- a/crates/tracedecay-cli/src/cli.rs +++ b/crates/tracedecay-cli/src/cli.rs @@ -11,8 +11,8 @@ mod package_hook; mod work; mod workflow; pub use automation::{ - AutomationAction, AutomationConfigAction, AutomationConfigScope, AutomationFactsAction, - AutomationRunsAction, AutomationSkillsAction, + AutomationAction, AutomationConfigAction, AutomationFactsAction, AutomationRunsAction, + AutomationSkillsAction, }; use help::*; pub use package_hook::{PackageHookAction, ScoopPackageHookAction}; @@ -114,45 +114,6 @@ pub enum FeedbackRollbackAction { }, } -#[derive(Clone, Debug, Subcommand)] -pub enum HostBundleAction { - /// List every host whose component-set lifecycle journal is awaiting recovery - Status, - /// Roll an interrupted host component transaction back to its pre-transaction state - /// - /// Recovery converges automatically when a second writer left the deployed - /// bytes equal to the pre-transaction backup or to the transaction's own - /// cataloged output. Genuinely foreign bytes stay fail-closed; pass - /// `--quarantine` to set the journal aside (backups are preserved) and - /// unblock the host. - Recover { - /// Recover only this agent's host journal (default: every pending host) - #[arg(long, value_parser = agent_value_parser())] - agent: Option, - /// Set aside a journal that convergent recovery cannot resolve - #[arg(long)] - quarantine: bool, - }, - /// Snapshot one installed component's managed artifact files. - /// - /// Requires the global `--yes` confirmation. The backup writer refuses an - /// unconfirmed receipt even though deployed files are not overwritten. - ArtifactBackup { - /// Agent whose selected component owns the managed artifacts - #[arg(long, value_parser = agent_value_parser())] - agent: String, - }, - /// Restore managed artifact files without changing host registration - ArtifactRestore { - /// Agent whose selected component owns the managed artifacts - #[arg(long, value_parser = agent_value_parser())] - agent: String, - /// Lowercase 32-character hexadecimal artifact-backup receipt id - #[arg(long)] - backup_id: String, - }, -} - /// Code intelligence for Rust codebases. #[derive(Parser)] #[command( @@ -162,24 +123,24 @@ pub enum HostBundleAction { version = crate::product_runtime::PRODUCT_BUILD_VERSION )] pub struct Cli { - /// Select one compiled first-party host component; without it, lifecycle commands apply - /// the host's canonical component set atomically + /// Select one compiled first-party host component; without it, lifecycle commands run + /// the same receipt-backed lifecycle over the host's whole canonical component set #[arg(long, global = true, value_enum)] pub component: Option, /// Verify and print the exact signed lifecycle plan without mutating. - /// Valid only alongside the agent-lifecycle commands; dispatch enforces the - /// `--component` pairing so this global flag never demands `--component` - /// from unrelated subcommands (e.g. `branch gc`, `storage report`). + /// Valid only alongside the agent-lifecycle commands; dispatch enforces + /// that scope so this global flag never leaks onto unrelated subcommands + /// (e.g. `branch gc`, `storage report`). #[arg(long, global = true, conflicts_with = "yes")] pub dry_run: bool, - /// Confirm a first-party component mutation, or a `wipe`. Scope is enforced + /// Confirm a host lifecycle mutation, or a `wipe`. Scope is enforced /// in dispatch, not by a global clap `requires`, so it does not leak onto /// other commands. #[arg(long, global = true)] pub yes: bool, /// Confirm taking ownership of an existing file that no - /// TraceDecay receipt records. Required alongside `--yes` for - /// `reinstall --component`; the previous bytes are always backed up first, + /// TraceDecay receipt records. Required alongside `--yes` for install, + /// update-plugin, or reinstall; the previous bytes are replaced and not kept, /// and a file another owner claims is refused regardless of this flag. #[arg(long, global = true)] pub adopt: bool, @@ -229,9 +190,6 @@ pub enum Commands { Sync { /// Project path (default: current directory) path: Option, - /// Compatibility flag that queues the same authoritative reconciliation - #[arg(short, long)] - force: bool, /// Folders to skip during indexing (can be repeated) #[arg(long = "skip-folder", num_args = 1..)] skip_folders: Vec, @@ -362,16 +320,12 @@ pub enum Commands { #[arg(long, value_parser = agent_value_parser(), requires = "local")] agent: Option, }, - /// Refresh generated plugin code/assets for detected installs without - /// touching agent config files. + /// Update every installed agent's component set to this binary /// - /// Rewrites only tracedecay-generated artifacts, the Hermes plugin - /// (.py files, schemas.json, dashboard page) for the user integration, - /// the Cursor plugin bundle, the Codex plugin bundle/cache, and the Kiro - /// managed agent, re-baking the current binary path and version. Config - /// files (Hermes config.yaml, mcp.json, settings, - /// prompt rules) are left byte-for-byte intact; use `tracedecay reinstall` - /// to refresh those. + /// Runs the receipt-backed update lifecycle over each tracked agent's + /// canonical component set (or the one `--component` names), re-baking + /// the current binary path and version into its artifacts and host + /// registration. #[command(name = "update-plugin", after_help = UPDATE_PLUGIN_AFTER_HELP)] UpdatePlugin { /// Update one project-local integration in the current directory @@ -405,16 +359,6 @@ pub enum Commands { #[command(subcommand)] action: FeedbackRollbackAction, }, - /// Inspect or recover an interrupted first-party host component transaction - #[command( - name = "host-bundle", - long_about = HOST_BUNDLE_LONG_ABOUT, - after_help = HOST_BUNDLE_AFTER_HELP - )] - HostBundle { - #[command(subcommand)] - action: HostBundleAction, - }, /// PreToolUse hook handler (called by Claude Code, not by users directly) #[command(name = "hook-pre-tool-use", hide = true)] HookPreToolUse, @@ -1000,9 +944,6 @@ pub enum AnalyticsAction { /// Skip the hook-JSONL import pass before summarizing #[arg(long)] no_sync: bool, - /// Keep compatibility with JSON-capable diagnostics commands. - #[arg(long)] - json: bool, }, /// Import hook_analytics.jsonl rows into the durable analytics_events table Sync, @@ -1125,10 +1066,10 @@ pub(crate) struct SessionsSearchArgs { #[arg(long, default_value_t = 10)] pub(crate) limit: usize, /// Inclusive minimum message timestamp. Accepts Unix seconds, RFC3339, YYYY-MM-DD, or relative time like "last hour" - #[arg(long, alias = "time-from", alias = "start-time")] + #[arg(long)] pub(crate) since: Option, /// Inclusive maximum message timestamp. Accepts Unix seconds, RFC3339, YYYY-MM-DD, or relative time like "last hour" - #[arg(long, alias = "time-to", alias = "end-time")] + #[arg(long)] pub(crate) until: Option, /// Registered project id whose session store should be searched #[arg(long)] @@ -1340,22 +1281,6 @@ pub enum ProfileStorageAction { #[arg(long)] restore: String, }, - /// Reset exactly one refused authority so the next open recreates it at - /// the canonical schema. Applies only to a store whose open failed with - /// the typed ResetRequired state naming that authority; healthy - /// authorities are refused and nothing else in the store is touched. - /// Requires the global `--yes` confirmation and an exclusive maintenance - /// lease (the daemon cannot open a refused store, so recovery runs - /// offline). - #[command(name = "reset-authority")] - ResetAuthority { - /// Authority named by the ResetRequired state ("observations"). - authority: String, - /// Sessions store carrying the refused authority (defaults to the - /// profile-scope user sessions store). - #[arg(long = "db")] - db: Option, - }, /// Reset a project graph store whose open failed with the typed /// ResetRequired state (an incompatible schema this binary cannot upgrade /// in place). Only the refused graph database is deleted; the store diff --git a/crates/tracedecay-cli/src/cli/automation.rs b/crates/tracedecay-cli/src/cli/automation.rs index 4214b56ae5..dd3411f45c 100644 --- a/crates/tracedecay-cli/src/cli/automation.rs +++ b/crates/tracedecay-cli/src/cli/automation.rs @@ -1,4 +1,4 @@ -use clap::{Subcommand, ValueEnum}; +use clap::Subcommand; #[allow(clippy::large_enum_variant)] #[derive(Subcommand)] @@ -25,20 +25,11 @@ pub enum AutomationAction { }, } -#[derive(Clone, Copy, Debug, PartialEq, Eq, ValueEnum)] -pub enum AutomationConfigScope { - Project, - Global, -} - #[allow(clippy::large_enum_variant)] #[derive(Subcommand)] pub enum AutomationConfigAction { /// Print effective automation config. Get { - /// Config scope to inspect. - #[arg(long, value_enum, default_value_t = AutomationConfigScope::Project)] - scope: AutomationConfigScope, /// Output as JSON. #[arg(long)] json: bool, @@ -48,9 +39,6 @@ pub enum AutomationConfigAction { }, /// Explain effective automation config, merge source, and backend availability. Explain { - /// Config scope to inspect. - #[arg(long, value_enum, default_value_t = AutomationConfigScope::Project)] - scope: AutomationConfigScope, /// Output as JSON. #[arg(long)] json: bool, @@ -60,27 +48,18 @@ pub enum AutomationConfigAction { }, /// Enable project automation. Enable { - /// Config scope to mutate. - #[arg(long, value_enum, default_value_t = AutomationConfigScope::Project)] - scope: AutomationConfigScope, /// Project path (default: current directory, with discovery). #[arg(short, long)] path: Option, }, /// Disable project automation. Disable { - /// Config scope to mutate. - #[arg(long, value_enum, default_value_t = AutomationConfigScope::Project)] - scope: AutomationConfigScope, /// Project path (default: current directory, with discovery). #[arg(short, long)] path: Option, }, /// Patch project automation config fields. Set { - /// Config scope to mutate. - #[arg(long, value_enum, default_value_t = AutomationConfigScope::Project)] - scope: AutomationConfigScope, /// Backend: disabled, codex-app-server. #[arg(long)] backend: Option, diff --git a/crates/tracedecay-cli/src/cli/help.rs b/crates/tracedecay-cli/src/cli/help.rs index 24c3dacf01..7ed4e2886b 100644 --- a/crates/tracedecay-cli/src/cli/help.rs +++ b/crates/tracedecay-cli/src/cli/help.rs @@ -56,14 +56,12 @@ pub(crate) const SYNC_LONG_ABOUT: &str = "\ Re-parses only files that changed since the last index and updates the code \ graph in place. Use after editing, switching branches, or pulling; agent \ hooks usually run it automatically. Incompatible derived lexical staging is \ -replaced automatically. The retained `--force` compatibility flag queues the \ -same authoritative reconciliation; it does not delete or fully rebuild the \ -project store. `--doctor`/`--verbose` explain what a sync actually did."; +replaced automatically. `--doctor`/`--verbose` explain what a sync actually \ +did."; pub(crate) const SYNC_AFTER_HELP: &str = "\ Examples: tracedecay sync Incremental refresh from cwd - tracedecay sync --force Compatible explicit refresh tracedecay sync --doctor List added/modified/removed files tracedecay sync --verbose Per-phase timings for slow syncs @@ -163,11 +161,11 @@ Examples: Related: tracedecay dashboard (uses these servers for diagnostics)."; pub(crate) const INSTALL_LONG_ABOUT: &str = "\ -Writes the MCP server registration, permissions, hooks, and prompt rules for \ -an agent host (Cursor, Codex, Claude Code, Hermes, Kiro, and others). \ -With --component, selects one compiled first-party Core or MCP companion \ -and uses the receipt-based host lifecycle instead of the compatibility installer. \ -Configures every detected agent when --agent is omitted, without prompting. \ +Installs an agent host's canonical first-party component set (MCP registration, \ +permissions, hooks, prompt rules) for Cursor, Codex, Claude Code, Hermes, Kiro, and \ +others through one receipt-backed host lifecycle. --component narrows that same \ +lifecycle to one named component; --dry-run prints the signed plan without mutating. \ +Configures every newly detected agent when --agent is omitted, without prompting. \ Safe to re-run; use it after installing a new agent or moving the tracedecay binary. \ Pass --git-hook to install the global post-commit sync hook; that flag is explicit \ because setting core.hooksPath can redirect every repository away from .git/hooks."; @@ -176,6 +174,7 @@ pub(crate) const INSTALL_AFTER_HELP: &str = "\ Examples: tracedecay install Configure every detected agent tracedecay install --agent cursor One agent only + tracedecay install --agent cursor --dry-run Preview the full component-set plan tracedecay install --git-hook Also install the post-commit hook tracedecay install --agent cursor --component core --dry-run tracedecay install --agent cursor --component core @@ -186,38 +185,40 @@ Examples: tracedecay install --agent hermes --profile dev tracedecay install --local Project-local config in cwd -Related: tracedecay uninstall, tracedecay reinstall (refresh settings), -tracedecay update-plugin (refresh generated assets only), tracedecay doctor."; +Related: tracedecay uninstall, tracedecay reinstall (repair installed agents), +tracedecay update-plugin (update installed agents), tracedecay doctor."; pub(crate) const REINSTALL_LONG_ABOUT: &str = "\ -Re-runs the installer for every agent that already has tracedecay configured, \ -rewriting MCP registrations, hooks, and prompt rules with current settings. \ -Use after upgrading the binary manually or when agent config drifted; it \ -never adds integration to agents that were not installed before."; +Repairs every agent that already has tracedecay configured by re-running its \ +component-set lifecycle, rewriting artifacts, MCP registrations, hooks, and prompt \ +rules with current settings. Use after upgrading the binary manually or when agent \ +config drifted; it never adds integration to agents that were not installed before. \ +--dry-run previews each tracked agent's repair plan without mutating."; pub(crate) const REINSTALL_AFTER_HELP: &str = "\ Examples: - tracedecay reinstall Refresh all installed agents + tracedecay reinstall Repair all installed agents + tracedecay reinstall --dry-run Preview every repair plan tracedecay reinstall --component core --dry-run tracedecay reinstall --component core Repair signed Core components Related: tracedecay install (add an agent), tracedecay update-plugin -(refresh generated plugin assets without touching config files)."; +(update installed agents to this binary)."; pub(crate) const UPDATE_PLUGIN_AFTER_HELP: &str = "\ Examples: - tracedecay update-plugin Refresh generated plugin assets + tracedecay update-plugin Update all installed agents tracedecay update-plugin --component context-mcp --dry-run tracedecay update-plugin --component context-mcp -Related: tracedecay reinstall (also rewrites agent config files), +Related: tracedecay reinstall (repair installed agents), tracedecay update (binary + plugins + daemon + health pass)."; pub(crate) const UNINSTALL_LONG_ABOUT: &str = "\ Removes tracedecay's MCP server registration, permissions, hooks, and prompt \ -rules from agent configuration. Removes every detected agent's integration \ -when --agent is omitted. Project indexes under .tracedecay/ are left intact. \ -use `tracedecay wipe` to delete data."; +rules from agent configuration through the same component-set lifecycle. Removes \ +every installed agent's integration when --agent is omitted. Project indexes under \ +.tracedecay/ are left intact. use `tracedecay wipe` to delete data."; pub(crate) const UNINSTALL_AFTER_HELP: &str = "\ Examples: @@ -244,34 +245,6 @@ Examples: Related: tracedecay doctor (surfaces restart-safe rollback state), tracedecay install / update-plugin (refresh Core feedback routes)."; -pub(crate) const HOST_BUNDLE_LONG_ABOUT: &str = "\ -Inspects and recovers interrupted first-party host component lifecycle \ -transactions, and snapshots or restores one component's managed artifact files. \ -Artifact backup/restore never captures or changes host registration and refuses \ -components whose lifecycle depends on registration state. Each host keeps its \ -own recovery journal; a host whose journal is pending refuses further mutation \ -until it is rolled back. Recovery converges automatically whenever deployed \ -bytes already equal the pre-transaction backup or this transaction's cataloged \ -output."; - -pub(crate) const HOST_BUNDLE_AFTER_HELP: &str = "\ -Examples: - tracedecay host-bundle status - tracedecay host-bundle recover --dry-run - tracedecay host-bundle recover --agent opencode --yes - tracedecay host-bundle recover --agent opencode --quarantine --yes - tracedecay host-bundle artifact-backup --agent opencode --component agent --yes - tracedecay host-bundle artifact-restore --agent opencode --component agent --backup-id <32-hex-id> --yes - -Quarantine moves the journal aside into the lifecycle control directory and -leaves every rollback backup on disk; nothing is deleted. - -Artifact restore changes only catalog-verified managed files. It fails closed -when native registration is part of the selected component lifecycle. - -Related: tracedecay doctor (surfaces the pending recovery boundary), -tracedecay reinstall (re-applies each host component set)."; - pub(crate) const DASHBOARD_LONG_ABOUT: &str = "\ Starts the local web dashboard: holographic memory curation, LCM session \ explorer, code-graph browser, analytics, and automation review UI. Binds to \ diff --git a/crates/tracedecay-cli/src/cli/parse_tests.rs b/crates/tracedecay-cli/src/cli/parse_tests.rs index a202b7724e..c5608e23ff 100644 --- a/crates/tracedecay-cli/src/cli/parse_tests.rs +++ b/crates/tracedecay-cli/src/cli/parse_tests.rs @@ -1,71 +1,6 @@ -use super::{ - AutomationAction, AutomationConfigAction, AutomationConfigScope, AutomationRunsAction, - AutomationSkillsAction, BranchAction, Cli, Commands, DaemonAction, FeedbackRollbackAction, - HostBundleAction, LspAction, MemoryAction, PackageHookAction, ProfileStorageAction, - RemoteAction, ScoopPackageHookAction, SessionsAction, SessionsRefreshAction, -}; +use super::{Cli, Commands, DaemonAction, RemoteAction}; use clap::{Parser, error::ErrorKind}; -fn strings(values: &[&str]) -> Vec { - values.iter().map(|value| value.to_string()).collect() -} - -#[test] -fn hidden_scoop_package_hook_contract_parses_both_operations() { - for operation in ["prepare", "restore"] { - let cli = Cli::try_parse_from([ - "tracedecay", - "package-hook", - "scoop", - operation, - "--package-id", - "tracedecay-beta", - "--state-file", - r"C:\state\scoop.json", - ]) - .expect("hidden Scoop package hook should parse"); - let Some(Commands::PackageHook { - action: - PackageHookAction::Scoop { - action: - ScoopPackageHookAction::Prepare { - package_id, - state_file, - } - | ScoopPackageHookAction::Restore { - package_id, - state_file, - }, - }, - }) = cli.command - else { - panic!("unexpected hidden Scoop package hook command"); - }; - assert_eq!(package_id, "tracedecay-beta"); - assert_eq!(state_file, std::path::Path::new(r"C:\state\scoop.json")); - } -} - -#[test] -fn first_class_git_hunks_carries_no_preview_binding_arguments() { - // The daemon captures exact repository state itself and mints the preview - // binding; the public CLI must not accept caller-supplied preview - // identities or snapshot digests. - let parsed = Cli::try_parse_from([ - "tracedecay", - "git", - "hunks", - "--preview-id", - "preview.manual", - ]); - - assert!( - parsed.is_err(), - "git hunks must reject caller-supplied preview bindings" - ); - assert!(Cli::try_parse_from(["tracedecay", "git", "hunks", "--scope", "staged"]).is_ok()); -} - #[test] fn tool_command_preserves_trailing_help_and_reserved_args() { let cli = Cli::try_parse_from([ @@ -127,67 +62,6 @@ fn workflow_command_binds_one_closed_typed_operation() { assert!(invocation.json); } -#[test] -fn removed_host_cli_aliases_are_invalid_subcommands() { - for alias in ["claude-install", "update-plugins", "claude-uninstall"] { - let error = match Cli::try_parse_from(["tracedecay", alias]) { - Ok(_) => panic!("removed host CLI alias must fail: {alias}"), - Err(error) => error, - }; - assert_eq!(error.kind(), ErrorKind::InvalidSubcommand, "alias: {alias}"); - } -} - -#[test] -fn removed_hermes_install_selectors_are_unknown_arguments() { - for args in [ - vec![ - "tracedecay", - "install", - "--agent", - "hermes", - "--profile", - "dev", - ], - vec![ - "tracedecay", - "install", - "--agent", - "hermes", - "--all-profiles", - ], - vec![ - "tracedecay", - "install", - "--agent", - "hermes", - "--project-root", - "/tmp/project", - ], - vec![ - "tracedecay", - "uninstall", - "--agent", - "hermes", - "--profile", - "dev", - ], - vec![ - "tracedecay", - "uninstall", - "--agent", - "hermes", - "--all-profiles", - ], - ] { - let error = match Cli::try_parse_from(args.clone()) { - Ok(_) => panic!("removed flag must fail: {args:?}"), - Err(error) => error, - }; - assert_eq!(error.kind(), ErrorKind::UnknownArgument, "args: {args:?}"); - } -} - #[test] fn project_local_lifecycle_commands_require_and_preserve_agent_scope() { let reinstall = @@ -199,426 +73,66 @@ fn project_local_lifecycle_commands_require_and_preserve_agent_scope() { agent: Some(ref agent) }) if agent == "opencode" )); - let update = - Cli::try_parse_from(["tracedecay", "update-plugin", "--local", "--agent", "kimi"]).unwrap(); - assert!(matches!( - update.command, - Some(Commands::UpdatePlugin { - local: true, - agent: Some(ref agent) - }) if agent == "kimi" - )); - let uninstall = - Cli::try_parse_from(["tracedecay", "uninstall", "--local", "--agent", "roo-code"]).unwrap(); - assert!(matches!( - uninstall.command, - Some(Commands::Uninstall { - local: true, - agent: Some(ref agent) - }) if agent == "roo-code" - )); - assert!(Cli::try_parse_from(["tracedecay", "reinstall", "--local"]).is_err()); -} - -#[test] -fn feedback_rollback_commands_parse_confirmation_and_state_paths() { - let dry_run = Cli::try_parse_from([ - "tracedecay", - "feedback-rollback", - "dry-run", - "--agent", - "kimi", - ]) - .unwrap(); - assert!(matches!( - dry_run.command, - Some(Commands::FeedbackRollback { - action: FeedbackRollbackAction::DryRun { ref agent } - }) if agent == "kimi" - )); - let apply = Cli::try_parse_from([ - "tracedecay", - "feedback-rollback", - "apply", - "--agent", - "opencode", - "--state", - ".tracedecay/feedback-opencode.json", - "--yes", - ]) - .unwrap(); - assert!(matches!( - apply.command, - Some(Commands::FeedbackRollback { - action: FeedbackRollbackAction::Apply { - ref agent, - ref state, - yes: true - } - }) if agent == "opencode" && state == ".tracedecay/feedback-opencode.json" - )); - assert!( - Cli::try_parse_from([ - "tracedecay", - "feedback-rollback", - "restore", - "--state", - "state.json" - ]) - .is_ok(), - "confirmation is enforced by the handler so dry parsing remains inspectable" - ); -} - -#[test] -fn host_bundle_recovery_commands_parse_agent_scope_and_quarantine() { - let status = Cli::try_parse_from(["tracedecay", "host-bundle", "status"]).unwrap(); - assert!(matches!( - status.command, - Some(Commands::HostBundle { - action: HostBundleAction::Status - }) - )); - let recover = Cli::try_parse_from([ - "tracedecay", - "host-bundle", - "recover", - "--agent", - "opencode", - "--quarantine", - "--yes", - ]) - .unwrap(); - assert!(recover.yes); - assert!(matches!( - recover.command, - Some(Commands::HostBundle { - action: HostBundleAction::Recover { - agent: Some(ref agent), - quarantine: true, - } - }) if agent == "opencode" - )); - let all_hosts = Cli::try_parse_from(["tracedecay", "host-bundle", "recover", "--dry-run"]) - .expect("--dry-run needs no --component on the recovery verb"); - assert!(all_hosts.dry_run); - assert!(matches!( - all_hosts.command, - Some(Commands::HostBundle { - action: HostBundleAction::Recover { - agent: None, - quarantine: false, - } - }) - )); -} - -#[test] -fn host_bundle_artifact_commands_parse_explicit_scope_and_confirmation() { - let backup = Cli::try_parse_from([ - "tracedecay", - "host-bundle", - "artifact-backup", - "--agent", - "opencode", - "--component", - "agent", - "--yes", - ]) - .expect("artifact backup is an explicit host-component command"); - assert!(backup.yes); - assert_eq!(backup.component, Some(super::HostBundleComponentArg::Agent)); - assert!(matches!( - backup.command, - Some(Commands::HostBundle { - action: HostBundleAction::ArtifactBackup { ref agent } - }) if agent == "opencode" - )); - - let restore = Cli::try_parse_from([ - "tracedecay", - "host-bundle", - "artifact-restore", - "--agent", - "opencode", - "--component", - "agent", - "--backup-id", - "01010101010101010101010101010101", - "--yes", - ]) - .expect("artifact restore names its durable backup receipt"); - assert!(restore.yes); - assert_eq!( - restore.component, - Some(super::HostBundleComponentArg::Agent) - ); - assert!(matches!( - restore.command, - Some(Commands::HostBundle { - action: HostBundleAction::ArtifactRestore { - ref agent, - ref backup_id, - } - }) if agent == "opencode" && backup_id == "01010101010101010101010101010101" - )); -} - -#[test] -fn upgrade_update_and_post_update_parse_no_reinstall_flag() { - let upgrade = Cli::try_parse_from(["tracedecay", "upgrade", "--no-reinstall"]) - .expect("upgrade --no-reinstall should parse"); - let update = Cli::try_parse_from(["tracedecay", "update", "--no-reinstall"]) - .expect("update --no-reinstall should parse"); - let post_update = Cli::try_parse_from(["tracedecay", "post-update", "--no-reinstall"]) - .expect("post-update --no-reinstall should parse"); - - assert!(matches!( - upgrade.command, - Some(Commands::Upgrade { no_reinstall: true }) - )); - assert!(matches!( - update.command, - Some(Commands::Update { no_reinstall: true }) - )); - assert!(matches!( - post_update.command, - Some(Commands::PostUpdate { - no_reinstall: true, - lifecycle_lease_token: None, - }) - )); -} - -#[test] -fn lsp_bridge_accepts_explicit_project_or_initialize_root() { - let cli = Cli::try_parse_from([ - "tracedecay", - "lsp", - "bridge", - "--stdio", - "--project", - "/workspace/project", - ]) - .expect("lsp bridge should parse"); - - assert!(matches!( - cli.command, - Some(Commands::Lsp { - action: LspAction::Bridge { - stdio: true, - project, - } - }) if project.as_deref() == Some("/workspace/project") - )); - let initialize_routed = Cli::try_parse_from(["tracedecay", "lsp", "bridge", "--stdio"]) - .expect("initialize-routed LSP bridge should parse"); - assert!(matches!( - initialize_routed.command, - Some(Commands::Lsp { - action: LspAction::Bridge { - stdio: true, - project: None, - } - }) - )); + let error = match Cli::try_parse_from(["tracedecay", "reinstall", "--local"]) { + Ok(_) => panic!("--local without --agent must fail admission"), + Err(error) => error, + }; + assert_eq!(error.kind(), ErrorKind::MissingRequiredArgument); } #[test] -fn daemon_install_service_command_parses_socket_and_no_start() { - let cli = Cli::try_parse_from([ - "tracedecay", - "daemon", - "install-service", - "--socket", - "/tmp/tracedecay.sock", - "--no-start", - ]) - .expect("daemon install-service should parse"); - - assert!(matches!( - cli.command, - Some(Commands::Daemon { - action: DaemonAction::InstallService { - socket, - no_start, - remote_listen: None, - remote_tls_cert: None, - remote_tls_key: None, - } - }) if socket.as_deref() == Some("/tmp/tracedecay.sock") && no_start - )); - - let remote = Cli::try_parse_from([ - "tracedecay", - "daemon", - "install-service", - "--remote-listen", - "192.0.2.10:7443", - "--remote-tls-cert", - "/run/tracedecay/remote.crt", - "--remote-tls-key", - "/run/tracedecay/remote.key", - ]) - .expect("managed Remote Brain TLS service should parse"); - assert!(matches!( - remote.command, - Some(Commands::Daemon { - action: DaemonAction::InstallService { - remote_listen: Some(listen), - remote_tls_cert: Some(certificate), - remote_tls_key: Some(private_key), - .. - } - }) if listen.to_string() == "192.0.2.10:7443" - && certificate == "/run/tracedecay/remote.crt" - && private_key == "/run/tracedecay/remote.key" - )); - - assert!( - Cli::try_parse_from([ +fn daemon_remote_tls_listener_requires_the_complete_triple() { + for action in ["run", "install-service"] { + let complete = Cli::try_parse_from([ "tracedecay", "daemon", - "install-service", + action, + "--remote-listen", + "192.0.2.10:7443", "--remote-tls-cert", "/run/tracedecay/remote.crt", + "--remote-tls-key", + "/run/tracedecay/remote.key", ]) - .is_err(), - "partial managed Remote Brain TLS configuration must fail admission" - ); -} - -#[test] -fn daemon_run_start_and_stop_commands_parse_lifecycle_options() { - let run = Cli::try_parse_from([ - "tracedecay", - "daemon", - "run", - "--profile-root", - r"C:\Users\trace\AppData\Local\TraceDecay", - ]) - .expect("daemon run profile root should parse"); - assert!(matches!( - run.command, - Some(Commands::Daemon { - action: DaemonAction::Run { - socket: None, - profile_root: Some(profile_root), - remote_listen: None, - remote_tls_cert: None, - remote_tls_key: None, - } - }) if profile_root == r"C:\Users\trace\AppData\Local\TraceDecay" - )); - - let remote = Cli::try_parse_from([ - "tracedecay", - "daemon", - "run", - "--remote-listen", - "192.0.2.10:7443", - "--remote-tls-cert", - "/run/tracedecay/remote.crt", - "--remote-tls-key", - "/run/tracedecay/remote.key", - ]) - .expect("complete Remote Brain TLS listener should parse"); - assert!(matches!( - remote.command, - Some(Commands::Daemon { - action: DaemonAction::Run { - remote_listen: Some(listen), - remote_tls_cert: Some(certificate), - remote_tls_key: Some(private_key), - .. - } - }) if listen.to_string() == "192.0.2.10:7443" - && certificate == "/run/tracedecay/remote.crt" - && private_key == "/run/tracedecay/remote.key" - )); + .unwrap_or_else(|error| panic!("complete daemon {action} TLS listener: {error}")); + let (listen, certificate, private_key) = match complete.command { + Some(Commands::Daemon { + action: + DaemonAction::Run { + remote_listen: Some(listen), + remote_tls_cert: Some(certificate), + remote_tls_key: Some(private_key), + .. + } + | DaemonAction::InstallService { + remote_listen: Some(listen), + remote_tls_cert: Some(certificate), + remote_tls_key: Some(private_key), + .. + }, + }) => (listen, certificate, private_key), + _ => panic!("daemon {action} did not bind the TLS listener"), + }; + assert_eq!(listen.to_string(), "192.0.2.10:7443"); + assert_eq!(certificate, "/run/tracedecay/remote.crt"); + assert_eq!(private_key, "/run/tracedecay/remote.key"); - assert!( - Cli::try_parse_from([ + let partial = match Cli::try_parse_from([ "tracedecay", "daemon", - "run", + action, "--remote-listen", "192.0.2.10:7443", - ]) - .is_err(), - "partial Remote Brain TLS configuration must fail during argument admission" - ); - - let start = - Cli::try_parse_from(["tracedecay", "daemon", "start"]).expect("daemon start should parse"); - assert!(matches!( - start.command, - Some(Commands::Daemon { - action: DaemonAction::Start - }) - )); - - let stop = - Cli::try_parse_from(["tracedecay", "daemon", "stop"]).expect("daemon stop should parse"); - assert!(matches!( - stop.command, - Some(Commands::Daemon { - action: DaemonAction::Stop - }) - )); -} - -#[test] -fn status_and_branch_add_commands_dispatch_to_expected_variants() { - let status = Cli::try_parse_from([ - "tracedecay", - "status", - "/tmp/project", - "--json", - "--short", - "--runtime", - ]) - .expect("status command should parse"); - assert!(matches!( - status.command, - Some(Commands::Status { - path, - project_id, - project_path, - json, - short, - runtime, - }) if path.as_deref() == Some("/tmp/project") - && project_id.is_none() - && project_path.is_none() - && json - && short - && runtime - )); - - let branch = Cli::try_parse_from([ - "tracedecay", - "branch", - "add", - "feature/dispatch-tests", - "--path", - "/tmp/project", - ]) - .expect("branch add should parse"); - assert!(matches!( - branch.command, - Some(Commands::Branch { - action: BranchAction::Add { name, path } - }) if name.as_deref() == Some("feature/dispatch-tests") - && path.as_deref() == Some("/tmp/project") - )); + ]) { + Ok(_) => panic!("partial daemon {action} TLS configuration must fail admission"), + Err(error) => error, + }; + assert_eq!(partial.kind(), ErrorKind::MissingRequiredArgument); + } } #[test] fn init_accepts_short_and_long_path_flag_like_dashboard_does() { - // `-p, --path` is documented (see TOP_LEVEL_AFTER_HELP) and already works - // on `dashboard`, `gitignore`, and `bench`; `init` previously only took - // PATH positionally and rejected `-p`/`--path` outright. let short = Cli::try_parse_from(["tracedecay", "init", "-p", "/tmp/project"]) .expect("init -p PATH should parse"); assert!(matches!( @@ -641,7 +155,6 @@ fn init_accepts_short_and_long_path_flag_like_dashboard_does() { }) if path_flag.as_deref() == Some("/tmp/project") )); - // The positional form keeps working unchanged. let positional = Cli::try_parse_from(["tracedecay", "init", "/tmp/project"]) .expect("init PATH should still parse positionally"); assert!(matches!( @@ -653,10 +166,8 @@ fn init_accepts_short_and_long_path_flag_like_dashboard_does() { }) if path.as_deref() == Some("/tmp/project") )); - // Supplying both the positional PATH and `-p`/`--path` is refused rather - // than silently picking one, clap's `conflicts_with` rejects it. // `Cli` does not derive `Debug`, so match directly instead of - // `.expect_err(...)` (which requires the `Ok` type to be `Debug`). + // `.expect_err(...)`. let conflict = match Cli::try_parse_from(["tracedecay", "init", "/tmp/project", "--path", "/tmp/other"]) { Ok(_) => panic!("init PATH and --path together should be rejected as a conflict"), @@ -666,7 +177,7 @@ fn init_accepts_short_and_long_path_flag_like_dashboard_does() { } #[test] -fn init_and_sync_parse_runtime_skip_and_include_folders() { +fn init_folder_flags_collect_multiple_values_until_the_next_flag() { let init = Cli::try_parse_from([ "tracedecay", "init", @@ -682,815 +193,18 @@ fn init_and_sync_parse_runtime_skip_and_include_folders() { init.command, Some(Commands::Init { path, - path_flag: None, - skip_folders, - include_folders, - adopt_project: None, - fresh: false, - }) if path.as_deref() == Some("/tmp/project") - && skip_folders == strings(&["vendor", "dist"]) - && include_folders == strings(&["dist/generated"]) - )); - - let sync = Cli::try_parse_from([ - "tracedecay", - "sync", - "/tmp/project", - "--force", - "--include-folder", - "dist", - "vendor/generated", - ]) - .expect("sync include folders should parse"); - assert!(matches!( - sync.command, - Some(Commands::Sync { - path, - force, skip_folders, include_folders, .. }) if path.as_deref() == Some("/tmp/project") - && force - && skip_folders.is_empty() - && include_folders == strings(&["dist", "vendor/generated"]) - )); -} - -#[test] -fn automation_config_commands_parse_project_sidecar_flags() { - let get = Cli::try_parse_from([ - "tracedecay", - "automation", - "config", - "get", - "--json", - "--path", - "/tmp/project", - ]) - .expect("automation config get should parse"); - assert!(matches!( - get.command, - Some(Commands::Automation { - action: - AutomationAction::Config { - action: - AutomationConfigAction::Get { - scope: AutomationConfigScope::Project, - json, - path - } - } - }) if json && path.as_deref() == Some("/tmp/project") - )); - - let explain = Cli::try_parse_from([ - "tracedecay", - "automation", - "config", - "explain", - "--json", - "--scope", - "global", - ]) - .expect("automation config explain should parse"); - assert!(matches!( - explain.command, - Some(Commands::Automation { - action: - AutomationAction::Config { - action: - AutomationConfigAction::Explain { - scope: AutomationConfigScope::Global, - json, - path - } - } - }) if json && path.is_none() - )); - - let enable = Cli::try_parse_from([ - "tracedecay", - "automation", - "config", - "enable", - "--scope", - "global", - ]) - .expect("automation config enable should parse"); - assert!(matches!( - enable.command, - Some(Commands::Automation { - action: - AutomationAction::Config { - action: - AutomationConfigAction::Enable { - scope: AutomationConfigScope::Global, - path - } - } - }) if path.is_none() - )); - - let disable = Cli::try_parse_from(["tracedecay", "automation", "config", "disable"]) - .expect("automation config disable should parse"); - assert!(matches!( - disable.command, - Some(Commands::Automation { - action: - AutomationAction::Config { - action: - AutomationConfigAction::Disable { - scope: AutomationConfigScope::Project, - path - } - } - }) if path.is_none() - )); - - let set = Cli::try_parse_from([ - "tracedecay", - "automation", - "config", - "set", - "--backend", - "codex-app-server", - "--host-mode", - "delegated-host", - "--timeout-secs", - "120", - "--scheduler-tick-secs", - "30", - "--memory-curator", - "true", - "--memory-curator-schedule", - "manual", - "--memory-curator-interval-secs", - "900", - "--memory-curator-cooldown-secs", - "300", - "--memory-curator-min-idle-secs", - "120", - "--memory-curator-stale-lock-secs", - "3600", - "--session-reflector", - "true", - "--session-reflector-schedule", - "interval", - "--session-reflector-interval-secs", - "1800", - "--session-reflector-cooldown-secs", - "600", - "--session-reflector-min-idle-secs", - "60", - "--session-reflector-stale-lock-secs", - "7200", - "--skill-writer", - "true", - "--skill-writer-schedule", - "manual", - "--skill-writer-interval-secs", - "", - "--skill-writer-cooldown-secs", - "none", - ]) - .expect("automation config set should parse"); - let Some(Commands::Automation { - action: - AutomationAction::Config { - action: - AutomationConfigAction::Set { - scope, - backend, - host_mode, - timeout_secs, - scheduler_tick_secs, - memory_curator, - memory_curator_schedule, - memory_curator_interval_secs, - memory_curator_cooldown_secs, - memory_curator_min_idle_secs, - memory_curator_stale_lock_secs, - session_reflector, - session_reflector_schedule, - session_reflector_interval_secs, - session_reflector_cooldown_secs, - session_reflector_min_idle_secs, - session_reflector_stale_lock_secs, - skill_writer, - skill_writer_schedule, - skill_writer_interval_secs, - skill_writer_cooldown_secs, - skill_writer_min_idle_secs, - skill_writer_stale_lock_secs, - path, - }, - }, - }) = set.command - else { - panic!("automation config set should parse into Set action"); - }; - assert_eq!(scope, AutomationConfigScope::Project); - assert_eq!(backend.as_deref(), Some("codex-app-server")); - assert_eq!(host_mode.as_deref(), Some("delegated-host")); - assert_eq!(timeout_secs, Some(120)); - assert_eq!(scheduler_tick_secs, Some(30)); - assert_eq!(memory_curator, Some(true)); - assert_eq!(memory_curator_schedule.as_deref(), Some("manual")); - assert_eq!(memory_curator_interval_secs.as_deref(), Some("900")); - assert_eq!(memory_curator_cooldown_secs.as_deref(), Some("300")); - assert_eq!(memory_curator_min_idle_secs.as_deref(), Some("120")); - assert_eq!(memory_curator_stale_lock_secs.as_deref(), Some("3600")); - assert_eq!(session_reflector, Some(true)); - assert_eq!(session_reflector_schedule.as_deref(), Some("interval")); - assert_eq!(session_reflector_interval_secs.as_deref(), Some("1800")); - assert_eq!(session_reflector_cooldown_secs.as_deref(), Some("600")); - assert_eq!(session_reflector_min_idle_secs.as_deref(), Some("60")); - assert_eq!(session_reflector_stale_lock_secs.as_deref(), Some("7200")); - assert_eq!(skill_writer, Some(true)); - assert_eq!(skill_writer_schedule.as_deref(), Some("manual")); - assert_eq!(skill_writer_interval_secs.as_deref(), Some("")); - assert_eq!(skill_writer_cooldown_secs.as_deref(), Some("none")); - assert!(skill_writer_min_idle_secs.is_none()); - assert!(skill_writer_stale_lock_secs.is_none()); - assert!(path.is_none()); -} - -#[test] -fn automation_runs_commands_parse_history_flags() { - let list = Cli::try_parse_from([ - "tracedecay", - "automation", - "runs", - "list", - "--limit", - "5", - "--json", - "--path", - "/tmp/project", - ]) - .expect("automation runs list should parse"); - - assert!(matches!( - list.command, - Some(Commands::Automation { - action: - AutomationAction::Runs { - action: - AutomationRunsAction::List { - limit, - json, - path, - } - } - }) if limit == 5 && json && path.as_deref() == Some("/tmp/project") - )); - - let view = Cli::try_parse_from([ - "tracedecay", - "automation", - "runs", - "view", - "run-123", - "--json", - "--path", - "/tmp/project", - ]) - .expect("automation runs view should parse"); - - assert!(matches!( - view.command, - Some(Commands::Automation { - action: - AutomationAction::Runs { - action: - AutomationRunsAction::View { run_id, json, path } - } - }) if run_id == "run-123" && json && path.as_deref() == Some("/tmp/project") - )); - - let artifact = Cli::try_parse_from([ - "tracedecay", - "automation", - "runs", - "artifact", - "run-123", - "codex_handoff", - "--json", - "--path", - "/tmp/project", - ]) - .expect("automation runs artifact should parse"); - - assert!(matches!( - artifact.command, - Some(Commands::Automation { - action: - AutomationAction::Runs { - action: - AutomationRunsAction::Artifact { - run_id, - kind, - json, - path - } - } - }) if run_id == "run-123" - && kind == "codex_handoff" - && json - && path.as_deref() == Some("/tmp/project") - )); -} - -#[test] -fn automation_skills_commands_parse_lifecycle_flags() { - let create = Cli::try_parse_from([ - "tracedecay", - "automation", - "skills", - "create", - "--id", - "repo-hygiene", - "--title", - "Repository hygiene", - "--summary", - "Keep checks focused", - "--routing-description", - "Use when selecting focused repository checks.", - "--category", - "maintenance", - "--body", - "Run focused tests.", - "--pinned", - ]) - .expect("automation skills create should parse"); - assert!(matches!( - create.command, - Some(Commands::Automation { - action: - AutomationAction::Skills { - action: - AutomationSkillsAction::Create { - id, - title, - summary, - routing_description, - category, - body, - pinned, - } - } - }) if id == "repo-hygiene" - && title == "Repository hygiene" - && summary == "Keep checks focused" - && routing_description == "Use when selecting focused repository checks." - && category == "maintenance" - && body == "Run focused tests." - && pinned - )); - - let update = Cli::try_parse_from([ - "tracedecay", - "automation", - "skills", - "update", - "repo-hygiene", - "--summary", - "Updated", - "--routing-description", - "Use when reviewing focused check selection.", - "--pinned", - "false", - ]) - .expect("automation skills update should parse"); - assert!(matches!( - update.command, - Some(Commands::Automation { - action: - AutomationAction::Skills { - action: - AutomationSkillsAction::Update { - id, - summary, - routing_description, - pinned, - .. - } - } - }) if id == "repo-hygiene" - && summary.as_deref() == Some("Updated") - && routing_description.as_deref() == Some("Use when reviewing focused check selection.") - && pinned == Some(false) - )); - - let approve = Cli::try_parse_from([ - "tracedecay", - "automation", - "skills", - "approve", - "repo-hygiene", - ]) - .err() - .expect("automation skills approve must stay removed"); - assert_eq!(approve.kind(), ErrorKind::InvalidSubcommand); - - let install = Cli::try_parse_from(["tracedecay", "automation", "skills", "install"]) - .err() - .expect("managed skills deploy automatically; manual install must stay removed"); - assert_eq!(install.kind(), ErrorKind::InvalidSubcommand); -} - -#[test] -fn project_selector_flags_parse_for_cli_read_surfaces() { - let status = - Cli::try_parse_from(["tracedecay", "status", "--project-id", "proj_123", "--json"]) - .expect("status project selector should parse"); - assert!(matches!( - status.command, - Some(Commands::Status { - path, - project_id, - project_path, - json, - .. - }) if path.is_none() - && project_id.as_deref() == Some("proj_123") - && project_path.is_none() - && json - )); - - let memory = Cli::try_parse_from([ - "tracedecay", - "memory", - "status", - "--project-path", - "/tmp/project", - ]) - .expect("memory status project selector should parse"); - assert!(matches!( - memory.command, - Some(Commands::Memory { - action: - MemoryAction::Status { - path, - project_id, - project_path, - .. - } - }) if path.is_none() - && project_id.is_none() - && project_path.as_deref() == Some("/tmp/project") - )); - - let sessions = Cli::try_parse_from([ - "tracedecay", - "sessions", - "search", - "needle", - "--project-id", - "proj_123", - ]) - .expect("sessions search project selector should parse"); - assert!(matches!( - sessions.command, - Some(Commands::Sessions { - action: SessionsAction::Search(args) - }) if args.project_id.as_deref() == Some("proj_123") && args.project_path.is_none() - )); -} - -#[test] -fn storage_subcommands_use_contextual_nouns_without_legacy_aliases() { - let report = Cli::try_parse_from([ - "tracedecay", - "storage", - "report", - "--profile-root", - "/tmp/profile", - "--project-id", - "proj_a", - "--project-root", - "/repos/a", - "--json", - ]) - .expect("storage report should parse"); - - assert!(matches!( - report.command, - Some(Commands::Storage { - action: ProfileStorageAction::StorageReport { - profile_root, - project_id, - project_root, - json, - } - }) if profile_root.as_deref() == Some("/tmp/profile") - && project_id.as_deref() == Some("proj_a") - && project_root.as_deref() == Some("/repos/a") - && json - )); - - let backup = Cli::try_parse_from([ - "tracedecay", - "storage", - "backup", - "--to", - "/tmp/backups", - "--backup-id", - "backup_2026_08_11", - ]) - .expect("storage backup should parse"); - - assert!(matches!( - backup.command, - Some(Commands::Storage { - action: ProfileStorageAction::BackupProfile { - to, - backup_id, - } - }) if to == "/tmp/backups" && backup_id == "backup_2026_08_11" - )); - - let rehearsal = Cli::try_parse_from([ - "tracedecay", - "storage", - "rehearse-backup", - "--backup", - "/tmp/backups/backup_2026_08_11", - "--restore", - "/tmp/restore", - ]) - .expect("storage rehearse-backup should parse"); - - assert!(matches!( - rehearsal.command, - Some(Commands::Storage { - action: ProfileStorageAction::RehearseProfileBackup { - backup, - restore, - } - }) if backup == "/tmp/backups/backup_2026_08_11" && restore == "/tmp/restore" - )); - - let reset = Cli::try_parse_from([ - "tracedecay", - "storage", - "reset-authority", - "observations", - "--db", - "/tmp/profile/user-sessions.db", - ]) - .expect("storage reset-authority should parse"); - - assert!(matches!( - reset.command, - Some(Commands::Storage { - action: ProfileStorageAction::ResetAuthority { authority, db } - }) if authority == "observations" - && db.as_deref() == Some("/tmp/profile/user-sessions.db") - )); - - for args in [ - vec![ - "tracedecay", - "storage", - "storage-report", - "--profile-root", - "/tmp/profile", - ], - vec![ - "tracedecay", - "storage", - "backup-profile", - "--to", - "/tmp/backups", - "--backup-id", - "backup_2026_08_11", - ], - vec![ - "tracedecay", - "storage", - "rehearse-profile-backup", - "--backup", - "/tmp/backups/backup_2026_08_11", - "--restore", - "/tmp/restore", - ], - ] { - let error = match Cli::try_parse_from(args.clone()) { - Ok(_) => panic!("legacy storage spelling must be rejected: {args:?}"), - Err(error) => error, - }; - assert_eq!(error.kind(), ErrorKind::InvalidSubcommand, "args: {args:?}"); - } -} - -#[test] -fn parses_sessions_import_and_search_commands() { - let import = Cli::try_parse_from(["tracedecay", "sessions", "import"]).unwrap(); - match import.command { - Some(Commands::Sessions { - action: - SessionsAction::Import { - project_id, - project_path, - }, - }) => { - assert!(project_id.is_none()); - assert!(project_path.is_none()); - } - _ => panic!("expected sessions import command"), - } - - let search = Cli::try_parse_from([ - "tracedecay", - "sessions", - "search", - "needle", - "--provider", - "codex", - "--limit", - "5", - ]) - .unwrap(); - match search.command { - Some(Commands::Sessions { - action: SessionsAction::Search(args), - }) => { - assert_eq!(args.query, "needle"); - assert_eq!(args.provider.as_deref(), Some("codex")); - assert_eq!(args.scope, "all"); - assert_eq!(args.message_type, "all"); - assert!(args.parent_session_id.is_none()); - assert_eq!(args.limit, 5); - assert!(args.project_id.is_none()); - assert!(args.project_path.is_none()); - assert!(args.since.is_none()); - assert!(args.until.is_none()); - assert!(args.branch.is_none()); - assert!(args.worktree.is_none()); - assert!(args.commit.is_none()); - } - _ => panic!("expected sessions search command"), - } - - let time_filtered_search = Cli::try_parse_from([ - "tracedecay", - "sessions", - "search", - "needle", - "--since", - "last hour", - "--until", - "2026-07-04T00:00:00Z", - ]) - .unwrap(); - match time_filtered_search.command { - Some(Commands::Sessions { - action: SessionsAction::Search(args), - }) => { - assert_eq!(args.since.as_deref(), Some("last hour")); - assert_eq!(args.until.as_deref(), Some("2026-07-04T00:00:00Z")); - } - _ => panic!("expected sessions search command"), - } - - let all_provider_search = - Cli::try_parse_from(["tracedecay", "sessions", "search", "needle"]).unwrap(); - match all_provider_search.command { - Some(Commands::Sessions { - action: SessionsAction::Search(args), - }) => { - assert_eq!(args.query, "needle"); - assert!(args.provider.is_none()); - assert_eq!(args.limit, 10); - } - _ => panic!("expected sessions search command"), - } - - let filtered_search = Cli::try_parse_from([ - "tracedecay", - "sessions", - "search", - "needle", - "--scope", - "subagents_only", - "--message-type", - "direct_user", - "--parent-session-id", - "parent-1", - ]) - .unwrap(); - assert!(matches!( - filtered_search.command, - Some(Commands::Sessions { - action: SessionsAction::Search(args) - }) if args.scope == "subagents_only" - && args.message_type == "direct_user" - && args.parent_session_id.as_deref() == Some("parent-1") + && skip_folders == ["vendor", "dist"] + && include_folders == ["dist/generated"] )); } -/// `--json` is not a sessions-search flag. Clap must reject it at parse -/// time; a hang after the usage error is a process-lifetime defect, not -/// this check. #[test] -fn sessions_search_rejects_json_at_parse() { - let error = match Cli::try_parse_from([ - "tracedecay", - "sessions", - "search", - "tracedecay", - "--limit", - "3", - "--json", - ]) { - Ok(_) => panic!("sessions search does not accept --json"), - Err(error) => error, - }; - assert_eq!(error.kind(), ErrorKind::UnknownArgument); -} - -#[test] -fn sessions_refresh_parses_exact_lifecycle_selectors() { - let begin = Cli::try_parse_from([ - "tracedecay", - "sessions", - "refresh", - "begin", - "--project-id", - "project.tracedecay", - "--session-id", - "session.refresh", - "--provider", - "cursor", - "--source", - "4", - "--target", - "9", - "--json", - ]) - .expect("project-scoped refresh begin should parse"); - assert!(matches!( - begin.command, - Some(Commands::Sessions { - action: - SessionsAction::Refresh { - action: SessionsRefreshAction::Begin(args) - } - }) if args.selectors.project_id.as_deref() == Some("project.tracedecay") - && args.selectors.project_path.is_none() - && !args.selectors.profile - && args.selectors.session_id == "session.refresh" - && args.selectors.provider == "cursor" - && args.selectors.source == 4 - && args.selectors.target == 9 - && args.json - )); - - let status = Cli::try_parse_from([ - "tracedecay", - "sessions", - "refresh", - "status", - "--profile", - "--session-id", - "session.refresh", - "--provider", - "cursor", - "--source", - "4", - "--target", - "9", - "--handle", - "refresh.abc", - ]) - .expect("profile-scoped refresh status should parse"); - assert!(matches!( - status.command, - Some(Commands::Sessions { - action: - SessionsAction::Refresh { - action: SessionsRefreshAction::Status(args) - } - }) if args.selectors.project_id.is_none() - && args.selectors.project_path.is_none() - && args.selectors.profile - && args.selectors.session_id == "session.refresh" - && args.selectors.provider == "cursor" - && args.selectors.source == 4 - && args.selectors.target == 9 - && args.handle == "refresh.abc" - && !args.json - )); - - let cancel = Cli::try_parse_from([ - "tracedecay", - "sessions", - "refresh", - "cancel", - "--project-path", - "/repo/tracedecay", +fn sessions_refresh_never_falls_back_to_the_current_directory() { + let selectors = [ "--session-id", "session.refresh", "--provider", @@ -1499,129 +213,30 @@ fn sessions_refresh_parses_exact_lifecycle_selectors() { "4", "--target", "9", - "--handle", - "refresh.abc", - "--json", - ]) - .expect("project-path refresh cancel should parse"); - assert!(matches!( - cancel.command, - Some(Commands::Sessions { - action: - SessionsAction::Refresh { - action: SessionsRefreshAction::Cancel(args) - } - }) if args.selectors.project_id.is_none() - && args.selectors.project_path.as_deref() == Some("/repo/tracedecay") - && !args.selectors.profile - && args.handle == "refresh.abc" - && args.json - )); -} - -#[test] -fn sessions_refresh_never_falls_back_to_the_current_directory() { - for args in [ - vec![ - "tracedecay", - "sessions", - "refresh", - "begin", - "--session-id", - "session.refresh", - "--provider", - "cursor", - "--source", - "4", - "--target", - "9", - ], - vec![ - "tracedecay", - "sessions", - "refresh", - "status", - "--session-id", - "session.refresh", - "--provider", - "cursor", - "--source", - "4", - "--target", - "9", - "--handle", - "refresh.abc", - ], - vec![ - "tracedecay", - "sessions", - "refresh", - "cancel", - "--session-id", - "session.refresh", - "--provider", - "cursor", - "--source", - "4", - "--target", - "9", - "--handle", - "refresh.abc", - ], + ]; + for (action, extra) in [ + ("begin", &[][..]), + ("status", &["--handle", "refresh.abc"][..]), + ("cancel", &["--handle", "refresh.abc"][..]), ] { - let error = match Cli::try_parse_from(args.clone()) { - Ok(_) => panic!("refresh must require a project or profile selector"), + let base: Vec<&str> = ["tracedecay", "sessions", "refresh", action] + .into_iter() + .chain(selectors) + .chain(extra.iter().copied()) + .collect(); + let error = match Cli::try_parse_from(base.clone()) { + Ok(_) => panic!("refresh {action} must require a project or profile selector"), Err(error) => error, }; - assert_eq!( - error.kind(), - ErrorKind::MissingRequiredArgument, - "args: {args:?}" - ); - } -} + assert_eq!(error.kind(), ErrorKind::MissingRequiredArgument); -#[test] -fn remote_protocol_actions_require_endpoint_credential_and_request_file() { - for action in [ - "enroll", - "capture", - "query", - "transfer-frame", - "replay", - "backup", - "restore", - "failover", - ] { - let error = match Cli::try_parse_from(["tracedecay", "remote", action]) { - Ok(_) => panic!("{action} must require authority flags"), - Err(error) => error, - }; - assert_eq!( - error.kind(), - ErrorKind::MissingRequiredArgument, - "{action} must require endpoint, credential-file, and request-file" + let mut with_project = base; + with_project.extend(["--project-path", "/repo/tracedecay"]); + assert!( + Cli::try_parse_from(with_project).is_ok(), + "refresh {action} with an explicit project selector must parse" ); } - - let enroll_without_enrollment_credential = match Cli::try_parse_from([ - "tracedecay", - "remote", - "enroll", - "--endpoint", - "https://brain.example/remote/", - "--credential-file", - "grant.bin", - "--request-file", - "enroll.json", - ]) { - Ok(_) => panic!("enroll must require --enrollment-credential-file"), - Err(error) => error, - }; - assert_eq!( - enroll_without_enrollment_credential.kind(), - ErrorKind::MissingRequiredArgument - ); } #[test] @@ -1660,76 +275,3 @@ fn remote_replay_parses_request_file_and_optional_trust_root() { assert_eq!(authority.request_file, std::path::Path::new("-")); assert!(authority.json); } - -#[test] -fn remote_capture_query_and_transfer_frame_parse_authority_flags() { - for (action, expected) in [ - ("capture", "capture"), - ("query", "query"), - ("transfer-frame", "transfer_frame"), - ] { - let cli = Cli::try_parse_from([ - "tracedecay", - "remote", - action, - "--endpoint", - "https://node.example/remote/", - "--credential-file", - "cred.bin", - "--request-file", - "request.json", - "--json", - ]) - .unwrap_or_else(|error| panic!("remote {action} should parse: {error}")); - - let Some(Commands::Remote { action: parsed }) = cli.command else { - panic!("unexpected remote {action} command"); - }; - let authority = match (&parsed, expected) { - (RemoteAction::Capture { authority }, "capture") - | (RemoteAction::Query { authority }, "query") - | (RemoteAction::TransferFrame { authority }, "transfer_frame") => authority, - _ => panic!("remote {action} parsed into the wrong action"), - }; - assert_eq!(authority.endpoint, "https://node.example/remote/"); - assert_eq!(authority.credential_file, std::path::Path::new("cred.bin")); - assert_eq!(authority.request_file, std::path::Path::new("request.json")); - assert!(authority.json); - } -} - -#[test] -fn remote_enroll_parses_both_credential_files() { - let cli = Cli::try_parse_from([ - "tracedecay", - "remote", - "enroll", - "--endpoint", - "https://brain.example/remote/", - "--credential-file", - "grant.bin", - "--enrollment-credential-file", - "enroll.bin", - "--request-file", - "enroll.json", - ]) - .expect("remote enroll should parse"); - - let Some(Commands::Remote { - action: - RemoteAction::Enroll { - authority, - enrollment_credential_file, - }, - }) = cli.command - else { - panic!("unexpected remote enroll command"); - }; - assert_eq!(authority.credential_file, std::path::Path::new("grant.bin")); - assert_eq!( - enrollment_credential_file, - std::path::Path::new("enroll.bin") - ); - assert_eq!(authority.request_file, std::path::Path::new("enroll.json")); - assert!(!authority.json); -} diff --git a/crates/tracedecay-cli/src/commands/branch.rs b/crates/tracedecay-cli/src/commands/branch.rs index 5217b579d5..4cb2bc9733 100644 --- a/crates/tracedecay-cli/src/commands/branch.rs +++ b/crates/tracedecay-cli/src/commands/branch.rs @@ -320,16 +320,12 @@ fn handle_branch_action_inner( for name in &report.removed_branches { eprintln!(" removed '{name}'"); } - for path in &report.removed_orphan_dbs { - eprintln!(" removed orphan '{}'", path.display()); - } eprintln!( - "\x1b[32m✔\x1b[0m Cleaned up {} stale branch(es) and {} orphan database(s).", + "\x1b[32m✔\x1b[0m Cleaned up {} stale branch(es).", report.removed_branches.len(), - report.removed_orphan_dbs.len() ); } else { - eprintln!("No stale branches or orphan databases to clean up."); + eprintln!("No stale branches to clean up."); } } BranchAction::Autotrack { action } => { @@ -543,7 +539,7 @@ async fn resolve_branch_data_root( project_path: &Path, ) -> tracedecay_domain::errors::Result { Ok( - tracedecay::project::TraceDecay::resolve_store_layout_for_identity(project_path) + tracedecay_project::project::TraceDecay::resolve_store_layout_for_identity(project_path) .await? .data_root, ) @@ -598,7 +594,6 @@ mod tests { let report = parse_daemon_branch_admin_report(&serde_json::json!({ "outcome": "removed", "removed_branches": ["feature/a"], - "removed_orphan_dbs": ["branches/orphan.db"], "default_branch": "main" })) .expect("valid branch admin response"); @@ -607,10 +602,6 @@ mod tests { tracedecay_runtime_core::branch::BranchAdminOutcome::Removed ); assert_eq!(report.removed_branches, vec!["feature/a"]); - assert_eq!( - report.removed_orphan_dbs, - vec![std::path::PathBuf::from("branches/orphan.db")] - ); assert_eq!(report.default_branch.as_deref(), Some("main")); } diff --git a/crates/tracedecay-cli/src/commands/daemon.rs b/crates/tracedecay-cli/src/commands/daemon.rs index bc97427486..3bb103ad6a 100644 --- a/crates/tracedecay-cli/src/commands/daemon.rs +++ b/crates/tracedecay-cli/src/commands/daemon.rs @@ -36,7 +36,9 @@ pub(crate) fn retained_tool_payload( ) -> tracedecay_domain::errors::Result { let payload = match retained_tool_outcome(tool_name, reply)? { ApplicationOutcome::Evidence(packet) => packet.payload, - ApplicationOutcome::Preview(_) | ApplicationOutcome::Effect(_) => { + ApplicationOutcome::Preview(_) + | ApplicationOutcome::Effect(_) + | ApplicationOutcome::Result(_) => { return Err(tracedecay_domain::errors::TraceDecayError::Config { message: format!("daemon tool {tool_name} returned a non-evidence outcome"), }); @@ -55,7 +57,9 @@ pub(crate) fn retained_effect_payload( ) -> tracedecay_domain::errors::Result { let payload = match retained_tool_outcome(tool_name, reply)? { ApplicationOutcome::Effect(effect) => effect.payload, - ApplicationOutcome::Evidence(_) | ApplicationOutcome::Preview(_) => { + ApplicationOutcome::Evidence(_) + | ApplicationOutcome::Preview(_) + | ApplicationOutcome::Result(_) => { return Err(tracedecay_domain::errors::TraceDecayError::Config { message: format!("daemon tool {tool_name} returned a non-effect outcome"), }); diff --git a/crates/tracedecay-cli/src/commands/index.rs b/crates/tracedecay-cli/src/commands/index.rs index bb7d31483e..3b7043d208 100644 --- a/crates/tracedecay-cli/src/commands/index.rs +++ b/crates/tracedecay-cli/src/commands/index.rs @@ -1,6 +1,6 @@ use std::path::{Path, PathBuf}; -use tracedecay::project::TraceDecay; +use tracedecay_project::project::TraceDecay; use super::daemon::daemon_tool_json; @@ -110,7 +110,7 @@ fn moved_store_adoption_request( adopt_project: Option, fresh: bool, assume_yes: bool, -) -> tracedecay_domain::errors::Result { +) -> tracedecay_domain::errors::Result { if fresh && adopt_project.is_some() { return Err(tracedecay_domain::errors::TraceDecayError::Config { message: "--fresh mints a new project identity and contradicts --adopt-project; \ @@ -119,10 +119,12 @@ fn moved_store_adoption_request( }); } Ok(match (adopt_project, fresh, assume_yes) { - (Some(project_id), _, _) => tracedecay::project::MovedStoreAdoption::AdoptNamed(project_id), - (None, true, _) => tracedecay::project::MovedStoreAdoption::Never, - (None, false, true) => tracedecay::project::MovedStoreAdoption::AdoptUnique, - (None, false, false) => tracedecay::project::MovedStoreAdoption::OfferCandidates, + (Some(project_id), _, _) => { + tracedecay_project::project::MovedStoreAdoption::AdoptNamed(project_id) + } + (None, true, _) => tracedecay_project::project::MovedStoreAdoption::Never, + (None, false, true) => tracedecay_project::project::MovedStoreAdoption::AdoptUnique, + (None, false, false) => tracedecay_project::project::MovedStoreAdoption::OfferCandidates, }) } @@ -380,7 +382,7 @@ mod init_bootstrap_tests { client_instance_id: "commands-init-test".to_string(), tool_list_changed_capable: false, catalog_version: String::new(), - moved_store_adoption: tracedecay::project::MovedStoreAdoption::Never, + moved_store_adoption: tracedecay_project::project::MovedStoreAdoption::Never, } } @@ -432,6 +434,13 @@ mod init_bootstrap_tests { let profile = temp.path().join("profile"); std::fs::create_dir_all(&project).unwrap(); let socket = temp.path().join("daemon.sock"); + let authority = tracedecay_daemon_identity::authority::DaemonAuthority::acquire( + temp.path(), + &tracedecay_daemon_protocol::DaemonEndpoint::Unix(socket.clone()), + env!("CARGO_PKG_VERSION"), + ) + .expect("publish the fixture daemon's authority record"); + let auth_token = authority.auth_token().to_owned(); let listener = tokio::net::UnixListener::bind(&socket).unwrap(); let _socket_env = SocketEnvGuard::set(&socket); @@ -444,6 +453,13 @@ mod init_bootstrap_tests { let (stream, _addr) = listener.accept().await.unwrap(); let (reader, mut writer) = stream.into_split(); let mut lines = tokio::io::BufReader::new(reader).lines(); + let preface = lines.next_line().await.unwrap().unwrap(); + assert!( + tracedecay_daemon_protocol::DaemonAuthPreface::from_line(preface.trim()) + .expect("auth preface") + .authenticate(&auth_token), + "init must present the daemon token" + ); let _handshake_line = lines.next_line().await.unwrap().unwrap(); let request_line = lines.next_line().await.unwrap().unwrap(); let request: serde_json::Value = serde_json::from_str(&request_line).unwrap(); @@ -598,7 +614,6 @@ mod init_bootstrap_tests { #[hotpath::measure(label = "cli.sync.run", future = true)] pub(crate) async fn handle_sync( path: Option, - force: bool, skip_folders: Vec, include_folders: Vec, doctor: bool, @@ -617,7 +632,7 @@ pub(crate) async fn handle_sync( let result = tracedecay::daemon::call_default_tool( &handshake, "tracedecay_admin_sync", - serde_json::json!({"force": force}), + serde_json::json!({}), ) .await?; if verbose { diff --git a/crates/tracedecay-cli/src/commands/profile_storage.rs b/crates/tracedecay-cli/src/commands/profile_storage.rs index 6dc517ceb4..7695e8b6ae 100644 --- a/crates/tracedecay-cli/src/commands/profile_storage.rs +++ b/crates/tracedecay-cli/src/commands/profile_storage.rs @@ -22,9 +22,6 @@ pub(crate) async fn handle_profile_storage_action( ProfileStorageAction::RehearseProfileBackup { backup, restore } => { handle_rehearse_profile_backup(backup, restore) } - ProfileStorageAction::ResetAuthority { authority, db } => { - handle_reset_authority(authority, db, assume_yes) - } ProfileStorageAction::ResetProjectStore { project_root, project_id, @@ -139,7 +136,7 @@ fn project_store_graph_db_paths( data_root: &Path, ) -> tracedecay_domain::errors::Result> { let mut candidates = Vec::new(); - let root_db = data_root.join(tracedecay::config::db_filename(data_root)); + let root_db = data_root.join(tracedecay_project::config::db_filename(data_root)); if root_db.is_file() { candidates.push(root_db); } @@ -248,7 +245,7 @@ fn reset_refused_project_graph_store( message: format!( "no project graph store exists at {}; nothing to reset", data_root - .join(tracedecay::config::db_filename(&data_root)) + .join(tracedecay_project::config::db_filename(&data_root)) .display() ), }); @@ -302,97 +299,6 @@ fn reset_refused_project_graph_store( }) } -/// Scoped operator recovery for a store whose open failed with the typed -/// `ResetRequired` state. The daemon cannot open a refused store, so the -/// reset runs offline under the profile's exclusive maintenance lease; the -/// next daemon open recreates the authority at the canonical schema and its -/// content re-derives from the preserved transcripts. -fn handle_reset_authority( - authority: String, - db: Option, - assume_yes: bool, -) -> tracedecay_domain::errors::Result<()> { - if authority != tracedecay_global_db::observation::OBSERVATION_AUTHORITY { - return Err(tracedecay_domain::errors::TraceDecayError::Config { - message: format!( - "no scoped reset exists for authority '{authority}'; the only \ - scoped-resettable authority is '{}'", - tracedecay_global_db::observation::OBSERVATION_AUTHORITY - ), - }); - } - if !assume_yes { - return Err(tracedecay_domain::errors::TraceDecayError::Config { - message: format!( - "resetting the '{authority}' authority drops its refused tables and \ - clears their recoverable derivations; re-run with --yes to confirm" - ), - }); - } - let profile_root = tracedecay_runtime_core::storage::default_profile_root()?; - let lifecycle_lease = tracedecay_runtime_core::lifecycle_lease::acquire_exclusive_for_profile( - &profile_root, - "reset-authority", - )?; - let _database_scope = tracedecay_runtime_core::db::enter_maintenance_database_scope( - &lifecycle_lease, - &profile_root, - "reset-authority", - )?; - let db_path = match db { - Some(path) => PathBuf::from(path), - None => tracedecay_sessions::runtime::user_sessions_db_path(&profile_root), - }; - if !db_path.is_file() { - return Err(tracedecay_domain::errors::TraceDecayError::Config { - message: format!( - "no sessions store exists at {}; nothing to reset", - db_path.display() - ), - }); - } - let mut connection = rusqlite::Connection::open(&db_path).map_err(|error| { - tracedecay_domain::errors::TraceDecayError::Database { - operation: "open sessions store for authority reset".to_string(), - message: error.to_string(), - } - })?; - let report = - tracedecay_global_db::observation::reset_refused_observation_authority(&mut connection)?; - println!( - "reset the refused '{authority}' authority in {}", - db_path.display() - ); - for table in &report.reset_tables { - println!(" recreated {table} empty at the canonical schema"); - } - println!( - " cleared {} recoverable session_messages row(s)", - report.cleared_session_message_rows - ); - println!( - " cleared {} observation-derived session-temporal row(s)", - report.cleared_derived_temporal_rows - ); - println!( - " cleared {} observation-bound retrieval anchor and alias row(s)", - report.cleared_retrieval_anchor_rows - ); - println!( - " cleared {} native-source scheduling cursor row(s)", - report.cleared_native_source_cursor_rows - ); - println!( - " cleared {} observation-derived external-source receipt row(s)", - report.cleared_external_source_rows - ); - println!( - "the authority content re-derives from the preserved transcripts at the \ - next daemon open" - ); - Ok(()) -} - async fn brokered_storage_report( project_id: Option<&str>, project_root: Option<&Path>, @@ -708,7 +614,7 @@ mod reset_project_store_tests { ) -> PathBuf { let data_root = tracedecay_runtime_core::storage::profile_sharded_data_root(profile_root, project_id); - let db_path = data_root.join(tracedecay::config::db_filename(&data_root)); + let db_path = data_root.join(tracedecay_project::config::db_filename(&data_root)); write_graph_db_with_user_version(&db_path, version); db_path } @@ -833,9 +739,9 @@ mod reset_project_store_tests { "profile storage exact-shape fixture", ) .unwrap(); - let graph = tracedecay::project::TraceDecay::init_with_exclusive_maintenance( + let graph = tracedecay_project::project::TraceDecay::init_with_exclusive_maintenance( &project_root, - tracedecay::project::TraceDecayOpenOptions { + tracedecay_project::project::TraceDecayOpenOptions { profile_root: Some(profile_root.clone()), global_db_path: Some(profile_root.join("global.db")), }, @@ -861,7 +767,7 @@ mod reset_project_store_tests { ); std::fs::create_dir_all(&incompatible_root).unwrap(); let incompatible_db = - incompatible_root.join(tracedecay::config::db_filename(&incompatible_root)); + incompatible_root.join(tracedecay_project::config::db_filename(&incompatible_root)); let source = rusqlite::Connection::open_with_flags( &healthy_db, rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY, @@ -917,7 +823,7 @@ mod reset_project_store_tests { "proj_not_sqlite", ); std::fs::create_dir_all(&data_root).unwrap(); - let db_path = data_root.join(tracedecay::config::db_filename(&data_root)); + let db_path = data_root.join(tracedecay_project::config::db_filename(&data_root)); std::fs::write(&db_path, b"not a database").unwrap(); let error = diff --git a/crates/tracedecay-cli/src/commands/settings.rs b/crates/tracedecay-cli/src/commands/settings.rs index 9e7e4dce39..68b10af5ac 100644 --- a/crates/tracedecay-cli/src/commands/settings.rs +++ b/crates/tracedecay-cli/src/commands/settings.rs @@ -1,5 +1,6 @@ use std::path::Path; +use tracedecay_contracts::now_micros; use tracedecay_contracts::request_identity::{GlobalRequestSurface, mint_global_request_id}; use tracedecay_contracts::{ ApplicationEnvelope, ApplicationOutcome, CancellationSignal, ComponentConfigurationState, @@ -11,7 +12,7 @@ use tracedecay_contracts::{ ConfigurationWireRequestV1, }; use tracedecay_daemon_protocol::ApplicationSurfaceRequest; -use tracedecay_daemon_protocol::{RequestedOutputFormat, invocation_now_micros}; +use tracedecay_daemon_protocol::RequestedOutputFormat; use tracedecay_domain::configuration::{ ConfigurationIdempotencyKey, ConfigurationLayerIdV1, ConfigurationRevisionId, ConfigurationValueV1, SettingKey, USER_UPLOAD_ENABLED_SETTING_KEY, UserProfileId, @@ -99,7 +100,7 @@ async fn invoke_configuration_surface( ) -> tracedecay_domain::errors::Result> { let request_id = mint_global_request_id(GlobalRequestSurface::Cli) .map_err(|error| configuration_error(error.to_string()))?; - let observed_at = invocation_now_micros(); + let observed_at = now_micros(); let deadline = configuration_deadline(operation, observed_at)?; let cancellation = CancellationSignal::active(format!("cancellation.cli.{}", request_id.as_str())) @@ -119,7 +120,7 @@ async fn invoke_configuration_surface( .await .map_err(|error| configuration_error(error.to_string()))?; if let Some(delay) = crate::cli::dispatch::surface_retry_delay(&result) { - let now = invocation_now_micros(); + let now = now_micros(); let remaining_micros = deadline.expires_at.0.saturating_sub(now.0); let remaining_micros = u64::try_from(remaining_micros) .map_err(|_| configuration_error("configuration deadline elapsed"))?; diff --git a/crates/tracedecay-cli/src/commands/storage.rs b/crates/tracedecay-cli/src/commands/storage.rs index 6765741bbd..c129a33f81 100644 --- a/crates/tracedecay-cli/src/commands/storage.rs +++ b/crates/tracedecay-cli/src/commands/storage.rs @@ -497,7 +497,7 @@ async fn wipe_under_profile_offline( let project_paths = if all { Vec::new() } else { - global::gather_target_projects(false, home_tracedecay).await? + global::gather_target_projects(false).await? }; let mut targets = Vec::new(); for path in &project_paths { @@ -632,8 +632,8 @@ fn handle_list_inner( Box::pin(async move { use tracedecay_runtime_core::text::format_token_count; - let home_tracedecay = tracedecay::config::user_data_dir(); - let project_paths = global::gather_target_projects(all, &home_tracedecay).await?; + let home_tracedecay = tracedecay_project::config::user_data_dir(); + let project_paths = global::gather_target_projects(all).await?; if !all && project_paths.is_empty() { println!("No tracedecay projects found in current folder, parents, or children."); @@ -826,7 +826,7 @@ fn append_orphan_manifest_rows( .collect(); let report = tracedecay_global_db::registry_maintenance::inspect_profile_store_orphans( profile_root, - tracedecay::project::current_timestamp(), + tracedecay_runtime_core::tracedecay::current_timestamp(), ); for plan in report.plans { if plan.status diff --git a/crates/tracedecay-cli/src/cost_cmd.rs b/crates/tracedecay-cli/src/cost_cmd.rs index 9d540f8230..9f78b3612f 100644 --- a/crates/tracedecay-cli/src/cost_cmd.rs +++ b/crates/tracedecay-cli/src/cost_cmd.rs @@ -15,7 +15,7 @@ pub(crate) async fn handle_cost( export: Option, ) -> tracedecay_domain::errors::Result<()> { let cwd = std::env::current_dir()?; - let project_root = tracedecay::config::discover_project_root(&cwd); + let project_root = tracedecay_project::config::discover_project_root(&cwd); let payload = daemon_tool_json( project_root.as_deref(), "tracedecay_admin_cli", diff --git a/crates/tracedecay-cli/src/global.rs b/crates/tracedecay-cli/src/global.rs index f461d60415..b078a2e250 100644 --- a/crates/tracedecay-cli/src/global.rs +++ b/crates/tracedecay-cli/src/global.rs @@ -184,7 +184,6 @@ pub(crate) fn tracedecay_dir_size(dir: &Path) -> u64 { /// registry. pub(crate) async fn gather_target_projects( all: bool, - home_tracedecay: &Option, ) -> tracedecay_domain::errors::Result> { if all { let payload = daemon_tool_json( @@ -199,7 +198,7 @@ pub(crate) async fn gather_target_projects( .await?; registry_project_roots(&payload) } else { - Ok(gather_local_projects(home_tracedecay)) + Ok(gather_local_projects()) } } @@ -230,74 +229,46 @@ fn registry_project_roots( .collect() } -/// Returns project roots whose `.tracedecay` data dir lives in cwd, an -/// ancestor, or a descendant. -pub(crate) fn gather_local_projects( - home_tracedecay: &Option, -) -> Vec { +/// Returns initialized project roots at cwd, an ancestor, or a descendant. +pub(crate) fn gather_local_projects() -> Vec { let Ok(cwd) = std::env::current_dir() else { return Vec::new(); }; - gather_local_projects_from(&cwd, home_tracedecay) + gather_local_projects_from(&cwd) } /// Same as [`gather_local_projects`] but takes the starting directory explicitly. /// -/// Pure (apart from filesystem reads), easier to test than the cwd-driven wrapper. -pub(crate) fn gather_local_projects_from( - cwd: &Path, - home_tracedecay: &Option, -) -> Vec { - use std::collections::HashSet; - use std::path::PathBuf; - - // Canonicalize the home data dir once so symlinked HOME paths still - // get correctly skipped during the ancestor + descendant walks. A user - // whose `$HOME` is `/Users/x` but whose canonical home is - // `/private/var/...` would otherwise leak the global DB into the wipe set. - let canon_home_ts: Option = - home_tracedecay.as_ref().and_then(|p| p.canonicalize().ok()); - - let mut out: Vec = Vec::new(); - let mut seen: HashSet = HashSet::new(); - - let is_home_tracedecay = |ts: &Path| -> bool { - if let Some(ref canon) = canon_home_ts - && ts.canonicalize().ok().as_ref() == Some(canon) +/// Ancestors count when they host a profile-sharded store or, at a worktree +/// root, the repository identity marker; ambient roots (filesystem root, the +/// user's home) never do. Descendants count by repository identity marker. +pub(crate) fn gather_local_projects_from(cwd: &Path) -> Vec { + let mut out = Vec::new(); + let mut seen = std::collections::HashSet::new(); + + for dir in cwd.ancestors() { + if tracedecay_runtime_core::config::is_initialized_project_root(dir) + && !tracedecay_runtime_core::config::is_ambient_project_root(dir) + && seen.insert(dir.to_path_buf()) { - return true; - } - false - }; - - let is_project_dir = |project_root: &Path, ts: &Path| -> bool { - !is_home_tracedecay(ts) && local_project_marker_exists(project_root, ts) - }; - - let mut cursor: Option<&Path> = Some(cwd); - while let Some(dir) = cursor { - let ts = dir.join(tracedecay::config::TRACEDECAY_DIR); - if is_project_dir(dir, &ts) && seen.insert(dir.to_path_buf()) { out.push(dir.to_path_buf()); } - cursor = dir.parent(); } - find_descendant_tracedecay(cwd, &canon_home_ts, &mut seen, &mut out); + find_descendant_tracedecay(cwd, &mut seen, &mut out); out } -/// Iteratively walks `start` looking for `.tracedecay/tracedecay.db` project -/// data dirs. +/// Iteratively walks `start` looking for repository roots carrying the +/// repository identity marker. /// -/// Skips common heavy directories (node_modules, target, .git, etc.) and never -/// descends into a data dir once found. Tracks canonicalized directories -/// to break symlink/junction cycles, and uses an explicit worklist instead of +/// Skips common heavy directories (node_modules, target, .git, etc.) and +/// `.tracedecay` data dirs. Tracks canonicalized directories to break +/// symlink/junction cycles, and uses an explicit worklist instead of /// recursion so deep trees can't overflow the stack. pub(crate) fn find_descendant_tracedecay( start: &Path, - canon_home_ts: &Option, seen: &mut std::collections::HashSet, out: &mut Vec, ) { @@ -330,22 +301,12 @@ pub(crate) fn find_descendant_tracedecay( let path = entry.path(); let name = entry.file_name(); let name_str = name.to_string_lossy(); - if name_str == tracedecay::config::TRACEDECAY_DIR { - // Only canonicalize when the entry could match the home skip; - // doing it for every dir entry would mean one syscall per - // entry on tree walks of arbitrary size. - if let Some(canon) = canon_home_ts - && path.canonicalize().ok().as_ref() == Some(canon) - { - continue; - } - if let Some(parent) = path.parent() - && local_project_marker_exists(parent, &path) - { - let pb = parent.to_path_buf(); - if seen.insert(pb.clone()) { - out.push(pb); - } + if name_str == tracedecay_project::config::TRACEDECAY_DIR { + continue; + } + if name_str == ".git" { + if repository_identity_root_exists(&dir) && seen.insert(dir.clone()) { + out.push(dir.clone()); } continue; } @@ -353,7 +314,6 @@ pub(crate) fn find_descendant_tracedecay( name_str.as_ref(), "node_modules" | "target" - | ".git" | "vendor" | "dist" | "build" @@ -368,26 +328,11 @@ pub(crate) fn find_descendant_tracedecay( } } -fn local_project_marker_exists(project_root: &Path, data_dir: &Path) -> bool { - if !data_dir.is_dir() { - return false; - } - if data_dir - .join(tracedecay::config::db_filename(data_dir)) - .exists() - { - return true; - } - // Wipe cleanup still recognizes retired repo-local enrollment markers so - // legacy `.tracedecay/` debris is removed alongside the profile store. - data_dir.file_name().is_some_and(|name| { - name == tracedecay::config::TRACEDECAY_DIR - && matches!( - tracedecay_runtime_core::storage::read_legacy_enrollment_marker(project_root), - Ok(Some(marker)) - if marker.storage_mode == tracedecay_runtime_core::storage::StorageMode::ProfileSharded - ) - }) +/// A repository checkout root whose `.git/` carries the repository identity +/// marker, the only identity a current install writes. +fn repository_identity_root_exists(dir: &Path) -> bool { + dir.join(".git").exists() + && tracedecay_runtime_core::storage::has_repository_identity_marker(dir) } /// Prints the big flashing warning shown before a wipe. @@ -459,52 +404,70 @@ mod gather_tests { use std::fs; #[cfg(unix)] use std::os::unix::fs::symlink; - use std::path::PathBuf; - /// Plant a `.tracedecay/tracedecay.db` marker so `is_project_dir` returns true. - fn make_project(root: &Path) { - let ts = root.join(".tracedecay"); - fs::create_dir_all(&ts).unwrap(); - fs::write(ts.join("tracedecay.db"), b"").unwrap(); + fn make_enrolled_project(root: &Path, project_id: &str) { + tracedecay_runtime_core::storage::pin_fixture_repository_identity(root, project_id) + .unwrap(); } - fn make_enrolled_project(root: &Path, project_id: &str) { - let ts = root.join(".tracedecay"); - fs::create_dir_all(&ts).unwrap(); - fs::write( - ts.join(tracedecay_runtime_core::storage::ENROLLMENT_FILENAME), - format!( - r#"{{ - "project_id": "{project_id}", - "storage_mode": "profile_sharded" -}}"# - ), + #[test] + fn finds_project_at_cwd() { + let _profile = tracedecay_runtime_core::config::PinnedUserDataDir::new(); + let dir = tempfile::tempdir().unwrap(); + let cwd = dir.path().canonicalize().unwrap(); + make_enrolled_project(&cwd, "proj_cwd"); + + let out = gather_local_projects_from(&cwd); + assert_eq!(out, vec![cwd]); + } + + #[test] + fn finds_profile_sharded_store_at_cwd() { + let _profile = tracedecay_runtime_core::config::PinnedUserDataDir::new(); + let dir = tempfile::tempdir().unwrap(); + let cwd = dir.path().canonicalize().unwrap(); + let store = tracedecay_runtime_core::storage::default_profile_sharded_layout( + &cwd, + &tracedecay_runtime_core::config::user_data_dir().unwrap(), ) .unwrap(); + fs::create_dir_all(&store.data_root).unwrap(); + fs::write(&store.graph_db_path, b"").unwrap(); + + assert_eq!(gather_local_projects_from(&cwd), vec![cwd]); } #[test] - fn finds_project_at_cwd() { + fn ignores_repo_local_graph_database_directories() { + let _profile = tracedecay_runtime_core::config::PinnedUserDataDir::new(); let dir = tempfile::tempdir().unwrap(); let cwd = dir.path().canonicalize().unwrap(); - make_project(&cwd); + let child = cwd.join("child"); + for root in [&cwd, &child] { + fs::create_dir_all(root.join(".tracedecay")).unwrap(); + fs::write(root.join(".tracedecay/tracedecay.db"), b"").unwrap(); + } - let out = gather_local_projects_from(&cwd, &None); - assert_eq!(out, vec![cwd]); + let out = gather_local_projects_from(&cwd); + assert!( + out.is_empty(), + "repo-local data dirs are not projects: {out:?}" + ); } #[test] fn finds_both_ancestor_and_descendant_dedup() { + let _profile = tracedecay_runtime_core::config::PinnedUserDataDir::new(); let dir = tempfile::tempdir().unwrap(); let root = dir.path().canonicalize().unwrap(); let cwd = root.join("mid"); fs::create_dir_all(&cwd).unwrap(); let child = cwd.join("child"); fs::create_dir_all(&child).unwrap(); - make_project(&root); - make_project(&child); + make_enrolled_project(&child, "proj_child"); + make_enrolled_project(&root, "proj_root"); - let out = gather_local_projects_from(&cwd, &None); + let out = gather_local_projects_from(&cwd); assert!(out.contains(&root)); assert!(out.contains(&child)); let unique: std::collections::HashSet<_> = out.iter().collect(); @@ -513,23 +476,44 @@ mod gather_tests { #[test] fn finds_profile_enrolled_projects_without_graph_db() { + let _profile = tracedecay_runtime_core::config::PinnedUserDataDir::new(); let dir = tempfile::tempdir().unwrap(); let root = dir.path().canonicalize().unwrap(); let cwd = root.join("mid"); let child = cwd.join("child"); + let unenrolled = cwd.join("unenrolled"); fs::create_dir_all(&child).unwrap(); - make_enrolled_project(&root, "proj_root"); + fs::create_dir_all(&unenrolled).unwrap(); + // Nested repositories first: pinning the outer root first would make + // the children resolve to its `.git/` instead of their own. make_enrolled_project(&child, "proj_child"); + make_enrolled_project(&root, "proj_root"); + let status = std::process::Command::new("git") + .args(["init", "--quiet"]) + .current_dir(&unenrolled) + .status() + .unwrap(); + assert!(status.success()); + fs::create_dir_all(unenrolled.join(".tracedecay")).unwrap(); + fs::write( + unenrolled.join(".tracedecay/enrollment.json"), + r#"{"project_id":"proj_legacy","storage_mode":"profile_sharded"}"#, + ) + .unwrap(); - let out = gather_local_projects_from(&cwd, &None); + let out = gather_local_projects_from(&cwd); assert!( out.contains(&root), - "ancestor enrollment marker must be detected, got {out:?}" + "ancestor repository identity marker must be detected, got {out:?}" ); assert!( out.contains(&child), - "descendant enrollment marker must be detected, got {out:?}" + "descendant repository identity marker must be detected, got {out:?}" + ); + assert!( + !out.contains(&unenrolled), + "a retired enrollment file is not an identity, got {out:?}" ); } @@ -587,69 +571,20 @@ mod gather_tests { #[test] fn skips_projects_inside_node_modules() { + let _profile = tracedecay_runtime_core::config::PinnedUserDataDir::new(); let dir = tempfile::tempdir().unwrap(); let cwd = dir.path().canonicalize().unwrap(); let buried = cwd.join("node_modules").join("pkg"); fs::create_dir_all(&buried).unwrap(); - make_project(&buried); + make_enrolled_project(&buried, "proj_buried"); - let out = gather_local_projects_from(&cwd, &None); + let out = gather_local_projects_from(&cwd); assert!( !out.contains(&buried), "projects inside node_modules must be skipped, got {out:?}" ); } - #[test] - fn skips_home_data_dir_via_canonical_path() { - // Simulate a symlinked HOME: `home_alias` → `home_real`. The user - // passes `home_alias/.tracedecay` as the skip path, but the descendant - // walk encounters the directory through `home_real/.tracedecay`. - // Canonicalization must resolve them as equal so the global DB - // directory is not picked up as a wipe target. - let dir = tempfile::tempdir().unwrap(); - let root = dir.path().canonicalize().unwrap(); - - let home_real = root.join("home_real"); - fs::create_dir_all(&home_real).unwrap(); - make_project(&home_real); // pretend `~/.tracedecay` is a project (it shouldn't be wiped) - - // Try to symlink: home_alias -> home_real. If the platform doesn't - // allow symlinks (e.g. Windows without dev mode) we just skip the - // canonical-equivalence check and verify the direct-path skip works. - let home_alias = root.join("home_alias"); - let symlink_ok = symlink_dir(&home_real, &home_alias).is_ok(); - - let cwd = root.clone(); - let alias_ts: PathBuf = if symlink_ok { - home_alias.join(".tracedecay") - } else { - home_real.join(".tracedecay") - }; - - let out = gather_local_projects_from(&cwd, &Some(alias_ts)); - assert!( - !out.contains(&home_real), - "home `.tracedecay` (canonical) must be skipped, got {out:?}" - ); - if symlink_ok { - assert!( - !out.contains(&home_alias), - "home `.tracedecay` (alias) must be skipped, got {out:?}" - ); - } - } - - #[cfg(unix)] - fn symlink_dir(src: &Path, dst: &Path) -> std::io::Result<()> { - std::os::unix::fs::symlink(src, dst) - } - - #[cfg(windows)] - fn symlink_dir(src: &Path, dst: &Path) -> std::io::Result<()> { - std::os::windows::fs::symlink_dir(src, dst) - } - #[test] fn registry_target_parser_rejects_malformed_rows() { let error = registry_project_roots(&serde_json::json!({ diff --git a/crates/tracedecay-cli/src/hook_capture_cmd.rs b/crates/tracedecay-cli/src/hook_capture_cmd.rs index 8d238c9a1a..13de3eb4ee 100644 --- a/crates/tracedecay-cli/src/hook_capture_cmd.rs +++ b/crates/tracedecay-cli/src/hook_capture_cmd.rs @@ -3,10 +3,11 @@ use std::io::{Read, Write}; use std::path::Path; use std::time::{SystemTime, UNIX_EPOCH}; +use tracedecay_domain::NativeHostIdentityV1; use tracedecay_domain::UtcMicros; use tracedecay_hooks::delivery_spool::HookDeliverySpoolError; use tracedecay_hooks::{ - HookDeliveryReceiptSpoolV1, HookHostV1, NativeHookCaptureOutcomeV1, NativeHookCaptureSourceV1, + HookDeliveryReceiptSpoolV1, NativeHookCaptureOutcomeV1, NativeHookCaptureSourceV1, }; use crate::cli::Commands; @@ -14,107 +15,107 @@ use crate::cli::Commands; const NATIVE_CAPTURE_COMMANDS: &[(&str, NativeHookCaptureSourceV1)] = &[ ( "hook-prompt-submit", - NativeHookCaptureSourceV1::Host(HookHostV1::ClaudeCode), + NativeHookCaptureSourceV1::Host(NativeHostIdentityV1::ClaudeCode), ), ( "hook-stop", - NativeHookCaptureSourceV1::Host(HookHostV1::ClaudeCode), + NativeHookCaptureSourceV1::Host(NativeHostIdentityV1::ClaudeCode), ), ( "hook-claude-session-start", - NativeHookCaptureSourceV1::Host(HookHostV1::ClaudeCode), + NativeHookCaptureSourceV1::Host(NativeHostIdentityV1::ClaudeCode), ), ( "hook-claude-post-tool-use", - NativeHookCaptureSourceV1::Host(HookHostV1::ClaudeCode), + NativeHookCaptureSourceV1::Host(NativeHostIdentityV1::ClaudeCode), ), ( "hook-claude-subagent-start", - NativeHookCaptureSourceV1::Host(HookHostV1::ClaudeCode), + NativeHookCaptureSourceV1::Host(NativeHostIdentityV1::ClaudeCode), ), ( "hook-kiro-pre-tool-use", - NativeHookCaptureSourceV1::Host(HookHostV1::Kiro), + NativeHookCaptureSourceV1::Host(NativeHostIdentityV1::Kiro), ), ( "hook-kiro-prompt-submit", - NativeHookCaptureSourceV1::Host(HookHostV1::Kiro), + NativeHookCaptureSourceV1::Host(NativeHostIdentityV1::Kiro), ), ( "hook-kiro-post-tool-use", - NativeHookCaptureSourceV1::Host(HookHostV1::Kiro), + NativeHookCaptureSourceV1::Host(NativeHostIdentityV1::Kiro), ), ( "hook-cursor-subagent-start", - NativeHookCaptureSourceV1::Host(HookHostV1::CursorDesktop), + NativeHookCaptureSourceV1::Host(NativeHostIdentityV1::CursorDesktop), ), ( "hook-cursor-post-tool-use", - NativeHookCaptureSourceV1::Host(HookHostV1::CursorDesktop), + NativeHookCaptureSourceV1::Host(NativeHostIdentityV1::CursorDesktop), ), ( "hook-cursor-before-submit-prompt", - NativeHookCaptureSourceV1::Host(HookHostV1::CursorDesktop), + NativeHookCaptureSourceV1::Host(NativeHostIdentityV1::CursorDesktop), ), ( "hook-cursor-pre-compact", - NativeHookCaptureSourceV1::Host(HookHostV1::CursorDesktop), + NativeHookCaptureSourceV1::Host(NativeHostIdentityV1::CursorDesktop), ), ( "hook-cursor-after-file-edit", - NativeHookCaptureSourceV1::Host(HookHostV1::CursorDesktop), + NativeHookCaptureSourceV1::Host(NativeHostIdentityV1::CursorDesktop), ), ( "hook-cursor-session-start", - NativeHookCaptureSourceV1::Host(HookHostV1::CursorDesktop), + NativeHookCaptureSourceV1::Host(NativeHostIdentityV1::CursorDesktop), ), ( "hook-cursor-session-end", - NativeHookCaptureSourceV1::Host(HookHostV1::CursorDesktop), + NativeHookCaptureSourceV1::Host(NativeHostIdentityV1::CursorDesktop), ), ( "hook-cursor-after-shell", - NativeHookCaptureSourceV1::Host(HookHostV1::CursorDesktop), + NativeHookCaptureSourceV1::Host(NativeHostIdentityV1::CursorDesktop), ), ( "hook-cursor-workspace-open", - NativeHookCaptureSourceV1::Host(HookHostV1::CursorDesktop), + NativeHookCaptureSourceV1::Host(NativeHostIdentityV1::CursorDesktop), ), ( "hook-cursor-stop", - NativeHookCaptureSourceV1::Host(HookHostV1::CursorDesktop), + NativeHookCaptureSourceV1::Host(NativeHostIdentityV1::CursorDesktop), ), ( "hook-codex-session-start", - NativeHookCaptureSourceV1::Host(HookHostV1::Codex), + NativeHookCaptureSourceV1::Host(NativeHostIdentityV1::Codex), ), ( "hook-codex-user-prompt-submit", - NativeHookCaptureSourceV1::Host(HookHostV1::Codex), + NativeHookCaptureSourceV1::Host(NativeHostIdentityV1::Codex), ), ( "hook-codex-subagent-start", - NativeHookCaptureSourceV1::Host(HookHostV1::Codex), + NativeHookCaptureSourceV1::Host(NativeHostIdentityV1::Codex), ), ( "hook-codex-post-tool-use", - NativeHookCaptureSourceV1::Host(HookHostV1::Codex), + NativeHookCaptureSourceV1::Host(NativeHostIdentityV1::Codex), ), ( "hook-codex-stop", - NativeHookCaptureSourceV1::Host(HookHostV1::Codex), + NativeHookCaptureSourceV1::Host(NativeHostIdentityV1::Codex), ), ( "hook-hermes-terminal-receipt", - NativeHookCaptureSourceV1::Host(HookHostV1::Hermes), + NativeHookCaptureSourceV1::Host(NativeHostIdentityV1::Hermes), ), ( "hook-kimi-event", - NativeHookCaptureSourceV1::Host(HookHostV1::KimiCode), + NativeHookCaptureSourceV1::Host(NativeHostIdentityV1::KimiCode), ), ( "hook-opencode-event", - NativeHookCaptureSourceV1::Host(HookHostV1::OpenCode), + NativeHookCaptureSourceV1::Host(NativeHostIdentityV1::OpenCode), ), ( "hook-opencode-tool-after", @@ -139,7 +140,7 @@ pub(crate) fn try_run(args: &[OsString]) -> Option { tracedecay_agent_hosts::hooks::record_native_capture_invoked( &tracedecay::hook_runtime(), std::env::current_dir().ok().as_deref(), - HookHostV1::ClaudeCode, + NativeHostIdentityV1::ClaudeCode, Some("preToolUse"), &std::env::var("TOOL_INPUT").unwrap_or_default(), ); @@ -240,7 +241,7 @@ fn capture_command_name(command: &Commands) -> Option<&'static str> { /// be refused for. The response hooks' output write waits the same way. fn open_delivery_receipt_spool( data_root: &Path, - host: HookHostV1, + host: NativeHostIdentityV1, ) -> Result { HookDeliveryReceiptSpoolV1::open_within( tracedecay_hooks::hook_delivery_receipt_spool_root(data_root, host), diff --git a/crates/tracedecay-cli/src/main.rs b/crates/tracedecay-cli/src/main.rs index 3c5a01c7aa..12d3ef7f93 100644 --- a/crates/tracedecay-cli/src/main.rs +++ b/crates/tracedecay-cli/src/main.rs @@ -70,7 +70,7 @@ use cli::*; use tracedecay_daemon_service::logging::StderrTracingDefault; pub(crate) fn current_unix_timestamp() -> i64 { - tracedecay::project::current_timestamp() + tracedecay_runtime_core::tracedecay::current_timestamp() } /// A self-animating spinner that ticks on a background thread. @@ -544,7 +544,7 @@ impl ProcessHotpathGuard { fn install(guard: hotpath::HotpathGuard) -> Result { let guard = Arc::new(Mutex::new(Some(guard))); let watchdog_guard = Arc::clone(&guard); - if !tracedecay::daemon::install_hotpath_shutdown_finalizer(move || { + if !tracedecay_daemon_service::shutdown::install_hotpath_shutdown_finalizer(move || { let guard = watchdog_guard .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) @@ -626,7 +626,9 @@ fn async_main() -> tracedecay_domain::errors::Result { // This binary is the sole generator of source provenance and the embedded // dashboard bundle; the composition library reads both through this // set-once registration. - tracedecay::register_product_runtime(crate::product_runtime::provider())?; + tracedecay_project::product_runtime::register_product_runtime( + crate::product_runtime::provider(), + )?; crate::cloud::admit_sync_probes(); // Every process-global runtime port the extracted crates invert back into // the composition root. Must precede argument parsing: hook, install, and @@ -685,7 +687,7 @@ fn async_main() -> tracedecay_domain::errors::Result { // Pin its profile before Tokio starts worker threads so every canonical // configuration authority observes the Task Scheduler argument. unsafe { - std::env::set_var(tracedecay::config::USER_DATA_DIR_ENV, profile_root); + std::env::set_var(tracedecay_project::config::USER_DATA_DIR_ENV, profile_root); } } // Route tracing events (degradation causes, ingest warnings) to stderr. @@ -872,7 +874,7 @@ async fn run_startup_preamble(command: &Commands) { && user_config.pending_upload > 0 && let Ok(cwd) = std::env::current_dir() && let Some(project_root) = - tracedecay::config::discover_project_root_with_identity(&cwd).await + tracedecay_project::config::discover_project_root_with_identity(&cwd).await { match commands::canonical_upload_enabled(&project_root).await { Ok(upload_enabled) => { @@ -996,8 +998,7 @@ impl CommandFamily { | Commands::Reinstall { .. } | Commands::UpdatePlugin { .. } | Commands::Uninstall { .. } - | Commands::FeedbackRollback { .. } - | Commands::HostBundle { .. } => Self::Agent, + | Commands::FeedbackRollback { .. } => Self::Agent, Commands::HookPreToolUse | Commands::HookPromptSubmit | Commands::HookStop @@ -1089,14 +1090,13 @@ fn validate_host_bundle_options( } return Ok(()); } - // The scoped storage resets destroy refused store state, so they REQUIRE - // the same `--yes` confirmation (their handlers refuse to run without it). - // Like `wipe`, they own no host component and have no preview. + // The scoped storage reset destroys refused store state, so it REQUIRES + // the same `--yes` confirmation (its handler refuses to run without it). + // Like `wipe`, it owns no host component and has no preview. if matches!( command, Commands::Storage { - action: ProfileStorageAction::ResetAuthority { .. } - | ProfileStorageAction::ResetProjectStore { .. }, + action: ProfileStorageAction::ResetProjectStore { .. }, } ) { if host_bundle.component.is_some() || host_bundle.dry_run || host_bundle.adopt { @@ -1146,18 +1146,6 @@ fn validate_host_bundle_options( Ok(()) } -fn is_full_component_set_adoption(command: &Commands, host_bundle: &HostBundleCliOptions) -> bool { - host_bundle.component.is_none() - && host_bundle.yes - && host_bundle.adopt - && matches!( - command, - Commands::Install { local: false, .. } - | Commands::Reinstall { local: false, .. } - | Commands::UpdatePlugin { local: false, .. } - ) -} - async fn dispatch_command( command: Commands, host_bundle: HostBundleCliOptions, @@ -1224,14 +1212,12 @@ async fn dispatch_project_command( } Commands::Sync { path, - force, skip_folders, include_folders, doctor, verbose, } => { - commands::handle_sync(path, force, skip_folders, include_folders, doctor, verbose) - .await?; + commands::handle_sync(path, skip_folders, include_folders, doctor, verbose).await?; } Commands::Status { path, @@ -1564,140 +1550,9 @@ async fn dispatch_agent_command( command: Commands, host_bundle: HostBundleCliOptions, ) -> tracedecay_domain::errors::Result<()> { - let full_reinstall_preflight = matches!( - &command, - Commands::Reinstall { - local: false, - agent: None, - } - ) && host_bundle.component.is_none() - && host_bundle.dry_run - && !host_bundle.yes; - let full_component_set_adoption = is_full_component_set_adoption(&command, &host_bundle); - // `--dry-run` / `--yes` preview or confirm a first-party component - // mutation, so they normally require `--component` to name the target. - // Full `reinstall --dry-run` is the read-only exception: it validates the - // same tracked integration set that post-update will refresh. - if !matches!( - command, - Commands::FeedbackRollback { .. } | Commands::HostBundle { .. } - ) && host_bundle.component.is_none() - && (host_bundle.dry_run || host_bundle.yes) - && !full_reinstall_preflight - && !full_component_set_adoption - { - return Err(tracedecay_domain::errors::TraceDecayError::Config { - message: "--dry-run and --yes require --component to select the target host component" - .to_string(), - }); - } - match command { - Commands::Install { - agent, - local, - no_dashboard, - automation, - git_hook, - } => { - if host_bundle.component.is_some() { - if local || automation || no_dashboard { - return Err(tracedecay_domain::errors::TraceDecayError::Config { - message: "--component cannot be combined with --local, --automation, or --no-dashboard" - .to_string(), - }); - } - agent_cmd::handle_host_bundle_component_command( - agent, - agent_cmd::HostBundleCliOperation::Install, - host_bundle, - ) - .await?; - if git_hook { - agent_cmd::install_requested_git_hook()?; - } - } else { - agent_cmd::handle_install_command( - agent, - local, - no_dashboard, - automation.then_some(agent_cmd::CodexAutomationInstall), - host_bundle.adopt, - git_hook, - ) - .await?; - } - } - Commands::Reinstall { local, agent } => { - if host_bundle.component.is_some() { - if local { - return Err(tracedecay_domain::errors::TraceDecayError::Config { - message: "--component cannot be combined with --local".to_string(), - }); - } - agent_cmd::handle_host_bundle_component_command( - None, - agent_cmd::HostBundleCliOperation::Repair, - host_bundle, - ) - .await?; - } else if host_bundle.dry_run { - agent_cmd::handle_reinstall_preflight_command()?; - } else if local { - agent_cmd::handle_project_local_lifecycle_command( - agent.expect("--local requires --agent"), - agent_cmd::HostBundleCliOperation::Repair, - ) - .await?; - } else { - agent_cmd::handle_reinstall_command(host_bundle.adopt).await?; - } - } - Commands::UpdatePlugin { local, agent } => { - if host_bundle.component.is_some() { - if local { - return Err(tracedecay_domain::errors::TraceDecayError::Config { - message: "--component cannot be combined with --local".to_string(), - }); - } - agent_cmd::handle_host_bundle_component_command( - None, - agent_cmd::HostBundleCliOperation::Update, - host_bundle, - ) - .await?; - } else if local { - agent_cmd::handle_project_local_lifecycle_command( - agent.expect("--local requires --agent"), - agent_cmd::HostBundleCliOperation::Update, - ) - .await?; - } else { - agent_cmd::handle_update_plugin_command(host_bundle.adopt).await?; - } - } - Commands::Uninstall { agent, local } => { - if host_bundle.component.is_some() { - if local { - return Err(tracedecay_domain::errors::TraceDecayError::Config { - message: "--component cannot be combined with --local".to_string(), - }); - } - agent_cmd::handle_host_bundle_component_command( - agent, - agent_cmd::HostBundleCliOperation::Uninstall, - host_bundle, - ) - .await?; - } else if local { - agent_cmd::handle_project_local_lifecycle_command( - agent.expect("--local requires --agent"), - agent_cmd::HostBundleCliOperation::Uninstall, - ) - .await?; - } else { - agent_cmd::handle_uninstall_command(agent).await?; - } - } + use agent_cmd::HostBundleCliOperation as Operation; + + let (operation, agent, local, no_dashboard, automation, git_hook) = match command { Commands::FeedbackRollback { mut action } => { if host_bundle.component.is_some() || host_bundle.dry_run { return Err(tracedecay_domain::errors::TraceDecayError::Config { @@ -1712,31 +1567,57 @@ async fn dispatch_agent_command( } crate::cli::FeedbackRollbackAction::DryRun { .. } => {} } - agent_cmd::handle_feedback_rollback_command(action).await?; - } - Commands::HostBundle { action } => { - if matches!( - &action, - crate::cli::HostBundleAction::ArtifactBackup { .. } - | crate::cli::HostBundleAction::ArtifactRestore { .. } - ) { - agent_cmd::handle_host_bundle_artifact_command(action, host_bundle).await?; - } else { - if host_bundle.component.is_some() { - return Err(tracedecay_domain::errors::TraceDecayError::Config { - message: "host-bundle recovery operates on the whole component set" - .to_string(), - }); - } - agent_cmd::handle_host_bundle_recovery_command( - action, - host_bundle.dry_run, - host_bundle.yes, - ) - .await?; - } + return agent_cmd::handle_feedback_rollback_command(action).await; + } + Commands::Install { + agent, + local, + no_dashboard, + automation, + git_hook, + } => ( + Operation::Install, + agent, + local, + no_dashboard, + automation, + git_hook, + ), + Commands::Reinstall { local, agent } => { + (Operation::Repair, agent, local, false, false, false) + } + Commands::UpdatePlugin { local, agent } => { + (Operation::Update, agent, local, false, false, false) + } + Commands::Uninstall { agent, local } => { + (Operation::Uninstall, agent, local, false, false, false) } _ => unreachable!("non-agent command passed to agent dispatcher"), + }; + if local { + if host_bundle.component.is_some() || host_bundle.dry_run { + return Err(tracedecay_domain::errors::TraceDecayError::Config { + message: "--component and --dry-run cannot be combined with --local".to_string(), + }); + } + let agent_id = agent.ok_or_else(|| tracedecay_domain::errors::TraceDecayError::Config { + message: "--local requires a project-capable --agent".to_string(), + })?; + agent_cmd::handle_project_local_lifecycle_command(agent_id, operation).await?; + } else { + agent_cmd::handle_host_lifecycle_command( + agent, + operation, + host_bundle, + no_dashboard, + automation.then_some(agent_cmd::CodexAutomationInstall), + ) + .await?; + } + if git_hook { + agent_cmd::install_requested_git_hook()?; + } else if operation == Operation::Install { + tracedecay_agent_hosts::agents::report_git_post_commit_hook_status(); } Ok(()) } @@ -1952,7 +1833,7 @@ async fn dispatch_knowledge_command(command: Commands) -> tracedecay_domain::err sessions_cmd::handle_sessions_action(action).await?; } Commands::Analytics { action } => match action { - AnalyticsAction::Diagnostics { all, no_sync, .. } => { + AnalyticsAction::Diagnostics { all, no_sync } => { hotpath::future!( analytics_cmd::run_analytics_diagnostics(all, no_sync), label = "cli.analytics.diagnostics" @@ -2004,7 +1885,6 @@ impl CommandStartupPolicy { | Commands::Reinstall { .. } | Commands::UpdatePlugin { .. } | Commands::FeedbackRollback { .. } - | Commands::HostBundle { .. } | Commands::Upgrade { .. } | Commands::Update { .. } | Commands::PostUpdate { .. } diff --git a/crates/tracedecay-cli/src/monitor_cmd.rs b/crates/tracedecay-cli/src/monitor_cmd.rs index a5458caeb0..1e9da8c5de 100644 --- a/crates/tracedecay-cli/src/monitor_cmd.rs +++ b/crates/tracedecay-cli/src/monitor_cmd.rs @@ -79,7 +79,7 @@ pub fn run() -> std::io::Result<()> { execute!(stdout, cursor::Show, LeaveAlternateScreen)?; terminal::disable_raw_mode()?; - let _ = lock_file.unlock(); + drop(lock_file); let _ = std::fs::remove_file(&lock_path); result diff --git a/crates/tracedecay-cli/src/product_runtime.rs b/crates/tracedecay-cli/src/product_runtime.rs index 6d4a202f8c..0f46d6c491 100644 --- a/crates/tracedecay-cli/src/product_runtime.rs +++ b/crates/tracedecay-cli/src/product_runtime.rs @@ -30,17 +30,18 @@ pub(crate) use generated::PRODUCT_FULL_SHA; /// fixture may call this unconditionally. #[cfg(test)] pub(crate) fn register_for_tests() { - match tracedecay::register_product_runtime(provider()) { - Ok(()) | Err(tracedecay::ProductRuntimeError::ConflictingProvider) => {} + match tracedecay_project::product_runtime::register_product_runtime(provider()) { + Ok(()) + | Err(tracedecay_project::product_runtime::ProductRuntimeError::ConflictingProvider) => {} Err(error) => panic!("register the CLI product runtime for tests: {error}"), } crate::cloud::admit_sync_probes(); } -pub(crate) fn provider() -> tracedecay::ProductRuntimeProvider { - tracedecay::ProductRuntimeProvider { +pub(crate) fn provider() -> tracedecay_project::product_runtime::ProductRuntimeProvider { + tracedecay_project::product_runtime::ProductRuntimeProvider { release_version: env!("CARGO_PKG_VERSION"), - source: tracedecay::ProductSourceProvenance { + source: tracedecay_project::product_runtime::ProductSourceProvenance { full_sha: generated::PRODUCT_FULL_SHA, dirty: generated::PRODUCT_SOURCE_DIRTY, }, diff --git a/crates/tracedecay-cli/src/project_cmd.rs b/crates/tracedecay-cli/src/project_cmd.rs index 35bcaab1ec..42708928c4 100644 --- a/crates/tracedecay-cli/src/project_cmd.rs +++ b/crates/tracedecay-cli/src/project_cmd.rs @@ -333,7 +333,7 @@ fn render_project_context_payload(payload: &Value) -> String { #[hotpath::measure(label = "cli.projects.request", future = true)] async fn call_registry_admin(arguments: Value) -> Result { let cwd = std::env::current_dir()?; - let project_root = tracedecay::config::discover_project_root(&cwd); + let project_root = tracedecay_project::config::discover_project_root(&cwd); let arguments = registry_admin_arguments(project_root, arguments); daemon_tool_json(None, "tracedecay_admin_cli", arguments).await } diff --git a/crates/tracedecay-cli/src/remote_command.rs b/crates/tracedecay-cli/src/remote_command.rs index 4f19838feb..c16791b32a 100644 --- a/crates/tracedecay-cli/src/remote_command.rs +++ b/crates/tracedecay-cli/src/remote_command.rs @@ -337,7 +337,9 @@ fn query_payload( match &response.result { Ok(envelope) => match &envelope.outcome { ApplicationOutcome::Evidence(packet) => packet.payload.as_ref(), - ApplicationOutcome::Preview(_) | ApplicationOutcome::Effect(_) => None, + ApplicationOutcome::Preview(_) + | ApplicationOutcome::Effect(_) + | ApplicationOutcome::Result(_) => None, }, Err(_) => None, } diff --git a/crates/tracedecay-cli/src/serve_cmd.rs b/crates/tracedecay-cli/src/serve_cmd.rs index a8e5c3cbea..3ca9506712 100644 --- a/crates/tracedecay-cli/src/serve_cmd.rs +++ b/crates/tracedecay-cli/src/serve_cmd.rs @@ -3,8 +3,8 @@ use std::io::Write; use std::path::Path; -use tracedecay::project::TraceDecay; use tracedecay_domain::errors::Result; +use tracedecay_project::project::TraceDecay; /// Returns the first plausible unexpanded `${...}` template variable in a /// `--path` argument (e.g. `${workspaceFolder}`), or `None` when the value @@ -99,7 +99,7 @@ fn proxy_serve_handshake( }; let ambient_discovery = - !explicit_path && tracedecay::config::is_ambient_project_root(&resolved_path); + !explicit_path && tracedecay_project::config::is_ambient_project_root(&resolved_path); let initialized = !ambient_discovery && TraceDecay::is_initialized(&resolved_path); // `serve` is a database-free proxy. It may consult only an already-pinned // in-memory snapshot; missing authority disables implicit auto-init rather @@ -112,13 +112,13 @@ fn proxy_serve_handshake( // mirrors the same default fallback in `resolve_daemon_initialize_route`. let auto_init_root = (!ambient_discovery && !initialized - && tracedecay::config::cached_sync_config(&resolved_path).map_or_else( + && tracedecay_project::config::cached_sync_config(&resolved_path).map_or_else( |_| tracedecay_configuration::SyncConfig::default().auto_init, |config| config.auto_init, )) .then(|| tracedecay_runtime_core::worktree::git_worktree_root(&resolved_path)) .flatten() - .filter(|root| !tracedecay::config::is_ambient_project_root(root)); + .filter(|root| !tracedecay_project::config::is_ambient_project_root(root)); if let Some(root) = auto_init_root.as_ref() { resolved_path.clone_from(root); } @@ -129,7 +129,7 @@ fn proxy_serve_handshake( .and_then(|project_path| serve_scope_prefix(original_cwd, project_path)); let telemetry_timings = timings || project_path.as_deref().is_some_and(|path| { - tracedecay::config::cached_telemetry_config(path) + tracedecay_project::config::cached_telemetry_config(path) .is_ok_and(|telemetry| telemetry.timings) }); let mut handshake = tracedecay::daemon::handshake_for_current_client( diff --git a/crates/tracedecay-cli/src/sessions_cmd.rs b/crates/tracedecay-cli/src/sessions_cmd.rs index f933967ab6..e0907d93fb 100644 --- a/crates/tracedecay-cli/src/sessions_cmd.rs +++ b/crates/tracedecay-cli/src/sessions_cmd.rs @@ -251,10 +251,7 @@ mod search_report_tests { fn base_result() -> serde_json::Value { json!({ - "catch_up": false, - "catch_up_failures": [], - "catch_up_performed": false, - "catch_up_provider": "all", + "require_fresh": false, "goals": false, "include_subagents": false, "message_type": "any", diff --git a/crates/tracedecay-cli/src/sessions_cmd/refresh.rs b/crates/tracedecay-cli/src/sessions_cmd/refresh.rs index 88c972e520..f808a3cee5 100644 --- a/crates/tracedecay-cli/src/sessions_cmd/refresh.rs +++ b/crates/tracedecay-cli/src/sessions_cmd/refresh.rs @@ -13,13 +13,13 @@ use std::pin::Pin; use serde_json::{Value, json}; use tracedecay_contracts::retained_surfaces::{ - RetainedErrorV1, RetainedOutcomeStatusV1, RetainedOutputFormatV1, - SessionRefreshActionRequestV1, SessionRefreshBeginResultV1, SessionRefreshCancelResultV1, - SessionRefreshFrontierV1, SessionRefreshGrainV1, SessionRefreshProgressV1, - SessionRefreshReceiptV1, SessionRefreshScopeV1, SessionRefreshSessionV1, - SessionRefreshSourceV1, SessionRefreshStatusResultV1, SessionRefreshTargetV1, - SessionRefreshTemporalModeV1, + RetainedErrorV1, RetainedOutcomeStatusV1, SessionRefreshActionRequestV1, + SessionRefreshBeginResultV1, SessionRefreshCancelResultV1, SessionRefreshFrontierV1, + SessionRefreshGrainV1, SessionRefreshProgressV1, SessionRefreshReceiptV1, + SessionRefreshScopeV1, SessionRefreshSessionV1, SessionRefreshSourceV1, + SessionRefreshStatusResultV1, SessionRefreshTargetV1, }; +use tracedecay_domain::TemporalModeV1; use tracedecay_domain::errors::{Result, TraceDecayError}; use crate::cli::{ @@ -233,12 +233,13 @@ where let handle = validated_refresh_handle(operation, handle)?; let scope = resolve_session_refresh_scope(transport, selectors).await?; - let request = session_refresh_request(selectors, &scope, handle); + let mut arguments = serde_json::to_value(session_refresh_request(selectors, &scope, handle))?; + arguments["format"] = json!("json"); let reply = transport .call( scope.project_root.as_deref(), operation.tool_name(), - serde_json::to_value(&request)?, + arguments, ) .await?; SessionRefreshOutcomeView::decode(operation, reply) @@ -359,7 +360,7 @@ fn session_refresh_request( scope: selectors.provider.clone(), }, target: SessionRefreshTargetV1 { - temporal_mode: SessionRefreshTemporalModeV1::Current, + temporal_mode: TemporalModeV1::Current, grain: SessionRefreshGrainV1::LogicalMessage, frontier: SessionRefreshFrontierV1 { observed_through: selectors.target, @@ -367,7 +368,6 @@ fn session_refresh_request( }, }, handle: handle.map(str::to_owned), - format: Some(RetainedOutputFormatV1::Json), } } diff --git a/crates/tracedecay-cli/src/sessions_cmd/refresh/tests.rs b/crates/tracedecay-cli/src/sessions_cmd/refresh/tests.rs index 729c76876a..ba3569444d 100644 --- a/crates/tracedecay-cli/src/sessions_cmd/refresh/tests.rs +++ b/crates/tracedecay-cli/src/sessions_cmd/refresh/tests.rs @@ -16,7 +16,8 @@ use tracedecay_contracts::{ RequestId, ResolvedScope, RetainedSurfaceExecutionContextV1, RetrievalEvidence, TemporalState, retained_receipts, retained_surface_application_operation, }; -use tracedecay_daemon_service::application_surface::retained::decode_request; +use tracedecay_daemon_protocol::decode_retained_request; +use tracedecay_daemon_protocol::{RequestedOutputFormat, separate_application_tool_request}; use tracedecay_domain::{ ActorId, ComponentVersion, ProjectId, RepositoryId, UtcMicros, WorktreeId, }; @@ -96,6 +97,9 @@ fn effect_reply(operation: RetainedSurfaceOperation, result: RetainedSurfaceResu request_id: context.request_id().clone(), scope: scope(), outcome, + touched_files: Vec::new(), + code_graph: None, + analytics: None, }) .unwrap() } @@ -346,14 +350,17 @@ impl SessionRefreshDaemonTransport for FakeDaemonTransport { } /// Every payload the CLI sends must be the exact canonical request the daemon -/// decodes for that operation: no `action`, no `profile` object, no -/// transport-only selector. +/// decodes for that operation once the JSON `format` transport key is +/// separated: no `action`, no `profile` object, no transport-only selector. fn assert_canonical(operation: RetainedSurfaceOperation, payload: &Value) { - let decoded = decode_request(operation, payload.clone()) + let separated = separate_application_tool_request(payload.clone()).unwrap(); + assert_eq!(separated.requested_format, RequestedOutputFormat::Json); + let decoded = decode_retained_request(operation, separated.request.clone()) .unwrap_or_else(|error| panic!("{} payload must decode: {error}", operation.as_str())); assert_eq!(decoded.operation(), operation); - let request: SessionRefreshActionRequestV1 = serde_json::from_value(payload.clone()).unwrap(); - assert_eq!(serde_json::to_value(&request).unwrap(), *payload); + let request: SessionRefreshActionRequestV1 = + serde_json::from_value(separated.request.clone()).unwrap(); + assert_eq!(serde_json::to_value(&request).unwrap(), separated.request); } #[tokio::test] diff --git a/crates/tracedecay-cli/src/startup_tests.rs b/crates/tracedecay-cli/src/startup_tests.rs index 926f8679e5..f9cb2cf606 100644 --- a/crates/tracedecay-cli/src/startup_tests.rs +++ b/crates/tracedecay-cli/src/startup_tests.rs @@ -1,14 +1,12 @@ use super::{ AnalyticsAction, AsyncRuntimeFlavor, Cli, CommandFamily, Commands, DAEMON_CPU_THREADS_ENV, - DEFAULT_MAX_DAEMON_CPU_THREADS, HostBundleCliOptions, HostBundleComponentArg, - PackageHookAction, ProfileStorageAction, ProjectsAction, RAYON_NUM_THREADS_ENV, + DEFAULT_MAX_DAEMON_CPU_THREADS, HostBundleCliOptions, PackageHookAction, RAYON_NUM_THREADS_ENV, ScoopPackageHookAction, StderrTracingDefault, async_runtime_flavor, command_profile_label, daemon_cpu_threads_from, hotpath_focus_is_valid, hotpath_output_format_is_none, hotpath_output_format_is_valid, hotpath_output_path_is_valid, - hotpath_requires_protocol_safe_output, is_full_component_set_adoption, - normalize_tool_reserved_global_flags, runs_worldwide_counter_flush, - should_skip_agent_install_check, should_skip_startup_maintenance, stderr_tracing_default, - validate_host_bundle_options, + hotpath_requires_protocol_safe_output, normalize_tool_reserved_global_flags, + runs_worldwide_counter_flush, should_skip_agent_install_check, should_skip_startup_maintenance, + stderr_tracing_default, validate_host_bundle_options, }; use clap::{CommandFactory, Parser}; use std::iter; @@ -137,87 +135,77 @@ fn hotpath_command_identity_uses_the_exact_clap_subcommand_path() { assert_eq!(parsed_command_profile_label(&["hook-stop"]), "hook-stop"); } -/// `wipe` owns no host component and has no preview, so the other two global -/// lifecycle flags stay rejected on it. -#[test] -fn wipe_still_rejects_component_and_dry_run() { - let command = Commands::Wipe { all: false }; - let family = CommandFamily::for_command(&command); - let dry_run = HostBundleCliOptions { - component: None, - dry_run: true, - yes: false, - adopt: false, - }; - assert!(validate_host_bundle_options(&command, family, &dry_run).is_err()); -} - -/// `projects forget` owns no host component, so the component/adopt lifecycle -/// flags stay rejected on it; and read-only `projects` verbs still reject the -/// confirmation flags entirely. -#[test] -fn projects_forget_rejects_component_and_projects_list_rejects_yes() { - let forget = Commands::Projects { - action: ProjectsAction::Forget { - selector: "proj_123".to_string(), - keep_store: false, - }, - }; - let component = HostBundleCliOptions { - component: Some(HostBundleComponentArg::Core), - dry_run: false, - yes: true, - adopt: false, - }; - assert!( - validate_host_bundle_options(&forget, CommandFamily::for_command(&forget), &component) - .is_err() - ); - - let list = Commands::Projects { - action: ProjectsAction::List { - limit: 25, - json: false, - }, - }; - let yes = HostBundleCliOptions { - component: None, - dry_run: false, - yes: true, - adopt: false, +fn validate_parsed_host_bundle_options(args: &[&str]) -> Result<(), String> { + let cli = Cli::try_parse_from(iter::once("tracedecay").chain(args.iter().copied())) + .unwrap_or_else(|error| panic!("{args:?} must parse before scoped validation: {error}")); + let command = cli.command.expect("subcommand must be present"); + let options = HostBundleCliOptions { + component: cli.component, + dry_run: cli.dry_run, + yes: cli.yes, + adopt: cli.adopt, }; - assert!(validate_host_bundle_options(&list, CommandFamily::for_command(&list), &yes).is_err()); + validate_host_bundle_options(&command, CommandFamily::for_command(&command), &options) + .map_err(|error| error.to_string()) } -/// The storage resets own no host component and have no preview, so the other -/// two global lifecycle flags stay rejected on them. #[test] -fn storage_resets_still_reject_component_and_dry_run() { - let command = Commands::Storage { - action: ProfileStorageAction::ResetProjectStore { - project_root: Some("/tmp/some-project".to_string()), - project_id: None, - }, - }; - let family = CommandFamily::for_command(&command); - let dry_run = HostBundleCliOptions { - component: None, - dry_run: true, - yes: true, - adopt: false, - }; - assert!(validate_host_bundle_options(&command, family, &dry_run).is_err()); - let component = HostBundleCliOptions { - component: Some(HostBundleComponentArg::Core), - dry_run: false, - yes: true, - adopt: false, - }; - assert!(validate_host_bundle_options(&command, family, &component).is_err()); +fn destructive_non_lifecycle_commands_accept_only_their_own_confirmation_flags() { + for accepted in [ + &["wipe", "--yes"][..], + &["projects", "forget", "proj_123", "--yes"][..], + &["projects", "forget", "proj_123", "--dry-run"][..], + &[ + "storage", + "reset-project-store", + "--project-root", + "/tmp/p", + "--yes", + ][..], + &["projects", "list"][..], + ] { + assert_eq!( + validate_parsed_host_bundle_options(accepted), + Ok(()), + "{accepted:?}" + ); + } + for (rejected, message) in [ + (&["wipe", "--dry-run"][..], "wipe accepts --yes to confirm"), + ( + &[ + "projects", + "forget", + "proj_123", + "--yes", + "--component", + "core", + ][..], + "projects forget accepts --yes to confirm and --dry-run to preview", + ), + ( + &[ + "storage", + "reset-project-store", + "--project-root", + "/tmp/p", + "--dry-run", + ][..], + "storage resets accept --yes to confirm", + ), + ( + &["projects", "list", "--yes"][..], + "--component, --dry-run, --yes, and --adopt are only valid with install", + ), + ] { + let error = validate_parsed_host_bundle_options(rejected) + .expect_err("lifecycle-only flag must be refused"); + assert!(error.contains(message), "{rejected:?}: {error}"); + } } #[test] -fn default_component_set_adoption_requires_confirmation_and_reaches_dispatch() { +fn default_component_set_adoption_requires_confirmation() { for args in [ &[ "tracedecay", @@ -240,11 +228,7 @@ fn default_component_set_adoption_requires_confirmation_and_reaches_dispatch() { }; validate_host_bundle_options(&command, CommandFamily::for_command(&command), &options) .expect("confirmed default component-set adoption must pass validation"); - assert!( - is_full_component_set_adoption(&command, &options), - "confirmed default adoption must reach the full component-set handler path" - ); - assert!(options.yes && options.adopt); + assert!(options.component.is_none() && options.yes && options.adopt); } } @@ -510,6 +494,7 @@ fn nested_inspection_commands_skip_agent_install_check() { &["branch", "autotrack", "status"][..], &["channel"][..], &["gitignore"][..], + &["automation", "config", "get", "--json"][..], ]; for args in commands { @@ -525,63 +510,6 @@ fn nested_inspection_commands_skip_agent_install_check() { } } -#[test] -fn read_only_automation_commands_skip_agent_install_check() { - for args in [ - &["automation", "config", "get", "--json"][..], - &["automation", "config", "explain", "--json"][..], - &["automation", "runs", "list", "--json"][..], - &["automation", "runs", "view", "run-123", "--json"][..], - &[ - "automation", - "runs", - "artifact", - "run-123", - "validation_gate", - "--json", - ][..], - &["automation", "skills", "list", "--json"][..], - &["automation", "skills", "view", "skill-123", "--json"][..], - &["automation", "facts", "list", "--json"][..], - &["automation", "facts", "view", "fact-123"][..], - ] { - let command = parse_command(args); - assert!( - should_skip_agent_install_check(&command), - "{args:?} must not run the unrelated agent-install check" - ); - assert!( - !should_skip_startup_maintenance(&command), - "{args:?} must retain ordinary startup maintenance" - ); - } -} - -#[test] -fn top_level_inspection_commands_skip_agent_install_check() { - for args in [ - &["status", "--json"][..], - &["channel"][..], - &["current-counter"][..], - &["gitignore"][..], - &["cost"][..], - &["bench", "--json"][..], - &["gain", "--json"][..], - &["monitor"][..], - &["list"][..], - ] { - let command = parse_command(args); - assert!( - should_skip_agent_install_check(&command), - "{args:?} must not run the unrelated agent-install check" - ); - assert!( - !should_skip_startup_maintenance(&command), - "{args:?} must retain ordinary startup maintenance" - ); - } -} - #[test] fn mutating_inspection_families_keep_full_startup_policy() { for args in [ diff --git a/crates/tracedecay-cli/src/status_cmd.rs b/crates/tracedecay-cli/src/status_cmd.rs index 6b48109c1b..90f4e24835 100644 --- a/crates/tracedecay-cli/src/status_cmd.rs +++ b/crates/tracedecay-cli/src/status_cmd.rs @@ -541,7 +541,7 @@ async fn handle_status_command_within( } if !tracedecay_configuration::is_in_gitignore(&project_path) { - let dir_name = tracedecay::config::active_data_dir_name(&project_path); + let dir_name = tracedecay_project::config::active_data_dir_name(&project_path); if stderr_is_terminal { eprintln!( "\n\x1b[33mWarning: {dir_name} is not in .gitignore. \ diff --git a/crates/tracedecay-cli/src/tool_command.rs b/crates/tracedecay-cli/src/tool_command.rs index 981ea66494..4bec672784 100644 --- a/crates/tracedecay-cli/src/tool_command.rs +++ b/crates/tracedecay-cli/src/tool_command.rs @@ -48,7 +48,6 @@ use serde_json::Value; use tokio::time::{Instant, timeout_at}; use tracedecay::daemon::call_default_tool_awaiting_project_open; -use tracedecay::mcp::server::TOKEN_ACCOUNTING_FOOTER_PREFIX; use tracedecay_contracts::request_identity::{GlobalRequestSurface, mint_global_request_id}; use tracedecay_contracts::{CancellationSignal, Deadline, RetainedSurfaceOperation}; use tracedecay_daemon_protocol::{ @@ -61,6 +60,9 @@ use tracedecay_daemon_protocol::{ use tracedecay_daemon_service::application_surface::observe_surface_argument_rejection; use tracedecay_domain::UtcMicros; use tracedecay_domain::errors::{Result, TraceDecayError}; +use tracedecay_mcp::tools::response_trailers::{ + TOKEN_ACCOUNTING_FOOTER_PREFIX, account_tool_result, +}; use tracedecay_mcp::{ RESERVED_FLAGS_FOOTER, ToolDefinition, get_tool_definitions, internal_daemon_tool_definition, render_tool_cli_help, short_tool_name, @@ -184,7 +186,7 @@ fn run_inner( let requested_operation = name .as_deref() .map(canonical_tool_name) - .and_then(|canonical| cli_application_operation(&canonical)); + .and_then(|canonical| ApplicationSurfaceOperation::from_tool_name(&canonical)); if let Some(operation) = requested_operation && let Some(parsed) = parse_whole_payload_invocation(&args)? { @@ -200,6 +202,43 @@ fn run_inner( .checked_add(tool_command_deadline()?) .ok_or_else(tool_deadline_range_error)?; let tool_name = operation.mcp_tool_name(); + if RetainedSurfaceOperation::from_application(operation).is_some() { + let mut tool_args = tool_args; + let dispatch = + DaemonToolDispatch::for_tool(explicit_project, tool_name, &mut tool_args); + if requests_profile_authority(tool_name, &tool_args) { + return dispatch_compatibility_tool( + dispatch, tool_name, tool_args, raw_json, deadline, + ) + .await; + } + return dispatch_cli_retained(operation, tool_args, dispatch, raw_json, deadline) + .await; + } + if operation.is_graph_tool() { + let project_path = + DaemonToolDispatch::project_scoped(explicit_project, tool_name).project_path; + return dispatch_cli_graph_tool( + operation, + tool_args, + project_path, + raw_json, + deadline, + ) + .await; + } + if tracedecay_daemon_protocol::is_source_edit_operation(operation) { + let project_path = + DaemonToolDispatch::project_scoped(explicit_project, tool_name).project_path; + return dispatch_cli_source_edit( + operation, + tool_args, + project_path, + raw_json, + deadline, + ) + .await; + } let (request, requested_format) = cli_surface_invocation(tool_name, tool_args, raw_json).map_err(|error| { TraceDecayError::Config { @@ -277,7 +316,40 @@ fn run_inner( let deadline = Instant::now() .checked_add(tool_command_deadline()?) .ok_or_else(tool_deadline_range_error)?; - if let Some(operation) = ApplicationSurfaceOperation::from_tool_name(&def.name) { + if let Some(operation) = ApplicationSurfaceOperation::from_tool_name(&def.name) + && RetainedSurfaceOperation::from_application(operation).is_some() + && !requests_profile_authority(&def.name, &tool_args) + { + let dispatch = + DaemonToolDispatch::for_tool(explicit_project, &def.name, &mut tool_args); + return dispatch_cli_retained(operation, tool_args, dispatch, raw_json, deadline).await; + } + if let Some(operation) = ApplicationSurfaceOperation::from_tool_name(&def.name) + && operation.is_graph_tool() + { + let project_path = + DaemonToolDispatch::project_scoped(explicit_project, &def.name).project_path; + return dispatch_cli_graph_tool(operation, tool_args, project_path, raw_json, deadline) + .await; + } + if let Some(operation) = ApplicationSurfaceOperation::from_tool_name(&def.name) + && tracedecay_daemon_protocol::is_source_edit_operation(operation) + { + let project_path = + DaemonToolDispatch::project_scoped(explicit_project, &def.name).project_path; + return dispatch_cli_source_edit( + operation, + tool_args, + project_path, + raw_json, + deadline, + ) + .await; + } + // Profile-authority retained calls stay on the daemon's profile route below. + if let Some(operation) = ApplicationSurfaceOperation::from_tool_name(&def.name) + && RetainedSurfaceOperation::from_application(operation).is_none() + { let (request, requested_format) = cli_surface_invocation(&def.name, tool_args, raw_json).map_err(|error| { TraceDecayError::Config { @@ -304,18 +376,6 @@ fn run_inner( }) } -/// Resolves a canonicalised `tracedecay tool` name to its application operation. -/// -/// The CLI answers to both spellings the catalog gives an operation: its CLI -/// binding name, which is the MCP tool spelling, and its canonical identity, -/// which every other surface and every rendered envelope reports. They differ -/// only for `diagnostics` / `diagnostics_read`, so neither spelling needs a -/// name table of its own. -fn cli_application_operation(canonical: &str) -> Option { - ApplicationSurfaceOperation::from_tool_name(canonical) - .or_else(|| ApplicationSurfaceOperation::from_catalog_name(short_tool_name(canonical))) -} - /// Dispatch one catalogued application-surface operation on behalf of a /// first-class CLI command (e.g. `tracedecay git status`). /// @@ -488,6 +548,199 @@ fn dispatch_cli_application_surface_inner( }) } +/// The request deadline and cancellation for one CLI application attempt. +fn cli_request_controls( + request_id: &tracedecay_contracts::RequestId, + deadline: Instant, +) -> Result<(Deadline, CancellationSignal)> { + let remaining = deadline.saturating_duration_since(Instant::now()); + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|duration| i64::try_from(duration.as_micros()).unwrap_or(i64::MAX)) + .unwrap_or(i64::MAX); + let request_deadline = Deadline::new(UtcMicros( + now.saturating_add(i64::try_from(remaining.as_micros()).unwrap_or(i64::MAX)), + )) + .map_err(|error| TraceDecayError::Config { + message: error.to_string(), + })?; + let cancellation = + CancellationSignal::active(format!("cancellation.cli.{}", request_id.as_str())).map_err( + |error| TraceDecayError::Config { + message: error.to_string(), + }, + )?; + Ok((request_deadline, cancellation)) +} + +/// Run one retained memory, session, or workflow tool through the application +/// surface and print the same tool result its MCP call returns. +#[hotpath::measure(label = "cli.tool.retained", future = true)] +async fn dispatch_cli_retained( + operation: ApplicationSurfaceOperation, + tool_args: Value, + dispatch: DaemonToolDispatch, + raw_json: bool, + deadline: Instant, +) -> Result<()> { + let tool_name = operation.mcp_tool_name(); + let request_id = + mint_global_request_id(GlobalRequestSurface::Cli).map_err(|_| TraceDecayError::Config { + message: "could not allocate an application surface request id".to_owned(), + })?; + let client = tracedecay_daemon_identity::invocation_client_for_current(dispatch.handshake()?)?; + // The mounting refusal precedes admission; re-send it until the deadline. + let execution = loop { + let (request_deadline, cancellation) = cli_request_controls(&request_id, deadline)?; + let execution = tracedecay::mcp::tools::execute_retained_surface_tool( + tracedecay_tool_catalog::BindingSurface::Cli, + operation, + tool_args.clone(), + Some(&client), + Some(request_id.clone()), + Some(request_deadline), + Some(cancellation), + ) + .await?; + let Some(delay) = execution + .result + .as_ref() + .err() + .and_then(|problem| problem.problem.owner_mount_resend_delay()) + else { + break execution; + }; + if deadline.saturating_duration_since(Instant::now()) <= delay { + break execution; + } + tokio::time::sleep(delay).await; + }; + let mut result = tracedecay::mcp::tools::render_retained_execution( + dispatch.project_path.as_deref(), + &execution, + )?; + tracedecay_mcp::tool_errors::mark_semantic_tool_error(&mut result); + print_tool_output(&result.value, raw_json); + tool_result_process_outcome(&result.value, tool_name) +} + +/// Run one source-edit tool through the application surface and print the +/// same tool result its MCP call returns. +#[hotpath::measure(label = "cli.tool.source_edit", future = true)] +async fn dispatch_cli_source_edit( + operation: ApplicationSurfaceOperation, + tool_args: Value, + project: Option, + raw_json: bool, + deadline: Instant, +) -> Result<()> { + let tool_name = operation.mcp_tool_name(); + let request_id = + mint_global_request_id(GlobalRequestSurface::Cli).map_err(|_| TraceDecayError::Config { + message: "could not allocate an application surface request id".to_owned(), + })?; + let handshake = + tracedecay::daemon::handshake_for_current_client(project.clone(), None, false, false)?; + let client = tracedecay_daemon_identity::invocation_client_for_current(handshake)?; + // A cold daemon refuses with the mounting problem while the project open + // warms; that refusal precedes admission, so it is re-sent until the CLI + // deadline like every other surface. + let outcome = loop { + let (request_deadline, cancellation) = cli_request_controls(&request_id, deadline)?; + let outcome = tracedecay_mcp::handlers::edit::run_source_edit( + tracedecay_tool_catalog::BindingSurface::Cli, + operation, + &tool_args, + tracedecay_mcp::handlers::edit::SourceEditInvocationContext { + executor: Some(&client), + target: tracedecay_contracts::InvocationTarget::CurrentProject, + request_id: Some(request_id.clone()), + deadline: Some(request_deadline), + cancellation: Some(cancellation), + }, + ) + .await?; + let Some(delay) = outcome + .as_ref() + .err() + .and_then(tracedecay_contracts::ApplicationProblemRecord::owner_mount_resend_delay) + else { + break outcome; + }; + if deadline.saturating_duration_since(Instant::now()) <= delay { + break outcome; + } + tokio::time::sleep(delay).await; + }; + let mut result = tracedecay_mcp::handlers::edit::render_source_edit_outcome( + project.as_deref(), + operation, + &tool_args, + outcome, + )?; + tracedecay_mcp::tool_errors::mark_semantic_tool_error(&mut result); + print_tool_output(&result.value, raw_json); + tool_result_process_outcome(&result.value, tool_name) +} + +/// Run one graph or port read through its project owner and print the same +/// tool result its MCP call returns. +#[hotpath::measure(label = "cli.tool.graph_tool", future = true)] +async fn dispatch_cli_graph_tool( + operation: ApplicationSurfaceOperation, + tool_args: Value, + project: Option, + raw_json: bool, + deadline: Instant, +) -> Result<()> { + let tool_name = operation.mcp_tool_name(); + let request_id = + mint_global_request_id(GlobalRequestSurface::Cli).map_err(|_| TraceDecayError::Config { + message: "could not allocate an application surface request id".to_owned(), + })?; + let handshake = + tracedecay::daemon::handshake_for_current_client(project.clone(), None, false, false)?; + let client = tracedecay_daemon_identity::invocation_client_for_current(handshake)?; + // A cold daemon refuses with the mounting problem while the project open + // warms; that refusal precedes admission, so it is re-sent until the CLI + // deadline like every other surface. + let completion = loop { + let (request_deadline, cancellation) = cli_request_controls(&request_id, deadline)?; + let outcome = tracedecay::mcp::tools::execute_graph_tool_surface( + tracedecay_tool_catalog::BindingSurface::Cli, + operation, + tool_args.clone(), + Some(&client), + Some(request_id.clone()), + Some(request_deadline), + Some(cancellation), + ) + .await; + let mounting = outcome.as_ref().err().is_some_and(|error| { + error.project_route_context().is_some_and(|(code, _, _)| { + code == tracedecay_contracts::RUNTIME_MOUNTING_REASON_CODE + }) + }); + if !mounting + || deadline.saturating_duration_since(Instant::now()) <= GRAPH_TOOL_RESEND_DELAY + { + break outcome?; + } + tokio::time::sleep(GRAPH_TOOL_RESEND_DELAY).await; + }; + let mut result = tracedecay_mcp::handlers::graph_tool::render_graph_tool( + project.as_deref(), + &tool_args, + completion, + )?; + account_tool_result(project.as_deref(), &mut result); + tracedecay_mcp::tool_errors::mark_semantic_tool_error(&mut result); + print_tool_output(&result.value, raw_json); + tool_result_process_outcome(&result.value, tool_name) +} + +const GRAPH_TOOL_RESEND_DELAY: Duration = Duration::from_millis(250); + fn print_cli_application_surface( result: ApplicationSurfaceInvocationResult, raw_json: bool, @@ -654,7 +907,7 @@ fn requests_profile_authority(tool_name: &str, tool_args: &Value) -> bool { } fn implicit_tool_project_path(cwd: &Path) -> Option { - tracedecay::config::discover_project_root(cwd) + tracedecay_project::config::discover_project_root(cwd) } /// `project_context` with an explicit uninitialised `--project` and no @@ -923,21 +1176,15 @@ fn group_for(def: &ToolDefinition) -> &'static str { || n == "tracedecay_dependency_depth" { "health" - } else if n == "tracedecay_callers" - || n == "tracedecay_callees" - || n == "tracedecay_callers_for" - || n == "tracedecay_call_chain" + } else if n == "tracedecay_call_chain" || n == "tracedecay_impact" || n == "tracedecay_file_dependents" || n == "tracedecay_by_qualified_name" || n == "tracedecay_signature" - || n == "tracedecay_impls" - || n == "tracedecay_implementations" || n == "tracedecay_derives" || n == "tracedecay_similar" || n == "tracedecay_rename_preview" || n == "tracedecay_find_exact_symbol" - || n == "tracedecay_type_hierarchy" { "graph" } else if n == "tracedecay_diagnose" diff --git a/crates/tracedecay-cli/src/tool_command/args.rs b/crates/tracedecay-cli/src/tool_command/args.rs index 9f5f3e09f9..62ba9022d7 100644 --- a/crates/tracedecay-cli/src/tool_command/args.rs +++ b/crates/tracedecay-cli/src/tool_command/args.rs @@ -6,9 +6,6 @@ use serde_json::{Map, Value}; use tracedecay_domain::errors::{Result, TraceDecayError}; use tracedecay_mcp::{ToolDefinition, resolve_property_schema, short_tool_name}; -/// Legacy CLI command names that do not match the MCP tool name. The right-hand -/// side is the canonical MCP suffix (without the `tracedecay_` prefix). -const NAME_ALIASES: &[(&str, &str)] = &[("query", "search")]; /// Result of CLI argument parsing: the JSON value to hand to the MCP handler, /// plus the reserved-flag side-effects. #[cfg_attr(test, derive(Debug))] @@ -21,17 +18,11 @@ pub(crate) struct ParsedInvocation { } /// Normalize a user-supplied tool name to the canonical `tracedecay_` -/// form used by the MCP registry. Accepts aliases (e.g. `query` → `search`), -/// strips a leading `tracedecay_` if present, and converts dashes to -/// underscores so `dead-code` and `dead_code` both work. +/// form used by the MCP registry. Strips a leading `tracedecay_` if present +/// and converts dashes to underscores so `dead-code` and `dead_code` both work. pub(crate) fn canonical_tool_name(raw: &str) -> String { let trimmed = raw.strip_prefix("tracedecay_").unwrap_or(raw); - let normalized = trimmed.replace('-', "_"); - let mapped = NAME_ALIASES - .iter() - .find(|(k, _)| *k == normalized) - .map_or(normalized.as_str(), |(_, v)| *v); - format!("tracedecay_{mapped}") + format!("tracedecay_{}", trimmed.replace('-', "_")) } /// Parse CLI args against the tool's JSON Schema. Returns the JSON object to diff --git a/crates/tracedecay-cli/src/tool_command/tests.rs b/crates/tracedecay-cli/src/tool_command/tests.rs index dd5d9fadde..26facff7bf 100644 --- a/crates/tracedecay-cli/src/tool_command/tests.rs +++ b/crates/tracedecay-cli/src/tool_command/tests.rs @@ -4,7 +4,7 @@ use tracedecay_contracts::{ ApplicationProblem, ApplicationProblemEnvelope, OpaqueCursor, PageRequest, RequestId, ResultContractRef, SafeDiagnostic, }; -use tracedecay_daemon_service::application_surface::retained::decode_request as decode_retained_request; +use tracedecay_daemon_protocol::decode_retained_request; use tracedecay_daemon_service::application_surface::{ parse_http_application_surface_request, resolve_application_surface_dispatch_with_controls, resolve_catalog_tool_binding, @@ -23,39 +23,7 @@ fn def(name: &str) -> ToolDefinition { } #[test] -fn fact_store_tool_lookup_rejects_broad_and_accepts_exact_routes() { - let definitions = defs(); - assert!( - definitions - .iter() - .all(|definition| definition.name != "tracedecay_fact_store") - ); - for name in [ - "fact_store_add", - "fact_store_search", - "fact_store_probe", - "fact_store_related", - "fact_store_reason", - "fact_store_contradict", - "fact_store_get", - "fact_store_update", - "fact_store_remove", - "fact_store_supersede", - "fact_store_list", - ] { - let canonical = canonical_tool_name(name); - assert!( - definitions - .iter() - .any(|definition| definition.name == canonical), - "{canonical} must resolve through the CLI catalog" - ); - } -} - -#[test] -fn canonicalizes_alias_and_strip_prefix() { - assert_eq!(canonical_tool_name("query"), "tracedecay_search"); +fn canonicalizes_prefix_and_dashes() { assert_eq!( canonical_tool_name("tracedecay_search"), "tracedecay_search" @@ -63,35 +31,6 @@ fn canonicalizes_alias_and_strip_prefix() { assert_eq!(canonical_tool_name("dead-code"), "tracedecay_dead_code"); } -#[test] -fn application_operations_resolve_by_identity_and_by_cli_spelling() { - for operation in ApplicationSurfaceOperation::ALL { - assert_eq!( - cli_application_operation(&canonical_tool_name(operation.as_str())), - Some(operation), - "{} must resolve by its canonical identity", - operation.as_str() - ); - assert_eq!( - cli_application_operation(&canonical_tool_name(operation.mcp_operation_name())), - Some(operation), - "{} must resolve by its CLI binding spelling", - operation.as_str() - ); - } - for spelling in ["diagnostics_read", "diagnostics", "tracedecay_diagnostics"] { - assert_eq!( - cli_application_operation(&canonical_tool_name(spelling)), - Some(ApplicationSurfaceOperation::DiagnosticsRead), - "{spelling}" - ); - } - assert_eq!( - cli_application_operation(&canonical_tool_name("totally-fake-tool")), - None - ); -} - #[test] fn whole_payload_invocation_parses_without_a_tool_definition() { let parsed = parse_whole_payload_invocation_with_stdin( @@ -381,10 +320,12 @@ const REGISTRY_READ_TOOLS: [&str; 3] = [ "tracedecay_project_context", ]; +/// An initialised root is one whose repository carries the `.git/`-side +/// identity marker (or a path-local profile store); the repo-local +/// `.tracedecay/tracedecay.db` layout no longer exists. fn mark_initialised_project(root: &Path) { - let store = root.join(".tracedecay"); - std::fs::create_dir_all(&store).expect("create project store dir"); - std::fs::write(store.join("tracedecay.db"), b"").expect("write project marker"); + tracedecay_runtime_core::storage::pin_fixture_repository_identity(root, "proj_dispatch") + .expect("pin fixture repository identity"); } #[test] @@ -491,10 +432,10 @@ fn registry_read_dispatch_honours_an_explicit_ambient_root_verbatim() { // process-wide profile discovery variables, and HOME is restored below. unsafe { std::env::set_var("HOME", home.path()) }; assert!( - tracedecay::config::is_ambient_project_root(home.path()), + tracedecay_project::config::is_ambient_project_root(home.path()), "fixture HOME must be an ambient root" ); - let discovered = tracedecay::config::discover_project_root(home.path()); + let discovered = tracedecay_project::config::discover_project_root(home.path()); let explicit = DaemonToolDispatch::for_tool(Some(home_arg), "tracedecay_project_list", &mut json!({})); match previous_home { @@ -753,38 +694,17 @@ fn dispatch_routing_keys_bypass_unknown_key_gate() { } #[test] -fn removed_storage_routing_keys_fail_validation() { - let d = def("fact_store_list"); - for removed in ["storage_scope", "hermes_home"] { - let payload = format!(r#"{{"{removed}":"removed"}}"#); - let error = parse_invocation(&d, &["--args".to_string(), payload]).unwrap_err(); - let flag = format!("--{}", removed.replace('_', "-")); - assert!( - error.to_string().contains("unknown parameter") && error.to_string().contains(&flag), - "removed argument should fail clearly: {error}" - ); - } -} +fn lcm_storage_scope_flag_lands_in_tool_args_and_rejects_unknown_scopes() { + let d = def("lcm_status"); + let parsed = parse_invocation(&d, &["--storage-scope".to_string(), "user".to_string()]) + .expect("storage scope flag should parse"); + assert_eq!(parsed.tool_args["storage_scope"], json!("user")); -#[test] -fn lcm_cli_help_exposes_scope_without_hermes_profile_routing() { - for tool_name in [ - "lcm_status", - "lcm_load_session", - "lcm_grep", - "lcm_describe", - "lcm_expand", - "lcm_expand_query", - "lcm_doctor", - "hermes_skill_bridge", - ] { - let help = render_tool_cli_help(&def(tool_name)); - if tool_name.starts_with("lcm_") { - assert!(help.contains("--storage-scope"), "{tool_name}: {help}"); - } - assert!(!help.contains("--hermes-home"), "{tool_name}: {help}"); - assert!(!help.contains("hermes_profile"), "{tool_name}: {help}"); - } + let err = + parse_invocation(&d, &["--storage-scope".to_string(), "shared".to_string()]).unwrap_err(); + let msg = format!("{err}"); + assert!(msg.contains("`shared` is not one of:"), "got: {msg}"); + assert!(msg.contains("project"), "got: {msg}"); } #[test] diff --git a/crates/tracedecay-cli/src/update_cmd.rs b/crates/tracedecay-cli/src/update_cmd.rs index 2077ecec7a..7849980cd9 100644 --- a/crates/tracedecay-cli/src/update_cmd.rs +++ b/crates/tracedecay-cli/src/update_cmd.rs @@ -1,18 +1,11 @@ -//! The `upgrade` / `update` / `post-update` / `update-plugin` flow: binary -//! upgrade via subprocess re-exec, generated-plugin refresh, daemon service -//! refresh, and the full tracked-agent reinstall that keeps config-managed -//! integrations in sync. +//! The `upgrade` / `update` / `post-update` flow: binary upgrade via +//! subprocess re-exec, daemon service refresh, and the tracked-agent +//! reinstall that keeps every host integration in sync. //! //! The post-update pass refreshes every already-configured agent integration -//! through its canonical lifecycle transaction and post-install action, so a -//! separate `tracedecay reinstall` is not needed after an upgrade. Pass -//! `--no-reinstall` to skip that agent-integration refresh. -//! -//! Hosts that own a canonical first-party component set are refreshed only by -//! that tracked-agent pass, which routes them through the receipt-backed -//! component-set transaction. The generated-plugin refresh deliberately skips -//! them: it is not part of the transaction, so rewriting a receipt-owned -//! artifact there would leave the receipt stale until the next reseal. +//! through its receipt-backed component-set transaction, the sole writer of +//! host artifacts, so a separate `tracedecay reinstall` is not needed after an +//! upgrade. Pass `--no-reinstall` to skip that agent-integration refresh. use std::path::{Path, PathBuf}; use std::time::Duration; @@ -25,114 +18,6 @@ use tracedecay_session_memory::user_config::UserConfig; // server-shutdown bounds with margin for service-manager/process-exit latency. const DAEMON_RESTART_LEASE_TIMEOUT: Duration = Duration::from_secs(90); -pub(crate) async fn refresh_generated_plugins() -> tracedecay_domain::errors::Result<()> { - let home = tracedecay_home_dir()?; - let tracedecay_bin = tracedecay_bin_for_generated_artifacts()?; - refresh_generated_plugins_at( - tracedecay_agent_hosts::agents::all_integrations(), - &home, - &tracedecay_bin, - ) -} - -/// Whether a host owns a canonical first-party component set. -/// -/// For those hosts the receipt-backed component-set transaction is the sole -/// writer of the deployed artifacts: `reinstall_agent_integrations` routes them -/// through `apply_default_canonical_component_set` and never calls -/// `update_plugin`. A second writer outside that transaction (this -/// generated-artifact refresh) rewrote the very files the receipt claims, -/// before the transaction resealed them, so every version bump left the -/// receipt stale and Doctor reported a component-ownership conflict. -/// -/// `integration_id_for_host` is many-to-one (CursorCloud and CursorDesktop both -/// map to `cursor`), so an id counts as canonical when ANY host behind it has a -/// non-empty default component set, the transaction owns that id's artifacts. -fn host_owns_canonical_component_set(agent_id: &str) -> bool { - tracedecay_agent_hosts::agents::host_bundle::stock_host_kinds() - .into_iter() - .any(|host| { - tracedecay_agent_hosts::agents::integration_id_for_host(host) == agent_id - && !tracedecay_agent_hosts::agents::host_bundle_registry::default_components(host) - .is_empty() - }) -} - -fn refresh_generated_plugins_at( - integrations: Vec>, - home: &Path, - tracedecay_bin: &str, -) -> tracedecay_domain::errors::Result<()> { - eprintln!( - "Refreshing tracedecay-generated plugin artifacts (supported user configs are preserved)" - ); - - // Detection-driven, not `installed_agents`-driven: each integration - // decides whether generated artifacts exist on this machine, so stale - // tracking state can neither skip a real install nor install anywhere new. - let mut refreshed_any = false; - let mut failures: Vec = Vec::new(); - for ag in integrations { - if host_owns_canonical_component_set(ag.id()) { - eprintln!( - " \x1b[2m·\x1b[0m {}: owned by the receipt-backed component-set transaction; \ - skipped here so the receipt is not left stale", - ag.id() - ); - continue; - } - let ctx = tracedecay_agent_hosts::agents::InstallContext { - home: home.to_path_buf(), - tracedecay_bin: tracedecay_bin.to_string(), - tool_permissions: tracedecay_agent_hosts::agents::expected_tool_perms()?, - project_root: None, - dashboard: true, - }; - let outcome = ag.update_plugin(&ctx); - match outcome { - Ok(tracedecay_agent_hosts::agents::UpdatePluginOutcome::Refreshed(paths)) => { - refreshed_any = true; - for path in paths { - eprintln!( - " \x1b[32m✔\x1b[0m {}: refreshed {}", - ag.id(), - path.display() - ); - } - } - Ok(tracedecay_agent_hosts::agents::UpdatePluginOutcome::NotInstalled) => {} - // Config-managed integrations (claude, copilot, …) are refreshed by - // the tracked-agent reinstall in `run_post_update_tasks`, so there - // is nothing to do, and nothing to nag about, here. - Ok(tracedecay_agent_hosts::agents::UpdatePluginOutcome::ConfigOnly) => {} - Ok(tracedecay_agent_hosts::agents::UpdatePluginOutcome::DeferredUserAction( - deferred, - )) => { - refreshed_any = true; - eprintln!( - " \x1b[33mwarning:\x1b[0m {} plugin activation deferred: {}", - ag.id(), - deferred.remediation - ); - for path in deferred.staged_paths { - eprintln!(" staged: {}", path.display()); - } - } - Err(e) => failures.push(format!("{}: {e}", ag.id())), - } - } - if !refreshed_any { - eprintln!("No generated plugin installs detected. Nothing to update."); - } - if !failures.is_empty() { - return Err(tracedecay_domain::errors::TraceDecayError::Config { - message: format!("update-plugin failed for {}", failures.join("; ")), - }); - } - - Ok(()) -} - /// Rewrites the installed daemon service while preserving its captured /// lifecycle state, returning the service path and socket or `None` when no /// service is installed. @@ -284,14 +169,6 @@ pub(crate) fn restart_daemon_service() -> tracedecay_domain::errors::Result<()> } } -fn tracedecay_home_dir() -> tracedecay_domain::errors::Result { - tracedecay_agent_hosts::agents::home_dir().ok_or_else(|| { - tracedecay_domain::errors::TraceDecayError::Config { - message: "could not determine home directory".to_string(), - } - }) -} - pub(crate) fn tracedecay_bin_on_path() -> tracedecay_domain::errors::Result { tracedecay_agent_hosts::agents::which_tracedecay().ok_or_else(|| { tracedecay_domain::errors::TraceDecayError::Config { @@ -300,15 +177,6 @@ pub(crate) fn tracedecay_bin_on_path() -> tracedecay_domain::errors::Result tracedecay_domain::errors::Result { - current_tracedecay_exe().map_or_else(tracedecay_bin_on_path, Ok) -} - -fn current_tracedecay_exe() -> Option { - let current = std::env::current_exe().ok()?; - current_tracedecay_exe_from(Some(¤t)) -} - fn current_tracedecay_exe_from(current: Option<&Path>) -> Option { let current = current?; let stem = current.file_stem()?.to_str()?; @@ -602,11 +470,9 @@ pub(crate) fn install_pass_covers_tracked_agents( tracked.iter().all(|id| refreshed.contains(id)) } -/// Re-runs the canonical component lifecycle for every tracked agent so tool -/// permissions, hooks, and MCP config stay in sync with the running binary, a -/// superset of `refresh_generated_plugins`, which rewrites generated artifacts -/// only. Mirrors the canonical `handle_reinstall_command` (global scope: -/// `project_root: None`). Continues past a failing agent; returns +/// Re-runs the canonical component lifecycle for every tracked agent so +/// artifacts, tool permissions, hooks, and MCP config stay in sync with the +/// running binary, exactly as `tracedecay reinstall` does. Continues past a failing agent; returns /// [`ReinstallOutcome::PartialFailure`] listing every failure (an empty tracked /// list is [`ReinstallOutcome::AllOk`]). If the home or binary cannot be /// resolved, no install runs and a descriptive failure is reported so the @@ -653,8 +519,6 @@ async fn run_post_update_mutations( no_reinstall: bool, lifecycle_lease: &tracedecay_runtime_core::lifecycle_lease::LifecycleLease, ) -> tracedecay_domain::errors::Result<()> { - refresh_generated_plugins().await?; - if no_reinstall { eprintln!("Skipping agent integration refresh (--no-reinstall)."); // `--no-reinstall` is a durable opt-out for THIS version, not a @@ -667,12 +531,10 @@ async fn run_post_update_mutations( return Ok(()); } - // The generated-artifact refresh above skips config-managed integrations - // (claude, copilot, …), but a version bump can change their tool - // permissions, hooks, or MCP config too. Run the same full tracked-agent - // install pass, then advance the version markers. On failure the markers - // stay put so the incomplete explicit lifecycle remains observable. - // + // A version bump can change any host's artifacts, permissions, hooks, or + // MCP config. Run the full tracked-agent pass, then advance the version + // markers. On failure the markers stay put so the incomplete explicit + // lifecycle remains observable. let mut config = UserConfig::load(); // Prune tracked ids that no longer resolve to an integration (a release // renamed/removed one, or a typo landed in `installed_agents`). @@ -760,9 +622,8 @@ mod tests { use super::{ RefreshPolicy, ReinstallOutcome, current_tracedecay_exe_from, - host_owns_canonical_component_set, install_pass_covers_tracked_agents, - partition_reinstall_results, post_update_binary, post_update_binary_from, - prepare_post_update_lease, refresh_generated_plugins_at, restart_daemon_service_with, + install_pass_covers_tracked_agents, partition_reinstall_results, post_update_binary, + post_update_binary_from, prepare_post_update_lease, restart_daemon_service_with, run_install_then_refresh, }; use crate::upgrade::UpgradeOutcome; @@ -800,10 +661,18 @@ mod tests { #[test] fn daemon_restart_forces_stopped_service_running() { + let order = RefCell::new(Vec::new()); let result = restart_daemon_service_with( - || Ok(daemon_control::DaemonServiceState::StoppedEnabled), - || Ok(()), + || { + order.borrow_mut().push("quiesce"); + Ok(daemon_control::DaemonServiceState::StoppedEnabled) + }, + || { + order.borrow_mut().push("acquire"); + Ok(()) + }, |state| { + order.borrow_mut().push("refresh"); assert_eq!(state, daemon_control::DaemonServiceState::RunningEnabled); Ok(Some((PathBuf::from("service"), PathBuf::from("socket")))) }, @@ -811,7 +680,11 @@ mod tests { ) .expect("restart orchestration"); - assert!(result.is_some()); + assert_eq!( + result, + Some((PathBuf::from("service"), PathBuf::from("socket"))) + ); + assert_eq!(order.into_inner(), ["quiesce", "acquire", "refresh"]); } #[test] @@ -904,91 +777,6 @@ mod tests { assert_eq!(current_tracedecay_exe_from(Some(current)), None); } - /// Kimi owns a canonical component set, so the receipt-backed transaction - /// (`reinstall_agent_integrations` → `apply_default_canonical_component_set`) - /// is its sole writer. The generated-artifact refresh must leave it alone. - /// including its staging directory, and must still succeed rather than - /// treating the skip as a failure that blocks maintenance. - #[test] - fn deferred_kimi_refresh_does_not_block_maintenance() { - let home = TempDir::new().unwrap(); - let installed_path = home.path().join(".kimi-code/plugins/installed.json"); - std::fs::create_dir_all(installed_path.parent().unwrap()).unwrap(); - let original = br#"{"version":1,"plugins":[{"id":"tracedecay","enabled":false}]} -"#; - std::fs::write(&installed_path, original).unwrap(); - - let result = refresh_generated_plugins_at( - vec![Box::new( - tracedecay_agent_hosts::agents::kimi::KimiIntegration, - )], - home.path(), - "new-tracedecay", - ); - - assert!(result.is_ok()); - assert_eq!(std::fs::read(installed_path).unwrap(), original); - assert!( - !home - .path() - .join(".tracedecay/host-bundle-stage/kimi/tracedecay/.kimi-plugin/plugin.json") - .exists(), - "the component-set transaction owns the Kimi staging bundle" - ); - } - - /// Post-update writer ordering. Every host with a canonical component set - /// is written exclusively by the receipt-backed transaction; a second - /// writer running before the transaction reseals the receipt is exactly - /// what left Cursor Core's receipt stale on every version bump and made - /// Doctor report a component-ownership conflict. - /// - /// Zed, Antigravity, and Vibe joined the receipt-backed lifecycle - /// (`default_components` is non-empty for every stock host except the two - /// typed-unavailable kinds, which share an id with a supported host), so - /// the roster this refresh is handed in production is now canonical end to - /// end. Assert that over the production roster itself rather than a frozen - /// copy of it, and keep one negative case so the predicate still has to - /// discriminate instead of answering `true` for anything. - #[test] - fn canonical_component_set_hosts_are_not_refreshed_by_a_second_writer() { - for integration in tracedecay_agent_hosts::agents::all_integrations() { - assert!( - host_owns_canonical_component_set(integration.id()), - "{} owns a canonical component set", - integration.id() - ); - } - assert!( - !host_owns_canonical_component_set("not-a-stock-host"), - "an id that names no stock host cannot own a canonical component set" - ); - } - - /// Cursor's receipt-owned plugin bundle must not be rewritten outside the - /// component-set transaction: `.cursor-plugin/plugin.json` carries the - /// stamped manifest version and `hooks/hooks.json` bakes the resolved - /// binary path, so a refresh here guarantees byte drift from the receipt. - #[test] - fn cursor_plugin_bundle_is_left_to_the_component_set_transaction() { - let home = TempDir::new().unwrap(); - let manifest_path = home - .path() - .join(".cursor/plugins/local/tracedecay/.cursor-plugin/plugin.json"); - std::fs::create_dir_all(manifest_path.parent().unwrap()).unwrap(); - let receipt_owned = br#"{"name":"tracedecay","version":"0.0.0-receipt"}"#; - std::fs::write(&manifest_path, receipt_owned).unwrap(); - - let result = refresh_generated_plugins_at( - vec![Box::new(tracedecay_agent_hosts::agents::CursorIntegration)], - home.path(), - "new-tracedecay", - ); - - assert!(result.is_ok()); - assert_eq!(std::fs::read(&manifest_path).unwrap(), receipt_owned); - } - #[test] fn partition_collects_only_failed_ids_in_order() { match partition_reinstall_results(vec![ok("claude"), err("cursor"), err("copilot")]) { diff --git a/crates/tracedecay-cli/src/upgrade.rs b/crates/tracedecay-cli/src/upgrade.rs index da334e24d1..e9d2db8baa 100644 --- a/crates/tracedecay-cli/src/upgrade.rs +++ b/crates/tracedecay-cli/src/upgrade.rs @@ -1187,6 +1187,27 @@ mod tests { // assertion setup; production upgrade code above is kept panic-free. use super::*; + /// Writes an executable script without this process ever holding it open + /// for writing. Linux refuses `execve` with `ETXTBSY` while any process + /// holds the file writable, and a sibling test thread that forks while a + /// write descriptor is open carries a copy into its child until that child + /// execs. The single-threaded `sh` that writes it here has no sibling to + /// fork, and has exited before the script runs. + #[cfg(unix)] + fn write_executable_script(path: &Path, contents: &str) { + let status = Command::new("/bin/sh") + .args([ + "-c", + r#"printf '%s' "$1" > "$2" && chmod 755 "$2""#, + "sh", + contents, + ]) + .arg(path) + .status() + .unwrap(); + assert!(status.success(), "writing {}: {status}", path.display()); + } + #[test] fn checksum_manifest_selects_the_exact_release_asset() { let digest = "a".repeat(64); @@ -1246,7 +1267,6 @@ mod tests { #[cfg(unix)] mod version_probe { use std::fs; - use std::os::unix::fs::PermissionsExt; use std::path::{Path, PathBuf}; use std::time::{Duration, Instant}; @@ -1256,11 +1276,11 @@ mod tests { UpgradeOutcome, VersionProbeError, finish_versioned_upgrade, installed_binary_version, installed_binary_version_within, }; + use super::write_executable_script; fn script(dir: &Path, body: &str) -> PathBuf { let path = dir.join("tracedecay"); - fs::write(&path, format!("#!/bin/sh\n{body}\n")).unwrap(); - fs::set_permissions(&path, fs::Permissions::from_mode(0o755)).unwrap(); + write_executable_script(&path, &format!("#!/bin/sh\n{body}\n")); path } @@ -1527,11 +1547,10 @@ mod tests { #[cfg(unix)] mod delegation { use std::cell::Cell; - use std::fs; - use std::os::unix::fs::PermissionsExt; use std::path::PathBuf; use super::super::{ManagerCommand, PackageManager, UpgradeOutcome, run_delegated_upgrade}; + use super::write_executable_script; fn sh(script: &str) -> ManagerCommand { ManagerCommand::new("sh", &["-c", script]) @@ -1541,12 +1560,10 @@ mod tests { /// ` like the real `--version`. fn fake_binary(dir: &std::path::Path, version: &str) -> PathBuf { let path = dir.join("tracedecay"); - fs::write( + write_executable_script( &path, - format!("#!/bin/sh\nprintf 'tracedecay %s\\n' '{version}'\n"), - ) - .unwrap(); - fs::set_permissions(&path, fs::Permissions::from_mode(0o755)).unwrap(); + &format!("#!/bin/sh\nprintf 'tracedecay %s\\n' '{version}'\n"), + ); path } diff --git a/crates/tracedecay-cli/src/work_cli.rs b/crates/tracedecay-cli/src/work_cli.rs index 9385069ed9..7211ebc67a 100644 --- a/crates/tracedecay-cli/src/work_cli.rs +++ b/crates/tracedecay-cli/src/work_cli.rs @@ -29,10 +29,9 @@ use tracedecay_domain::UtcMicros; use tracedecay_domain::WorkDuplicateAdjudicationCommandV1; use tracedecay_tool_catalog::OperationId; +use tracedecay_contracts::now_micros; use tracedecay_contracts::request_identity::{GlobalRequestSurface, mint_global_request_id}; -use tracedecay_daemon_protocol::{ - DaemonInvocationDelivery, InvocationCancellationPolicy, invocation_now_micros, -}; +use tracedecay_daemon_protocol::{DaemonInvocationDelivery, InvocationCancellationPolicy}; use tracedecay_daemon_protocol::{ DaemonInvocationOutcome, DaemonInvocationRequest, WorkApplicationInvocationV1, WorkApplicationOutcomeV1, @@ -353,7 +352,7 @@ pub async fn invoke_work_cli_with_delivery( mint_global_request_id(GlobalRequestSurface::Cli).map_err(|_| TraceDecayError::Config { message: "could not allocate a Work CLI request id".to_owned(), })?; - let observed_at = invocation_now_micros(); + let observed_at = now_micros(); let deadline = Deadline::new(UtcMicros( observed_at.0.saturating_add(WORK_CLI_DEADLINE_MICROS), )) @@ -420,6 +419,9 @@ pub async fn invoke_work_cli_with_delivery( request_id: request_id.clone(), scope, outcome: erase_work_outcome(outcome)?, + touched_files: Vec::new(), + code_graph: None, + analytics: None, }) } DaemonInvocationOutcome::ApplicationProblem { problem } => Err( diff --git a/crates/tracedecay-cli/src/workflow_cli.rs b/crates/tracedecay-cli/src/workflow_cli.rs index 897c203436..8528d3854e 100644 --- a/crates/tracedecay-cli/src/workflow_cli.rs +++ b/crates/tracedecay-cli/src/workflow_cli.rs @@ -20,12 +20,13 @@ use tracedecay_contracts::{ use tracedecay_domain::UtcMicros; use tracedecay_tool_catalog::OperationId; +use tracedecay_contracts::now_micros; use tracedecay_contracts::request_identity::{GlobalRequestSurface, mint_global_request_id}; +use tracedecay_daemon_protocol::InvocationCancellationPolicy; use tracedecay_daemon_protocol::{ DaemonInvocationOutcome, DaemonInvocationRequest, WorkflowApplicationInvocation, WorkflowApplicationOutcome, }; -use tracedecay_daemon_protocol::{InvocationCancellationPolicy, invocation_now_micros}; use tracedecay_domain::errors::{Result, TraceDecayError}; use crate::application_cli::{WORKFLOW, config_error}; @@ -185,7 +186,7 @@ pub async fn invoke_workflow_cli( mint_global_request_id(GlobalRequestSurface::Cli).map_err(|_| TraceDecayError::Config { message: "could not allocate a Workflow CLI request id".to_owned(), })?; - let observed_at = invocation_now_micros(); + let observed_at = now_micros(); let deadline = deadline_from_maximum_millis(maximum_millis, observed_at)?; let cancellation = CancellationSignal::active(format!("cancellation.cli.{}", request_id.as_str())) @@ -235,6 +236,9 @@ pub async fn invoke_workflow_cli( request_id, scope, outcome: erase_workflow_outcome(outcome)?, + touched_files: Vec::new(), + code_graph: None, + analytics: None, })) } DaemonInvocationOutcome::ApplicationProblem { problem } => Ok(Err( diff --git a/crates/tracedecay-cli/tests/core_cli_suite/build_version_test.rs b/crates/tracedecay-cli/tests/core_cli_suite/build_version_test.rs index c348a68773..67d14a5c78 100644 --- a/crates/tracedecay-cli/tests/core_cli_suite/build_version_test.rs +++ b/crates/tracedecay-cli/tests/core_cli_suite/build_version_test.rs @@ -11,7 +11,7 @@ use std::path::Path; use std::process::Command; -use tracedecay::version::PACKAGE_VERSION; +use tracedecay_project::version::PACKAGE_VERSION; /// The version `tracedecay --version` reports, with the `tracedecay ` prefix /// clap prints stripped off. diff --git a/crates/tracedecay-cli/tests/core_cli_suite/cli_boundary.rs b/crates/tracedecay-cli/tests/core_cli_suite/cli_boundary.rs index 827cdaece2..41a2e6c5d0 100644 --- a/crates/tracedecay-cli/tests/core_cli_suite/cli_boundary.rs +++ b/crates/tracedecay-cli/tests/core_cli_suite/cli_boundary.rs @@ -69,8 +69,22 @@ fn compiled_hotpath_profiles_without_a_runtime_environment_gate() { .expect("run feature-on profiling binary"); assert!(output.status.success(), "{output:?}"); - let bytes = std::fs::read(&report).expect("feature-on binary must write a report"); - assert!(!bytes.is_empty(), "Hotpath report must not be empty"); + let text = std::fs::read_to_string(&report).expect("feature-on binary must write a report"); + let report: serde_json::Value = serde_json::from_str(&text) + .unwrap_or_else(|error| panic!("Hotpath report is not JSON: {error}\n{text}")); + let measured = report + .pointer("/functions_timing/data") + .and_then(serde_json::Value::as_array) + .unwrap_or_else(|| panic!("report has no functions_timing data: {report}")) + .iter() + .filter_map(|entry| entry.get("name").and_then(serde_json::Value::as_str)) + .collect::>(); + assert!( + measured + .iter() + .any(|name| name.contains("cli.hotpath.install_shutdown_finalizer")), + "process guard installation must be measured: {measured:?}" + ); } #[cfg(feature = "hotpath")] diff --git a/crates/tracedecay-cli/tests/core_cli_suite/cli_non_interactive_test.rs b/crates/tracedecay-cli/tests/core_cli_suite/cli_non_interactive_test.rs index bca627ffff..235f53489f 100644 --- a/crates/tracedecay-cli/tests/core_cli_suite/cli_non_interactive_test.rs +++ b/crates/tracedecay-cli/tests/core_cli_suite/cli_non_interactive_test.rs @@ -11,18 +11,17 @@ use crate::provision_host_cli_fixture; #[cfg(unix)] use std::os::unix::fs::PermissionsExt; use tempfile::TempDir; -use tracedecay::test_support::host_admission::HostAdmissionTestRuntimeV1; use tracedecay_agent_hosts::PRODUCT_VERSION; use tracedecay_automation_runtime::automation::run_ledger::{ AutomationRunArtifactKind, AutomationRunLedgerRecord, append_run_record, write_run_artifact, }; use tracedecay_domain::ProjectId; use tracedecay_global_db::StoreInstanceUpsert; +use tracedecay_project::test_support::host_admission::HostAdmissionTestRuntimeV1; use tracedecay_runtime_core::branch_meta::BranchMeta; use tracedecay_runtime_core::storage::{ - EnrollmentMarker, STORE_MANIFEST_FILENAME, STORE_MANIFEST_SCHEMA_VERSION, StorageMode, - StoreKind, StoreManifest, default_profile_project_id, profile_sharded_data_root, - profile_sharded_layout, write_repository_identity_marker, write_store_manifest, + STORE_MANIFEST_FILENAME, STORE_MANIFEST_SCHEMA_VERSION, StorageMode, StoreKind, StoreManifest, + default_profile_project_id, profile_sharded_data_root, write_repository_identity_marker, }; use tracedecay_sessions::admission::HostAdmissionScope; @@ -532,7 +531,7 @@ fn write_profile_sharded_fixture(home: &std::path::Path, project: &std::path::Pa // mount; a non-empty wrong-shape file trips the workflow persisted-shape // gate and is refused as reset-required. write_empty_sqlite_fixture(&shard_root.join("sessions.db")); - write_branch_meta(&shard_root, &[], false); + write_branch_meta(&shard_root, &[]); let manifest = StoreManifest { schema_version: STORE_MANIFEST_SCHEMA_VERSION, project_id: Some("proj_cli".to_string()), @@ -591,18 +590,10 @@ async fn register_profile_sharded_store( .expect("store instance should upsert"); } -fn write_branch_meta( - shard_root: &std::path::Path, - tracked_branches: &[(&str, &str)], - create_branch_dbs: bool, -) { +fn write_branch_meta(shard_root: &std::path::Path, tracked_branches: &[&str]) { let mut meta = BranchMeta::new_for_dir(shard_root, "main"); - for (name, rel_db_path) in tracked_branches { - meta.add_branch(name, rel_db_path, "main"); - if create_branch_dbs { - let db_path = shard_root.join(rel_db_path); - write_empty_sqlite_fixture(&db_path); - } + for name in tracked_branches { + meta.add_branch(name, "main"); } std::fs::write( shard_root.join("branch-meta.json"), @@ -747,6 +738,40 @@ fn explicit_kimi_install_fails_with_interactive_remediation() { assert!(!kimi_home.join("plugins/installed.json").exists()); } +#[test] +fn detected_install_continues_past_a_failing_host_and_reports_it() { + let home = TempDir::new().unwrap(); + let project = TempDir::new().unwrap(); + let home_path = canonical_temp_path(home.path()); + let kimi_home = home_path.join(".kimi-code"); + std::fs::create_dir_all(&kimi_home).unwrap(); + std::fs::create_dir_all(home_path.join(".vibe")).unwrap(); + let mut install = tracedecay_command_without_daemon(home.path(), project.path()); + let _shim = add_tracedecay_path_shim(&mut install, home.path()); + install + .env( + tracedecay_agent_hosts::agents::kimi::KIMI_CODE_HOME_ENV, + &kimi_home, + ) + .arg("install"); + + let output = run_with_timeout(install, cli_timeout()); + + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + !output.status.success(), + "a failed host must fail the pass\nstderr:\n{stderr}" + ); + assert!( + stderr.contains("agent install failed for: kimi"), + "{stderr}" + ); + assert!( + home_path.join(".vibe/config.toml").is_file(), + "Vibe is detected after Kimi and must still be installed\nstderr:\n{stderr}" + ); +} + #[test] fn install_without_any_detected_agent_succeeds_with_a_notice() { let home = TempDir::new().unwrap(); @@ -1008,42 +1033,6 @@ fn automation_config_enable_writes_canonical_project_setting_noninteractively() ); } -#[test] -fn automation_config_rejects_retired_global_scope() { - let home = TempDir::new().unwrap(); - let project = TempDir::new().unwrap(); - std::fs::create_dir_all(project.path()).unwrap(); - - let mut set = tracedecay_command(home.path(), project.path()); - set.args([ - "automation", - "config", - "set", - "--scope", - "global", - "--backend", - "codex-app-server", - "--timeout-secs", - "75", - "--session-reflector", - "true", - "--session-reflector-schedule", - "interval", - "--session-reflector-interval-secs", - "1800", - ]); - let output = run_with_timeout(set, cli_timeout()); - assert!( - !output.status.success(), - "automation config global set should be rejected\nstdout:\n{}\nstderr:\n{}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ); - assert!( - String::from_utf8_lossy(&output.stderr).contains("automation settings are project-scoped") - ); -} - #[test] fn automation_config_set_rejects_unimplemented_external_backend() { let home = TempDir::new().unwrap(); @@ -1452,69 +1441,6 @@ fn status_reports_uninitialized_project_without_creating_it() { ); } -#[tokio::test] -async fn status_surfaces_split_identity_conflict_without_suggesting_init() { - let home = TempDir::new().unwrap(); - let project = TempDir::new().unwrap(); - let project_root = canonical_temp_path(project.path()); - git(&project_root, &["init", "-b", "main"]); - - for project_id in ["proj_status_selected", "proj_status_legacy"] { - let layout = profile_sharded_layout( - &project_root, - &profile_root(home.path()), - &EnrollmentMarker { - project_id: project_id.to_string(), - storage_mode: StorageMode::ProfileSharded, - }, - ) - .unwrap(); - let (db, _) = crate::common::initialize_test_database(&layout.graph_db_path) - .await - .unwrap(); - db.checkpoint().await.unwrap(); - db.close(); - write_store_manifest(&layout).unwrap(); - } - // Only the repository identity marker, deliberately: an enrollment marker - // is a current-generation authority that resolves this checkout outright, - // short-circuiting the legacy-candidate scan that detects the split. A - // cutover conflict can only exist on a pre-enrollment checkout, so writing - // one here would model a state in which the conflict cannot arise. - write_repository_identity_marker(&project_root, "proj_status_selected").unwrap(); - let runtime = HostAdmissionTestRuntimeV1::profile(profile_root(home.path())) - .await - .unwrap(); - register_profile_sharded_store(&runtime, &project_root, "proj_status_selected").await; - runtime.checkpoint_profile_database_for_test().await; - drop(runtime); - - let selected_db = profile_root(home.path()).join("projects/proj_status_selected/tracedecay.db"); - let legacy_db = profile_root(home.path()).join("projects/proj_status_legacy/tracedecay.db"); - let selected_before = std::fs::read(&selected_db).unwrap(); - let legacy_before = std::fs::read(&legacy_db).unwrap(); - - let mut command = tracedecay_command(home.path(), &project_root); - command.arg("status"); - let output = run_with_timeout(command, cli_timeout()); - let stderr = String::from_utf8_lossy(&output.stderr); - - assert!( - !output.status.success(), - "status should fail safely\n{stderr}" - ); - assert!(stderr.contains("identity cutover conflict"), "{stderr}"); - assert!(stderr.contains("proj_status_selected"), "{stderr}"); - assert!(stderr.contains("proj_status_legacy"), "{stderr}"); - assert!( - stderr.contains("choose one shard and retire the other"), - "{stderr}" - ); - assert!(!stderr.contains("run `tracedecay init`"), "{stderr}"); - assert_eq!(std::fs::read(selected_db).unwrap(), selected_before); - assert_eq!(std::fs::read(legacy_db).unwrap(), legacy_before); -} - #[tokio::test] async fn list_all_reports_profile_sharded_store_without_stale_label() { let home = TempDir::new().unwrap(); @@ -1898,14 +1824,6 @@ fn wipe_local_returns_failure_when_a_selected_store_cannot_be_deleted() { let blocked = data_root.join("blocked"); std::fs::create_dir_all(&blocked).unwrap(); std::fs::write(blocked.join("retry-authority"), b"preserve").unwrap(); - let marker = EnrollmentMarker { - project_id: "proj_cli".to_string(), - storage_mode: StorageMode::ProfileSharded, - }; - let marker_path = - tracedecay_runtime_core::storage::legacy_enrollment_marker_path(project.path()); - std::fs::create_dir_all(marker_path.parent().unwrap()).unwrap(); - std::fs::write(&marker_path, serde_json::to_vec(&marker).unwrap()).unwrap(); std::fs::set_permissions(&blocked, std::fs::Permissions::from_mode(0o000)).unwrap(); let mut command = tracedecay_command_without_daemon(home.path(), project.path()); @@ -1985,7 +1903,7 @@ fn list_all_reports_orphan_manifest_reconstructable_store() { let report = tracedecay_global_db::registry_maintenance::inspect_profile_store_orphans( &profile_root(home.path()), - tracedecay::project::current_timestamp(), + tracedecay_runtime_core::tracedecay::current_timestamp(), ); assert_eq!(report.plans.len(), 1, "{report:#?}"); assert_eq!( @@ -2113,19 +2031,6 @@ fn write_wedged_generation_debris(shard_root: &Path) { std::fs::write(wal.join("segment"), b"graph wal segment").unwrap(); } -/// Plants the repo-local enrollment marker that makes the profile shard a -/// local wipe target, exactly as -/// `wipe_local_returns_failure_when_a_selected_store_cannot_be_deleted` does. -fn write_profile_sharded_enrollment_marker(project: &Path) { - let marker = EnrollmentMarker { - project_id: "proj_cli".to_string(), - storage_mode: StorageMode::ProfileSharded, - }; - let marker_path = tracedecay_runtime_core::storage::legacy_enrollment_marker_path(project); - std::fs::create_dir_all(marker_path.parent().unwrap()).unwrap(); - std::fs::write(&marker_path, serde_json::to_vec(&marker).unwrap()).unwrap(); -} - /// The #765 operator journey: the managed daemon holds its lifetime shared /// lease and is wedged in a terminal activation retry loop, so it never /// exits. Without an installed service to stop, the holder never releases. @@ -2136,7 +2041,6 @@ fn wipe_refuses_within_bound_when_profile_lease_never_releases() { let home = TempDir::new().unwrap(); let project = TempDir::new().unwrap(); write_profile_sharded_fixture(home.path(), project.path()); - write_profile_sharded_enrollment_marker(project.path()); let shard_root = profile_shard_root(home.path()); write_wedged_generation_debris(&shard_root); let profile = profile_root(home.path()); @@ -2184,7 +2088,6 @@ fn wipe_completes_within_bound_once_the_wedged_holder_stops() { let home = TempDir::new().unwrap(); let project = TempDir::new().unwrap(); write_profile_sharded_fixture(home.path(), project.path()); - write_profile_sharded_enrollment_marker(project.path()); let shard_root = profile_shard_root(home.path()); write_wedged_generation_debris(&shard_root); let profile = profile_root(home.path()); @@ -2377,25 +2280,17 @@ async fn branch_list_reads_profile_sharded_branch_meta() { let tracked_branches = (0..300) .map(|index| { - ( - format!("feature/branch-{index:03}-with-enough-detail-to-exercise-status-bounds"), - format!("branches/feature_branch_{index:03}.db"), - ) + format!("feature/branch-{index:03}-with-enough-detail-to-exercise-status-bounds") }) .collect::>(); - for (name, _) in &tracked_branches { + for name in &tracked_branches { git(project.path(), &["branch", name]); } let tracked_branch_refs = tracked_branches .iter() - .map(|(name, path)| (name.as_str(), path.as_str())) + .map(String::as_str) .collect::>(); - write_branch_meta(&shard_root, &tracked_branch_refs, false); - for (_, path) in &tracked_branches { - let db_path = shard_root.join(path); - std::fs::create_dir_all(db_path.parent().unwrap()).unwrap(); - std::fs::write(db_path, b"branch fixture").unwrap(); - } + write_branch_meta(&shard_root, &tracked_branch_refs); let mut command = tracedecay_command_without_daemon(home.path(), project.path()); command.args(["branch", "list"]); @@ -2596,11 +2491,6 @@ fn branch_add_admits_background_publication_and_remove_retires_its_exact_artifac .branches .get("feature/new") .expect("branch add must track the branch"); - assert!( - entry.served_by_project_store(), - "tracked branch must be served by the single project store, found '{}'", - entry.db_file - ); let source = entry .graph_source .as_ref() @@ -2653,10 +2543,6 @@ fn branch_add_admits_background_publication_and_remove_retires_its_exact_artifac String::from_utf8_lossy(&sealed_head.stdout).trim(), "the stored OID must belong to the recorded source worktree" ); - assert!( - !shard_root.join("branches").exists(), - "branch add must not create a per-branch database" - ); git( &project_root, @@ -2842,7 +2728,7 @@ fn branch_search_serves_a_committed_generation_behind_dirty_worktree_state() { } #[tokio::test] -async fn branch_remove_deletes_branch_local_memory_without_cutover_receipt() { +async fn branch_removeall_retires_every_tracked_branch() { let home = TempDir::new().unwrap(); let project = TempDir::new().unwrap(); write_git_fixture(project.path()); @@ -2856,59 +2742,7 @@ async fn branch_remove_deletes_branch_local_memory_without_cutover_receipt() { drop(runtime); seed_canonical_configuration(home.path(), project.path()); let shard_root = profile_shard_root(home.path()); - write_branch_meta( - &shard_root, - &[("feature/legacy-memory", "branches/feature_legacy_memory.db")], - true, - ); - let branch_db = shard_root.join("branches/feature_legacy_memory.db"); - rusqlite::Connection::open(&branch_db) - .unwrap() - .execute_batch( - "CREATE TABLE memory_facts (fact_id TEXT PRIMARY KEY); - INSERT INTO memory_facts (fact_id) VALUES ('branch-local');", - ) - .unwrap(); - - let mut command = tracedecay_command(home.path(), project.path()); - command.args(["branch", "remove", "feature/legacy-memory"]); - let output = run_with_timeout(command, cli_timeout()); - - assert!( - output.status.success(), - "branch remove should not require a migration receipt\nstdout:\n{}\nstderr:\n{}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ); - assert!( - !branch_db.exists(), - "branch remove should delete obsolete branch-local memory with its branch database" - ); -} - -#[tokio::test] -async fn branch_removeall_deletes_profile_shard_branch_dbs() { - let home = TempDir::new().unwrap(); - let project = TempDir::new().unwrap(); - write_git_fixture(project.path()); - write_profile_sharded_fixture(home.path(), project.path()); - write_repository_identity_marker(project.path(), "proj_cli").unwrap(); - let runtime = HostAdmissionTestRuntimeV1::profile(profile_root(home.path())) - .await - .unwrap(); - register_profile_sharded_store(&runtime, project.path(), "proj_cli").await; - runtime.checkpoint_profile_database_for_test().await; - drop(runtime); - seed_canonical_configuration(home.path(), project.path()); - let shard_root = profile_shard_root(home.path()); - write_branch_meta( - &shard_root, - &[ - ("feature/one", "branches/feature_one.db"), - ("feature/two", "branches/feature_two.db"), - ], - true, - ); + write_branch_meta(&shard_root, &["feature/one", "feature/two"]); let mut command = tracedecay_command(home.path(), project.path()); command.args(["branch", "removeall"]); @@ -2920,11 +2754,13 @@ async fn branch_removeall_deletes_profile_shard_branch_dbs() { String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr) ); - assert!( - !shard_root.join("branches/feature_one.db").exists() - && !shard_root.join("branches/feature_two.db").exists(), - "branch removeall should delete all non-default branch DBs from profile shard" + let meta = tracedecay_runtime_core::branch_meta::load_branch_meta(&shard_root).unwrap(); + assert_eq!( + meta.branches.keys().collect::>(), + vec!["main"], + "branch removeall should retire every non-default tracked branch" ); + assert!(shard_root.join("tracedecay.db").exists()); } #[tokio::test] @@ -2942,11 +2778,7 @@ async fn branch_gc_preserves_profile_shard_without_repository_evidence() { drop(runtime); seed_canonical_configuration(home.path(), project.path()); let shard_root = profile_shard_root(home.path()); - write_branch_meta( - &shard_root, - &[("feature/stale", "branches/feature_stale.db")], - true, - ); + write_branch_meta(&shard_root, &["feature/stale"]); let mut command = tracedecay_command(home.path(), project.path()); command.args(["branch", "gc"]); @@ -2959,7 +2791,9 @@ async fn branch_gc_preserves_profile_shard_without_repository_evidence() { String::from_utf8_lossy(&output.stderr) ); assert!( - shard_root.join("branches/feature_stale.db").exists(), + tracedecay_runtime_core::branch_meta::load_branch_meta(&shard_root) + .unwrap() + .is_tracked("feature/stale"), "branch gc must fail closed without repository branch evidence" ); } diff --git a/crates/tracedecay-cli/tests/core_cli_suite/config_test.rs b/crates/tracedecay-cli/tests/core_cli_suite/config_test.rs index 10da5001ae..c941adfb3a 100644 --- a/crates/tracedecay-cli/tests/core_cli_suite/config_test.rs +++ b/crates/tracedecay-cli/tests/core_cli_suite/config_test.rs @@ -1,76 +1,5 @@ use tempfile::TempDir; -use tracedecay_configuration::{ - TraceDecayConfig, get_config_path, is_excluded, is_excluded_dir, is_in_gitignore, load_config, -}; - -#[test] -fn legacy_config_fixture_load_does_not_rewrite_input() { - let dir = TempDir::new().unwrap(); - let config = TraceDecayConfig::default(); - let config_path = get_config_path(dir.path()); - std::fs::create_dir_all(config_path.parent().unwrap()).unwrap(); - let source = serde_json::to_string_pretty(&config).unwrap(); - std::fs::write(&config_path, &source).unwrap(); - let loaded = load_config(dir.path()).unwrap(); - assert_eq!(config.version, loaded.version); - assert_eq!(config.exclude, loaded.exclude); - assert_eq!(std::fs::read_to_string(config_path).unwrap(), source); -} - -#[test] -fn test_is_excluded() { - let config = TraceDecayConfig::default(); - assert!(!is_excluded("src/main.rs", &config)); - assert!(is_excluded("target/debug/foo", &config)); - assert!(is_excluded("node_modules/foo.rs", &config)); - assert!(is_excluded("build/classes/App.class", &config)); - assert!(is_excluded("packages/web/dist/main.js", &config)); - assert!(is_excluded("packages/web/coverage/lcov.js", &config)); - assert!(is_excluded("packages/web/.next/server/app.js", &config)); - assert!(is_excluded("tools/.cache/generated.py", &config)); -} - -#[test] -fn default_generated_excludes_prune_nested_dirs() { - let config = TraceDecayConfig::default(); - for path in [ - "packages/web/dist", - "packages/web/coverage", - "packages/web/.next", - "packages/web/.turbo", - "tools/.cache", - "backend/.venv", - "backend/__pycache__", - ] { - assert!( - is_excluded_dir(path, &config), - "expected default excludes to prune {path}" - ); - } -} - -#[test] -fn test_legacy_config_with_include_field_still_loads() { - let dir = TempDir::new().unwrap(); - let tracedecay_dir = dir.path().join(".tracedecay"); - std::fs::create_dir_all(&tracedecay_dir).unwrap(); - // Simulate an old config that still has an "include" field - let legacy_json = r#"{ - "version": 1, - "root_dir": ".", - "include": ["**/*.rs"], - "exclude": ["target/**", ".git/**", ".tracedecay/**"], - "max_file_size": 1048576, - "extract_docstrings": true, - "track_call_sites": true, - "enable_embeddings": false - }"#; - std::fs::write(tracedecay_dir.join("config.json"), legacy_json).unwrap(); - let loaded = load_config(dir.path()).unwrap(); - assert_eq!(loaded.version, 1); - assert!(loaded.exclude.contains(&"target/**".to_string())); - assert!(loaded.git_ignore); -} +use tracedecay_configuration::is_in_gitignore; // ── is_in_gitignore ───────────────────────────────────────────────────────── @@ -113,18 +42,18 @@ fn test_is_in_gitignore_no_file() { fn test_discover_project_root_finds_parent() { let dir = tempfile::TempDir::new().unwrap(); let root = dir.path(); - std::fs::create_dir_all(root.join(".tracedecay")).unwrap(); - std::fs::write(root.join(".tracedecay/tracedecay.db"), b"fake").unwrap(); + tracedecay_runtime_core::storage::pin_fixture_repository_identity(root, "proj_discover_parent") + .unwrap(); let child = root.join("src/mcp"); std::fs::create_dir_all(&child).unwrap(); - let found = tracedecay::config::discover_project_root(&child); + let found = tracedecay_project::config::discover_project_root(&child); assert_eq!(found, Some(root.to_path_buf())); } #[test] fn test_discover_project_root_returns_none() { let dir = tempfile::TempDir::new().unwrap(); - let found = tracedecay::config::discover_project_root(dir.path()); + let found = tracedecay_project::config::discover_project_root(dir.path()); assert!(found.is_none()); } diff --git a/crates/tracedecay-cli/tests/core_cli_suite/gain_test.rs b/crates/tracedecay-cli/tests/core_cli_suite/gain_test.rs index dc2044005e..1c66e25e7f 100644 --- a/crates/tracedecay-cli/tests/core_cli_suite/gain_test.rs +++ b/crates/tracedecay-cli/tests/core_cli_suite/gain_test.rs @@ -1,7 +1,7 @@ use std::fs; use tempfile::TempDir; -use tracedecay::test_support::host_admission::HostAdmissionTestRuntimeV1; +use tracedecay_project::test_support::host_admission::HostAdmissionTestRuntimeV1; async fn open_isolated_runtime(tmp: &TempDir) -> HostAdmissionTestRuntimeV1 { HostAdmissionTestRuntimeV1::profile(tmp.path().join(".tracedecay")) diff --git a/crates/tracedecay-cli/tests/core_cli_suite/main.rs b/crates/tracedecay-cli/tests/core_cli_suite/main.rs index 3b4210b9ce..e453d472c0 100644 --- a/crates/tracedecay-cli/tests/core_cli_suite/main.rs +++ b/crates/tracedecay-cli/tests/core_cli_suite/main.rs @@ -22,8 +22,6 @@ mod gain_test; mod monitor_test; // Mounted for cli_non_interactive_test, which exercises the compiled // host-CLI fixture provisioner. -#[cfg(unix)] -mod observation_reset_recovery_test; #[path = "../../build-support/provision_host_cli_fixture.rs"] mod provision_host_cli_fixture; mod source_provenance_test; diff --git a/crates/tracedecay-cli/tests/core_cli_suite/observation_reset_recovery_test.rs b/crates/tracedecay-cli/tests/core_cli_suite/observation_reset_recovery_test.rs deleted file mode 100644 index 12132e8ff5..0000000000 --- a/crates/tracedecay-cli/tests/core_cli_suite/observation_reset_recovery_test.rs +++ /dev/null @@ -1,538 +0,0 @@ -//! Retained-store recovery through the supported observation-authority reset: -//! ingest → refuse → `storage reset-authority observations` → reopen → -//! converge → the same session is describable and searchable again with the -//! LCM content it had before the reset, and the doctor names where the -//! re-derivation stands on the way there and reports complete/current after. - -use std::path::{Path, PathBuf}; -use std::time::{Duration, Instant}; - -use serde_json::{Value, json}; -use tempfile::TempDir; -use tracedecay_global_db::observation::OBSERVATION_NATIVE_SOURCE_SCHEME_MIGRATION; - -use crate::common::{ - canonical_existing_path, git_program, initialize_tracedecay_cli_project, - spawn_tracedecay_daemon_with, stop_managed_daemon, tracedecay_command_with_home, -}; - -/// How long the daemon may take to re-derive the reset store from the -/// preserved transcripts (open the project, re-admit every rollout, run the -/// temporal refresh) before the journey is judged broken. -const CONVERGENCE_TIMEOUT: Duration = Duration::from_secs(120); - -/// Filler Codex rollouts beside the session under test, so the reopened -/// authority re-derives a corpus rather than one transcript and search has to -/// select the recovered session out of it. -const ROLLOUT_COUNT: usize = 24; - -const SESSION_ID: &str = "codex-reset-recovery-session"; -const NEEDLE: &str = "orchard billing pipeline regression"; - -fn git(project: &Path, args: &[&str]) { - let output = std::process::Command::new(git_program()) - .args(args) - .current_dir(project) - .output() - .unwrap_or_else(|error| panic!("git {args:?} should run: {error}")); - assert!( - output.status.success(), - "git {args:?} failed\nstdout:\n{}\nstderr:\n{}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ); -} - -fn seed_committed_project(project: &Path) { - std::fs::create_dir_all(project.join("src")).unwrap(); - std::fs::write( - project.join("src/lib.rs"), - "pub fn billing_pipeline() -> u32 { 42 }\n", - ) - .unwrap(); - git(project, &["init", "-b", "main"]); - git(project, &["add", "."]); - git( - project, - &[ - "-c", - "user.name=TraceDecay Tests", - "-c", - "user.email=tests@tracedecay.local", - "commit", - "-m", - "seed reset recovery fixture", - ], - ); -} - -fn codex_rollout_path(home: &Path, session: &str) -> PathBuf { - home.join(".codex/sessions/2026/01/01") - .join(format!("rollout-2026-01-01T00-00-00-{session}.jsonl")) -} - -fn codex_rollout_contents(project: &Path, session: &str, message: &str) -> String { - format!( - "{}\n{}\n{}\n", - json!({ - "timestamp": "2026-01-01T00:00:00.000Z", - "type": "session_meta", - "payload": {"id": session, "cwd": project.to_string_lossy(), "model": "gpt-5.5"} - }), - json!({ - "timestamp": "2026-01-01T00:00:01.000Z", - "type": "event_msg", - "payload": {"type": "user_message", "message": format!("Investigate the {message}")} - }), - json!({ - "timestamp": "2026-01-01T00:00:02.000Z", - "type": "event_msg", - "payload": { - "type": "agent_message", - "message": format!("The {message} is fixed by the retained change.") - } - }), - ) -} - -fn write_codex_rollouts(home: &Path, project: &Path) { - let path = codex_rollout_path(home, SESSION_ID); - std::fs::create_dir_all(path.parent().unwrap()).unwrap(); - std::fs::write(&path, codex_rollout_contents(project, SESSION_ID, NEEDLE)).unwrap(); - for index in 0..ROLLOUT_COUNT { - let session = format!("codex-reset-recovery-filler-{index:02}"); - std::fs::write( - codex_rollout_path(home, &session), - codex_rollout_contents(project, &session, &format!("filler topic {index}")), - ) - .unwrap(); - } -} - -/// Replaces the target rollout with byte-identical content under a new file -/// identity, what a restore, a copy, or a host rewrite does to a transcript. -/// Every observation the file yields now carries a new source generation, so -/// its anchors no longer verify against the ones the pre-reset admission wrote. -fn replace_codex_rollout_identity(home: &Path) { - let path = codex_rollout_path(home, SESSION_ID); - let contents = std::fs::read(&path).unwrap(); - let staged = path.with_extension("jsonl.replaced"); - std::fs::write(&staged, contents).unwrap(); - std::fs::rename(&staged, &path).unwrap(); -} - -fn project_sessions_db(home: &Path) -> PathBuf { - let projects = home.join(".tracedecay/projects"); - let mut stores = std::fs::read_dir(&projects) - .unwrap_or_else(|error| panic!("{} should list: {error}", projects.display())) - .map(|entry| entry.unwrap().path().join("sessions.db")) - .filter(|path| path.is_file()) - .collect::>(); - assert_eq!( - stores.len(), - 1, - "exactly one project sessions store is expected under {}: {stores:?}", - projects.display() - ); - stores.remove(0) -} - -/// Turns a healthy store into the exact shape admission refuses with the -/// typed `ResetRequired` state: rows written under the superseded Cline-like -/// native-source scheme, with no enrollment marker. -fn make_observation_authority_refused(db: &Path) { - let connection = rusqlite::Connection::open(db).unwrap(); - connection - .pragma_update(None, "foreign_keys", false) - .unwrap(); - connection - .execute( - "INSERT INTO source_cursors(source_json, scope_json, cursor_json) - VALUES ('{\"provider\":\"cline\",\"session_id\":\"cline.refused\"}', - '{\"kind\":\"profile\"}', '{}')", - [], - ) - .unwrap(); - connection - .execute( - "DELETE FROM global_schema_migrations WHERE migration = ?1", - [OBSERVATION_NATIVE_SOURCE_SCHEME_MIGRATION], - ) - .unwrap(); -} - -fn count(db: &Path, table: &str) -> i64 { - rusqlite::Connection::open(db) - .unwrap() - .query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |row| { - row.get(0) - }) - .unwrap() -} - -/// Native-source scheduling cursors (host coverage verdicts, discovery -/// frontiers, the Codex corpus epoch) that tell the next pass what it may skip. -fn scheduling_cursor_count(db: &Path) -> i64 { - rusqlite::Connection::open(db) - .unwrap() - .query_row( - "SELECT COUNT(*) FROM parse_offsets WHERE file_path NOT LIKE 'hook_analytics:%'", - [], - |row| row.get(0), - ) - .unwrap() -} - -/// Runs one tool through the shipped CLI and returns the retained evidence or -/// problem envelope the tool answered with. -fn tool_envelope(home: &Path, project: &Path, tool: &str, args: Value) -> Value { - let mut args = args; - args["format"] = json!("json"); - let output = tracedecay_command_with_home(home) - .current_dir(project) - .args([ - "tool", - "--project", - &project.to_string_lossy(), - tool, - "--json", - "--args", - &args.to_string(), - ]) - .output() - .unwrap_or_else(|error| panic!("tracedecay tool {tool} should run: {error}")); - let stdout = String::from_utf8_lossy(&output.stdout); - let stderr = String::from_utf8_lossy(&output.stderr); - let wire: Value = serde_json::from_str(stdout.trim()).unwrap_or_else(|error| { - panic!("{tool} must answer JSON: {error}\nstdout:\n{stdout}\nstderr:\n{stderr}") - }); - wire["content"] - .as_array() - .and_then(|items| { - items - .iter() - .find_map(|item| serde_json::from_str::(item["text"].as_str()?).ok()) - }) - .unwrap_or_else(|| panic!("{tool} must carry a JSON envelope: {wire}")) -} - -fn describe(home: &Path, project: &Path) -> Value { - tool_envelope( - home, - project, - "tracedecay_lcm_describe", - json!({"provider": "codex", "session_id": SESSION_ID}), - ) -} - -fn doctor(home: &Path, project: &Path) -> Value { - tool_envelope(home, project, "tracedecay_lcm_doctor", json!({})) -} - -fn message_search(home: &Path, project: &Path, catch_up: bool) -> Value { - tool_envelope( - home, - project, - "tracedecay_message_search", - json!({"query": NEEDLE, "provider": "codex", "limit": 5, "catch_up": catch_up}), - ) -} - -fn is_evidence(envelope: &Value) -> bool { - envelope.pointer("/outcome/outcome").and_then(Value::as_str) == Some("evidence") -} - -fn payload(envelope: &Value) -> &Value { - envelope - .pointer("/outcome/value/payload") - .unwrap_or_else(|| panic!("evidence envelope must carry a payload: {envelope}")) -} - -fn problem_message(envelope: &Value) -> &str { - envelope - .pointer("/problem/message") - .and_then(Value::as_str) - .unwrap_or_else(|| panic!("a non-evidence envelope must carry a problem: {envelope}")) -} - -fn served_generation(envelope: &Value) -> Option { - envelope - .pointer("/outcome/value/payload/temporal/watermarks/generation") - .and_then(Value::as_u64) - .filter(|generation| *generation > 0) -} - -/// Polls describe until the session serves from an active temporal -/// generation. A complete description at generation zero is the window -/// between the projection drain and the temporal refresh, not evidence. -fn wait_for_described_session(home: &Path, project: &Path, phase: &str) -> Value { - let deadline = Instant::now() + CONVERGENCE_TIMEOUT; - loop { - let envelope = describe(home, project); - if is_evidence(&envelope) && served_generation(&envelope).is_some() { - let payload = payload(&envelope); - assert_eq!(payload["status"], "ok", "{phase}: {payload}"); - assert_eq!(payload["description"]["session_id"], SESSION_ID, "{phase}"); - assert_eq!( - payload["description"]["raw_message_count"], 2, - "{phase}: {payload}" - ); - return envelope; - } - assert!( - Instant::now() < deadline, - "{phase}: the session never became describable; last answer: {envelope}" - ); - std::thread::sleep(Duration::from_millis(250)); - } -} - -/// A search without the freshness precondition serves the retained projection -/// and must answer evidence at once. The deprecated `catch_up` flag now only -/// requires fresh data: while projection catches up it may return a typed -/// stale result or a historical-convergence refusal, and must settle to the -/// retained session within the convergence bound. -fn assert_search_hits(home: &Path, project: &Path, catch_up: bool, phase: &str) { - let deadline = Instant::now() + CONVERGENCE_TIMEOUT; - let envelope = loop { - let envelope = message_search(home, project, catch_up); - if is_evidence(&envelope) { - let payload = payload(&envelope); - let has_session = payload["results"].as_array().is_some_and(|hits| { - hits.iter() - .any(|hit| hit["session"]["session_id"] == SESSION_ID) - }); - let has_generation = payload["temporal"]["watermarks"]["generation"] - .as_u64() - .is_some_and(|generation| generation > 0); - if has_session && has_generation { - break envelope; - } - assert!( - catch_up, - "{phase}: message_search must answer retained evidence: {envelope}" - ); - assert_eq!(payload["status"], "stale", "{phase}: {payload}"); - assert_eq!(payload["outcome"], "stale", "{phase}: {payload}"); - assert_eq!(payload["refresh_required"], true, "{phase}: {payload}"); - } else { - assert!( - catch_up, - "{phase}: message_search must answer evidence: {envelope}" - ); - let message = problem_message(&envelope); - assert!( - message.contains("HistoricalRetry") || message.contains("HistoricalConvergence"), - "{phase}: a refused fresh read must name the historical state: {envelope}" - ); - } - assert!( - Instant::now() < deadline, - "{phase}: fresh search never reached retained evidence; last answer: {envelope}" - ); - std::thread::sleep(Duration::from_millis(250)); - }; - let payload = payload(&envelope); - let hits = payload["results"] - .as_array() - .unwrap_or_else(|| panic!("{phase}: message_search must list results: {payload}")); - assert!( - hits.iter() - .any(|hit| hit["session"]["session_id"] == SESSION_ID), - "{phase}: the recovered session must be searchable: {payload}" - ); - assert!( - payload["temporal"]["watermarks"]["generation"] - .as_u64() - .is_some_and(|generation| generation > 0), - "{phase}: a served search page must carry a non-zero generation watermark: {payload}" - ); -} - -#[test] -fn observation_authority_reset_recovers_the_retained_temporal_authority() { - let home = TempDir::new().unwrap(); - let project_dir = TempDir::new().unwrap(); - let home = canonical_existing_path(home.path()); - let project = canonical_existing_path(project_dir.path()); - seed_committed_project(&project); - write_codex_rollouts(&home, &project); - initialize_tracedecay_cli_project(&home, &project); - - // Baseline: the retained authority serves the ingested session. - let baseline = wait_for_described_session(&home, &project, "before reset"); - assert_search_hits(&home, &project, false, "before reset"); - let sessions_db = project_sessions_db(&home); - assert!(count(&sessions_db, "observations") > 0); - assert!(count(&sessions_db, "retrieval_anchor_aliases") > 0); - assert!( - scheduling_cursor_count(&sessions_db) > 0, - "a converged store records the swept Codex corpus and provider coverage" - ); - - // Reopening unchanged history must settle without a spurious conflict. - assert_search_hits(&home, &project, true, "baseline catch_up"); - stop_managed_daemon(&home); - let reopen_log = home.join("ordinary-reopen.log"); - let daemon = spawn_tracedecay_daemon_with(&home, |command| { - command.stderr(std::fs::File::create(&reopen_log).unwrap()); - }); - wait_for_described_session(&home, &project, "ordinary reopen"); - assert_search_hits(&home, &project, true, "ordinary reopen catch_up"); - drop(daemon); - assert_no_replay_conflicts(&reopen_log); - - // Recovery runs offline: the daemon cannot open a refused store. - make_observation_authority_refused(&sessions_db); - replace_codex_rollout_identity(&home); - - let reset = tracedecay_command_with_home(&home) - .current_dir(&project) - .args([ - "storage", - "--yes", - "reset-authority", - "observations", - "--db", - &sessions_db.to_string_lossy(), - ]) - .output() - .expect("storage reset-authority should run"); - let reset_stdout = String::from_utf8_lossy(&reset.stdout); - assert!( - reset.status.success(), - "the scoped reset must succeed\nstdout:\n{reset_stdout}\nstderr:\n{}", - String::from_utf8_lossy(&reset.stderr) - ); - assert!( - reset_stdout.contains("recreated observations empty"), - "the reset must report the refused authority: {reset_stdout}" - ); - assert_eq!(count(&sessions_db, "observations"), 0); - assert_eq!(count(&sessions_db, "session_temporal_generations"), 0); - assert_eq!( - count(&sessions_db, "retrieval_anchor_aliases"), - 0, - "native-record aliases bound by the reset stream must not outlive it" - ); - assert_eq!( - scheduling_cursor_count(&sessions_db), - 0, - "a swept-corpus frontier or complete coverage verdict must not tell the rebuilt \ - authority to skip the transcripts it has to re-read" - ); - assert!( - count(&sessions_db, "lcm_raw_messages") > 0, - "the reset preserves LCM content" - ); - - // Reopen: the runtime re-derives the projection from the preserved - // transcripts. The doctor answers evidence throughout, so it is the - // surface that names where that stands: a store still converging is - // partial evidence whose projection carries the historical state, and a - // store that has already converged is complete and current, and only - // once its generations are rebuilt. Neither reading may be an unavailable - // projection or the reset store read as converged. - let reset_log = home.join("reset-reopen.log"); - let daemon = spawn_tracedecay_daemon_with(&home, |command| { - command.stderr(std::fs::File::create(&reset_log).unwrap()); - }); - let reopened = doctor(&home, &project); - assert!( - is_evidence(&reopened), - "doctor must answer evidence: {reopened}" - ); - let report = payload(&reopened); - match report["projection"]["state"].as_str() { - Some("stale") => { - assert_eq!( - report["status"], "partial", - "a converging store is partial diagnostic evidence: {report}" - ); - assert!( - report["projection"]["reason"] - .as_str() - .is_some_and(|reason| reason.starts_with("historical_")), - "the doctor must name the historical convergence state: {report}" - ); - } - Some("current") => { - assert_eq!(report["status"], "complete", "{report}"); - assert!( - count(&sessions_db, "session_temporal_generations") > 0, - "a current projection must be a rebuilt one: {report}" - ); - } - other => { - panic!("the reopened store must be converging or current, not {other:?}: {report}") - } - } - - // `tracedecay tool` honours the daemon's after-delay retry directive: a - // describe issued while history converges rides out the typed converging - // refusal inside the tool deadline and answers once the projection is - // current, so the journey reads the converged answer here rather than - // polling for the transient. The recovered session must carry the LCM - // content the reset preserved, byte for byte. - let recovered = wait_for_described_session(&home, &project, "after reset"); - assert_eq!( - payload(&recovered)["description"], - payload(&baseline)["description"], - "the rebuilt authority must serve the session's preserved LCM content unchanged" - ); - assert_search_hits(&home, &project, false, "after reset"); - // Historical catch-up must reach a terminal state: the frontier the pass - // persists is the whole reason a second pass has nothing left to do. - assert_search_hits(&home, &project, true, "after reset with catch_up"); - - let converged = doctor(&home, &project); - assert!(is_evidence(&converged), "{converged}"); - let report = payload(&converged); - assert_eq!(report["status"], "complete", "{report}"); - assert_eq!(report["health"]["status"], "complete", "{report}"); - assert_eq!(report["projection"]["state"], "current", "{report}"); - - let import = tracedecay_command_with_home(&home) - .current_dir(&project) - .args([ - "sessions", - "import", - "--project-path", - &project.to_string_lossy(), - ]) - .output() - .expect("sessions import should run"); - assert!( - import.status.success(), - "explicit import must reach a terminal state once history converged\nstdout:\n{}\nstderr:\n{}", - String::from_utf8_lossy(&import.stdout), - String::from_utf8_lossy(&import.stderr) - ); - - assert!(count(&sessions_db, "observations") > 0); - assert!( - count(&sessions_db, "session_temporal_generations") > 0, - "the temporal projection must be rebuilt from the preserved transcripts" - ); - assert!( - scheduling_cursor_count(&sessions_db) > 0, - "the rebuilt authority must record the swept Codex corpus and provider coverage again" - ); - drop(daemon); - assert_no_replay_conflicts(&reset_log); -} - -fn assert_no_replay_conflicts(log: &Path) { - let output = std::fs::read_to_string(log).unwrap(); - for reason in [ - "authority_write_failed", - "external_source_commit_failed", - "observation repository provenance collision", - "external source idempotency key conflicts", - ] { - assert!( - !output.contains(reason), - "reopen must settle on its first pass: {output}" - ); - } -} diff --git a/crates/tracedecay-cli/tests/core_cli_suite/test_profile_isolation_test.rs b/crates/tracedecay-cli/tests/core_cli_suite/test_profile_isolation_test.rs index c362a0e574..8f19fbb1a4 100644 --- a/crates/tracedecay-cli/tests/core_cli_suite/test_profile_isolation_test.rs +++ b/crates/tracedecay-cli/tests/core_cli_suite/test_profile_isolation_test.rs @@ -8,7 +8,7 @@ use std::path::{Path, PathBuf}; -use tracedecay::config::{USER_DATA_DIR_ENV, user_data_dir}; +use tracedecay_project::config::{USER_DATA_DIR_ENV, user_data_dir}; fn canonical(path: &Path) -> PathBuf { path.canonicalize().unwrap_or_else(|_| path.to_path_buf()) diff --git a/crates/tracedecay-cli/tests/core_cli_suite/tool_daemon_test.rs b/crates/tracedecay-cli/tests/core_cli_suite/tool_daemon_test.rs index e9b5617f83..c6c8bf735e 100644 --- a/crates/tracedecay-cli/tests/core_cli_suite/tool_daemon_test.rs +++ b/crates/tracedecay-cli/tests/core_cli_suite/tool_daemon_test.rs @@ -14,18 +14,23 @@ use crate::common::{ }; use serde_json::{Value, json}; use tempfile::TempDir; +use tracedecay_contracts::retained_surfaces::RetainedSurfaceRequestV1; use tracedecay_contracts::{ - ApplicationProblem, ApplicationProblemEnvelope, RUNTIME_MOUNTING_REASON_CODE, RequestId, - ResultContractRef, SafeDiagnostic, + ApplicationProblem, RUNTIME_MOUNTING_REASON_CODE, ResolvedScope, SafeDiagnostic, }; +use tracedecay_daemon_identity::authority::DaemonAuthority; +use tracedecay_daemon_protocol::{ + DAEMON_INVOCATION_PROTOCOL, DaemonAuthPreface, DaemonEndpoint, DaemonInvocationPayload, + DaemonInvocationRequest, DaemonInvocationResponse, +}; +use tracedecay_domain::NativeHostIdentityV1; use tracedecay_domain::UtcMicros; -use tracedecay_hooks::{HookEventV2, HookHostV1, HookSpoolConfigV1, HookSpoolV1}; +use tracedecay_domain::{FactCategoryV1, ProjectId, RepositoryId, WorktreeId}; +use tracedecay_hooks::{HookEventV2, HookSpoolConfigV1, HookSpoolV1}; use tracedecay_runtime_core::storage::{ EnrollmentMarker, StorageMode, default_profile_project_id, pin_fixture_repository_identity, profile_sharded_data_root, profile_sharded_layout, }; -use tracedecay_tool_catalog::SchemaId; - /// Bound for waits that depend on spawning and running the real `tracedecay` /// CLI as a child process: connecting to the fake daemon socket and forwarding /// the observed request back to the test thread. Under nextest's @@ -127,6 +132,19 @@ fn cli_build_version() -> &'static str { }) } +/// Publishes the authority record beside `socket_path` that the CLI resolves +/// a fake daemon through: the record `TRACEDECAY_DAEMON_SOCKET` routes read +/// or, for a fake bound on the profile's own socket, the profile record the +/// typed retained route reads. Hold it for as long as the fake daemon serves. +fn seed_fake_daemon_authority(socket_path: &Path) -> DaemonAuthority { + DaemonAuthority::acquire( + socket_path.parent().expect("socket parent"), + &DaemonEndpoint::Unix(socket_path.to_path_buf()), + "fake-daemon", + ) + .expect("seed fake daemon authority") +} + fn spawn_scripted_daemon( socket_path: PathBuf, expected_tool_name: &'static str, @@ -137,6 +155,7 @@ fn spawn_scripted_daemon( std::thread::spawn(move || { let _ = std::fs::remove_file(&socket_path); + let authority = seed_fake_daemon_authority(&socket_path); let listener = UnixListener::bind(&socket_path).expect("bind fake daemon socket"); listener .set_nonblocking(true) @@ -177,6 +196,13 @@ fn spawn_scripted_daemon( Ok(0) | Err(_) => break None, Ok(_) => {} } + if let Ok(preface) = DaemonAuthPreface::from_line(line.trim()) { + assert!( + preface.authenticate(authority.auth_token()), + "the CLI must present the daemon token" + ); + continue; + } let value: Value = serde_json::from_str(line.trim()).expect("fake daemon preamble JSON"); if value.get("method").is_some() { @@ -527,23 +553,6 @@ fn wait_for_daemon_socket(socket_path: &Path) { ); } -fn spawn_sentinel_daemon( - socket_path: PathBuf, - expected_tool_name: &'static str, - expect_project_path: bool, - expect_allow_init: bool, - sentinel: &'static str, -) -> mpsc::Receiver { - spawn_sentinel_daemon_with_notification( - socket_path, - expected_tool_name, - expect_project_path, - expect_allow_init, - sentinel, - false, - ) -} - fn spawn_sentinel_daemon_with_notification( socket_path: PathBuf, expected_tool_name: &'static str, @@ -557,6 +566,7 @@ fn spawn_sentinel_daemon_with_notification( std::thread::spawn(move || { let _ = std::fs::remove_file(&socket_path); + let authority = seed_fake_daemon_authority(&socket_path); let listener = UnixListener::bind(&socket_path).expect("bind fake daemon socket"); listener .set_nonblocking(true) @@ -582,6 +592,14 @@ fn spawn_sentinel_daemon_with_notification( .expect("write timeout"); let mut reader = BufReader::new(stream.try_clone().expect("clone fake daemon stream")); + let mut preface = String::new(); + reader.read_line(&mut preface).expect("read auth preface"); + assert!( + DaemonAuthPreface::from_line(preface.trim()) + .expect("fake daemon auth preface") + .authenticate(authority.auth_token()), + "the CLI must present the daemon token" + ); let mut handshake = String::new(); reader .read_line(&mut handshake) @@ -603,6 +621,14 @@ fn spawn_sentinel_daemon_with_notification( let request: Value = serde_json::from_str(request.trim()).expect("request JSON"); assert_eq!(request["method"], "tools/call"); assert_eq!(request["params"]["name"], expected_tool_name); + // The one-shot call has no `initialize` session; the daemon serves it + // over rmcp only because it carries SEP-2575 per-request context. + let meta = &request["params"]["_meta"]; + assert!( + meta["io.modelcontextprotocol/protocolVersion"].is_string() + && meta["io.modelcontextprotocol/clientCapabilities"].is_object(), + "one-shot tools/call omitted its per-request MCP context: {request}" + ); request_tx .send(request.clone()) .expect("send observed JSON-RPC request"); @@ -697,11 +723,11 @@ fn run_native_capture_hook( .expect("hook command should run") } -fn native_capture_spool_root(data_root: &Path, host: HookHostV1) -> PathBuf { +fn native_capture_spool_root(data_root: &Path, host: NativeHostIdentityV1) -> PathBuf { data_root.join("hook-v2-spool").join(host.hook_key()) } -fn native_capture_pending_records(data_root: &Path, host: HookHostV1) -> u32 { +fn native_capture_pending_records(data_root: &Path, host: NativeHostIdentityV1) -> u32 { HookSpoolV1::open( native_capture_spool_root(data_root, host), HookSpoolConfigV1::stock(host), @@ -742,7 +768,7 @@ fn cursor_after_file_edit_hook_captures_bound_spool_record() { let project = TempDir::new().unwrap(); let home_path = canonical_existing_path(home.path()); let project_path = canonical_existing_path(project.path()); - let host = HookHostV1::CursorDesktop; + let host = NativeHostIdentityV1::CursorDesktop; let data_root = enroll_native_capture_project( &home_path, &project_path, @@ -826,7 +852,7 @@ fn cursor_after_shell_hook_is_typed_unsupported_without_spool_record() { let project = TempDir::new().unwrap(); let home_path = canonical_existing_path(home.path()); let project_path = canonical_existing_path(project.path()); - let host = HookHostV1::CursorDesktop; + let host = NativeHostIdentityV1::CursorDesktop; // Bind every family Cursor natively supports so the absence of a spool // record is attributable to the unsupported event, not a missing binding. let data_root = @@ -935,7 +961,7 @@ fn kiro_hooks_capture_prompt_boundary_and_type_post_tool_use_unsupported() { let project = TempDir::new().unwrap(); let home_path = canonical_existing_path(home.path()); let project_path = canonical_existing_path(project.path()); - let host = HookHostV1::Kiro; + let host = NativeHostIdentityV1::Kiro; let data_root = enroll_native_capture_project(&home_path, &project_path, "proj_kiro_capture"); std::fs::create_dir_all(project_path.join("src")).unwrap(); std::fs::write( @@ -1183,39 +1209,29 @@ fn tool_cli_skips_daemon_notifications_until_matching_response() { .expect("fake daemon should receive tools/call request"); } +/// A retained store tool travels to the profile's daemon as one typed +/// invocation: the explicit `--project` rides the handshake with first-touch +/// init allowed, and the decoded request carries the caller's exact fields. #[test] -fn fact_store_cli_accepts_exact_route_and_rejects_broad_router() { +fn fact_store_cli_routes_exact_tool_through_daemon() { let home = TempDir::new().unwrap(); let project = TempDir::new().unwrap(); - let socket_dir = TempDir::new().unwrap(); let home_path = canonical_existing_path(home.path()); let project_path = canonical_existing_path(project.path()); - let broad = tracedecay_command_with_home(&home_path) - .current_dir(&project_path) - .args(["tool", "fact_store", "--help"]) - .output() - .expect("broad fact-store lookup should return"); - assert!(!broad.status.success(), "broad fact-store route must fail"); - assert!( - String::from_utf8_lossy(&broad.stderr).contains("unknown tool: 'fact_store'"), - "broad lookup must fail as unknown:\n{}", - String::from_utf8_lossy(&broad.stderr) - ); - - let sentinel = "first-touch daemon response"; - let socket_path = socket_dir.path().join("tracedecay.sock"); - let observed_request = spawn_sentinel_daemon( - socket_path.clone(), - "tracedecay_fact_store_add", - true, - true, - sentinel, + let daemon = spawn_scripted_retained_daemon( + &home_path, + vec![ApplicationProblem::unavailable( + SafeDiagnostic::new( + "application.retained.authority-unavailable", + "first-touch daemon response", + ) + .expect("diagnostic"), + )], ); let project_arg = project_path.to_string_lossy().to_string(); let output = tracedecay_command_with_home(&home_path) .current_dir(&project_path) - .env("TRACEDECAY_DAEMON_SOCKET", &socket_path) .args([ "tool", "--project", @@ -1228,29 +1244,31 @@ fn fact_store_cli_accepts_exact_route_and_rejects_broad_router() { .output() .expect("tracedecay tool should run"); - assert!( - output.status.success(), - "first-touch store tool CLI should accept daemon response\nstdout:\n{}\nstderr:\n{}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ); let stdout = String::from_utf8_lossy(&output.stdout); assert!( - stdout.contains(sentinel), - "tool CLI should print daemon response, got:\n{stdout}" + stdout.contains("first-touch daemon response"), + "tool CLI should print the daemon's answer, got:\n{stdout}\nstderr:\n{}", + String::from_utf8_lossy(&output.stderr) ); - let request = observed_request + let (handshake, request) = daemon + .requests .recv_timeout(CLI_ROUNDTRIP_TIMEOUT) - .expect("fake daemon should receive first-touch tools/call request"); - assert_eq!(request["params"]["name"], "tracedecay_fact_store_add"); + .expect("fake daemon should receive the first-touch invocation"); assert_eq!( - request["params"]["arguments"]["content"], - "first touch via daemon" - ); - assert!( - request["params"]["arguments"].get("action").is_none(), - "exact route payload must not carry the deleted broad action selector" - ); + handshake["project_path"], + json!(project_path.to_string_lossy()), + "{handshake}" + ); + assert_eq!(handshake["allow_init"], true, "{handshake}"); + let DaemonInvocationPayload::RetainedApplication { + request: RetainedSurfaceRequestV1::FactStoreAdd(add), + .. + } = &request.payload + else { + panic!("fact_store_add must travel as a retained invocation: {request:?}"); + }; + assert_eq!(add.content, "first touch via daemon"); + assert_eq!(add.category, Some(FactCategoryV1::Decision)); } #[test] @@ -1414,7 +1432,7 @@ fn doctor_keeps_live_daemon_database_healthy_without_compaction() { &home_path.join(".tracedecay"), &default_profile_project_id(&project_path), ); - let db_path = data_root.join(tracedecay::config::db_filename(&data_root)); + let db_path = data_root.join(tracedecay_project::config::db_filename(&data_root)); common::create_runtime().block_on(async { let (db, _) = crate::common::open_test_database(&db_path) .await @@ -1489,29 +1507,29 @@ fn daemon_project_handshake_uses_client_profile_identity() { ); } +/// Retained tools reach the daemon through the client profile's own authority +/// record, so the daemon serving that profile sees the profile's stale +/// `config.json` and must neither read it into the configuration authority +/// nor rewrite it. #[test] fn daemon_first_touch_uses_registered_runtime_without_rewriting_legacy_config() { - let daemon_home = TempDir::new().unwrap(); - let client_home = TempDir::new().unwrap(); + let home = TempDir::new().unwrap(); let project = TempDir::new().unwrap(); - let daemon_home_path = canonical_existing_path(daemon_home.path()); - let client_home_path = canonical_existing_path(client_home.path()); + let home_path = canonical_existing_path(home.path()); let project_path = canonical_existing_path(project.path()); - init_project_with_cli(&client_home_path, &project_path); + init_project_with_cli(&home_path, &project_path); let project_id = default_profile_project_id(&project_path); - let config_path = client_home_path + let config_path = home_path .join(".tracedecay/projects") .join(project_id) .join("config.json"); std::fs::write(&config_path, b"{not json").unwrap(); - let _daemon = spawn_tracedecay_daemon(&daemon_home_path); - let socket_path = common::daemon_socket_path(&daemon_home_path); + let _daemon = spawn_tracedecay_daemon(&home_path); let project_arg = project_path.to_string_lossy().to_string(); - let output = tracedecay_command_with_home(&client_home_path) + let output = tracedecay_command_with_home(&home_path) .current_dir(&project_path) - .env("TRACEDECAY_DAEMON_SOCKET", &socket_path) .args([ "tool", "--project", @@ -1830,28 +1848,34 @@ fn status_command_times_out_when_daemon_never_replies() { ); } -/// The `tools/call` requests a [`spawn_scripted_result_sequence_daemon`] -/// observed. Dropping it stops the daemon thread. -struct ScriptedResultSequenceDaemon { - requests: mpsc::Receiver, +/// The handshake and typed invocation each request a +/// [`spawn_scripted_retained_daemon`] observed. Dropping it stops the daemon +/// thread. +struct ScriptedRetainedDaemon { + requests: mpsc::Receiver<(Value, DaemonInvocationRequest)>, stop: Arc, } -impl Drop for ScriptedResultSequenceDaemon { +impl Drop for ScriptedRetainedDaemon { fn drop(&mut self) { self.stop.store(true, Ordering::Release); } } -/// A fake daemon that answers every `tools/call` for `expected_tool_name` with -/// the next scripted `result`, repeating the last one once the script is -/// exhausted, so a test can hand the CLI a typed retryable state for as many -/// attempts as it makes and then either the answer or nothing else. -fn spawn_scripted_result_sequence_daemon( - socket_path: PathBuf, - expected_tool_name: &'static str, - results: Vec, -) -> ScriptedResultSequenceDaemon { +/// A fake daemon serving `home`'s profile on that profile's own socket and +/// authority record, speaking the typed invocation protocol the CLI's +/// retained tools use. It answers every invocation with the next scripted +/// problem as the retained owner's completed answer, repeating the last one +/// once the script is exhausted, so a test can hand the CLI a typed +/// retryable state for as many attempts as it makes and then the answer. +/// +/// The CLI pools its invocation connection, so one accepted stream keeps +/// serving requests until the client releases it. +fn spawn_scripted_retained_daemon( + home: &Path, + problems: Vec, +) -> ScriptedRetainedDaemon { + let socket_path = common::daemon_socket_path(home); let (ready_tx, ready_rx) = mpsc::channel(); let (request_tx, request_rx) = mpsc::channel(); let stop = Arc::new(AtomicBool::new(false)); @@ -1859,13 +1883,21 @@ fn spawn_scripted_result_sequence_daemon( std::thread::spawn(move || { let _ = std::fs::remove_file(&socket_path); + let authority = seed_fake_daemon_authority(&socket_path); + let scope = ResolvedScope::new( + ProjectId::new("project.scripted-daemon").expect("project id"), + RepositoryId::new("repository.scripted-daemon").expect("repository id"), + WorktreeId::new("worktree.scripted-daemon").expect("worktree id"), + None, + ) + .expect("scripted daemon scope"); let listener = UnixListener::bind(&socket_path).expect("bind fake daemon socket"); listener .set_nonblocking(true) .expect("set listener nonblocking"); ready_tx.send(()).expect("notify fake daemon readiness"); - let mut results = results.into_iter(); - let mut current = results.next().expect("at least one scripted result"); + let mut problems = problems.into_iter(); + let mut current = problems.next().expect("at least one scripted problem"); while !daemon_stop.load(Ordering::Acquire) { let stream = match listener.accept() { @@ -1884,92 +1916,76 @@ fn spawn_scripted_result_sequence_daemon( .expect("write timeout"); let _ = stream.set_read_timeout(Some(CLI_ROUNDTRIP_TIMEOUT)); let mut reader = BufReader::new(stream.try_clone().expect("clone fake daemon stream")); - let request = loop { + let mut writer = stream; + let mut handshake: Option = None; + loop { let mut line = String::new(); match reader.read_line(&mut line) { - Ok(0) | Err(_) => break None, + Ok(0) | Err(_) => break, Ok(_) => {} } - let value: Value = - serde_json::from_str(line.trim()).expect("fake daemon preamble JSON"); - if value.get("method").is_some() { - break Some(value); + let line = line.trim(); + if let Ok(preface) = DaemonAuthPreface::from_line(line) { + assert!( + preface.authenticate(authority.auth_token()), + "the CLI must present the daemon token" + ); + continue; } - }; - let Some(request) = request else { - continue; - }; - let result = if request["method"] == "initialize" { - json!({ - "serverInfo": { - "name": "tracedecay", - "version": cli_build_version(), - } - }) - } else { - assert_eq!(request["method"], "tools/call"); - assert_eq!(request["params"]["name"], expected_tool_name); - if request_tx.send(request.clone()).is_err() { - break; + let frame: Value = serde_json::from_str(line).expect("fake daemon frame JSON"); + if frame["protocol"] != DAEMON_INVOCATION_PROTOCOL { + handshake = Some(frame); + continue; } - let served = current.clone(); - if let Some(next) = results.next() { + let request: DaemonInvocationRequest = + serde_json::from_value(frame).expect("typed daemon invocation request"); + let request_id = request.request_id.clone(); + let observed_handshake = handshake + .clone() + .expect("the handshake precedes every invocation"); + if request_tx.send((observed_handshake, request)).is_err() { + return; + } + let response = DaemonInvocationResponse::retained_application_problem( + request_id, + scope.clone(), + current.clone(), + ); + if let Some(next) = problems.next() { current = next; } - served - }; - let response = json!({ - "jsonrpc": "2.0", - "id": request["id"].clone(), - "result": result, - }); - let mut writer = stream; - writeln!(writer, "{}", serde_json::to_string(&response).unwrap()) + writeln!( + writer, + "{}", + serde_json::to_string(&response).expect("response JSON") + ) .expect("write fake daemon response"); + } } }); ready_rx .recv_timeout(LOCAL_READY_TIMEOUT) .expect("fake daemon should become ready"); - ScriptedResultSequenceDaemon { + ScriptedRetainedDaemon { requests: request_rx, stop, } } -/// The MCP tool result the daemon renders for a completed pre-admission -/// problem whose retry directive is `after_delay`. -fn retry_directed_tool_result(code: &str, message: &str, retry_after_millis: u64) -> Value { - let envelope = ApplicationProblemEnvelope::new( - ResultContractRef::new( - SchemaId::new("schema.retained.fact_store_add.result").expect("schema id"), - 1, - ) - .expect("result contract"), - RequestId::new("request.cli.tool.mounting-owner").expect("request id"), - ApplicationProblem::unavailable(SafeDiagnostic::new(code, message).expect("diagnostic")), - ) - .expect("retry-directed envelope") - .with_retry_after_millis(Some(retry_after_millis)) - .expect("retry delay"); - json!({ - "content": [{ - "type": "text", - "text": serde_json::to_string(&envelope).expect("envelope JSON"), - }], - "isError": true, - "problem": serde_json::to_value(envelope.problem.as_ref()).expect("problem record"), - }) +/// The retained owner's refusal while it is still mounting behind the core +/// publication; the CLI re-sends the same request after the delay the daemon +/// side's problem record names. +fn mounting_owner_problem() -> ApplicationProblem { + ApplicationProblem::runtime_mounting() } -/// The MCP tool result for a project route whose retained owner is still -/// mounting behind the core publication. -fn mounting_owner_tool_result(retry_after_millis: u64) -> Value { - retry_directed_tool_result( - RUNTIME_MOUNTING_REASON_CODE, - "The project runtime for this operation is still mounting", - retry_after_millis, +/// A completed authority problem: the owner's answer, carrying its own +/// `after_delay` directive for the caller. +fn completed_authority_problem(message: &str) -> ApplicationProblem { + ApplicationProblem::unavailable( + SafeDiagnostic::new("application.retained.authority-unavailable", message) + .expect("diagnostic"), ) } @@ -1984,17 +2000,11 @@ fn fact_store_add_args() -> String { .to_string() } -fn fact_store_add_command( - home: &Path, - project: &Path, - socket: &Path, - deadline_ms: &str, -) -> Command { +fn fact_store_add_command(home: &Path, project: &Path, deadline_ms: &str) -> Command { let project_arg = project.to_string_lossy().to_string(); let mut command = tracedecay_command_with_home(home); command .current_dir(project) - .env("TRACEDECAY_DAEMON_SOCKET", socket) .env("TRACEDECAY_TOOL_DEADLINE_MS", deadline_ms) .args([ "tool", @@ -2010,48 +2020,50 @@ fn fact_store_add_command( /// A one-shot `tracedecay tool` call holds a deadline, so a typed /// `retry: after_delay` unavailable from an owner still mounting is progress -/// to wait through on the delay the directive names, not the answer. +/// to wait through on the delay the problem record names, not the answer. The +/// owner's eventual answer here is a completed typed problem, which the CLI +/// returns on first observation exactly like a success. #[test] fn tool_waits_through_an_after_delay_unavailable_within_its_deadline() { let home = TempDir::new().unwrap(); let project = TempDir::new().unwrap(); - let socket_dir = TempDir::new().unwrap(); let home_path = canonical_existing_path(home.path()); let project_path = canonical_existing_path(project.path()); - init_project_with_cli(&home_path, &project_path); - const RETRY_AFTER_MILLIS: u64 = 100; - let socket_path = socket_dir.path().join("tracedecay.sock"); - let daemon = spawn_scripted_result_sequence_daemon( - socket_path.clone(), - "tracedecay_fact_store_add", + let daemon = spawn_scripted_retained_daemon( + &home_path, vec![ - mounting_owner_tool_result(RETRY_AFTER_MILLIS), - mounting_owner_tool_result(RETRY_AFTER_MILLIS), - mounting_owner_tool_result(RETRY_AFTER_MILLIS), - mounting_owner_tool_result(RETRY_AFTER_MILLIS), - json!({ - "content": [{ - "type": "text", - "text": json!({"outcome": {"outcome": "effect", "marker": "mounted-answer"}}).to_string(), - }] - }), + mounting_owner_problem(), + mounting_owner_problem(), + mounting_owner_problem(), + mounting_owner_problem(), + completed_authority_problem("mounted-answer: history is not available"), ], ); let started = Instant::now(); let output = run_command_with_timeout( - fact_store_add_command(&home_path, &project_path, &socket_path, "10000"), + fact_store_add_command(&home_path, &project_path, "10000"), CLI_ROUNDTRIP_TIMEOUT, ); let elapsed = started.elapsed(); assert!( - output.status.success(), - "the tool must return the owner's answer once it mounts\nstdout:\n{}\nstderr:\n{}", + !output.status.success(), + "the owner's completed problem must fail typed\nstdout:\n{}\nstderr:\n{}", String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr) ); let stdout = String::from_utf8_lossy(&output.stdout); + let printed: Value = serde_json::from_str(&stdout).unwrap_or_else(|error| { + panic!( + "the owner's typed answer must be printed as JSON ({error}):\n{stdout}\nstderr:\n{}", + String::from_utf8_lossy(&output.stderr) + ) + }); + assert_eq!( + printed["problem"]["code"], "application.retained.authority-unavailable", + "stdout must carry the mounted owner's answer, got:\n{stdout}" + ); assert!( stdout.contains("mounted-answer"), "stdout must carry the mounted owner's answer, got:\n{stdout}" @@ -2065,8 +2077,14 @@ fn tool_waits_through_an_after_delay_unavailable_within_its_deadline() { attempts, 5, "the CLI must re-send the same mounting request until the owner answers" ); + // The mounting refusal and the final answer are both `after_delay` + // problems built by the one problem-record authority, so the printed + // delay is the delay each ridden-out refusal waited. + let retry_after_millis = printed["problem"]["retry_after_millis"] + .as_u64() + .expect("an after_delay problem names its delay"); assert!( - elapsed >= Duration::from_millis(4 * RETRY_AFTER_MILLIS), + elapsed >= Duration::from_millis(4 * retry_after_millis), "each retry must wait the delay the directive names, took {elapsed:?}" ); } @@ -2077,24 +2095,18 @@ fn tool_waits_through_an_after_delay_unavailable_within_its_deadline() { fn tool_returns_a_completed_authority_result_without_resending() { let home = TempDir::new().unwrap(); let project = TempDir::new().unwrap(); - let socket_dir = TempDir::new().unwrap(); let home_path = canonical_existing_path(home.path()); let project_path = canonical_existing_path(project.path()); - init_project_with_cli(&home_path, &project_path); - let socket_path = socket_dir.path().join("tracedecay.sock"); - let daemon = spawn_scripted_result_sequence_daemon( - socket_path.clone(), - "tracedecay_fact_store_add", - vec![retry_directed_tool_result( - "application.retained.authority-unavailable", + let daemon = spawn_scripted_retained_daemon( + &home_path, + vec![completed_authority_problem( "The retained operation authority is unavailable: history is not available", - 250, )], ); let started = Instant::now(); let output = run_command_with_timeout( - fact_store_add_command(&home_path, &project_path, &socket_path, "10000"), + fact_store_add_command(&home_path, &project_path, "10000"), CLI_CHILD_KILL_TIMEOUT, ); let elapsed = started.elapsed(); @@ -2137,6 +2149,7 @@ fn spawn_handshake_capturing_daemon(socket_path: PathBuf) -> mpsc::Receiver mpsc::Receiver HostCase { @@ -277,6 +265,14 @@ impl IsolatedCli { self.command(args).output().unwrap() } + /// Runs with only the isolated bin dir on `PATH`, so no host CLI the + /// machine happens to carry can resolve. + fn run_without_host_clis(&self, args: &[&str]) -> Output { + let mut command = self.command(args); + command.env("PATH", &self.bin_dir); + command.output().unwrap() + } + fn run_with_env(&self, args: &[&str], key: &str, value: &str) -> Output { let mut command = self.command(args); command.env(key, value); @@ -757,24 +753,7 @@ fn production_cli_completes_deterministic_lifecycle_for_config_native_hosts() { "{} interrupted repair did not preserve its durable receipt", case.id ); - assert_success( - case.id, - "interruption recovery", - cli.run(&["host-bundle", "recover", "--agent", case.id, "--yes"]), - ); - assert_eq!( - owned_bytes(&cli, &repaired_receipt, &originals), - before_interruption, - "{} recovery did not preserve rolled-back configs/artifacts", - case.id - ); - assert_eq!( - serde_json::to_vec(&latest_receipt(&cli, case.host)).unwrap(), - receipt_before_interruption, - "{} recovery did not preserve the pre-interruption receipt", - case.id - ); - assert_success(case.id, "post-recovery repair", cli.run(&["reinstall"])); + assert_success(case.id, "post-interruption repair", cli.run(&["reinstall"])); let repaired_receipt = latest_receipt(&cli, case.host); assert_receipt_digests(&cli, &repaired_receipt); @@ -965,6 +944,7 @@ fn hermes_dashboard_opt_out_survives_install_update_and_reinstall() { let case = host_case(HostKindV1::Hermes); seed_host(case, &cli); + let bin_literal = serde_json::to_string(&cli.bin_dir.join("tracedecay")).unwrap(); let assert_dashboard_absent = || { for plugin in [ cli.home.path().join(".hermes/plugins/tracedecay"), @@ -972,10 +952,33 @@ fn hermes_dashboard_opt_out_survives_install_update_and_reinstall() { .path() .join(".hermes/profiles/review/plugins/tracedecay"), ] { + let manifest = fs::read_to_string(plugin.join("plugin.yaml")).unwrap(); + assert!( + manifest.starts_with("name: tracedecay\nkind: standalone\n"), + "{}: {manifest}", + plugin.display() + ); + let tools = fs::read_to_string(plugin.join("tools.py")).unwrap(); + assert!( + tools.contains(&format!( + r#"TRACEDECAY_BIN = os.environ.get("TRACEDECAY_BIN") or {bin_literal}"# + )), + "{}: tools.py must invoke the installed binary", + plugin.display() + ); + let skill = fs::read_to_string(plugin.join("skills/tracedecay/SKILL.md")).unwrap(); + assert!(skill.starts_with("---\nname: tracedecay\n"), "{skill}"); assert!(!plugin.join("dashboard/manifest.json").exists()); assert!(!plugin.join("dashboard/plugin_api.py").exists()); assert!(!plugin.join("dashboard/dist/index.js").exists()); } + let config: toml::Value = + toml::from_str(&fs::read_to_string(cli.profile.join("config.toml")).unwrap()).unwrap(); + assert_eq!( + config["agent_dashboard_enabled"]["hermes"].as_bool(), + Some(false), + "the opt-out must persist: {config}" + ); }; assert_success( @@ -1078,112 +1081,143 @@ fn feedback_policy_failure_precedes_apply_and_restore_mutations() { assert_eq!(owned_bytes(&cli, &receipt, &originals), before_apply); } +/// Codex activation is part of the component transaction: without its CLI +/// the transaction rolls back and leaves nothing staged out of band; with it, +/// install and a stale-cache update both converge through `codex plugin add`. +#[cfg(unix)] #[test] -fn codex_stale_cache_remediation_executes_on_the_current_stock_cli_and_converges_update() { +fn codex_lifecycle_activates_through_the_stock_cli_inside_the_transaction() { let cli = IsolatedCli::new(); let case = host_case(HostKindV1::Codex); let originals = seed_host(case, &cli); + let home = cli.home.path(); - let staged = cli.run(&["install", "--agent", case.id]); - assert!(!staged.status.success()); + let refused = cli.run_without_host_clis(&["install", "--agent", case.id]); + assert!(!refused.status.success()); + let stderr = String::from_utf8_lossy(&refused.stderr); + assert!( + stderr.contains("Install the `codex` CLI"), + "missing-CLI refusal must name the host CLI: {stderr}" + ); assert_seeded_bytes(&cli, &originals); assert!( - cli.home - .path() + !home .join(".codex/plugins/tracedecay/.codex-plugin/plugin.json") - .is_file(), - "Codex remediation has no staged plugin source" + .exists(), + "a refused activation left the plugin source behind" ); assert!( - cli.home - .path() - .join(".agents/plugins/marketplace.json") - .is_file(), - "Codex remediation has no staged marketplace entry" + !home.join(".agents/plugins/marketplace.json").exists(), + "a refused activation left the marketplace entry behind" ); assert!( latest_host_component_set_receipt_at(&cli.lifecycle_root(), case.host) .unwrap() - .is_none(), - "staging Codex activation published a lifecycle receipt" + .is_none() ); - apply_current_codex_plugin_remediation(cli.home.path(), remediation_command(&staged.stderr)) - .unwrap(); + + install_current_codex_cli(&cli.bin_dir); assert_success( case.id, - "receipt-backed install after native activation", + "install through the stock plugin CLI", cli.run(&["install", "--agent", case.id]), ); - - let cache_manifest = cli - .home - .path() + assert_receipt_digests(&cli, &latest_receipt(&cli, case.host)); + // The stock CLI appends its activation record and TraceDecay appends only + // its hook-trust tables: every operator byte stays in place. + let config_path = home.join(".codex/config.toml"); + let installed_config = fs::read(&config_path).unwrap(); + assert!( + installed_config.starts_with(&originals[&PathBuf::from(".codex/config.toml")]), + "install rewrote operator bytes in config.toml:\n{}", + String::from_utf8_lossy(&installed_config) + ); + let source_manifest = home.join(".codex/plugins/tracedecay/.codex-plugin/plugin.json"); + let cache_manifest = home .join(".codex/plugins/cache/personal/tracedecay") .join(tracedecay_agent_hosts::PRODUCT_VERSION) .join(".codex-plugin/plugin.json"); + assert_eq!( + fs::read(&cache_manifest).unwrap(), + fs::read(&source_manifest).unwrap() + ); + fs::write( &cache_manifest, br#"{"name":"tracedecay","version":"stale"}"#, ) .unwrap(); - - let stale_update = cli.run(&["update-plugin"]); - assert!(!stale_update.status.success()); - apply_current_codex_plugin_remediation( - cli.home.path(), - remediation_command(&stale_update.stderr), - ) - .unwrap(); assert_success( case.id, - "update after current stock remediation", + "update re-drives the stock plugin CLI over a stale cache", cli.run(&["update-plugin"]), ); + assert_eq!( + fs::read(&cache_manifest).unwrap(), + fs::read(&source_manifest).unwrap() + ); + assert_eq!( + fs::read(&config_path).unwrap(), + installed_config, + "re-driving the converged Codex activation rewrote config.toml" + ); } +/// Claude's marketplace source is receipt-owned from the first install: the +/// transaction deploys it and then drives the stock `claude plugin` grammar, +/// or rolls both back when that CLI is absent. #[cfg(unix)] #[test] -fn claude_lifecycle_tracks_assets_only_after_native_activation() { +fn claude_lifecycle_activates_through_the_stock_cli_inside_the_transaction() { let cli = IsolatedCli::new(); let case = host_case(HostKindV1::ClaudeCode); let originals = seed_host(case, &cli); + let home = cli.home.path(); + let source_manifest = + home.join(".claude/plugins/marketplaces/tracedecay/.claude-plugin/plugin.json"); - let deferred = cli.run(&["install", "--agent", case.id]); - assert!(!deferred.status.success()); - let stderr = String::from_utf8_lossy(&deferred.stderr); + let refused = cli.run_without_host_clis(&["install", "--agent", case.id]); + assert!(!refused.status.success()); + let stderr = String::from_utf8_lossy(&refused.stderr); assert!( - stderr.contains("Claude Code owns marketplace registration"), - "Claude deferral omitted its native activation boundary: {stderr}" + stderr.contains("host CLI is unavailable"), + "missing-CLI refusal must name the host CLI: {stderr}" ); assert!( - cli.home - .path() - .join(".claude/plugins/marketplaces/tracedecay/.claude-plugin/marketplace.json") - .is_file(), - "Claude deferral did not stage the verified marketplace source" + !source_manifest.exists(), + "a refused activation left the marketplace source behind" ); assert_seeded_bytes(&cli, &originals); assert!( latest_host_component_set_receipt_at(&cli.lifecycle_root(), case.host) .unwrap() - .is_none(), - "staging native activation published a lifecycle receipt" + .is_none() ); - set_claude_native_activation(cli.home.path(), true); - let settings_path = cli.home.path().join(".claude/settings.json"); - let marketplaces_path = cli - .home - .path() - .join(".claude/plugins/known_marketplaces.json"); + let claude_invocations = install_current_claude_cli(home, &cli.bin_dir); + let settings_path = home.join(".claude/settings.json"); + let marketplaces_path = home.join(".claude/plugins/known_marketplaces.json"); let settings_before_install: serde_json::Value = serde_json::from_slice(&fs::read(&settings_path).unwrap()).unwrap(); - let marketplaces_before_install = fs::read(&marketplaces_path).unwrap(); + let marketplaces_before_install: serde_json::Value = + serde_json::from_slice(&fs::read(&marketplaces_path).unwrap()).unwrap(); assert_success( case.id, - "receipt-backed install after native activation", + "install through the stock plugin CLI", cli.run(&["install", "--agent", case.id]), ); + assert_eq!( + recorded_claude_invocations(&claude_invocations), + [ + format!( + "plugin marketplace add {}", + home.join(".claude/plugins/marketplaces/tracedecay") + .display() + ), + "plugin install tracedecay@tracedecay".to_string(), + ], + "Claude activation must use the current stock plugin lifecycle grammar" + ); let install_receipt = latest_receipt(&cli, case.host); assert_receipt_digests(&cli, &install_receipt); let installed_settings: serde_json::Value = @@ -1198,18 +1232,22 @@ fn claude_lifecycle_tracks_assets_only_after_native_activation() { serde_json::json!(["Read", "mcp__plugin_tracedecay_graph__*"]), "catalog install must add the one managed permission without replacing foreign grants" ); - assert_eq!( - fs::read(&marketplaces_path).unwrap(), - marketplaces_before_install - ); - let active_native_state = [ - fs::read(&settings_path).unwrap(), - fs::read(&marketplaces_path).unwrap(), - ]; + let installed_marketplaces: serde_json::Value = + serde_json::from_slice(&fs::read(&marketplaces_path).unwrap()).unwrap(); + for (name, entry) in marketplaces_before_install.as_object().unwrap() { + assert_eq!( + &installed_marketplaces[name], entry, + "foreign marketplace {name}" + ); + } + let native_values = || { + [&settings_path, &marketplaces_path].map(|path| { + serde_json::from_slice::(&fs::read(path).unwrap()).unwrap() + }) + }; + let converged_activation = native_values(); - let cache_manifest = cli - .home - .path() + let cache_manifest = home .join(".claude/plugins/cache/tracedecay/tracedecay") .join(tracedecay_agent_hosts::PRODUCT_VERSION) .join(".claude-plugin/plugin.json"); @@ -1218,31 +1256,26 @@ fn claude_lifecycle_tracks_assets_only_after_native_activation() { br#"{"name":"tracedecay","version":"stale"}"#, ) .unwrap(); - let before_stale_update = serde_json::to_vec(&latest_receipt(&cli, case.host)).unwrap(); - let stale_update = cli.run(&["update-plugin"]); - assert!(!stale_update.status.success()); - assert!( - String::from_utf8_lossy(&stale_update.stderr).contains("loaded TraceDecay cache is stale"), - "Claude stale cache did not produce native-update remediation: {}", - String::from_utf8_lossy(&stale_update.stderr) - ); - assert_eq!( - serde_json::to_vec(&latest_receipt(&cli, case.host)).unwrap(), - before_stale_update, - "stale native cache changed the component receipt" - ); - fs::copy( - cli.home - .path() - .join(".claude/plugins/marketplaces/tracedecay/.claude-plugin/plugin.json"), - &cache_manifest, - ) - .unwrap(); assert_success( case.id, - "catalog update after native cache refresh", + "update re-drives the stock plugin CLI over a stale cache", cli.run(&["update-plugin"]), ); + assert_eq!( + fs::read(&cache_manifest).unwrap(), + fs::read(&source_manifest).unwrap() + ); + // The re-driven `claude plugin install` rewrites these host-owned files in + // the host's own serialization; only their values are TraceDecay's to keep. + assert_eq!( + native_values(), + converged_activation, + "re-driving the stock plugin CLI changed the converged Claude activation state" + ); + let active_native_state = [ + fs::read(&settings_path).unwrap(), + fs::read(&marketplaces_path).unwrap(), + ]; assert_success(case.id, "catalog repair", cli.run(&["reinstall"])); for (phase, entrypoint, fixture) in native_feedback(case) { assert_success(case.id, phase, cli.run_with_stdin(&[entrypoint], &fixture)); @@ -1256,7 +1289,7 @@ fn claude_lifecycle_tracks_assets_only_after_native_activation() { "catalog maintenance changed the converged Claude activation state" ); - let claude_invocations = install_current_claude_cli(cli.home.path(), &cli.bin_dir); + fs::remove_file(&claude_invocations).unwrap(); assert_success( case.id, "stock CLI-backed uninstall", @@ -1305,14 +1338,27 @@ fn kimi_lifecycle_reports_official_activation_deferral() { assert!(!output.status.success()); let stderr = String::from_utf8_lossy(&output.stderr); + let staged = cli + .home + .path() + .join(".tracedecay/host-bundle-stage/kimi/tracedecay"); assert!( - stderr.contains("Kimi") && stderr.contains("plugin"), + stderr.contains(&format!("/plugins install {}", staged.display())), "Kimi deferral omitted its official activation boundary: {stderr}" ); + // The staged source that `/plugins install` consumes is receipt-owned; + // Kimi's own registry is never written. + assert!(staged.join(".kimi-plugin/plugin.json").is_file()); assert!( latest_host_component_set_receipt_at(&cli.lifecycle_root(), case.host) .unwrap() - .is_none() + .is_some() + ); + assert!( + !cli.home + .path() + .join(".kimi-code/plugins/installed.json") + .exists() ); } @@ -1347,76 +1393,15 @@ fn unadmitted_catalog_hosts_never_fall_back_to_direct_installers() { } } +/// Rollback bytes live only in the process that staged them, so a killed +/// install leaves whatever it wrote and nothing to recover from: no journal, +/// backup, or copy of any host config. The next install converges over it. #[test] -fn killed_registration_mutation_recovers_exact_pre_effect_state() { +fn killed_install_keeps_no_rollback_state_and_the_next_install_converges() { let cli = IsolatedCli::new(); let case = host_case(HostKindV1::OpenCode); let originals = seed_host(case, &cli); - assert_success( - case.id, - "initial install", - cli.run(&["install", "--agent", case.id]), - ); - let receipt = latest_receipt(&cli, case.host); let config_path = cli.home.path().join(".config/opencode/opencode.json"); - let mut config: serde_json::Value = - serde_json::from_slice(&fs::read(&config_path).unwrap()).unwrap(); - config["mcp"]["tracedecay"]["command"] = serde_json::json!(["operator-owned", "pending"]); - fs::write(&config_path, serde_json::to_vec_pretty(&config).unwrap()).unwrap(); - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - - fs::set_permissions(&config_path, fs::Permissions::from_mode(0o640)).unwrap(); - } - let before = owned_bytes(&cli, &receipt, &originals); - let receipt_before = serde_json::to_vec(&latest_receipt(&cli, case.host)).unwrap(); - - let killed = cli.run_with_env( - &["reinstall"], - "TRACEDECAY_TEST_ABORT_AFTER_HOST_CONFIG_WRITE", - "1", - ); - assert!(!killed.status.success(), "fault subprocess did not abort"); - assert_ne!( - fs::read(&config_path).unwrap(), - before[&PathBuf::from(".config/opencode/opencode.json")], - "fault boundary did not cross a real host-config mutation" - ); - - assert_success( - case.id, - "restart recovery", - cli.run(&["host-bundle", "recover", "--agent", case.id, "--yes"]), - ); - assert_eq!(owned_bytes(&cli, &receipt, &originals), before); - assert_eq!( - serde_json::to_vec(&latest_receipt(&cli, case.host)).unwrap(), - receipt_before - ); - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - - assert_eq!( - fs::metadata(&config_path).unwrap().permissions().mode() & 0o777, - 0o640 - ); - } -} - -#[test] -fn killed_install_recovers_with_original_journal_operation() { - let cli = IsolatedCli::new(); - let case = host_case(HostKindV1::OpenCode); - let originals = seed_host(case, &cli); - let config_path = cli.home.path().join(".config/opencode/opencode.json"); - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - - fs::set_permissions(&config_path, fs::Permissions::from_mode(0o640)).unwrap(); - } let killed = cli.run_with_env( &["install", "--agent", case.id], "TRACEDECAY_TEST_ABORT_AFTER_HOST_CONFIG_WRITE", @@ -1430,112 +1415,30 @@ fn killed_install_recovers_with_original_journal_operation() { fs::read(&config_path).unwrap(), originals[&PathBuf::from(".config/opencode/opencode.json")] ); - assert_success( - case.id, - "install restart recovery", - cli.run(&["host-bundle", "recover", "--agent", case.id, "--yes"]), - ); - assert_seeded_bytes(&cli, &originals); - assert!( - latest_host_component_set_receipt_at(&cli.lifecycle_root(), case.host) - .unwrap() - .is_none() - ); - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - - assert_eq!( - fs::metadata(&config_path).unwrap().permissions().mode() & 0o777, - 0o640 - ); + let control = cli.lifecycle_root().join(".tracedecay-host-bundle-v1"); + if let Ok(entries) = fs::read_dir(&control) { + for entry in entries { + let name = entry.unwrap().file_name().to_string_lossy().into_owned(); + assert!( + name.starts_with("receipt.") || name.starts_with("writer."), + "a killed install must leave no rollback state: {name}" + ); + } } -} - -#[cfg(target_os = "linux")] -#[test] -fn recovery_rejects_foreign_metadata_drift_with_unchanged_bytes() { - use std::os::unix::fs::PermissionsExt; - - let cli = IsolatedCli::new(); - let case = host_case(HostKindV1::OpenCode); - seed_host(case, &cli); - let config_path = cli.home.path().join(".config/opencode/opencode.json"); - fs::set_permissions(&config_path, fs::Permissions::from_mode(0o640)).unwrap(); - let mut original_acl = 2_u32.to_le_bytes().to_vec(); - for (tag, permissions, id) in [ - (0x01_u16, 0x06_u16, u32::MAX), - (0x02, 0x04, 65_534), - (0x04, 0x04, u32::MAX), - (0x10, 0x04, u32::MAX), - (0x20, 0x00, u32::MAX), - ] { - original_acl.extend_from_slice(&tag.to_le_bytes()); - original_acl.extend_from_slice(&permissions.to_le_bytes()); - original_acl.extend_from_slice(&id.to_le_bytes()); + for entry in fs::read_dir(config_path.parent().unwrap()).unwrap() { + let name = entry.unwrap().file_name().to_string_lossy().into_owned(); + assert!( + !name.ends_with(".bak") && !name.ends_with(".tracedecay-original"), + "no copy of the host config may remain beside it: {name}" + ); } - xattr::set(&config_path, "system.posix_acl_access", &original_acl).unwrap(); - let killed = cli.run_with_env( - &["install", "--agent", case.id], - "TRACEDECAY_TEST_ABORT_AFTER_HOST_CONFIG_WRITE", - "1", - ); - assert!(!killed.status.success()); - fs::set_permissions(&config_path, fs::Permissions::from_mode(0o600)).unwrap(); - - let bytes_after_kill = fs::read(&config_path).unwrap(); - let acl_after_drift = xattr::get(&config_path, "system.posix_acl_access").unwrap(); - let refused = cli.run(&["host-bundle", "recover", "--agent", case.id, "--yes"]); - assert!(!refused.status.success()); - assert_eq!(fs::read(&config_path).unwrap(), bytes_after_kill); - assert_eq!( - fs::metadata(&config_path).unwrap().permissions().mode() & 0o777, - 0o600, - "recovery must not overwrite foreign metadata drift" - ); - assert_eq!( - xattr::get(&config_path, "system.posix_acl_access").unwrap(), - acl_after_drift - ); - assert_ne!(acl_after_drift, Some(original_acl)); -} - -#[test] -fn interrupted_registration_rollback_converges_across_two_restarts() { - let cli = IsolatedCli::new(); - let case = host_case(HostKindV1::OpenCode); - let originals = seed_host(case, &cli); - let config_path = cli.home.path().join(".config/opencode/opencode.json"); - let killed = cli.run_with_env( - &["install", "--agent", case.id], - "TRACEDECAY_TEST_ABORT_AFTER_HOST_CONFIG_WRITE", - "1", - ); - assert!(!killed.status.success()); - - let mut recovery = cli.command(&["host-bundle", "recover", "--agent", case.id, "--yes"]); - let interrupted = recovery - .env( - "TRACEDECAY_TEST_ABORT_AFTER_REGISTRATION_ROLLBACK_WRITE_PATH", - &config_path, - ) - .output() - .unwrap(); - assert!(!interrupted.status.success()); - assert_seeded_bytes(&cli, &originals); assert_success( case.id, - "rollback restart", - cli.run(&["host-bundle", "recover", "--agent", case.id, "--yes"]), - ); - assert_seeded_bytes(&cli, &originals); - assert_success( - case.id, - "idempotent rollback restart", - cli.run(&["host-bundle", "recover", "--agent", case.id, "--yes"]), + "install after the killed install", + cli.run(&["install", "--agent", case.id]), ); - assert_seeded_bytes(&cli, &originals); + assert_receipt_digests(&cli, &latest_receipt(&cli, case.host)); } #[cfg(target_os = "linux")] @@ -1608,44 +1511,6 @@ fn claude_install_rejects_empty_symlinked_config_directory() { assert_eq!(fs::read_dir(outside.path()).unwrap().count(), 0); } -#[test] -fn killed_install_recovery_refuses_later_operator_edit() { - let cli = IsolatedCli::new(); - let case = host_case(HostKindV1::OpenCode); - let originals = seed_host(case, &cli); - let killed = cli.run_with_env( - &["install", "--agent", case.id], - "TRACEDECAY_TEST_ABORT_AFTER_HOST_CONFIG_WRITE", - "1", - ); - assert!(!killed.status.success()); - - let config_path = cli.home.path().join(".config/opencode/opencode.json"); - let mut config: serde_json::Value = - serde_json::from_slice(&fs::read(&config_path).unwrap()).unwrap(); - config["operatorAfterKill"] = serde_json::json!(true); - fs::write(&config_path, serde_json::to_vec_pretty(&config).unwrap()).unwrap(); - let config_before = fs::read(&config_path).unwrap(); - let receipt_before = - latest_host_component_set_receipt_at(&cli.lifecycle_root(), case.host).unwrap(); - - let refused = cli.run(&["host-bundle", "recover", "--agent", case.id, "--yes"]); - assert!(!refused.status.success()); - assert_eq!(fs::read(&config_path).unwrap(), config_before); - assert_eq!( - latest_host_component_set_receipt_at(&cli.lifecycle_root(), case.host).unwrap(), - receipt_before - ); - for relative in originals.keys() { - if relative != &PathBuf::from(".config/opencode/opencode.json") { - assert_eq!( - fs::read(cli.home.path().join(relative)).unwrap(), - originals[relative] - ); - } - } -} - #[test] fn stale_feedback_registration_refuses_before_artifact_or_receipt_effects() { let cli = IsolatedCli::new(); diff --git a/crates/tracedecay-cli/tests/host_journeys_suite/host_lifecycle_cli_acceptance/native_plugin_fixture.rs b/crates/tracedecay-cli/tests/host_journeys_suite/host_lifecycle_cli_acceptance/native_plugin_fixture.rs index 6a744dad70..546b1af0f4 100644 --- a/crates/tracedecay-cli/tests/host_journeys_suite/host_lifecycle_cli_acceptance/native_plugin_fixture.rs +++ b/crates/tracedecay-cli/tests/host_journeys_suite/host_lifecycle_cli_acceptance/native_plugin_fixture.rs @@ -1,109 +1,8 @@ use std::fs; use std::path::{Path, PathBuf}; -pub fn copy_test_bundle(source: &Path, destination: &Path) { - for entry in fs::read_dir(source).unwrap() { - let entry = entry.unwrap(); - let source_path = entry.path(); - let destination_path = destination.join(entry.file_name()); - if entry.file_type().unwrap().is_dir() { - fs::create_dir_all(&destination_path).unwrap(); - copy_test_bundle(&source_path, &destination_path); - } else { - fs::create_dir_all(destination_path.parent().unwrap()).unwrap(); - fs::copy(source_path, destination_path).unwrap(); - } - } -} - -pub fn set_claude_native_activation(home: &Path, active: bool) { - let settings_path = home.join(".claude/settings.json"); - let mut settings: serde_json::Value = - serde_json::from_slice(&fs::read(&settings_path).unwrap()).unwrap(); - let enabled_plugins = settings - .get_mut("enabledPlugins") - .and_then(serde_json::Value::as_object_mut) - .unwrap(); - if active { - enabled_plugins.insert("tracedecay@tracedecay".to_string(), true.into()); - } else { - enabled_plugins.remove("tracedecay@tracedecay"); - } - fs::write( - &settings_path, - serde_json::to_vec_pretty(&settings).unwrap(), - ) - .unwrap(); - - let marketplaces_path = home.join(".claude/plugins/known_marketplaces.json"); - let mut marketplaces: serde_json::Value = - serde_json::from_slice(&fs::read(&marketplaces_path).unwrap()).unwrap(); - let marketplaces = marketplaces.as_object_mut().unwrap(); - if active { - let deploy_dir = home.join(".claude/plugins/marketplaces/tracedecay"); - marketplaces.insert( - "tracedecay".to_string(), - serde_json::json!({ - "source": { "source": "directory", "path": deploy_dir }, - "installLocation": deploy_dir - }), - ); - } else { - marketplaces.remove("tracedecay"); - } - fs::write( - &marketplaces_path, - serde_json::to_vec_pretty(&marketplaces).unwrap(), - ) - .unwrap(); - - let cache_root = home - .join(".claude/plugins/cache/tracedecay/tracedecay") - .join(tracedecay_agent_hosts::PRODUCT_VERSION); - if active { - fs::create_dir_all(&cache_root).unwrap(); - copy_test_bundle( - &home.join(".claude/plugins/marketplaces/tracedecay"), - &cache_root, - ); - } else if cache_root.exists() { - fs::remove_dir_all(cache_root).unwrap(); - } -} - -pub fn apply_current_codex_plugin_remediation(home: &Path, command: &str) -> Result<(), String> { - let ["codex", "plugin", "add", "tracedecay@personal"] = - command.split_whitespace().collect::>().as_slice() - else { - return Err(format!( - "Codex 0.147.0 cannot execute lifecycle remediation `{command}`" - )); - }; - - let config_path = home.join(".codex/config.toml"); - let mut config = fs::read_to_string(&config_path).unwrap_or_default(); - if !config.contains("[plugins.\"tracedecay@personal\"]") { - config.push_str("\n[plugins.\"tracedecay@personal\"]\nenabled = true\n"); - fs::write(&config_path, config).unwrap(); - } - - let source = home.join(".codex/plugins/tracedecay"); - let cache = home - .join(".codex/plugins/cache/personal/tracedecay") - .join(tracedecay_agent_hosts::PRODUCT_VERSION); - fs::create_dir_all(&cache).unwrap(); - copy_test_bundle(&source, &cache); - Ok(()) -} - -pub fn remediation_command(stderr: &[u8]) -> &str { - std::str::from_utf8(stderr) - .unwrap() - .split('`') - .nth(1) - .expect("lifecycle failure must provide one executable remediation command") -} - +/// A stock-grammar `claude` that performs the plugin lifecycle the component +/// transaction drives: marketplace registration, cache install, removal. #[cfg(unix)] pub fn install_current_claude_cli(home: &Path, bin_dir: &Path) -> PathBuf { use std::os::unix::fs::PermissionsExt; @@ -123,7 +22,25 @@ args = sys.argv[1:] with (home / ".claude-test-invocations").open("a") as log: log.write(" ".join(args) + "\n") -if args == ["plugin", "uninstall", "tracedecay"]: +deploy_dir = home / ".claude/plugins/marketplaces/tracedecay" +if args == ["plugin", "marketplace", "add", str(deploy_dir)]: + marketplace_path = home / ".claude/plugins/known_marketplaces.json" + marketplaces = json.loads(marketplace_path.read_text()) if marketplace_path.exists() else {} + marketplaces["tracedecay"] = { + "source": {"source": "directory", "path": str(deploy_dir)}, + "installLocation": str(deploy_dir), + } + marketplace_path.write_text(json.dumps(marketplaces, indent=2)) +elif args == ["plugin", "install", "tracedecay@tracedecay"]: + manifest = json.loads((deploy_dir / ".claude-plugin/plugin.json").read_text()) + cache = home / ".claude/plugins/cache/tracedecay/tracedecay" / manifest["version"] + shutil.rmtree(cache, ignore_errors=True) + shutil.copytree(deploy_dir, cache) + settings_path = home / ".claude/settings.json" + settings = json.loads(settings_path.read_text()) if settings_path.exists() else {} + settings.setdefault("enabledPlugins", {})["tracedecay@tracedecay"] = True + settings_path.write_text(json.dumps(settings, indent=2)) +elif args == ["plugin", "uninstall", "tracedecay"]: settings_path = home / ".claude/settings.json" settings = json.loads(settings_path.read_text()) settings["enabledPlugins"].pop("tracedecay@tracedecay", None) @@ -154,3 +71,45 @@ pub fn recorded_claude_invocations(path: &Path) -> Vec { .map(str::to_string) .collect() } + +/// A stock-grammar `codex` whose `plugin add` enables the plugin and copies +/// the catalog-deployed source into its versioned cache, as Codex does. +#[cfg(unix)] +pub fn install_current_codex_cli(bin_dir: &Path) { + use std::os::unix::fs::PermissionsExt; + + let cli = bin_dir.join("codex"); + fs::write( + &cli, + r##"#!/usr/bin/env python3 +import json +import os +import pathlib +import shutil +import sys + +home = pathlib.Path(os.environ["HOME"]) +args = sys.argv[1:] +if args[:2] == ["plugin", "add"] and len(args) >= 3 and args[2].startswith("tracedecay@") and args[3:] in ([], ["--json"]): + marketplace = args[2].split("@", 1)[1] + source = home / ".codex/plugins/tracedecay" + manifest = json.loads((source / ".codex-plugin/plugin.json").read_text()) + cache = home / ".codex/plugins/cache" / marketplace / "tracedecay" / manifest["version"] + shutil.rmtree(cache, ignore_errors=True) + shutil.copytree(source, cache) + config_path = home / ".codex/config.toml" + config = config_path.read_text() if config_path.exists() else "" + header = '[plugins."' + args[2] + '"]' + if header not in config: + config_path.write_text(config + "\n" + header + "\nenabled = true\n") + print(json.dumps({"pluginId": args[2], "enabled": True})) +else: + print("unsupported fake Codex lifecycle command: " + " ".join(args), file=sys.stderr) + sys.exit(2) +"##, + ) + .unwrap(); + let mut permissions = fs::metadata(&cli).unwrap().permissions(); + permissions.set_mode(0o755); + fs::set_permissions(&cli, permissions).unwrap(); +} diff --git a/crates/tracedecay-cli/tests/host_journeys_suite/opencode_one_analyzer_journey.rs b/crates/tracedecay-cli/tests/host_journeys_suite/opencode_one_analyzer_journey.rs index 5acb3051c6..239383150e 100644 --- a/crates/tracedecay-cli/tests/host_journeys_suite/opencode_one_analyzer_journey.rs +++ b/crates/tracedecay-cli/tests/host_journeys_suite/opencode_one_analyzer_journey.rs @@ -6,7 +6,7 @@ //! uninstall while TraceDecay findings still project. //! //! The journey drives the real CLI lifecycle (`install`, `reinstall`, a -//! killed mutation recovered by `host-bundle recover`, `uninstall`) against an +//! killed mutation converged by the next `reinstall`, `uninstall`) against an //! isolated home whose OpenCode configuration already declares a real //! pre-existing analyzer (`rust-analyzer` for `.rs`), and at every stage //! proves single ownership through both consumption paths: @@ -303,24 +303,15 @@ fn opencode_keeps_exactly_one_analyzer_through_install_repair_rollback_uninstall HostAnalyzerOwnership::from_opencode_config(&repaired), ); - // ROLLBACK: a mutation killed mid-write is rolled back by recovery to the - // exact pre-effect state, which must still hold single ownership. - let pre_fault = fs::read(cli.host_config_path()).unwrap(); + // INTERRUPTION: a mutation killed mid-write leaves what it wrote; the next + // reinstall converges, and the result must still hold single ownership. let killed = cli.run_with_env( &["reinstall"], "TRACEDECAY_TEST_ABORT_AFTER_HOST_CONFIG_WRITE", "1", ); assert!(!killed.status.success(), "fault subprocess did not abort"); - assert_success( - "rollback recovery", - cli.run(&["host-bundle", "recover", "--agent", "opencode", "--yes"]), - ); - assert_eq!( - fs::read(cli.host_config_path()).unwrap(), - pre_fault, - "recovery must restore the exact pre-effect registration" - ); + assert_success("converging reinstall", cli.run(&["reinstall"])); let recovered = cli.host_config(); assert_registration_retains_host_analyzer(&recovered); assert_broker_enforces_single_ownership( diff --git a/crates/tracedecay-cli/tests/work_loop_journey.rs b/crates/tracedecay-cli/tests/work_loop_journey.rs index d158e47e66..773c439da9 100644 --- a/crates/tracedecay-cli/tests/work_loop_journey.rs +++ b/crates/tracedecay-cli/tests/work_loop_journey.rs @@ -50,11 +50,11 @@ use std::time::{Duration, Instant}; use serde_json::{Value, json}; use sha2::{Digest, Sha256}; use tempfile::TempDir; -use tracedecay::config::USER_DATA_DIR_ENV; use tracedecay_domain::{ CommitId, ManifestDigest, ProposalId, RefId, TaskId, WorkEffectStateV1, WorkGraphVersionV1, WorkRelationReplanProposalV1, WorkflowOperationRef, }; +use tracedecay_project::config::USER_DATA_DIR_ENV; use tracedecay_runtime_core::storage::PrivateStoreIo; /// Pins the global database away from the operator's profile. The production diff --git a/crates/tracedecay-cli/tests/work_route_exposure_conformance.rs b/crates/tracedecay-cli/tests/work_route_exposure_conformance.rs index 4d7030ac42..2042ec855b 100644 --- a/crates/tracedecay-cli/tests/work_route_exposure_conformance.rs +++ b/crates/tracedecay-cli/tests/work_route_exposure_conformance.rs @@ -57,7 +57,6 @@ use axum::http::{Request, StatusCode}; use serde_json::{Map, Value}; use tempfile::TempDir; use tower::ServiceExt; -use tracedecay::config::USER_DATA_DIR_ENV; use tracedecay_application::operation_stream::OperationEventAuthority; use tracedecay_contracts::{ EXECUTION_TOPOLOGY_DESCRIPTOR_REVISION_V1, EXECUTION_TOPOLOGY_METRIC_DESCRIPTORS_V1, @@ -65,6 +64,7 @@ use tracedecay_contracts::{ }; use tracedecay_daemon_service::application_surface::http_application_router; use tracedecay_domain::ProjectId; +use tracedecay_project::config::USER_DATA_DIR_ENV; use tracedecay_runtime_core::storage::PrivateStoreIo; use tracedecay_tool_catalog::RouteExposureV1; @@ -889,25 +889,6 @@ fn the_work_surface_answers_real_requests_on_both_published_mounts() { .expect("created event observation time"); work_evidence::assert_live_task_rooted_retrieval(&agent, &fixture, &dashboard, observed_at); - for retired in ["snapshot", "delta", "replan-dependencies", "accept-task"] { - let (status, body) = post_envelope( - &agent, - &fixture.external_url(&format!("/application/work/{retired}")), - &fixture, - &serde_json::json!({}), - ); - assert_eq!(status, 404, "retired daemon Work route {retired}: {body}"); - let (status, body) = post_dashboard_envelope( - &agent, - &format!("{}/api/work/{retired}", dashboard.base_url), - &serde_json::json!({}), - ); - assert_eq!( - status, 404, - "retired dashboard Work route {retired}: {body}" - ); - } - // -- Product publication binds an empty attempt page. -------------------- // Once a product task exists, the product graph supplies the canonical // generation for every attempt-page reader. With no admitted attempts the diff --git a/crates/tracedecay-code-extraction/src/common.rs b/crates/tracedecay-code-extraction/src/common.rs index cc6bc5ece5..08624ab874 100644 --- a/crates/tracedecay-code-extraction/src/common.rs +++ b/crates/tracedecay-code-extraction/src/common.rs @@ -137,37 +137,31 @@ pub(crate) fn local_node_id( ) -> String { let start = node.start_position(); let line = start.row as u32; + if begins_line(source, node) { + generate_node_id(file_path, kind, name, line) + } else { + generate_node_id_at(file_path, kind, name, line, start.column as u32) + } +} + +/// Whether only blanks precede `node` on its first line. +fn begins_line(source: &[u8], node: TsNode<'_>) -> bool { let start_byte = node.start_byte().min(source.len()); let line_start = source[..start_byte] .iter() .rposition(|byte| *byte == b'\n') .map_or(0, |newline| newline + 1); - let begins_line = source[line_start..start_byte] + source[line_start..start_byte] .iter() - .all(|byte| matches!(byte, b' ' | b'\t' | b'\r')); - if begins_line { - generate_node_id(file_path, kind, name, line) - } else { - generate_node_id_at(file_path, kind, name, line, start.column as u32) - } + .all(|byte| matches!(byte, b' ' | b'\t' | b'\r')) } -/// Strip comment markers from a single C-style comment text -/// (`//` line comments and `/* ... */` block comments). +/// Strip comment markers from a single C-style comment text: `//` and `///` +/// line comments and `/* ... */` block comments. pub(crate) fn clean_c_comment(comment: &str) -> String { - clean_c_line_or_block_comment(comment, &["//"]) -} - -/// Strip comment markers from a single C-style comment text, including -/// `///` doc comments. -pub(crate) fn clean_c_doc_comment(comment: &str) -> String { - // Longer prefixes first so `///` is not stripped as `//`. - clean_c_line_or_block_comment(comment, &["///", "//"]) -} - -fn clean_c_line_or_block_comment(comment: &str, line_prefixes: &[&str]) -> String { let trimmed = comment.trim(); - for prefix in line_prefixes { + // Longer prefix first so `///` is not stripped as `//`. + for prefix in ["///", "//"] { if let Some(stripped) = trimmed.strip_prefix(prefix) { return stripped.strip_prefix(' ').unwrap_or(stripped).to_string(); } @@ -193,20 +187,33 @@ fn clean_c_line_or_block_comment(comment: &str, line_prefixes: &[&str]) -> Strin /// Extract a docstring from the run of `comment` siblings immediately /// preceding `node`, cleaning each comment with `clean`. +/// +/// A comment belongs to the run only when it starts its own line and no blank +/// line separates it from what follows, so a trailing `} // namespace x` or a +/// detached section banner never documents the next declaration. pub(crate) fn docstring_from_preceding_comments( source: &[u8], node: TsNode<'_>, clean: fn(&str) -> String, ) -> Option { let mut comments = Vec::new(); + let mut following_row = node.start_position().row; let mut current = node.prev_named_sibling(); while let Some(sibling) = current { - if sibling.kind() == "comment" { - comments.push(node_text(source, sibling)); - current = sibling.prev_named_sibling(); + // Some grammars end a line comment after its newline, at column 0. + let end = sibling.end_position(); + let last_row = if end.column == 0 && end.row > sibling.start_position().row { + end.row - 1 } else { + end.row + }; + let adjacent = last_row + 1 >= following_row; + if sibling.kind() != "comment" || !adjacent || !begins_line(source, sibling) { break; } + comments.push(node_text(source, sibling)); + following_row = sibling.start_position().row; + current = sibling.prev_named_sibling(); } if comments.is_empty() { return None; @@ -246,6 +253,31 @@ pub(crate) fn docstring_from_hash_comments(source: &[u8], node: TsNode<'_>) -> O Some(comments.join("\n")) } +/// The full identifier lexeme around `identifier`. +/// +/// The BASIC grammars lex `_` inside a name as an error token, splitting +/// `MAX_RETRIES` into sibling fragments and sometimes dropping text, so the +/// name is recovered from source by extending the node over adjacent +/// identifier characters. Type suffixes (`$`, `%`) and member access (`.`) +/// stay outside the lexeme. +#[cfg(any( + feature = "lang-gwbasic", + feature = "lang-msbasic2", + feature = "lang-qbasic" +))] +pub(crate) fn basic_identifier_text<'s>(source: &'s [u8], identifier: TsNode<'_>) -> &'s str { + let is_ident = |byte: u8| byte.is_ascii_alphanumeric() || byte == b'_'; + let mut start = identifier.start_byte().min(source.len()); + let mut end = identifier.end_byte().min(source.len()); + while start > 0 && is_ident(source[start - 1]) { + start -= 1; + } + while end < source.len() && is_ident(source[end]) { + end += 1; + } + std::str::from_utf8(&source[start..end]).unwrap_or("") +} + /// Recursively find `call_expression` nodes and create unresolved Calls /// references, taking the callee name from the first named child. pub(crate) fn extract_call_expression_sites( diff --git a/crates/tracedecay-code-extraction/src/cpp_extractor/metadata.rs b/crates/tracedecay-code-extraction/src/cpp_extractor/metadata.rs index 2dcab2716b..21c7a02499 100644 --- a/crates/tracedecay-code-extraction/src/cpp_extractor/metadata.rs +++ b/crates/tracedecay-code-extraction/src/cpp_extractor/metadata.rs @@ -5,7 +5,7 @@ use tree_sitter::Node as TsNode; use super::{CppExtractor, ExtractionState}; use crate::{ common::{ - clean_c_doc_comment, docstring_from_preceding_comments, extract_call_expression_sites, + clean_c_comment, docstring_from_preceding_comments, extract_call_expression_sites, local_node_id, }, traversal::{find_descendant_by_kind, find_direct_child_by_kind}, @@ -247,7 +247,7 @@ impl CppExtractor { } pub(super) fn extract_docstring(state: &ExtractionState, node: TsNode<'_>) -> Option { - docstring_from_preceding_comments(state.source, node, clean_c_doc_comment) + docstring_from_preceding_comments(state.source, node, clean_c_comment) } pub(super) fn has_storage_class( diff --git a/crates/tracedecay-code-extraction/src/gwbasic_extractor.rs b/crates/tracedecay-code-extraction/src/gwbasic_extractor.rs index f91fc36962..f9d437faa8 100644 --- a/crates/tracedecay-code-extraction/src/gwbasic_extractor.rs +++ b/crates/tracedecay-code-extraction/src/gwbasic_extractor.rs @@ -12,7 +12,7 @@ use tree_sitter::{Node as TsNode, Tree}; use crate::basic_common::{ BasicLine, derive_function_name, find_subroutine_ranges, for_each_top_level_line, }; -use crate::common::{ExtractionState, local_node_id}; +use crate::common::{ExtractionState, basic_identifier_text, local_node_id}; use crate::traversal::find_direct_child_by_kind; use crate::types::{ ComplexityAnalysisV1, Edge, EdgeKind, ExtractionResult, Node, NodeKind, UnresolvedRef, @@ -179,7 +179,7 @@ impl GwBasicExtractor { let Some(fn_name_node) = find_direct_child_by_kind(def_fn, "user_function") else { continue; }; - let fn_name = state.node_text(fn_name_node); + let fn_name = basic_identifier_text(state.source, fn_name_node); let start_line = basic_line.node.start_position().row as u32; let end_line = basic_line.node.end_position().row as u32; @@ -275,7 +275,7 @@ impl GwBasicExtractor { let Some(id_node) = find_direct_child_by_kind(var_node, "identifier") else { return; }; - let name = state.node_text(id_node); + let name = basic_identifier_text(state.source, id_node); let start_line = basic_line.node.start_position().row as u32; let end_line = basic_line.node.end_position().row as u32; diff --git a/crates/tracedecay-code-extraction/src/incremental/extraction.rs b/crates/tracedecay-code-extraction/src/incremental/extraction.rs index 0111f08bc5..9e9f08239f 100644 --- a/crates/tracedecay-code-extraction/src/incremental/extraction.rs +++ b/crates/tracedecay-code-extraction/src/incremental/extraction.rs @@ -1,4 +1,4 @@ -use tracedecay_domain::{ExtractionResult, NodeKind, SourceSpan}; +use tracedecay_domain::{NodeKind, SourceSpan}; use super::{ ParseCompleteness, ParseError, ParseInputEdit, ParseReport, ParseResetReason, ParseReuse, @@ -7,31 +7,12 @@ use super::{ use crate::LanguageExtractor; use crate::extraction_artifact::{ExtractedImportEvidenceV1, ExtractionArtifactV1}; use crate::parsed_extraction::{ - ParsedExtraction, ParsedExtractionArtifactV1, ParsedExtractionDisposition, - ParsedExtractionResetReason, ParsedExtractionScope, ParsedTraversalMetrics, - merge_changed_extraction, superseded_previous_nodes, + ParsedExtractionArtifactV1, ParsedExtractionDisposition, ParsedExtractionResetReason, + ParsedExtractionScope, ParsedTraversalMetrics, merge_changed_extraction, + superseded_previous_nodes, }; impl RetainedParseDocument { - /// Produce a complete canonical legacy graph from the retained tree. - pub fn extract_canonical( - &self, - extractor: &dyn LanguageExtractor, - report: &ParseReport, - previous: Option<&ExtractionResult>, - ) -> Result { - // Only the Noop and Incremental paths read the prior extraction; the - // Initial and Reset paths must not pay its deep clone. - let previous_artifact = match report.reuse { - ParseReuse::Noop | ParseReuse::Incremental => { - previous.cloned().map(ExtractionArtifactV1::from_result) - } - ParseReuse::Initial | ParseReuse::Reset { .. } => None, - }; - self.extract_canonical_artifact(extractor, report, previous_artifact.as_ref()) - .map(ParsedExtractionArtifactV1::into_parsed) - } - /// Produce a complete graph and structured evidence artifact from the /// current retained tree. Incremental deltas replace every affected import /// statement's rows, including when deletion produces no replacement row. @@ -184,7 +165,9 @@ impl RetainedParseDocument { // fallback does not re-enter the full traversal span. crate::hotpath_observe::measure_markdown_composite_fallback(|| { ExtractionArtifactV1::from_result( - extractor.extract(self.identity.logical_path(), &self.source), + extractor + .extract_artifact(self.identity.logical_path(), &self.source) + .result, ) }), ParsedExtractionResetReason::CompositeGrammar, diff --git a/crates/tracedecay-code-extraction/src/lib.rs b/crates/tracedecay-code-extraction/src/lib.rs index df019ba1b5..761a7fc378 100644 --- a/crates/tracedecay-code-extraction/src/lib.rs +++ b/crates/tracedecay-code-extraction/src/lib.rs @@ -92,9 +92,6 @@ mod lean_extractor; mod lua_extractor; #[cfg(feature = "lang-markdown")] mod markdown_extractor; -/// Grammar-free; always compiled so the retrieval layer can read section -/// structure without linking a tree-sitter bundle. -pub mod markdown_structure; #[cfg(feature = "lang-metal")] mod metal_extractor; #[cfg(feature = "lang-msbasic2")] @@ -346,12 +343,6 @@ pub trait LanguageExtractor: Send + Sync { crate::hotpath_observe::ExtractOutputCounts::from_artifact, ) } - - /// Nodes, edges, and unresolved refs of the whole document, parsed with - /// this extractor's own grammar. - fn extract(&self, file_path: &str, source: &str) -> ExtractionResult { - self.extract_artifact(file_path, source).result - } } /// Registry of all available language extractors. diff --git a/crates/tracedecay-code-extraction/src/markdown_structure.rs b/crates/tracedecay-code-extraction/src/markdown_structure.rs deleted file mode 100644 index ca34a9a1c7..0000000000 --- a/crates/tracedecay-code-extraction/src/markdown_structure.rs +++ /dev/null @@ -1,417 +0,0 @@ -//! Load-bearing structure *inside* a markdown section. -//! -//! The symbol graph stays heading-level: sections are the only markdown -//! symbols, and bullets are never exploded into graph nodes. But the content -//! that makes a plan document a *work ledger*, task-list checkboxes and their -//! checked state, nested bullets, ordered steps, tables, block quotes, fenced -//! code and its language tag, has to survive into retrieval as structure, not -//! as one flat text blob. Otherwise "which checklist items under 'Remaining -//! work' are still unchecked" is unanswerable without re-reading the file. -//! -//! This parser is deliberately line-based and grammar-free: it compiles with -//! no tree-sitter feature enabled, so the retrieval layer can depend on it -//! without pulling a grammar bundle. Fenced code is tracked so that a `- [ ]` -//! or `| a | b |` line inside a code sample is never mistaken for real -//! structure. - -use serde::{Deserialize, Serialize}; - -/// Spaces of indentation per nesting level for list items. -const INDENT_PER_LEVEL: u32 = 2; - -/// One `- [ ]` / `- [x]` task-list entry. -#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq, Serialize)] -pub struct MarkdownChecklistItem { - /// Absolute line in the enclosing file, in whatever base the caller passed - /// as `start_line` to [`parse_section_structure`]. Retrieval passes 1-based - /// lines so published items address the same rows `tracedecay_read - /// mode=lines` does. - pub line: u32, - /// Nesting depth, zero for a top-level item. - pub depth: u32, - /// `true` for `[x]` / `[X]`, `false` for `[ ]`. - pub checked: bool, - /// Item text with the marker and checkbox removed. - pub text: String, -} - -/// One bullet or ordered list entry that is not a task-list entry. -#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq, Serialize)] -pub struct MarkdownListItem { - pub line: u32, - pub depth: u32, - pub text: String, -} - -/// A pipe table, kept as a typed block with its shape rather than its cells. -#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq, Serialize)] -pub struct MarkdownTable { - pub start_line: u32, - pub end_line: u32, - /// Columns in the header row. - pub columns: u32, - /// Body rows, excluding the header and the delimiter row. - pub rows: u32, -} - -/// A contiguous run of `>` quoted lines. -#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq, Serialize)] -pub struct MarkdownBlockQuote { - pub start_line: u32, - pub end_line: u32, - pub lines: u32, -} - -/// A fenced code block and its info-string language, when tagged. -#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq, Serialize)] -pub struct MarkdownCodeBlock { - pub start_line: u32, - pub end_line: u32, - pub language: Option, -} - -/// Everything structural found in one section body. -#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq, Serialize)] -pub struct MarkdownSectionStructure { - pub checklist: Vec, - pub bullets: Vec, - pub ordered: Vec, - pub tables: Vec, - pub block_quotes: Vec, - pub code_blocks: Vec, -} - -impl MarkdownSectionStructure { - /// `true` when the section carries no structure worth publishing. - pub fn is_empty(&self) -> bool { - self.checklist.is_empty() - && self.bullets.is_empty() - && self.ordered.is_empty() - && self.tables.is_empty() - && self.block_quotes.is_empty() - && self.code_blocks.is_empty() - } - - /// Checklist items still unchecked. The question plan documents are - /// actually asked. - pub fn unchecked(&self) -> impl Iterator { - self.checklist.iter().filter(|item| !item.checked) - } -} - -/// Fence state for one `` ``` `` / `~~~` block. -struct OpenFence { - marker: u8, - width: usize, - start_line: u32, - language: Option, -} - -/// Parse the structure of `body`, whose first line is `start_line` in the -/// enclosing file. Reported line numbers are absolute in the caller's base: -/// pass a 0-based row for extractor-native numbering, or a 1-based row so the -/// results address the same lines a source read reports. -pub fn parse_section_structure(body: &str, start_line: u32) -> MarkdownSectionStructure { - let mut structure = MarkdownSectionStructure::default(); - let mut fence: Option = None; - let mut quote_run: Option<(u32, u32)> = None; - let mut table_run: Option = None; - - for (offset, raw_line) in body.lines().enumerate() { - let line_number = start_line.saturating_add(offset as u32); - let line = raw_line.trim_end_matches('\r'); - let trimmed = line.trim_start(); - - if let Some(open) = &fence { - if closes_fence(trimmed, open) { - structure.code_blocks.push(MarkdownCodeBlock { - start_line: open.start_line, - end_line: line_number, - language: open.language.clone(), - }); - fence = None; - } - continue; - } - if let Some(open) = open_fence(trimmed, line_number) { - flush_quote(&mut structure, &mut quote_run); - flush_table(&mut structure, &mut table_run); - fence = Some(open); - continue; - } - - if trimmed.starts_with('>') { - flush_table(&mut structure, &mut table_run); - match &mut quote_run { - Some((_, end)) => *end = line_number, - None => quote_run = Some((line_number, line_number)), - } - continue; - } - flush_quote(&mut structure, &mut quote_run); - - if let Some(cells) = table_row_cells(trimmed) { - match &mut table_run { - Some(table) => { - if is_table_delimiter(trimmed) { - // The delimiter row confirms the pending header; it is - // not itself a body row. - table.columns = table.columns.max(cells); - } else { - table.rows += 1; - } - table.end_line = line_number; - } - None => { - table_run = Some(MarkdownTable { - start_line: line_number, - end_line: line_number, - columns: cells, - rows: 0, - }); - } - } - continue; - } - flush_table(&mut structure, &mut table_run); - - let indent = leading_spaces(line); - let depth = indent / INDENT_PER_LEVEL; - if let Some(rest) = bullet_body(trimmed) { - match checkbox(rest) { - Some((checked, text)) => structure.checklist.push(MarkdownChecklistItem { - line: line_number, - depth, - checked, - text: text.trim().to_owned(), - }), - None => structure.bullets.push(MarkdownListItem { - line: line_number, - depth, - text: rest.trim().to_owned(), - }), - } - continue; - } - if let Some(rest) = ordered_body(trimmed) { - match checkbox(rest) { - Some((checked, text)) => structure.checklist.push(MarkdownChecklistItem { - line: line_number, - depth, - checked, - text: text.trim().to_owned(), - }), - None => structure.ordered.push(MarkdownListItem { - line: line_number, - depth, - text: rest.trim().to_owned(), - }), - } - } - } - - // An unterminated fence still describes real content: close it at the end - // of the section rather than dropping the block. - if let Some(open) = fence { - let end = start_line.saturating_add(body.lines().count().saturating_sub(1) as u32); - structure.code_blocks.push(MarkdownCodeBlock { - start_line: open.start_line, - end_line: end.max(open.start_line), - language: open.language, - }); - } - flush_quote(&mut structure, &mut quote_run); - flush_table(&mut structure, &mut table_run); - structure -} - -fn flush_quote(structure: &mut MarkdownSectionStructure, run: &mut Option<(u32, u32)>) { - if let Some((start, end)) = run.take() { - structure.block_quotes.push(MarkdownBlockQuote { - start_line: start, - end_line: end, - lines: end - start + 1, - }); - } -} - -fn flush_table(structure: &mut MarkdownSectionStructure, run: &mut Option) { - if let Some(table) = run.take() { - // A single pipe-bearing line is prose, not a table. - if table.end_line > table.start_line { - structure.tables.push(table); - } - } -} - -fn leading_spaces(line: &str) -> u32 { - let mut spaces = 0u32; - for ch in line.chars() { - match ch { - ' ' => spaces += 1, - '\t' => spaces += INDENT_PER_LEVEL, - _ => break, - } - } - spaces -} - -fn open_fence(trimmed: &str, line_number: u32) -> Option { - let marker = trimmed.as_bytes().first().copied()?; - if marker != b'`' && marker != b'~' { - return None; - } - let width = trimmed.bytes().take_while(|byte| *byte == marker).count(); - if width < 3 { - return None; - } - let info = trimmed[width..].trim(); - let language = info - .split_whitespace() - .next() - .filter(|language| !language.is_empty()) - .map(str::to_owned); - Some(OpenFence { - marker, - width, - start_line: line_number, - language, - }) -} - -fn closes_fence(trimmed: &str, open: &OpenFence) -> bool { - let width = trimmed - .bytes() - .take_while(|byte| *byte == open.marker) - .count(); - width >= open.width && trimmed[width..].trim().is_empty() -} - -/// The text after a `-`/`*`/`+` bullet marker, when the line is a bullet. -fn bullet_body(trimmed: &str) -> Option<&str> { - let mut chars = trimmed.chars(); - let marker = chars.next()?; - if !matches!(marker, '-' | '*' | '+') { - return None; - } - let rest = chars.as_str(); - // `---` is a setext underline or thematic break, not a list. - if rest.starts_with(marker) { - return None; - } - match rest.strip_prefix([' ', '\t']) { - Some(body) => Some(body), - None => None, - } -} - -/// The text after an `1.` / `1)` marker, when the line is an ordered item. -fn ordered_body(trimmed: &str) -> Option<&str> { - let digits = trimmed.bytes().take_while(u8::is_ascii_digit).count(); - if digits == 0 { - return None; - } - let rest = &trimmed[digits..]; - let rest = rest.strip_prefix('.').or_else(|| rest.strip_prefix(')'))?; - rest.strip_prefix([' ', '\t']) -} - -/// `(checked, remaining text)` when a list body opens with a task checkbox. -fn checkbox(body: &str) -> Option<(bool, &str)> { - let rest = body.strip_prefix('[')?; - let mut chars = rest.chars(); - let state = chars.next()?; - let rest = chars.as_str().strip_prefix(']')?; - match state { - ' ' => Some((false, rest)), - 'x' | 'X' => Some((true, rest)), - _ => None, - } -} - -/// Cell count when the line looks like a pipe-table row. -fn table_row_cells(trimmed: &str) -> Option { - if !trimmed.contains('|') { - return None; - } - let body = trimmed - .trim_start_matches('|') - .trim_end_matches('|') - .trim_end(); - if body.is_empty() { - return None; - } - Some(body.split('|').count() as u32) -} - -fn is_table_delimiter(trimmed: &str) -> bool { - let body = trimmed.trim_start_matches('|').trim_end_matches('|'); - body.contains('-') - && body - .chars() - .all(|ch| matches!(ch, '-' | ':' | '|' | ' ' | '\t')) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn parses_checklists_lists_tables_quotes_and_fences() { - let body = "\ -- [x] done -- [ ] open - - nested bullet -1. first step - -| a | b | -| - | - | -| 1 | 2 | - -> quoted - -```rust -fn x() {} -``` -"; - let structure = parse_section_structure(body, 10); - assert_eq!(structure.checklist.len(), 2); - assert!(structure.checklist[0].checked); - assert!(!structure.checklist[1].checked); - assert_eq!(structure.checklist[0].line, 10); - assert_eq!(structure.bullets.len(), 1); - assert_eq!(structure.bullets[0].depth, 1); - assert_eq!(structure.ordered.len(), 1); - assert_eq!(structure.tables.len(), 1); - assert_eq!(structure.tables[0].columns, 2); - assert_eq!(structure.tables[0].rows, 1); - assert_eq!(structure.block_quotes.len(), 1); - assert_eq!(structure.code_blocks.len(), 1); - assert_eq!(structure.code_blocks[0].language.as_deref(), Some("rust")); - assert_eq!(structure.unchecked().count(), 1); - } - - #[test] - fn fenced_code_does_not_mint_fake_structure() { - let body = "\ -``` -- [ ] not a task -| a | b | -``` -"; - let structure = parse_section_structure(body, 0); - assert!(structure.checklist.is_empty()); - assert!(structure.tables.is_empty()); - assert_eq!(structure.code_blocks.len(), 1); - } - - #[test] - fn unterminated_fence_closes_at_section_end() { - let body = "```python\nprint(1)\nprint(2)\n"; - let structure = parse_section_structure(body, 3); - assert_eq!(structure.code_blocks.len(), 1); - assert_eq!(structure.code_blocks[0].start_line, 3); - assert_eq!(structure.code_blocks[0].end_line, 5); - assert_eq!(structure.code_blocks[0].language.as_deref(), Some("python")); - } -} diff --git a/crates/tracedecay-code-extraction/src/msbasic2_extractor.rs b/crates/tracedecay-code-extraction/src/msbasic2_extractor.rs index 6455b3c7f2..30351064bf 100644 --- a/crates/tracedecay-code-extraction/src/msbasic2_extractor.rs +++ b/crates/tracedecay-code-extraction/src/msbasic2_extractor.rs @@ -13,7 +13,7 @@ use tree_sitter::{Node as TsNode, Tree}; use crate::basic_common::{ BasicLine, derive_function_name, find_subroutine_ranges, for_each_top_level_line, }; -use crate::common::{ExtractionState, local_node_id}; +use crate::common::{ExtractionState, basic_identifier_text, local_node_id}; use crate::traversal::find_direct_child_by_kind; use crate::types::{ ComplexityAnalysisV1, Edge, EdgeKind, ExtractionResult, Node, NodeKind, UnresolvedRef, @@ -194,7 +194,7 @@ impl MsBasic2Extractor { let Some(id_node) = find_direct_child_by_kind(var_node, "identifier") else { return; }; - let name = state.node_text(id_node); + let name = basic_identifier_text(state.source, id_node); let start_line = basic_line.node.start_position().row as u32; let end_line = basic_line.node.end_position().row as u32; diff --git a/crates/tracedecay-code-extraction/src/objc_extractor.rs b/crates/tracedecay-code-extraction/src/objc_extractor.rs index f5ff046285..c33849caad 100644 --- a/crates/tracedecay-code-extraction/src/objc_extractor.rs +++ b/crates/tracedecay-code-extraction/src/objc_extractor.rs @@ -6,7 +6,7 @@ use std::time::Instant; use tree_sitter::{Node as TsNode, Tree}; -use crate::common::{clean_c_doc_comment, docstring_from_preceding_comments, local_node_id}; +use crate::common::{clean_c_comment, docstring_from_preceding_comments, local_node_id}; use crate::complexity::{OBJC_COMPLEXITY, count_complexity}; use crate::traversal::{find_descendant_by_kind, find_direct_child_by_kind}; use crate::types::{ @@ -1030,7 +1030,7 @@ impl ObjcExtractor { /// Extract docstring for an `implementation_definition` by looking at preceding /// sibling comments within the `class_implementation`. fn extract_impl_method_docstring(state: &ExtractionState, node: TsNode<'_>) -> Option { - docstring_from_preceding_comments(state.source, node, clean_c_doc_comment) + docstring_from_preceding_comments(state.source, node, clean_c_comment) } /// Extract a method definition (has a body). @@ -1345,7 +1345,7 @@ impl ObjcExtractor { /// Extract docstrings from preceding comment nodes. fn extract_docstring(state: &ExtractionState, node: TsNode<'_>) -> Option { - docstring_from_preceding_comments(state.source, node, clean_c_doc_comment) + docstring_from_preceding_comments(state.source, node, clean_c_comment) } /// Extract first line of text as a signature. diff --git a/crates/tracedecay-code-extraction/src/parsed_extraction.rs b/crates/tracedecay-code-extraction/src/parsed_extraction.rs index bcaad7dbfb..660c6e8da1 100644 --- a/crates/tracedecay-code-extraction/src/parsed_extraction.rs +++ b/crates/tracedecay-code-extraction/src/parsed_extraction.rs @@ -117,14 +117,6 @@ impl ParsedExtractionArtifactV1 { metrics: parsed.metrics, } } - - pub(crate) fn into_parsed(self) -> ParsedExtraction { - ParsedExtraction { - result: self.artifact.result, - disposition: self.disposition, - metrics: self.metrics, - } - } } impl ParsedExtraction { diff --git a/crates/tracedecay-code-extraction/src/proto_extractor.rs b/crates/tracedecay-code-extraction/src/proto_extractor.rs index c7ea37cd3c..621757e2b2 100644 --- a/crates/tracedecay-code-extraction/src/proto_extractor.rs +++ b/crates/tracedecay-code-extraction/src/proto_extractor.rs @@ -5,7 +5,7 @@ use std::time::Instant; use tree_sitter::{Node as TsNode, Tree}; -use crate::common::local_node_id; +use crate::common::{clean_c_comment, docstring_from_preceding_comments, local_node_id}; use crate::traversal::find_direct_child_by_kind; use crate::types::{ ComplexityAnalysisV1, Edge, EdgeKind, ExtractionResult, Node, NodeKind, Visibility, @@ -915,28 +915,9 @@ impl ProtoExtractor { } } - /// Extract docstrings from `// comment` lines preceding definitions. - /// - /// Protobuf uses line comments (`//`) as documentation. We look for `comment` - /// sibling nodes that immediately precede the given definition node. + /// Protobuf documents definitions with the comments directly above them. fn extract_docstring(state: &ExtractionState, node: TsNode<'_>) -> Option { - let mut comments: Vec = Vec::new(); - let mut prev = node.prev_named_sibling(); - while let Some(prev_node) = prev { - if prev_node.kind() == "comment" { - let text = state.node_text(prev_node); - let stripped = text.trim_start_matches("//").trim().to_string(); - comments.push(stripped); - prev = prev_node.prev_named_sibling(); - } else { - break; - } - } - if comments.is_empty() { - return None; - } - comments.reverse(); - Some(comments.join("\n")) + docstring_from_preceding_comments(state.source, node, clean_c_comment) } fn build_artifact(state: ExtractionState<'_>, start: Instant) -> ExtractionArtifactV1 { diff --git a/crates/tracedecay-code-extraction/src/qbasic_extractor.rs b/crates/tracedecay-code-extraction/src/qbasic_extractor.rs index db99a15bb8..ecbb44128d 100644 --- a/crates/tracedecay-code-extraction/src/qbasic_extractor.rs +++ b/crates/tracedecay-code-extraction/src/qbasic_extractor.rs @@ -11,7 +11,7 @@ use std::time::Instant; use tree_sitter::{Node as TsNode, Tree}; -use crate::common::{ExtractionState, local_node_id}; +use crate::common::{ExtractionState, basic_identifier_text, local_node_id}; use crate::complexity::{ComplexityMetrics, QBASIC_COMPLEXITY, count_complexity}; use crate::traversal::find_direct_child_by_kind; use crate::types::{ @@ -156,7 +156,7 @@ impl QBasicExtractor { let Some(id_node) = find_direct_child_by_kind(const_stmt, "identifier") else { return; }; - let name = state.node_text(id_node); + let name = basic_identifier_text(state.source, id_node); let start_line = line.start_position().row as u32; let end_line = line.end_position().row as u32; @@ -223,7 +223,7 @@ impl QBasicExtractor { let Some(id_node) = find_direct_child_by_kind(dim_var, "identifier") else { return; }; - let name = state.node_text(id_node); + let name = basic_identifier_text(state.source, id_node); let start_line = line.start_position().row as u32; let end_line = line.end_position().row as u32; @@ -280,7 +280,7 @@ impl QBasicExtractor { let Some(name_node) = node.child_by_field_name("name") else { return; }; - let name = state.node_text(name_node); + let name = basic_identifier_text(state.source, name_node); let start_line = node.start_position().row as u32; let end_line = node.end_position().row as u32; @@ -355,7 +355,7 @@ impl QBasicExtractor { /// Visit a `type_member` inside a TYPE block and emit a Field node. fn visit_type_member(state: &mut ExtractionState, member: TsNode<'_>) { let name = match find_direct_child_by_kind(member, "identifier") { - Some(id_node) => state.node_text(id_node), + Some(id_node) => basic_identifier_text(state.source, id_node), None => return, }; @@ -420,7 +420,7 @@ impl QBasicExtractor { let Some(name_node) = node.child_by_field_name("name") else { return; }; - let name = state.node_text(name_node); + let name = basic_identifier_text(state.source, name_node); let start_line = node.start_position().row as u32; let end_line = node.end_position().row as u32; @@ -497,7 +497,7 @@ impl QBasicExtractor { let Some(name_node) = node.child_by_field_name("name") else { return; }; - let name = state.node_text(name_node); + let name = basic_identifier_text(state.source, name_node); let start_line = node.start_position().row as u32; let end_line = node.end_position().row as u32; @@ -566,7 +566,7 @@ impl QBasicExtractor { /// Extract a call reference from a `call_statement` node. fn extract_call_from_call_statement(state: &mut ExtractionState, call_stmt: TsNode<'_>) { let target_name = match find_direct_child_by_kind(call_stmt, "identifier") { - Some(id_node) => state.node_text(id_node), + Some(id_node) => basic_identifier_text(state.source, id_node), None => return, }; diff --git a/crates/tracedecay-code-extraction/tests/extract_alloc.rs b/crates/tracedecay-code-extraction/tests/extract_alloc.rs index 644b939ba8..6b71f1d24c 100644 --- a/crates/tracedecay-code-extraction/tests/extract_alloc.rs +++ b/crates/tracedecay-code-extraction/tests/extract_alloc.rs @@ -294,7 +294,7 @@ fn extract_both_and_compare( source: &str, grammar_key: &str, ) -> (ParsedExtraction, usize) { - let full = extractor.extract(file_path, source); + let full = extractor.extract_artifact(file_path, source).result; assert!(full.errors.is_empty(), "extract errors: {:?}", full.errors); let tree = parse_with_grammar(grammar_key, source); @@ -653,7 +653,10 @@ fn representative_language_walks_allocate_by_changed_region() { case.file_path, case.source.len() ); - let cold = case.extractor.extract(case.file_path, &case.source); + let cold = case + .extractor + .extract_artifact(case.file_path, &case.source) + .result; assert!( cold.errors.is_empty(), "{} cold errors: {:?}", @@ -741,7 +744,7 @@ fn representative_language_walks_allocate_by_changed_region() { #[test] fn bash_changed_region_reset_walk_does_not_copy_source() { let source = regional_fixture("kept() { echo kept; }\nkept\ntiny() { echo tiny; }\n"); - let cold = BashExtractor.extract("region.sh", &source); + let cold = BashExtractor.extract_artifact("region.sh", &source).result; let tree = parse_with_grammar("bash", &source); let region = trailing_region(&source, "tiny()"); let (incremental, walk_bytes) = measure_allocation(|| { @@ -1196,7 +1199,10 @@ fn every_migrated_language_walk_allocates_by_changed_region() { case.file_path, source.len() ); - let cold = case.extractor.extract(case.file_path, &source); + let cold = case + .extractor + .extract_artifact(case.file_path, &source) + .result; let tree = parse_with_grammar(case.grammar_key, &source); let region = trailing_region(&source, case.needle); let regions = [region]; diff --git a/crates/tracedecay-code-extraction/tests/main/astro.rs b/crates/tracedecay-code-extraction/tests/main/astro.rs index 3289b9f10f..70d14e57f2 100644 --- a/crates/tracedecay-code-extraction/tests/main/astro.rs +++ b/crates/tracedecay-code-extraction/tests/main/astro.rs @@ -5,7 +5,7 @@ use tracedecay_domain::*; #[test] fn test_astro_file_node() { let source = "---\nconst x = 1;\n---\n

hi

"; - let result = AstroExtractor.extract("page.astro", source); + let result = AstroExtractor.extract_artifact("page.astro", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let files: Vec<_> = result .nodes @@ -24,7 +24,7 @@ export function formatTitle(t: string): string { } --- "#; - let result = AstroExtractor.extract("page.astro", source); + let result = AstroExtractor.extract_artifact("page.astro", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let fmt = result .nodes @@ -38,7 +38,9 @@ export function formatTitle(t: string): string { fn test_astro_line_numbers_are_original_file_positions() { // `greet` is on line 2 (0-indexed) in the full .astro file. let source = "---\n\nexport function greet(): void {}\n---\n

hi

"; - let result = AstroExtractor.extract("greet.astro", source); + let result = AstroExtractor + .extract_artifact("greet.astro", source) + .result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let greet = result.nodes.iter().find(|n| n.name == "greet").unwrap(); assert_eq!( @@ -51,7 +53,9 @@ fn test_astro_line_numbers_are_original_file_positions() { #[test] fn test_astro_no_frontmatter_returns_file_node_only() { let source = "

Static

"; - let result = AstroExtractor.extract("static.astro", source); + let result = AstroExtractor + .extract_artifact("static.astro", source) + .result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let non_file: Vec<_> = result .nodes @@ -65,7 +69,7 @@ fn test_astro_no_frontmatter_returns_file_node_only() { fn test_astro_template_markup_does_not_produce_symbols() { // HTML after the closing `---` must not leak TypeScript symbols. let source = "---\nconst greeting = 'hello';\n---\n

{greeting}

\n"; - let result = AstroExtractor.extract("page.astro", source); + let result = AstroExtractor.extract_artifact("page.astro", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let fns: Vec<_> = result .nodes @@ -82,7 +86,9 @@ fn test_astro_template_markup_does_not_produce_symbols() { #[test] fn test_astro_fixture() { let source = include_str!("../../fixtures/sample.astro"); - let result = AstroExtractor.extract("sample.astro", source); + let result = AstroExtractor + .extract_artifact("sample.astro", source) + .result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let names: Vec<_> = result.nodes.iter().map(|n| n.name.as_str()).collect(); assert!( diff --git a/crates/tracedecay-code-extraction/tests/main/bash.rs b/crates/tracedecay-code-extraction/tests/main/bash.rs index 5ff298ddbe..a6045d2f23 100644 --- a/crates/tracedecay-code-extraction/tests/main/bash.rs +++ b/crates/tracedecay-code-extraction/tests/main/bash.rs @@ -24,7 +24,7 @@ fn bash_overlay(version: i64, content: &str) -> ParseDocumentIdentity { fn test_bash_call_sites() { let source = std::fs::read_to_string("../../tests/fixtures/sample.sh").unwrap(); let extractor = BashExtractor; - let result = extractor.extract("sample.sh", &source); + let result = extractor.extract_artifact("sample.sh", &source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let call_refs: Vec<_> = result @@ -37,7 +37,6 @@ fn test_bash_call_sites() { .iter() .find(|node| node.kind == NodeKind::Module && node.name == "sample") .expect("script execution scope"); - assert!(!call_refs.is_empty(), "should have call refs"); assert!( call_refs.iter().any(|r| r.reference_name == "echo"), "should find echo call" @@ -96,7 +95,7 @@ fn test_bash_incremental_edit_rebuilds_script_scope() { RetainedParseDocument::open(bash_overlay(1, "1"), "bash", before, ParseLimits::default()) .expect("initial Bash parse"); let initial = document - .extract_canonical(&BashExtractor, &opened, None) + .extract_canonical_artifact(&BashExtractor, &opened, None) .expect("initial Bash extraction"); let report = document @@ -104,7 +103,7 @@ fn test_bash_incremental_edit_rebuilds_script_scope() { .expect("incremental Bash parse"); assert_eq!(report.reuse, ParseReuse::Incremental); let updated = document - .extract_canonical(&BashExtractor, &report, Some(&initial.result)) + .extract_canonical_artifact(&BashExtractor, &report, Some(&initial.artifact)) .expect("updated Bash extraction"); assert_eq!( updated.disposition, @@ -114,12 +113,14 @@ fn test_bash_incremental_edit_rebuilds_script_scope() { ); let script = updated + .artifact .result .nodes .iter() .find(|node| node.kind == NodeKind::Module) .expect("script module"); let mut functions = updated + .artifact .result .nodes .iter() @@ -129,6 +130,7 @@ fn test_bash_incremental_edit_rebuilds_script_scope() { functions.sort_unstable(); assert_eq!(functions, ["kept", "newfn", "oldfn"]); let top_level_calls = updated + .artifact .result .unresolved_refs .iter() @@ -144,7 +146,7 @@ fn test_bash_incremental_edit_rebuilds_script_scope() { fn test_bash_docstrings() { let source = std::fs::read_to_string("../../tests/fixtures/sample.sh").unwrap(); let extractor = BashExtractor; - let result = extractor.extract("sample.sh", &source); + let result = extractor.extract_artifact("sample.sh", &source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let log_fn = result @@ -192,20 +194,3 @@ fn test_bash_docstrings() { main_fn.docstring ); } - -#[test] -fn test_bash_contains_edges() { - let source = std::fs::read_to_string("../../tests/fixtures/sample.sh").unwrap(); - let extractor = BashExtractor; - let result = extractor.extract("sample.sh", &source); - let contains: Vec<_> = result - .edges - .iter() - .filter(|e| e.kind == EdgeKind::Contains) - .collect(); - assert!( - contains.len() >= 8, - "should have >= 8 Contains edges, got {}", - contains.len() - ); -} diff --git a/crates/tracedecay-code-extraction/tests/main/batch.rs b/crates/tracedecay-code-extraction/tests/main/batch.rs index e0849cf809..b42ac64189 100644 --- a/crates/tracedecay-code-extraction/tests/main/batch.rs +++ b/crates/tracedecay-code-extraction/tests/main/batch.rs @@ -6,7 +6,7 @@ use tracedecay_domain::*; fn test_batch_call_sites() { let source = std::fs::read_to_string("../../tests/fixtures/sample.bat").unwrap(); let extractor = BatchExtractor; - let result = extractor.extract("sample.bat", &source); + let result = extractor.extract_artifact("sample.bat", &source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let call_refs: Vec<_> = result @@ -14,7 +14,6 @@ fn test_batch_call_sites() { .iter() .filter(|r| r.reference_kind == EdgeKind::Calls) .collect(); - assert!(!call_refs.is_empty(), "should have call refs"); assert!( call_refs.iter().any(|r| r.reference_name == "Log"), "should find Log call" @@ -39,7 +38,7 @@ fn test_batch_call_sites() { fn test_batch_docstrings() { let source = std::fs::read_to_string("../../tests/fixtures/sample.bat").unwrap(); let extractor = BatchExtractor; - let result = extractor.extract("sample.bat", &source); + let result = extractor.extract_artifact("sample.bat", &source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let log_fn = result @@ -47,15 +46,9 @@ fn test_batch_docstrings() { .iter() .find(|n| n.kind == NodeKind::Function && n.name == "Log") .expect("Log function not found"); - assert!(log_fn.docstring.is_some(), "Log should have docstring"); - assert!( - log_fn - .docstring - .as_ref() - .unwrap() - .contains("Logs a message"), - "docstring: {:?}", - log_fn.docstring + assert_eq!( + log_fn.docstring.as_deref(), + Some("Logs a message with timestamp.") ); let vc_fn = result @@ -88,20 +81,3 @@ fn test_batch_docstrings() { main_fn.docstring ); } - -#[test] -fn test_batch_contains_edges() { - let source = std::fs::read_to_string("../../tests/fixtures/sample.bat").unwrap(); - let extractor = BatchExtractor; - let result = extractor.extract("sample.bat", &source); - let contains: Vec<_> = result - .edges - .iter() - .filter(|e| e.kind == EdgeKind::Contains) - .collect(); - assert!( - contains.len() >= 7, - "should have >= 7 Contains edges, got {}", - contains.len() - ); -} diff --git a/crates/tracedecay-code-extraction/tests/main/c.rs b/crates/tracedecay-code-extraction/tests/main/c.rs index 2c58ebb083..30536e5d91 100644 --- a/crates/tracedecay-code-extraction/tests/main/c.rs +++ b/crates/tracedecay-code-extraction/tests/main/c.rs @@ -6,6 +6,8 @@ use tracedecay_domain::*; // into this module's own namespace, so the tests below call them unqualified // without each extractor module re-declaring the support module. include!("support/docstrings.rs"); +include!("support/edges.rs"); + #[test] fn test_c_file_node_is_root() { let source = r#" @@ -14,7 +16,7 @@ int main() { } "#; let extractor = CExtractor; - let result = extractor.extract("test.c", source); + let result = extractor.extract_artifact("test.c", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let files: Vec<_> = result .nodes @@ -33,7 +35,7 @@ int add(int a, int b) { } "#; let extractor = CExtractor; - let result = extractor.extract("math.c", source); + let result = extractor.extract_artifact("math.c", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let fns: Vec<_> = result .nodes @@ -54,7 +56,7 @@ int add(int a, int b); void process(const char *data); "#; let extractor = CExtractor; - let result = extractor.extract("math.h", source); + let result = extractor.extract_artifact("math.h", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let fns: Vec<_> = result .nodes @@ -76,7 +78,7 @@ struct Point { }; "#; let extractor = CExtractor; - let result = extractor.extract("point.h", source); + let result = extractor.extract_artifact("point.h", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let structs: Vec<_> = result .nodes @@ -106,7 +108,7 @@ union Data { }; "#; let extractor = CExtractor; - let result = extractor.extract("data.h", source); + let result = extractor.extract_artifact("data.h", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let union_node = result @@ -146,7 +148,7 @@ enum Color { }; "#; let extractor = CExtractor; - let result = extractor.extract("color.h", source); + let result = extractor.extract_artifact("color.h", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let enums: Vec<_> = result .nodes @@ -173,7 +175,7 @@ fn test_c_typedef() { typedef unsigned long ulong; "#; let extractor = CExtractor; - let result = extractor.extract("types.h", source); + let result = extractor.extract_artifact("types.h", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let typedefs: Vec<_> = result .nodes @@ -191,7 +193,7 @@ fn test_c_preprocessor_define() { #define PI 3.14159 "#; let extractor = CExtractor; - let result = extractor.extract("defs.h", source); + let result = extractor.extract_artifact("defs.h", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let macros: Vec<_> = result .nodes @@ -210,7 +212,7 @@ fn test_c_include() { #include "myheader.h" "#; let extractor = CExtractor; - let result = extractor.extract("main.c", source); + let result = extractor.extract_artifact("main.c", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let includes: Vec<_> = result .nodes @@ -228,7 +230,7 @@ static int helper(int x) { } "#; let extractor = CExtractor; - let result = extractor.extract("utils.c", source); + let result = extractor.extract_artifact("utils.c", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let fns: Vec<_> = result .nodes @@ -248,7 +250,7 @@ int public_func(void) { } "#; let extractor = CExtractor; - let result = extractor.extract("api.c", source); + let result = extractor.extract_artifact("api.c", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let fns: Vec<_> = result .nodes @@ -289,7 +291,7 @@ int mul(int a, int b) { for (style, source, expected) in cases { let extractor = CExtractor; - let result = extractor.extract("math.c", source); + let result = extractor.extract_artifact("math.c", source).result; assert_node_docstring(style, &result, NodeKind::Function, None, expected); } } @@ -308,17 +310,19 @@ int main() { } "#; let extractor = CExtractor; - let result = extractor.extract("main.c", source); + let result = extractor.extract_artifact("main.c", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let call_refs: Vec<_> = result .unresolved_refs .iter() .filter(|r| r.reference_kind == EdgeKind::Calls) .collect(); - assert!( - call_refs.len() >= 2, - "should have call refs for helper and printf, got: {:?}", + assert_eq!( call_refs + .iter() + .map(|x| x.reference_name.as_str()) + .collect::>(), + ["helper", "printf"] ); assert!(call_refs.iter().any(|r| r.reference_name == "helper")); assert!(call_refs.iter().any(|r| r.reference_name == "printf")); @@ -331,18 +335,11 @@ void foo() {} void bar() {} "#; let extractor = CExtractor; - let result = extractor.extract("test.c", source); + let result = extractor.extract_artifact("test.c", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); - let contains: Vec<_> = result - .edges - .iter() - .filter(|e| e.kind == EdgeKind::Contains) - .collect(); - // File -> foo, File -> bar = at least 2 - assert!( - contains.len() >= 2, - "should have Contains edges from File to Functions, got: {}", - contains.len() + assert_eq!( + edge_pairs(&result, EdgeKind::Contains), + [("test.c", "foo"), ("test.c", "bar")] ); } @@ -355,7 +352,7 @@ struct Rect { }; "#; let extractor = CExtractor; - let result = extractor.extract("rect.h", source); + let result = extractor.extract_artifact("rect.h", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let struct_node = result @@ -382,7 +379,7 @@ fn test_c_function_pointer_typedef() { typedef int (*compare_fn)(const void *, const void *); "#; let extractor = CExtractor; - let result = extractor.extract("types.h", source); + let result = extractor.extract_artifact("types.h", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let typedefs: Vec<_> = result .nodes @@ -402,7 +399,7 @@ typedef struct { } Point; "#; let extractor = CExtractor; - let result = extractor.extract("point.h", source); + let result = extractor.extract_artifact("point.h", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); // Should have a Typedef node for Point @@ -442,7 +439,7 @@ enum LogLevel { }; "#; let extractor = CExtractor; - let result = extractor.extract("log.h", source); + let result = extractor.extract_artifact("log.h", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let variants: Vec<_> = result .nodes @@ -459,7 +456,7 @@ fn test_c_global_variable_docstring() { int global_counter = 0; "#; let extractor = CExtractor; - let result = extractor.extract("globals.c", source); + let result = extractor.extract_artifact("globals.c", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let statics: Vec<_> = result .nodes @@ -480,7 +477,7 @@ fn test_c_static_global_variable() { static int counter = 0; "#; let extractor = CExtractor; - let result = extractor.extract("state.c", source); + let result = extractor.extract_artifact("state.c", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let statics: Vec<_> = result .nodes diff --git a/crates/tracedecay-code-extraction/tests/main/clone_body_tokens.rs b/crates/tracedecay-code-extraction/tests/main/clone_body_tokens.rs index 76f1e114da..451bfb87c9 100644 --- a/crates/tracedecay-code-extraction/tests/main/clone_body_tokens.rs +++ b/crates/tracedecay-code-extraction/tests/main/clone_body_tokens.rs @@ -258,7 +258,7 @@ fn bodies_above_the_token_maximum_are_excluded_without_streams() { let body = &under.clone_bodies[0]; assert!(body.non_trivia_token_count <= MAX_AUTOMATIC_CLONE_BODY_TOKENS_V1); assert_eq!(body.eligibility, CloneBodyEligibilityV1::Eligible); - assert!(!body.conservative_tokens.is_empty()); + assert_eq!(body.conservative_tokens.len(), 10164); } #[test] @@ -292,7 +292,8 @@ fn body_bytes_are_bounded_before_a_large_literal_is_tokenized() { #[test] fn clone_bodies_bind_to_method_and_stable_arrow_occurrences() { - for (artifact, expected_kind, expected_language) in [ + // Both bodies are the `{ load(); }` block. + for (artifact, expected_kind, expected_language, expected_span) in [ ( RustExtractor.extract_artifact( "src/store.rs", @@ -300,11 +301,13 @@ fn clone_bodies_bind_to_method_and_stable_arrow_occurrences() { ), NodeKind::Method, "rust", + (42, 53), ), ( TypeScriptExtractor.extract_artifact("src/store.ts", "const read = () => { load(); };"), NodeKind::ArrowFunction, "typescript", + (19, 30), ), ] { let body = artifact.clone_bodies.first().expect("clone body"); @@ -317,7 +320,10 @@ fn clone_bodies_bind_to_method_and_stable_arrow_occurrences() { assert_eq!(body.symbol_occurrence_id, callable.id); assert_eq!(body.symbol_kind, expected_kind); assert_eq!(body.language, expected_language); - assert!(!body.body_span.is_empty()); + assert_eq!( + (body.body_span.start_byte, body.body_span.end_byte), + expected_span + ); } } @@ -325,7 +331,20 @@ fn clone_bodies_bind_to_method_and_stable_arrow_occurrences() { fn extracted_token_kinds_borrow_the_grammar_table_without_changing_the_wire_shape() { let source = "pub fn publish(input: &str) -> bool {\n let trimmed = input.trim();\n let ready = !trimmed.is_empty();\n let flagged = trimmed.starts_with('!');\n let long = trimmed.len() > 4;\n ready && long && !flagged\n}\n"; let emitted = tokens(&RustExtractor, "borrowed.rs", source); - assert!(!emitted.is_empty()); + assert_eq!(emitted.len(), 92); + let texts: Vec<&str> = emitted + .iter() + .filter_map(|token| match token { + ConservativeCloneTokenV1::Syntax { text, .. } => Some(text.as_str()), + _ => None, + }) + .collect(); + assert_eq!( + texts.join(" "), + "{ let trimmed = input . trim ( ) ; let ready = ! trimmed . is_empty ( ) ; \ + let flagged = trimmed . starts_with ( '!' ) ; let long = trimmed . len ( ) > 4 ; \ + ready && long && ! flagged }" + ); for token in &emitted { let kind = match token { ConservativeCloneTokenV1::StructureStart { syntax_kind } @@ -339,9 +358,9 @@ fn extracted_token_kinds_borrow_the_grammar_table_without_changing_the_wire_shap } let encoded = serde_json::to_string(&emitted[0]).expect("token encodes"); - assert!( - encoded.contains("\"syntax_kind\":\""), - "the persisted clone-token shape changed: {encoded}" + assert_eq!( + encoded, r#"{"kind":"structure_start","syntax_kind":"block"}"#, + "the persisted clone-token shape changed" ); let decoded: ConservativeCloneTokenV1 = serde_json::from_str(&encoded).expect("token decodes"); assert_eq!(decoded, emitted[0]); diff --git a/crates/tracedecay-code-extraction/tests/main/cobol.rs b/crates/tracedecay-code-extraction/tests/main/cobol.rs index a81df0e914..4b0d143382 100644 --- a/crates/tracedecay-code-extraction/tests/main/cobol.rs +++ b/crates/tracedecay-code-extraction/tests/main/cobol.rs @@ -5,7 +5,7 @@ use tracedecay_domain::*; fn extract_fixture() -> ExtractionResult { let source = std::fs::read_to_string("../../tests/fixtures/sample.cob").unwrap(); let extractor = CobolExtractor; - let result = extractor.extract("sample.cob", &source); + let result = extractor.extract_artifact("sample.cob", &source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); result } @@ -60,7 +60,6 @@ fn test_cobol_perform_calls() { .iter() .filter(|r| r.reference_kind == EdgeKind::Calls) .collect(); - assert!(!calls.is_empty(), "expected call site refs"); assert!( calls.iter().any(|r| r.reference_name == "VALIDATE-CONFIG"), "expected call to VALIDATE-CONFIG, got: {:?}", @@ -85,51 +84,27 @@ fn test_cobol_perform_calls() { #[test] fn test_cobol_docstrings() { let result = extract_fixture(); - let validate = result - .nodes - .iter() - .find(|n| n.kind == NodeKind::Function && n.name == "VALIDATE-CONFIG"); - assert!(validate.is_some(), "VALIDATE-CONFIG not found"); - assert!( - validate.unwrap().docstring.is_some(), - "VALIDATE-CONFIG should have docstring" - ); - - let log_msg = result - .nodes - .iter() - .find(|n| n.kind == NodeKind::Function && n.name == "LOG-MESSAGE"); - assert!(log_msg.is_some(), "LOG-MESSAGE not found"); - assert!( - log_msg.unwrap().docstring.is_some(), - "LOG-MESSAGE should have docstring" - ); - - let connect = result + let docs: Vec<(&str, &str)> = result .nodes .iter() - .find(|n| n.kind == NodeKind::Function && n.name == "CONNECT-SERVER"); - assert!(connect.is_some(), "CONNECT-SERVER not found"); - assert!( - connect.unwrap().docstring.is_some(), - "CONNECT-SERVER should have docstring" - ); - - let disconnect = result - .nodes - .iter() - .find(|n| n.kind == NodeKind::Function && n.name == "DISCONNECT-SERVER"); - assert!(disconnect.is_some(), "DISCONNECT-SERVER not found"); - assert!( - disconnect.unwrap().docstring.is_some(), - "DISCONNECT-SERVER should have docstring" - ); - - let max_retries = result.nodes.iter().find(|n| n.name == "WS-MAX-RETRIES"); - assert!(max_retries.is_some(), "WS-MAX-RETRIES not found"); - assert!( - max_retries.unwrap().docstring.is_some(), - "WS-MAX-RETRIES should have docstring" + .filter_map(|n| Some((n.name.as_str(), n.docstring.as_deref()?))) + .collect(); + assert_eq!( + docs, + [ + ("WS-MAX-RETRIES", "Maximum number of retries."), + ("WS-DEFAULT-PORT", "Default port number."), + ("WS-HOST", "Connection host name."), + ("WS-PORT", "Connection port."), + ("WS-CONNECTED", "Connection status flag."), + ("WS-LOG-LEVEL", "Log level."), + ("WS-LOG-MESSAGE", "Log message text."), + ("WS-RETRY-COUNT", "Retry counter."), + ("VALIDATE-CONFIG", "Validates the configuration."), + ("LOG-MESSAGE", "Logs a message with timestamp."), + ("CONNECT-SERVER", "Connects to the remote server."), + ("DISCONNECT-SERVER", "Disconnects from the server."), + ] ); } diff --git a/crates/tracedecay-code-extraction/tests/main/complexity_budget.rs b/crates/tracedecay-code-extraction/tests/main/complexity_budget.rs index 386f8292ce..a86349f642 100644 --- a/crates/tracedecay-code-extraction/tests/main/complexity_budget.rs +++ b/crates/tracedecay-code-extraction/tests/main/complexity_budget.rs @@ -142,7 +142,9 @@ fn deep_nesting_is_measured_rather_than_asserted_against() { fn extracted_nodes_carry_the_analysis_state() { let statements = TRAVERSAL_BUDGET / 4; let over_budget = body_with_statements(statements); - let result = RustExtractor.extract("huge.rs", &over_budget); + let result = RustExtractor + .extract_artifact("huge.rs", &over_budget) + .result; let body = result .nodes .iter() @@ -153,7 +155,9 @@ fn extracted_nodes_carry_the_analysis_state() { ComplexityAnalysisV1::TraversalBudgetExhausted ); - let small = RustExtractor.extract("small.rs", &body_with_statements(3)); + let small = RustExtractor + .extract_artifact("small.rs", &body_with_statements(3)) + .result; let body = small .nodes .iter() diff --git a/crates/tracedecay-code-extraction/tests/main/cpp.rs b/crates/tracedecay-code-extraction/tests/main/cpp.rs index 99eb1e9b79..884a23a842 100644 --- a/crates/tracedecay-code-extraction/tests/main/cpp.rs +++ b/crates/tracedecay-code-extraction/tests/main/cpp.rs @@ -6,6 +6,8 @@ use tracedecay_domain::*; // into this module's own namespace, so the tests below call them unqualified // without each extractor module re-declaring the support module. include!("support/docstrings.rs"); +include!("support/edges.rs"); + #[test] fn test_cpp_file_node_is_root() { let source = r#" @@ -14,7 +16,7 @@ int main() { } "#; let extractor = CppExtractor; - let result = extractor.extract("test.cpp", source); + let result = extractor.extract_artifact("test.cpp", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let files: Vec<_> = result .nodes @@ -33,7 +35,7 @@ int add(int a, int b) { } "#; let extractor = CppExtractor; - let result = extractor.extract("math.cpp", source); + let result = extractor.extract_artifact("math.cpp", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let fns: Vec<_> = result .nodes @@ -61,7 +63,7 @@ private: }; "#; let extractor = CppExtractor; - let result = extractor.extract("dog.cpp", source); + let result = extractor.extract_artifact("dog.cpp", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let classes: Vec<_> = result @@ -109,7 +111,7 @@ public: }; "#; let extractor = CppExtractor; - let result = extractor.extract("foo.cpp", source); + let result = extractor.extract_artifact("foo.cpp", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let constructors: Vec<_> = result @@ -117,10 +119,12 @@ public: .iter() .filter(|n| n.kind == NodeKind::Constructor) .collect(); - assert!( - constructors.len() >= 2, - "should have 2 constructors, got: {:?}", + assert_eq!( constructors + .iter() + .map(|x| x.name.as_str()) + .collect::>(), + ["Foo", "Foo"] ); // Destructor can also be a Method with special name @@ -141,7 +145,7 @@ namespace mylib { } "#; let extractor = CppExtractor; - let result = extractor.extract("lib.cpp", source); + let result = extractor.extract_artifact("lib.cpp", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let namespaces: Vec<_> = result @@ -153,16 +157,13 @@ namespace mylib { assert_eq!(namespaces[0].name, "mylib"); // Namespace should contain the function - let ns_id = &namespaces[0].id; - let contains_from_ns: Vec<_> = result - .edges - .iter() - .filter(|e| e.kind == EdgeKind::Contains && e.source == *ns_id) - .collect(); - assert!( - !contains_from_ns.is_empty(), - "namespace should contain children, got: {:?}", - contains_from_ns + assert_eq!( + edge_pairs(&result, EdgeKind::Contains), + [ + ("lib.cpp", "mylib"), + ("mylib", "helper"), + ("mylib", "value") + ] ); let fns: Vec<_> = result @@ -183,7 +184,7 @@ T maximum(T a, T b) { } "#; let extractor = CppExtractor; - let result = extractor.extract("tmpl.cpp", source); + let result = extractor.extract_artifact("tmpl.cpp", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let templates: Vec<_> = result @@ -205,7 +206,7 @@ public: }; "#; let extractor = CppExtractor; - let result = extractor.extract("shape.cpp", source); + let result = extractor.extract_artifact("shape.cpp", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let methods: Vec<_> = result @@ -247,7 +248,7 @@ private: }; "#; let extractor = CppExtractor; - let result = extractor.extract("widget.cpp", source); + let result = extractor.extract_artifact("widget.cpp", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let fields: Vec<_> = result @@ -296,7 +297,7 @@ public: }; "#; let extractor = CppExtractor; - let result = extractor.extract("animals.cpp", source); + let result = extractor.extract_artifact("animals.cpp", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let classes: Vec<_> = result @@ -312,11 +313,6 @@ public: .iter() .filter(|r| r.reference_kind == EdgeKind::Extends) .collect(); - assert!( - !extends_refs.is_empty(), - "should have Extends refs, got: {:?}", - extends_refs - ); assert!(extends_refs.iter().any(|r| r.reference_name == "Animal")); } @@ -330,7 +326,7 @@ struct Point { }; "#; let extractor = CppExtractor; - let result = extractor.extract("point.cpp", source); + let result = extractor.extract_artifact("point.cpp", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let structs: Vec<_> = result @@ -367,7 +363,7 @@ enum Color { }; "#; let extractor = CppExtractor; - let result = extractor.extract("color.cpp", source); + let result = extractor.extract_artifact("color.cpp", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let enums: Vec<_> = result @@ -399,7 +395,7 @@ union Data { }; "#; let extractor = CppExtractor; - let result = extractor.extract("data.cpp", source); + let result = extractor.extract_artifact("data.cpp", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let unions: Vec<_> = result @@ -417,7 +413,7 @@ fn test_cpp_typedef() { typedef unsigned long ulong; "#; let extractor = CppExtractor; - let result = extractor.extract("types.hpp", source); + let result = extractor.extract_artifact("types.hpp", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let typedefs: Vec<_> = result @@ -437,7 +433,7 @@ fn test_cpp_preprocessor_and_include() { #include "myheader.h" "#; let extractor = CppExtractor; - let result = extractor.extract("main.cpp", source); + let result = extractor.extract_artifact("main.cpp", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let macros: Vec<_> = result @@ -462,7 +458,7 @@ fn test_cpp_using_declaration() { using namespace std; "#; let extractor = CppExtractor; - let result = extractor.extract("main.cpp", source); + let result = extractor.extract_artifact("main.cpp", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let uses: Vec<_> = result @@ -513,7 +509,7 @@ int divide(int a, int b) { for (style, source, expected) in cases { let extractor = CppExtractor; - let result = extractor.extract("math.cpp", source); + let result = extractor.extract_artifact("math.cpp", source).result; assert_node_docstring(style, &result, NodeKind::Function, None, expected); } } @@ -531,7 +527,7 @@ int main() { } "#; let extractor = CppExtractor; - let result = extractor.extract("main.cpp", source); + let result = extractor.extract_artifact("main.cpp", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let call_refs: Vec<_> = result @@ -540,11 +536,9 @@ int main() { .filter(|r| r.reference_kind == EdgeKind::Calls) .collect(); assert!( - !call_refs.is_empty(), - "should have call refs for helper, got: {:?}", - call_refs + call_refs.iter().any(|r| r.reference_name == "helper"), + "should have call refs for helper, got: {call_refs:?}" ); - assert!(call_refs.iter().any(|r| r.reference_name == "helper")); } #[test] @@ -554,18 +548,12 @@ void foo() {} void bar() {} "#; let extractor = CppExtractor; - let result = extractor.extract("test.cpp", source); + let result = extractor.extract_artifact("test.cpp", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); - let contains: Vec<_> = result - .edges - .iter() - .filter(|e| e.kind == EdgeKind::Contains) - .collect(); - assert!( - contains.len() >= 2, - "should have Contains edges from File to Functions, got: {}", - contains.len() + assert_eq!( + edge_pairs(&result, EdgeKind::Contains), + [("test.cpp", "foo"), ("test.cpp", "bar")] ); } @@ -580,7 +568,7 @@ public: }; "#; let extractor = CppExtractor; - let result = extractor.extract("rect.cpp", source); + let result = extractor.extract_artifact("rect.cpp", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let class_node = result @@ -611,7 +599,7 @@ static int helper(int x) { } "#; let extractor = CppExtractor; - let result = extractor.extract("utils.cpp", source); + let result = extractor.extract_artifact("utils.cpp", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let fns: Vec<_> = result @@ -632,7 +620,7 @@ int public_func() { } "#; let extractor = CppExtractor; - let result = extractor.extract("api.cpp", source); + let result = extractor.extract_artifact("api.cpp", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let fns: Vec<_> = result @@ -656,7 +644,7 @@ public: }; "#; let extractor = CppExtractor; - let result = extractor.extract("container.hpp", source); + let result = extractor.extract_artifact("container.hpp", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let templates: Vec<_> = result @@ -679,7 +667,7 @@ enum class Direction { }; "#; let extractor = CppExtractor; - let result = extractor.extract("direction.cpp", source); + let result = extractor.extract_artifact("direction.cpp", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let enums: Vec<_> = result @@ -706,7 +694,7 @@ class B {}; class C : public A, public B {}; "#; let extractor = CppExtractor; - let result = extractor.extract("multi.cpp", source); + let result = extractor.extract_artifact("multi.cpp", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let extends_refs: Vec<_> = result @@ -714,10 +702,12 @@ class C : public A, public B {}; .iter() .filter(|r| r.reference_kind == EdgeKind::Extends) .collect(); - assert!( - extends_refs.len() >= 2, - "should have 2 Extends refs, got: {:?}", + assert_eq!( extends_refs + .iter() + .map(|x| x.reference_name.as_str()) + .collect::>(), + ["A", "B"] ); assert!(extends_refs.iter().any(|r| r.reference_name == "A")); assert!(extends_refs.iter().any(|r| r.reference_name == "B")); @@ -736,7 +726,7 @@ class [[nodiscard]] Result { }; "#; let extractor = CppExtractor; - let result = extractor.extract("attr.cpp", source); + let result = extractor.extract_artifact("attr.cpp", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); // Should have 3 AnnotationUsage nodes: nodiscard, deprecated, nodiscard @@ -745,23 +735,21 @@ class [[nodiscard]] Result { .iter() .filter(|n| n.kind == NodeKind::AnnotationUsage) .collect(); - assert!( - annots.len() >= 3, - "expected at least 3 annotations, got: {:?}", - annots.iter().map(|a| &a.name).collect::>() + assert_eq!( + annots.iter().map(|x| x.name.as_str()).collect::>(), + ["nodiscard", "deprecated", "nodiscard"] ); assert!(annots.iter().any(|a| a.name == "nodiscard")); assert!(annots.iter().any(|a| a.name == "deprecated")); // Should have Annotates edges. - let annotates_edges: Vec<_> = result - .edges - .iter() - .filter(|e| e.kind == EdgeKind::Annotates) - .collect(); - assert!( - annotates_edges.len() >= 3, - "expected at least 3 Annotates edges" + assert_eq!( + edge_pairs(&result, EdgeKind::Annotates), + [ + ("nodiscard", "getValue"), + ("deprecated", "oldFunc"), + ("nodiscard", "Result") + ] ); // Should have Annotates unresolved refs. @@ -770,7 +758,13 @@ class [[nodiscard]] Result { .iter() .filter(|r| r.reference_kind == EdgeKind::Annotates) .collect(); - assert!(annot_refs.len() >= 3, "expected at least 3 Annotates refs"); + assert_eq!( + annot_refs + .iter() + .map(|x| x.reference_name.as_str()) + .collect::>(), + ["nodiscard", "deprecated", "nodiscard"] + ); } #[test] @@ -779,7 +773,7 @@ fn test_cpp_function_pointer_typedef() { typedef int (*compare_fn)(const void *, const void *); "#; let extractor = CppExtractor; - let result = extractor.extract("types.hpp", source); + let result = extractor.extract_artifact("types.hpp", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let typedefs: Vec<_> = result @@ -790,3 +784,16 @@ typedef int (*compare_fn)(const void *, const void *); assert_eq!(typedefs.len(), 1, "typedef nodes: {:?}", typedefs); assert_eq!(typedefs[0].name, "compare_fn"); } + +#[test] +fn test_cpp_docstring_needs_an_adjacent_own_line_comment() { + let source = "// Section banner\n\nint detached() { return 0; }\nint x = 1; // trailing note\nint trailing() { return 0; }\n/// Adjacent doc.\nint documented() { return 0; }\n"; + let result = CppExtractor.extract_artifact("docs.cpp", source).result; + assert!(result.errors.is_empty(), "errors: {:?}", result.errors); + let docs: Vec<(&str, &str)> = result + .nodes + .iter() + .filter_map(|n| Some((n.name.as_str(), n.docstring.as_deref()?))) + .collect(); + assert_eq!(docs, [("documented", "Adjacent doc.")]); +} diff --git a/crates/tracedecay-code-extraction/tests/main/csharp.rs b/crates/tracedecay-code-extraction/tests/main/csharp.rs index b969bb16a3..4ab7cde1de 100644 --- a/crates/tracedecay-code-extraction/tests/main/csharp.rs +++ b/crates/tracedecay-code-extraction/tests/main/csharp.rs @@ -2,11 +2,13 @@ use tracedecay_code_extraction::CSharpExtractor; use tracedecay_code_extraction::LanguageExtractor; use tracedecay_domain::*; +include!("support/edges.rs"); + #[test] fn test_cs_file_node_is_root() { let source = "public class Main {}"; let extractor = CSharpExtractor; - let result = extractor.extract("src/Main.cs", source); + let result = extractor.extract_artifact("src/Main.cs", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let files: Vec<_> = result .nodes @@ -26,7 +28,7 @@ namespace MyApp.Models } "#; let extractor = CSharpExtractor; - let result = extractor.extract("test.cs", source); + let result = extractor.extract_artifact("test.cs", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let namespaces: Vec<_> = result .nodes @@ -47,7 +49,7 @@ using System.Linq; public class Foo {} "#; let extractor = CSharpExtractor; - let result = extractor.extract("test.cs", source); + let result = extractor.extract_artifact("test.cs", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let uses: Vec<_> = result .nodes @@ -74,7 +76,7 @@ namespace TestApp } "#; let extractor = CSharpExtractor; - let result = extractor.extract("Calculator.cs", source); + let result = extractor.extract_artifact("Calculator.cs", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let classes: Vec<_> = result .nodes @@ -105,7 +107,7 @@ public struct Point } "#; let extractor = CSharpExtractor; - let result = extractor.extract("test.cs", source); + let result = extractor.extract_artifact("test.cs", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let structs: Vec<_> = result .nodes @@ -127,7 +129,7 @@ public interface IDrawable } "#; let extractor = CSharpExtractor; - let result = extractor.extract("test.cs", source); + let result = extractor.extract_artifact("test.cs", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let ifaces: Vec<_> = result .nodes @@ -149,7 +151,7 @@ public enum Color } "#; let extractor = CSharpExtractor; - let result = extractor.extract("test.cs", source); + let result = extractor.extract_artifact("test.cs", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let enums: Vec<_> = result .nodes @@ -182,7 +184,7 @@ public class Person } "#; let extractor = CSharpExtractor; - let result = extractor.extract("test.cs", source); + let result = extractor.extract_artifact("test.cs", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let constructors: Vec<_> = result .nodes @@ -203,7 +205,7 @@ public class Config } "#; let extractor = CSharpExtractor; - let result = extractor.extract("test.cs", source); + let result = extractor.extract_artifact("test.cs", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let props: Vec<_> = result .nodes @@ -228,7 +230,7 @@ public class Config } "#; let extractor = CSharpExtractor; - let result = extractor.extract("test.cs", source); + let result = extractor.extract_artifact("test.cs", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let fields: Vec<_> = result .nodes @@ -255,7 +257,7 @@ fn test_cs_record() { public record Person(string Name, int Age); "#; let extractor = CSharpExtractor; - let result = extractor.extract("test.cs", source); + let result = extractor.extract_artifact("test.cs", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let records: Vec<_> = result .nodes @@ -273,7 +275,7 @@ fn test_cs_delegate() { public delegate void EventHandler(object sender, EventArgs e); "#; let extractor = CSharpExtractor; - let result = extractor.extract("test.cs", source); + let result = extractor.extract_artifact("test.cs", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let delegates: Vec<_> = result .nodes @@ -294,7 +296,7 @@ public class Button } "#; let extractor = CSharpExtractor; - let result = extractor.extract("test.cs", source); + let result = extractor.extract_artifact("test.cs", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let events: Vec<_> = result .nodes @@ -328,24 +330,19 @@ public class Foo } "#; let extractor = CSharpExtractor; - let result = extractor.extract("test.cs", source); + let result = extractor.extract_artifact("test.cs", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); - let annots: Vec<_> = result + let annots: Vec<&str> = result .nodes .iter() .filter(|n| n.kind == NodeKind::AnnotationUsage) + .map(|n| n.name.as_str()) .collect(); - assert!( - annots.len() >= 2, - "should extract attribute usages, got: {:?}", - annots.iter().map(|a| &a.name).collect::>() + assert_eq!(annots, ["Obsolete", "Serializable"]); + assert_eq!( + edge_pairs(&result, EdgeKind::Annotates), + [("Obsolete", "OldMethod"), ("Serializable", "NewMethod")] ); - let has_annotates = result.edges.iter().any(|e| e.kind == EdgeKind::Annotates) - || result - .unresolved_refs - .iter() - .any(|r| r.reference_kind == EdgeKind::Annotates); - assert!(has_annotates, "should have Annotates edges"); } #[test] @@ -367,7 +364,7 @@ public class Dog : Animal, IAnimal } "#; let extractor = CSharpExtractor; - let result = extractor.extract("test.cs", source); + let result = extractor.extract_artifact("test.cs", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let has_extends = result.edges.iter().any(|e| e.kind == EdgeKind::Extends) || result @@ -395,7 +392,7 @@ public class Foo } "#; let extractor = CSharpExtractor; - let result = extractor.extract("test.cs", source); + let result = extractor.extract_artifact("test.cs", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let methods: Vec<_> = result .nodes @@ -430,7 +427,7 @@ public class Foo } "#; let extractor = CSharpExtractor; - let result = extractor.extract("test.cs", source); + let result = extractor.extract_artifact("test.cs", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let methods: Vec<_> = result .nodes @@ -465,7 +462,7 @@ public class Service } "#; let extractor = CSharpExtractor; - let result = extractor.extract("test.cs", source); + let result = extractor.extract_artifact("test.cs", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let methods: Vec<_> = result .nodes @@ -489,18 +486,16 @@ public class Foo } "#; let extractor = CSharpExtractor; - let result = extractor.extract("test.cs", source); + let result = extractor.extract_artifact("test.cs", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); - let contains: Vec<_> = result - .edges - .iter() - .filter(|e| e.kind == EdgeKind::Contains) - .collect(); - // File contains: Class; Class contains: Field, Property, Method - assert!( - contains.len() >= 4, - "should have Contains edges: {}", - contains.len() + assert_eq!( + edge_pairs(&result, EdgeKind::Contains), + [ + ("test.cs", "Foo"), + ("Foo", "_x"), + ("Foo", "Name"), + ("Foo", "Bar") + ] ); } @@ -516,7 +511,7 @@ namespace MyApp } "#; let extractor = CSharpExtractor; - let result = extractor.extract("src/Service.cs", source); + let result = extractor.extract_artifact("src/Service.cs", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let methods: Vec<_> = result .nodes diff --git a/crates/tracedecay-code-extraction/tests/main/dart.rs b/crates/tracedecay-code-extraction/tests/main/dart.rs index 9a8ff4f3eb..1ad07b8dc1 100644 --- a/crates/tracedecay-code-extraction/tests/main/dart.rs +++ b/crates/tracedecay-code-extraction/tests/main/dart.rs @@ -1,8 +1,10 @@ use tracedecay_code_extraction::{DartExtractor, LanguageExtractor}; use tracedecay_domain::*; +include!("support/edges.rs"); + fn extract(source: &str) -> ExtractionResult { - DartExtractor.extract("test.dart", source) + DartExtractor.extract_artifact("test.dart", source).result } #[test] @@ -222,44 +224,22 @@ fn test_dart_async_function_detection() { fn test_dart_call_site_tracking() { let result = extract("void main() {\n print('hello');\n greet('world');\n}"); assert!(result.errors.is_empty(), "errors: {:?}", result.errors); - let calls: Vec<_> = result + let calls: Vec<&str> = result .unresolved_refs .iter() .filter(|r| r.reference_kind == EdgeKind::Calls) + .map(|r| r.reference_name.as_str()) .collect(); - assert!( - calls.len() >= 2, - "Expected at least 2 calls, got {}: {:?}", - calls.len(), - calls - ); - assert!( - calls.iter().any(|c| c.reference_name == "print"), - "Expected a call to 'print', calls: {:?}", - calls - ); - assert!( - calls.iter().any(|c| c.reference_name == "greet"), - "Expected a call to 'greet', calls: {:?}", - calls - ); + assert_eq!(calls, ["print", "greet"]); } #[test] fn test_dart_contains_edges() { let result = extract("class Foo {\n void bar() {}\n}"); assert!(result.errors.is_empty(), "errors: {:?}", result.errors); - let contains_edges: Vec<_> = result - .edges - .iter() - .filter(|e| e.kind == EdgeKind::Contains) - .collect(); - // File -> Class, Class -> Method - assert!( - contains_edges.len() >= 2, - "Expected at least 2 Contains edges, got {}: {:?}", - contains_edges.len(), - contains_edges + assert_eq!( + edge_pairs(&result, EdgeKind::Contains), + [("test.dart", "Foo"), ("Foo", "bar")] ); } @@ -292,7 +272,7 @@ fn test_dart_private_field_visibility() { .iter() .filter(|n| n.kind == NodeKind::Field) .collect(); - assert!(!fields.is_empty()); + assert_eq!(fields.len(), 1); let private_field = fields.iter().find(|n| n.name == "_count").unwrap(); assert_eq!(private_field.visibility, Visibility::Private); } @@ -421,10 +401,6 @@ class OldWidget { .iter() .filter(|e| e.kind == EdgeKind::Annotates) .collect(); - assert!( - !annotates_edges.is_empty(), - "expected Annotates edges, found none" - ); assert_eq!( annotates_edges.len(), annots.len(), diff --git a/crates/tracedecay-code-extraction/tests/main/dockerfile.rs b/crates/tracedecay-code-extraction/tests/main/dockerfile.rs index 804500ca58..3bfd70f586 100644 --- a/crates/tracedecay-code-extraction/tests/main/dockerfile.rs +++ b/crates/tracedecay-code-extraction/tests/main/dockerfile.rs @@ -4,10 +4,14 @@ use tracedecay_code_extraction::DockerfileExtractor; use tracedecay_code_extraction::LanguageExtractor; use tracedecay_domain::*; +include!("support/edges.rs"); + #[test] fn test_dockerfile_file_node_is_root() { let source = std::fs::read_to_string("../../tests/fixtures/sample.dockerfile").unwrap(); - let result = DockerfileExtractor.extract("sample.dockerfile", &source); + let result = DockerfileExtractor + .extract_artifact("sample.dockerfile", &source) + .result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let files: Vec<_> = result .nodes @@ -21,7 +25,9 @@ fn test_dockerfile_file_node_is_root() { #[test] fn test_dockerfile_extract_from_stages() { let source = std::fs::read_to_string("../../tests/fixtures/sample.dockerfile").unwrap(); - let result = DockerfileExtractor.extract("sample.dockerfile", &source); + let result = DockerfileExtractor + .extract_artifact("sample.dockerfile", &source) + .result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let modules: Vec<_> = result .nodes @@ -41,7 +47,9 @@ fn test_dockerfile_extract_from_stages() { #[test] fn test_dockerfile_extract_env_vars() { let source = std::fs::read_to_string("../../tests/fixtures/sample.dockerfile").unwrap(); - let result = DockerfileExtractor.extract("sample.dockerfile", &source); + let result = DockerfileExtractor + .extract_artifact("sample.dockerfile", &source) + .result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let consts: Vec<_> = result .nodes @@ -60,7 +68,9 @@ fn test_dockerfile_extract_env_vars() { #[test] fn test_dockerfile_extract_arg_vars() { let source = std::fs::read_to_string("../../tests/fixtures/sample.dockerfile").unwrap(); - let result = DockerfileExtractor.extract("sample.dockerfile", &source); + let result = DockerfileExtractor + .extract_artifact("sample.dockerfile", &source) + .result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let consts: Vec<_> = result .nodes @@ -76,7 +86,9 @@ fn test_dockerfile_extract_arg_vars() { #[test] fn test_dockerfile_extract_expose_ports() { let source = std::fs::read_to_string("../../tests/fixtures/sample.dockerfile").unwrap(); - let result = DockerfileExtractor.extract("sample.dockerfile", &source); + let result = DockerfileExtractor + .extract_artifact("sample.dockerfile", &source) + .result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); // EXPOSE -> Field node (port declaration) let fields: Vec<_> = result @@ -93,7 +105,9 @@ fn test_dockerfile_extract_expose_ports() { #[test] fn test_dockerfile_extract_labels() { let source = std::fs::read_to_string("../../tests/fixtures/sample.dockerfile").unwrap(); - let result = DockerfileExtractor.extract("sample.dockerfile", &source); + let result = DockerfileExtractor + .extract_artifact("sample.dockerfile", &source) + .result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let fields: Vec<_> = result .nodes @@ -110,23 +124,32 @@ fn test_dockerfile_extract_labels() { #[test] fn test_dockerfile_contains_edges() { let source = std::fs::read_to_string("../../tests/fixtures/sample.dockerfile").unwrap(); - let result = DockerfileExtractor.extract("sample.dockerfile", &source); + let result = DockerfileExtractor + .extract_artifact("sample.dockerfile", &source) + .result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); - let contains: Vec<_> = result - .edges - .iter() - .filter(|e| e.kind == EdgeKind::Contains) - .collect(); - assert!( - !contains.is_empty(), - "should have Contains edges from file/stage to children" + assert_eq!( + edge_pairs(&result, EdgeKind::Contains), + [ + ("sample.dockerfile", "builder"), + ("builder", "APP_VERSION"), + ("builder", "CARGO_HOME"), + ("builder", "8080"), + ("sample.dockerfile", "runtime"), + ("runtime", "maintainer"), + ("runtime", "version"), + ("runtime", "APP_PORT"), + ("runtime", "LOG_LEVEL") + ] ); } #[test] fn test_dockerfile_copy_from_creates_uses_edge() { let source = std::fs::read_to_string("../../tests/fixtures/sample.dockerfile").unwrap(); - let result = DockerfileExtractor.extract("sample.dockerfile", &source); + let result = DockerfileExtractor + .extract_artifact("sample.dockerfile", &source) + .result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let builder = result .nodes @@ -139,10 +162,7 @@ fn test_dockerfile_copy_from_creates_uses_edge() { .iter() .filter(|e| e.kind == EdgeKind::Uses && e.target == builder.id) .collect(); - assert!( - !uses_edges.is_empty(), - "COPY --from=builder should create a Uses edge to the builder stage node" - ); + assert_eq!(uses_edges.len(), 1); } #[test] @@ -153,7 +173,9 @@ RUN apk add --no-cache curl FROM scratch COPY --from=0 /bin/curl /bin/curl "#; - let result = DockerfileExtractor.extract("Dockerfile", source); + let result = DockerfileExtractor + .extract_artifact("Dockerfile", source) + .result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let first_stage = result @@ -189,7 +211,9 @@ fn test_dockerfile_copy_from_external_image_creates_backed_use_node() { FROM alpine AS runtime COPY --from=nginx:alpine /etc/nginx/nginx.conf /tmp/nginx.conf "#; - let result = DockerfileExtractor.extract("Dockerfile", source); + let result = DockerfileExtractor + .extract_artifact("Dockerfile", source) + .result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let external = result .nodes diff --git a/crates/tracedecay-code-extraction/tests/main/fixture.rs b/crates/tracedecay-code-extraction/tests/main/fixture.rs index 06cf639221..3f386058ef 100644 --- a/crates/tracedecay-code-extraction/tests/main/fixture.rs +++ b/crates/tracedecay-code-extraction/tests/main/fixture.rs @@ -3,18 +3,61 @@ use tracedecay_code_extraction::LanguageExtractor; use tracedecay_domain::*; +include!("support/edges.rs"); + fn read_fixture(name: &str) -> String { let path = format!("../../tests/fixtures/{}", name); std::fs::read_to_string(&path).unwrap_or_else(|e| panic!("Failed to read {}: {}", path, e)) } +fn names<'a>(nodes: &[&'a Node]) -> Vec<&'a str> { + nodes.iter().map(|n| n.name.as_str()).collect() +} + +fn kind_names(result: &ExtractionResult, kind: NodeKind) -> Vec<&str> { + result + .nodes + .iter() + .filter(|n| n.kind == kind) + .map(|n| n.name.as_str()) + .collect() +} + +fn docstring_of<'a>(result: &'a ExtractionResult, kind: NodeKind, name: &str) -> Option<&'a str> { + result + .nodes + .iter() + .find(|n| n.kind == kind && n.name == name) + .unwrap_or_else(|| panic!("{kind:?} {name} not extracted")) + .docstring + .as_deref() +} + +fn ref_names(result: &ExtractionResult, kind: EdgeKind) -> Vec<&str> { + result + .unresolved_refs + .iter() + .filter(|r| r.reference_kind == kind) + .map(|r| r.reference_name.as_str()) + .collect() +} + +/// Names of the nodes `parent` directly contains, in emission order. +fn contained_children<'a>(result: &'a ExtractionResult, parent: &str) -> Vec<&'a str> { + edge_pairs(result, EdgeKind::Contains) + .into_iter() + .filter(|(source, _)| *source == parent) + .map(|(_, child)| child) + .collect() +} + // ── TypeScript ────────────────────────────────────────────────────────────── #[test] fn test_fixture_typescript() { let source = read_fixture("sample.ts"); let extractor = tracedecay_code_extraction::TypeScriptExtractor; - let result = extractor.extract("sample.ts", &source); + let result = extractor.extract_artifact("sample.ts", &source).result; assert!(result.errors.is_empty(), "TS errors: {:?}", result.errors); // File root @@ -26,11 +69,7 @@ fn test_fixture_typescript() { .iter() .filter(|n| n.kind == NodeKind::Use) .collect(); - assert!( - imports.len() >= 2, - "expected >= 2 imports, got {}", - imports.len() - ); + assert_eq!(names(&imports), ["events", "path"]); // Const let consts: Vec<_> = result @@ -76,9 +115,22 @@ fn test_fixture_typescript() { let class = result .nodes .iter() - .find(|n| n.kind == NodeKind::Class && n.name == "UserService"); - assert!(class.is_some(), "UserService class not found"); - assert_eq!(class.unwrap().visibility, Visibility::Pub); + .find(|n| n.kind == NodeKind::Class && n.name == "UserService") + .expect("UserService class not found"); + assert_eq!(class.visibility, Visibility::Pub); + assert_eq!( + contained_children(&result, "UserService"), + [ + "id", + "name", + "_cache", + "settings", + "constructor", + "getDisplayName", + "fetchProfile", + "resetCache", + ] + ); // Methods including async let methods: Vec<_> = result @@ -86,10 +138,20 @@ fn test_fixture_typescript() { .iter() .filter(|n| n.kind == NodeKind::Method) .collect(); - assert!(methods.len() >= 2, "expected >= 2 methods"); - let fetch = methods.iter().find(|m| m.name == "fetchProfile"); - assert!(fetch.is_some(), "fetchProfile method not found"); - assert!(fetch.unwrap().is_async, "fetchProfile should be async"); + assert_eq!( + names(&methods), + [ + "getDisplayName", + "getDisplayName", + "fetchProfile", + "resetCache" + ] + ); + let fetch = methods + .iter() + .find(|m| m.name == "fetchProfile") + .expect("fetchProfile method not found"); + assert!(fetch.is_async, "fetchProfile should be async"); // Arrow function (export const createUser = ...) assert!( @@ -99,29 +161,20 @@ fn test_fixture_typescript() { .any(|n| n.kind == NodeKind::ArrowFunction && n.name == "createUser") ); - // Call sites - assert!( - !result.unresolved_refs.is_empty(), - "expected call site refs" - ); - assert!( - result - .unresolved_refs - .iter() - .any(|r| r.reference_kind == EdgeKind::Calls) - ); - - // Contains edges - assert!(result.edges.iter().any(|e| e.kind == EdgeKind::Contains)); - - // Extends edge (UserService extends EventEmitter) - assert!( - result - .unresolved_refs - .iter() - .any(|r| r.reference_kind == EdgeKind::Extends), - "expected Extends ref for UserService" - ); + assert_eq!( + ref_names(&result, EdgeKind::Calls), + [ + "console.log", + "super", + "fetch", + "response.json", + "this._cache.set", + "log", + "this._cache.clear", + ] + ); + assert_eq!(ref_names(&result, EdgeKind::Extends), ["EventEmitter"]); + assert_eq!(ref_names(&result, EdgeKind::Implements), ["IUser"]); } // ── JavaScript ────────────────────────────────────────────────────────────── @@ -130,7 +183,7 @@ fn test_fixture_typescript() { fn test_fixture_javascript() { let source = read_fixture("sample.js"); let extractor = tracedecay_code_extraction::TypeScriptExtractor; - let result = extractor.extract("sample.js", &source); + let result = extractor.extract_artifact("sample.js", &source).result; assert!(result.errors.is_empty(), "JS errors: {:?}", result.errors); assert!( @@ -148,9 +201,9 @@ fn test_fixture_javascript() { let fetch_fn = result .nodes .iter() - .find(|n| n.kind == NodeKind::Function && n.name == "fetchData"); - assert!(fetch_fn.is_some()); - assert!(fetch_fn.unwrap().is_async); + .find(|n| n.kind == NodeKind::Function && n.name == "fetchData") + .expect("fetchData function"); + assert!(fetch_fn.is_async); assert!( result .nodes @@ -165,7 +218,7 @@ fn test_fixture_javascript() { fn test_fixture_python() { let source = read_fixture("sample.py"); let extractor = tracedecay_code_extraction::PythonExtractor; - let result = extractor.extract("sample.py", &source); + let result = extractor.extract_artifact("sample.py", &source).result; assert!( result.errors.is_empty(), "Python errors: {:?}", @@ -181,10 +234,9 @@ fn test_fixture_python() { .iter() .filter(|n| n.kind == NodeKind::Use) .collect(); - assert!( - imports.len() >= 3, - "expected >= 3 imports, got {}", - imports.len() + assert_eq!( + names(&imports), + ["os", "pathlib.Path", "typing.List", "typing.Optional"] ); // Module-level constants @@ -213,10 +265,18 @@ fn test_fixture_python() { .iter() .find(|n| n.kind == NodeKind::Function && n.name == "log") .unwrap(); - assert!(log_fn.docstring.is_some(), "log() should have docstring"); + assert_eq!( + log_fn.docstring.as_deref(), + Some("Log a message to stdout.") + ); // Decorator - assert!(result.nodes.iter().any(|n| n.kind == NodeKind::Decorator)); + let decorators: Vec<_> = result + .nodes + .iter() + .filter(|n| n.kind == NodeKind::Decorator) + .collect(); + assert_eq!(names(&decorators), ["retry", "property"]); // Classes assert!( @@ -244,61 +304,77 @@ fn test_fixture_python() { .iter() .find(|n| n.kind == NodeKind::Class && n.name == "Connection") .unwrap(); - assert!(conn.docstring.is_some(), "Connection should have docstring"); + assert_eq!( + conn.docstring.as_deref(), + Some("Manages a network connection.") + ); - // Methods + // Methods, with the nested Config class owned by Connection + assert_eq!( + contained_children(&result, "Base"), + ["__init__", "__repr__", "_internal_method"] + ); + assert_eq!( + contained_children(&result, "Connection"), + [ + "__init__", + "connect", + "disconnect", + "is_connected", + "Config" + ] + ); + assert_eq!( + contained_children(&result, "Pool"), + ["__init__", "acquire", "release"] + ); let methods: Vec<_> = result .nodes .iter() .filter(|n| n.kind == NodeKind::Method) .collect(); - assert!( - methods.len() >= 5, - "expected >= 5 methods, got {}", - methods.len() - ); // Async method - let connect = methods.iter().find(|m| m.name == "connect"); - assert!(connect.is_some(), "connect method not found"); - assert!(connect.unwrap().is_async, "connect should be async"); + let connect = methods + .iter() + .find(|m| m.name == "connect") + .expect("connect method not found"); + assert!(connect.is_async, "connect should be async"); // Visibility: _internal_method is private - let internal = methods.iter().find(|m| m.name == "_internal_method"); - assert!(internal.is_some()); - assert_eq!(internal.unwrap().visibility, Visibility::Private); - - // Nested class - assert!( - result - .nodes - .iter() - .any(|n| n.kind == NodeKind::Class && n.name == "Config") - ); + let internal = methods + .iter() + .find(|m| m.name == "_internal_method") + .expect("_internal_method"); + assert_eq!(internal.visibility, Visibility::Private); // Inheritance - assert!( - result - .unresolved_refs - .iter() - .any(|r| r.reference_kind == EdgeKind::Extends), - "expected Extends refs for class inheritance" + assert_eq!( + ref_names(&result, EdgeKind::Extends), + ["Base", "Connection"] ); // Call sites - assert!( - result - .unresolved_refs - .iter() - .any(|r| r.reference_kind == EdgeKind::Calls) + assert_eq!( + ref_names(&result, EdgeKind::Calls), + [ + "print", + "super", + "super().__init__", + "log", + "super", + "super().__init__", + "self._connections.pop", + "Connection", + "conn.connect", + "self._connections.append", + ] ); // Signature with type annotations should not be truncated - let log_sig = log_fn.signature.as_ref().unwrap(); - assert!( - log_sig.contains("message"), - "log signature should contain 'message', got: {}", - log_sig + assert_eq!( + log_fn.signature.as_deref(), + Some("def log(message: str) -> None") ); } @@ -308,100 +384,85 @@ fn test_fixture_python() { fn test_fixture_c() { let source = read_fixture("sample.c"); let extractor = tracedecay_code_extraction::CExtractor; - let result = extractor.extract("sample.c", &source); + let result = extractor.extract_artifact("sample.c", &source).result; assert!(result.errors.is_empty(), "C errors: {:?}", result.errors); - // Includes - let includes: Vec<_> = result - .nodes - .iter() - .filter(|n| n.kind == NodeKind::Include) - .collect(); - assert!(includes.len() >= 3, "expected >= 3 includes"); - - // Preprocessor defines - let defs: Vec<_> = result - .nodes - .iter() - .filter(|n| n.kind == NodeKind::PreprocessorDef) - .collect(); - assert!(defs.iter().any(|n| n.name == "MAX_BUFFER_SIZE")); - - // Typedef struct - assert!( - result - .nodes - .iter() - .any(|n| n.kind == NodeKind::Typedef && n.name == "Point") + assert_eq!( + kind_names(&result, NodeKind::Include), + ["stdio.h", "stdlib.h", "string.h"] ); - - // Struct with fields - let fields: Vec<_> = result - .nodes - .iter() - .filter(|n| n.kind == NodeKind::Field) - .collect(); - assert!(fields.len() >= 2, "expected struct fields"); - - // Union - assert!(result.nodes.iter().any(|n| n.kind == NodeKind::Union)); - - // Enum - assert!(result.nodes.iter().any(|n| n.kind == NodeKind::Enum)); - let variants: Vec<_> = result - .nodes - .iter() - .filter(|n| n.kind == NodeKind::EnumVariant) - .collect(); - assert!(variants.len() >= 4, "expected >= 4 enum variants"); - - // Function pointer typedef - assert!( - result - .nodes - .iter() - .any(|n| n.kind == NodeKind::Typedef && n.name == "Callback") + assert_eq!( + kind_names(&result, NodeKind::PreprocessorDef), + ["MAX_BUFFER_SIZE"] ); - - // Functions - assert!( - result - .nodes - .iter() - .any(|n| n.kind == NodeKind::Function && n.name == "point_distance") + assert_eq!( + kind_names(&result, NodeKind::Typedef), + ["Point", "Variant", "Status", "Callback"] ); - assert!( - result - .nodes - .iter() - .any(|n| n.kind == NodeKind::Function && n.name == "main") + assert_eq!(kind_names(&result, NodeKind::Struct), ["Point", "Color"]); + assert_eq!( + kind_names(&result, NodeKind::Field), + [ + "x", + "y", + "r", + "g", + "b", + "a", + "int_val", + "float_val", + "str_val", + ] + ); + assert_eq!(kind_names(&result, NodeKind::Union), ["Variant"]); + assert_eq!(kind_names(&result, NodeKind::Enum), ["Status"]); + assert_eq!( + kind_names(&result, NodeKind::EnumVariant), + [ + "STATUS_OK", + "STATUS_ERROR", + "STATUS_PENDING", + "STATUS_TIMEOUT", + ] + ); + assert_eq!( + kind_names(&result, NodeKind::Function), + [ + "point_distance", + "point_new", + "set_error", + "process_variant", + "main", + ] ); // Static function is private let set_err = result .nodes .iter() - .find(|n| n.kind == NodeKind::Function && n.name == "set_error"); - assert!(set_err.is_some()); - assert_eq!(set_err.unwrap().visibility, Visibility::Private); + .find(|n| n.kind == NodeKind::Function && n.name == "set_error") + .expect("set_error function"); + assert_eq!(set_err.visibility, Visibility::Private); - // Docstrings - let dist_fn = result - .nodes - .iter() - .find(|n| n.name == "point_distance") - .unwrap(); - assert!( - dist_fn.docstring.is_some(), - "point_distance should have docstring" + assert_eq!( + docstring_of(&result, NodeKind::Function, "point_distance"), + Some("Compute the distance between two points.") ); - // Call sites - assert!( - result - .unresolved_refs - .iter() - .any(|r| r.reference_kind == EdgeKind::Calls) + assert_eq!( + ref_names(&result, EdgeKind::Calls), + [ + "sqrt", + "strncpy", + "cb", + "printf", + "set_error", + "point_new", + "point_new", + "point_distance", + "printf", + "process_variant", + ] ); } @@ -411,7 +472,7 @@ fn test_fixture_c() { fn test_fixture_c_header() { let source = read_fixture("sample.h"); let extractor = tracedecay_code_extraction::CExtractor; - let result = extractor.extract("sample.h", &source); + let result = extractor.extract_artifact("sample.h", &source).result; assert!( result.errors.is_empty(), "C header errors: {:?}", @@ -437,8 +498,11 @@ fn test_fixture_c_header() { .any(|n| n.kind == NodeKind::Typedef && n.name == "Rect") ); - // Enum - assert!(result.nodes.iter().any(|n| n.kind == NodeKind::Enum)); + assert_eq!(kind_names(&result, NodeKind::Enum), ["LogLevel"]); + assert_eq!( + kind_names(&result, NodeKind::Function), + ["rect_new", "rect_area", "rect_contains", "log_init"] + ); } // ── C++ ───────────────────────────────────────────────────────────────────── @@ -447,7 +511,7 @@ fn test_fixture_c_header() { fn test_fixture_cpp() { let source = read_fixture("sample.cpp"); let extractor = tracedecay_code_extraction::CppExtractor; - let result = extractor.extract("sample.cpp", &source); + let result = extractor.extract_artifact("sample.cpp", &source).result; assert!(result.errors.is_empty(), "C++ errors: {:?}", result.errors); // Namespace @@ -488,42 +552,36 @@ fn test_fixture_cpp() { .any(|n| n.kind == NodeKind::Class && n.name == "Rectangle") ); - // Template class - assert!(result.nodes.iter().any(|n| n.kind == NodeKind::Template)); - - // Methods - let methods: Vec<_> = result - .nodes - .iter() - .filter(|n| n.kind == NodeKind::Method) - .collect(); - assert!(methods.len() >= 4, "expected >= 4 methods"); - - // Enum class - assert!(result.nodes.iter().any(|n| n.kind == NodeKind::Enum)); - - // Union - assert!(result.nodes.iter().any(|n| n.kind == NodeKind::Union)); - - // Typedef - assert!( - result - .nodes - .iter() - .any(|n| n.kind == NodeKind::Typedef && n.name == "EntityId") + assert_eq!(kind_names(&result, NodeKind::Template), ["FixedBuffer"]); + assert_eq!( + kind_names(&result, NodeKind::Method), + [ + "length", + "~Shape", + "name", + "area", + "perimeter", + "center", + "radius", + "~Rectangle", + "area", + "perimeter", + ] ); - - // Typedef - assert!( - result - .nodes - .iter() - .any(|n| n.kind == NodeKind::Typedef && n.name == "EntityId") + assert_eq!( + kind_names(&result, NodeKind::AbstractMethod), + ["area", "perimeter"] + ); + assert_eq!(kind_names(&result, NodeKind::Enum), ["Color"]); + // `} // namespace geom` trails code two lines above, so it documents nothing. + assert_eq!(docstring_of(&result, NodeKind::Enum, "Color"), None); + assert_eq!(kind_names(&result, NodeKind::Union), ["Number"]); + assert_eq!(kind_names(&result, NodeKind::Typedef), ["EntityId"]); + assert_eq!( + kind_names(&result, NodeKind::Include), + ["iostream", "string", "vector", "memory"] ); - // Includes - assert!(result.nodes.iter().any(|n| n.kind == NodeKind::Include)); - // Preprocessor def assert!( result @@ -536,25 +594,26 @@ fn test_fixture_cpp() { let helper = result .nodes .iter() - .find(|n| n.kind == NodeKind::Function && n.name == "internal_helper"); - assert!(helper.is_some()); - assert_eq!(helper.unwrap().visibility, Visibility::Private); + .find(|n| n.kind == NodeKind::Function && n.name == "internal_helper") + .expect("internal_helper function"); + assert_eq!(helper.visibility, Visibility::Private); - // Inheritance edges - assert!( - result - .unresolved_refs - .iter() - .any(|r| r.reference_kind == EdgeKind::Extends), - "expected Extends refs for class inheritance" - ); + // Circle and Rectangle both extend Shape + assert_eq!(ref_names(&result, EdgeKind::Extends), ["Shape", "Shape"]); - // Call sites - assert!( - result - .unresolved_refs - .iter() - .any(|r| r.reference_kind == EdgeKind::Calls) + assert_eq!( + ref_names(&result, EdgeKind::Calls), + [ + "std::sqrt", + "shape.name", + "shape.area", + "shape.perimeter", + "print_shape", + "print_shape", + "buffer.push", + "buffer.push", + "internal_helper", + ] ); } @@ -564,119 +623,84 @@ fn test_fixture_cpp() { fn test_fixture_kotlin() { let source = read_fixture("sample.kt"); let extractor = tracedecay_code_extraction::KotlinExtractor; - let result = extractor.extract("sample.kt", &source); + let result = extractor.extract_artifact("sample.kt", &source).result; assert!( result.errors.is_empty(), "Kotlin errors: {:?}", result.errors ); - // Package - assert!( - result - .nodes - .iter() - .any(|n| n.kind == NodeKind::KotlinPackage) + assert_eq!( + kind_names(&result, NodeKind::KotlinPackage), + ["com.example.app"] ); - - // Imports - let imports: Vec<_> = result - .nodes - .iter() - .filter(|n| n.kind == NodeKind::Use) - .collect(); - assert!(imports.len() >= 2, "expected >= 2 imports"); - - // Data class - assert!( - result - .nodes - .iter() - .any(|n| n.kind == NodeKind::DataClass && n.name == "Point") + assert_eq!( + kind_names(&result, NodeKind::Use), + ["kotlin.math.sqrt", "java.time.Instant"] ); - - // Sealed class - assert!(result.nodes.iter().any(|n| n.kind == NodeKind::SealedClass)); - - // Interface - let iface = result - .nodes - .iter() - .find(|n| n.kind == NodeKind::Interface || n.kind == NodeKind::Trait); - assert!(iface.is_some(), "Repository interface not found"); - - // Annotation (may be Decorator or AnnotationUsage depending on extractor) - let has_annotation = result - .nodes - .iter() - .any(|n| n.kind == NodeKind::Decorator || n.kind == NodeKind::AnnotationUsage); - assert!(has_annotation, "expected annotation nodes"); - - // Abstract class - assert!( - result - .nodes - .iter() - .any(|n| n.kind == NodeKind::Class && n.name == "Entity") + assert_eq!( + kind_names(&result, NodeKind::DataClass), + ["Point", "Success", "Failure"] ); - - // Regular class with properties - assert!( - result - .nodes - .iter() - .any(|n| n.kind == NodeKind::Class && n.name == "User") + assert_eq!(kind_names(&result, NodeKind::SealedClass), ["Result"]); + assert_eq!(kind_names(&result, NodeKind::Trait), ["Repository"]); + assert_eq!( + kind_names(&result, NodeKind::AnnotationUsage), + ["Target", "Retention", "Cacheable"] ); - let properties: Vec<_> = result - .nodes - .iter() - .filter(|n| n.kind == NodeKind::Property) - .collect(); - assert!(properties.len() >= 2, "expected >= 2 properties"); - - // Companion object - assert!( - result - .nodes - .iter() - .any(|n| n.kind == NodeKind::CompanionObject) + assert_eq!( + kind_names(&result, NodeKind::Class), + ["Cacheable", "Entity", "User"] ); - - // Enum class - assert!(result.nodes.iter().any(|n| n.kind == NodeKind::Enum)); - - // Object declaration (singleton) - assert!( - result - .nodes - .iter() - .any(|n| n.kind == NodeKind::KotlinObject && n.name == "Logger") + assert_eq!( + kind_names(&result, NodeKind::Property), + ["MAX_RETRIES", "APP_NAME", "createdAt", "lastLogin"] ); - - // Extension function - assert!( - result - .nodes - .iter() - .any(|n| n.kind == NodeKind::Function && n.name.contains("toSlug")) + assert_eq!( + kind_names(&result, NodeKind::CompanionObject), + ["Companion"] + ); + assert_eq!(kind_names(&result, NodeKind::Enum), ["Role"]); + assert_eq!( + kind_names(&result, NodeKind::KotlinObject), + ["Loading", "Logger"] + ); + assert_eq!( + kind_names(&result, NodeKind::Function), + ["toSlug", "processUser", "helperFunction"] ); // Visibility: protected helper - let helper = result.nodes.iter().find(|n| n.name == "helperFunction"); - if let Some(h) = helper { - assert_eq!( - h.visibility, - Visibility::PubSuper, - "protected should be PubSuper" - ); - } + let helper = result + .nodes + .iter() + .find(|n| n.name == "helperFunction") + .expect("helperFunction"); + assert_eq!( + helper.visibility, + Visibility::PubSuper, + "protected should be PubSuper" + ); - // Call sites - assert!( - result - .unresolved_refs - .iter() - .any(|r| r.reference_kind == EdgeKind::Calls) + assert_eq!( + ref_names(&result, EdgeKind::Calls), + [ + "sqrt", + "name.isNotBlank", + "email.contains", + "println", + "mapOf", + "User", + "println", + "println", + "this.lowercase", + "this.lowercase().replace", + "repo.count", + "Logger.info", + "Result.Success", + "User.guest", + "Logger.info", + ] ); } @@ -687,26 +711,17 @@ fn test_fixture_kotlin() { fn test_fixture_dart() { let source = read_fixture("sample.dart"); let extractor = tracedecay_code_extraction::DartExtractor; - let result = extractor.extract("sample.dart", &source); + let result = extractor.extract_artifact("sample.dart", &source).result; assert!(result.errors.is_empty(), "Dart errors: {:?}", result.errors); - // Library - assert!(result.nodes.iter().any(|n| n.kind == NodeKind::Library)); - - // Imports - let imports: Vec<_> = result - .nodes - .iter() - .filter(|n| n.kind == NodeKind::Use) - .collect(); - assert!(imports.len() >= 2, "expected >= 2 imports"); - - // Enum - assert!(result.nodes.iter().any(|n| n.kind == NodeKind::Enum)); - - // Abstract class (may map to Interface or Class) - let serializable = result.nodes.iter().find(|n| n.name == "Serializable"); - assert!(serializable.is_some(), "Serializable not found"); + assert_eq!(kind_names(&result, NodeKind::Library), ["sample"]); + assert_eq!( + kind_names(&result, NodeKind::Use), + ["dart:async", "dart:convert"] + ); + assert_eq!(kind_names(&result, NodeKind::Enum), ["LogLevel"]); + // The abstract class is modelled as an interface + assert_eq!(kind_names(&result, NodeKind::Interface), ["Serializable"]); // Mixin assert!( @@ -732,36 +747,51 @@ fn test_fixture_dart() { .any(|n| n.kind == NodeKind::Extension && n.name == "StringUtils") ); - // Methods - let methods: Vec<_> = result - .nodes - .iter() - .filter(|n| n.kind == NodeKind::Method) - .collect(); - assert!(methods.len() >= 2, "expected >= 2 methods"); - - // Constructor - assert!(result.nodes.iter().any(|n| n.kind == NodeKind::Constructor)); + assert_eq!( + kind_names(&result, NodeKind::Method), + [ + "toJson", + "toJsonString", + "createdAt", + "updatedAt", + "age", + "toJson", + "fetchProfile", + "_isValid", + "_logAction", + "toSlug", + "isBlank", + ] + ); + assert_eq!( + kind_names(&result, NodeKind::Constructor), + ["User", "User.guest"] + ); - // Private visibility (_email, _isValid, _logAction) + // Underscore-prefixed members are library-private let privates: Vec<_> = result .nodes .iter() - .filter(|n| n.visibility == Visibility::Private) + .filter(|n| n.visibility == Visibility::Private && n.name.starts_with('_')) .collect(); - assert!(!privates.is_empty(), "expected private members"); + assert_eq!(names(&privates), ["_email", "_isValid", "_logAction"]); // Async function - let process = result.nodes.iter().find(|n| n.name == "processUsers"); - if let Some(p) = process { - assert!(p.is_async, "processUsers should be async"); - } - - // Typedef - assert!(result.nodes.iter().any(|n| n.kind == NodeKind::TypeAlias)); + let process = result + .nodes + .iter() + .find(|n| n.name == "processUsers") + .expect("processUsers function"); + assert!(process.is_async, "processUsers should be async"); - // Contains edges - assert!(result.edges.iter().any(|e| e.kind == EdgeKind::Contains)); + assert_eq!( + kind_names(&result, NodeKind::TypeAlias), + ["JsonMap", "Callback"] + ); + assert_eq!( + contained_children(&result, "StringUtils"), + ["toSlug", "isBlank"] + ); } // ── C# ────────────────────────────────────────────────────────────────────── @@ -770,266 +800,187 @@ fn test_fixture_dart() { fn test_fixture_csharp() { let source = read_fixture("sample.cs"); let extractor = tracedecay_code_extraction::CSharpExtractor; - let result = extractor.extract("sample.cs", &source); + let result = extractor.extract_artifact("sample.cs", &source).result; assert!(result.errors.is_empty(), "C# errors: {:?}", result.errors); - // Namespace - assert!(result.nodes.iter().any(|n| n.kind == NodeKind::Namespace)); - - // Using directives - let usings: Vec<_> = result - .nodes - .iter() - .filter(|n| n.kind == NodeKind::Use) - .collect(); - assert!(usings.len() >= 3, "expected >= 3 using directives"); - - // Enum - assert!(result.nodes.iter().any(|n| n.kind == NodeKind::Enum)); - - // Record - assert!( - result - .nodes - .iter() - .any(|n| n.kind == NodeKind::Record && n.name == "AppConfig") - ); - - // Delegate - assert!(result.nodes.iter().any(|n| n.kind == NodeKind::Delegate)); - - // Interfaces - let ifaces: Vec<_> = result - .nodes - .iter() - .filter(|n| n.kind == NodeKind::Interface) - .collect(); - assert!(ifaces.len() >= 2, "expected >= 2 interfaces"); - - // Attribute (decorator) - assert!( - result - .nodes - .iter() - .any(|n| n.kind == NodeKind::AnnotationUsage || n.kind == NodeKind::Decorator) - ); - - // Abstract class - assert!( - result - .nodes - .iter() - .any(|n| n.kind == NodeKind::Class && n.name == "Entity") - ); - - // Class with methods - assert!( - result - .nodes - .iter() - .any(|n| n.kind == NodeKind::Class && n.name == "User") - ); - let methods: Vec<_> = result - .nodes - .iter() - .filter(|n| n.kind == NodeKind::Method) - .collect(); - assert!(methods.len() >= 3, "expected >= 3 methods"); - - // Constructor - assert!(result.nodes.iter().any(|n| n.kind == NodeKind::Constructor)); - - // Properties - let props: Vec<_> = result - .nodes - .iter() - .filter(|n| n.kind == NodeKind::CSharpProperty) - .collect(); - assert!(props.len() >= 2, "expected >= 2 properties"); - - // Event - assert!(result.nodes.iter().any(|n| n.kind == NodeKind::Event)); - - // Fields - assert!(result.nodes.iter().any(|n| n.kind == NodeKind::Field)); - - // Struct - assert!( - result - .nodes - .iter() - .any(|n| n.kind == NodeKind::Struct && n.name == "Point") - ); - - // Visibility: private, internal, protected - assert!( - result - .nodes - .iter() - .any(|n| n.visibility == Visibility::Private) - ); - assert!( - result - .nodes - .iter() - .any(|n| n.visibility == Visibility::PubCrate) - ); // internal - - // Async method - let fetch = methods.iter().find(|m| m.name == "FetchProfileAsync"); - if let Some(f) = fetch { - assert!(f.is_async, "FetchProfileAsync should be async"); - } - - // Inheritance - assert!( - result - .unresolved_refs - .iter() - .any(|r| r.reference_kind == EdgeKind::Extends - || r.reference_kind == EdgeKind::Implements), - "expected inheritance refs" - ); - - // Call sites - assert!( - result - .unresolved_refs - .iter() - .any(|r| r.reference_kind == EdgeKind::Calls) - ); -} - -// ── PHP ───────────────────────────────────────────────────────────────────── - -#[cfg(feature = "lang-php")] -#[test] -fn test_fixture_php() { - let source = read_fixture("sample.php"); - let extractor = tracedecay_code_extraction::PhpExtractor; - let result = extractor.extract("sample.php", &source); - assert!(result.errors.is_empty(), "PHP errors: {:?}", result.errors); - - // File root node - assert!(result.nodes.iter().any(|n| n.kind == NodeKind::File)); - - // Namespace (mapped to NodeKind::Module in PHP extractor) - assert!( - result.nodes.iter().any(|n| n.kind == NodeKind::Module), - "expected a namespace/module node" + assert_eq!( + kind_names(&result, NodeKind::Namespace), + ["SampleApp.Models"] ); - - // Use nodes: the PHP extractor extracts trait `use` declarations inside class bodies as - // NodeKind::Use. Namespace-level `use` imports use a different grammar node - // (namespace_use_declaration) that is not yet mapped. We expect >= 2 Use nodes - // because both Connection (use Timestamps) and Pool (use Loggable) have trait uses. - let imports: Vec<_> = result - .nodes - .iter() - .filter(|n| n.kind == NodeKind::Use) - .collect(); - assert!( - imports.len() >= 2, - "expected >= 2 Use nodes (trait uses), got {}", - imports.len() + assert_eq!( + kind_names(&result, NodeKind::Use), + [ + "System", + "System.Collections.Generic", + "System.Threading.Tasks", + ] + ); + assert_eq!(kind_names(&result, NodeKind::Enum), ["LogLevel"]); + assert_eq!(kind_names(&result, NodeKind::Record), ["AppConfig"]); + assert_eq!( + kind_names(&result, NodeKind::Delegate), + ["StatusChangedHandler"] ); - - // Interface and Trait (both mapped to NodeKind::Trait) - let traits: Vec<_> = result - .nodes - .iter() - .filter(|n| n.kind == NodeKind::Trait) - .collect(); - assert!( - traits.len() >= 2, - "expected >= 2 Trait nodes (interface + trait), got {}", - traits.len() + assert_eq!( + kind_names(&result, NodeKind::Interface), + ["IEntity", "IRepository"] ); - assert!( - traits.iter().any(|n| n.name == "ConnectionInterface"), - "ConnectionInterface not found" + assert_eq!( + kind_names(&result, NodeKind::AnnotationUsage), + ["AttributeUsage", "Cacheable"] ); - assert!( - traits.iter().any(|n| n.name == "Timestamps"), - "Timestamps trait not found" + assert_eq!( + kind_names(&result, NodeKind::Class), + ["CacheableAttribute", "Entity", "User"] ); - - // Classes - assert!( - result - .nodes - .iter() - .any(|n| n.kind == NodeKind::Class && n.name == "Connection"), - "Connection class not found" + assert_eq!( + kind_names(&result, NodeKind::Method), + [ + "Validate", + "FindByIdAsync", + "GetAllAsync", + "Validate", + "Validate", + "FetchProfileAsync", + "LogAction", + "DistanceTo", + ] ); - assert!( - result - .nodes - .iter() - .any(|n| n.kind == NodeKind::Class && n.name == "Pool"), - "Pool class not found" + assert_eq!( + kind_names(&result, NodeKind::Constructor), + ["CacheableAttribute", "Entity", "User", "Point"] ); - - // Methods (>= 3) - let methods: Vec<_> = result - .nodes - .iter() - .filter(|n| n.kind == NodeKind::Method) - .collect(); - assert!( - methods.len() >= 3, - "expected >= 3 methods, got {}", - methods.len() + assert_eq!( + kind_names(&result, NodeKind::CSharpProperty), + [ + "Id", + "Count", + "TtlSeconds", + "Id", + "CreatedAt", + "Name", + "Level", + "IsActive", + "InstanceCount", + "X", + "Y", + ] + ); + assert_eq!(kind_names(&result, NodeKind::Event), ["StatusChanged"]); + assert_eq!( + kind_names(&result, NodeKind::Field), + ["_email", "_instanceCount"] ); + assert_eq!(kind_names(&result, NodeKind::Struct), ["Point"]); - // Enum - assert!( - result + // Visibility: private fields, internal property + for field in ["_email", "_instanceCount"] { + let node = result .nodes .iter() - .any(|n| n.kind == NodeKind::Enum && n.name == "ConnectionState"), - "ConnectionState enum not found" - ); - - // Fields (properties) - let fields: Vec<_> = result + .find(|n| n.kind == NodeKind::Field && n.name == field) + .expect("field"); + assert_eq!(node.visibility, Visibility::Private, "{field}"); + } + let internal: Vec<_> = result .nodes .iter() - .filter(|n| n.kind == NodeKind::Field) + .filter(|n| n.visibility == Visibility::PubCrate) .collect(); - assert!(!fields.is_empty(), "expected property/field nodes"); + assert_eq!(names(&internal), ["Level"]); - // Visibility: has private members - assert!( - result - .nodes - .iter() - .any(|n| n.visibility == Visibility::Private), - "expected at least one private member" + // Async method + let fetch = result + .nodes + .iter() + .find(|m| m.kind == NodeKind::Method && m.name == "FetchProfileAsync") + .expect("FetchProfileAsync method"); + assert!(fetch.is_async, "FetchProfileAsync should be async"); + + assert_eq!( + ref_names(&result, EdgeKind::Extends), + ["Attribute", "IEntity", "Entity"] + ); + assert_eq!( + ref_names(&result, EdgeKind::Calls), + [ + "string.IsNullOrWhiteSpace", + "_email.Contains", + "Task.Delay", + "Console.WriteLine", + "StatusChanged?.Invoke", + "new Dictionary", + "Level.ToString", + "Console.WriteLine", + "Math.Sqrt", + ] ); +} - // Inheritance: Extends refs (Pool extends Connection) - assert!( - result - .unresolved_refs - .iter() - .any(|r| r.reference_kind == EdgeKind::Extends), - "expected Extends ref for Pool extends Connection" +// ── PHP ───────────────────────────────────────────────────────────────────── + +#[cfg(feature = "lang-php")] +#[test] +fn test_fixture_php() { + let source = read_fixture("sample.php"); + let extractor = tracedecay_code_extraction::PhpExtractor; + let result = extractor.extract_artifact("sample.php", &source).result; + assert!(result.errors.is_empty(), "PHP errors: {:?}", result.errors); + + // File root node + assert!(result.nodes.iter().any(|n| n.kind == NodeKind::File)); + + // Namespace (mapped to NodeKind::Module in PHP extractor) + assert_eq!(kind_names(&result, NodeKind::Module), [r"App\Http"]); + + // Use nodes are the trait `use` declarations inside class bodies: + // Connection uses Timestamps and Pool uses Loggable. + assert_eq!( + kind_names(&result, NodeKind::Use), + ["Timestamps", "Loggable"] ); - // Call sites - assert!( - result - .unresolved_refs - .iter() - .any(|r| r.reference_kind == EdgeKind::Calls), - "expected Calls refs" + // Interface and Trait (both mapped to NodeKind::Trait) + assert_eq!( + kind_names(&result, NodeKind::Trait), + ["ConnectionInterface", "Timestamps", "Loggable"] + ); + assert_eq!(kind_names(&result, NodeKind::Class), ["Connection", "Pool"]); + assert_eq!( + contained_children(&result, "Connection"), + [ + "Timestamps", + "host", + "port", + "connected", + "__construct", + "connect", + "disconnect", + "validatePort", + ] + ); + assert_eq!(kind_names(&result, NodeKind::Enum), ["ConnectionState"]); + assert_eq!( + kind_names(&result, NodeKind::Field), + ["connectedAt", "host", "port", "connected", "size"] ); - // Contains edges - assert!( - result.edges.iter().any(|e| e.kind == EdgeKind::Contains), - "expected Contains edges" + // Visibility: private members + let privates: Vec<_> = result + .nodes + .iter() + .filter(|n| n.visibility == Visibility::Private && n.kind != NodeKind::Use) + .collect(); + assert_eq!( + names(&privates), + ["connectedAt", "port", "connected", "validatePort", "size"] + ); + + // Pool extends Connection + assert_eq!(ref_names(&result, EdgeKind::Extends), ["Connection"]); + assert_eq!( + ref_names(&result, EdgeKind::Calls), + ["error_log", "log_message", "log_message"] ); } @@ -1040,23 +991,15 @@ fn test_fixture_php() { fn test_fixture_pascal() { let source = read_fixture("sample.pas"); let extractor = tracedecay_code_extraction::PascalExtractor; - let result = extractor.extract("sample.pas", &source); + let result = extractor.extract_artifact("sample.pas", &source).result; assert!( result.errors.is_empty(), "Pascal errors: {:?}", result.errors ); - // Unit declaration - assert!(result.nodes.iter().any(|n| n.kind == NodeKind::PascalUnit)); - - // Uses clause - let uses: Vec<_> = result - .nodes - .iter() - .filter(|n| n.kind == NodeKind::Use) - .collect(); - assert!(uses.len() >= 2, "expected >= 2 uses"); + assert_eq!(kind_names(&result, NodeKind::PascalUnit), ["SampleUnit"]); + assert_eq!(kind_names(&result, NodeKind::Use), ["SysUtils", "Classes"]); // Constants assert!( @@ -1074,26 +1017,22 @@ fn test_fixture_pascal() { .any(|n| n.kind == NodeKind::PascalRecord && n.name == "TPoint") ); - // Interface type - assert!(result.nodes.iter().any(|n| n.kind == NodeKind::Interface)); - - // Classes - assert!( - result - .nodes - .iter() - .any(|n| n.kind == NodeKind::Class && n.name == "TEntity") - ); - assert!( - result - .nodes - .iter() - .any(|n| n.kind == NodeKind::Class && n.name == "TUser") + assert_eq!(kind_names(&result, NodeKind::Interface), ["ISerializable"]); + assert_eq!(kind_names(&result, NodeKind::Class), ["TEntity", "TUser"]); + assert_eq!( + contained_children(&result, "TEntity"), + [ + "FId", + "FCreatedAt", + "GetId", + "Create", + "Destroy", + "Validate", + "Id", + "CreatedAt", + ] ); - // Constructor - assert!(result.nodes.iter().any(|n| n.kind == NodeKind::Constructor)); - // Functions and procedures assert!( result @@ -1108,32 +1047,22 @@ fn test_fixture_pascal() { .any(|n| n.kind == NodeKind::Procedure && n.name == "LogMessage") ); - // Methods - let methods: Vec<_> = result - .nodes - .iter() - .filter(|n| n.kind == NodeKind::Method) - .collect(); - assert!(methods.len() >= 2, "expected >= 2 methods"); + assert_eq!( + kind_names(&result, NodeKind::Property), + ["Id", "CreatedAt", "Name", "Level"] + ); + assert_eq!(contained_children(&result, "TPoint"), ["X", "Y"]); - // Properties - let properties: Vec<_> = result + // Visibility: fields declared in `private` sections + let private_fields: Vec<_> = result .nodes .iter() - .filter(|n| n.kind == NodeKind::Property) + .filter(|n| n.kind == NodeKind::Field && n.visibility == Visibility::Private) .collect(); - assert!(!properties.is_empty(), "expected >= 1 property"); - - // Visibility: private members - assert!( - result - .nodes - .iter() - .any(|n| n.visibility == Visibility::Private) + assert_eq!( + names(&private_fields), + ["FId", "FCreatedAt", "FName", "FEmail", "FLevel"] ); - - // Contains edges - assert!(result.edges.iter().any(|e| e.kind == EdgeKind::Contains)); } // ── Ruby ──────────────────────────────────────────────────────────────────── @@ -1143,7 +1072,7 @@ fn test_fixture_pascal() { fn test_fixture_ruby() { let source = read_fixture("sample.rb"); let extractor = tracedecay_code_extraction::RubyExtractor; - let result = extractor.extract("sample.rb", &source); + let result = extractor.extract_artifact("sample.rb", &source).result; assert!(result.errors.is_empty(), "Ruby errors: {:?}", result.errors); // File root node @@ -1206,48 +1135,45 @@ fn test_fixture_ruby() { "nested Config class not found" ); - // Methods (>= 3) - let methods: Vec<_> = result - .nodes - .iter() - .filter(|n| n.kind == NodeKind::Method) - .collect(); - assert!( - methods.len() >= 3, - "expected >= 3 methods, got {}", - methods.len() + // `log` is defined inside a module, so it is a Method of Networking. + assert_eq!( + contained_children(&result, "Networking"), + [ + "MAX_CONNECTIONS", + "DEFAULT_TIMEOUT", + "log", + "Base", + "Connection", + "Pool", + ] ); - - // Top-level function (log is defined inside a module, class_depth > 0, so it's a Method; - // but `log` is at module level, class_depth is incremented for modules too). - // Accept either Function or Method for `log`. - assert!( - result.nodes.iter().any( - |n| (n.kind == NodeKind::Function || n.kind == NodeKind::Method) && n.name == "log" - ), - "log function/method not found" + assert_eq!( + contained_children(&result, "Connection"), + [ + "initialize", + "connect", + "disconnect", + "connected?", + "Config" + ] + ); + assert_eq!( + contained_children(&result, "Config"), + ["initialize", "valid?"] ); // Inheritance: Connection < Base, Pool < Connection - assert!( - result - .unresolved_refs - .iter() - .any(|r| r.reference_kind == EdgeKind::Extends), - "expected Extends refs for class inheritance" + assert_eq!( + ref_names(&result, EdgeKind::Extends), + ["Base", "Connection"] ); - - // Call sites - assert!( - result - .unresolved_refs - .iter() - .any(|r| r.reference_kind == EdgeKind::Calls), - "expected Calls refs" + assert_eq!( + ref_names(&result, EdgeKind::Calls), + [ + "puts", "class", "name", "raise", "super", "log", "super", "empty?", "new", "connect", + "pop", "push", + ] ); - - // Contains edges - assert!(result.edges.iter().any(|e| e.kind == EdgeKind::Contains)); } // -- Swift ──────────────────────────────────────────────────────────────────── @@ -1256,7 +1182,7 @@ fn test_fixture_ruby() { fn test_fixture_swift() { let source = read_fixture("sample.swift"); let extractor = tracedecay_code_extraction::SwiftExtractor; - let result = extractor.extract("sample.swift", &source); + let result = extractor.extract_artifact("sample.swift", &source).result; assert!( result.errors.is_empty(), "Swift errors: {:?}", @@ -1272,13 +1198,7 @@ fn test_fixture_swift() { .iter() .filter(|n| n.kind == NodeKind::Use) .collect(); - assert!( - imports.len() >= 2, - "expected >= 2 imports, got {}", - imports.len() - ); - assert!(imports.iter().any(|n| n.name == "Foundation")); - assert!(imports.iter().any(|n| n.name == "UIKit")); + assert_eq!(names(&imports), ["Foundation", "UIKit"]); // Top-level constant assert!( @@ -1313,11 +1233,7 @@ fn test_fixture_swift() { .iter() .filter(|n| n.kind == NodeKind::EnumVariant) .collect(); - assert!( - variants.len() >= 4, - "expected >= 4 enum variants, got {}", - variants.len() - ); + assert_eq!(names(&variants), ["debug", "info", "warning", "error"]); // Protocol as Interface assert!( @@ -1362,22 +1278,19 @@ fn test_fixture_swift() { "String extension not found" ); - // Constructor - assert!( - result.nodes.iter().any(|n| n.kind == NodeKind::Constructor), - "expected at least one Constructor node" - ); - - // Methods (>= 3: description, validate, connect, disconnect, distance, toSlug, toJson, toJsonString) - let methods: Vec<_> = result - .nodes - .iter() - .filter(|n| n.kind == NodeKind::Method) - .collect(); - assert!( - methods.len() >= 3, - "expected >= 3 methods, got {}", - methods.len() + assert_eq!(kind_names(&result, NodeKind::Constructor), ["init", "init"]); + assert_eq!( + kind_names(&result, NodeKind::Method), + [ + "toJson", + "toJsonString", + "description", + "validate", + "connect", + "disconnect", + "distance", + "toSlug", + ] ); // Top-level function @@ -1395,62 +1308,58 @@ fn test_fixture_swift() { .iter() .filter(|n| n.kind == NodeKind::Property) .collect(); - assert!( - props.len() >= 2, - "expected >= 2 properties, got {}", - props.len() + assert_eq!( + names(&props), + ["name", "port", "connected", "isConnected", "x", "y"] ); - // Docstrings - let base = result - .nodes - .iter() - .find(|n| n.kind == NodeKind::Class && n.name == "Base") - .unwrap(); - assert!(base.docstring.is_some(), "Base class should have docstring"); + assert_eq!( + docstring_of(&result, NodeKind::Class, "Base"), + Some("Base class with shared functionality.") + ); // Inheritance: Connection extends Base - assert!( - result - .unresolved_refs - .iter() - .any(|r| r.reference_kind == EdgeKind::Extends), - "expected Extends refs for class inheritance" - ); - assert!( - result - .unresolved_refs - .iter() - .any(|r| r.reference_kind == EdgeKind::Extends && r.reference_name == "Base"), - "expected Extends ref to Base" + assert_eq!(ref_names(&result, EdgeKind::Extends), ["Base"]); + assert_eq!( + ref_names(&result, EdgeKind::Calls), + [ + "type", + "assert", + "init", + "print", + "squareRoot", + "lowercased", + "replacingOccurrences", + "map", + "description", + ] ); - - // Call sites - assert!( - result - .unresolved_refs - .iter() - .any(|r| r.reference_kind == EdgeKind::Calls), - "expected Calls refs" + assert_eq!( + contained_children(&result, "Connection"), + [ + "port", + "connected", + "init", + "connect", + "disconnect", + "isConnected", + ] ); - // Contains edges - assert!(result.edges.iter().any(|e| e.kind == EdgeKind::Contains)); - // Async method - let connect = result.nodes.iter().find(|n| n.name == "connect"); - if let Some(c) = connect { - assert!(c.is_async, "connect should be async"); - } + let connect = result + .nodes + .iter() + .find(|n| n.kind == NodeKind::Method && n.name == "connect") + .expect("connect method"); + assert!(connect.is_async, "connect should be async"); - // Private visibility - assert!( - result - .nodes - .iter() - .any(|n| n.visibility == Visibility::Private), - "expected at least one private member" - ); + let privates: Vec<_> = result + .nodes + .iter() + .filter(|n| n.visibility == Visibility::Private && n.kind != NodeKind::Use) + .collect(); + assert_eq!(names(&privates), ["validate", "connected"]); } // ── Bash ──────────────────────────────────────────────────────────────────── @@ -1460,7 +1369,7 @@ fn test_fixture_swift() { fn test_fixture_bash() { let source = read_fixture("sample.sh"); let extractor = tracedecay_code_extraction::BashExtractor; - let result = extractor.extract("sample.sh", &source); + let result = extractor.extract_artifact("sample.sh", &source).result; assert!(result.errors.is_empty(), "Bash errors: {:?}", result.errors); // File root node @@ -1504,19 +1413,52 @@ fn test_fixture_bash() { .iter() .find(|n| n.kind == NodeKind::Function && n.name == "log") .unwrap(); - assert!(log_fn.docstring.is_some(), "log should have docstring"); - - // Call sites - assert!( - result - .unresolved_refs - .iter() - .any(|r| r.reference_kind == EdgeKind::Calls), - "expected Calls refs" + assert_eq!( + log_fn.docstring.as_deref(), + Some("Logs a message with timestamp.") ); - // Contains edges - assert!(result.edges.iter().any(|e| e.kind == EdgeKind::Contains)); + assert_eq!( + ref_names(&result, EdgeKind::Calls), + [ + "source", + "echo", + "date", + "log", + "return", + "log", + "return", + "return", + "log", + "seq", + "curl", + "log", + "return", + "log", + "sleep", + "return", + "log", + "validate_config", + "connect", + "log", + "exit", + "disconnect", + "main", + ] + ); + assert_eq!( + contained_children(&result, "sample"), + [ + "MAX_RETRIES", + "DEFAULT_PORT", + "./utils.sh", + "log", + "validate_config", + "connect", + "disconnect", + "main", + ] + ); } // ── Lua ───────────────────────────────────────────────────────────────────── @@ -1526,7 +1468,7 @@ fn test_fixture_bash() { fn test_fixture_lua() { let source = read_fixture("sample.lua"); let extractor = tracedecay_code_extraction::LuaExtractor; - let result = extractor.extract("sample.lua", &source); + let result = extractor.extract_artifact("sample.lua", &source).result; assert!(result.errors.is_empty(), "Lua errors: {:?}", result.errors); // File root node @@ -1585,19 +1527,46 @@ fn test_fixture_lua() { .iter() .find(|n| n.kind == NodeKind::Function && n.name == "log") .unwrap(); - assert!(lua_log_fn.docstring.is_some(), "log should have docstring"); - - // Call sites - assert!( - result - .unresolved_refs - .iter() - .any(|r| r.reference_kind == EdgeKind::Calls), - "expected Calls refs" + assert_eq!( + lua_log_fn.docstring.as_deref(), + Some( + "Logs a message with the given level.\n\ + @param level string The log level\n\ + @param message string The message to log" + ) ); - // Contains edges - assert!(result.edges.iter().any(|e| e.kind == EdgeKind::Contains)); + assert_eq!( + ref_names(&result, EdgeKind::Calls), + [ + "print", + "string.format", + "setmetatable", + "log", + "setmetatable", + "table.remove", + "Connection.new", + "conn:connect", + "table.insert", + ] + ); + assert_eq!( + contained_children(&result, "sample.lua"), + [ + "json", + "socket", + "MAX_RETRIES", + "DEFAULT_PORT", + "log", + "new", + "connect", + "disconnect", + "isConnected", + "new", + "acquire", + "release", + ] + ); } // ── Zig ───────────────────────────────────────────────────────────────────── @@ -1607,7 +1576,7 @@ fn test_fixture_lua() { fn test_fixture_zig() { let source = read_fixture("sample.zig"); let extractor = tracedecay_code_extraction::ZigExtractor; - let result = extractor.extract("sample.zig", &source); + let result = extractor.extract_artifact("sample.zig", &source).result; assert!(result.errors.is_empty(), "Zig errors: {:?}", result.errors); // File root node @@ -1681,36 +1650,23 @@ fn test_fixture_zig() { ); // Fields - let fields: Vec<_> = result - .nodes - .iter() - .filter(|n| n.kind == NodeKind::Field) - .collect(); - assert!( - fields.len() >= 5, - "expected >= 5 fields, got {}", - fields.len() + // Fields and methods nest under their structs + assert_eq!( + contained_children(&result, "Point"), + ["x", "y", "distance", "origin"] ); - assert!(fields.iter().any(|f| f.name == "x")); - assert!(fields.iter().any(|f| f.name == "host")); - - // Methods inside structs (distance, origin, init, connect, disconnect, isConnected) - let methods: Vec<_> = result - .nodes - .iter() - .filter(|n| n.kind == NodeKind::Method) - .collect(); - assert!( - methods.len() >= 6, - "expected >= 6 methods, got {}", - methods.len() + assert_eq!( + contained_children(&result, "Connection"), + [ + "host", + "port", + "connected", + "init", + "connect", + "disconnect", + "isConnected", + ] ); - assert!(methods.iter().any(|m| m.name == "distance")); - assert!(methods.iter().any(|m| m.name == "origin")); - assert!(methods.iter().any(|m| m.name == "init")); - assert!(methods.iter().any(|m| m.name == "connect")); - assert!(methods.iter().any(|m| m.name == "disconnect")); - assert!(methods.iter().any(|m| m.name == "isConnected")); // Top-level functions (log, processConnections) let fns: Vec<_> = result @@ -1757,20 +1713,18 @@ fn test_fixture_zig() { .iter() .find(|n| n.kind == NodeKind::Struct && n.name == "Point") .unwrap(); - assert!(point.docstring.is_some(), "Point should have docstring"); - assert!(point.docstring.as_ref().unwrap().contains("2D point")); + assert_eq!(point.docstring.as_deref(), Some("A 2D point.")); - // Call sites - assert!( - result - .unresolved_refs - .iter() - .any(|r| r.reference_kind == EdgeKind::Calls), - "expected Calls refs" + assert_eq!( + ref_names(&result, EdgeKind::Calls), + [ + "std.debug.print", + "std.debug.print", + "conn.connect", + "p1.distance", + "std.testing.expectEqual", + ] ); - - // Contains edges - assert!(result.edges.iter().any(|e| e.kind == EdgeKind::Contains)); } // ── Protobuf ──────────────────────────────────────────────────────────────── @@ -1780,7 +1734,7 @@ fn test_fixture_zig() { fn test_fixture_proto() { let source = read_fixture("sample.proto"); let extractor = tracedecay_code_extraction::ProtoExtractor; - let result = extractor.extract("sample.proto", &source); + let result = extractor.extract_artifact("sample.proto", &source).result; assert!( result.errors.is_empty(), "Proto errors: {:?}", @@ -1818,14 +1772,18 @@ fn test_fixture_proto() { .iter() .filter(|n| n.kind == NodeKind::ProtoMessage) .collect(); - assert!( - msgs.len() >= 7, - "expected >= 7 messages, got {}", - msgs.len() + assert_eq!( + names(&msgs), + [ + "Endpoint", + "ConnectionConfig", + "AuthConfig", + "ConnectionStatus", + "DisconnectRequest", + "HealthCheckRequest", + "HealthCheckResponse", + ] ); - assert!(msgs.iter().any(|m| m.name == "Endpoint")); - assert!(msgs.iter().any(|m| m.name == "ConnectionConfig")); - assert!(msgs.iter().any(|m| m.name == "AuthConfig")); // nested // Enum + variants assert!( @@ -1866,49 +1824,40 @@ fn test_fixture_proto() { assert!(rpcs.iter().any(|r| r.name == "Disconnect")); assert!(rpcs.iter().any(|r| r.name == "HealthCheck")); - // Fields - let fields: Vec<_> = result - .nodes - .iter() - .filter(|n| n.kind == NodeKind::Field) - .collect(); - assert!( - fields.len() >= 15, - "expected >= 15 fields, got {}", - fields.len() + // Fields, with the nested AuthConfig message owned by ConnectionConfig + assert_eq!( + contained_children(&result, "Endpoint"), + ["host", "port", "tls"] ); - - // Docstrings - let endpoint = result - .nodes - .iter() - .find(|n| n.kind == NodeKind::ProtoMessage && n.name == "Endpoint") - .unwrap(); - assert!( - endpoint.docstring.is_some(), - "Endpoint should have docstring" + assert_eq!( + contained_children(&result, "ConnectionConfig"), + [ + "endpoint", + "max_retries", + "timeout_ms", + "log_level", + "AuthConfig", + "auth", + "round_robin", + "least_connections", + ] ); - - let log_level = result - .nodes - .iter() - .find(|n| n.kind == NodeKind::Enum && n.name == "LogLevel") - .unwrap(); - assert!( - log_level.docstring.is_some(), - "LogLevel should have docstring" + assert_eq!( + contained_children(&result, "AuthConfig"), + ["token", "username"] + ); + assert_eq!( + contained_children(&result, "ConnectionService"), + ["Connect", "Disconnect", "HealthCheck"] ); - // Contains edges - let contains: Vec<_> = result - .edges - .iter() - .filter(|e| e.kind == EdgeKind::Contains) - .collect(); - assert!( - contains.len() >= 10, - "expected >= 10 Contains edges, got {}", - contains.len() + assert_eq!( + docstring_of(&result, NodeKind::ProtoMessage, "Endpoint"), + Some("A network endpoint.") + ); + assert_eq!( + docstring_of(&result, NodeKind::Enum, "LogLevel"), + Some("Represents the log level.") ); } @@ -1919,7 +1868,7 @@ fn test_fixture_proto() { fn test_fixture_nix() { let source = read_fixture("sample.nix"); let extractor = tracedecay_code_extraction::NixExtractor; - let result = extractor.extract("sample.nix", &source); + let result = extractor.extract_artifact("sample.nix", &source).result; assert!(result.errors.is_empty(), "Nix errors: {:?}", result.errors); // File root @@ -1971,34 +1920,31 @@ fn test_fixture_nix() { // Docstrings let log_fn = fns.iter().find(|f| f.name == "log").unwrap(); - assert!(log_fn.docstring.is_some(), "log should have docstring"); - - let net = result - .nodes - .iter() - .find(|n| n.kind == NodeKind::Module && n.name == "networking") - .unwrap(); - assert!(net.docstring.is_some(), "networking should have docstring"); - - // Call sites - assert!( - result - .unresolved_refs - .iter() - .any(|r| r.reference_kind == EdgeKind::Calls), - "expected call refs" + assert_eq!(log_fn.docstring.as_deref(), Some("Formats a log message.")); + assert_eq!( + docstring_of(&result, NodeKind::Module, "networking"), + Some("Networking utilities.") ); - // Contains edges - let contains: Vec<_> = result - .edges - .iter() - .filter(|e| e.kind == EdgeKind::Contains) - .collect(); - assert!( - contains.len() >= 5, - "expected >= 5 Contains edges, got {}", - contains.len() + assert_eq!( + ref_names(&result, EdgeKind::Calls), + [ + "builtins.trace", + "builtins.trace", + "toString", + "toString", + "builtins.genList", + "builtins.genList", + "mkConnection", + ] + ); + assert_eq!( + contained_children(&result, "networking"), + ["mkPool", "validateConfig", "defaultConfig"] + ); + assert_eq!( + contained_children(&result, "defaultConfig"), + ["host", "port", "tls"] ); // Inherit (Use) nodes @@ -2030,7 +1976,7 @@ fn test_fixture_nix() { fn test_fixture_vbnet() { let source = read_fixture("sample.vb"); let extractor = tracedecay_code_extraction::VbNetExtractor; - let result = extractor.extract("sample.vb", &source); + let result = extractor.extract_artifact("sample.vb", &source).result; // File root assert!(result.nodes.iter().any(|n| n.kind == NodeKind::File)); @@ -2077,14 +2023,7 @@ fn test_fixture_vbnet() { .iter() .filter(|n| n.kind == NodeKind::EnumVariant) .collect(); - assert!( - variants.len() >= 4, - "expected >= 4 enum variants, got {}", - variants.len() - ); - assert!(variants.iter().any(|v| v.name == "Debug")); - assert!(variants.iter().any(|v| v.name == "Info")); - assert!(variants.iter().any(|v| v.name == "Warning")); + assert_eq!(names(&variants), ["Debug", "Info", "Warning", "[Error]"]); // Interface assert!( @@ -2129,44 +2068,30 @@ fn test_fixture_vbnet() { "Helpers module not found" ); - // Methods - let methods: Vec<_> = result - .nodes - .iter() - .filter(|n| n.kind == NodeKind::Method) - .collect(); - assert!( - methods.len() >= 5, - "expected >= 5 methods, got {}", - methods.len() - ); - - // Constructor - assert!( - result.nodes.iter().any(|n| n.kind == NodeKind::Constructor), - "expected at least one Constructor node" + assert_eq!( + contained_children(&result, "Base"), + ["Name", "New", "Description", "Validate"] ); + assert_eq!( + contained_children(&result, "Connection"), + [ + "Port", + "_connected", + "New", + "Connect", + "Disconnect", + "IsConnected", + "ToJson", + ] + ); + assert_eq!(contained_children(&result, "Point"), ["X", "Y", "Distance"]); + assert_eq!(contained_children(&result, "Helpers"), ["LogMessage"]); - // Properties - let props: Vec<_> = result - .nodes - .iter() - .filter(|n| n.kind == NodeKind::Property) - .collect(); - assert!( - props.len() >= 2, - "expected >= 2 properties, got {}", - props.len() + assert_eq!( + docstring_of(&result, NodeKind::Class, "Base"), + Some("Base class with shared functionality.") ); - // Docstrings on classes - let base = result - .nodes - .iter() - .find(|n| n.kind == NodeKind::Class && n.name == "Base") - .unwrap(); - assert!(base.docstring.is_some(), "Base class should have docstring"); - // Inheritance: Connection extends Base assert!( result @@ -2184,17 +2109,18 @@ fn test_fixture_vbnet() { "expected Implements ref to ISerializable" ); - // Call sites - assert!( - result - .unresolved_refs - .iter() - .any(|r| r.reference_kind == EdgeKind::Calls), - "expected Calls refs" + assert_eq!( + ref_names(&result, EdgeKind::Calls), + [ + "Me.GetType", + "Debug.Assert", + "String.IsNullOrEmpty", + "MyBase.New", + "Console.WriteLine", + "Math.Sqrt", + "Console.WriteLine", + ] ); - - // Contains edges - assert!(result.edges.iter().any(|e| e.kind == EdgeKind::Contains)); } // ── PowerShell ────────────────────────────────────────────────────────────── @@ -2204,7 +2130,7 @@ fn test_fixture_vbnet() { fn test_fixture_powershell() { let source = read_fixture("sample.ps1"); let extractor = tracedecay_code_extraction::PowerShellExtractor; - let result = extractor.extract("sample.ps1", &source); + let result = extractor.extract_artifact("sample.ps1", &source).result; assert!( result.errors.is_empty(), "PowerShell errors: {:?}", @@ -2244,31 +2170,49 @@ fn test_fixture_powershell() { .filter(|n| n.kind == NodeKind::Use) .collect(); assert_eq!(uses.len(), 2, "expected 2 Use nodes, got {}", uses.len()); - assert!(uses.iter().any(|n| n.name == "ActiveDirectory")); - assert!(uses.iter().any(|n| n.name.contains("Utils.ps1"))); + assert_eq!(names(&uses), ["ActiveDirectory", r".\Utils.ps1"]); - // Docstrings - let write_log = result - .nodes - .iter() - .find(|n| n.kind == NodeKind::Function && n.name == "Write-Log") - .unwrap(); - assert!( - write_log.docstring.is_some(), - "Write-Log should have docstring" + assert_eq!( + docstring_of(&result, NodeKind::Function, "Write-Log"), + Some( + ".SYNOPSIS\n Logs a message with the given level.\n\ + .PARAMETER Level\n The log level.\n\ + .PARAMETER Message\n The message to log." + ) ); - // Call sites - assert!( - result - .unresolved_refs - .iter() - .any(|r| r.reference_kind == EdgeKind::Calls), - "expected Calls refs" + assert_eq!( + ref_names(&result, EdgeKind::Calls), + [ + "Write-Host", + "Get-Date", + "Write-Log", + "Write-Log", + "Write-Log", + "Test-Connection", + "Write-Log", + "Write-Log", + "Start-Sleep", + "Write-Log", + "Test-Config", + "Connect-Server", + "Disconnect-Server", + ] + ); + assert_eq!( + contained_children(&result, "sample.ps1"), + [ + "ActiveDirectory", + r".\Utils.ps1", + "MaxRetries", + "DefaultPort", + "Write-Log", + "Test-Config", + "Connect-Server", + "Disconnect-Server", + "Main", + ] ); - - // Contains edges - assert!(result.edges.iter().any(|e| e.kind == EdgeKind::Contains)); } // ── Batch ─────────────────────────────────────────────────────────────────── @@ -2278,7 +2222,7 @@ fn test_fixture_powershell() { fn test_fixture_batch() { let source = read_fixture("sample.bat"); let extractor = tracedecay_code_extraction::BatchExtractor; - let result = extractor.extract("sample.bat", &source); + let result = extractor.extract_artifact("sample.bat", &source).result; assert!( result.errors.is_empty(), "Batch errors: {:?}", @@ -2317,19 +2261,37 @@ fn test_fixture_batch() { .iter() .find(|n| n.kind == NodeKind::Function && n.name == "Log") .unwrap(); - assert!(log_fn.docstring.is_some(), "Log should have docstring"); - - // Call sites - assert!( - result - .unresolved_refs - .iter() - .any(|r| r.reference_kind == EdgeKind::Calls), - "expected Calls refs" + assert_eq!( + log_fn.docstring.as_deref(), + Some("Logs a message with timestamp.") ); - // Contains edges - assert!(result.edges.iter().any(|e| e.kind == EdgeKind::Contains)); + assert_eq!( + ref_names(&result, EdgeKind::Calls), + [ + "Log", + "Log", + "Log", + "Log", + "Log", + "Log", + "ValidateConfig", + "Connect", + "Disconnect", + ] + ); + assert_eq!( + contained_children(&result, "sample.bat"), + [ + "MAX_RETRIES", + "DEFAULT_PORT", + "Log", + "ValidateConfig", + "Connect", + "Disconnect", + "Main", + ] + ); } // ── Perl ──────────────────────────────────────────────────────────────────── @@ -2339,7 +2301,7 @@ fn test_fixture_batch() { fn test_fixture_perl() { let source = read_fixture("sample.pl"); let extractor = tracedecay_code_extraction::PerlExtractor; - let result = extractor.extract("sample.pl", &source); + let result = extractor.extract_artifact("sample.pl", &source).result; assert!(result.errors.is_empty(), "Perl errors: {:?}", result.errors); // File root node @@ -2422,40 +2384,33 @@ fn test_fixture_perl() { .iter() .find(|n| n.kind == NodeKind::Function && n.name == "log_message") .unwrap(); - assert!( - log_fn.docstring.is_some(), - "log_message should have docstring" + assert_eq!( + log_fn.docstring.as_deref(), + Some("Logs a message with the given level.") ); - - let max_retries = result - .nodes - .iter() - .find(|n| n.kind == NodeKind::Const && n.name == "MAX_RETRIES") - .unwrap(); - assert!( - max_retries.docstring.is_some(), - "MAX_RETRIES should have docstring" + assert_eq!( + docstring_of(&result, NodeKind::Const, "MAX_RETRIES"), + Some("Maximum number of retries.") ); - // Call sites - assert!( - result - .unresolved_refs - .iter() - .any(|r| r.reference_kind == EdgeKind::Calls), - "expected Calls refs" + assert_eq!( + ref_names(&result, EdgeKind::Calls), + [ + "log_message", + "main::log_message", + "Connection->new", + "$conn->connect", + "croak", + "croak", + ] ); - - // Contains edges - let contains: Vec<_> = result - .edges - .iter() - .filter(|e| e.kind == EdgeKind::Contains) - .collect(); - assert!( - contains.len() >= 15, - "expected >= 15 Contains edges, got {}", - contains.len() + assert_eq!( + contained_children(&result, "Connection"), + ["new", "connect", "disconnect", "is_connected"] + ); + assert_eq!( + contained_children(&result, "Pool"), + ["new", "acquire", "release"] ); } @@ -2466,7 +2421,7 @@ fn test_fixture_perl() { fn test_fixture_objc() { let source = read_fixture("sample.m"); let extractor = tracedecay_code_extraction::ObjcExtractor; - let result = extractor.extract("sample.m", &source); + let result = extractor.extract_artifact("sample.m", &source).result; // File root assert!(result.nodes.iter().any(|n| n.kind == NodeKind::File)); @@ -2536,38 +2491,46 @@ fn test_fixture_objc() { .iter() .find(|n| n.kind == NodeKind::Class && n.name == "Base") .unwrap(); - assert!(base.docstring.is_some(), "Base should have docstring"); - - // Implementation blocks - let impls: Vec<_> = result - .nodes - .iter() - .filter(|n| n.kind == NodeKind::Impl) - .collect(); - assert_eq!(impls.len(), 2, "expected 2 implementation blocks"); + assert_eq!( + base.docstring.as_deref(), + Some("Base class with shared functionality.") + ); - // Properties - let props: Vec<_> = result - .nodes - .iter() - .filter(|n| n.kind == NodeKind::Property) - .collect(); - assert!( - props.len() >= 3, - "expected >= 3 properties, got {}", - props.len() + assert_eq!(kind_names(&result, NodeKind::Impl), ["Base", "Connection"]); + assert_eq!( + kind_names(&result, NodeKind::Property), + ["name", "port", "connected"] ); - // Methods (from both @interface declarations and @implementation definitions) - let methods: Vec<_> = result - .nodes - .iter() - .filter(|n| n.kind == NodeKind::Method) - .collect(); - assert!( - methods.len() >= 6, - "expected >= 6 methods, got {}", - methods.len() + // Methods come from both the @interface declaration and the + // @implementation definition of each class, which share its name. + assert_eq!( + contained_children(&result, "Base"), + [ + "name", + "initWithName", + "description", + "initWithName", + "description", + "validate", + ] + ); + assert_eq!( + contained_children(&result, "Connection"), + [ + "port", + "connected", + "initWithHost", + "connect", + "disconnect", + "connectionWithHost", + "initWithHost", + "connect", + "disconnect", + "connectionWithHost", + "toJson", + "toJsonString", + ] ); // C function @@ -2582,54 +2545,37 @@ fn test_fixture_objc() { .iter() .find(|n| n.kind == NodeKind::Function && n.name == "logMessage") .unwrap(); - assert!( - log_fn.docstring.is_some(), - "logMessage should have docstring" - ); - - // Inheritance - assert!( - result - .unresolved_refs - .iter() - .any(|r| r.reference_kind == EdgeKind::Extends && r.reference_name == "NSObject"), - "expected Extends ref to NSObject" - ); - assert!( - result - .unresolved_refs - .iter() - .any(|r| r.reference_kind == EdgeKind::Extends && r.reference_name == "Base"), - "expected Extends ref to Base" - ); - - // Protocol conformance - assert!( - result - .unresolved_refs - .iter() - .any(|r| r.reference_kind == EdgeKind::Implements), - "expected Implements refs for protocol conformance" + assert_eq!( + log_fn.docstring.as_deref(), + Some("Top-level C function for logging.") ); - // Call sites - assert!( - result - .unresolved_refs - .iter() - .any(|r| r.reference_kind == EdgeKind::Calls) + // Base extends NSObject and Connection extends Base; both conform to + // protocols. + assert_eq!(ref_names(&result, EdgeKind::Extends), ["NSObject", "Base"]); + assert_eq!( + ref_names(&result, EdgeKind::Implements), + ["NSObject", "Serializable"] ); - - // Contains edges - let contains: Vec<_> = result - .edges - .iter() - .filter(|e| e.kind == EdgeKind::Contains) - .collect(); - assert!( - contains.len() >= 15, - "expected >= 15 Contains edges, got {}", - contains.len() + assert_eq!( + ref_names(&result, EdgeKind::Calls), + [ + "super.init", + "name.copy", + "NSString.stringWithFormat", + "NSStringFromClass", + "self.class", + "NSAssert", + "super.initWithName", + "NSLog", + "[self alloc].initWithHost", + "self.alloc", + "NSJSONSerialization.dataWithJSONObject", + "self.toJson", + "[NSString alloc].initWithData", + "NSString.alloc", + "NSLog", + ] ); } @@ -2640,7 +2586,7 @@ fn test_fixture_objc() { fn test_fixture_fortran() { let source = read_fixture("sample.f90"); let extractor = tracedecay_code_extraction::FortranExtractor; - let result = extractor.extract("sample.f90", &source); + let result = extractor.extract_artifact("sample.f90", &source).result; assert!( result.errors.is_empty(), "Fortran errors: {:?}", @@ -2700,17 +2646,11 @@ fn test_fixture_fortran() { "PooledEndpoint type not found" ); - // Fields - let fields: Vec<_> = result - .nodes - .iter() - .filter(|n| n.kind == NodeKind::Field) - .collect(); - assert!( - fields.len() >= 4, - "expected >= 4 fields, got {}", - fields.len() + assert_eq!( + contained_children(&result, "Endpoint"), + ["host", "port", "connected"] ); + assert_eq!(contained_children(&result, "PooledEndpoint"), ["pool_size"]); // Interface assert!( @@ -2750,9 +2690,9 @@ fn test_fixture_fortran() { // Docstrings let log_msg = fns.iter().find(|f| f.name == "log_message").unwrap(); - assert!( - log_msg.docstring.is_some(), - "log_message should have docstring" + assert_eq!( + log_msg.docstring.as_deref(), + Some("Logs a message with the given level.") ); // Use imports @@ -2773,25 +2713,35 @@ fn test_fixture_fortran() { "expected Extends ref for PooledEndpoint -> Endpoint" ); - // Call sites - assert!( - result - .unresolved_refs - .iter() - .any(|r| r.reference_kind == EdgeKind::Calls) - ); - - // Contains edges - let contains: Vec<_> = result - .edges - .iter() - .filter(|e| e.kind == EdgeKind::Contains) - .collect(); - assert!( - contains.len() >= 5, - "expected >= 5 Contains edges, got {}", - contains.len() + assert_eq!( + ref_names(&result, EdgeKind::Calls), + [ + "trim", + "trim", + "present", + "log_message", + "trim", + "create_endpoint", + "connect_endpoint", + "disconnect_endpoint", + ] ); + assert_eq!( + contained_children(&result, "networking"), + [ + "MAX_RETRIES", + "DEFAULT_PORT", + "Endpoint", + "PooledEndpoint", + "Connectable", + "log_message", + "create_endpoint", + "connect_endpoint", + "disconnect_endpoint", + "is_connected", + ] + ); + assert_eq!(contained_children(&result, "main"), ["networking"]); } // -- COBOL ──────────────────────────────────────────────────────────────────── @@ -2801,7 +2751,7 @@ fn test_fixture_fortran() { fn test_fixture_cobol() { let source = read_fixture("sample.cob"); let extractor = tracedecay_code_extraction::CobolExtractor; - let result = extractor.extract("sample.cob", &source); + let result = extractor.extract_artifact("sample.cob", &source).result; assert!( result.errors.is_empty(), "COBOL errors: {:?}", @@ -2875,30 +2825,41 @@ fn test_fixture_cobol() { // Docstrings let validate = fns.iter().find(|f| f.name == "VALIDATE-CONFIG").unwrap(); - assert!( - validate.docstring.is_some(), - "VALIDATE-CONFIG should have docstring" + assert_eq!( + validate.docstring.as_deref(), + Some("Validates the configuration.") ); - // Call sites - assert!( - result - .unresolved_refs - .iter() - .any(|r| r.reference_kind == EdgeKind::Calls), - "expected Calls refs" + // PERFORM targets + assert_eq!( + ref_names(&result, EdgeKind::Calls), + [ + "VALIDATE-CONFIG", + "CONNECT-SERVER", + "DISCONNECT-SERVER", + "LOG-MESSAGE", + "LOG-MESSAGE", + "LOG-MESSAGE", + "LOG-MESSAGE", + ] ); - - // Contains edges - let contains: Vec<_> = result - .edges - .iter() - .filter(|e| e.kind == EdgeKind::Contains) - .collect(); - assert!( - contains.len() >= 10, - "expected >= 10 Contains edges, got {}", - contains.len() + assert_eq!( + contained_children(&result, "NETWORKING"), + [ + "WS-MAX-RETRIES", + "WS-DEFAULT-PORT", + "WS-HOST", + "WS-PORT", + "WS-CONNECTED", + "WS-LOG-LEVEL", + "WS-LOG-MESSAGE", + "WS-RETRY-COUNT", + "MAIN-PROGRAM", + "VALIDATE-CONFIG", + "LOG-MESSAGE", + "CONNECT-SERVER", + "DISCONNECT-SERVER", + ] ); } @@ -2909,7 +2870,7 @@ fn test_fixture_cobol() { fn test_fixture_msbasic2() { let source = read_fixture("sample.bas"); let extractor = tracedecay_code_extraction::MsBasic2Extractor; - let result = extractor.extract("sample.bas", &source); + let result = extractor.extract_artifact("sample.bas", &source).result; assert!( result.errors.is_empty(), "MS BASIC 2.0 errors: {:?}", @@ -2951,9 +2912,9 @@ fn test_fixture_msbasic2() { // Docstrings let log_fn = fns.iter().find(|f| f.name == "LOG_A_MESSAGE").unwrap(); - assert!( - log_fn.docstring.is_some(), - "LOG_A_MESSAGE should have docstring" + assert_eq!( + log_fn.docstring.as_deref(), + Some("LOG A MESSAGE\nPARAMS: L$=LEVEL, M$=MESSAGE") ); // Complexity: CONNECT_TO_SERVER has a FOR loop @@ -2963,25 +2924,20 @@ fn test_fixture_msbasic2() { "CONNECT_TO_SERVER should have >= 1 loop" ); - // Call sites (GOSUB references) - assert!( - result - .unresolved_refs - .iter() - .any(|r| r.reference_kind == EdgeKind::Calls), - "expected Calls refs" + // GOSUB targets are line numbers + assert_eq!( + ref_names(&result, EdgeKind::Calls), + ["200", "300", "400", "200", "200"] ); - - // Contains edges - let contains: Vec<_> = result - .edges - .iter() - .filter(|e| e.kind == EdgeKind::Contains) - .collect(); - assert!( - contains.len() >= 5, - "expected >= 5 Contains edges, got {}", - contains.len() + assert_eq!( + contained_children(&result, "sample.bas"), + [ + "MR", + "DP", + "LOG_A_MESSAGE", + "CONNECT_TO_SERVER", + "DISCONNECT" + ] ); } @@ -2992,7 +2948,7 @@ fn test_fixture_msbasic2() { fn test_fixture_gwbasic() { let source = read_fixture("sample.gw"); let extractor = tracedecay_code_extraction::GwBasicExtractor; - let result = extractor.extract("sample.gw", &source); + let result = extractor.extract_artifact("sample.gw", &source).result; assert!( result.errors.is_empty(), "GW-BASIC errors: {:?}", @@ -3018,19 +2974,14 @@ fn test_fixture_gwbasic() { .iter() .filter(|n| n.kind == NodeKind::Function) .collect(); - assert!(fns.len() >= 4, "expected >= 4 functions, got {}", fns.len()); - assert!(fns.iter().any(|f| f.name == "FNLOG"), "FNLOG not found"); - assert!( - fns.iter().any(|f| f.name == "VALIDATE_CONFIGURATION"), - "VALIDATE_CONFIGURATION not found" - ); - assert!( - fns.iter().any(|f| f.name == "CONNECT_TO_SERVER"), - "CONNECT_TO_SERVER not found" - ); - assert!( - fns.iter().any(|f| f.name == "DISCONNECT"), - "DISCONNECT not found" + assert_eq!( + names(&fns), + [ + "FNLOG", + "VALIDATE_CONFIGURATION", + "CONNECT_TO_SERVER", + "DISCONNECT", + ] ); // Docstrings @@ -3038,9 +2989,9 @@ fn test_fixture_gwbasic() { .iter() .find(|f| f.name == "VALIDATE_CONFIGURATION") .unwrap(); - assert!( - validate_fn.docstring.is_some(), - "VALIDATE_CONFIGURATION should have docstring" + assert_eq!( + validate_fn.docstring.as_deref(), + Some("VALIDATE CONFIGURATION") ); // Complexity: CONNECT_TO_SERVER has a WHILE loop @@ -3050,25 +3001,21 @@ fn test_fixture_gwbasic() { "CONNECT_TO_SERVER should have >= 1 loop" ); - // Call sites (GOSUB references) - assert!( - result - .unresolved_refs - .iter() - .any(|r| r.reference_kind == EdgeKind::Calls), - "expected Calls refs" + // GOSUB targets are line numbers + assert_eq!( + ref_names(&result, EdgeKind::Calls), + ["1000", "2000", "3000"] ); - - // Contains edges - let contains: Vec<_> = result - .edges - .iter() - .filter(|e| e.kind == EdgeKind::Contains) - .collect(); - assert!( - contains.len() >= 6, - "expected >= 6 Contains edges, got {}", - contains.len() + assert_eq!( + contained_children(&result, "sample.gw"), + [ + "MR", + "DP", + "FNLOG", + "VALIDATE_CONFIGURATION", + "CONNECT_TO_SERVER", + "DISCONNECT", + ] ); } @@ -3079,7 +3026,7 @@ fn test_fixture_gwbasic() { fn test_fixture_qbasic() { let source = read_fixture("sample.qb"); let extractor = tracedecay_code_extraction::QBasicExtractor; - let result = extractor.extract("sample.qb", &source); + let result = extractor.extract_artifact("sample.qb", &source).result; assert!( result.errors.is_empty(), "QBasic errors: {:?}", @@ -3109,10 +3056,10 @@ fn test_fixture_qbasic() { .iter() .filter(|n| n.kind == NodeKind::Field && n.qualified_name.contains("Endpoint")) .collect(); - assert!( - struct_fields.len() >= 3, - "expected >= 3 Endpoint fields, got {}", - struct_fields.len() + assert_eq!(names(&struct_fields), ["host", "port", "connected"]); + assert_eq!( + contained_children(&result, "Endpoint"), + ["host", "port", "connected"] ); // SUBs and FUNCTION as Function nodes @@ -3121,33 +3068,22 @@ fn test_fixture_qbasic() { .iter() .filter(|n| n.kind == NodeKind::Function) .collect(); - assert!(fns.len() >= 5, "expected >= 5 functions, got {}", fns.len()); - assert!( - fns.iter().any(|f| f.name == "LogMessage"), - "LogMessage not found" - ); - assert!( - fns.iter().any(|f| f.name == "ValidateConfig"), - "ValidateConfig not found" - ); - assert!( - fns.iter().any(|f| f.name == "ConnectServer"), - "ConnectServer not found" - ); - assert!( - fns.iter().any(|f| f.name == "DisconnectServer"), - "DisconnectServer not found" - ); - assert!( - fns.iter().any(|f| f.name == "IsConnected"), - "IsConnected not found" + assert_eq!( + names(&fns), + [ + "LogMessage", + "ValidateConfig", + "ConnectServer", + "DisconnectServer", + "IsConnected", + ] ); // Docstrings on functions let log_fn = fns.iter().find(|f| f.name == "LogMessage").unwrap(); - assert!( - log_fn.docstring.is_some(), - "LogMessage should have docstring" + assert_eq!( + log_fn.docstring.as_deref(), + Some("Logs a message with the given level.") ); // Complexity: ValidateConfig has IF branches, ConnectServer has FOR loop @@ -3159,32 +3095,24 @@ fn test_fixture_qbasic() { let connect_fn = fns.iter().find(|f| f.name == "ConnectServer").unwrap(); assert!(connect_fn.loops >= 1, "ConnectServer should have >= 1 loop"); - // CONST nodes - let consts: Vec<_> = result - .nodes - .iter() - .filter(|n| n.kind == NodeKind::Const) - .collect(); - assert!(!consts.is_empty(), "expected at least 1 CONST node"); - - // Call sites (CALL references) - assert!( - result - .unresolved_refs - .iter() - .any(|r| r.reference_kind == EdgeKind::Calls), - "expected Calls refs" + assert_eq!( + kind_names(&result, NodeKind::Const), + ["MAX_RETRIES", "DEFAULT_PORT"] ); - // Contains edges - let contains: Vec<_> = result - .edges - .iter() - .filter(|e| e.kind == EdgeKind::Contains) - .collect(); - assert!( - contains.len() >= 10, - "expected >= 10 Contains edges, got {}", - contains.len() + assert_eq!( + ref_names(&result, EdgeKind::Calls), + [ + "ValidateConfig", + "ConnectServer", + "DisconnectServer", + "LogMessage", + "LogMessage", + "LogMessage", + "LogMessage", + "LogMessage", + "LogMessage", + "LogMessage", + ] ); } diff --git a/crates/tracedecay-code-extraction/tests/main/fortran.rs b/crates/tracedecay-code-extraction/tests/main/fortran.rs index f8ba18f5ae..df8510df24 100644 --- a/crates/tracedecay-code-extraction/tests/main/fortran.rs +++ b/crates/tracedecay-code-extraction/tests/main/fortran.rs @@ -5,7 +5,7 @@ use tracedecay_domain::*; fn extract_fixture() -> ExtractionResult { let source = std::fs::read_to_string("../../tests/fixtures/sample.f90").unwrap(); let extractor = FortranExtractor; - let result = extractor.extract("sample.f90", &source); + let result = extractor.extract_artifact("sample.f90", &source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); result } @@ -80,7 +80,6 @@ fn test_fortran_call_sites() { .iter() .filter(|r| r.reference_kind == EdgeKind::Calls) .collect(); - assert!(!calls.is_empty(), "expected call site refs"); assert!( calls.iter().any(|r| r.reference_name == "log_message"), "expected call to log_message, got: {:?}", @@ -105,34 +104,32 @@ fn test_fortran_call_sites() { #[test] fn test_fortran_docstrings() { let result = extract_fixture(); - let log_msg = result - .nodes - .iter() - .find(|n| n.kind == NodeKind::Function && n.name == "log_message"); - assert!(log_msg.is_some(), "log_message not found"); - assert!( - log_msg.unwrap().docstring.is_some(), - "log_message should have docstring" - ); - - let create_ep = result - .nodes - .iter() - .find(|n| n.kind == NodeKind::Function && n.name == "create_endpoint"); - assert!(create_ep.is_some(), "create_endpoint not found"); - assert!( - create_ep.unwrap().docstring.is_some(), - "create_endpoint should have docstring" - ); - - let ep = result + let docs: Vec<(&str, &str)> = result .nodes .iter() - .find(|n| n.kind == NodeKind::Struct && n.name == "Endpoint"); - assert!(ep.is_some(), "Endpoint not found"); - assert!( - ep.unwrap().docstring.is_some(), - "Endpoint should have docstring" + .filter_map(|n| Some((n.name.as_str(), n.docstring.as_deref()?))) + .collect(); + assert_eq!( + docs, + [ + ( + "networking", + "Sample Fortran file exercising extractor features." + ), + ("MAX_RETRIES", "Maximum number of retries."), + ("DEFAULT_PORT", "Default port for connections."), + ("Endpoint", "Represents a network endpoint."), + ( + "PooledEndpoint", + "Extends Endpoint with pool functionality." + ), + ("Connectable", "Interface for connectable types."), + ("log_message", "Logs a message with the given level."), + ("create_endpoint", "Creates a new endpoint."), + ("connect_endpoint", "Connects an endpoint."), + ("disconnect_endpoint", "Disconnects an endpoint."), + ("is_connected", "Checks if endpoint is connected."), + ] ); } diff --git a/crates/tracedecay-code-extraction/tests/main/general.rs b/crates/tracedecay-code-extraction/tests/main/general.rs index b064b343be..e954a528a0 100644 --- a/crates/tracedecay-code-extraction/tests/main/general.rs +++ b/crates/tracedecay-code-extraction/tests/main/general.rs @@ -7,20 +7,16 @@ fn test_extract_derive_macros() { #[derive(Debug, Clone, Serialize)] pub struct Config { pub name: String } "#; - let result = RustExtractor.extract("src/config.rs", source); + let result = RustExtractor + .extract_artifact("src/config.rs", source) + .result; let derives: Vec<_> = result .unresolved_refs .iter() .filter(|r| r.reference_kind == EdgeKind::DerivesMacro) .collect(); - assert!( - !derives.is_empty(), - "should have derives_macro unresolved refs" - ); let names: Vec<&str> = derives.iter().map(|r| r.reference_name.as_str()).collect(); - assert!(names.contains(&"Debug")); - assert!(names.contains(&"Clone")); - assert!(names.contains(&"Serialize")); + assert_eq!(names, ["Clone", "Debug", "Serialize"]); } #[test] @@ -30,32 +26,23 @@ mod server { pub fn handle_request() {} } "#; - let result = RustExtractor.extract("src/lib.rs", source); + let result = RustExtractor.extract_artifact("src/lib.rs", source).result; let fns: Vec<_> = result .nodes .iter() .filter(|n| n.kind == NodeKind::Function) .collect(); assert_eq!(fns.len(), 1); - assert!(fns[0].qualified_name.contains("server")); - assert!(fns[0].qualified_name.contains("handle_request")); + assert_eq!(fns[0].qualified_name, "src/lib.rs::server::handle_request"); } #[test] -fn test_language_registry_finds_scala_extractor() { +fn test_language_registry_dispatches_by_extension() { let registry = LanguageRegistry::new(); - assert!(registry.extractor_for_file("Main.scala").is_some()); - assert!( - registry - .extractor_for_file("src/com/example/App.scala") - .is_some() - ); - assert!(registry.extractor_for_file("script.sc").is_some()); -} - -#[test] -fn test_language_registry_returns_none_for_unknown() { - let registry = LanguageRegistry::new(); - assert!(registry.extractor_for_file("style.css").is_none()); - assert!(registry.extractor_for_file("README.unknown").is_none()); + let language = |path: &str| registry.extractor_for_file(path).map(|e| e.language_name()); + assert_eq!(language("Main.scala"), Some("Scala")); + assert_eq!(language("src/com/example/App.scala"), Some("Scala")); + assert_eq!(language("script.sc"), Some("Scala")); + assert_eq!(language("style.css"), None); + assert_eq!(language("README.unknown"), None); } diff --git a/crates/tracedecay-code-extraction/tests/main/glsl.rs b/crates/tracedecay-code-extraction/tests/main/glsl.rs index 37076767ee..562d2463c6 100644 --- a/crates/tracedecay-code-extraction/tests/main/glsl.rs +++ b/crates/tracedecay-code-extraction/tests/main/glsl.rs @@ -4,10 +4,14 @@ use tracedecay_code_extraction::GlslExtractor; use tracedecay_code_extraction::LanguageExtractor; use tracedecay_domain::*; +include!("support/edges.rs"); + #[test] fn test_glsl_file_node_is_root() { let source = std::fs::read_to_string("../../tests/fixtures/sample.glsl").unwrap(); - let result = GlslExtractor.extract("sample.glsl", &source); + let result = GlslExtractor + .extract_artifact("sample.glsl", &source) + .result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let files: Vec<_> = result .nodes @@ -21,7 +25,9 @@ fn test_glsl_file_node_is_root() { #[test] fn test_glsl_extract_functions() { let source = std::fs::read_to_string("../../tests/fixtures/sample.glsl").unwrap(); - let result = GlslExtractor.extract("sample.glsl", &source); + let result = GlslExtractor + .extract_artifact("sample.glsl", &source) + .result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let fns: Vec<_> = result .nodes @@ -51,7 +57,9 @@ fn test_glsl_extract_functions() { #[test] fn test_glsl_extract_structs() { let source = std::fs::read_to_string("../../tests/fixtures/sample.glsl").unwrap(); - let result = GlslExtractor.extract("sample.glsl", &source); + let result = GlslExtractor + .extract_artifact("sample.glsl", &source) + .result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let structs: Vec<_> = result .nodes @@ -72,7 +80,9 @@ fn test_glsl_extract_structs() { #[test] fn test_glsl_extract_struct_fields() { let source = std::fs::read_to_string("../../tests/fixtures/sample.glsl").unwrap(); - let result = GlslExtractor.extract("sample.glsl", &source); + let result = GlslExtractor + .extract_artifact("sample.glsl", &source) + .result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let fields: Vec<_> = result .nodes @@ -98,7 +108,9 @@ fn test_glsl_extract_struct_fields() { #[test] fn test_glsl_extract_uniforms() { let source = std::fs::read_to_string("../../tests/fixtures/sample.glsl").unwrap(); - let result = GlslExtractor.extract("sample.glsl", &source); + let result = GlslExtractor + .extract_artifact("sample.glsl", &source) + .result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let consts: Vec<_> = result .nodes @@ -125,7 +137,9 @@ fn test_glsl_extract_uniforms() { #[test] fn test_glsl_extract_in_out_declarations() { let source = std::fs::read_to_string("../../tests/fixtures/sample.glsl").unwrap(); - let result = GlslExtractor.extract("sample.glsl", &source); + let result = GlslExtractor + .extract_artifact("sample.glsl", &source) + .result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let fields: Vec<_> = result .nodes @@ -157,7 +171,9 @@ fn test_glsl_extract_in_out_declarations() { #[test] fn test_glsl_extract_preproc_defines() { let source = std::fs::read_to_string("../../tests/fixtures/sample.glsl").unwrap(); - let result = GlslExtractor.extract("sample.glsl", &source); + let result = GlslExtractor + .extract_artifact("sample.glsl", &source) + .result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let consts: Vec<_> = result .nodes @@ -174,7 +190,9 @@ fn test_glsl_extract_preproc_defines() { #[test] fn test_glsl_extract_const_globals() { let source = std::fs::read_to_string("../../tests/fixtures/sample.glsl").unwrap(); - let result = GlslExtractor.extract("sample.glsl", &source); + let result = GlslExtractor + .extract_artifact("sample.glsl", &source) + .result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let consts: Vec<_> = result .nodes @@ -188,32 +206,27 @@ fn test_glsl_extract_const_globals() { #[test] fn test_glsl_function_docstrings() { let source = std::fs::read_to_string("../../tests/fixtures/sample.glsl").unwrap(); - let result = GlslExtractor.extract("sample.glsl", &source); + let result = GlslExtractor + .extract_artifact("sample.glsl", &source) + .result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let fresnel = result .nodes .iter() .find(|n| n.name == "fresnelSchlick") .unwrap(); - assert!( - fresnel.docstring.is_some(), - "fresnelSchlick should have a docstring" - ); - assert!( - fresnel - .docstring - .as_ref() - .unwrap() - .contains("Fresnel-Schlick"), - "docstring: {:?}", - fresnel.docstring + assert_eq!( + fresnel.docstring.as_deref(), + Some("Compute the Fresnel-Schlick approximation.") ); } #[test] fn test_glsl_function_signatures() { let source = std::fs::read_to_string("../../tests/fixtures/sample.glsl").unwrap(); - let result = GlslExtractor.extract("sample.glsl", &source); + let result = GlslExtractor + .extract_artifact("sample.glsl", &source) + .result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let dist = result .nodes @@ -234,20 +247,68 @@ fn test_glsl_function_signatures() { #[test] fn test_glsl_contains_edges() { let source = std::fs::read_to_string("../../tests/fixtures/sample.glsl").unwrap(); - let result = GlslExtractor.extract("sample.glsl", &source); + let result = GlslExtractor + .extract_artifact("sample.glsl", &source) + .result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); - let contains: Vec<_> = result - .edges + let contains = edge_pairs(&result, EdgeKind::Contains); + let nested: Vec<_> = contains .iter() - .filter(|e| e.kind == EdgeKind::Contains) + .filter(|(parent, _)| *parent != "sample.glsl") + .copied() .collect(); - assert!(!contains.is_empty(), "should have Contains edges"); + assert_eq!( + nested, + [ + ("PointLight", "position"), + ("PointLight", "color"), + ("PointLight", "intensity"), + ("PointLight", "radius"), + ("Material", "albedo"), + ("Material", "metallic"), + ("Material", "roughness"), + ] + ); + let top_level: Vec<&str> = contains + .iter() + .filter(|(parent, _)| *parent == "sample.glsl") + .map(|(_, child)| *child) + .collect(); + assert_eq!( + top_level, + [ + "MAX_LIGHTS", + "aPosition", + "aNormal", + "aTexCoord", + "vWorldPos", + "vNormal", + "vTexCoord", + "uModelMatrix", + "uViewMatrix", + "uProjectionMatrix", + "uTime", + "PointLight", + "Material", + "uLights", + "uNumLights", + "uMaterial", + "PI", + "fresnelSchlick", + "distributionGGX", + "geometrySchlickGGX", + "calculatePointLight", + "main", + ] + ); } #[test] fn test_glsl_call_sites() { let source = std::fs::read_to_string("../../tests/fixtures/sample.glsl").unwrap(); - let result = GlslExtractor.extract("sample.glsl", &source); + let result = GlslExtractor + .extract_artifact("sample.glsl", &source) + .result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let calls: Vec<_> = result .unresolved_refs @@ -276,7 +337,9 @@ fn test_glsl_call_sites() { #[test] fn test_glsl_complexity_metrics() { let source = std::fs::read_to_string("../../tests/fixtures/sample.glsl").unwrap(); - let result = GlslExtractor.extract("sample.glsl", &source); + let result = GlslExtractor + .extract_artifact("sample.glsl", &source) + .result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let calc = result .nodes diff --git a/crates/tracedecay-code-extraction/tests/main/go.rs b/crates/tracedecay-code-extraction/tests/main/go.rs index b0f6c8b516..752c9d5623 100644 --- a/crates/tracedecay-code-extraction/tests/main/go.rs +++ b/crates/tracedecay-code-extraction/tests/main/go.rs @@ -2,6 +2,8 @@ use tracedecay_code_extraction::GoExtractor; use tracedecay_code_extraction::LanguageExtractor; use tracedecay_domain::*; +include!("support/edges.rs"); + #[test] fn test_go_extract_package() { let source = r#"package main @@ -13,7 +15,7 @@ func main() { } "#; let extractor = GoExtractor; - let result = extractor.extract("main.go", source); + let result = extractor.extract_artifact("main.go", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let pkgs: Vec<_> = result .nodes @@ -36,7 +38,7 @@ func Add(a, b int) int { func helper() {} "#; let extractor = GoExtractor; - let result = extractor.extract("math.go", source); + let result = extractor.extract_artifact("math.go", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let fns: Vec<_> = result .nodes @@ -69,7 +71,7 @@ type Point struct { } "#; let extractor = GoExtractor; - let result = extractor.extract("model/point.go", source); + let result = extractor.extract_artifact("model/point.go", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let structs: Vec<_> = result .nodes @@ -102,14 +104,21 @@ type Config struct { } "#; let extractor = GoExtractor; - let result = extractor.extract("model/config.go", source); + let result = extractor.extract_artifact("model/config.go", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let tags: Vec<_> = result .nodes .iter() .filter(|n| n.kind == NodeKind::StructTag) + .map(|n| (n.name.as_str(), n.signature.as_deref())) .collect(); - assert!(tags.len() >= 2, "should extract struct tags"); + assert_eq!( + tags, + [ + ("Name:tag", Some(r#"`json:"name" yaml:"name"`"#)), + ("Port:tag", Some(r#"`json:"port"`"#)), + ] + ); } #[test] @@ -122,7 +131,7 @@ type Reader interface { } "#; let extractor = GoExtractor; - let result = extractor.extract("io/reader.go", source); + let result = extractor.extract_artifact("io/reader.go", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let ifaces: Vec<_> = result .nodes @@ -152,7 +161,7 @@ func (c Circle) String() string { } "#; let extractor = GoExtractor; - let result = extractor.extract("model/circle.go", source); + let result = extractor.extract_artifact("model/circle.go", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let methods: Vec<_> = result .nodes @@ -161,14 +170,9 @@ func (c Circle) String() string { .collect(); assert_eq!(methods.len(), 2); // Check Receives edges - let receives: Vec<_> = result - .edges - .iter() - .filter(|e| e.kind == EdgeKind::Receives) - .collect(); - assert!( - !receives.is_empty(), - "should have Receives edges for methods with receivers" + assert_eq!( + edge_pairs(&result, EdgeKind::Receives), + [("Area", "Circle"), ("String", "Circle")] ); } @@ -183,7 +187,7 @@ import ( ) "#; let extractor = GoExtractor; - let result = extractor.extract("main.go", source); + let result = extractor.extract_artifact("main.go", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let uses: Vec<_> = result .nodes @@ -202,7 +206,7 @@ const MaxSize = 1024 var counter int "#; let extractor = GoExtractor; - let result = extractor.extract("main.go", source); + let result = extractor.extract_artifact("main.go", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let consts: Vec<_> = result .nodes @@ -235,14 +239,15 @@ func main() { } "#; let extractor = GoExtractor; - let result = extractor.extract("main.go", source); + let result = extractor.extract_artifact("main.go", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let call_refs: Vec<_> = result .unresolved_refs .iter() .filter(|r| r.reference_kind == EdgeKind::Calls) + .map(|r| r.reference_name.as_str()) .collect(); - assert!(!call_refs.is_empty(), "should have call refs"); + assert_eq!(call_refs, ["fmt.Println", "greet"]); } #[test] @@ -252,7 +257,7 @@ fn test_go_extract_type_alias() { type StringSlice = []string "#; let extractor = GoExtractor; - let result = extractor.extract("main.go", source); + let result = extractor.extract_artifact("main.go", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let aliases: Vec<_> = result .nodes @@ -277,7 +282,7 @@ type ReadWriter interface { } "#; let extractor = GoExtractor; - let result = extractor.extract("io/io.go", source); + let result = extractor.extract_artifact("io/io.go", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); // Should have an Extends edge or unresolved ref for Reader embedded in ReadWriter let has_extends = result.edges.iter().any(|e| e.kind == EdgeKind::Extends) @@ -301,7 +306,7 @@ func Map[T any, U any](s []T, f func(T) U) []U { } "#; let extractor = GoExtractor; - let result = extractor.extract("main.go", source); + let result = extractor.extract_artifact("main.go", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let fns: Vec<_> = result .nodes @@ -315,9 +320,9 @@ func Map[T any, U any](s []T, f func(T) U) []U { .iter() .filter(|n| n.kind == NodeKind::GenericParam) .collect(); - assert!( - generics.len() >= 2, - "should extract generic type params T and U" + assert_eq!( + generics.iter().map(|x| x.name.as_str()).collect::>(), + ["T", "U"] ); } @@ -328,7 +333,7 @@ fn test_go_file_node_is_root() { func main() {} "#; let extractor = GoExtractor; - let result = extractor.extract("main.go", source); + let result = extractor.extract_artifact("main.go", source).result; let files: Vec<_> = result .nodes .iter() @@ -349,17 +354,15 @@ type Foo struct { func (f Foo) Baz() {} "#; let extractor = GoExtractor; - let result = extractor.extract("main.go", source); - let contains: Vec<_> = result - .edges - .iter() - .filter(|e| e.kind == EdgeKind::Contains) - .collect(); - // File contains: GoPackage, Struct, StructMethod; Struct contains: Field - assert!( - contains.len() >= 4, - "should have Contains edges: {:?}", - contains.len() + let result = extractor.extract_artifact("main.go", source).result; + assert_eq!( + edge_pairs(&result, EdgeKind::Contains), + [ + ("main.go", "main"), + ("main.go", "Foo"), + ("Foo", "Bar"), + ("main.go", "Baz") + ] ); } @@ -370,7 +373,9 @@ fn test_go_qualified_names() { func HandleRequest() {} "#; let extractor = GoExtractor; - let result = extractor.extract("pkg/server/handler.go", source); + let result = extractor + .extract_artifact("pkg/server/handler.go", source) + .result; let fns: Vec<_> = result .nodes .iter() diff --git a/crates/tracedecay-code-extraction/tests/main/gwbasic.rs b/crates/tracedecay-code-extraction/tests/main/gwbasic.rs index c7543ed115..d24cc8f857 100644 --- a/crates/tracedecay-code-extraction/tests/main/gwbasic.rs +++ b/crates/tracedecay-code-extraction/tests/main/gwbasic.rs @@ -5,7 +5,7 @@ use tracedecay_domain::*; fn extract_fixture() -> ExtractionResult { let source = std::fs::read_to_string("../../tests/fixtures/sample.gw").unwrap(); let extractor = GwBasicExtractor; - let result = extractor.extract("sample.gw", &source); + let result = extractor.extract_artifact("sample.gw", &source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); result } @@ -18,8 +18,6 @@ fn test_gwbasic_gosub_calls() { .iter() .filter(|r| r.reference_kind == EdgeKind::Calls) .collect(); - assert!(!calls.is_empty(), "expected call site refs"); - assert!( calls.iter().any(|r| r.reference_name == "1000"), "expected GOSUB 1000 call, got: {:?}", @@ -38,44 +36,18 @@ fn test_gwbasic_gosub_calls() { #[test] fn test_gwbasic_docstrings() { let result = extract_fixture(); - - let validate_fn = result - .nodes - .iter() - .find(|n| n.kind == NodeKind::Function && n.name == "VALIDATE_CONFIGURATION") - .expect("VALIDATE_CONFIGURATION function not found"); - assert!( - validate_fn.docstring.is_some(), - "VALIDATE_CONFIGURATION should have docstring" - ); - assert!( - validate_fn - .docstring - .as_ref() - .unwrap() - .contains("VALIDATE CONFIGURATION"), - "docstring: {:?}", - validate_fn.docstring - ); - - let connect_fn = result - .nodes - .iter() - .find(|n| n.kind == NodeKind::Function && n.name == "CONNECT_TO_SERVER") - .expect("CONNECT_TO_SERVER function not found"); - assert!( - connect_fn.docstring.is_some(), - "CONNECT_TO_SERVER should have docstring" - ); - - let disconnect_fn = result + let docs: Vec<(&str, &str)> = result .nodes .iter() - .find(|n| n.kind == NodeKind::Function && n.name == "DISCONNECT") - .expect("DISCONNECT function not found"); - assert!( - disconnect_fn.docstring.is_some(), - "DISCONNECT should have docstring" + .filter_map(|n| Some((n.name.as_str(), n.docstring.as_deref()?))) + .collect(); + assert_eq!( + docs, + [ + ("VALIDATE_CONFIGURATION", "VALIDATE CONFIGURATION"), + ("CONNECT_TO_SERVER", "CONNECT TO SERVER"), + ("DISCONNECT", "DISCONNECT"), + ] ); } @@ -122,3 +94,17 @@ fn test_gwbasic_subroutine_signatures() { validate_fn.signature ); } + +#[test] +fn test_gwbasic_let_name_keeps_underscores() { + let result = GwBasicExtractor + .extract_artifact("names.gw", "10 LET MAX_RETRIES = 3\n20 LET MR = 1\n") + .result; + let consts: Vec<&str> = result + .nodes + .iter() + .filter(|n| n.kind == NodeKind::Const) + .map(|n| n.name.as_str()) + .collect(); + assert_eq!(consts, ["MAX_RETRIES", "MR"]); +} diff --git a/crates/tracedecay-code-extraction/tests/main/incremental_parse.rs b/crates/tracedecay-code-extraction/tests/main/incremental_parse.rs index 6c200d4e09..cd32c93b04 100644 --- a/crates/tracedecay-code-extraction/tests/main/incremental_parse.rs +++ b/crates/tracedecay-code-extraction/tests/main/incremental_parse.rs @@ -337,7 +337,7 @@ fn canonical_reextraction_visits_only_changed_top_level_syntax() { ) .expect("initial parse"); let initial = document - .extract_canonical(&RustExtractor, &opened, None) + .extract_canonical_artifact(&RustExtractor, &opened, None) .expect("initial canonical extraction"); assert_eq!( initial.disposition, @@ -351,7 +351,7 @@ fn canonical_reextraction_visits_only_changed_top_level_syntax() { ) .expect("incremental parse"); let increment = document - .extract_canonical(&RustExtractor, &report, Some(&initial.result)) + .extract_canonical_artifact(&RustExtractor, &report, Some(&initial.artifact)) .expect("incremental canonical extraction"); assert_eq!( @@ -361,6 +361,7 @@ fn canonical_reextraction_visits_only_changed_top_level_syntax() { assert_eq!(increment.metrics.visited_top_level_nodes, 1); assert!(increment.metrics.visited_bytes < after.len()); let edited = increment + .artifact .result .nodes .iter() @@ -368,7 +369,7 @@ fn canonical_reextraction_visits_only_changed_top_level_syntax() { .expect("edited function"); assert!(edited.is_async); assert!(matches!( - document.extract_canonical(&RustExtractor, &opened, Some(&initial.result)), + document.extract_canonical_artifact(&RustExtractor, &opened, Some(&initial.artifact)), Err(ParseError::StaleReport) )); } @@ -431,7 +432,7 @@ fn same_line_column_shifts_reextract_following_top_level_syntax() { ) .expect("initial parse"); let initial = document - .extract_canonical(&RustExtractor, &opened, None) + .extract_canonical_artifact(&RustExtractor, &opened, None) .expect("initial canonical extraction"); let report = document @@ -441,9 +442,10 @@ fn same_line_column_shifts_reextract_following_top_level_syntax() { ) .expect("same-line incremental parse"); let incremental = document - .extract_canonical(&RustExtractor, &report, Some(&initial.result)) + .extract_canonical_artifact(&RustExtractor, &report, Some(&initial.artifact)) .expect("same-line canonical extraction"); let following = incremental + .artifact .result .nodes .iter() @@ -472,7 +474,7 @@ fn same_line_method_edit_keeps_both_methods_distinct_after_merge() { ) .expect("initial parse"); let initial = document - .extract_canonical(&RustExtractor, &opened, None) + .extract_canonical_artifact(&RustExtractor, &opened, None) .expect("initial canonical extraction"); let report = document @@ -483,7 +485,7 @@ fn same_line_method_edit_keeps_both_methods_distinct_after_merge() { .expect("same-line incremental parse"); assert_eq!(report.reuse, ParseReuse::Incremental); let incremental = document - .extract_canonical(&RustExtractor, &report, Some(&initial.result)) + .extract_canonical_artifact(&RustExtractor, &report, Some(&initial.artifact)) .expect("same-line canonical extraction"); assert_eq!( incremental.disposition, @@ -491,8 +493,8 @@ fn same_line_method_edit_keeps_both_methods_distinct_after_merge() { ); assert_eq!(incremental.metrics.visited_top_level_nodes, 1); - let mut cold = RustExtractor.extract("src/lib.rs", after); - let mut merged = incremental.result; + let mut cold = RustExtractor.extract_artifact("src/lib.rs", after).result; + let mut merged = incremental.artifact.result; for result in [&mut cold, &mut merged] { result.duration_ms = 0; for node in &mut result.nodes { @@ -576,16 +578,16 @@ fn file_root_span_matches_cold_extraction_for_every_line_ending_shape() { ) .expect("initial parse"); let initial = document - .extract_canonical(&RustExtractor, &opened, None) + .extract_canonical_artifact(&RustExtractor, &opened, None) .expect("initial canonical extraction"); - let cold_before = RustExtractor.extract("src/lib.rs", before); + let cold_before = RustExtractor.extract_artifact("src/lib.rs", before).result; assert_eq!( - timeless_rows(&initial.result), + timeless_rows(&initial.artifact.result), timeless_rows(&cold_before), "initial rows for {before:?}" ); assert_eq!( - file_root_end_line(&initial.result), + file_root_end_line(&initial.artifact.result), before.lines().count().saturating_sub(1) as u32, "initial file root for {before:?}" ); @@ -602,21 +604,21 @@ fn file_root_span_matches_cold_extraction_for_every_line_ending_shape() { "{before:?} -> {after:?}" ); let incremental = document - .extract_canonical(&RustExtractor, &report, Some(&initial.result)) + .extract_canonical_artifact(&RustExtractor, &report, Some(&initial.artifact)) .expect("same-line canonical extraction"); assert_eq!( incremental.disposition, ParsedExtractionDisposition::ChangedRegions, "{before:?} -> {after:?}" ); - let cold_after = RustExtractor.extract("src/lib.rs", after); + let cold_after = RustExtractor.extract_artifact("src/lib.rs", after).result; assert_eq!( - timeless_rows(&incremental.result), + timeless_rows(&incremental.artifact.result), timeless_rows(&cold_after), "same-line merged rows for {before:?} -> {after:?}" ); assert_eq!( - file_root_end_line(&incremental.result), + file_root_end_line(&incremental.artifact.result), after.lines().count().saturating_sub(1) as u32, "same-line file root for {after:?}" ); @@ -629,7 +631,7 @@ fn file_root_span_matches_cold_extraction_for_every_line_ending_shape() { ) .expect("multiline incremental parse"); let reset = document - .extract_canonical(&RustExtractor, &report, Some(&incremental.result)) + .extract_canonical_artifact(&RustExtractor, &report, Some(&incremental.artifact)) .expect("multiline canonical extraction"); assert_eq!( reset.disposition, @@ -638,14 +640,16 @@ fn file_root_span_matches_cold_extraction_for_every_line_ending_shape() { }, "{after:?} -> {multiline:?}" ); - let cold_multiline = RustExtractor.extract("src/lib.rs", &multiline); + let cold_multiline = RustExtractor + .extract_artifact("src/lib.rs", &multiline) + .result; assert_eq!( - timeless_rows(&reset.result), + timeless_rows(&reset.artifact.result), timeless_rows(&cold_multiline), "reset rows for {multiline:?}" ); assert_eq!( - file_root_end_line(&reset.result), + file_root_end_line(&reset.artifact.result), multiline.lines().count().saturating_sub(1) as u32, "reset file root for {multiline:?}" ); diff --git a/crates/tracedecay-code-extraction/tests/main/java.rs b/crates/tracedecay-code-extraction/tests/main/java.rs index 867ae57fac..7a4dd640fa 100644 --- a/crates/tracedecay-code-extraction/tests/main/java.rs +++ b/crates/tracedecay-code-extraction/tests/main/java.rs @@ -2,6 +2,8 @@ use tracedecay_code_extraction::JavaExtractor; use tracedecay_code_extraction::LanguageExtractor; use tracedecay_domain::*; +include!("support/edges.rs"); + #[test] fn test_java_empty_javadoc_no_panic() { let source = r#" @@ -13,7 +15,9 @@ public class PanicReproduction { } "#; let extractor = JavaExtractor; - let result = extractor.extract("PanicReproduction.java", source); + let result = extractor + .extract_artifact("PanicReproduction.java", source) + .result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let enums: Vec<_> = result @@ -35,7 +39,7 @@ public class Main { } "#; let extractor = JavaExtractor; - let result = extractor.extract("src/Main.java", source); + let result = extractor.extract_artifact("src/Main.java", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let pkgs: Vec<_> = result .nodes @@ -60,7 +64,7 @@ public class Calculator { } "#; let extractor = JavaExtractor; - let result = extractor.extract("Calculator.java", source); + let result = extractor.extract_artifact("Calculator.java", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let classes: Vec<_> = result .nodes @@ -89,7 +93,7 @@ public class Foo { } "#; let extractor = JavaExtractor; - let result = extractor.extract("Foo.java", source); + let result = extractor.extract_artifact("Foo.java", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let methods: Vec<_> = result .nodes @@ -116,7 +120,7 @@ public class Person { } "#; let extractor = JavaExtractor; - let result = extractor.extract("Person.java", source); + let result = extractor.extract_artifact("Person.java", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let constructors: Vec<_> = result .nodes @@ -136,7 +140,7 @@ public interface Drawable { } "#; let extractor = JavaExtractor; - let result = extractor.extract("Drawable.java", source); + let result = extractor.extract_artifact("Drawable.java", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let ifaces: Vec<_> = result .nodes @@ -163,7 +167,7 @@ public enum Color { } "#; let extractor = JavaExtractor; - let result = extractor.extract("Color.java", source); + let result = extractor.extract_artifact("Color.java", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let enums: Vec<_> = result .nodes @@ -189,7 +193,7 @@ public class Config { } "#; let extractor = JavaExtractor; - let result = extractor.extract("Config.java", source); + let result = extractor.extract_artifact("Config.java", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let fields: Vec<_> = result .nodes @@ -211,7 +215,7 @@ import static java.lang.Math.PI; public class Foo {} "#; let extractor = JavaExtractor; - let result = extractor.extract("Foo.java", source); + let result = extractor.extract_artifact("Foo.java", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let uses: Vec<_> = result .nodes @@ -231,7 +235,7 @@ class Worker extends Base implements Runnable { } "#; let extractor = JavaExtractor; - let result = extractor.extract("Worker.java", source); + let result = extractor.extract_artifact("Worker.java", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let has_extends = result.edges.iter().any(|e| e.kind == EdgeKind::Extends) || result @@ -257,7 +261,7 @@ public class Foo { } "#; let extractor = JavaExtractor; - let result = extractor.extract("Foo.java", source); + let result = extractor.extract_artifact("Foo.java", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let annots: Vec<_> = result @@ -295,7 +299,7 @@ public class Outer { } "#; let extractor = JavaExtractor; - let result = extractor.extract("Outer.java", source); + let result = extractor.extract_artifact("Outer.java", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let inners: Vec<_> = result .nodes @@ -317,7 +321,7 @@ public class Registry { } "#; let extractor = JavaExtractor; - let result = extractor.extract("Registry.java", source); + let result = extractor.extract_artifact("Registry.java", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let init_blocks: Vec<_> = result .nodes @@ -336,7 +340,7 @@ public abstract class Shape { } "#; let extractor = JavaExtractor; - let result = extractor.extract("Shape.java", source); + let result = extractor.extract_artifact("Shape.java", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let abstract_methods: Vec<_> = result .nodes @@ -356,14 +360,15 @@ public class Box { } "#; let extractor = JavaExtractor; - let result = extractor.extract("Box.java", source); + let result = extractor.extract_artifact("Box.java", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let generics: Vec<_> = result .nodes .iter() .filter(|n| n.kind == NodeKind::GenericParam) + .map(|n| n.name.as_str()) .collect(); - assert!(!generics.is_empty(), "should extract generic type param T"); + assert_eq!(generics, ["T"]); } #[test] @@ -379,14 +384,18 @@ public class App { } "#; let extractor = JavaExtractor; - let result = extractor.extract("App.java", source); + let result = extractor.extract_artifact("App.java", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let call_refs: Vec<_> = result .unresolved_refs .iter() .filter(|r| r.reference_kind == EdgeKind::Calls) + .map(|r| r.reference_name.as_str()) .collect(); - assert!(!call_refs.is_empty(), "should have call refs"); + assert_eq!( + call_refs, + ["System.out.println", "helper", "new ArrayList<>"] + ); } #[test] @@ -397,7 +406,9 @@ public @interface MyAnnotation { } "#; let extractor = JavaExtractor; - let result = extractor.extract("MyAnnotation.java", source); + let result = extractor + .extract_artifact("MyAnnotation.java", source) + .result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let annots: Vec<_> = result .nodes @@ -412,7 +423,7 @@ public @interface MyAnnotation { fn test_java_file_node_is_root() { let source = "public class Main {}"; let extractor = JavaExtractor; - let result = extractor.extract("src/Main.java", source); + let result = extractor.extract_artifact("src/Main.java", source).result; let files: Vec<_> = result .nodes .iter() @@ -431,17 +442,10 @@ public class Foo { } "#; let extractor = JavaExtractor; - let result = extractor.extract("Foo.java", source); - let contains: Vec<_> = result - .edges - .iter() - .filter(|e| e.kind == EdgeKind::Contains) - .collect(); - // File contains: Class; Class contains: Field, Method - assert!( - contains.len() >= 3, - "should have Contains edges: {}", - contains.len() + let result = extractor.extract_artifact("Foo.java", source).result; + assert_eq!( + edge_pairs(&result, EdgeKind::Contains), + [("Foo.java", "Foo"), ("Foo", "x"), ("Foo", "bar")] ); } @@ -455,7 +459,7 @@ public class App { } "#; let extractor = JavaExtractor; - let result = extractor.extract("src/App.java", source); + let result = extractor.extract_artifact("src/App.java", source).result; let methods: Vec<_> = result .nodes .iter() diff --git a/crates/tracedecay-code-extraction/tests/main/kotlin.rs b/crates/tracedecay-code-extraction/tests/main/kotlin.rs index 4960fa4bd3..47e5478389 100644 --- a/crates/tracedecay-code-extraction/tests/main/kotlin.rs +++ b/crates/tracedecay-code-extraction/tests/main/kotlin.rs @@ -4,13 +4,15 @@ use tracedecay_domain::*; fn extract(source: &str) -> ExtractionResult { let extractor = KotlinExtractor; - extractor.extract("test.kt", source) + extractor.extract_artifact("test.kt", source).result } // ----------------------------------------------------------------------- // File node // ----------------------------------------------------------------------- +include!("support/edges.rs"); + #[test] fn test_kt_file_node_is_root() { let result = extract("fun main() {}"); @@ -287,14 +289,9 @@ fn test_kt_annotation() { assert_eq!(annots[0].name, "Deprecated"); // Should have an Annotates edge. - let annotates_edges: Vec<_> = result - .edges - .iter() - .filter(|e| e.kind == EdgeKind::Annotates) - .collect(); - assert!( - !annotates_edges.is_empty(), - "expected at least one Annotates edge" + assert_eq!( + edge_pairs(&result, EdgeKind::Annotates), + [("Deprecated", "oldFunc")] ); } @@ -338,12 +335,7 @@ fn test_kt_kdoc() { .filter(|n| n.kind == NodeKind::Function) .collect(); assert_eq!(fns.len(), 1); - assert!(fns[0].docstring.is_some(), "expected docstring"); - assert!( - fns[0].docstring.as_ref().unwrap().contains("KDoc comment"), - "docstring: {:?}", - fns[0].docstring - ); + assert_eq!(fns[0].docstring.as_deref(), Some("This is a KDoc comment")); } // ----------------------------------------------------------------------- @@ -415,7 +407,6 @@ fn test_kt_call_site() { .iter() .filter(|r| r.reference_kind == EdgeKind::Calls) .collect(); - assert!(!calls.is_empty(), "expected at least one call site"); assert!( calls.iter().any(|c| c.reference_name == "println"), "expected println call, got: {:?}", @@ -432,16 +423,9 @@ fn test_kt_contains_edges() { let source = "class MyClass {\n fun hello() {}\n}"; let result = extract(source); assert!(result.errors.is_empty(), "errors: {:?}", result.errors); - let contains_edges: Vec<_> = result - .edges - .iter() - .filter(|e| e.kind == EdgeKind::Contains) - .collect(); - // File->Class and Class->Method at minimum - assert!( - contains_edges.len() >= 2, - "expected at least 2 Contains edges, got {}", - contains_edges.len() + assert_eq!( + edge_pairs(&result, EdgeKind::Contains), + [("test.kt", "MyClass"), ("MyClass", "hello")] ); } diff --git a/crates/tracedecay-code-extraction/tests/main/lean.rs b/crates/tracedecay-code-extraction/tests/main/lean.rs index 930220d689..3374d1bb0b 100644 --- a/crates/tracedecay-code-extraction/tests/main/lean.rs +++ b/crates/tracedecay-code-extraction/tests/main/lean.rs @@ -3,7 +3,7 @@ use tracedecay_code_extraction::LeanExtractor; use tracedecay_domain::*; fn extract(source: &str) -> ExtractionResult { - LeanExtractor.extract("Demo.lean", source) + LeanExtractor.extract_artifact("Demo.lean", source).result } fn names_of(result: &ExtractionResult, kind: NodeKind) -> Vec { diff --git a/crates/tracedecay-code-extraction/tests/main/lua.rs b/crates/tracedecay-code-extraction/tests/main/lua.rs index a1be25b493..6be7d4a488 100644 --- a/crates/tracedecay-code-extraction/tests/main/lua.rs +++ b/crates/tracedecay-code-extraction/tests/main/lua.rs @@ -6,7 +6,7 @@ use tracedecay_domain::*; fn test_lua_call_sites() { let source = std::fs::read_to_string("../../tests/fixtures/sample.lua").unwrap(); let extractor = LuaExtractor; - let result = extractor.extract("sample.lua", &source); + let result = extractor.extract_artifact("sample.lua", &source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let call_refs: Vec<_> = result @@ -14,8 +14,6 @@ fn test_lua_call_sites() { .iter() .filter(|r| r.reference_kind == EdgeKind::Calls) .collect(); - assert!(!call_refs.is_empty(), "should have call refs"); - assert!( call_refs.iter().any(|r| r.reference_name == "print"), "should find print call" @@ -57,68 +55,35 @@ fn test_lua_call_sites() { #[test] fn test_lua_docstrings() { let source = std::fs::read_to_string("../../tests/fixtures/sample.lua").unwrap(); - let extractor = LuaExtractor; - let result = extractor.extract("sample.lua", &source); + let result = LuaExtractor.extract_artifact("sample.lua", &source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); - - let log_fn = result - .nodes - .iter() - .find(|n| n.kind == NodeKind::Function && n.name == "log") - .expect("log function not found"); - assert!(log_fn.docstring.is_some(), "log should have docstring"); - let doc = log_fn.docstring.as_ref().unwrap(); - assert!( - doc.contains("Logs a message"), - "docstring should contain 'Logs a message', got: {}", - doc - ); - - let connect_method = result + let docs: Vec<(&str, &str)> = result .nodes .iter() - .find(|n| n.kind == NodeKind::Method && n.name == "connect") - .expect("connect method not found"); - assert!( - connect_method - .docstring - .as_ref() - .unwrap() - .contains("Connects to the remote host"), - "docstring: {:?}", - connect_method.docstring - ); - - let max_retries = result - .nodes - .iter() - .find(|n| n.kind == NodeKind::Const && n.name == "MAX_RETRIES") - .expect("MAX_RETRIES not found"); - assert!( - max_retries - .docstring - .as_ref() - .unwrap() - .contains("Maximum number of retries"), - "docstring: {:?}", - max_retries.docstring - ); -} - -#[test] -fn test_lua_contains_edges() { - let source = std::fs::read_to_string("../../tests/fixtures/sample.lua").unwrap(); - let extractor = LuaExtractor; - let result = extractor.extract("sample.lua", &source); - let contains: Vec<_> = result - .edges - .iter() - .filter(|e| e.kind == EdgeKind::Contains) + .filter_map(|n| Some((n.name.as_str(), n.docstring.as_deref()?))) .collect(); - assert!( - contains.len() >= 12, - "should have >= 12 Contains edges, got {}", - contains.len() + assert_eq!( + docs, + [ + ("MAX_RETRIES", "Maximum number of retries."), + ("DEFAULT_PORT", "Default port for connections."), + ( + "log", + "Logs a message with the given level.\n\ + @param level string The log level\n\ + @param message string The message to log" + ), + ( + "new", + "Creates a new Connection.\n\ + @param host string The host to connect to\n\ + @param port number The port number\n\ + @return Connection" + ), + ("connect", "Connects to the remote host."), + ("disconnect", "Disconnects from the remote host."), + ("isConnected", "Checks if the connection is active."), + ] ); } @@ -126,7 +91,7 @@ fn test_lua_contains_edges() { fn test_lua_local_function_is_private() { let source = std::fs::read_to_string("../../tests/fixtures/sample.lua").unwrap(); let extractor = LuaExtractor; - let result = extractor.extract("sample.lua", &source); + let result = extractor.extract_artifact("sample.lua", &source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let log_fn = result @@ -145,7 +110,7 @@ fn test_lua_local_function_is_private() { fn test_lua_dot_function_qualified_name() { let source = std::fs::read_to_string("../../tests/fixtures/sample.lua").unwrap(); let extractor = LuaExtractor; - let result = extractor.extract("sample.lua", &source); + let result = extractor.extract_artifact("sample.lua", &source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let conn_new_fns: Vec<_> = result @@ -169,7 +134,7 @@ fn test_lua_dot_function_qualified_name() { fn test_lua_signatures() { let source = std::fs::read_to_string("../../tests/fixtures/sample.lua").unwrap(); let extractor = LuaExtractor; - let result = extractor.extract("sample.lua", &source); + let result = extractor.extract_artifact("sample.lua", &source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let log_fn = result diff --git a/crates/tracedecay-code-extraction/tests/main/markdown.rs b/crates/tracedecay-code-extraction/tests/main/markdown.rs index 6d3b71d3df..14c3d09e70 100644 --- a/crates/tracedecay-code-extraction/tests/main/markdown.rs +++ b/crates/tracedecay-code-extraction/tests/main/markdown.rs @@ -5,7 +5,9 @@ use tracedecay_domain::*; #[test] fn test_markdown_header_hierarchy() { let source = "# Top\n\n## Section1\n\n### Deep\n\n## Section2"; - let result = MarkdownExtractor.extract("README.md", source); + let result = MarkdownExtractor + .extract_artifact("README.md", source) + .result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); // Section1 and Section2 should be children of Top // Deep should be child of Section1 @@ -39,7 +41,9 @@ fn test_markdown_header_hierarchy() { #[test] fn test_markdown_skips_external_links() { let source = "Check [Google](https://google.com) for more."; - let result = MarkdownExtractor.extract("README.md", source); + let result = MarkdownExtractor + .extract_artifact("README.md", source) + .result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let uses_edges: Vec<_> = result .edges @@ -55,7 +59,9 @@ fn test_markdown_skips_external_links() { #[test] fn test_markdown_skips_non_code_links() { let source = "See [image](docs/image.png) for diagram."; - let result = MarkdownExtractor.extract("README.md", source); + let result = MarkdownExtractor + .extract_artifact("README.md", source) + .result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); // .png is not a code extension, so no Uses edge let uses_edges: Vec<_> = result @@ -72,7 +78,9 @@ fn test_markdown_skips_non_code_links() { #[test] fn test_markdown_handles_empty_file() { let source = ""; - let result = MarkdownExtractor.extract("README.md", source); + let result = MarkdownExtractor + .extract_artifact("README.md", source) + .result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); // Should still have a File node let files: Vec<_> = result @@ -86,7 +94,9 @@ fn test_markdown_handles_empty_file() { #[test] fn test_markdown_handles_no_headers() { let source = "Just some plain text without any headers."; - let result = MarkdownExtractor.extract("README.md", source); + let result = MarkdownExtractor + .extract_artifact("README.md", source) + .result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let modules: Vec<_> = result .nodes @@ -99,7 +109,9 @@ fn test_markdown_handles_no_headers() { #[test] fn test_markdown_multiple_links_same_line() { let source = "See [main](src/main.rs) and [lib](src/lib.rs)."; - let result = MarkdownExtractor.extract("README.md", source); + let result = MarkdownExtractor + .extract_artifact("README.md", source) + .result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let uses_edges: Vec<_> = result .edges @@ -112,7 +124,9 @@ fn test_markdown_multiple_links_same_line() { #[test] fn test_markdown_handles_header_with_punctuation() { let source = "# Hello, World! (2024)"; - let result = MarkdownExtractor.extract("README.md", source); + let result = MarkdownExtractor + .extract_artifact("README.md", source) + .result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let modules: Vec<_> = result .nodes @@ -128,7 +142,9 @@ fn test_markdown_link_inside_heading_emits_uses_edge() { // `## See [main](src/main.rs)`. The link inside the heading should // be captured as a Uses edge parented to that heading. let source = "## See [main](src/main.rs)\n"; - let result = MarkdownExtractor.extract("README.md", source); + let result = MarkdownExtractor + .extract_artifact("README.md", source) + .result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let uses_edges: Vec<_> = result @@ -156,7 +172,9 @@ fn test_markdown_link_in_heading_does_not_double_count_body_links() { // A heading with a link, plus a body paragraph with another link, // produces exactly two Uses edges. One per link. let source = "# [foo](src/foo.rs)\n\nSee also [bar](src/bar.rs).\n"; - let result = MarkdownExtractor.extract("README.md", source); + let result = MarkdownExtractor + .extract_artifact("README.md", source) + .result; let uses_edges: Vec<_> = result .edges .iter() diff --git a/crates/tracedecay-code-extraction/tests/main/markdown_modern_grammar.rs b/crates/tracedecay-code-extraction/tests/main/markdown_modern_grammar.rs index fdcdd86c56..2d940b5a4f 100644 --- a/crates/tracedecay-code-extraction/tests/main/markdown_modern_grammar.rs +++ b/crates/tracedecay-code-extraction/tests/main/markdown_modern_grammar.rs @@ -6,19 +6,23 @@ //! grammar produces an opaque `(minus_metadata)` node so the body's //! markdown rules never see the YAML. use std::time::{Duration, Instant}; +use tracedecay_domain::NodeKind; -fn timed_extract(source: String, timeout: Duration) -> Option<(f64, usize, usize)> { +fn timed_extract(source: String, timeout: Duration) -> Option<(f64, Vec<(NodeKind, String)>)> { let (tx, rx) = std::sync::mpsc::channel(); std::thread::spawn(move || { let t0 = Instant::now(); let res = tracedecay_code_extraction::MarkdownExtractor::extract_markdown("t.md", &source); - let _ = tx.send((t0.elapsed().as_secs_f64(), res.nodes.len(), res.edges.len())); + let nodes = res.nodes.into_iter().map(|n| (n.kind, n.name)).collect(); + let _ = tx.send((t0.elapsed().as_secs_f64(), nodes)); }); rx.recv_timeout(timeout).ok() } /// 4.4 KB / 113-line YAML-frontmatter-heavy file that hung the old grammar /// indefinitely. With the new grammar it must parse in well under a second. +/// The fixture's frontmatter is never closed, so the whole file is metadata +/// and its YAML list items must not surface as document structure. #[test] fn yaml_frontmatter_hang_reproducer() { let path = concat!( @@ -26,10 +30,11 @@ fn yaml_frontmatter_hang_reproducer() { "/../../tests/fixtures/markdown_yaml_frontmatter_hang.md" ); let src = std::fs::read_to_string(path).expect("fixture missing"); - match timed_extract(src, Duration::from_secs(5)) { - Some((t, n, _)) => assert!(t < 1.0, "should parse fast, took {t:.3}s ({n} nodes)"), - None => panic!("hang reproducer still hung > 5s"), - } + let Some((t, nodes)) = timed_extract(src, Duration::from_secs(5)) else { + panic!("hang reproducer still hung > 5s"); + }; + assert!(t < 1.0, "should parse fast, took {t:.3}s"); + assert_eq!(nodes, vec![(NodeKind::File, "t.md".to_owned())]); } #[test] diff --git a/crates/tracedecay-code-extraction/tests/main/metal.rs b/crates/tracedecay-code-extraction/tests/main/metal.rs index 67a58f05ce..47febf3a48 100644 --- a/crates/tracedecay-code-extraction/tests/main/metal.rs +++ b/crates/tracedecay-code-extraction/tests/main/metal.rs @@ -41,7 +41,9 @@ public: "#; fn extract() -> ExtractionResult { - let result = MetalExtractor.extract("shader.metal", SHADER); + let result = MetalExtractor + .extract_artifact("shader.metal", SHADER) + .result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); result } @@ -128,7 +130,7 @@ fn metal_retained_tree_extraction_matches_cold_extraction() { assert_eq!(report.reuse, ParseReuse::Initial); let retained = document - .extract_canonical(&MetalExtractor, &report, None) + .extract_canonical_artifact(&MetalExtractor, &report, None) .expect("retained extraction"); assert_eq!( retained.disposition, @@ -136,7 +138,7 @@ fn metal_retained_tree_extraction_matches_cold_extraction() { ); let mut cold = extract(); - let mut retained = retained.result; + let mut retained = retained.artifact.result; for result in [&mut cold, &mut retained] { result.duration_ms = 0; for node in &mut result.nodes { diff --git a/crates/tracedecay-code-extraction/tests/main/msbasic2.rs b/crates/tracedecay-code-extraction/tests/main/msbasic2.rs index f82b55e89f..7e8d1d22ab 100644 --- a/crates/tracedecay-code-extraction/tests/main/msbasic2.rs +++ b/crates/tracedecay-code-extraction/tests/main/msbasic2.rs @@ -5,7 +5,7 @@ use tracedecay_domain::*; fn extract_fixture() -> ExtractionResult { let source = std::fs::read_to_string("../../tests/fixtures/sample.bas").unwrap(); let extractor = MsBasic2Extractor; - let result = extractor.extract("sample.bas", &source); + let result = extractor.extract_artifact("sample.bas", &source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); result } @@ -18,8 +18,6 @@ fn test_msbasic2_gosub_calls() { .iter() .filter(|r| r.reference_kind == EdgeKind::Calls) .collect(); - assert!(!calls.is_empty(), "expected call site refs"); - assert!( calls.iter().any(|r| r.reference_name == "200"), "expected GOSUB 200 call, got: {:?}", @@ -38,49 +36,21 @@ fn test_msbasic2_gosub_calls() { #[test] fn test_msbasic2_docstrings() { let result = extract_fixture(); - - let log_fn = result - .nodes - .iter() - .find(|n| n.kind == NodeKind::Function && n.name == "LOG_A_MESSAGE") - .expect("LOG_A_MESSAGE function not found"); - assert!( - log_fn.docstring.is_some(), - "LOG_A_MESSAGE should have docstring" - ); - assert!( - log_fn.docstring.as_ref().unwrap().contains("LOG A MESSAGE"), - "docstring: {:?}", - log_fn.docstring - ); - - let connect_fn = result + let docs: Vec<(&str, &str)> = result .nodes .iter() - .find(|n| n.kind == NodeKind::Function && n.name == "CONNECT_TO_SERVER") - .expect("CONNECT_TO_SERVER function not found"); - assert!( - connect_fn.docstring.is_some(), - "CONNECT_TO_SERVER should have docstring" - ); - assert!( - connect_fn - .docstring - .as_ref() - .unwrap() - .contains("CONNECT TO SERVER"), - "docstring: {:?}", - connect_fn.docstring - ); - - let disconnect_fn = result - .nodes - .iter() - .find(|n| n.kind == NodeKind::Function && n.name == "DISCONNECT") - .expect("DISCONNECT function not found"); - assert!( - disconnect_fn.docstring.is_some(), - "DISCONNECT should have docstring" + .filter_map(|n| Some((n.name.as_str(), n.docstring.as_deref()?))) + .collect(); + assert_eq!( + docs, + [ + ( + "LOG_A_MESSAGE", + "LOG A MESSAGE\nPARAMS: L$=LEVEL, M$=MESSAGE" + ), + ("CONNECT_TO_SERVER", "CONNECT TO SERVER"), + ("DISCONNECT", "DISCONNECT"), + ] ); } @@ -117,3 +87,17 @@ fn test_msbasic2_subroutine_internal_calls() { gosub_200_count ); } + +#[test] +fn test_msbasic2_let_name_keeps_underscores() { + let result = MsBasic2Extractor + .extract_artifact("names.bas", "10 LET MAX_RETRIES = 3\n20 LET MR = 1\n") + .result; + let consts: Vec<&str> = result + .nodes + .iter() + .filter(|n| n.kind == NodeKind::Const) + .map(|n| n.name.as_str()) + .collect(); + assert_eq!(consts, ["MAX_RETRIES", "MR"]); +} diff --git a/crates/tracedecay-code-extraction/tests/main/nix.rs b/crates/tracedecay-code-extraction/tests/main/nix.rs index 7c1b1c7d2f..29704644cc 100644 --- a/crates/tracedecay-code-extraction/tests/main/nix.rs +++ b/crates/tracedecay-code-extraction/tests/main/nix.rs @@ -6,7 +6,7 @@ fn extract_sample() -> ExtractionResult { let source = std::fs::read_to_string("../../tests/fixtures/sample.nix") .expect("failed to read sample.nix"); let extractor = NixExtractor; - extractor.extract("sample.nix", &source) + extractor.extract_artifact("sample.nix", &source).result } #[test] @@ -39,51 +39,23 @@ fn test_nix_nested_functions() { #[test] fn test_nix_docstrings() { let result = extract_sample(); - - let dp = result - .nodes - .iter() - .find(|n| n.name == "defaultPort") - .unwrap(); - assert!(dp.docstring.is_some(), "defaultPort should have docstring"); - assert!( - dp.docstring.as_ref().unwrap().contains("Default port"), - "docstring: {:?}", - dp.docstring - ); - - let log_fn = result - .nodes - .iter() - .find(|n| n.kind == NodeKind::Function && n.name == "log") - .unwrap(); - assert!( - log_fn.docstring.is_some(), - "log function should have docstring" - ); - assert!( - log_fn - .docstring - .as_ref() - .unwrap() - .contains("Formats a log message"), - "docstring: {:?}", - log_fn.docstring - ); - - let net = result + let docs: Vec<(&str, &str)> = result .nodes .iter() - .find(|n| n.kind == NodeKind::Module && n.name == "networking") - .unwrap(); - assert!(net.docstring.is_some(), "networking should have docstring"); - assert!( - net.docstring - .as_ref() - .unwrap() - .contains("Networking utilities"), - "docstring: {:?}", - net.docstring + .filter_map(|n| Some((n.name.as_str(), n.docstring.as_deref()?))) + .collect(); + assert_eq!( + docs, + [ + ("defaultPort", "Default port for the service."), + ("maxRetries", "Maximum retry count."), + ("log", "Formats a log message."), + ("mkConnection", "Builds a connection configuration."), + ("networking", "Networking utilities."), + ("mkPool", "Creates a connection pool."), + ("validateConfig", "Validates a connection config."), + ("service", "Package definition."), + ] ); } @@ -95,7 +67,6 @@ fn test_nix_call_sites() { .iter() .filter(|r| r.reference_kind == EdgeKind::Calls) .collect(); - assert!(!call_refs.is_empty(), "should have call site refs"); assert!( call_refs.iter().any(|r| r.reference_name == "mkConnection"), "should find mkConnection call, got: {:?}", @@ -155,7 +126,7 @@ fn extract_flake() -> ExtractionResult { let source = std::fs::read_to_string("../../tests/fixtures/sample-flake.nix") .expect("failed to read sample-flake.nix"); let extractor = NixExtractor; - extractor.extract("flake.nix", &source) + extractor.extract_artifact("flake.nix", &source).result } // ------------------------------------------------------------------- @@ -165,37 +136,20 @@ fn extract_flake() -> ExtractionResult { #[test] fn test_nix_import_path_resolution() { let result = extract_sample(); - - let uses: Vec<_> = result + let import_nodes: Vec<(&NodeKind, &str)> = result .nodes .iter() - .filter(|n| n.kind == NodeKind::Use && n.name == "./utils.nix") + .filter(|n| n.kind != NodeKind::File && n.name.ends_with(".nix")) + .map(|n| (&n.kind, n.name.as_str())) .collect(); - assert!( - !uses.is_empty(), - "should have Use node for import ./utils.nix, got uses: {:?}", - result - .nodes - .iter() - .filter(|n| n.kind == NodeKind::Use) - .map(|n| &n.name) - .collect::>() - ); - - let uses_refs: Vec<_> = result + assert_eq!(import_nodes, [(&NodeKind::Use, "./utils.nix")]); + let import_refs: Vec<(&EdgeKind, &str)> = result .unresolved_refs .iter() - .filter(|r| r.reference_kind == EdgeKind::Uses && r.reference_name == "./utils.nix") + .filter(|r| r.reference_name.ends_with(".nix")) + .map(|r| (&r.reference_kind, r.reference_name.as_str())) .collect(); - assert!( - !uses_refs.is_empty(), - "should have unresolved Uses ref for ./utils.nix, got: {:?}", - result - .unresolved_refs - .iter() - .map(|r| (&r.reference_kind, &r.reference_name)) - .collect::>() - ); + assert_eq!(import_refs, [(&EdgeKind::Uses, "./utils.nix")]); } // ------------------------------------------------------------------- diff --git a/crates/tracedecay-code-extraction/tests/main/objc.rs b/crates/tracedecay-code-extraction/tests/main/objc.rs index 475d38fcc5..4fbe60561e 100644 --- a/crates/tracedecay-code-extraction/tests/main/objc.rs +++ b/crates/tracedecay-code-extraction/tests/main/objc.rs @@ -2,6 +2,8 @@ use tracedecay_code_extraction::LanguageExtractor; use tracedecay_code_extraction::ObjcExtractor; use tracedecay_domain::*; +include!("support/edges.rs"); + #[test] fn test_objc_extract_imports() { let source = r#"#import @@ -9,7 +11,7 @@ fn test_objc_extract_imports() { #include "#; let extractor = ObjcExtractor; - let result = extractor.extract("sample.m", source); + let result = extractor.extract_artifact("sample.m", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let includes: Vec<_> = result .nodes @@ -28,7 +30,7 @@ fn test_objc_extract_preprocessor_defines() { #define DEFAULT_PORT 8080 "#; let extractor = ObjcExtractor; - let result = extractor.extract("sample.m", source); + let result = extractor.extract_artifact("sample.m", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let defs: Vec<_> = result .nodes @@ -50,7 +52,7 @@ fn test_objc_extract_ns_enum() { }; "#; let extractor = ObjcExtractor; - let result = extractor.extract("sample.m", source); + let result = extractor.extract_artifact("sample.m", source).result; // NS_ENUM may produce parse errors but we still extract useful data let enums: Vec<_> = result .nodes @@ -94,7 +96,7 @@ fn test_objc_extract_protocol() { @end "#; let extractor = ObjcExtractor; - let result = extractor.extract("sample.m", source); + let result = extractor.extract_artifact("sample.m", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let protocols: Vec<_> = result @@ -141,7 +143,7 @@ fn test_objc_extract_class_interface() { @end "#; let extractor = ObjcExtractor; - let result = extractor.extract("sample.m", source); + let result = extractor.extract_artifact("sample.m", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let classes: Vec<_> = result @@ -198,7 +200,7 @@ fn test_objc_extract_class_with_protocol_conformance() { @end "#; let extractor = ObjcExtractor; - let result = extractor.extract("sample.m", source); + let result = extractor.extract_artifact("sample.m", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let classes: Vec<_> = result @@ -271,7 +273,7 @@ fn test_objc_extract_implementation() { @end "#; let extractor = ObjcExtractor; - let result = extractor.extract("sample.m", source); + let result = extractor.extract_artifact("sample.m", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let impls: Vec<_> = result @@ -312,15 +314,29 @@ fn test_objc_extract_implementation() { .unresolved_refs .iter() .filter(|r| r.reference_kind == EdgeKind::Calls) + .map(|r| r.reference_name.as_str()) .collect(); - assert!(!calls.is_empty(), "expected call site refs"); + assert_eq!( + calls, + [ + "super.init", + "name.copy", + "NSString.stringWithFormat", + "NSStringFromClass", + "self.class", + "NSAssert", + ] + ); - let contains: Vec<_> = result - .edges - .iter() - .filter(|e| e.kind == EdgeKind::Contains) - .collect(); - assert!(contains.len() >= 3, "expected >= 3 Contains edges"); + assert_eq!( + edge_pairs(&result, EdgeKind::Contains), + [ + ("sample.m", "Base"), + ("Base", "initWithName"), + ("Base", "description"), + ("Base", "validate"), + ] + ); } #[test] @@ -331,7 +347,7 @@ void logMessage(LogLevel level, NSString *message) { } "#; let extractor = ObjcExtractor; - let result = extractor.extract("sample.m", source); + let result = extractor.extract_artifact("sample.m", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let fns: Vec<_> = result @@ -356,8 +372,9 @@ void logMessage(LogLevel level, NSString *message) { .unresolved_refs .iter() .filter(|r| r.reference_kind == EdgeKind::Calls) + .map(|r| r.reference_name.as_str()) .collect(); - assert!(!calls.is_empty(), "expected call site refs from NSLog"); + assert_eq!(calls, ["NSLog"]); } #[test] @@ -373,7 +390,7 @@ fn test_objc_message_expression_calls() { @end "#; let extractor = ObjcExtractor; - let result = extractor.extract("sample.m", source); + let result = extractor.extract_artifact("sample.m", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let calls: Vec<_> = result @@ -381,10 +398,12 @@ fn test_objc_message_expression_calls() { .iter() .filter(|r| r.reference_kind == EdgeKind::Calls) .collect(); - assert!( - calls.len() >= 3, - "expected >= 3 call refs, got {}", - calls.len() + assert_eq!( + calls + .iter() + .map(|x| x.reference_name.as_str()) + .collect::>(), + ["self.doSomething", "NSString.stringWithFormat", "NSLog"] ); // Message sends create receiver.method format assert!( diff --git a/crates/tracedecay-code-extraction/tests/main/pascal.rs b/crates/tracedecay-code-extraction/tests/main/pascal.rs index 27ef098e36..8b4a844cca 100644 --- a/crates/tracedecay-code-extraction/tests/main/pascal.rs +++ b/crates/tracedecay-code-extraction/tests/main/pascal.rs @@ -6,9 +6,10 @@ use tracedecay_domain::*; // into this module's own namespace, so the tests below call them unqualified // without each extractor module re-declaring the support module. include!("support/docstrings.rs"); +include!("support/edges.rs"); fn extract(source: &str) -> ExtractionResult { let extractor = PascalExtractor; - extractor.extract("test.pas", source) + extractor.extract_artifact("test.pas", source).result } // ---------------------------- @@ -359,10 +360,9 @@ end."#, .iter() .filter(|n| n.kind == NodeKind::Constructor) .collect(); - assert!( - !ctors.is_empty(), - "Should have at least one constructor, got {}", - ctors.len() + assert_eq!( + ctors.iter().map(|x| x.name.as_str()).collect::>(), + ["Create", "Create"] ); assert!(ctors.iter().any(|n| n.name == "Create")); } @@ -395,10 +395,9 @@ end."#, .iter() .filter(|n| n.kind == NodeKind::Method && n.name == "Destroy") .collect(); - assert!( - !methods.is_empty(), - "Should have at least one destructor-as-method, got {}", - methods.len() + assert_eq!( + methods.iter().map(|x| x.name.as_str()).collect::>(), + ["Destroy", "Destroy"] ); } @@ -818,9 +817,9 @@ end."#, .iter() .filter(|n| n.kind == NodeKind::Method && n.name == "DoSomething") .collect(); - assert!( - !methods.is_empty(), - "Should have at least one Method node for DoSomething" + assert_eq!( + methods.iter().map(|x| x.name.as_str()).collect::>(), + ["DoSomething"] ); } @@ -941,9 +940,9 @@ end."#; .iter() .filter(|n| n.kind == NodeKind::Use) .collect(); - assert!( - uses.len() >= 3, - "Should have at least 3 uses (SysUtils, Classes, Math)" + assert_eq!( + uses.iter().map(|x| x.name.as_str()).collect::>(), + ["SysUtils", "Classes", "Math"] ); // Should have the class. @@ -1002,11 +1001,31 @@ end."#; .any(|n| n.kind == NodeKind::Field && n.name == "FName") ); - // Should have Contains edges. - assert!(!result.edges.is_empty(), "Should have Contains edges"); - assert!( - result.edges.iter().any(|e| e.kind == EdgeKind::Contains), - "Should have at least one Contains edge" + assert_eq!( + edge_pairs(&result, EdgeKind::Contains), + [ + ("test.pas", "MyUnit"), + ("MyUnit", "SysUtils"), + ("MyUnit", "Classes"), + ("MyUnit", "TMyClass"), + ("TMyClass", "FName"), + ("TMyClass", "Create"), + ("TMyClass", "Destroy"), + ("TMyClass", "DoSomething"), + ("TMyClass", "GetName"), + ("TMyClass", "Name"), + ("MyUnit", "TMyRecord"), + ("TMyRecord", "X"), + ("TMyRecord", "Y"), + ("MyUnit", "TMyAlias"), + ("MyUnit", "MAX_VALUE"), + ("MyUnit", "GlobalVar"), + ("MyUnit", "Math"), + ("MyUnit", "Create"), + ("MyUnit", "Destroy"), + ("MyUnit", "DoSomething"), + ("MyUnit", "GetName") + ] ); } diff --git a/crates/tracedecay-code-extraction/tests/main/perl.rs b/crates/tracedecay-code-extraction/tests/main/perl.rs index cf1be033c7..d10dfbe870 100644 --- a/crates/tracedecay-code-extraction/tests/main/perl.rs +++ b/crates/tracedecay-code-extraction/tests/main/perl.rs @@ -6,7 +6,7 @@ use tracedecay_domain::*; fn test_perl_call_sites() { let source = std::fs::read_to_string("../../tests/fixtures/sample.pl").unwrap(); let extractor = PerlExtractor; - let result = extractor.extract("sample.pl", &source); + let result = extractor.extract_artifact("sample.pl", &source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let call_refs: Vec<_> = result @@ -14,8 +14,6 @@ fn test_perl_call_sites() { .iter() .filter(|r| r.reference_kind == EdgeKind::Calls) .collect(); - assert!(!call_refs.is_empty(), "should have call refs"); - // connect method calls main::log_message (qualified call) assert!( call_refs @@ -63,54 +61,26 @@ fn test_perl_call_sites() { #[test] fn test_perl_docstrings() { let source = std::fs::read_to_string("../../tests/fixtures/sample.pl").unwrap(); - let extractor = PerlExtractor; - let result = extractor.extract("sample.pl", &source); + let result = PerlExtractor.extract_artifact("sample.pl", &source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); - - let log_fn = result - .nodes - .iter() - .find(|n| n.kind == NodeKind::Function && n.name == "log_message") - .expect("log_message function not found"); - assert!( - log_fn.docstring.is_some(), - "log_message should have docstring" - ); - let doc = log_fn.docstring.as_ref().unwrap(); - assert!( - doc.contains("Logs a message"), - "docstring should contain 'Logs a message', got: {}", - doc - ); - - let max_retries = result - .nodes - .iter() - .find(|n| n.kind == NodeKind::Const && n.name == "MAX_RETRIES") - .expect("MAX_RETRIES not found"); - assert!( - max_retries - .docstring - .as_ref() - .unwrap() - .contains("Maximum number of retries"), - "docstring: {:?}", - max_retries.docstring - ); - - let connect = result + let docs: Vec<(&str, &str)> = result .nodes .iter() - .find(|n| n.kind == NodeKind::Method && n.name == "connect") - .expect("connect method not found"); - assert!( - connect - .docstring - .as_ref() - .unwrap() - .contains("Connects to the remote host"), - "docstring: {:?}", - connect.docstring + .filter_map(|n| Some((n.name.as_str(), n.docstring.as_deref()?))) + .collect(); + assert_eq!( + docs, + [ + ("MAX_RETRIES", "Maximum number of retries."), + ("DEFAULT_PORT", "Default port for connections."), + ("log_message", "Logs a message with the given level."), + ("new", "Creates a new Connection object."), + ("connect", "Connects to the remote host."), + ("disconnect", "Disconnects from the remote host."), + ("is_connected", "Checks if the connection is active."), + ("new", "Creates a new Pool."), + ("acquire", "Acquires a connection from the pool."), + ] ); } @@ -118,7 +88,7 @@ fn test_perl_docstrings() { fn test_perl_signatures() { let source = std::fs::read_to_string("../../tests/fixtures/sample.pl").unwrap(); let extractor = PerlExtractor; - let result = extractor.extract("sample.pl", &source); + let result = extractor.extract_artifact("sample.pl", &source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let log_fn = result diff --git a/crates/tracedecay-code-extraction/tests/main/php.rs b/crates/tracedecay-code-extraction/tests/main/php.rs index ac086046a6..bfd0e70a87 100644 --- a/crates/tracedecay-code-extraction/tests/main/php.rs +++ b/crates/tracedecay-code-extraction/tests/main/php.rs @@ -13,7 +13,7 @@ function add(int $a, int $b): int { } "#; let extractor = PhpExtractor; - let result = extractor.extract("math.php", source); + let result = extractor.extract_artifact("math.php", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let fns: Vec<_> = result .nodes @@ -44,7 +44,7 @@ class User { } "#; let extractor = PhpExtractor; - let result = extractor.extract("user.php", source); + let result = extractor.extract_artifact("user.php", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let classes: Vec<_> = result @@ -60,10 +60,9 @@ class User { .iter() .filter(|n| n.kind == NodeKind::Method) .collect(); - assert!( - methods.len() >= 2, - "expected >= 2 methods, got {}", - methods.len() + assert_eq!( + methods.iter().map(|x| x.name.as_str()).collect::>(), + ["__construct", "getName", "validate"] ); assert!(methods.iter().any(|m| m.name == "getName")); @@ -86,9 +85,9 @@ class User { .iter() .filter(|n| n.kind == NodeKind::Field) .collect(); - assert!( - !fields.is_empty(), - "expected field nodes for class properties" + assert_eq!( + fields.iter().map(|x| x.name.as_str()).collect::>(), + ["name"] ); // Contains edges @@ -103,7 +102,7 @@ interface Loggable { } "#; let extractor = PhpExtractor; - let result = extractor.extract("loggable.php", source); + let result = extractor.extract_artifact("loggable.php", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let traits: Vec<_> = result .nodes @@ -124,7 +123,7 @@ trait Timestamps { } "#; let extractor = PhpExtractor; - let result = extractor.extract("timestamps.php", source); + let result = extractor.extract_artifact("timestamps.php", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let traits: Vec<_> = result .nodes @@ -143,7 +142,7 @@ namespace App\Models; class Item {} "#; let extractor = PhpExtractor; - let result = extractor.extract("item.php", source); + let result = extractor.extract_artifact("item.php", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); assert!( result.nodes.iter().any(|n| n.kind == NodeKind::Module), @@ -161,7 +160,7 @@ enum Status { } "#; let extractor = PhpExtractor; - let result = extractor.extract("status.php", source); + let result = extractor.extract_artifact("status.php", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let enums: Vec<_> = result .nodes @@ -190,16 +189,16 @@ class Service { } "#; let extractor = PhpExtractor; - let result = extractor.extract("service.php", source); + let result = extractor.extract_artifact("service.php", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let uses: Vec<_> = result .nodes .iter() .filter(|n| n.kind == NodeKind::Use) .collect(); - assert!( - !uses.is_empty(), - "expected Use node for `use Logger` inside class" + assert_eq!( + uses.iter().map(|x| x.name.as_str()).collect::>(), + ["Logger"] ); } @@ -211,7 +210,7 @@ class Widget { } "#; let extractor = PhpExtractor; - let result = extractor.extract("widget.php", source); + let result = extractor.extract_artifact("widget.php", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); // PHP extractor maps __construct as a regular Method let methods: Vec<_> = result @@ -239,7 +238,7 @@ class MyController { } "#; let extractor = PhpExtractor; - let result = extractor.extract("attr.php", source); + let result = extractor.extract_artifact("attr.php", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); // Should have 4 AnnotationUsage nodes: Route, Deprecated, Override, AllowDynamicProperties @@ -279,7 +278,7 @@ class MyController { #[test] fn test_php_empty_source() { let extractor = PhpExtractor; - let result = extractor.extract("empty.php", " = result .nodes diff --git a/crates/tracedecay-code-extraction/tests/main/powershell.rs b/crates/tracedecay-code-extraction/tests/main/powershell.rs index d4809d3835..6e09ac2d92 100644 --- a/crates/tracedecay-code-extraction/tests/main/powershell.rs +++ b/crates/tracedecay-code-extraction/tests/main/powershell.rs @@ -6,7 +6,7 @@ use tracedecay_domain::*; fn test_powershell_call_sites() { let source = std::fs::read_to_string("../../tests/fixtures/sample.ps1").unwrap(); let extractor = PowerShellExtractor; - let result = extractor.extract("sample.ps1", &source); + let result = extractor.extract_artifact("sample.ps1", &source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let call_refs: Vec<_> = result @@ -14,7 +14,6 @@ fn test_powershell_call_sites() { .iter() .filter(|r| r.reference_kind == EdgeKind::Calls) .collect(); - assert!(!call_refs.is_empty(), "should have call refs"); assert!( call_refs.iter().any(|r| r.reference_name == "Write-Host"), "should find Write-Host call" @@ -39,7 +38,7 @@ fn test_powershell_call_sites() { fn test_powershell_docstrings() { let source = std::fs::read_to_string("../../tests/fixtures/sample.ps1").unwrap(); let extractor = PowerShellExtractor; - let result = extractor.extract("sample.ps1", &source); + let result = extractor.extract_artifact("sample.ps1", &source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); // Write-Log should have a block comment docstring. @@ -89,20 +88,3 @@ fn test_powershell_docstrings() { main_fn.docstring ); } - -#[test] -fn test_powershell_contains_edges() { - let source = std::fs::read_to_string("../../tests/fixtures/sample.ps1").unwrap(); - let extractor = PowerShellExtractor; - let result = extractor.extract("sample.ps1", &source); - let contains: Vec<_> = result - .edges - .iter() - .filter(|e| e.kind == EdgeKind::Contains) - .collect(); - assert!( - contains.len() >= 9, - "should have >= 9 Contains edges, got {}", - contains.len() - ); -} diff --git a/crates/tracedecay-code-extraction/tests/main/proto.rs b/crates/tracedecay-code-extraction/tests/main/proto.rs index 41b913f38d..8200321456 100644 --- a/crates/tracedecay-code-extraction/tests/main/proto.rs +++ b/crates/tracedecay-code-extraction/tests/main/proto.rs @@ -8,7 +8,7 @@ fn extract_sample() -> ExtractionResult { let source = std::fs::read_to_string("../../tests/fixtures/sample.proto") .expect("failed to read sample.proto"); let extractor = ProtoExtractor; - extractor.extract("sample.proto", &source) + extractor.extract_artifact("sample.proto", &source).result } #[test] @@ -138,11 +138,17 @@ fn test_proto_messages() { .iter() .filter(|n| n.kind == NodeKind::ProtoMessage) .collect(); - assert!( - msgs.len() >= 7, - "expected >= 7 messages, got {} : {:?}", - msgs.len(), - msgs.iter().map(|m| &m.name).collect::>() + assert_eq!( + msgs.iter().map(|x| x.name.as_str()).collect::>(), + [ + "Endpoint", + "ConnectionConfig", + "AuthConfig", + "ConnectionStatus", + "DisconnectRequest", + "HealthCheckRequest", + "HealthCheckResponse" + ] ); assert!(msgs.iter().any(|m| m.name == "Endpoint")); assert!(msgs.iter().any(|m| m.name == "ConnectionConfig")); @@ -185,14 +191,9 @@ fn test_proto_enum() { .collect(); assert_eq!(enums.len(), 1); assert_eq!(enums[0].name, "LogLevel"); - assert!( - enums[0].docstring.is_some(), - "LogLevel should have docstring" - ); - assert!( - enums[0].docstring.as_ref().unwrap().contains("log level"), - "docstring: {:?}", - enums[0].docstring + assert_eq!( + enums[0].docstring.as_deref(), + Some("Represents the log level.") ); } @@ -228,9 +229,9 @@ fn test_proto_service() { .collect(); assert_eq!(services.len(), 1); assert_eq!(services[0].name, "ConnectionService"); - assert!( - services[0].docstring.is_some(), - "ConnectionService should have docstring" + assert_eq!( + services[0].docstring.as_deref(), + Some("Manages network connections.") ); } @@ -254,9 +255,9 @@ fn test_proto_rpcs() { assert!(rpcs.iter().any(|r| r.name == "HealthCheck")); let connect = rpcs.iter().find(|r| r.name == "Connect").unwrap(); - assert!( - connect.docstring.is_some(), - "Connect rpc should have docstring" + assert_eq!( + connect.docstring.as_deref(), + Some("Establishes a new connection.") ); } @@ -268,10 +269,28 @@ fn test_proto_fields() { .iter() .filter(|n| n.kind == NodeKind::Field) .collect(); - assert!( - fields.len() >= 15, - "expected >= 15 fields, got {}", - fields.len() + assert_eq!( + fields.iter().map(|x| x.name.as_str()).collect::>(), + [ + "host", + "port", + "tls", + "endpoint", + "max_retries", + "timeout_ms", + "log_level", + "token", + "username", + "auth", + "round_robin", + "least_connections", + "connected", + "connection_id", + "connection_id", + "connection_id", + "healthy", + "latency_ms" + ] ); assert!(fields.iter().any(|f| f.name == "host")); assert!(fields.iter().any(|f| f.name == "port")); @@ -310,19 +329,7 @@ fn test_proto_docstrings() { .iter() .find(|n| n.kind == NodeKind::ProtoMessage && n.name == "Endpoint") .unwrap(); - assert!( - endpoint.docstring.is_some(), - "Endpoint should have docstring" - ); - assert!( - endpoint - .docstring - .as_ref() - .unwrap() - .contains("network endpoint"), - "docstring: {:?}", - endpoint.docstring - ); + assert_eq!(endpoint.docstring.as_deref(), Some("A network endpoint.")); } #[test] diff --git a/crates/tracedecay-code-extraction/tests/main/python.rs b/crates/tracedecay-code-extraction/tests/main/python.rs index 90af028a27..536a42032b 100644 --- a/crates/tracedecay-code-extraction/tests/main/python.rs +++ b/crates/tracedecay-code-extraction/tests/main/python.rs @@ -7,6 +7,8 @@ use tracedecay_domain::*; // without each extractor module re-declaring the support module. include!("support/docstrings.rs"); +include!("support/edges.rs"); + #[test] fn test_py_function_declaration() { let source = r#" @@ -17,7 +19,7 @@ def helper(): pass "#; let extractor = PythonExtractor; - let result = extractor.extract("math.py", source); + let result = extractor.extract_artifact("math.py", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let fns: Vec<_> = result .nodes @@ -40,7 +42,7 @@ async def fetch_data(url): pass "#; let extractor = PythonExtractor; - let result = extractor.extract("async_mod.py", source); + let result = extractor.extract_artifact("async_mod.py", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let fns: Vec<_> = result .nodes @@ -62,7 +64,7 @@ class MyClass: pass "#; let extractor = PythonExtractor; - let result = extractor.extract("classes.py", source); + let result = extractor.extract_artifact("classes.py", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let classes: Vec<_> = result .nodes @@ -85,7 +87,7 @@ class Dog: return item "#; let extractor = PythonExtractor; - let result = extractor.extract("dog.py", source); + let result = extractor.extract_artifact("dog.py", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let methods: Vec<_> = result .nodes @@ -107,7 +109,7 @@ def my_func(): pass "#; let extractor = PythonExtractor; - let result = extractor.extract("decorators.py", source); + let result = extractor.extract_artifact("decorators.py", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let decorators: Vec<_> = result .nodes @@ -138,26 +140,23 @@ class MyClass: self._name = value "#; let extractor = PythonExtractor; - let result = extractor.extract("props.py", source); + let result = extractor.extract_artifact("props.py", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let decorators: Vec<_> = result .nodes .iter() .filter(|n| n.kind == NodeKind::Decorator) .collect(); - assert!( - decorators.len() >= 2, - "should have at least 2 decorators, got {}", - decorators.len() + assert_eq!( + decorators + .iter() + .map(|x| x.name.as_str()) + .collect::>(), + ["property", "name.setter"] ); - let annotates: Vec<_> = result - .edges - .iter() - .filter(|e| e.kind == EdgeKind::Annotates) - .collect(); - assert!( - annotates.len() >= 2, - "should have at least 2 Annotates edges" + assert_eq!( + edge_pairs(&result, EdgeKind::Annotates), + [("property", "name"), ("name.setter", "name")] ); } @@ -168,7 +167,7 @@ import os import sys "#; let extractor = PythonExtractor; - let result = extractor.extract("imports.py", source); + let result = extractor.extract_artifact("imports.py", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let uses: Vec<_> = result .nodes @@ -187,7 +186,7 @@ from os.path import join, exists from collections import defaultdict "#; let extractor = PythonExtractor; - let result = extractor.extract("imports.py", source); + let result = extractor.extract_artifact("imports.py", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let uses: Vec<_> = result .nodes @@ -249,7 +248,7 @@ def process(): for (shape, path, source, kind, expected) in cases { let extractor = PythonExtractor; - let result = extractor.extract(path, source); + let result = extractor.extract_artifact(path, source).result; assert_node_docstring(shape, &result, kind, None, expected); } } @@ -264,7 +263,7 @@ def public_func(): pass "#; let extractor = PythonExtractor; - let result = extractor.extract("vis.py", source); + let result = extractor.extract_artifact("vis.py", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let fns: Vec<_> = result .nodes @@ -291,7 +290,7 @@ class MyClass: pass "#; let extractor = PythonExtractor; - let result = extractor.extract("vis2.py", source); + let result = extractor.extract_artifact("vis2.py", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let methods: Vec<_> = result .nodes @@ -322,7 +321,7 @@ MIN_VALUE = 0 some_var = "hello" "#; let extractor = PythonExtractor; - let result = extractor.extract("consts.py", source); + let result = extractor.extract_artifact("consts.py", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let consts: Vec<_> = result .nodes @@ -347,20 +346,19 @@ def main(): some_func(42) "#; let extractor = PythonExtractor; - let result = extractor.extract("main.py", source); + let result = extractor.extract_artifact("main.py", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let call_refs: Vec<_> = result .unresolved_refs .iter() .filter(|r| r.reference_kind == EdgeKind::Calls) .collect(); - assert!( - call_refs.len() >= 2, - "should have call refs for print and some_func, got: {:?}", + assert_eq!( call_refs .iter() - .map(|r| &r.reference_name) - .collect::>() + .map(|x| x.reference_name.as_str()) + .collect::>(), + ["print", "some_func"] ); } @@ -373,7 +371,7 @@ class Outer: pass "#; let extractor = PythonExtractor; - let result = extractor.extract("nested.py", source); + let result = extractor.extract_artifact("nested.py", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let classes: Vec<_> = result .nodes @@ -396,18 +394,15 @@ def standalone(): pass "#; let extractor = PythonExtractor; - let result = extractor.extract("edges.py", source); + let result = extractor.extract_artifact("edges.py", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); - let contains: Vec<_> = result - .edges - .iter() - .filter(|e| e.kind == EdgeKind::Contains) - .collect(); - // File → Class, File → Function, Class → Method - assert!( - contains.len() >= 3, - "should have at least 3 Contains edges, got {}", - contains.len() + assert_eq!( + edge_pairs(&result, EdgeKind::Contains), + [ + ("edges.py", "Dog"), + ("Dog", "bark"), + ("edges.py", "standalone") + ] ); } @@ -421,7 +416,7 @@ class Dog(Animal): pass "#; let extractor = PythonExtractor; - let result = extractor.extract("inherit.py", source); + let result = extractor.extract_artifact("inherit.py", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let has_extends = result.edges.iter().any(|e| e.kind == EdgeKind::Extends) || result @@ -452,20 +447,19 @@ class Child(Base, Mixin): pass "#; let extractor = PythonExtractor; - let result = extractor.extract("multi.py", source); + let result = extractor.extract_artifact("multi.py", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let extends_refs: Vec<_> = result .unresolved_refs .iter() .filter(|r| r.reference_kind == EdgeKind::Extends) .collect(); - assert!( - extends_refs.len() >= 2, - "should have Extends refs for Base and Mixin, got: {:?}", + assert_eq!( extends_refs .iter() - .map(|r| &r.reference_name) - .collect::>() + .map(|x| x.reference_name.as_str()) + .collect::>(), + ["Base", "Mixin"] ); } @@ -477,7 +471,7 @@ class MyClass: pass "#; let extractor = PythonExtractor; - let result = extractor.extract("pkg/module.py", source); + let result = extractor.extract_artifact("pkg/module.py", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let methods: Vec<_> = result .nodes @@ -510,7 +504,7 @@ class Server: pass "#; let extractor = PythonExtractor; - let result = extractor.extract("server.py", source); + let result = extractor.extract_artifact("server.py", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let methods: Vec<_> = result .nodes diff --git a/crates/tracedecay-code-extraction/tests/main/qbasic.rs b/crates/tracedecay-code-extraction/tests/main/qbasic.rs index 00d757997f..7db2ff0c0a 100644 --- a/crates/tracedecay-code-extraction/tests/main/qbasic.rs +++ b/crates/tracedecay-code-extraction/tests/main/qbasic.rs @@ -5,7 +5,7 @@ use tracedecay_domain::*; fn extract_fixture() -> ExtractionResult { let source = std::fs::read_to_string("../../tests/fixtures/sample.qb").unwrap(); let extractor = QBasicExtractor; - let result = extractor.extract("sample.qb", &source); + let result = extractor.extract_artifact("sample.qb", &source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); result } @@ -13,29 +13,13 @@ fn extract_fixture() -> ExtractionResult { #[test] fn test_qbasic_type_fields() { let result = extract_fixture(); - let fields: Vec<_> = result + let fields: Vec<&str> = result .nodes .iter() .filter(|n| n.kind == NodeKind::Field && n.qualified_name.contains("Endpoint")) + .map(|n| n.name.as_str()) .collect(); - assert!( - fields.len() >= 3, - "expected >= 3 fields in Endpoint (host, port, connected), got {}: {:?}", - fields.len(), - fields.iter().map(|n| &n.name).collect::>() - ); - assert!( - fields.iter().any(|n| n.name == "host"), - "host field not found" - ); - assert!( - fields.iter().any(|n| n.name == "port"), - "port field not found" - ); - assert!( - fields.iter().any(|n| n.name == "connected"), - "connected field not found" - ); + assert_eq!(fields, ["host", "port", "connected"]); } #[test] @@ -46,8 +30,6 @@ fn test_qbasic_call_sites() { .iter() .filter(|r| r.reference_kind == EdgeKind::Calls) .collect(); - assert!(!calls.is_empty(), "expected call site refs"); - assert!( calls.iter().any(|r| r.reference_name == "ValidateConfig"), "expected CALL ValidateConfig, got: {:?}", @@ -71,64 +53,20 @@ fn test_qbasic_call_sites() { #[test] fn test_qbasic_docstrings() { let result = extract_fixture(); - - let log_fn = result - .nodes - .iter() - .find(|n| n.kind == NodeKind::Function && n.name == "LogMessage") - .expect("LogMessage function not found"); - assert!( - log_fn.docstring.is_some(), - "LogMessage should have docstring" - ); - assert!( - log_fn - .docstring - .as_ref() - .unwrap() - .contains("Logs a message"), - "docstring: {:?}", - log_fn.docstring - ); - - let validate_fn = result + let docs: Vec<(&str, &str)> = result .nodes .iter() - .find(|n| n.kind == NodeKind::Function && n.name == "ValidateConfig") - .expect("ValidateConfig function not found"); - assert!( - validate_fn.docstring.is_some(), - "ValidateConfig should have docstring" - ); - - let connect_fn = result - .nodes - .iter() - .find(|n| n.kind == NodeKind::Function && n.name == "ConnectServer") - .expect("ConnectServer function not found"); - assert!( - connect_fn.docstring.is_some(), - "ConnectServer should have docstring" - ); - - let disconnect_fn = result - .nodes - .iter() - .find(|n| n.kind == NodeKind::Function && n.name == "DisconnectServer") - .expect("DisconnectServer function not found"); - assert!( - disconnect_fn.docstring.is_some(), - "DisconnectServer should have docstring" - ); - - let is_connected_fn = result - .nodes - .iter() - .find(|n| n.kind == NodeKind::Function && n.name == "IsConnected") - .expect("IsConnected function not found"); - assert!( - is_connected_fn.docstring.is_some(), - "IsConnected should have docstring" + .filter_map(|n| Some((n.name.as_str(), n.docstring.as_deref()?))) + .collect(); + assert_eq!( + docs, + [ + ("LogMessage", "Logs a message with the given level."), + ("ValidateConfig", "Validates the configuration."), + ("ConnectServer", "Connects to the remote server."), + ("DisconnectServer", "Disconnects from the server."), + ("IsConnected", "Checks if connected."), + ] ); } @@ -166,27 +104,30 @@ fn test_qbasic_signatures() { #[test] fn test_qbasic_dim_shared_fields() { let result = extract_fixture(); - let dim_fields: Vec<_> = result + let dim_fields: Vec<&str> = result .nodes .iter() .filter(|n| n.kind == NodeKind::Field && !n.qualified_name.contains("Endpoint")) + .map(|n| n.name.as_str()) .collect(); - assert!( - dim_fields.len() >= 3, - "expected >= 3 DIM SHARED fields (conn, logLevel, logMsg), got {}: {:?}", - dim_fields.len(), - dim_fields.iter().map(|n| &n.name).collect::>() - ); - assert!( - dim_fields.iter().any(|n| n.name == "conn"), - "conn field not found" - ); - assert!( - dim_fields.iter().any(|n| n.name == "logLevel"), - "logLevel field not found" - ); - assert!( - dim_fields.iter().any(|n| n.name == "logMsg"), - "logMsg field not found" + assert_eq!(dim_fields, ["conn", "logLevel", "logMsg"]); +} + +#[test] +fn test_qbasic_names_keep_underscores() { + let source = "CONST MAX_RETRIES = 3\nCONST DEFAULT_PORT = 8080\n"; + let result = QBasicExtractor.extract_artifact("names.qb", source).result; + let named: Vec<(&NodeKind, &str)> = result + .nodes + .iter() + .filter(|n| n.kind != NodeKind::File) + .map(|n| (&n.kind, n.name.as_str())) + .collect(); + assert_eq!( + named, + [ + (&NodeKind::Const, "MAX_RETRIES"), + (&NodeKind::Const, "DEFAULT_PORT") + ] ); } diff --git a/crates/tracedecay-code-extraction/tests/main/quickbasic.rs b/crates/tracedecay-code-extraction/tests/main/quickbasic.rs index 11506678bf..863f9d1d9a 100644 --- a/crates/tracedecay-code-extraction/tests/main/quickbasic.rs +++ b/crates/tracedecay-code-extraction/tests/main/quickbasic.rs @@ -5,10 +5,12 @@ mod quickbasic_tests { use tracedecay_code_extraction::QuickBasicExtractor; use tracedecay_domain::*; + include!("support/edges.rs"); + fn extract_fixture() -> ExtractionResult { let source = std::fs::read_to_string("../../tests/fixtures/sample.bi").unwrap(); let extractor = QuickBasicExtractor; - let result = extractor.extract("sample.bi", &source); + let result = extractor.extract_artifact("sample.bi", &source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); result } @@ -28,21 +30,13 @@ mod quickbasic_tests { #[test] fn test_quickbasic_sub_functions() { let result = extract_fixture(); - let fns: Vec<_> = result + let fns: Vec<&str> = result .nodes .iter() .filter(|n| n.kind == NodeKind::Function) + .map(|n| n.name.as_str()) .collect(); - assert!( - fns.len() >= 4, - "expected >= 4 functions, got {}: {:?}", - fns.len(), - fns.iter().map(|n| &n.name).collect::>() - ); - assert!(fns.iter().any(|n| n.name == "InitSystem")); - assert!(fns.iter().any(|n| n.name == "Shutdown")); - assert!(fns.iter().any(|n| n.name == "GetStatus")); - assert!(fns.iter().any(|n| n.name == "LogInit")); + assert_eq!(fns, ["InitSystem", "Shutdown", "GetStatus", "LogInit"]); } #[test] @@ -60,31 +54,24 @@ mod quickbasic_tests { #[test] fn test_quickbasic_type_fields() { let result = extract_fixture(); - let fields: Vec<_> = result - .nodes - .iter() - .filter(|n| n.kind == NodeKind::Field && n.qualified_name.contains("Config")) + let fields: Vec<&str> = edge_pairs(&result, EdgeKind::Contains) + .into_iter() + .filter(|(parent, _)| *parent == "Config") + .map(|(_, child)| child) .collect(); - assert!( - fields.len() >= 3, - "expected >= 3 fields in Config, got {}: {:?}", - fields.len(), - fields.iter().map(|n| &n.name).collect::>() - ); - assert!(fields.iter().any(|n| n.name == "name")); - assert!(fields.iter().any(|n| n.name == "value")); - assert!(fields.iter().any(|n| n.name == "active")); + assert_eq!(fields, ["name", "value", "active"]); } #[test] fn test_quickbasic_const_nodes() { let result = extract_fixture(); - let consts: Vec<_> = result + let consts: Vec<&str> = result .nodes .iter() .filter(|n| n.kind == NodeKind::Const) + .map(|n| n.name.as_str()) .collect(); - assert!(!consts.is_empty(), "expected at least 1 CONST node"); + assert_eq!(consts, ["VERSION", "MAX_ITEMS"]); } #[test] @@ -95,7 +82,6 @@ mod quickbasic_tests { .iter() .filter(|r| r.reference_kind == EdgeKind::Calls) .collect(); - assert!(!calls.is_empty(), "expected call site refs"); assert!( calls.iter().any(|r| r.reference_name == "LogInit"), "expected CALL LogInit from InitSystem" @@ -110,44 +96,46 @@ mod quickbasic_tests { .iter() .find(|n| n.kind == NodeKind::Function && n.name == "GetStatus") .expect("GetStatus function not found"); - assert!( - get_status.branches >= 1, - "GetStatus should have >= 1 branch (IF), got {}", - get_status.branches - ); + assert_eq!(get_status.branches, 1); } #[test] fn test_quickbasic_docstrings() { let result = extract_fixture(); - let init_fn = result + let docs: Vec<(&str, &str)> = result .nodes .iter() - .find(|n| n.kind == NodeKind::Function && n.name == "InitSystem") - .expect("InitSystem not found"); - assert!( - init_fn.docstring.is_some(), - "InitSystem should have a docstring" - ); - assert!( - init_fn.docstring.as_ref().unwrap().contains("Initializes"), - "docstring: {:?}", - init_fn.docstring + .filter_map(|n| Some((n.name.as_str(), n.docstring.as_deref()?))) + .collect(); + assert_eq!( + docs, + [ + ("InitSystem", "Initializes the system."), + ("Shutdown", "Shuts down the system."), + ("GetStatus", "Returns the current status."), + ("LogInit", "Logs initialization.") + ] ); } #[test] fn test_quickbasic_contains_edges() { let result = extract_fixture(); - let contains: Vec<_> = result - .edges - .iter() - .filter(|e| e.kind == EdgeKind::Contains) - .collect(); - assert!( - contains.len() >= 8, - "should have >= 8 Contains edges, got {}", - contains.len() + assert_eq!( + edge_pairs(&result, EdgeKind::Contains), + [ + ("sample.bi", "VERSION"), + ("sample.bi", "MAX_ITEMS"), + ("sample.bi", "Config"), + ("Config", "name"), + ("Config", "value"), + ("Config", "active"), + ("sample.bi", "appConfig"), + ("sample.bi", "InitSystem"), + ("sample.bi", "Shutdown"), + ("sample.bi", "GetStatus"), + ("sample.bi", "LogInit") + ] ); } @@ -161,7 +149,7 @@ SUB Test END SUB "#; let extractor = QuickBasicExtractor; - let result = extractor.extract("test.bi", source); + let result = extractor.extract_artifact("test.bi", source).result; assert!( result.errors.is_empty(), "QB4.5 statements should parse without errors: {:?}", diff --git a/crates/tracedecay-code-extraction/tests/main/quint.rs b/crates/tracedecay-code-extraction/tests/main/quint.rs index 6875dd5566..df3577f940 100644 --- a/crates/tracedecay-code-extraction/tests/main/quint.rs +++ b/crates/tracedecay-code-extraction/tests/main/quint.rs @@ -3,7 +3,7 @@ use tracedecay_code_extraction::QuintExtractor; use tracedecay_domain::*; fn extract(source: &str) -> ExtractionResult { - QuintExtractor.extract("spec.qnt", source) + QuintExtractor.extract_artifact("spec.qnt", source).result } fn names_of(result: &ExtractionResult, kind: NodeKind) -> Vec { diff --git a/crates/tracedecay-code-extraction/tests/main/ruby.rs b/crates/tracedecay-code-extraction/tests/main/ruby.rs index 09f9c8e9a8..a6bd077550 100644 --- a/crates/tracedecay-code-extraction/tests/main/ruby.rs +++ b/crates/tracedecay-code-extraction/tests/main/ruby.rs @@ -13,7 +13,7 @@ def greet(name) end "#; let extractor = RubyExtractor; - let result = extractor.extract("greet.rb", source); + let result = extractor.extract_artifact("greet.rb", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let fns: Vec<_> = result .nodes @@ -42,7 +42,7 @@ class Dog end "#; let extractor = RubyExtractor; - let result = extractor.extract("dog.rb", source); + let result = extractor.extract_artifact("dog.rb", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let classes: Vec<_> = result @@ -58,10 +58,9 @@ end .iter() .filter(|n| n.kind == NodeKind::Method) .collect(); - assert!( - methods.len() >= 2, - "expected >= 2 methods, got {}", - methods.len() + assert_eq!( + methods.iter().map(|x| x.name.as_str()).collect::>(), + ["initialize", "bark", "species"] ); assert!(methods.iter().any(|m| m.name == "bark")); @@ -79,7 +78,7 @@ module Utils end "#; let extractor = RubyExtractor; - let result = extractor.extract("utils.rb", source); + let result = extractor.extract_artifact("utils.rb", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let modules: Vec<_> = result .nodes @@ -104,7 +103,7 @@ class Cat < Animal end "#; let extractor = RubyExtractor; - let result = extractor.extract("animals.rb", source); + let result = extractor.extract_artifact("animals.rb", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let classes: Vec<_> = result .nodes @@ -130,7 +129,7 @@ module Config end "#; let extractor = RubyExtractor; - let result = extractor.extract("config.rb", source); + let result = extractor.extract_artifact("config.rb", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let consts: Vec<_> = result .nodes @@ -157,7 +156,7 @@ class Outer end "#; let extractor = RubyExtractor; - let result = extractor.extract("nested.rb", source); + let result = extractor.extract_artifact("nested.rb", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let classes: Vec<_> = result .nodes @@ -172,7 +171,7 @@ end #[test] fn test_ruby_empty_source() { let extractor = RubyExtractor; - let result = extractor.extract("empty.rb", ""); + let result = extractor.extract_artifact("empty.rb", "").result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let files: Vec<_> = result .nodes diff --git a/crates/tracedecay-code-extraction/tests/main/rust.rs b/crates/tracedecay-code-extraction/tests/main/rust.rs index bd9fc4cf10..f9b64aa3e8 100644 --- a/crates/tracedecay-code-extraction/tests/main/rust.rs +++ b/crates/tracedecay-code-extraction/tests/main/rust.rs @@ -4,6 +4,8 @@ use tracedecay_code_extraction::{ use tracedecay_domain::*; use tree_sitter::Parser; +include!("support/edges.rs"); + #[test] fn test_rust_cfg_attribute_in_struct_pattern_field() { let source = r#" @@ -39,7 +41,7 @@ fn destructure(context: Context) { fn test_rust_file_node_is_root() { let source = r#"fn main() {}"#; let extractor = RustExtractor; - let result = extractor.extract("test.rs", source); + let result = extractor.extract_artifact("test.rs", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let files: Vec<_> = result .nodes @@ -61,7 +63,7 @@ pub fn add(a: i32, b: i32) -> i32 { fn helper() {} "#; let extractor = RustExtractor; - let result = extractor.extract("math.rs", source); + let result = extractor.extract_artifact("math.rs", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let fns: Vec<_> = result .nodes @@ -100,7 +102,7 @@ pub enum Mode { Fast, } "#; - let result = RustExtractor.extract("mode.rs", source); + let result = RustExtractor.extract_artifact("mode.rs", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let safe = result .nodes @@ -124,7 +126,7 @@ pub async fn fetch_data() -> String { } "#; let extractor = RustExtractor; - let result = extractor.extract("async.rs", source); + let result = extractor.extract_artifact("async.rs", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let fns: Vec<_> = result .nodes @@ -145,7 +147,7 @@ pub struct Point { } "#; let extractor = RustExtractor; - let result = extractor.extract("types.rs", source); + let result = extractor.extract_artifact("types.rs", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let structs: Vec<_> = result .nodes @@ -189,7 +191,7 @@ pub enum Color { } "#; let extractor = RustExtractor; - let result = extractor.extract("color.rs", source); + let result = extractor.extract_artifact("color.rs", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let enums: Vec<_> = result .nodes @@ -222,7 +224,7 @@ pub trait Drawable { } "#; let extractor = RustExtractor; - let result = extractor.extract("draw.rs", source); + let result = extractor.extract_artifact("draw.rs", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let traits: Vec<_> = result .nodes @@ -261,7 +263,7 @@ impl Rect { } "#; let extractor = RustExtractor; - let result = extractor.extract("rect.rs", source); + let result = extractor.extract_artifact("rect.rs", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let impls: Vec<_> = result .nodes @@ -390,7 +392,7 @@ pub const MAX_SIZE: usize = 1024; static COUNTER: u32 = 0; "#; let extractor = RustExtractor; - let result = extractor.extract("consts.rs", source); + let result = extractor.extract_artifact("consts.rs", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let consts: Vec<_> = result .nodes @@ -414,7 +416,7 @@ fn test_rust_type_alias() { pub type Result = std::result::Result; "#; let extractor = RustExtractor; - let result = extractor.extract("types.rs", source); + let result = extractor.extract_artifact("types.rs", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let aliases: Vec<_> = result .nodes @@ -433,7 +435,7 @@ pub mod inner { } "#; let extractor = RustExtractor; - let result = extractor.extract("lib.rs", source); + let result = extractor.extract_artifact("lib.rs", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let modules: Vec<_> = result .nodes @@ -467,7 +469,7 @@ mod tests { } "#; let extractor = RustExtractor; - let result = extractor.extract("src/lib.rs", source); + let result = extractor.extract_artifact("src/lib.rs", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let modules: Vec<_> = result @@ -483,10 +485,7 @@ mod tests { .iter() .filter(|e| e.kind == EdgeKind::Annotates && e.target == modules[0].id) .collect(); - assert!( - !cfg_annotations.is_empty(), - "expected #[cfg(test)] to annotate the 'tests' module" - ); + assert_eq!(cfg_annotations.len(), 1); let cfg_source = result .nodes .iter() @@ -514,10 +513,7 @@ mod tests { .iter() .filter(|e| e.kind == EdgeKind::Annotates && e.target == test_fn.id) .collect(); - assert!( - !test_annotations.is_empty(), - "expected #[test] to annotate the test function" - ); + assert_eq!(test_annotations.len(), 1); let test_annot = result .nodes .iter() @@ -542,7 +538,7 @@ fn complex(x: i32) -> i32 { } "#; let extractor = RustExtractor; - let result = extractor.extract("complex.rs", source); + let result = extractor.extract_artifact("complex.rs", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let fns: Vec<_> = result .nodes @@ -576,7 +572,7 @@ fn risky(v: Option) -> i32 { } "#; let extractor = RustExtractor; - let result = extractor.extract("risky.rs", source); + let result = extractor.extract_artifact("risky.rs", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let fns: Vec<_> = result .nodes @@ -607,7 +603,7 @@ fn caller() { fn helper() {} "#; let extractor = RustExtractor; - let result = extractor.extract("calls.rs", source); + let result = extractor.extract_artifact("calls.rs", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); assert!( result @@ -657,7 +653,7 @@ fn assemble(args: &HiArgs, raw: Vec) -> Widget { types.clone().matched(); } "#; - let result = RustExtractor.extract("typed.rs", source); + let result = RustExtractor.extract_artifact("typed.rs", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let names = call_names(&result, "assemble"); // `Vec::len` is stated too: which owners never bind cross-file is the @@ -706,7 +702,7 @@ fn assemble() { r.run(); } "#; - let result = RustExtractor.extract("factory.rs", source); + let result = RustExtractor.extract_artifact("factory.rs", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let names = call_names(&result, "assemble"); assert!( @@ -744,7 +740,9 @@ fn assemble() -> Questioned { Questioned {} } "#; - let result = RustExtractor.extract("initializers.rs", source); + let result = RustExtractor + .extract_artifact("initializers.rs", source) + .result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let names = call_names(&result, "assemble"); assert!( @@ -778,7 +776,7 @@ fn rebound(builders: Vec, pair: (Builder, Builder), maybe: Option = result .nodes @@ -832,7 +830,7 @@ impl Greet for Bot { #[test] fn test_rust_empty_source() { let extractor = RustExtractor; - let result = extractor.extract("empty.rs", ""); + let result = extractor.extract_artifact("empty.rs", "").result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let files: Vec<_> = result .nodes @@ -862,7 +860,7 @@ pub struct Config { } "#; let extractor = RustExtractor; - let result = extractor.extract("attrs.rs", source); + let result = extractor.extract_artifact("attrs.rs", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let annots: Vec<_> = result @@ -916,9 +914,15 @@ pub struct Config { .iter() .filter(|e| e.kind == EdgeKind::Annotates) .collect(); - assert!( - !annotates_edges.is_empty(), - "expected Annotates edges, found none" + assert_eq!( + edge_pairs(&result, EdgeKind::Annotates), + [ + ("test", "my_test"), + ("cfg", "guarded_fn"), + ("allow", "guarded_fn"), + ("inline", "fast_add"), + ("serde", "Config") + ] ); assert_eq!( annotates_edges.len(), @@ -953,7 +957,7 @@ pub fn no_attrs(y: i32) -> i32 { y } "#; - let result = RustExtractor.extract("doc.rs", source); + let result = RustExtractor.extract_artifact("doc.rs", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let double = result .nodes @@ -995,7 +999,7 @@ pub struct Container { count: usize, } "#; - let result = RustExtractor.extract("c.rs", source); + let result = RustExtractor.extract_artifact("c.rs", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let type_of_refs: Vec<_> = result .unresolved_refs @@ -1024,7 +1028,7 @@ pub fn make(name: String, count: usize) -> Result { todo!() } "#; - let result = RustExtractor.extract("f.rs", source); + let result = RustExtractor.extract_artifact("f.rs", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let type_of: Vec<_> = result .unresolved_refs @@ -1068,7 +1072,7 @@ fn test_check() { } "#; let extractor = RustExtractor; - let result = extractor.extract("src/lib.rs", source); + let result = extractor.extract_artifact("src/lib.rs", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let test_fn = result @@ -1107,7 +1111,7 @@ fn use_foo() { } "#; let extractor = RustExtractor; - let result = extractor.extract("src/lib.rs", source); + let result = extractor.extract_artifact("src/lib.rs", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let use_fn = result @@ -1174,7 +1178,7 @@ fn caller(items: Vec, rows: Rows) { } fn make() -> Vec { Vec::new() } "#; - let result = RustExtractor.extract("src/lib.rs", source); + let result = RustExtractor.extract_artifact("src/lib.rs", source).result; assert!(result.errors.is_empty(), "{:?}", result.errors); let from = |name: &str| { @@ -1279,7 +1283,7 @@ fn ambiguous(processor: &T, input: u32) -> u32 { processor.process(input) } "#; - let result = RustExtractor.extract("src/lib.rs", source); + let result = RustExtractor.extract_artifact("src/lib.rs", source).result; assert!(result.errors.is_empty(), "{:?}", result.errors); let from = |name: &str| { let function = result @@ -1345,7 +1349,7 @@ impl Local for ::Item { fn span(&self) -> usize { self.len() } } "#; - let result = RustExtractor.extract("src/lib.rs", source); + let result = RustExtractor.extract_artifact("src/lib.rs", source).result; assert!(result.errors.is_empty(), "{:?}", result.errors); let from = |qualified: &str| { @@ -1386,10 +1390,12 @@ impl Local for ::Item { #[test] fn wildcard_imports_retain_unresolved_dependencies_alongside_named_bindings() { - let result = RustExtractor.extract( - "src/lib.rs", - "use crate::one::*;\nuse crate::two::{Item, *};", - ); + let result = RustExtractor + .extract_artifact( + "src/lib.rs", + "use crate::one::*;\nuse crate::two::{Item, *};", + ) + .result; assert!(result.errors.is_empty(), "{:?}", result.errors); let uses: Vec<_> = result .unresolved_refs diff --git a/crates/tracedecay-code-extraction/tests/main/same_line_identity.rs b/crates/tracedecay-code-extraction/tests/main/same_line_identity.rs index 81aee4a04b..08690e6741 100644 --- a/crates/tracedecay-code-extraction/tests/main/same_line_identity.rs +++ b/crates/tracedecay-code-extraction/tests/main/same_line_identity.rs @@ -67,7 +67,9 @@ fn pairs(items: &[(&str, &str)]) -> BTreeSet<(String, String)> { #[test] fn rust_same_line_methods_bind_their_own_calls_and_containers() { - let result = RustExtractor.extract("test.rs", SAME_LINE_RUST); + let result = RustExtractor + .extract_artifact("test.rs", SAME_LINE_RUST) + .result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let runs = result @@ -89,7 +91,9 @@ fn rust_same_line_methods_bind_their_own_calls_and_containers() { #[test] fn typescript_same_line_methods_and_fields_stay_distinct() { - let result = TypeScriptExtractor.extract("test.ts", SAME_LINE_TYPESCRIPT); + let result = TypeScriptExtractor + .extract_artifact("test.ts", SAME_LINE_TYPESCRIPT) + .result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let fields = result @@ -116,13 +120,15 @@ fn line_leading_constructs_keep_line_keyed_ids_regardless_of_indentation() { let compact = "impl A {\nfn run() {}\n}\n"; let indented = "impl A {\n fn run() {}\n}\n"; let compact_ids: BTreeSet = RustExtractor - .extract("test.rs", compact) + .extract_artifact("test.rs", compact) + .result .nodes .into_iter() .map(|node| node.id) .collect(); let indented_ids: BTreeSet = RustExtractor - .extract("test.rs", indented) + .extract_artifact("test.rs", indented) + .result .nodes .into_iter() .map(|node| node.id) @@ -136,7 +142,9 @@ fn line_leading_constructs_keep_line_keyed_ids_regardless_of_indentation() { #[test] fn only_constructs_sharing_a_line_with_earlier_source_carry_a_column() { - let result = RustExtractor.extract("test.rs", SAME_LINE_RUST); + let result = RustExtractor + .extract_artifact("test.rs", SAME_LINE_RUST) + .result; let struct_a = result .nodes .iter() diff --git a/crates/tracedecay-code-extraction/tests/main/scala.rs b/crates/tracedecay-code-extraction/tests/main/scala.rs index 7460a0f2e8..4e903fe1f0 100644 --- a/crates/tracedecay-code-extraction/tests/main/scala.rs +++ b/crates/tracedecay-code-extraction/tests/main/scala.rs @@ -1,8 +1,10 @@ use tracedecay_code_extraction::{LanguageExtractor, ScalaExtractor}; use tracedecay_domain::{EdgeKind, NodeKind}; +include!("support/edges.rs"); + fn extract(source: &str) -> tracedecay_domain::ExtractionResult { - ScalaExtractor.extract("test.scala", source) + ScalaExtractor.extract_artifact("test.scala", source).result } #[test] @@ -171,7 +173,10 @@ fn test_scala_extract_class_params_as_fields() { .filter(|n| n.kind == NodeKind::ValField) .collect(); // x and y are val params, z is a plain param (also extracted as ValField but private) - assert!(vals.len() >= 2); + assert_eq!( + vals.iter().map(|x| x.name.as_str()).collect::>(), + ["x", "y", "z"] + ); assert!(vals.iter().any(|n| n.name == "x")); assert!(vals.iter().any(|n| n.name == "y")); } @@ -179,13 +184,10 @@ fn test_scala_extract_class_params_as_fields() { #[test] fn test_scala_contains_edges() { let result = extract("object Main {\n def hello(): Unit = ()\n}"); - let contains_edges: Vec<_> = result - .edges - .iter() - .filter(|e| e.kind == EdgeKind::Contains) - .collect(); - // File → Object, Object → Method - assert!(contains_edges.len() >= 2); + assert_eq!( + edge_pairs(&result, EdgeKind::Contains), + [("test.scala", "Main"), ("Main", "hello")] + ); } #[test] @@ -197,7 +199,6 @@ fn test_scala_extract_call_sites() { .iter() .filter(|r| r.reference_kind == EdgeKind::Calls) .collect(); - assert!(calls.len() >= 2); assert!(calls.iter().any(|c| c.reference_name == "println")); assert!(calls.iter().any(|c| c.reference_name == "foo")); } diff --git a/crates/tracedecay-code-extraction/tests/main/support/edges.rs b/crates/tracedecay-code-extraction/tests/main/support/edges.rs new file mode 100644 index 0000000000..cf65f0cee1 --- /dev/null +++ b/crates/tracedecay-code-extraction/tests/main/support/edges.rs @@ -0,0 +1,25 @@ +// Shared edge assertions for the extractor test suite. +// +// Paths stay fully qualified so this file can be `include!`d next to +// `docstrings.rs` without duplicate imports. + +/// Edges of `kind` as `(source name, target name)` pairs in emission order. +pub fn edge_pairs( + result: &tracedecay_domain::ExtractionResult, + kind: tracedecay_domain::EdgeKind, +) -> Vec<(&str, &str)> { + let name_of = |id: &str| { + result + .nodes + .iter() + .find(|n| n.id == id) + .map(|n| n.name.as_str()) + .unwrap_or_else(|| panic!("edge endpoint {id} is not an extracted node")) + }; + result + .edges + .iter() + .filter(|e| e.kind == kind) + .map(|e| (name_of(&e.source), name_of(&e.target))) + .collect() +} diff --git a/crates/tracedecay-code-extraction/tests/main/svelte.rs b/crates/tracedecay-code-extraction/tests/main/svelte.rs index a54cfd26d0..6a76e58c6e 100644 --- a/crates/tracedecay-code-extraction/tests/main/svelte.rs +++ b/crates/tracedecay-code-extraction/tests/main/svelte.rs @@ -8,7 +8,9 @@ fn test_svelte_file_node() { export function greet(): void {}

Hello

"#; - let result = SvelteExtractor.extract("Page.svelte", source); + let result = SvelteExtractor + .extract_artifact("Page.svelte", source) + .result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let files: Vec<_> = result .nodes @@ -28,7 +30,9 @@ export function increment(n: number): number { function internal(): void {} "#; - let result = SvelteExtractor.extract("Counter.svelte", source); + let result = SvelteExtractor + .extract_artifact("Counter.svelte", source) + .result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let fns: Vec<_> = result .nodes @@ -46,7 +50,9 @@ function internal(): void {} fn test_svelte_line_numbers_are_original_file_positions() { // `greet` is on line 2 (0-indexed) in the full .svelte file. let source = "\n

hi

"; - let result = SvelteExtractor.extract("greet.svelte", source); + let result = SvelteExtractor + .extract_artifact("greet.svelte", source) + .result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let greet = result.nodes.iter().find(|n| n.name == "greet").unwrap(); assert_eq!( @@ -59,7 +65,9 @@ fn test_svelte_line_numbers_are_original_file_positions() { #[test] fn test_svelte_no_script_block_returns_file_node_only() { let source = "

Hello

\n

World

"; - let result = SvelteExtractor.extract("Static.svelte", source); + let result = SvelteExtractor + .extract_artifact("Static.svelte", source) + .result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); // Only the File node. No symbols to extract. let non_file: Vec<_> = result @@ -73,7 +81,9 @@ fn test_svelte_no_script_block_returns_file_node_only() { #[test] fn test_svelte_fixture() { let source = include_str!("../../fixtures/sample.svelte"); - let result = SvelteExtractor.extract("sample.svelte", source); + let result = SvelteExtractor + .extract_artifact("sample.svelte", source) + .result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let names: Vec<_> = result.nodes.iter().map(|n| n.name.as_str()).collect(); assert!( diff --git a/crates/tracedecay-code-extraction/tests/main/swift.rs b/crates/tracedecay-code-extraction/tests/main/swift.rs index 9da4bd03cd..a1dede970b 100644 --- a/crates/tracedecay-code-extraction/tests/main/swift.rs +++ b/crates/tracedecay-code-extraction/tests/main/swift.rs @@ -2,13 +2,15 @@ use tracedecay_code_extraction::LanguageExtractor; use tracedecay_code_extraction::SwiftExtractor; use tracedecay_domain::*; +include!("support/edges.rs"); + #[test] fn test_swift_extract_imports() { let source = r#"import Foundation import UIKit "#; let extractor = SwiftExtractor; - let result = extractor.extract("sample.swift", source); + let result = extractor.extract_artifact("sample.swift", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let uses: Vec<_> = result .nodes @@ -36,7 +38,7 @@ class Base { } "#; let extractor = SwiftExtractor; - let result = extractor.extract("base.swift", source); + let result = extractor.extract_artifact("base.swift", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let classes: Vec<_> = result @@ -63,7 +65,7 @@ fn test_swift_class_inheritance() { class Connection: Base {} "#; let extractor = SwiftExtractor; - let result = extractor.extract("conn.swift", source); + let result = extractor.extract_artifact("conn.swift", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let extends: Vec<_> = result @@ -71,7 +73,13 @@ class Connection: Base {} .iter() .filter(|r| r.reference_kind == EdgeKind::Extends) .collect(); - assert!(!extends.is_empty(), "expected Extends refs for inheritance"); + assert_eq!( + extends + .iter() + .map(|x| x.reference_name.as_str()) + .collect::>(), + ["Base"] + ); assert!( extends.iter().any(|r| r.reference_name == "Base"), "expected Extends ref to Base" @@ -87,7 +95,7 @@ class Foo { } "#; let extractor = SwiftExtractor; - let result = extractor.extract("funcs.swift", source); + let result = extractor.extract_artifact("funcs.swift", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let fns: Vec<_> = result @@ -119,7 +127,7 @@ fn test_swift_struct_with_fields_and_methods() { } "#; let extractor = SwiftExtractor; - let result = extractor.extract("point.swift", source); + let result = extractor.extract_artifact("point.swift", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let structs: Vec<_> = result @@ -135,10 +143,9 @@ fn test_swift_struct_with_fields_and_methods() { .iter() .filter(|n| n.kind == NodeKind::Property) .collect(); - assert!( - props.len() >= 2, - "expected >= 2 properties, got {}", - props.len() + assert_eq!( + props.iter().map(|x| x.name.as_str()).collect::>(), + ["x", "y"] ); let methods: Vec<_> = result @@ -160,7 +167,7 @@ fn test_swift_enum_with_variants() { } "#; let extractor = SwiftExtractor; - let result = extractor.extract("log.swift", source); + let result = extractor.extract_artifact("log.swift", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let enums: Vec<_> = result @@ -192,7 +199,7 @@ protocol Serializable { } "#; let extractor = SwiftExtractor; - let result = extractor.extract("proto.swift", source); + let result = extractor.extract_artifact("proto.swift", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let ifaces: Vec<_> = result @@ -229,7 +236,7 @@ fn test_swift_extension() { } "#; let extractor = SwiftExtractor; - let result = extractor.extract("ext.swift", source); + let result = extractor.extract_artifact("ext.swift", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let exts: Vec<_> = result @@ -256,7 +263,7 @@ fn test_swift_constructor() { } "#; let extractor = SwiftExtractor; - let result = extractor.extract("foo.swift", source); + let result = extractor.extract_artifact("foo.swift", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let ctors: Vec<_> = result @@ -279,7 +286,7 @@ func main() { } "#; let extractor = SwiftExtractor; - let result = extractor.extract("main.swift", source); + let result = extractor.extract_artifact("main.swift", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let call_refs: Vec<_> = result @@ -287,7 +294,6 @@ func main() { .iter() .filter(|r| r.reference_kind == EdgeKind::Calls) .collect(); - assert!(!call_refs.is_empty(), "should have call refs"); assert!( call_refs.iter().any(|r| r.reference_name == "print"), "should find print call" @@ -304,7 +310,7 @@ fn test_swift_docstrings() { func setup() {} "#; let extractor = SwiftExtractor; - let result = extractor.extract("doc.swift", source); + let result = extractor.extract_artifact("doc.swift", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let fns: Vec<_> = result @@ -333,17 +339,10 @@ fn test_swift_contains_edges() { } "#; let extractor = SwiftExtractor; - let result = extractor.extract("foo.swift", source); - let contains: Vec<_> = result - .edges - .iter() - .filter(|e| e.kind == EdgeKind::Contains) - .collect(); - // File contains: Class; Class contains: Property, Method - assert!( - contains.len() >= 3, - "should have >= 3 Contains edges, got {}", - contains.len() + let result = extractor.extract_artifact("foo.swift", source).result; + assert_eq!( + edge_pairs(&result, EdgeKind::Contains), + [("foo.swift", "Foo"), ("Foo", "bar"), ("Foo", "baz")] ); } @@ -352,7 +351,7 @@ fn test_swift_typealias() { let source = r#"typealias CompletionHandler = (Bool) -> Void "#; let extractor = SwiftExtractor; - let result = extractor.extract("alias.swift", source); + let result = extractor.extract_artifact("alias.swift", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let aliases: Vec<_> = result @@ -369,7 +368,7 @@ fn test_swift_top_level_const() { let source = r#"let maxConnections = 100 "#; let extractor = SwiftExtractor; - let result = extractor.extract("const.swift", source); + let result = extractor.extract_artifact("const.swift", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let consts: Vec<_> = result @@ -389,7 +388,7 @@ fn test_swift_visibility_private() { } "#; let extractor = SwiftExtractor; - let result = extractor.extract("vis.swift", source); + let result = extractor.extract_artifact("vis.swift", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let secret = result @@ -416,7 +415,7 @@ fn test_swift_async_function() { } "#; let extractor = SwiftExtractor; - let result = extractor.extract("async.swift", source); + let result = extractor.extract_artifact("async.swift", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let connect = result @@ -443,7 +442,7 @@ fn test_swift_annotation_extraction() { } "#; let extractor = SwiftExtractor; - let result = extractor.extract("attrs.swift", source); + let result = extractor.extract_artifact("attrs.swift", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let annots: Vec<_> = result @@ -453,50 +452,20 @@ fn test_swift_annotation_extraction() { .collect(); let annot_names: Vec<&str> = annots.iter().map(|a| a.name.as_str()).collect(); - - assert!( - annot_names.contains(&"objc"), - "expected 'objc' annotation, got: {:?}", - annot_names - ); - - assert!( - annot_names.contains(&"discardableResult"), - "expected 'discardableResult' annotation, got: {:?}", - annot_names - ); - - assert!( - annot_names.contains(&"available"), - "expected 'available' annotation, got: {:?}", - annot_names - ); - - // Verify Annotates edges exist - let annotates_edges: Vec<_> = result - .edges - .iter() - .filter(|e| e.kind == EdgeKind::Annotates) - .collect(); - assert!( - !annotates_edges.is_empty(), - "expected Annotates edges, found none" - ); + assert_eq!(annot_names, ["objc", "discardableResult", "available"]); assert_eq!( - annotates_edges.len(), - annots.len(), - "each AnnotationUsage should have an Annotates edge" + edge_pairs(&result, EdgeKind::Annotates), + [ + ("objc", "MyController"), + ("discardableResult", "doWork"), + ("available", "newFeature") + ] ); - - // Verify Annotates unresolved refs exist - let annotates_refs: Vec<_> = result + let annot_refs: Vec<&str> = result .unresolved_refs .iter() .filter(|r| r.reference_kind == EdgeKind::Annotates) + .map(|r| r.reference_name.as_str()) .collect(); - assert_eq!( - annotates_refs.len(), - annots.len(), - "each AnnotationUsage should have an Annotates unresolved ref" - ); + assert_eq!(annot_refs, ["objc", "discardableResult", "available"]); } diff --git a/crates/tracedecay-code-extraction/tests/main/toml.rs b/crates/tracedecay-code-extraction/tests/main/toml.rs index 5e5f9edab2..5f1c944813 100644 --- a/crates/tracedecay-code-extraction/tests/main/toml.rs +++ b/crates/tracedecay-code-extraction/tests/main/toml.rs @@ -3,7 +3,7 @@ use tracedecay_code_extraction::TomlExtractor; use tracedecay_domain::*; fn extract(source: &str) -> ExtractionResult { - TomlExtractor.extract("Cargo.toml", source) + TomlExtractor.extract_artifact("Cargo.toml", source).result } fn names_of(result: &ExtractionResult, kind: NodeKind) -> Vec { diff --git a/crates/tracedecay-code-extraction/tests/main/typescript.rs b/crates/tracedecay-code-extraction/tests/main/typescript.rs index d0ba2d6db1..1998b8be7d 100644 --- a/crates/tracedecay-code-extraction/tests/main/typescript.rs +++ b/crates/tracedecay-code-extraction/tests/main/typescript.rs @@ -6,7 +6,7 @@ use tracedecay_domain::*; fn test_ts_file_node_is_root() { let source = r#"function main() {}"#; let extractor = TypeScriptExtractor; - let result = extractor.extract("test.ts", source); + let result = extractor.extract_artifact("test.ts", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let files: Vec<_> = result .nodes @@ -28,7 +28,7 @@ export function greet(name: string): string { function internal(): void {} "#; let extractor = TypeScriptExtractor; - let result = extractor.extract("greet.ts", source); + let result = extractor.extract_artifact("greet.ts", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let fns: Vec<_> = result .nodes @@ -49,7 +49,7 @@ fn test_ts_empty_jsdoc_comment_does_not_panic() { export function documented(): void {} "#; let extractor = TypeScriptExtractor; - let result = extractor.extract("empty-jsdoc.ts", source); + let result = extractor.extract_artifact("empty-jsdoc.ts", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let function = result .nodes @@ -67,7 +67,9 @@ class C { } "#; let extractor = TypeScriptExtractor; - let result = extractor.extract("anonymous-generator-method.ts", source); + let result = extractor + .extract_artifact("anonymous-generator-method.ts", source) + .result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); assert!( result @@ -84,7 +86,9 @@ fn test_ts_empty_decorator_name_does_not_panic() { class C {} "#; let extractor = TypeScriptExtractor; - let result = extractor.extract("empty-decorator.ts", source); + let result = extractor + .extract_artifact("empty-decorator.ts", source) + .result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); assert!( result @@ -104,7 +108,7 @@ export const multiply = (a: number, b: number) => { }; "#; let extractor = TypeScriptExtractor; - let result = extractor.extract("arrow.ts", source); + let result = extractor.extract_artifact("arrow.ts", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let arrows: Vec<_> = result .nodes @@ -137,7 +141,7 @@ export class MyClass { } "#; let extractor = TypeScriptExtractor; - let result = extractor.extract("class.ts", source); + let result = extractor.extract_artifact("class.ts", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); // Check class @@ -202,7 +206,7 @@ export interface Printable { } "#; let extractor = TypeScriptExtractor; - let result = extractor.extract("iface.ts", source); + let result = extractor.extract_artifact("iface.ts", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let interfaces: Vec<_> = result @@ -233,7 +237,7 @@ export enum Color { } "#; let extractor = TypeScriptExtractor; - let result = extractor.extract("color.ts", source); + let result = extractor.extract_artifact("color.ts", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let enums: Vec<_> = result @@ -264,7 +268,7 @@ import { foo, bar } from './utils'; import * as path from 'path'; "#; let extractor = TypeScriptExtractor; - let result = extractor.extract("imports.ts", source); + let result = extractor.extract_artifact("imports.ts", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let uses: Vec<_> = result @@ -292,7 +296,7 @@ import type { Foo, Bar as Baz } from "pkg"; import { localThing } from "./local"; "#; let extractor = TypeScriptExtractor; - let result = extractor.extract("imports.ts", source); + let result = extractor.extract_artifact("imports.ts", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let use_refs: Vec<_> = result @@ -320,7 +324,7 @@ export async function fetchData(url: string): Promise { function syncHelper(): void {} "#; let extractor = TypeScriptExtractor; - let result = extractor.extract("async.ts", source); + let result = extractor.extract_artifact("async.ts", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let fns: Vec<_> = result @@ -343,7 +347,7 @@ class Service { } "#; let extractor = TypeScriptExtractor; - let result = extractor.extract("service.ts", source); + let result = extractor.extract_artifact("service.ts", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let decorators: Vec<_> = result @@ -379,7 +383,7 @@ namespace MyNamespace { } "#; let extractor = TypeScriptExtractor; - let result = extractor.extract("ns.ts", source); + let result = extractor.extract_artifact("ns.ts", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let namespaces: Vec<_> = result @@ -410,7 +414,7 @@ function add(a: number, b: number): number { } "#; let extractor = TypeScriptExtractor; - let result = extractor.extract("doc.ts", source); + let result = extractor.extract_artifact("doc.ts", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let fns: Vec<_> = result @@ -435,7 +439,7 @@ export function greet(name: string): string { } "#; let extractor = TypeScriptExtractor; - let result = extractor.extract("doc_export.ts", source); + let result = extractor.extract_artifact("doc_export.ts", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let fns: Vec<_> = result @@ -466,7 +470,7 @@ function main(): void { } "#; let extractor = TypeScriptExtractor; - let result = extractor.extract("calls.ts", source); + let result = extractor.extract_artifact("calls.ts", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let call_refs: Vec<_> = result @@ -474,18 +478,11 @@ function main(): void { .iter() .filter(|r| r.reference_kind == EdgeKind::Calls) .collect(); - assert!(!call_refs.is_empty(), "should have call refs"); - // Should have: console.log from greet, greet from main - assert!( - call_refs.iter().any(|r| r.reference_name.contains("greet")), - "should have a call to greet" - ); - assert!( - call_refs - .iter() - .any(|r| r.reference_name.contains("console.log")), - "should have a call to console.log" - ); + let callees: Vec<&str> = call_refs + .iter() + .map(|r| r.reference_name.as_str()) + .collect(); + assert_eq!(callees, ["console.log", "greet"]); } #[test] @@ -495,7 +492,7 @@ export type StringOrNum = string | number; type ID = string; "#; let extractor = TypeScriptExtractor; - let result = extractor.extract("types.ts", source); + let result = extractor.extract_artifact("types.ts", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let aliases: Vec<_> = result @@ -524,7 +521,7 @@ export class Child extends Base implements Printable { } "#; let extractor = TypeScriptExtractor; - let result = extractor.extract("inherit.ts", source); + let result = extractor.extract_artifact("inherit.ts", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); // Check for Extends unresolved ref @@ -533,7 +530,6 @@ export class Child extends Base implements Printable { .iter() .filter(|r| r.reference_kind == EdgeKind::Extends) .collect(); - assert!(!extends_refs.is_empty(), "should have Extends ref for Base"); assert!(extends_refs.iter().any(|r| r.reference_name == "Base")); // Check for Implements unresolved ref @@ -542,10 +538,6 @@ export class Child extends Base implements Printable { .iter() .filter(|r| r.reference_kind == EdgeKind::Implements) .collect(); - assert!( - !impl_refs.is_empty(), - "should have Implements ref for Printable" - ); assert!(impl_refs.iter().any(|r| r.reference_name == "Printable")); } @@ -555,7 +547,7 @@ fn test_ts_contains_edges() { function foo(): void {} "#; let extractor = TypeScriptExtractor; - let result = extractor.extract("edges.ts", source); + let result = extractor.extract_artifact("edges.ts", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let file_node = result @@ -593,7 +585,7 @@ export default class Foo { } "#; let extractor = TypeScriptExtractor; - let result = extractor.extract("test.js", source); + let result = extractor.extract_artifact("test.js", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let fns: Vec<_> = result @@ -626,7 +618,9 @@ export default class Foo { #[test] fn test_ts_unknown_language_key_surfaces_parse_error() { let extractor = TypeScriptExtractor; - let result = extractor.extract("test.definitely-not-registered", "const value = 1;"); + let result = extractor + .extract_artifact("test.definitely-not-registered", "const value = 1;") + .result; assert!( result @@ -652,7 +646,7 @@ export function App() { } "#; let extractor = TypeScriptExtractor; - let result = extractor.extract("app.jsx", source); + let result = extractor.extract_artifact("app.jsx", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let fns: Vec<_> = result @@ -680,7 +674,7 @@ export const Greeting: React.FC = ({ name }) => { }; "#; let extractor = TypeScriptExtractor; - let result = extractor.extract("greeting.tsx", source); + let result = extractor.extract_artifact("greeting.tsx", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let interfaces: Vec<_> = result @@ -709,7 +703,7 @@ export const MAX_SIZE = 1024; const SECRET = "hidden"; "#; let extractor = TypeScriptExtractor; - let result = extractor.extract("consts.ts", source); + let result = extractor.extract_artifact("consts.ts", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let consts: Vec<_> = result @@ -732,7 +726,7 @@ const fetchData = async (url: string) => { }; "#; let extractor = TypeScriptExtractor; - let result = extractor.extract("async_arrow.ts", source); + let result = extractor.extract_artifact("async_arrow.ts", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let arrows: Vec<_> = result @@ -786,7 +780,7 @@ describe('math', () => { }); "#; let extractor = TypeScriptExtractor; - let result = extractor.extract("math.test.ts", source); + let result = extractor.extract_artifact("math.test.ts", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let fns = ts_functions(&result); @@ -835,7 +829,7 @@ describe('suite', () => { }); "#; let extractor = TypeScriptExtractor; - let result = extractor.extract("s.test.ts", source); + let result = extractor.extract_artifact("s.test.ts", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); // Helper function inside describe becomes its own Function node. @@ -880,7 +874,7 @@ describe('mods', () => { }); "#; let extractor = TypeScriptExtractor; - let result = extractor.extract("mods.test.ts", source); + let result = extractor.extract_artifact("mods.test.ts", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let fns = ts_functions(&result); @@ -919,7 +913,7 @@ describe('todos', () => { }); "#; let extractor = TypeScriptExtractor; - let result = extractor.extract("todo.test.ts", source); + let result = extractor.extract_artifact("todo.test.ts", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); // it.todo has no callback -> a node is still emitted for the title, // but no calls are attributed. @@ -937,7 +931,7 @@ describe('fnexpr', function () { }); "#; let extractor = TypeScriptExtractor; - let result = extractor.extract("fnexpr.test.ts", source); + let result = extractor.extract_artifact("fnexpr.test.ts", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); // The template-literal title is captured (backticks stripped). @@ -955,7 +949,7 @@ fn test_ts_expression_bodied_arrow_extracts_calls() { export const compute = (x: number) => transform(x); "#; let extractor = TypeScriptExtractor; - let result = extractor.extract("compute.ts", source); + let result = extractor.extract_artifact("compute.ts", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let arrow = result .nodes @@ -981,14 +975,15 @@ test.describe("", () => { }); "#; let extractor = TypeScriptExtractor; - let result = extractor.extract("integration/fs-routes-test.ts", source); + let result = extractor + .extract_artifact("integration/fs-routes-test.ts", source) + .result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let fns = ts_functions(&result); - assert!( - fns.iter().all(|f| !f.name.is_empty()), - "no extracted node may carry an empty name; got {:?}", - fns.iter().map(|f| &f.name).collect::>() + assert_eq!( + fns.iter().map(|x| x.name.as_str()).collect::>(), + ["", "adds"] ); let suite = fns .iter() diff --git a/crates/tracedecay-code-extraction/tests/main/vbnet.rs b/crates/tracedecay-code-extraction/tests/main/vbnet.rs index ab3dcb65a3..18abcaa394 100644 --- a/crates/tracedecay-code-extraction/tests/main/vbnet.rs +++ b/crates/tracedecay-code-extraction/tests/main/vbnet.rs @@ -12,18 +12,13 @@ Class MyClass End Class "#; let extractor = VbNetExtractor; - let result = extractor.extract("test.vb", source); + let result = extractor.extract_artifact("test.vb", source).result; let class = result .nodes .iter() .find(|n| n.kind == NodeKind::Class && n.name == "MyClass") .expect("MyClass not found"); - assert!(class.docstring.is_some(), "Expected docstring on MyClass"); - assert!( - class.docstring.as_ref().unwrap().contains("test class"), - "Docstring should contain 'test class', got: {:?}", - class.docstring - ); + assert_eq!(class.docstring.as_deref(), Some("A test class.")); } #[test] @@ -34,7 +29,7 @@ Interface ISerializable End Interface "#; let extractor = VbNetExtractor; - let result = extractor.extract("test.vb", source); + let result = extractor.extract_artifact("test.vb", source).result; let interfaces: Vec<_> = result .nodes .iter() @@ -53,7 +48,7 @@ Structure Point End Structure "#; let extractor = VbNetExtractor; - let result = extractor.extract("test.vb", source); + let result = extractor.extract_artifact("test.vb", source).result; let structs: Vec<_> = result .nodes .iter() @@ -73,7 +68,7 @@ Module Helpers End Module "#; let extractor = VbNetExtractor; - let result = extractor.extract("test.vb", source); + let result = extractor.extract_artifact("test.vb", source).result; let modules: Vec<_> = result .nodes .iter() @@ -93,7 +88,7 @@ Enum LogLevel End Enum "#; let extractor = VbNetExtractor; - let result = extractor.extract("test.vb", source); + let result = extractor.extract_artifact("test.vb", source).result; let enums: Vec<_> = result .nodes @@ -128,17 +123,16 @@ Class Foo End Class "#; let extractor = VbNetExtractor; - let result = extractor.extract("test.vb", source); + let result = extractor.extract_artifact("test.vb", source).result; let methods: Vec<_> = result .nodes .iter() .filter(|n| n.kind == NodeKind::Method) .collect(); - assert!( - methods.len() >= 2, - "expected >= 2 methods, got {}", - methods.len() + assert_eq!( + methods.iter().map(|x| x.name.as_str()).collect::>(), + ["GetValue", "DoWork"] ); assert!(methods.iter().any(|m| m.name == "GetValue")); assert!(methods.iter().any(|m| m.name == "DoWork")); @@ -154,7 +148,7 @@ Class Foo End Class "#; let extractor = VbNetExtractor; - let result = extractor.extract("test.vb", source); + let result = extractor.extract_artifact("test.vb", source).result; let ctors: Vec<_> = result .nodes @@ -174,17 +168,16 @@ Class Foo End Class "#; let extractor = VbNetExtractor; - let result = extractor.extract("test.vb", source); + let result = extractor.extract_artifact("test.vb", source).result; let props: Vec<_> = result .nodes .iter() .filter(|n| n.kind == NodeKind::Property) .collect(); - assert!( - props.len() >= 2, - "expected >= 2 properties, got {}", - props.len() + assert_eq!( + props.iter().map(|x| x.name.as_str()).collect::>(), + ["Name", "Id"] ); assert!(props.iter().any(|p| p.name == "Name")); assert!(props.iter().any(|p| p.name == "Id")); @@ -196,7 +189,7 @@ fn test_vb_const() { Const MaxConnections As Integer = 100 "#; let extractor = VbNetExtractor; - let result = extractor.extract("test.vb", source); + let result = extractor.extract_artifact("test.vb", source).result; let consts: Vec<_> = result .nodes @@ -219,7 +212,7 @@ Class Foo End Class "#; let extractor = VbNetExtractor; - let result = extractor.extract("test.vb", source); + let result = extractor.extract_artifact("test.vb", source).result; let pub_method = result .nodes @@ -244,7 +237,7 @@ Class Foo End Class "#; let extractor = VbNetExtractor; - let result = extractor.extract("test.vb", source); + let result = extractor.extract_artifact("test.vb", source).result; let fields: Vec<_> = result .nodes @@ -270,7 +263,7 @@ Class MyClass End Class "#; let extractor = VbNetExtractor; - let result = extractor.extract("attr.vb", source); + let result = extractor.extract_artifact("attr.vb", source).result; // Should have 3 AnnotationUsage nodes: Serializable, Obsolete, TestMethod let annots: Vec<_> = result diff --git a/crates/tracedecay-code-extraction/tests/main/zig.rs b/crates/tracedecay-code-extraction/tests/main/zig.rs index c933295bce..143220005d 100644 --- a/crates/tracedecay-code-extraction/tests/main/zig.rs +++ b/crates/tracedecay-code-extraction/tests/main/zig.rs @@ -2,13 +2,15 @@ use tracedecay_code_extraction::LanguageExtractor; use tracedecay_code_extraction::ZigExtractor; use tracedecay_domain::*; +include!("support/edges.rs"); + #[test] fn test_zig_extract_imports() { let source = r#"const std = @import("std"); const mem = @import("std").mem; "#; let extractor = ZigExtractor; - let result = extractor.extract("sample.zig", source); + let result = extractor.extract_artifact("sample.zig", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let uses: Vec<_> = result .nodes @@ -30,7 +32,7 @@ const LogLevel = enum { }; "#; let extractor = ZigExtractor; - let result = extractor.extract("log.zig", source); + let result = extractor.extract_artifact("log.zig", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let enums: Vec<_> = result @@ -67,7 +69,7 @@ const Foo = struct { }; "#; let extractor = ZigExtractor; - let result = extractor.extract("funcs.zig", source); + let result = extractor.extract_artifact("funcs.zig", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let fns: Vec<_> = result @@ -93,7 +95,7 @@ fn test_zig_const_extraction() { const max_connections: u32 = 100; "#; let extractor = ZigExtractor; - let result = extractor.extract("const.zig", source); + let result = extractor.extract_artifact("const.zig", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let consts: Vec<_> = result @@ -125,7 +127,7 @@ pub fn publicFn() void {} fn privateFn() void {} "#; let extractor = ZigExtractor; - let result = extractor.extract("vis.zig", source); + let result = extractor.extract_artifact("vis.zig", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let public_method = result @@ -167,7 +169,7 @@ test "point distance" { } "#; let extractor = ZigExtractor; - let result = extractor.extract("test.zig", source); + let result = extractor.extract_artifact("test.zig", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); // test declarations are mapped as Function nodes @@ -196,7 +198,7 @@ pub fn main() void { } "#; let extractor = ZigExtractor; - let result = extractor.extract("main.zig", source); + let result = extractor.extract_artifact("main.zig", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let call_refs: Vec<_> = result @@ -204,7 +206,6 @@ pub fn main() void { .iter() .filter(|r| r.reference_kind == EdgeKind::Calls) .collect(); - assert!(!call_refs.is_empty(), "should have call refs"); assert!( call_refs.iter().any(|r| r.reference_name.contains("print")), "should find print call, got: {:?}", @@ -226,7 +227,7 @@ fn test_zig_docstrings() { pub fn setup() void {} "#; let extractor = ZigExtractor; - let result = extractor.extract("doc.zig", source); + let result = extractor.extract_artifact("doc.zig", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let fns: Vec<_> = result @@ -253,17 +254,10 @@ fn test_zig_contains_edges() { }; "#; let extractor = ZigExtractor; - let result = extractor.extract("foo.zig", source); - let contains: Vec<_> = result - .edges - .iter() - .filter(|e| e.kind == EdgeKind::Contains) - .collect(); - // File -> Struct, Struct -> Field, Struct -> Method = 3 minimum - assert!( - contains.len() >= 3, - "should have >= 3 Contains edges, got {}", - contains.len() + let result = extractor.extract_artifact("foo.zig", source).result; + assert_eq!( + edge_pairs(&result, EdgeKind::Contains), + [("foo.zig", "Foo"), ("Foo", "x"), ("Foo", "bar")] ); } @@ -298,7 +292,7 @@ fn test_zig_struct_with_multiple_methods() { }; "#; let extractor = ZigExtractor; - let result = extractor.extract("conn.zig", source); + let result = extractor.extract_artifact("conn.zig", source).result; assert!(result.errors.is_empty(), "errors: {:?}", result.errors); let structs: Vec<_> = result diff --git a/crates/tracedecay-code-index-retention/Cargo.toml b/crates/tracedecay-code-index-retention/Cargo.toml index 7d63fb7a83..eeb836a55c 100644 --- a/crates/tracedecay-code-index-retention/Cargo.toml +++ b/crates/tracedecay-code-index-retention/Cargo.toml @@ -18,6 +18,7 @@ sha2 = "0.11" thiserror = "2" tracedecay-code-index = { path = "../tracedecay-code-index", version = "0.1.0", default-features = false } tracedecay-domain = { path = "../tracedecay-domain", version = "0.1.0" } +tracedecay-graph-db = { path = "../tracedecay-graph-db", version = "0.1.0" } tracedecay-private-fs = { path = "../tracedecay-private-fs", version = "0.1.0" } tracedecay-runtime-core = { path = "../tracedecay-runtime-core", version = "0.1.0" } tracedecay-store = { path = "../tracedecay-store", version = "0.1.0" } diff --git a/crates/tracedecay-code-index-retention/src/code_index_generations.rs b/crates/tracedecay-code-index-retention/src/code_index_generations.rs index 4f14d84089..0a83543b0e 100644 --- a/crates/tracedecay-code-index-retention/src/code_index_generations.rs +++ b/crates/tracedecay-code-index-retention/src/code_index_generations.rs @@ -46,9 +46,11 @@ pub use graph_replay_release::{ CodeGenerationGraphReplayReleasePageV1, CodeGenerationGraphReplayReleaseV1, code_generation_graph_replay_release_page, complete_code_generation_graph_replay_release, }; +use locking::acquire_generation_segments_lock_checked; pub use locking::{ CodeGenerationStoreLockV1, acquire_code_generation_store_lock, - try_acquire_code_generation_store_lock, try_acquire_code_generation_store_read_lock, + acquire_generation_segments_publication_lock, try_acquire_code_generation_store_lock, + try_acquire_code_generation_store_read_lock, }; pub use scope_roots::{ RefusedCodeIndexScopeV1, SCOPE_ROOT_RECORD_FILE, ScopeRootAuthorityReceiptV1, @@ -62,7 +64,7 @@ pub use scope_roots::{ resolve_live_code_index_roots, }; pub use text_artifacts::{ - attach_verified_text_artifact_under_lock, replace_verified_text_artifact_under_lock, + attach_verified_text_artifact_under_lock, find_shared_text_artifact, withdraw_verified_text_artifact_under_lock, }; @@ -110,6 +112,7 @@ pub const MAX_DURABLE_GENERATION_INDEX_BYTES_V1: u64 = 8 * 1024 * 1024 * 1024; pub const MAX_DURABLE_GENERATION_INDEX_TTL_MICROS_V1: i64 = 7 * 24 * 60 * 60 * 1_000_000; pub const MAX_DURABLE_PUBLICATION_POINTER_BYTES_V1: u64 = 512 * 1024; pub const CODE_TEXT_ARTIFACTS_DIRECTORY_V1: &str = "code-text-artifacts-v1"; +pub const CODE_TEXT_ARTIFACT_STAGING_DIRECTORY_V1: &str = "code-text-artifact-staging-v1"; /// How long a code-index scope root must have been untouched before it can be /// classified as stranded and collected. A worktree can be unmounted, moved, or @@ -214,6 +217,37 @@ pub struct DurableCodeTextArtifactDescriptorV1 { pub artifact_file: String, pub artifact_digest: ManifestDigest, pub artifact_size_bytes: u64, + /// The key of the content the artifact was built from, which lets a + /// sibling worktree sealing the same content adopt the artifact instead + /// of rebuilding it. + pub content_key: ManifestDigest, +} + +/// A text artifact descriptor published before artifacts carried a content +/// key. Every such artifact predates the current format, so it is never +/// opened or adopted; the slot stays in the durable index, byte for byte, only +/// until its generation seals a current artifact or leaves the index. +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct RetiredCodeTextArtifactDescriptorV1 { + pub generation_id: CodeGenerationId, + pub artifact_file: String, + pub artifact_digest: ManifestDigest, + pub artifact_size_bytes: u64, +} + +/// What a durable index entry records about its generation's text artifact. +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(untagged)] +pub enum DurableTextArtifactSlotV1 { + Current(DurableCodeTextArtifactDescriptorV1), + Retired(RetiredCodeTextArtifactDescriptorV1), +} + +impl From for DurableTextArtifactSlotV1 { + fn from(descriptor: DurableCodeTextArtifactDescriptorV1) -> Self { + Self::Current(descriptor) + } } #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] @@ -242,7 +276,19 @@ pub struct DurableGenerationIndexEntryV1 { #[serde(default, skip_serializing_if = "Option::is_none")] pub cardinality: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub text_artifact: Option, + pub text_artifact: Option, +} + +impl DurableGenerationIndexEntryV1 { + /// The generation's current-format text artifact; a retired descriptor + /// names nothing a reader may open. + #[must_use] + pub fn text_artifact(&self) -> Option<&DurableCodeTextArtifactDescriptorV1> { + match self.text_artifact.as_ref()? { + DurableTextArtifactSlotV1::Current(descriptor) => Some(descriptor), + DurableTextArtifactSlotV1::Retired(_) => None, + } + } } /// Apply the durable exact-generation history bounds in canonical oldest-first @@ -381,7 +427,7 @@ impl<'entries> GenerationIndexByteAccountingV1<'entries> { let total = total .saturating_add(entry.size_bytes) .saturating_add(entry.segment_bytes); - match entry.text_artifact.as_ref() { + match entry.text_artifact() { Some(artifact) if self.artifacts_seen.insert(artifact.artifact_file.as_str()) => { total.saturating_add(artifact.artifact_size_bytes) } @@ -507,7 +553,6 @@ pub struct CodeGenerationRetentionGenerationV1 { pub enum CodeTextArtifactRetentionKindV1 { Completed, Staging, - Corrupt, } #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] @@ -543,6 +588,10 @@ pub struct CodeGenerationRetentionPlanV1 { /// pass's selected debris candidates. A descriptor shared by retained /// generations is counted once by its canonical artifact path. text_artifact_inventory_bytes: u64, + /// The active generation already names a text artifact and its resumable + /// staging file exists, so a successor build is in flight whose + /// publication replaces that artifact and leaves it unreferenced. + active_text_replacement_in_flight: bool, /// How thoroughly this plan proved generation integrity. Apply-mode /// execution refuses anything but [`GenerationDigestVerificationV1::Full`]. pub verification: GenerationDigestVerificationV1, @@ -584,6 +633,22 @@ impl CodeGenerationRetentionPlanV1 { pub const fn generation_segment_census(&self) -> GenerationSegmentCensusV1 { self.collectable_generation_segments } + + /// Whether superseded bytes are held live only by a holder that lets go + /// without waking maintenance: a superseded generation named by + /// `transient_pins` (the serving and text slots, which move when the + /// successor seats), or an in-flight replacement of the active + /// generation's text artifact. Maintenance keeps its short cadence while + /// this holds so the release is collected when it happens rather than at + /// the next full interval. + #[must_use] + pub fn awaits_transient_release(&self, transient_pins: &BTreeSet) -> bool { + self.active_text_replacement_in_flight + || self + .superseded_generations + .iter() + .any(|generation| transient_pins.contains(&generation.generation_id)) + } } #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] @@ -705,6 +770,7 @@ struct CodeTextArtifactRetentionTransactionV1 { struct CodeTextArtifactRetentionInventoryV1 { candidates: Vec, unique_bytes: u64, + active_text_replacement_in_flight: bool, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -722,9 +788,52 @@ pub fn scoped_code_index_store_root(store_root: &Path, canonical_project_root: & store_root.join(code_index_scope_hash(canonical_project_root)) } +/// The directory that owns a store's content-addressed generation segments. +/// +/// A store root named by its scope hash is one worktree's scope inside a +/// project's `code-index-v1/`, and its segments live beside every other scope +/// of that project, so worktrees that seal identical files store them once. +/// Any other store root is a project directory of its own. Writers, readers, +/// and the sweep resolve segments only through this. +#[must_use] +pub fn code_generation_segments_root(store_root: &Path) -> PathBuf { + generation_segment_project_root(store_root).join(GENERATION_SEGMENTS_DIRECTORY) +} + +fn generation_segment_project_root(store_root: &Path) -> &Path { + match ( + store_root.file_name().and_then(|name| name.to_str()), + store_root.parent(), + ) { + (Some(name), Some(project_root)) + if is_code_index_scope_hash(name) && !project_root.as_os_str().is_empty() => + { + project_root + } + _ => store_root, + } +} + +/// Whether `store_root` shares its project's segments with sibling scopes. +fn shares_project_generation_segments(store_root: &Path) -> bool { + generation_segment_project_root(store_root) != store_root +} + +/// The directory that owns a store's completed, content-addressed text +/// artifacts: the project's, beside its shared generation segments, so +/// worktrees that seal identical trees store one artifact. Retention marks +/// every scope's descriptors before collecting one. #[must_use] pub fn code_text_artifacts_root(store_root: &Path) -> PathBuf { - store_root.join(CODE_TEXT_ARTIFACTS_DIRECTORY_V1) + generation_segment_project_root(store_root).join(CODE_TEXT_ARTIFACTS_DIRECTORY_V1) +} + +/// The directory that owns one scope's resumable text-artifact staging. +/// Staging is named by the scope's sealed generation, so it stays with the +/// scope that builds it. +#[must_use] +pub fn code_text_artifact_staging_root(store_root: &Path) -> PathBuf { + store_root.join(CODE_TEXT_ARTIFACT_STAGING_DIRECTORY_V1) } pub fn code_text_artifact_path( @@ -792,6 +901,7 @@ fn unpublished_store_plan( collectable_text_artifacts: Vec::new(), collectable_generation_segments: GenerationSegmentCensusV1::NoneFound, text_artifact_inventory_bytes: 0, + active_text_replacement_in_flight: false, verification: GenerationDigestVerificationV1::Full, active_pointer: None, } @@ -1155,6 +1265,8 @@ fn plan_code_generation_retention_with_verification_cancellable( collectable_text_artifacts: text_artifact_inventory.candidates, collectable_generation_segments, text_artifact_inventory_bytes: text_artifact_inventory.unique_bytes, + active_text_replacement_in_flight: text_artifact_inventory + .active_text_replacement_in_flight, verification, active_pointer, }) @@ -1204,12 +1316,10 @@ fn sweep_unreferenced_generation_segments( apply: bool, is_cancelled: &dyn Fn() -> bool, ) -> Result<(bool, u64, bool), CodeGenerationRetentionErrorV1> { - let segments_root = store_root.join(GENERATION_SEGMENTS_DIRECTORY); + let segments_root = code_generation_segments_root(store_root); let entries = match std::fs::read_dir(&segments_root) { Ok(entries) => entries, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => { - return Ok((false, 0, false)); - } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok((false, 0, false)), Err(error) => return Err(storage(error)), }; let mut live_segments = BTreeSet::new(); @@ -1246,6 +1356,15 @@ fn sweep_unreferenced_generation_segments( })? .to_owned() } else { + if let Some(digests) = + tracedecay_graph_db::sealed_read_bundle_manifest_artifact_digests(&path) + .map_err(|error| { + CodeGenerationRetentionErrorV1::UnsafeState(error.to_string()) + })? + { + live_segments.extend(digests); + continue; + } let Some(file_name) = generation_file_name(&path) else { continue; }; @@ -1300,7 +1419,9 @@ fn sweep_unreferenced_generation_segments( } Ok(()) }; - mark_root(&store_root.join(GENERATIONS_DIRECTORY), false)?; + for directory in segment_manifest_directories(store_root)? { + mark_root(&directory, false)?; + } if let Some(pool_root) = graph_replay_pool_root { mark_root(pool_root, true)?; } @@ -1321,10 +1442,12 @@ fn sweep_unreferenced_generation_segments( .strip_prefix("segment-") .and_then(|name| name.strip_suffix(".json")) .filter(|digest| is_lowercase_hex(digest, 64)) + .map(|digest| format!("sha256:{digest}")) + .or_else(|| tracedecay_graph_db::sealed_read_bundle_artifact_file_digest(file_name)) else { continue; }; - if live_segments.contains(&format!("sha256:{digest}")) { + if live_segments.contains(&digest) { continue; } let metadata = path.symlink_metadata().map_err(deferred_if_absent)?; @@ -1355,6 +1478,58 @@ fn sweep_unreferenced_generation_segments( )) } +fn subdirectories(root: &Path) -> Result, CodeGenerationRetentionErrorV1> { + let entries = match std::fs::read_dir(root) { + Ok(entries) => entries, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(error) => return Err(storage(error)), + }; + let mut directories = Vec::new(); + for entry in entries { + let entry = entry.map_err(storage)?; + if entry.file_type().map_err(storage)?.is_dir() { + directories.push(entry.path()); + } + } + Ok(directories) +} + +/// The scopes whose manifests can name the store's segments: the store alone, +/// or in a shared project every worktree scope, including scopes quarantined +/// by scope collection, which a recovery may still restore. +fn segment_scope_roots(store_root: &Path) -> Result, CodeGenerationRetentionErrorV1> { + if !shares_project_generation_segments(store_root) { + return Ok(vec![store_root.to_path_buf()]); + } + let project_root = generation_segment_project_root(store_root); + let is_scope = |path: &PathBuf| { + path.file_name() + .and_then(|name| name.to_str()) + .is_some_and(is_code_index_scope_hash) + }; + let mut scopes = subdirectories(project_root)? + .into_iter() + .filter(is_scope) + .collect::>(); + for stage in subdirectories(&project_root.join(SCOPE_RETENTION_QUARANTINE_DIRECTORY))? { + scopes.extend(subdirectories(&stage)?.into_iter().filter(is_scope)); + } + Ok(scopes) +} + +/// Every directory a manifest naming the store's segments can occupy: each +/// scope's generations and the stages its retention quarantine may restore. +fn segment_manifest_directories( + store_root: &Path, +) -> Result, CodeGenerationRetentionErrorV1> { + let mut directories = Vec::new(); + for scope in segment_scope_roots(store_root)? { + directories.push(scope.join(GENERATIONS_DIRECTORY)); + directories.extend(subdirectories(&scope.join(QUARANTINE_DIRECTORY))?); + } + Ok(directories) +} + fn replay_generation_file_digest(file_name: &str) -> Option<&str> { generation_file_digest(file_name).or_else(|| { let (digest, suffix) = file_name @@ -1376,7 +1551,7 @@ fn store_may_hold_generation_segments( store_root: &Path, is_cancelled: &dyn Fn() -> bool, ) -> Result { - let mut entries = match std::fs::read_dir(store_root.join(GENERATION_SEGMENTS_DIRECTORY)) { + let mut entries = match std::fs::read_dir(code_generation_segments_root(store_root)) { Ok(entries) => entries, Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(false), Err(error) => return Err(storage(error)), @@ -1472,6 +1647,10 @@ pub fn execute_code_generation_retention_cancellable( let vector_readable_sources = plan.vector_readable_sources.clone(); let _store_lock = try_acquire_code_generation_store_lock(store_root)? .ok_or(CodeGenerationRetentionErrorV1::GenerationStoreBusy)?; + // Retiring manifests and sweeping segments both change which segments + // are named; neither may interleave with another scope's publication, + // retention, or scope collection over the shared segment directory. + let _segments_lock = acquire_generation_segments_lock_checked(store_root, is_cancelled)?; if observe_cancel(is_cancelled) { return Err(CodeGenerationRetentionErrorV1::Cancelled); } @@ -1687,6 +1866,9 @@ fn recover_code_generation_retention_cancellable( } let _store_lock = try_acquire_code_generation_store_lock(store_root)? .ok_or(CodeGenerationRetentionErrorV1::GenerationStoreBusy)?; + // Recovery may restore quarantined manifests into the generations + // directory, which a concurrent sweep over shared segments must not miss. + let _segments_lock = acquire_generation_segments_lock_checked(store_root, is_cancelled)?; if observe_cancel(is_cancelled) { return Err(CodeGenerationRetentionErrorV1::Cancelled); } @@ -1942,7 +2124,7 @@ fn validate_durable_generation_index( .to_owned(), )); } - if let Some(artifact) = entry.text_artifact.as_ref() { + if let Some(artifact) = entry.text_artifact() { validate_text_artifact_descriptor(artifact)?; if artifact.generation_id.as_str() != entry.generation_id { return Err(CodeGenerationRetentionErrorV1::UnsafeState( diff --git a/crates/tracedecay-code-index-retention/src/code_index_generations/locking.rs b/crates/tracedecay-code-index-retention/src/code_index_generations/locking.rs index 14e39d92d4..dc8b490918 100644 --- a/crates/tracedecay-code-index-retention/src/code_index_generations/locking.rs +++ b/crates/tracedecay-code-index-retention/src/code_index_generations/locking.rs @@ -2,6 +2,8 @@ use std::fs::{File, OpenOptions}; use std::path::{Path, PathBuf}; use std::time::Instant; +use tracedecay_private_fs::FileLease; + use super::{ CodeGenerationRetentionErrorV1, GRAPH_REPLAY_POOL_ACQUIRE_BUDGET, GRAPH_REPLAY_POOL_ACQUIRE_POLL, SCOPE_RETENTION_LOCK_FILE, STORE_LOCK_FILE, storage, @@ -9,8 +11,10 @@ use super::{ #[cfg(windows)] use super::{SCOPE_RETENTION_TRANSACTION_FILE, is_code_index_scope_hash, journal, scope_roots}; +const CODE_GENERATION_LEASE_LABEL: &str = "code_index_retention.store"; + pub struct CodeGenerationStoreLockV1 { - file: File, + _file: FileLease, store_root: PathBuf, generation_store: bool, shared: bool, @@ -41,12 +45,6 @@ impl CodeGenerationStoreLockV1 { } } -impl Drop for CodeGenerationStoreLockV1 { - fn drop(&mut self) { - let _ = self.file.unlock(); - } -} - pub fn acquire_code_generation_store_lock( store_root: &Path, ) -> Result { @@ -87,7 +85,7 @@ pub fn try_acquire_code_generation_store_read_lock( let lock = open_lock_file(&store_root.join(STORE_LOCK_FILE))?; match lock.try_lock_shared().map_err(std::io::Error::from) { Ok(()) => Ok(Some(CodeGenerationStoreLockV1 { - file: lock, + _file: FileLease::held(lock, CODE_GENERATION_LEASE_LABEL), store_root, generation_store: true, shared: true, @@ -135,7 +133,7 @@ fn try_acquire_code_generation_store_lock_unfenced( let lock = open_lock_file(&store_root.join(STORE_LOCK_FILE))?; match lock.try_lock().map_err(std::io::Error::from) { Ok(()) => Ok(Some(CodeGenerationStoreLockV1 { - file: lock, + _file: FileLease::held(lock, CODE_GENERATION_LEASE_LABEL), store_root, generation_store: true, shared: false, @@ -159,6 +157,57 @@ pub(super) fn acquire_scope_retention_lock( ) } +/// Exclusive hold on the project's shared generation segments. +/// +/// It is the project's scope-retention lock: every change that can make a +/// manifest stop or start naming segments, retiring generations, sweeping +/// segments, and collecting or restoring whole scopes, runs under it, so a +/// sweep's mark phase observes every manifest that can still be read. +pub(super) fn acquire_generation_segments_lock_checked( + store_root: &Path, + is_cancelled: &dyn Fn() -> bool, +) -> Result { + lock_file( + super::generation_segment_project_root(store_root), + SCOPE_RETENTION_LOCK_FILE, + false, + Instant::now() + GRAPH_REPLAY_POOL_ACQUIRE_BUDGET, + is_cancelled, + ) +} + +/// Shared hold a publication keeps from its first segment write until its +/// manifest is durable, so no sweep can collect a segment a manifest is about +/// to name. Publications of different worktrees share it; the exclusive +/// holders are bounded passes, so this waits for them instead of failing the +/// publication, observing `is_cancelled` between probes. +pub fn acquire_generation_segments_publication_lock( + store_root: &Path, + is_cancelled: &dyn Fn() -> bool, +) -> Result { + let project_root = canonical_store_root(super::generation_segment_project_root(store_root))?; + loop { + if is_cancelled() { + return Err(CodeGenerationRetentionErrorV1::Cancelled); + } + let lock = open_lock_file(&project_root.join(SCOPE_RETENTION_LOCK_FILE))?; + match lock.try_lock_shared().map_err(std::io::Error::from) { + Ok(()) => { + return Ok(CodeGenerationStoreLockV1 { + _file: FileLease::held(lock, CODE_GENERATION_LEASE_LABEL), + store_root: project_root, + generation_store: false, + shared: true, + }); + } + Err(error) if tracedecay_private_fs::is_lock_contended(&error) => { + std::thread::park_timeout(GRAPH_REPLAY_POOL_ACQUIRE_POLL); + } + Err(error) => return Err(storage(error)), + } + } +} + #[hotpath::measure(label = "code_index_retention.lock")] fn lock_file( store_root: &Path, @@ -192,7 +241,7 @@ fn lock_file( match lock.try_lock().map_err(std::io::Error::from) { Ok(()) => { return Ok(CodeGenerationStoreLockV1 { - file: lock, + _file: FileLease::held(lock, CODE_GENERATION_LEASE_LABEL), store_root, generation_store, shared: false, diff --git a/crates/tracedecay-code-index-retention/src/code_index_generations/tests.rs b/crates/tracedecay-code-index-retention/src/code_index_generations/tests.rs index a32a09ca85..b708630e8e 100644 --- a/crates/tracedecay-code-index-retention/src/code_index_generations/tests.rs +++ b/crates/tracedecay-code-index-retention/src/code_index_generations/tests.rs @@ -56,6 +56,8 @@ fn text_artifact( artifact_digest: ManifestDigest::new(format!("sha256:{sequence:064x}")) .expect("artifact digest"), artifact_size_bytes, + content_key: ManifestDigest::new(format!("sha256:{}", "c".repeat(64))) + .expect("content key"), } } @@ -70,6 +72,8 @@ fn text_artifact_for_bytes( artifact_file: format!("text-artifact-{digest}.bin"), artifact_digest: ManifestDigest::from_sha256_bytes(&digest_bytes).expect("artifact digest"), artifact_size_bytes: u64::try_from(bytes.len()).expect("artifact byte count"), + content_key: ManifestDigest::new(format!("sha256:{}", "c".repeat(64))) + .expect("content key"), } } @@ -162,11 +166,8 @@ fn durable_index_counts_text_bytes_and_never_evicts_the_active_text_head() { let mut text_head = indexed_generation(1, now - 3, 32, true); let text_head_id = CodeGenerationId::new(text_head.generation_id.clone()).expect("text-head generation id"); - text_head.text_artifact = Some(text_artifact( - &text_head_id, - 1, - MAX_DURABLE_GENERATION_INDEX_BYTES_V1, - )); + text_head.text_artifact = + Some(text_artifact(&text_head_id, 1, MAX_DURABLE_GENERATION_INDEX_BYTES_V1).into()); let mut entries = vec![ indexed_generation(0, now - 4, 32, true), text_head.clone(), @@ -202,7 +203,7 @@ fn reference_retain_bounded_generation_index( }); let mut artifacts = BTreeSet::new(); entries.iter().fold(generation_bytes, |total, entry| { - let Some(artifact) = entry.text_artifact.as_ref() else { + let Some(artifact) = entry.text_artifact() else { return total; }; if artifacts.insert(artifact.artifact_file.as_str()) { @@ -267,7 +268,7 @@ fn uneven_generation_history() -> ( let text_head_id = CodeGenerationId::new(text_head.generation_id.clone()).expect("text-head generation id"); let head_artifact = text_artifact(&text_head_id, 1, MAX_DURABLE_GENERATION_INDEX_BYTES_V1 / 4); - text_head.text_artifact = Some(head_artifact.clone()); + text_head.text_artifact = Some(head_artifact.clone().into()); let mut entries = vec![ indexed_generation( 1, @@ -298,7 +299,7 @@ fn uneven_generation_history() -> ( 8, sequence % 2 == 0, ); - entry.text_artifact = Some(shared_artifact.clone()); + entry.text_artifact = Some(shared_artifact.clone().into()); entries.push(entry); } let mut segment_heavy = indexed_generation(10, now - 70, 1, true); @@ -307,7 +308,7 @@ fn uneven_generation_history() -> ( // One removable generation shares the protected head's artifact, so its // eviction never releases those bytes. let mut head_sharer = indexed_generation(30, now - 60, 8, false); - head_sharer.text_artifact = Some(head_artifact); + head_sharer.text_artifact = Some(head_artifact.into()); entries.push(head_sharer); for sequence in 100..(100 + MAX_DURABLE_GENERATION_INDEX_ENTRIES_V1 + 4) { entries.push(indexed_generation( @@ -421,7 +422,7 @@ fn single_pass_sweep_matches_the_remove_recompute_loop_on_randomized_histories() other => u64::try_from(other).expect("ordinal") * 1_000, }; entry.text_artifact = - Some(text_artifact(&artifact_owner, shared, artifact_size)); + Some(text_artifact(&artifact_owner, shared, artifact_size).into()); } entry }) @@ -474,11 +475,14 @@ fn single_pass_sweep_accounting_is_linear_on_thousands_of_shared_artifact_entrie for sequence in 1..=entry_count { let mut entry = indexed_generation(sequence, now - 1_000 + sequence as i64, 1, false); let owner = CodeGenerationId::new(entry.generation_id.clone()).expect("generation id"); - entry.text_artifact = Some(text_artifact( - &owner, - sequence / 8, - MAX_DURABLE_GENERATION_INDEX_BYTES_V1 / 2, - )); + entry.text_artifact = Some( + text_artifact( + &owner, + sequence / 8, + MAX_DURABLE_GENERATION_INDEX_BYTES_V1 / 2, + ) + .into(), + ); entries.push(entry); } let mut reference = entries.clone(); @@ -646,55 +650,12 @@ fn verified_text_artifact_attachment_is_durable_and_idempotent_under_the_store_l read_active_pointer(fixture.store.path()) .expect("durable active pointer") .generation_index[0] - .text_artifact, + .text_artifact() + .cloned(), Some(descriptor) ); } -#[test] -fn verified_text_artifact_replacement_is_exact_durable_and_never_clears_the_head() { - let fixture = text_artifact_mutation_fixture(); - let prior = text_artifact(&fixture.generation_id, 7, 4096); - let replacement = text_artifact(&fixture.generation_id, 8, 8192); - let lock = - acquire_code_generation_store_lock(fixture.store.path()).expect("generation store lock"); - let attached = attach_verified_text_artifact_under_lock( - &lock, - &fixture.pointer, - &fixture.sealed_identity, - prior.clone(), - ) - .expect("attach prior artifact"); - - let replaced = replace_verified_text_artifact_under_lock( - &lock, - &attached, - &fixture.sealed_identity, - &prior, - replacement.clone(), - ) - .expect("replace exact artifact"); - let repeated = replace_verified_text_artifact_under_lock( - &lock, - &replaced, - &fixture.sealed_identity, - &prior, - replacement.clone(), - ) - .expect("repeat replacement"); - drop(lock); - - assert_eq!(repeated, replaced); - assert_eq!( - replaced.generation_index[0].text_artifact, - Some(replacement) - ); - assert_eq!( - read_active_pointer(fixture.store.path()).expect("durable pointer"), - replaced - ); -} - #[test] fn verified_text_artifact_attachment_retires_history_before_enforcing_byte_bound() { let (store, generations) = fixture_store(2); @@ -716,7 +677,7 @@ fn verified_text_artifact_attachment_retires_history_before_enforcing_byte_bound source_revision: None, source_tree: None, cardinality: None, - text_artifact: Some(prior_artifact), + text_artifact: Some(prior_artifact.into()), }, ); pointer.generation_index_digest = Some( @@ -755,7 +716,10 @@ fn verified_text_artifact_attachment_retires_history_before_enforcing_byte_bound updated.generation_index[0].generation_id, active.id.as_str() ); - assert_eq!(updated.generation_index[0].text_artifact, Some(descriptor)); + assert_eq!( + updated.generation_index[0].text_artifact(), + Some(&descriptor) + ); assert_eq!( read_active_pointer(store.path()).expect("durable pointer"), updated @@ -827,27 +791,26 @@ fn text_artifact_retention_preserves_references_and_collects_orphans() { let active = generations.last().expect("active generation"); let referenced = attach_fixture_text_artifact(&store, active, b"durably referenced"); let artifacts_root = code_text_artifacts_root(store.path()); + let staging_root = code_text_artifact_staging_root(store.path()); + std::fs::create_dir_all(&staging_root).expect("create staging root"); let orphan = text_artifact_for_bytes(&active.id, b"unreferenced completed bytes"); let orphan_path = write_text_artifact(&store, &orphan, b"unreferenced completed bytes"); let staging_name = format!(".text-artifact-{}.staging", "a".repeat(64)); - let staging_path = artifacts_root.join(&staging_name); + let staging_path = staging_root.join(&staging_name); std::fs::write(&staging_path, b"abandoned staging").expect("write stale staging"); let active_staging_name = format!( ".text-artifact-{}.staging", sha256_hex_suffix(&active.state_digest).expect("active sealed digest") ); - let active_staging_path = artifacts_root.join(&active_staging_name); + let active_staging_path = staging_root.join(&active_staging_name); std::fs::write(&active_staging_path, b"resumable active staging") .expect("write active staging"); - let corrupt_name = format!("text-artifact-{}.corrupt-incident", "b".repeat(64)); - let corrupt_path = artifacts_root.join(&corrupt_name); - std::fs::write(&corrupt_path, b"corrupt backup").expect("write corrupt backup"); let plan = plan_code_generation_retention(store.path(), &BTreeSet::new()) .expect("plan artifact retention"); assert!(plan.collectable_generations.is_empty()); - assert_eq!(plan.collectable_text_artifacts.len(), 3); + assert_eq!(plan.collectable_text_artifacts.len(), 2); assert!(plan.has_collectable_work()); assert_eq!( total_text_artifact_bytes(&plan.collectable_text_artifacts), @@ -859,11 +822,6 @@ fn text_artifact_retention_preserves_references_and_collects_orphans() { .expect("staging metadata") .len(), ) - .saturating_add( - std::fs::metadata(&corrupt_path) - .expect("corrupt metadata") - .len(), - ) ); assert_eq!( plan.text_artifact_inventory_bytes, @@ -888,11 +846,11 @@ fn text_artifact_retention_preserves_references_and_collects_orphans() { ) .expect("collect artifact debris"); assert_eq!(report.deleted_generations.len(), 0); - assert_eq!(report.deleted_text_artifacts.len(), 3); + assert_eq!(report.deleted_text_artifacts.len(), 2); let receipt = report .text_artifact_receipt .expect("durable artifact retention receipt"); - assert_eq!(receipt.deleted_artifacts.len(), 3); + assert_eq!(receipt.deleted_artifacts.len(), 2); assert_eq!( receipt.reclaimed_bytes, total_text_artifact_bytes(&receipt.deleted_artifacts), @@ -917,7 +875,6 @@ fn text_artifact_retention_preserves_references_and_collects_orphans() { ); assert!(!orphan_path.exists()); assert!(!staging_path.exists()); - assert!(!corrupt_path.exists()); assert!( active_staging_path.is_file(), "only the active generation's resumable staging evidence is preserved" @@ -962,7 +919,7 @@ fn text_artifact_retention_collects_empty_publish_crash_placeholder() { #[test] fn text_artifact_retention_collects_staging_database_sidecars_with_their_owner() { let (store, _generations) = fixture_store(1); - let artifacts_root = code_text_artifacts_root(store.path()); + let artifacts_root = code_text_artifact_staging_root(store.path()); std::fs::create_dir_all(&artifacts_root).expect("create artifact root"); let staging_name = format!(".text-artifact-{}.staging", "c".repeat(64)); let paths = [ @@ -1005,7 +962,7 @@ fn text_artifact_retention_collects_staging_database_sidecars_with_their_owner() #[test] fn text_artifact_inventory_skips_an_entry_reclaimed_during_the_scan() { let store = tempfile::TempDir::new().expect("artifact store"); - let artifacts_root = code_text_artifacts_root(store.path()); + let artifacts_root = code_text_artifact_staging_root(store.path()); std::fs::create_dir_all(&artifacts_root).expect("create artifact root"); let staging_family = ["a", "b", "c"] .into_iter() @@ -1153,12 +1110,12 @@ fn cancellable_artifact_apply_stops_rehash_before_quarantine_and_retries() { #[test] fn text_artifact_retention_uses_bounded_restartable_batches() { let (store, _generations) = fixture_store(1); - let artifacts_root = code_text_artifacts_root(store.path()); - std::fs::create_dir_all(&artifacts_root).expect("create artifact root"); + let staging_root = code_text_artifact_staging_root(store.path()); + std::fs::create_dir_all(&staging_root).expect("create staging root"); for sequence in 0..(MAX_CODE_TEXT_ARTIFACT_RETENTION_BATCH_V1 + 2) { - let path = artifacts_root.join(format!("text-artifact-{sequence:064x}.corrupt-restart")); + let path = staging_root.join(format!(".text-artifact-{sequence:064x}.staging")); std::fs::write(path, [u8::try_from(sequence).expect("small sequence")]) - .expect("write corrupt backup"); + .expect("write abandoned staging"); } let first = @@ -1188,8 +1145,8 @@ fn text_artifact_retention_uses_bounded_restartable_batches() { ) .expect("apply second bounded page"); assert!( - std::fs::read_dir(&artifacts_root) - .expect("read empty artifact root") + std::fs::read_dir(&staging_root) + .expect("read empty staging root") .next() .is_none(), "a resumed page must reach later orphan artifacts" @@ -1199,9 +1156,9 @@ fn text_artifact_retention_uses_bounded_restartable_batches() { #[test] fn text_artifact_inventory_honors_cancellation_before_marking_or_mutation() { let (store, _generations) = fixture_store(1); - let artifacts_root = code_text_artifacts_root(store.path()); - std::fs::create_dir_all(&artifacts_root).expect("create artifact root"); - let orphan = artifacts_root.join(format!("text-artifact-{}.corrupt-cancel", "c".repeat(64))); + let staging_root = code_text_artifact_staging_root(store.path()); + std::fs::create_dir_all(&staging_root).expect("create staging root"); + let orphan = staging_root.join(format!(".text-artifact-{}.staging", "c".repeat(64))); std::fs::write(&orphan, b"uncollected").expect("write artifact debris"); let pointer = read_active_pointer(store.path()).expect("pointer"); let checks = std::sync::atomic::AtomicUsize::new(0); @@ -2756,7 +2713,9 @@ fn metadata_only_census_matches_full_verification() { let referenced = attach_fixture_text_artifact(&store, active, b"metadata parity live"); let orphan = text_artifact_for_bytes(&active.id, b"metadata parity orphan"); write_text_artifact(&store, &orphan, b"metadata parity orphan"); - let stale_staging = code_text_artifacts_root(store.path()) + std::fs::create_dir_all(code_text_artifact_staging_root(store.path())) + .expect("create staging root"); + let stale_staging = code_text_artifact_staging_root(store.path()) .join(format!(".text-artifact-{}.staging", "e".repeat(64))); std::fs::write(&stale_staging, b"metadata parity staging").expect("write stale staging"); @@ -2849,7 +2808,8 @@ fn applied_retention_refuses_a_metadata_only_plan() { fn unpublished_store_retention_reclaims_orphaned_partial_generations() { let (store, generations) = fixture_store(2); std::fs::remove_file(store.path().join(ACTIVE_POINTER_FILE)).expect("sever the active pointer"); - let artifacts_root = code_text_artifacts_root(store.path()); + let artifacts_root = code_text_artifact_staging_root(store.path()); + std::fs::create_dir_all(&artifacts_root).expect("create staging root"); let orphan = text_artifact_for_bytes(&generations[0].id, b"orphan completed bytes"); let orphan_path = write_text_artifact(&store, &orphan, b"orphan completed bytes"); let staging_name = format!(".text-artifact-{}.staging", "a".repeat(64)); @@ -2954,7 +2914,7 @@ fn unpublished_store_execution_refuses_when_a_pointer_appears() { fn staging_sidecars_share_their_staging_artifact_liveness() { let (store, generations) = fixture_store(1); let active = generations.last().expect("active generation"); - let artifacts_root = code_text_artifacts_root(store.path()); + let artifacts_root = code_text_artifact_staging_root(store.path()); std::fs::create_dir_all(&artifacts_root).expect("create artifact root"); let active_digest = sha256_hex_suffix(&active.state_digest).expect("active sealed digest"); let active_staging = artifacts_root.join(format!(".text-artifact-{active_digest}.staging")); @@ -2986,6 +2946,56 @@ fn staging_sidecars_share_their_staging_artifact_liveness() { assert!(!orphan_sidecar.exists()); } +/// A seated successor releases the serving pin on its predecessor, and a +/// published successor text artifact orphans the incumbent, without waking +/// maintenance. The plan must report each holder while it holds and stop the +/// moment it lets go, or maintenance either sleeps a day on debris or never +/// leaves its short cadence. +#[test] +fn transient_holders_keep_retention_awake_exactly_until_release() { + let (store, generations) = fixture_store(2); + let (superseded, active) = (&generations[0], &generations[1]); + let none = BTreeSet::new(); + let plan = |pins: &BTreeSet| { + plan_code_generation_retention(store.path(), pins).expect("plan retention") + }; + + let serving_pin = BTreeSet::from([superseded.id.clone()]); + let pinned = plan(&serving_pin); + assert!(pinned.collectable_generations.is_empty()); + assert!(pinned.awaits_transient_release(&serving_pin)); + + let released = plan(&none); + assert_eq!(released.collectable_generations.len(), 1); + assert_eq!( + released.collectable_generations[0].generation_id, + superseded.id + ); + assert!(!released.awaits_transient_release(&none)); + + let active_pin = BTreeSet::from([active.id.clone()]); + assert!( + !plan(&active_pin).awaits_transient_release(&active_pin), + "a pin on the active generation is steady state, not a pending release" + ); + + let artifacts_root = code_text_artifact_staging_root(store.path()); + std::fs::create_dir_all(&artifacts_root).expect("create artifact root"); + let active_digest = sha256_hex_suffix(&active.state_digest).expect("active sealed digest"); + let staging = artifacts_root.join(format!(".text-artifact-{active_digest}.staging")); + std::fs::write(&staging, b"cold build").expect("write active staging"); + assert!( + !plan(&none).awaits_transient_release(&none), + "a first build orphans nothing when it publishes" + ); + + attach_fixture_text_artifact(&store, active, b"incumbent artifact"); + assert!(plan(&none).awaits_transient_release(&none)); + + std::fs::remove_file(&staging).expect("successor publication consumes staging"); + assert!(!plan(&none).awaits_transient_release(&none)); +} + /// A production store's publication pointer names its whole retained history, /// not just the active generation. `fixture_store` writes the minimal pointer, /// which is exactly the shape that hid #897: with an index of one entry, no @@ -3341,3 +3351,76 @@ fn missing_store_is_an_unpublished_plan_not_a_storage_failure() { assert!(plan.collectable_generations.is_empty()); assert!(!plan.has_collectable_work()); } + +/// A descriptor published before artifacts carried a content key keeps the +/// durable index readable and byte-identical, names nothing a reader may open +/// or retention must keep, and is replaced by the next attached artifact. +#[test] +fn a_pre_key_text_artifact_descriptor_reads_as_retired_and_is_replaced() { + let (store, generations) = fixture_store(1); + let active = generations.last().expect("active generation"); + let mut pointer = read_active_pointer(store.path()).expect("fixture pointer"); + let retired = text_artifact_for_bytes(&active.id, b"pre-key artifact"); + write_text_artifact(&store, &retired, b"pre-key artifact"); + let generation_id = pointer.generation_id.clone(); + let entry = pointer + .generation_index + .iter_mut() + .find(|entry| entry.generation_id == generation_id) + .expect("active entry"); + entry.text_artifact = Some(DurableTextArtifactSlotV1::Retired( + RetiredCodeTextArtifactDescriptorV1 { + generation_id: retired.generation_id.clone(), + artifact_file: retired.artifact_file.clone(), + artifact_digest: retired.artifact_digest.clone(), + artifact_size_bytes: retired.artifact_size_bytes, + }, + )); + pointer.generation_index_digest = Some( + durable_generation_index_digest( + &pointer.generation_index, + pointer.generation_index_truncated, + ) + .expect("index digest"), + ); + write_active_pointer(store.path(), "pre-key-fixture", &pointer).expect("write pointer"); + let stored = std::fs::read_to_string(store.path().join(ACTIVE_POINTER_FILE)) + .expect("read stored pointer"); + assert!( + stored.contains(&retired.artifact_file) && !stored.contains("content_key"), + "the fixture stores the pre-key descriptor shape" + ); + let reread = read_active_pointer(store.path()).expect("pre-key pointer stays readable"); + assert_eq!(reread, pointer); + assert!( + reread + .generation_index + .iter() + .all(|entry| entry.text_artifact().is_none()), + "a pre-key descriptor names no openable artifact" + ); + let report = run_code_generation_retention( + store.path(), + &BTreeSet::new(), + CodeGenerationRetentionModeV1::Apply, + UtcMicros(99), + None, + ) + .expect("retention over a pre-key descriptor"); + assert!( + report + .deleted_text_artifacts + .iter() + .any(|candidate| candidate.artifact_file == retired.artifact_file), + "the file a pre-key descriptor names is collected" + ); + let attached = attach_fixture_text_artifact(&store, active, b"current artifact"); + let after = read_active_pointer(store.path()).expect("pointer after attach"); + assert!( + after + .generation_index + .iter() + .any(|entry| entry.text_artifact() == Some(&attached)), + "attaching a current artifact replaces the retired slot" + ); +} diff --git a/crates/tracedecay-code-index-retention/src/code_index_generations/text_artifacts.rs b/crates/tracedecay-code-index-retention/src/code_index_generations/text_artifacts.rs index b1a938e25d..50921c8e34 100644 --- a/crates/tracedecay-code-index-retention/src/code_index_generations/text_artifacts.rs +++ b/crates/tracedecay-code-index-retention/src/code_index_generations/text_artifacts.rs @@ -26,12 +26,13 @@ use super::{ MAX_CODE_TEXT_ARTIFACT_RETENTION_BATCH_V1, MAX_DURABLE_PUBLICATION_POINTER_BYTES_V1, MAX_TRANSACTION_BYTES, TEXT_ARTIFACT_QUARANTINE_DIRECTORY, TEXT_ARTIFACT_RECEIPT_SCHEMA, TEXT_ARTIFACT_RECEIPTS_DIRECTORY, TEXT_ARTIFACT_TRANSACTION_FILE, - TEXT_ARTIFACT_TRANSACTION_SCHEMA, code_text_artifacts_root, durable_generation_index_digest, - generation_file_digest, observe_cancel, open_file_sha256_hex_cancellable, - path_still_names_open_file, read_active_pointer, read_optional_active_pointer, - regular_file_exists, remove_empty_stage_root, retain_bounded_generation_index_with_text_head, - sha256_file_component, storage, sync_directory, validate_durable_generation_index, - validate_sealed_generation_identity, validate_text_artifact_descriptor, + TEXT_ARTIFACT_TRANSACTION_SCHEMA, code_text_artifact_staging_root, code_text_artifacts_root, + durable_generation_index_digest, generation_file_digest, observe_cancel, + open_file_sha256_hex_cancellable, path_still_names_open_file, read_active_pointer, + read_optional_active_pointer, regular_file_exists, remove_empty_stage_root, + retain_bounded_generation_index_with_text_head, sha256_file_component, storage, sync_directory, + validate_durable_generation_index, validate_sealed_generation_identity, + validate_text_artifact_descriptor, }; pub(super) const TEXT_ARTIFACT_TRANSACTION_JOURNAL: BoundedJournalSpec< @@ -54,11 +55,6 @@ enum VerifiedTextArtifactMutationV1<'a> { sealed_identity: &'a DurableSealedCodeGenerationIdentityV1, descriptor: DurableCodeTextArtifactDescriptorV1, }, - Replace { - sealed_identity: &'a DurableSealedCodeGenerationIdentityV1, - expected: &'a DurableCodeTextArtifactDescriptorV1, - replacement: DurableCodeTextArtifactDescriptorV1, - }, Withdraw { expected: &'a DurableCodeTextArtifactDescriptorV1, }, @@ -83,25 +79,6 @@ fn mutate_verified_text_artifact_under_lock( true, ) } - VerifiedTextArtifactMutationV1::Replace { - sealed_identity, - expected, - replacement, - } => { - validate_sealed_generation_identity(sealed_identity)?; - validate_text_artifact_descriptor(expected)?; - validate_text_artifact_descriptor(replacement)?; - if expected.generation_id != replacement.generation_id { - return Err(CodeGenerationRetentionErrorV1::Conflict( - "text-artifact replacement changed generation identity".to_owned(), - )); - } - ( - replacement.generation_id.clone(), - Some(*sealed_identity), - false, - ) - } VerifiedTextArtifactMutationV1::Withdraw { expected } => { validate_text_artifact_descriptor(expected)?; (expected.generation_id.clone(), None, false) @@ -133,46 +110,24 @@ fn mutate_verified_text_artifact_under_lock( )); } match mutation { - VerifiedTextArtifactMutationV1::Attach { descriptor, .. } => { - match entry.text_artifact.as_ref() { - Some(existing) if existing == &descriptor => return Ok(pointer), - Some(_) => { - return Err(CodeGenerationRetentionErrorV1::Conflict( - "sealed generation already names a different text artifact".to_owned(), - )); - } - None => entry.text_artifact = Some(descriptor), - } - } - VerifiedTextArtifactMutationV1::Replace { - expected, - replacement, - .. - } => match entry.text_artifact.as_ref() { - Some(existing) if existing == &replacement => return Ok(pointer), - Some(existing) if existing == expected => entry.text_artifact = Some(replacement), + VerifiedTextArtifactMutationV1::Attach { descriptor, .. } => match entry.text_artifact() { + Some(existing) if existing == &descriptor => return Ok(pointer), Some(_) => { return Err(CodeGenerationRetentionErrorV1::Conflict( - "sealed generation names a newer text artifact".to_owned(), + "sealed generation already names a different text artifact".to_owned(), )); } - None => { + None => entry.text_artifact = Some(descriptor.into()), + }, + VerifiedTextArtifactMutationV1::Withdraw { expected } => match entry.text_artifact() { + Some(existing) if existing == expected => entry.text_artifact = None, + Some(_) => { return Err(CodeGenerationRetentionErrorV1::Conflict( - "sealed generation has no text artifact to replace".to_owned(), + "sealed generation names a newer text artifact".to_owned(), )); } + None => return Ok(pointer), }, - VerifiedTextArtifactMutationV1::Withdraw { expected } => { - match entry.text_artifact.as_ref() { - Some(existing) if existing == expected => entry.text_artifact = None, - Some(_) => { - return Err(CodeGenerationRetentionErrorV1::Conflict( - "sealed generation names a newer text artifact".to_owned(), - )); - } - None => return Ok(pointer), - } - } } if retain_text_head { let active_generation_id = pointer.generation_id.clone(); @@ -238,26 +193,6 @@ pub fn attach_verified_text_artifact_under_lock( ) } -/// Atomically replace one exact text-artifact descriptor without clearing -/// the generation's readable attachment between versions. -pub fn replace_verified_text_artifact_under_lock( - lock: &CodeGenerationStoreLockV1, - expected_pointer: &DurablePublicationPointerV1, - sealed_identity: &DurableSealedCodeGenerationIdentityV1, - expected_descriptor: &DurableCodeTextArtifactDescriptorV1, - replacement: DurableCodeTextArtifactDescriptorV1, -) -> Result { - mutate_verified_text_artifact_under_lock( - lock, - expected_pointer, - VerifiedTextArtifactMutationV1::Replace { - sealed_identity, - expected: expected_descriptor, - replacement, - }, - ) -} - /// Withdraw one exact derived text-artifact attachment under the canonical /// generation-store lock. /// @@ -279,11 +214,13 @@ pub fn withdraw_verified_text_artifact_under_lock( ) } -/// Select one bounded page of derived text-artifact debris from the canonical -/// artifact root. The durable generation index is the only completed-artifact -/// liveness authority. An in-progress builder names its staging database with -/// the sealed generation digest, so every staging path whose source digest is -/// still retained is preserved rather than guessed dead by wall-clock age. +/// Select one bounded page of derived text-artifact debris. Completed +/// artifacts live in the project's shared directory and stay live while any +/// scope's durable index, live or quarantined, names them; this scope's own +/// descriptors are verified before anything is planned. An in-progress +/// builder names its staging database, in this scope's staging directory, +/// with the sealed generation digest, so the active generation's staging is +/// preserved rather than guessed dead by wall-clock age. pub(super) fn plan_collectable_text_artifacts_cancellable( store_root: &Path, active_pointer: Option<&DurablePublicationPointerV1>, @@ -294,18 +231,22 @@ pub(super) fn plan_collectable_text_artifacts_cancellable( return Err(CodeGenerationRetentionErrorV1::Cancelled); } // An unpublished store (`None`) has no durable index and no resumable - // build authority: every completed, staging, sidecar, and corrupt file - // under its artifact root is crash debris and therefore a candidate. + // build authority: every staging and sidecar file in its + // staging directory is crash debris and therefore a candidate. let mut referenced = BTreeMap::new(); for entry in active_pointer .map(|pointer| pointer.generation_index.as_slice()) .unwrap_or_default() { - if let Some(descriptor) = entry.text_artifact.as_ref() { + if let Some(descriptor) = entry.text_artifact() { validate_text_artifact_descriptor(descriptor)?; + // Generations that sealed the same content name one artifact. if referenced .insert(descriptor.artifact_file.as_str(), descriptor) - .is_some_and(|prior| prior != descriptor) + .is_some_and(|prior| { + (&prior.artifact_digest, prior.artifact_size_bytes) + != (&descriptor.artifact_digest, descriptor.artifact_size_bytes) + }) { return Err(CodeGenerationRetentionErrorV1::UnsafeState( "publication-pointer text artifact path has conflicting identity".to_owned(), @@ -313,6 +254,7 @@ pub(super) fn plan_collectable_text_artifacts_cancellable( } } } + let marked = scope_text_artifact_marks(store_root)?; let active_staging_source = active_pointer .map(|pointer| { generation_file_digest(&pointer.generation_file).ok_or_else(|| { @@ -325,29 +267,14 @@ pub(super) fn plan_collectable_text_artifacts_cancellable( .transpose()?; let root = code_text_artifacts_root(store_root); - let root_metadata = match std::fs::symlink_metadata(&root) { - Ok(metadata) => metadata, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => { - if referenced.is_empty() { - return Ok(CodeTextArtifactRetentionInventoryV1 { - candidates: Vec::new(), - unique_bytes: 0, - }); - } - return Err(CodeGenerationRetentionErrorV1::UnsafeState( - "durable publication pointer references text artifacts but their root is missing" - .to_owned(), - )); - } - Err(error) => return Err(storage(error)), - }; - if !root_metadata.file_type().is_dir() { - return Err(CodeGenerationRetentionErrorV1::UnsafeState(format!( - "code text artifact root '{}' is not a directory", - root.display() - ))); + let staging_root = code_text_artifact_staging_root(store_root); + let root_present = private_directory_exists(&root)?; + if !root_present && !referenced.is_empty() { + return Err(CodeGenerationRetentionErrorV1::UnsafeState( + "durable publication pointer references text artifacts but their root is missing" + .to_owned(), + )); } - // The index can name at most 32 completed artifacts, and only the active // generation has a resumable build authority. Verify the completed // liveness set directly before scanning debris, so an early candidate page @@ -369,12 +296,19 @@ pub(super) fn plan_collectable_text_artifacts_cancellable( descriptor.artifact_size_bytes, ); } + let active_entry_has_artifact = active_pointer.is_some_and(|pointer| { + pointer.generation_index.iter().any(|entry| { + entry.generation_id == pointer.generation_id && entry.text_artifact().is_some() + }) + }); + let mut active_text_replacement_in_flight = false; if let Some(active_staging_source) = active_staging_source { let active_staging_file = format!(".text-artifact-{active_staging_source}.staging"); - let active_staging_path = root.join(&active_staging_file); + let active_staging_path = staging_root.join(&active_staging_file); match std::fs::symlink_metadata(&active_staging_path) { Ok(metadata) if metadata.file_type().is_file() => { inventory.insert(active_staging_file, metadata.len()); + active_text_replacement_in_flight = active_entry_has_artifact; } Ok(_) => { return Err(CodeGenerationRetentionErrorV1::UnsafeState(format!( @@ -387,120 +321,110 @@ pub(super) fn plan_collectable_text_artifacts_cancellable( } } - let mut entries = std::fs::read_dir(&root).map_err(storage)?; let mut candidates = BTreeMap::new(); - for _ in 0..MAX_CODE_TEXT_ARTIFACT_INVENTORY_ENTRIES_V1 { - if observe_cancel(is_cancelled) { - return Err(CodeGenerationRetentionErrorV1::Cancelled); + let mut remaining = MAX_CODE_TEXT_ARTIFACT_INVENTORY_ENTRIES_V1; + for (directory, shared) in [(&root, true), (&staging_root, false)] { + if !private_directory_exists(directory)? { + continue; } - let Some(entry) = entries.next() else { - break; - }; - let entry = entry.map_err(storage)?; - let file_name = entry.file_name().into_string().map_err(|_| { - CodeGenerationRetentionErrorV1::UnsafeState( - "code text artifact inventory filename is not UTF-8".to_owned(), - ) - })?; - let path = entry.path(); - // This inventory reads the artifact root without the generation-store - // lock, so an entry the listing just named can already be gone: the - // text-artifact builder retires a `.staging` family (the staging - // database and its `-journal`/`-wal`/`-shm` sidecars) under that lock - // while this scan runs. A vanished entry is reclaimed, which is what - // this inventory would have planned anyway, so it is not a candidate - // and not a failure. Failing the plan here turned every publish that - // raced a maintenance tick into a loud `retention_plan_failed` pass - // (master run 35422072661, `Storage("No such file or directory")`). - // A completed artifact the durable index *references* is verified - // above, before this scan, and stays fail-closed if it disappears. - let metadata = match std::fs::symlink_metadata(&path) { - Ok(metadata) => metadata, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, - Err(error) => return Err(storage(error)), - }; - if !metadata.file_type().is_file() { - return Err(CodeGenerationRetentionErrorV1::UnsafeState(format!( - "code text artifact inventory path '{}' is not a regular file", - path.display() - ))); - } - - let candidate = if let Some(digest) = completed_text_artifact_digest(&file_name) { - if referenced.contains_key(file_name.as_str()) { - None - } else { - // A completed SQLite artifact can never be empty. A zero-byte - // file at its final content-addressed path is the only state - // left when publication created the destination but failed - // before writing any bytes. It contains no recoverable data, - // so retain the regular-file/inode/size checks while allowing - // retention to collect that publish-crash placeholder. Every - // non-empty candidate still requires its full content proof. - let candidate_verification = if metadata.len() == 0 { - GenerationDigestVerificationV1::MetadataOnly - } else { - verification - }; - if !verify_unreferenced_completed_text_artifact( - &path, - digest, - metadata.len(), - candidate_verification, - is_cancelled, - )? { - continue; - } - Some(CodeTextArtifactRetentionCandidateV1 { - artifact_file: file_name, - kind: CodeTextArtifactRetentionKindV1::Completed, - size_bytes: metadata.len(), - }) + let mut entries = std::fs::read_dir(directory).map_err(storage)?; + while remaining > 0 && candidates.len() < MAX_CODE_TEXT_ARTIFACT_RETENTION_BATCH_V1 { + remaining -= 1; + if observe_cancel(is_cancelled) { + return Err(CodeGenerationRetentionErrorV1::Cancelled); } - } else if let Some(source_digest) = staging_text_artifact_source_digest(&file_name) { - if Some(source_digest) == active_staging_source { - None - } else { - Some(CodeTextArtifactRetentionCandidateV1 { - artifact_file: file_name, - kind: CodeTextArtifactRetentionKindV1::Staging, - size_bytes: metadata.len(), - }) + let Some(entry) = entries.next() else { + break; + }; + let entry = entry.map_err(storage)?; + let file_name = entry.file_name().into_string().map_err(|_| { + CodeGenerationRetentionErrorV1::UnsafeState( + "code text artifact inventory filename is not UTF-8".to_owned(), + ) + })?; + let path = entry.path(); + // This inventory reads the artifact directories without the + // generation-store lock, so an entry the listing just named can + // already be gone: the text-artifact builder retires a `.staging` + // family (the staging database and its `-journal`/`-wal`/`-shm` + // sidecars) under that lock while this scan runs. A vanished entry + // is reclaimed, which is what this inventory would have planned + // anyway, so it is not a candidate and not a failure. Failing the + // plan here turned every publish that raced a maintenance tick + // into a loud `retention_plan_failed` pass (master run + // 35422072661, `Storage("No such file or directory")`). A + // completed artifact this scope's index *references* is verified + // above, before this scan, and stays fail-closed if it disappears. + let metadata = match std::fs::symlink_metadata(&path) { + Ok(metadata) => metadata, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, + Err(error) => return Err(storage(error)), + }; + if !metadata.file_type().is_file() { + return Err(CodeGenerationRetentionErrorV1::UnsafeState(format!( + "code text artifact inventory path '{}' is not a regular file", + path.display() + ))); } - } else if let Some(source_digest) = staging_sidecar_text_artifact_source_digest(&file_name) - { - // SQLite sidecars of the staging database (`-journal`, `-wal`, - // `-shm`). They live and die with their staging file: the active - // build's sidecars are the builder's property, while an orphaned - // staging file's sidecars are the same crash debris it is. Before - // this arm they were "unrecognized regular file" failures that - // poisoned every retention plan for the scope. - if Some(source_digest) == active_staging_source { - None - } else { - Some(CodeTextArtifactRetentionCandidateV1 { + let staging_source = staging_text_artifact_source_digest(&file_name) + .or_else(|| staging_sidecar_text_artifact_source_digest(&file_name)); + let completed = completed_text_artifact_digest(&file_name); + let candidate = match (shared, staging_source, completed) { + (true, None, Some(digest)) => { + if marked.contains(file_name.as_str()) { + None + } else { + // A completed SQLite artifact can never be empty. A + // zero-byte file at its final content-addressed path + // is the only state left when publication created the + // destination but failed before writing any bytes. It + // contains no recoverable data, so retain the + // regular-file/inode/size checks while allowing + // retention to collect that publish-crash placeholder. + // Every non-empty candidate still requires its full + // content proof. + let candidate_verification = if metadata.len() == 0 { + GenerationDigestVerificationV1::MetadataOnly + } else { + verification + }; + if !verify_unreferenced_completed_text_artifact( + &path, + digest, + metadata.len(), + candidate_verification, + is_cancelled, + )? { + continue; + } + Some(CodeTextArtifactRetentionCandidateV1 { + artifact_file: file_name, + kind: CodeTextArtifactRetentionKindV1::Completed, + size_bytes: metadata.len(), + }) + } + } + // The active build's staging family is the builder's + // property; any other staging database or sidecar is crash + // debris. + (false, Some(source_digest), _) if Some(source_digest) == active_staging_source => { + None + } + (false, Some(_), _) => Some(CodeTextArtifactRetentionCandidateV1 { artifact_file: file_name, kind: CodeTextArtifactRetentionKindV1::Staging, size_bytes: metadata.len(), - }) - } - } else if is_corrupt_text_artifact_file(&file_name) { - Some(CodeTextArtifactRetentionCandidateV1 { - artifact_file: file_name, - kind: CodeTextArtifactRetentionKindV1::Corrupt, - size_bytes: metadata.len(), - }) - } else { - return Err(CodeGenerationRetentionErrorV1::UnsafeState(format!( - "code text artifact inventory contains unrecognized regular file '{}'", - path.display() - ))); - }; - if let Some(candidate) = candidate { - inventory.insert(candidate.artifact_file.clone(), candidate.size_bytes); - candidates.insert(candidate.artifact_file.clone(), candidate); - if candidates.len() == MAX_CODE_TEXT_ARTIFACT_RETENTION_BATCH_V1 { - break; + }), + _ => { + return Err(CodeGenerationRetentionErrorV1::UnsafeState(format!( + "code text artifact inventory contains unrecognized regular file '{}'", + path.display() + ))); + } + }; + if let Some(candidate) = candidate { + inventory.insert(candidate.artifact_file.clone(), candidate.size_bytes); + candidates.insert(candidate.artifact_file.clone(), candidate); } } } @@ -509,9 +433,83 @@ pub(super) fn plan_collectable_text_artifacts_cancellable( unique_bytes: inventory .values() .fold(0_u64, |total, bytes| total.saturating_add(*bytes)), + active_text_replacement_in_flight, }) } +/// A completed artifact some scope of this store's project published under +/// `content_key` and whose file is still in the shared directory. The file +/// is not verified here: the caller opens it against its content address +/// and its own projection, and rebuilds when that fails. Call it under the +/// project lock held shared, so no retention collects the file before the +/// caller's descriptor is durable. +pub fn find_shared_text_artifact( + store_root: &Path, + content_key: &tracedecay_domain::ManifestDigest, +) -> Result, CodeGenerationRetentionErrorV1> { + let root = code_text_artifacts_root(store_root); + for scope in super::segment_scope_roots(store_root)? { + let Some(pointer) = read_optional_active_pointer(&scope)? else { + continue; + }; + for descriptor in pointer + .generation_index + .iter() + .filter_map(|entry| entry.text_artifact()) + .filter(|descriptor| descriptor.content_key == *content_key) + { + validate_text_artifact_descriptor(descriptor)?; + if regular_file_exists(&root.join(&descriptor.artifact_file))? { + return Ok(Some(descriptor.clone())); + } + } + } + Ok(None) +} + +/// Every completed artifact a scope of this store's project names: its live +/// worktree scopes and any scope collection quarantined, which a recovery may +/// restore. A store outside a shared project marks only itself. +fn scope_text_artifact_marks( + store_root: &Path, +) -> Result, CodeGenerationRetentionErrorV1> { + let mut marked = BTreeSet::new(); + for scope in super::segment_scope_roots(store_root)? { + let Some(pointer) = read_optional_active_pointer(&scope)? else { + continue; + }; + validate_durable_generation_index(&pointer)?; + marked.extend( + pointer + .generation_index + .iter() + .filter_map(|entry| entry.text_artifact()) + .map(|descriptor| descriptor.artifact_file.clone()), + ); + } + Ok(marked) +} + +fn private_directory_exists(path: &Path) -> Result { + match std::fs::symlink_metadata(path) { + Ok(metadata) if metadata.file_type().is_dir() => Ok(true), + Ok(_) => Err(CodeGenerationRetentionErrorV1::UnsafeState(format!( + "code text artifact directory '{}' is not a directory", + path.display() + ))), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(storage(error)), + } +} + +/// The directory a candidate of `kind` lives in. +fn candidate_root(store_root: &Path, kind: CodeTextArtifactRetentionKindV1) -> PathBuf { + match kind { + CodeTextArtifactRetentionKindV1::Completed => code_text_artifacts_root(store_root), + CodeTextArtifactRetentionKindV1::Staging => code_text_artifact_staging_root(store_root), + } +} + pub(super) fn completed_text_artifact_digest(file_name: &str) -> Option<&str> { file_name .strip_prefix("text-artifact-")? @@ -538,17 +536,6 @@ pub(super) fn staging_sidecar_text_artifact_source_digest(file_name: &str) -> Op is_lowercase_hex(digest, 64).then_some(digest) } -pub(super) fn is_corrupt_text_artifact_file(file_name: &str) -> bool { - let Some(value) = file_name.strip_prefix("text-artifact-") else { - return false; - }; - let Some((digest, suffix)) = value.split_once(".corrupt-") else { - return false; - }; - let digest = digest.strip_suffix(".bin").unwrap_or(digest); - !suffix.is_empty() && is_lowercase_hex(digest, 64) -} - pub(super) fn verify_completed_text_artifact( path: &Path, descriptor: &DurableCodeTextArtifactDescriptorV1, @@ -805,9 +792,6 @@ pub(super) fn validate_text_artifact_candidate( staging_text_artifact_source_digest(&candidate.artifact_file).is_some() || staging_sidecar_text_artifact_source_digest(&candidate.artifact_file).is_some() } - CodeTextArtifactRetentionKindV1::Corrupt => { - is_corrupt_text_artifact_file(&candidate.artifact_file) - } }; if !direct_name || candidate.artifact_file.contains(['/', '\\']) || !kind_matches_name { return Err(CodeGenerationRetentionErrorV1::UnsafeState( @@ -830,7 +814,6 @@ pub(super) fn stage_collectable_text_artifacts_cancellable( transaction: &CodeTextArtifactRetentionTransactionV1, is_cancelled: &dyn Fn() -> bool, ) -> Result<(), CodeGenerationRetentionErrorV1> { - let artifacts_root = code_text_artifacts_root(store_root); let stage_root = text_artifact_transaction_stage_root(store_root, &transaction.receipt); std::fs::create_dir_all(&stage_root).map_err(storage)?; sync_directory(stage_root.parent().ok_or_else(|| { @@ -843,6 +826,7 @@ pub(super) fn stage_collectable_text_artifacts_cancellable( return Err(CodeGenerationRetentionErrorV1::Cancelled); } validate_text_artifact_candidate(candidate)?; + let artifacts_root = candidate_root(store_root, candidate.kind); let source = artifacts_root.join(&candidate.artifact_file); let staged = stage_root.join(&candidate.artifact_file); match (regular_file_exists(&source)?, regular_file_exists(&staged)?) { @@ -914,9 +898,9 @@ pub(super) fn rollback_staged_text_artifact_transaction( store_root: &Path, transaction: &CodeTextArtifactRetentionTransactionV1, ) -> Result<(), CodeGenerationRetentionErrorV1> { - let artifacts_root = code_text_artifacts_root(store_root); let stage_root = text_artifact_transaction_stage_root(store_root, &transaction.receipt); for candidate in &transaction.receipt.deleted_artifacts { + let artifacts_root = candidate_root(store_root, candidate.kind); let source = artifacts_root.join(&candidate.artifact_file); let staged = stage_root.join(&candidate.artifact_file); match (regular_file_exists(&source)?, regular_file_exists(&staged)?) { @@ -948,10 +932,9 @@ pub(super) fn cleanup_committed_text_artifact_transaction( transaction: &CodeTextArtifactRetentionTransactionV1, ) -> Result<(), CodeGenerationRetentionErrorV1> { ensure_text_artifact_transaction_liveness(store_root, transaction)?; - let artifacts_root = code_text_artifacts_root(store_root); let stage_root = text_artifact_transaction_stage_root(store_root, &transaction.receipt); for candidate in &transaction.receipt.deleted_artifacts { - let source = artifacts_root.join(&candidate.artifact_file); + let source = candidate_root(store_root, candidate.kind).join(&candidate.artifact_file); if regular_file_exists(&source)? { return Err(CodeGenerationRetentionErrorV1::UnsafeState(format!( "text-artifact receipt is durable but '{}' returned to its source root", @@ -971,25 +954,17 @@ pub(super) fn ensure_text_artifact_transaction_liveness( store_root: &Path, transaction: &CodeTextArtifactRetentionTransactionV1, ) -> Result<(), CodeGenerationRetentionErrorV1> { - // Liveness is proven against the *current* pointer: a publish may have - // landed since the transaction was staged (including the first publish - // into a previously unpublished store), and no durable descriptor target - // it names may be removed. - let Some(current) = read_optional_active_pointer(store_root)? else { - return Ok(()); - }; - validate_durable_generation_index(¤t)?; - let deleted = transaction + // Liveness is proven against every scope's *current* pointer: a publish + // may have landed since the transaction was staged, in this scope or in + // a sibling that shares the completed artifact, and no durable + // descriptor target any of them names may be removed. + let marked = scope_text_artifact_marks(store_root)?; + if transaction .receipt .deleted_artifacts .iter() - .map(|candidate| candidate.artifact_file.as_str()) - .collect::>(); - if current - .generation_index - .iter() - .filter_map(|entry| entry.text_artifact.as_ref()) - .any(|descriptor| deleted.contains(descriptor.artifact_file.as_str())) + .filter(|candidate| candidate.kind != CodeTextArtifactRetentionKindV1::Staging) + .any(|candidate| marked.contains(candidate.artifact_file.as_str())) { return Err(CodeGenerationRetentionErrorV1::UnsafeState( "text-artifact retention recovery would remove a durable descriptor target".to_owned(), diff --git a/crates/tracedecay-code-index-runtime/src/code_index_executor.rs b/crates/tracedecay-code-index-runtime/src/code_index_executor.rs index a45245ff44..dae985fa9d 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_executor.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_executor.rs @@ -1477,44 +1477,6 @@ where return unavailable(code_search::CodeIndexSearchUnavailableReasonV1::Internal); } } - // Clone backfill is retained-worker work after the seat. Kick that - // wake before any inline slice so a quiet daemon does not strand - // the successor on this request thread. Match the admission, - // terminal Corrupt must fail closed before the inline slice. - match schedulers.request_query_background_reconcile(&scope).await { - code_index_scheduler::CodeIndexReconcileAdmissionV1::Accepted - | code_index_scheduler::CodeIndexReconcileAdmissionV1::Unavailable => {} - code_index_scheduler::CodeIndexReconcileAdmissionV1::PublicationAuthorityCorrupt( - _, - ) => { - return unavailable( - code_search::CodeIndexSearchUnavailableReasonV1::CorruptionResetRequired, - ); - } - } - match generation.finish_clone_similarity_warmup_for_request(control.as_ref()) { - Ok(code_index_scheduler::CloneSimilarityWarmupForRequestV1::Ready) => {} - Ok(code_index_scheduler::CloneSimilarityWarmupForRequestV1::Pending) => { - // One bounded slice ran; retained worker owns the rest. - // Surface warming, not a hard GenerationUnavailable miss. - return unavailable( - code_search::CodeIndexSearchUnavailableReasonV1::GenerationUnverified, - ); - } - // A contended or already-retired staging artifact leaves this - // generation's clone projection unfinished. That is the same - // state `Pending` reports above, so it keeps `Pending`'s - // retryable verdict; `Internal` told callers never to retry a - // window that resolves itself within one background pass. - Err(RetrievalPortError::AuthorityUnavailable(_)) => { - return unavailable( - code_search::CodeIndexSearchUnavailableReasonV1::GenerationUnverified, - ); - } - Err(_) => { - return unavailable(code_search::CodeIndexSearchUnavailableReasonV1::Internal); - } - } let owners = match generation.production_query_owners_with_budget( &code_index_scheduler::queries::maximum_retrieval_budget(), ) { @@ -1689,38 +1651,6 @@ where return unavailable(code_search::CodeIndexSearchUnavailableReasonV1::Internal); } } - match schedulers.request_query_background_reconcile(&scope).await { - code_index_scheduler::CodeIndexReconcileAdmissionV1::Accepted - | code_index_scheduler::CodeIndexReconcileAdmissionV1::Unavailable => {} - code_index_scheduler::CodeIndexReconcileAdmissionV1::PublicationAuthorityCorrupt( - _, - ) => { - return unavailable( - code_search::CodeIndexSearchUnavailableReasonV1::CorruptionResetRequired, - ); - } - } - match generation.finish_clone_similarity_warmup_for_request(control.as_ref()) { - Ok(code_index_scheduler::CloneSimilarityWarmupForRequestV1::Ready) => {} - Ok(code_index_scheduler::CloneSimilarityWarmupForRequestV1::Pending) => { - return unavailable( - code_search::CodeIndexSearchUnavailableReasonV1::GenerationUnverified, - ); - } - // A contended or already-retired staging artifact leaves this - // generation's clone projection unfinished. That is the same - // state `Pending` reports above, so it keeps `Pending`'s - // retryable verdict; `Internal` told callers never to retry a - // window that resolves itself within one background pass. - Err(RetrievalPortError::AuthorityUnavailable(_)) => { - return unavailable( - code_search::CodeIndexSearchUnavailableReasonV1::GenerationUnverified, - ); - } - Err(_) => { - return unavailable(code_search::CodeIndexSearchUnavailableReasonV1::Internal); - } - } let owners = match generation.production_query_owners_with_budget( &code_index_scheduler::queries::maximum_retrieval_budget(), ) { diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler.rs index 788aa597d1..601ebfa87b 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler.rs @@ -8,7 +8,7 @@ use std::time::Duration; use tracedecay_domain::{ ContentDigest, FileOccurrenceId, ManifestDigest, ProjectionKeyV1, ProjectionKindV1, RepositoryId, SanitizationReceiptId, SanitizedCodeFileV1, SnapshotFileDispositionV1, - WorktreeId, canonical_text::sha256_hex, + canonical_text::sha256_hex, }; use crate::code_index::chunks::content_digest; @@ -66,9 +66,13 @@ where .map_err(|error| CodeIndexSchedulerErrorV1::Identity(error.to_string())) } +/// File (and so symbol) occurrences name repository content, not the +/// worktree holding it: linked worktrees sealing identical files mint +/// identical occurrences and share every artifact derived from them. Each +/// worktree's manifest and generation bind its snapshot authority, and every +/// read selects its exact worktree route before resolving an occurrence. fn file_occurrence_id( repository: &RepositoryId, - worktree: &WorktreeId, logical_path: &str, digest: &ContentDigest, receipt: &SanitizationReceiptId, @@ -77,9 +81,8 @@ fn file_occurrence_id( "file.daemon.{}", sha256_hex( format!( - "{}\0{}\0{logical_path}\0{}\0{}", + "{}\0{logical_path}\0{}\0{}", repository.as_str(), - worktree.as_str(), digest.as_str(), receipt.as_str(), ) @@ -90,7 +93,6 @@ fn file_occurrence_id( fn omitted_file_occurrence_id( repository: &RepositoryId, - worktree: &WorktreeId, logical_path: &str, digest: &ContentDigest, disposition: SnapshotFileDispositionV1, @@ -112,9 +114,8 @@ fn omitted_file_occurrence_id( "file.daemon.omitted.{}", sha256_hex( format!( - "{}\0{}\0{logical_path}\0{}\0{disposition}", + "{}\0{logical_path}\0{}\0{disposition}", repository.as_str(), - worktree.as_str(), digest.as_str(), ) .as_bytes() @@ -238,21 +239,19 @@ pub use reconcile::{ ReconcilePassGuard, }; pub(crate) use reconcile::{ServingSourceWitnessV1, SourceFreshnessFenceV1}; -pub use serving::{ - CloneSimilarityWarmupForRequestV1, CodeIndexBuildProgressSlotStateV1, - CodeIndexBuildProgressSlotV1, DaemonCodeTextArtifactStoreV1, LatestCodeTextGenerationV1, - LatestCompleteCodeIndexV1, ProductionCodeIndexQueryOwnersV1, -}; use serving::{ CodeGraphActivationStateV1, CodeGraphServingAuthorityV1, CodeIndexBuildProgressStateV1, CodeTextProjectionStateV1, DurableActiveSealedGenerationBindingV1, GenerationServingCachesV1, GenerationTextControlV1, TEXT_ARTIFACT_MAXIMUM_WORK_PER_ADVANCE_V1, try_publish_build_progress, }; +pub use serving::{ + CodeIndexBuildProgressSlotStateV1, CodeIndexBuildProgressSlotV1, DaemonCodeTextArtifactStoreV1, + LatestCodeTextGenerationV1, LatestCompleteCodeIndexV1, ProductionCodeIndexQueryOwnersV1, +}; #[cfg(test)] use serving::{ CodeIndexCommittedProgressSampleV1, CodeTextProjectionSlotV1, TEXT_ARTIFACT_PAGE_CHUNKS_V1, - clone_successor_source_batch_limits_from_charges, map_sealed_page_source_error, - sha256_private_file_and_size, text_artifact_admitted_build_budget, - text_artifact_builder_budget, text_artifact_resident_memory_charges, - text_artifact_source_batch_limits, + map_sealed_page_source_error, sha256_private_file_and_size, + text_artifact_admitted_build_budget, text_artifact_builder_budget, + text_artifact_resident_memory_charges, text_artifact_source_batch_limits, }; diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/activation_tests.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/activation_tests.rs index 6176720ccf..9af5c9ccd8 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/activation_tests.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/activation_tests.rs @@ -13,13 +13,19 @@ //! Plus the invariant none of the above may weaken: a corrupt sealed store //! still fails every request, with no memoized verdict. +use std::collections::BTreeSet; use std::fs; use std::path::Path; use std::process::Command; use std::sync::Arc; use tempfile::TempDir; -use tracedecay_domain::{CodeGenerationId, ProjectId, SanitizerRevision, sha256_hex_suffix}; +use tracedecay_code_index_retention::code_index_generations::{ + CodeGenerationRetentionModeV1, run_code_generation_retention, +}; +use tracedecay_domain::{ + CodeGenerationId, ProjectId, SanitizerRevision, UtcMicros, sha256_hex_suffix, +}; use tracedecay_runtime_core::path_safety::{ canonical_root_identity, plain_git_args, plain_host_path, }; @@ -497,7 +503,28 @@ fn superseded_generation_churn_never_evicts_the_pinned_active_generation() { .latest_complete() .expect("active generation after churn"); assert_eq!(served.generation().manifest().generation_id, active); - assert!(!served.exact().expect("exact admission").is_empty()); + let exact = served.exact().expect("exact admission"); + let files = &served.generation().snapshot().files; + let exact_paths: BTreeSet<&str> = exact + .iter() + .map(|admitted| { + files + .iter() + .find(|file| file.file_occurrence_id == admitted.chunk().anchor.file_occurrence_id) + .map(|file| file.logical_path.as_str()) + .expect("exact chunk names a snapshot file") + }) + .collect(); + assert_eq!(exact_paths, BTreeSet::from(["src/lib.rs"])); + let latest_body = format!("activation_revision() -> u32 {{ {} }}", revisions - 1); + assert!( + exact.iter().any(|admitted| admitted + .chunk() + .sanitized_text + .as_str() + .contains(&latest_body)), + "the active generation must serve the last published revision" + ); assert_eq!( scheduler.sealed_decode_count(), after_activation + pinned_decodes, @@ -567,3 +594,61 @@ fn a_corrupt_sealed_generation_fails_closed_on_every_request() { "the next request must repeat the full check rather than trust a verdict" ); } + +/// The decoded-generation LRU is an in-memory decode cache, never a retention +/// mark: a superseded generation it still holds loses its sealed manifest and +/// the segment only it named on the first retention pass. +#[test] +fn decoded_generation_cache_never_keeps_a_superseded_generation_on_disk() { + let project = fixture(); + let store = TempDir::new().expect("store root"); + let mut scheduler = open(project.path(), store.path()); + let superseded = publish(&mut scheduler); + write(project.path(), "src/lib.rs", 1); + let active = publish(&mut scheduler); + assert_ne!(superseded, active); + scheduler + .generation(&superseded) + .expect("pinned generation read") + .expect("superseded generation is decoded into the cache"); + + let file_count = |directory: &str| { + fs::read_dir(store.path().join(directory)) + .expect("read store directory") + .count() + }; + let segments_before = file_count("code-generation-segments-v1"); + assert_eq!(file_count("code-generations-v1"), 2); + + let report = run_code_generation_retention( + store.path(), + &BTreeSet::new(), + CodeGenerationRetentionModeV1::Apply, + UtcMicros(1), + None, + ) + .expect("apply retention"); + + assert_eq!( + report + .deleted_generations + .iter() + .map(|generation| &generation.generation_id) + .collect::>(), + vec![&superseded] + ); + assert_eq!(file_count("code-generations-v1"), 1); + assert!( + file_count("code-generation-segments-v1") < segments_before, + "the edited file's superseded segment is collected with its manifest" + ); + assert_eq!( + scheduler + .latest_complete() + .expect("active generation still serves") + .generation() + .manifest() + .generation_id, + active + ); +} diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs index 359c0ed7ce..f1cf4aa5d3 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs @@ -186,10 +186,9 @@ impl WorktreeChangeClassificationV1 { /// /// `TraceDecay` no longer writes anything into a checkout, but projects enrolled /// before the working-tree cutover may still carry a legacy -/// `.tracedecay/` directory (e.g. `enrollment.json`, see -/// `tracedecay_runtime_core::storage::identity::legacy_enrollment_marker_path`). -/// Those bytes were produced by `TraceDecay`, are never checkout content, and -/// are not indexable source. Unless the user happens to ignore +/// `.tracedecay/` directory (e.g. `enrollment.json`). Those bytes were +/// produced by `TraceDecay`, are never checkout content, and are not +/// indexable source. Unless the user happens to ignore /// `.tracedecay/`, gix reports them as untracked, so treating them as a /// worktree change makes *every* legacy checkout look permanently dirty: no /// capture can then seal an exact HEAD tree, no published generation carries a diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/git_tree_capture.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/git_tree_capture.rs index 5b195ea01a..9f9ec02b50 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/git_tree_capture.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/git_tree_capture.rs @@ -353,13 +353,8 @@ impl CodeIndexWorktreeSchedulerV1 { reason: String, ) -> Result { let digest = content_digest(raw_bytes); - let occurrence = omitted_file_occurrence_id( - &self.repository_id, - &self.worktree_id, - logical_path, - &digest, - disposition, - )?; + let occurrence = + omitted_file_occurrence_id(&self.repository_id, logical_path, &digest, disposition)?; Ok(WithheldSourceV1 { logical_path: logical_path.to_owned(), file: Some(SanitizedCodeFileV1 { @@ -401,13 +396,8 @@ impl CodeIndexWorktreeSchedulerV1 { privacy::sanitize_code_file(&descriptor.language, raw_bytes)?; let (digest, shared) = self.byte_pool.intern(sanitized_bytes); let retained_reservation = self.reserve_snapshot_memory(&digest, shared.len())?; - let occurrence = file_occurrence_id( - &self.repository_id, - &self.worktree_id, - logical_path, - &digest, - &receipt_id, - )?; + let occurrence = + file_occurrence_id(&self.repository_id, logical_path, &digest, &receipt_id)?; Ok(Some(CapturedCandidateV1 { file: SanitizedCodeFileV1 { file_occurrence_id: occurrence.clone(), diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/memory_tests.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/memory_tests.rs index 165d08e943..bd426df6f5 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/memory_tests.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/memory_tests.rs @@ -198,7 +198,23 @@ fn latest_complete_reuses_the_immutable_generation_allocation() { std::ptr::eq(first.generation(), second.generation()), "readers must share the sealed generation instead of deep-cloning it" ); - assert!(!first.exact().expect("exact chunks").is_empty()); + let exact = first.exact().expect("exact chunks"); + let files = &first.generation().snapshot().files; + for admitted in exact.iter() { + let path = files + .iter() + .find(|file| file.file_occurrence_id == admitted.chunk().anchor.file_occurrence_id) + .map(|file| file.logical_path.as_str()); + assert_eq!(path, Some("src/lib.rs")); + } + assert!( + exact.iter().any(|admitted| admitted + .chunk() + .sanitized_text + .as_str() + .contains("pub fn retained_generation() -> u32 { 1 }")), + "exact chunks must carry the committed fixture source" + ); let generation_id = first.generation().manifest().generation_id.clone(); drop(first); drop(second); diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/publication_store.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/publication_store.rs index 92cf1f063d..7db32e1700 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/publication_store.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/publication_store.rs @@ -16,11 +16,13 @@ use same_file::Handle; use sha2::{Digest, Sha256}; use tracedecay_application::code_index::DaemonCodeIndexControlV1; use tracedecay_code_index_retention::code_index_generations::{ - CodeGenerationStoreLockV1, DurableGenerationCardinalityV1, DurableGenerationIndexEntryV1, - DurablePublicationPointerV1, DurableSealedCodeGenerationIdentityV1, - MAX_DURABLE_GENERATION_INDEX_BYTES_V1, MAX_DURABLE_GENERATION_INDEX_ENTRIES_V1, - durable_generation_index_digest, retain_bounded_generation_index, - try_acquire_code_generation_store_lock, try_acquire_code_generation_store_read_lock, + CodeGenerationRetentionErrorV1, CodeGenerationStoreLockV1, DurableGenerationCardinalityV1, + DurableGenerationIndexEntryV1, DurablePublicationPointerV1, + DurableSealedCodeGenerationIdentityV1, MAX_DURABLE_GENERATION_INDEX_BYTES_V1, + MAX_DURABLE_GENERATION_INDEX_ENTRIES_V1, acquire_generation_segments_publication_lock, + code_generation_segments_root, durable_generation_index_digest, + retain_bounded_generation_index, try_acquire_code_generation_store_lock, + try_acquire_code_generation_store_read_lock, }; use tracedecay_domain::{ CodeGenerationId, ContentDigest, ManifestDigest, ProjectionBatchRequestV1, @@ -36,7 +38,7 @@ use crate::code_index::{ CodeIndexInterruptionV1, CodeIndexProductionErrorV1, CodeIndexPublicationStoreErrorV1, CodeIndexPublishedGenerationV1, SealedGenerationSegmentPublicationV1, SealedGenerationSegmentReadV1, SharedPhysicalCodeArtifactPoolV1, - UninterruptibleCodeIndexControlV1, VerifiedSealedTextGenerationMetadataV1, + VerifiedSealedTextGenerationMetadataV1, }, projection::{ ChunkProjectionDecisionV1, CodeChunkProjectionSink, ProjectionReceiptBuilderV1, @@ -469,6 +471,9 @@ pub struct DaemonCodeIndexPublicationStoreV1 { active_path: PathBuf, pub(super) generations_root: PathBuf, segments_root: PathBuf, + /// Names this store's temporaries in a segment directory other worktree + /// scopes of the project publish into as well. + segment_temporary_prefix: String, pub(super) project_root: PathBuf, expected_sanitizer_revision: SanitizerRevision, disposition: CodeIndexPublicationDispositionV1, @@ -748,8 +753,18 @@ impl DaemonCodeIndexPublicationStoreV1 { ) -> Result { let generations_root = store_root.join("code-generations-v1"); std::fs::create_dir_all(&generations_root)?; - let segments_root = store_root.join("code-generation-segments-v1"); + let segments_root = code_generation_segments_root(store_root); std::fs::create_dir_all(&segments_root)?; + let segment_temporary_prefix = format!( + "{}.{}", + store_root + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| std::io::Error::other( + "code-generation store root has no UTF-8 name" + ))?, + std::process::id() + ); let _store_lock = try_acquire_code_generation_store_lock(store_root) .map_err(|error| std::io::Error::other(error.to_string()))? .ok_or_else(|| std::io::Error::other("code-generation store has an active owner"))?; @@ -760,7 +775,7 @@ impl DaemonCodeIndexPublicationStoreV1 { store_root, project_root, )?; - Self::remove_abandoned_evidence_packs(&segments_root) + Self::remove_abandoned_evidence_packs(&segments_root, store_root) .map_err(|error| std::io::Error::other(error.to_string()))?; Ok(Self { cache: Arc::new(DecodedGenerationCacheV1::default()), @@ -772,6 +787,7 @@ impl DaemonCodeIndexPublicationStoreV1 { active_path: store_root.join("active-code-generation-v1.json"), generations_root, segments_root, + segment_temporary_prefix, project_root: project_root.to_path_buf(), expected_sanitizer_revision, disposition: CodeIndexPublicationDispositionV1::Active, @@ -910,9 +926,19 @@ impl DaemonCodeIndexPublicationStoreV1 { .ok_or_else(|| Self::unavailable("generation store read lock is contended")) } + /// Only this scope's temporaries: the store lock held by the caller proves + /// no publication of this scope is in flight, and other scopes' are theirs. fn remove_abandoned_evidence_packs( segments_root: &Path, + store_root: &Path, ) -> Result<(), CodeIndexPublicationStoreErrorV1> { + let prefix = format!( + ".evidence-pack-publication.{}.", + store_root + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| Self::unavailable("code-generation store root has no UTF-8 name"))? + ); let mut removed = false; for entry in std::fs::read_dir(segments_root).map_err(Self::unavailable)? { let entry = entry.map_err(Self::unavailable)?; @@ -920,7 +946,7 @@ impl DaemonCodeIndexPublicationStoreV1 { let Some(name) = name.to_str() else { continue; }; - if !name.starts_with(".evidence-pack-publication.") || !name.ends_with(".tmp") { + if !name.starts_with(&prefix) || !name.ends_with(".tmp") { continue; } let metadata = entry.path().symlink_metadata().map_err(Self::unavailable)?; @@ -998,8 +1024,7 @@ impl DaemonCodeIndexPublicationStoreV1 { } let temporary_path = self.segments_root.join(format!( ".segment-publication.{}.{}.tmp", - std::process::id(), - digest_hex + self.segment_temporary_prefix, digest_hex )); match temporary_path.symlink_metadata() { Ok(metadata) if metadata.file_type().is_file() => { @@ -1602,8 +1627,11 @@ impl DaemonCodeIndexPublicationStoreV1 { )); } match CodeIndexPublishedGenerationV1::partitioned_text_metadata(&bytes) { - Ok(metadata) => Ok(metadata), - Err(CodeIndexProductionErrorV1::SourceCommitmentsUnavailable) => Ok(None), + Ok(metadata) => Ok(Some(metadata)), + Err( + CodeIndexProductionErrorV1::SourceCommitmentsUnavailable + | CodeIndexProductionErrorV1::SupersededSealedGenerationRevision(_), + ) => Ok(None), Err(error) => Err(Self::corruption(error.to_string())), } } @@ -1616,51 +1644,25 @@ impl DaemonCodeIndexPublicationStoreV1 { expected_file_digest: &ManifestDigest, lifetime_lock: CodeGenerationStoreLockV1, ) -> Result, CodeIndexProductionErrorV1> { - let monolithic = match CodeIndexPublishedGenerationV1::decode_sealed_seek_reader( - &mut *file, - admitted_len, - Some(expected_file_digest), - &UninterruptibleCodeIndexControlV1, - ) { - Ok(monolithic) => monolithic, - // A generation is a pure function of its source tree, so an - // envelope revision this build no longer reads is refused rather - // than repaired: abstain the way an incompatible generation does - // and let the scheduler rebuild it. - Err(CodeIndexProductionErrorV1::SupersededSealedGenerationRevision(revision)) => { - tracing::warn!( - target: "tracedecay::code_index", - sealed_format_revision = revision, - "{}", - CodeIndexProductionErrorV1::SupersededSealedGenerationRevision(revision) - ); - return Ok(None); - } - Err(error @ CodeIndexProductionErrorV1::SealedRowContractRefused { revision, .. }) => { - tracing::warn!( - target: "tracedecay::code_index", - sealed_format_revision = revision, - "{error}" - ); - return Ok(None); - } - Err(CodeIndexProductionErrorV1::SourceCommitmentsUnavailable) => return Ok(None), - Err(error) => return Err(error), - }; - if monolithic.is_some() { - return Ok(monolithic); - } file.seek(SeekFrom::Start(0)).map_err(|error| { CodeIndexProductionErrorV1::Contract(format!( "sealed generation manifest seek failed: {error}" )) })?; let mut manifest = Vec::new(); - file.read_to_end(&mut manifest).map_err(|error| { - CodeIndexProductionErrorV1::Contract(format!( - "sealed generation manifest read failed: {error}" - )) - })?; + Read::by_ref(file) + .take(admitted_len) + .read_to_end(&mut manifest) + .map_err(|error| { + CodeIndexProductionErrorV1::Contract(format!( + "sealed generation manifest read failed: {error}" + )) + })?; + if u64::try_from(manifest.len()).ok() != Some(admitted_len) { + return Err(CodeIndexProductionErrorV1::Contract( + "sealed generation length does not match its admitted length".to_owned(), + )); + } if Self::state_digest(&manifest) != expected_file_digest.as_str() { return Err(CodeIndexProductionErrorV1::Contract( "sealed generation manifest filename digest does not match its bytes".to_owned(), @@ -1696,11 +1698,10 @@ impl DaemonCodeIndexPublicationStoreV1 { }, ) { Err(CodeIndexProductionErrorV1::SourceCommitmentsUnavailable) => Ok(None), - // The manifest revision is refused for the same reason a retired - // monolithic envelope is, and on the same terms: the generation is - // re-derivable from its source tree, so the scheduler rebuilds it - // instead of treating a shape this build no longer writes as - // corruption. + // A generation is a pure function of its source tree, so a + // manifest revision this build no longer reads is refused rather + // than repaired: abstain and let the scheduler rebuild it instead + // of treating a shape this build no longer writes as corruption. Err(CodeIndexProductionErrorV1::SupersededSealedGenerationRevision(revision)) => { tracing::warn!( target: "tracedecay::code_index", @@ -1710,7 +1711,7 @@ impl DaemonCodeIndexPublicationStoreV1 { ); Ok(None) } - result => result, + result => result.map(Some), } } @@ -2358,9 +2359,20 @@ impl CodeIndexAtomicPublicationPort for DaemonCodeIndexPublicationStoreV1 { Err(error) => return Err(Self::unavailable(error)), } let mut temporary = TemporaryGenerationFileV1::new(temporary_path); + // From the first segment write until the manifest naming them is + // durable, no sweep over the project's shared segments may run. + let segments_lock = acquire_generation_segments_publication_lock(store_root, &|| { + self.seal_checkpoint().is_err() + }) + .map_err(|error| match error { + CodeGenerationRetentionErrorV1::Cancelled => { + CodeIndexPublicationStoreErrorV1::CompareAndSwap + } + error => Self::unavailable(error), + })?; let evidence_temporary_path = self.segments_root.join(format!( ".evidence-pack-publication.{}.tmp", - std::process::id() + self.segment_temporary_prefix )); let mut evidence_pack = TemporaryEvidencePackV1::create(evidence_temporary_path)?; let mut referenced_segment_bytes = 0_u64; @@ -2548,6 +2560,7 @@ impl CodeIndexAtomicPublicationPort for DaemonCodeIndexPublicationStoreV1 { } }; evidence_pack.attach_to_manifest(); + drop(segments_lock); let exact_git_evidence = self.exact_git_evidence(&generation)?; let mut generation_index = prior_pointer diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/queries.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/queries.rs index 880d0f8bfc..8607f27ba5 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/queries.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/queries.rs @@ -35,10 +35,10 @@ use tracedecay_contracts::{ use tracedecay_domain::{ AuthorizationRevision, CodeGenerationId, CodeSearchChunkId, ComponentRevision, ExactAdmissionRuleRevision, FileOccurrenceId, FreshnessVectorDigest, ManifestDigest, NodeKind, - PrincipalId, QueryNormalizationRevision, RelationEdgeKindV1, RetrievalAnchorId, - RetrievalBudget, RetrievalBudgetUsage, RetrievalFailure, RetrievalRequest, RetrievalScope, - RetrievalSnapshot, SanitizerRevision, ScoreDomainId, SingleRootScopeV1, SourceOccurrenceId, - SymbolOccurrenceId, TemporalModeV1, UtcMicros, VectorWatermark, canonical_sha256, + PrincipalId, QueryNormalizationRevision, RelationEdgeKindV1, RetrievalBudget, + RetrievalBudgetUsage, RetrievalFailure, RetrievalRequest, RetrievalScope, RetrievalSnapshot, + SanitizerRevision, ScoreDomainId, SingleRootScopeV1, SymbolOccurrenceId, TemporalModeV1, + UtcMicros, VectorWatermark, canonical_sha256, }; use tracedecay_tool_catalog::SortContractId; @@ -54,21 +54,18 @@ use tracedecay_query::code_search; use tracedecay_query::retrieval::exact::{ CentralExactAdmissionAuthorityV1, ExactAdmissionAuthority, ExactLaneRequest, }; -use tracedecay_query::retrieval::graph::{ - GraphLaneRequest, GraphLaneRetriever, graph_read_cancellation, -}; +use tracedecay_query::retrieval::graph::graph_read_cancellation; use tracedecay_query::retrieval::lexical::{ LexicalFieldFilterV1, LexicalFieldV1, LexicalLaneRequest, lexical_query_parts, }; use tracedecay_query::retrieval::ports::{ - CodeCandidateBindingV1, CodeOccurrenceRefV1, RetrievalExecutionControl, RetrievalPortError, + CodeCandidateBindingV1, RetrievalExecutionControl, RetrievalPortError, }; use tracedecay_query::retrieval::{ - AdmittedGenerationContextV1, NativeCodeOccurrenceV1, NativeExactRecordV1, NativeGraphRecordV1, - NativeLaneOutcomeV1, NativeLanePageV1, NativeLexicalRecordV1, NativeRecordReadPortV1, - NativeSymbolRecordV1, PreparedQueryBindingsV1, PreparedQueryErrorV1, - PreparedQueryRoutingBindingsV1, PreparedQueryV1, QueryExecutionContractErrorV1, - route_authenticated_prepared_query_cursor, + AdmittedGenerationContextV1, NativeCodeOccurrenceV1, NativeExactRecordV1, NativeLaneOutcomeV1, + NativeLanePageV1, NativeLexicalRecordV1, NativeRecordReadPortV1, NativeSymbolRecordV1, + PreparedQueryBindingsV1, PreparedQueryErrorV1, PreparedQueryRoutingBindingsV1, PreparedQueryV1, + QueryExecutionContractErrorV1, route_authenticated_prepared_query_cursor, }; const CALLABLE_CODE_SORT: &str = "sort.application.code-index.v1"; @@ -1165,11 +1162,6 @@ impl NativeRecordReadPortV1 for LatestCompleteCodeIndexV1 { } } -struct GraphProjectionNativeRecordReadPortV1 { - generation: CodeGenerationId, - reader: CodeGraphInteractiveReader, -} - fn graph_projection_symbol_record( summary: tracedecay_code_index::graph_projection::CodeGraphSymbolSummaryV1, symbol: &SymbolOccurrenceId, @@ -1218,32 +1210,6 @@ fn graph_projection_symbol_record( }) } -impl NativeRecordReadPortV1 for GraphProjectionNativeRecordReadPortV1 { - fn generation(&self) -> &CodeGenerationId { - &self.generation - } - - fn occurrence( - &self, - _binding: &CodeCandidateBindingV1, - ) -> Result { - Err(QueryExecutionContractErrorV1::RecordUnavailable) - } - - fn symbol( - &self, - symbol: &SymbolOccurrenceId, - file: &FileOccurrenceId, - ) -> Result { - let summary = self - .reader - .symbol_summary(symbol, Arc::new(tracedecay_graph_db::NeverCancelled)) - .map_err(|_| QueryExecutionContractErrorV1::RecordUnavailable)? - .ok_or(QueryExecutionContractErrorV1::RecordUnavailable)?; - graph_projection_symbol_record(summary, symbol, file) - } -} - struct TextArtifactNativeRecordReadPortV1 { generation: CodeGenerationId, owners: std::sync::Arc, @@ -1315,19 +1281,6 @@ fn application_symbol_record(record: NativeSymbolRecordV1) -> SymbolPrimitiveRec } } -fn application_graph_record(record: NativeGraphRecordV1) -> SymbolRelationRecord { - SymbolRelationRecord { - symbol: application_symbol_record(record.symbol), - edge_kind: record.edge_kind.map_or_else( - || "unknown".to_owned(), - |edge| relation_edge_kind_name(edge).to_owned(), - ), - dispatch_via_trait: false, - dispatch_from: None, - depth: Some(record.depth), - } -} - #[derive(Clone, Copy, Debug, Eq, PartialEq)] enum DispatchExpansionStop { Cancelled, @@ -1372,7 +1325,6 @@ fn visit_trait_dispatch_targets( scope: &tracedecay_contracts::CodeQueryScope, budget: RetrievalBudget, control: &Arc, - examined: &mut u64, mut visit: impl FnMut( &tracedecay_code_index::graph_projection::CodeGraphSymbolSummaryV1, ) -> Result, @@ -1393,9 +1345,7 @@ fn visit_trait_dispatch_targets( .ok_or(DispatchExpansionStop::Unavailable)? .simple_name .clone(); - let relation_limit = usize::try_from(budget.max_candidates_per_lane) - .unwrap_or(usize::MAX) - .max(1); + let relation_limit = MAX_RELATION_CANDIDATE_KEYS; check_dispatch_control(control.as_ref(), budget)?; let parent_batches = match reader.callers( std::slice::from_ref(callee), @@ -1408,14 +1358,9 @@ fn visit_trait_dispatch_targets( tracedecay_code_index::graph_projection::CodeGraphProjectionError::BudgetExhausted { .. }, - ) => { - *examined = examined.saturating_add(relation_limit as u64); - return Ok(false); - } + ) => return Ok(false), Err(_) => return Err(dispatch_read_stop(control.as_ref(), budget)), }; - let parent_count = parent_batches.iter().map(Vec::len).sum::(); - *examined = examined.saturating_add(parent_count as u64); let mut traits = Vec::new(); for edge in parent_batches.into_iter().flatten() { let metadata = edge @@ -1445,14 +1390,9 @@ fn visit_trait_dispatch_targets( tracedecay_code_index::graph_projection::CodeGraphProjectionError::BudgetExhausted { .. }, - ) => { - *examined = examined.saturating_add(relation_limit as u64); - return Ok(false); - } + ) => return Ok(false), Err(_) => return Err(dispatch_read_stop(control.as_ref(), budget)), }; - let implementor_count = implementor_batches.iter().map(Vec::len).sum::(); - *examined = examined.saturating_add(implementor_count as u64); let implementors = implementor_batches .into_iter() .flatten() @@ -1473,14 +1413,9 @@ fn visit_trait_dispatch_targets( tracedecay_code_index::graph_projection::CodeGraphProjectionError::BudgetExhausted { .. }, - ) => { - *examined = examined.saturating_add(relation_limit as u64); - return Ok(false); - } + ) => return Ok(false), Err(_) => return Err(dispatch_read_stop(control.as_ref(), budget)), }; - let child_count = child_batches.iter().map(Vec::len).sum::(); - *examined = examined.saturating_add(child_count as u64); for child in child_batches .into_iter() .flatten() @@ -1510,162 +1445,98 @@ fn visit_trait_dispatch_targets( Ok(true) } -fn callee_dispatch_usage( - page: &NativeLanePageV1, - examined: u64, - control: &dyn RetrievalExecutionControl, -) -> RetrievalBudgetUsage { - RetrievalBudgetUsage { - candidates_examined: page.coverage.examined.saturating_add(examined), - candidates_returned: u64::try_from(page.items.len()).unwrap_or(u64::MAX), - hydrated_results: u64::try_from(page.items.len()).unwrap_or(u64::MAX), - hydration_bytes: 0, - elapsed_micros: control.elapsed_micros(), - } -} - -fn augment_callee_dispatch_page( +/// Appends the concrete impl methods reachable through each direct callee's +/// trait as compact keys (`dispatch_from` names the trait method), so they +/// page and hydrate like every other relation. Dispatch keys follow the +/// direct keys in canonical order; a trait fan-out past the key ceiling +/// leaves the listing incomplete rather than claiming full dispatch. +fn augment_callee_dispatch_keys( reader: &CodeGraphInteractiveReader, - page: NativeLanePageV1, + found: &mut GraphRelationKeysV1, scope: &tracedecay_contracts::CodeQueryScope, budget: RetrievalBudget, control: &Arc, -) -> Result, Box>> -{ - let candidate_cap = usize::try_from(budget.max_candidates_per_lane).unwrap_or(usize::MAX); - let mut page = NativeLanePageV1 { - generation: page.generation, - items: page - .items - .into_iter() - .map(application_graph_record) - .collect(), - total_eligible: page.total_eligible, - coverage: page.coverage, - }; - let direct = page.items.clone(); - let mut seen = direct +) -> Result<(), DispatchExpansionStop> { + let mut seen = found + .keys .iter() - .map(|record| record.symbol.node_id.clone()) + .map(|key| key.occurrence.clone()) .collect::>(); - let mut examined = 0_u64; - let mut eligible = 0_u64; - - 'callees: for callee in direct { - match check_dispatch_control(control.as_ref(), budget) { - Ok(()) => {} - Err(DispatchExpansionStop::Cancelled) => { - return Err(Box::new(NativeLaneOutcomeV1::Cancelled)); - } - Err(DispatchExpansionStop::TimedOut) => { - return Err(Box::new(NativeLaneOutcomeV1::TimedOut( - callee_dispatch_usage(&page, examined, control.as_ref()), - ))); - } - Err(DispatchExpansionStop::Unavailable) => { - return Err(Box::new(NativeLaneOutcomeV1::Unavailable( - RetrievalFailure::AuthorityUnavailable { - detail: "verified code graph dispatch authority is unavailable".to_owned(), - }, - ))); - } - } - if page.items.len() >= candidate_cap { - page.coverage.unknown = page.coverage.unknown.saturating_add(1); - break; - } - let Ok(callee_id) = SymbolOccurrenceId::new(callee.symbol.node_id.clone()) else { - continue; - }; + let mut dispatch = Vec::new(); + for callee in &found.keys { let exhausted = visit_trait_dispatch_targets( reader, - &callee_id, + &callee.occurrence, scope, budget, control, - &mut examined, |target| { - if !seen.insert(target.occurrence.as_str().to_owned()) { - return Ok(true); + if seen.insert(target.occurrence.clone()) { + dispatch.push(RelationKeyV1 { + occurrence: target.occurrence.clone(), + edge_kind: RelationEdgeKindV1::Calls, + dispatch_from: Some(callee.occurrence.clone()), + depth: callee.depth, + }); } - let file = target - .binding - .as_ref() - .map(|binding| &binding.file) - .ok_or(DispatchExpansionStop::Unavailable)?; - let symbol = - graph_projection_symbol_record(target.clone(), &target.occurrence, file) - .map(application_symbol_record) - .map_err(|_| DispatchExpansionStop::Unavailable)?; - eligible = eligible.saturating_add(1); - page.items.push(SymbolRelationRecord { - symbol, - edge_kind: relation_edge_kind_name(RelationEdgeKindV1::Calls).to_owned(), - dispatch_via_trait: true, - dispatch_from: Some(callee.symbol.node_id.clone()), - depth: callee.depth, - }); - Ok(page.items.len() < candidate_cap) + Ok(found.keys.len() + dispatch.len() < MAX_RELATION_CANDIDATE_KEYS) }, - ); - match exhausted { - Ok(true) => {} - Ok(false) => { - page.coverage.unknown = page.coverage.unknown.saturating_add(1); - break 'callees; - } - Err(DispatchExpansionStop::Cancelled) => { - return Err(Box::new(NativeLaneOutcomeV1::Cancelled)); - } - Err(DispatchExpansionStop::TimedOut) => { - return Err(Box::new(NativeLaneOutcomeV1::TimedOut( - callee_dispatch_usage(&page, examined, control.as_ref()), - ))); - } - Err(DispatchExpansionStop::Unavailable) => { - return Err(Box::new(NativeLaneOutcomeV1::Unavailable( - RetrievalFailure::AuthorityUnavailable { - detail: "verified code graph dispatch authority is unavailable".to_owned(), - }, - ))); - } + )?; + if !exhausted { + found.complete = false; + break; } } - page.total_eligible = page.total_eligible.saturating_add(eligible); - page.coverage.examined = page.coverage.examined.saturating_add(examined); - page.coverage.eligible = page.coverage.eligible.saturating_add(eligible); - Ok(page) + dispatch.sort_by(|left, right| { + left.depth + .cmp(&right.depth) + .then(left.occurrence.cmp(&right.occurrence)) + }); + found.keys.append(&mut dispatch); + Ok(()) } -fn augment_callee_dispatch( - reader: &CodeGraphInteractiveReader, - outcome: NativeLaneOutcomeV1, - scope: &tracedecay_contracts::CodeQueryScope, - budget: RetrievalBudget, +/// A relation read that stopped is typed by what stopped it: the request's +/// cancellation, its deadline, or the graph authority. +fn relation_read_stop( + prepared: &impl PreparedCallableQueryStateV1, + stop: DispatchExpansionStop, control: &Arc, -) -> NativeLaneOutcomeV1 { - match outcome { - NativeLaneOutcomeV1::Complete(page) => { - match augment_callee_dispatch_page(reader, page, scope, budget, control) { - Ok(page) => NativeLaneOutcomeV1::Complete(page), - Err(terminal) => *terminal, - } +) -> RetrievalPortOutcome> { + let finished_at = query_finished_at(); + let generation = prepared.generation().clone(); + match stop { + DispatchExpansionStop::Cancelled => { + let mut evidence = + terminal_lane_evidence(finished_at, generation, OmissionReason::Cancelled); + evidence.cancellation = Some(CancellationObservation { + stage: CancellationStage::DuringRead, + observed_at: finished_at, + }); + RetrievalPortOutcome::Cancelled(evidence) } - NativeLaneOutcomeV1::Partial { page, reason } => { - match augment_callee_dispatch_page(reader, page, scope, budget, control) { - Ok(page) => NativeLaneOutcomeV1::Partial { page, reason }, - Err(terminal) => *terminal, - } + DispatchExpansionStop::TimedOut => { + let mut evidence = + terminal_lane_evidence(finished_at, generation, OmissionReason::TimedOut); + evidence.budget.elapsed_micros = control.elapsed_micros(); + RetrievalPortOutcome::TimedOut(evidence) } - NativeLaneOutcomeV1::Unavailable(reason) => NativeLaneOutcomeV1::Unavailable(reason), - NativeLaneOutcomeV1::Denied => NativeLaneOutcomeV1::Denied, - NativeLaneOutcomeV1::Stale(source) => NativeLaneOutcomeV1::Stale(source), - NativeLaneOutcomeV1::BudgetExceeded(usage) => NativeLaneOutcomeV1::BudgetExceeded(usage), - NativeLaneOutcomeV1::TimedOut(usage) => NativeLaneOutcomeV1::TimedOut(usage), - NativeLaneOutcomeV1::Cancelled => NativeLaneOutcomeV1::Cancelled, + DispatchExpansionStop::Unavailable => unavailable_for_generation(finished_at, generation), } } +fn relation_read_failure( + prepared: &impl PreparedCallableQueryStateV1, + control: &Arc, + budget: RetrievalBudget, +) -> RetrievalPortOutcome> { + relation_read_stop( + prepared, + dispatch_read_stop(control.as_ref(), budget), + control, + ) +} + fn symbol_record( latest: &LatestCompleteCodeIndexV1, symbol: &SymbolOccurrenceId, @@ -2265,6 +2136,25 @@ struct GraphRelationKeysV1 { complete: bool, } +/// Compact relation keys one navigation query enumerates before it reports +/// the remainder as unknown (`Partial`, `Budget` omission, `total` = the +/// admitted count the cursor walks). +/// +/// This is a runaway guard, not the page budget: the fusion profile's +/// `max_candidates_per_lane` (32) ranks search candidates and must not bound +/// an exhaustive relation listing, which is what made `code_callers` and +/// `code_callees` answer a 104-relation symbol with `total: 32`. Keys are +/// identities only and rows hydrate per page, so `total` is the true relation +/// count whenever the neighborhood fits under this ceiling. +/// +/// Enumeration costs about 60 µs per relation (Hotpath +/// `query.graph.relation_keys`: 2.1 ms at 32 relations, 5.7 ms at 104, 60 ms +/// at 1,000, 125 ms at 2,000; two `graph_db` entity reads per edge, the far +/// symbol's only for the scope path) and is paid again on every page, so this +/// ceiling keeps a page under one second while the request deadline +/// (`TimedOut`) remains the bound on the rest. +const MAX_RELATION_CANDIDATE_KEYS: usize = 10_000; + fn graph_summary_symbol_record( summary: tracedecay_code_index::graph_projection::CodeGraphSymbolSummaryV1, ) -> Result { @@ -2279,6 +2169,7 @@ fn graph_summary_symbol_record( .map_err(|_| PreparedQueryErrorV1::Unavailable) } +#[hotpath::measure(label = "query.graph.relation_keys")] fn graph_relation_keys( reader: &CodeGraphInteractiveReader, start: &SymbolOccurrenceId, @@ -2286,9 +2177,9 @@ fn graph_relation_keys( reverse: bool, maximum_depth: u32, scope: &tracedecay_contracts::CodeQueryScope, - cap: usize, cancellation: Arc, ) -> Result { + let cap = MAX_RELATION_CANDIDATE_KEYS; let mut queue = VecDeque::from([(start.clone(), 0_u32)]); let mut visited = BTreeSet::from([start.clone()]); let mut keys = Vec::new(); @@ -2358,6 +2249,7 @@ fn graph_relation_keys( Ok(GraphRelationKeysV1 { keys, complete }) } +#[hotpath::measure(label = "query.graph.relation_hydrate")] fn hydrate_graph_relation_records( reader: &CodeGraphInteractiveReader, keys: &[RelationKeyV1], @@ -2374,7 +2266,7 @@ fn hydrate_graph_relation_records( Ok(SymbolRelationRecord { symbol: graph_summary_symbol_record(summary)?, edge_kind: relation_edge_kind_name(key.edge_kind).to_owned(), - dispatch_via_trait: key.edge_kind == RelationEdgeKindV1::Implements, + dispatch_via_trait: key.dispatch_from.is_some(), dispatch_from: key .dispatch_from .as_ref() @@ -2804,7 +2696,7 @@ impl CallableCodeQueryPort for CodeIndexSchedulerRegistryV1 { request: &'a CodeRelationRequest, ) -> CallableCodeQueryFuture<'a, SymbolRelationRecord> { Box::pin(async move { - let (prepared, query_binding_digest) = prepare_graph_callable_query_or_return!( + let (prepared, binding) = prepare_graph_callable_query_or_return!( self, context, request, @@ -2819,104 +2711,52 @@ impl CallableCodeQueryPort for CodeIndexSchedulerRegistryV1 { &request.meta.order, ) ); - let latest = &prepared.latest; - let served_generation = latest.metadata().manifest().generation_id.clone(); - let finished_at = query_finished_at(); - let base = prepared.query.request(); - let Ok(symbol) = typed::(request.node_id.clone()) else { - return unavailable(finished_at); - }; - let Ok(Some(summary)) = prepared - .reader - .symbol_summary(&symbol, Arc::new(tracedecay_graph_db::NeverCancelled)) - else { - return unavailable_for_generation(finished_at, served_generation); - }; - let Some(symbol_binding) = summary.binding else { - return unavailable(finished_at); - }; - let source_occurrence = - SourceOccurrenceId::new(format!("code-symbol:{}", symbol.as_str())) - .unwrap_or_else(|_| panic!("validated symbol creates source occurrence")); - let seed = CodeCandidateBindingV1 { - candidate_anchor: RetrievalAnchorId::new(format!( - "code-symbol:{}", - symbol.as_str() - )) - .unwrap_or_else(|_| panic!("validated symbol creates anchor")), - occurrence: CodeOccurrenceRefV1 { - generation: served_generation.clone(), - file: symbol_binding.file, - symbol: Some(symbol), - chunk: symbol_binding.chunk, - }, - language_descriptor_revision: symbol_binding.language_descriptor_revision, - matched_term_kinds: Vec::new(), - source_occurrence, - }; - let graph_budget = graph_budget_for_request(base.budget, context.request); - let lane_request = GraphLaneRequest { - generation: served_generation.clone(), - seed_anchors: vec![seed], - edge_kinds: vec![RelationEdgeKindV1::Calls], - max_depth: request.maximum_depth, - budget: graph_budget, - base: base.clone(), - }; - let Ok(graph_serving) = latest.production_graph_serving() else { - return unavailable(finished_at); - }; - let records = GraphProjectionNativeRecordReadPortV1 { - generation: served_generation.clone(), - reader: prepared.reader.clone(), - }; - let Ok(native_context) = - AdmittedGenerationContextV1::admit(served_generation.clone(), &records) - else { - return unavailable_for_generation(finished_at, served_generation); - }; + let graph_budget = + graph_budget_for_request(prepared.query.request().budget, context.request); let graph_control = CallableRetrievalExecutionControl::for_request(context.request); - let outcome = graph_serving - .graph - .retrieve_graph(&lane_request, Arc::clone(&graph_control)); - match outcome { - Ok(outcome) => { - let Ok(outcome) = native_context.graph(outcome, |path| { - path_is_in_code_query_scope(path, &request.scope) - }) else { - return unavailable(finished_at); - }; - if request.resolve_trait_dispatch { - let outcome = augment_callee_dispatch( - &prepared.reader, - outcome, - &request.scope, - graph_budget, - &graph_control, - ); - finish_native_lane_query( - &prepared, - &context, - "code_callees", - query_binding_digest, - &request.meta.page, - outcome, - |record| record, - ) - } else { - finish_native_lane_query( - &prepared, - &context, - "code_callees", - query_binding_digest, - &request.meta.page, - outcome, - application_graph_record, - ) - } - } - Err(_) => unavailable(finished_at), + let cancellation = + graph_read_cancellation(Arc::clone(&graph_control), graph_budget.deadline_micros); + let start = resolve_graph_start_symbol!(prepared, request.node_id, cancellation); + let Ok(mut found) = graph_relation_keys( + &prepared.reader, + &start, + &[RelationEdgeKindV1::Calls], + false, + request.maximum_depth, + &request.scope, + Arc::clone(&cancellation), + ) else { + return relation_read_failure(&prepared, &graph_control, graph_budget); + }; + if request.resolve_trait_dispatch + && let Err(stop) = augment_callee_dispatch_keys( + &prepared.reader, + &mut found, + &request.scope, + graph_budget, + &graph_control, + ) + { + return relation_read_stop(&prepared, stop, &graph_control); } + finish_generation_candidate_page( + &prepared, + &context, + "code_callees", + binding, + found.keys, + |slice| { + hydrate_graph_relation_records( + &prepared.reader, + slice, + cancellation, + &self.relation_symbol_hydrations, + ) + }, + &request.meta.page, + "callees", + found.complete, + ) }) } @@ -3147,8 +2987,9 @@ impl CallableCodeQueryPort for CodeIndexSchedulerRegistryV1 { let graph_budget = graph_budget_for_request(prepared.query.request().budget, context.request); let graph_control = CallableRetrievalExecutionControl::for_request(context.request); - let cancellation = graph_read_cancellation(graph_control, graph_budget.deadline_micros); - let cap = graph_budget.max_candidates_per_lane as usize; + let cancellation = + graph_read_cancellation(Arc::clone(&graph_control), graph_budget.deadline_micros); + let cap = MAX_RELATION_CANDIDATE_KEYS; let selector_simple = selector .rsplit_once("::") .map_or(selector.as_str(), |(_, name)| name); @@ -3167,10 +3008,7 @@ impl CallableCodeQueryPort for CodeIndexSchedulerRegistryV1 { Arc::clone(&cancellation), ), ) else { - return unavailable_for_generation( - query_finished_at(), - prepared.generation().clone(), - ); + return relation_read_failure(&prepared, &graph_control, graph_budget); }; let mut complete = targets.len() <= cap && simple_targets.len() <= cap; targets.extend(simple_targets); @@ -3182,8 +3020,7 @@ impl CallableCodeQueryPort for CodeIndexSchedulerRegistryV1 { } let mut keys = Vec::new(); for target in targets { - let remaining = cap.saturating_sub(keys.len()); - if remaining == 0 { + if keys.len() >= cap { complete = false; break; } @@ -3194,19 +3031,19 @@ impl CallableCodeQueryPort for CodeIndexSchedulerRegistryV1 { true, 1, &request.scope, - remaining, Arc::clone(&cancellation), ) else { - return unavailable_for_generation( - query_finished_at(), - prepared.generation().clone(), - ); + return relation_read_failure(&prepared, &graph_control, graph_budget); }; complete &= found.complete; keys.extend(found.keys); } keys.sort_by(|left, right| left.occurrence.cmp(&right.occurrence)); keys.dedup_by(|left, right| left.occurrence == right.occurrence); + if keys.len() > cap { + keys.truncate(cap); + complete = false; + } finish_generation_candidate_page( &prepared, &context, @@ -3251,7 +3088,8 @@ impl CallableCodeQueryPort for CodeIndexSchedulerRegistryV1 { let graph_budget = graph_budget_for_request(prepared.query.request().budget, context.request); let graph_control = CallableRetrievalExecutionControl::for_request(context.request); - let cancellation = graph_read_cancellation(graph_control, graph_budget.deadline_micros); + let cancellation = + graph_read_cancellation(Arc::clone(&graph_control), graph_budget.deadline_micros); let start = resolve_graph_start_symbol!(prepared, request.node_id, cancellation); let Ok(found) = graph_relation_keys( &prepared.reader, @@ -3260,13 +3098,9 @@ impl CallableCodeQueryPort for CodeIndexSchedulerRegistryV1 { false, request.maximum_depth, &request.scope, - graph_budget.max_candidates_per_lane as usize, Arc::clone(&cancellation), ) else { - return unavailable_for_generation( - query_finished_at(), - prepared.generation().clone(), - ); + return relation_read_failure(&prepared, &graph_control, graph_budget); }; let parent_node_id = request.node_id.clone(); finish_generation_candidate_page( @@ -3325,7 +3159,8 @@ impl CallableCodeQueryPort for CodeIndexSchedulerRegistryV1 { let graph_budget = graph_budget_for_request(prepared.query.request().budget, context.request); let graph_control = CallableRetrievalExecutionControl::for_request(context.request); - let cancellation = graph_read_cancellation(graph_control, graph_budget.deadline_micros); + let cancellation = + graph_read_cancellation(Arc::clone(&graph_control), graph_budget.deadline_micros); let start = resolve_graph_start_symbol!(prepared, request.node_id, cancellation); let Ok(found) = graph_relation_keys( &prepared.reader, @@ -3334,13 +3169,9 @@ impl CallableCodeQueryPort for CodeIndexSchedulerRegistryV1 { true, request.maximum_depth, &request.scope, - graph_budget.max_candidates_per_lane as usize, Arc::clone(&cancellation), ) else { - return unavailable_for_generation( - query_finished_at(), - prepared.generation().clone(), - ); + return relation_read_failure(&prepared, &graph_control, graph_budget); }; let mut traversed = vec![start]; traversed.extend( @@ -3425,7 +3256,8 @@ impl CallableCodeQueryPort for CodeIndexSchedulerRegistryV1 { let graph_budget = graph_budget_for_request(prepared.query.request().budget, context.request); let graph_control = CallableRetrievalExecutionControl::for_request(context.request); - let cancellation = graph_read_cancellation(graph_control, graph_budget.deadline_micros); + let cancellation = + graph_read_cancellation(Arc::clone(&graph_control), graph_budget.deadline_micros); let start = resolve_graph_start_symbol!(prepared, request.node_id, cancellation); let Ok(found) = graph_relation_keys( &prepared.reader, @@ -3442,13 +3274,9 @@ impl CallableCodeQueryPort for CodeIndexSchedulerRegistryV1 { true, request.maximum_depth, &request.scope, - graph_budget.max_candidates_per_lane as usize, Arc::clone(&cancellation), ) else { - return unavailable_for_generation( - query_finished_at(), - prepared.generation().clone(), - ); + return relation_read_failure(&prepared, &graph_control, graph_budget); }; finish_generation_candidate_page( &prepared, @@ -3500,8 +3328,9 @@ impl CallableCodeQueryPort for CodeIndexSchedulerRegistryV1 { let graph_budget = graph_budget_for_request(prepared.query.request().budget, context.request); let graph_control = CallableRetrievalExecutionControl::for_request(context.request); - let cancellation = graph_read_cancellation(graph_control, graph_budget.deadline_micros); - let cap = graph_budget.max_candidates_per_lane as usize; + let cancellation = + graph_read_cancellation(Arc::clone(&graph_control), graph_budget.deadline_micros); + let cap = MAX_RELATION_CANDIDATE_KEYS; let Ok(mut summaries) = prepared.reader.find_symbols( &|_, binding, metadata| { let Some(path) = binding.and_then(|binding| binding.logical_path.as_deref()) @@ -3515,10 +3344,7 @@ impl CallableCodeQueryPort for CodeIndexSchedulerRegistryV1 { cap.saturating_add(1), Arc::clone(&cancellation), ) else { - return unavailable_for_generation( - query_finished_at(), - prepared.generation().clone(), - ); + return relation_read_failure(&prepared, &graph_control, graph_budget); }; let complete = summaries.len() <= cap; summaries.truncate(cap); @@ -3795,7 +3621,8 @@ impl CallableCodeQueryPort for CodeIndexSchedulerRegistryV1 { let graph_budget = graph_budget_for_request(prepared.query.request().budget, context.request); let graph_control = CallableRetrievalExecutionControl::for_request(context.request); - let cancellation = graph_read_cancellation(graph_control, graph_budget.deadline_micros); + let cancellation = + graph_read_cancellation(Arc::clone(&graph_control), graph_budget.deadline_micros); let start = resolve_graph_start_symbol!(prepared, request.node_id, cancellation); let Ok(found) = graph_relation_keys( &prepared.reader, @@ -3809,13 +3636,9 @@ impl CallableCodeQueryPort for CodeIndexSchedulerRegistryV1 { true, 1, &request.scope, - graph_budget.max_candidates_per_lane as usize, Arc::clone(&cancellation), ) else { - return unavailable_for_generation( - query_finished_at(), - prepared.generation().clone(), - ); + return relation_read_failure(&prepared, &graph_control, graph_budget); }; finish_generation_candidate_page( &prepared, @@ -3863,7 +3686,8 @@ fn navigation_symbol_query<'a>( let graph_budget = graph_budget_for_request(prepared.query.request().budget, context.request); let graph_control = CallableRetrievalExecutionControl::for_request(context.request); - let cancellation = graph_read_cancellation(graph_control, graph_budget.deadline_micros); + let cancellation = + graph_read_cancellation(Arc::clone(&graph_control), graph_budget.deadline_micros); let start = resolve_graph_start_symbol!(prepared, request.node_id, cancellation); let mut items = Vec::new(); let summary = match prepared @@ -3901,13 +3725,9 @@ fn navigation_symbol_query<'a>( false, 1, &request.scope, - graph_budget.max_candidates_per_lane as usize, Arc::clone(&cancellation), ) else { - return unavailable_for_generation( - query_finished_at(), - prepared.generation().clone(), - ); + return relation_read_failure(&prepared, &graph_control, graph_budget); }; return finish_generation_candidate_page( &prepared, diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/query_runtime.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/query_runtime.rs index 3b5efa071c..d7b393d1ec 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/query_runtime.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/query_runtime.rs @@ -715,14 +715,9 @@ where let sanitized = RawRetrievalRequestV1::new(input.query, request) .sanitize(input.sanitizer_revision, input.normalization_revision)?; let readiness = text.query_owner_readiness(); - // Only this search's own owners decide whether it needs the worker. - // A clone-fingerprint successor keeps `text_projection_needs_work` - // true long after exact and lexical are ready, and asking for it here - // stamped the pending-wake slot on a seat whose source proof was - // current: the freshness ladder reads that slot as - // `refresh_in_flight`, so the very response that stamped it answered - // `verifying`. The successor is the worker's own continuation, and - // the similarity path that consumes it still requests it directly. + // Only this search's own owners decide whether it needs the worker: + // stamping the pending-wake slot on a seat whose source proof is + // current makes the freshness ladder answer `verifying`. if !matches!(&readiness, CodeTextQueryOwnerReadinessV1::Ready(_)) { match schedulers.request_query_background_reconcile(scope).await { CodeIndexReconcileAdmissionV1::PublicationAuthorityCorrupt(_) => { diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs index a83c866680..41f40d37bd 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs @@ -612,8 +612,8 @@ impl SourceFreshnessFenceV1 { /// Whether the last completed proof was sealed from exactly this snapshot. /// - /// Clock age is not part of the answer. A seal or clone backfill can - /// outlive the admission window without the snapshot changing identity. + /// Clock age is not part of the answer. A seal can outlive the admission + /// window without the snapshot changing identity. pub(super) fn proof_describes_snapshot( &self, snapshot_content_identity: &ContentDigest, @@ -2908,8 +2908,7 @@ impl CodeIndexWorktreeSchedulerV1 { /// Bind a sealed snapshot to the source proof, renewing an expired clock /// when the sealed digests still match. /// - /// The admission window is 30s. This does not move the clone-successor - /// copy off the publication advance. It only stops an expired clock, or a + /// The admission window is 30s. This only stops an expired clock, or a /// predecessor disk witness, from clearing the generation those digests /// already name. A hook epoch or a digest mismatch still refuses. pub(super) fn currency_witness_for_sealed_snapshot( @@ -3765,7 +3764,6 @@ impl CodeIndexWorktreeSchedulerV1 { for receipt in &active.snapshot().sanitization_receipts { if file_occurrence_id( &self.repository_id, - &self.worktree_id, &file.logical_path, &file.content_digest, receipt, @@ -3781,7 +3779,6 @@ impl CodeIndexWorktreeSchedulerV1 { for file in &files { if file_occurrence_id( &self.repository_id, - &self.worktree_id, &file.logical_path, &file.content_digest, &receipt, diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry.rs index 3b632a139e..254f4f0924 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry.rs @@ -146,11 +146,12 @@ const TEXT_PROJECTION_MAXIMUM_ACTIVATION_ADVANCES_V1: usize = 10_000; /// generation, so a complete sealed generation sat on disk with zero seat /// attempts and no log line, because a missing prepare is not a refusal. The /// gate is now the text owner, not the tree: a publication prepares on its own -/// pass once its lightweight text owner has finished, and an unchanged pass -/// prepares as soon as a retained owner exists to recover a verified head. -/// Fresh graph publication and any retained full replay follow text projection -/// because both are corpus-sized consumers of the sealed source and process -/// memory. Every skip names itself. +/// pass alongside its lightweight text owner's projection, and an unchanged +/// pass prepares as soon as a retained owner exists to recover a verified head. +/// Both are corpus-sized consumers of the sealed source and process memory, so +/// fresh graph publication starts only once that projection holds its build +/// reservation, and any retained full replay follows text projection. Every +/// skip names itself. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum GraphSeatGateV1 { /// Prepare, decode, activate, and swap this generation into serving. @@ -183,9 +184,11 @@ enum PublishedTextProjectionOutcomeV1 { impl GraphSeatGateV1 { /// `text_owner_admitted_for_graph` means a publication's replacement - /// owner is ready, or an unchanged pass has a retained owner from which it - /// can first try to recover an already-verified graph head. A retained full - /// replay is gated separately on text readiness after that recovery attempt. + /// owner is ready or its projection runs in this pass (the serving swap + /// joins it before seating), or an unchanged pass has a retained owner + /// from which it can first try to recover an already-verified graph head. + /// A retained full replay is gated separately on text readiness after that + /// recovery attempt. #[hotpath::skip] pub const fn decide( activation_enabled: bool, @@ -355,11 +358,8 @@ impl ServingSwapOutcomeV1 { } } -/// An unfinished text projection withholds the serving seat only when exact -/// or lexical owners are still missing. -/// -/// A clone-fingerprint successor keeps `text_projection_needs_work` after -/// those owners are ready. That is not `published_text_owner_unfinished`. +/// An unfinished text projection withholds the serving seat only when its +/// query owners are still missing. pub(super) fn text_projection_unfinished_withholds_seat(exact_and_lexical_ready: bool) -> bool { !exact_and_lexical_ready } @@ -2298,9 +2298,12 @@ impl CodeIndexSchedulerRegistryV1 { /// advance at a time, until exact and lexical serving are ready or the /// projection stops typed. /// - /// The caller chooses ordering. Fresh publications await this before graph - /// work because text and graph compete for the same sealed source and - /// resident-memory headroom. Retained owners may still run on their own + /// The caller chooses ordering. Text and graph compete for the same sealed + /// source and resident-memory headroom, so a fresh publication starts + /// graph work only once `opened` fires: the first advance succeeded and + /// the build holds its reservation. A projection that stops before that + /// (parked, failed, or already ready) drops the sender instead, and graph + /// then waits for ready owners. Retained owners may still run on their own /// task while the scheduler recovers an already-verified graph head. The /// advance itself is single-flight on the owner's projection slot, so /// scheduler wakes that race it wait, never double drive. @@ -2323,6 +2326,7 @@ impl CodeIndexSchedulerRegistryV1 { shutting_down: Arc, convergence_park: Arc>>, installed: Option>>>, + mut opened: Option>, #[cfg(test)] project_root: PathBuf, ) -> PublishedTextProjectionOutcomeV1 { #[cfg(test)] @@ -2335,11 +2339,6 @@ impl CodeIndexSchedulerRegistryV1 { return PublishedTextProjectionOutcomeV1::Shutdown; } // A publication's pass waits only for the owners the seat needs. - // Once the admission artifact serves exact and lexical, the slot - // may still hold the clone-fingerprint successor: that backfill - // re-decodes the whole sealed source into a second artifact and - // is not a seat precondition, so it continues on the retained - // driver of a later pass instead of holding graph activation. if installed.is_none() && text.query_owners_are_ready() { break; } @@ -2354,14 +2353,19 @@ impl CodeIndexSchedulerRegistryV1 { break; } let advancing = text.clone(); - match hotpath::future!( + let advance = hotpath::future!( tokio::task::spawn_blocking( move || advancing.advance_text_serving(TEXT_PROJECTION_DOCUMENTS_PER_PASS_V1) ), label = "daemon.code_index.text_projection" ) - .await + .await; + if matches!(advance, Ok(Ok(_))) + && let Some(opened) = opened.take() { + let _ = opened.send(()); + } + match advance { Ok(Ok(true)) => { clear_convergence_park(&convergence_park); break; diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/convergence_park_tests.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/convergence_park_tests.rs index 14413122a2..ed7538a200 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/convergence_park_tests.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/convergence_park_tests.rs @@ -10,10 +10,10 @@ //! socket-directory variant of the same contract refuses fast and typed at //! daemon bootstrap; background convergence must be just as truthful. //! -//! Green means: an owned legacy mode self-heals (with the store converging to -//! owner-private and serving), and an unhealable violation surfaces as a typed -//! `parked` freshness state whose reason names the violation, while removing -//! the violation lets the ordinary wake cadence resume without a remount. +//! Green means: every violation, including a permissive mode, surfaces as a +//! typed `parked` freshness state whose reason names the violation and is +//! never rewritten in place, while removing the violation lets the ordinary +//! wake cadence resume without a remount. use std::fs; use std::os::unix::fs::PermissionsExt; @@ -23,7 +23,7 @@ use std::time::Duration; use tempfile::TempDir; use tracedecay_code_index_retention::code_index_generations::{ - code_text_artifacts_root, scoped_code_index_store_root, + code_text_artifact_staging_root, scoped_code_index_store_root, }; use super::super::graph_activation::{ @@ -94,7 +94,7 @@ impl Fixture { canonical_existing_identity(&project).expect("canonical project root"); let scoped = scoped_code_index_store_root(&store, &canonical_project); tracedecay_private_fs::create_private_directory(&scoped).expect("create scoped root"); - let artifacts_root = code_text_artifacts_root(&scoped); + let artifacts_root = code_text_artifact_staging_root(&scoped); poison(&artifacts_root); let registry = CodeIndexSchedulerRegistryV1::with_background_reconcile_permits(1, 1); @@ -179,14 +179,14 @@ impl Fixture { } } -/// An owned legacy artifacts root with a permissive mode is exactly the state -/// older binaries left behind. Ownership is provable, so the worker heals it -/// to owner-private in place and serving converges, no operator chmod, no -/// parked state, no indefinite warming. +/// A permissive artifacts root violates the owner-privacy contract; the +/// worker never re-permissions it. It parks typed, leaves the mode exactly as +/// found, and resumes on the ordinary wake cadence once the operator restores +/// owner-only access. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn a_legacy_permissive_text_artifacts_root_self_heals_and_serves() { +async fn a_permissive_text_artifacts_root_parks_typed_without_rewriting_its_mode() { let fixture = Fixture::mount_with_poisoned_artifacts_root( - "project.text-artifacts-root-self-heal", + "project.text-artifacts-root-permissive", |artifacts_root| { fs::create_dir_all(artifacts_root).expect("create artifacts root"); fs::set_permissions(artifacts_root, fs::Permissions::from_mode(0o775)) @@ -195,47 +195,62 @@ async fn a_legacy_permissive_text_artifacts_root_self_heals_and_serves() { ) .await; - let observed = fixture - .wait_for_freshness(|freshness| { - freshness.staleness_state - == Some( - tracedecay_contracts::code_index_freshness::CodeIndexStalenessStateV1::Fresh, - ) - }) + let parked = fixture + .wait_for_freshness(|freshness| freshness.parked.is_some()) .await .expect("freshness projection for the mounted worktree"); - assert_eq!( - observed.staleness_state, - Some(tracedecay_contracts::code_index_freshness::CodeIndexStalenessStateV1::Fresh), - "the healed store must converge to serving instead of warming forever: {observed:?}" + parked.staleness_state, + Some(tracedecay_contracts::code_index_freshness::CodeIndexStalenessStateV1::Parked), + "a permissive root must park instead of serving: {parked:?}" ); + let park = parked.parked.as_ref().expect("typed parked state"); assert!( - observed.parked.is_none(), - "a healed root must not stay parked: {:?}", - observed.parked + park.reason.contains("code text artifacts root") && park.reason.contains("mode 775"), + "the parked reason must name the violated contract and observed mode: {}", + park.reason ); - let mode = fs::metadata(&fixture.artifacts_root) - .expect("artifacts root metadata") - .permissions() - .mode() - & 0o777; + let mode = |root: &Path| { + fs::metadata(root) + .expect("artifacts root metadata") + .permissions() + .mode() + & 0o777 + }; assert_eq!( - mode, 0o700, - "self-heal must tighten the legacy root to owner-private" + mode(&fixture.artifacts_root), + 0o775, + "the worker must not rewrite a root it did not create" + ); + + fs::set_permissions(&fixture.artifacts_root, fs::Permissions::from_mode(0o700)) + .expect("operator restores owner-only access"); + let recovered = fixture + .wait_for_freshness(|freshness| { + freshness.parked.is_none() + && freshness.staleness_state + == Some( + tracedecay_contracts::code_index_freshness::CodeIndexStalenessStateV1::Fresh, + ) + }) + .await + .expect("freshness projection for the mounted worktree"); + assert!( + recovered.parked.is_none(), + "the park must clear once the operator fixes the mode: {recovered:?}" ); fixture.registry.shutdown().await; } -/// A violation ownership cannot prove away, here a foreign regular file -/// squatting on the artifacts-root path, must park typed: the freshness +/// A foreign regular file squatting on the artifacts-root path must park +/// typed: the freshness /// projection names the exact violation and remediation instead of reporting /// "indexing" (surfaced as "warming") forever. Removing the violation lets /// the ordinary wake cadence resume without a remount, proving parked is /// visible-but-recoverable rather than permanently dead. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn an_unhealable_text_artifacts_root_parks_typed_and_recovers_when_fixed() { +async fn a_squatted_text_artifacts_root_parks_typed_and_recovers_when_fixed() { let fixture = Fixture::mount_with_poisoned_artifacts_root( "project.text-artifacts-root-typed-park", |artifacts_root| { @@ -312,7 +327,7 @@ async fn an_unhealable_text_artifacts_root_parks_typed_and_recovers_when_fixed() /// consumers of the sealed generation. Starting graph work while text is /// parked can hold the source text needs, cross the process RSS watermark, /// and then prevent text from reacquiring its reservation indefinitely. A -/// text owner parked on an unhealable artifacts root makes the required +/// text owner parked on an invalid artifacts root makes the required /// ordering observable: fresh graph activation must not start. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn fresh_graph_activation_waits_while_the_published_text_owner_is_parked() { @@ -338,7 +353,7 @@ async fn fresh_graph_activation_waits_while_the_published_text_owner_is_parked() assert_eq!( parked.staleness_state, Some(tracedecay_contracts::code_index_freshness::CodeIndexStalenessStateV1::Parked), - "the text owner must park on the unhealable root: {parked:?}" + "the text owner must park on the invalid root: {parked:?}" ); assert!( @@ -370,20 +385,13 @@ async fn fresh_graph_activation_waits_while_the_published_text_owner_is_parked() fixture.registry.shutdown().await; } -/// The published pass waits for the owners the seat needs, exact and -/// lexical, and nothing more. The clone-fingerprint successor that follows -/// the admission artifact re-decodes the whole sealed source into a second -/// artifact; on the 772-file lifecycle fixture that pass alone held graph -/// activation back by ~27 s (#1103). Fresh graph activation must start while -/// that successor is still pending, and the successor must still finish on a -/// later pass. +/// A publication's graph activation overlaps its own text projection once +/// that projection has opened its build: text readiness never waits on graph +/// activation, and the serving swap still waits for both. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn fresh_graph_activation_starts_while_the_clone_successor_is_pending() { - let (fixture, admission) = Fixture::mount_with_poisoned_artifacts_root_held( - "project.graph-before-clone-successor", - |_| {}, - ) - .await; +async fn fresh_graph_activation_never_delays_the_published_text_owner() { + let (fixture, admission) = + Fixture::mount_with_poisoned_artifacts_root_held("project.graph-beside-text", |_| {}).await; let scope = fixture .registry .serving_code_scope(&fixture.project) @@ -394,7 +402,7 @@ async fn fresh_graph_activation_starts_while_the_clone_successor_is_pending() { tokio::time::timeout(CONVERGENCE_DEADLINE, gate.wait_until_started()) .await - .expect("fresh graph activation starts once exact and lexical owners are ready"); + .expect("fresh graph activation starts beside the opened text projection"); let canonical = canonical_existing_identity(&fixture.project).expect("canonical project"); let text = { let mounted = fixture.registry.mounted.lock().await; @@ -407,29 +415,38 @@ async fn fresh_graph_activation_starts_while_the_clone_successor_is_pending() { .clone() } .expect("the publication installed its text owner before activation"); + tokio::time::timeout(CONVERGENCE_DEADLINE, async { + while !text.query_owners_are_ready() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("the text owner finishes while graph activation is held"); assert!( - text.query_owners_are_ready(), - "graph activation must not start before exact and lexical owners are ready" - ); - assert!( - text.text_projection_needs_work(), - "the clone-fingerprint successor must still be pending when activation starts" + fixture + .registry + .serving_code_scope(&fixture.project) + .await + .expect("mounted scope") + .serving_generation + .is_none(), + "the seat waits for the held graph activation" ); gate.release(); - - let deadline = tokio::time::Instant::now() + CONVERGENCE_DEADLINE; - while text.text_projection_needs_work() { - assert!( - tokio::time::Instant::now() < deadline, - "the clone successor must finish on a follow-up pass after the seat" - ); - fixture.wake_without_new_input().await; - tokio::time::sleep(POLL_SPACING).await; - } - assert!( - text.query_owners_are_ready(), - "finishing the successor must keep exact and lexical owners ready" - ); + tokio::time::timeout(CONVERGENCE_DEADLINE, async { + while fixture + .registry + .serving_code_scope(&fixture.project) + .await + .expect("mounted scope") + .serving_generation + .is_none() + { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("the publication seats once graph activation and text are done"); fixture.registry.shutdown().await; } diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/mount.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/mount.rs index f7741c5801..4207f7724a 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/mount.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/mount.rs @@ -406,9 +406,9 @@ impl CodeIndexSchedulerRegistryV1 { // late complete-generation request starts a successor pass; that // successor must neither detach nor duplicate the text owner. let mut retained_text_projection = None; - // Whether the retained projection in flight started with exact - // and lexical owners already serving, so it only backfills clone - // fingerprints and its finish owes the worker no successor pass. + // Whether the retained projection in flight started with its + // query owners already serving, so its finish changes no owner the + // seat reads and owes the worker no successor pass. let mut retained_projection_successor_only = false; loop { hotpath::future!( @@ -540,7 +540,7 @@ impl CodeIndexSchedulerRegistryV1 { let mut reconcile_pass = Some(super::super::ReconcilePassGuard::enter( &worker_reconcile_in_progress, )); - let mut clone_backfill_waiting_for_source = false; + let mut retained_work_waiting_for_source = false; let mut text_generation = worker_text_generation .read() .unwrap_or_else(std::sync::PoisonError::into_inner) @@ -588,13 +588,12 @@ impl CodeIndexSchedulerRegistryV1 { && latest.text_projection_needs_work() && graph_activation_enabled { - // Exact/lexical warming always runs here. Clone-fingerprint - // backfill is demand-driven *after* the seated generation - // matches this text owner and the source is current: starting - // it while the serving slot is empty, mismatched, or the - // checkout is dirty races the publish/seat path that still - // owns the receipt bound (cc-22286: phase=ready, graph - // pending, rebuild_in_flight, clone 0/N at the 45 s bound). + // Warming a missing owner always runs here. Work left behind + // an owner that already serves runs only once the seated + // generation matches this text owner and the source is + // current: starting it while the serving slot is empty, + // mismatched, or the checkout is dirty races the + // publish/seat path that still owns the receipt bound. let owners_ready = latest.query_owners_are_ready(); let serving_matches_text = worker_serving_generation .read() @@ -606,9 +605,9 @@ impl CodeIndexSchedulerRegistryV1 { }); let source_current = worker_source_freshness .ready_without_stat(&worker_project_root, &worker_shutting_down); - // This flag schedules the successor; that successor - // re-checks `serving_matches_text` before doing backfill. - clone_backfill_waiting_for_source = owners_ready && !source_current; + // This flag schedules the successor pass; that pass + // re-checks `serving_matches_text` before driving it. + retained_work_waiting_for_source = owners_ready && !source_current; let drive_retained = !owners_ready || (serving_matches_text && source_current); if drive_retained { // The retained owner projects on its own task, exactly as @@ -627,13 +626,11 @@ impl CodeIndexSchedulerRegistryV1 { let gated_root = worker_project_root.clone(); // The pass guard is what `rebuild_in_flight`, the // `verifying` freshness state and a read's busy fence - // consult: it means exact or lexical serving is still - // being produced. An owner whose query owners already - // serve has only the clone-fingerprint backfill left, and - // holding the guard for that reported a complete current - // generation as verifying / partial_source_verification - // for the whole backfill (#1103). The worker still owns - // and joins the task, so shutdown sees the work. + // consult: it means query serving is still being + // produced. Holding it for work behind owners that + // already serve reported a complete current generation + // as verifying. The worker still owns and joins + // the task, so shutdown sees the work. retained_projection_successor_only = owners_ready; let projection_pass = (!retained_projection_successor_only).then(|| { super::super::ReconcilePassGuard::enter(&worker_reconcile_in_progress) @@ -649,6 +646,7 @@ impl CodeIndexSchedulerRegistryV1 { shutting_down, park, Some(installed), + None, #[cfg(test)] gated_root, ) @@ -1080,13 +1078,16 @@ impl CodeIndexSchedulerRegistryV1 { Ok(Ok(CodeIndexReconcileOutcomeV1::Published(_))) ); let mut graph_text = retained_text.clone(); - // The replacement owner's bounded projection must finish - // before a fresh graph publication starts. Both consume the - // same sealed generation and are corpus-sized: overlapping - // them lets graph replay hold the source while text opens it, - // then leaves text unable to reacquire its reservation after - // graph publication reaches the process RSS watermark. + // The replacement owner's bounded projection runs on its own + // task while this pass prepares and activates the graph. Both + // consume the same sealed generation and are corpus-sized, so + // graph starts only after text's first advance has opened the + // build and taken its reservation: graph replay can then no + // longer hold the source or the RSS headroom text needs to + // open. The serving swap still waits for the text outcome. let mut published_text_projection_outcome = None; + let mut published_text_projection = None; + let mut published_text_opened = None; if published_pass { *worker_text_generation .write() @@ -1128,54 +1129,69 @@ impl CodeIndexSchedulerRegistryV1 { } Ok(Ok(Ok(None)) | Err(_)) | Err(_) => None, }; - // Finish the replacement text owner before optional - // O(store) graph work below. Exact and lexical are the - // required fresh-index product; graph activation is an - // optional projection and must not consume the source or - // resident-memory headroom needed to build them. - // Yielding back to the loop instead would hand the next - // pass a checkout that has already moved, and on a shared - // repository that pass publishes again - which is exactly - // how a sealed generation stayed unseated forever. + // Drive the replacement text owner in this pass. Exact and + // lexical are the required fresh-index product; graph + // activation is an optional projection that must not take + // the source or resident-memory headroom text needs to + // open. Yielding back to the loop instead would hand the + // next pass a checkout that has already moved, and on a + // shared repository that pass publishes again - which is + // exactly how a sealed generation stayed unseated forever. if graph_activation_enabled && !graph_activation_deferred && let Some(text) = graph_text.clone() { - // `reconcile_pass` is held across this projection, so - // the pointer rename is inside the pass a reader - // samples. Taking the admission permit back here - // instead would deadlock against an - // ignored-dependency owner that already holds it and - // is waiting for `_build_publication`. - let projection = tokio::spawn(Self::drive_text_projection( - text, - Arc::clone(&worker_shutting_down), - Arc::clone(&worker_convergence_park), - None, - #[cfg(test)] - worker_project_root.clone(), - )); - published_text_projection_outcome = Some(match projection.await { - Ok(outcome) => outcome, - Err(error) => { - if let Some(text) = graph_text.as_ref() { - text.mark_text_serving_failed(); + // `reconcile_pass` covers this projection, so the + // pointer rename is inside the pass a reader samples. + // The task releases it when text stops, after stamping + // the continuation the projection still owes: graph + // work that outlives text is not query serving. Taking + // the admission permit back here instead would + // deadlock against an ignored-dependency owner that + // already holds it and is waiting for + // `_build_publication`. + let projection_pass = if retained_text_projection.is_none() + || retained_projection_successor_only + { + reconcile_pass.take() + } else { + None + }; + let (opened, text_opened) = tokio::sync::oneshot::channel(); + published_text_opened = Some(text_opened); + let shutting_down = Arc::clone(&worker_shutting_down); + let park = Arc::clone(&worker_convergence_park); + let projection_pending_wake = Arc::clone(&worker_pending_wake); + let projection_wake = Arc::clone(&worker_wake); + #[cfg(test)] + let project_root = worker_project_root.clone(); + published_text_projection = Some(tokio::spawn(async move { + let _projection_pass = projection_pass; + let outcome = Self::drive_text_projection( + text.clone(), + shutting_down, + park, + None, + Some(opened), + #[cfg(test)] + project_root, + ) + .await; + let schedule_continuation = match outcome { + PublishedTextProjectionOutcomeV1::Finished => { + text.text_projection_needs_work() } - park_convergence( - &worker_convergence_park, - format!("code text projection task failed abnormally: {error}"), - CONVERGENCE_PARK_TASK_FAILURE_REMEDIATION_V1, - None, - false, - ); - tracing::warn!( - event = "code_index_text_projection_task_failed", - error = %error, - "published text projection task failed before graph seating" + PublishedTextProjectionOutcomeV1::Unfinished => true, + PublishedTextProjectionOutcomeV1::Shutdown => false, + }; + if schedule_continuation { + Self::note_worker_continuation( + &projection_pending_wake, + &projection_wake, ); - PublishedTextProjectionOutcomeV1::Unfinished } - }); + outcome + })); } else if graph_text .as_ref() .is_none_or(LatestCodeTextGenerationV1::text_projection_needs_work) @@ -1196,30 +1212,15 @@ impl CodeIndexSchedulerRegistryV1 { // unchanged pass still seats the retained Ready text owner's // generation. Source reconciliation is complete either way: // release its public freshness guard before the optional - // O(store) full decode and native graph activation begin, - // after this pass finishes the publication's text projection. - // Optional graph must not hold it: each graph step that must take the scheduler - // re-enters the pass around that acquisition (see - // `lock_scheduler_for_graph_step`); only the unlocked decode - // and native activation run outside it. + // O(store) full decode and native graph activation begin; a + // publication's projection task holds its own share until text + // stops. Optional graph must not hold it: each graph step that + // must take the scheduler re-enters the pass around that + // acquisition (see `lock_scheduler_for_graph_step`); only the + // unlocked decode and native activation run outside it. // A successor-only retained projection holds no pass guard of // its own; keeping the worker's guard through graph seat would - // report rebuild_in_flight for clone backfill that is not - // exact/lexical work. Stamp the continuation this projection - // already owes before that drop: the slot, not a later note, - // is what an idle reader observes. - if let Some(outcome) = published_text_projection_outcome.as_ref() { - let schedule_continuation = match outcome { - PublishedTextProjectionOutcomeV1::Finished => graph_text - .as_ref() - .is_some_and(LatestCodeTextGenerationV1::text_projection_needs_work), - PublishedTextProjectionOutcomeV1::Unfinished => true, - PublishedTextProjectionOutcomeV1::Shutdown => false, - }; - if schedule_continuation { - Self::note_worker_continuation(&worker_pending_wake, &worker_wake); - } - } + // report rebuild_in_flight for work that is not query serving. if retained_text_projection.is_none() || retained_projection_successor_only { drop(reconcile_pass.take()); } @@ -1229,7 +1230,8 @@ impl CodeIndexSchedulerRegistryV1 { matches!(&source_result, Ok(Ok(_))), published_pass, if published_pass { - exact_and_lexical_ready_for_graph(graph_text.as_ref()) + published_text_projection.is_some() + || exact_and_lexical_ready_for_graph(graph_text.as_ref()) } else { graph_text.is_some() }, @@ -1498,8 +1500,11 @@ impl CodeIndexSchedulerRegistryV1 { // decode and replay below. Otherwise a failed fresh text pass // becomes a retained pass on its next wake and recreates the // same source and resident-memory contention we avoid above. - // Same named predicate as the published seat gate above. + // A publication's own projection is the exception: it already + // holds its build reservation once `opened` fires, and the + // swap joins it before seating. if prepare_graph + && published_text_projection.is_none() && !graph_already_serves && !exact_and_lexical_ready_for_graph(graph_text.as_ref()) { @@ -1511,6 +1516,22 @@ impl CodeIndexSchedulerRegistryV1 { "full graph replay waits for exact and lexical projection" ); } + // A projection that stopped before opening its build (parked, + // failed, or already ready) admits graph only on ready owners. + if prepare_graph + && let Some(opened) = published_text_opened.take() + && opened.await.is_err() + { + prepare_graph = exact_and_lexical_ready_for_graph(graph_text.as_ref()); + if !prepare_graph { + tracing::debug!( + event = "code_index_graph_seat_skipped", + reason = "text_projection_unfinished", + published_pass, + "fresh graph publication waits for a text projection that did not open" + ); + } + } let mut result = match source_result { Ok(mut outcome) if prepare_graph => { let graph_scheduler = Arc::clone(&worker_scheduler); @@ -1772,9 +1793,32 @@ impl CodeIndexSchedulerRegistryV1 { } } } - // Process the fresh text outcome at the existing source-proof - // and serving-swap boundary. Graph work above ran only when - // the outcome was ready. + // Join the publication's projection, then process its outcome + // at the existing source-proof and serving-swap boundary. Graph + // work above overlapped it; nothing seats before text is done. + if let Some(projection) = published_text_projection.take() { + published_text_projection_outcome = Some(match projection.await { + Ok(outcome) => outcome, + Err(error) => { + if let Some(text) = graph_text.as_ref() { + text.mark_text_serving_failed(); + } + park_convergence( + &worker_convergence_park, + format!("code text projection task failed abnormally: {error}"), + CONVERGENCE_PARK_TASK_FAILURE_REMEDIATION_V1, + None, + false, + ); + tracing::warn!( + event = "code_index_text_projection_task_failed", + error = %error, + "published text projection task failed before graph seating" + ); + PublishedTextProjectionOutcomeV1::Unfinished + } + }); + } if let Some(outcome) = published_text_projection_outcome.take() { // A clone-fingerprint successor is still `Unfinished` work // after exact and lexical owners are ready. That must not @@ -1790,15 +1834,11 @@ impl CodeIndexSchedulerRegistryV1 { }; match outcome { PublishedTextProjectionOutcomeV1::Finished => { - // The seat needs only the ready exact/lexical - // owners. A clone-fingerprint successor left in - // the slot is retained-owner work on the next - // pass (no pass guard once owners already serve). - // Schedule that pass here: there is no periodic - // cadence timer, and leaving the successor parked - // until the first similar/redundancy request made - // that request own the whole backfill inline on a - // Tokio worker thread (#1339 change-risk S1). + // The seat needs only the ready owners. Text work + // left in the slot is retained-owner work on the + // next pass (no pass guard once owners already + // serve); schedule that pass here, since there is + // no periodic cadence timer. if graph_text .as_ref() .is_some_and(LatestCodeTextGenerationV1::text_projection_needs_work) @@ -2043,17 +2083,9 @@ impl CodeIndexSchedulerRegistryV1 { ), ServingSwapOutcomeV1::Offered => {} } - // A seated owner whose exact and lexical serving - // are ready still owes its clone-fingerprint - // backfill, and this worker owns that slice. - // Leaving it for query demand only looked free: - // the worker has no cadence timer, it blocks on - // `wake.notified()`, so the next search had to - // stamp the pending-wake slot to deliver it, and - // the freshness ladder reads that slot as - // `refresh_in_flight` and answered `verifying` - // for a seat whose source proof was current. - // Stamp what the two sibling publication sites + // Text work a seated owner still owes is this + // worker's slice: the worker has no cadence timer, + // so stamp what the two sibling publication sites // above already stamp. if text_latest.text_projection_needs_work() { Self::note_visible_worker_continuation( @@ -2089,10 +2121,10 @@ impl CodeIndexSchedulerRegistryV1 { } // The source proof and serving witness are now published as // one lifecycle. Optional receipts do not keep source - // verification in flight. A clone-backfill continuation this + // verification in flight. A retained-work continuation this // pass already knows about is stamped first, so the drop is // not an empty slot. - if clone_backfill_waiting_for_source + if retained_work_waiting_for_source && matches!( &result, Ok((Ok(CodeIndexReconcileOutcomeV1::Noop(_)), _, _)) @@ -2145,7 +2177,7 @@ impl CodeIndexSchedulerRegistryV1 { // scheduler lock; announce the change now that the // proof is public. worker_serving_generation_changed.send_replace(()); - // The clone-backfill continuation was stamped before + // The retained-work continuation was stamped before // this pass dropped `reconcile_in_progress`. } } else { @@ -2373,8 +2405,8 @@ impl CodeIndexSchedulerRegistryV1 { // retained text task was still running. Give the // now-ready owner one bounded successor pass so a // full replay can proceed without overlapping it. - // A clone-fingerprint backfill changed no owner - // the seat reads, so it owes no such pass. + // Work behind owners that already served changed + // no owner the seat reads, so it owes no such pass. Self::note_worker_continuation(&worker_pending_wake, &worker_wake); } PublishedTextProjectionOutcomeV1::Finished => { @@ -2478,7 +2510,7 @@ impl CodeIndexSchedulerRegistryV1 { /// Sole exact/lexical-ready bit for the published graph seat gate and the /// full sealed-generation replay skip. Delegates to /// [`LatestCodeTextGenerationV1::query_owners_are_ready`] so those two sites -/// cannot fork; clone backfill is not part of this bit. +/// cannot fork. #[inline] fn exact_and_lexical_ready_for_graph(text: Option<&LatestCodeTextGenerationV1>) -> bool { text.is_some_and(LatestCodeTextGenerationV1::query_owners_are_ready) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/seat_swap_tests.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/seat_swap_tests.rs index cb0ad8e762..873e323a54 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/seat_swap_tests.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/seat_swap_tests.rs @@ -1,10 +1,9 @@ -/// Clone-fingerprint backfill is still unfinished after exact and lexical -/// owners are ready. That successor is not `published_text_owner_unfinished`. +/// Only missing query owners withhold the seat. #[test] -fn unfinished_clone_fingerprint_successor_is_not_text_projection_unfinished() { +fn only_missing_query_owners_withhold_the_seat() { assert!( !super::text_projection_unfinished_withholds_seat(true), - "ready exact and lexical owners must still seat while the clone successor runs" + "ready query owners must seat" ); assert!( super::text_projection_unfinished_withholds_seat(false), diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/serving.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/serving.rs index b817a30c4f..e9a2c17db2 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/serving.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/serving.rs @@ -4,9 +4,8 @@ mod family_report; use std::{ collections::VecDeque, - fs::File, io::Read, - num::{NonZeroU64, NonZeroUsize}, + num::NonZeroU64, path::{Path, PathBuf}, sync::{ Arc, Condvar, Mutex, MutexGuard, OnceLock, PoisonError, RwLock, @@ -24,8 +23,9 @@ use tracedecay_code_extraction::{ }; use tracedecay_code_index_retention::code_index_generations::{ CodeGenerationStoreLockV1, DurableCodeTextArtifactDescriptorV1, DurablePublicationPointerV1, - DurableSealedCodeGenerationIdentityV1, attach_verified_text_artifact_under_lock, - code_text_artifact_path, code_text_artifacts_root, replace_verified_text_artifact_under_lock, + DurableSealedCodeGenerationIdentityV1, acquire_generation_segments_publication_lock, + attach_verified_text_artifact_under_lock, code_text_artifact_path, + code_text_artifact_staging_root, code_text_artifacts_root, find_shared_text_artifact, try_acquire_code_generation_store_lock, withdraw_verified_text_artifact_under_lock, }; use tracedecay_contracts::{ @@ -42,9 +42,7 @@ use tracedecay_domain::{ RetrieverOutcome, ScoreDomainId, WorktreeId, canonical_text::encode_lowercase_hex, sha256_hex_suffix, }; -use tracedecay_private_fs::{ - make_private_directory, open_private_file, validate_private_directory, -}; +use tracedecay_private_fs::{open_private_file, validate_private_directory}; use tracedecay_runtime_core::resident_memory::{ ProcessResidentMemoryV1, ResidentMemoryComponentIdV1, ResidentMemoryKeyV1, ResidentMemoryReservationV1, sampled_process_resident_bytes_v1, @@ -57,10 +55,9 @@ use crate::{ CodeIndexExecutionControlV1, CodeIndexProductionErrorV1, CodeIndexPublicationStoreErrorV1, CodeIndexPublishedGenerationV1, SealedGenerationSegmentReadV1, VerifiedSealedLexicalCursorRestoreErrorV1, - VerifiedSealedLexicalCursorV1, VerifiedSealedLexicalPageBatchBoundsV1, - VerifiedSealedLexicalPageBatchReadV1, VerifiedSealedLexicalPageReadV1, - VerifiedSealedLexicalPageSourceV1, VerifiedSealedLexicalPageV1, - VerifiedSealedLexicalSourceReceiptV1, VerifiedSealedTextGenerationMetadataV1, + VerifiedSealedLexicalPageBatchBoundsV1, VerifiedSealedLexicalPageBatchReadV1, + VerifiedSealedLexicalPageSourceV1, VerifiedSealedLexicalSourceReceiptV1, + VerifiedSealedTextGenerationMetadataV1, }, }, query::retrieval::{ @@ -74,15 +71,14 @@ use crate::{ CLONE_FINGERPRINT_POSTING_ROW_BUDGET_V1, CLONE_NEAR_MATCH_BODY_COMPARISON_BUDGET_V1, CLONE_NEAR_MATCH_MINIMUM_COVERAGE_MILLIONTHS_V1, CLONE_NEAR_MATCH_TOKEN_WORK_BUDGET_V1, CODE_LEXICAL_ARTIFACT_BUILD_MEMORY_BUDGET_BYTES_V1, - CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, - CODE_LEXICAL_ARTIFACT_SQLITE_CACHE_BYTES_V1, CodeExactLexicalArtifactReaderV1, + CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, CodeExactLexicalArtifactReaderV1, CodeLexicalArtifactBuilderV1, CodeLexicalArtifactErrorV1, CodeLexicalArtifactFinalizationPhaseV1, CodeLexicalArtifactFinalizationStepV1, CodeLexicalArtifactOccurrenceV1, CodeLexicalArtifactReaderV1, - CodeLexicalArtifactWriterRevisionV1, CodeLexicalCloneIndexCensusV1, - CodeLexicalCloneSuccessorV1, CodeLexicalProjectionMetadataV1, LexicalLane, - LexicalLaneEvidence, LexicalLaneRequest, LexicalLaneRetriever, - code_lexical_artifact_build_memory_budget_for, + CodeLexicalCloneIndexCensusV1, CodeLexicalCloneRouteV1, + CodeLexicalProjectionMetadataV1, LexicalLane, LexicalLaneEvidence, LexicalLaneRequest, + LexicalLaneRetriever, PreparedCodeLexicalArtifactPageV1, + code_lexical_artifact_build_memory_budget_for, code_lexical_artifact_content_key, }, ports::{RETRIEVAL_CANDIDATE_BATCH_SIZE, RetrievalPortError}, }, @@ -94,12 +90,8 @@ use super::{DaemonCodeIndexPublicationStoreV1, ProfiledStdMutex, queries}; /// text artifact. One page is one bounded unit of background build progress. pub(super) const TEXT_ARTIFACT_PAGE_CHUNKS_V1: usize = RETRIEVAL_CANDIDATE_BATCH_SIZE; const TEXT_ARTIFACT_PAGE_BYTES_V1: usize = 4 * 1024 * 1024; -const CLONE_SUCCESSOR_MEMORY_BUDGET_BYTES_V1: usize = 128 * 1024 * 1024; const TEXT_ARTIFACT_BASE_BATCH_PAGES_V1: usize = 64; const TEXT_ARTIFACT_BASE_BATCH_BYTES_V1: usize = 64 * 1024 * 1024; -/// Live JSON copies `append_clone_rows` holds for one clone body while the -/// staged pages remain live: payload, occurrence, stored payload, comparison. -const CLONE_SUCCESSOR_APPEND_LIVE_COPIES_V1: usize = 4; const TEXT_ARTIFACT_MAXIMUM_BATCH_SCALE_V1: usize = 8; /// One synchronous activation advances only this many page/finalization /// operations. Larger caller hints are clamped so work accounting cannot @@ -119,60 +111,6 @@ pub(super) fn text_artifact_source_batch_limits( (pages, bytes, pages * 2) } -/// Clone-successor page-batch bounds from the 128 MiB reservation ledger. -/// -/// `open_builder_connection` grants the full `SQLite` cache. `append_clone_rows` -/// then serializes payload/occurrence, reads the stored payload, and compares -/// a second serialization while the batch's pages stay live, and the builder -/// retains metadata. The batch byte ceiling is the remainder after those -/// charges so resident-memory admission is not filled to the last byte. -/// -/// The ceiling is capped at the scale-1 first-pass batch (64 MiB), while -/// the first pass itself scales up to 8x, so a single page retained above -/// this ceiling would be admitted there and refused here as a `Contract` -/// error. That is unreachable in practice only because the sealed source -/// caps every page at `TEXT_ARTIFACT_PAGE_BYTES_V1` (4 MiB) serialized. -pub(super) fn clone_successor_source_batch_limits( - metadata: &CodeLexicalProjectionMetadataV1, -) -> Result<(usize, usize), RetrievalPortError> { - clone_successor_source_batch_limits_from_charges(metadata.retained_owned_bytes()) -} - -pub(super) fn clone_successor_source_batch_limits_from_charges( - metadata_bytes: usize, -) -> Result<(usize, usize), RetrievalPortError> { - let scratch = CLONE_SUCCESSOR_APPEND_LIVE_COPIES_V1 - .checked_mul(TEXT_ARTIFACT_PAGE_BYTES_V1) - .ok_or_else(|| { - RetrievalPortError::Contract( - "clone-successor append-scratch ledger overflowed".to_owned(), - ) - })?; - let fixed = CODE_LEXICAL_ARTIFACT_SQLITE_CACHE_BYTES_V1 - .checked_add(metadata_bytes) - .and_then(|bytes| bytes.checked_add(scratch)) - .ok_or_else(|| { - RetrievalPortError::Contract("clone-successor fixed ledger overflowed".to_owned()) - })?; - if fixed >= CLONE_SUCCESSOR_MEMORY_BUDGET_BYTES_V1 { - return Err(RetrievalPortError::Contract(format!( - "clone-successor SQLite cache, metadata, and append scratch exhaust the {CLONE_SUCCESSOR_MEMORY_BUDGET_BYTES_V1}-byte reservation" - ))); - } - let remaining = CLONE_SUCCESSOR_MEMORY_BUDGET_BYTES_V1 - fixed; - if remaining < TEXT_ARTIFACT_PAGE_BYTES_V1 { - return Err(RetrievalPortError::Contract(format!( - "clone-successor reservation leaves {remaining} bytes for pages, under the {TEXT_ARTIFACT_PAGE_BYTES_V1}-byte source page bound" - ))); - } - let bytes = remaining.min(TEXT_ARTIFACT_BASE_BATCH_BYTES_V1); - let slot = std::mem::size_of::(); - let pages = TEXT_ARTIFACT_BASE_BATCH_PAGES_V1 - .min(bytes / slot.max(1)) - .max(1); - Ok((pages, bytes)) -} - /// Cancellation-checkpoint cadence for a wake parked behind another wake's /// corpus-sized verified head open. The parked wake re-checks its typed /// cancellation state at this interval, so shutdown or supersession surfaces @@ -180,24 +118,11 @@ pub(super) fn clone_successor_source_batch_limits_from_charges( /// digest call that has not yet reached its own checkpoint. const TEXT_HEAD_OPEN_CANCELLATION_CHECK_INTERVAL_V1: Duration = Duration::from_millis(100); const TEXT_ARTIFACT_MAXIMUM_OWNER_WARMUP_ADVANCES_V1: usize = 10_000; -/// Clone-fingerprint backfill slices a request may drive inline. The retained -/// worker owns the rest after `request_query_background_reconcile`; more than -/// one advance here re-owns the whole successor encode on a Tokio thread. -const TEXT_ARTIFACT_MAXIMUM_CLONE_WARMUP_ADVANCES_V1: usize = 1; /// Rows digested by one scheduler finalization operation. The builder persists /// its exact section/row cursor after this bounded slice, avoiding both a /// corpus-sized wake and one scheduler wake per individual `SQLite` row. const TEXT_ARTIFACT_FINALIZATION_ROWS_PER_OPERATION_V1: usize = 4 * 1024; -/// Outcome of the one-slice clone-fingerprint warmup on a similar/redundancy -/// request. `Pending` means the retained worker owns remaining backfill, -/// never collapse that into a hard `GenerationUnavailable` miss. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum CloneSimilarityWarmupForRequestV1 { - Ready, - Pending, -} - pub(super) type GenerationServingCachesV1 = ( CodeGenerationId, Arc>>>, @@ -515,8 +440,7 @@ pub struct LatestCodeTextGenerationV1 { /// A cold graph-off bind authenticates the sealed source once and hands /// that same reader to the artifact build. The full generation is never /// decoded merely to discover text metadata or source layout. - pub(super) preopened_source: - Arc>>>, + pub(super) preopened_source: Arc>>, pub(super) publication_binding: Option>, } @@ -563,8 +487,6 @@ struct CloneIndexArtifactSnapshotV1 { census: Option, format_revision: u32, bytes_on_disk: u64, - source_pages: u64, - has_fingerprints: bool, } impl ProductionCodeIndexQueryOwnersV1 { @@ -601,8 +523,6 @@ impl ProductionCodeIndexQueryOwnersV1 { .map(|census| census.as_ref().clone()), format_revision: self.hydration.artifact_format_revision(), bytes_on_disk: self.hydration.verified_artifact().file_size_bytes(), - source_pages: self.hydration.verified_artifact().page_count(), - has_fingerprints: self.hydration.has_clone_fingerprints(), }) } @@ -794,8 +714,10 @@ pub(super) enum CodeTextQueryOwnerReadinessV1 { /// verified page source over this generation's durable sealed file. pub(super) struct CodeTextArtifactBuildV1 { pub(super) builder: CodeLexicalArtifactBuilderV1, - pub(super) source: VerifiedSealedLexicalPageSourceV1, + pub(super) source: VerifiedSealedLexicalPageSourceV1, sealed_identity: DurableSealedCodeGenerationIdentityV1, + /// The key the artifact is published under, derived before the build. + content_key: ManifestDigest, source_receipt: Option, pub(super) staging_path: PathBuf, /// Holds the builder's advertised memory ceiling reserved in the @@ -803,23 +725,6 @@ pub(super) struct CodeTextArtifactBuildV1 { _build_reservation: ResidentMemoryReservationV1, } -pub(super) struct CodeTextCloneSuccessorBuildV1 { - builder: Option, - source: VerifiedSealedLexicalPageSourceV1, - source_position: CloneSuccessorSourcePositionV1, - sealed_identity: DurableSealedCodeGenerationIdentityV1, - source_receipt: Option, - prior: tracedecay_query::retrieval::lexical::VerifiedCodeLexicalArtifactV1, - prior_descriptor: DurableCodeTextArtifactDescriptorV1, - staging_path: PathBuf, - build_reservation: Option, -} - -enum CloneSuccessorSourcePositionV1 { - Appending, - Revalidating(VerifiedSealedLexicalCursorV1), -} - /// Singleflight authority for one generation's durable text projection. /// /// The slot is the generation-owned partial-state authority; the condvar @@ -847,8 +752,6 @@ pub(super) enum CodeTextProjectionSlotV1 { /// The resumable staging build; each wake advances one bounded slice /// under the slot lock. Building(Box), - CloneSuccessorPending, - BuildingCloneSuccessor(Box), } impl CodeTextProjectionStateV1 { @@ -864,17 +767,6 @@ impl CodeTextProjectionStateV1 { self.slot.lock().unwrap_or_else(PoisonError::into_inner) }) } - - fn retain_clone_successor_retry(&self) -> Result<(), RetrievalPortError> { - let mut slot = self.lock_slot(); - if !matches!(&*slot, CodeTextProjectionSlotV1::HeadOpening) { - return Err(RetrievalPortError::Contract( - "clone-successor retry requires an active head-open claim".to_owned(), - )); - } - *slot = CodeTextProjectionSlotV1::CloneSuccessorPending; - Ok(()) - } } /// One wake's exclusive claim on a corpus-sized verified head open. @@ -887,11 +779,6 @@ struct TextHeadOpenClaimV1<'a> { armed: bool, } -enum TextHeadOpenBuildV1 { - Artifact(Box), - CloneSuccessor(Box), -} - impl<'a> TextHeadOpenClaimV1<'a> { /// The caller must already have transitioned the slot to `HeadOpening` /// and released the lock; this guard owns restoring it. @@ -899,15 +786,13 @@ impl<'a> TextHeadOpenClaimV1<'a> { Self { state, armed: true } } - fn install(&mut self, build: TextHeadOpenBuildV1) -> MutexGuard<'a, CodeTextProjectionSlotV1> { + fn install( + &mut self, + build: Box, + ) -> MutexGuard<'a, CodeTextProjectionSlotV1> { self.armed = false; let mut slot = self.state.lock_slot(); - *slot = match build { - TextHeadOpenBuildV1::Artifact(build) => CodeTextProjectionSlotV1::Building(build), - TextHeadOpenBuildV1::CloneSuccessor(build) => { - CodeTextProjectionSlotV1::BuildingCloneSuccessor(build) - } - }; + *slot = CodeTextProjectionSlotV1::Building(build); self.state.ready.notify_all(); slot } @@ -936,7 +821,6 @@ pub(super) enum TextHeadOpenOutcomeV1 { /// No published head was servable; the resumable staging build begins /// (or resumes) from its durable staging file. Build(Box), - BuildCloneSuccessor(Box), } fn map_text_artifact_error(error: CodeLexicalArtifactErrorV1) -> RetrievalPortError { @@ -1252,17 +1136,17 @@ impl DaemonCodeTextArtifactStoreV1 { .generation_index .iter() .find(|entry| entry.generation_id == generation_id.as_str()) - .and_then(|entry| entry.text_artifact.clone())) + .and_then(|entry| entry.text_artifact().cloned())) } /// Withdraw one exact missing/corrupt derived artifact so the immutable - /// sealed generation can rebuild it. A corrupt regular file is moved out - /// of the content-addressed namespace before the durable pointer is - /// cleared; non-regular objects are preserved and refused fail-closed. + /// sealed generation can rebuild it. A corrupt regular file is deleted + /// before the durable pointer is cleared; non-regular objects are + /// preserved and refused fail-closed. fn withdraw_unavailable_descriptor( &self, descriptor: &DurableCodeTextArtifactDescriptorV1, - quarantine_corrupt_file: bool, + delete_corrupt_file: bool, control: &dyn CodeIndexExecutionControlV1, ) -> Result<(), RetrievalPortError> { checkpoint_text_artifact_control(control)?; @@ -1280,15 +1164,17 @@ impl DaemonCodeTextArtifactStoreV1 { .generation_index .iter() .find(|entry| entry.generation_id == descriptor.generation_id.as_str()) - .and_then(|entry| entry.text_artifact.as_ref()); + .and_then(|entry| entry.text_artifact()); if current != Some(descriptor) { return Err(RetrievalPortError::AuthorityUnavailable( "durable text-artifact attachment changed during repair".to_owned(), )); } - let mut quarantined = None; - if quarantine_corrupt_file { + // A corrupt file is deleted before its descriptor is withdrawn: a crash + // between the two leaves a descriptor naming a missing file, which the + // next open withdraws, and never a second copy of damaged bytes. + if delete_corrupt_file { let path = code_text_artifact_path(&self.store_root, descriptor) .map_err(text_artifact_unavailable)?; let metadata = path.symlink_metadata().map_err(text_artifact_unavailable)?; @@ -1297,29 +1183,14 @@ impl DaemonCodeTextArtifactStoreV1 { "corrupt code text artifact is not a regular file".to_owned(), )); } - let quarantine = - path.with_extension(format!("corrupt-{}-{}", std::process::id(), now_micros().0)); - std::fs::rename(&path, &quarantine).map_err(text_artifact_unavailable)?; + std::fs::remove_file(&path).map_err(text_artifact_unavailable)?; DaemonCodeIndexPublicationStoreV1::sync_directory(path.parent().ok_or_else(|| { RetrievalPortError::Contract("code text artifact path has no parent".to_owned()) })?) .map_err(text_artifact_unavailable)?; - quarantined = Some(quarantine); } - withdraw_verified_text_artifact_under_lock(&lock, &pointer, descriptor) .map_err(text_artifact_unavailable)?; - if let Some(quarantine) = quarantined { - std::fs::remove_file(&quarantine).map_err(text_artifact_unavailable)?; - DaemonCodeIndexPublicationStoreV1::sync_directory(quarantine.parent().ok_or_else( - || { - RetrievalPortError::Contract( - "quarantined code text artifact has no parent".to_owned(), - ) - }, - )?) - .map_err(text_artifact_unavailable)?; - } Ok(()) } @@ -1333,7 +1204,7 @@ impl DaemonCodeTextArtifactStoreV1 { control: &dyn CodeIndexExecutionControlV1, ) -> Result<(), RetrievalPortError> { checkpoint_text_artifact_control(control)?; - let artifacts_root = code_text_artifacts_root(&self.store_root); + let artifacts_root = code_text_artifact_staging_root(&self.store_root); if staging_path.parent() != Some(artifacts_root.as_path()) { return Err(RetrievalPortError::Contract( "text-artifact staging path is outside its canonical root".to_owned(), @@ -1401,13 +1272,13 @@ impl DaemonCodeTextArtifactStoreV1 { } /// Open the exact durable sealed file after the caller has admitted the - /// build's resident-memory ceiling. The lexical source verifies the whole - /// file content address during its one bounded structural scan. + /// build's resident-memory ceiling. The manifest is verified against its + /// content address here; each segment authenticates as the source reads it. pub(super) fn open_sealed_source( &self, identity: &DurableSealedCodeGenerationIdentityV1, control: &dyn CodeIndexExecutionControlV1, - ) -> Result, RetrievalPortError> { + ) -> Result { self.open_sealed_source_with_progress(identity, control, |_, _| {}) } @@ -1416,7 +1287,7 @@ impl DaemonCodeTextArtifactStoreV1 { identity: &DurableSealedCodeGenerationIdentityV1, control: &dyn CodeIndexExecutionControlV1, mut progress: F, - ) -> Result, RetrievalPortError> + ) -> Result where F: FnMut(u64, u64), { @@ -1441,11 +1312,9 @@ impl DaemonCodeTextArtifactStoreV1 { )); } progress(identity.size_bytes, identity.size_bytes); - let manifest = File::open(path).map_err(text_artifact_unavailable)?; let publication = self.publication.clone(); let source_identity = identity.clone(); VerifiedSealedLexicalPageSourceV1::open_partitioned_sealed( - manifest, &manifest_bytes, identity.digest.clone(), move |digest, expected_size, buffer, control| { @@ -1462,12 +1331,7 @@ impl DaemonCodeTextArtifactStoreV1 { TEXT_ARTIFACT_PAGE_CHUNKS_V1, TEXT_ARTIFACT_PAGE_BYTES_V1, ) - .map_err(map_sealed_page_source_error)? - .ok_or_else(|| { - RetrievalPortError::Contract( - "partitioned sealed lexical source is incompatible".to_owned(), - ) - }) + .map_err(map_sealed_page_source_error) } /// Durably publish one finalized staging artifact: content-address it, @@ -1478,17 +1342,7 @@ impl DaemonCodeTextArtifactStoreV1 { staging_path: &Path, generation_id: &CodeGenerationId, sealed_identity: &DurableSealedCodeGenerationIdentityV1, - control: &dyn CodeIndexExecutionControlV1, - ) -> Result { - self.publish_with_prior(staging_path, generation_id, sealed_identity, None, control) - } - - fn publish_with_prior( - &self, - staging_path: &Path, - generation_id: &CodeGenerationId, - sealed_identity: &DurableSealedCodeGenerationIdentityV1, - prior: Option<&DurableCodeTextArtifactDescriptorV1>, + content_key: &ManifestDigest, control: &dyn CodeIndexExecutionControlV1, ) -> Result { hotpath::measure_block!("query.artifact.store.publish", { @@ -1500,6 +1354,14 @@ impl DaemonCodeTextArtifactStoreV1 { // a plan made before the descriptor was attached. checkpoint_text_artifact_control(control)?; let lock = self.acquire_store_write_lock()?; + // The completed artifact is the project's: from the moment this + // publication finds or places it until its descriptor is durable, + // no scope's retention may collect it. + let _project_lock = + acquire_generation_segments_publication_lock(&self.store_root, &|| { + checkpoint_text_artifact_control(control).is_err() + }) + .map_err(text_artifact_unavailable)?; let (artifact_sha256, artifact_size_bytes) = hotpath::measure_block!( "query.artifact.store.state_digest", sha256_private_file_and_size(staging_path, control) @@ -1515,39 +1377,41 @@ impl DaemonCodeTextArtifactStoreV1 { artifact_digest: ManifestDigest::from_sha256_bytes(&artifact_sha256) .map_err(text_artifact_unavailable)?, artifact_size_bytes, + content_key: content_key.clone(), }; let final_path = artifacts_root.join(&descriptor.artifact_file); - match final_path.symlink_metadata() { - Ok(_) => { - // A digest-derived name is not proof that an existing filesystem - // object contains the named bytes. Verify the stable destination - // before withdrawing staging evidence; a symlink, non-regular - // object, truncated file, or same-name collision fails closed. - let (existing_sha256, existing_size_bytes) = hotpath::measure_block!( - "query.artifact.store.dedupe_compare", - sha256_private_file_and_size(&final_path, control) - )?; - if existing_size_bytes != artifact_size_bytes { - return Err(RetrievalPortError::Contract( - "existing code text artifact does not match its content address" - .to_owned(), - )); - } - if existing_sha256 != artifact_sha256 { - return Err(RetrievalPortError::Contract( - "existing code text artifact contains different bytes".to_owned(), - )); - } - retire_text_artifact_staging_family(staging_path) - .map_err(text_artifact_unavailable)?; + // A digest-derived name is not proof that an existing filesystem + // object contains the named bytes. A regular file that does not + // hash to its name is a damaged copy some worktree published: the + // verified staging file replaces it and its bytes are gone. A + // symlink or other non-regular object fails closed. + let existing = match final_path.symlink_metadata() { + Ok(_) => Some(hotpath::measure_block!( + "query.artifact.store.dedupe_compare", + sha256_private_file_and_size(&final_path, control) + )?), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => None, + Err(error) => return Err(text_artifact_unavailable(error)), + }; + if existing == Some((artifact_sha256, artifact_size_bytes)) { + retire_text_artifact_staging_family(staging_path) + .map_err(text_artifact_unavailable)?; + } else { + // Renaming over the damaged file unlinks it atomically. + if existing.is_some() { + tracing::warn!( + event = "code_index_shared_text_artifact_replaced", + artifact = %descriptor.artifact_file, + "a text artifact that did not match its content address was replaced" + ); } - Err(error) if error.kind() == std::io::ErrorKind::NotFound => { - std::fs::rename(staging_path, &final_path) - .map_err(text_artifact_unavailable)?; - clear_text_artifact_staging_sidecars(staging_path) + std::fs::rename(staging_path, &final_path).map_err(text_artifact_unavailable)?; + clear_text_artifact_staging_sidecars(staging_path) + .map_err(text_artifact_unavailable)?; + if let Some(staging_root) = staging_path.parent() { + DaemonCodeIndexPublicationStoreV1::sync_directory(staging_root) .map_err(text_artifact_unavailable)?; } - Err(error) => return Err(text_artifact_unavailable(error)), } hotpath::measure_block!( "query.artifact.store.seal_fsync", @@ -1565,21 +1429,12 @@ impl DaemonCodeTextArtifactStoreV1 { ) })?; hotpath::measure_block!("query.artifact.store.pointer_commit", { - match prior { - Some(prior) => replace_verified_text_artifact_under_lock( - &lock, - &pointer, - sealed_identity, - prior, - descriptor.clone(), - ), - None => attach_verified_text_artifact_under_lock( - &lock, - &pointer, - sealed_identity, - descriptor.clone(), - ), - } + attach_verified_text_artifact_under_lock( + &lock, + &pointer, + sealed_identity, + descriptor.clone(), + ) .map_err(text_artifact_unavailable) })?; Ok(descriptor) @@ -1711,10 +1566,9 @@ impl LatestCodeTextGenerationV1 { /// Exact and lexical query owners are installed for this generation. /// /// This is the sole readiness predicate for a publication's graph seat - /// gate and for admitting a full sealed-generation graph replay. Clone - /// fingerprint backfill may still be unfinished when this returns true, - /// that remaining work is `Self::text_projection_needs_work`, not a - /// seat or replay precondition. + /// gate and for admitting a full sealed-generation graph replay. The + /// artifact seals its clone index with its lexical rows, so ready owners + /// serve clone lookups too. pub fn query_owners_are_ready(&self) -> bool { matches!( self.query_owner_readiness(), @@ -1745,9 +1599,8 @@ impl LatestCodeTextGenerationV1 { return true; } // A read probe must not queue behind an advance: a wake holds the - // slot lock for its whole bounded slice, and a clone-fingerprint - // backfill slice over a large sealed source runs for seconds. A - // contended slot is by definition work in progress. + // slot lock for its whole bounded slice. A contended slot is by + // definition work in progress. match self.text_projection_build.slot.try_lock() { Ok(slot) => !matches!(&*slot, CodeTextProjectionSlotV1::Idle), Err(std::sync::TryLockError::WouldBlock) => true, @@ -1780,15 +1633,6 @@ impl LatestCodeTextGenerationV1 { }; } }; - let successor = match self.clone_successor_progress() { - CloneSuccessorProgressReadV1::Idle => None, - CloneSuccessorProgressReadV1::Backfilling(progress) => Some(progress), - CloneSuccessorProgressReadV1::Busy => { - return CodeCloneIndexStatusV1::Unavailable { - reason: "clone-index status is being updated".to_owned(), - }; - } - }; let artifact = match owners.clone_index_artifact() { Ok(artifact) => artifact, Err(error) => { @@ -1797,28 +1641,7 @@ impl LatestCodeTextGenerationV1 { }; } }; - let (completed_source_pages, total_source_pages, bytes_on_disk) = successor.map_or( - ( - artifact.source_pages, - artifact.source_pages, - Some(artifact.bytes_on_disk), - ), - |progress| { - ( - progress.completed_source_pages, - progress.total_source_pages, - progress.bytes_on_disk.or(Some(artifact.bytes_on_disk)), - ) - }, - ); - let observation = clone_index_observation( - self, - &artifact, - update, - completed_source_pages, - total_source_pages, - bytes_on_disk, - ); + let observation = clone_index_observation(self, &artifact, update); if source_is_stale { return CodeCloneIndexStatusV1::Stale { observation, @@ -1826,9 +1649,6 @@ impl LatestCodeTextGenerationV1 { .to_owned(), }; } - if successor.is_some() { - return CodeCloneIndexStatusV1::Backfilling { observation }; - } let omission_reasons = clone_index_omission_reasons(&artifact); if omission_reasons.is_empty() { CodeCloneIndexStatusV1::Ready { observation } @@ -1840,51 +1660,6 @@ impl LatestCodeTextGenerationV1 { } } - fn clone_successor_progress(&self) -> CloneSuccessorProgressReadV1 { - let slot = match self.text_projection_build.slot.try_lock() { - Ok(slot) => slot, - Err(std::sync::TryLockError::WouldBlock) => { - return CloneSuccessorProgressReadV1::Busy; - } - Err(std::sync::TryLockError::Poisoned(poisoned)) => poisoned.into_inner(), - }; - match &*slot { - CodeTextProjectionSlotV1::CloneSuccessorPending => { - let owners = match self.query_owner_readiness() { - CodeTextQueryOwnerReadinessV1::Ready(owners) => owners, - CodeTextQueryOwnerReadinessV1::Pending - | CodeTextQueryOwnerReadinessV1::Invalid => { - return CloneSuccessorProgressReadV1::Idle; - } - }; - CloneSuccessorProgressReadV1::Backfilling(CloneSuccessorProgressV1 { - completed_source_pages: 0, - total_source_pages: owners.hydration.verified_artifact().page_count(), - bytes_on_disk: None, - }) - } - CodeTextProjectionSlotV1::BuildingCloneSuccessor(build) => { - let completed_source_pages = build - .builder - .as_ref() - .and_then(|builder| builder.next_cursor().ok().flatten()) - .map_or(0, |cursor| cursor.next_page_ordinal()); - CloneSuccessorProgressReadV1::Backfilling(CloneSuccessorProgressV1 { - completed_source_pages, - total_source_pages: build.prior.page_count(), - bytes_on_disk: build - .staging_path - .metadata() - .ok() - .map(|metadata| metadata.len()), - }) - } - CodeTextProjectionSlotV1::Idle - | CodeTextProjectionSlotV1::HeadOpening - | CodeTextProjectionSlotV1::Building(_) => CloneSuccessorProgressReadV1::Idle, - } - } - pub(super) fn same_text_owner(&self, other: &Self) -> bool { Arc::ptr_eq(&self.text_projection_build, &other.text_projection_build) } @@ -1894,26 +1669,10 @@ impl LatestCodeTextGenerationV1 { } } -#[derive(Clone, Copy)] -struct CloneSuccessorProgressV1 { - completed_source_pages: u64, - total_source_pages: u64, - bytes_on_disk: Option, -} - -enum CloneSuccessorProgressReadV1 { - Idle, - Backfilling(CloneSuccessorProgressV1), - Busy, -} - fn clone_index_observation( text: &LatestCodeTextGenerationV1, artifact: &CloneIndexArtifactSnapshotV1, update: Option, - completed_source_pages: u64, - total_source_pages: u64, - bytes_on_disk: Option, ) -> CodeCloneIndexObservationV1 { let census = artifact.census.as_ref(); let peak_scratch_memory_bytes = text @@ -1944,30 +1703,16 @@ fn clone_index_observation( unique_payloads: census.map(|census| census.unique_payloads), payloads_reused: update.and_then(|update| update.payloads_reused), exact_postings: census.map(|census| census.exact_postings), - near_fingerprint_bodies: artifact - .has_fingerprints - .then(|| census.map(|census| census.near_fingerprint_bodies)) - .flatten(), - near_fingerprint_postings: artifact - .has_fingerprints - .then(|| census.map(|census| census.near_fingerprint_postings)) - .flatten(), - hot_postings_skipped: artifact - .has_fingerprints - .then(|| census.map(|census| census.hot_postings)) - .flatten(), - hot_posting_rows_skipped: artifact - .has_fingerprints - .then(|| census.map(|census| census.hot_posting_rows)) - .flatten(), + near_fingerprint_bodies: census.map(|census| census.near_fingerprint_bodies), + near_fingerprint_postings: census.map(|census| census.near_fingerprint_postings), + hot_postings_skipped: census.map(|census| census.hot_postings), + hot_posting_rows_skipped: census.map(|census| census.hot_posting_rows), excluded_too_small_bodies: census.map(|census| census.excluded_too_small_bodies), excluded_too_large_bodies: census.map(|census| census.excluded_too_large_bodies), excluded_incomplete_tokenization_bodies: census .map(|census| census.excluded_incomplete_tokenization_bodies), rename_partial_bodies: census.map(|census| census.rename_partial_bodies), rename_unsupported_bodies: census.map(|census| census.rename_unsupported_bodies), - completed_source_pages, - total_source_pages, }, budgets: CodeCloneIndexBudgetsV1 { posting_rows: CLONE_FINGERPRINT_POSTING_ROW_BUDGET_V1, @@ -1982,7 +1727,7 @@ fn clone_index_observation( ), }, resources: CodeCloneIndexResourcesV1 { - bytes_on_disk, + bytes_on_disk: Some(artifact.bytes_on_disk), peak_scratch_memory_bytes, changed_symbol_update_micros: update .and_then(|update| update.changed_symbol_update_micros), @@ -2002,9 +1747,7 @@ fn clone_index_omission_reasons(artifact: &CloneIndexArtifactSnapshotV1) -> Vec< "conservative normalization postings do not cover every eligible body".to_owned(), ); } - if !artifact.has_fingerprints { - reasons.push("positional fingerprint backfill is not active".to_owned()); - } else if census.near_fingerprint_bodies != census.eligible_source_bodies { + if census.near_fingerprint_bodies != census.eligible_source_bodies { reasons.push("positional fingerprints do not cover every eligible body".to_owned()); } reasons @@ -2037,7 +1780,7 @@ impl LatestCompleteCodeIndexV1 { } impl LatestCodeTextGenerationV1 { - /// Return exact and lexical owners without waiting for clone backfill. + /// Warm and return the exact, lexical, and clone query owners. #[cfg(any(test, feature = "test-helpers"))] #[cfg_attr(not(test), allow(dead_code))] pub fn production_query_owners( @@ -2074,37 +1817,6 @@ impl LatestCodeTextGenerationV1 { Ok(true) } - /// Advance text projection until clone successor fingerprints are sealed. - /// - /// Lexical owners can be Ready while clone backfill is still background - /// work. `tracedecay_similar` needs those postings; ordinary search does not - /// wait here. Drive at most one bounded slice inline and leave the rest to - /// the retained worker wake the caller must have requested, owning the - /// whole successor on the request thread was the #1339 S1 regression. - pub(crate) fn finish_clone_similarity_warmup_for_request( - &self, - request_control: &dyn CodeIndexExecutionControlV1, - ) -> Result { - let mut advances = 0_usize; - while self.text_projection_needs_work() { - self.advance_text_serving_for_request( - TEXT_ARTIFACT_MAXIMUM_WORK_PER_ADVANCE_V1, - request_control, - )?; - advances += 1; - if advances >= TEXT_ARTIFACT_MAXIMUM_CLONE_WARMUP_ADVANCES_V1 { - return Ok(if self.text_projection_needs_work() { - // Background owns the remainder; do not collapse this into - // a hard GenerationUnavailable miss at the executor. - CloneSimilarityWarmupForRequestV1::Pending - } else { - CloneSimilarityWarmupForRequestV1::Ready - }); - } - } - Ok(CloneSimilarityWarmupForRequestV1::Ready) - } - pub(crate) fn production_query_owners_with_budget( &self, _build_budget: &RetrievalBudget, @@ -2299,6 +2011,11 @@ impl LatestCodeTextGenerationV1 { tracedecay_query::retrieval::QUERY_EXACT_SCORE_DOMAIN_V1, ) .map_err(|error| RetrievalPortError::Contract(error.to_string()))?, + clone_route: Some(CodeLexicalCloneRouteV1 { + project_id: self.metadata.manifest().project_id.clone(), + worktree_id: self.metadata.snapshot().worktree.clone(), + snapshot_digest: self.metadata.manifest().snapshot_digest.clone(), + }), }) } @@ -2422,11 +2139,11 @@ impl LatestCodeTextGenerationV1 { &self, reader: &CodeLexicalArtifactReaderV1, sealed_identity: &DurableSealedCodeGenerationIdentityV1, - source: &VerifiedSealedLexicalPageSourceV1, + source: &VerifiedSealedLexicalPageSourceV1, ) -> Result { let artifact = reader.verified_artifact(); let generation_id = &self.metadata.manifest().generation_id; - if artifact.generation() != generation_id { + if &reader.metadata().generation != generation_id { return Err(RetrievalPortError::GenerationMismatch); } let elapsed_micros = self @@ -2646,7 +2363,7 @@ impl LatestCodeTextGenerationV1 { &self, sealed_identity: &DurableSealedCodeGenerationIdentityV1, control: &dyn CodeIndexExecutionControlV1, - ) -> Result, RetrievalPortError> { + ) -> Result { if let Some(source) = self .preopened_source .lock() @@ -2655,101 +2372,8 @@ impl LatestCodeTextGenerationV1 { { return Ok(source); } - let mut source = self - .text_artifact_store - .open_sealed_source(sealed_identity, control)?; - if let Ok(Some(published)) = self - .text_artifact_store - .publication - .active_already_decoded() - && published.manifest().generation_id == self.metadata.manifest().generation_id - { - let _ = source.attach_published_files(&published); - } - Ok(source) - } - - fn begin_clone_successor( - &self, - prior_descriptor: DurableCodeTextArtifactDescriptorV1, - prior: tracedecay_query::retrieval::lexical::VerifiedCodeLexicalArtifactV1, - sealed_identity: DurableSealedCodeGenerationIdentityV1, - source: VerifiedSealedLexicalPageSourceV1, - control: &dyn CodeIndexExecutionControlV1, - ) -> Result, RetrievalPortError> { - let generation_id = &self.metadata.manifest().generation_id; - // The successor's working set stays at - // `CLONE_SUCCESSOR_MEMORY_BUDGET_BYTES_V1`. The charge is at least - // the reader budget so publication can transfer it onto the reader - // component. A fresh reader admission at that boundary is refused - // when graph replay is already on the RSS watermark. - let publication_charge = CLONE_SUCCESSOR_MEMORY_BUDGET_BYTES_V1 - .max(CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1); - let reservation = self.text_artifact_store.reserve_resident_memory( - generation_id, - "code-text-clone-successor", - publication_charge, - )?; - let artifacts_root = code_text_artifacts_root(self.text_artifact_store.store_root()); - ensure_private_text_artifacts_root(&artifacts_root)?; - let prior_path = - code_text_artifact_path(self.text_artifact_store.store_root(), &prior_descriptor) - .map_err(text_artifact_unavailable)?; - let sealed_hex = sha256_hex_suffix(sealed_identity.digest.as_str()).ok_or_else(|| { - RetrievalPortError::Contract( - "clone-successor sealed generation digest is not SHA-256".to_owned(), - ) - })?; - let staging_path = artifacts_root.join(format!(".text-artifact-{sealed_hex}.staging")); - prepare_absent_text_artifact_staging(&staging_path).map_err(text_artifact_unavailable)?; - let metadata = self.text_projection_metadata()?; - let open_builder = || { - CodeLexicalCloneSuccessorV1::open_or_create( - &prior_path, - &staging_path, - prior.clone(), - metadata.clone(), - CLONE_SUCCESSOR_MEMORY_BUDGET_BYTES_V1, - ) - }; - let mut builder = match open_builder() { - Ok(builder) => builder, - Err( - CodeLexicalArtifactErrorV1::Incompatible(_) - | CodeLexicalArtifactErrorV1::Corrupt(_), - ) => { - self.text_artifact_store - .discard_incompatible_staging(&staging_path, control)?; - open_builder().map_err(map_text_artifact_error)? - } - Err(error) => return Err(map_text_artifact_error(error)), - }; - let source_position = match builder.next_cursor() { - Ok(Some(cursor)) => CloneSuccessorSourcePositionV1::Revalidating(cursor), - Ok(None) => CloneSuccessorSourcePositionV1::Appending, - Err( - CodeLexicalArtifactErrorV1::Incompatible(_) - | CodeLexicalArtifactErrorV1::Corrupt(_), - ) => { - drop(builder); - self.text_artifact_store - .discard_incompatible_staging(&staging_path, control)?; - builder = open_builder().map_err(map_text_artifact_error)?; - CloneSuccessorSourcePositionV1::Appending - } - Err(error) => return Err(map_text_artifact_error(error)), - }; - Ok(Box::new(CodeTextCloneSuccessorBuildV1 { - builder: Some(builder), - source, - source_position, - sealed_identity, - source_receipt: None, - prior, - prior_descriptor, - staging_path, - build_reservation: Some(reservation), - })) + self.text_artifact_store + .open_sealed_source(sealed_identity, control) } fn open_published_text_artifact( @@ -2770,62 +2394,18 @@ impl LatestCodeTextGenerationV1 { path, &descriptor.artifact_digest, descriptor.artifact_size_bytes, + &self.text_projection_metadata()?, CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, control, - ) - .and_then(|reader| { - let expected_metadata = self - .text_projection_metadata() - .map_err(|error| CodeLexicalArtifactErrorV1::Contract(error.to_string()))?; - if reader.metadata() != &expected_metadata { - return Err(CodeLexicalArtifactErrorV1::Incompatible( - "published lexical metadata does not match the current projection".to_owned(), - )); - } - Ok(reader) - }); + ); match reader { Ok(reader) => { let sealed_identity = store.sealed_identity(generation_id)?; let source = self.take_preopened_source_or_open(&sealed_identity, control)?; let ready_progress = self.ready_text_progress_snapshot(&reader, &sealed_identity, &source)?; - let needs_clone_successor = !reader.has_clone_fingerprints(); - let prior = reader.verified_artifact().clone(); self.install_artifact_owners(reader, reader_reservation)?; self.publish_text_progress_snapshot(ready_progress); - if needs_clone_successor { - // `begin_clone_successor` copies the whole prior lexical - // artifact with the slot lock released, so this wake's - // head-open claim has to span it. Parking - // `CloneSuccessorPending` before the copy published a - // takeable state mid-claim: `advance_artifact_text_serving` - // leaves its park loop on that state, so a concurrent wake - // took a second `HeadOpening` on top of this open and both - // drove the same staging database. Whichever open resolved - // second then found the slot already reset and failed the - // clone lane closed. A successful begin resolves the claim - // to `BuildingCloneSuccessor` anyway, so only a failed one - // needs the retry marker: the owners installed above would - // otherwise let the next wake short-circuit on a plain - // `Idle` and never owe the successor again. - return match self.begin_clone_successor( - descriptor, - prior, - sealed_identity, - source, - control, - ) { - Ok(build) => Ok(Some(TextHeadOpenOutcomeV1::BuildCloneSuccessor(build))), - Err(error) => { - // The claim still owns `HeadOpening`, so this only - // parks the marker; the begin failure is the one - // worth reporting. - let _ = self.text_projection_build.retain_clone_successor_retry(); - Err(error) - } - }; - } drop(source); Ok(Some(TextHeadOpenOutcomeV1::Served)) } @@ -2848,6 +2428,102 @@ impl LatestCodeTextGenerationV1 { } } + /// Serve an artifact a sibling worktree published for the same content + /// instead of building one. The content key is derived before any build + /// from the sealed source and the projection; a descriptor under that key + /// is adopted only after its file opens against its content address and + /// this generation's projection, and anything else falls through to the + /// build. The project lock is held shared from the lookup until the + /// descriptor is durable, so no retention collects the file meanwhile. + fn adopt_shared_text_artifact( + &self, + generation_id: &CodeGenerationId, + control: &dyn CodeIndexExecutionControlV1, + ) -> Result, RetrievalPortError> { + let store = &self.text_artifact_store; + let sealed_identity = store.sealed_identity(generation_id)?; + let reader_reservation = store.reserve_resident_memory( + generation_id, + "code-text-artifact-reader", + CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, + )?; + let source = self.take_preopened_source_or_open(&sealed_identity, control)?; + let metadata = self.text_projection_metadata()?; + let content_key = text_artifact_content_key(&source, &metadata)?; + let adopted = (|| { + let lock = store.acquire_store_write_lock()?; + let _project_lock = + acquire_generation_segments_publication_lock(store.store_root(), &|| { + checkpoint_text_artifact_control(control).is_err() + }) + .map_err(text_artifact_unavailable)?; + let Some(shared) = find_shared_text_artifact(store.store_root(), &content_key) + .map_err(text_artifact_unavailable)? + else { + return Ok(None); + }; + let path = code_text_artifact_path(store.store_root(), &shared) + .map_err(text_artifact_unavailable)?; + let reader = match CodeLexicalArtifactReaderV1::open_content_addressed( + path, + &shared.artifact_digest, + shared.artifact_size_bytes, + &metadata, + CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, + control, + ) { + Ok(reader) => reader, + Err(error @ CodeLexicalArtifactErrorV1::Interrupted(_)) => { + return Err(map_text_artifact_error(error)); + } + Err(error) => { + tracing::warn!( + event = "code_index_shared_text_artifact_refused", + artifact = %shared.artifact_file, + %error, + "a shared text artifact under this content key failed verification; building one" + ); + return Ok(None); + } + }; + let descriptor = DurableCodeTextArtifactDescriptorV1 { + generation_id: generation_id.clone(), + artifact_file: shared.artifact_file, + artifact_digest: shared.artifact_digest, + artifact_size_bytes: shared.artifact_size_bytes, + content_key: content_key.clone(), + }; + let pointer = store + .publication + .read_publication_pointer() + .map_err(text_artifact_unavailable)? + .ok_or_else(|| { + RetrievalPortError::AuthorityUnavailable( + "no durable publication pointer exists for text-artifact attachment" + .to_owned(), + ) + })?; + attach_verified_text_artifact_under_lock(&lock, &pointer, &sealed_identity, descriptor) + .map_err(text_artifact_unavailable)?; + Ok(Some(reader)) + })(); + let Some(reader) = adopted? else { + // The build reads the same source; hand it over rather than + // reopen it. + *self + .preopened_source + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(source); + return Ok(None); + }; + hotpath::gauge!("query.artifact.shared_adoptions_total").inc(1u64); + let ready_progress = + self.ready_text_progress_snapshot(&reader, &sealed_identity, &source)?; + self.install_artifact_owners(reader, reader_reservation)?; + self.publish_text_progress_snapshot(ready_progress); + Ok(Some(TextHeadOpenOutcomeV1::Served)) + } + /// One claimed head-open pass, run with the slot lock released: reopen /// the published durable head when one exists, otherwise authenticate the /// sealed source and begin (or resume) the staging build. Fail-closed: @@ -2869,6 +2545,9 @@ impl LatestCodeTextGenerationV1 { { return Ok(outcome); } + if let Some(outcome) = self.adopt_shared_text_artifact(&generation_id, control)? { + return Ok(outcome); + } // The builder's advertised memory ceiling is reserved through the // process resident-memory authority before the build allocates. The // host-scaled figure is a preferred ceiling, not a minimum: under a @@ -2894,14 +2573,15 @@ impl LatestCodeTextGenerationV1 { "durable sealed lexical source digest is not SHA-256".to_owned(), ) })?; - let artifacts_root = code_text_artifacts_root(store.store_root()); - ensure_private_text_artifacts_root(&artifacts_root)?; - let staging_path = artifacts_root.join(format!(".text-artifact-{sealed_hex}.staging")); + let staging_root = code_text_artifact_staging_root(store.store_root()); + ensure_private_text_artifacts_root(&staging_root)?; + let staging_path = staging_root.join(format!(".text-artifact-{sealed_hex}.staging")); prepare_absent_text_artifact_staging(&staging_path).map_err(text_artifact_unavailable)?; let mut source = self.take_preopened_source_or_open(&sealed_identity, control)?; let builder_budget = text_artifact_builder_budget(build_memory_budget, source.staging_window_bytes())?; let metadata = self.text_projection_metadata()?; + let content_key = text_artifact_content_key(&source, &metadata)?; let mut builder = if staging_path.exists() { match CodeLexicalArtifactBuilderV1::open_or_resume_with_memory_budget_and_control( &staging_path, @@ -2912,24 +2592,39 @@ impl LatestCodeTextGenerationV1 { Ok(builder) => Ok(builder), Err(CodeLexicalArtifactErrorV1::Incompatible(_)) => { store.discard_incompatible_staging(&staging_path, control)?; - CodeLexicalArtifactBuilderV1::create_with_memory_budget_and_format_revision( + CodeLexicalArtifactBuilderV1::create_with_memory_budget( &staging_path, metadata.clone(), builder_budget, - CodeLexicalArtifactWriterRevisionV1::V14, ) } Err(error) => Err(error), } } else { - CodeLexicalArtifactBuilderV1::create_with_memory_budget_and_format_revision( + CodeLexicalArtifactBuilderV1::create_with_memory_budget( &staging_path, metadata.clone(), builder_budget, - CodeLexicalArtifactWriterRevisionV1::V14, ) } .map_err(map_text_artifact_error)?; + // A staging file sealed before its publication was interrupted keeps + // no source cursor; it is complete and is published as it stands. + if builder + .sealed_receipt() + .map_err(map_text_artifact_error)? + .is_some() + { + drop(builder); + drop(source); + return self.publish_sealed_text_artifact( + &staging_path, + &sealed_identity, + &content_key, + build_reservation, + control, + ); + } let mut progress = builder.progress().map_err(map_text_artifact_error)?; if let Some(cursor) = progress.next_cursor.as_ref() { match source.restore_cursor_classified(cursor, control) { @@ -2937,14 +2632,10 @@ impl LatestCodeTextGenerationV1 { Err(VerifiedSealedLexicalCursorRestoreErrorV1::IncompatiblePosition) => { drop(builder); store.discard_incompatible_staging(&staging_path, control)?; - // Keep the same V14 admission writer as the cold create path. - // Defaulting to V16 here would admit clone fingerprints on the - // rebuild lane and diverge from the successor-backed cutover. - builder = CodeLexicalArtifactBuilderV1::create_with_memory_budget_and_format_revision( + builder = CodeLexicalArtifactBuilderV1::create_with_memory_budget( &staging_path, metadata, builder_budget, - CodeLexicalArtifactWriterRevisionV1::V14, ) .map_err(map_text_artifact_error)?; progress = builder.progress().map_err(map_text_artifact_error)?; @@ -2958,6 +2649,7 @@ impl LatestCodeTextGenerationV1 { builder, source, sealed_identity, + content_key, source_receipt: None, staging_path, _build_reservation: build_reservation, @@ -2996,10 +2688,7 @@ impl LatestCodeTextGenerationV1 { } let guard = self.text_projection_build.lock_slot(); match &*guard { - CodeTextProjectionSlotV1::Idle - | CodeTextProjectionSlotV1::CloneSuccessorPending - | CodeTextProjectionSlotV1::Building(_) - | CodeTextProjectionSlotV1::BuildingCloneSuccessor(_) => { + CodeTextProjectionSlotV1::Idle | CodeTextProjectionSlotV1::Building(_) => { break guard; } CodeTextProjectionSlotV1::HeadOpening => { @@ -3019,11 +2708,8 @@ impl LatestCodeTextGenerationV1 { } } }; - if matches!( - &*slot, - CodeTextProjectionSlotV1::Idle | CodeTextProjectionSlotV1::CloneSuccessorPending - ) { - if matches!(&*slot, CodeTextProjectionSlotV1::Idle) && self.query_owners_are_ready() { + if matches!(&*slot, CodeTextProjectionSlotV1::Idle) { + if self.query_owners_are_ready() { return Ok(true); } *slot = CodeTextProjectionSlotV1::HeadOpening; @@ -3036,29 +2722,9 @@ impl LatestCodeTextGenerationV1 { match outcome { TextHeadOpenOutcomeV1::Served => return Ok(true), TextHeadOpenOutcomeV1::Build(initialized) => { - slot = claim.install(TextHeadOpenBuildV1::Artifact(initialized)); - } - TextHeadOpenOutcomeV1::BuildCloneSuccessor(initialized) => { - slot = claim.install(TextHeadOpenBuildV1::CloneSuccessor(initialized)); - } - } - } - if matches!(&*slot, CodeTextProjectionSlotV1::BuildingCloneSuccessor(_)) { - let done = match &mut *slot { - CodeTextProjectionSlotV1::BuildingCloneSuccessor(successor) => { - self.advance_clone_successor(successor, maximum_work, control)? - } - _ => { - return Err(RetrievalPortError::Contract( - "clone-successor build state changed under its lock".to_owned(), - )); + slot = claim.install(initialized); } - }; - if done { - *slot = CodeTextProjectionSlotV1::Idle; - self.text_projection_build.ready.notify_all(); } - return Ok(done); } let CodeTextProjectionSlotV1::Building(artifact_build) = &mut *slot else { return Err(RetrievalPortError::Contract( @@ -3086,6 +2752,7 @@ impl LatestCodeTextGenerationV1 { self.publish_text_progress_phase(CodeIndexBuildPhaseV1::SourceScan, 0, 0); let mut durable_progress = None; let mut commit_latency_micros = None; + let mut clone_scratch_bytes = 0_usize; let admitted = { let (source, builder) = (&mut artifact_build.source, &mut artifact_build.builder); source.next_page_batch_if(control, bounds, |pages| { @@ -3138,6 +2805,12 @@ impl LatestCodeTextGenerationV1 { batch_pages, batch_payload_bytes, ); + clone_scratch_bytes = prepared + .prepared_pages() + .iter() + .map(PreparedCodeLexicalArtifactPageV1::clone_body_peak_bytes) + .max() + .unwrap_or(0); let commit_started = Instant::now(); let progress = builder .append_prepared_pages(prepared.prepared_pages(), control)?; @@ -3204,6 +2877,12 @@ impl LatestCodeTextGenerationV1 { true, ) )?; + self.text_progress_state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .observe_clone_scratch( + u64::try_from(clone_scratch_bytes).unwrap_or(u64::MAX), + ); #[cfg(feature = "hotpath")] { let committed_lexical_units = artifact_build @@ -3310,19 +2989,9 @@ impl LatestCodeTextGenerationV1 { } CodeLexicalArtifactFinalizationStepV1::Ready(_) => CodeIndexBuildPhaseV1::Verification, }; - let progress = artifact_build - .builder - .progress() - .map_err(map_text_artifact_error)?; - self.publish_text_progress_boundary( - artifact_build, - &progress, - finalization_phase, - 0, - 0, - None, - false, - )?; + // The sealed file keeps no source cursor, so the phase is published + // over the last boundary's counters. + self.publish_text_progress_phase(finalization_phase, 0, 0); // The publication tail content-addresses the finalized staging file // and reopens it verified, corpus-sized digest work, so it runs // under a fresh `HeadOpening` claim with the slot lock released, the @@ -3343,6 +3012,7 @@ impl LatestCodeTextGenerationV1 { builder, source, sealed_identity, + content_key, source_receipt: _, staging_path, _build_reservation: build_reservation, @@ -3351,10 +3021,32 @@ impl LatestCodeTextGenerationV1 { // finalized staging file. drop(builder); drop(source); - let descriptor = store.publish( + self.publish_sealed_text_artifact( &staging_path, - &self.metadata.manifest().generation_id, &sealed_identity, + &content_key, + build_reservation, + control, + )?; + Ok(true) + } + + /// Publish one sealed staging file, open it verified, and install its + /// owners. + fn publish_sealed_text_artifact( + &self, + staging_path: &Path, + sealed_identity: &DurableSealedCodeGenerationIdentityV1, + content_key: &ManifestDigest, + build_reservation: ResidentMemoryReservationV1, + control: &dyn CodeIndexExecutionControlV1, + ) -> Result { + let store = &self.text_artifact_store; + let descriptor = store.publish( + staging_path, + &self.metadata.manifest().generation_id, + sealed_identity, + content_key, control, )?; // The reader is a smaller charge of the bytes this build already @@ -3368,256 +3060,18 @@ impl LatestCodeTextGenerationV1 { final_path, &descriptor.artifact_digest, descriptor.artifact_size_bytes, + &self.text_projection_metadata()?, CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, control, ) .map_err(map_text_artifact_error)?; - let needs_clone_successor = !reader.has_clone_fingerprints(); // Match the cold-open path: install owners first, then publish Ready. // Publishing Ready before a failed install (admission ceiling / shrink) // would leave dashboard/MCP progress claiming a ready generation that // cannot serve queries. self.install_artifact_owners(reader, reader_reservation)?; self.publish_text_progress_phase(CodeIndexBuildPhaseV1::Ready, 0, 0); - if needs_clone_successor { - // `begin_clone_successor` copies the whole prior lexical artifact - // before the first page walk. Doing that here kept this advance, - // and the publication pass awaiting it, inside `reconcile_in_progress` - // for the copy. Exact and lexical serving are already installed; - // the copy is not a freshness precondition. Leave the slot pending - // so the retained driver starts the successor after the seat, - // without the receipt guard. The claim stays armed: its drop - // restores only `HeadOpening`, so `CloneSuccessorPending` survives - // and parked wakes are notified. - self.text_projection_build.retain_clone_successor_retry()?; - return Ok(false); - } - Ok(true) - } - - fn advance_clone_successor( - &self, - build: &mut CodeTextCloneSuccessorBuildV1, - maximum_work: usize, - control: &dyn CodeIndexExecutionControlV1, - ) -> Result { - if build.builder.is_none() { - self.rebuild_clone_successor(build, control)?; - } - let (batch_pages, batch_bytes) = clone_successor_source_batch_limits( - build - .builder - .as_ref() - .ok_or_else(|| { - RetrievalPortError::Contract("clone-successor builder is missing".to_owned()) - })? - .projection_metadata(), - )?; - let mut remaining = maximum_work.max(1); - while remaining > 0 && build.source_receipt.is_none() { - if let CloneSuccessorSourcePositionV1::Revalidating(target) = &build.source_position { - // Resume revalidation replays already-committed pages one at a - // time until the persisted successor cursor is reached. - let target = target.clone(); - let read = build - .source - .next_page(control) - .map_err(map_sealed_page_source_error)?; - match read { - VerifiedSealedLexicalPageReadV1::Page(page) => { - if !self.revalidate_clone_successor_page(build, target, &page, control)? { - return Ok(false); - } - remaining -= 1; - } - VerifiedSealedLexicalPageReadV1::Complete(_) => { - self.rebuild_clone_successor(build, control)?; - return Ok(false); - } - } - continue; - } - let bounds = VerifiedSealedLexicalPageBatchBoundsV1::new( - remaining.clamp(1, batch_pages), - batch_bytes, - ) - .map_err(map_sealed_page_source_error)?; - match self.append_clone_successor_batch(build, bounds, control)? { - VerifiedSealedLexicalPageBatchReadV1::Pages(batch) => { - remaining = remaining.checked_sub(batch.len()).ok_or_else(|| { - RetrievalPortError::Contract( - "clone-successor batch delivered more pages than the remaining work bound" - .to_owned(), - ) - })?; - } - VerifiedSealedLexicalPageBatchReadV1::Complete(receipt) => { - build.source_receipt = Some(receipt); - } - } - } - let Some(source_receipt) = build.source_receipt.as_ref() else { - return Ok(false); - }; - if remaining == 0 { - return Ok(false); - } - let _receipt = build - .builder - .as_mut() - .ok_or_else(|| { - RetrievalPortError::Contract("clone-successor builder is missing".to_owned()) - })? - .finish(source_receipt, control) - .map_err(map_text_artifact_error)?; - drop(build.builder.take()); - let held_reservation = build.build_reservation.take().ok_or_else(|| { - RetrievalPortError::Contract( - "clone-successor resident-memory charge disappeared before publication".to_owned(), - ) - })?; - let descriptor = self.text_artifact_store.publish_with_prior( - &build.staging_path, - &self.metadata.manifest().generation_id, - &build.sealed_identity, - Some(&build.prior_descriptor), - control, - )?; - let reader_reservation = reader_charge_from_held_reservation(held_reservation)?; - let final_path = - code_text_artifact_path(self.text_artifact_store.store_root(), &descriptor) - .map_err(text_artifact_unavailable)?; - let reader = CodeLexicalArtifactReaderV1::open_content_addressed( - final_path, - &descriptor.artifact_digest, - descriptor.artifact_size_bytes, - CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, - control, - ) - .map_err(map_text_artifact_error)?; - self.install_artifact_owners(reader, reader_reservation)?; - Ok(true) - } - - /// Stage one bounded ordered page batch from the sealed source and append - /// it to the clone successor under one durable commit. The source cursor - /// only advances through pages the successor accepted, so a failed batch - /// leaves both authorities at their pre-batch position. - fn append_clone_successor_batch( - &self, - build: &mut CodeTextCloneSuccessorBuildV1, - bounds: VerifiedSealedLexicalPageBatchBoundsV1, - control: &dyn CodeIndexExecutionControlV1, - ) -> Result { - let (source, builder) = (&mut build.source, &mut build.builder); - let builder = builder.as_mut().ok_or_else(|| { - RetrievalPortError::Contract("clone-successor builder is missing".to_owned()) - })?; - source - .next_page_batch_if(control, bounds, |pages| { - for page in pages { - self.observe_clone_page_scratch(page)?; - } - builder - .append_pages(pages, control) - .map_err(map_text_artifact_error)?; - NonZeroUsize::new(pages.len()).ok_or_else(|| { - RetrievalPortError::Contract( - "sealed lexical source staged an empty clone-successor batch".to_owned(), - ) - }) - }) - .map_err(map_sealed_page_source_error)? - } - - fn observe_clone_page_scratch( - &self, - page: &VerifiedSealedLexicalPageV1, - ) -> Result<(), RetrievalPortError> { - let scratch_bytes = page.clone_bodies().iter().try_fold(0_u64, |peak, body| { - let payload = serde_json::to_vec(&body.payload) - .map_err(|error| RetrievalPortError::Contract(error.to_string()))?; - let occurrence = serde_json::to_vec(&body.occurrence) - .map_err(|error| RetrievalPortError::Contract(error.to_string()))?; - let bytes = u64::try_from(payload.len().saturating_add(occurrence.len())) - .map_err(|error| RetrievalPortError::Contract(error.to_string()))?; - Ok::<_, RetrievalPortError>(peak.max(bytes)) - })?; - self.text_progress_state - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .observe_clone_scratch(scratch_bytes); - Ok(()) - } - - /// Replay one already-committed page against the resumed successor while - /// it is `Revalidating` toward `target`. Returns `false` when the - /// successor had to be rebuilt and the slice must stop. - fn revalidate_clone_successor_page( - &self, - build: &mut CodeTextCloneSuccessorBuildV1, - target: VerifiedSealedLexicalCursorV1, - page: &VerifiedSealedLexicalPageV1, - control: &dyn CodeIndexExecutionControlV1, - ) -> Result { - self.observe_clone_page_scratch(page)?; - let verification = build - .builder - .as_ref() - .ok_or_else(|| { - RetrievalPortError::Contract("clone-successor builder is missing".to_owned()) - })? - .verify_resumed_page(page, control); - if matches!( - verification, - Err(CodeLexicalArtifactErrorV1::Incompatible(_) - | CodeLexicalArtifactErrorV1::Corrupt(_)) - ) || page.next_cursor().next_page_ordinal() > target.next_page_ordinal() - { - self.rebuild_clone_successor(build, control)?; - return Ok(false); - } - verification.map_err(map_text_artifact_error)?; - if page.next_cursor().next_page_ordinal() == target.next_page_ordinal() { - if page.next_cursor() != &target { - self.rebuild_clone_successor(build, control)?; - return Ok(false); - } - build.source_position = CloneSuccessorSourcePositionV1::Appending; - } - Ok(true) - } - - fn rebuild_clone_successor( - &self, - build: &mut CodeTextCloneSuccessorBuildV1, - control: &dyn CodeIndexExecutionControlV1, - ) -> Result<(), RetrievalPortError> { - drop(build.builder.take()); - self.text_artifact_store - .discard_incompatible_staging(&build.staging_path, control)?; - prepare_absent_text_artifact_staging(&build.staging_path) - .map_err(text_artifact_unavailable)?; - let prior_path = code_text_artifact_path( - self.text_artifact_store.store_root(), - &build.prior_descriptor, - ) - .map_err(text_artifact_unavailable)?; - let builder = CodeLexicalCloneSuccessorV1::open_or_create( - prior_path, - &build.staging_path, - build.prior.clone(), - self.text_projection_metadata()?, - CLONE_SUCCESSOR_MEMORY_BUDGET_BYTES_V1, - ) - .map_err(map_text_artifact_error)?; - build.source = self - .text_artifact_store - .open_sealed_source(&build.sealed_identity, control)?; - build.source_position = CloneSuccessorSourcePositionV1::Appending; - build.source_receipt = None; - build.builder = Some(builder); - Ok(()) + Ok(TextHeadOpenOutcomeV1::Served) } fn install_artifact_owners( @@ -3763,42 +3217,34 @@ pub(super) fn sha256_private_file_and_size( Ok((hasher.finalize().into(), file_metadata.len())) } +/// The content key the text artifact of `source` under `metadata` is +/// published with. +fn text_artifact_content_key( + source: &VerifiedSealedLexicalPageSourceV1, + metadata: &CodeLexicalProjectionMetadataV1, +) -> Result { + code_lexical_artifact_content_key( + &source.content_key().map_err(map_sealed_page_source_error)?, + metadata, + ) + .map_err(map_text_artifact_error) +} + fn ensure_private_text_artifacts_root(path: &Path) -> Result<(), RetrievalPortError> { match tracedecay_private_fs::create_private_directory(path) { Ok(()) => Ok(()), Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => { - let validation_error = match validate_private_directory(path) { - Ok(()) => return Ok(()), - Err(error) => error, - }; - // A pre-existing root that fails owner-privacy validation is most - // often a legacy directory an older binary created under a - // permissive umask. Ownership is the proof this process may - // tighten it in place; a root it does not own, or that is not a - // directory at all, stays a typed deterministic contract - // violation for the operator instead of an endless silent retry. - match make_private_directory(path) { - Ok(receipt) => { - let previous_mode = receipt - .previous_unix_mode - .map_or_else(|| "platform-acl".to_owned(), |mode| format!("{mode:o}")); - tracing::info!( - event = "code_index_text_artifacts_root_privacy_healed", - previous_mode = %previous_mode, - "legacy code text artifacts root was re-permissioned to owner-private" - ); - Ok(()) - } - Err(heal_error) => Err(RetrievalPortError::Contract(format!( + validate_private_directory(path).map_err(|validation_error| { + RetrievalPortError::Contract(format!( "code text artifacts root '{}' is not owner-private{}: {validation_error}; \ - self-heal refused: {heal_error}; restore owner-only access (chmod 700 and \ - chown to the daemon user) or re-enroll the store", + restore owner-only access (chmod 700 and chown to the daemon user) or \ + re-enroll the store", path.display(), observed_unix_mode(path) .map(|mode| format!(" (mode {mode:o}, need 700)")) .unwrap_or_default(), - ))), - } + )) + }) } Err(error) => Err(text_artifact_unavailable(error)), } @@ -3834,7 +3280,7 @@ fn checkpoint_text_artifact_control( /// /// DELETE-mode recovery applies `path-journal` into whatever file is later /// opened at `path`. A crash that unlinked the database and left the journal -/// would otherwise roll that journal into the next successor copy. +/// would otherwise roll that journal into the next staging database. fn prepare_absent_text_artifact_staging(staging_path: &Path) -> std::io::Result<()> { match staging_path.symlink_metadata() { Ok(metadata) if metadata.file_type().is_file() => Ok(()), @@ -3929,7 +3375,7 @@ mod staging_sidecar_tests { prepare_absent_text_artifact_staging(&staging).expect("clear orphan journal"); assert!(!journal.exists()); - std::fs::write(&staging, b"prior-copy").expect("fresh successor copy"); + std::fs::write(&staging, b"staged").expect("fresh staging database"); std::fs::write(&journal, b"rollback").expect("replant journal"); retire_text_artifact_staging_family(&staging).expect("retire family"); assert!(!staging.exists()); diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/mod.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/mod.rs index e33b11a62d..9ca7d5d20d 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/mod.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/mod.rs @@ -716,13 +716,6 @@ fn install_verified_graph_store_on_text( } fn query_authority(privacy_domain: PrivacyDomainId) -> Arc { - query_authority_with_candidate_cap(privacy_domain, 32) -} - -fn query_authority_with_candidate_cap( - privacy_domain: PrivacyDomainId, - max_candidates_per_lane: u32, -) -> Arc { let id = |value: &str| value.to_owned(); let profile = FusionProfile { profile_id: id("profile.code-index.fixture") @@ -791,7 +784,7 @@ fn query_authority_with_candidate_cap( .try_into() .expect("diversity id"), retrieval_budget: RetrievalBudget { - max_candidates_per_lane, + max_candidates_per_lane: 32, max_fused_candidates: 32, max_hydrated_results: 32, max_hydration_bytes: 32 * 65_536, @@ -862,14 +855,13 @@ fn active_text_artifact_path(store_root: &Path) -> PathBuf { let artifact_file = entry["text_artifact"]["artifact_file"] .as_str() .expect("attached text artifact descriptor"); - store_root - .join("code-text-artifacts-v1") + tracedecay_code_index_retention::code_index_generations::code_text_artifacts_root(store_root) .join(artifact_file) } fn rewrite_active_text_artifact_format_revision(store_root: &Path, revision: u64) -> PathBuf { use tracedecay_code_index_retention::code_index_generations::{ - DurablePublicationPointerV1, durable_generation_index_digest, + DurablePublicationPointerV1, DurableTextArtifactSlotV1, durable_generation_index_digest, }; let pointer_path = store_root.join("active-code-generation-v1.json"); @@ -882,10 +874,9 @@ fn rewrite_active_text_artifact_format_revision(store_root: &Path, revision: u64 .iter_mut() .find(|entry| entry.generation_id == pointer.generation_id) .expect("active generation entry"); - let descriptor = entry - .text_artifact - .as_mut() - .expect("active text artifact descriptor"); + let Some(DurableTextArtifactSlotV1::Current(descriptor)) = entry.text_artifact.as_mut() else { + panic!("active text artifact descriptor"); + }; let old_path = store_root .join("code-text-artifacts-v1") .join(&descriptor.artifact_file); @@ -1228,24 +1219,19 @@ async fn wait_for_settled_owner(registry: &CodeIndexSchedulerRegistryV1, path: & } } -/// Drive the seated owner's clone-fingerprint backfill to completion. -/// -/// The seat no longer waits for that successor: exact and lexical serve as -/// soon as the admission artifact is ready and the backfill runs on a later -/// pass. A query over pending clone work requests that pass, so a test that -/// pins query admission or wake accounting against a *settled* seat drains -/// the backfill first with plain wakes. +/// Settle the mounted owner's text projection and the worker's owed passes. /// -/// It returns only once the pending-wake slot reads empty under held -/// admission, so a caller that then seats a crafted owner cannot lose to a -/// worker tail that was still owed a pass. -async fn drain_clone_backfill(registry: &CodeIndexSchedulerRegistryV1, path: &Path) { +/// A test that pins query admission or wake accounting against a *settled* +/// seat waits here with plain wakes. It returns only once the pending-wake +/// slot reads empty under held admission, so a caller that then seats a +/// crafted owner cannot lose to a worker tail that was still owed a pass. +async fn settle_text_projection(registry: &CodeIndexSchedulerRegistryV1, path: &Path) { let canonical = canonical_existing_identity(path).expect("canonical project"); let deadline = Instant::now() + SERVING_SEAT_FAILURE_CEILING; loop { assert!( Instant::now() <= deadline, - "the clone backfill for {} never finished", + "the text projection for {} never settled", path.display() ); let text = { @@ -1260,21 +1246,19 @@ async fn drain_clone_backfill(registry: &CodeIndexSchedulerRegistryV1, path: &Pa }; if text.is_none_or(|text| !text.text_projection_needs_work()) { let admission = quiesced_background_reconcile_admission(registry, path).await; - // A settled owner is not a settled worktree. A successor-only - // clone projection releases the worker's pass guard before it - // awaits the task, so its tail reads as an idle worker while it - // still owes a continuation. The tail stamps that continuation - // before the guard drops, so the slot, not the pass counter, is - // what an outstanding tail shows up in. Observe it empty under - // held admission. A stamped slot means the worker still owes the - // pass that clears it, so hand the permit back and let it run. + // A settled owner is not a settled worktree: a worker tail stamps + // its continuation in the pending-wake slot, so the slot, not the + // pass counter, is what an outstanding tail shows up in. Observe + // it empty under held admission. A stamped slot means the worker + // still owes the pass that clears it, so hand the permit back and + // let it run. if registry.pending_wake_micros_for_root(path).await == Some(0) { return; } drop(admission); } else { // Complete-generation demand is an ordinary wake; the pass it - // starts drives the pending successor on the retained path. + // starts drives the pending projection on the retained path. registry.request_complete_generation(path).await; } tokio::time::sleep(Duration::from_millis(5)).await; @@ -1448,6 +1432,32 @@ fn caller_star_sources() -> Vec<(String, String)> { files } +/// One `fanout` function calling `relations` distinct leaves, one statement +/// per call so the extractor sees every site, spread over the star's files. +fn callee_fanout_sources(relations: usize) -> Vec<(String, String)> { + let per_file = relations.div_ceil(CALLER_STAR_FILES); + let mut lib = String::new(); + let mut fanout = String::from("pub fn fanout() {\n"); + let mut files = Vec::new(); + for (index, chunk) in (0..relations) + .collect::>() + .chunks(per_file) + .enumerate() + { + let _ = writeln!(lib, "pub mod leaves_{index:02};"); + let mut module = String::new(); + for leaf in chunk { + let _ = writeln!(module, "pub fn leaf_{leaf:04}() {{}}"); + let _ = writeln!(fanout, " crate::leaves_{index:02}::leaf_{leaf:04}();"); + } + files.push((format!("src/leaves_{index:02}.rs"), module)); + } + fanout.push_str("}\n"); + lib.push_str(&fanout); + files.insert(0, ("src/lib.rs".to_owned(), lib)); + files +} + fn callers_page_meta(page_size: u32, cursor: Option) -> RetrievalRequestMeta { RetrievalRequestMeta::current( PageRequest::new(page_size, cursor).expect("callers page"), @@ -1687,10 +1697,9 @@ async fn wait_for_dashboard_ready(registry: &CodeIndexSchedulerRegistryV1, path: && freshness.coverage == tracedecay_contracts::code_index_freshness::CodeIndexFreshnessCoverageV1::Complete }); - // A seat can leave a continuation queued (the clone-fingerprint - // successor runs on a later pass), and the ladder reports - // Verifying for as long as that pass runs. Ready means no pass - // is running and none is pending. + // A seat can leave a continuation queued, and the ladder + // reports Verifying for as long as that pass runs. Ready means + // no pass is running and none is pending. if still_ready && !registry.reconcile_in_progress_for_test(path).await && registry.pending_wake_micros_for_root(path).await == Some(0) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/noop_reconcile_tests.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/noop_reconcile_tests.rs index 7c1c03a2b0..691e7f7fe2 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/noop_reconcile_tests.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/noop_reconcile_tests.rs @@ -14,13 +14,12 @@ async fn unchanged_reconcile_does_not_reactivate_the_serving_generation() { .await .expect("mount"); wait_for_initial_generation(®istry, fixture.path()).await; - // The seat is published mid-pass and the seat no longer waits for the - // clone successor, so the mount's own receipt lands after the in-progress - // guard drops and the leftover backfill drains on later wakes that post - // receipts of their own. Settle that whole chain first: a wake still - // pending when the overflow arrives keeps its earlier arrival instant, and - // the pass would then answer for both. - drain_clone_backfill(®istry, fixture.path()).await; + // The seat is published mid-pass, so the mount's own receipt lands after + // the in-progress guard drops and later wakes post receipts of their own. + // Settle that whole chain first: a wake still pending when the overflow + // arrives keeps its earlier arrival instant, and the pass would then + // answer for both. + settle_text_projection(®istry, fixture.path()).await; wait_for_settled_owner(®istry, fixture.path()).await; wait_for_event_to_ready(®istry).await; let admission = quiesced_background_reconcile_admission(®istry, fixture.path()).await; diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/publication_store.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/publication_store.rs index f5285d04d0..5740eabc48 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/publication_store.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/publication_store.rs @@ -13,15 +13,20 @@ use tempfile::TempDir; use tracedecay_code_index_retention::code_index_generations::{ CodeGenerationRetentionErrorV1, CodeGenerationRetentionModeV1, DurableGenerationIndexEntryV1, DurablePublicationPointerV1, MAX_CODE_GENERATION_RETENTION_BATCH_V1, - acquire_code_generation_store_lock, durable_generation_index_digest, + acquire_code_generation_store_lock, code_generation_segments_root, + code_text_artifact_staging_root, code_text_artifacts_root, durable_generation_index_digest, execute_code_generation_retention_cancellable, prepare_next_code_generation_retention_cancellable, run_code_generation_retention, - try_acquire_code_generation_store_read_lock, + try_acquire_code_generation_store_read_lock, withdraw_verified_text_artifact_under_lock, }; use tracedecay_domain::{ - CodeGenerationId, ManifestDigest, SanitizerRevision, UtcMicros, encode_lowercase_hex, - sha256_hex_suffix, + AuthorizationRevision, CodeGenerationId, ComponentRevision, EphemeralSanitizedQueryViewV1, + FreshnessVectorDigest, ManifestDigest, PrincipalId, QueryNormalizationRevision, + RetrievalBudget, RetrievalRequest, RetrievalScope, RetrievalSnapshot, RetrieverOutcome, + SanitizerRevision, ScoreDomainId, SingleRootScopeV1, TemporalModeV1, UtcMicros, + VectorWatermark, encode_lowercase_hex, sha256_hex_suffix, }; +use tracedecay_query::retrieval::lexical::LexicalLaneRequest; use tracedecay_query::retrieval::ports::RetrievalPortError; use super::{ @@ -38,7 +43,10 @@ use crate::{ SealedGenerationSegmentReadV1, UninterruptibleCodeIndexControlV1, VerifiedSealedLexicalPageReadV1, }, - code_index_scheduler::{CodeIndexWorktreeSchedulerV1, SharedCodeIndexBytePoolV1}, + code_index_scheduler::{ + CodeIndexSchedulerRegistryV1, CodeIndexWorktreeSchedulerV1, SharedCodeIndexBytePoolV1, + scoped_code_index_store_root, + }, }; struct CancelledCodeIndexControlV1; @@ -241,17 +249,14 @@ fn partitioned_reclamation_is_bounded_and_preserves_retained_segments() { .map(|(_, size)| *size) .sum::(), ); - let monolithic_second_bytes = scheduler - .latest_complete_already_decoded() - .expect("second generation remains decoded") - .generation - .encode_sealed() - .expect("encode monolithic comparison") - .len() as u64; + let full_second_bytes = std::fs::metadata(&second_manifest_path) + .expect("second manifest metadata") + .len() + .saturating_add(second_components.values().sum::()); assert!( - second_generation_growth.saturating_mul(2) < monolithic_second_bytes, + second_generation_growth.saturating_mul(2) < full_second_bytes, "one-line edit added {second_generation_growth} physical bytes versus a \ - {monolithic_second_bytes}-byte monolithic rewrite" + {full_second_bytes}-byte full rewrite" ); let segment_sizes = std::fs::read_dir(&segment_root) .expect("list content-addressed segments") @@ -623,16 +628,44 @@ fn generation_decode_shares_store_and_refuses_exclusive_writer_contention() { ); } -#[test] -fn multi_page_evidence_uses_one_durable_pack_and_survives_restart() { - let source = (0..1_600).fold(String::new(), |mut source, index| { +/// 1,600 functions named `{prefix}_{index}` whose bodies apply `operator`. +/// A clean generation's evidence is implied by its own symbols and chunks +/// and fits one page; a successor that changes every body keeps one whole +/// lineage row per function, which spans several. +fn evidence_fixture_source(prefix: &str, operator: char) -> String { + (0..1_600).fold(String::new(), |mut source, index| { writeln!( source, - "pub fn evidence_{index}(value: usize) -> usize {{ value + {index} }}" + "pub fn {prefix}_{index}(value: usize) -> usize {{ value {operator} {index} }}" ) .expect("write generated fixture source"); source - }); + }) +} + +/// Publish the fixture, then a successor that changes every function body. +fn publish_multi_page_evidence( + fixture: &GitFixture, + scheduler: &mut CodeIndexWorktreeSchedulerV1, + prefix: &str, +) { + published( + scheduler + .reconcile_now() + .expect("publish the clean generation"), + ); + fixture.edit("src/evidence.rs", &evidence_fixture_source(prefix, '*')); + fixture.commit_all("change every evidence body"); + published( + scheduler + .reconcile_now() + .expect("publish multi-page generation"), + ); +} + +#[test] +fn multi_page_evidence_uses_one_durable_pack_and_survives_restart() { + let source = evidence_fixture_source("evidence", '+'); let fixture = GitFixture::new(&[("src/evidence.rs", source.as_str())]); let store = TempDir::new().expect("store root"); let (generation_id, evidence_pack_path) = { @@ -641,11 +674,7 @@ fn multi_page_evidence_uses_one_durable_pack_and_survives_restart() { store.path().to_path_buf(), Arc::new(SharedCodeIndexBytePoolV1::default()), ); - published( - scheduler - .reconcile_now() - .expect("publish multi-page generation"), - ); + publish_multi_page_evidence(&fixture, &mut scheduler, "evidence"); let latest = scheduler .latest_complete_already_decoded() .expect("multi-page generation remains decoded"); @@ -671,17 +700,22 @@ fn multi_page_evidence_uses_one_durable_pack_and_survives_restart() { .expect("evidence page descriptors"); assert!(pages.len() > 1, "the production fixture must span pages"); let segments_root = store.path().join("code-generation-segments-v1"); - let file_segment_count = manifest["generation"]["file_segments"] + for descriptor in manifest["generation"]["file_segments"] .as_array() .expect("file segment descriptors") - .len(); - assert_eq!( - std::fs::read_dir(&segments_root) - .expect("read segment objects") - .count(), - file_segment_count + 1, - "pages must be ranges in one pack, never separate filesystem objects" - ); + .iter() + .map(|descriptor| &descriptor["segment_digest"]) + .chain([&manifest["generation"]["generation_evidence"]["segment_digest"]]) + { + let digest = sha256_hex_suffix(descriptor.as_str().expect("segment digest")) + .expect("tagged segment digest"); + assert!( + segments_root + .join(format!("segment-{digest}.json")) + .is_file(), + "every file segment and the one evidence pack are durable objects" + ); + } for page in pages { let page_digest = sha256_hex_suffix(page["page_digest"].as_str().expect("page digest")) .expect("tagged page digest"); @@ -784,7 +818,16 @@ fn failed_and_crashed_evidence_pack_temporaries_are_removed() { "a failure after N pages must remove the incomplete pack" ); - std::fs::write(&temporary_path, b"crash orphan").expect("write crash orphan"); + let scope = store + .path() + .file_name() + .and_then(|name| name.to_str()) + .expect("store scope name"); + let orphan_path = segments_root.join(format!(".evidence-pack-publication.{scope}.4242.tmp")); + std::fs::write(&orphan_path, b"crash orphan").expect("write crash orphan"); + // Linked worktrees share this directory; a sibling's pack may be in flight. + let sibling_path = segments_root.join(".evidence-pack-publication.sibling.4242.tmp"); + std::fs::write(&sibling_path, b"sibling in flight").expect("write sibling pack"); let _reopened = super::super::DaemonCodeIndexPublicationStoreV1::new( store.path(), fixture.path(), @@ -793,9 +836,13 @@ fn failed_and_crashed_evidence_pack_temporaries_are_removed() { ) .expect("restart publication store"); assert!( - !temporary_path.exists(), + !orphan_path.exists(), "restart must durably clean an abandoned evidence pack" ); + assert!( + sibling_path.exists(), + "restart must not remove another worktree scope's evidence pack" + ); } #[test] @@ -968,14 +1015,7 @@ fn publishing_many_new_segments_syncs_the_segments_directory_once() { #[test] fn evidence_pack_failure_after_pages_never_publishes_manifest_or_pointer() { - let source = (0..1_600).fold(String::new(), |mut source, index| { - writeln!( - source, - "pub fn failed_evidence_{index}(value: usize) -> usize {{ value + {index} }}" - ) - .expect("write generated fixture source"); - source - }); + let source = evidence_fixture_source("failed_evidence", '+'); let fixture = GitFixture::new(&[("src/evidence.rs", source.as_str())]); let source_store = TempDir::new().expect("source store root"); let generation = { @@ -984,11 +1024,7 @@ fn evidence_pack_failure_after_pages_never_publishes_manifest_or_pointer() { source_store.path().to_path_buf(), Arc::new(SharedCodeIndexBytePoolV1::default()), ); - published( - scheduler - .reconcile_now() - .expect("build multi-page generation"), - ); + publish_multi_page_evidence(&fixture, &mut scheduler, "failed_evidence"); Arc::clone( &scheduler .latest_complete_already_decoded() @@ -1907,7 +1943,7 @@ fn restart_rejects_corrupt_sealed_generation() { #[derive(Debug, PartialEq, Eq)] enum RestartDecodeStatusV1 { - Abstained { refused_revision: Option }, + Abstained, Decoded, SourceCommitmentRefused, } @@ -1921,7 +1957,6 @@ enum RestartIdentityStatusV1 { #[derive(Debug, PartialEq, Eq)] struct RestartDecodeCensusV1 { - monolithic: RestartDecodeStatusV1, partitioned: RestartDecodeStatusV1, sanitizer: RestartIdentityStatusV1, pointer: RestartIdentityStatusV1, @@ -1935,30 +1970,7 @@ fn restart_decode_census( .join("code-generations-v1") .join(&pointer.generation_file); let generation_bytes = std::fs::read(&generation_path).expect("read generation manifest"); - let generation_size = u64::try_from(generation_bytes.len()).expect("generation byte size"); - let expected_digest = - ManifestDigest::new(pointer.state_digest.clone()).expect("generation digest"); - let monolithic = match CodeIndexPublishedGenerationV1::decode_sealed_seek_reader( - File::open(&generation_path).expect("open generation manifest"), - generation_size, - Some(&expected_digest), - &UninterruptibleCodeIndexControlV1, - ) { - Ok(None) => RestartDecodeStatusV1::Abstained { - refused_revision: None, - }, - Ok(Some(_)) => RestartDecodeStatusV1::Decoded, - Err(CodeIndexProductionErrorV1::SupersededSealedGenerationRevision(revision)) => { - RestartDecodeStatusV1::Abstained { - refused_revision: Some(revision), - } - } - Err(CodeIndexProductionErrorV1::SourceCommitmentsUnavailable) => { - RestartDecodeStatusV1::SourceCommitmentRefused - } - Err(error) => panic!("monolithic restart census failed: {error}"), - }; - let segments = store.join("code-generation-segments-v1"); + let segments = code_generation_segments_root(store); let partitioned = CodeIndexPublishedGenerationV1::decode_partitioned_sealed( &generation_bytes, |request, buffer| { @@ -2014,20 +2026,16 @@ fn restart_decode_census( }, ); let generation = match partitioned { - Ok(Some(generation)) => generation, - Ok(None) => { + Ok(generation) => generation, + Err(CodeIndexProductionErrorV1::SupersededSealedGenerationRevision(_)) => { return RestartDecodeCensusV1 { - monolithic, - partitioned: RestartDecodeStatusV1::Abstained { - refused_revision: None, - }, + partitioned: RestartDecodeStatusV1::Abstained, sanitizer: RestartIdentityStatusV1::NotReached, pointer: RestartIdentityStatusV1::NotReached, }; } Err(CodeIndexProductionErrorV1::SourceCommitmentsUnavailable) => { return RestartDecodeCensusV1 { - monolithic, partitioned: RestartDecodeStatusV1::SourceCommitmentRefused, sanitizer: RestartIdentityStatusV1::NotReached, pointer: RestartIdentityStatusV1::NotReached, @@ -2047,7 +2055,6 @@ fn restart_decode_census( && generation.projection().publication_digest().as_str() == pointer.publication_digest && generation.manifest().seal.sealed_at.0 == pointer.sealed_at_micros; RestartDecodeCensusV1 { - monolithic, partitioned: RestartDecodeStatusV1::Decoded, sanitizer, pointer: if pointer_matches { @@ -2079,14 +2086,11 @@ fn restart_decode_census_reaches_partitioned_decode_and_matches_durable_identity assert_eq!( restart_decode_census(store.path(), &pointer), RestartDecodeCensusV1 { - monolithic: RestartDecodeStatusV1::Abstained { - refused_revision: None, - }, partitioned: RestartDecodeStatusV1::Decoded, sanitizer: RestartIdentityStatusV1::Matched, pointer: RestartIdentityStatusV1::Matched, }, - "shared premise: a current partitioned generation must survive the monolithic probe and reach exact sanitizer and pointer checks" + "shared premise: a current partitioned generation must decode and reach exact sanitizer and pointer checks" ); } @@ -2703,3 +2707,668 @@ fn stale_pointer_commit_does_not_replace_a_changed_active_pointer() { "the truncated pointer must still be the file" ); } + +/// The segment digests one scope's active manifest names: its file segments +/// and, last, its evidence pack. +fn active_segment_digests(scope: &Path) -> Vec { + let pointer: DurablePublicationPointerV1 = serde_json::from_slice( + &std::fs::read(scope.join("active-code-generation-v1.json")).expect("active pointer"), + ) + .expect("decode active pointer"); + let manifest = std::fs::read( + scope + .join("code-generations-v1") + .join(&pointer.generation_file), + ) + .expect("active manifest"); + CodeIndexPublishedGenerationV1::partitioned_segment_identities(&manifest) + .expect("segment identities") + .into_iter() + .map(|identity| { + sha256_hex_suffix(identity.digest.as_str()) + .expect("sha256 segment digest") + .to_owned() + }) + .collect() +} + +fn segment_files(segments_root: &Path) -> BTreeSet { + std::fs::read_dir(segments_root) + .expect("read shared segments") + .map(|entry| { + entry + .expect("segment entry") + .file_name() + .into_string() + .expect("UTF-8 segment name") + }) + .filter_map(|name| { + name.strip_prefix("segment-") + .and_then(|name| name.strip_suffix(".json")) + .map(str::to_owned) + }) + .collect() +} + +/// A primary checkout and a linked worktree of it, mounted as two scopes of +/// one project's `code-index-v1/`, both published at the same tree. +struct LinkedWorktreeScopesV1 { + _first: GitFixture, + _linked_root: TempDir, + linked: PathBuf, + _store: TempDir, + code_index_root: PathBuf, + first_scope: PathBuf, + linked_scope: PathBuf, +} + +fn publish_linked_worktree_scopes(files: &[(&str, &str)]) -> LinkedWorktreeScopesV1 { + let first = GitFixture::new(files); + let linked_root = TempDir::new_in(super::canonical_temp_root()).expect("linked root"); + let linked = linked_root.path().join("linked"); + super::git( + first.path(), + &[ + "worktree", + "add", + "-q", + "-b", + "linked", + linked.to_str().expect("linked path"), + "main", + ], + ); + let store = TempDir::new().expect("project store"); + let code_index_root = store.path().join("code-index-v1"); + std::fs::create_dir_all(&code_index_root).expect("project code-index root"); + let first_scope = scoped_code_index_store_root(&code_index_root, first.path()); + let linked_scope = scoped_code_index_store_root(&code_index_root, &linked); + let registry = CodeIndexSchedulerRegistryV1::new(2); + for (root, scope) in [ + (first.path(), &first_scope), + (linked.as_path(), &linked_scope), + ] { + let mut scheduler = registry + .open_worktree(test_project_id(), root, scope.clone()) + .expect("open worktree scheduler"); + published(scheduler.reconcile_now().expect("publish worktree")); + } + LinkedWorktreeScopesV1 { + _first: first, + _linked_root: linked_root, + linked, + _store: store, + code_index_root, + first_scope, + linked_scope, + } +} + +#[test] +fn linked_worktrees_that_seal_identical_files_share_one_segment_per_file() { + // Several bodies per file: in memory they sort by worktree-specific + // symbol occurrences, so the segment must not persist that order. + let scopes = publish_linked_worktree_scopes(&[ + ( + "src/lib.rs", + "pub fn alpha(value: u32) -> u32 { value + 1 }\n\ + pub fn beta(value: u32) -> u32 { value * 2 }\n\ + pub fn gamma(value: u32) -> u32 { value - 3 }\n\ + pub fn delta(value: u32) -> u32 { value / 4 }\n\ + pub fn epsilon(value: u32) -> u32 { value % 5 }\n\ + pub fn zeta(value: u32) -> u32 { value ^ 6 }\n\ + pub fn eta(value: u32) -> u32 { value | 7 }\n\ + pub fn theta(value: u32) -> u32 { value & 8 }\n", + ), + ( + "src/other.rs", + "pub fn other(value: u32) -> u32 { value + 1 }\n", + ), + ]); + let segments_root = scopes.code_index_root.join("code-generation-segments-v1"); + assert_eq!( + code_generation_segments_root(&scopes.first_scope), + segments_root + ); + assert_eq!( + code_generation_segments_root(&scopes.linked_scope), + segments_root + ); + for scope in [&scopes.first_scope, &scopes.linked_scope] { + assert!( + !scope.join("code-generation-segments-v1").exists(), + "a worktree scope holds no segments of its own" + ); + } + + let mut first = active_segment_digests(&scopes.first_scope); + let mut linked = active_segment_digests(&scopes.linked_scope); + let first_evidence = first.pop().expect("first evidence pack"); + let linked_evidence = linked.pop().expect("linked evidence pack"); + assert_eq!(first.len(), 2, "one file segment per source file"); + assert_eq!( + first, linked, + "identical files seal to identical, worktree-independent segments" + ); + assert_ne!(first_evidence, linked_evidence); + let mut expected = first.into_iter().collect::>(); + expected.extend([first_evidence, linked_evidence]); + assert_eq!( + segment_files(&segments_root), + expected, + "the project stores each shared file segment exactly once" + ); +} + +#[test] +fn retiring_one_worktree_keeps_the_segments_its_sibling_still_names() { + let scopes = + publish_linked_worktree_scopes(&[("src/lib.rs", "pub fn shared() -> u32 { 7 }\n")]); + // The linked worktree also seals a file the primary checkout does not. + super::write( + &scopes.linked, + "src/only_linked.rs", + "pub fn only_linked() {}\n", + ); + super::git(&scopes.linked, &["add", "-A"]); + super::git(&scopes.linked, &["commit", "-qm", "linked-only file"]); + let registry = CodeIndexSchedulerRegistryV1::new(1); + let mut linked_scheduler = registry + .open_worktree( + test_project_id(), + &scopes.linked, + scopes.linked_scope.clone(), + ) + .expect("reopen linked scheduler"); + linked_scheduler.notify_path(scopes.linked.join("src/only_linked.rs")); + published( + linked_scheduler + .reconcile_now() + .expect("publish linked-only file"), + ); + drop(linked_scheduler); + + let segments_root = code_generation_segments_root(&scopes.first_scope); + let first = active_segment_digests(&scopes.first_scope); + let linked = active_segment_digests(&scopes.linked_scope); + let retain = |scope: &Path| { + run_code_generation_retention( + scope, + &BTreeSet::new(), + CodeGenerationRetentionModeV1::Apply, + UtcMicros(unix_now_secs() * 1_000_000), + None, + ) + .expect("retention over shared segments") + }; + // The linked scope's superseded generation retires, and a sweep from + // the primary scope must still mark everything the linked manifest names. + retain(&scopes.linked_scope); + retain(&scopes.first_scope); + let present = segment_files(&segments_root); + for digest in first.iter().chain(&linked) { + assert!( + present.contains(digest), + "a sweep from one scope must keep what a sibling scope names" + ); + } + + // Collecting the linked scope strands only what it alone named. + std::fs::remove_dir_all(&scopes.linked_scope).expect("collect linked scope"); + retain(&scopes.first_scope); + let present = segment_files(&segments_root); + for digest in &first { + assert!( + present.contains(digest), + "the primary scope keeps its segments" + ); + } + let linked_only = linked + .iter() + .filter(|digest| !first.contains(digest)) + .collect::>(); + assert!( + linked_only.len() >= 2, + "the linked scope named at least its own file segment and evidence pack" + ); + for digest in linked_only { + assert!( + !present.contains(digest), + "a segment only the collected scope named is swept" + ); + } +} + +/// Serve one scope's text artifact to completion and return the descriptor +/// its active generation names. +fn serve_scope_text( + worktree: &Path, + scope: &Path, +) -> tracedecay_code_index_retention::code_index_generations::DurableCodeTextArtifactDescriptorV1 { + serve_scope_text_with_hits(worktree, scope, "alpha").0 +} + +/// [`serve_scope_text`], plus the route-independent identity of every +/// lexical hit the served artifact returns for `term`. +fn serve_scope_text_with_hits( + worktree: &Path, + scope: &Path, + term: &str, +) -> ( + tracedecay_code_index_retention::code_index_generations::DurableCodeTextArtifactDescriptorV1, + Vec, +) { + let registry = CodeIndexSchedulerRegistryV1::new(1); + let mut scheduler = registry + .open_worktree(test_project_id(), worktree, scope.to_path_buf()) + .expect("open worktree scheduler"); + scheduler + .reconcile_now() + .expect("adopt published generation"); + let latest = scheduler.latest_complete().expect("published generation"); + let mut passes = 0_usize; + while !latest + .advance_text_serving(64) + .expect("advance text-artifact build") + { + passes += 1; + assert!(passes < 10_000, "the text-artifact build never completed"); + } + let generation = latest.generation().manifest().generation_id.clone(); + let owners = latest.production_query_owners().expect("text query owners"); + let base = RetrievalRequest { + principal: PrincipalId::new("principal.shared-artifact").expect("principal"), + scope: RetrievalScope { + privacy_domain: latest.generation().manifest().privacy_domain.clone(), + root: SingleRootScopeV1 { + repository: latest.generation().snapshot().repository.clone(), + worktree: latest.generation().snapshot().worktree.clone(), + reference: latest.generation().snapshot().reference.clone(), + }, + }, + temporal_mode: TemporalModeV1::Current, + snapshot: RetrievalSnapshot { + watermarks: VectorWatermark::default(), + freshness_digest: FreshnessVectorDigest::new(format!("sha256:{}", "f".repeat(64))) + .expect("freshness digest"), + authorization_revision: AuthorizationRevision::new("authorization.shared.v1") + .expect("authorization revision"), + captured_at: UtcMicros(1), + }, + profile_id: "profile.shared-artifact.v1" + .to_owned() + .try_into() + .expect("profile"), + budget: RetrievalBudget { + max_candidates_per_lane: 16, + max_fused_candidates: 16, + max_hydrated_results: 16, + max_hydration_bytes: 65_536, + deadline_micros: None, + }, + }; + let query_view = EphemeralSanitizedQueryViewV1::sanitize( + term, + SanitizerRevision::new("sanitizer.shared-artifact.v1").expect("sanitizer"), + QueryNormalizationRevision::new("normalization.shared-artifact.v1").expect("normalization"), + ) + .expect("query view"); + let RetrieverOutcome::Complete(batch) = owners + .retrieve_lexical(&LexicalLaneRequest { + query_view: &query_view, + generation, + whole_terms: std::borrow::Cow::Owned(vec![term.to_owned()]), + subtokens: std::borrow::Cow::Owned(vec![term.to_owned()]), + phrases: std::borrow::Cow::Owned(Vec::new()), + proximities: std::borrow::Cow::Owned(Vec::new()), + field_filters: std::borrow::Cow::Owned(Vec::new()), + fuzzy_budget: 0, + lexical_profile_revision: ComponentRevision::new( + tracedecay_query::retrieval::QUERY_LEXICAL_PROFILE_REVISION_V1, + ) + .expect("lexical profile revision"), + score_domain: ScoreDomainId::new( + tracedecay_query::retrieval::QUERY_LEXICAL_SCORE_DOMAIN_V1, + ) + .expect("lexical score domain"), + budget: base.budget, + base, + control: &super::ReadyRetrievalControlV1, + }) + .expect("lexical retrieval") + else { + panic!("the served artifact must complete the lexical retrieval"); + }; + let mut hits = batch + .evidence_by_occurrence + .values() + .map(|evidence| { + format!( + "{} {:?} {:?} {:?} {:?}", + evidence.binding.occurrence.file, + evidence.binding.occurrence.symbol, + evidence.binding.occurrence.chunk, + evidence.field_scores_micros, + evidence.matched_whole_terms, + ) + }) + .collect::>(); + hits.sort(); + (active_text_descriptor(scope), hits) +} + +fn active_pointer(scope: &Path) -> DurablePublicationPointerV1 { + serde_json::from_slice( + &std::fs::read(scope.join("active-code-generation-v1.json")).expect("read pointer"), + ) + .expect("decode pointer") +} + +fn active_text_descriptor( + scope: &Path, +) -> tracedecay_code_index_retention::code_index_generations::DurableCodeTextArtifactDescriptorV1 { + let pointer = active_pointer(scope); + pointer + .generation_index + .iter() + .find(|entry| entry.generation_id == pointer.generation_id) + .and_then(|entry| entry.text_artifact().cloned()) + .expect("active generation names a text artifact") +} + +fn completed_text_artifacts(root: &Path) -> BTreeSet { + std::fs::read_dir(root) + .expect("read text artifact root") + .map(|entry| { + entry + .expect("artifact entry") + .file_name() + .into_string() + .expect("utf-8") + }) + .filter(|name| name.starts_with("text-artifact-") && name.ends_with(".bin")) + .collect() +} + +/// The content metadata an artifact stores, which lists its logical paths. +fn artifact_content_metadata(path: &Path) -> String { + let metadata: Vec = rusqlite::Connection::open(path) + .expect("open artifact") + .query_row( + "SELECT metadata FROM artifact_state WHERE singleton = 1", + [], + |row| row.get(0), + ) + .expect("read artifact metadata"); + String::from_utf8(metadata).expect("utf-8 metadata") +} + +const CLONE_FIXTURE_SOURCE: &str = "pub fn alpha(value: u32) -> u32 { let a = value + 1; let b = a * 2; let c = b - 3; let d = c / 4; a + b + c + d }\n\ + pub fn beta(input: u32) -> u32 { let a = input + 1; let b = a * 2; let c = b - 3; let d = c / 4; a + b + c + d }\n"; + +#[test] +fn linked_worktrees_that_index_identical_trees_share_one_text_artifact() { + let scopes = publish_linked_worktree_scopes(&[ + ("src/lib.rs", CLONE_FIXTURE_SOURCE), + ( + "src/other.rs", + "pub fn other(value: u32) -> u32 { value + 1 }\n", + ), + ]); + let (first, first_hits) = + serve_scope_text_with_hits(scopes._first.path(), &scopes.first_scope, "alpha"); + assert!( + code_text_artifact_staging_root(&scopes.first_scope).is_dir(), + "the first worktree builds the artifact" + ); + let (linked, linked_hits) = + serve_scope_text_with_hits(&scopes.linked, &scopes.linked_scope, "alpha"); + assert!( + !code_text_artifact_staging_root(&scopes.linked_scope).exists(), + "a worktree sealing content a sibling already published adopts it without building" + ); + assert_eq!(first.content_key, linked.content_key); + assert!(!first_hits.is_empty()); + assert_eq!( + first_hits, linked_hits, + "the adopted artifact serves identical results" + ); + let shared_root = code_text_artifacts_root(&scopes.first_scope); + assert_eq!( + shared_root, + scopes.code_index_root.join("code-text-artifacts-v1") + ); + assert_eq!(code_text_artifacts_root(&scopes.linked_scope), shared_root); + assert_ne!( + first.generation_id, linked.generation_id, + "each worktree seals its own generation" + ); + assert_eq!( + ( + &first.artifact_file, + &first.artifact_digest, + first.artifact_size_bytes + ), + ( + &linked.artifact_file, + &linked.artifact_digest, + linked.artifact_size_bytes + ), + "identical trees seal byte-identical text artifacts" + ); + assert_eq!( + completed_text_artifacts(&shared_root), + BTreeSet::from([first.artifact_file.clone()]), + "the project stores the shared artifact exactly once" + ); + for scope in [&scopes.first_scope, &scopes.linked_scope] { + assert!( + !scope.join("code-text-artifacts-v1").exists(), + "a worktree scope holds no completed artifact of its own" + ); + } +} + +#[test] +fn a_file_that_diverges_in_one_worktree_is_never_served_to_its_sibling() { + let scopes = publish_linked_worktree_scopes(&[("src/lib.rs", CLONE_FIXTURE_SOURCE)]); + super::write( + &scopes.linked, + "src/only_linked.rs", + "pub fn only_linked() -> u32 { 7 }\n", + ); + super::git(&scopes.linked, &["add", "-A"]); + super::git(&scopes.linked, &["commit", "-qm", "linked-only file"]); + { + let registry = CodeIndexSchedulerRegistryV1::new(1); + let mut linked_scheduler = registry + .open_worktree( + test_project_id(), + &scopes.linked, + scopes.linked_scope.clone(), + ) + .expect("reopen linked scheduler"); + linked_scheduler.notify_path(scopes.linked.join("src/only_linked.rs")); + published( + linked_scheduler + .reconcile_now() + .expect("publish linked-only file"), + ); + } + let first = serve_scope_text(scopes._first.path(), &scopes.first_scope); + let linked = serve_scope_text(&scopes.linked, &scopes.linked_scope); + assert_ne!(first.content_key, linked.content_key); + assert!( + code_text_artifact_staging_root(&scopes.linked_scope).is_dir(), + "one differing file forces the worktree to build its own artifact" + ); + assert_ne!( + first.artifact_digest, linked.artifact_digest, + "diverged trees seal different artifacts" + ); + let shared_root = code_text_artifacts_root(&scopes.first_scope); + assert!( + artifact_content_metadata(&shared_root.join(&linked.artifact_file)) + .contains("src/only_linked.rs") + ); + assert!( + !artifact_content_metadata(&shared_root.join(&first.artifact_file)) + .contains("src/only_linked.rs"), + "the primary worktree's artifact carries none of its sibling's divergent file" + ); +} + +#[test] +fn a_shared_artifact_that_fails_verification_is_rebuilt_not_adopted() { + let scopes = publish_linked_worktree_scopes(&[("src/lib.rs", CLONE_FIXTURE_SOURCE)]); + let first = serve_scope_text(scopes._first.path(), &scopes.first_scope); + // The shared file keeps its name and size but no longer holds the bytes + // its content address names. + let shared = code_text_artifacts_root(&scopes.first_scope).join(&first.artifact_file); + let mut bytes = std::fs::read(&shared).expect("read shared artifact"); + let last = bytes.len() - 1; + bytes[last] ^= 0xff; + std::fs::write(&shared, &bytes).expect("damage shared artifact"); + let linked = serve_scope_text(&scopes.linked, &scopes.linked_scope); + assert!( + code_text_artifact_staging_root(&scopes.linked_scope).is_dir(), + "a key match that fails verification builds instead of adopting" + ); + assert_eq!(linked.content_key, first.content_key); + assert_eq!(linked.artifact_file, first.artifact_file); + assert_eq!( + Some(encode_lowercase_hex(&Sha256::digest( + std::fs::read(&shared).expect("read rebuilt artifact") + ))) + .as_deref(), + sha256_hex_suffix(first.artifact_digest.as_str()), + "the rebuild restores the bytes the content address names" + ); +} + +#[test] +fn retiring_one_worktree_keeps_the_text_artifact_its_sibling_references() { + let scopes = publish_linked_worktree_scopes(&[("src/lib.rs", CLONE_FIXTURE_SOURCE)]); + let first = serve_scope_text(scopes._first.path(), &scopes.first_scope); + let linked = serve_scope_text(&scopes.linked, &scopes.linked_scope); + assert_eq!(first.artifact_file, linked.artifact_file); + let shared = code_text_artifacts_root(&scopes.first_scope).join(&first.artifact_file); + let retain = |scope: &Path| { + run_code_generation_retention( + scope, + &BTreeSet::new(), + CodeGenerationRetentionModeV1::Apply, + UtcMicros(unix_now_secs() * 1_000_000), + None, + ) + .expect("retention over shared text artifacts") + }; + let withdraw = |scope: &Path| { + let lock = acquire_code_generation_store_lock(scope).expect("scope store lock"); + let pointer = active_pointer(scope); + let descriptor = active_text_descriptor(scope); + withdraw_verified_text_artifact_under_lock(&lock, &pointer, &descriptor) + .expect("withdraw text artifact"); + }; + // A scope that stops naming the shared artifact must not collect it + // while its sibling still does. + withdraw(&scopes.first_scope); + retain(&scopes.first_scope); + assert!( + shared.exists(), + "retention from one scope keeps an artifact a sibling scope names" + ); + // Collecting the linked scope leaves the artifact unnamed. + std::fs::remove_dir_all(&scopes.linked_scope).expect("collect linked scope"); + retain(&scopes.first_scope); + assert!(!shared.exists(), "an artifact no scope names is collected"); +} + +#[test] +fn a_sweep_never_collects_segments_a_publication_has_not_yet_named() { + let fixture = GitFixture::new(&[ + ("src/a.rs", "pub fn a() -> u32 { 1 }\n"), + ("src/b.rs", "pub fn b() -> u32 { 2 }\n"), + ("src/c.rs", "pub fn c() -> u32 { 3 }\n"), + ]); + let source_store = TempDir::new().expect("source store root"); + let generation = { + let mut scheduler = scheduler( + &fixture, + source_store.path().to_path_buf(), + Arc::new(SharedCodeIndexBytePoolV1::default()), + ); + published(scheduler.reconcile_now().expect("build generation")); + Arc::clone( + &scheduler + .latest_complete_already_decoded() + .expect("generation remains decoded") + .generation, + ) + }; + let project_store = TempDir::new().expect("project store"); + let code_index_root = project_store.path().join("code-index-v1"); + let target_scope = scoped_code_index_store_root(&code_index_root, fixture.path()); + // A sibling worktree scope of the same project whose maintenance pass + // runs while the target publication is between segments and manifest. + let sibling_scope = code_index_root.join("a".repeat(64)); + std::fs::create_dir_all(sibling_scope.join("code-generations-v1")).expect("sibling scope"); + std::fs::create_dir_all(&target_scope).expect("target scope"); + let sweeps = Arc::new(std::sync::Mutex::new(Vec::new())); + let observed_sweeps = Arc::clone(&sweeps); + let observed_sibling = sibling_scope.clone(); + let mut publication = super::super::DaemonCodeIndexPublicationStoreV1::new( + &target_scope, + fixture.path(), + SanitizerRevision::new(tracedecay_privacy::CODE_SOURCE_SANITIZER_VERSION_V1) + .expect("sanitizer revision"), + ) + .expect("open target publication store") + .with_seal_segment_observer_for_test(Arc::new(move || { + let outcome = run_code_generation_retention( + &observed_sibling, + &BTreeSet::new(), + CodeGenerationRetentionModeV1::Apply, + UtcMicros(1), + None, + ) + .map(|_| ()); + observed_sweeps + .lock() + .expect("sweep observations") + .push(outcome); + })); + + publication + .publish_atomically(&generation.sealed_scope(), None, Arc::clone(&generation)) + .expect("publish while sibling sweeps run"); + + let sweeps = sweeps.lock().expect("sweep observations"); + assert_eq!( + sweeps.len(), + 3, + "one sibling sweep per written file segment" + ); + assert!( + sweeps.iter().all(|sweep| matches!( + sweep, + Err(CodeGenerationRetentionErrorV1::GenerationStoreBusy) + )), + "a sweep that saw unnamed segments must wait out the publication: {sweeps:?}" + ); + let segments_root = code_generation_segments_root(&target_scope); + let present = segment_files(&segments_root); + for digest in active_segment_digests(&target_scope) { + assert!(present.contains(&digest), "every named segment survived"); + } + // With the manifest durable the same sweep runs and keeps everything. + run_code_generation_retention( + &sibling_scope, + &BTreeSet::new(), + CodeGenerationRetentionModeV1::Apply, + UtcMicros(1), + None, + ) + .expect("sweep after publication"); + assert_eq!(segment_files(&segments_root), present); +} diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/reconcile.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/reconcile.rs index 25d49aa0a0..1892999250 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/reconcile.rs @@ -25,12 +25,12 @@ use tracedecay_runtime_core::resident_memory::{ use super::{ ALPHA_LIB_V1, GitFixture, RETAINED_REVISION_0, SERVING_SEAT_FAILURE_CEILING, advance_pointer_to_unseated_successor, application_context, clear_pending_wake_until_quiet, - committed_capture_corpus_files, core_search_request, drain_clone_backfill, git, git_stdout, - hold_scheduler_for_root, mounted_core_query_worktree, - mounted_core_query_worktree_with_one_permit, published, query_authority, query_meta, - quiesced_background_reconcile_admission, replace_scheduler_chunker_revision, - replace_scheduler_policy_revision, rewrite_active_rust_extractor_revision, - rewrite_preserving_stat, scheduler, scheduler_with_policy, served_lexical_texts, + committed_capture_corpus_files, core_search_request, git, git_stdout, hold_scheduler_for_root, + mounted_core_query_worktree, mounted_core_query_worktree_with_one_permit, published, + query_authority, query_meta, quiesced_background_reconcile_admission, + replace_scheduler_chunker_revision, replace_scheduler_policy_revision, + rewrite_active_rust_extractor_revision, rewrite_preserving_stat, scheduler, + scheduler_with_policy, served_lexical_texts, settle_text_projection, settled_owner_with_idle_admission, test_project_id, wait_for_dashboard_ready, wait_for_event_to_ready, wait_for_generation_change, wait_for_initial_generation, wait_for_live_complete_generation, wait_for_live_complete_generation_by_polling, @@ -42,8 +42,8 @@ use crate::{ code_index::{ chunks::content_digest, production::{ - CodeIndexExecutionControlV1, DAEMON_CODE_INDEX_CHUNKER_REVISION, - UninterruptibleCodeIndexControlV1, + CodeIndexExecutionControlV1, CodeIndexPublishedGenerationV1, + DAEMON_CODE_INDEX_CHUNKER_REVISION, UninterruptibleCodeIndexControlV1, }, }, code_index_scheduler::{ @@ -675,11 +675,9 @@ async fn registry_feeds_publications_and_bounded_freshness_reads() { /// Poll a mounted worktree's dashboard clone-index status until it reports /// ready coverage. /// -/// `clone_index_status` reads the clone-successor slot with `try_lock` so a -/// freshness read never joins a running backfill. A single sample therefore -/// reports `Unavailable { "clone-index status is being updated" }` whenever a -/// freshly published generation's successor still holds the slot, which is a -/// truthful transient, not the settled answer a caller is asking for. +/// Clone status is `Unavailable` until the published generation's owners +/// serve, which is a truthful transient, not the settled answer a caller is +/// asking for. async fn wait_for_ready_clone_index( registry: &CodeIndexSchedulerRegistryV1, path: &Path, @@ -697,7 +695,7 @@ async fn wait_for_ready_clone_index( }) => return observation, transient => assert!( Instant::now() <= deadline, - "the V16 artifact never reported ready clone coverage: {transient:?}" + "the artifact never reported ready clone coverage: {transient:?}" ), } tokio::time::sleep(Duration::from_millis(5)).await; @@ -1462,9 +1460,16 @@ fn occurrence_graph_store_is_available_before_catalog_warm() { ); } +/// Linked worktrees sealing identical content share its occurrence identity +/// and physical artifacts; each worktree still seals its own snapshot and +/// generation, and a worktree's generation never names content only its +/// sibling holds. #[test] -fn cross_worktree_byte_reuse_without_identity_alias() { - let first = GitFixture::new(&[("src/lib.rs", "pub fn shared() -> u32 { 7 }\n")]); +fn linked_worktrees_share_identity_for_identical_content_and_never_serve_divergent_content() { + let first = GitFixture::new(&[ + ("src/lib.rs", "pub fn shared() -> u32 { 7 }\n"), + ("src/other.rs", "pub fn other() -> u32 { 9 }\n"), + ]); let linked_root = TempDir::new().expect("linked worktree root"); let linked = linked_root.path().join("linked"); let linked_arg = linked.to_str().expect("linked worktree path"); @@ -1500,72 +1505,103 @@ fn cross_worktree_byte_reuse_without_identity_alias() { "matching parse/chunk artifacts must be physically shared" ); assert_eq!(first_publish.repository_id, second_publish.repository_id); - assert_eq!(first_generation.manifest().project_id, project_id); - assert_eq!(second_generation.manifest().project_id, project_id); - assert_ne!( - first_generation.snapshot().worktree, - second_generation.snapshot().worktree + assert_eq!( + first_publish.file_occurrence_ids, second_publish.file_occurrence_ids, + "identical content shares its occurrence identity across linked worktrees" ); + let symbol_occurrences = |generation: &CodeIndexPublishedGenerationV1| { + generation + .symbols() + .symbols + .iter() + .map(|symbol| symbol.occurrence.clone()) + .collect::>() + }; assert_eq!( - first_publish.snapshot_content_identity, - second_publish.snapshot_content_identity + symbol_occurrences(&first_generation), + symbol_occurrences(&second_generation) ); + // Snapshot authority stays with each worktree's own generation. assert_ne!( - first_publish.file_occurrence_ids, second_publish.file_occurrence_ids, - "shared artifacts must never alias worktree occurrence identity" + first_generation.snapshot().worktree, + second_generation.snapshot().worktree ); assert_ne!(first_publish.generation_id, second_publish.generation_id); assert_ne!( first_generation.manifest().snapshot_digest, second_generation.manifest().snapshot_digest ); - assert_eq!( - first_generation.capability().manifest_digest, - second_generation.capability().manifest_digest, - "byte-identical capability evidence is generation-free; generation, occurrence, \ - snapshot, and publication identities remain worktree-local above and below" - ); assert_ne!( first_generation.projection().publication_digest(), second_generation.projection().publication_digest(), "publication identity remains generation-local" ); - git(&linked, &["mv", "src/lib.rs", "src/renamed.rs"]); + // Divergent content in the linked worktree mints its own identity and is + // never part of what the primary worktree's generation serves. + write( + &linked, + "src/lib.rs", + "pub fn only_in_linked() -> u32 { 8 }\n", + ); second_scheduler.notify_path(linked.join("src/lib.rs")); - second_scheduler.notify_path(linked.join("src/renamed.rs")); - published( + let diverged_publish = published( second_scheduler .reconcile_now() - .expect("renamed linked-worktree publish"), + .expect("diverged linked-worktree publish"), ); - let after_rename = registry.byte_pool_stats(); + let shared = first_publish + .file_occurrence_ids + .iter() + .filter(|occurrence| diverged_publish.file_occurrence_ids.contains(occurrence)) + .count(); assert_eq!( - after_rename.parse_chunk_reused, reuse.parse_chunk_reused, - "same content at a new logical path must not reuse path-bound parse/chunk artifacts" + shared, 1, + "only the unchanged file keeps a shared identity: {:?} vs {:?}", + first_publish.file_occurrence_ids, diverged_publish.file_occurrence_ids ); + let names = |generation: &CodeIndexPublishedGenerationV1| { + generation + .symbols() + .symbols + .iter() + .map(|symbol| symbol.simple_name.clone()) + .collect::>() + }; + let primary = first_scheduler + .latest_complete() + .expect("first worktree remains current") + .generation; + let diverged = second_scheduler + .latest_complete() + .expect("diverged linked generation") + .generation; + assert_eq!( + primary.manifest().generation_id, + first_publish.generation_id, + "editing one linked worktree must not invalidate its sibling" + ); + assert!(names(&diverged).contains("only_in_linked")); + assert!( + !names(&primary).contains("only_in_linked"), + "a read routed to the primary worktree's generation never sees linked-only symbols" + ); + assert!(names(&primary).contains("shared")); + assert!(!names(&diverged).contains("shared")); - write(&linked, "src/renamed.rs", "pub fn shared() -> u32 { 8 }\n"); + git(&linked, &["mv", "src/lib.rs", "src/renamed.rs"]); + second_scheduler.notify_path(linked.join("src/lib.rs")); second_scheduler.notify_path(linked.join("src/renamed.rs")); + let before_rename = registry.byte_pool_stats(); published( second_scheduler .reconcile_now() - .expect("edited linked-worktree publish"), - ); - let after_edit = registry.byte_pool_stats(); - assert_eq!( - after_edit.parse_chunk_reused, after_rename.parse_chunk_reused, - "changed source content must not reuse the prior parse/chunk artifact" + .expect("renamed linked-worktree publish"), ); assert_eq!( - first_scheduler - .latest_complete() - .expect("first worktree remains current") - .generation - .manifest() - .generation_id, - first_publish.generation_id, - "editing one linked worktree must not invalidate its sibling" + registry.byte_pool_stats().parse_chunk_reused, + before_rename.parse_chunk_reused, + "same content at a new logical path must not reuse path-bound parse/chunk artifacts" ); } @@ -2688,9 +2724,9 @@ async fn long_text_projection_renews_source_before_seating_and_noop_follow_up_se }) .await; let generation = ready.generation().manifest().generation_id.clone(); - // The seat precedes the clone-fingerprint backfill; settle it so the pass - // observed below is the source-verification Noop alone. - drain_clone_backfill(®istry, fixture.path()).await; + // Settle the mount-era passes so the pass observed below is the + // source-verification Noop alone. + settle_text_projection(®istry, fixture.path()).await; // Exercise the ordinary expiry path too. The existing seat keeps its exact // witness while the source-verification Noop renews the proof. @@ -2887,7 +2923,7 @@ async fn ignored_dependency_waits_for_global_admission_before_publication_gate() // Draining leaves the busy follow-up wake armed, and every pass it starts // owns the single global admission permit this test needs idle. Settle // that chain and burn the banked permit behind it before sampling. - drain_clone_backfill(®istry, fixture.path()).await; + settle_text_projection(®istry, fixture.path()).await; settled_owner_with_idle_admission(®istry, fixture.path()).await; let generation = latest.generation(); let verified_import = generation @@ -3705,13 +3741,11 @@ async fn dashboard_progress_does_not_wait_for_the_scheduler_mutex() { wait_for_initial_generation(®istry, fixture.path()).await; wait_for_dashboard_ready(®istry, fixture.path()).await; // `refresh_in_flight` is the pass counter *or* the pending-wake slot, and - // `wait_for_dashboard_ready` only joins the running pass. The seat no - // longer waits for the clone successor, so the mount leaves backfill work - // behind, and the wakes that drain it leave a banked permit whose no-op - // pass projects Verifying instead of Fresh. Settle the whole mount-era - // chain, then hold the admission so no further pass can start under the - // sample below. - drain_clone_backfill(®istry, fixture.path()).await; + // `wait_for_dashboard_ready` only joins the running pass; a banked + // permit's no-op pass projects Verifying instead of Fresh. Settle the + // whole mount-era chain, then hold the admission so no further pass can + // start under the sample below. + settle_text_projection(®istry, fixture.path()).await; settled_owner_with_idle_admission(®istry, fixture.path()).await; let _quiet_owner = quiesced_background_reconcile_admission(®istry, fixture.path()).await; let canonical_root = @@ -3802,7 +3836,7 @@ async fn busy_query_does_not_rearm_dashboard_verification() { .expect("mount daemon-owned scheduler"); wait_for_initial_generation(®istry, fixture.path()).await; wait_for_dashboard_ready(®istry, fixture.path()).await; - drain_clone_backfill(®istry, fixture.path()).await; + settle_text_projection(®istry, fixture.path()).await; settled_owner_with_idle_admission(®istry, fixture.path()).await; let admission = quiesced_background_reconcile_admission(®istry, fixture.path()).await; let canonical_root = @@ -3998,12 +4032,11 @@ async fn unchanged_background_freshness_probe_posts_no_overflow_wake() { .await .expect("mount daemon-owned scheduler"); wait_for_initial_generation(®istry, fixture.path()).await; - // The seat no longer waits for the clone successor, so the mount leaves - // pending backfill behind. Draining it is a wake of its own, and every - // wake posts its own receipt, so settle the whole mount-era chain first: - // a pass that ends with a wake still pending re-arms a busy follow-up - // whose receipt would otherwise land inside the probe's window below. - drain_clone_backfill(®istry, fixture.path()).await; + // Every wake posts its own receipt, so settle the whole mount-era chain + // first: a pass that ends with a wake still pending re-arms a busy + // follow-up whose receipt would otherwise land inside the probe's window + // below. + settle_text_projection(®istry, fixture.path()).await; wait_for_settled_owner(®istry, fixture.path()).await; wait_for_event_to_ready(®istry).await; let canonical = canonical_existing_identity(fixture.path()).expect("canonical fixture"); @@ -4179,12 +4212,11 @@ async fn elapsed_freshness_window_alone_does_not_make_dashboard_state_stale() { .expect("mount daemon-owned scheduler"); wait_for_initial_generation(®istry, fixture.path()).await; wait_for_dashboard_ready(®istry, fixture.path()).await; - // The mount leaves clone backfill behind, and the wakes that drain it - // leave a banked permit whose no-op pass projects `Verifying` instead of - // `Fresh` (CI run 35425541839). Settle the mount-era chain, hold the - // admission so no pass can start under the sample, and prove the - // pending-wake slot stays empty, exactly as the text-progress test does. - drain_clone_backfill(®istry, fixture.path()).await; + // A banked permit's no-op pass projects `Verifying` instead of `Fresh`. + // Settle the mount-era chain, hold the admission so no pass can start + // under the sample, and prove the pending-wake slot stays empty, exactly + // as the text-progress test does. + settle_text_projection(®istry, fixture.path()).await; settled_owner_with_idle_admission(®istry, fixture.path()).await; let _quiet_owner = quiesced_background_reconcile_admission(®istry, fixture.path()).await; let canonical = canonical_existing_identity(fixture.path()).expect("canonical fixture"); @@ -4237,16 +4269,10 @@ async fn dashboard_freshness_reports_pending_rebuild_liveness() { .expect("mount daemon-owned scheduler"); wait_for_initial_generation(®istry, fixture.path()).await; wait_for_dashboard_ready(®istry, fixture.path()).await; - // The mount seats exact/lexical before the clone successor is built, so - // `wait_for_dashboard_ready` returns with that backfill still pending, and - // the admission below only parks a *new* pass at its dequeue point. A - // backfill slice advances under the clone-successor slot lock, which - // `clone_index_status` takes with `try_lock`: a sample that lands inside - // one reports `Unavailable { "clone-index status is being updated" }` - // before the source-stale branch can answer `Stale` (CI run 35432037843). - // Drain the mount-era backfill and burn the wake permits it banks, so the + // The admission below only parks a *new* pass at its dequeue point. + // Settle the mount-era passes and burn the wake permits they bank, so the // held admission is the only scheduling this sample can observe. - drain_clone_backfill(®istry, fixture.path()).await; + settle_text_projection(®istry, fixture.path()).await; settled_owner_with_idle_admission(®istry, fixture.path()).await; let admission = registry @@ -4296,9 +4322,9 @@ async fn a_fresh_seat_declines_query_admission_during_source_verification() { let store = TempDir::new().expect("store root"); let (registry, scope) = mounted_core_query_worktree(&fixture, &store).await; wait_for_dashboard_ready(®istry, fixture.path()).await; - // Pending clone work is a reason to admit a background pass; settle it so - // the freshness gate alone decides this admission. - drain_clone_backfill(®istry, fixture.path()).await; + // Settle the mount-era passes so the freshness gate alone decides this + // admission. + settle_text_projection(®istry, fixture.path()).await; registry.clear_pending_wake_for_scope(&scope).await; let pass = registry @@ -5420,10 +5446,10 @@ async fn concurrent_query_admissions_claim_one_pending_wake_before_worker_coales let fixture = GitFixture::new(&[("src/main.rs", "fn main() {}\n")]); let store = TempDir::new().expect("store root"); let (registry, scope) = mounted_core_query_worktree_with_one_permit(&fixture, &store).await; - // Clone backfill owns the same coalesced pending-wake slot. This test is - // about simultaneous query admissions, so finish that independent - // production journey before establishing the empty-slot precondition. - drain_clone_backfill(®istry, fixture.path()).await; + // The mount's own passes own the same coalesced pending-wake slot. This + // test is about simultaneous query admissions, so settle them before + // establishing the empty-slot precondition. + settle_text_projection(®istry, fixture.path()).await; // Take the shared admission first, through the helper that also waits out // an in-flight pass: from here no new pass can start, so the quiet window // established below stays quiet. A raw `acquire_owned` returns the instant @@ -5718,9 +5744,9 @@ async fn foreign_wake_arriving_during_query_claim_drop_is_retained() { let fixture = GitFixture::new(&[("src/main.rs", "fn main() {}\n")]); let store = TempDir::new().expect("store root"); let (registry, scope) = mounted_core_query_worktree_with_one_permit(&fixture, &store).await; - // A query over pending clone work is admitted for that work and never - // reaches the claim gate under test; settle the backfill first. - drain_clone_backfill(®istry, fixture.path()).await; + // A query over pending text work is admitted for that work and never + // reaches the claim gate under test; settle it first. + settle_text_projection(®istry, fixture.path()).await; let admission = quiesced_background_reconcile_admission(®istry, fixture.path()).await; // Same hang: a tail's `BusyFollowUp` stamp declines the request before the // claim gate this test waits on. @@ -9723,34 +9749,46 @@ async fn retryable_graph_activation_does_not_block_changed_text_generation() { tracedecay_contracts::code_index_freshness::CodeIndexBuildPhaseV1::Ready ); } - assert!( - registry - .latest_complete_serving_for_scope(&scope) - .await - .is_none(), - "retryable graph activation must not expose an unactivated graph owner" + // A retryable failure still seats the changed generation; its graph stays + // typed pending until a retry activates it. + let seat_deadline = std::time::Instant::now() + Duration::from_secs(10); + let seated = loop { + if let Some(latest) = registry.latest_complete_serving_for_scope(&scope).await + && latest.generation().manifest().generation_id == refreshed_generation_id + { + break latest; + } + assert!( + std::time::Instant::now() <= seat_deadline, + "graph retry backoff withheld the changed generation's seat" + ); + tokio::time::sleep(Duration::from_millis(10)).await; + }; + assert_eq!( + seated.code_graph_serving_readiness(), + tracedecay_contracts::code_index_freshness::CodeGraphServingReadinessV1::Pending, + "retryable graph activation must not expose an activated graph" ); // Clearing the injected failure lets the scheduled backoff activate the - // changed generation without resealing it. + // seated generation without resealing it. super::super::graph_activation::set_injected_activation_failures(&sealed_worktree_id, 0); let deadline = std::time::Instant::now() + Duration::from_secs(10); - loop { - if registry - .latest_complete_serving_for_scope(&scope) - .await - .is_some_and(|latest| { - latest.generation().manifest().generation_id == refreshed_generation_id - }) - { - break; - } + while seated.code_graph_serving_readiness() + != tracedecay_contracts::code_index_freshness::CodeGraphServingReadinessV1::Ready + { assert!( std::time::Instant::now() <= deadline, "the backoff retry did not activate the sealed generation" ); tokio::time::sleep(Duration::from_millis(25)).await; } + assert_eq!( + registry.latest_generation_id(fixture.path()).await, + Some(refreshed_generation_id), + "activation must not reseal the changed generation" + ); + assert_eq!(generation_files(&scoped_store), 2); registry.shutdown().await; } @@ -10080,6 +10118,20 @@ async fn busy_admission_schedules_follow_up_cadence_wake() { let _ = release_rx.recv(); }); held_rx.recv().expect("lock acquired"); + // Busy means a pass owns the worktree: wake one and let it park on the + // held scheduler before the read. + let _ = registry.notify_hook_overflow(fixture.path()).await; + let busy_deadline = std::time::Instant::now() + Duration::from_secs(5); + while !registry + .reconcile_in_progress_for_test(fixture.path()) + .await + { + assert!( + std::time::Instant::now() <= busy_deadline, + "the woken pass never took the worktree" + ); + tokio::time::sleep(Duration::from_millis(5)).await; + } let latest = tokio::time::timeout( Duration::from_millis(250), diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/retained_configuration_tests.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/retained_configuration_tests.rs index fa66cfaccf..2aea99e22e 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/retained_configuration_tests.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/retained_configuration_tests.rs @@ -1,6 +1,7 @@ use std::sync::Arc; use tempfile::TempDir; +use tracedecay_code_index_retention::code_index_generations::code_generation_segments_root; use super::{ CodeIndexSchedulerRegistryV1, GitFixture, SharedCodeIndexBytePoolV1, published, @@ -44,7 +45,7 @@ async fn partitioned_restart_rebuilds_incompatible_retained_generation() { generation }; rewrite_active_rust_extractor_revision(&scoped_store, "extractor.rust.v3"); - let segment_path = std::fs::read_dir(scoped_store.join("code-generation-segments-v1")) + let segment_path = std::fs::read_dir(code_generation_segments_root(&scoped_store)) .expect("read retained segment directory") .find_map(|entry| { let path = entry.expect("read retained segment entry").path(); diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/search_permit_release.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/search_permit_release.rs index 86c72b9d58..b6e11e62cb 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/search_permit_release.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/search_permit_release.rs @@ -553,11 +553,9 @@ async fn a_family_read_that_loses_the_permit_race_waits_for_the_permit() { let store = TempDir::new().expect("store root"); let (registry, scope) = mounted_core_query_worktree(&fixture, &store).await; let latest = wait_for_live_complete_generation(®istry, fixture.path()).await; - // A family read serves clone postings, and the seat precedes the - // clone-fingerprint backfill. Leaving that backfill in flight made the - // reads below report the warming generation rather than the permit - // handover this test is about. - drain_clone_backfill(®istry, fixture.path()).await; + // Settle the mount's worker first so the reads below observe the permit + // handover this test is about, not a warming generation. + settle_text_projection(®istry, fixture.path()).await; let source = crate::code_index_branch_diff::generation_symbols( latest.generation(), Some("src/first.rs"), diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/serving.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/serving.rs index 1dc5ec30a1..75aa44bdd0 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/serving.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/serving.rs @@ -11,14 +11,14 @@ use std::{ use sha2::{Digest, Sha256}; use tempfile::TempDir; use tracedecay_code_index_retention::code_index_generations::{ - acquire_code_generation_store_lock, code_text_artifacts_root, + acquire_code_generation_store_lock, code_text_artifact_staging_root, code_text_artifacts_root, }; use tracedecay_contracts::{ CallableCodeOperationKind, CallableCodeQueryPort, CodeQueryScope, CodeRelationRequest, - CodeSymbolSearchRequest, ExactOccurrenceRequest, OmissionReason, OpaqueCursor, PageRequest, - PhraseSearchRequest, QualifiedNameRequest, ResolvedScope, ResultProjection, RetrievalOrder, - RetrievalPortContext, RetrievalPortOutcome, RetrievalRequestMeta, SourceMetadataRequest, - callable_code_operation, + CodeSymbolSearchRequest, CoverageCompleteness, ExactOccurrenceRequest, OmissionReason, + OpaqueCursor, PageRequest, PhraseSearchRequest, QualifiedNameRequest, ResolvedScope, + ResultProjection, RetrievalOrder, RetrievalPortContext, RetrievalPortOutcome, + RetrievalRequestMeta, SourceMetadataRequest, callable_code_operation, retrieval::{ CodeFacetDimension, CodeFacetRequest, CodeHierarchyRequest, CodeImpactRequest, CodeImplementationsRequest, CodeNavigationRequest, CodeTimelineRequest, @@ -27,38 +27,41 @@ use tracedecay_contracts::{ }; use tracedecay_domain::{ AuthorizationRevision, CodeGenerationId, ComponentRevision, EphemeralSanitizedQueryViewV1, - ExactAdmissionRuleRevision, FreshnessVectorDigest, PrincipalId, PrivacyDomainId, ProjectId, - ProviderEvaluationStateV1, PublicRetrieverStatus, QueryNormalizationRevision, - RelationEdgeKindV1, RetrievalBudget, RetrievalRequest, RetrievalScope, RetrievalSnapshot, - RetrieverKind, RetrieverOutcome, SanitizerRevision, ScoreDomainId, SingleRootScopeV1, - TemporalModeV1, UtcMicros, VectorWatermark, encode_lowercase_hex, sha256_hex_suffix, + ExactAdmissionRuleRevision, FreshnessVectorDigest, ManifestDigest, PrincipalId, + PrivacyDomainId, ProjectId, ProviderEvaluationStateV1, PublicRetrieverStatus, + QueryNormalizationRevision, RelationEdgeKindV1, RetrievalBudget, RetrievalRequest, + RetrievalScope, RetrievalSnapshot, RetrieverKind, RetrieverOutcome, SanitizerRevision, + ScoreDomainId, SingleRootScopeV1, TemporalModeV1, UtcMicros, VectorWatermark, + encode_lowercase_hex, sha256_hex_suffix, }; use tracedecay_query::retrieval::{ RetrievalPortError, exact::{CentralExactAdmissionAuthorityV1, ExactAdmissionAuthority, ExactLaneRequest}, lexical::{ CODE_LEXICAL_ARTIFACT_BUILD_MEMORY_BUDGET_BYTES_V1, - CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, - CODE_LEXICAL_ARTIFACT_SQLITE_CACHE_BYTES_V1, CodeLexicalArtifactBuilderV1, + CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, CodeLexicalArtifactBuilderV1, CodeLexicalArtifactFinalizationStepV1, CodeLexicalArtifactReaderV1, LexicalLaneRequest, LexicalRouteKindV1, LexicalRoutingV1, }, }; use tracedecay_runtime_core::resident_memory::{ - DEFAULT_PROCESS_RESIDENT_MEMORY_LIMIT_V1, ProcessResidentMemoryV1, ResidentMemoryComponentIdV1, - ResidentMemoryPressureV1, sampled_process_resident_bytes_v1, + DEFAULT_PROCESS_RESIDENT_MEMORY_LIMIT_V1, ProcessResidentMemoryV1, ResidentMemoryPressureV1, + sampled_process_resident_bytes_v1, }; +use tracedecay_session_temporal_store::SessionTemporalAccess; + use super::{ - ALPHA_LIB_V1, CALLER_PAGE, GitFixture, ReadyRetrievalControlV1, active_text_artifact_path, - application_context, build_progress_snapshot, caller_star_sources, callers_page_meta, - core_search_request, decode_hex, drain_clone_backfill, git, install_verified_graph_store, - install_verified_graph_store_on_text, mount_core_query_authority, mount_query_authority, - mounted_core_query_worktree, mounted_core_query_worktree_with_one_permit, - moved_reference_scope, progress_snapshot_for_generation, published, query_authority, - query_authority_with_candidate_cap, query_meta, quiesced_background_reconcile_admission, - ranked_symbol_names, ranks_symbol, rewrite_active_text_artifact_format_revision, - routed_core_search_request, scheduler, test_project_id, wait_for_live_complete_generation, + ALPHA_LIB_V1, CALLER_PAGE, CALLER_STAR, GitFixture, ReadyRetrievalControlV1, + active_text_artifact_path, application_context, build_progress_snapshot, callee_fanout_sources, + caller_star_sources, callers_page_meta, core_search_request, decode_hex, git, + install_verified_graph_store, install_verified_graph_store_on_text, mount_core_query_authority, + mount_query_authority, mounted_core_query_worktree, + mounted_core_query_worktree_with_one_permit, moved_reference_scope, + progress_snapshot_for_generation, published, query_authority, query_meta, + quiesced_background_reconcile_admission, ranked_symbol_names, ranks_symbol, + rewrite_active_text_artifact_format_revision, routed_core_search_request, scheduler, + settle_text_projection, test_project_id, wait_for_live_complete_generation, wait_for_queryable_text_generation, wait_for_queryable_text_generation_change, }; use crate::{ @@ -74,6 +77,11 @@ use crate::{ }; use tracedecay_runtime_core::path_safety::canonical_existing_identity; +/// A content key for publications whose adoption the test does not exercise. +fn test_content_key() -> ManifestDigest { + ManifestDigest::new(format!("sha256:{}", "c".repeat(64))).expect("content key") +} + #[test] fn text_artifact_source_batches_scale_with_build_memory() { assert_eq!( @@ -90,42 +98,6 @@ fn text_artifact_source_batches_scale_with_build_memory() { ); } -#[test] -fn clone_successor_batches_leave_scratch_and_metadata_headroom() { - const RESERVATION: usize = 128 * 1024 * 1024; - const PAGE: usize = 4 * 1024 * 1024; - const SCRATCH: usize = 4 * PAGE; - let (pages, bytes) = super::super::clone_successor_source_batch_limits_from_charges(0) - .expect("empty metadata fits the successor reservation"); - assert_eq!(pages, 64); - assert_eq!( - bytes, - RESERVATION - CODE_LEXICAL_ARTIFACT_SQLITE_CACHE_BYTES_V1 - SCRATCH - ); - assert_eq!( - CODE_LEXICAL_ARTIFACT_SQLITE_CACHE_BYTES_V1 + SCRATCH + bytes, - RESERVATION - ); - assert!( - bytes < 64 * 1024 * 1024, - "a 64 MiB page batch would consume the entire remainder after the SQLite cache" - ); - - let metadata = 8 * 1024 * 1024; - let (_, with_metadata) = - super::super::clone_successor_source_batch_limits_from_charges(metadata) - .expect("modest metadata still leaves a page batch"); - assert_eq!(with_metadata, bytes - metadata); - - let too_large = RESERVATION - .saturating_sub(CODE_LEXICAL_ARTIFACT_SQLITE_CACHE_BYTES_V1) - .saturating_sub(SCRATCH); - assert!( - super::super::clone_successor_source_batch_limits_from_charges(too_large).is_err(), - "metadata that fills the remainder after cache and scratch must fail closed" - ); -} - /// Foreground query admission never performs the O(store) text projection. /// Only the scheduler-owned builder may advance it, one bounded document /// window at a time, until the immutable owners become visible atomically. @@ -676,14 +648,16 @@ fn text_artifact_publication_serializes_pointer_attachment_with_retention() { impl CodeIndexExecutionControlV1 for PauseAfterExistingArtifactRead { fn is_cancelled(&self) -> bool { - // One short staging file and one short existing artifact each - // checkpoint before open and after their single bounded read. The - // fourth checkpoint is therefore after the destination's bytes - // were verified but before publication can attach its descriptor. + // Publication checkpoints once before the store lock and once + // while taking the project's shared segment lock; then one short + // staging file and one short existing artifact each checkpoint + // before open and after their single bounded read. The sixth + // checkpoint is therefore after the destination's bytes were + // verified but before publication can attach its descriptor. if self .checkpoints .fetch_add(1, std::sync::atomic::Ordering::AcqRel) - == 3 + == 5 { let mut state = self.state.lock().expect("publication pause state"); state.0 = true; @@ -716,10 +690,24 @@ fn text_artifact_publication_serializes_pointer_attachment_with_retention() { .expect("sealed generation identity"); let sealed_hex = sha256_hex_suffix(sealed_identity.digest.as_str()).expect("sealed SHA-256 digest"); - let artifacts_root = store.path().join("code-text-artifacts-v1"); - tracedecay_private_fs::create_private_directory(&artifacts_root) - .expect("create private artifacts root"); - let staging = artifacts_root.join(format!(".text-artifact-{sealed_hex}.staging")); + // Staging stays with the scope that builds it; the completed artifact is + // the project's, beside its shared generation segments. A store root that + // is not a scope hash is a project of its own, so both live under it. + let ensure_private_root = + |path: &Path| match tracedecay_private_fs::create_private_directory(path) { + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {} + result => result.expect("create private root"), + }; + let staging_root = code_text_artifact_staging_root(store.path()); + assert_eq!( + staging_root, + store.path().join("code-text-artifact-staging-v1") + ); + ensure_private_root(&staging_root); + let artifacts_root = code_text_artifacts_root(store.path()); + assert_eq!(artifacts_root, store.path().join("code-text-artifacts-v1")); + ensure_private_root(&artifacts_root); + let staging = staging_root.join(format!(".text-artifact-{sealed_hex}.staging")); let artifact_bytes = b"already content-addressed artifact"; let mut staging_file = tracedecay_private_fs::create_private_file(&staging).expect("create private staging file"); @@ -745,6 +733,7 @@ fn text_artifact_publication_serializes_pointer_attachment_with_retention() { &staging, &generation.manifest().generation_id, &sealed_identity, + &test_content_key(), publish_control.as_ref(), ) }); @@ -798,12 +787,15 @@ fn text_artifact_publication_serializes_pointer_attachment_with_retention() { ); } -#[cfg(unix)] +/// The artifact seals its clone index with its lexical rows, so the owners +/// that serve search serve clone lookups at once: no work is left behind +/// the first seal, and the clone status is ready (stale only when the source +/// moved on). #[test] -fn clone_successor_keeps_lexical_owners_ready_and_cas_replaces_v14() { +fn clone_index_is_ready_when_the_artifact_first_seals() { let fixture = GitFixture::new(&[( "src/lib.rs", - "pub fn alpha() { one(); two(); three(); four(); five(); six(); seven(); eight(); nine(); ten(); }\n", + "pub fn alpha() { one(); two(); three(); four(); five(); six(); seven(); eight(); nine(); ten(); }\npub fn beta() { one(); two(); three(); four(); five(); six(); seven(); eight(); nine(); ten(); }\n", )]); let store = TempDir::new().expect("store root"); let mut scheduler = scheduler( @@ -813,283 +805,66 @@ fn clone_successor_keeps_lexical_owners_ready_and_cas_replaces_v14() { ); published(scheduler.reconcile_now().expect("publish generation")); let latest = scheduler.latest_complete().expect("latest generation"); - + assert!(matches!( + latest.clone_index_status(false, None), + tracedecay_contracts::code_index_freshness::CodeCloneIndexStatusV1::Unavailable { .. } + )); while !latest.query_owners_are_ready() { - latest.advance_text_serving(1).expect("advance V14 build"); + latest.advance_text_serving(1).expect("advance text build"); } assert!( - latest.text_projection_needs_work(), - "clone successor must continue after lexical owners are seated" - ); - latest - .production_query_owners() - .expect("lexical owners serve during clone successor"); - let v14_path = active_text_artifact_path(store.path()); - let v14_revision: i64 = rusqlite::Connection::open(&v14_path) - .expect("open V14 artifact") - .query_row( - "SELECT format_revision FROM artifact_state WHERE singleton = 1", - [], - |row| row.get(0), - ) - .expect("read V14 revision"); - assert_eq!(v14_revision, 14); - - while latest.text_projection_needs_work() { - latest - .advance_text_serving(16) - .expect("advance clone successor"); - } - assert!(latest.query_owners_are_ready()); - let v16_path = active_text_artifact_path(store.path()); - assert_ne!(v16_path, v14_path); - let v16_revision: i64 = rusqlite::Connection::open(v16_path) - .expect("open V16 artifact") - .query_row( - "SELECT format_revision FROM artifact_state WHERE singleton = 1", - [], - |row| row.get(0), - ) - .expect("read V16 revision"); - assert_eq!(v16_revision, 16); -} - -/// Exact and lexical readiness is not the clone-successor copy. -/// -/// The publication advance that installs those owners used to call -/// `begin_clone_successor` before returning, and that call copies the whole -/// prior lexical artifact. The freshness receipt awaits that advance, so -/// status stayed non-current for the copy. The successor must still be -/// reported as backfill, and the next advance is what writes its staging file. -#[test] -fn lexical_readiness_leaves_the_clone_successor_uncopied() { - let fixture = GitFixture::new(&[( - "src/lib.rs", - "pub fn alpha() { one(); two(); three(); four(); five(); six(); seven(); eight(); nine(); ten(); }\n", - )]); - let store = TempDir::new().expect("store root"); - let mut scheduler = scheduler( - &fixture, - store.path().to_path_buf(), - Arc::new(SharedCodeIndexBytePoolV1::default()), + !latest.text_projection_needs_work(), + "the first seal must leave no clone work behind" ); - published(scheduler.reconcile_now().expect("publish generation")); - let latest = scheduler.latest_complete().expect("latest generation"); - while !latest.query_owners_are_ready() { - latest.advance_text_serving(1).expect("advance V14 build"); - } - let tracedecay_contracts::code_index_freshness::CodeCloneIndexStatusV1::Backfilling { - observation, - } = latest.clone_index_status(false, None) + let tracedecay_contracts::code_index_freshness::CodeCloneIndexStatusV1::Ready { observation } = + latest.clone_index_status(false, None) else { panic!( - "a generation without clone fingerprints must report backfill once lexical owners serve, got {:?}", + "clone data must be available when the artifact first seals, got {:?}", latest.clone_index_status(false, None) ); }; - assert_eq!(observation.coverage.completed_source_pages, 0); assert!( - observation.coverage.total_source_pages > 0, - "the pending successor must name the sealed page count it has not visited" + observation + .coverage + .source_bodies + .is_some_and(|bodies| bodies > 0) ); - // Status falls back to the published artifact's bytes when the successor - // has not created a staging file, so the bytes field cannot prove the - // copy stayed off this advance. The slot and the artifacts directory can. - assert!( - matches!( - &*latest.text_projection_build.lock_slot(), - super::super::CodeTextProjectionSlotV1::CloneSuccessorPending - ), - "owner readiness must leave the successor pending" - ); - let staging_names = |root: &std::path::Path| { - std::fs::read_dir(code_text_artifacts_root(root)) - .expect("artifacts root") - .map(|entry| entry.expect("artifact entry").file_name()) - .filter(|name| name.to_string_lossy().ends_with(".staging")) - .collect::>() - }; - assert!( - staging_names(store.path()).is_empty(), - "owner readiness copied the prior lexical artifact: {:?}", - staging_names(store.path()) - ); - - latest - .advance_text_serving(1) - .expect("the retained successor advance copies the prior artifact"); - assert!(latest.query_owners_are_ready()); - assert!( - !matches!( - &*latest.text_projection_build.lock_slot(), - super::super::CodeTextProjectionSlotV1::CloneSuccessorPending - ), - "the next advance must take the pending successor" + assert_eq!( + observation.coverage.near_fingerprint_bodies, observation.coverage.eligible_source_bodies, + "positional fingerprints cover every eligible body at the first seal" ); - - while latest.text_projection_needs_work() { - latest - .advance_text_serving(16) - .expect("finish clone successor"); - } + assert!(observation.resources.peak_scratch_memory_bytes.is_some()); let revision: i64 = rusqlite::Connection::open(active_text_artifact_path(store.path())) - .expect("open finished artifact") + .expect("open sealed artifact") .query_row( "SELECT format_revision FROM artifact_state WHERE singleton = 1", [], |row| row.get(0), ) - .expect("read finished revision"); - assert_eq!(revision, 16); -} - -/// Only one wake at a time may own a head-open claim. -/// -/// `open_published_text_artifact` used to park `CloneSuccessorPending` -/// before `begin_clone_successor` copied the whole prior artifact, and -/// `advance_artifact_text_serving` leaves its park loop on that state. A -/// concurrent wake took a second `HeadOpening` on top of the first open, -/// both drove the same staging database, and whichever open resolved second -/// found the slot already reset and refused with `clone-successor retry -/// requires an active head-open claim`. The clone lanes report that refusal -/// as a non-retryable `search_failed`. -#[test] -fn concurrent_wakes_never_overlap_the_clone_successor_head_open() { - let sources = (0..24) - .map(|index| { - ( - format!("src/module_{index}.rs"), - format!( - "pub fn alpha_{index}() {{ one(); two(); three(); four(); five(); six(); seven(); eight(); nine(); ten(); }}\npub fn beta_{index}() {{ one(); two(); three(); four(); five(); six(); seven(); eight(); nine(); ten(); }}\n" - ), - ) - }) - .collect::>(); - let files = sources - .iter() - .map(|(path, contents)| (path.as_str(), contents.as_str())) + .expect("read sealed revision"); + assert_eq!(revision, 26); + let staging = std::fs::read_dir(code_text_artifact_staging_root(store.path())) + .expect("artifacts root") + .map(|entry| entry.expect("artifact entry").file_name()) + .filter(|name| name.to_string_lossy().ends_with(".staging")) .collect::>(); - let fixture = GitFixture::new(&files); - let store = TempDir::new().expect("store root"); - let mut scheduler = scheduler( - &fixture, - store.path().to_path_buf(), - Arc::new(SharedCodeIndexBytePoolV1::default()), - ); - published(scheduler.reconcile_now().expect("publish generation")); - let latest = scheduler.latest_complete().expect("latest generation"); - while !latest.query_owners_are_ready() { - latest - .advance_text_serving(1) - .expect("advance lexical build"); - } assert!( - matches!( - &*latest.text_projection_build.lock_slot(), - super::super::CodeTextProjectionSlotV1::CloneSuccessorPending - ), - "the successor must still be owed when the wakes start" + staging.is_empty(), + "no second build follows the seal: {staging:?}" ); - - let workers = (0..4) - .map(|_| { - let latest = latest.clone(); - thread::spawn(move || { - let mut advances = 0_usize; - while latest.text_projection_needs_work() && advances < 400 { - latest.advance_text_serving(1)?; - advances += 1; - } - Ok(()) - }) - }) - .collect::>(); - for worker in workers { - worker - .join() - .expect("wake thread joins") - .unwrap_or_else(|error: RetrievalPortError| { - panic!("a concurrent wake failed the text projection: {error}") - }); - } - - assert!(!latest.text_projection_needs_work()); - let revision: i64 = rusqlite::Connection::open(active_text_artifact_path(store.path())) - .expect("open finished artifact") - .query_row( - "SELECT format_revision FROM artifact_state WHERE singleton = 1", - [], - |row| row.get(0), - ) - .expect("read finished revision"); - assert_eq!(revision, 16, "the clone successor must have sealed"); -} - -#[test] -fn clone_status_distinguishes_unavailable_backfill_partial_ready_and_stale() { - let fixture = GitFixture::new(&[( - "src/lib.rs", - "pub fn alpha() { one(); two(); three(); four(); five(); six(); seven(); eight(); nine(); ten(); }\n", - )]); - let store = TempDir::new().expect("store root"); - let mut scheduler = scheduler( - &fixture, - store.path().to_path_buf(), - Arc::new(SharedCodeIndexBytePoolV1::default()), - ); - published(scheduler.reconcile_now().expect("publish generation")); - let latest = scheduler.latest_complete().expect("latest generation"); - assert!(matches!( - latest.clone_index_status(false, None), - tracedecay_contracts::code_index_freshness::CodeCloneIndexStatusV1::Unavailable { .. } - )); - while !latest.query_owners_are_ready() { - latest.advance_text_serving(1).expect("advance V14 build"); - } - assert!(matches!( - latest.clone_index_status(false, None), - tracedecay_contracts::code_index_freshness::CodeCloneIndexStatusV1::Backfilling { .. } - )); - let successor = { - let mut slot = latest.text.text_projection_build.lock_slot(); - std::mem::replace(&mut *slot, super::super::CodeTextProjectionSlotV1::Idle) - }; - - let tracedecay_contracts::code_index_freshness::CodeCloneIndexStatusV1::Partial { - observation, - omission_reasons, - } = latest.clone_index_status(false, None) - else { - panic!("missing successor must report partial clone coverage"); - }; - assert_eq!(observation.coverage.source_bodies, None); - assert!( - omission_reasons - .iter() - .any(|reason| reason.contains("clone rows are missing")) - ); - *latest.text.text_projection_build.lock_slot() = successor; - while latest.text_projection_needs_work() { - latest - .advance_text_serving(16) - .expect("finish clone successor"); - } - assert!(matches!( - latest.clone_index_status(false, None), - tracedecay_contracts::code_index_freshness::CodeCloneIndexStatusV1::Ready { .. } - )); assert!(matches!( latest.clone_index_status(true, None), tracedecay_contracts::code_index_freshness::CodeCloneIndexStatusV1::Stale { .. } )); } -// Holding the clone-successor slot across the await is the scenario, not an -// oversight: the read under test must answer without joining the backfill that +// Holding the text projection slot across the await is the scenario, not an +// oversight: the read under test must answer without joining a slice that // owns the slot. The guard is released before shutdown. #[allow(clippy::await_holding_lock)] #[tokio::test] -async fn dashboard_freshness_does_not_join_a_clone_backfill_slice() { +async fn dashboard_freshness_does_not_join_a_text_projection_slice() { let fixture = GitFixture::new(&[( "src/lib.rs", "pub fn alpha() { one(); two(); three(); four(); five(); six(); seven(); eight(); nine(); ten(); }\n", @@ -1108,621 +883,132 @@ async fn dashboard_freshness_does_not_join_a_clone_backfill_slice() { while !latest.query_owners_are_ready() { latest.advance_text_serving(1).expect("advance text build"); } + // The first status read computes the reader's clone census; warm it so + // the timed read below measures only whether it joins the held slot. + latest.clone_index_status(false, None); let held_slot = latest.text_projection_build.lock_slot(); - let freshness = tokio::time::timeout( - Duration::from_millis(100), - registry.dashboard_freshness(fixture.path()), - ) - .await - .expect("dashboard freshness must not wait for the clone backfill slice") - .expect("mounted dashboard freshness"); - assert!(matches!( - freshness.clone_index, - Some( - tracedecay_contracts::code_index_freshness::CodeCloneIndexStatusV1::Unavailable { - reason - } - ) if reason == "clone-index status is being updated" - )); - - drop(held_slot); - registry.shutdown().await; -} - -#[tokio::test] -async fn a_proven_seat_serves_v14_without_asking_for_the_pending_clone_successor() { - let fixture = GitFixture::new(&[( - "src/lib.rs", - "pub fn alpha() { one(); two(); three(); four(); five(); six(); seven(); eight(); nine(); ten(); }\n", - )]); - let retained_store = TempDir::new().expect("retained store root"); - let mut scheduler = scheduler( - &fixture, - retained_store.path().to_path_buf(), - Arc::new(SharedCodeIndexBytePoolV1::default()), - ); - published(scheduler.reconcile_now().expect("publish generation")); - let latest = scheduler.latest_complete().expect("latest generation"); - while !latest.query_owners_are_ready() { - latest.advance_text_serving(1).expect("advance V14 build"); - } - assert!( - latest.text_projection_needs_work(), - "the query must enter admission while clone successor work remains" - ); - - let registry_store = TempDir::new().expect("registry store root"); - let (registry, scope) = - mounted_core_query_worktree_with_one_permit(&fixture, ®istry_store).await; - // The registry's own owner owes a clone successor too, and the worker - // keeps a continuation queued for it. Settle that first so the slot this - // test reads belongs to the query alone. - drain_clone_backfill(®istry, fixture.path()).await; - let admission = quiesced_background_reconcile_admission(®istry, fixture.path()).await; - { - let mounted = registry.mounted.lock().await; - let worktree = mounted - .get(&canonical_existing_identity(fixture.path()).expect("canonical root")) - .expect("mounted worktree"); - // Generation identity binds the capture instant (`captured_at` is in - // the intake digest), so the crafted owner and the registry's own - // capture of the same checkout never share an id. Seat the crafted - // owner too: a text owner that is not the seated generation is a state - // the daemon never produces, and the worker's clone-backfill gate - // (`serving_matches_text`) refuses to drive it. - *worktree - .serving_generation - .write() - .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(latest.clone()); - *worktree - .text_generation - .write() - .unwrap_or_else(std::sync::PoisonError::into_inner) = - Some(latest.text_generation_handle()); - let serving_generation = worktree - .serving_generation - .read() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .as_ref() - .expect("the injected text owner has a serving seat") - .generation() - .manifest() - .generation_id - .clone(); - assert_eq!( - serving_generation, - latest.metadata().manifest().generation_id, - "the test must exercise pending backfill on the seated generation" - ); - } - - // Bind the seat's source proof, the state a settled mount reaches. The - // read below then has nothing left to verify. - let source_freshness = registry - .source_freshness_for_root(fixture.path()) - .await - .expect("mounted source fence"); - let witness = source_freshness - .source_currency_witness_for( - &latest.metadata().manifest().generation_id, - &latest.metadata().snapshot().content_identity, - ) - .expect("the seated owner's snapshot is the one the fence proved"); - *registry - .serving_source_witness_for_root(fixture.path()) - .await - .expect("mounted serving witness") - .write() - .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(witness); - // Not cleared: the slot is the only place an outstanding worker tail is - // visible, and wiping it here would hide the very state this test reads. - assert_eq!( - registry.pending_wake_micros_for_scope(&scope).await, - Some(0), - "the fixture must reach the search with nothing outstanding" - ); - - let executed = registry - .execute_query_search(&scope, core_search_request("alpha")) - .await - .expect("V14 exact and lexical owners remain admissible"); - assert!( - ranks_symbol(&ranked_symbol_names(&executed, &latest), "alpha"), - "the query must return the V14 alpha symbol" - ); - assert_eq!( - registry.pending_wake_micros_for_scope(&scope).await, - Some(0), - "a search whose owners are ready under a current proof must ask the \ - worker for nothing: the pending-wake slot is what the freshness \ - ladder reports as `verifying`" - ); - - drop(admission); - registry.shutdown().await; -} - -#[tokio::test] -async fn expired_source_proof_reschedules_pending_clone_backfill() { - let fixture = GitFixture::new(&[( - "src/lib.rs", - "pub fn alpha() { one(); two(); three(); four(); five(); six(); seven(); eight(); nine(); ten(); }\n", - )]); - let retained_store = TempDir::new().expect("retained store root"); - let mut scheduler = scheduler( - &fixture, - retained_store.path().to_path_buf(), - Arc::new(SharedCodeIndexBytePoolV1::default()), - ); - published(scheduler.reconcile_now().expect("publish generation")); - let latest = scheduler.latest_complete().expect("latest generation"); - while !latest.query_owners_are_ready() { - latest.advance_text_serving(1).expect("advance V14 build"); - } - assert!( - latest.text_projection_needs_work(), - "the query must enter admission while clone successor work remains" - ); - - let registry_store = TempDir::new().expect("registry store root"); - let (registry, scope) = - mounted_core_query_worktree_with_one_permit(&fixture, ®istry_store).await; - let admission = quiesced_background_reconcile_admission(®istry, fixture.path()).await; - registry.clear_pending_wake_for_scope(&scope).await; - { - let mounted = registry.mounted.lock().await; - let worktree = mounted - .get(&canonical_existing_identity(fixture.path()).expect("canonical root")) - .expect("mounted worktree"); - // Seat the crafted owner alongside its text handle: the worker's - // clone-backfill gate only drives a text owner that is the seated - // generation, and a daemon never holds one that is not. - *worktree - .serving_generation - .write() - .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(latest.clone()); - *worktree - .text_generation - .write() - .unwrap_or_else(std::sync::PoisonError::into_inner) = - Some(latest.text_generation_handle()); - } - - let source_freshness = registry - .source_freshness_for_root(fixture.path()) - .await - .expect("mounted source fence"); - { - let mut state = source_freshness.state.lock().unwrap(); - state.last_reconciled_at = Instant::now() - .checked_sub(state.staleness_threshold + Duration::from_secs(1)) - .expect("expire source proof"); - } - assert!(matches!( - registry.request_query_background_reconcile(&scope).await, - CodeIndexReconcileAdmissionV1::Accepted - )); - drop(admission); - // No second query or external wake: refreshing the proof must hand the - // pending clone work to a successor pass by itself. - let settled = tokio::time::timeout(Duration::from_secs(10), async { - loop { - if registry - .latest_text_serving_for_root(fixture.path()) - .await - .is_some_and(|text| !text.text_projection_needs_work()) - { - break; - } - tokio::time::sleep(Duration::from_millis(5)).await; - } - }) - .await; - assert!( - settled.is_ok(), - "the source refresh stranded clone backfill: pending_wake={:?} reconcile_in_progress={} source_age={:?} receipts={:?}", - registry.pending_wake_micros_for_scope(&scope).await, - registry - .reconcile_in_progress_for_test(fixture.path()) - .await, - source_freshness - .state - .lock() - .unwrap() - .last_reconciled_at - .elapsed(), - registry.event_to_ready_receipts(), - ); - registry.shutdown().await; -} - -#[test] -fn transient_clone_successor_reservation_refusal_retries_without_cooling_v14_owners() { - let fixture = GitFixture::new(&[( - "src/lib.rs", - "pub fn alpha() { one(); two(); three(); four(); five(); six(); seven(); eight(); nine(); ten(); }\n", - )]); - let store = TempDir::new().expect("store root"); - { - let mut scheduler = scheduler( - &fixture, - store.path().to_path_buf(), - Arc::new(SharedCodeIndexBytePoolV1::default()), - ); - published(scheduler.reconcile_now().expect("publish generation")); - let latest = scheduler.latest_complete().expect("latest generation"); - while !latest.query_owners_are_ready() { - latest.advance_text_serving(1).expect("publish V14 head"); - } - } - - let limit = NonZeroU64::new(4 * 1024 * 1024 * 1024).expect("resident-memory limit"); - let pressure = Arc::new(ResidentMemoryPressureV1::new(limit)); - let admission_headroom = limit.get().saturating_sub(pressure.high_watermark_bytes()); - let held_bytes = limit - .get() - .saturating_sub(admission_headroom) - .saturating_sub( - u64::try_from(CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1) - .expect("reader budget fits u64"), - ) - .saturating_sub(64 * 1024 * 1024); - let resident_memory = Arc::new(ProcessResidentMemoryV1::with_pressure(limit, pressure)); - let held = resident_memory - .reserve_process_shared( - ResidentMemoryComponentIdV1::new("test.clone-successor-transient") - .expect("test component"), - NonZeroU64::new(held_bytes).expect("temporary reservation"), - ) - .expect("hold transient competing memory"); - let mut scheduler = scheduler( - &fixture, - store.path().to_path_buf(), - Arc::new(SharedCodeIndexBytePoolV1::default()), - ); - scheduler.bind_resident_memory(Arc::clone(&resident_memory)); - let latest = scheduler.latest_complete().expect("restored generation"); - assert!( - matches!( - latest.advance_text_serving(1), - Err(tracedecay_query::retrieval::RetrievalPortError::AuthorityUnavailable(_)) - ), - "the competing reservation must deny the first successor admission" - ); - latest - .production_query_owners() - .expect("V14 owners remain queryable after successor refusal"); - assert!( - latest.text_projection_needs_work(), - "the refused successor must remain pending for a later scheduler wake" - ); - - drop(held); - while latest.text_projection_needs_work() { - latest - .advance_text_serving(16) - .expect("retry clone successor after transient memory clears"); - } - latest - .production_query_owners() - .expect("successor retry leaves query owners ready"); - let revision: i64 = rusqlite::Connection::open(active_text_artifact_path(store.path())) - .expect("open successor artifact") - .query_row( - "SELECT format_revision FROM artifact_state WHERE singleton = 1", - [], - |row| row.get(0), - ) - .expect("read successor revision"); - assert_eq!(revision, 16); -} - -fn start_partial_clone_successor( - fixture: &GitFixture, - store: &TempDir, -) -> (PathBuf, PathBuf, u64, u64) { - let mut scheduler = scheduler( - fixture, - store.path().to_path_buf(), - Arc::new(SharedCodeIndexBytePoolV1::default()), - ); - published(scheduler.reconcile_now().expect("publish generation")); - let latest = scheduler.latest_complete().expect("latest generation"); - while !latest.query_owners_are_ready() { - latest.advance_text_serving(1).expect("advance V14 build"); - } - let v14_path = active_text_artifact_path(store.path()); - assert!(latest.text_projection_needs_work()); - assert!( - !latest - .advance_text_serving(1) - .expect("append one clone-successor page"), - "one page must leave a resumable clone successor" - ); - latest - .production_query_owners() - .expect("V14 owners remain readable"); - let staging_path = std::fs::read_dir(store.path().join("code-text-artifacts-v1")) - .expect("read text artifact root") - .map(|entry| entry.expect("read text artifact entry").path()) - .find(|path| { - path.file_name() - .and_then(|name| name.to_str()) - .is_some_and(|name| name.ends_with(".staging")) - }) - .expect("partial clone-successor staging"); - let connection = - rusqlite::Connection::open(&staging_path).expect("open clone-successor staging"); - let next_page: i64 = connection - .query_row( - "SELECT next_page_ordinal FROM clone_successor_state WHERE singleton = 1", - [], - |row| row.get(0), - ) - .expect("read clone-successor cursor"); - let source_pages: i64 = connection - .query_row("SELECT COUNT(*) FROM source_pages", [], |row| row.get(0)) - .expect("count clone-successor source pages"); - assert!(next_page > 0 && next_page <= source_pages); - ( - v14_path, - staging_path, - u64::try_from(next_page).expect("nonnegative clone-successor cursor"), - u64::try_from(source_pages).expect("nonnegative source page count"), - ) -} - -#[test] -fn clone_successor_restart_revalidates_its_source_cursor_and_keeps_v14_readable() { - let fixture = GitFixture::new(&[( - "src/lib.rs", - "pub fn alpha() { one(); two(); three(); four(); five(); six(); seven(); eight(); nine(); ten(); }\n", - )]); - let store = TempDir::new().expect("store root"); - let (v14_path, staging_path, next_page, source_pages) = - start_partial_clone_successor(&fixture, &store); - - let scheduler = scheduler( - &fixture, - store.path().to_path_buf(), - Arc::new(SharedCodeIndexBytePoolV1::default()), - ); - let latest = scheduler.latest_complete().expect("restored generation"); - let completed = latest - .advance_text_serving(1) - .expect("resume clone successor after restart"); - latest - .production_query_owners() - .expect("V14 owners serve during resumed successor"); - assert!( - !completed, - "the first resumed slice must authenticate persisted rows before publishing" - ); - let resumed_page: i64 = rusqlite::Connection::open(&staging_path) - .expect("open resumed clone successor") - .query_row( - "SELECT next_page_ordinal FROM clone_successor_state WHERE singleton = 1", - [], - |row| row.get(0), - ) - .expect("read resumed clone-successor cursor"); - assert_eq!( - u64::try_from(resumed_page).expect("nonnegative resumed cursor"), - next_page, - "source replay must not append past the unauthenticated durable cursor" - ); - assert!(next_page <= source_pages); - while latest.text_projection_needs_work() { - latest - .advance_text_serving(16) - .expect("finish resumed clone successor"); - } - assert_ne!(active_text_artifact_path(store.path()), v14_path); -} - -#[test] -fn corrupt_clone_successor_staging_is_rebuilt_without_cooling_v14_owners() { - let fixture = GitFixture::new(&[( - "src/lib.rs", - "pub fn alpha() { one(); two(); three(); four(); five(); six(); seven(); eight(); nine(); ten(); }\n", - )]); - let store = TempDir::new().expect("store root"); - let (v14_path, staging_path, _, _) = start_partial_clone_successor(&fixture, &store); - rusqlite::Connection::open(&staging_path) - .expect("open clone-successor staging") - .execute( - "UPDATE clone_successor_state SET next_cursor = ?1 WHERE singleton = 1", - [vec![0xff_u8]], - ) - .expect("corrupt clone-successor cursor"); - - let scheduler = scheduler( - &fixture, - store.path().to_path_buf(), - Arc::new(SharedCodeIndexBytePoolV1::default()), - ); - let latest = scheduler.latest_complete().expect("restored generation"); - latest - .advance_text_serving(1) - .expect("discard corrupt clone successor and rebuild"); - latest - .production_query_owners() - .expect("V14 owners serve while successor rebuilds"); - while latest.text_projection_needs_work() { - latest - .advance_text_serving(16) - .expect("finish rebuilt clone successor"); - } - assert_ne!(active_text_artifact_path(store.path()), v14_path); + let freshness = tokio::time::timeout( + Duration::from_millis(100), + registry.dashboard_freshness(fixture.path()), + ) + .await + .expect("dashboard freshness must not wait for the text projection slice") + .expect("mounted dashboard freshness"); + assert!(matches!( + freshness.clone_index, + Some(tracedecay_contracts::code_index_freshness::CodeCloneIndexStatusV1::Ready { .. }) + )); + + drop(held_slot); + registry.shutdown().await; } -#[test] -fn tampered_resumed_clone_rows_are_rebuilt_from_the_sealed_source() { +#[tokio::test] +async fn a_proven_seat_serves_admission_without_asking_the_worker() { let fixture = GitFixture::new(&[( "src/lib.rs", "pub fn alpha() { one(); two(); three(); four(); five(); six(); seven(); eight(); nine(); ten(); }\n", )]); - let store = TempDir::new().expect("store root"); - let (_, staging_path, _, _) = start_partial_clone_successor(&fixture, &store); - let connection = - rusqlite::Connection::open(&staging_path).expect("open clone-successor staging"); - let (occurrence_id, original_occurrence): (String, Vec) = connection - .query_row( - "SELECT symbol_occurrence_id, occurrence FROM clone_occurrences ORDER BY symbol_occurrence_id LIMIT 1", - [], - |row| Ok((row.get(0)?, row.get(1)?)), - ) - .expect("read staged clone occurrence"); - let original_posting: (i64, i64, String, String, String) = connection - .query_row( - "SELECT class, normalization_revision, digest, symbol_occurrence_id, payload_digest FROM clone_exact_postings ORDER BY class, normalization_revision, digest, symbol_occurrence_id LIMIT 1", - [], - |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?, row.get(4)?)), - ) - .expect("read staged clone posting"); - let original_fingerprint: (String, i64, i64, i64, String, i64, String, String) = connection - .query_row( - "SELECT language, class, normalization_revision, fingerprint, symbol_occurrence_id, token_position, payload_digest, body_digest FROM clone_fingerprint_postings ORDER BY language, class, normalization_revision, fingerprint, symbol_occurrence_id, token_position LIMIT 1", - [], - |row| { - Ok(( - row.get(0)?, - row.get(1)?, - row.get(2)?, - row.get(3)?, - row.get(4)?, - row.get(5)?, - row.get(6)?, - row.get(7)?, - )) - }, - ) - .expect("read staged clone fingerprint"); - connection - .execute_batch( - "DROP TRIGGER immutable_clone_occurrences_update; - DROP TRIGGER immutable_clone_exact_postings_delete; - DROP TRIGGER builder_gate_clone_exact_postings_insert; - DROP TRIGGER immutable_clone_fingerprint_postings_delete; - DROP TRIGGER builder_gate_clone_fingerprint_postings_insert;", - ) - .expect("remove staging mutation guards for tamper injection"); - connection - .execute( - "UPDATE clone_occurrences SET occurrence = X'5B5D' WHERE symbol_occurrence_id = ?1", - [&occurrence_id], - ) - .expect("alter one persisted occurrence"); - connection - .execute( - "DELETE FROM clone_exact_postings WHERE class = ?1 AND normalization_revision = ?2 AND digest = ?3 AND symbol_occurrence_id = ?4", - rusqlite::params![ - original_posting.0, - original_posting.1, - original_posting.2, - original_posting.3 - ], - ) - .expect("delete one persisted posting"); - connection - .execute( - "INSERT INTO clone_exact_postings(class, normalization_revision, digest, symbol_occurrence_id, payload_digest) VALUES (?1, ?2, ?3, ?4, ?5)", - rusqlite::params![ - original_posting.0, - original_posting.1, - "sha256:0000000000000000000000000000000000000000000000000000000000000000", - original_posting.3, - original_posting.4 - ], - ) - .expect("replace the posting while preserving counts and references"); - connection - .execute( - "DELETE FROM clone_fingerprint_postings WHERE language = ?1 AND class = ?2 AND normalization_revision = ?3 AND fingerprint = ?4 AND symbol_occurrence_id = ?5 AND token_position = ?6", - rusqlite::params![ - original_fingerprint.0, - original_fingerprint.1, - original_fingerprint.2, - original_fingerprint.3, - original_fingerprint.4, - original_fingerprint.5, - ], - ) - .expect("delete one persisted fingerprint"); - connection - .execute( - "INSERT INTO clone_fingerprint_postings(language, class, normalization_revision, fingerprint, symbol_occurrence_id, token_position, payload_digest, body_digest) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)", - rusqlite::params![ - original_fingerprint.0, - original_fingerprint.1, - original_fingerprint.2, - original_fingerprint.3, - original_fingerprint.4, - original_fingerprint.5 + 1, - original_fingerprint.6, - original_fingerprint.7, - ], - ) - .expect("replace fingerprint at another position"); - drop(connection); - - let scheduler = scheduler( + let retained_store = TempDir::new().expect("retained store root"); + let mut scheduler = scheduler( &fixture, - store.path().to_path_buf(), + retained_store.path().to_path_buf(), Arc::new(SharedCodeIndexBytePoolV1::default()), ); - let latest = scheduler.latest_complete().expect("restored generation"); - while latest.text_projection_needs_work() { - latest - .advance_text_serving(16) - .expect("revalidate or rebuild resumed clone rows"); + published(scheduler.reconcile_now().expect("publish generation")); + let latest = scheduler.latest_complete().expect("latest generation"); + while !latest.query_owners_are_ready() { + latest.advance_text_serving(1).expect("advance text build"); } - let published = - rusqlite::Connection::open(active_text_artifact_path(store.path())).expect("open V16 head"); - let occurrence: Vec = published - .query_row( - "SELECT occurrence FROM clone_occurrences WHERE symbol_occurrence_id = ?1", - [&occurrence_id], - |row| row.get(0), + + let registry_store = TempDir::new().expect("registry store root"); + let (registry, scope) = + mounted_core_query_worktree_with_one_permit(&fixture, ®istry_store).await; + // Settle the registry's own worker first so the pending-wake slot this + // test reads belongs to the query alone. + settle_text_projection(®istry, fixture.path()).await; + let admission = quiesced_background_reconcile_admission(®istry, fixture.path()).await; + { + let mounted = registry.mounted.lock().await; + let worktree = mounted + .get(&canonical_existing_identity(fixture.path()).expect("canonical root")) + .expect("mounted worktree"); + // Generation identity binds the capture instant (`captured_at` is in + // the intake digest), so the crafted owner and the registry's own + // capture of the same checkout never share an id. Seat the crafted + // owner too: a text owner that is not the seated generation is a state + // the daemon never produces. + *worktree + .serving_generation + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(latest.clone()); + *worktree + .text_generation + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner) = + Some(latest.text_generation_handle()); + let serving_generation = worktree + .serving_generation + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .as_ref() + .expect("the injected text owner has a serving seat") + .generation() + .manifest() + .generation_id + .clone(); + assert_eq!( + serving_generation, + latest.metadata().manifest().generation_id, + "the test must search the seated generation" + ); + } + + // Bind the seat's source proof, the state a settled mount reaches. The + // read below then has nothing left to verify. + let source_freshness = registry + .source_freshness_for_root(fixture.path()) + .await + .expect("mounted source fence"); + let witness = source_freshness + .source_currency_witness_for( + &latest.metadata().manifest().generation_id, + &latest.metadata().snapshot().content_identity, ) - .expect("read rebuilt clone occurrence"); - assert_eq!(occurrence, original_occurrence); + .expect("the seated owner's snapshot is the one the fence proved"); + *registry + .serving_source_witness_for_root(fixture.path()) + .await + .expect("mounted serving witness") + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(witness); + // Not cleared: the slot is the only place an outstanding worker tail is + // visible, and wiping it here would hide the very state this test reads. assert_eq!( - published - .query_row( - "SELECT COUNT(*) FROM clone_exact_postings WHERE class = ?1 AND normalization_revision = ?2 AND digest = ?3 AND symbol_occurrence_id = ?4 AND payload_digest = ?5", - rusqlite::params![ - original_posting.0, - original_posting.1, - original_posting.2, - original_posting.3, - original_posting.4 - ], - |row| row.get::<_, i64>(0), - ) - .expect("read rebuilt clone posting"), - 1 + registry.pending_wake_micros_for_scope(&scope).await, + Some(0), + "the fixture must reach the search with nothing outstanding" + ); + + let executed = registry + .execute_query_search(&scope, core_search_request("alpha")) + .await + .expect("the seated owners are admissible"); + assert!( + ranks_symbol(&ranked_symbol_names(&executed, &latest), "alpha"), + "the query must return the alpha symbol" ); assert_eq!( - published - .query_row( - "SELECT COUNT(*) FROM clone_fingerprint_postings WHERE language = ?1 AND class = ?2 AND normalization_revision = ?3 AND fingerprint = ?4 AND symbol_occurrence_id = ?5 AND token_position = ?6 AND payload_digest = ?7 AND body_digest = ?8", - rusqlite::params![ - original_fingerprint.0, - original_fingerprint.1, - original_fingerprint.2, - original_fingerprint.3, - original_fingerprint.4, - original_fingerprint.5, - original_fingerprint.6, - original_fingerprint.7, - ], - |row| row.get::<_, i64>(0), - ) - .expect("read rebuilt clone fingerprint"), - 1 + registry.pending_wake_micros_for_scope(&scope).await, + Some(0), + "a search whose owners are ready under a current proof must ask the \ + worker for nothing: the pending-wake slot is what the freshness \ + ladder reports as `verifying`" ); + + drop(admission); + registry.shutdown().await; } #[cfg(unix)] @@ -1744,7 +1030,7 @@ fn text_artifact_builder_creates_an_owner_private_artifacts_root() { .advance_text_serving(1) .expect("start text-artifact build"); - let mode = std::fs::symlink_metadata(store.path().join("code-text-artifacts-v1")) + let mode = std::fs::symlink_metadata(code_text_artifact_staging_root(store.path())) .expect("artifacts-root metadata") .permissions() .mode() @@ -1796,6 +1082,7 @@ fn text_artifact_publish_rejects_a_permissive_artifacts_root() { &staging, &generation.manifest().generation_id, &sealed_identity, + &test_content_key(), &NeverCancelled, ), Err(tracedecay_query::retrieval::RetrievalPortError::Contract(_)) @@ -1842,6 +1129,7 @@ fn text_artifact_publish_refuses_a_busy_generation_store_and_retries() { &publish_staging, &publish_generation, &publish_identity, + &test_content_key(), &UninterruptibleCodeIndexControlV1, )) .expect("return publication outcome"); @@ -1860,6 +1148,7 @@ fn text_artifact_publish_refuses_a_busy_generation_store_and_retries() { &staging, &generation_id, &sealed_identity, + &test_content_key(), &UninterruptibleCodeIndexControlV1, ) .expect("publication retries after the store owner releases"); @@ -1908,6 +1197,7 @@ fn text_artifact_publish_rejects_a_symlink_artifacts_root() { &staging, &generation.manifest().generation_id, &sealed_identity, + &test_content_key(), &NeverCancelled, ), Err(tracedecay_query::retrieval::RetrievalPortError::Contract(_)) @@ -1961,7 +1251,7 @@ fn missing_durable_text_artifact_is_withdrawn_and_rebuilt() { } #[test] -fn cold_owner_warmup_seats_query_owners_before_clone_backfill() { +fn cold_owner_warmup_seats_every_query_owner_in_one_build() { let fixture = GitFixture::new(&[ ("src/lib.rs", "pub fn cold_activation() {}\n"), ("src/second.rs", "pub fn second_unit() -> usize { 2 }\n"), @@ -1988,22 +1278,15 @@ fn cold_owner_warmup_seats_query_owners_before_clone_backfill() { "owner warmup must leave the text serving owners installed" ); assert!( - latest.text_projection_needs_work(), - "lexical warmup must leave clone backfill as background work" + !latest.text_projection_needs_work(), + "owner warmup leaves no clone work behind" ); - while latest.text_projection_needs_work() { - latest - .advance_text_serving(64) - .expect("finish clone successor"); - } } /// `query_owners_are_ready` is the sole exact/lexical bit for the published /// seat gate and the full graph-replay skip, both directions. /// -/// Owners-ready with clone backfill still unfinished must admit seat/replay; -/// lexical-incomplete (owners absent) must refuse both. Clone completeness is -/// `text_projection_needs_work`, not a fork of this predicate. +/// Owners-ready must admit seat/replay; owners absent must refuse both. #[test] fn query_owners_ready_admits_seat_and_replay_both_directions() { use super::super::registry::GraphSeatGateV1; @@ -2041,23 +1324,12 @@ fn query_owners_ready_admits_seat_and_replay_both_directions() { .production_query_owners() .expect("cold owner warmup must install exact/lexical owners"); - // Direction: owners-ready while clone backfill remains → seat admits, replay admits. - assert!( - latest.query_owners_are_ready(), - "exact/lexical owners ready" - ); - assert!( - latest.text_projection_needs_work(), - "clone backfill must still be unfinished so the two predicates stay distinct" - ); + // Direction: owners-ready → seat admits, replay admits. + assert!(latest.query_owners_are_ready(), "query owners ready"); assert_eq!( GraphSeatGateV1::decide(true, false, true, true, latest.query_owners_are_ready()), GraphSeatGateV1::Prepare, - "published seat gate admits on owners-ready without waiting for clone backfill" - ); - assert!( - latest.query_owners_are_ready(), - "full graph replay skip clears on the same owners-ready bit; clone backfill is not a wait" + "published seat gate admits on owners-ready" ); } @@ -2184,7 +1456,7 @@ fn published_text_artifact_with_stale_search_revision_is_rebuilt() { let root = store.path().join("code-text-artifacts-v1"); tracedecay_private_fs::create_private_directory(&root).expect("private artifacts root"); let staging = root.join(".previous-search.staging"); - let mut builder = CodeLexicalArtifactBuilderV1::create(&staging, metadata) + let mut builder = CodeLexicalArtifactBuilderV1::create(&staging, metadata.clone()) .expect("previous-revision artifact builder"); let source_receipt = loop { match source.next_page(&control).expect("verified page") { @@ -2209,13 +1481,20 @@ fn published_text_artifact_with_stale_search_revision_is_rebuilt() { CodeLexicalArtifactReaderV1::open_with_control( &staging, &verified, + &metadata, CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, &control, ) .expect("historical metadata remains readable"); latest .text_artifact_store - .publish(&staging, &generation_id, &sealed_identity, &control) + .publish( + &staging, + &generation_id, + &sealed_identity, + &test_content_key(), + &control, + ) .expect("publish previous-revision artifact"); active_text_artifact_path(store.path()) }; @@ -2336,7 +1615,7 @@ fn incompatible_partial_text_artifact_is_discarded_and_rebuilt() { "one page must leave resumable staging state" ); } - let artifacts_root = store.path().join("code-text-artifacts-v1"); + let artifacts_root = code_text_artifact_staging_root(store.path()); let staging_path = std::fs::read_dir(&artifacts_root) .expect("read artifacts root") .map(|entry| entry.expect("artifact entry").path()) @@ -2410,7 +1689,7 @@ fn invalid_partial_text_artifact_cursor_is_discarded_and_rebuilt() { latest.metadata().snapshot().clone(), ) }; - let artifacts_root = store.path().join("code-text-artifacts-v1"); + let artifacts_root = code_text_artifact_staging_root(store.path()); let staging_path = std::fs::read_dir(&artifacts_root) .expect("read artifacts root") .map(|entry| entry.expect("artifact entry").path()) @@ -2425,7 +1704,7 @@ fn invalid_partial_text_artifact_cursor_is_discarded_and_rebuilt() { rusqlite::Connection::open(&staging_path).expect("open partial staging database"); let cursor_bytes: Vec = connection .query_row( - "SELECT next_cursor FROM source_pages ORDER BY page_ordinal DESC LIMIT 1", + "SELECT next_cursor FROM source_page_cursors ORDER BY page_ordinal DESC LIMIT 1", [], |row| row.get(0), ) @@ -2484,13 +1763,13 @@ fn invalid_partial_text_artifact_cursor_is_discarded_and_rebuilt() { )); let cursor_bytes = serde_json::to_vec(&cursor).expect("encode invalid persisted cursor"); connection - .execute_batch("DROP TRIGGER immutable_source_pages_update") + .execute_batch("DROP TRIGGER immutable_source_page_cursors_update") .expect("open immutable page fixture for corruption"); assert_eq!( connection .execute( - "UPDATE source_pages SET next_cursor = ?1 WHERE page_ordinal = \ - (SELECT MAX(page_ordinal) FROM source_pages)", + "UPDATE source_page_cursors SET next_cursor = ?1 WHERE page_ordinal = \ + (SELECT MAX(page_ordinal) FROM source_page_cursors)", [cursor_bytes], ) .expect("rewrite persisted cursor"), @@ -2498,8 +1777,8 @@ fn invalid_partial_text_artifact_cursor_is_discarded_and_rebuilt() { ); connection .execute_batch( - "CREATE TRIGGER immutable_source_pages_update BEFORE UPDATE ON source_pages \ - BEGIN SELECT RAISE(ABORT, 'immutable lexical source pages'); END", + "CREATE TRIGGER immutable_source_page_cursors_update BEFORE UPDATE ON source_page_cursors \ + BEGIN SELECT RAISE(ABORT, 'immutable lexical source page cursors'); END", ) .expect("restore immutable page contract"); } @@ -5161,56 +4440,6 @@ async fn callable_application_operations_consume_exact_lexical_and_graph_owners( ); assert!(second_continuation.next_cursor.is_none()); - registry - .mount_query_authority( - fixture.path(), - graph_context.scope(), - query_authority_with_candidate_cap( - latest.generation.manifest().privacy_domain.clone(), - 2, - ), - ) - .await - .expect("mount candidate-capped query authority"); - let capped_dispatch_request = CodeRelationRequest { - node_id: continuation_request.node_id.clone(), - maximum_depth: 1, - resolve_trait_dispatch: true, - scope: scope.clone(), - meta: query_meta(), - }; - let capped_dispatch = registry - .callees( - RetrievalPortContext { - request: &graph_context, - operation: &graph_operation, - }, - &capped_dispatch_request, - ) - .await; - let RetrievalPortOutcome::Partial(capped_dispatch) = capped_dispatch else { - panic!("candidate-capped trait dispatch must report partial coverage"); - }; - let capped_page = capped_dispatch - .payload - .expect("candidate-capped trait dispatch page"); - assert_eq!(capped_page.items.len(), 1); - assert_eq!(capped_page.items[0].symbol.node_id, trait_method); - assert!(!capped_page.items[0].dispatch_via_trait); - assert!( - capped_dispatch - .omissions - .iter() - .any(|omission| omission.reason == OmissionReason::Budget) - ); - mount_query_authority( - ®istry, - fixture.path(), - &graph_context, - latest.generation.manifest().privacy_domain.clone(), - ) - .await; - let qualified_name = latest .generation .symbols() @@ -5468,8 +4697,7 @@ async fn callable_application_operations_consume_exact_lexical_and_graph_owners( ) .await .expect("cursor key runtime"); - let cursor_keys = cursor_runtime - .profile_database() + let cursor_keys = SessionTemporalAccess::new(cursor_runtime.profile_database()) .load_session_cursor_key_provider_result() .await .expect("cursor keys"); @@ -5622,8 +4850,11 @@ async fn callable_application_operations_consume_exact_lexical_and_graph_owners( registry.shutdown().await; } +/// A relation page reports the true relation count, not the fusion lane's +/// 32-candidate budget, and its cursor walks the whole set while each page +/// hydrates only its own slice. #[tokio::test] -async fn callers_page_reports_candidate_cap_and_hydrates_only_the_requested_slice() { +async fn callers_page_reports_the_true_relation_count_and_hydrates_only_the_requested_slice() { let sources = caller_star_sources(); let files = sources .iter() @@ -5684,16 +4915,19 @@ async fn callers_page_reports_candidate_cap_and_hydrates_only_the_requested_slic ) .await; let first_page = match first { - RetrievalPortOutcome::Partial(evidence) => { - assert_eq!(evidence.coverage.eligible, Some(33)); - assert_eq!(evidence.omissions.len(), 1); - assert_eq!(evidence.omissions[0].reason, OmissionReason::Budget); + RetrievalPortOutcome::Completed(evidence) => { + assert_eq!( + evidence.coverage.completeness, + CoverageCompleteness::Complete + ); + assert_eq!(evidence.coverage.eligible, Some(CALLER_STAR as u64)); + assert!(evidence.omissions.is_empty(), "{:?}", evidence.omissions); evidence.payload.expect("first callers page") } - other => panic!("expected capped callers page, got {other:?}"), + other => panic!("expected a complete callers page, got {other:?}"), }; assert_eq!(first_page.items.len(), CALLER_PAGE as usize); - assert_eq!(first_page.total, Some(32)); + assert_eq!(first_page.total, Some(CALLER_STAR as u64)); let page1_hydrations = registry.take_relation_symbol_hydrations(); assert_eq!( page1_hydrations, @@ -5719,9 +4953,10 @@ async fn callers_page_reports_candidate_cap_and_hydrates_only_the_requested_slic ) .await; let second_page = match second { - RetrievalPortOutcome::Partial(evidence) => evidence.payload.expect("second callers page"), - other => panic!("expected capped callers continuation, got {other:?}"), + RetrievalPortOutcome::Completed(evidence) => evidence.payload.expect("second callers page"), + other => panic!("expected a complete callers continuation, got {other:?}"), }; + assert_eq!(second_page.total, Some(CALLER_STAR as u64)); assert_eq!(second_page.items.len(), CALLER_PAGE as usize); assert!( second_page @@ -5737,16 +4972,18 @@ async fn callers_page_reports_candidate_cap_and_hydrates_only_the_requested_slic "page 2 must hydrate only the returned slice; observed {page2_hydrations}" ); - let mut collected = first_page.items.clone(); - collected.extend(second_page.items); - let mut cursor = second_page.next_cursor; - while let Some(next) = cursor { + // Every page re-enumerates the relation set before hydrating its slice, so + // the full walk uses wider pages than the slice assertions above. + const WALK_PAGE: u32 = 200; + let mut collected = Vec::new(); + let mut cursor = None; + loop { let page_request = CodeRelationRequest { node_id: hub.occurrence.as_str().to_owned(), maximum_depth: 1, resolve_trait_dispatch: false, scope: scope.clone(), - meta: callers_page_meta(CALLER_PAGE, Some(next)), + meta: callers_page_meta(WALK_PAGE, cursor.take()), }; let page = match registry .callers( @@ -5758,23 +4995,145 @@ async fn callers_page_reports_candidate_cap_and_hydrates_only_the_requested_slic ) .await { - RetrievalPortOutcome::Partial(evidence) => evidence.payload.expect("callers page"), - other => panic!("expected capped callers page, got {other:?}"), + RetrievalPortOutcome::Completed(evidence) => evidence.payload.expect("callers page"), + other => panic!("expected a complete callers page, got {other:?}"), }; + assert_eq!(page.total, Some(CALLER_STAR as u64)); + assert_eq!(page.items.len(), WALK_PAGE as usize); collected.extend(page.items); cursor = page.next_cursor; + if cursor.is_none() { + break; + } } let _ = registry.take_relation_symbol_hydrations(); assert_eq!( collected.len(), - 32, - "the declared candidate cap is enforced" + CALLER_STAR, + "the cursor walks every relation, not a lane budget's prefix" ); let identities = collected .iter() .map(|record| record.symbol.node_id.as_str()) .collect::>(); - assert_eq!(identities.len(), collected.len(), "capped rows stay unique"); + assert_eq!( + identities.len(), + collected.len(), + "each relation is served once" + ); + registry.shutdown().await; +} + +/// `code_callees` shares the compact-key page with `code_callers`: 104 callees +/// answer as a ten-row first page with `total: 104`, a minted cursor, and +/// complete coverage, and the cursors walk all 104 exactly once. The fusion +/// lane's 32-candidate budget used to answer this as `total: 32` without a +/// continuation. +#[tokio::test] +async fn callees_page_reports_the_true_relation_count_and_walks_every_relation() { + const RELATIONS: usize = 104; + let sources = callee_fanout_sources(RELATIONS); + let files = sources + .iter() + .map(|(path, source)| (path.as_str(), source.as_str())) + .collect::>(); + let fixture = GitFixture::new(&files); + let store = TempDir::new().expect("store root"); + let registry = CodeIndexSchedulerRegistryV1::new(1); + registry + .mount_worktree( + test_project_id(), + fixture.path(), + store.path().to_path_buf(), + ) + .await + .expect("mount daemon-owned scheduler"); + let latest = wait_for_live_complete_generation(®istry, fixture.path()).await; + install_verified_graph_store(&latest); + let generation = latest.generation.manifest().generation_id.clone(); + let repository = latest.generation.snapshot().repository.clone(); + let worktree = latest + .generation + .snapshot() + .worktree + .clone() + .expect("worktree identity"); + let scope = CodeQueryScope::new(generation.clone(), None).expect("query scope"); + let fanout = latest + .generation + .symbols() + .symbols + .iter() + .find(|record| record.qualified_name.ends_with("fanout")) + .expect("fanout symbol"); + let operation = callable_code_operation(CallableCodeOperationKind::Callees).expect("operation"); + let context = application_context(&operation, repository, worktree); + mount_query_authority( + ®istry, + fixture.path(), + &context, + latest.generation.manifest().privacy_domain.clone(), + ) + .await; + let page_request = |cursor: Option| CodeRelationRequest { + node_id: fanout.occurrence.as_str().to_owned(), + maximum_depth: 1, + resolve_trait_dispatch: false, + scope: scope.clone(), + meta: callers_page_meta(CALLER_PAGE, cursor), + }; + let mut cursor = None; + let mut walked = Vec::new(); + let mut pages = 0; + loop { + let outcome = registry + .callees( + RetrievalPortContext { + request: &context, + operation: &operation, + }, + &page_request(cursor.take()), + ) + .await; + let RetrievalPortOutcome::Completed(evidence) = outcome else { + panic!("expected a complete callees page, got {outcome:?}"); + }; + assert_eq!( + evidence.coverage.completeness, + CoverageCompleteness::Complete + ); + assert_eq!(evidence.coverage.eligible, Some(RELATIONS as u64)); + assert!(evidence.omissions.is_empty(), "{:?}", evidence.omissions); + let page = evidence.payload.expect("callees page"); + assert_eq!(page.total, Some(RELATIONS as u64)); + pages += 1; + if pages == 1 { + assert_eq!(page.items.len(), CALLER_PAGE as usize); + assert!(page.next_cursor.is_some(), "page 1 must mint a cursor"); + } + assert_eq!( + registry.take_relation_symbol_hydrations(), + page.items.len() as u64, + "each page hydrates only its own rows" + ); + walked.extend(page.items); + cursor = page.next_cursor; + if cursor.is_none() { + break; + } + } + assert_eq!(pages, RELATIONS.div_ceil(CALLER_PAGE as usize)); + assert_eq!(walked.len(), RELATIONS); + let identities = walked + .iter() + .map(|record| record.symbol.node_id.as_str()) + .collect::>(); + assert_eq!(identities.len(), RELATIONS, "each callee is served once"); + assert!( + walked + .iter() + .all(|record| record.edge_kind == "calls" && !record.dispatch_via_trait) + ); registry.shutdown().await; } @@ -5854,11 +5213,11 @@ async fn graph_cursor_holds_its_generation_until_the_cursor_expires() { ) .await; let (page, expires_at) = match first { - RetrievalPortOutcome::Partial(evidence) => { + RetrievalPortOutcome::Completed(evidence) => { let expires_at = evidence.page.expires_at.expect("minted cursor expiry"); (evidence.payload.expect("first callers page"), expires_at) } - other => panic!("expected capped callers page, got {other:?}"), + other => panic!("expected a complete callers page, got {other:?}"), }; assert!(page.next_cursor.is_some(), "page 1 must mint a cursor"); assert_eq!( diff --git a/crates/tracedecay-code-index-runtime/src/git_transactions/owner.rs b/crates/tracedecay-code-index-runtime/src/git_transactions/owner.rs index 9fab04d3db..8c34a362f7 100644 --- a/crates/tracedecay-code-index-runtime/src/git_transactions/owner.rs +++ b/crates/tracedecay-code-index-runtime/src/git_transactions/owner.rs @@ -4,10 +4,10 @@ use std::collections::{BTreeSet, HashMap}; use std::path::{Path, PathBuf}; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; -use std::time::{SystemTime, UNIX_EPOCH}; use tracedecay_contracts::{ - GitIndexApplyRequestV1, GitIndexOperationBindingV1, GitIndexTransactionPortError, ResolvedScope, + GitIndexApplyRequestV1, GitIndexOperationBindingV1, GitIndexTransactionPortError, + ResolvedScope, now_micros, }; use tracedecay_domain::configuration::{ ACCESS_RULES_SETTING_KEY, AuthorityRef, CapabilityResolutionContextV1, ConfigurationValueV1, @@ -27,8 +27,8 @@ use super::{ GitIndexTransactionStoreRegistry, RepositoryMutationQueue, SharedDaemonGitIndexTransactionStore, canonicalize_repository_root, }; -use crate::ports::ApplicationCatalogSnapshotErrorV1; use tracedecay_application::ProjectSourceAccessSnapshot; +use tracedecay_contracts::catalog_composition::CatalogCompositionError; use tracedecay_global_db::RegisteredGlobalDbLeaseV1; use tracedecay_global_db::configuration::OwnedGlobalDbConfigurationControlStore; use tracedecay_global_db::configuration::contracts::ConfigurationControlStore; @@ -38,7 +38,7 @@ const GIT_POLICY_REVISION: u64 = 2; type ProfiledStdRwLock = hotpath::rw_locks::RwLock; type ProfiledTokioMutex = hotpath::wrap::tokio::sync::Mutex; type ApplicationCatalogComposer = - Arc Result + Send + Sync>; + Arc Result + Send + Sync>; #[derive(Clone, Debug)] pub struct DaemonGitAuthorityStateV1 { @@ -83,7 +83,7 @@ impl DaemonGitAuthoritySource for ProductionDaemonGitAuthoritySource { &self, capability_id: &CapabilityId, ) -> Result { - let evaluated_at = current_micros(); + let evaluated_at = now_micros(); if evaluated_at >= self.access.grant_expires_at { return Err(GitIndexTransactionPortError::PolicyDenied); } @@ -367,17 +367,6 @@ fn scope_matches_snapshot( } } -fn current_micros() -> UtcMicros { - UtcMicros( - i64::try_from( - SystemTime::now() - .duration_since(UNIX_EPOCH) - .map_or(0, |duration| duration.as_micros()), - ) - .unwrap_or(i64::MAX), - ) -} - pub type DaemonProjectGitIndexTransactionService = DaemonGitIndexTransactionService< SharedDaemonGitIndexTransactionStore, FixedDaemonGitIndexExecutor, @@ -454,10 +443,7 @@ impl DaemonGitIndexTransactionServiceRegistry { /// mounts resolves capability manifests through it, so there is no window /// in which an owner exists without one. pub fn new( - catalog: impl Fn() -> Result - + Send - + Sync - + 'static, + catalog: impl Fn() -> Result + Send + Sync + 'static, ) -> Self { Self { catalog: Arc::new(catalog), diff --git a/crates/tracedecay-code-index-runtime/src/git_transactions/tests.rs b/crates/tracedecay-code-index-runtime/src/git_transactions/tests.rs index 5aa3d92b16..c443f84d3d 100644 --- a/crates/tracedecay-code-index-runtime/src/git_transactions/tests.rs +++ b/crates/tracedecay-code-index-runtime/src/git_transactions/tests.rs @@ -6,6 +6,7 @@ use std::time::Duration; use std::collections::BTreeSet; use std::sync::Mutex; +use tracedecay_contracts::catalog_composition::CatalogCompositionError; use tracedecay_contracts::{ CancellationStage, GitIndexApplyRequestV1, GitIndexTransactionPort, GitIndexTransactionPortError, OperationTermination, @@ -22,7 +23,7 @@ use tracedecay_store::{ GitIndexTransactionBeginResultV1, GitIndexTransactionStore, GitIndexTransactionStoreError, GitIndexTransactionStoreResult, GitIndexTransactionTerminalWriteV1, }; -use tracedecay_tool_catalog::CapabilityId; +use tracedecay_tool_catalog::{CapabilityId, CatalogSnapshotBuilderV1, CatalogSnapshotV1}; use super::owner::{DaemonGitAuthoritySource, DaemonGitIndexPolicyRecheck, preview_conflict_risk}; use super::queue::{RepositoryMutationQueue, RepositoryMutationQueueError}; @@ -42,13 +43,8 @@ use tracedecay_global_db::tests::harness::RegisteredGlobalDbHarness; /// Registry owners take their catalog composer by construction, so these /// fixtures compose a real (contribution-free) snapshot rather than relying on /// whatever some other test installed first. -fn test_catalog_snapshot() -> Result< - tracedecay_tool_catalog::CatalogSnapshotV1, - crate::ports::ApplicationCatalogSnapshotErrorV1, -> { - tracedecay_tool_catalog::CatalogSnapshotBuilderV1::new() - .build() - .map_err(|error| crate::ports::ApplicationCatalogSnapshotErrorV1::new(error.to_string())) +fn test_catalog_snapshot() -> Result { + Ok(CatalogSnapshotBuilderV1::new().build()?) } #[test] diff --git a/crates/tracedecay-code-index-runtime/src/lib.rs b/crates/tracedecay-code-index-runtime/src/lib.rs index a0be6ef891..160ee736cd 100644 --- a/crates/tracedecay-code-index-runtime/src/lib.rs +++ b/crates/tracedecay-code-index-runtime/src/lib.rs @@ -65,8 +65,8 @@ pub use code_graph_seat::{ pub use code_index_scheduler::CodeIndexSchedulerRegistryV1; pub use code_index_scheduler::identity::resolved_scope_for_project; pub use ports::{ - AdmissionParkLeaseV1, ApplicationCatalogSnapshotErrorV1, CONNECTION_ADMISSION, - GitWatchMaintenanceWakeV1, GitWatchSyncConfigV1, park_admission, + AdmissionParkLeaseV1, CONNECTION_ADMISSION, GitWatchMaintenanceWakeV1, GitWatchSyncConfigV1, + park_admission, }; /// Installs the registered global/session schema into the kernel's fail-closed diff --git a/crates/tracedecay-code-index-runtime/src/ports.rs b/crates/tracedecay-code-index-runtime/src/ports.rs index ee1edcc156..34b0e127bd 100644 --- a/crates/tracedecay-code-index-runtime/src/ports.rs +++ b/crates/tracedecay-code-index-runtime/src/ports.rs @@ -8,7 +8,7 @@ use tokio::time::{Duration, timeout}; /// Watcher knobs the git-metadata watcher needs from resolved sync config. /// -/// Root maps `tracedecay::config::SyncConfig` into this type at construction. +/// Root maps `tracedecay_configuration::SyncConfig` into this type at construction. /// The usecases `SyncConfig` is a different, smaller PR-autotrack type. #[derive(Clone, Debug, PartialEq, Eq)] pub struct GitWatchSyncConfigV1 { @@ -60,19 +60,6 @@ impl Default for GitWatchMaintenanceWakeV1 { } } -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct ApplicationCatalogSnapshotErrorV1 { - pub message: String, -} - -impl ApplicationCatalogSnapshotErrorV1 { - pub fn new(message: impl Into) -> Self { - Self { - message: message.into(), - } - } -} - /// Connection-admission lease the scheduler parks behind on blocking work. pub trait AdmissionParkLeaseV1: Send + Sync { fn release(&self) -> bool; diff --git a/crates/tracedecay-code-index-runtime/src/project_reads.rs b/crates/tracedecay-code-index-runtime/src/project_reads.rs index 5f779710a2..dc02bd74c9 100644 --- a/crates/tracedecay-code-index-runtime/src/project_reads.rs +++ b/crates/tracedecay-code-index-runtime/src/project_reads.rs @@ -168,7 +168,7 @@ impl ProjectCodeGraphServingAuthorityV1 { })?; Ok(ProjectCodeGraphServingProjectionV1 { generation_id: latest.metadata().manifest().generation_id.clone(), - statistics: latest.metadata().generation_statistics().cloned(), + statistics: Some(latest.metadata().generation_statistics().clone()), store, freshness, }) diff --git a/crates/tracedecay-code-index/Cargo.toml b/crates/tracedecay-code-index/Cargo.toml index 261f638d68..87fe347d17 100644 --- a/crates/tracedecay-code-index/Cargo.toml +++ b/crates/tracedecay-code-index/Cargo.toml @@ -71,6 +71,7 @@ lang-lean = ["tracedecay-code-extraction/lang-lean"] [dependencies] ast-grep-core = "0.44" +flate2 = "1" hex = "0.4" hotpath.workspace = true ignore = "0.4" @@ -89,6 +90,9 @@ tracedecay-runtime-core = { path = "../tracedecay-runtime-core", version = "0.1. tree-sitter = "0.26" [dev-dependencies] +# Self dev-dependency: the integration suite forces indexing width through +# the `test-helpers` hooks; production builds never select them. +tracedecay-code-index = { path = ".", default-features = false, features = ["test-helpers"] } criterion = "0.5" rusqlite = { version = "0.40.1", default-features = false, features = ["backup"] } tempfile = "3" @@ -122,6 +126,11 @@ harness = false test = false required-features = ["hotpath"] +[[bench]] +name = "sealed_storage" +harness = false +test = false + [[bench]] name = "restore_generation" harness = false diff --git a/crates/tracedecay-code-index/benches/code_index_chunks.rs b/crates/tracedecay-code-index/benches/code_index_chunks.rs index 037410732a..3f1617c104 100644 --- a/crates/tracedecay-code-index/benches/code_index_chunks.rs +++ b/crates/tracedecay-code-index/benches/code_index_chunks.rs @@ -9,10 +9,7 @@ use std::time::{Instant, SystemTime, UNIX_EPOCH}; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; -use tracedecay_code_index::chunks::{ - CodeChunker, CodeFileChunksV1, DeterministicCodeChunker, content_digest, -}; -use tracedecay_code_index::extract::ExtractionBatchV1; +use tracedecay_code_index::chunks::{CodeFileChunksV1, DeterministicCodeChunker, content_digest}; use tracedecay_code_index::extract::{LanguageExtractor, NeverCancelled, TreeSitterExtractor}; use tracedecay_code_index::incremental::{GenerationChunkManifestV1, plan_chunk_increment}; use tracedecay_code_index::intake::{CodeIndexIntake, ReceiptBoundCodeFileV1, SanitizedCodeIntake}; @@ -27,8 +24,8 @@ use tracedecay_domain::{ LanguageDescriptorV1, LanguageId, ManifestDigest, PolicyRevisionId, ProjectId, ProjectionBatchRequestV1, ProjectionKeyV1, ProjectionKindV1, ProjectionOperationV1, ProjectionOutcomeV1, ProjectionReplayReasonV1, RepositoryId, SanitizationReceiptId, - SanitizedCodeFileV1, SanitizedCodeSnapshotV1, SanitizerRevision, SnapshotFileDispositionV1, - UtcMicros, ValidatedCodeFileV1, + SanitizedCodeFileV1, SanitizedCodeSnapshotV1, SanitizerRevision, SensitivityLevelV1, + SnapshotFileDispositionV1, UtcMicros, ValidatedCodeFileV1, }; const WORKLOAD_PATH: &str = concat!( @@ -193,7 +190,6 @@ struct WorkloadFile { #[derive(Clone)] struct FileArtifact { source: WorkloadFile, - extraction: ExtractionBatchV1, chunks: CodeFileChunksV1, } @@ -695,11 +691,12 @@ fn execute_case( ProjectionReplayReasonV1::VerificationReplay, ), CaseName::ChunkerReplay => ( - rechunk( - prior.as_ref().expect("prior generation"), + build_fresh( + &sources, generation(2)?, chunker_revision(CHUNKER_REPLAY)?, &descriptor, + &extractor, )?, sources.len() as u64, corpus_bytes, @@ -823,14 +820,18 @@ fn build_artifact( let extraction = extractor .extract(&file, descriptor, &NeverCancelled) .map_err(|error| format!("extract {}: {error:?}", source.logical_path))?; - let chunks = chunker - .chunk_file(&file, extraction.batch(), descriptor, &NeverCancelled) + let (artifacts, _) = chunker + .index_file_with_authority_from_extraction( + &file, + &extraction, + descriptor, + SensitivityLevelV1::Public, + &NeverCancelled, + ) .map_err(|error| format!("chunk {}: {error}", source.logical_path))?; - let extraction = extraction.batch().clone(); Ok(FileArtifact { source: source.clone(), - extraction, - chunks, + chunks: artifacts.chunks, }) } @@ -876,8 +877,6 @@ fn rebind_artifact( generation_id: &CodeGenerationId, ) -> Result<(), String> { let occurrence = file_occurrence(&artifact.source.logical_path, generation_id)?; - artifact.extraction.generation_id = generation_id.clone(); - artifact.extraction.file_occurrence_id = occurrence.clone(); artifact.chunks.document.generation_id = generation_id.clone(); artifact.chunks.document.file_occurrence_id = occurrence.clone(); for chunk in &mut artifact.chunks.chunks { @@ -891,34 +890,6 @@ fn rebind_artifact( .map_err(|error| format!("validate carried {}: {error}", artifact.source.logical_path)) } -fn rechunk( - prior: &BuiltCorpus, - generation_id: CodeGenerationId, - chunker_revision: ChunkerRevision, - descriptor: &LanguageDescriptorV1, -) -> Result { - let chunker = chunker(generation_id.clone(), chunker_revision)?; - let mut artifacts = BTreeMap::new(); - for artifact in prior.artifacts.values() { - let mut extraction = artifact.extraction.clone(); - let file = receipt_bound_file(&artifact.source, &generation_id)?; - extraction.generation_id = generation_id.clone(); - extraction.file_occurrence_id = file.file.file_occurrence_id.clone(); - let chunks = chunker - .chunk_file(&file, &extraction, descriptor, &NeverCancelled) - .map_err(|error| format!("rechunk {}: {error}", artifact.source.logical_path))?; - artifacts.insert( - artifact.source.logical_path.clone(), - FileArtifact { - source: artifact.source.clone(), - extraction, - chunks, - }, - ); - } - BuiltCorpus::from_artifacts(generation_id, artifacts) -} - fn chunker( generation_id: CodeGenerationId, chunker_revision: ChunkerRevision, @@ -929,7 +900,6 @@ fn chunker( id::("sanitizer.query-benchmark.v1")?, id::("policy.query-benchmark.v1")?, chunker_revision, - tracedecay_code_extraction::LanguageRegistry::new(), )) } diff --git a/crates/tracedecay-code-index/benches/partitioned_codec.rs b/crates/tracedecay-code-index/benches/partitioned_codec.rs index f73af4f0b9..bbd76fa097 100644 --- a/crates/tracedecay-code-index/benches/partitioned_codec.rs +++ b/crates/tracedecay-code-index/benches/partitioned_codec.rs @@ -3,7 +3,6 @@ use std::{ error::Error, fmt::Debug, hint::black_box, - io::Cursor, path::PathBuf, sync::Arc, time::Instant, @@ -485,10 +484,7 @@ fn decode_and_open(fixture: &EncodedFixture) -> Result<(), CodeIndexProductionEr buffer.extend_from_slice(&bytes[start..end]); Ok(()) }, - )? - .ok_or_else(|| { - CodeIndexProductionErrorV1::Contract("benchmark manifest is incompatible".to_owned()) - })?; + )?; black_box(decoded); black_box(open_lexical(fixture)?); Ok(()) @@ -497,12 +493,11 @@ fn decode_and_open(fixture: &EncodedFixture) -> Result<(), CodeIndexProductionEr #[hotpath::measure(label = "code_index.lexical.open")] fn open_lexical( fixture: &EncodedFixture, -) -> Result>>, CodeIndexProductionErrorV1> { +) -> Result { let source_digest = ManifestDigest::from_sha256_bytes(&Sha256::digest(&fixture.manifest)) .map_err(|error| CodeIndexProductionErrorV1::Contract(error.to_string()))?; let segments = Arc::clone(&fixture.segments); - let source = VerifiedSealedLexicalPageSourceV1::open_partitioned_sealed( - Cursor::new(Vec::::new()), + VerifiedSealedLexicalPageSourceV1::open_partitioned_sealed( &fixture.manifest, source_digest, move |digest, _, buffer, _control| { @@ -515,11 +510,7 @@ fn open_lexical( }, 256, 1024 * 1024, - )? - .ok_or_else(|| { - CodeIndexProductionErrorV1::Contract("benchmark manifest is incompatible".to_owned()) - })?; - Ok(source) + ) } fn drain_lexical(fixture: &EncodedFixture) -> Result<(), CodeIndexProductionErrorV1> { diff --git a/crates/tracedecay-code-index/benches/restore_generation.rs b/crates/tracedecay-code-index/benches/restore_generation.rs index 101a341872..523ea9b5ad 100644 --- a/crates/tracedecay-code-index/benches/restore_generation.rs +++ b/crates/tracedecay-code-index/benches/restore_generation.rs @@ -187,8 +187,7 @@ fn restore_once( CodeIndexPublishedGenerationV1::decode_partitioned_sealed(manifest, |request, buffer| { read_segment(segments, request, buffer) }) - .map_err(|error| error.to_string())? - .ok_or_else(|| "generation manifest is not a partitioned manifest".to_owned())?; + .map_err(|error| error.to_string())?; let mut coverage = restored .analysis_coverage() .map(|(path, _)| path.to_owned()) diff --git a/crates/tracedecay-code-index/benches/sealed_storage.rs b/crates/tracedecay-code-index/benches/sealed_storage.rs new file mode 100644 index 0000000000..e6d1fc6126 --- /dev/null +++ b/crates/tracedecay-code-index/benches/sealed_storage.rs @@ -0,0 +1,778 @@ +//! Sealed-generation storage bytes per section, plus a decoded-content digest. +//! +//! Builds a clean generation over the blobs of one git revision, then a +//! successor after a deterministic edit of every seventeenth file, seals both +//! through the partitioned codec, and reports what a store would hold: file +//! segments (deduplicated across the two generations by content address, as +//! the segment directory is), each generation's evidence pack, and manifests. +//! File segment bytes are also split by payload section after undoing the +//! stored encoding, so a codec change shows which section it moved. +//! +//! `decoded_digest` hashes what readers observe after restore: every lexical +//! page's chunks and clone bodies, the lineage roster, symbols, and edges. +//! A storage change that preserves behaviour leaves it unchanged. +//! +//! ```text +//! cargo bench -p tracedecay-code-index --bench sealed_storage +//! SEALED_STORAGE_REPO= SEALED_STORAGE_REV= cargo bench ... +//! ``` +//! +//! The corpus is read from git objects, never from the working tree, so two +//! runs at one revision admit identical bytes regardless of local edits. + +use std::{ + collections::{BTreeMap, BTreeSet}, + error::Error, + fmt::Debug, + io::{BufRead, BufReader, Read, Write}, + path::PathBuf, + process::{Command, Stdio}, + sync::{Arc, Mutex}, + time::Instant, +}; + +use serde::Serialize; +use sha2::{Digest, Sha256}; +use tracedecay_code_index::{ + chunks::content_digest, + graph_projection::{ + CODE_GRAPH_PROJECTOR_REVISION, build_published_code_graph_manifest_checked, + code_graph_projection_identity, write_interactive_catalog_artifact, + }, + languages::{LanguageRegistry, StaticLanguageRegistry}, + lineage::LineageKindV1, + production::{ + CodeIndexAtomicPublicationPort, CodeIndexBuildRequestV1, CodeIndexCapturedFileV1, + CodeIndexExecutionControlV1, CodeIndexGenerationScopeV1, CodeIndexProductionConfigV1, + CodeIndexProductionErrorV1, CodeIndexProductionOwnerV1, CodeIndexPublicationStoreErrorV1, + CodeIndexPublishedGenerationV1, CodeIndexRepositoryParseIdentityV1, + SealedGenerationSegmentPublicationV1, SealedGenerationSegmentReadV1, + VerifiedSealedLexicalPageReadV1, VerifiedSealedLexicalPageSourceV1, + }, + projection::{ + ChunkProjectionDecisionV1, CodeChunkProjectionSink, ProjectionReceiptBuilderV1, + ProjectionSinkErrorV1, ProjectionSinkReceiptV1, + }, +}; +use tracedecay_domain::{ + ChunkerRevision, CodeGenerationId, FileOccurrenceId, LanguageId, ManifestDigest, + PolicyRevisionId, PrivacyDomainId, ProjectId, ProjectionBatchRequestV1, ProjectionKeyV1, + ProjectionKindV1, ProjectionOperationV1, ProjectionOutcomeV1, RepositoryDirtyStateV1, + RepositoryId, SanitizationReceiptId, SanitizedCodeFileV1, SanitizedCodeSnapshotV1, + SanitizerRevision, SensitivityLevelV1, SnapshotFileDispositionV1, TreeId, UtcMicros, + WorktreeId, +}; +use tracedecay_graph_db::{GraphNamespace, GraphProjectorRevision, NeverCancelled}; + +const REPO_ENV: &str = "SEALED_STORAGE_REPO"; +const REV_ENV: &str = "SEALED_STORAGE_REV"; +/// Matches `tracedecay_index_bench`: prime, so edits spread across languages. +const EDIT_STRIDE: usize = 17; +const EDIT_APPEND: &[u8] = b"\n// sealed-storage successor edit\n"; +/// The daemon's text-artifact page bounds; a real repository holds clone +/// bodies larger than a smaller page admits. +const LEXICAL_PAGE_CHUNKS: usize = 256; +const LEXICAL_PAGE_BYTES: usize = 4 * 1024 * 1024; + +struct SourceFile { + logical_path: String, + language: LanguageId, + bytes: Arc<[u8]>, +} + +#[derive(Default, Clone)] +struct MemoryPublication { + active: Arc>>>, +} + +impl CodeIndexAtomicPublicationPort for MemoryPublication { + fn load_active( + &self, + scope: &CodeIndexGenerationScopeV1, + ) -> Result>, CodeIndexPublicationStoreErrorV1> { + Ok(self + .active + .lock() + .map_err(|_| CodeIndexPublicationStoreErrorV1::CompareAndSwap)? + .get(scope) + .cloned()) + } + + fn publish_atomically( + &mut self, + scope: &CodeIndexGenerationScopeV1, + expected_active_generation: Option<&CodeGenerationId>, + generation: Arc, + ) -> Result<(), CodeIndexPublicationStoreErrorV1> { + let mut active = self + .active + .lock() + .map_err(|_| CodeIndexPublicationStoreErrorV1::CompareAndSwap)?; + if active + .get(scope) + .map(|current| current.manifest().generation_id.clone()) + .as_ref() + != expected_active_generation + { + return Err(CodeIndexPublicationStoreErrorV1::CompareAndSwap); + } + active.insert(scope.clone(), generation); + Ok(()) + } +} + +struct ApplyingProjection; + +impl CodeChunkProjectionSink for ApplyingProjection { + fn project_changed_chunks( + &mut self, + request: &ProjectionBatchRequestV1, + receipt_builder: ProjectionReceiptBuilderV1<'_>, + ) -> Result { + let changes = &request.changes; + let decisions = changes + .added_or_changed + .iter() + .map(|change| ChunkProjectionDecisionV1 { + chunk_id: change.chunk_id.clone(), + prior_chunk_digest: change.prior_digest.clone(), + current_chunk_digest: change.current_digest.clone(), + operation: if change.prior_digest.is_some() { + ProjectionOperationV1::Updated + } else { + ProjectionOperationV1::Added + }, + outcome: ProjectionOutcomeV1::Applied, + output_digest: change.current_digest.clone(), + }) + .chain( + changes + .deleted + .iter() + .map(|change| ChunkProjectionDecisionV1 { + chunk_id: change.chunk_id.clone(), + prior_chunk_digest: change.prior_digest.clone(), + current_chunk_digest: None, + operation: ProjectionOperationV1::Deleted, + outcome: ProjectionOutcomeV1::Applied, + output_digest: None, + }), + ) + .collect::>(); + receipt_builder + .build(&decisions) + .map_err(|error| ProjectionSinkErrorV1::Rejected(error.to_string())) + } +} + +struct ActiveControl; + +impl CodeIndexExecutionControlV1 for ActiveControl { + fn is_cancelled(&self) -> bool { + false + } + + fn is_deadline_exceeded(&self) -> bool { + false + } +} + +struct Sealed { + manifest: Vec, + file_segments: BTreeMap>, + evidence: (String, Vec), +} + +#[derive(Serialize)] +struct GenerationReport { + seal_ms: u128, + /// Restore through `decode_partitioned_sealed` alone. + restore_ms: u128, + manifest_bytes: usize, + file_segments: usize, + file_segments_written: usize, + file_segment_bytes_written: usize, + evidence_bytes: usize, + lineage_rows: usize, + lineage_unchanged_rows: usize, +} + +#[derive(Serialize)] +struct Report { + repository: String, + revision: String, + corpus_files: usize, + corpus_bytes: usize, + edited_files: usize, + clean: GenerationReport, + successor: GenerationReport, + /// What the segment directory holds for both generations. + store_file_segments: usize, + store_file_segment_bytes: usize, + store_evidence_bytes: usize, + store_manifest_bytes: usize, + store_total_bytes: usize, + /// Distinct file segments split by payload section, measured on the + /// canonical JSON the stored encoding decodes to. + decoded_section_bytes: BTreeMap, + decoded_digest: String, + linked_worktree: LinkedWorktreeReport, + read_bundle: ReadBundleReport, +} + +/// The interactive-catalog artifact a sealed read bundle stores for the +/// primary checkout and for a linked worktree of the same tree, and what the +/// two cost when identical catalogs are stored once. +#[derive(Serialize)] +struct ReadBundleReport { + primary_catalog_bytes: usize, + linked_catalog_identical_to_primary: bool, + two_worktree_catalog_bytes: usize, +} + +/// The catalog artifact bytes a graph seal writes for `generation`. +fn interactive_catalog( + generation: &CodeIndexPublishedGenerationV1, +) -> Result, Box> { + let manifest = build_published_code_graph_manifest_checked( + code_graph_projection_identity(GraphNamespace::new("sealed-storage")?)?, + generation, + &GraphProjectorRevision::try_from(CODE_GRAPH_PROJECTOR_REVISION.to_owned())?, + &|| Ok(()), + )?; + let mut bytes = Vec::new(); + write_interactive_catalog_artifact(&manifest, &mut bytes, &NeverCancelled)?; + Ok(bytes) +} + +/// The clean generation sealed again as a linked worktree of the same +/// project: every file segment it writes, and what two worktree scopes cost +/// when each stores its own segments versus sharing the project's. +#[derive(Serialize)] +struct LinkedWorktreeReport { + file_segments_written: usize, + file_segments_shared_with_primary: usize, + manifest_bytes: usize, + evidence_bytes: usize, + per_worktree_segments_total_bytes: usize, + shared_segments_total_bytes: usize, +} + +fn main() -> Result<(), Box> { + let repository = std::env::var_os(REPO_ENV) + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from(concat!(env!("CARGO_MANIFEST_DIR"), "/../.."))); + let revision = std::env::var(REV_ENV).unwrap_or_else(|_| "HEAD".to_owned()); + let revision = git_output(&repository, &["rev-parse", "--verify", &revision])?; + let sources = git_sources(&repository, &revision)?; + let corpus_bytes = sources.iter().map(|source| source.bytes.len()).sum(); + + let mut owner = CodeIndexProductionOwnerV1::new( + config()?, + MemoryPublication::default(), + ApplyingProjection, + )?; + let all_paths = sources + .iter() + .map(|source| source.logical_path.clone()) + .collect::>(); + let clean = owner.build_and_publish( + request( + &sources, + all_paths.clone(), + "tree.sealed-storage.clean", + 2_000_000, + None, + )?, + &ActiveControl, + )?; + // The same tree sealed again as a linked worktree of the project. + let linked = CodeIndexProductionOwnerV1::new( + config()?, + MemoryPublication::default(), + ApplyingProjection, + )? + .build_and_publish( + request( + &sources, + all_paths, + "tree.sealed-storage.clean", + 2_000_000, + Some(id::("worktree.sealed-storage.linked")?), + )?, + &ActiveControl, + )?; + let (edited, edited_paths) = edit(&sources); + let edited_files = edited_paths.len(); + let successor = owner.build_and_publish( + request( + &edited, + edited_paths, + "tree.sealed-storage.successor", + 4_000_000, + None, + )?, + &ActiveControl, + )?; + + let started = Instant::now(); + let clean_sealed = seal(&clean, None)?; + let clean_seal_ms = started.elapsed().as_millis(); + let started = Instant::now(); + let successor_sealed = seal(&successor, Some(&clean_sealed.manifest))?; + let successor_seal_ms = started.elapsed().as_millis(); + let linked_sealed = seal(&linked, None)?; + let file_segment_bytes = + |segments: &BTreeMap>| segments.values().map(Vec::len).sum::(); + let scope_bytes = |sealed: &Sealed| sealed.manifest.len() + sealed.evidence.1.len(); + let mut shared_segments = clean_sealed.file_segments.clone(); + shared_segments.extend( + linked_sealed + .file_segments + .iter() + .map(|(digest, bytes)| (digest.clone(), bytes.clone())), + ); + let linked_worktree = LinkedWorktreeReport { + file_segments_written: linked_sealed.file_segments.len(), + file_segments_shared_with_primary: linked_sealed + .file_segments + .keys() + .filter(|digest| clean_sealed.file_segments.contains_key(*digest)) + .count(), + manifest_bytes: linked_sealed.manifest.len(), + evidence_bytes: linked_sealed.evidence.1.len(), + per_worktree_segments_total_bytes: file_segment_bytes(&clean_sealed.file_segments) + + file_segment_bytes(&linked_sealed.file_segments) + + scope_bytes(&clean_sealed) + + scope_bytes(&linked_sealed), + shared_segments_total_bytes: file_segment_bytes(&shared_segments) + + scope_bytes(&clean_sealed) + + scope_bytes(&linked_sealed), + }; + let primary_catalog = interactive_catalog(&clean)?; + let linked_catalog = interactive_catalog(&linked)?; + let read_bundle = ReadBundleReport { + primary_catalog_bytes: primary_catalog.len(), + linked_catalog_identical_to_primary: linked_catalog == primary_catalog, + two_worktree_catalog_bytes: if linked_catalog == primary_catalog { + primary_catalog.len() + } else { + primary_catalog.len() + linked_catalog.len() + }, + }; + let mut store_segments = clean_sealed.file_segments.clone(); + store_segments.extend( + successor_sealed + .file_segments + .iter() + .map(|(digest, bytes)| (digest.clone(), bytes.clone())), + ); + + let mut decoded = Sha256::new(); + let clean_restore_ms = + digest_decoded(&clean_sealed, &clean_sealed.file_segments, &mut decoded)?; + let successor_restore_ms = digest_decoded(&successor_sealed, &store_segments, &mut decoded)?; + + let mut sections = BTreeMap::new(); + for bytes in store_segments.values() { + add_sections(bytes, &mut sections)?; + } + for sealed in [&clean_sealed, &successor_sealed] { + add_evidence_sections(&sealed.evidence.1, &mut sections)?; + } + let store_file_segment_bytes = store_segments.values().map(Vec::len).sum::(); + let store_evidence_bytes = clean_sealed.evidence.1.len() + successor_sealed.evidence.1.len(); + let store_manifest_bytes = clean_sealed.manifest.len() + successor_sealed.manifest.len(); + let report = Report { + repository: repository.display().to_string(), + revision, + corpus_files: sources.len(), + corpus_bytes, + edited_files, + clean: generation_report(&clean, &clean_sealed, clean_seal_ms, clean_restore_ms), + successor: generation_report( + &successor, + &successor_sealed, + successor_seal_ms, + successor_restore_ms, + ), + store_file_segments: store_segments.len(), + store_file_segment_bytes, + store_evidence_bytes, + store_manifest_bytes, + store_total_bytes: store_file_segment_bytes + store_evidence_bytes + store_manifest_bytes, + decoded_section_bytes: sections, + decoded_digest: format!("sha256:{}", hex::encode(decoded.finalize())), + linked_worktree, + read_bundle, + }; + println!("{}", serde_json::to_string_pretty(&report)?); + Ok(()) +} + +fn generation_report( + generation: &CodeIndexPublishedGenerationV1, + sealed: &Sealed, + seal_ms: u128, + restore_ms: u128, +) -> GenerationReport { + GenerationReport { + seal_ms, + restore_ms, + manifest_bytes: sealed.manifest.len(), + file_segments: generation.snapshot().files.len(), + file_segments_written: sealed.file_segments.len(), + file_segment_bytes_written: sealed.file_segments.values().map(Vec::len).sum(), + evidence_bytes: sealed.evidence.1.len(), + lineage_rows: generation.lineage().len(), + lineage_unchanged_rows: generation + .lineage() + .iter() + .filter(|row| row.kind == LineageKindV1::Unchanged) + .count(), + } +} + +fn git_output(repository: &PathBuf, args: &[&str]) -> Result> { + let output = Command::new("git") + .arg("-C") + .arg(repository) + .args(args) + .output()?; + if !output.status.success() { + return Err(format!("git {args:?}: {}", String::from_utf8_lossy(&output.stderr)).into()); + } + Ok(String::from_utf8(output.stdout)?.trim().to_owned()) +} + +/// Every blob at `revision` whose extension a compiled extractor parses. +fn git_sources(repository: &PathBuf, revision: &str) -> Result, Box> { + let registry = StaticLanguageRegistry::new(); + let listing = git_output(repository, &["ls-tree", "-r", "--full-tree", revision])?; + let mut wanted = Vec::new(); + for line in listing.lines() { + let (meta, path) = line.split_once('\t').ok_or("malformed ls-tree line")?; + let mut meta = meta.split_whitespace(); + let (Some(_mode), Some("blob"), Some(object)) = (meta.next(), meta.next(), meta.next()) + else { + continue; + }; + let Some(extension) = path.rsplit_once('.').map(|(_, extension)| extension) else { + continue; + }; + let Some(descriptor) = registry.descriptor_for_extension(&extension.to_lowercase()) else { + continue; + }; + if !descriptor.capabilities.extraction { + continue; + } + wanted.push(( + object.to_owned(), + path.to_owned(), + descriptor.language.clone(), + )); + } + let mut child = Command::new("git") + .arg("-C") + .arg(repository) + .args(["cat-file", "--batch"]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn()?; + let mut stdin = child.stdin.take().ok_or("git cat-file stdin")?; + let objects = wanted + .iter() + .map(|(object, _, _)| format!("{object}\n")) + .collect::(); + let writer = std::thread::spawn(move || stdin.write_all(objects.as_bytes())); + let mut stdout = BufReader::new(child.stdout.take().ok_or("git cat-file stdout")?); + let mut sources = Vec::with_capacity(wanted.len()); + for (_, path, language) in wanted { + let mut header = String::new(); + stdout.read_line(&mut header)?; + let size = header + .split_whitespace() + .nth(2) + .ok_or("malformed cat-file header")? + .parse::()?; + let mut bytes = vec![0; size + 1]; + stdout.read_exact(&mut bytes)?; + bytes.pop(); + if std::str::from_utf8(&bytes).is_err() { + continue; + } + sources.push(SourceFile { + logical_path: path, + language, + bytes: bytes.into(), + }); + } + writer + .join() + .map_err(|_| "git cat-file writer panicked")??; + child.wait()?; + sources.sort_by(|left, right| left.logical_path.cmp(&right.logical_path)); + Ok(sources) +} + +fn edit(sources: &[SourceFile]) -> (Vec, BTreeSet) { + let mut changed = BTreeSet::new(); + let edited = sources + .iter() + .enumerate() + .map(|(index, source)| { + let bytes = if index % EDIT_STRIDE == 0 { + changed.insert(source.logical_path.clone()); + let mut bytes = source.bytes.to_vec(); + bytes.extend_from_slice(EDIT_APPEND); + bytes.into() + } else { + Arc::clone(&source.bytes) + }; + SourceFile { + logical_path: source.logical_path.clone(), + language: source.language.clone(), + bytes, + } + }) + .collect(); + (edited, changed) +} + +fn request( + sources: &[SourceFile], + changed: BTreeSet, + tree: &str, + sealed_at: i64, + worktree: Option, +) -> Result> { + let mut files = Vec::with_capacity(sources.len()); + let mut captured_files = Vec::new(); + let mut identity = Sha256::new(); + for source in sources { + let digest = content_digest(&source.bytes); + identity.update(digest.as_str().as_bytes()); + // Like the daemon's, occurrences name content, not the worktree. + let occurrence = id::(&format!( + "file.sealed-storage.{}", + &hex::encode(Sha256::digest( + format!("{}\0{}", source.logical_path, digest.as_str()).as_bytes() + ))[..32] + ))?; + files.push(SanitizedCodeFileV1 { + file_occurrence_id: occurrence.clone(), + logical_path: source.logical_path.clone(), + language: Some(source.language.clone()), + content_digest: digest, + disposition: SnapshotFileDispositionV1::Present, + }); + if changed.contains(&source.logical_path) { + captured_files.push(CodeIndexCapturedFileV1 { + file_occurrence_id: occurrence, + sanitized_bytes: Arc::clone(&source.bytes), + sensitivity_level: SensitivityLevelV1::Public, + }); + } + } + Ok(CodeIndexBuildRequestV1 { + snapshot: SanitizedCodeSnapshotV1 { + repository: id("repository.sealed-storage")?, + worktree, + reference: None, + source_revision: None, + sanitizer_revision: id("sanitizer.sealed-storage.v1")?, + sanitization_receipts: vec![id::("receipt.sealed-storage")?], + content_identity: content_digest(&identity.finalize()), + captured_at: UtcMicros(sealed_at - 1_000_000), + files, + }, + captured_files, + changed_files: changed, + invalidations: BTreeSet::new(), + ignored_source_admissions: Vec::new(), + repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + tree: Some(id::(tree)?), + dirty: RepositoryDirtyStateV1::Clean, + }, + sealed_at: UtcMicros(sealed_at), + target_projection_key: ProjectionKeyV1 { + kind: ProjectionKindV1::Lexical, + schema_revision: "lexical.sealed-storage.v1".to_owned(), + profile_digest: ManifestDigest::from_sha256_bytes(&Sha256::digest(b"sealed-storage"))?, + }, + }) +} + +fn config() -> Result> { + Ok(CodeIndexProductionConfigV1 { + project_id: id::("project.sealed-storage")?, + repository: id::("repository.sealed-storage")?, + sanitizer_revision: id::("sanitizer.sealed-storage.v1")?, + policy_revision: id::("policy.sealed-storage.v1")?, + chunker_revision: id::("chunker.sealed-storage.v1")?, + privacy_domain: id::("privacy.sealed-storage")?, + privacy_key_epoch: 1, + max_snapshot_age_micros: None, + }) +} + +fn seal( + generation: &CodeIndexPublishedGenerationV1, + parent: Option<&[u8]>, +) -> Result { + let mut file_segments = BTreeMap::new(); + let mut pack = Vec::new(); + let mut evidence = None; + let manifest = generation.encode_partitioned_sealed_with_parent(parent, |publication| { + match publication { + SealedGenerationSegmentPublicationV1::File { digest, bytes } => { + file_segments.insert(digest.as_str().to_owned(), bytes.to_vec()); + } + SealedGenerationSegmentPublicationV1::GenerationEvidencePage { bytes, .. } => { + pack.extend_from_slice(bytes); + } + SealedGenerationSegmentPublicationV1::GenerationEvidenceCommit { + segment_digest, + .. + } => { + evidence = Some(( + segment_digest.as_str().to_owned(), + std::mem::take(&mut pack), + )); + } + } + Ok(()) + })?; + Ok(Sealed { + manifest, + file_segments, + evidence: evidence.ok_or_else(|| { + CodeIndexProductionErrorV1::Contract("sealed generation has no evidence".to_owned()) + })?, + }) +} + +/// Restore `sealed` from `segments` (its own plus any it reuses from a +/// parent) and hash everything restore and the lexical drain hand readers. +fn digest_decoded( + sealed: &Sealed, + segments: &BTreeMap>, + digest: &mut Sha256, +) -> Result> { + let mut segments = segments.clone(); + segments.insert(sealed.evidence.0.clone(), sealed.evidence.1.clone()); + let segments = Arc::new(segments); + let segment = |digest: &ManifestDigest| { + segments + .get(digest.as_str()) + .map(Vec::as_slice) + .ok_or_else(|| CodeIndexProductionErrorV1::Contract("segment is missing".to_owned())) + }; + let started = Instant::now(); + let generation = CodeIndexPublishedGenerationV1::decode_partitioned_sealed( + &sealed.manifest, + |request, buffer| { + let (bytes, offset, length) = match request { + SealedGenerationSegmentReadV1::Whole { digest, size_bytes } => { + (segment(digest)?, 0, size_bytes) + } + SealedGenerationSegmentReadV1::Range { + digest, + offset, + length, + .. + } => (segment(digest)?, offset, length), + }; + buffer.clear(); + buffer.extend_from_slice(&bytes[offset as usize..(offset + length) as usize]); + Ok(()) + }, + )?; + let restore_ms = started.elapsed().as_millis(); + digest.update(serde_json::to_vec(generation.lineage())?); + digest.update(serde_json::to_vec(&generation.symbols().symbols)?); + digest.update(serde_json::to_vec(generation.edges())?); + digest.update(serde_json::to_vec(generation.imports())?); + + let state_digest = ManifestDigest::from_sha256_bytes(&Sha256::digest(&sealed.manifest))?; + let lexical_segments = Arc::clone(&segments); + let mut source = VerifiedSealedLexicalPageSourceV1::open_partitioned_sealed( + &sealed.manifest, + state_digest, + move |digest, _, buffer, _control| { + let bytes = lexical_segments.get(digest.as_str()).ok_or_else(|| { + CodeIndexProductionErrorV1::Contract("segment is missing".to_owned()) + })?; + buffer.clear(); + buffer.extend_from_slice(bytes); + Ok(()) + }, + LEXICAL_PAGE_CHUNKS, + LEXICAL_PAGE_BYTES, + )?; + loop { + match source.next_page(&ActiveControl)? { + VerifiedSealedLexicalPageReadV1::Page(page) => { + for chunk in page.chunks() { + digest.update(serde_json::to_vec(chunk.chunk())?); + } + digest.update(serde_json::to_vec(page.clone_bodies())?); + digest.update(serde_json::to_vec(page.imports())?); + } + VerifiedSealedLexicalPageReadV1::Complete(receipt) => { + receipt.verify_completion(Some(source.cursor()))?; + return Ok(restore_ms); + } + } + } +} + +/// Split one stored file segment by payload section. The stored bytes are +/// either the canonical JSON itself or its raw DEFLATE stream. +fn add_sections( + bytes: &[u8], + sections: &mut BTreeMap, +) -> Result<(), Box> { + let json = if bytes.first() == Some(&b'{') { + bytes.to_vec() + } else { + let mut inflated = Vec::new(); + flate2::read::DeflateDecoder::new(bytes).read_to_end(&mut inflated)?; + inflated + }; + *sections.entry("total".to_owned()).or_default() += json.len(); + let value: serde_json::Value = serde_json::from_slice(&json)?; + let file = value.get("file").ok_or("segment has no file payload")?; + for (key, section) in file.as_object().ok_or("file payload is not an object")? { + if key == "artifacts" { + for (artifact, section) in section.as_object().ok_or("artifacts is not an object")? { + *sections.entry(format!("artifacts.{artifact}")).or_default() += + serde_json::to_vec(section)?.len(); + } + } else { + *sections.entry(key.clone()).or_default() += serde_json::to_vec(section)?.len(); + } + } + Ok(()) +} + +/// Split one evidence pack by top-level field. +fn add_evidence_sections( + bytes: &[u8], + sections: &mut BTreeMap, +) -> Result<(), Box> { + let value = serde_json::from_slice::(bytes)?; + for (key, section) in value.as_object().ok_or("evidence is not an object")? { + *sections.entry(format!("evidence.{key}")).or_default() += + serde_json::to_vec(section)?.len(); + } + Ok(()) +} + +fn id(value: &str) -> Result +where + T: TryFrom, + T::Error: Debug, +{ + T::try_from(value.to_owned()) +} diff --git a/crates/tracedecay-code-index/examples/sealed_decode_profile.rs b/crates/tracedecay-code-index/examples/sealed_decode_profile.rs deleted file mode 100644 index c0dd9ceb79..0000000000 --- a/crates/tracedecay-code-index/examples/sealed_decode_profile.rs +++ /dev/null @@ -1,108 +0,0 @@ -//! Decode one sealed generation file at corpus scale and report wall time, -//! resident-set peaks, and the restored corpus shape. -//! -//! Usage: `sealed_decode_profile ` -//! -//! Build with `--features hotpath` (or `hotpath-alloc`) for span/allocation -//! attribution; the plain build reports OS-level numbers only. - -use std::io::BufReader; -use std::process::ExitCode; -use std::time::Instant; - -use tracedecay_code_index::production::{ - CodeIndexPublishedGenerationV1, UninterruptibleCodeIndexControlV1, -}; - -#[cfg(feature = "hotpath-alloc")] -#[global_allocator] -static HOTPATH_ALLOCATOR: hotpath::CountingAllocator = hotpath::CountingAllocator::new(); - -fn proc_status_kib(field: &str) -> Option { - let status = std::fs::read_to_string("/proc/self/status").ok()?; - status.lines().find_map(|line| { - line.strip_prefix(field)? - .trim_start_matches(':') - .trim() - .strip_suffix(" kB")? - .parse() - .ok() - }) -} - -fn report_memory(stage: &str) { - let rss = proc_status_kib("VmRSS").unwrap_or(0); - let peak = proc_status_kib("VmHWM").unwrap_or(0); - println!( - "{stage}: VmRSS {} MiB, VmHWM {} MiB", - rss / 1024, - peak / 1024 - ); -} - -fn main() -> ExitCode { - #[cfg(feature = "hotpath")] - if std::env::var_os("HOTPATH_METRICS_SERVER_OFF").is_none() { - // The profiling run must open no socket; set before any thread exists. - unsafe { - std::env::set_var("HOTPATH_METRICS_SERVER_OFF", "1"); - } - } - #[cfg(feature = "hotpath")] - let _hotpath = hotpath::HotpathGuardBuilder::new("sealed-decode-profile").build(); - - let Some(path) = std::env::args().nth(1) else { - eprintln!("usage: sealed_decode_profile "); - return ExitCode::FAILURE; - }; - let file = match std::fs::File::open(&path) { - Ok(file) => file, - Err(error) => { - eprintln!("sealed_decode_profile: open {path}: {error}"); - return ExitCode::FAILURE; - } - }; - let admitted_len = match file.metadata() { - Ok(metadata) => metadata.len(), - Err(error) => { - eprintln!("sealed_decode_profile: metadata {path}: {error}"); - return ExitCode::FAILURE; - } - }; - println!("sealed bytes: {admitted_len}"); - report_memory("before decode"); - - let started = Instant::now(); - let decoded = CodeIndexPublishedGenerationV1::decode_sealed_seek_reader( - BufReader::with_capacity(1024 * 1024, file), - admitted_len, - None, - &UninterruptibleCodeIndexControlV1, - ); - let elapsed = started.elapsed(); - - let generation = match decoded { - Ok(Some(generation)) => generation, - Ok(None) => { - eprintln!("sealed_decode_profile: incompatible format revision"); - return ExitCode::FAILURE; - } - Err(error) => { - eprintln!("sealed_decode_profile: decode failed: {error}"); - return ExitCode::FAILURE; - } - }; - println!("decode wall: {:.3}s", elapsed.as_secs_f64()); - println!( - "generation {}: {} chunks, {} symbols, {} edges", - generation.manifest().generation_id.as_str(), - generation.chunks().chunks().len(), - generation.symbols().symbols.len(), - generation.edges().len(), - ); - report_memory("after decode"); - - drop(generation); - report_memory("after drop"); - ExitCode::SUCCESS -} diff --git a/crates/tracedecay-code-index/src/capabilities.rs b/crates/tracedecay-code-index/src/capabilities.rs index 9bc5598283..53caf72712 100644 --- a/crates/tracedecay-code-index/src/capabilities.rs +++ b/crates/tracedecay-code-index/src/capabilities.rs @@ -72,44 +72,12 @@ struct SealPayload<'a> { source_commitments: &'a Option, } -#[derive(Serialize)] -struct LegacySealPayload<'a> { - separator: &'static str, - generation_id: &'a CodeGenerationId, - snapshot_digest: &'a ManifestDigest, - registry_revision: &'a LanguageRegistryRevision, - grammar_revisions: &'a [(LanguageId, GrammarRevision)], - extractor_revisions: &'a [(LanguageId, ExtractorRevision)], - sanitizer_revision: &'a SanitizerRevision, - chunker_revision: &'a ChunkerRevision, - privacy_domain: &'a PrivacyDomainId, - privacy_key_epoch: u64, - parent_generation: &'a Option, - source_commitments: &'a Option, -} - /// The canonical expected digest a generation planner must seal before handing /// rows and the expected digest to the store publication port. The emitter /// verifies this digest before emitting capabilities for the generation. pub fn expected_seal_digest( generation: &CodeGenerationManifestV1, ) -> Result { - if generation.uses_legacy_v1_identity()? { - return canonical_sha256(&LegacySealPayload { - separator: GENERATION_SEAL_SEPARATOR, - generation_id: &generation.generation_id, - snapshot_digest: &generation.snapshot_digest, - registry_revision: &generation.registry_revision, - grammar_revisions: &generation.grammar_revisions, - extractor_revisions: &generation.extractor_revisions, - sanitizer_revision: &generation.sanitizer_revision, - chunker_revision: &generation.chunker_revision, - privacy_domain: &generation.privacy_domain, - privacy_key_epoch: generation.privacy_key_epoch, - parent_generation: &generation.parent_generation, - source_commitments: &generation.source_commitments, - }); - } canonical_sha256(&SealPayload { separator: GENERATION_SEAL_SEPARATOR, generation_id: &generation.generation_id, @@ -382,8 +350,11 @@ mod tests { .collect(); let mut manifest = CodeGenerationManifestV1 { project_id: tracedecay_domain::ProjectId::new("project.fixture").expect("valid id"), - generation_id: CodeGenerationId::new("generation.v1.aaaaaaaa.00000001") - .expect("valid id"), + generation_id: CodeGenerationId::new(format!( + "generation.v1.aaaaaaaa.00000001.{}", + "c".repeat(64) + )) + .expect("valid id"), snapshot_digest: digest('a'), invalidation_digest: digest('c'), registry_revision: registry.registry_revision(), @@ -401,9 +372,6 @@ mod tests { planner: tracedecay_domain::ComponentVersion::new("planner.v1").expect("valid id"), }, }; - manifest.invalidation_digest = manifest - .expected_legacy_invalidation_digest() - .expect("legacy invalidation digest computes"); manifest.seal.expected_digest = expected_seal_digest(&manifest).expect("seal digest computes"); manifest @@ -439,10 +407,7 @@ mod tests { let mut mixed = generation_manifest(); mixed.parent_generation = Some(mixed.generation_id.clone()); - // Recompute the integrity inputs so only self-supersession is wrong. - mixed.invalidation_digest = mixed - .expected_legacy_invalidation_digest() - .expect("mixed invalidation digest"); + // Recompute the seal so only self-supersession is wrong. mixed.seal.expected_digest = expected_seal_digest(&mixed).expect("seal"); assert_eq!( emitter().emit(&mixed), diff --git a/crates/tracedecay-code-index/src/chunks.rs b/crates/tracedecay-code-index/src/chunks.rs index 0320e16221..1920510c97 100644 --- a/crates/tracedecay-code-index/src/chunks.rs +++ b/crates/tracedecay-code-index/src/chunks.rs @@ -58,9 +58,6 @@ pub enum CodeSearchEligibilityV1 { Partial { reason: String, }, - Unsupported { - reason: String, - }, } /// One generation-bound file manifest, the scheduling/checkpoint unit. @@ -94,21 +91,6 @@ pub enum ChunkingFailureV1 { WorkerPanic { index: usize, message: String }, } -/// The deterministic chunker contract (Plan 25: `src/code_index/chunks.rs` -/// builds chunks and their parent/child hierarchy). -pub trait CodeChunker { - /// Build every chunk for one receipt-bound file plus its extraction batch, - /// covering every eligible sanitized byte with a declared chunk or an - /// explicit unsupported/excluded range. - fn chunk_file( - &self, - file: &ReceiptBoundCodeFileV1, - batch: &ExtractionBatchV1, - descriptor: &LanguageDescriptorV1, - cancellation: &dyn ExtractionCancellation, - ) -> Result; -} - /// The chunks produced for one file: the generation-bound document manifest /// plus its chunks in deterministic order (Plan 25). /// @@ -523,12 +505,10 @@ pub const EXACT_EXTRACTION_AUTHORITY_SEPARATOR: &str = "tracedecay.exact-extract /// The deterministic five-grain chunker. /// -/// The compatibility `CodeChunker` port accepts only extraction evidence and -/// therefore re-parses. Production indexing uses -/// [`Self::index_file_with_authority_from_extraction`] to consume the exact -/// sanitized parser rows that produced that evidence, avoiding a second parse. -/// Both paths validate batch, descriptor, and file identity before structural -/// work and share the same canonical materialization. +/// [`Self::index_file_with_authority_from_extraction`] consumes the exact +/// sanitized parser rows that produced the extraction evidence, so chunking +/// never re-parses. Batch, descriptor, and file identity are validated before +/// structural work. /// /// Construct one chunker per generation: generation identity, repository /// identity, sanitizer revision, policy revision, and chunker revision are @@ -538,50 +518,25 @@ pub struct DeterministicCodeChunker { repository: RepositoryId, sanitizer_revision: SanitizerRevision, policy_revision: PolicyRevisionId, - sensitivity_level: SensitivityLevelV1, chunker_revision: ChunkerRevision, - extractors: Arc, } impl DeterministicCodeChunker { - /// Create a chunker bound to one generation. Chunks default to - /// `SensitivityLevelV1::Public` under `policy_revision`; application - /// policy output refines this via `with_sensitivity_level`. + /// Create a chunker bound to one generation. Each indexing call supplies + /// the sensitivity level its chunks carry under `policy_revision`. pub fn new( generation_id: CodeGenerationId, repository: RepositoryId, sanitizer_revision: SanitizerRevision, policy_revision: PolicyRevisionId, chunker_revision: ChunkerRevision, - extractors: tracedecay_code_extraction::LanguageRegistry, - ) -> Self { - Self::from_shared_registry( - generation_id, - repository, - sanitizer_revision, - policy_revision, - chunker_revision, - Arc::new(extractors), - ) - } - - /// Create a generation-bound chunker over a shared parser registry. - pub fn from_shared_registry( - generation_id: CodeGenerationId, - repository: RepositoryId, - sanitizer_revision: SanitizerRevision, - policy_revision: PolicyRevisionId, - chunker_revision: ChunkerRevision, - extractors: Arc, ) -> Self { Self { generation_id, repository, sanitizer_revision, policy_revision, - sensitivity_level: SensitivityLevelV1::Public, chunker_revision, - extractors, } } @@ -590,41 +545,6 @@ impl DeterministicCodeChunker { &self.generation_id } - /// Index one receipt-bound file and retain its symbols, canonical graph - /// edges, and typed edge abstentions alongside its chunks. - pub fn index_file( - &self, - file: &ReceiptBoundCodeFileV1, - batch: &ExtractionBatchV1, - descriptor: &LanguageDescriptorV1, - cancellation: &dyn ExtractionCancellation, - ) -> Result { - let mut clone_build = ClonePayloadBuildContextV1::new(None); - self.build_file_artifacts_with_parse( - file, - batch, - descriptor, - None, - self.sensitivity_level, - cancellation, - &mut clone_build, - ) - } - - /// Index one receipt-bound file and return the opaque capability required - /// to re-admit its exact extraction evidence into lexical projection. - pub fn index_file_with_authority( - &self, - file: &ReceiptBoundCodeFileV1, - batch: &ExtractionBatchV1, - descriptor: &LanguageDescriptorV1, - cancellation: &dyn ExtractionCancellation, - ) -> Result<(CodeFileIndexArtifactsV1, ExactExtractionAuthorityV1), ChunkingFailureV1> { - let result = self.index_file(file, batch, descriptor, cancellation)?; - let authority = ExactExtractionAuthorityV1::mint(&result.chunks.chunks); - Ok((result, authority)) - } - /// Index one file from the parser rows that produced its extraction batch. /// The opaque output type prevents callers from pairing a batch with rows /// from a different parse. @@ -669,7 +589,7 @@ impl DeterministicCodeChunker { file, extraction.batch(), descriptor, - Some(extraction.parse_artifact()), + extraction.parse_artifact(), sensitivity_level, cancellation, &mut clone_build, @@ -681,20 +601,6 @@ impl DeterministicCodeChunker { Ok((result, authority, clone_build.stats())) } - /// Chunk one receipt-bound file and return the opaque capability required - /// to re-admit its exact extraction evidence into lexical projection. - pub fn chunk_file_with_authority( - &self, - file: &ReceiptBoundCodeFileV1, - batch: &ExtractionBatchV1, - descriptor: &LanguageDescriptorV1, - cancellation: &dyn ExtractionCancellation, - ) -> Result<(CodeFileChunksV1, ExactExtractionAuthorityV1), ChunkingFailureV1> { - let (result, authority) = - self.index_file_with_authority(file, batch, descriptor, cancellation)?; - Ok((result.chunks, authority)) - } - fn file_identity(&self, logical_path: &str) -> Result { code_file_identity(self.repository.as_str(), logical_path) } @@ -1120,20 +1026,6 @@ struct PendingChunk { parent: Option<(usize, Vec)>, } -impl CodeChunker for DeterministicCodeChunker { - #[hotpath::measure(label = "code_index.chunk.file")] - fn chunk_file( - &self, - file: &ReceiptBoundCodeFileV1, - batch: &ExtractionBatchV1, - descriptor: &LanguageDescriptorV1, - cancellation: &dyn ExtractionCancellation, - ) -> Result { - self.index_file(file, batch, descriptor, cancellation) - .map(|artifacts| artifacts.chunks) - } -} - impl DeterministicCodeChunker { #[allow(clippy::too_many_arguments)] fn build_file_artifacts_with_parse( @@ -1141,7 +1033,7 @@ impl DeterministicCodeChunker { file: &ReceiptBoundCodeFileV1, batch: &ExtractionBatchV1, descriptor: &LanguageDescriptorV1, - parse_artifact: Option<&ExtractionArtifactV1>, + parse_artifact: &ExtractionArtifactV1, sensitivity_level: SensitivityLevelV1, cancellation: &dyn ExtractionCancellation, clone_build: &mut ClonePayloadBuildContextV1<'_>, @@ -1166,50 +1058,10 @@ impl DeterministicCodeChunker { return Err(ChunkingFailureV1::GenerationMismatch); } - // A failed, timed-out, or cancelled extraction attests no structure: - // the document is explicitly unsupported and every byte is covered by - // the batch's error/unsupported evidence, not by invented chunks. - let parse_reason = match &batch.parse_outcome { - ParseOutcomeV1::Complete => None, - ParseOutcomeV1::Partial { reason } => { - return self.build_partial_artifacts( - file, - authority, - batch, - descriptor, - parse_artifact, - sensitivity_level, - cancellation, - reason.clone(), - clone_build, - ); - } - ParseOutcomeV1::TimedOut => { - Some("Tree-sitter parsing exceeded the bounded per-file parse budget".to_owned()) - } - ParseOutcomeV1::Cancelled => { - Some("extraction was cancelled before parsing completed".to_owned()) - } - ParseOutcomeV1::Failed { reason } => { - Some(format!("Tree-sitter parsing failed: {reason}")) - } + let partial_reason = match &batch.parse_outcome { + ParseOutcomeV1::Complete => String::new(), + ParseOutcomeV1::Partial { reason } => reason.clone(), }; - if let Some(reason) = parse_reason { - let document = CodeSearchDocumentV1 { - generation_id: self.generation_id.clone(), - file_occurrence_id: file.file.file_occurrence_id.clone(), - content_digest: file.file.content_digest.clone(), - eligibility: CodeSearchEligibilityV1::Unsupported { reason }, - chunk_ids: Vec::new(), - }; - return CodeFileIndexArtifactsV1::without_parser_rows( - CodeFileChunksV1 { - document, - chunks: Vec::new(), - }, - batch, - ); - } self.build_partial_artifacts( file, authority, @@ -1218,7 +1070,7 @@ impl DeterministicCodeChunker { parse_artifact, sensitivity_level, cancellation, - String::new(), + partial_reason, clone_build, ) } @@ -1232,7 +1084,7 @@ impl DeterministicCodeChunker { authority: &crate::intake::ReceiptBoundCodeFileAuthorityV1, batch: &ExtractionBatchV1, descriptor: &LanguageDescriptorV1, - parse_artifact: Option<&ExtractionArtifactV1>, + artifact: &ExtractionArtifactV1, sensitivity_level: SensitivityLevelV1, cancellation: &dyn ExtractionCancellation, partial_reason: String, @@ -1281,28 +1133,6 @@ impl DeterministicCodeChunker { )); } let source = &full_source[..parsed_prefix_end]; - let mut reparsed; - let artifact = if let Some(parse_artifact) = parse_artifact { - parse_artifact - } else { - let extractor = self - .extractors - .extractor_for_file(&file.file.logical_path) - .or_else(|| { - descriptor.extensions.iter().find_map(|extension| { - self.extractors - .extractor_for_file(&format!("probe.{extension}")) - }) - }) - .ok_or(ChunkingFailureV1::DescriptorMismatch)?; - if cancellation.is_cancelled() { - return Err(ChunkingFailureV1::Cancelled); - } - reparsed = extractor.extract_artifact(&file.file.logical_path, source); - reparsed.result.sanitize(); - reparsed.result.canonicalize_order(); - &reparsed - }; let result = &artifact.result; if cancellation.is_cancelled() { return Err(ChunkingFailureV1::Cancelled); @@ -2572,7 +2402,7 @@ fn edge_abstention( CodeIndexEdgeAbstentionV1 { source_node_id: edge.source.clone(), target_node_id: edge.target.clone(), - legacy_kind: edge.kind.as_str().to_owned(), + kind: edge.kind, reason, } } @@ -2750,11 +2580,10 @@ mod tests { use std::time::{Duration, Instant}; use super::*; - use crate::extract::ExtractionCoverageV1; use tracedecay_domain::{ BoundedSanitizedText, ChunkerRevision, CodeGenerationId, CodeIndexWorkerSelectionV1, CodeSearchChunkAnchorV1, CodeSearchChunkGrainV1, CodeSearchChunkId, ContentDigest, - FileOccurrenceId, GrammarRevision, LanguageDescriptorRevision, LanguageId, ManifestDigest, + FileOccurrenceId, GrammarRevision, LanguageDescriptorRevision, LanguageId, PolicyRevisionId, ProjectId, SanitizationReceiptId, SanitizedCodeFileV1, SanitizedCodeSnapshotV1, SanitizerRevision, SensitivityDecision, SensitivityLevelV1, SnapshotFileDispositionV1, SourceSpan, SymbolOccurrenceId, UtcMicros, ValidatedCodeFileV1, @@ -3043,7 +2872,6 @@ mod tests { id("sanitizer.v1"), id("policy.v1"), id("chunker.v1"), - tracedecay_code_extraction::LanguageRegistry::new(), ) } @@ -3087,47 +2915,22 @@ mod tests { .expect("receipt-bound file") } - fn batch_for(file: &ReceiptBoundCodeFileV1, outcome: ParseOutcomeV1) -> ExtractionBatchV1 { - let descriptor = rust_descriptor(); - // The parser import digest is the extractor's to state, never the - // fixture's: chunking re-derives the rows and refuses a batch that - // declares different ones. An outcome that attests no structure - // carries no rows at all, which is what the unsupported-document path - // builds its artifacts from. - let parser_import_rows_digest = match &outcome { - ParseOutcomeV1::Complete | ParseOutcomeV1::Partial { .. } => TreeSitterExtractor::new() - .extract(file, &descriptor, &NeverCancelled) - .expect("fixture extraction") - .batch() - .parser_import_rows_digest - .clone(), - ParseOutcomeV1::Failed { .. } - | ParseOutcomeV1::TimedOut - | ParseOutcomeV1::Cancelled => crate::extract::parser_import_rows_digest(&[]) - .expect("empty parser import rows digest"), - }; - ExtractionBatchV1 { - generation_id: file.generation_id.clone(), - file_occurrence_id: file.file.file_occurrence_id.clone(), - language: descriptor.language.clone(), - descriptor_revision: descriptor.descriptor_revision.clone(), - grammar_revision: descriptor.grammar_revision.clone(), - extractor_revision: descriptor.extractor_revision.clone(), - content_digest: file.file.content_digest.clone(), - parse_outcome: outcome, - parsed_ranges: vec![SourceSpan { - start_byte: 0, - end_byte: file.sanitized_bytes.len() as u64, - }], - error_ranges: Vec::new(), - unsupported_ranges: Vec::new(), - coverage: ExtractionCoverageV1 { - parsed_bytes: file.sanitized_bytes.len() as u64, - ..ExtractionCoverageV1::default() - }, - parser_import_rows_digest, - rows_digest: id::(&digest('d')), - } + fn extract_rust(file: &ReceiptBoundCodeFileV1) -> ExtractedCodeFileV1 { + TreeSitterExtractor::new() + .extract(file, &rust_descriptor(), &NeverCancelled) + .expect("fixture extraction") + } + + fn index_rust( + file: &ReceiptBoundCodeFileV1, + ) -> Result<(CodeFileIndexArtifactsV1, ExactExtractionAuthorityV1), ChunkingFailureV1> { + chunker().index_file_with_authority_from_extraction( + file, + &extract_rust(file), + &rust_descriptor(), + SensitivityLevelV1::Public, + &NeverCancelled, + ) } fn rust_descriptor() -> tracedecay_domain::LanguageDescriptorV1 { @@ -3139,13 +2942,7 @@ mod tests { fn chunk_source(source: &str) -> CodeFileChunksV1 { let file = validated_file("src/lib.rs", source.as_bytes()); - let descriptor = rust_descriptor(); - let extracted = TreeSitterExtractor::new() - .extract(&file, &descriptor, &NeverCancelled) - .expect("extract source"); - chunker() - .chunk_file(&file, extracted.batch(), &descriptor, &NeverCancelled) - .expect("chunking succeeds") + index_rust(&file).expect("chunking succeeds").0.chunks } /// A generation-sized chunk set, built from real extraction rather than @@ -3829,85 +3626,57 @@ mod tests { #[test] fn descriptor_and_generation_mismatch_are_typed_failures() { let file = validated_file("src/lib.rs", RUST_SOURCE.as_bytes()); - let batch = batch_for(&file, ParseOutcomeV1::Complete); + let extraction = extract_rust(&file); + let failure = |chunker: DeterministicCodeChunker, + file: &ReceiptBoundCodeFileV1, + descriptor: &LanguageDescriptorV1, + cancellation: &dyn ExtractionCancellation| { + chunker + .index_file_with_authority_from_extraction( + file, + &extraction, + descriptor, + SensitivityLevelV1::Public, + cancellation, + ) + .err() + }; - // Descriptor mismatch: python descriptor against a rust batch. + // Descriptor mismatch: python descriptor against a rust extraction. let python = StaticLanguageRegistry::new() .descriptor(&id::("python")) .expect("python descriptor") .clone(); assert_eq!( - chunker().chunk_file(&file, &batch, &python, &NeverCancelled), - Err(ChunkingFailureV1::DescriptorMismatch) + failure(chunker(), &file, &python, &NeverCancelled), + Some(ChunkingFailureV1::DescriptorMismatch) ); - // Generation mismatch: batch attests a different content digest. - let mut stale_batch = batch.clone(); - stale_batch.content_digest = id::(&digest('f')); + // Generation mismatch: extraction attests a different content digest. + let edited = validated_file("src/lib.rs", b"pub fn edited() {}\n"); assert_eq!( - chunker().chunk_file(&file, &stale_batch, &rust_descriptor(), &NeverCancelled), - Err(ChunkingFailureV1::GenerationMismatch) + failure(chunker(), &edited, &rust_descriptor(), &NeverCancelled), + Some(ChunkingFailureV1::GenerationMismatch) ); - // Generation mismatch: batch belongs to another generation. - let mut other_generation = batch.clone(); - other_generation.generation_id = id("generation.other"); - assert_eq!( - chunker().chunk_file( - &file, - &other_generation, - &rust_descriptor(), - &NeverCancelled - ), - Err(ChunkingFailureV1::GenerationMismatch) + // Generation mismatch: extraction belongs to another generation. + let other_generation = DeterministicCodeChunker::new( + id("generation.other"), + id("repo.fixture"), + id("sanitizer.v1"), + id("policy.v1"), + id("chunker.v1"), ); - - // Cancellation is a typed failure. assert_eq!( - chunker().chunk_file(&file, &batch, &rust_descriptor(), &AlwaysCancelled), - Err(ChunkingFailureV1::Cancelled) - ); - } - - #[test] - fn failed_parse_yields_an_explicit_unsupported_document() { - let file = validated_file("src/lib.rs", RUST_SOURCE.as_bytes()); - let batch = batch_for( - &file, - ParseOutcomeV1::Failed { - reason: "grammar crashed".to_owned(), - }, + failure(other_generation, &file, &rust_descriptor(), &NeverCancelled), + Some(ChunkingFailureV1::GenerationMismatch) ); - let result = chunker() - .chunk_file(&file, &batch, &rust_descriptor(), &NeverCancelled) - .expect("failed parse is evidence, not an error"); - assert!(result.chunks.is_empty()); - assert!(matches!( - result.document.eligibility, - CodeSearchEligibilityV1::Unsupported { .. } - )); - result.validate().expect("unsupported document validates"); - } - #[test] - fn partial_parse_is_declared_on_the_document() { - let file = validated_file("src/lib.rs", RUST_SOURCE.as_bytes()); - let batch = batch_for( - &file, - ParseOutcomeV1::Partial { - reason: "bounded traversal cap reached".to_owned(), - }, - ); - let result = chunker() - .chunk_file(&file, &batch, &rust_descriptor(), &NeverCancelled) - .expect("partial parse still chunks"); + // Cancellation is a typed failure. assert_eq!( - result.document.eligibility, - CodeSearchEligibilityV1::Partial { - reason: "bounded traversal cap reached".to_owned() - } + failure(chunker(), &file, &rust_descriptor(), &AlwaysCancelled), + Some(ChunkingFailureV1::Cancelled) ); - assert!(!result.chunks.is_empty()); } #[test] @@ -4166,10 +3935,10 @@ pub fn real_symbol() {} &missing_endpoint.reason, CodeIndexEdgeAbstentionReasonV1::MissingSymbolEndpoint )); - assert_eq!(missing_endpoint.legacy_kind, EdgeKind::Calls.as_str()); + assert_eq!(missing_endpoint.kind, EdgeKind::Calls); let unsupported_kind = abstentions .iter() - .find(|abstention| abstention.legacy_kind == EdgeKind::DerivesMacro.as_str()) + .find(|abstention| abstention.kind == EdgeKind::DerivesMacro) .expect("unsupported kind abstention"); assert!(matches!( &unsupported_kind.reason, @@ -4187,10 +3956,7 @@ pub fn real_symbol() {} let index = |source: &str| { let file = validated_file("src/lib.rs", source.as_bytes()); - let batch = batch_for(&file, ParseOutcomeV1::Complete); - chunker() - .index_file(&file, &batch, &rust_descriptor(), &NeverCancelled) - .expect("indexing succeeds") + index_rust(&file).expect("indexing succeeds").0 }; let relations = |artifacts: &CodeFileIndexArtifactsV1| { let name_of = |occurrence: &SymbolOccurrenceId| { @@ -4296,10 +4062,7 @@ pub fn real_symbol() {} fn no_call_refresh_probe_seals_zero_relation_edges() { let index = |source: &str| { let file = validated_file("src/refresh_batch/file_0000.rs", source.as_bytes()); - let batch = batch_for(&file, ParseOutcomeV1::Complete); - chunker() - .index_file(&file, &batch, &rust_descriptor(), &NeverCancelled) - .expect("indexing succeeds") + index_rust(&file).expect("indexing succeeds").0 }; let probe = "pub fn refresh_probe_0000_000(input: u32) -> u32 { input + 0 }\n"; @@ -4313,7 +4076,7 @@ pub fn real_symbol() {} assert!( artifacts.edge_abstentions.iter().all(|abstention| { abstention.reason == CodeIndexEdgeAbstentionReasonV1::MissingSymbolEndpoint - && abstention.legacy_kind == EdgeKind::Contains.as_str() + && abstention.kind == EdgeKind::Contains && abstention.source_node_id.starts_with("file:") }), "file Contains must abstain, not vanish: {:?}", @@ -4355,15 +4118,13 @@ pub fn real_symbol() {} #[test] fn incomplete_complexity_walk_reaches_lineage_records_as_unavailable_counters() { let mut source = String::from("pub fn huge(mut x: u64) -> u64 {\n"); + // Dense statements keep the file under the extraction byte cap. for _ in 0..tracedecay_code_extraction::complexity::TRAVERSAL_BUDGET / 4 { - source.push_str(" x += 1;\n"); + source.push_str("x+=1;"); } source.push_str(" if x > 3 { return x; }\n x\n}\n\npub fn small(x: u64) -> u64 {\n if x > 3 { return x; }\n x\n}\n"); let file = validated_file("src/lib.rs", source.as_bytes()); - let batch = batch_for(&file, ParseOutcomeV1::Complete); - let artifacts = chunker() - .index_file(&file, &batch, &rust_descriptor(), &NeverCancelled) - .expect("indexing succeeds"); + let artifacts = index_rust(&file).expect("indexing succeeds").0; let record = |name: &str| { artifacts .symbols @@ -4405,10 +4166,7 @@ pub fn real_symbol() {} fn resolved_calls_keep_each_parser_observed_invocation_span() { let source = "pub fn target() {}\npub fn caller() {\n let _label = \"λ\"; target();\n target();\n}\n"; let file = validated_file("src/lib.rs", source.as_bytes()); - let batch = batch_for(&file, ParseOutcomeV1::Complete); - let artifacts = chunker() - .index_file(&file, &batch, &rust_descriptor(), &NeverCancelled) - .expect("indexing succeeds"); + let artifacts = index_rust(&file).expect("indexing succeeds").0; let occurrence = |name: &str| { artifacts .symbols @@ -4471,10 +4229,7 @@ pub fn real_symbol() {} "fn make() -> Vec { Vec::new() }\n", ); let file = validated_file("src/lib.rs", source.as_bytes()); - let batch = batch_for(&file, ParseOutcomeV1::Complete); - let artifacts = chunker() - .index_file(&file, &batch, &rust_descriptor(), &NeverCancelled) - .expect("indexing succeeds"); + let artifacts = index_rust(&file).expect("indexing succeeds").0; let qualified = |occurrence: &SymbolOccurrenceId| { artifacts .symbols @@ -4533,10 +4288,7 @@ pub fn real_symbol() {} fn dotted_chain_references_keep_distinct_method_token_spans() { let source = "fn nested(builder: &WalkBuilder) { builder.repeat().repeat(); }\n"; let file = validated_file("src/lib.rs", source.as_bytes()); - let batch = batch_for(&file, ParseOutcomeV1::Complete); - let artifacts = chunker() - .index_file(&file, &batch, &rust_descriptor(), &NeverCancelled) - .expect("real Rust extraction"); + let artifacts = index_rust(&file).expect("real Rust extraction").0; let receiver_sites = artifacts .unresolved_references .iter() @@ -4564,10 +4316,7 @@ pub fn real_symbol() {} "fn caller(args: &Args) { args.walk_builder()?. /* decoy.unused */ build::(); }\n", ] { let file = validated_file("src/lib.rs", source.as_bytes()); - let batch = batch_for(&file, ParseOutcomeV1::Complete); - let artifacts = chunker() - .index_file(&file, &batch, &rust_descriptor(), &NeverCancelled) - .expect("real Rust extraction"); + let artifacts = index_rust(&file).expect("real Rust extraction").0; let member = artifacts .unresolved_references .iter() @@ -4599,10 +4348,7 @@ pub fn real_symbol() {} "pub fn ambiguous(processor: &T, input: u32) -> u32 { processor.process(input) }\n", ); let file = validated_file("src/lib.rs", source.as_bytes()); - let batch = batch_for(&file, ParseOutcomeV1::Complete); - let artifacts = chunker() - .index_file(&file, &batch, &rust_descriptor(), &NeverCancelled) - .expect("indexing succeeds"); + let artifacts = index_rust(&file).expect("indexing succeeds").0; let qualified = |occurrence: &SymbolOccurrenceId| { artifacts .symbols @@ -4662,10 +4408,7 @@ pub fn real_symbol() {} "}\n", ); let file = validated_file("src/lib.rs", source.as_bytes()); - let batch = batch_for(&file, ParseOutcomeV1::Complete); - let artifacts = chunker() - .index_file(&file, &batch, &rust_descriptor(), &NeverCancelled) - .expect("indexing succeeds"); + let artifacts = index_rust(&file).expect("indexing succeeds").0; let qualified = |occurrence: &SymbolOccurrenceId| { artifacts .symbols @@ -4751,10 +4494,7 @@ pub fn real_symbol() {} "}\n", ); let file = validated_file("src/walk.rs", source.as_bytes()); - let batch = batch_for(&file, ParseOutcomeV1::Complete); - let artifacts = chunker() - .index_file(&file, &batch, &rust_descriptor(), &NeverCancelled) - .expect("indexing succeeds"); + let artifacts = index_rust(&file).expect("indexing succeeds").0; let from_method = artifacts .symbols .iter() @@ -4801,10 +4541,7 @@ pub fn real_symbol() {} "}\n", ); let file = validated_file("src/lib.rs", source.as_bytes()); - let batch = batch_for(&file, ParseOutcomeV1::Complete); - let artifacts = chunker() - .index_file(&file, &batch, &rust_descriptor(), &NeverCancelled) - .expect("indexing succeeds"); + let artifacts = index_rust(&file).expect("indexing succeeds").0; let inherent = artifacts .symbols .iter() @@ -4849,10 +4586,7 @@ pub fn real_symbol() {} "pub struct Holder { pub processor: Doubler }\n", ); let file = validated_file("src/lib.rs", source.as_bytes()); - let batch = batch_for(&file, ParseOutcomeV1::Complete); - let artifacts = chunker() - .index_file(&file, &batch, &rust_descriptor(), &NeverCancelled) - .expect("indexing succeeds"); + let artifacts = index_rust(&file).expect("indexing succeeds").0; let holder_field = artifacts .symbols .iter() @@ -5165,11 +4899,9 @@ pub fn real_symbol() {} fn extraction_authority_rejects_matching_occurrence_forgery() { let source = "pub fn real_symbol() {\n // comment_fake\n}\n"; let file = validated_file("src/lib.rs", source.as_bytes()); - let batch = batch_for(&file, ParseOutcomeV1::Complete); - let (result, authority) = chunker() - .chunk_file_with_authority(&file, &batch, &rust_descriptor(), &NeverCancelled) - .expect("parser-backed chunks"); + let (result, authority) = index_rust(&file).expect("parser-backed chunks"); let mut chunk = result + .chunks .chunks .into_iter() .find(|chunk| { @@ -5225,11 +4957,19 @@ pub fn real_symbol() {} #[test] fn grammar_revision_in_descriptor_must_match_the_batch() { let file = validated_file("src/lib.rs", RUST_SOURCE.as_bytes()); - let mut batch = batch_for(&file, ParseOutcomeV1::Complete); - batch.grammar_revision = GrammarRevision::new("grammar.other.v1").expect("valid id"); + let mut descriptor = rust_descriptor(); + descriptor.grammar_revision = GrammarRevision::new("grammar.other.v1").expect("valid id"); assert_eq!( - chunker().chunk_file(&file, &batch, &rust_descriptor(), &NeverCancelled), - Err(ChunkingFailureV1::DescriptorMismatch) + chunker() + .index_file_with_authority_from_extraction( + &file, + &extract_rust(&file), + &descriptor, + SensitivityLevelV1::Public, + &NeverCancelled, + ) + .err(), + Some(ChunkingFailureV1::DescriptorMismatch) ); } } diff --git a/crates/tracedecay-code-index/src/chunks/artifacts.rs b/crates/tracedecay-code-index/src/chunks/artifacts.rs index 024941fcd7..0c42376654 100644 --- a/crates/tracedecay-code-index/src/chunks/artifacts.rs +++ b/crates/tracedecay-code-index/src/chunks/artifacts.rs @@ -9,7 +9,7 @@ use tracedecay_code_extraction::{ import_module_kind, }; use tracedecay_domain::{ - CanonicalRelationEdgeV1, CodeGenerationId, FileOccurrenceId, ManifestDigest, + CanonicalRelationEdgeV1, CodeGenerationId, EdgeKind, FileOccurrenceId, ManifestDigest, RelationEdgeKindV1, SourceSpan, SymbolOccurrenceId, }; @@ -226,7 +226,7 @@ pub enum CodeIndexEdgeAbstentionReasonV1 { pub struct CodeIndexEdgeAbstentionV1 { pub source_node_id: String, pub target_node_id: String, - pub legacy_kind: String, + pub kind: EdgeKind, pub reason: CodeIndexEdgeAbstentionReasonV1, } @@ -263,24 +263,6 @@ impl CodeFileIndexArtifactsV1 { Ok(artifacts) } - pub(crate) fn without_parser_rows( - chunks: CodeFileChunksV1, - extraction: &ExtractionBatchV1, - ) -> Result { - let artifacts = Self::from_parts( - chunks, - Vec::new(), - Vec::new(), - Vec::new(), - Vec::new(), - Vec::new(), - None, - Vec::new(), - )?; - artifacts.validate_generation_import_authority(extraction)?; - Ok(artifacts) - } - #[allow(clippy::too_many_arguments)] fn from_parts( chunks: CodeFileChunksV1, diff --git a/crates/tracedecay-code-index/src/clones.rs b/crates/tracedecay-code-index/src/clones.rs index ba989e0a53..de0d9be534 100644 --- a/crates/tracedecay-code-index/src/clones.rs +++ b/crates/tracedecay-code-index/src/clones.rs @@ -4,12 +4,10 @@ use std::sync::Arc; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; +use tracedecay_code_extraction::ExtractedCloneBodyV1; pub use tracedecay_code_extraction::{ - CloneBodyEligibilityV1, CloneBodyRenameStatusV1, ConservativeCloneTokenV1, -}; -use tracedecay_code_extraction::{ - CloneBodyRenameIssueV1, CloneBodyTokenizationIssueV1, CloneBodyTokenizationStatusV1, - ExtractedCloneBodyV1, + CloneBodyEligibilityV1, CloneBodyRenameIssueV1, CloneBodyRenameStatusV1, + CloneBodyTokenizationIssueV1, CloneBodyTokenizationStatusV1, ConservativeCloneTokenV1, }; use tracedecay_domain::{ CodeGenerationId, ManifestDigest, ProjectId, RepositoryId, SourceSpan, SymbolOccurrenceId, @@ -409,40 +407,72 @@ impl CodeIndexCloneBodyV1 { } } +/// Everything a clone payload holds except its digests, which are a pure +/// function of these fields. +pub struct CloneBodyPayloadPartsV1 { + pub language: String, + pub symbol_kind: String, + pub token_count: u32, + pub conservative_normalization_revision: u16, + pub conservative_tokens: Arc<[ConservativeCloneTokenV1]>, + pub tokenization_status: CloneBodyTokenizationStatusV1, + pub tokenization_issues: Vec, + pub rename_normalization_revision: Option, + pub rename_tokens: Option>, + pub rename_coverage: CloneBodyRenameStatusV1, + pub rename_issues: Vec, +} + impl CloneBodyPayloadV1 { pub fn from_extracted(body: &ExtractedCloneBodyV1) -> Result { - let digests = clone_payload_digests(ClonePayloadDigestInputV1 { - language: body.language.as_str(), - symbol_kind: body.symbol_kind.as_str(), - token_count: body.non_trivia_token_count, - conservative_revision: body.normalization_revision, - conservative_tokens: &body.conservative_tokens, - tokenization_status: body.tokenization_status, - tokenization_issues: &body.tokenization_issues, - rename_revision: body.rename_normalization_revision, - rename_tokens: body.rename_tokens.as_deref(), - rename_coverage: body.rename_status, - rename_issues: &body.rename_issues, - })?; - Ok(Self { - payload_digest: digests.payload, + Self::from_parts(CloneBodyPayloadPartsV1 { language: body.language.clone(), symbol_kind: body.symbol_kind.as_str().to_owned(), - body_digest: digests.body, token_count: body.non_trivia_token_count, conservative_normalization_revision: body.normalization_revision, - conservative_digest: digests.conservative, conservative_tokens: Arc::clone(&body.conservative_tokens), tokenization_status: body.tokenization_status, tokenization_issues: body.tokenization_issues.clone(), rename_normalization_revision: body.rename_normalization_revision, - rename_digest: digests.rename, rename_tokens: body.rename_tokens.clone(), rename_coverage: body.rename_status, rename_issues: body.rename_issues.clone(), }) } + pub fn from_parts(parts: CloneBodyPayloadPartsV1) -> Result { + let digests = clone_payload_digests(ClonePayloadDigestInputV1 { + language: &parts.language, + symbol_kind: &parts.symbol_kind, + token_count: parts.token_count, + conservative_revision: parts.conservative_normalization_revision, + conservative_tokens: &parts.conservative_tokens, + tokenization_status: parts.tokenization_status, + tokenization_issues: &parts.tokenization_issues, + rename_revision: parts.rename_normalization_revision, + rename_tokens: parts.rename_tokens.as_deref(), + rename_coverage: parts.rename_coverage, + rename_issues: &parts.rename_issues, + })?; + Ok(Self { + payload_digest: digests.payload, + language: parts.language, + symbol_kind: parts.symbol_kind, + body_digest: digests.body, + token_count: parts.token_count, + conservative_normalization_revision: parts.conservative_normalization_revision, + conservative_digest: digests.conservative, + conservative_tokens: parts.conservative_tokens, + tokenization_status: parts.tokenization_status, + tokenization_issues: parts.tokenization_issues, + rename_normalization_revision: parts.rename_normalization_revision, + rename_digest: digests.rename, + rename_tokens: parts.rename_tokens, + rename_coverage: parts.rename_coverage, + rename_issues: parts.rename_issues, + }) + } + pub fn exact_keys(&self, eligibility: CloneBodyEligibilityV1) -> Vec { if eligibility != CloneBodyEligibilityV1::Eligible || self.tokenization_status != CloneBodyTokenizationStatusV1::Complete diff --git a/crates/tracedecay-code-index/src/diagnostics.rs b/crates/tracedecay-code-index/src/diagnostics.rs index 9f54235cf9..56093fdf43 100644 --- a/crates/tracedecay-code-index/src/diagnostics.rs +++ b/crates/tracedecay-code-index/src/diagnostics.rs @@ -1,9 +1,9 @@ //! Generation/content-exact attachment of Plan 35 diagnostic records. //! //! Plan 35 owns diagnostic identity, producer provenance, persistence, and -//! clearing/supersession semantics. This module neither stores nor translates +//! clearing semantics. This module neither stores nor translates //! diagnostics. It validates one clean-generation watermark and emits only -//! exact current attachments; stale, cleared, superseded, incomplete, and +//! exact current attachments; stale, cleared, incomplete, and //! unsupported evidence remains explicitly typed. use std::collections::BTreeMap; @@ -70,9 +70,6 @@ pub enum GenerationDiagnosticDispositionV1 { Current { attachment: GenerationDiagnosticAttachmentV1, }, - Superseded { - successor_generation: CodeGenerationId, - }, Cleared { cleared_in_generation: CodeGenerationId, }, @@ -223,13 +220,6 @@ fn disposition_for( return GenerationDiagnosticDispositionV1::StaleScope; } match &record.state { - DiagnosticRecordStateV1::Superseded { - successor_generation, - } => { - return GenerationDiagnosticDispositionV1::Superseded { - successor_generation: successor_generation.clone(), - }; - } DiagnosticRecordStateV1::Cleared { cleared_in_generation, } => { diff --git a/crates/tracedecay-code-index/src/extract.rs b/crates/tracedecay-code-index/src/extract.rs index 1dacf8fdd0..52b3e9fd53 100644 --- a/crates/tracedecay-code-index/src/extract.rs +++ b/crates/tracedecay-code-index/src/extract.rs @@ -9,8 +9,6 @@ //! constructs are preserved as evidence; extraction never invents successful //! structure. -use std::sync::Arc; - use serde::{Deserialize, Serialize}; use tracedecay_code_extraction::{ ExtractedImportEvidenceV1, ExtractedSchemaEvidenceV1, ExtractionArtifactV1, @@ -63,9 +61,6 @@ pub struct ExtractionBatchV1 { pub enum ParseOutcomeV1 { Complete, Partial { reason: String }, - TimedOut, - Cancelled, - Failed { reason: String }, } /// Extraction coverage and ambiguity evidence. These are canonical raw @@ -223,24 +218,17 @@ pub const MAX_EXTRACTION_SOURCE_BYTES: usize = 1024 * 1024; /// canonically ordered before hashing, so identical sanitized input under /// identical descriptor revisions produces identical digests. pub struct TreeSitterExtractor { - parsers: Arc, + parsers: tracedecay_code_extraction::LanguageRegistry, } impl TreeSitterExtractor { /// Create the adapter over a freshly built extraction registry. pub fn new() -> Self { Self { - parsers: Arc::new(tracedecay_code_extraction::LanguageRegistry::new()), + parsers: tracedecay_code_extraction::LanguageRegistry::new(), } } - /// Share one generation-scoped registry with downstream chunking. - pub fn from_shared_registry( - parsers: Arc, - ) -> Self { - Self { parsers } - } - /// Resolve the parser for one file, falling back to the descriptor's /// declared extensions when the logical path carries no (recognized) /// extension. diff --git a/crates/tracedecay-code-index/src/generations.rs b/crates/tracedecay-code-index/src/generations.rs index a1bff3552e..20fb3b06f5 100644 --- a/crates/tracedecay-code-index/src/generations.rs +++ b/crates/tracedecay-code-index/src/generations.rs @@ -642,22 +642,22 @@ fn repository_discriminator( .collect()) } -/// Parse an identity minted by this planner. Legacy -/// `generation.v1..` parents remain accepted; current identities -/// include a SHA-256 invalidation fingerprint suffix. +/// Parse an identity minted by this planner: +/// `generation.v1...`, where the fingerprint is the +/// 64-hex SHA-256 invalidation suffix. Any other shape is not ours. fn parse_minted_generation_id(generation_id: &CodeGenerationId) -> Option<(String, u64)> { let mut parts = generation_id.as_str().split('.'); let scheme = parts.next()?; let version = parts.next()?; let discriminator = parts.next()?; let sequence = parts.next()?; - let fingerprint = parts.next(); - if scheme != "generation" || version != "v1" || parts.next().is_some() { - return None; - } - if fingerprint.is_some_and(|fingerprint| { - fingerprint.len() != 64 || !fingerprint.bytes().all(|byte| byte.is_ascii_hexdigit()) - }) { + let fingerprint = parts.next()?; + if scheme != "generation" + || version != "v1" + || parts.next().is_some() + || fingerprint.len() != 64 + || !fingerprint.bytes().all(|byte| byte.is_ascii_hexdigit()) + { return None; } Some((discriminator.to_owned(), sequence.parse().ok()?)) @@ -958,11 +958,14 @@ mod tests { // Foreign parent: an identity this planner never minted. let mut foreign = genesis.clone(); - foreign.generation_id = id("generation.v1.00000002.00000001"); + let fingerprint = genesis + .generation_id + .as_str() + .rsplit('.') + .next() + .expect("minted fingerprint"); + foreign.generation_id = id(&format!("generation.v1.00000002.00000001.{fingerprint}")); foreign.parent_generation = None; - foreign.invalidation_digest = foreign - .expected_legacy_invalidation_digest() - .expect("foreign invalidation digest"); foreign.seal.expected_digest = expected_seal_digest(&foreign).expect("foreign reseal"); assert_eq!( planner.plan_generation(&snapshot, Some(&foreign), UtcMicros(4_000)), diff --git a/crates/tracedecay-code-index/src/graph_projection/interactive.rs b/crates/tracedecay-code-index/src/graph_projection/interactive.rs index 87224aac46..7d02e42780 100644 --- a/crates/tracedecay-code-index/src/graph_projection/interactive.rs +++ b/crates/tracedecay-code-index/src/graph_projection/interactive.rs @@ -177,19 +177,9 @@ impl CodeGraphProjectionStore { if cancellation.is_cancelled() { return Err(CodeGraphProjectionError::Cancelled); } - let expected_generation = crate::graph_projection::code_graph_generation_id( - &self.generation, - &tracedecay_graph_db::GraphProjectorRevision::try_from( - crate::graph_projection::CODE_GRAPH_PROJECTOR_REVISION.to_owned(), - )?, - )?; let catalog = hotpath::measure_block!( "code_graph.catalog.bundle_install", - artifact::decode_interactive_catalog_artifact( - bytes, - expected_generation.as_str(), - cancellation.as_ref(), - ) + artifact::decode_interactive_catalog_artifact(bytes, cancellation.as_ref()) )?; let mut state = self .interactive_catalog diff --git a/crates/tracedecay-code-index/src/graph_projection/interactive/artifact.rs b/crates/tracedecay-code-index/src/graph_projection/interactive/artifact.rs index 88cffd2f06..50258ff291 100644 --- a/crates/tracedecay-code-index/src/graph_projection/interactive/artifact.rs +++ b/crates/tracedecay-code-index/src/graph_projection/interactive/artifact.rs @@ -30,7 +30,10 @@ use crate::lineage::LineageSymbolRecordV1; /// Bundle artifact name of the interactive catalog. pub const INTERACTIVE_CATALOG_ARTIFACT_NAME: &str = "interactive-catalog"; -const INTERACTIVE_CATALOG_ARTIFACT_FORMAT_V1: &str = "tracedecay.code-graph-interactive-catalog.v2"; +/// v3 names no generation: the catalog is a pure function of the graph's +/// rows, so linked worktrees sealing the same graph share one artifact, and +/// the per-generation bundle manifest carries the identity binding. +const INTERACTIVE_CATALOG_ARTIFACT_FORMAT_V1: &str = "tracedecay.code-graph-interactive-catalog.v3"; #[derive(Debug, Deserialize, Serialize)] #[serde(deny_unknown_fields)] @@ -45,9 +48,6 @@ struct CatalogSymbolRowV1 { #[serde(deny_unknown_fields)] struct InteractiveCatalogArtifactV1 { format: String, - /// The graph generation this catalog was derived from, for a cheap - /// self-description check on top of the envelope's identity binding. - graph_generation: String, symbols: Vec, files: Vec, imports: Vec, @@ -118,7 +118,6 @@ where #[derive(Serialize)] struct InteractiveCatalogArtifactViewV1<'a> { format: &'static str, - graph_generation: &'a str, symbols: CatalogSymbolRowsV1<'a>, files: CancellableRowsV1<'a, SanitizedCodeFileV1>, imports: CancellableRowsV1<'a, CodeIndexImportEvidenceV1>, @@ -284,7 +283,6 @@ pub fn write_interactive_catalog_artifact( check_cancelled(cancellation)?; let artifact = InteractiveCatalogArtifactViewV1 { format: INTERACTIVE_CATALOG_ARTIFACT_FORMAT_V1, - graph_generation: manifest.generation.as_str(), symbols: CatalogSymbolRowsV1 { manifest, cancellation, @@ -314,7 +312,6 @@ pub fn write_interactive_catalog_artifact( /// catalog, revalidating structural invariants row by row. pub(super) fn decode_interactive_catalog_artifact( bytes: &[u8], - expected_graph_generation: &str, cancellation: &dyn GraphCancellation, ) -> Result { let artifact: InteractiveCatalogArtifactV1 = @@ -329,9 +326,6 @@ pub(super) fn decode_interactive_catalog_artifact( artifact.format ))); } - if artifact.graph_generation != expected_graph_generation { - return Err(CodeGraphProjectionError::GenerationMismatch); - } let mut catalog = InteractiveCatalog::empty(); for file in artifact.files { check_cancelled(cancellation)?; diff --git a/crates/tracedecay-code-index/src/graph_projection/interactive/tests/bundle_artifact.rs b/crates/tracedecay-code-index/src/graph_projection/interactive/tests/bundle_artifact.rs index 0dc4cf9f7a..24f394f615 100644 --- a/crates/tracedecay-code-index/src/graph_projection/interactive/tests/bundle_artifact.rs +++ b/crates/tracedecay-code-index/src/graph_projection/interactive/tests/bundle_artifact.rs @@ -1,6 +1,7 @@ //! Seal-time catalog artifact: the manifest-derived bundle artifact must //! install as a ready catalog identical to the one the projection warm scan -//! builds, and a foreign or corrupt artifact must be a typed refusal. +//! builds, carry no generation of its own, and a corrupt artifact must be a +//! typed refusal. use std::io::{self, Write}; use std::sync::atomic::{AtomicBool, Ordering}; @@ -237,24 +238,26 @@ fn catalog_with_reversed_receiver_span_is_refused() { )); } +/// Linked worktrees seal the same graph under their own generations; the +/// artifact must not tell them apart, or they could never share it. The +/// generation binding is the per-generation bundle manifest's job. #[test] -fn artifact_for_a_foreign_generation_is_a_typed_mismatch() { +fn the_same_graph_under_another_generation_encodes_identical_bytes() { let bytes = encoded_fixture_artifact(); - let foreign = code_graph_generation_id( + let mut other = production_manifest(); + other.generation = code_graph_generation_id( &id::("generation.interactive.other"), &GraphProjectorRevision::try_from(CODE_GRAPH_PROJECTOR_REVISION.to_owned()) .expect("projector revision"), ) - .expect("foreign generation id"); - let error = match decode_interactive_catalog_artifact(&bytes, foreign.as_str(), &NeverCancelled) - { - Ok(_) => panic!("foreign generation must be refused"), - Err(error) => error, - }; - assert!(matches!( - error, - CodeGraphProjectionError::GenerationMismatch - )); + .expect("other generation id"); + assert_ne!(other.generation, production_manifest().generation); + let mut other_bytes = Vec::new(); + write_interactive_catalog_artifact(&other, &mut other_bytes, &NeverCancelled) + .expect("encode catalog artifact under another generation"); + assert_eq!(bytes, other_bytes); + decode_interactive_catalog_artifact(&other_bytes, &NeverCancelled) + .expect("the shared artifact decodes for either generation"); } #[test] diff --git a/crates/tracedecay-code-index/src/hotpath_observe.rs b/crates/tracedecay-code-index/src/hotpath_observe.rs index 4dffc12492..aca1b9e9a7 100644 --- a/crates/tracedecay-code-index/src/hotpath_observe.rs +++ b/crates/tracedecay-code-index/src/hotpath_observe.rs @@ -298,18 +298,6 @@ pub(crate) fn record_pages(count: u64) { } } -#[inline(always)] -pub(crate) fn record_seal_bytes(bytes: u64) { - #[cfg(feature = "hotpath")] - { - hotpath::gauge!("code_index_seal_bytes").set(bytes); - } - #[cfg(not(feature = "hotpath"))] - { - let _ = bytes; - } -} - /// Start of one production-owner generation build. The matching observation /// ends only after the immutable generation has been published and is /// queryable through that owner; daemon scheduling/wake latency is measured diff --git a/crates/tracedecay-code-index/src/languages.rs b/crates/tracedecay-code-index/src/languages.rs index 73140e9bb3..c3a115cd34 100644 --- a/crates/tracedecay-code-index/src/languages.rs +++ b/crates/tracedecay-code-index/src/languages.rs @@ -219,12 +219,17 @@ impl StaticLanguageRegistry { // callee. Only re-extraction removes the poisoned record. Rust v13 // retains unresolved receiver-call evidence at the parser's member // token; re-extracting Rust does not perturb other languages' rows. - let extractor_revision = if language == "rust" { - 13 - } else if matches!(language.as_str(), "typescript" | "protobuf" | "sql") { - 6 - } else { - 5 + // The C-comment docstring languages moved one revision when a + // docstring stopped absorbing trailing or blank-line-detached + // comments and `///` lost its stray `/`; QBasic dialects moved when + // CONST names stopped losing their text before an underscore. + let extractor_revision = match language.as_str() { + "rust" => 13, + "protobuf" => 7, + "typescript" | "sql" => 6, + "c" | "cpp" | "metal" | "objc" | "go" | "glsl" | "pascal" | "qbasic" + | "quickbasic" => 6, + _ => 5, }; let descriptor = LanguageDescriptorV1 { language: LanguageId::new(language.clone()) @@ -415,8 +420,6 @@ mod tests { #[test] fn descriptor_lookups_are_canonical_and_deterministic() { let registry = StaticLanguageRegistry::new(); - let again = StaticLanguageRegistry::new(); - assert_eq!(registry.registry_revision(), again.registry_revision()); let rust = registry .descriptor(&language("rust")) @@ -448,8 +451,11 @@ mod tests { assert!(registry.descriptor(&language("cobol-nope")).is_none()); assert!(registry.descriptor_for_extension("nope").is_none()); assert_eq!( - registry.descriptor_revision(&language("rust")), - Some(rust.descriptor_revision.clone()) + registry + .descriptor_revision(&language("rust")) + .as_ref() + .map(|revision| revision.as_str()), + Some("descriptor.rust.v1") ); // Canonical language-identity order. @@ -463,6 +469,73 @@ mod tests { assert_eq!(ids, sorted); } + #[test] + #[cfg(feature = "full")] + fn full_tier_registers_every_compiled_extractor_language() { + let registry = StaticLanguageRegistry::new(); + let ids: Vec<&str> = registry + .descriptors() + .iter() + .map(|d| d.language.as_str()) + .collect(); + assert_eq!( + ids, + [ + "astro", + "bash", + "batch", + "c", + "clojure", + "cobol", + "cpp", + "csharp", + "dart", + "dockerfile", + "elixir", + "erlang", + "fortran", + "fsharp", + "glsl", + "go", + "gwbasic", + "haskell", + "hlsl", + "java", + "julia", + "kotlin", + "lean", + "lua", + "markdown", + "metal", + "msbasic2", + "nix", + "objc", + "ocaml", + "pascal", + "perl", + "php", + "powershell", + "protobuf", + "python", + "qbasic", + "quickbasic", + "quint", + "r", + "ruby", + "rust", + "scala", + "sql", + "svelte", + "swift", + "toml", + "typescript", + "vbnet", + "wgsl", + "zig", + ] + ); + } + #[test] fn descriptor_set_rejects_duplicate_languages_aliases_and_extensions() { let rust = StaticLanguageRegistry::new() diff --git a/crates/tracedecay-code-index/src/lineage.rs b/crates/tracedecay-code-index/src/lineage.rs index 8de82d2589..82a29cd967 100644 --- a/crates/tracedecay-code-index/src/lineage.rs +++ b/crates/tracedecay-code-index/src/lineage.rs @@ -44,6 +44,39 @@ pub struct SymbolLineageCandidateV1 { pub abstention: Option, } +impl SymbolLineageCandidateV1 { + /// The candidate the resolver emits when `symbol` continues with the + /// same content from `prior_occurrence`, the prior occurrence of its + /// exact identity tuple. Sealed evidence leaves such rows implicit and + /// rebuilds them here. + pub(crate) fn exact_unchanged( + prior_generation: &CodeGenerationId, + current_generation: &CodeGenerationId, + prior_occurrence: &SymbolOccurrenceId, + symbol: &LineageSymbolRecordV1, + ) -> Result { + let kind = LineageKindV1::Unchanged; + let method = LineageMethodV1::ExactIdentityTuple; + Ok(Self { + prior_occurrence: prior_occurrence.clone(), + current_occurrence: symbol.occurrence.clone(), + kind, + method, + evidence: evidence_from( + prior_generation, + current_generation, + symbol, + Some((prior_occurrence, &symbol.content_digest)), + kind, + method, + )?, + confidence: LineageConfidenceKindV1::Exact, + alternatives: Vec::new(), + abstention: None, + }) + } +} + #[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)] #[serde(rename_all = "snake_case")] pub enum LineageKindV1 { @@ -1053,13 +1086,32 @@ fn evidence( kind: LineageKindV1, method: LineageMethodV1, ) -> Result { - let prior_digest = ancestor.map(|record| record.content_digest.clone()); + evidence_from( + prior_generation, + current_generation, + symbol, + ancestor.map(|record| (&record.occurrence, &record.content_digest)), + kind, + method, + ) +} + +/// [`evidence`] from the only two facts it reads of the ancestor. +fn evidence_from( + prior_generation: &CodeGenerationId, + current_generation: &CodeGenerationId, + symbol: &LineageSymbolRecordV1, + ancestor: Option<(&SymbolOccurrenceId, &ContentDigest)>, + kind: LineageKindV1, + method: LineageMethodV1, +) -> Result { + let prior_digest = ancestor.map(|(_, digest)| digest.clone()); let evidence_digest = canonical_sha256(&( LINEAGE_EVIDENCE_SEPARATOR, prior_generation, current_generation, &symbol.occurrence, - ancestor.map(|record| &record.occurrence), + ancestor.map(|(occurrence, _)| occurrence), kind, method, &prior_digest, diff --git a/crates/tracedecay-code-index/src/parallelism.rs b/crates/tracedecay-code-index/src/parallelism.rs index d0c8e7191a..e01e08c562 100644 --- a/crates/tracedecay-code-index/src/parallelism.rs +++ b/crates/tracedecay-code-index/src/parallelism.rs @@ -514,15 +514,18 @@ impl fmt::Display for CodeIndexParallelismErrorV1 { impl std::error::Error for CodeIndexParallelismErrorV1 {} +#[cfg(any(test, feature = "test-helpers"))] thread_local! { - /// Test-only worker width. Thread-scoped so one equivalence or batching + /// Forced worker width. Thread-scoped so one equivalence or batching /// test cannot change a sibling test's scheduling policy. static FORCED_WORKERS: std::cell::Cell = const { std::cell::Cell::new(0) }; +} - /// Test-only: force [`install`] on this thread to return +#[cfg(test)] +thread_local! { + /// Forces [`install`] on this thread to return /// [`CodeIndexParallelismErrorV1::PoolBuild`]. Thread-scoped so a fault /// test cannot leak into sibling tests running in the same process. - /// Visible to integration tests; production callers leave it false. static FORCE_INSTALL_FAILURE: std::cell::Cell = const { std::cell::Cell::new(false) }; } @@ -530,13 +533,14 @@ thread_local! { /// inline". #[must_use] pub fn indexing_workers() -> usize { - match FORCED_WORKERS.with(std::cell::Cell::get) { - 0 => WORKER_RUNTIME.get().map_or_else( - || indexing_worker_target(detected_cores()), - |runtime| runtime.plan.effective_workers, - ), - forced => forced, + #[cfg(any(test, feature = "test-helpers"))] + if let forced @ 1.. = FORCED_WORKERS.with(std::cell::Cell::get) { + return forced; } + WORKER_RUNTIME.get().map_or_else( + || indexing_worker_target(detected_cores()), + |runtime| runtime.plan.effective_workers, + ) } /// Force the indexing width for an equivalence test. @@ -544,23 +548,22 @@ pub fn indexing_workers() -> usize { /// Width is sizing policy, never semantics: the same inputs must produce the /// same generation bytes at any width. This exists so one test process can /// build a fixture at width 1 and at full width and compare the sealed -/// digests directly. It is not a supported runtime control, production sizing -/// comes from [`indexing_workers`]. -#[doc(hidden)] +/// digests directly. Production sizing comes from [`indexing_workers`]. +#[cfg(any(test, feature = "test-helpers"))] pub fn force_indexing_workers_for_test(workers: usize) { FORCED_WORKERS.with(|forced| forced.set(workers.max(1))); } /// Restore production sizing after [`force_indexing_workers_for_test`]. -#[doc(hidden)] +#[cfg(any(test, feature = "test-helpers"))] pub fn clear_forced_indexing_workers_for_test() { FORCED_WORKERS.with(|forced| forced.set(0)); } /// Force [`install`] to fail so callers can assert operational pool errors stay /// typed as parallelism failures instead of identity corruption. -#[doc(hidden)] -pub fn force_install_failure_for_test(force: bool) { +#[cfg(test)] +pub(crate) fn force_install_failure_for_test(force: bool) { FORCE_INSTALL_FAILURE.with(|flag| flag.set(force)); } @@ -599,6 +602,7 @@ where R: Send, { hotpath::gauge!("code_index_worker_count").set(indexing_workers()); + #[cfg(test)] if FORCE_INSTALL_FAILURE.with(std::cell::Cell::get) { return Err(CodeIndexParallelismErrorV1::PoolBuild { message: "forced code-index worker pool failure for test".to_owned(), diff --git a/crates/tracedecay-code-index/src/production/clone_rows.rs b/crates/tracedecay-code-index/src/production/clone_rows.rs new file mode 100644 index 0000000000..035dffd33a --- /dev/null +++ b/crates/tracedecay-code-index/src/production/clone_rows.rs @@ -0,0 +1,564 @@ +//! Persisted clone-body rows of one file segment. +//! +//! Token streams dominate a file segment when every token is its own JSON +//! object carrying its syntax kind and text. This row form interns each +//! syntax kind and token text of the file once and writes every stream as +//! integer codes. A rename stream agrees with its conservative stream token +//! for token except where an identifier was renamed, so it is written as +//! those renamed positions. Payload digests and the occurrence fields the +//! file authority already fixes (project, repository, worktree, source +//! generation, path) are not stored: restore recomputes the digests from the +//! tokens and rebinds the occurrence to the authority it is validated +//! against. + +use std::borrow::Cow; +use std::collections::HashMap; +use std::sync::Arc; + +use serde::{Deserialize, Serialize}; +use tracedecay_code_extraction::{ + CloneBodyRenameIssueV1, CloneBodyTokenizationIssueV1, CloneBodyTokenizationStatusV1, +}; +use tracedecay_domain::{ManifestDigest, SourceSpan, SymbolOccurrenceId}; + +use super::CodeIndexProductionErrorV1; +use crate::clones::{ + CloneBodyEligibilityV1, CloneBodyOccurrenceV1, CloneBodyPayloadPartsV1, CloneBodyPayloadV1, + CloneBodyRenameStatusV1, CodeIndexCloneBodyV1, ConservativeCloneTokenV1, +}; +use crate::extract::ExtractionBatchV1; +use crate::intake::ReceiptBoundCodeFileAuthorityV1; + +/// Token codes. `0` closes the innermost open structure; any other code `c` +/// names string `(c - 1) >> 2` as the syntax kind and carries tag +/// `(c - 1) & 3`. A syntax token whose text differs from its kind is followed +/// by the string index of its text. +const CLOSE_INNERMOST: u32 = 0; +const TAG_START: u32 = 0; +const TAG_END: u32 = 1; +const TAG_SYNTAX_KIND_TEXT: u32 = 2; +const TAG_SYNTAX: u32 = 3; + +fn contract(message: &str) -> CodeIndexProductionErrorV1 { + CodeIndexProductionErrorV1::Contract(message.to_owned()) +} + +#[derive(Serialize)] +pub(super) struct PersistedCloneBodiesRefV1<'a> { + /// The first body's language and snapshot digest; a body carries its own + /// only where it differs. + #[serde(skip_serializing_if = "Option::is_none")] + language: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + snapshot_digest: Option<&'a ManifestDigest>, + strings: Vec<&'a str>, + bodies: Vec>, +} + +#[derive(Serialize)] +struct PersistedCloneBodyRefV1<'a> { + symbol_occurrence_id: &'a SymbolOccurrenceId, + #[serde(skip_serializing_if = "Option::is_none")] + language: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + snapshot_digest: Option<&'a ManifestDigest>, + symbol_kind: &'a str, + body_span: SourceSpan, + eligibility: CloneBodyEligibilityV1, + token_count: u32, + conservative_normalization_revision: u16, + tokenization_status: CloneBodyTokenizationStatusV1, + #[serde(skip_serializing_if = "<[_]>::is_empty")] + tokenization_issues: &'a [CloneBodyTokenizationIssueV1], + conservative_tokens: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + rename_normalization_revision: Option, + rename_coverage: CloneBodyRenameStatusV1, + #[serde(skip_serializing_if = "<[_]>::is_empty")] + rename_issues: &'a [CloneBodyRenameIssueV1], + #[serde(skip_serializing_if = "Option::is_none")] + rename_tokens: Option, +} + +#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case", deny_unknown_fields)] +enum PersistedRenameTokensV1 { + /// The conservative stream itself. + Conservative, + /// The conservative stream with `[position, text]` replacements. + Renamed(Vec<[u32; 2]>), + /// A stream that does not align with the conservative one. + Tokens(Vec), +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct PersistedCloneBodiesV1 { + #[serde(default)] + language: Option, + #[serde(default)] + snapshot_digest: Option, + strings: Vec, + bodies: Vec, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct PersistedCloneBodyV1 { + symbol_occurrence_id: SymbolOccurrenceId, + #[serde(default)] + language: Option, + #[serde(default)] + snapshot_digest: Option, + symbol_kind: String, + body_span: SourceSpan, + eligibility: CloneBodyEligibilityV1, + token_count: u32, + conservative_normalization_revision: u16, + tokenization_status: CloneBodyTokenizationStatusV1, + #[serde(default)] + tokenization_issues: Vec, + conservative_tokens: Vec, + #[serde(default)] + rename_normalization_revision: Option, + rename_coverage: CloneBodyRenameStatusV1, + #[serde(default)] + rename_issues: Vec, + #[serde(default)] + rename_tokens: Option, +} + +impl<'a> PersistedCloneBodiesRefV1<'a> { + /// Refuses a body whose occurrence disagrees with the file authority it + /// would be rebound to on restore, rather than persisting a row that + /// restores differently. Rows and the string table follow `bodies` order, + /// so a caller that wants worktree-independent bytes passes an order that + /// does not depend on occurrence identities. + pub(super) fn new( + authority: &ReceiptBoundCodeFileAuthorityV1, + extraction: &ExtractionBatchV1, + bodies: &[&'a CodeIndexCloneBodyV1], + ) -> Result { + let language = bodies.first().map(|body| body.payload.language.as_str()); + let snapshot_digest = bodies.first().map(|body| &body.occurrence.snapshot_digest); + let mut strings = StringTableV1::default(); + let mut rows = Vec::with_capacity(bodies.len()); + for &body in bodies { + let occurrence = &body.occurrence; + let payload = &*body.payload; + if occurrence.project_id != authority.project_id + || occurrence.repository_id != authority.repository_id + || occurrence.worktree_id != authority.worktree_id + || occurrence.path != authority.logical_path + || occurrence.source_generation != extraction.generation_id + || occurrence.payload_digest != payload.payload_digest + { + return Err(contract( + "sealed clone body occurrence disagrees with its file authority", + )); + } + let conservative_tokens = strings.encode(&payload.conservative_tokens)?; + let rename_tokens = payload + .rename_tokens + .as_deref() + .map(|rename| strings.encode_rename(&payload.conservative_tokens, rename)) + .transpose()?; + rows.push(PersistedCloneBodyRefV1 { + symbol_occurrence_id: &occurrence.symbol_occurrence_id, + language: (Some(payload.language.as_str()) != language) + .then_some(payload.language.as_str()), + snapshot_digest: (Some(&occurrence.snapshot_digest) != snapshot_digest) + .then_some(&occurrence.snapshot_digest), + symbol_kind: &payload.symbol_kind, + body_span: occurrence.body_span, + eligibility: occurrence.eligibility, + token_count: payload.token_count, + conservative_normalization_revision: payload.conservative_normalization_revision, + tokenization_status: payload.tokenization_status, + tokenization_issues: &payload.tokenization_issues, + conservative_tokens, + rename_normalization_revision: payload.rename_normalization_revision, + rename_coverage: payload.rename_coverage, + rename_issues: &payload.rename_issues, + rename_tokens, + }); + } + Ok(Self { + language, + snapshot_digest, + strings: strings.strings, + bodies: rows, + }) + } +} + +impl PersistedCloneBodiesV1 { + pub(super) fn expand( + self, + authority: &ReceiptBoundCodeFileAuthorityV1, + extraction: &ExtractionBatchV1, + ) -> Result, CodeIndexProductionErrorV1> { + let Self { + language, + snapshot_digest, + strings, + bodies, + } = self; + bodies + .into_iter() + .map(|body| { + let conservative_tokens: Arc<[ConservativeCloneTokenV1]> = + decode_tokens(&body.conservative_tokens, &strings)?.into(); + let rename_tokens = match body.rename_tokens { + None => None, + Some(PersistedRenameTokensV1::Conservative) => { + Some(Arc::clone(&conservative_tokens)) + } + Some(PersistedRenameTokensV1::Renamed(renamed)) => { + let mut tokens = conservative_tokens.to_vec(); + for [position, text] in renamed { + let Some(ConservativeCloneTokenV1::Syntax { text: slot, .. }) = + usize::try_from(position) + .ok() + .and_then(|position| tokens.get_mut(position)) + else { + return Err(contract( + "sealed clone rename renames a position that is not a syntax token", + )); + }; + string(&strings, text)?.clone_into(slot); + } + Some(tokens.into()) + } + Some(PersistedRenameTokensV1::Tokens(codes)) => { + Some(decode_tokens(&codes, &strings)?.into()) + } + }; + let payload = CloneBodyPayloadV1::from_parts(CloneBodyPayloadPartsV1 { + language: body.language.or_else(|| language.clone()).ok_or_else(|| { + contract("sealed clone body omits its language without a file default") + })?, + symbol_kind: body.symbol_kind, + token_count: body.token_count, + conservative_normalization_revision: body.conservative_normalization_revision, + conservative_tokens, + tokenization_status: body.tokenization_status, + tokenization_issues: body.tokenization_issues, + rename_normalization_revision: body.rename_normalization_revision, + rename_tokens, + rename_coverage: body.rename_coverage, + rename_issues: body.rename_issues, + }) + .map_err(CodeIndexProductionErrorV1::Contract)?; + Ok(CodeIndexCloneBodyV1 { + occurrence: CloneBodyOccurrenceV1 { + project_id: authority.project_id.clone(), + repository_id: authority.repository_id.clone(), + worktree_id: authority.worktree_id.clone(), + source_generation: extraction.generation_id.clone(), + snapshot_digest: body + .snapshot_digest + .or_else(|| snapshot_digest.clone()) + .ok_or_else(|| { + contract( + "sealed clone body omits its snapshot digest without a file default", + ) + })?, + symbol_occurrence_id: body.symbol_occurrence_id, + path: authority.logical_path.clone(), + body_span: body.body_span, + payload_digest: payload.payload_digest.clone(), + eligibility: body.eligibility, + }, + payload: Arc::new(payload), + }) + }) + .collect() + } +} + +#[derive(Default)] +struct StringTableV1<'a> { + strings: Vec<&'a str>, + index: HashMap<&'a str, u32>, +} + +impl<'a> StringTableV1<'a> { + fn intern(&mut self, value: &'a str) -> Result { + if let Some(index) = self.index.get(value) { + return Ok(*index); + } + let index = u32::try_from(self.strings.len()) + .map_err(|_| contract("sealed clone string table exceeds u32"))?; + self.strings.push(value); + self.index.insert(value, index); + Ok(index) + } + + fn code(&mut self, kind: &'a str, tag: u32) -> Result { + self.intern(kind)? + .checked_mul(4) + .and_then(|code| code.checked_add(tag + 1)) + .ok_or_else(|| contract("sealed clone token code exceeds u32")) + } + + fn encode( + &mut self, + tokens: &'a [ConservativeCloneTokenV1], + ) -> Result, CodeIndexProductionErrorV1> { + let mut codes = Vec::with_capacity(tokens.len()); + let mut open = Vec::new(); + for token in tokens { + match token { + ConservativeCloneTokenV1::StructureStart { syntax_kind } => { + open.push(syntax_kind.as_ref()); + codes.push(self.code(syntax_kind, TAG_START)?); + } + ConservativeCloneTokenV1::StructureEnd { syntax_kind } => { + if open.last() == Some(&syntax_kind.as_ref()) { + open.pop(); + codes.push(CLOSE_INNERMOST); + } else { + codes.push(self.code(syntax_kind, TAG_END)?); + } + } + ConservativeCloneTokenV1::Syntax { syntax_kind, text } => { + if text == syntax_kind { + codes.push(self.code(syntax_kind, TAG_SYNTAX_KIND_TEXT)?); + } else { + codes.push(self.code(syntax_kind, TAG_SYNTAX)?); + codes.push(self.intern(text)?); + } + } + } + } + Ok(codes) + } + + fn encode_rename( + &mut self, + conservative: &'a [ConservativeCloneTokenV1], + rename: &'a [ConservativeCloneTokenV1], + ) -> Result { + if rename == conservative { + return Ok(PersistedRenameTokensV1::Conservative); + } + if rename.len() != conservative.len() { + return self.encode(rename).map(PersistedRenameTokensV1::Tokens); + } + let mut renamed = Vec::new(); + for (position, (left, right)) in conservative.iter().zip(rename).enumerate() { + match (left, right) { + _ if left == right => {} + ( + ConservativeCloneTokenV1::Syntax { + syntax_kind: left_kind, + .. + }, + ConservativeCloneTokenV1::Syntax { syntax_kind, text }, + ) if left_kind == syntax_kind => { + let position = u32::try_from(position) + .map_err(|_| contract("sealed clone rename position exceeds u32"))?; + renamed.push([position, self.intern(text)?]); + } + _ => return self.encode(rename).map(PersistedRenameTokensV1::Tokens), + } + } + Ok(PersistedRenameTokensV1::Renamed(renamed)) + } +} + +fn string(strings: &[String], index: u32) -> Result<&str, CodeIndexProductionErrorV1> { + usize::try_from(index) + .ok() + .and_then(|index| strings.get(index)) + .map(String::as_str) + .ok_or_else(|| contract("sealed clone token names a string outside its table")) +} + +fn decode_tokens( + codes: &[u32], + strings: &[String], +) -> Result, CodeIndexProductionErrorV1> { + let owned = |kind: &str| Cow::Owned(kind.to_owned()); + let mut tokens = Vec::with_capacity(codes.len()); + let mut open = Vec::new(); + let mut codes = codes.iter().copied(); + while let Some(code) = codes.next() { + let Some(value) = code.checked_sub(1) else { + let kind = open.pop().ok_or_else(|| { + contract("sealed clone token stream closes a structure it never opened") + })?; + tokens.push(ConservativeCloneTokenV1::StructureEnd { + syntax_kind: owned(kind), + }); + continue; + }; + let kind = string(strings, value >> 2)?; + tokens.push(match value & 3 { + TAG_START => { + open.push(kind); + ConservativeCloneTokenV1::StructureStart { + syntax_kind: owned(kind), + } + } + TAG_END => ConservativeCloneTokenV1::StructureEnd { + syntax_kind: owned(kind), + }, + TAG_SYNTAX_KIND_TEXT => ConservativeCloneTokenV1::Syntax { + syntax_kind: owned(kind), + text: kind.to_owned(), + }, + _ => { + let text = codes + .next() + .ok_or_else(|| contract("sealed clone syntax token is missing its text"))?; + ConservativeCloneTokenV1::Syntax { + syntax_kind: owned(kind), + text: string(strings, text)?.to_owned(), + } + } + }); + } + Ok(tokens) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn start(kind: &'static str) -> ConservativeCloneTokenV1 { + ConservativeCloneTokenV1::StructureStart { + syntax_kind: Cow::Borrowed(kind), + } + } + + fn end(kind: &'static str) -> ConservativeCloneTokenV1 { + ConservativeCloneTokenV1::StructureEnd { + syntax_kind: Cow::Borrowed(kind), + } + } + + fn syntax(kind: &'static str, text: &str) -> ConservativeCloneTokenV1 { + ConservativeCloneTokenV1::Syntax { + syntax_kind: Cow::Borrowed(kind), + text: text.to_owned(), + } + } + + fn round_trip(tokens: &[ConservativeCloneTokenV1]) -> (Vec, Vec) { + let mut table = StringTableV1::default(); + let codes = table.encode(tokens).expect("encode"); + let strings = table + .strings + .iter() + .map(|value| (*value).to_owned()) + .collect::>(); + assert_eq!( + decode_tokens(&codes, &strings).expect("decode"), + tokens, + "every token stream must restore exactly" + ); + (codes, strings) + } + + #[test] + fn token_streams_restore_exactly_without_storing_kind_equal_text() { + let tokens = vec![ + start("block"), + start("let_declaration"), + syntax("let", "let"), + syntax("identifier", "value"), + syntax("=", "="), + syntax("identifier", "value"), + end("let_declaration"), + end("block"), + ]; + let (codes, strings) = round_trip(&tokens); + assert_eq!( + strings, + [ + "block", + "let_declaration", + "let", + "identifier", + "value", + "=" + ], + "each kind and text is interned once" + ); + assert_eq!( + codes + .iter() + .filter(|code| **code == CLOSE_INNERMOST) + .count(), + 2 + ); + assert_eq!( + codes.len(), + tokens.len() + 2, + "only differing texts add a code" + ); + } + + #[test] + fn unbalanced_structure_markers_restore_exactly() { + round_trip(&[ + end("orphan"), + start("outer"), + start("inner"), + end("outer"), + end("inner"), + syntax("identifier", "tail"), + ]); + } + + #[test] + fn damaged_streams_are_refused() { + let strings = vec!["identifier".to_owned()]; + assert!(decode_tokens(&[CLOSE_INNERMOST], &strings).is_err()); + assert!(decode_tokens(&[1 + TAG_SYNTAX], &strings).is_err()); + assert!(decode_tokens(&[1 + (5 << 2)], &strings).is_err()); + } + + #[test] + fn rename_streams_keep_only_renamed_positions() { + let conservative = vec![ + syntax("identifier", "input"), + syntax("(", "("), + syntax("identifier", "input"), + ]; + let mut table = StringTableV1::default(); + assert_eq!( + table + .encode_rename(&conservative, &conservative) + .expect("rename"), + PersistedRenameTokensV1::Conservative + ); + let renamed = vec![ + syntax("identifier", "$0"), + syntax("(", "("), + syntax("identifier", "$0"), + ]; + let PersistedRenameTokensV1::Renamed(pairs) = table + .encode_rename(&conservative, &renamed) + .expect("rename") + else { + panic!("an aligned rename keeps only its renamed positions"); + }; + assert_eq!( + pairs + .iter() + .map(|[position, _]| *position) + .collect::>(), + [0, 2] + ); + let misaligned = vec![syntax("(", "("), syntax("identifier", "$0")]; + assert!(matches!( + table + .encode_rename(&conservative, &misaligned) + .expect("rename"), + PersistedRenameTokensV1::Tokens(_) + )); + } +} diff --git a/crates/tracedecay-code-index/src/production/helpers.rs b/crates/tracedecay-code-index/src/production/helpers.rs index cf6500eb00..680ee083a7 100644 --- a/crates/tracedecay-code-index/src/production/helpers.rs +++ b/crates/tracedecay-code-index/src/production/helpers.rs @@ -267,7 +267,6 @@ pub(crate) fn coverage_summary( CodeSearchEligibilityV1::Eligible => {} CodeSearchEligibilityV1::Excluded { .. } => coverage.files_excluded += 1, CodeSearchEligibilityV1::Partial { .. } => coverage.files_partial += 1, - CodeSearchEligibilityV1::Unsupported { .. } => coverage.files_unsupported += 1, } } coverage @@ -1955,11 +1954,15 @@ mod tests { #[test] fn rust_crate_qualified_names_stay_inside_their_cargo_source_root() { - let call = RustExtractor.extract( - "src/alpha/mod.rs", - "pub fn run() -> i32 { crate::beta::run() }", - ); - let target = RustExtractor.extract("src/beta/mod.rs", "pub fn run() -> i32 { 1 }"); + let call = RustExtractor + .extract_artifact( + "src/alpha/mod.rs", + "pub fn run() -> i32 { crate::beta::run() }", + ) + .result; + let target = RustExtractor + .extract_artifact("src/beta/mod.rs", "pub fn run() -> i32 { 1 }") + .result; assert!( call.unresolved_refs .iter() diff --git a/crates/tracedecay-code-index/src/production/lexical_page_source.rs b/crates/tracedecay-code-index/src/production/lexical_page_source.rs index e587d326a2..8ccf1c3e4c 100644 --- a/crates/tracedecay-code-index/src/production/lexical_page_source.rs +++ b/crates/tracedecay-code-index/src/production/lexical_page_source.rs @@ -1,9 +1,4 @@ -use std::{ - collections::BTreeMap, - io::{Read, Seek, SeekFrom}, - num::NonZeroUsize, - sync::Arc, -}; +use std::{collections::BTreeMap, num::NonZeroUsize, sync::Arc}; use sha2::{Digest, Sha256}; use tracedecay_domain::{ @@ -17,10 +12,6 @@ use crate::{ }; use super::partitioned_codec::PartitionedLexicalFileSourceV1; -use super::sealed_codec::{ - MINIMUM_SEALED_GENERATION_FORMAT_REVISION, MONOLITHIC_SEALED_GENERATION_FORMAT_REVISION, - PersistedFileGenerationArtifactsV1, superseded_sealed_generation_revision, -}; use super::{FileGenerationArtifactsV1, *}; const PAGE_DIGEST_DOMAIN: &[u8] = b"tracedecay.sealed-lexical-page.v1\0"; @@ -37,15 +28,13 @@ const IMPORT_DICTIONARY_CHAIN_RECORD_DOMAIN: &[u8] = const CURSOR_DIGEST_DOMAIN: &[u8] = b"tracedecay.sealed-lexical-cursor.v1\0"; const INVALID_CURSOR_POSITION_DETAIL: &str = "sealed lexical cursor is not a valid position in its next file"; -const LAYOUT_PROGRESS_INTERVAL_BYTES: u64 = 16 * 1024 * 1024; -const MAX_LEXICAL_GENERATION_METADATA_BYTES: u64 = 64 * 1024 * 1024; /// Concurrent exact-read/decode window. Same 64 MiB retain cap as one /// lexical page batch, so prefetch cannot exceed a window the builder /// already admits for staged pages. /// Bound on the sealed file bytes read ahead of one decode window: the /// lexical source's admitted-file prefetch and the partitioned decoder's -/// segment window share it so neither holds more than this in raw segment -/// bytes while the pool decodes them. +/// segment window share it so neither holds more than this in decoded +/// segment bytes while the pool decodes them. pub(super) const LEXICAL_FILE_PREFETCH_BYTES_V1: u64 = 64 * 1024 * 1024; /// Files admitted per worker in one restore window, mirroring the encode /// side's `SEALED_ENCODE_WINDOW_FILES_PER_WORKER_V1`. A bare `workers`-sized @@ -1124,22 +1113,20 @@ enum StagedSealedLexicalPageBatchReadV1 { }, } -/// Seekable, bounded lexical projection source over a verified v5/v6 seal. +/// Bounded lexical projection source over an authenticated partitioned +/// sealed generation. /// -/// Opening performs a streaming structural scan and verifies the exact raw -/// generation digest. Layout records every file byte range so source_scan -/// can exact-read and decode on the indexing pool instead of walking the -/// files array a second time one byte at a time. Page minting stays serial -/// because the cumulative digest is a chain. Raw sealed bytes never cross -/// this interface. +/// Opening authenticates the manifest; file segments are read and verified +/// one bounded admission window at a time and decoded on the indexing pool. +/// File ranges are file ordinals. Page minting stays serial because the +/// cumulative digest is a chain. Raw sealed bytes never cross this interface. #[derive(Debug)] -pub struct VerifiedSealedLexicalPageSourceV1 { - reader: R, +pub struct VerifiedSealedLexicalPageSourceV1 { file_count: u64, first_file_offset: u64, files_end_offset: u64, file_ranges: Vec<(u64, u64)>, - partitioned_lexical_byte_offsets: Option>, + lexical_byte_offsets: Vec, total_lexical_units: u64, maximum_file_bytes: u64, source_state_digest: ManifestDigest, @@ -1149,29 +1136,22 @@ pub struct VerifiedSealedLexicalPageSourceV1 { maximum_page_bytes: usize, cursor: VerifiedSealedLexicalCursorV1, admitted_window: BTreeMap>, - /// Durable partitioned descriptors or same-process published file authority. - /// Partitioned sources load only the next bounded admission window; their - /// cursors retain stable file ordinals across process restarts. - file_source: Option, -} - -#[derive(Debug)] -pub(super) enum SealedLexicalFilesV1 { - Published(Vec>), - Partitioned(PartitionedLexicalFileSourceV1), + /// Durable partitioned descriptors. Only the next bounded admission + /// window is loaded; cursors retain stable file ordinals across process + /// restarts. + file_source: PartitionedLexicalFileSourceV1, } /// Authenticated generation metadata needed by exact and lexical serving. /// /// The full sealed generation can be gigabytes. This projection retains only -/// the manifest and sanitized snapshot header that precede the files array; -/// the layout scan authenticates the complete content-addressed seal before -/// this value becomes observable. +/// the manifest, sanitized snapshot, and statistics the partitioned manifest +/// carries; the manifest is authenticated before this value is observable. #[derive(Clone, Debug)] pub struct VerifiedSealedTextGenerationMetadataV1 { manifest: CodeGenerationManifestV1, snapshot: SanitizedCodeSnapshotV1, - statistics: Option, + statistics: CodeIndexGenerationStatisticsV1, } impl VerifiedSealedTextGenerationMetadataV1 { @@ -1179,14 +1159,14 @@ impl VerifiedSealedTextGenerationMetadataV1 { Self { manifest: generation.manifest().clone(), snapshot: generation.snapshot().clone(), - statistics: Some(generation.statistics.clone()), + statistics: generation.statistics.clone(), } } pub(super) fn from_partitioned_manifest( manifest: CodeGenerationManifestV1, snapshot: SanitizedCodeSnapshotV1, - statistics: Option, + statistics: CodeIndexGenerationStatisticsV1, ) -> Result { if manifest.source_commitments.is_none() { return Err(CodeIndexProductionErrorV1::SourceCommitmentsUnavailable); @@ -1239,16 +1219,28 @@ impl VerifiedSealedTextGenerationMetadataV1 { &self.snapshot } - pub fn generation_statistics(&self) -> Option<&CodeIndexGenerationStatisticsV1> { - self.statistics.as_ref() + pub fn generation_statistics(&self) -> &CodeIndexGenerationStatisticsV1 { + &self.statistics } } -impl VerifiedSealedLexicalPageSourceV1 { +impl VerifiedSealedLexicalPageSourceV1 { + /// The content key of what this source emits: its format and the + /// content-only identity of every file it reads. Two sources with one + /// key emit the same records apart from route identity, which a lexical + /// artifact keeps out of its rows. + pub fn content_key(&self) -> Result { + let mut hasher = Sha256::new(); + hasher.update(b"tracedecay.sealed-lexical-source-content.v1\0"); + hasher.update(self.format_revision.to_le_bytes()); + self.file_source.content_digest(&mut hasher); + ManifestDigest::from_sha256_bytes(&hasher.finalize()) + .map_err(|error| CodeIndexProductionErrorV1::Contract(error.to_string())) + } + // Every argument is a distinct authority the constructor binds together - // exactly once: the reader, the manifest, the sanitized snapshot, the - // optional statistics, the partitioned file source, its state digest, and - // the two page bounds. Grouping any of them into a parameter struct would + // exactly once: the manifest, the sanitized snapshot, the statistics, the + // partitioned file source, its state digest, and the two page bounds. Grouping any of them into a parameter struct would // invent a type with one construction site and hide which authority a // caller failed to supply. #[allow( @@ -1256,10 +1248,9 @@ impl VerifiedSealedLexicalPageSourceV1 { reason = "each argument is a separate authority bound once at construction" )] pub(super) fn open_partitioned_parts( - reader: R, manifest: CodeGenerationManifestV1, snapshot: SanitizedCodeSnapshotV1, - statistics: Option, + statistics: CodeIndexGenerationStatisticsV1, source: PartitionedLexicalFileSourceV1, source_state_digest: ManifestDigest, maximum_page_chunks: usize, @@ -1281,24 +1272,20 @@ impl VerifiedSealedLexicalPageSourceV1 { let file_ranges = (0..file_count) .map(|file| (file, file.saturating_add(1))) .collect::>(); - let partitioned_lexical_byte_offsets = source.lexical_byte_offsets()?; - let total_lexical_units = partitioned_lexical_byte_offsets - .last() - .copied() - .ok_or_else(|| { - CodeIndexProductionErrorV1::Contract( - "partitioned lexical byte offsets are empty".to_owned(), - ) - })?; + let lexical_byte_offsets = source.lexical_byte_offsets()?; + let total_lexical_units = lexical_byte_offsets.last().copied().ok_or_else(|| { + CodeIndexProductionErrorV1::Contract( + "partitioned lexical byte offsets are empty".to_owned(), + ) + })?; let maximum_file_bytes = source.maximum_file_bytes(); let cursor = VerifiedSealedLexicalCursorV1::initial(source_state_digest.clone(), 0)?; Ok(Self { - reader, file_count, first_file_offset: 0, files_end_offset: file_count, file_ranges, - partitioned_lexical_byte_offsets: Some(partitioned_lexical_byte_offsets), + lexical_byte_offsets, total_lexical_units, maximum_file_bytes, source_state_digest, @@ -1308,153 +1295,7 @@ impl VerifiedSealedLexicalPageSourceV1 { maximum_page_bytes, cursor, admitted_window: BTreeMap::new(), - file_source: Some(SealedLexicalFilesV1::Partitioned(source)), - }) - } - - #[hotpath::measure(label = "code_index.restore.open")] - pub fn open( - mut reader: R, - admitted_len: u64, - expected_state_digest: ManifestDigest, - maximum_page_chunks: usize, - maximum_page_bytes: usize, - control: &dyn CodeIndexExecutionControlV1, - ) -> Result { - if maximum_page_chunks == 0 || maximum_page_bytes == 0 { - return Err(CodeIndexProductionErrorV1::Contract( - "sealed lexical page bounds must be non-zero".to_owned(), - )); - } - let layout = scan_layout(&mut reader, admitted_len, None, control)?; - if layout.state_digest != expected_state_digest { - return Err(CodeIndexProductionErrorV1::Contract( - "sealed generation state digest does not match the admitted source".to_owned(), - )); - } - let cursor = VerifiedSealedLexicalCursorV1::initial( - layout.state_digest.clone(), - layout.first_file_offset, - )?; - let total_lexical_units = layout - .files_end_offset - .checked_sub(layout.first_file_offset) - .ok_or_else(|| { - CodeIndexProductionErrorV1::Contract( - "sealed lexical files array has an invalid byte span".to_owned(), - ) - })?; - let metadata = read_verified_text_metadata(&mut reader, &layout, control)?; - Ok(Self { - reader, - file_count: layout.file_count, - first_file_offset: layout.first_file_offset, - files_end_offset: layout.files_end_offset, - file_ranges: layout.file_ranges, - partitioned_lexical_byte_offsets: None, - total_lexical_units, - maximum_file_bytes: layout.maximum_file_bytes, - source_state_digest: layout.state_digest, - format_revision: layout.format_revision, - metadata, - maximum_page_chunks, - maximum_page_bytes, - cursor, - admitted_window: BTreeMap::new(), - file_source: None, - }) - } - - /// Open a durable sealed source through its content address. - /// - /// Unlike [`Self::open`], whose caller already holds the envelope's inner - /// state digest, this journey binds the complete file bytes to the digest - /// in the durable generation index while the same bounded scan discovers - /// the lexical layout. The caller can therefore pass a `File` directly; - /// no whole-generation `Vec` is required merely to authenticate it. - #[hotpath::measure(label = "code_index.restore.open_content_addressed")] - pub fn open_content_addressed( - reader: R, - admitted_len: u64, - expected_file_digest: ManifestDigest, - maximum_page_chunks: usize, - maximum_page_bytes: usize, - control: &dyn CodeIndexExecutionControlV1, - ) -> Result { - if maximum_page_chunks == 0 || maximum_page_bytes == 0 { - return Err(CodeIndexProductionErrorV1::Contract( - "sealed lexical page bounds must be non-zero".to_owned(), - )); - } - Self::open_content_addressed_with_progress( - reader, - admitted_len, - expected_file_digest, - maximum_page_chunks, - maximum_page_bytes, - control, - |_, _| {}, - ) - } - - /// Open a content-addressed source while reporting authenticated scan - /// bytes. The callback is invoked at zero, bounded byte intervals, and - /// exactly once with the admitted total before metadata is exposed. - #[hotpath::measure(label = "code_index.restore.open_content_addressed")] - pub fn open_content_addressed_with_progress( - mut reader: R, - admitted_len: u64, - expected_file_digest: ManifestDigest, - maximum_page_chunks: usize, - maximum_page_bytes: usize, - control: &dyn CodeIndexExecutionControlV1, - mut progress: F, - ) -> Result - where - F: FnMut(u64, u64), - { - if maximum_page_chunks == 0 || maximum_page_bytes == 0 { - return Err(CodeIndexProductionErrorV1::Contract( - "sealed lexical page bounds must be non-zero".to_owned(), - )); - } - let layout = scan_layout_with_progress( - &mut reader, - admitted_len, - Some(&expected_file_digest), - control, - &mut progress, - )?; - let cursor = VerifiedSealedLexicalCursorV1::initial( - layout.state_digest.clone(), - layout.first_file_offset, - )?; - let total_lexical_units = layout - .files_end_offset - .checked_sub(layout.first_file_offset) - .ok_or_else(|| { - CodeIndexProductionErrorV1::Contract( - "sealed lexical files array has an invalid byte span".to_owned(), - ) - })?; - let metadata = read_verified_text_metadata(&mut reader, &layout, control)?; - Ok(Self { - reader, - file_count: layout.file_count, - first_file_offset: layout.first_file_offset, - files_end_offset: layout.files_end_offset, - file_ranges: layout.file_ranges, - partitioned_lexical_byte_offsets: None, - total_lexical_units, - maximum_file_bytes: layout.maximum_file_bytes, - source_state_digest: layout.state_digest, - format_revision: layout.format_revision, - metadata, - maximum_page_chunks, - maximum_page_bytes, - cursor, - admitted_window: BTreeMap::new(), - file_source: None, + file_source: source, }) } @@ -1466,68 +1307,6 @@ impl VerifiedSealedLexicalPageSourceV1 { self.format_revision } - /// Admit later pages from an already-decoded published generation. - /// - /// The sealed file remains the layout and cursor authority. This only - /// replaces per-file JSON decode when the in-memory files match the - /// scanned ranges one-for-one. Partitioned sources validate the supplied - /// identity but keep their bounded durable reader, avoiding retention of - /// the complete decoded generation. Mismatches are rejected. - pub fn attach_published_files( - &mut self, - generation: &CodeIndexPublishedGenerationV1, - ) -> Result<(), CodeIndexProductionErrorV1> { - if generation.manifest() != self.metadata.manifest() - || generation.snapshot() != self.metadata.snapshot() - { - return Err(CodeIndexProductionErrorV1::Contract( - "published generation does not match the authenticated sealed lexical source" - .to_owned(), - )); - } - if generation.files.len() != self.file_ranges.len() { - return Err(CodeIndexProductionErrorV1::Contract( - "published generation file count does not match the sealed lexical layout" - .to_owned(), - )); - } - // Partitioned readers keep bounded durable file authority after the - // supplied generation's identity has been checked above. - if matches!(self.file_source, Some(SealedLexicalFilesV1::Partitioned(_))) { - return Ok(()); - } - generation.validate()?; - self.file_source = Some(SealedLexicalFilesV1::Published(generation.files.clone())); - Ok(()) - } - - /// Reopen an authenticated durable source at an accepted persisted cursor. - /// - /// The layout scan authenticates the raw content address but does not - /// deserialize file artifacts. Resume validates only the cursor's next - /// artifact and emits that page first; previously admitted artifacts are - /// never decoded or replayed on the reopen path. - pub fn open_content_addressed_at( - reader: R, - admitted_len: u64, - expected_file_digest: ManifestDigest, - cursor: VerifiedSealedLexicalCursorV1, - maximum_page_chunks: usize, - maximum_page_bytes: usize, - control: &dyn CodeIndexExecutionControlV1, - ) -> Result { - let mut source = Self::open_content_addressed( - reader, - admitted_len, - expected_file_digest, - maximum_page_chunks, - maximum_page_bytes, - control, - )?; - source.restore_cursor(&cursor, control)?; - Ok(source) - } - /// Adopt a persisted cursor after binding it to this source and validating /// its first unread file. This deliberately never walks earlier files. pub fn restore_cursor( @@ -1647,24 +1426,17 @@ impl VerifiedSealedLexicalPageSourceV1 { /// cursor. A partially consumed file counts only after its final chunk and /// imports are committed, matching `completed_files`. pub fn completed_lexical_units(&self) -> Result { - if let Some(offsets) = &self.partitioned_lexical_byte_offsets { - let completed = usize::try_from(self.cursor.next_file_ordinal()).map_err(|_| { - CodeIndexProductionErrorV1::Contract( - "sealed lexical completed file count exceeds usize".to_owned(), - ) - })?; - return offsets.get(completed).copied().ok_or_else(|| { - CodeIndexProductionErrorV1::Contract( - "sealed lexical cursor exceeds partitioned byte bounds".to_owned(), - ) - }); - } - self.cursor - .next_file_offset - .checked_sub(self.first_file_offset) + let completed = usize::try_from(self.cursor.next_file_ordinal()).map_err(|_| { + CodeIndexProductionErrorV1::Contract( + "sealed lexical completed file count exceeds usize".to_owned(), + ) + })?; + self.lexical_byte_offsets + .get(completed) + .copied() .ok_or_else(|| { CodeIndexProductionErrorV1::Contract( - "sealed lexical cursor precedes the files-array start".to_owned(), + "sealed lexical cursor exceeds partitioned byte bounds".to_owned(), ) }) } @@ -1721,13 +1493,7 @@ impl VerifiedSealedLexicalPageSourceV1 { /// Compact retained file positions and partitioned content identities. /// These scale with file count; decoded chunks only occupy the admission window. pub fn retained_layout_bytes(&self) -> usize { - let source_bytes = match &self.file_source { - Some(SealedLexicalFilesV1::Published(files)) => files - .capacity() - .saturating_mul(std::mem::size_of::>()), - Some(SealedLexicalFilesV1::Partitioned(source)) => source.retained_layout_bytes(), - None => 0, - }; + let source_bytes = self.file_source.retained_layout_bytes(); std::mem::size_of::() .saturating_mul(4) .saturating_add( @@ -1736,13 +1502,9 @@ impl VerifiedSealedLexicalPageSourceV1 { .saturating_mul(std::mem::size_of::<(u64, u64)>()), ) .saturating_add( - self.partitioned_lexical_byte_offsets - .as_ref() - .map_or(0, |offsets| { - offsets - .capacity() - .saturating_mul(std::mem::size_of::()) - }), + self.lexical_byte_offsets + .capacity() + .saturating_mul(std::mem::size_of::()), ) .saturating_add(source_bytes) } @@ -2210,147 +1972,18 @@ impl VerifiedSealedLexicalPageSourceV1 { // A rejected batch or restored cursor may revisit a range before the // current prefetch window. Keep one window, including during retries. self.admitted_window.clear(); - match self.file_source { - Some(SealedLexicalFilesV1::Published(_)) => { - self.fill_admitted_window_from_memory(file_offset, control) - } - Some(SealedLexicalFilesV1::Partitioned(_)) => { - self.fill_admitted_window_from_segments(file_offset, control) - } - None => self.fill_admitted_window(file_offset, control), - } - } - - fn fill_admitted_window( - &mut self, - file_offset: u64, - control: &dyn CodeIndexExecutionControlV1, - ) -> Result<(), CodeIndexProductionErrorV1> { - let snapshot_digest = self.metadata.manifest().snapshot_digest.clone(); - let start_index = self.file_range_index(file_offset)?; - let workers = crate::parallelism::indexing_workers().max(1); - let window_files = workers.saturating_mul(LEXICAL_DECODE_WINDOW_FILES_PER_WORKER_V1); - let mut prefetch_bytes = 0u64; - let mut inputs = Vec::new(); - for (index, &(start, end)) in self.file_ranges[start_index..].iter().enumerate() { - let file_bytes = end.checked_sub(start).ok_or_else(|| { - CodeIndexProductionErrorV1::Contract( - "sealed lexical file byte range is invalid".to_owned(), - ) - })?; - if index > 0 - && (inputs.len() >= window_files - || prefetch_bytes - .checked_add(file_bytes) - .is_some_and(|total| total > LEXICAL_FILE_PREFETCH_BYTES_V1)) - { - break; - } - checkpoint(control)?; - let bytes = read_file_bytes_at_range( - &mut self.reader, - start, - end, - self.files_end_offset, - self.maximum_file_bytes, - control, - )?; - let next_file_offset = self - .file_ranges - .get(start_index + index + 1) - .map(|(next_start, _)| *next_start) - .unwrap_or(self.files_end_offset); - prefetch_bytes = prefetch_bytes.saturating_add(file_bytes); - inputs.push((start, bytes, next_file_offset)); - } - if inputs.is_empty() { - return Err(CodeIndexProductionErrorV1::Contract( - "sealed lexical file range produced no readable files".to_owned(), - )); - } - let admitted = - super::collect_bounded_ordered(&inputs, |(_start, bytes, next_offset), _| { - admit_persisted_file_bytes(bytes, &snapshot_digest, *next_offset, control) - })?; - for ((start, _, _), admitted) in inputs.into_iter().zip(admitted) { - self.admitted_window.insert(start, Arc::new(admitted)); - } - Ok(()) - } - - fn fill_admitted_window_from_memory( - &mut self, - file_offset: u64, - control: &dyn CodeIndexExecutionControlV1, - ) -> Result<(), CodeIndexProductionErrorV1> { - let snapshot_digest = self.metadata.manifest().snapshot_digest.clone(); - let Some(SealedLexicalFilesV1::Published(files)) = self.file_source.as_ref() else { - return Err(CodeIndexProductionErrorV1::Contract( - "sealed lexical memory admit ran without published files".to_owned(), - )); - }; - let start_index = self.file_range_index(file_offset)?; - let workers = crate::parallelism::indexing_workers().max(1); - let window_files = workers.saturating_mul(LEXICAL_DECODE_WINDOW_FILES_PER_WORKER_V1); - let mut prefetch_bytes = 0u64; - let mut inputs = Vec::new(); - for (index, file) in files[start_index..].iter().enumerate() { - let &(start, end) = self.file_ranges.get(start_index + index).ok_or_else(|| { - CodeIndexProductionErrorV1::Contract( - "published generation file is missing a sealed lexical range".to_owned(), - ) - })?; - let file_bytes = end.checked_sub(start).ok_or_else(|| { - CodeIndexProductionErrorV1::Contract( - "sealed lexical file byte range is invalid".to_owned(), - ) - })?; - if index > 0 - && (inputs.len() >= window_files - || prefetch_bytes - .checked_add(file_bytes) - .is_some_and(|total| total > LEXICAL_FILE_PREFETCH_BYTES_V1)) - { - break; - } - checkpoint(control)?; - let next_file_offset = self - .file_ranges - .get(start_index + index + 1) - .map(|(next_start, _)| *next_start) - .unwrap_or(self.files_end_offset); - prefetch_bytes = prefetch_bytes.saturating_add(file_bytes); - inputs.push((start, Arc::clone(file), next_file_offset)); - } - if inputs.is_empty() { - return Err(CodeIndexProductionErrorV1::Contract( - "sealed lexical memory range produced no published files".to_owned(), - )); - } - let admitted = - super::collect_bounded_ordered(&inputs, |(_start, file, next_offset), _| { - admit_file_generation_artifacts(file, &snapshot_digest, *next_offset, control) - })?; - for ((start, _, _), admitted) in inputs.into_iter().zip(admitted) { - self.admitted_window.insert(start, Arc::new(admitted)); - } - Ok(()) + self.fill_admitted_window(file_offset, control) } #[hotpath::measure(label = "code_index.restore.partitioned_window")] - fn fill_admitted_window_from_segments( + fn fill_admitted_window( &mut self, file_offset: u64, control: &dyn CodeIndexExecutionControlV1, ) -> Result<(), CodeIndexProductionErrorV1> { let snapshot_digest = self.metadata.manifest().snapshot_digest.clone(); let start = self.file_range_index(file_offset)?; - let Some(SealedLexicalFilesV1::Partitioned(source)) = self.file_source.as_mut() else { - return Err(CodeIndexProductionErrorV1::Contract( - "sealed lexical segment admit ran without segment authority".to_owned(), - )); - }; - let files = source.read_window( + let files = self.file_source.read_window( start, crate::parallelism::indexing_workers() .max(1) @@ -2522,762 +2155,6 @@ struct AdmittedSealedLexicalFileV1 { next_file_offset: u64, } -pub(super) struct SealedLexicalLayoutV1 { - pub(super) state_digest: ManifestDigest, - pub(super) format_revision: u32, - file_count: u64, - first_file_offset: u64, - files_end_offset: u64, - file_ranges: Vec<(u64, u64)>, - maximum_file_bytes: u64, - manifest_range: Option<(u64, u64)>, - snapshot_range: Option<(u64, u64)>, -} - -#[hotpath::measure(label = "code_index.restore.scan")] -pub(super) fn scan_layout( - reader: &mut R, - admitted_len: u64, - expected_file_digest: Option<&ManifestDigest>, - control: &dyn CodeIndexExecutionControlV1, -) -> Result { - scan_layout_with_progress( - reader, - admitted_len, - expected_file_digest, - control, - &mut |_, _| {}, - ) -} - -fn scan_layout_with_progress( - reader: &mut R, - admitted_len: u64, - expected_file_digest: Option<&ManifestDigest>, - control: &dyn CodeIndexExecutionControlV1, - progress: &mut dyn FnMut(u64, u64), -) -> Result { - if admitted_len > MAX_SEALED_CODE_GENERATION_BYTES_V1 { - return Err(CodeIndexProductionErrorV1::Contract( - "sealed generation exceeds the canonical byte limit".to_owned(), - )); - } - reader.seek(SeekFrom::Start(0)).map_err(|error| { - CodeIndexProductionErrorV1::Contract(format!("sealed lexical source seek failed: {error}")) - })?; - hotpath::gauge!("code_index_lexical_layout_scan_attempts").inc(1); - hotpath::gauge!("code_index_lexical_layout_bytes_total").set(admitted_len); - hotpath::gauge!("code_index_lexical_layout_bytes_scanned").set(0); - progress(0, admitted_len); - let mut scanner = LayoutScanner::default(); - let mut file_hasher = expected_file_digest.map(|_| Sha256::new()); - let read_limit = admitted_len.checked_add(1).ok_or_else(|| { - CodeIndexProductionErrorV1::Contract("sealed generation length overflowed".to_owned()) - })?; - let mut remaining = read_limit; - let mut observed = 0u64; - let mut next_progress = LAYOUT_PROGRESS_INTERVAL_BYTES; - let mut buffer = [0u8; 64 * 1024]; - while remaining > 0 { - checkpoint(control)?; - let requested = usize::try_from(remaining.min(buffer.len() as u64)).map_err(|_| { - CodeIndexProductionErrorV1::Contract( - "sealed lexical read window exceeds the platform limit".to_owned(), - ) - })?; - let read = reader.read(&mut buffer[..requested]).map_err(|error| { - CodeIndexProductionErrorV1::Contract(format!( - "sealed lexical source read failed: {error}" - )) - })?; - if read == 0 { - break; - } - let read_bytes = u64::try_from(read).map_err(|_| { - CodeIndexProductionErrorV1::Contract( - "sealed lexical source read exceeds u64".to_owned(), - ) - })?; - // Only bytes below the admitted length are hashed and scanned; split - // the buffer at that boundary and feed whole slices, not single bytes. - let admitted = usize::try_from(read_bytes.min(admitted_len.saturating_sub(observed))) - .map_err(|_| { - CodeIndexProductionErrorV1::Contract( - "sealed lexical read window exceeds the platform limit".to_owned(), - ) - })?; - if admitted > 0 { - if let Some(hasher) = file_hasher.as_mut() { - hasher.update(&buffer[..admitted]); - } - scanner.observe_slice(&buffer[..admitted], observed)?; - } - observed = observed.checked_add(read_bytes).ok_or_else(|| { - CodeIndexProductionErrorV1::Contract( - "sealed lexical source length overflowed".to_owned(), - ) - })?; - let admitted_observed = observed.min(admitted_len); - if admitted_observed >= next_progress || admitted_observed == admitted_len { - hotpath::gauge!("code_index_lexical_layout_bytes_scanned").set(admitted_observed); - progress(admitted_observed, admitted_len); - next_progress = admitted_observed.saturating_add(LAYOUT_PROGRESS_INTERVAL_BYTES); - } - remaining -= read_bytes; - } - if observed != admitted_len { - return Err(CodeIndexProductionErrorV1::Contract( - "sealed generation length does not match its admitted length".to_owned(), - )); - } - if let (Some(expected), Some(hasher)) = (expected_file_digest, file_hasher) - && digest_hasher(hasher)? != *expected - { - return Err(CodeIndexProductionErrorV1::Contract( - "sealed lexical source bytes do not match their durable content address".to_owned(), - )); - } - scanner.finish() -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum LayoutKey { - StateDigest, - Generation, - Files, - FormatRevision, - Manifest, - Snapshot, -} - -impl LayoutKey { - fn from_bytes(bytes: &[u8]) -> Option { - match bytes { - b"state_digest" => Some(Self::StateDigest), - b"generation" => Some(Self::Generation), - b"files" => Some(Self::Files), - b"format_revision" => Some(Self::FormatRevision), - b"manifest" => Some(Self::Manifest), - b"snapshot" => Some(Self::Snapshot), - _ => None, - } - } -} - -struct LayoutScanner { - brace_depth: usize, - bracket_depth: usize, - in_string: bool, - escaped: bool, - string: [u8; 128], - string_len: usize, - string_overflowed: bool, - completed_key: Option, - pending_key: Option, - capture_state_digest: bool, - state_digest: Option, - format_revision: Option, - generation_depth: Option, - generation_hasher: Option, - generation_digest: Option, - files_depth: Option, - current_file_start: Option, - first_file_offset: Option, - files_end_offset: Option, - file_count: u64, - file_ranges: Vec<(u64, u64)>, - maximum_file_bytes: u64, - captured_metadata_object: Option<(LayoutKey, u64, usize)>, - manifest_range: Option<(u64, u64)>, - snapshot_range: Option<(u64, u64)>, -} - -impl Default for LayoutScanner { - fn default() -> Self { - Self { - brace_depth: 0, - bracket_depth: 0, - in_string: false, - escaped: false, - string: [0; 128], - string_len: 0, - string_overflowed: false, - completed_key: None, - pending_key: None, - capture_state_digest: false, - state_digest: None, - format_revision: None, - generation_depth: None, - generation_hasher: None, - generation_digest: None, - files_depth: None, - current_file_start: None, - first_file_offset: None, - files_end_offset: None, - file_count: 0, - file_ranges: Vec::new(), - maximum_file_bytes: 0, - captured_metadata_object: None, - manifest_range: None, - snapshot_range: None, - } - } -} - -/// Transition of the generation-payload hash span produced by one observed -/// byte. -enum GenerationSpanEvent { - None, - Opened, - Closed, -} - -impl LayoutScanner { - /// Observe one contiguous run of admitted bytes starting at `base_offset`. - /// - /// The generation hasher receives one update per contiguous in-generation - /// byte range instead of one update per byte; the hashed bytes and their - /// order are identical. - fn observe_slice( - &mut self, - bytes: &[u8], - base_offset: u64, - ) -> Result<(), CodeIndexProductionErrorV1> { - let mut active_from = self.generation_hasher.is_some().then_some(0usize); - let mut index = 0usize; - while index < bytes.len() { - if self.in_string && !self.escaped { - let relative_end = first_json_string_control(&bytes[index..]); - let end = relative_end.map_or(bytes.len(), |relative| index + relative); - if end > index { - self.observe_string_run(&bytes[index..end]); - index = end; - if index == bytes.len() { - break; - } - } - } - let offset = u64::try_from(index) - .ok() - .and_then(|index| base_offset.checked_add(index)) - .ok_or_else(|| { - CodeIndexProductionErrorV1::Contract( - "sealed lexical source length overflowed".to_owned(), - ) - })?; - match self.observe(bytes[index], offset)? { - GenerationSpanEvent::None => {} - GenerationSpanEvent::Opened => active_from = Some(index), - GenerationSpanEvent::Closed => { - let start = active_from.take().ok_or_else(|| { - CodeIndexProductionErrorV1::Contract( - "sealed generation digest state is missing".to_owned(), - ) - })?; - let mut hasher = self.generation_hasher.take().ok_or_else(|| { - CodeIndexProductionErrorV1::Contract( - "sealed generation digest state is missing".to_owned(), - ) - })?; - hasher.update(&bytes[start..=index]); - self.generation_digest = Some(digest_hasher(hasher)?); - } - } - index += 1; - } - if let Some(hasher) = self.generation_hasher.as_mut() { - let start = active_from.ok_or_else(|| { - CodeIndexProductionErrorV1::Contract( - "sealed generation digest state is missing".to_owned(), - ) - })?; - hasher.update(&bytes[start..]); - } - Ok(()) - } - - /// Consume bytes that cannot alter JSON string state in one bounded step. - /// Only a key or the envelope state digest is retained, and both are - /// capped at the scanner's existing 128-byte contract. - fn observe_string_run(&mut self, bytes: &[u8]) { - let remaining = self.string.len().saturating_sub(self.string_len); - let retained = remaining.min(bytes.len()); - let retained_end = self.string_len + retained; - self.string[self.string_len..retained_end].copy_from_slice(&bytes[..retained]); - self.string_len = retained_end; - if retained < bytes.len() { - self.string_overflowed = true; - } - } - - fn observe_string_byte(&mut self, byte: u8) { - if self.string_len < self.string.len() { - self.string[self.string_len] = byte; - self.string_len += 1; - } else { - self.string_overflowed = true; - } - } - - fn observe( - &mut self, - byte: u8, - offset: u64, - ) -> Result { - if self.in_string { - if self.escaped { - self.escaped = false; - self.observe_string_byte(byte); - return Ok(GenerationSpanEvent::None); - } - match byte { - b'\\' => self.escaped = true, - b'"' => { - self.in_string = false; - if self.capture_state_digest { - let value = String::from_utf8(self.string[..self.string_len].to_vec()) - .map_err(|_| { - CodeIndexProductionErrorV1::Contract( - "sealed generation state digest is not UTF-8".to_owned(), - ) - })?; - self.state_digest = Some(ManifestDigest::new(value).map_err(|error| { - CodeIndexProductionErrorV1::Contract(error.to_string()) - })?); - self.capture_state_digest = false; - self.pending_key = None; - } else if !self.string_overflowed { - std::str::from_utf8(&self.string[..self.string_len]).map_err(|_| { - CodeIndexProductionErrorV1::Contract( - "sealed generation key is not UTF-8".to_owned(), - ) - })?; - self.completed_key = LayoutKey::from_bytes(&self.string[..self.string_len]); - } else { - self.completed_key = None; - } - self.string_len = 0; - self.string_overflowed = false; - } - _ => self.observe_string_byte(byte), - } - return Ok(GenerationSpanEvent::None); - } - - let mut event = GenerationSpanEvent::None; - match byte { - b'"' => { - self.in_string = true; - self.string_len = 0; - self.string_overflowed = false; - self.capture_state_digest = - self.pending_key == Some(LayoutKey::StateDigest) && self.brace_depth == 1; - } - b':' => { - self.pending_key = self.completed_key.take(); - if self.pending_key == Some(LayoutKey::FormatRevision) - && self.generation_depth == Some(self.brace_depth) - { - self.format_revision = None; - } - } - b'{' => { - if self.pending_key == Some(LayoutKey::Generation) && self.brace_depth == 1 { - self.generation_depth = Some(self.brace_depth + 1); - self.generation_hasher = Some(Sha256::new()); - event = GenerationSpanEvent::Opened; - } - if matches!( - self.pending_key, - Some(LayoutKey::Manifest | LayoutKey::Snapshot) - ) && self.generation_depth == Some(self.brace_depth) - { - let key = self.pending_key.ok_or_else(|| { - CodeIndexProductionErrorV1::Contract( - "sealed text metadata key disappeared".to_owned(), - ) - })?; - if self.captured_metadata_object.is_some() { - return Err(CodeIndexProductionErrorV1::Contract( - "sealed text metadata objects overlap".to_owned(), - )); - } - self.captured_metadata_object = Some((key, offset, self.brace_depth + 1)); - } - if self.files_depth == Some(self.bracket_depth) - && self.generation_depth == Some(self.brace_depth) - && self.current_file_start.is_none() - { - self.current_file_start = Some(offset); - } - self.brace_depth += 1; - self.pending_key = None; - } - b'}' => { - if let Some((key, start, depth)) = self.captured_metadata_object - && depth == self.brace_depth - { - let end = offset.checked_add(1).ok_or_else(|| { - CodeIndexProductionErrorV1::Contract( - "sealed text metadata end offset overflowed".to_owned(), - ) - })?; - match key { - LayoutKey::Manifest => self.manifest_range = Some((start, end)), - LayoutKey::Snapshot => self.snapshot_range = Some((start, end)), - _ => { - return Err(CodeIndexProductionErrorV1::Contract( - "sealed text metadata capture has an invalid key".to_owned(), - )); - } - } - self.captured_metadata_object = None; - } - if let Some(start) = self.current_file_start - && self - .generation_depth - .is_some_and(|depth| self.brace_depth == depth + 1) - { - let end = offset.checked_add(1).ok_or_else(|| { - CodeIndexProductionErrorV1::Contract( - "sealed lexical file end offset overflowed".to_owned(), - ) - })?; - let byte_len = end.checked_sub(start).ok_or_else(|| { - CodeIndexProductionErrorV1::Contract( - "sealed lexical file byte range is invalid".to_owned(), - ) - })?; - self.first_file_offset.get_or_insert(start); - self.maximum_file_bytes = self.maximum_file_bytes.max(byte_len); - self.file_count = self.file_count.checked_add(1).ok_or_else(|| { - CodeIndexProductionErrorV1::Contract( - "sealed lexical file count overflowed".to_owned(), - ) - })?; - self.file_ranges.push((start, end)); - self.current_file_start = None; - } - if self.generation_depth == Some(self.brace_depth) { - event = GenerationSpanEvent::Closed; - } - self.brace_depth = self.brace_depth.checked_sub(1).ok_or_else(|| { - CodeIndexProductionErrorV1::Contract( - "sealed generation object nesting is invalid".to_owned(), - ) - })?; - self.pending_key = None; - } - b'[' => { - if self.pending_key == Some(LayoutKey::Files) - && self.generation_depth == Some(self.brace_depth) - { - self.files_depth = Some(self.bracket_depth + 1); - } - self.bracket_depth += 1; - self.pending_key = None; - } - b']' => { - if self.files_depth == Some(self.bracket_depth) { - self.files_end_offset = Some(offset); - self.files_depth = None; - } - self.bracket_depth = self.bracket_depth.checked_sub(1).ok_or_else(|| { - CodeIndexProductionErrorV1::Contract( - "sealed generation array nesting is invalid".to_owned(), - ) - })?; - self.pending_key = None; - } - b'0'..=b'9' - if self.pending_key == Some(LayoutKey::FormatRevision) - && self.generation_depth == Some(self.brace_depth) => - { - self.format_revision = Some( - self.format_revision - .unwrap_or_default() - .checked_mul(10) - .and_then(|revision| revision.checked_add(u32::from(byte - b'0'))) - .ok_or_else(|| { - CodeIndexProductionErrorV1::Contract( - "sealed generation format revision exceeds u32".to_owned(), - ) - })?, - ); - } - b',' => { - self.completed_key = None; - self.pending_key = None; - } - byte if byte.is_ascii_whitespace() => {} - _ => self.completed_key = None, - } - Ok(event) - } - - fn finish(self) -> Result { - if self.in_string - || self.brace_depth != 0 - || self.bracket_depth != 0 - || self.current_file_start.is_some() - || self.captured_metadata_object.is_some() - { - return Err(CodeIndexProductionErrorV1::Contract( - "sealed lexical source has incomplete JSON structure".to_owned(), - )); - } - let state_digest = self.state_digest.ok_or_else(|| { - CodeIndexProductionErrorV1::Contract( - "sealed generation state digest is missing".to_owned(), - ) - })?; - let generation_digest = self.generation_digest.ok_or_else(|| { - CodeIndexProductionErrorV1::Contract("sealed generation payload is missing".to_owned()) - })?; - if generation_digest != state_digest { - return Err(CodeIndexProductionErrorV1::Contract( - "sealed generation state digest does not match its payload".to_owned(), - )); - } - let format_revision = self.format_revision.ok_or_else(|| { - CodeIndexProductionErrorV1::Contract( - "sealed generation format revision is missing".to_owned(), - ) - })?; - // A superseded envelope is refused, not scanned: the caller rebuilds - // the generation instead of falling through to another decoder that - // would report these bytes as corrupt. - if format_revision < MINIMUM_SEALED_GENERATION_FORMAT_REVISION { - return Err(superseded_sealed_generation_revision(format_revision)); - } - if format_revision != MONOLITHIC_SEALED_GENERATION_FORMAT_REVISION { - return Err(CodeIndexProductionErrorV1::Contract( - "sealed generation format revision is incompatible".to_owned(), - )); - } - let files_end_offset = self.files_end_offset.ok_or_else(|| { - CodeIndexProductionErrorV1::Contract( - "sealed generation files array is missing".to_owned(), - ) - })?; - let first_file_offset = self.first_file_offset.unwrap_or(files_end_offset); - if u64::try_from(self.file_ranges.len()).unwrap_or(u64::MAX) != self.file_count { - return Err(CodeIndexProductionErrorV1::Contract( - "sealed lexical file ranges do not match the admitted file count".to_owned(), - )); - } - Ok(SealedLexicalLayoutV1 { - state_digest, - format_revision, - file_count: self.file_count, - first_file_offset, - files_end_offset, - file_ranges: self.file_ranges, - maximum_file_bytes: self.maximum_file_bytes, - manifest_range: self.manifest_range, - snapshot_range: self.snapshot_range, - }) - } -} - -/// Locate the next quote or escape marker with eight-byte candidate probes. -/// Every input byte is still authenticated by the outer SHA-256 stream; this -/// helper only avoids interpreting ordinary string payload bytes one by one. -fn first_json_string_control(bytes: &[u8]) -> Option { - const LOW_BITS: u64 = 0x0101_0101_0101_0101; - const HIGH_BITS: u64 = 0x8080_8080_8080_8080; - const QUOTES: u64 = u64::from_ne_bytes([b'"'; 8]); - const ESCAPES: u64 = u64::from_ne_bytes([b'\\'; 8]); - - fn contains_zero_byte(value: u64) -> bool { - value.wrapping_sub(LOW_BITS) & !value & HIGH_BITS != 0 - } - - let mut chunks = bytes.chunks_exact(8); - for (chunk_index, chunk) in chunks.by_ref().enumerate() { - let word = u64::from_ne_bytes([ - chunk[0], chunk[1], chunk[2], chunk[3], chunk[4], chunk[5], chunk[6], chunk[7], - ]); - if contains_zero_byte(word ^ QUOTES) || contains_zero_byte(word ^ ESCAPES) { - let base = chunk_index * 8; - return chunk - .iter() - .position(|byte| matches!(*byte, b'"' | b'\\')) - .map(|relative| base + relative); - } - } - let tail_base = bytes.len() - chunks.remainder().len(); - chunks - .remainder() - .iter() - .position(|byte| matches!(*byte, b'"' | b'\\')) - .map(|relative| tail_base + relative) -} - -#[hotpath::measure(label = "code_index.restore.metadata")] -fn read_verified_text_metadata( - reader: &mut R, - layout: &SealedLexicalLayoutV1, - control: &dyn CodeIndexExecutionControlV1, -) -> Result { - fn decode_range( - reader: &mut R, - range: (u64, u64), - label: &'static str, - control: &dyn CodeIndexExecutionControlV1, - ) -> Result { - checkpoint(control)?; - let length = range.1.checked_sub(range.0).ok_or_else(|| { - CodeIndexProductionErrorV1::Contract(format!( - "sealed {label} metadata range is invalid" - )) - })?; - if length == 0 || length > MAX_LEXICAL_GENERATION_METADATA_BYTES { - return Err(CodeIndexProductionErrorV1::Contract(format!( - "sealed {label} metadata exceeds its byte bound" - ))); - } - let length = usize::try_from(length).map_err(|_| { - CodeIndexProductionErrorV1::Contract(format!( - "sealed {label} metadata exceeds the platform limit" - )) - })?; - reader.seek(SeekFrom::Start(range.0)).map_err(|error| { - CodeIndexProductionErrorV1::Contract(format!( - "sealed {label} metadata seek failed: {error}" - )) - })?; - let mut bytes = vec![0; length]; - reader.read_exact(&mut bytes).map_err(|error| { - CodeIndexProductionErrorV1::Contract(format!( - "sealed {label} metadata read failed: {error}" - )) - })?; - checkpoint(control)?; - serde_json::from_slice(&bytes).map_err(|error| { - CodeIndexProductionErrorV1::Contract(format!( - "sealed {label} metadata decoding failed: {error}" - )) - }) - } - - let manifest: CodeGenerationManifestV1 = hotpath::measure_block!( - "code_index.restore.metadata.manifest_decode", - decode_range( - reader, - layout.manifest_range.ok_or_else(|| { - CodeIndexProductionErrorV1::Contract( - "sealed generation manifest metadata is missing".to_owned(), - ) - })?, - "manifest", - control, - ) - )?; - if manifest.source_commitments.is_none() { - return Err(CodeIndexProductionErrorV1::SourceCommitmentsUnavailable); - } - let snapshot: SanitizedCodeSnapshotV1 = hotpath::measure_block!( - "code_index.restore.metadata.snapshot_decode", - decode_range( - reader, - layout.snapshot_range.ok_or_else(|| { - CodeIndexProductionErrorV1::Contract( - "sealed generation snapshot metadata is missing".to_owned(), - ) - })?, - "snapshot", - control, - ) - )?; - hotpath::measure_block!("code_index.restore.metadata.snapshot_validate", { - snapshot - .validate() - .map_err(|error| CodeIndexProductionErrorV1::Contract(error.to_string())) - })?; - hotpath::measure_block!("code_index.restore.metadata.digest_verify", { - let snapshot_digest = canonical_sha256(&(INTAKE_DIGEST_SEPARATOR, &snapshot)) - .map_err(|error| CodeIndexProductionErrorV1::Contract(error.to_string()))?; - if snapshot_digest != manifest.snapshot_digest { - return Err(CodeIndexProductionErrorV1::Contract( - "sealed text metadata snapshot digest does not match the manifest".to_owned(), - )); - } - let seal_digest = expected_seal_digest(&manifest) - .map_err(|error| CodeIndexProductionErrorV1::Contract(error.to_string()))?; - if seal_digest != manifest.seal.expected_digest { - return Err(CodeIndexProductionErrorV1::Contract( - "sealed text metadata manifest seal is invalid".to_owned(), - )); - } - Ok::<_, CodeIndexProductionErrorV1>(()) - })?; - VerifiedSealedTextGenerationMetadataV1::from_partitioned_manifest(manifest, snapshot, None) -} - -fn read_file_bytes_at_range( - reader: &mut R, - start: u64, - end: u64, - files_end_offset: u64, - maximum_file_bytes: u64, - control: &dyn CodeIndexExecutionControlV1, -) -> Result, CodeIndexProductionErrorV1> { - checkpoint(control)?; - if start >= files_end_offset || end > files_end_offset || end <= start { - return Err(CodeIndexProductionErrorV1::Contract( - "sealed lexical file range is outside the admitted source".to_owned(), - )); - } - let file_bytes = end - start; - if file_bytes > maximum_file_bytes { - return Err(CodeIndexProductionErrorV1::Contract( - "sealed lexical file exceeds its admitted decode window".to_owned(), - )); - } - let len = usize::try_from(file_bytes).map_err(|_| { - CodeIndexProductionErrorV1::Contract( - "sealed lexical file window exceeds the platform limit".to_owned(), - ) - })?; - reader.seek(SeekFrom::Start(start)).map_err(|error| { - CodeIndexProductionErrorV1::Contract(format!("sealed lexical source seek failed: {error}")) - })?; - let mut bytes = vec![0u8; len]; - reader.read_exact(&mut bytes).map_err(|error| { - CodeIndexProductionErrorV1::Contract(format!("sealed lexical file read failed: {error}")) - })?; - Ok(bytes) -} - -fn admit_persisted_file_bytes( - bytes: &[u8], - snapshot_digest: &ManifestDigest, - next_file_offset: u64, - control: &dyn CodeIndexExecutionControlV1, -) -> Result { - checkpoint(control)?; - let file: PersistedFileGenerationArtifactsV1 = - hotpath::measure_block!("code_index.restore.file_decode", { - serde_json::from_slice(bytes).map_err(|error| { - CodeIndexProductionErrorV1::Contract(format!( - "sealed lexical file decoding failed: {error}" - )) - }) - })?; - let exact_authority = ExactExtractionAuthorityV1::restore(&file.artifacts.chunks) - .map_err(CodeIndexProductionErrorV1::Chunk)?; - admit_validated_file_parts( - &file.authority, - &file.extraction, - &file.artifacts, - &exact_authority, - snapshot_digest, - next_file_offset, - control, - ) -} - fn admit_file_generation_artifacts( file: &FileGenerationArtifactsV1, snapshot_digest: &ManifestDigest, diff --git a/crates/tracedecay-code-index/src/production/lexical_page_source_tests.rs b/crates/tracedecay-code-index/src/production/lexical_page_source_tests.rs index 8bbe4b011a..bdf5034266 100644 --- a/crates/tracedecay-code-index/src/production/lexical_page_source_tests.rs +++ b/crates/tracedecay-code-index/src/production/lexical_page_source_tests.rs @@ -1,6 +1,5 @@ use std::{ collections::BTreeSet, - io::Cursor, sync::{ Arc, atomic::{AtomicUsize, Ordering}, @@ -144,28 +143,43 @@ impl CodeIndexExecutionControlV1 for ActiveControl { } } +/// A partitioned sealed generation held in memory: the manifest, its content +/// address, and every published file segment under its digest. struct SealedSourceFixture { - sealed: Vec, + manifest: Vec, + segments: Arc>>, state_digest: ManifestDigest, generation: Arc, } impl SealedSourceFixture { - fn open(&self) -> VerifiedSealedLexicalPageSourceV1>> { + fn open(&self) -> VerifiedSealedLexicalPageSourceV1 { self.open_with_page_chunks(1) } fn open_with_page_chunks( &self, maximum_page_chunks: usize, - ) -> VerifiedSealedLexicalPageSourceV1>> { - VerifiedSealedLexicalPageSourceV1::open( - Cursor::new(self.sealed.clone()), - u64::try_from(self.sealed.len()).expect("sealed fixture length fits u64"), + ) -> VerifiedSealedLexicalPageSourceV1 { + self.open_with_bounds(maximum_page_chunks, 1024 * 1024) + } + + fn open_with_bounds( + &self, + maximum_page_chunks: usize, + maximum_page_bytes: usize, + ) -> VerifiedSealedLexicalPageSourceV1 { + let segments = Arc::clone(&self.segments); + VerifiedSealedLexicalPageSourceV1::open_partitioned_sealed( + &self.manifest, self.state_digest.clone(), + move |digest, _, buffer, _control| { + buffer.clear(); + buffer.extend_from_slice(segments.get(digest).expect("sealed segment exists")); + Ok(()) + }, maximum_page_chunks, - 1024 * 1024, - &ActiveControl, + maximum_page_bytes, ) .expect("real sealed fixture source opens") } @@ -187,74 +201,12 @@ fn fixture() -> SealedSourceFixture { fixture_for_source(BATCH_FIXTURE_SOURCE) } -#[test] -fn published_memory_files_admit_the_same_pages_as_sealed_decode() { - let fixture = fixture(); - let disk = one_page_expectations(&fixture); - let mut source = fixture.open(); - source - .attach_published_files(&fixture.generation) - .expect("published files attach onto the scanned layout"); - let mut memory = Vec::new(); - loop { - match source - .next_page(&ActiveControl) - .expect("memory-admitted page") - { - VerifiedSealedLexicalPageReadV1::Page(page) => { - memory.push(expectation(&page)); - } - VerifiedSealedLexicalPageReadV1::Complete(receipt) => { - receipt - .verify_completion(Some(source.cursor())) - .expect("memory-admitted receipt verifies"); - break; - } - } - } - assert_eq!(disk, memory); -} - -#[test] -fn published_clone_rows_refuse_foreign_generation_authority_before_paging() { - let fixture = fixture(); - let mut generation = (*fixture.generation).clone(); - let file = Arc::make_mut(&mut generation.files[0]); - let body = file - .artifacts - .clone_bodies - .first_mut() - .expect("fixture clone body"); - body.occurrence.project_id = ProjectId::new("project.foreign").expect("foreign project"); - - let mut source = fixture.open(); - source - .attach_published_files(&generation) - .expect("generation identity still attaches"); - assert!(matches!( - source.next_page(&ActiveControl), - Err(CodeIndexProductionErrorV1::Chunk( - crate::chunks::ChunkingFailureV1::GenerationMismatch - )) - )); -} - #[test] fn partitioned_reopen_reports_encoded_byte_progress_and_bounds_prefetch() { let fixture = fixture_for_source_files(BATCH_FIXTURE_SOURCE, "src/batch_fixture.rs", "rust", 25); - let mut segments = BTreeMap::new(); - let manifest = fixture - .generation - .encode_partitioned_sealed(|request| { - if let super::super::SealedGenerationSegmentPublicationV1::File { digest, bytes } = - request - { - segments.insert(digest.clone(), bytes.to_vec()); - } - Ok(()) - }) - .expect("partitioned generation encodes"); + let manifest = fixture.manifest.clone(); + let segments = Arc::clone(&fixture.segments); let manifest_value: serde_json::Value = serde_json::from_slice(&manifest).expect("partitioned manifest envelope"); let segment_sizes = manifest_value["generation"]["file_segments"] @@ -267,12 +219,20 @@ fn partitioned_reopen_reports_encoded_byte_progress_and_bounds_prefetch() { .expect("partitioned segment size") }) .collect::>(); - let segments = Arc::new(segments); + let decoded_segment_bytes = manifest_value["generation"]["file_segments"] + .as_array() + .expect("partitioned file descriptors") + .iter() + .map(|descriptor| { + descriptor["decoded_size_bytes"] + .as_u64() + .expect("partitioned decoded segment size") + }) + .sum::(); let read_segments = Arc::clone(&segments); let reads = Arc::new(AtomicUsize::new(0)); let read_count = Arc::clone(&reads); let mut source = VerifiedSealedLexicalPageSourceV1::open_partitioned_sealed( - Cursor::new(Vec::::new()), &manifest, fixture.state_digest.clone(), move |digest, _, buffer, _control| { @@ -284,8 +244,7 @@ fn partitioned_reopen_reports_encoded_byte_progress_and_bounds_prefetch() { 1, 1 << 20, ) - .expect("partitioned source opens") - .expect("partitioned format"); + .expect("partitioned source opens"); let total_segment_bytes = segment_sizes.iter().sum::(); assert_eq!(source.total_lexical_units(), total_segment_bytes); assert!(source.total_lexical_units() > source.total_files()); @@ -296,8 +255,9 @@ fn partitioned_reopen_reports_encoded_byte_progress_and_bounds_prefetch() { "opening a page source must not decode every file before the first page" ); assert!( - source.retained_layout_bytes() < fixture.sealed.len() / 8, - "compact file identities must remain below an eighth of the decoded corpus encoding" + source.retained_layout_bytes() + < usize::try_from(decoded_segment_bytes).expect("segment bytes fit usize") / 8, + "compact file identities must remain below an eighth of the decoded corpus" ); source.next_page(&ActiveControl).expect("first page admits"); assert!( @@ -348,7 +308,6 @@ fn partitioned_reopen_reports_encoded_byte_progress_and_bounds_prefetch() { "cancelled reads preserve accepted progress" ); let mut corrupt = VerifiedSealedLexicalPageSourceV1::open_partitioned_sealed( - Cursor::new(Vec::::new()), &manifest, fixture.state_digest.clone(), move |digest, _, buffer, _control| { @@ -360,8 +319,7 @@ fn partitioned_reopen_reports_encoded_byte_progress_and_bounds_prefetch() { 1, 1 << 20, ) - .expect("lazy source authenticates manifest") - .expect("partitioned format"); + .expect("lazy source authenticates manifest"); let initial = corrupt.cursor().clone(); assert!(matches!( corrupt.next_page(&ActiveControl), @@ -478,7 +436,7 @@ fn draining_a_many_file_generation_amortizes_install_calls_by_the_multiplier() { } #[test] -fn foreign_memory_files_cannot_mint_an_import_cursor_for_a_sealed_source() { +fn accepted_import_cursor_resumes_a_fresh_source_after_cancellation() { let imports = (0..128) .map(|ordinal| format!("import type {{ Type{ordinal} }} from \"module-{ordinal}\";\n")) .collect::(); @@ -490,78 +448,39 @@ fn foreign_memory_files_cannot_mint_an_import_cursor_for_a_sealed_source() { }) .collect::() ); - let foreign_source = - format!("{imports}export function foreignItem(): number {{ return 1; }}\n"); let target = fixture_for_typescript_source(&target_source); - let foreign = fixture_for_typescript_source(&foreign_source); - assert!( - target - .generation - .admitted_chunks() - .expect("target generation exposes chunks") - .len() - > foreign - .generation - .admitted_chunks() - .expect("foreign generation exposes chunks") - .len(), - "the authenticated target must have more chunks than the foreign memory source" - ); - assert!( - foreign.generation.imports().len() > 1, - "the foreign source must reach a partial import position" - ); - let maximum_page_bytes = [&target.generation, &foreign.generation] - .into_iter() - .map(|generation| { - let admitted = admit_file_generation_artifacts( - generation.files[0].as_ref(), - &generation.manifest().snapshot_digest, - 1, - &ActiveControl, - ) - .expect("fixture file admits"); - admitted - .serialized_chunks - .iter() - .zip(&admitted.serialized_displays) - .map(|(chunk, display)| { - chunk - .len() - .saturating_add(display.as_ref().map_or(0, Vec::len)) - }) - .chain(admitted.serialized_imports.iter().map(Vec::len)) - .max() - .expect("fixture exposes lexical records") + let generation = &target.generation; + let admitted = admit_file_generation_artifacts( + generation.files[0].as_ref(), + &generation.manifest().snapshot_digest, + 1, + &ActiveControl, + ) + .expect("fixture file admits"); + let maximum_page_bytes = admitted + .serialized_chunks + .iter() + .zip(&admitted.serialized_displays) + .map(|(chunk, display)| { + chunk + .len() + .saturating_add(display.as_ref().map_or(0, Vec::len)) }) + .chain(admitted.serialized_imports.iter().map(Vec::len)) .max() - .expect("fixtures expose lexical records") + .expect("fixture exposes lexical records") .saturating_add(1); - let foreign_import_bytes = foreign.generation.files[0] - .artifacts - .imports + let import_bytes = admitted + .serialized_imports .iter() - .map(|evidence| { - serde_json::to_vec(evidence) - .expect("import serializes") - .len() - }) + .map(Vec::len) .sum::(); assert!( - foreign_import_bytes > maximum_page_bytes, + import_bytes > maximum_page_bytes, "imports must span more than one bounded page" ); - let mut source = VerifiedSealedLexicalPageSourceV1::open( - Cursor::new(target.sealed.clone()), - u64::try_from(target.sealed.len()).expect("target sealed length fits u64"), - target.state_digest.clone(), - usize::MAX, - maximum_page_bytes, - &ActiveControl, - ) - .expect("authenticated target source opens"); - let foreign_was_rejected = source.attach_published_files(&foreign.generation).is_err(); + let mut source = target.open_with_bounds(usize::MAX, maximum_page_bytes); let boundary_cursor = loop { let previous_cursor = source.cursor().clone(); @@ -604,15 +523,7 @@ fn foreign_memory_files_cannot_mint_an_import_cursor_for_a_sealed_source() { let restored = VerifiedSealedLexicalCursorV1::restore_persisted(&persisted) .expect("accepted import cursor restores"); - let mut resumed = VerifiedSealedLexicalPageSourceV1::open( - Cursor::new(target.sealed.clone()), - u64::try_from(target.sealed.len()).expect("target sealed length fits u64"), - target.state_digest.clone(), - usize::MAX, - maximum_page_bytes, - &ActiveControl, - ) - .expect("fresh authenticated target source opens"); + let mut resumed = target.open_with_bounds(usize::MAX, maximum_page_bytes); resumed .restore_cursor(&restored, &ActiveControl) .expect("an accepted cursor must resume its authenticated source"); @@ -631,10 +542,6 @@ fn foreign_memory_files_cannot_mint_an_import_cursor_for_a_sealed_source() { resumed_page.next_cursor().next_import_ordinal() > restored.next_import_ordinal(), "resumed acceptance must advance the import position" ); - assert!( - foreign_was_rejected, - "decoded files from another generation must not replace sealed source authority" - ); } fn fixture_for_source(source: &str) -> SealedSourceFixture { @@ -748,19 +655,22 @@ fn fixture_for_source_files( let generation = owner .build_and_publish(request, &ActiveControl) .expect("fixture generation publishes"); - let sealed = generation - .encode_sealed() + let mut segments = BTreeMap::new(); + let manifest = generation + .encode_partitioned_sealed(|request| { + if let super::super::SealedGenerationSegmentPublicationV1::File { digest, bytes } = + request + { + segments.insert(digest.clone(), bytes.to_vec()); + } + Ok(()) + }) .expect("fixture generation seals"); - let envelope: serde_json::Value = - serde_json::from_slice(&sealed).expect("fixture sealed envelope decodes"); - let state_digest = ManifestDigest::new( - envelope["state_digest"] - .as_str() - .expect("fixture sealed state digest"), - ) - .expect("fixture state digest is canonical"); + let state_digest = ManifestDigest::from_sha256_bytes(&Sha256::digest(&manifest)) + .expect("fixture manifest digest is canonical"); SealedSourceFixture { - sealed, + manifest, + segments: Arc::new(segments), state_digest, generation, } @@ -1113,100 +1023,3 @@ fn cancellation_during_staging_keeps_the_exact_pre_batch_cursor() { cursor_before, ); } - -#[test] -fn layout_scan_preserves_digest_and_file_boundaries_across_escaped_syntax() { - let first_file = r#"{"payload":"escaped \\\" quote and { [ ] } syntax"}"#; - let second_file = format!(r#"{{"payload":"{}"}}"#, "y".repeat(96 * 1024)); - let generation = format!( - r#"{{"format_revision":{MONOLITHIC_SEALED_GENERATION_FORMAT_REVISION},"files":[{first_file},{second_file}],"tail":"done"}}"# - ); - let state_digest = ManifestDigest::from_sha256_bytes(&Sha256::digest(generation.as_bytes())) - .expect("synthetic generation digest is canonical"); - let sealed = format!( - r#"{{"state_digest":"{}","generation":{generation}}}"#, - state_digest.as_str() - ) - .into_bytes(); - let file_digest = ManifestDigest::from_sha256_bytes(&Sha256::digest(&sealed)) - .expect("synthetic file digest is canonical"); - let first_file_offset = sealed - .windows(first_file.len()) - .position(|window| window == first_file.as_bytes()) - .expect("first synthetic file is present"); - let files_end_offset = first_file_offset + first_file.len() + 1 + second_file.len(); - - let layout = scan_layout( - &mut Cursor::new(&sealed), - u64::try_from(sealed.len()).expect("synthetic seal length fits u64"), - Some(&file_digest), - &ActiveControl, - ) - .expect("escaped syntax does not alter the authenticated layout"); - - assert_eq!(layout.state_digest, state_digest); - assert_eq!(layout.file_count, 2); - assert_eq!(layout.first_file_offset, first_file_offset as u64); - assert_eq!(layout.files_end_offset, files_end_offset as u64); - assert_eq!(layout.maximum_file_bytes, second_file.len() as u64); - assert_eq!( - layout.file_ranges, - [ - ( - first_file_offset as u64, - (first_file_offset + first_file.len()) as u64 - ), - ( - (first_file_offset + first_file.len() + 1) as u64, - files_end_offset as u64 - ) - ] - ); -} - -#[test] -fn layout_scan_rejects_cancelled_and_corrupted_sources() { - let file = format!(r#"{{"payload":"{}"}}"#, "z".repeat(512 * 1024)); - let generation = format!( - r#"{{"format_revision":{MONOLITHIC_SEALED_GENERATION_FORMAT_REVISION},"files":[{file}]}}"# - ); - let state_digest = ManifestDigest::from_sha256_bytes(&Sha256::digest(generation.as_bytes())) - .expect("synthetic generation digest is canonical"); - let sealed = format!( - r#"{{"state_digest":"{}","generation":{generation}}}"#, - state_digest.as_str() - ) - .into_bytes(); - - let cancellation = CancelDuringStaging::new(); - let cancelled = match scan_layout( - &mut Cursor::new(&sealed), - sealed.len() as u64, - None, - &cancellation, - ) { - Ok(_) => panic!("layout opening must honor bounded read checkpoints"), - Err(error) => error, - }; - assert!(matches!( - cancelled, - CodeIndexProductionErrorV1::Interrupted(CodeIndexInterruptionV1::Cancelled) - )); - - let mut corrupted = sealed; - let payload = corrupted - .windows(b"zzzz".len()) - .position(|window| window == b"zzzz") - .expect("synthetic payload is present"); - corrupted[payload] = b'x'; - let error = match scan_layout( - &mut Cursor::new(&corrupted), - corrupted.len() as u64, - None, - &ActiveControl, - ) { - Ok(_) => panic!("payload corruption must fail the exact generation digest"), - Err(error) => error, - }; - assert!(matches!(error, CodeIndexProductionErrorV1::Contract(_))); -} diff --git a/crates/tracedecay-code-index/src/production/lineage_rows.rs b/crates/tracedecay-code-index/src/production/lineage_rows.rs new file mode 100644 index 0000000000..007643f848 --- /dev/null +++ b/crates/tracedecay-code-index/src/production/lineage_rows.rs @@ -0,0 +1,328 @@ +//! Persisted lineage rows of one generation's evidence. +//! +//! Nearly every lineage row of a successor generation says that a symbol +//! continued unchanged from the prior occurrence of its exact identity +//! tuple, and such a row is a pure function of the two generation ids, that +//! prior occurrence, and the current symbol record. The persisted form names +//! those rows by the current symbol's position in the generation's +//! occurrence-ordered symbol roster (as runs when the prior occurrence is the +//! current one) and keeps every other row whole. Encoding admits a compact +//! row only when rebuilding it reproduces the resolver's candidate exactly. + +use std::collections::HashMap; +use std::sync::Arc; + +use serde::{Deserialize, Serialize}; +use tracedecay_domain::{CodeGenerationId, SymbolOccurrenceId}; + +use super::{CodeIndexProductionErrorV1, collect_bounded_ordered}; +use crate::lineage::{ + LineageKindV1, LineageResolutionErrorV1, LineageSymbolRecordV1, SymbolLineageCandidateV1, +}; + +/// Rows verified or rebuilt per unit of pool work. +const LINEAGE_BATCH_ROWS_V1: usize = 4096; + +#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct PersistedLineageV1 { + #[serde(default, skip_serializing_if = "Option::is_none")] + prior_generation: Option, + rows: Vec, +} + +#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case", deny_unknown_fields)] +enum PersistedLineageRowV1 { + /// Roster symbols `start..start + count`, each unchanged from itself. + Unchanged { + start: u32, + count: u32, + }, + /// Roster symbol `current`, unchanged from the prior occurrence `prior`. + UnchangedFrom { + current: u32, + prior: SymbolOccurrenceId, + }, + Candidate(Box), +} + +/// A generation's symbols in canonical occurrence order. Encoding and +/// restore both derive it from the file artifacts, so positions agree. +pub(super) fn occurrence_roster<'a>( + symbols: impl Iterator>, +) -> Vec<&'a LineageSymbolRecordV1> { + let mut roster = symbols.map(Arc::as_ref).collect::>(); + roster.sort_by(|left, right| left.occurrence.cmp(&right.occurrence)); + roster +} + +fn lineage_error(error: LineageResolutionErrorV1) -> CodeIndexProductionErrorV1 { + CodeIndexProductionErrorV1::Lineage(error) +} + +impl PersistedLineageV1 { + pub(super) fn compact( + lineage: &[SymbolLineageCandidateV1], + current_generation: &CodeGenerationId, + roster: &[&LineageSymbolRecordV1], + ) -> Result { + let prior_generation = lineage + .first() + .map(|row| row.evidence.prior_generation.clone()); + let positions = roster + .iter() + .enumerate() + .map(|(position, symbol)| { + u32::try_from(position) + .map(|position| (&symbol.occurrence, position)) + .map_err(|_| { + CodeIndexProductionErrorV1::Contract( + "sealed lineage roster exceeds u32".to_owned(), + ) + }) + }) + .collect::, _>>()?; + let batches = lineage.chunks(LINEAGE_BATCH_ROWS_V1).collect::>(); + let implicit = collect_bounded_ordered(&batches, |batch, _worker| { + batch + .iter() + .map(|candidate| { + let (Some(prior_generation), Some(&position)) = ( + prior_generation.as_ref(), + positions.get(&candidate.current_occurrence), + ) else { + return Ok(None); + }; + if candidate.kind != LineageKindV1::Unchanged { + return Ok(None); + } + let rebuilt = SymbolLineageCandidateV1::exact_unchanged( + prior_generation, + current_generation, + &candidate.prior_occurrence, + roster[position as usize], + ) + .map_err(lineage_error)?; + Ok((rebuilt == *candidate).then_some(position)) + }) + .collect::, CodeIndexProductionErrorV1>>() + })?; + let mut rows = Vec::new(); + for (candidate, position) in lineage.iter().zip(implicit.into_iter().flatten()) { + match position { + Some(position) if candidate.prior_occurrence == candidate.current_occurrence => { + match rows.last_mut() { + Some(PersistedLineageRowV1::Unchanged { start, count }) + if start.checked_add(*count) == Some(position) => + { + *count += 1; + } + _ => rows.push(PersistedLineageRowV1::Unchanged { + start: position, + count: 1, + }), + } + } + Some(position) => rows.push(PersistedLineageRowV1::UnchangedFrom { + current: position, + prior: candidate.prior_occurrence.clone(), + }), + None => rows.push(PersistedLineageRowV1::Candidate(Box::new( + candidate.clone(), + ))), + } + } + Ok(Self { + prior_generation, + rows, + }) + } + + pub(super) fn expand( + self, + current_generation: &CodeGenerationId, + roster: &[&LineageSymbolRecordV1], + ) -> Result, CodeIndexProductionErrorV1> { + enum RowV1 { + Implicit(usize, Option), + Whole(SymbolLineageCandidateV1), + } + let symbol = |position: u32| { + usize::try_from(position) + .ok() + .filter(|position| *position < roster.len()) + .ok_or_else(|| { + CodeIndexProductionErrorV1::Contract( + "sealed lineage row names a symbol outside its roster".to_owned(), + ) + }) + }; + let mut work = Vec::new(); + for row in self.rows { + match row { + PersistedLineageRowV1::Unchanged { start, count } => { + let end = start.checked_add(count).ok_or_else(|| { + CodeIndexProductionErrorV1::Contract( + "sealed lineage run exceeds u32".to_owned(), + ) + })?; + if count > 0 { + symbol(end - 1)?; + } + work.extend( + (start..end).map(|position| RowV1::Implicit(position as usize, None)), + ); + } + PersistedLineageRowV1::UnchangedFrom { current, prior } => { + work.push(RowV1::Implicit(symbol(current)?, Some(prior))); + } + PersistedLineageRowV1::Candidate(candidate) => work.push(RowV1::Whole(*candidate)), + } + } + let prior_generation = self.prior_generation; + let batches = work.chunks(LINEAGE_BATCH_ROWS_V1).collect::>(); + let expanded = collect_bounded_ordered(&batches, |batch, _worker| { + batch + .iter() + .map(|row| match row { + RowV1::Whole(candidate) => Ok(candidate.clone()), + RowV1::Implicit(position, prior) => { + let prior_generation = prior_generation.as_ref().ok_or_else(|| { + CodeIndexProductionErrorV1::Contract( + "sealed lineage has implicit rows without a prior generation" + .to_owned(), + ) + })?; + let symbol = roster[*position]; + SymbolLineageCandidateV1::exact_unchanged( + prior_generation, + current_generation, + prior.as_ref().unwrap_or(&symbol.occurrence), + symbol, + ) + .map_err(lineage_error) + } + }) + .collect::, CodeIndexProductionErrorV1>>() + })?; + Ok(expanded.into_iter().flatten().collect()) + } +} + +#[cfg(test)] +mod tests { + use tracedecay_domain::{ + ComplexityAnalysisV1, ContentDigest, FileIdentityDigest, SymbolIdentityDigest, + }; + + use super::*; + use crate::lineage::{GenerationSymbolIndexV1, SymbolLineageResolver}; + + fn digest(byte: char) -> String { + format!("sha256:{}", byte.to_string().repeat(64)) + } + + fn generation(sequence: u64) -> CodeGenerationId { + CodeGenerationId::new(format!("generation.v1.aaaaaaaa.{sequence:08}")) + .expect("valid generation id") + } + + fn record(occurrence: &str, identity: char, content: char) -> Arc { + Arc::new(LineageSymbolRecordV1 { + occurrence: SymbolOccurrenceId::new(occurrence).expect("occurrence"), + identity: SymbolIdentityDigest::new(digest(identity)).expect("identity"), + qualified_name: format!("crate::{identity}"), + simple_name: identity.to_string(), + kind: "function".to_owned(), + visibility: "private".to_owned(), + branches: 0, + loops: 0, + max_nesting: 0, + complexity_analysis: ComplexityAnalysisV1::Complete, + line_span: 1, + start_line: 0, + signature: None, + docstring: None, + is_async: false, + derives: Vec::new(), + skip_test_coverage: false, + file_identity: FileIdentityDigest::new(digest('f')).expect("file identity"), + content_digest: ContentDigest::new(digest(content)).expect("content"), + }) + } + + #[test] + fn continuity_rows_are_implicit_and_every_row_restores_exactly() { + let prior = GenerationSymbolIndexV1::new( + generation(1), + vec![ + record("sym.a", 'a', '0'), + record("sym.b", 'b', '1'), + record("sym.c", 'c', '2'), + record("sym.e", 'e', '4'), + record("sym.f", 'f', '5'), + ], + ) + .expect("prior"); + let current = GenerationSymbolIndexV1::new( + generation(2), + vec![ + record("sym.a", 'a', '0'), + // The same identity and content under a new occurrence. + record("sym.b2", 'b', '1'), + // The same identity with new content. + record("sym.c", 'c', '3'), + // No ancestor at all, so no row; the runs around it split. + record("sym.d", 'd', '9'), + record("sym.e", 'e', '4'), + record("sym.f", 'f', '5'), + ], + ) + .expect("current"); + let lineage = SymbolLineageResolver::new() + .resolve(&prior, ¤t) + .expect("lineage"); + assert_eq!(lineage.len(), 5); + let roster = occurrence_roster(current.symbols.iter()); + + let persisted = PersistedLineageV1::compact(&lineage, ¤t.generation_id, &roster) + .expect("compact"); + assert_eq!(persisted.prior_generation, Some(generation(1))); + assert_eq!( + persisted.rows, + [ + PersistedLineageRowV1::Unchanged { start: 0, count: 1 }, + PersistedLineageRowV1::UnchangedFrom { + current: 1, + prior: SymbolOccurrenceId::new("sym.b").expect("occurrence"), + }, + PersistedLineageRowV1::Candidate(Box::new(lineage[2].clone())), + PersistedLineageRowV1::Unchanged { start: 4, count: 2 }, + ] + ); + + let bytes = serde_json::to_vec(&persisted).expect("serialize"); + let restored = serde_json::from_slice::(&bytes) + .expect("deserialize") + .expand(¤t.generation_id, &roster) + .expect("expand"); + assert_eq!(restored, lineage); + } + + #[test] + fn rows_outside_the_roster_or_without_a_prior_generation_are_refused() { + let symbols = [record("sym.a", 'a', '0')]; + let roster = occurrence_roster(symbols.iter()); + let outside = PersistedLineageV1 { + prior_generation: Some(generation(1)), + rows: vec![PersistedLineageRowV1::Unchanged { start: 0, count: 2 }], + }; + assert!(outside.expand(&generation(2), &roster).is_err()); + let unanchored = PersistedLineageV1 { + prior_generation: None, + rows: vec![PersistedLineageRowV1::Unchanged { start: 0, count: 1 }], + }; + assert!(unanchored.expand(&generation(2), &roster).is_err()); + } +} diff --git a/crates/tracedecay-code-index/src/production/mod.rs b/crates/tracedecay-code-index/src/production/mod.rs index 42d2ac7f05..501cf0e40e 100644 --- a/crates/tracedecay-code-index/src/production/mod.rs +++ b/crates/tracedecay-code-index/src/production/mod.rs @@ -64,7 +64,10 @@ use super::{ }; mod canonical_json; +mod clone_rows; mod helpers; +mod lineage_rows; +mod projection_rows; pub use helpers::generation_language_revisions_are_current; use helpers::*; mod ignored_sources; @@ -101,9 +104,8 @@ pub use partitioned_codec::{ }; mod sealed_codec; pub use sealed_codec::{ - MAX_SEALED_CODE_GENERATION_BYTES_V1, MINIMUM_SEALED_GENERATION_FORMAT_REVISION, - SEALED_GENERATION_FORMAT_REVISION_V1, sealed_generation_format_revision_is_compatible, - sealed_generation_payload_digest, superseded_sealed_generation_revision, + MAX_SEALED_CODE_GENERATION_BYTES_V1, SEALED_GENERATION_FORMAT_REVISION_V1, + superseded_sealed_generation_revision, }; /// Current daemon chunker identity shared by production indexing and native @@ -2022,15 +2024,13 @@ where }; lexical_page_source::checkpoint(control)?; - let parser_registry = Arc::new(tracedecay_code_extraction::LanguageRegistry::new()); - let extractor = TreeSitterExtractor::from_shared_registry(Arc::clone(&parser_registry)); - let chunker = DeterministicCodeChunker::from_shared_registry( + let extractor = TreeSitterExtractor::new(); + let chunker = DeterministicCodeChunker::new( manifest.generation_id.clone(), self.config.repository.clone(), self.config.sanitizer_revision.clone(), self.config.policy_revision.clone(), self.config.chunker_revision.clone(), - parser_registry, ); crate::hotpath_observe::record_rebuild_state(match increment.as_ref() { Some(plan) if plan.is_full_rebuild() => "rebuild", diff --git a/crates/tracedecay-code-index/src/production/partitioned_codec.rs b/crates/tracedecay-code-index/src/production/partitioned_codec.rs index 47f5cb30e1..8082776b55 100644 --- a/crates/tracedecay-code-index/src/production/partitioned_codec.rs +++ b/crates/tracedecay-code-index/src/production/partitioned_codec.rs @@ -38,16 +38,17 @@ use std::collections::BTreeMap; use std::collections::{BTreeSet, HashMap}; use std::fmt::Write as _; -use std::io::{Read, Seek, Write as IoWrite}; +use std::io::{Read, Write as IoWrite}; use std::sync::{Mutex, PoisonError}; -use serde::{Deserialize, Deserializer, Serialize}; +use flate2::Compression; +use flate2::read::DeflateDecoder; +use flate2::write::DeflateEncoder; +use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; use sha2::{Digest, Sha256}; use tracedecay_domain::{ - CodeChunkProjectionReceiptV1, CodeSearchChunkId, ContentDigest, FileOccurrenceId, - ManifestDigest, ProjectionOperationV1, ProjectionOutcomeV1, SymbolIdentityDigest, - SymbolOccurrenceId, + FileOccurrenceId, ManifestDigest, SymbolIdentityDigest, SymbolOccurrenceId, }; use super::canonical_json::{ @@ -55,30 +56,31 @@ use super::canonical_json::{ write_json_string, }; use super::lexical_page_source::{LEXICAL_FILE_PREFETCH_BYTES_V1, checkpoint}; +use super::lineage_rows::{PersistedLineageV1, occurrence_roster}; +use super::projection_rows::{ + PersistedBatchReceiptRefV1, PersistedBatchReceiptV1, PersistedProjectionRequestRefV1, + PersistedProjectionRequestV1, chunk_roster, +}; use super::sealed_codec::{ - MONOLITHIC_SEALED_GENERATION_FORMAT_REVISION, PersistedFileGenerationArtifactsRefV2, - PersistedFileGenerationArtifactsV1, PersistedFileGenerationArtifactsV2, - SEALED_GENERATION_FORMAT_REVISION_V1, StreamingPersistedPublishedGenerationV1, - StreamingRestoredFilesV1, assemble_published_generation, restore_file_pages, + FileScopeIdentityV1, PersistedFileGenerationArtifactsRefV2, PersistedFileGenerationArtifactsV1, + PersistedFileGenerationArtifactsV2, SEALED_GENERATION_FORMAT_REVISION_V1, + StreamingPersistedPublishedGenerationV1, assemble_published_generation, restore_file_pages, superseded_sealed_generation_revision, }; use super::*; -/// Revision 1 persisted the file record as its in-memory serialization; -/// revision 2 persists the row form described on -/// [`PersistedFileGenerationArtifactsRefV2`]. Both decode, so a generation -/// may reuse revision-1 segments from its parent beside revision-2 segments -/// of its own. -#[cfg(test)] -const FILE_SEGMENT_FORMAT_REVISION_V1: u32 = 1; -#[cfg(test)] -const FILE_SEGMENT_FORMAT_REVISION_V2: u32 = 2; -const FILE_SEGMENT_FORMAT_REVISION_V3: u32 = 3; +/// The row form described on [`PersistedFileGenerationArtifactsRefV2`], +/// stored as a raw DEFLATE stream of its canonical JSON. Only generations +/// of the current manifest revision address segments, so earlier segment +/// revisions are never read. +const FILE_SEGMENT_FORMAT_REVISION: u32 = 5; +/// The zlib default. Changing it changes stored bytes and therefore every +/// segment's content address, which only costs one generation's reuse. +const FILE_SEGMENT_COMPRESSION_LEVEL: u32 = 6; const GENERATION_ID_MARKER: &str = "$tracedecay:g"; const SNAPSHOT_DIGEST_MARKER: &str = "$tracedecay:snapshot"; const FILE_OCCURRENCE_ID_MARKER: &str = "$tracedecay:f"; const SYMBOL_OCCURRENCE_ID_MARKER_PREFIX: &str = "$tracedecay:s:"; -const CHUNK_ID_MARKER_PREFIX: &str = "$tracedecay:c:"; const GENERATION_EVIDENCE_PAGE_MAX_BYTES_V1: usize = 256 * 1024; #[derive(Clone, Debug, Serialize, Deserialize)] @@ -87,10 +89,38 @@ struct PartitionedFileSegmentDescriptorV1 { file_key: u32, segment_digest: ManifestDigest, segment_size_bytes: u64, + /// Length of the canonical JSON the stored bytes inflate to. Restore + /// windows budget on it, and inflation must end at exactly this length. + decoded_size_bytes: u64, file_occurrence_id: FileOccurrenceId, - symbol_identities: Vec, - #[serde(skip)] - symbol_occurrences: Vec, + /// Digest of the ordered symbol identities the segment carries, so a + /// successor can decide reuse without reading the segment. + symbol_identities_digest: ManifestDigest, +} + +fn symbol_identities_digest( + identities: &[SymbolIdentityDigest], +) -> Result { + let mut hasher = Sha256::new(); + for identity in identities { + hasher.update(identity.as_str().as_bytes()); + hasher.update(b"\n"); + } + ManifestDigest::from_sha256_bytes(&hasher.finalize()) + .map_err(|error| CodeIndexProductionErrorV1::Contract(error.to_string())) +} + +/// A segment's symbol keys resolve to these occurrences, rebound to the +/// file occurrence of the generation that addresses it. +fn bind_symbol_occurrences( + file_occurrence_id: &FileOccurrenceId, + identities: &[SymbolIdentityDigest], +) -> Result, CodeIndexProductionErrorV1> { + identities + .iter() + .map(|identity| crate::chunks::symbol_occurrence_id(file_occurrence_id, identity)) + .collect::, _>>() + .map_err(|error| CodeIndexProductionErrorV1::Contract(error.to_string())) } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -101,68 +131,12 @@ struct PartitionedEvidencePageDescriptorV1 { page_size_bytes: u64, } -/// How the evidence segment's typed JSON stream is shaped. -/// -/// The publication receipt is the bulk of the evidence: one row per chunk, -/// and every row repeats the batch's projection key, request digest, and -/// generation watermarks and restates the request's own chunk digests. -/// `CompactReceipts` persists only what the request cannot reproduce, the -/// projector's decision per chunk, and the reader rebuilds the full -/// receipt, whose publication digest then has to recompute exactly as it -/// did when the batch was sealed. Manifests written before the tag existed -/// carry the full receipt rows and decode as `Typed`. -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -enum PartitionedEvidenceEncodingV1 { - #[default] - Typed, - CompactReceipts, -} - -#[derive(Clone, Debug, Serialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] struct PartitionedGenerationEvidenceDescriptorV1 { segment_digest: ManifestDigest, segment_size_bytes: u64, pages: Vec, - encoding: PartitionedEvidenceEncodingV1, - #[serde(skip)] - legacy_unpaged: bool, -} - -#[derive(Deserialize)] -#[serde(deny_unknown_fields)] -struct PartitionedGenerationEvidenceDescriptorWireV1 { - segment_digest: ManifestDigest, - segment_size_bytes: u64, - #[serde(default, deserialize_with = "deserialize_present_vec")] - pages: Option>, - #[serde(default)] - encoding: PartitionedEvidenceEncodingV1, -} - -fn deserialize_present_vec<'de, D, T>(deserializer: D) -> Result>, D::Error> -where - D: Deserializer<'de>, - T: Deserialize<'de>, -{ - Vec::::deserialize(deserializer).map(Some) -} - -impl<'de> Deserialize<'de> for PartitionedGenerationEvidenceDescriptorV1 { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let wire = PartitionedGenerationEvidenceDescriptorWireV1::deserialize(deserializer)?; - let legacy_unpaged = wire.pages.is_none(); - Ok(Self { - segment_digest: wire.segment_digest, - segment_size_bytes: wire.segment_size_bytes, - pages: wire.pages.unwrap_or_default(), - encoding: wire.encoding, - legacy_unpaged, - }) - } } #[derive(Clone, Debug, PartialEq, Eq)] @@ -221,7 +195,9 @@ struct PartitionedPublishedGenerationRefV1<'a> { #[derive(Deserialize)] #[serde(deny_unknown_fields)] struct PartitionedPublishedGenerationV1 { - format_revision: u32, + /// Gated by the revision probe before this strict parse runs. + #[serde(rename = "format_revision")] + _format_revision: u32, manifest: CodeGenerationManifestV1, snapshot: SanitizedCodeSnapshotV1, statistics: CodeIndexGenerationStatisticsV1, @@ -291,12 +267,15 @@ struct PartitionedFileSegmentIdentityV1 { file_occurrence_id: FileOccurrenceId, } +/// Retention projects the descriptor before its revision gate, so a retired +/// manifest without a page table must still reach that gate and abstain; a +/// current-revision manifest without one fails the shared layout validator. #[derive(Deserialize)] struct PartitionedEvidenceSegmentIdentityV1 { segment_digest: ManifestDigest, segment_size_bytes: u64, - #[serde(default, deserialize_with = "deserialize_present_vec")] - pages: Option>, + #[serde(default)] + pages: Vec, } #[derive(Deserialize)] @@ -313,39 +292,33 @@ struct PartitionedEvidencePageIdentityV1 { #[serde(deny_unknown_fields)] struct PartitionedRawFileSegmentV1<'a> { format_revision: u32, + /// The identities the payload's symbol keys index, in key order. + symbol_identities: Vec, #[serde(borrow)] file: &'a RawValue, } -#[derive(Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -struct PartitionedGenerationEvidenceV1 { - lineage: Vec, - projection_request: ProjectionBatchRequestV1, - projection_receipt: ProjectionBatchReceiptV1, -} - -/// The [`PartitionedEvidenceEncodingV1::CompactReceipts`] stream: the same -/// lineage and request, with the receipt reduced to its batch header and one -/// decision row per chunk. +/// The evidence stream. Every part is in a persisted row form whose rows +/// index the generation's own symbols and chunks, so restore expands it only +/// after the file segments supply those rosters: lineage per +/// [`super::lineage_rows`], and the projection request and receipt per +/// [`super::projection_rows`]. #[derive(Deserialize)] #[serde(deny_unknown_fields)] -struct PartitionedCompactGenerationEvidenceV1 { +struct PartitionedGenerationEvidenceV1 { #[serde(deserialize_with = "deserialize_evidence_lineage")] - lineage: Vec, + lineage: PersistedLineageV1, #[serde(deserialize_with = "deserialize_evidence_projection_request")] - projection_request: ProjectionBatchRequestV1, - #[serde(deserialize_with = "deserialize_evidence_compact_receipt")] - projection_receipt: PartitionedCompactBatchReceiptV1, + projection_request: PersistedProjectionRequestV1, + #[serde(deserialize_with = "deserialize_evidence_projection_receipt")] + projection_receipt: PersistedBatchReceiptV1, } -/// The generation evidence stream is the largest segment a restore reads and -/// it decodes on one thread, so each of its three payloads is measured -/// separately: the lineage roster, the projection request's per-chunk change -/// rows, and the projector's per-chunk decision rows. +/// The generation evidence stream decodes on one thread, so each of its +/// three payloads is measured separately. fn deserialize_evidence_lineage<'de, D: serde::Deserializer<'de>>( deserializer: D, -) -> Result, D::Error> { +) -> Result { hotpath::measure_block!( "code_index.restore.evidence_lineage", Deserialize::deserialize(deserializer) @@ -354,16 +327,16 @@ fn deserialize_evidence_lineage<'de, D: serde::Deserializer<'de>>( fn deserialize_evidence_projection_request<'de, D: serde::Deserializer<'de>>( deserializer: D, -) -> Result { +) -> Result { hotpath::measure_block!( "code_index.restore.evidence_projection_request", Deserialize::deserialize(deserializer) ) } -fn deserialize_evidence_compact_receipt<'de, D: serde::Deserializer<'de>>( +fn deserialize_evidence_projection_receipt<'de, D: serde::Deserializer<'de>>( deserializer: D, -) -> Result { +) -> Result { hotpath::measure_block!( "code_index.restore.evidence_projection_receipt", Deserialize::deserialize(deserializer) @@ -371,141 +344,40 @@ fn deserialize_evidence_compact_receipt<'de, D: serde::Deserializer<'de>>( } #[derive(Serialize)] -struct PartitionedCompactGenerationEvidenceRefV1<'a> { - lineage: &'a [SymbolLineageCandidateV1], - projection_request: &'a ProjectionBatchRequestV1, - projection_receipt: PartitionedCompactBatchReceiptRefV1<'a>, +struct PartitionedGenerationEvidenceRefV1<'a> { + lineage: PersistedLineageV1, + projection_request: PersistedProjectionRequestRefV1<'a>, + projection_receipt: PersistedBatchReceiptRefV1<'a>, } -#[derive(Deserialize)] -#[serde(deny_unknown_fields)] -struct PartitionedCompactBatchReceiptV1 { - target_projection_key: ProjectionKeyV1, - request_digest: ManifestDigest, - source_generation: CodeGenerationId, - source_manifest_digest: ManifestDigest, - receipts: Vec, - reused_count: u64, - publication_digest: ManifestDigest, -} - -#[derive(Serialize)] -struct PartitionedCompactBatchReceiptRefV1<'a> { - target_projection_key: &'a ProjectionKeyV1, - request_digest: &'a ManifestDigest, - source_generation: &'a CodeGenerationId, - source_manifest_digest: &'a ManifestDigest, - receipts: Vec>, - reused_count: u64, - publication_digest: &'a ManifestDigest, -} - -/// The projector's decision for one chunk. The batch header supplies the -/// projection key, request digest, and generation watermarks; the request's -/// change row for `chunk_id` supplies the prior and current chunk digests. -#[derive(Deserialize)] -#[serde(deny_unknown_fields)] -struct PartitionedCompactChunkReceiptV1 { - chunk_id: CodeSearchChunkId, - operation: ProjectionOperationV1, - outcome: ProjectionOutcomeV1, - #[serde(default)] - output_digest: Option, -} - -#[derive(Serialize)] -struct PartitionedCompactChunkReceiptRefV1<'a> { - chunk_id: &'a CodeSearchChunkId, - operation: ProjectionOperationV1, - outcome: &'a ProjectionOutcomeV1, - #[serde(skip_serializing_if = "Option::is_none")] - output_digest: Option<&'a ContentDigest>, -} - -impl<'a> PartitionedCompactGenerationEvidenceRefV1<'a> { - fn new(generation: &'a CodeIndexPublishedGenerationV1) -> Self { - let receipt = generation.projection.receipt(); - Self { - lineage: &generation.lineage, - projection_request: generation.projection.request(), - projection_receipt: PartitionedCompactBatchReceiptRefV1 { - target_projection_key: &receipt.target_projection_key, - request_digest: &receipt.request_digest, - source_generation: &receipt.source_generation, - source_manifest_digest: &receipt.source_manifest_digest, - receipts: receipt - .receipts - .iter() - .map(|receipt| PartitionedCompactChunkReceiptRefV1 { - chunk_id: &receipt.chunk_id, - operation: receipt.operation, - outcome: &receipt.outcome, - output_digest: receipt.output_digest.as_ref(), - }) - .collect(), - reused_count: receipt.reused_count, - publication_digest: &receipt.publication_digest, - }, - } - } -} - -impl PartitionedCompactGenerationEvidenceV1 { - /// Rebuild the full receipt rows from the request. Every field restored - /// here is one the receipt verifier requires to equal the request, so the - /// batch's publication digest recomputes over the same bytes it sealed; - /// a decision row naming a chunk the request does not carry is a - /// contract failure, not a receipt the verifier gets to judge. - fn expand(self) -> Result { - let request = self.projection_request; - let compact = self.projection_receipt; - let changes = &request.changes; - let change_digests = changes - .added_or_changed - .iter() - .chain(&changes.deleted) - .map(|change| { - ( - &change.chunk_id, - (&change.prior_digest, &change.current_digest), - ) - }) - .collect::>(); - let mut receipts = Vec::with_capacity(compact.receipts.len()); - for receipt in compact.receipts { - let (prior_chunk_digest, current_chunk_digest) = - change_digests.get(&receipt.chunk_id).ok_or_else(|| { - CodeIndexProductionErrorV1::Contract( - "sealed generation receipt names a chunk outside its projection request" - .to_owned(), - ) - })?; - receipts.push(CodeChunkProjectionReceiptV1 { - projection_key: compact.target_projection_key.clone(), - request_digest: compact.request_digest.clone(), - prior_generation: changes.from_generation.clone(), - source_generation: compact.source_generation.clone(), - source_manifest_digest: compact.source_manifest_digest.clone(), - chunk_id: receipt.chunk_id, - prior_chunk_digest: (*prior_chunk_digest).clone(), - current_chunk_digest: (*current_chunk_digest).clone(), - operation: receipt.operation, - outcome: receipt.outcome, - output_digest: receipt.output_digest, - }); - } - Ok(PartitionedGenerationEvidenceV1 { - lineage: self.lineage, - projection_request: request, - projection_receipt: ProjectionBatchReceiptV1 { - target_projection_key: compact.target_projection_key, - request_digest: compact.request_digest, - source_generation: compact.source_generation, - source_manifest_digest: compact.source_manifest_digest, - receipts, - reused_count: compact.reused_count, - publication_digest: compact.publication_digest, - }, +impl<'a> PartitionedGenerationEvidenceRefV1<'a> { + fn new( + generation: &'a CodeIndexPublishedGenerationV1, + ) -> Result { + let symbols = occurrence_roster( + generation + .files + .iter() + .flat_map(|file| file.artifacts.symbols.iter()), + ); + let chunks = chunk_roster( + generation + .files + .iter() + .flat_map(|file| file.artifacts.chunks.chunks.iter()), + ); + let request = generation.projection.request(); + Ok(Self { + lineage: PersistedLineageV1::compact( + &generation.lineage, + &generation.manifest.generation_id, + &symbols, + )?, + projection_request: PersistedProjectionRequestRefV1::new(request, &chunks)?, + projection_receipt: PersistedBatchReceiptRefV1::new( + request, + generation.projection.receipt(), + )?, }) } } @@ -797,8 +669,6 @@ where segment_digest, segment_size_bytes: self.segment_size_bytes, pages: std::mem::take(&mut self.descriptors), - encoding: PartitionedEvidenceEncodingV1::CompactReceipts, - legacy_unpaged: false, }) } @@ -876,19 +746,13 @@ struct PartitionedEvidencePageReaderV1<'a, R> { page: Vec, page_offset: usize, next_page: usize, - segment_offset: u64, - /// The aggregate segment identity, computed only for a pre-paging segment. - /// - /// A paged segment's manifest carries a digest per page, and the manifest - /// itself is authenticated before one page is requested. Every byte the - /// stream yields therefore arrives inside a page this reader already - /// verified against that manifest, the page table's sizes must sum to the + /// The aggregate segment digest is never recomputed. The manifest carries + /// a digest per page and is itself authenticated before one page is + /// requested, so every byte the stream yields arrives inside a page this + /// reader already verified; the page table's sizes must sum to the /// segment size, and [`Self::finish`] refuses unless every page was read - /// and drained, so re-hashing the concatenation attests nothing the page - /// digests have not already attested. A pre-paging segment has no page - /// table, so there the aggregate identity is the only attestation and is - /// still computed and compared. - segment_hasher: Option, + /// and drained. + segment_offset: u64, read_error: Option, } @@ -910,7 +774,6 @@ where page_offset: 0, next_page: 0, segment_offset: 0, - segment_hasher: descriptor.legacy_unpaged.then(Sha256::new), read_error: None, } } @@ -920,56 +783,7 @@ where std::io::Error::other("sealed generation evidence page read failed") } - /// A pre-paging segment carries no page table, so it is read in the same - /// bounded ranges a paged segment would have used. Only the aggregate - /// digest authenticates it, there are no per-page digests to check, and - /// `finish` still refuses a segment whose bytes do not hash to its - /// manifest identity. - fn load_next_legacy_chunk(&mut self) -> std::io::Result { - let Some(remaining) = self - .descriptor - .segment_size_bytes - .checked_sub(self.segment_offset) - .filter(|remaining| *remaining > 0) - else { - return Ok(false); - }; - let page_max = u64::try_from(GENERATION_EVIDENCE_PAGE_MAX_BYTES_V1).map_err(|_| { - self.remember_error(CodeIndexProductionErrorV1::Contract( - "sealed generation evidence page bound exceeds u64".to_owned(), - )) - })?; - let length = remaining.min(page_max); - self.page.clear(); - self.page_offset = 0; - if let Err(error) = (self.read_segment)( - SealedGenerationSegmentReadV1::Range { - digest: &self.descriptor.segment_digest, - size_bytes: self.descriptor.segment_size_bytes, - offset: self.segment_offset, - length, - }, - &mut self.page, - ) { - return Err(self.remember_error(error)); - } - if u64::try_from(self.page.len()).is_ok_and(|read| read == length) { - if let Some(hasher) = self.segment_hasher.as_mut() { - hasher.update(&self.page); - } - self.next_page += 1; - self.segment_offset += length; - return Ok(true); - } - Err(self.remember_error(CodeIndexProductionErrorV1::Contract( - "sealed generation evidence byte size does not match its manifest".to_owned(), - ))) - } - fn load_next_page(&mut self) -> std::io::Result { - if self.descriptor.legacy_unpaged { - return self.load_next_legacy_chunk(); - } let Some(descriptor) = self.descriptor.pages.get(self.next_page) else { return Ok(false); }; @@ -1017,9 +831,7 @@ where if let Some(error) = self.read_error.take() { return Err(error); } - let pages_drained = - self.descriptor.legacy_unpaged || self.next_page == self.descriptor.pages.len(); - if !pages_drained + if self.next_page != self.descriptor.pages.len() || self.page_offset != self.page.len() || self.segment_offset != self.descriptor.segment_size_bytes { @@ -1028,16 +840,6 @@ where .to_owned(), )); } - let Some(hasher) = self.segment_hasher else { - return Ok(()); - }; - let segment_digest = ManifestDigest::from_sha256_bytes(&hasher.finalize()) - .map_err(|error| CodeIndexProductionErrorV1::Contract(error.to_string()))?; - if segment_digest != self.descriptor.segment_digest { - return Err(CodeIndexProductionErrorV1::Contract( - "sealed generation evidence segment digest does not match its manifest".to_owned(), - )); - } Ok(()) } } @@ -1134,6 +936,7 @@ impl PartitionedSegmentEncoderV1 { fn encode_file_segment( &mut self, generation_id: &CodeGenerationId, + scope: &FileScopeIdentityV1, file: &FileGenerationArtifactsV1, file_key: u32, ) -> Result { @@ -1141,10 +944,11 @@ impl PartitionedSegmentEncoderV1 { serde_json::to_writer( &mut self.payload, &PersistedFileGenerationArtifactsRefV2::new( + scope, &file.authority, &file.extraction, &file.artifacts, - ), + )?, ) .map_err(|error| { CodeIndexProductionErrorV1::Contract(format!( @@ -1158,7 +962,7 @@ impl PartitionedSegmentEncoderV1 { .map(|symbol| (symbol.occurrence.as_str(), &symbol.identity)) .collect::>(); self.encode_serialized_file_segment( - FILE_SEGMENT_FORMAT_REVISION_V3, + FILE_SEGMENT_FORMAT_REVISION, generation_id, file.artifacts .clone_bodies @@ -1171,8 +975,10 @@ impl PartitionedSegmentEncoderV1 { } /// Rewrite the serialization already staged in `payload` into one canonical - /// segment. The typed and test entry points share this single authority so - /// production never carries a second encoder. + /// segment and compress it into `segment`. The typed and test entry points + /// share this single authority so production never carries a second + /// encoder. The content address covers the stored bytes, which the store + /// and retention verify without inflating them. #[hotpath::measure(label = "code_index.sealed_encode.file_rewrite")] fn encode_serialized_file_segment<'s>( &mut self, @@ -1199,11 +1005,6 @@ impl PartitionedSegmentEncoderV1 { }, )?; let symbol_identities = ordered_identities.into_iter().cloned().collect::>(); - let symbol_occurrences = symbol_identities - .iter() - .map(|identity| crate::chunks::symbol_occurrence_id(&file_occurrence_id, identity)) - .collect::, _>>() - .map_err(|error| CodeIndexProductionErrorV1::Contract(error.to_string()))?; let identity_keys = symbol_identities .iter() .enumerate() @@ -1227,28 +1028,47 @@ impl PartitionedSegmentEncoderV1 { }; segment.clear(); segment.extend_from_slice(b"{\"format_revision\":"); - serde_json::to_writer(&mut *segment, &format_revision).map_err(|error| { + let serialization_failed = |error: serde_json::Error| { CodeIndexProductionErrorV1::Contract(format!( "sealed file segment serialization failed: {error}" )) - })?; + }; + serde_json::to_writer(&mut *segment, &format_revision).map_err(serialization_failed)?; + segment.extend_from_slice(b",\"symbol_identities\":"); + serde_json::to_writer(&mut *segment, &symbol_identities).map_err(serialization_failed)?; segment.extend_from_slice(b",\"file\":"); canonicalize_json_into(payload, &mut policy, segment)?; segment.push(b'}'); + let length = |bytes: &[u8]| { + u64::try_from(bytes.len()).map_err(|_| { + CodeIndexProductionErrorV1::Contract( + "sealed file segment length exceeds u64".to_owned(), + ) + }) + }; + let decoded_size_bytes = length(segment)?; + let compression_failed = |error: std::io::Error| { + CodeIndexProductionErrorV1::Contract(format!( + "sealed file segment compression failed: {error}" + )) + }; + payload.clear(); + let mut encoder = DeflateEncoder::new( + std::mem::take(payload), + Compression::new(FILE_SEGMENT_COMPRESSION_LEVEL), + ); + encoder.write_all(segment).map_err(compression_failed)?; + *payload = encoder.finish().map_err(compression_failed)?; + std::mem::swap(payload, segment); let segment_digest = ManifestDigest::from_sha256_bytes(&Sha256::digest(&*segment)) .map_err(|error| CodeIndexProductionErrorV1::Contract(error.to_string()))?; - let segment_size_bytes = u64::try_from(segment.len()).map_err(|_| { - CodeIndexProductionErrorV1::Contract( - "sealed file segment length exceeds u64".to_owned(), - ) - })?; Ok(PartitionedFileSegmentDescriptorV1 { file_key, segment_digest, - segment_size_bytes, + segment_size_bytes: length(segment)?, + decoded_size_bytes, file_occurrence_id, - symbol_identities, - symbol_occurrences, + symbol_identities_digest: symbol_identities_digest(&symbol_identities)?, }) } @@ -1265,11 +1085,9 @@ impl PartitionedSegmentEncoderV1 { // one bounded page plus its compact content-address descriptors. drop(std::mem::take(&mut self.payload)); drop(std::mem::take(&mut self.segment)); + let evidence = PartitionedGenerationEvidenceRefV1::new(generation)?; let mut writer = PartitionedEvidencePageWriterV1::new(&mut publish); - let encoded = serde_json::to_writer( - &mut writer, - &PartitionedCompactGenerationEvidenceRefV1::new(generation), - ); + let encoded = serde_json::to_writer(&mut writer, &evidence); if let Some(error) = writer.take_publish_error() { return Err(error); } @@ -1311,6 +1129,7 @@ fn decode_file_segment( descriptor: &PartitionedFileSegmentDescriptorV1, generation_id: &CodeGenerationId, snapshot_digest: &ManifestDigest, + scope: &FileScopeIdentityV1, bytes: &[u8], restored: &mut Vec, ) -> Result { @@ -1327,1009 +1146,153 @@ fn decode_file_segment( )?; hotpath::measure_block!( "code_index.restore.segment_decode", - decode_verified_file_segment(descriptor, generation_id, snapshot_digest, bytes, restored,) + decode_verified_file_segment( + descriptor, + generation_id, + snapshot_digest, + scope, + bytes, + restored, + ) ) } +/// Inflate stored segment bytes into exactly `decoded_size_bytes` of +/// canonical JSON; the manifest-authenticated length also bounds the output. +fn inflate_file_segment( + bytes: &[u8], + decoded_size_bytes: u64, + out: &mut Vec, +) -> Result<(), CodeIndexProductionErrorV1> { + let expected = usize::try_from(decoded_size_bytes).map_err(|_| { + CodeIndexProductionErrorV1::Contract( + "sealed file segment decoded size exceeds addressable memory".to_owned(), + ) + })?; + out.clear(); + out.try_reserve_exact(expected).map_err(|error| { + CodeIndexProductionErrorV1::Contract(format!( + "sealed file segment decoded size cannot be allocated: {error}" + )) + })?; + DeflateDecoder::new(bytes) + .take(decoded_size_bytes.saturating_add(1)) + .read_to_end(out) + .map_err(|error| { + CodeIndexProductionErrorV1::Contract(format!( + "sealed file segment does not inflate: {error}" + )) + })?; + if out.len() != expected { + return Err(CodeIndexProductionErrorV1::Contract( + "sealed file segment does not inflate to its manifest size".to_owned(), + )); + } + Ok(()) +} + /// Decode a segment whose bytes already verified against the manifest. fn decode_verified_file_segment( descriptor: &PartitionedFileSegmentDescriptorV1, generation_id: &CodeGenerationId, snapshot_digest: &ManifestDigest, + scope: &FileScopeIdentityV1, bytes: &[u8], restored: &mut Vec, ) -> Result { hotpath::gauge!("code_index.restore.segment_bytes_total").inc(bytes.len()); + let mut canonical = Vec::new(); + hotpath::measure_block!( + "code_index.restore.segment_inflate", + inflate_file_segment(bytes, descriptor.decoded_size_bytes, &mut canonical) + )?; let segment: PartitionedRawFileSegmentV1 = hotpath::measure_block!( "code_index.restore.segment_parse", - serde_json::from_slice(bytes).map_err(|error| { + serde_json::from_slice(&canonical).map_err(|error| { CodeIndexProductionErrorV1::Contract(format!( "sealed file segment decoding failed: {error}" )) }) )?; - if segment.format_revision != FILE_SEGMENT_FORMAT_REVISION_V3 { - return Err(CodeIndexProductionErrorV1::SealedRowContractRefused { - revision: segment.format_revision, - message: "sealed file segment predates clone body rows".to_owned(), - }); - } - let mut policy = FileSegmentDecodePolicyV1 { - generation_id: generation_id.as_str(), - snapshot_digest: snapshot_digest.as_str(), - file_occurrence_id: descriptor.file_occurrence_id.as_str(), - symbol_occurrences: &descriptor.symbol_occurrences, - }; - restored.clear(); - let identity_restore = hotpath::measure_block!( - "code_index.restore.segment_identity_restore", - canonicalize_json_into(segment.file.get().as_bytes(), &mut policy, restored) - ); - hotpath::gauge!("code_index.restore.identity_restored_bytes_total").inc(restored.len()); - identity_restore?; - let payload_decoding_failed = |error: serde_json::Error| { - // The payload already parsed as canonical JSON under its verified - // digest, so a data-shaped refusal (missing or unknown field) is an - // older writer's row contract, not damaged bytes. - if error.classify() == serde_json::error::Category::Data { - return CodeIndexProductionErrorV1::SealedRowContractRefused { - revision: segment.format_revision, - message: format!("sealed file segment payload decoding failed: {error}"), - }; - } - CodeIndexProductionErrorV1::Contract(format!( - "sealed file segment payload decoding failed: {error}" - )) - }; - let mut file: PersistedFileGenerationArtifactsV1 = hotpath::measure_block!( - "code_index.restore.segment_typed_deserialize_expand", - serde_json::from_slice::(restored) - .map_err(payload_decoding_failed) - .and_then(PersistedFileGenerationArtifactsV2::expand) - )?; - hotpath::measure_block!("code_index.restore.segment_artifact_sorts", { - file.artifacts - .symbols - .sort_by(|left, right| left.occurrence.cmp(&right.occurrence)); - file.artifacts.edges.sort_by(|left, right| { - crate::chunks::canonical_edge_key(left).cmp(&crate::chunks::canonical_edge_key(right)) - }); - file.artifacts.clone_bodies.sort_by(|left, right| { - left.occurrence - .symbol_occurrence_id - .cmp(&right.occurrence.symbol_occurrence_id) - }); - file.artifacts.unresolved_references.sort(); - }); - Ok(file) -} - -fn legacy_generation_identity_field(key: &str) -> bool { - matches!( - key, - "generation_id" - | "from_generation" - | "to_generation" - | "prior_generation" - | "source_generation" - ) -} - -fn legacy_symbol_identity_field(key: &str) -> bool { - matches!( - key, - "occurrence" - | "from_occurrence" - | "to_occurrence" - | "prior_occurrence" - | "current_occurrence" - | "alternatives" - | "symbol_occurrence_id" - | "symbol_occurrence_ids" - ) -} - -fn legacy_chunk_identity_field(key: &str) -> bool { - matches!(key, "chunk_id" | "chunk_ids" | "parent_chunk_id") -} - -fn legacy_identity_marker_indices( - identity: &str, - prefix: &str, - invalid_key_message: &'static str, -) -> Result<(usize, usize), CodeIndexProductionErrorV1> { - let (file_key, item_key) = identity - .strip_prefix(prefix) - .and_then(|marker| marker.split_once(':')) - .and_then(|(file_key, item_key)| Some((file_key.parse().ok()?, item_key.parse().ok()?))) - .ok_or_else(|| CodeIndexProductionErrorV1::Contract(invalid_key_message.to_owned()))?; - Ok((file_key, item_key)) -} - -fn legacy_symbol_identity<'a>( - identity: &str, - file_segments: &'a [PartitionedFileSegmentDescriptorV1], -) -> Result<&'a str, CodeIndexProductionErrorV1> { - let (file_key, symbol_key) = legacy_identity_marker_indices( - identity, - SYMBOL_OCCURRENCE_ID_MARKER_PREFIX, - "sealed generation evidence contains an invalid symbol key", - )?; - file_segments - .get(file_key) - .and_then(|descriptor| descriptor.symbol_occurrences.get(symbol_key)) - .map(SymbolOccurrenceId::as_str) - .ok_or_else(|| { - CodeIndexProductionErrorV1::Contract( - "sealed generation evidence contains an invalid symbol key".to_owned(), - ) - }) -} - -fn legacy_chunk_identity<'a>( - identity: &str, - files: &'a [PersistedFileGenerationArtifactsV1], -) -> Result<&'a str, CodeIndexProductionErrorV1> { - let (file_key, chunk_key) = legacy_identity_marker_indices( - identity, - CHUNK_ID_MARKER_PREFIX, - "sealed generation evidence contains an invalid chunk key", - )?; - files - .get(file_key) - .and_then(|file| file.artifacts.chunks.chunks.get(chunk_key)) - .map(|chunk| chunk.id.as_str()) - .ok_or_else(|| { - CodeIndexProductionErrorV1::Contract( - "sealed generation evidence contains an invalid chunk key".to_owned(), - ) - }) -} - -/// Streaming identity restoration for the pre-paging evidence segment. -/// -/// The shipped restore read the whole evidence segment, parsed it into a -/// `serde_json::Value`, substituted identities in that tree, then deserialized -/// the tree into the typed payload, peak memory was the segment plus a DOM -/// plus the payload, measured at 2.35x the on-disk generation and linear in -/// corpus size. This module runs the identical substitution as a `serde` -/// transcoder wrapped around the same bounded page reader the paged form uses, -/// so a legacy restore retains one page and the typed payload and nothing else. -/// -/// The classification rules are the replaced DOM walk's, unchanged: a string is -/// substituted by the object key that encloses it, the classification resets at -/// every object member and is inherited through arrays. -mod legacy_identity { - use std::cell::Cell; - use std::fmt; - - use serde::de::{ - self, DeserializeOwned, DeserializeSeed, Deserializer, EnumAccess, MapAccess, SeqAccess, - VariantAccess, Visitor, - }; - - use super::{ - CHUNK_ID_MARKER_PREFIX, CodeIndexProductionErrorV1, GENERATION_ID_MARKER, - PartitionedFileSegmentDescriptorV1, PersistedFileGenerationArtifactsV1, - SYMBOL_OCCURRENCE_ID_MARKER_PREFIX, legacy_chunk_identity, legacy_chunk_identity_field, - legacy_generation_identity_field, legacy_symbol_identity, legacy_symbol_identity_field, - }; - - #[derive(Clone, Copy)] - enum LegacyIdentityFieldV1 { - Other, - Generation, - SymbolOccurrence, - Chunk, - } - - fn field_for_key(key: &str) -> LegacyIdentityFieldV1 { - if legacy_generation_identity_field(key) { - LegacyIdentityFieldV1::Generation - } else if legacy_symbol_identity_field(key) { - LegacyIdentityFieldV1::SymbolOccurrence - } else if legacy_chunk_identity_field(key) { - LegacyIdentityFieldV1::Chunk - } else { - LegacyIdentityFieldV1::Other - } - } - - /// The index the markers address: identities are resolved by position, so - /// the lookup borrows from the already-restored manifest and files instead - /// of building a map. - pub(super) struct LegacyIdentityIndexV1<'a> { - pub(super) generation_id: &'a str, - pub(super) file_segments: &'a [PartitionedFileSegmentDescriptorV1], - pub(super) files: &'a [PersistedFileGenerationArtifactsV1], - } - - impl<'a> LegacyIdentityIndexV1<'a> { - fn restore( - &self, - field: LegacyIdentityFieldV1, - value: &str, - ) -> Result, CodeIndexProductionErrorV1> { - match field { - LegacyIdentityFieldV1::Generation if value == GENERATION_ID_MARKER => { - Ok(Some(self.generation_id)) - } - LegacyIdentityFieldV1::SymbolOccurrence - if value.starts_with(SYMBOL_OCCURRENCE_ID_MARKER_PREFIX) => - { - legacy_symbol_identity(value, self.file_segments).map(Some) - } - LegacyIdentityFieldV1::Chunk if value.starts_with(CHUNK_ID_MARKER_PREFIX) => { - legacy_chunk_identity(value, self.files).map(Some) - } - LegacyIdentityFieldV1::Other - | LegacyIdentityFieldV1::Generation - | LegacyIdentityFieldV1::SymbolOccurrence - | LegacyIdentityFieldV1::Chunk => Ok(None), - } - } - } - - /// The transcoder's shared state. `failure` carries a restore rejection out - /// of `serde`'s error type unchanged; `captured_key` hands one object key's - /// classification from the key seed back to the map that read it, which is - /// sound because JSON object keys are strings and never nest. - struct RestoreContextV1<'a> { - index: LegacyIdentityIndexV1<'a>, - failure: Cell>, - captured_key: Cell, - } - - impl<'a> RestoreContextV1<'a> { - fn restore( - &self, - field: LegacyIdentityFieldV1, - value: &str, - ) -> Result, E> { - self.index.restore(field, value).map_err(|error| { - let message = error.to_string(); - self.failure.set(Some(error)); - E::custom(message) - }) - } - } - - /// Deserialize `reader` into `T`, restoring legacy identity markers as the - /// stream is read. The restore rejection, when there is one, is returned - /// beside the `serde` error so the caller reports the original contract - /// message rather than its stringified form. - pub(super) fn deserialize_restored( - reader: R, - index: LegacyIdentityIndexV1<'_>, - ) -> ( - Result, - Option, - ) - where - T: DeserializeOwned, - R: std::io::Read, - { - let context = RestoreContextV1 { - index, - failure: Cell::new(None), - captured_key: Cell::new(LegacyIdentityFieldV1::Other), - }; - let mut deserializer = serde_json::Deserializer::from_reader(reader); - // `serde_json::from_reader` is `T::deserialize` followed by `end()`; - // keep the trailing-byte rejection the replaced call had. - let decoded = T::deserialize(RestoringDeserializerV1 { - inner: &mut deserializer, - field: LegacyIdentityFieldV1::Other, - capture_key: false, - context: &context, - }) - .and_then(|value| deserializer.end().map(|()| value)); - (decoded, context.failure.take()) - } - - struct RestoringDeserializerV1<'c, 'a, D> { - inner: D, - field: LegacyIdentityFieldV1, - capture_key: bool, - context: &'c RestoreContextV1<'a>, - } - - macro_rules! forward_deserialize { - ($($method:ident),* $(,)?) => { - $( - fn $method(self, visitor: V) -> Result - where - V: Visitor<'de>, - { - let Self { inner, field, capture_key, context } = self; - inner.$method(RestoringVisitorV1 { inner: visitor, field, capture_key, context }) - } - )* - }; - } - - impl<'de, 'c, 'a, D> Deserializer<'de> for RestoringDeserializerV1<'c, 'a, D> - where - D: Deserializer<'de>, - { - type Error = D::Error; - - forward_deserialize!( - deserialize_any, - deserialize_bool, - deserialize_i8, - deserialize_i16, - deserialize_i32, - deserialize_i64, - deserialize_i128, - deserialize_u8, - deserialize_u16, - deserialize_u32, - deserialize_u64, - deserialize_u128, - deserialize_f32, - deserialize_f64, - deserialize_char, - deserialize_str, - deserialize_string, - deserialize_bytes, - deserialize_byte_buf, - deserialize_option, - deserialize_unit, - deserialize_seq, - deserialize_map, - deserialize_identifier, - deserialize_ignored_any, - ); - - fn deserialize_unit_struct( - self, - name: &'static str, - visitor: V, - ) -> Result - where - V: Visitor<'de>, - { - let Self { - inner, - field, - capture_key, - context, - } = self; - inner.deserialize_unit_struct( - name, - RestoringVisitorV1 { - inner: visitor, - field, - capture_key, - context, - }, - ) - } - - fn deserialize_newtype_struct( - self, - name: &'static str, - visitor: V, - ) -> Result - where - V: Visitor<'de>, - { - let Self { - inner, - field, - capture_key, - context, - } = self; - inner.deserialize_newtype_struct( - name, - RestoringVisitorV1 { - inner: visitor, - field, - capture_key, - context, - }, - ) - } - - fn deserialize_tuple(self, len: usize, visitor: V) -> Result - where - V: Visitor<'de>, - { - let Self { - inner, - field, - capture_key, - context, - } = self; - inner.deserialize_tuple( - len, - RestoringVisitorV1 { - inner: visitor, - field, - capture_key, - context, - }, - ) - } - - fn deserialize_tuple_struct( - self, - name: &'static str, - len: usize, - visitor: V, - ) -> Result - where - V: Visitor<'de>, - { - let Self { - inner, - field, - capture_key, - context, - } = self; - inner.deserialize_tuple_struct( - name, - len, - RestoringVisitorV1 { - inner: visitor, - field, - capture_key, - context, - }, - ) - } - - fn deserialize_struct( - self, - name: &'static str, - fields: &'static [&'static str], - visitor: V, - ) -> Result - where - V: Visitor<'de>, - { - let Self { - inner, - field, - capture_key, - context, - } = self; - inner.deserialize_struct( - name, - fields, - RestoringVisitorV1 { - inner: visitor, - field, - capture_key, - context, - }, - ) - } - - fn deserialize_enum( - self, - name: &'static str, - variants: &'static [&'static str], - visitor: V, - ) -> Result - where - V: Visitor<'de>, - { - let Self { - inner, - field, - capture_key, - context, - } = self; - inner.deserialize_enum( - name, - variants, - RestoringVisitorV1 { - inner: visitor, - field, - capture_key, - context, - }, - ) - } - - fn is_human_readable(&self) -> bool { - self.inner.is_human_readable() - } - } - - struct RestoringVisitorV1<'c, 'a, V> { - inner: V, - field: LegacyIdentityFieldV1, - capture_key: bool, - context: &'c RestoreContextV1<'a>, - } - - macro_rules! forward_visit { - ($($method:ident($argument:ty)),* $(,)?) => { - $( - fn $method(self, value: $argument) -> Result - where - E: de::Error, - { - self.inner.$method(value) - } - )* - }; - } - - impl<'de, 'c, 'a, V> Visitor<'de> for RestoringVisitorV1<'c, 'a, V> - where - V: Visitor<'de>, - { - type Value = V::Value; - - fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - self.inner.expecting(formatter) - } - - forward_visit!( - visit_bool(bool), - visit_i8(i8), - visit_i16(i16), - visit_i32(i32), - visit_i64(i64), - visit_i128(i128), - visit_u8(u8), - visit_u16(u16), - visit_u32(u32), - visit_u64(u64), - visit_u128(u128), - visit_f32(f32), - visit_f64(f64), - visit_char(char), - visit_bytes(&[u8]), - visit_borrowed_bytes(&'de [u8]), - visit_byte_buf(Vec), - ); - - fn visit_none(self) -> Result - where - E: de::Error, - { - self.inner.visit_none() - } - - fn visit_unit(self) -> Result - where - E: de::Error, - { - self.inner.visit_unit() - } - - fn visit_str(self, value: &str) -> Result - where - E: de::Error, - { - let Self { - inner, - field, - capture_key, - context, - } = self; - if capture_key { - context.captured_key.set(field_for_key(value)); - return inner.visit_str(value); - } - match context.restore::(field, value)? { - Some(restored) => inner.visit_str(restored), - None => inner.visit_str(value), - } - } - - fn visit_borrowed_str(self, value: &'de str) -> Result - where - E: de::Error, - { - let Self { - inner, - field, - capture_key, - context, - } = self; - if capture_key { - context.captured_key.set(field_for_key(value)); - return inner.visit_borrowed_str(value); - } - match context.restore::(field, value)? { - Some(restored) => inner.visit_str(restored), - None => inner.visit_borrowed_str(value), - } - } - - fn visit_string(self, value: String) -> Result - where - E: de::Error, - { - let Self { - inner, - field, - capture_key, - context, - } = self; - if capture_key { - context.captured_key.set(field_for_key(value.as_str())); - return inner.visit_string(value); - } - let restored = context.restore::(field, value.as_str())?; - match restored { - Some(restored) => inner.visit_str(restored), - None => inner.visit_string(value), - } - } - - fn visit_some(self, deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let Self { - inner, - field, - context, - .. - } = self; - inner.visit_some(RestoringDeserializerV1 { - inner: deserializer, - field, - capture_key: false, - context, - }) - } - - fn visit_newtype_struct(self, deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let Self { - inner, - field, - context, - .. - } = self; - inner.visit_newtype_struct(RestoringDeserializerV1 { - inner: deserializer, - field, - capture_key: false, - context, - }) - } - - fn visit_seq(self, seq: A) -> Result - where - A: SeqAccess<'de>, - { - let Self { - inner, - field, - context, - .. - } = self; - inner.visit_seq(RestoringSeqV1 { - inner: seq, - field, - context, - }) - } - - fn visit_map(self, map: A) -> Result - where - A: MapAccess<'de>, - { - let Self { inner, context, .. } = self; - inner.visit_map(RestoringMapV1 { - inner: map, - field: LegacyIdentityFieldV1::Other, - context, - }) - } - - fn visit_enum(self, data: A) -> Result - where - A: EnumAccess<'de>, - { - let Self { inner, context, .. } = self; - inner.visit_enum(RestoringEnumV1 { - inner: data, - context, - }) - } - } - - struct RestoringSeedV1<'c, 'a, T> { - inner: T, - field: LegacyIdentityFieldV1, - capture_key: bool, - context: &'c RestoreContextV1<'a>, - } - - impl<'de, 'c, 'a, T> DeserializeSeed<'de> for RestoringSeedV1<'c, 'a, T> - where - T: DeserializeSeed<'de>, - { - type Value = T::Value; - - fn deserialize(self, deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let Self { - inner, - field, - capture_key, - context, - } = self; - inner.deserialize(RestoringDeserializerV1 { - inner: deserializer, - field, - capture_key, - context, - }) - } - } - - /// Array elements inherit the enclosing key's classification, matching the - /// replaced `Value::Array` arm. - struct RestoringSeqV1<'c, 'a, A> { - inner: A, - field: LegacyIdentityFieldV1, - context: &'c RestoreContextV1<'a>, - } - - impl<'de, 'c, 'a, A> SeqAccess<'de> for RestoringSeqV1<'c, 'a, A> - where - A: SeqAccess<'de>, - { - type Error = A::Error; - - fn next_element_seed(&mut self, seed: T) -> Result, A::Error> - where - T: DeserializeSeed<'de>, - { - self.inner.next_element_seed(RestoringSeedV1 { - inner: seed, - field: self.field, - capture_key: false, - context: self.context, - }) - } - - fn size_hint(&self) -> Option { - self.inner.size_hint() - } - } - - /// Object members reset the classification to their own key, matching the - /// replaced `Value::Object` arm. - struct RestoringMapV1<'c, 'a, A> { - inner: A, - field: LegacyIdentityFieldV1, - context: &'c RestoreContextV1<'a>, - } - - impl<'de, 'c, 'a, A> MapAccess<'de> for RestoringMapV1<'c, 'a, A> - where - A: MapAccess<'de>, - { - type Error = A::Error; - - fn next_key_seed(&mut self, seed: K) -> Result, A::Error> - where - K: DeserializeSeed<'de>, - { - self.context.captured_key.set(LegacyIdentityFieldV1::Other); - let key = self.inner.next_key_seed(RestoringSeedV1 { - inner: seed, - field: LegacyIdentityFieldV1::Other, - capture_key: true, - context: self.context, - })?; - self.field = self.context.captured_key.get(); - Ok(key) - } - - fn next_value_seed(&mut self, seed: T) -> Result - where - T: DeserializeSeed<'de>, - { - self.inner.next_value_seed(RestoringSeedV1 { - inner: seed, - field: self.field, - capture_key: false, - context: self.context, - }) - } - - fn size_hint(&self) -> Option { - self.inner.size_hint() - } - } - - /// An externally tagged variant is one object member, so its content is - /// classified by the variant name exactly as an object key would classify - /// it. - struct RestoringEnumV1<'c, 'a, A> { - inner: A, - context: &'c RestoreContextV1<'a>, - } - - impl<'de, 'c, 'a, A> EnumAccess<'de> for RestoringEnumV1<'c, 'a, A> - where - A: EnumAccess<'de>, - { - type Error = A::Error; - type Variant = RestoringVariantV1<'c, 'a, A::Variant>; - - fn variant_seed(self, seed: T) -> Result<(T::Value, Self::Variant), A::Error> - where - T: DeserializeSeed<'de>, - { - self.context.captured_key.set(LegacyIdentityFieldV1::Other); - let (value, variant) = self.inner.variant_seed(RestoringSeedV1 { - inner: seed, - field: LegacyIdentityFieldV1::Other, - capture_key: true, - context: self.context, - })?; - Ok(( - value, - RestoringVariantV1 { - inner: variant, - field: self.context.captured_key.get(), - context: self.context, - }, - )) - } - } - - struct RestoringVariantV1<'c, 'a, A> { - inner: A, - field: LegacyIdentityFieldV1, - context: &'c RestoreContextV1<'a>, - } - - impl<'de, 'c, 'a, A> VariantAccess<'de> for RestoringVariantV1<'c, 'a, A> - where - A: VariantAccess<'de>, - { - type Error = A::Error; - - fn unit_variant(self) -> Result<(), A::Error> { - self.inner.unit_variant() - } - - fn newtype_variant_seed(self, seed: T) -> Result - where - T: DeserializeSeed<'de>, - { - self.inner.newtype_variant_seed(RestoringSeedV1 { - inner: seed, - field: self.field, - capture_key: false, - context: self.context, - }) - } - - fn tuple_variant(self, len: usize, visitor: V) -> Result - where - V: Visitor<'de>, - { - self.inner.tuple_variant( - len, - RestoringVisitorV1 { - inner: visitor, - field: self.field, - capture_key: false, - context: self.context, - }, - ) - } - - fn struct_variant( - self, - fields: &'static [&'static str], - visitor: V, - ) -> Result - where - V: Visitor<'de>, - { - self.inner.struct_variant( - fields, - RestoringVisitorV1 { - inner: visitor, - field: self.field, - capture_key: false, - context: self.context, - }, - ) - } + if segment.format_revision != FILE_SEGMENT_FORMAT_REVISION { + return Err(CodeIndexProductionErrorV1::SealedRowContractRefused { + revision: segment.format_revision, + message: "sealed file segment revision is not the one this build writes".to_owned(), + }); } + let symbol_occurrences = + bind_symbol_occurrences(&descriptor.file_occurrence_id, &segment.symbol_identities)?; + let mut policy = FileSegmentDecodePolicyV1 { + generation_id: generation_id.as_str(), + snapshot_digest: snapshot_digest.as_str(), + file_occurrence_id: descriptor.file_occurrence_id.as_str(), + symbol_occurrences: &symbol_occurrences, + }; + restored.clear(); + let identity_restore = hotpath::measure_block!( + "code_index.restore.segment_identity_restore", + canonicalize_json_into(segment.file.get().as_bytes(), &mut policy, restored) + ); + hotpath::gauge!("code_index.restore.identity_restored_bytes_total").inc(restored.len()); + identity_restore?; + let payload_decoding_failed = |error: serde_json::Error| { + // The payload already parsed as canonical JSON under its verified + // digest, so a data-shaped refusal (missing or unknown field) is an + // older writer's row contract, not damaged bytes. + if error.classify() == serde_json::error::Category::Data { + return CodeIndexProductionErrorV1::SealedRowContractRefused { + revision: segment.format_revision, + message: format!("sealed file segment payload decoding failed: {error}"), + }; + } + CodeIndexProductionErrorV1::Contract(format!( + "sealed file segment payload decoding failed: {error}" + )) + }; + let mut file: PersistedFileGenerationArtifactsV1 = hotpath::measure_block!( + "code_index.restore.segment_typed_deserialize_expand", + serde_json::from_slice::(restored) + .map_err(payload_decoding_failed) + .and_then(|file| file.expand(scope)) + )?; + hotpath::measure_block!("code_index.restore.segment_artifact_sorts", { + file.artifacts + .symbols + .sort_by(|left, right| left.occurrence.cmp(&right.occurrence)); + file.artifacts.edges.sort_by(|left, right| { + crate::chunks::canonical_edge_key(left).cmp(&crate::chunks::canonical_edge_key(right)) + }); + file.artifacts.clone_bodies.sort_by(|left, right| { + left.occurrence + .symbol_occurrence_id + .cmp(&right.occurrence.symbol_occurrence_id) + }); + file.artifacts.unresolved_references.sort(); + }); + Ok(file) } fn decode_generation_evidence( descriptor: &PartitionedGenerationEvidenceDescriptorV1, - generation_id: &CodeGenerationId, - file_segments: &[PartitionedFileSegmentDescriptorV1], - files: &[PersistedFileGenerationArtifactsV1], mut read_segment: impl FnMut( SealedGenerationSegmentReadV1<'_>, &mut Vec, ) -> Result<(), CodeIndexProductionErrorV1>, ) -> Result { let mut reader = PartitionedEvidencePageReaderV1::new(descriptor, &mut read_segment); - // A pre-paging segment carries identity markers; restore them while the - // stream is read rather than materializing the segment and a DOM. let decoding_failure = |error: serde_json::Error| { CodeIndexProductionErrorV1::Contract(format!( "sealed generation evidence payload decoding failed: {error}" )) }; - // Paging and receipt shape are orthogonal: the pre-paging segment is read - // through the identity-restoring transcoder, and either segment form then - // decodes as the shape its descriptor names. - let (decoded, restore_failure) = if descriptor.legacy_unpaged { - let index = legacy_identity::LegacyIdentityIndexV1 { - generation_id: generation_id.as_str(), - file_segments, - files, - }; - match descriptor.encoding { - PartitionedEvidenceEncodingV1::Typed => { - let (decoded, restore_failure) = legacy_identity::deserialize_restored::< - PartitionedGenerationEvidenceV1, - _, - >(&mut reader, index); - (decoded.map_err(decoding_failure), restore_failure) - } - PartitionedEvidenceEncodingV1::CompactReceipts => { - let (decoded, restore_failure) = legacy_identity::deserialize_restored::< - PartitionedCompactGenerationEvidenceV1, - _, - >(&mut reader, index); - ( - decoded - .map_err(decoding_failure) - .and_then(PartitionedCompactGenerationEvidenceV1::expand), - restore_failure, - ) - } - } - } else { - match descriptor.encoding { - PartitionedEvidenceEncodingV1::Typed => ( - serde_json::from_reader::<_, PartitionedGenerationEvidenceV1>(&mut reader) - .map_err(decoding_failure), - None, - ), - PartitionedEvidenceEncodingV1::CompactReceipts => ( - hotpath::measure_block!( - "code_index.restore.evidence_stream", - serde_json::from_reader::<_, PartitionedCompactGenerationEvidenceV1>( - &mut reader - ) - .map_err(decoding_failure) - ) - .and_then(|compact| { - hotpath::measure_block!( - "code_index.restore.evidence_receipt_expand", - compact.expand() - ) - }), - None, - ), - } - }; + let decoded = hotpath::measure_block!( + "code_index.restore.evidence_stream", + serde_json::from_reader::<_, PartitionedGenerationEvidenceV1>(&mut reader) + .map_err(decoding_failure) + ); if let Some(error) = reader.take_read_error() { return Err(error); } - if let Some(error) = restore_failure { - return Err(error); - } let evidence = decoded?; reader.finish()?; Ok(evidence) @@ -2343,7 +1306,7 @@ fn validate_partitioned_generation_layout<'a, I, J, K>( file_segments: I, snapshot_files: J, evidence_segment_size_bytes: u64, - pages: Option, + pages: K, ) -> Result<(), CodeIndexProductionErrorV1> where I: ExactSizeIterator, @@ -2370,14 +1333,6 @@ where )); } } - let Some(pages) = pages else { - if evidence_segment_size_bytes == 0 { - return Err(CodeIndexProductionErrorV1::Contract( - "legacy sealed generation evidence segment is empty".to_owned(), - )); - } - return Ok(()); - }; if pages.len() == 0 { return Err(CodeIndexProductionErrorV1::Contract( "sealed generation evidence has no pages".to_owned(), @@ -2419,7 +1374,7 @@ where fn parse_partitioned_manifest( bytes: &[u8], -) -> Result, CodeIndexProductionErrorV1> { +) -> Result { let raw: PartitionedRawEnvelopeV1 = hotpath::measure_block!( "code_index.restore.manifest_envelope_parse", serde_json::from_slice(bytes).map_err(|error| { @@ -2448,8 +1403,6 @@ fn parse_partitioned_manifest( )?; match probe.format_revision { SEALED_GENERATION_FORMAT_REVISION_V1 => {} - // The monolithic envelope, which its own decoder owns. - MONOLITHIC_SEALED_GENERATION_FORMAT_REVISION => return Ok(None), // Every other revision is a manifest this build refuses to read. A // retired one names a shape the writer no longer emits, so the caller // rebuilds the generation from its source tree instead of decoding @@ -2485,31 +1438,13 @@ fn parse_partitioned_manifest( .filter(|(_, file)| file.disposition == SnapshotFileDispositionV1::Present) .map(|(key, file)| (key, &file.file_occurrence_id)), generation.generation_evidence.segment_size_bytes, - (!generation.generation_evidence.legacy_unpaged).then(|| { - generation - .generation_evidence - .pages - .iter() - .map(|page| (page.page_ordinal, page.page_size_bytes)) - }), - )?; - Ok(Some(generation)) -} - -fn bind_file_segment_occurrences( - descriptors: &mut [PartitionedFileSegmentDescriptorV1], -) -> Result<(), CodeIndexProductionErrorV1> { - for descriptor in descriptors { - descriptor.symbol_occurrences = descriptor - .symbol_identities + generation + .generation_evidence + .pages .iter() - .map(|identity| { - crate::chunks::symbol_occurrence_id(&descriptor.file_occurrence_id, identity) - }) - .collect::, _>>() - .map_err(|error| CodeIndexProductionErrorV1::Contract(error.to_string()))?; - } - Ok(()) + .map(|page| (page.page_ordinal, page.page_size_bytes)), + )?; + Ok(generation) } fn snapshot_file_keys<'a>( @@ -2542,6 +1477,7 @@ type LexicalSegmentReaderV1 = dyn FnMut( pub(super) struct PartitionedLexicalFileSourceV1 { generation_id: CodeGenerationId, snapshot_digest: ManifestDigest, + scope: FileScopeIdentityV1, descriptors: Vec, read_segment: Box, } @@ -2561,6 +1497,26 @@ impl PartitionedLexicalFileSourceV1 { self.descriptors.len() } + /// Every input the content of the pages this source emits depends on + /// apart from the generation's route identity: each file segment in + /// order (its key, occurrence, content address, and symbol identities). + /// Worktrees that seal identical trees agree on it; where pages are cut + /// can still differ with route identity widths, which changes no row. + pub(super) fn content_digest(&self, hasher: &mut Sha256) { + hasher.update((self.descriptors.len() as u64).to_le_bytes()); + for descriptor in &self.descriptors { + hasher.update(descriptor.file_key.to_le_bytes()); + for field in [ + descriptor.file_occurrence_id.as_str(), + descriptor.segment_digest.as_str(), + descriptor.symbol_identities_digest.as_str(), + ] { + hasher.update((field.len() as u64).to_le_bytes()); + hasher.update(field.as_bytes()); + } + } + } + pub(super) fn lexical_byte_offsets(&self) -> Result, CodeIndexProductionErrorV1> { let mut offsets = Vec::with_capacity(self.descriptors.len().saturating_add(1)); offsets.push(0_u64); @@ -2582,7 +1538,7 @@ impl PartitionedLexicalFileSourceV1 { pub(super) fn maximum_file_bytes(&self) -> u64 { self.descriptors .iter() - .map(|descriptor| descriptor.segment_size_bytes) + .map(|descriptor| descriptor.decoded_size_bytes) .max() .unwrap_or(0) } @@ -2592,37 +1548,13 @@ impl PartitionedLexicalFileSourceV1 { self.descriptors .capacity() .saturating_mul(std::mem::size_of::()) - .saturating_add(self.generation_id.as_str().len()), + .saturating_add(self.generation_id.as_str().len()) + .saturating_add(self.scope.retained_bytes()), |bytes, descriptor| { bytes .saturating_add(descriptor.segment_digest.as_str().len()) .saturating_add(descriptor.file_occurrence_id.as_str().len()) - .saturating_add( - descriptor - .symbol_identities - .capacity() - .saturating_mul(std::mem::size_of::()), - ) - .saturating_add( - descriptor - .symbol_identities - .iter() - .map(|id| id.as_str().len()) - .sum::(), - ) - .saturating_add( - descriptor - .symbol_occurrences - .capacity() - .saturating_mul(std::mem::size_of::()), - ) - .saturating_add( - descriptor - .symbol_occurrences - .iter() - .map(|id| id.as_str().len()) - .sum::(), - ) + .saturating_add(descriptor.symbol_identities_digest.as_str().len()) }, ) } @@ -2637,6 +1569,7 @@ impl PartitionedLexicalFileSourceV1 { let Self { generation_id, snapshot_digest, + scope, descriptors, read_segment, } = self; @@ -2671,13 +1604,15 @@ impl PartitionedLexicalFileSourceV1 { &buffers[..read], generation_id, snapshot_digest, + scope, )?) } } /// Read the next window of segment bytes on the calling thread into /// `buffers`, one slot per file: at least one file, then as many as fit within -/// `buffers.len()` and `maximum_bytes`. Returns how many leading descriptors +/// `buffers.len()` and `maximum_bytes` of decoded segment JSON, the bytes the +/// window's decode materializes. Returns how many leading descriptors /// were read. Callers keep the same slots across windows, so a decode never /// holds more than `buffers.len()` segments and each slot grows only to the /// largest segment it has read (the bound @@ -2695,7 +1630,7 @@ fn read_segment_window( let mut read = 0; let mut bytes = 0u64; for (descriptor, buffer) in descriptors.iter().zip(buffers) { - if read > 0 && bytes.saturating_add(descriptor.segment_size_bytes) > maximum_bytes { + if read > 0 && bytes.saturating_add(descriptor.decoded_size_bytes) > maximum_bytes { break; } buffer.clear(); @@ -2703,7 +1638,7 @@ fn read_segment_window( "code_index.restore.segment_read", read_segment(descriptor, buffer) )?; - bytes = bytes.saturating_add(descriptor.segment_size_bytes); + bytes = bytes.saturating_add(descriptor.decoded_size_bytes); read += 1; } Ok(read) @@ -2722,6 +1657,7 @@ fn decode_segment_window( segments: &[Vec], generation_id: &CodeGenerationId, snapshot_digest: &ManifestDigest, + scope: &FileScopeIdentityV1, ) -> Result, CodeIndexProductionErrorV1> { let window = descriptors.iter().zip(segments).collect::>(); collect_bounded_ordered(&window, |(descriptor, segment), _worker| { @@ -2730,15 +1666,15 @@ fn decode_segment_window( descriptor, generation_id, snapshot_digest, + scope, segment, &mut restored, ) }) } -impl VerifiedSealedLexicalPageSourceV1 { +impl VerifiedSealedLexicalPageSourceV1 { pub fn open_partitioned_sealed( - reader: R, manifest_bytes: &[u8], source_state_digest: ManifestDigest, read_segment: impl FnMut( @@ -2751,28 +1687,24 @@ impl VerifiedSealedLexicalPageSourceV1 { + 'static, maximum_page_chunks: usize, maximum_page_bytes: usize, - ) -> Result, CodeIndexProductionErrorV1> { - let Some(mut generation) = parse_partitioned_manifest(manifest_bytes)? else { - return Ok(None); - }; - bind_file_segment_occurrences(&mut generation.file_segments)?; + ) -> Result { + let generation = parse_partitioned_manifest(manifest_bytes)?; let source = PartitionedLexicalFileSourceV1 { generation_id: generation.manifest.generation_id.clone(), snapshot_digest: generation.manifest.snapshot_digest.clone(), + scope: FileScopeIdentityV1::of(&generation.manifest, &generation.snapshot), descriptors: generation.file_segments, read_segment: Box::new(read_segment), }; Self::open_partitioned_parts( - reader, generation.manifest, generation.snapshot, - Some(generation.statistics), + generation.statistics, source, source_state_digest, maximum_page_chunks, maximum_page_bytes, ) - .map(Some) } } @@ -2803,7 +1735,7 @@ impl CodeIndexPublishedGenerationV1 { .map(parse_partitioned_manifest) .transpose() { - Ok(parent) => parent.flatten(), + Ok(parent) => parent, Err(CodeIndexProductionErrorV1::SupersededSealedGenerationRevision(_)) => None, Err(error) => return Err(error), }; @@ -2838,6 +1770,7 @@ impl CodeIndexPublishedGenerationV1 { )?; let mut file_segments = Vec::with_capacity(self.files.len()); let buffers = SealedEncodeBufferPoolV1::default(); + let scope = FileScopeIdentityV1::of(&self.manifest, &self.snapshot); let plan_file = |file: &FileGenerationArtifactsV1| -> Result< FileSegmentPlanV1, CodeIndexProductionErrorV1, @@ -2855,8 +1788,21 @@ impl CodeIndexPublishedGenerationV1 { "sealed generation file key is outside its snapshot".to_owned(), ) })?; - let reused = parent_segments - .get(¤t_snapshot_file.file_occurrence_id) + let prior = parent_segments.get(¤t_snapshot_file.file_occurrence_id); + let current_identities_digest = prior + .map(|_| { + let mut identities = file + .artifacts + .symbols + .iter() + .map(|symbol| symbol.identity.clone()) + .collect::>(); + identities.sort(); + identities.dedup(); + symbol_identities_digest(&identities) + }) + .transpose()?; + let reused = prior .and_then(|(prior_file, prior_descriptor)| { (*prior_file == current_snapshot_file).then_some(())?; let language = current_snapshot_file.language.as_ref()?; @@ -2874,18 +1820,9 @@ impl CodeIndexPublishedGenerationV1 { .find(|(candidate, _)| candidate == language) .map(|(_, revision)| revision)?; (prior_extractor_revision == current_extractor_revision).then_some(())?; - let mut current_identities = file - .artifacts - .symbols - .iter() - .map(|symbol| symbol.identity.clone()) - .collect::>(); - current_identities.sort(); - current_identities.dedup(); - if current_identities.as_slice() != prior_descriptor.symbol_identities.as_slice() - { - return None; - } + (current_identities_digest.as_ref() + == Some(&prior_descriptor.symbol_identities_digest)) + .then_some(())?; let mut descriptor = (*prior_descriptor).clone(); descriptor.file_key = key; Some(descriptor) @@ -2898,7 +1835,7 @@ impl CodeIndexPublishedGenerationV1 { segment: buffers.take(), }; let descriptor = - encoder.encode_file_segment(&self.manifest.generation_id, file, key)?; + encoder.encode_file_segment(&self.manifest.generation_id, &scope, file, key)?; buffers.give(std::mem::take(&mut encoder.payload)); Ok(FileSegmentPlanV1::Encoded(descriptor, encoder.segment)) }; @@ -3011,20 +1948,17 @@ impl CodeIndexPublishedGenerationV1 { SealedGenerationSegmentReadV1<'_>, &mut Vec, ) -> Result<(), CodeIndexProductionErrorV1>, - ) -> Result, CodeIndexProductionErrorV1> { - let Some(mut generation) = hotpath::measure_block!( + ) -> Result { + let generation = hotpath::measure_block!( "code_index.restore.manifest", parse_partitioned_manifest(bytes) - )? - else { - return Ok(None); - }; - bind_file_segment_occurrences(&mut generation.file_segments)?; + )?; let mut files = Vec::with_capacity(generation.file_segments.len()); // One segment buffer per window slot, reused across windows: the // decode holds at most `partitioned_decode_window_files()` segments, // each buffer grown only to the largest segment its slot has read. let mut buffers = vec![Vec::new(); Self::partitioned_decode_window_files()]; + let scope = FileScopeIdentityV1::of(&generation.manifest, &generation.snapshot); while files.len() < generation.file_segments.len() { let pending = &generation.file_segments[files.len()..]; let read = read_segment_window( @@ -3046,35 +1980,42 @@ impl CodeIndexPublishedGenerationV1 { &buffers[..read], &generation.manifest.generation_id, &generation.manifest.snapshot_digest, + &scope, )?); } let evidence = hotpath::measure_block!( "code_index.restore.generation_evidence", - decode_generation_evidence( - &generation.generation_evidence, - &generation.manifest.generation_id, - &generation.file_segments, - &files, - read_segment, - ) + decode_generation_evidence(&generation.generation_evidence, read_segment) )?; + let (lineage, projection_request, projection_receipt) = + hotpath::measure_block!("code_index.restore.evidence_expand", { + let symbols = + occurrence_roster(files.iter().flat_map(|file| file.artifacts.symbols.iter())); + let chunks = chunk_roster( + files + .iter() + .flat_map(|file| file.artifacts.chunks.chunks.iter()), + ); + let request = evidence.projection_request.expand(&chunks)?; + let receipt = evidence.projection_receipt.expand(&request)?; + let lineage = evidence + .lineage + .expand(&generation.manifest.generation_id, &symbols)?; + Ok::<_, CodeIndexProductionErrorV1>((lineage, request, receipt)) + })?; assemble_published_generation(StreamingPersistedPublishedGenerationV1 { - format_revision: super::sealed_codec::CompatibleSealedFormatRevisionV1( - generation.format_revision, - ), manifest: generation.manifest, snapshot: generation.snapshot, repository_parse_identity: generation.repository_parse_identity, ignored_source_admissions: generation.ignored_source_admissions, ignored_source_admissions_digest: generation.ignored_source_admissions_digest, - files: StreamingRestoredFilesV1 { files }, - lineage: evidence.lineage, + files, + lineage, coverage: generation.coverage, capability: generation.capability, - projection_request: evidence.projection_request, - projection_receipt: evidence.projection_receipt, + projection_request, + projection_receipt, }) - .map(Some) } /// Authenticate only the tiny partitioned manifest and return the metadata @@ -3083,24 +2024,19 @@ impl CodeIndexPublishedGenerationV1 { /// have their own verified durable artifacts. pub fn partitioned_text_metadata( bytes: &[u8], - ) -> Result, CodeIndexProductionErrorV1> { - let Some(generation) = parse_partitioned_manifest(bytes)? else { - return Ok(None); - }; + ) -> Result { + let generation = parse_partitioned_manifest(bytes)?; VerifiedSealedTextGenerationMetadataV1::from_partitioned_manifest( generation.manifest, generation.snapshot, - Some(generation.statistics), + generation.statistics, ) - .map(Some) } pub fn partitioned_segment_identities( bytes: &[u8], - ) -> Result>, CodeIndexProductionErrorV1> { - let Some(generation) = parse_partitioned_manifest(bytes)? else { - return Ok(None); - }; + ) -> Result, CodeIndexProductionErrorV1> { + let generation = parse_partitioned_manifest(bytes)?; let mut identities = generation .file_segments .into_iter() @@ -3113,7 +2049,7 @@ impl CodeIndexPublishedGenerationV1 { digest: generation.generation_evidence.segment_digest, size_bytes: generation.generation_evidence.segment_size_bytes, }); - Ok(Some(identities)) + Ok(identities) } /// Stream only current-revision segment descriptors from a generation @@ -3163,11 +2099,11 @@ impl CodeIndexPublishedGenerationV1 { .filter(|(_, file)| file.disposition == SnapshotFileDispositionV1::Present) .map(|(key, file)| (key, &file.file_occurrence_id)), generation.generation_evidence.segment_size_bytes, - generation.generation_evidence.pages.as_ref().map(|pages| { - pages - .iter() - .map(|page| (page.page_ordinal, page.page_size_bytes)) - }), + generation + .generation_evidence + .pages + .iter() + .map(|page| (page.page_ordinal, page.page_size_bytes)), )?; let mut identities = Vec::with_capacity(generation.file_segments.len().saturating_add(1)); for segment in generation.file_segments { @@ -3189,10 +2125,8 @@ impl CodeIndexPublishedGenerationV1 { SealedGenerationSegmentReadV1<'_>, &mut Vec, ) -> Result<(), CodeIndexProductionErrorV1>, - ) -> Result { - let Some(generation) = parse_partitioned_manifest(bytes)? else { - return Ok(false); - }; + ) -> Result<(), CodeIndexProductionErrorV1> { + let generation = parse_partitioned_manifest(bytes)?; let mut segment = Vec::new(); for descriptor in &generation.file_segments { segment.clear(); @@ -3223,8 +2157,7 @@ impl CodeIndexPublishedGenerationV1 { )) }) })?; - evidence.finish()?; - Ok(true) + evidence.finish() } } @@ -3277,34 +2210,44 @@ mod tests { } #[test] - fn missing_evidence_pages_is_legacy_but_explicit_null_is_rejected() { + fn missing_or_null_evidence_pages_are_rejected_by_both_readers() { let missing = serde_json::json!({ "segment_digest": "sha256:56f954431e92b5e2ef9b1355bc229acf516a8d3409b7e48e9cd9fb7856411f29", "segment_size_bytes": 1 }); - let descriptor: PartitionedGenerationEvidenceDescriptorV1 = - serde_json::from_value(missing.clone()).expect("missing pages is historical format"); - assert!(descriptor.legacy_unpaged); - let identity: PartitionedEvidenceSegmentIdentityV1 = - serde_json::from_value(missing).expect("retention accepts historical format"); - assert!(identity.pages.is_none()); - let explicit_null = serde_json::json!({ "segment_digest": "sha256:56f954431e92b5e2ef9b1355bc229acf516a8d3409b7e48e9cd9fb7856411f29", "segment_size_bytes": 1, "pages": null }); - assert!( - serde_json::from_value::( - explicit_null.clone() - ) - .is_err(), - "the full parser must not treat explicit null as historical format" - ); + for descriptor in [missing.clone(), explicit_null.clone()] { + assert!( + serde_json::from_value::( + descriptor.clone() + ) + .is_err(), + "the full parser requires a page table: {descriptor}" + ); + } assert!( serde_json::from_value::(explicit_null).is_err(), - "the retention reader must not treat explicit null as historical format" + "the retention reader refuses an explicit null page table" ); + let identity: PartitionedEvidenceSegmentIdentityV1 = + serde_json::from_value(missing).expect("retention reaches its revision gate"); + assert!(identity.pages.is_empty()); + let file = FileOccurrenceId::new("file.partitioned.only").unwrap(); + let error = validate_partitioned_generation_layout( + [(0, &file)].into_iter(), + [&file].into_iter().enumerate(), + identity.segment_size_bytes, + identity + .pages + .iter() + .map(|page| (page.page_ordinal, page.page_size_bytes)), + ) + .expect_err("a current descriptor without pages is malformed"); + assert!(error.to_string().contains("has no pages"), "{error}"); } #[test] @@ -3317,75 +2260,61 @@ mod tests { [(0, &first), (1, &second)].into_iter(), files.into_iter().enumerate(), 9, - Some(valid_pages.into_iter()), + valid_pages.into_iter(), ) .expect("current paged descriptor is canonical"); - validate_partitioned_generation_layout( - [(0, &first), (1, &second)].into_iter(), - files.into_iter().enumerate(), - 9, - None::>, - ) - .expect("historical unpaged descriptor is canonical"); for (segments, snapshot, size, pages, expected) in [ ( vec![(0, &first)], vec![&first, &second], 4, - Some(vec![(0, 4)]), + vec![(0, 4)], "segment count", ), ( vec![(0, &second), (1, &first)], vec![&first, &second], 4, - Some(vec![(0, 4)]), + vec![(0, 4)], "canonically keyed", ), - ( - vec![(0, &first), (1, &second)], - vec![&first, &second], - 0, - None, - "legacy sealed generation evidence segment is empty", - ), ( vec![(0, &first), (1, &second)], vec![&first, &second], 4, - Some(vec![]), + vec![], "has no pages", ), ( vec![(0, &first), (1, &second)], vec![&first, &second], 4, - Some(vec![(1, 4)]), + vec![(1, 4)], "canonically bounded and ordered", ), ( vec![(0, &first), (1, &second)], vec![&first, &second], 4, - Some(vec![(0, 0)]), + vec![(0, 0)], "canonically bounded and ordered", ), ( vec![(0, &first), (1, &second)], vec![&first, &second], 4, - Some(vec![( + vec![( 0, u64::try_from(GENERATION_EVIDENCE_PAGE_MAX_BYTES_V1).unwrap() + 1, - )]), + )], "canonically bounded and ordered", ), ( vec![(0, &first), (1, &second)], vec![&first, &second], 5, - Some(vec![(0, 4)]), + vec![(0, 4)], "byte size does not match", ), ] { @@ -3393,7 +2322,7 @@ mod tests { segments.into_iter(), snapshot.into_iter().enumerate(), size, - pages.map(Vec::into_iter), + pages.into_iter(), ) .expect_err("malformed descriptor must be rejected by both readers"); assert!( @@ -3412,6 +2341,7 @@ mod tests { #[derive(Serialize)] pub(super) struct ReferenceFileSegmentV1 { pub(super) format_revision: u32, + pub(super) symbol_identities: Vec, pub(super) file: Value, } @@ -3522,7 +2452,11 @@ mod tests { } drop(symbol_keys); let bytes = serde_json::to_vec(&ReferenceFileSegmentV1 { - format_revision: FILE_SEGMENT_FORMAT_REVISION_V1, + format_revision: FILE_SEGMENT_FORMAT_REVISION, + symbol_identities: ordered_symbols + .keys() + .map(|identity| (*identity).to_owned()) + .collect(), file: value, }) .expect("reference segment bytes"); @@ -3738,7 +2672,7 @@ mod tests { serde_json::to_writer(&mut encoder.payload, &payload).expect("streamed payload"); let descriptor = encoder .encode_serialized_file_segment( - FILE_SEGMENT_FORMAT_REVISION_V1, + FILE_SEGMENT_FORMAT_REVISION, &CodeGenerationId::new(FIXTURE_GENERATION).expect("fixture generation identity"), None, FileOccurrenceId::new(FIXTURE_FILE).expect("fixture file identity"), @@ -3758,32 +2692,63 @@ mod tests { &fixture_stable_symbols(), ); - let (streamed_bytes, descriptor) = streamed_file_segment(); + let (stored_bytes, descriptor) = streamed_file_segment(); + let mut streamed_bytes = Vec::new(); + inflate_file_segment( + &stored_bytes, + descriptor.decoded_size_bytes, + &mut streamed_bytes, + ) + .expect("stored segment inflates to its recorded size"); assert_eq!( String::from_utf8(streamed_bytes.clone()).expect("streamed segment is UTF-8"), String::from_utf8(reference_bytes.clone()).expect("reference segment is UTF-8"), - "the streaming writer must reproduce the shipped segment bytes" + "the streaming writer must reproduce the canonical segment bytes" + ); + assert!( + stored_bytes.len() < streamed_bytes.len(), + "the stored segment is compressed" ); + assert!( + inflate_file_segment( + &stored_bytes, + descriptor.decoded_size_bytes - 1, + &mut Vec::new() + ) + .is_err(), + "inflation past the recorded size is refused" + ); + let segment = serde_json::from_slice::>(&streamed_bytes) + .expect("segment envelope"); assert_eq!( - descriptor - .symbol_occurrences + bind_symbol_occurrences(&descriptor.file_occurrence_id, &segment.symbol_identities) + .expect("segment symbol occurrences") .iter() .map(|occurrence| occurrence.as_str().to_owned()) .collect::>(), reference_occurrences, "the symbol key assignment must not move" ); + assert_eq!( + descriptor.symbol_identities_digest, + symbol_identities_digest(&segment.symbol_identities).expect("identities digest"), + "reuse compares the digest of exactly the identities the segment carries" + ); assert_eq!(descriptor.file_key, 7); assert_eq!( - descriptor.segment_size_bytes, + descriptor.decoded_size_bytes, u64::try_from(reference_bytes.len()).expect("reference length"), ); + assert_eq!( + descriptor.segment_size_bytes, + u64::try_from(stored_bytes.len()).expect("stored length"), + ); assert_eq!( descriptor.segment_digest, - ManifestDigest::from_sha256_bytes(&Sha256::digest(&reference_bytes)) - .expect("reference digest"), - "the segment content address must not move" + ManifestDigest::from_sha256_bytes(&Sha256::digest(&stored_bytes)) + .expect("stored digest"), + "the content address covers the stored bytes" ); assert!( streamed_bytes @@ -4010,80 +2975,6 @@ mod tests { ); } - /// A pre-paging segment carries no page table, so its aggregate identity - /// is the only attestation of its bytes and stays computed and compared. - #[test] - fn pre_paging_evidence_is_refused_by_its_aggregate_identity() { - let evidence = FixtureEvidence { - lineage: Vec::new(), - projection_request: FixtureProjectionRequest { - generation_id: FIXTURE_GENERATION.to_owned(), - chunk_ids: Vec::new(), - parent_chunk_id: None, - }, - padding: "p".repeat(GENERATION_EVIDENCE_PAGE_MAX_BYTES_V1 + 23), - }; - let stream = serde_json::to_vec(&evidence).expect("reference evidence stream"); - let descriptor: PartitionedGenerationEvidenceDescriptorV1 = - serde_json::from_value(serde_json::json!({ - "segment_digest": ManifestDigest::from_sha256_bytes(&Sha256::digest(&stream)) - .expect("stream digest") - .as_str(), - "segment_size_bytes": stream.len(), - })) - .expect("a manifest without a page table is the pre-paging format"); - assert!(descriptor.legacy_unpaged); - - fn read_range( - source: &[u8], - request: SealedGenerationSegmentReadV1<'_>, - buffer: &mut Vec, - ) -> Result<(), CodeIndexProductionErrorV1> { - let SealedGenerationSegmentReadV1::Range { offset, length, .. } = request else { - panic!("evidence reader must request a range") - }; - let start = usize::try_from(offset).expect("range offset"); - let end = start + usize::try_from(length).expect("range length"); - buffer.clear(); - buffer.extend_from_slice(&source[start..end]); - Ok(()) - } - - let mut read = |request: SealedGenerationSegmentReadV1<'_>, buffer: &mut Vec| { - read_range(&stream, request, buffer) - }; - let mut reader = PartitionedEvidencePageReaderV1::new(&descriptor, &mut read); - let restored: FixtureEvidence = - serde_json::from_reader(&mut reader).expect("pre-paging evidence decode"); - reader.finish().expect("aggregate identity verifies"); - assert_eq!(restored, evidence); - - // A padding byte, so the stream still decodes and only the aggregate - // identity can refuse it. It has to be past the first bounded range so - // the refusal cannot come from a short first read. - let mut tampered = stream.clone(); - let padding_byte = stream - .windows(GENERATION_EVIDENCE_PAGE_MAX_BYTES_V1 + 1) - .position(|window| window.iter().all(|byte| *byte == b'p')) - .expect("the fixture padding must span more than one bounded range") - + GENERATION_EVIDENCE_PAGE_MAX_BYTES_V1; - tampered[padding_byte] = b'q'; - let mut read = |request: SealedGenerationSegmentReadV1<'_>, buffer: &mut Vec| { - read_range(&tampered, request, buffer) - }; - let mut reader = PartitionedEvidencePageReaderV1::new(&descriptor, &mut read); - let restored: FixtureEvidence = - serde_json::from_reader(&mut reader).expect("tampered padding still decodes"); - assert_ne!(restored, evidence); - let error = reader - .finish() - .expect_err("pre-paging bytes must fail their aggregate identity"); - assert!( - error.to_string().contains("segment digest"), - "unexpected pre-paging tamper error: {error}" - ); - } - #[test] fn evidence_failure_after_a_page_never_emits_a_pack_commit() { let evidence = FixtureEvidence { @@ -4226,64 +3117,6 @@ mod tests { .join("tests/fixtures/partitioned_pre_paging") } - /// The descriptors that address the archival carrier's file segments. - /// - /// The carrier's own envelope names a retired manifest revision, so the - /// decoding readers refuse it (see - /// [`tests::archival_carrier_revision_is_refused_for_rebuild`]). What the - /// fixture exists to prove lives one level below that envelope, the - /// revision-1 and revision-2 *segment* bytes, which no manifest revision - /// rewrites, so the tests below read the descriptors addressing them - /// straight from the archival bytes instead of asking a current decoder - /// to admit a shape this build no longer writes. - #[derive(Deserialize)] - struct ArchivalSegmentCarrierV1 { - generation: ArchivalSegmentCarrierWireGenerationV1, - } - - #[derive(Deserialize)] - struct ArchivalSegmentCarrierWireGenerationV1 { - manifest: CodeGenerationManifestV1, - file_segments: Vec, - } - - #[derive(Deserialize)] - struct ArchivalFileSegmentDescriptorV1 { - file_key: u32, - segment_digest: ManifestDigest, - segment_size_bytes: u64, - file_occurrence_id: FileOccurrenceId, - symbol_occurrences: Vec, - } - - struct ArchivalSegmentCarrierGenerationV1 { - manifest: CodeGenerationManifestV1, - file_segments: Vec, - } - - fn archival_segment_carrier() -> ArchivalSegmentCarrierGenerationV1 { - let manifest = std::fs::read(historical_fixture_root().join("manifest.json")) - .expect("historical manifest"); - let carrier = serde_json::from_slice::(&manifest) - .expect("archival carrier segment descriptors") - .generation; - ArchivalSegmentCarrierGenerationV1 { - manifest: carrier.manifest, - file_segments: carrier - .file_segments - .into_iter() - .map(|descriptor| PartitionedFileSegmentDescriptorV1 { - file_key: descriptor.file_key, - segment_digest: descriptor.segment_digest, - segment_size_bytes: descriptor.segment_size_bytes, - file_occurrence_id: descriptor.file_occurrence_id, - symbol_identities: Vec::new(), - symbol_occurrences: descriptor.symbol_occurrences, - }) - .collect(), - } - } - /// The archival carrier was sealed at revision seven, which named a /// manifest both with and without its census and is therefore retired. /// Every decoding reader refuses it with the typed rebuild error rather @@ -4308,79 +3141,4 @@ mod tests { "a retired revision must tell the operator it rebuilds: {error}" ); } - - /// Historical file segments predate clone-body rows and are rebuilt. - #[test] - fn revision_one_historical_segments_are_refused_without_clone_evidence() { - let fixture = historical_fixture_root(); - let generation = archival_segment_carrier(); - let mut refused_segments = 0; - for descriptor in &generation.file_segments { - let name = descriptor - .segment_digest - .hex_suffix() - .expect("sha256 segment digest"); - let bytes = std::fs::read(fixture.join("segments").join(format!("{name}.json"))) - .expect("historical segment bytes"); - let probe: PartitionedFormatProbeV1 = - serde_json::from_slice(&bytes).expect("segment format probe"); - assert_eq!(probe.format_revision, FILE_SEGMENT_FORMAT_REVISION_V1); - let mut restored = Vec::new(); - let error = decode_file_segment( - descriptor, - &generation.manifest.generation_id, - &generation.manifest.snapshot_digest, - &bytes, - &mut restored, - ) - .expect_err("historical rows without clone evidence must be refused"); - assert!( - matches!( - &error, - CodeIndexProductionErrorV1::SealedRowContractRefused { revision, message } - if *revision == FILE_SEGMENT_FORMAT_REVISION_V1 - && message.contains("predates clone body rows") - ), - "unexpected error: {error}" - ); - refused_segments += 1; - } - assert!(refused_segments > 0); - } - - /// Revision two is refused before any of its now-incomplete rows decode. - #[test] - fn revision_two_rows_are_refused_without_clone_evidence() { - let generation = archival_segment_carrier(); - let prior = &generation.file_segments[0]; - let bytes = br#"{"format_revision":2,"file":{}}"#; - let descriptor = PartitionedFileSegmentDescriptorV1 { - file_key: prior.file_key, - segment_digest: ManifestDigest::from_sha256_bytes(&Sha256::digest(bytes)) - .expect("segment digest"), - segment_size_bytes: u64::try_from(bytes.len()).expect("segment size"), - file_occurrence_id: prior.file_occurrence_id.clone(), - symbol_identities: prior.symbol_identities.clone(), - symbol_occurrences: Vec::new(), - }; - let mut restored = Vec::new(); - let error = decode_file_segment( - &descriptor, - &generation.manifest.generation_id, - &generation.manifest.snapshot_digest, - bytes, - &mut restored, - ) - .expect_err("revision two must be refused"); - assert!( - matches!( - &error, - CodeIndexProductionErrorV1::SealedRowContractRefused { - revision, - .. - } if *revision == FILE_SEGMENT_FORMAT_REVISION_V2 - ), - "unexpected error: {error}" - ); - } } diff --git a/crates/tracedecay-code-index/src/production/projection_rows.rs b/crates/tracedecay-code-index/src/production/projection_rows.rs new file mode 100644 index 0000000000..8e52d307d6 --- /dev/null +++ b/crates/tracedecay-code-index/src/production/projection_rows.rs @@ -0,0 +1,397 @@ +//! Persisted projection request and receipt rows of one generation's evidence. +//! +//! A request's added-or-changed rows name chunks of the generation being +//! sealed, and each row's current digest is that chunk's content digest, so +//! the persisted form names those rows by position in the generation's +//! chunk-id-ordered roster (as runs when the chunk is new) and keeps every +//! other row whole. A receipt answers exactly the request's rows in chunk +//! order, and a row the projector applied as the request says is a pure +//! function of that row, so only the other decisions are persisted. +//! Restore re-verifies the request, manifest, and publication digests over +//! the rebuilt rows, so any disagreement fails closed. + +use std::collections::{BTreeMap, HashMap}; +use std::sync::Arc; + +use serde::{Deserialize, Serialize}; +use tracedecay_domain::{ + ChangedCodeChunkSetV1, ChangedCodeChunkV1, CodeChunkProjectionReceiptV1, CodeGenerationId, + CodeSearchChunkId, CodeSearchChunkV1, ContentDigest, ManifestDigest, ProjectionBatchReceiptV1, + ProjectionBatchRequestV1, ProjectionKeyV1, ProjectionOperationV1, ProjectionOutcomeV1, + ProjectionReplayReasonV1, +}; + +use super::CodeIndexProductionErrorV1; + +fn contract(message: &str) -> CodeIndexProductionErrorV1 { + CodeIndexProductionErrorV1::Contract(message.to_owned()) +} + +/// A generation's chunks in canonical chunk-id order. Encoding and restore +/// both derive it from the file artifacts, so positions agree. +pub(super) fn chunk_roster<'a>( + chunks: impl Iterator>, +) -> Vec<&'a CodeSearchChunkV1> { + let mut roster = chunks.map(Arc::as_ref).collect::>(); + roster.sort_by(|left, right| left.id.cmp(&right.id)); + roster +} + +#[derive(Serialize)] +pub(super) struct PersistedProjectionRequestRefV1<'a> { + request_digest: &'a ManifestDigest, + changes: PersistedChangeSetRefV1<'a>, + previous_projection_key: &'a Option, + target_projection_key: &'a ProjectionKeyV1, + replay_reason: ProjectionReplayReasonV1, +} + +#[derive(Serialize)] +struct PersistedChangeSetRefV1<'a> { + from_generation: &'a Option, + to_generation: &'a CodeGenerationId, + manifest_digest: &'a ManifestDigest, + added_or_changed: Vec>, + deleted: &'a [ChangedCodeChunkV1], + reused_count: u64, + reused_digest: &'a ManifestDigest, +} + +#[derive(Serialize)] +#[serde(rename_all = "snake_case")] +enum PersistedChangeRowRefV1<'a> { + Added { + start: u32, + count: u32, + }, + Changed { + current: u32, + prior: &'a ContentDigest, + }, + Row(&'a ChangedCodeChunkV1), +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct PersistedProjectionRequestV1 { + request_digest: ManifestDigest, + changes: PersistedChangeSetV1, + previous_projection_key: Option, + target_projection_key: ProjectionKeyV1, + replay_reason: ProjectionReplayReasonV1, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct PersistedChangeSetV1 { + from_generation: Option, + to_generation: CodeGenerationId, + manifest_digest: ManifestDigest, + added_or_changed: Vec, + deleted: Vec, + reused_count: u64, + reused_digest: ManifestDigest, +} + +#[derive(Deserialize)] +#[serde(rename_all = "snake_case", deny_unknown_fields)] +enum PersistedChangeRowV1 { + /// Roster chunks `start..start + count`, each new in this generation. + Added { + start: u32, + count: u32, + }, + /// Roster chunk `current`, changed from content digest `prior`. + Changed { + current: u32, + prior: ContentDigest, + }, + Row(ChangedCodeChunkV1), +} + +impl<'a> PersistedProjectionRequestRefV1<'a> { + pub(super) fn new( + request: &'a ProjectionBatchRequestV1, + roster: &[&CodeSearchChunkV1], + ) -> Result { + let positions = roster + .iter() + .enumerate() + .map(|(position, chunk)| { + u32::try_from(position) + .map(|position| (&chunk.id, (position, &chunk.content_digest))) + .map_err(|_| contract("sealed chunk roster exceeds u32")) + }) + .collect::, _>>()?; + let changes = &request.changes; + let mut rows = Vec::new(); + for change in &changes.added_or_changed { + let position = positions + .get(&change.chunk_id) + .filter(|(_, digest)| change.current_digest.as_ref() == Some(*digest)) + .map(|(position, _)| *position); + match (position, &change.prior_digest) { + (Some(position), None) => match rows.last_mut() { + Some(PersistedChangeRowRefV1::Added { start, count }) + if start.checked_add(*count) == Some(position) => + { + *count += 1; + } + _ => rows.push(PersistedChangeRowRefV1::Added { + start: position, + count: 1, + }), + }, + (Some(position), Some(prior)) => rows.push(PersistedChangeRowRefV1::Changed { + current: position, + prior, + }), + (None, _) => rows.push(PersistedChangeRowRefV1::Row(change)), + } + } + Ok(Self { + request_digest: &request.request_digest, + changes: PersistedChangeSetRefV1 { + from_generation: &changes.from_generation, + to_generation: &changes.to_generation, + manifest_digest: &changes.manifest_digest, + added_or_changed: rows, + deleted: &changes.deleted, + reused_count: changes.reused_count, + reused_digest: &changes.reused_digest, + }, + previous_projection_key: &request.previous_projection_key, + target_projection_key: &request.target_projection_key, + replay_reason: request.replay_reason, + }) + } +} + +impl PersistedProjectionRequestV1 { + pub(super) fn expand( + self, + roster: &[&CodeSearchChunkV1], + ) -> Result { + let chunk = |position: u32| { + usize::try_from(position) + .ok() + .and_then(|position| roster.get(position)) + .copied() + .ok_or_else(|| contract("sealed projection row names a chunk outside its roster")) + }; + let changes = self.changes; + let mut added_or_changed = Vec::new(); + for row in changes.added_or_changed { + match row { + PersistedChangeRowV1::Added { start, count } => { + let end = start + .checked_add(count) + .ok_or_else(|| contract("sealed projection run exceeds u32"))?; + for position in start..end { + let chunk = chunk(position)?; + added_or_changed.push(ChangedCodeChunkV1 { + chunk_id: chunk.id.clone(), + prior_digest: None, + current_digest: Some(chunk.content_digest.clone()), + }); + } + } + PersistedChangeRowV1::Changed { current, prior } => { + let chunk = chunk(current)?; + added_or_changed.push(ChangedCodeChunkV1 { + chunk_id: chunk.id.clone(), + prior_digest: Some(prior), + current_digest: Some(chunk.content_digest.clone()), + }); + } + PersistedChangeRowV1::Row(change) => added_or_changed.push(change), + } + } + Ok(ProjectionBatchRequestV1 { + request_digest: self.request_digest, + changes: ChangedCodeChunkSetV1 { + from_generation: changes.from_generation, + to_generation: changes.to_generation, + manifest_digest: changes.manifest_digest, + added_or_changed, + deleted: changes.deleted, + reused_count: changes.reused_count, + reused_digest: changes.reused_digest, + }, + previous_projection_key: self.previous_projection_key, + target_projection_key: self.target_projection_key, + replay_reason: self.replay_reason, + }) + } +} + +/// The receipt's batch header and the decisions that are not the default +/// for their request row. +#[derive(Serialize)] +pub(super) struct PersistedBatchReceiptRefV1<'a> { + target_projection_key: &'a ProjectionKeyV1, + request_digest: &'a ManifestDigest, + source_generation: &'a CodeGenerationId, + source_manifest_digest: &'a ManifestDigest, + exceptions: Vec>, + reused_count: u64, + publication_digest: &'a ManifestDigest, +} + +#[derive(Serialize)] +struct PersistedChunkReceiptRefV1<'a> { + chunk_id: &'a CodeSearchChunkId, + operation: ProjectionOperationV1, + outcome: &'a ProjectionOutcomeV1, + #[serde(skip_serializing_if = "Option::is_none")] + output_digest: Option<&'a ContentDigest>, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct PersistedBatchReceiptV1 { + target_projection_key: ProjectionKeyV1, + request_digest: ManifestDigest, + source_generation: CodeGenerationId, + source_manifest_digest: ManifestDigest, + exceptions: Vec, + reused_count: u64, + publication_digest: ManifestDigest, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct PersistedChunkReceiptV1 { + chunk_id: CodeSearchChunkId, + operation: ProjectionOperationV1, + outcome: ProjectionOutcomeV1, + #[serde(default)] + output_digest: Option, +} + +/// The request's rows in the chunk order a receipt answers them in. +fn answered_rows( + request: &ProjectionBatchRequestV1, +) -> BTreeMap<&CodeSearchChunkId, &ChangedCodeChunkV1> { + request + .changes + .added_or_changed + .iter() + .chain(&request.changes.deleted) + .map(|change| (&change.chunk_id, change)) + .collect() +} + +/// The operation a receipt must record for `change`, with the outcome and +/// output a projector that applied it as requested records. +fn applied(change: &ChangedCodeChunkV1) -> ProjectionOperationV1 { + match (&change.prior_digest, &change.current_digest) { + (_, None) => ProjectionOperationV1::Deleted, + (None, Some(_)) => ProjectionOperationV1::Added, + (Some(_), Some(_)) => ProjectionOperationV1::Updated, + } +} + +impl<'a> PersistedBatchReceiptRefV1<'a> { + pub(super) fn new( + request: &ProjectionBatchRequestV1, + receipt: &'a ProjectionBatchReceiptV1, + ) -> Result { + let rows = answered_rows(request); + if receipt.receipts.len() != rows.len() + || receipt + .receipts + .iter() + .zip(rows.keys()) + .any(|(receipt, chunk_id)| &receipt.chunk_id != *chunk_id) + { + return Err(contract( + "sealed projection receipt does not answer its request rows in chunk order", + )); + } + let exceptions = receipt + .receipts + .iter() + .zip(rows.values()) + .filter(|(receipt, change)| { + receipt.operation != applied(change) + || receipt.outcome != ProjectionOutcomeV1::Applied + || receipt.output_digest != change.current_digest + }) + .map(|(receipt, _)| PersistedChunkReceiptRefV1 { + chunk_id: &receipt.chunk_id, + operation: receipt.operation, + outcome: &receipt.outcome, + output_digest: receipt.output_digest.as_ref(), + }) + .collect(); + Ok(Self { + target_projection_key: &receipt.target_projection_key, + request_digest: &receipt.request_digest, + source_generation: &receipt.source_generation, + source_manifest_digest: &receipt.source_manifest_digest, + exceptions, + reused_count: receipt.reused_count, + publication_digest: &receipt.publication_digest, + }) + } +} + +impl PersistedBatchReceiptV1 { + /// Rebuild the full receipt rows from the request. Every field restored + /// here is one the receipt verifier requires to equal the request, so the + /// batch's publication digest recomputes over the same bytes it sealed. + pub(super) fn expand( + self, + request: &ProjectionBatchRequestV1, + ) -> Result { + let rows = answered_rows(request); + let mut exceptions = self + .exceptions + .into_iter() + .map(|exception| (exception.chunk_id.clone(), exception)) + .collect::>(); + let mut receipts = Vec::with_capacity(rows.len()); + for (chunk_id, change) in rows { + let (operation, outcome, output_digest) = match exceptions.remove(chunk_id) { + Some(exception) => ( + exception.operation, + exception.outcome, + exception.output_digest, + ), + None => ( + applied(change), + ProjectionOutcomeV1::Applied, + change.current_digest.clone(), + ), + }; + receipts.push(CodeChunkProjectionReceiptV1 { + projection_key: self.target_projection_key.clone(), + request_digest: self.request_digest.clone(), + prior_generation: request.changes.from_generation.clone(), + source_generation: self.source_generation.clone(), + source_manifest_digest: self.source_manifest_digest.clone(), + chunk_id: chunk_id.clone(), + prior_chunk_digest: change.prior_digest.clone(), + current_chunk_digest: change.current_digest.clone(), + operation, + outcome, + output_digest, + }); + } + if !exceptions.is_empty() { + return Err(contract( + "sealed generation receipt names a chunk outside its projection request", + )); + } + Ok(ProjectionBatchReceiptV1 { + target_projection_key: self.target_projection_key, + request_digest: self.request_digest, + source_generation: self.source_generation, + source_manifest_digest: self.source_manifest_digest, + receipts, + reused_count: self.reused_count, + publication_digest: self.publication_digest, + }) + } +} diff --git a/crates/tracedecay-code-index/src/production/sealed_codec.rs b/crates/tracedecay-code-index/src/production/sealed_codec.rs index 0689922eda..c0ad6f7377 100644 --- a/crates/tracedecay-code-index/src/production/sealed_codec.rs +++ b/crates/tracedecay-code-index/src/production/sealed_codec.rs @@ -1,104 +1,57 @@ use std::collections::HashMap; -use std::fmt; -use std::io::{BufReader, BufWriter, Read, Seek, SeekFrom, Write}; use std::sync::{Arc, OnceLock}; -use serde::de::{SeqAccess, Visitor}; use serde::{Deserialize, Serialize}; -use serde_json::value::RawValue; -use sha2::{Digest, Sha256}; use tracedecay_code_extraction::ExtractedSchemaEvidenceV1; use tracedecay_domain::{ BoundedSanitizedText, ChunkerRevision, CodeSearchChunkAnchorV1, CodeSearchChunkGrainV1, - CodeSearchChunkId, CodeSearchChunkV1, ContentDigest, ExactTechnicalTermV1, - LanguageDescriptorRevision, SensitivityDecision, SourceSpan, + CodeSearchChunkId, CodeSearchChunkV1, ContentDigest, ExactTechnicalTermKindV1, + ExactTechnicalTermV1, LanguageDescriptorRevision, SensitivityDecision, SourceSpan, }; use crate::chunks::{ CodeFileChunksV1, CodeIndexUnresolvedReferenceV1, CodeSearchDocumentV1, CodeSearchEligibilityV1, }; -use crate::clones::CodeIndexCloneBodyV1; use crate::extract::ExtractionBatchV1; +use crate::intake::content_digest; use crate::lineage::LineageSymbolRecordV1; use crate::parallelism; -use super::lexical_page_source::scan_layout; +use super::clone_rows::{PersistedCloneBodiesRefV1, PersistedCloneBodiesV1}; use super::*; -/// The monolithic sealed-generation envelope revision. Every reader that -/// gates on the monolithic format, the publication store, the worker probe, -/// and code-generation retention, must gate on this one value. -pub(super) const MONOLITHIC_SEALED_GENERATION_FORMAT_REVISION: u32 = 9; /// The partitioned generation manifest revision, which the daemon publishes. /// -/// Revisions through eight predate required clone-body source rows and are -/// rebuilt rather than interpreted as successful empty clone evidence. -/// Revision 10 stored generation-bound `symbol_occurrences` on each file -/// segment descriptor; revision 11 stores generation-independent -/// `symbol_identities` and rebinds occurrences at restore so one-file seal -/// reuse no longer SHA-256-rebounds every unchanged file's symbols. Revision -/// 12 seals `full_replay_digest` as a parent-delta (optional parent binding). -/// Revision 11 bytes omit that field; decoding them as 12 would fail -/// `validate_for_changes` as contract corruption instead of the typed rebuild -/// refusal, so revision 11 is retired rather than migrated. -pub const SEALED_GENERATION_FORMAT_REVISION_V1: u32 = 12; - -/// The oldest sealed envelope revision this build decodes. Anything below it, -/// and any retired revision between it and -/// [`SEALED_GENERATION_FORMAT_REVISION_V1`], is refused by -/// [`superseded_sealed_generation_revision`] instead of being migrated, a -/// generation is re-derivable from its source tree, so the daemon rebuilds -/// rather than carrying a decoder per retired shape. -pub const MINIMUM_SEALED_GENERATION_FORMAT_REVISION: u32 = - MONOLITHIC_SEALED_GENERATION_FORMAT_REVISION; +/// File segments are compact (DEFLATE-compressed, clone token streams +/// interned, chunk text stored once per file, symbol identities inside the +/// segment), each described by its decoded size and an identity digest. They +/// live in the project's `code-index-v1/`, shared by every worktree scope, and +/// carry no scope identity. File occurrences are minted without the worktree, +/// so identical trees in linked worktrees derive identical artifacts. Each +/// segment descriptor holds generation-independent `symbol_identities`, and +/// restore rebinds occurrences, so a one-file seal does not SHA-256-rebind +/// every unchanged file's symbols. `full_replay_digest` is sealed as a +/// parent delta (optional parent binding). Evidence lineage, request, and +/// receipt rows leave implicit what the generation's own symbols and chunks +/// imply. +/// +/// Every other revision is refused through +/// [`superseded_sealed_generation_revision`], and the generation is rebuilt +/// from source rather than migrated. Revisions through eight also predate +/// required clone-body source rows, so the rebuild keeps them from reading as +/// successful empty clone evidence. +pub const SEALED_GENERATION_FORMAT_REVISION_V1: u32 = 15; /// The typed refusal for a sealed generation this build no longer reads. pub fn superseded_sealed_generation_revision(revision: u32) -> CodeIndexProductionErrorV1 { CodeIndexProductionErrorV1::SupersededSealedGenerationRevision(revision) } -/// One bound, enforced on both sides of the sealed store: encoding refuses to -/// publish a generation larger than this, and decoding refuses to admit one. -/// The bound previously applied only to reads while publication happily wrote -/// larger envelopes, so a large repository sealed generations (~1.5 GB here) -/// that every later load refused as "corrupt", permanently denying its own -/// graph. Two GiB admits those real generations while keeping decode memory -/// bounded. +/// The largest sealed generation file readers admit. Two GiB admits real +/// large-repository generations while keeping decode memory bounded, and the +/// graph write batch bound is sized to cover it. pub const MAX_SEALED_CODE_GENERATION_BYTES_V1: u64 = 2 * 1024 * 1024 * 1024; -fn admit_sealed_generation_len(len: u64) -> Result<(), CodeIndexProductionErrorV1> { - if len > MAX_SEALED_CODE_GENERATION_BYTES_V1 { - return Err(CodeIndexProductionErrorV1::Contract( - "sealed generation exceeds the canonical byte limit".to_owned(), - )); - } - Ok(()) -} - -pub const fn sealed_generation_format_revision_is_compatible(revision: u32) -> bool { - matches!( - revision, - MONOLITHIC_SEALED_GENERATION_FORMAT_REVISION | SEALED_GENERATION_FORMAT_REVISION_V1 - ) -} - -pub fn sealed_generation_payload_digest( - format_revision: u32, - generation: &T, -) -> Result { - match format_revision { - MONOLITHIC_SEALED_GENERATION_FORMAT_REVISION | SEALED_GENERATION_FORMAT_REVISION_V1 => { - json_generation_bytes_and_digest(generation).map(|(_, digest)| digest) - } - revision if revision < MINIMUM_SEALED_GENERATION_FORMAT_REVISION => { - Err(superseded_sealed_generation_revision(revision)) - } - _ => Err(CodeIndexProductionErrorV1::Contract( - "sealed generation format revision is incompatible".to_owned(), - )), - } -} - #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub(super) struct PersistedFileGenerationArtifactsV1 { @@ -107,13 +60,6 @@ pub(super) struct PersistedFileGenerationArtifactsV1 { pub(super) artifacts: CodeFileIndexArtifactsV1, } -#[derive(Serialize)] -pub(super) struct PersistedFileGenerationArtifactsRefV1<'a> { - pub(super) authority: &'a ReceiptBoundCodeFileAuthorityV1, - pub(super) extraction: &'a ExtractionBatchV1, - pub(super) artifacts: &'a CodeFileIndexArtifactsV1, -} - /// The revision-2 file segment payload: the same file record with its chunk /// rows reduced to what the file does not already say. /// @@ -127,9 +73,15 @@ pub(super) struct PersistedFileGenerationArtifactsRefV1<'a> { /// Decoding expands back into [`PersistedFileGenerationArtifactsV1`], and /// every restored row then passes the same chunk validation as a revision-1 /// row before it can be served. +/// +/// Every chunk's text is a span of the file's one sanitized source, and a +/// chunk's exact terms are spans of its text, so the file stores the source +/// its chunks cover once ([`ChunkTextBaseV1`]) and rows keep only spans. A +/// chunk's content digest is the digest of that text and is recomputed. Clone +/// bodies use the row form in [`super::clone_rows`]. #[derive(Serialize)] pub(super) struct PersistedFileGenerationArtifactsRefV2<'a> { - authority: &'a ReceiptBoundCodeFileAuthorityV1, + authority: PersistedFileAuthorityRefV1<'a>, extraction: &'a ExtractionBatchV1, artifacts: PersistedFileIndexArtifactsRefV2<'a>, } @@ -137,11 +89,59 @@ pub(super) struct PersistedFileGenerationArtifactsRefV2<'a> { #[derive(Deserialize)] #[serde(deny_unknown_fields)] pub(super) struct PersistedFileGenerationArtifactsV2 { - authority: ReceiptBoundCodeFileAuthorityV1, + authority: PersistedFileAuthorityV1, extraction: ExtractionBatchV1, artifacts: PersistedFileIndexArtifactsV2, } +/// The identity every file authority of one generation repeats, which +/// validation requires to equal the generation's manifest and snapshot. +/// Segments leave it to the generation that addresses them, so worktrees of +/// one project that seal the same file share that file's segment. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct FileScopeIdentityV1 { + project_id: ProjectId, + repository_id: RepositoryId, + worktree_id: Option, + reference: Option, +} + +impl FileScopeIdentityV1 { + pub(super) fn of( + manifest: &CodeGenerationManifestV1, + snapshot: &SanitizedCodeSnapshotV1, + ) -> Self { + Self { + project_id: manifest.project_id.clone(), + repository_id: snapshot.repository.clone(), + worktree_id: snapshot.worktree.clone(), + reference: snapshot.reference.clone(), + } + } + + pub(super) fn retained_bytes(&self) -> usize { + self.project_id + .as_str() + .len() + .saturating_add(self.repository_id.as_str().len()) + .saturating_add(self.worktree_id.as_ref().map_or(0, |id| id.as_str().len())) + .saturating_add(self.reference.as_ref().map_or(0, |id| id.as_str().len())) + } +} + +#[derive(Serialize)] +struct PersistedFileAuthorityRefV1<'a> { + logical_path: &'a str, + content_digest: &'a ContentDigest, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct PersistedFileAuthorityV1 { + logical_path: String, + content_digest: ContentDigest, +} + #[derive(Serialize)] struct PersistedFileIndexArtifactsRefV2<'a> { chunks: PersistedFileChunksRefV2<'a>, @@ -149,7 +149,7 @@ struct PersistedFileIndexArtifactsRefV2<'a> { edges: &'a [CanonicalRelationEdgeV1], edge_abstentions: &'a [CodeIndexEdgeAbstentionV1], imports: &'a [CodeIndexImportEvidenceV1], - clone_bodies: &'a [CodeIndexCloneBodyV1], + clone_bodies: PersistedCloneBodiesRefV1<'a>, #[serde(skip_serializing_if = "Option::is_none")] schema_evidence: Option<&'a ExtractedSchemaEvidenceV1>, unresolved_references: &'a [CodeIndexUnresolvedReferenceV1], @@ -163,9 +163,8 @@ struct PersistedFileIndexArtifactsV2 { edges: Vec, edge_abstentions: Vec, imports: Vec, - clone_bodies: Vec, + clone_bodies: PersistedCloneBodiesV1, schema_evidence: Option, - #[serde(default)] unresolved_references: Vec, } @@ -177,6 +176,8 @@ struct PersistedFileChunksRefV2<'a> { eligibility: &'a CodeSearchEligibilityV1, #[serde(skip_serializing_if = "Option::is_none")] chunk_defaults: Option>, + #[serde(flatten)] + text: ChunkTextBaseV1, chunks: Vec>, } @@ -189,9 +190,198 @@ struct PersistedFileChunksV2 { eligibility: CodeSearchEligibilityV1, #[serde(default)] chunk_defaults: Option, + text_ranges: Vec<[u64; 2]>, + text: String, chunks: Vec, } +/// The maximal source ranges a file's chunks cover, and the sanitized text of +/// those ranges concatenated in order. Overlapping chunks (a body, its +/// members, its signature) therefore share one stored copy of their bytes. +#[derive(Serialize)] +struct ChunkTextBaseV1 { + text_ranges: Vec<[u64; 2]>, + text: String, +} + +impl ChunkTextBaseV1 { + /// The base for `rows`, and per row whether its text is the base's bytes + /// at its span. A row whose text is not keeps it explicitly, so the form + /// stays lossless. + fn build(rows: &[Arc]) -> (Self, Vec) { + let mut order = (0..rows.len()).collect::>(); + order.sort_by_key(|&index| { + let span = rows[index].anchor.source_span; + (span.start_byte, span.end_byte) + }); + let mut base = Self { + text_ranges: Vec::new(), + text: String::new(), + }; + let mut derived = vec![false; rows.len()]; + for index in order { + let chunk = &rows[index]; + derived[index] = base.admit(chunk.anchor.source_span, chunk.sanitized_text.as_str()); + } + (base, derived) + } + + /// Extend the base with `text` at `span`, or report that the bytes it + /// already holds there disagree. Spans arrive in ascending start order. + fn admit(&mut self, span: SourceSpan, text: &str) -> bool { + let (Ok(start), Ok(end)) = ( + usize::try_from(span.start_byte), + usize::try_from(span.end_byte), + ) else { + return false; + }; + if end.checked_sub(start) != Some(text.len()) { + return false; + } + let last = self + .text_ranges + .last() + .copied() + .and_then(|[range_start, range_end]| { + Some(( + usize::try_from(range_start).ok()?, + usize::try_from(range_end).ok()?, + )) + }); + let Some((range_start, range_end)) = last.filter(|(_, range_end)| start <= *range_end) + else { + self.text_ranges.push([span.start_byte, span.end_byte]); + self.text.push_str(text); + return true; + }; + let Some(offset) = start.checked_sub(range_start) else { + return false; + }; + let covered = range_end.min(end) - start; + let base_start = self.text.len() - (range_end - range_start) + offset; + if self.text.as_bytes().get(base_start..base_start + covered) + != text.as_bytes().get(..covered) + { + return false; + } + if end > range_end { + let Some(tail) = text.get(covered..) else { + return false; + }; + self.text.push_str(tail); + if let Some(range) = self.text_ranges.last_mut() { + range[1] = span.end_byte; + } + } + true + } +} + +/// Restores chunk text from a decoded [`ChunkTextBaseV1`]. +struct ChunkTextSlicesV1<'a> { + ranges: &'a [[u64; 2]], + offsets: Vec, + text: &'a str, +} + +impl<'a> ChunkTextSlicesV1<'a> { + fn new(ranges: &'a [[u64; 2]], text: &'a str) -> Result { + let mut offsets = Vec::with_capacity(ranges.len()); + let mut total = 0_u64; + let mut previous_end = None; + for [start, end] in ranges { + if end <= start || previous_end.is_some_and(|previous| previous > *start) { + return Err(CodeIndexProductionErrorV1::Contract( + "sealed file segment text ranges are not ascending and disjoint".to_owned(), + )); + } + offsets.push(total); + total = total.checked_add(end - start).ok_or_else(|| { + CodeIndexProductionErrorV1::Contract( + "sealed file segment text length exceeds u64".to_owned(), + ) + })?; + previous_end = Some(*end); + } + if u64::try_from(text.len()).ok() != Some(total) { + return Err(CodeIndexProductionErrorV1::Contract( + "sealed file segment text does not match its ranges".to_owned(), + )); + } + Ok(Self { + ranges, + offsets, + text, + }) + } + + fn slice(&self, span: SourceSpan) -> Result<&'a str, CodeIndexProductionErrorV1> { + let text = self.text; + self.ranges + .partition_point(|range| range[0] <= span.start_byte) + .checked_sub(1) + .filter(|index| span.end_byte <= self.ranges[*index][1]) + .and_then(|index| { + let from = + self.offsets[index].checked_add(span.start_byte - self.ranges[index][0])?; + let to = from.checked_add(span.end_byte.checked_sub(span.start_byte)?)?; + text.get(usize::try_from(from).ok()?..usize::try_from(to).ok()?) + }) + .ok_or_else(|| { + CodeIndexProductionErrorV1::Contract( + "sealed file segment chunk span is outside its stored text".to_owned(), + ) + }) + } +} + +/// An exact term row: its bytes are the chunk text at `span`. +#[derive(Serialize)] +struct PersistedExactTermRefV1<'a> { + kind: ExactTechnicalTermKindV1, + span: SourceSpan, + #[serde(skip_serializing_if = "Option::is_none")] + symbol_occurrence_id: Option<&'a SymbolOccurrenceId>, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct PersistedExactTermV1 { + kind: ExactTechnicalTermKindV1, + span: SourceSpan, + #[serde(default)] + symbol_occurrence_id: Option, +} + +impl<'a> PersistedExactTermRefV1<'a> { + fn new( + chunk: &CodeSearchChunkV1, + term: &'a ExactTechnicalTermV1, + ) -> Result { + if term_bytes( + chunk.sanitized_text.as_str(), + chunk.anchor.source_span, + term.span(), + ) != Some(term.original_bytes()) + { + return Err(CodeIndexProductionErrorV1::Contract( + "sealed chunk exact term is not its chunk text at its span".to_owned(), + )); + } + Ok(Self { + kind: term.kind(), + span: term.span(), + symbol_occurrence_id: term.symbol_occurrence_id(), + }) + } +} + +fn term_bytes(text: &str, chunk_span: SourceSpan, term_span: SourceSpan) -> Option<&[u8]> { + let from = usize::try_from(term_span.start_byte.checked_sub(chunk_span.start_byte)?).ok()?; + let to = usize::try_from(term_span.end_byte.checked_sub(chunk_span.start_byte)?).ok()?; + text.as_bytes().get(from..to) +} + #[derive(Serialize)] struct PersistedChunkDefaultsRefV2<'a> { language_descriptor_revision: &'a LanguageDescriptorRevision, @@ -223,7 +413,9 @@ struct PersistedChunkRefV2<'a> { source_span: SourceSpan, grain: CodeSearchChunkGrainV1, ordinal: u32, - content_digest: &'a ContentDigest, + /// Present only when it is not the digest of the chunk's text. + #[serde(skip_serializing_if = "Option::is_none")] + content_digest: Option<&'a ContentDigest>, #[serde(skip_serializing_if = "Option::is_none")] language_descriptor_revision: Option<&'a LanguageDescriptorRevision>, #[serde(skip_serializing_if = "Option::is_none")] @@ -232,9 +424,11 @@ struct PersistedChunkRefV2<'a> { sanitizer_revision: Option<&'a SanitizerRevision>, #[serde(skip_serializing_if = "Option::is_none")] sensitivity: Option<&'a SensitivityDecision>, - exact_terms: &'a [ExactTechnicalTermV1], + exact_terms: Vec>, subtokens: &'a [String], - sanitized_text: &'a BoundedSanitizedText, + /// Present only when the file's text base does not hold it at its span. + #[serde(skip_serializing_if = "Option::is_none")] + sanitized_text: Option<&'a BoundedSanitizedText>, } #[derive(Deserialize)] @@ -250,7 +444,8 @@ struct PersistedChunkV2 { source_span: SourceSpan, grain: CodeSearchChunkGrainV1, ordinal: u32, - content_digest: ContentDigest, + #[serde(default)] + content_digest: Option, #[serde(default)] language_descriptor_revision: Option, #[serde(default)] @@ -259,18 +454,32 @@ struct PersistedChunkV2 { sanitizer_revision: Option, #[serde(default)] sensitivity: Option, - exact_terms: Vec, + exact_terms: Vec, subtokens: Vec, - sanitized_text: BoundedSanitizedText, + #[serde(default)] + sanitized_text: Option, } impl<'a> PersistedFileGenerationArtifactsRefV2<'a> { + /// Refuses a file whose authority names another scope than its + /// generation, rather than persisting a row that restores differently. pub(super) fn new( + scope: &FileScopeIdentityV1, authority: &'a ReceiptBoundCodeFileAuthorityV1, extraction: &'a ExtractionBatchV1, artifacts: &'a CodeFileIndexArtifactsV1, - ) -> Self { + ) -> Result { + if authority.project_id != scope.project_id + || authority.repository_id != scope.repository_id + || authority.worktree_id != scope.worktree_id + || authority.reference != scope.reference + { + return Err(CodeIndexProductionErrorV1::Contract( + "sealed file authority names another scope than its generation".to_owned(), + )); + } let rows = &artifacts.chunks.chunks; + let (text, derived_text) = ChunkTextBaseV1::build(rows); let defaults = rows.first().map(|first| PersistedChunkDefaultsRefV2 { language_descriptor_revision: &first.language_descriptor_revision, chunker_revision: &first.chunker_revision, @@ -284,14 +493,15 @@ impl<'a> PersistedFileGenerationArtifactsRefV2<'a> { .collect::>(); let chunks = rows .iter() - .map(|chunk| { + .zip(derived_text) + .map(|(chunk, derived_text)| { let parent = chunk .anchor .parent_chunk_id .as_ref() .and_then(|parent| row_index.get(parent)) .and_then(|index| u32::try_from(*index).ok()); - PersistedChunkRefV2 { + Ok(PersistedChunkRefV2 { id: &chunk.id, symbol_occurrence_id: chunk.anchor.symbol_occurrence_id.as_ref(), parent, @@ -303,7 +513,9 @@ impl<'a> PersistedFileGenerationArtifactsRefV2<'a> { source_span: chunk.anchor.source_span, grain: chunk.anchor.grain, ordinal: chunk.anchor.ordinal, - content_digest: &chunk.content_digest, + content_digest: (chunk.content_digest + != content_digest(chunk.sanitized_text.as_str().as_bytes())) + .then_some(&chunk.content_digest), language_descriptor_revision: own_unless_default( &chunk.language_descriptor_revision, defaults @@ -324,14 +536,21 @@ impl<'a> PersistedFileGenerationArtifactsRefV2<'a> { &chunk.sensitivity, defaults.as_ref().map(|defaults| defaults.sensitivity), ), - exact_terms: &chunk.exact_terms, + exact_terms: chunk + .exact_terms + .iter() + .map(|term| PersistedExactTermRefV1::new(chunk, term)) + .collect::>()?, subtokens: &chunk.subtokens, - sanitized_text: &chunk.sanitized_text, - } + sanitized_text: (!derived_text).then_some(&chunk.sanitized_text), + }) }) - .collect(); - Self { - authority, + .collect::>()?; + Ok(Self { + authority: PersistedFileAuthorityRefV1 { + logical_path: &authority.logical_path, + content_digest: &authority.content_digest, + }, extraction, artifacts: PersistedFileIndexArtifactsRefV2 { chunks: PersistedFileChunksRefV2 { @@ -340,20 +559,51 @@ impl<'a> PersistedFileGenerationArtifactsRefV2<'a> { content_digest: &artifacts.chunks.document.content_digest, eligibility: &artifacts.chunks.document.eligibility, chunk_defaults: defaults, + text, chunks, }, symbols: &artifacts.symbols, edges: &artifacts.edges, edge_abstentions: &artifacts.edge_abstentions, imports: &artifacts.imports, - clone_bodies: &artifacts.clone_bodies, + clone_bodies: PersistedCloneBodiesRefV1::new( + authority, + extraction, + &clone_bodies_by_symbol_identity(artifacts), + )?, schema_evidence: artifacts.schema_evidence.as_ref(), unresolved_references: &artifacts.unresolved_references, }, - } + }) } } +/// Clone bodies in memory sort by symbol occurrence, which hashes the +/// worktree's file occurrence; persisting them by symbol identity instead +/// lets identical files in linked worktrees seal to one segment. Restore +/// re-sorts by occurrence. +fn clone_bodies_by_symbol_identity( + artifacts: &CodeFileIndexArtifactsV1, +) -> Vec<&CodeIndexCloneBodyV1> { + let identities = artifacts + .symbols + .iter() + .map(|symbol| (&symbol.occurrence, &symbol.identity)) + .collect::>(); + let mut bodies = artifacts.clone_bodies.iter().collect::>(); + bodies.sort_by(|left, right| { + let (left, right) = ( + &left.occurrence.symbol_occurrence_id, + &right.occurrence.symbol_occurrence_id, + ); + identities + .get(left) + .cmp(&identities.get(right)) + .then_with(|| left.cmp(right)) + }); + bodies +} + /// A row carries its own value only where it differs from the file default. fn own_unless_default<'a, T: PartialEq>(value: &'a T, default: Option<&'a T>) -> Option<&'a T> { (default != Some(value)).then_some(value) @@ -365,10 +615,23 @@ impl PersistedFileGenerationArtifactsV2 { /// then decides whether the expanded file is admissible. pub(super) fn expand( self, + scope: &FileScopeIdentityV1, ) -> Result { + let authority = ReceiptBoundCodeFileAuthorityV1 { + project_id: scope.project_id.clone(), + repository_id: scope.repository_id.clone(), + worktree_id: scope.worktree_id.clone(), + reference: scope.reference.clone(), + logical_path: self.authority.logical_path, + content_digest: self.authority.content_digest, + }; let artifacts = self.artifacts; + let clone_bodies = artifacts + .clone_bodies + .expand(&authority, &self.extraction)?; let file = artifacts.chunks; let defaults = file.chunk_defaults; + let text = ChunkTextSlicesV1::new(&file.text_ranges, &file.text)?; let ids = file .chunks .iter() @@ -394,6 +657,30 @@ impl PersistedFileGenerationArtifactsV2 { )); } }; + let sanitized_text = match chunk.sanitized_text { + Some(explicit) => explicit, + None => BoundedSanitizedText::new(text.slice(chunk.source_span)?) + .map_err(|error| CodeIndexProductionErrorV1::Contract(error.to_string()))?, + }; + let exact_terms = chunk + .exact_terms + .into_iter() + .map(|term| { + let bytes = term_bytes(sanitized_text.as_str(), chunk.source_span, term.span) + .ok_or_else(|| { + CodeIndexProductionErrorV1::Contract( + "sealed chunk exact term span is outside its chunk text".to_owned(), + ) + })?; + ExactTechnicalTermV1::from_persisted_parts( + term.kind, + bytes.to_vec(), + term.span, + term.symbol_occurrence_id, + ) + .map_err(|error| CodeIndexProductionErrorV1::Contract(error.to_string())) + }) + .collect::, _>>()?; chunks.push(Arc::new(CodeSearchChunkV1 { id: chunk.id, anchor: CodeSearchChunkAnchorV1 { @@ -405,7 +692,9 @@ impl PersistedFileGenerationArtifactsV2 { grain: chunk.grain, ordinal: chunk.ordinal, }, - content_digest: chunk.content_digest, + content_digest: chunk + .content_digest + .unwrap_or_else(|| content_digest(sanitized_text.as_str().as_bytes())), language_descriptor_revision: chunk .language_descriptor_revision .or_else(|| { @@ -438,13 +727,13 @@ impl PersistedFileGenerationArtifactsV2 { .map(|defaults| defaults.sensitivity.clone()) }) .ok_or_else(|| missing_default("sensitivity"))?, - exact_terms: chunk.exact_terms, + exact_terms, subtokens: chunk.subtokens, - sanitized_text: chunk.sanitized_text, + sanitized_text, })); } Ok(PersistedFileGenerationArtifactsV1 { - authority: self.authority, + authority, extraction: self.extraction, artifacts: CodeFileIndexArtifactsV1 { chunks: CodeFileChunksV1 { @@ -461,7 +750,7 @@ impl PersistedFileGenerationArtifactsV2 { edges: artifacts.edges, edge_abstentions: artifacts.edge_abstentions, imports: artifacts.imports, - clone_bodies: artifacts.clone_bodies, + clone_bodies, schema_evidence: artifacts.schema_evidence, unresolved_references: artifacts.unresolved_references, }, @@ -469,93 +758,13 @@ impl PersistedFileGenerationArtifactsV2 { } } -#[derive(Clone, Copy, Debug)] -pub(super) struct CompatibleSealedFormatRevisionV1(pub(super) u32); - -impl Serialize for CompatibleSealedFormatRevisionV1 { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - serializer.serialize_u32(self.0) - } -} - -impl<'de> Deserialize<'de> for CompatibleSealedFormatRevisionV1 { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - let revision = u32::deserialize(deserializer)?; - if !sealed_generation_format_revision_is_compatible(revision) { - return Err(serde::de::Error::custom( - // Retirement runs from below the decode floor up to the - // revision the writer emits, so a revision this build once - // wrote and has since retired is refused for rebuild rather - // than reported as a shape only a newer build produces. - if revision < SEALED_GENERATION_FORMAT_REVISION_V1 { - superseded_sealed_generation_revision(revision).to_string() - } else { - "sealed generation format revision is incompatible".to_owned() - }, - )); - } - Ok(Self(revision)) - } -} - -/// The sealed `files` array, decoded page by page. The visitor is pure -/// decode: each persist page accumulates exactly once (the pages are the -/// restored corpus), and the CPU-bound authority reconstruction is deferred -/// to [`assemble_published_generation`]'s pool fan-out so the deserializer -/// thread never serializes corpus-scale digest work. -pub(super) struct StreamingRestoredFilesV1 { - pub(super) files: Vec, -} - -impl<'de> Deserialize<'de> for StreamingRestoredFilesV1 { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - struct FilesVisitor; - - impl<'de> Visitor<'de> for FilesVisitor { - type Value = StreamingRestoredFilesV1; - - fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter.write_str("a sealed generation files array") - } - - fn visit_seq(self, mut seq: A) -> Result - where - A: SeqAccess<'de>, - { - let mut files = Vec::new(); - if let Some(hint) = seq.size_hint() { - files.reserve(hint); - } - while let Some(file) = seq.next_element::()? { - files.push(file); - } - Ok(StreamingRestoredFilesV1 { files }) - } - } - - deserializer.deserialize_seq(FilesVisitor) - } -} - -#[derive(Deserialize)] -#[serde(deny_unknown_fields)] pub(super) struct StreamingPersistedPublishedGenerationV1 { - pub(super) format_revision: CompatibleSealedFormatRevisionV1, pub(super) manifest: CodeGenerationManifestV1, pub(super) snapshot: SanitizedCodeSnapshotV1, pub(super) repository_parse_identity: CodeIndexRepositoryParseIdentityV1, pub(super) ignored_source_admissions: Vec, pub(super) ignored_source_admissions_digest: ManifestDigest, - pub(super) files: StreamingRestoredFilesV1, + pub(super) files: Vec, pub(super) lineage: Vec, pub(super) coverage: CoverageSummaryV1, pub(super) capability: CodeIndexCapabilityManifestV1, @@ -563,13 +772,6 @@ pub(super) struct StreamingPersistedPublishedGenerationV1 { pub(super) projection_receipt: ProjectionBatchReceiptV1, } -#[derive(Deserialize)] -#[serde(deny_unknown_fields)] -struct StreamingSealedEnvelopeV1 { - state_digest: ManifestDigest, - generation: StreamingPersistedPublishedGenerationV1, -} - /// Rebuild every file's parser-backed exact authority on the indexing pool, /// then move each persist page into its published artifact. /// @@ -604,13 +806,12 @@ pub(super) fn assemble_published_generation( generation: StreamingPersistedPublishedGenerationV1, ) -> Result { let StreamingPersistedPublishedGenerationV1 { - format_revision: _, manifest, snapshot, repository_parse_identity, ignored_source_admissions, ignored_source_admissions_digest, - files: StreamingRestoredFilesV1 { files }, + files, lineage, coverage, capability, @@ -718,745 +919,10 @@ pub(super) fn assemble_published_generation( Ok(published) } -#[derive(Serialize)] -struct PersistedPublishedGenerationRefV1<'a> { - format_revision: u32, - manifest: &'a CodeGenerationManifestV1, - snapshot: &'a SanitizedCodeSnapshotV1, - repository_parse_identity: &'a CodeIndexRepositoryParseIdentityV1, - ignored_source_admissions: &'a [CodeIndexIgnoredSourceAdmissionV1], - ignored_source_admissions_digest: &'a ManifestDigest, - /// Pre-encoded file JSON. Each file is serialized on the indexing pool - /// before the envelope is stitched so a 700+ file generation does not - /// pay a single-threaded `to_writer` of the whole files array. - files: Vec>, - lineage: &'a [SymbolLineageCandidateV1], - coverage: CoverageSummaryV1, - capability: &'a CodeIndexCapabilityManifestV1, - projection_request: &'a ProjectionBatchRequestV1, - projection_receipt: &'a ProjectionBatchReceiptV1, -} - -#[derive(Deserialize)] -struct SealedPublishedGenerationRawEnvelopeV1<'a> { - state_digest: ManifestDigest, - #[serde(borrow)] - generation: &'a RawValue, -} - -#[derive(Deserialize)] -struct SealedPublishedGenerationFormatProbeV1 { - generation: PersistedPublishedGenerationFormatProbeV1, -} - -#[derive(Deserialize)] -struct PersistedPublishedGenerationFormatProbeV1 { - format_revision: u32, -} - -/// Materialize one sealed monolithic envelope with the fewest corpus-scale -/// passes: `None` means the bytes belong to another decoder (a partitioned -/// manifest), and a superseded revision is refused outright. -/// -/// The happy path is exactly one boundary parse (isolating the payload -/// bytes), one payload digest, and one typed materialization. The standalone -/// format probe runs only when that single-pass decode cannot accept the -/// bytes, where [`classify_unaccepted_envelope`] decides whether the -/// interrupting rejection is real or the bytes are simply not monolithic. The -/// digest is computed and compared before the payload is materialized, so a -/// corrupt envelope is still rejected without building corpus-scale -/// structures from unverified bytes. -fn materialize_compatible_envelope( - bytes: &[u8], -) -> Result, CodeIndexProductionErrorV1> { - let raw: SealedPublishedGenerationRawEnvelopeV1 = match hotpath::measure_block!( - "code_index.generation.decode.raw_envelope_parse", - serde_json::from_slice(bytes) - ) { - Ok(raw) => raw, - Err(error) => { - return classify_unaccepted_envelope( - bytes, - CodeIndexProductionErrorV1::Contract(format!( - "sealed generation decoding failed: {error}" - )), - ); - } - }; - let payload_digest = hotpath::measure_block!( - "code_index.sealed_decode.v6_payload_digest", - json_generation_digest(raw.generation.get().as_bytes()) - )?; - if payload_digest != raw.state_digest { - // Corrupt under the monolithic raw-bytes rule, but the probe, never - // the digest, decides which revision owns these bytes, so the - // format-revision gate stays authoritative. - return classify_unaccepted_envelope( - bytes, - CodeIndexProductionErrorV1::Contract( - "sealed generation state digest does not match its payload".to_owned(), - ), - ); - } - let streamed: Result = hotpath::measure_block!( - "code_index.sealed_decode.persisted_materialization", - serde_json::from_str(raw.generation.get()) - ); - match streamed { - Ok(generation) - if generation.format_revision.0 == MONOLITHIC_SEALED_GENERATION_FORMAT_REVISION => - { - Ok(Some(generation)) - } - // The only other admitted revision is the partitioned manifest, which - // this decoder does not own. - Ok(_) => Ok(None), - Err(error) => classify_unaccepted_envelope( - bytes, - CodeIndexProductionErrorV1::Contract(format!( - "sealed generation payload decoding failed: {error}" - )), - ), - } -} - -/// Probe-first classification for bytes the single-pass decode did not -/// accept: a monolithic revision keeps the exact typed rejection that -/// interrupted the decode, a superseded revision is refused so the caller -/// rebuilds from source, and anything else abstains for another decoder. -fn classify_unaccepted_envelope( - bytes: &[u8], - monolithic_rejection: CodeIndexProductionErrorV1, -) -> Result, CodeIndexProductionErrorV1> { - let probe: SealedPublishedGenerationFormatProbeV1 = hotpath::measure_block!( - "code_index.generation.decode.format_probe", - serde_json::from_slice(bytes).map_err(|error| { - CodeIndexProductionErrorV1::Contract(format!( - "sealed generation format probe failed: {error}" - )) - }) - )?; - match probe.generation.format_revision { - MONOLITHIC_SEALED_GENERATION_FORMAT_REVISION => Err(monolithic_rejection), - revision if revision < MINIMUM_SEALED_GENERATION_FORMAT_REVISION => { - Err(superseded_sealed_generation_revision(revision)) - } - _ => Ok(None), - } -} - -fn admit_sealed_generation_bytes( - bytes: &[u8], - admitted_len: u64, -) -> Result<&[u8], CodeIndexProductionErrorV1> { - admit_sealed_generation_len(admitted_len)?; - let actual_len = u64::try_from(bytes.len()).map_err(|_| { - CodeIndexProductionErrorV1::Contract("sealed generation length exceeds u64".to_owned()) - })?; - if actual_len != admitted_len { - return Err(CodeIndexProductionErrorV1::Contract( - "sealed generation length does not match its admitted length".to_owned(), - )); - } - Ok(bytes) -} - -const SEALED_GENERATION_WRITE_CHUNK_BYTES_V1: usize = 1024 * 1024; - -struct BoundedChunkWriterV1<'a, W> { - writer: &'a mut W, - written: u64, - byte_limit: u64, - maximum_write: usize, - limit_exceeded: bool, -} - -impl Write for BoundedChunkWriterV1<'_, W> { - fn write(&mut self, bytes: &[u8]) -> std::io::Result { - if bytes.is_empty() { - return Ok(0); - } - let remaining = self.byte_limit.saturating_sub(self.written); - if remaining == 0 { - self.limit_exceeded = true; - return Err(std::io::Error::new( - std::io::ErrorKind::FileTooLarge, - "sealed generation exceeds the canonical byte limit", - )); - } - let remaining = usize::try_from(remaining).unwrap_or(usize::MAX); - let admitted = bytes.len().min(self.maximum_write).min(remaining); - let written = self.writer.write(&bytes[..admitted])?; - self.written = self - .written - .checked_add(u64::try_from(written).map_err(std::io::Error::other)?) - .ok_or_else(|| std::io::Error::other("sealed generation length overflowed"))?; - Ok(written) - } - - fn flush(&mut self) -> std::io::Result<()> { - self.writer.flush() - } -} - -struct GenerationDigestWriterV1<'writer, 'sink, W> { - writer: &'writer mut BoundedChunkWriterV1<'sink, W>, - hasher: Sha256, -} - -impl Write for GenerationDigestWriterV1<'_, '_, W> { - fn write(&mut self, bytes: &[u8]) -> std::io::Result { - let written = self.writer.write(bytes)?; - self.hasher.update(&bytes[..written]); - Ok(written) - } - - fn flush(&mut self) -> std::io::Result<()> { - self.writer.flush() - } -} - -fn byte_limit_error() -> CodeIndexProductionErrorV1 { - CodeIndexProductionErrorV1::Contract( - "sealed generation exceeds the canonical byte limit".to_owned(), - ) -} - -fn write_chunked( - writer: &mut W, - mut bytes: &[u8], - maximum_write: usize, -) -> std::io::Result<()> { - while !bytes.is_empty() { - let written = writer.write(&bytes[..bytes.len().min(maximum_write)])?; - if written == 0 { - return Err(std::io::ErrorKind::WriteZero.into()); - } - bytes = &bytes[written..]; - } - Ok(()) -} - -fn write_generation_envelope_with_limits( - generation: &T, - writer: &mut W, - byte_limit: u64, - maximum_write: usize, -) -> Result { - if maximum_write == 0 { - return Err(CodeIndexProductionErrorV1::Contract( - "sealed generation write chunk must be non-zero".to_owned(), - )); - } - let placeholder = ManifestDigest::from_sha256_bytes(&[0; 32]) - .map_err(|error| CodeIndexProductionErrorV1::Contract(error.to_string()))?; - let envelope_start = writer.stream_position().map_err(|error| { - CodeIndexProductionErrorV1::Contract(format!( - "sealed generation writer position failed: {error}" - )) - })?; - let mut writer = BufWriter::with_capacity(maximum_write, writer); - let (digest_start, digest_end, generation_hash, written) = { - let mut bounded = BoundedChunkWriterV1 { - writer: &mut writer, - written: 0, - byte_limit, - maximum_write, - limit_exceeded: false, - }; - bounded.write_all(b"{\"state_digest\":").map_err(|error| { - if bounded.limit_exceeded { - byte_limit_error() - } else { - CodeIndexProductionErrorV1::Contract(format!( - "sealed generation serialization failed: {error}" - )) - } - })?; - let digest_start = envelope_start.checked_add(bounded.written).ok_or_else(|| { - CodeIndexProductionErrorV1::Contract( - "sealed generation writer position overflowed".to_owned(), - ) - })?; - if let Err(error) = serde_json::to_writer(&mut bounded, &placeholder) { - return Err(if bounded.limit_exceeded { - byte_limit_error() - } else { - CodeIndexProductionErrorV1::Contract(format!( - "sealed generation digest serialization failed: {error}" - )) - }); - } - let digest_end = envelope_start.checked_add(bounded.written).ok_or_else(|| { - CodeIndexProductionErrorV1::Contract( - "sealed generation writer position overflowed".to_owned(), - ) - })?; - bounded.write_all(b",\"generation\":").map_err(|error| { - if bounded.limit_exceeded { - byte_limit_error() - } else { - CodeIndexProductionErrorV1::Contract(format!( - "sealed generation serialization failed: {error}" - )) - } - })?; - let generation_hash = { - let mut generation_writer = GenerationDigestWriterV1 { - writer: &mut bounded, - hasher: Sha256::new(), - }; - if let Err(error) = serde_json::to_writer(&mut generation_writer, generation) { - return Err(if generation_writer.writer.limit_exceeded { - byte_limit_error() - } else { - CodeIndexProductionErrorV1::Contract(format!( - "sealed generation serialization failed: {error}" - )) - }); - } - generation_writer.hasher.finalize() - }; - bounded.write_all(b"}").map_err(|error| { - if bounded.limit_exceeded { - byte_limit_error() - } else { - CodeIndexProductionErrorV1::Contract(format!( - "sealed generation serialization failed: {error}" - )) - } - })?; - bounded.flush().map_err(|error| { - CodeIndexProductionErrorV1::Contract(format!( - "sealed generation serialization flush failed: {error}" - )) - })?; - (digest_start, digest_end, generation_hash, bounded.written) - }; - - let state_digest = ManifestDigest::from_sha256_bytes(&generation_hash) - .map_err(|error| CodeIndexProductionErrorV1::Contract(error.to_string()))?; - let digest_bytes = serde_json::to_vec(&state_digest).map_err(|error| { - CodeIndexProductionErrorV1::Contract(format!( - "sealed generation digest serialization failed: {error}" - )) - })?; - let digest_width = digest_end - .checked_sub(digest_start) - .and_then(|width| usize::try_from(width).ok()) - .ok_or_else(|| { - CodeIndexProductionErrorV1::Contract( - "sealed generation digest width overflowed".to_owned(), - ) - })?; - if digest_bytes.len() != digest_width { - return Err(CodeIndexProductionErrorV1::Contract( - "sealed generation digest width changed during encoding".to_owned(), - )); - } - writer - .seek(SeekFrom::Start(digest_start)) - .map_err(|error| { - CodeIndexProductionErrorV1::Contract(format!( - "sealed generation digest seek failed: {error}" - )) - })?; - write_chunked(&mut writer, &digest_bytes, maximum_write).map_err(|error| { - CodeIndexProductionErrorV1::Contract(format!( - "sealed generation digest serialization failed: {error}" - )) - })?; - writer.flush().map_err(|error| { - CodeIndexProductionErrorV1::Contract(format!( - "sealed generation digest flush failed: {error}" - )) - })?; - let envelope_end = envelope_start.checked_add(written).ok_or_else(|| { - CodeIndexProductionErrorV1::Contract( - "sealed generation writer position overflowed".to_owned(), - ) - })?; - writer - .seek(SeekFrom::Start(envelope_end)) - .map_err(|error| { - CodeIndexProductionErrorV1::Contract(format!( - "sealed generation final seek failed: {error}" - )) - })?; - writer.flush().map_err(|error| { - CodeIndexProductionErrorV1::Contract(format!( - "sealed generation final flush failed: {error}" - )) - })?; - Ok(written) -} - -fn encode_persisted_files_parallel( - files: &[Arc], -) -> Result>, CodeIndexProductionErrorV1> { - hotpath::measure_block!("code_index.sealed_encode.files", { - super::collect_bounded_ordered(files, |file, _| { - let persisted = PersistedFileGenerationArtifactsRefV1 { - authority: &file.authority, - extraction: &file.extraction, - artifacts: &file.artifacts, - }; - serde_json::value::to_raw_value(&persisted).map_err(|error| { - CodeIndexProductionErrorV1::Contract(format!( - "sealed generation file serialization failed: {error}" - )) - }) - }) - }) -} - -fn json_generation_digest( - generation_bytes: &[u8], -) -> Result { - ManifestDigest::from_sha256_bytes(&Sha256::digest(generation_bytes)) - .map_err(|error| CodeIndexProductionErrorV1::Contract(error.to_string())) -} - -fn json_generation_bytes_and_digest( - generation: &T, -) -> Result<(Vec, ManifestDigest), CodeIndexProductionErrorV1> { - let generation_bytes = serde_json::to_vec(generation).map_err(|error| { - CodeIndexProductionErrorV1::Contract(format!( - "sealed generation serialization failed: {error}" - )) - })?; - let state_digest = json_generation_digest(&generation_bytes)?; - Ok((generation_bytes, state_digest)) -} - -impl CodeIndexPublishedGenerationV1 { - /// Stream the complete sealed generation into one seekable immutable-store - /// sink. Writes and the total envelope are bounded independently, and the - /// payload digest is patched in place after the generation has been hashed. - #[hotpath::measure(label = "code_index.sealed_encode.write")] - pub fn write_sealed( - &self, - writer: &mut W, - ) -> Result { - self.validate()?; - let files = encode_persisted_files_parallel(&self.files)?; - let generation = PersistedPublishedGenerationRefV1 { - format_revision: MONOLITHIC_SEALED_GENERATION_FORMAT_REVISION, - manifest: &self.manifest, - snapshot: &self.snapshot, - repository_parse_identity: &self.repository_parse_identity, - ignored_source_admissions: self.ignored_source_roster.admissions(), - ignored_source_admissions_digest: self.ignored_source_roster.digest(), - files, - lineage: &self.lineage, - coverage: self.coverage, - capability: &self.capability, - projection_request: self.projection.request(), - projection_receipt: self.projection.receipt(), - }; - let written = write_generation_envelope_with_limits( - &generation, - writer, - MAX_SEALED_CODE_GENERATION_BYTES_V1, - SEALED_GENERATION_WRITE_CHUNK_BYTES_V1, - )?; - crate::hotpath_observe::record_seal_bytes(written); - Ok(written) - } - - /// Encode the complete sealed generation in memory for callers that need - /// an owned wire payload. Durable publication uses [`Self::write_sealed`] - /// so it never materializes a corpus-sized intermediate buffer. - pub fn encode_sealed(&self) -> Result, CodeIndexProductionErrorV1> { - let mut sealed = std::io::Cursor::new(Vec::new()); - self.write_sealed(&mut sealed)?; - Ok(sealed.into_inner()) - } - - /// Restore and revalidate a complete sealed generation. - #[hotpath::measure(label = "code_index.sealed_decode")] - pub fn decode_sealed(bytes: &[u8]) -> Result { - Self::decode_sealed_if_compatible(bytes)?.ok_or_else(|| { - CodeIndexProductionErrorV1::Contract( - "sealed generation format revision is incompatible".to_owned(), - ) - }) - } - - /// Restore one compatible sealed generation without a separate format - /// probe over the same corpus-sized byte slice. - #[hotpath::measure(label = "code_index.generation.decode")] - pub fn decode_sealed_if_compatible( - bytes: &[u8], - ) -> Result, CodeIndexProductionErrorV1> { - let admitted_len = u64::try_from(bytes.len()).map_err(|_| { - CodeIndexProductionErrorV1::Contract("sealed generation length exceeds u64".to_owned()) - })?; - Self::decode_admitted_sealed_bytes_if_compatible(bytes, admitted_len) - } - - fn decode_admitted_sealed_bytes_if_compatible( - bytes: &[u8], - admitted_len: u64, - ) -> Result, CodeIndexProductionErrorV1> { - let bytes = hotpath::measure_block!( - "code_index.sealed_decode.input_admission", - admit_sealed_generation_bytes(bytes, admitted_len) - )?; - crate::hotpath_observe::record_seal_bytes(admitted_len); - match materialize_compatible_envelope(bytes)? { - Some(generation) => assemble_published_generation(generation).map(Some), - None => Ok(None), - } - } - - /// Stream one compatible sealed generation from a seekable reader without - /// holding the envelope bytes or the persist corpus in memory at once. - /// - /// The layout scan proves the envelope digest incrementally (the same - /// `sha256(generation_json)` rule as [`Self::decode_sealed_if_compatible`]) - /// and optionally the durable file digest. Each `files` element is then - /// restored and dropped before the next is decoded. - #[hotpath::measure(label = "code_index.generation.decode.seek")] - pub fn decode_sealed_seek_reader( - mut reader: R, - admitted_len: u64, - expected_file_digest: Option<&ManifestDigest>, - control: &dyn CodeIndexExecutionControlV1, - ) -> Result, CodeIndexProductionErrorV1> { - admit_sealed_generation_len(admitted_len)?; - crate::hotpath_observe::record_seal_bytes(admitted_len); - let layout = match scan_layout(&mut reader, admitted_len, expected_file_digest, control) { - Ok(layout) => layout, - Err(CodeIndexProductionErrorV1::Contract(message)) - if message.contains("format revision is incompatible") => - { - return Ok(None); - } - Err(error) => return Err(error), - }; - reader.seek(SeekFrom::Start(0)).map_err(|error| { - CodeIndexProductionErrorV1::Contract(format!( - "sealed generation decode seek failed: {error}" - )) - })?; - match layout.format_revision { - MONOLITHIC_SEALED_GENERATION_FORMAT_REVISION => { - let envelope: StreamingSealedEnvelopeV1 = hotpath::measure_block!( - "code_index.sealed_decode.persisted_materialization", - serde_json::from_reader(BufReader::with_capacity(64 * 1024, reader)) - ) - .map_err(|error| { - CodeIndexProductionErrorV1::Contract(format!( - "sealed generation payload decoding failed: {error}" - )) - })?; - if envelope.state_digest != layout.state_digest { - return Err(CodeIndexProductionErrorV1::Contract( - "sealed generation state digest does not match its payload".to_owned(), - )); - } - assemble_published_generation(envelope.generation).map(Some) - } - // Every other revision the layout scan admits belongs to the - // partitioned decoder; a superseded one never reaches here because - // `scan_layout` refuses it outright. - _ => Ok(None), - } - } - - pub fn sealed_format_is_compatible(bytes: &[u8]) -> Result { - let probe: SealedPublishedGenerationFormatProbeV1 = - serde_json::from_slice(bytes).map_err(|error| { - CodeIndexProductionErrorV1::Contract(format!( - "sealed generation format probe failed: {error}" - )) - })?; - Ok(sealed_generation_format_revision_is_compatible( - probe.generation.format_revision, - )) - } -} - #[cfg(test)] mod tests { - use std::alloc::{GlobalAlloc, Layout, System}; - use std::cell::Cell; - use super::*; - #[test] - fn format_gate_accepts_only_the_monolithic_and_partitioned_revisions() { - assert!(sealed_generation_format_revision_is_compatible(9)); - assert!(sealed_generation_format_revision_is_compatible(12)); - assert!(!sealed_generation_format_revision_is_compatible(8)); - // Revision 10 stored generation-bound symbol occurrence lists. - // Revision 11 sealed generation-independent symbol identities but a - // full-corpus `full_replay_digest`. Revision 12 seals that digest as - // a parent-delta, so 11 is rebuilt rather than decoded as corruption. - assert!(!sealed_generation_format_revision_is_compatible(10)); - assert!(!sealed_generation_format_revision_is_compatible(11)); - assert!(!sealed_generation_format_revision_is_compatible(13)); - } - - struct LargestAllocationRecorderV1; - - thread_local! { - static LARGEST_ALLOCATION_BYTES: Cell = const { Cell::new(0) }; - } - - unsafe impl GlobalAlloc for LargestAllocationRecorderV1 { - unsafe fn alloc(&self, layout: Layout) -> *mut u8 { - LARGEST_ALLOCATION_BYTES.with(|largest| largest.set(largest.get().max(layout.size()))); - unsafe { System.alloc(layout) } - } - - unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { - unsafe { System.dealloc(ptr, layout) } - } - - unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { - LARGEST_ALLOCATION_BYTES.with(|largest| largest.set(largest.get().max(layout.size()))); - unsafe { System.alloc_zeroed(layout) } - } - - unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { - LARGEST_ALLOCATION_BYTES.with(|largest| largest.set(largest.get().max(new_size))); - unsafe { System.realloc(ptr, layout, new_size) } - } - } - - #[global_allocator] - static TEST_ALLOCATOR: LargestAllocationRecorderV1 = LargestAllocationRecorderV1; - - fn measure_largest_allocation(work: impl FnOnce() -> T) -> (T, usize) { - LARGEST_ALLOCATION_BYTES.with(|largest| largest.set(0)); - let value = work(); - let largest = LARGEST_ALLOCATION_BYTES.with(Cell::get); - (value, largest) - } - - struct MaximumWriteSink { - inner: std::io::Cursor>, - maximum_write: usize, - write_calls: usize, - largest_write: usize, - } - - impl Write for MaximumWriteSink { - fn write(&mut self, bytes: &[u8]) -> std::io::Result { - if bytes.len() > self.maximum_write { - return Err(std::io::Error::other("write exceeded the fixture bound")); - } - self.write_calls += 1; - self.largest_write = self.largest_write.max(bytes.len()); - self.inner.write(bytes) - } - - fn flush(&mut self) -> std::io::Result<()> { - self.inner.flush() - } - } - - impl std::io::Seek for MaximumWriteSink { - fn seek(&mut self, position: std::io::SeekFrom) -> std::io::Result { - self.inner.seek(position) - } - } - - #[derive(Serialize)] - struct EnvelopeParityFixture<'a> { - state_digest: &'a ManifestDigest, - generation: &'a serde_json::Value, - } - - #[test] - fn direct_envelope_encoding_matches_canonical_serde_bytes() { - let generation = serde_json::json!({ - "format_revision": SEALED_GENERATION_FORMAT_REVISION_V1, - "manifest": {"generation_id": "generation.parity", "payload": "x".repeat(256)} - }); - let mut assembled = MaximumWriteSink { - inner: std::io::Cursor::new(Vec::new()), - maximum_write: 7, - write_calls: 0, - largest_write: 0, - }; - write_generation_envelope_with_limits(&generation, &mut assembled, u64::MAX, 7) - .expect("direct sealed envelope encoding"); - let assembled = assembled.inner.into_inner(); - let generation_bytes = - serde_json::to_vec(&generation).expect("generation fixture serialization"); - let state_digest = - json_generation_digest(&generation_bytes).expect("generation fixture digest"); - let prior = serde_json::to_vec(&EnvelopeParityFixture { - state_digest: &state_digest, - generation: &generation, - }) - .expect("serde envelope serialization"); - - assert_eq!(assembled, prior); - } - - #[test] - fn direct_envelope_encoding_coalesces_small_serialization_writes() { - const WRITE_BOUND: usize = 1024 * 1024; - let generation = serde_json::json!({ - "format_revision": SEALED_GENERATION_FORMAT_REVISION_V1, - "payload": vec![1_u8; WRITE_BOUND] - }); - let mut assembled = MaximumWriteSink { - inner: std::io::Cursor::new(Vec::new()), - maximum_write: WRITE_BOUND, - write_calls: 0, - largest_write: 0, - }; - - write_generation_envelope_with_limits(&generation, &mut assembled, u64::MAX, WRITE_BOUND) - .expect("direct sealed envelope encoding"); - - assert!( - assembled.write_calls <= 8, - "a two-megabyte seal must use coalesced writes, observed {}", - assembled.write_calls - ); - assert!(assembled.largest_write <= WRITE_BOUND); - } - - #[test] - fn direct_envelope_encoding_refuses_before_exceeding_its_byte_limit() { - let generation = serde_json::json!({ - "format_revision": SEALED_GENERATION_FORMAT_REVISION_V1, - "payload": "x".repeat(256) - }); - let generation_bytes = - serde_json::to_vec(&generation).expect("generation fixture serialization"); - let state_digest = - json_generation_digest(&generation_bytes).expect("generation fixture digest"); - let canonical = serde_json::to_vec(&EnvelopeParityFixture { - state_digest: &state_digest, - generation: &generation, - }) - .expect("canonical envelope serialization"); - let byte_limit = u64::try_from(canonical.len() - 1).expect("fixture length fits u64"); - let mut refused = MaximumWriteSink { - inner: std::io::Cursor::new(Vec::new()), - maximum_write: 7, - write_calls: 0, - largest_write: 0, - }; - - let error = write_generation_envelope_with_limits(&generation, &mut refused, byte_limit, 7) - .expect_err("an oversized envelope must be refused"); - - assert!(error.to_string().contains("canonical byte limit")); - assert!( - u64::try_from(refused.inner.get_ref().len()).expect("fixture length fits u64") - <= byte_limit, - "a refused stream must never write beyond its admitted limit" - ); - } - /// Publishing a sealed generation re-encodes its content as one canonical /// graph write batch, with record payloads JSON-escaped into string /// properties (at most doubling the bytes). A batch bound below that @@ -1471,190 +937,70 @@ mod tests { ); } - /// Encode and decode share one admission bound, so publication can never - /// seal a generation that every later load would refuse as corrupt. - #[test] - fn sealed_generation_byte_bound_is_symmetric() { - assert!(matches!( - admit_sealed_generation_len(MAX_SEALED_CODE_GENERATION_BYTES_V1), - Ok(()) - )); - assert!(matches!( - admit_sealed_generation_len(MAX_SEALED_CODE_GENERATION_BYTES_V1 + 1), - Err(CodeIndexProductionErrorV1::Contract(message)) - if message == "sealed generation exceeds the canonical byte limit" - )); - } - - #[test] - fn admitted_bytes_reject_extra_and_missing_bytes() { - let extra = admit_sealed_generation_bytes(b"{} ", 2); - let missing = admit_sealed_generation_bytes(b"{}", 3); - - assert!(matches!( - extra, - Err(CodeIndexProductionErrorV1::Contract(message)) - if message.contains("admitted length") - )); - assert!(matches!( - missing, - Err(CodeIndexProductionErrorV1::Contract(message)) - if message.contains("admitted length") - )); - } - - fn sealed_fixture(state_digest: &ManifestDigest, generation: &str) -> Vec { - format!( - "{{\"state_digest\":{},\"generation\":{}}}", - serde_json::to_string(state_digest).expect("fixture digest serialization"), - generation - ) - .into_bytes() - } - - /// An incompatible revision must stay `Ok(None)` on every classification - /// path: with a payload digest that matches the V1 raw-bytes rule (the - /// single-pass decode fails inside materialization) and with one that does - /// not (the digest gate fails first). - #[test] - fn incompatible_revision_stays_none_with_and_without_a_matching_payload_digest() { - let generation = format!( - "{{\"format_revision\":{}}}", - SEALED_GENERATION_FORMAT_REVISION_V1 + 1 - ); - let matching = json_generation_digest(generation.as_bytes()).expect("fixture digest"); - let mismatched = ManifestDigest::from_sha256_bytes(&[0; 32]).expect("fixture digest"); - - for state_digest in [matching, mismatched] { - let sealed = sealed_fixture(&state_digest, &generation); - assert!(matches!( - CodeIndexPublishedGenerationV1::decode_sealed_if_compatible(&sealed), - Ok(None) - )); - assert!(matches!( - CodeIndexPublishedGenerationV1::decode_sealed(&sealed), - Err(CodeIndexProductionErrorV1::Contract(message)) - if message.contains("format revision is incompatible") - )); + fn span(start_byte: u64, end_byte: u64) -> SourceSpan { + SourceSpan { + start_byte, + end_byte, } } - /// A superseded revision is refused with the typed rebuild error on every - /// classification path, never silently abstained like a revision this - /// decoder simply does not own, because abstention would hand the bytes to - /// the partitioned decoder and surface them as corruption. #[test] - fn superseded_revision_is_refused_with_the_rebuild_error() { - let generation = format!( - "{{\"format_revision\":{}}}", - MINIMUM_SEALED_GENERATION_FORMAT_REVISION - 1 - ); - let matching = json_generation_digest(generation.as_bytes()).expect("fixture digest"); - let mismatched = ManifestDigest::from_sha256_bytes(&[0; 32]).expect("fixture digest"); - - for state_digest in [matching, mismatched] { - let sealed = sealed_fixture(&state_digest, &generation); - for decoded in [ - CodeIndexPublishedGenerationV1::decode_sealed_if_compatible(&sealed).err(), - CodeIndexPublishedGenerationV1::decode_sealed(&sealed).err(), - ] { - let error = decoded.expect("a superseded revision must be refused"); - assert!( - matches!( - error, - CodeIndexProductionErrorV1::SupersededSealedGenerationRevision(revision) - if revision == MINIMUM_SEALED_GENERATION_FORMAT_REVISION - 1 - ), - "superseded revision reached the wrong rejection: {error}" - ); - assert!( - error.to_string().contains("will be rebuilt from source"), - "superseded rejection must tell the operator it rebuilds: {error}" - ); - } + fn chunk_text_base_stores_overlapping_chunks_once_and_restores_each_span() { + let source = "fn a() { let x = 1; }\n// gap\nfn b() {}\n"; + let text = |start: u64, end: u64| &source[start as usize..end as usize]; + let mut base = ChunkTextBaseV1 { + text_ranges: Vec::new(), + text: String::new(), + }; + assert_eq!(source.len(), 39); + let chunks = [ + span(0, 6), + span(0, 22), + span(7, 21), + span(29, 38), + span(29, 39), + ]; + for chunk in chunks { + assert!(base.admit(chunk, text(chunk.start_byte, chunk.end_byte))); + } + assert_eq!(base.text_ranges, [[0, 22], [29, 39]]); + assert_eq!(base.text, format!("{}{}", text(0, 22), text(29, 39))); + + let slices = ChunkTextSlicesV1::new(&base.text_ranges, &base.text).expect("slices"); + for chunk in chunks { + assert_eq!( + slices.slice(chunk).expect("slice"), + text(chunk.start_byte, chunk.end_byte) + ); } - } - - #[test] - fn undecodable_bytes_are_rejected_as_a_format_probe_failure() { - let error = CodeIndexPublishedGenerationV1::decode_sealed_if_compatible(b"not sealed json") - .expect_err("garbage bytes must not decode"); - assert!( - error.to_string().contains("format probe failed"), - "garbage bytes reached the wrong rejection: {error}" + slices.slice(span(20, 31)).is_err(), + "a span across a gap is refused" ); - } - - /// A current monolithic envelope whose digest verifies but whose payload does - /// not materialize must keep the payload-decoding rejection, never the - /// probe or digest one. - #[test] - fn v1_payload_that_fails_materialization_keeps_the_payload_rejection() { - let generation = - format!("{{\"format_revision\":{MONOLITHIC_SEALED_GENERATION_FORMAT_REVISION}}}"); - let state_digest = json_generation_digest(generation.as_bytes()).expect("fixture digest"); - let sealed = sealed_fixture(&state_digest, &generation); - - let error = CodeIndexPublishedGenerationV1::decode_sealed(&sealed) - .expect_err("an incomplete current payload must not decode"); - assert!( - error.to_string().contains("payload decoding failed"), - "incomplete current payload reached the wrong rejection: {error}" + slices.slice(span(38, 41)).is_err(), + "a span past the text is refused" ); - } - #[test] - fn borrowed_decode_does_not_allocate_a_second_corpus_sized_buffer() { - const PADDING_BYTES: usize = 8 * 1024 * 1024; - let wrong_digest = ManifestDigest::from_sha256_bytes(&[0; 32]).expect("fixture digest"); - let mut sealed = format!( - "{{\"state_digest\":{},\"generation\":{{\"format_revision\":{},\"padding\":\"", - serde_json::to_string(&wrong_digest).expect("fixture digest serialization"), - MONOLITHIC_SEALED_GENERATION_FORMAT_REVISION, - ) - .into_bytes(); - sealed.resize(sealed.len() + PADDING_BYTES, b'x'); - sealed.extend_from_slice(b"\"}}"); - - let (result, largest_allocation) = - measure_largest_allocation(|| CodeIndexPublishedGenerationV1::decode_sealed(&sealed)); - - assert!(matches!( - result, - Err(CodeIndexProductionErrorV1::Contract(message)) - if message.contains("state digest does not match") - )); assert!( - largest_allocation < sealed.len() / 2, - "borrowed decode allocated {largest_allocation} bytes for a {} byte sealed input", - sealed.len() + !base.admit(span(33, 38), "XXXXX"), + "a chunk whose text disagrees with the stored bytes keeps its own text" + ); + assert!( + !base.admit(span(35, 38), "ab"), + "a text whose length is not its span keeps its own text" + ); + assert_eq!( + base.text_ranges, + [[0, 22], [29, 39]], + "refused chunks do not extend the base" ); } #[test] - fn raw_v6_payload_borrows_the_callers_admitted_bytes() { - const PADDING_BYTES: usize = 4 * 1024 * 1024; - let digest = ManifestDigest::from_sha256_bytes(&[0; 32]).expect("fixture digest"); - let mut sealed = format!( - "{{\"state_digest\":{},\"generation\":{{\"format_revision\":{},\"padding\":\"", - serde_json::to_string(&digest).expect("fixture digest serialization"), - MONOLITHIC_SEALED_GENERATION_FORMAT_REVISION, - ) - .into_bytes(); - sealed.resize(sealed.len() + PADDING_BYTES, b'x'); - sealed.extend_from_slice(b"\"}}"); - - let raw: SealedPublishedGenerationRawEnvelopeV1 = - serde_json::from_slice(&sealed).expect("raw envelope parses"); - let payload_start = raw.generation.get().as_ptr() as usize; - let admitted_start = sealed.as_ptr() as usize; - let admitted_end = admitted_start + sealed.len(); - - assert!( - (admitted_start..admitted_end).contains(&payload_start), - "the raw payload must point into the caller's admitted byte slice" - ); + fn chunk_text_slices_refuse_ranges_that_disagree_with_their_text() { + assert!(ChunkTextSlicesV1::new(&[[0, 4]], "abc").is_err()); + assert!(ChunkTextSlicesV1::new(&[[0, 4], [2, 6]], "abcdefgh").is_err()); + assert!(ChunkTextSlicesV1::new(&[[4, 4]], "").is_err()); } } diff --git a/crates/tracedecay-code-index/src/production/worker_tests.rs b/crates/tracedecay-code-index/src/production/worker_tests.rs index 482d40f201..8af3a7f867 100644 --- a/crates/tracedecay-code-index/src/production/worker_tests.rs +++ b/crates/tracedecay-code-index/src/production/worker_tests.rs @@ -1,5 +1,7 @@ use std::sync::atomic::{AtomicUsize, Ordering}; +use sha2::Digest as _; + use tracedecay_domain::{ ChunkerRevision, ExtractorRevision, LanguageId, PrivacyDomainId, ProjectionKeyV1, ProjectionKindV1, ProjectionOperationV1, ProjectionOutcomeV1, SanitizationReceiptId, @@ -211,11 +213,10 @@ fn restored_generation_resolves_seal_references_once() { &UninterruptibleCodeIndexControlV1, ) .expect("fresh generation"); - let sealed = published.encode_sealed().expect("sealed generation bytes"); + let (manifest, segments) = partitioned_seal(&published); super::helpers::take_seal_reference_resolutions(); - let restored = - CodeIndexPublishedGenerationV1::decode_sealed(&sealed).expect("restored generation"); + let restored = partitioned_restore(&manifest, &segments); assert_eq!(super::helpers::take_seal_reference_resolutions(), 1); assert_eq!(restored.edges, published.edges); @@ -282,9 +283,8 @@ fn arc_share_increment_restores_under_parentless_validate_fresh() { "fixture must Arc-share the unchanged file page" ); - let sealed = next.encode_sealed().expect("arc-share generation seals"); - let restored = - CodeIndexPublishedGenerationV1::decode_sealed(&sealed).expect("parentless restore"); + let (manifest, segments) = partitioned_seal(&next); + let restored = partitioned_restore(&manifest, &segments); assert_eq!(restored.manifest.generation_id, next.manifest.generation_id); assert_eq!( restored.projection.request().changes.reused_digest, @@ -491,18 +491,86 @@ fn incremental_carry_forward_rejects_a_stale_extractor_revision() { #[test] fn prior_sealed_generation_is_rejected_before_manifest_decode() { - let prior = br#"{"generation":{"format_revision":4}}"#; + for revision in [4, 9] { + let generation = format!(r#"{{"format_revision":{revision}}}"#); + let digest = + ManifestDigest::from_sha256_bytes(&sha2::Sha256::digest(generation.as_bytes())) + .expect("prior generation digest"); + let prior = format!( + r#"{{"state_digest":"{}","generation":{generation}}}"#, + digest.as_str() + ); + let error = + CodeIndexPublishedGenerationV1::decode_partitioned_sealed(prior.as_bytes(), |_, _| { + panic!("a retired manifest must be refused before any segment read") + }) + .expect_err("prior generation must require a rebuild"); + assert!( + matches!( + error, + CodeIndexProductionErrorV1::SupersededSealedGenerationRevision(refused) + if refused == revision + ), + "revision {revision} reached the wrong rejection: {error}" + ); + assert!(error.to_string().contains("will be rebuilt from source")); + } +} - assert!( - !CodeIndexPublishedGenerationV1::sealed_format_is_compatible(prior) - .expect("prior format probe") - ); - let error = CodeIndexPublishedGenerationV1::decode_sealed_if_compatible(prior) - .expect_err("a caller that accepts incompatible durable state must not materialize it"); - assert!(error.to_string().contains("will be rebuilt from source")); - let error = CodeIndexPublishedGenerationV1::decode_sealed(prior) - .expect_err("prior generation must require a rebuild"); - assert!(error.to_string().contains("will be rebuilt from source")); +/// Seal `generation` partitioned, keeping every segment in memory with the +/// evidence pages assembled under their pack digest. +fn partitioned_seal( + generation: &CodeIndexPublishedGenerationV1, +) -> (Vec, std::collections::BTreeMap>) { + let mut segments = std::collections::BTreeMap::new(); + let mut evidence_pack = Vec::new(); + let manifest = generation + .encode_partitioned_sealed(|publication| { + match publication { + SealedGenerationSegmentPublicationV1::File { digest, bytes } => { + segments.insert(digest.as_str().to_owned(), bytes.to_vec()); + } + SealedGenerationSegmentPublicationV1::GenerationEvidencePage { bytes, .. } => { + evidence_pack.extend_from_slice(bytes); + } + SealedGenerationSegmentPublicationV1::GenerationEvidenceCommit { + segment_digest, + .. + } => { + segments.insert( + segment_digest.as_str().to_owned(), + std::mem::take(&mut evidence_pack), + ); + } + } + Ok(()) + }) + .expect("generation seals"); + (manifest, segments) +} + +fn partitioned_restore( + manifest: &[u8], + segments: &std::collections::BTreeMap>, +) -> CodeIndexPublishedGenerationV1 { + CodeIndexPublishedGenerationV1::decode_partitioned_sealed(manifest, |request, buffer| { + let (digest, offset, length) = match request { + SealedGenerationSegmentReadV1::Whole { digest, size_bytes } => (digest, 0, size_bytes), + SealedGenerationSegmentReadV1::Range { + digest, + offset, + length, + .. + } => (digest, offset, length), + }; + let bytes = &segments[digest.as_str()]; + let start = usize::try_from(offset).expect("segment offset"); + let end = start + usize::try_from(length).expect("segment length"); + buffer.clear(); + buffer.extend_from_slice(&bytes[start..end]); + Ok(()) + }) + .expect("generation restores") } #[test] diff --git a/crates/tracedecay-code-index/tests/code_index_suite/diagnostic_generation.rs b/crates/tracedecay-code-index/tests/code_index_suite/diagnostic_generation.rs index 12431c0b95..45dab4e8b3 100644 --- a/crates/tracedecay-code-index/tests/code_index_suite/diagnostic_generation.rs +++ b/crates/tracedecay-code-index/tests/code_index_suite/diagnostic_generation.rs @@ -112,16 +112,8 @@ fn diagnostic( } #[test] -fn superseded_and_cleared_records_remain_typed_historical_evidence() { +fn cleared_records_remain_typed_historical_evidence() { let (snapshot, manifest) = generation(); - let superseded = diagnostic( - "generation.prior.1", - "anchor.diagnostic.superseded", - 'a', - DiagnosticRecordStateV1::Superseded { - successor_generation: manifest.generation_id.clone(), - }, - ); let cleared = diagnostic( "generation.prior.2", "anchor.diagnostic.cleared", @@ -134,7 +126,7 @@ fn superseded_and_cleared_records_remain_typed_historical_evidence() { let joined = GenerationDiagnosticJoinV1::join( &manifest, &snapshot, - &[superseded, cleared], + &[cleared], &watermark( &snapshot, &manifest, @@ -143,20 +135,11 @@ fn superseded_and_cleared_records_remain_typed_historical_evidence() { ) .expect("historical evidence remains inspectable"); + assert_eq!(joined.records.len(), 1); assert!(matches!( joined.records[0].disposition, GenerationDiagnosticDispositionV1::Cleared { .. } - | GenerationDiagnosticDispositionV1::Superseded { .. } - )); - assert!(matches!( - joined.records[1].disposition, - GenerationDiagnosticDispositionV1::Cleared { .. } - | GenerationDiagnosticDispositionV1::Superseded { .. } )); - assert!(joined.records.iter().all(|record| !matches!( - record.disposition, - GenerationDiagnosticDispositionV1::Current { .. } - ))); } #[test] @@ -166,8 +149,8 @@ fn out_of_scope_historical_record_is_not_classified_as_lifecycle_history() { "generation.prior", "anchor.diagnostic.out-of-scope", 'a', - DiagnosticRecordStateV1::Superseded { - successor_generation: manifest.generation_id.clone(), + DiagnosticRecordStateV1::Cleared { + cleared_in_generation: manifest.generation_id.clone(), }, ); record.reference = Some(id("ref.other")); diff --git a/crates/tracedecay-code-index/tests/code_index_suite/generations.rs b/crates/tracedecay-code-index/tests/code_index_suite/generations.rs index fe803b8056..08a4cf787c 100644 --- a/crates/tracedecay-code-index/tests/code_index_suite/generations.rs +++ b/crates/tracedecay-code-index/tests/code_index_suite/generations.rs @@ -8,10 +8,9 @@ use tracedecay_code_index::intake::INTAKE_DIGEST_SEPARATOR; use tracedecay_code_index::intake::ValidatedCodeSnapshotV1; use tracedecay_code_index::languages::StaticLanguageRegistry; use tracedecay_domain::{ - ChunkerRevision, CodeGenerationManifestV1, ContentDigest, DomainError, FileOccurrenceId, - LanguageId, ManifestDigest, PrivacyDomainId, RepositoryId, SanitizationReceiptId, - SanitizedCodeFileV1, SanitizedCodeSnapshotV1, SanitizerRevision, SnapshotFileDispositionV1, - UtcMicros, canonical_sha256, + ChunkerRevision, ContentDigest, DomainError, FileOccurrenceId, LanguageId, PrivacyDomainId, + RepositoryId, SanitizationReceiptId, SanitizedCodeFileV1, SanitizedCodeSnapshotV1, + SanitizerRevision, SnapshotFileDispositionV1, UtcMicros, canonical_sha256, }; use tracedecay_domain::test_fixtures::id; @@ -238,64 +237,3 @@ fn resealing_cannot_hide_a_generation_fingerprint_mismatch() { )) ); } - -#[test] -fn legacy_v1_manifest_deserialization_migrates_and_remains_a_valid_parent() { - let planner = planner(); - let snapshot = validated(snapshot(vec![file("file.a", "src/a.rs", 'a')])); - let current = planner - .plan_generation(&snapshot, None, UtcMicros(3_000)) - .expect("current manifest"); - let mut legacy_identity = current - .generation_id - .as_str() - .split('.') - .take(4) - .collect::>() - .join("."); - assert_eq!(legacy_identity.matches('.').count(), 3); - - let mut wire = serde_json::to_value(¤t).expect("manifest wire"); - let object = wire.as_object_mut().expect("manifest object"); - object.insert( - "generation_id".to_owned(), - serde_json::Value::String(std::mem::take(&mut legacy_identity)), - ); - object.remove("invalidation_digest"); - object - .get_mut("seal") - .and_then(serde_json::Value::as_object_mut) - .expect("seal object") - .insert( - "expected_digest".to_owned(), - serde_json::Value::String(format!("sha256:{}", "0".repeat(64))), - ); - - let mut migrated: CodeGenerationManifestV1 = - serde_json::from_value(wire).expect("legacy wire migrates"); - migrated.seal.expected_digest = expected_seal_digest(&migrated).expect("legacy seal digest"); - let mut legacy_wire = serde_json::to_value(&migrated).expect("migrated wire"); - legacy_wire - .as_object_mut() - .expect("manifest object") - .remove("invalidation_digest"); - let legacy_parent: CodeGenerationManifestV1 = - serde_json::from_value(legacy_wire).expect("legacy fixture deserializes"); - - legacy_parent.validate().expect("legacy parent validates"); - assert_eq!( - legacy_parent.invalidation_digest, - migrated.invalidation_digest - ); - assert_ne!( - legacy_parent.invalidation_digest, - id::(&format!("sha256:{}", "0".repeat(64))) - ); - let child = planner - .plan_generation(&snapshot, Some(&legacy_parent), UtcMicros(4_000)) - .expect("legacy parent accepted"); - assert_eq!( - child.parent_generation.as_ref(), - Some(&legacy_parent.generation_id) - ); -} diff --git a/crates/tracedecay-code-index/tests/code_index_suite/graph_projection_publication.rs b/crates/tracedecay-code-index/tests/code_index_suite/graph_projection_publication.rs index 6039b4d930..4af2f6a6c2 100644 --- a/crates/tracedecay-code-index/tests/code_index_suite/graph_projection_publication.rs +++ b/crates/tracedecay-code-index/tests/code_index_suite/graph_projection_publication.rs @@ -23,7 +23,7 @@ use crate::{ production_orchestration::{ ActiveControl, ApplyingProjectionSink, SharedPublicationStore, config, request_with_source, }, - support::id, + support::{PartitionedSealV1, id}, }; const IMPORT_SOURCE: &str = concat!( @@ -274,9 +274,7 @@ fn sealed_generation_replay_rebuilds_identical_import_manifest_and_digest() { let generation = published_import_generation(); let revision = current_projector_revision(); let original = projection_manifest(&generation, &revision); - let sealed = generation.encode_sealed().expect("generation seals"); - let restored = CodeIndexPublishedGenerationV1::decode_sealed(&sealed) - .expect("sealed import generation restores"); + let restored = PartitionedSealV1::of(&generation).restored(); assert_eq!(restored.imports(), generation.imports()); let replayed = projection_manifest(&restored, &revision); diff --git a/crates/tracedecay-code-index/tests/code_index_suite/ignored_source_admissions.rs b/crates/tracedecay-code-index/tests/code_index_suite/ignored_source_admissions.rs index 2a41d59806..ccf7b42651 100644 --- a/crates/tracedecay-code-index/tests/code_index_suite/ignored_source_admissions.rs +++ b/crates/tracedecay-code-index/tests/code_index_suite/ignored_source_admissions.rs @@ -6,8 +6,7 @@ use tracedecay_code_index::{ production::{ CodeIndexAtomicPublicationPort, CodeIndexBuildRequestV1, CodeIndexCapturedFileV1, CodeIndexGenerationScopeV1, CodeIndexIgnoredSourceAdmissionV1, CodeIndexProductionOwnerV1, - CodeIndexPublishedGenerationV1, MINIMUM_SEALED_GENERATION_FORMAT_REVISION, - SEALED_GENERATION_FORMAT_REVISION_V1, sealed_generation_payload_digest, + CodeIndexPublishedGenerationV1, }, }; use tracedecay_domain::{ @@ -19,7 +18,7 @@ use crate::{ production_orchestration::{ ActiveControl, ApplyingProjectionSink, SharedPublicationStore, config, request_with_source, }, - support::id, + support::{PartitionedSealV1, id, reseal_manifest}, }; const PRIMARY_IGNORED_PATH: &str = "node_modules/alpha/index.ts"; @@ -128,28 +127,6 @@ fn assert_rejected_before_publication(request: CodeIndexBuildRequestV1) { ); } -fn sealed_envelope(generation: &CodeIndexPublishedGenerationV1) -> Value { - serde_json::from_slice( - &generation - .encode_sealed() - .expect("ignored-source generation seals"), - ) - .expect("sealed ignored-source generation JSON") -} - -fn reseal_outer_state(mut envelope: Value) -> Vec { - let format_revision = u32::try_from( - envelope["generation"]["format_revision"] - .as_u64() - .expect("forged generation format revision"), - ) - .expect("format revision fits u32"); - let state_digest = sealed_generation_payload_digest(format_revision, &envelope["generation"]) - .expect("forged generation has a digest"); - envelope["state_digest"] = Value::String(state_digest.as_str().to_owned()); - serde_json::to_vec(&envelope).expect("forged sealed-generation JSON") -} - #[test] fn sorted_unique_ignored_source_roster_round_trips_with_present_snapshot_membership() { let generation = publish(request_with_ignored_sources(vec![ @@ -176,11 +153,8 @@ fn sorted_unique_ignored_source_roster_round_trips_with_present_snapshot_members })); } - let sealed = generation - .encode_sealed() - .expect("ignored-source generation seals"); - let restored = CodeIndexPublishedGenerationV1::decode_sealed(&sealed) - .expect("ignored-source generation restores"); + let sealed = PartitionedSealV1::of(&generation); + let restored = sealed.restored(); assert_eq!( restored .ignored_source_admissions() @@ -193,10 +167,7 @@ fn sorted_unique_ignored_source_roster_round_trips_with_present_snapshot_members restored.repository_parse_identity().dirty, RepositoryDirtyStateV1::Dirty ); - assert_eq!( - restored.encode_sealed().expect("restored generation seals"), - sealed - ); + assert_eq!(PartitionedSealV1::of(&restored).manifest, sealed.manifest); } #[test] @@ -294,14 +265,15 @@ fn sealed_ignored_source_roster_rejects_semantic_tampering_after_outer_reseal() let generation = publish(request_with_ignored_sources(vec![admission( PRIMARY_IGNORED_PATH, )])); - let mut envelope = sealed_envelope(&generation); + let seal = PartitionedSealV1::of(&generation); + let mut envelope = seal.envelope(); let roster = envelope["generation"]["ignored_source_admissions"] .as_array_mut() .expect("sealed generation carries the required ignored-source roster"); assert_eq!(roster.len(), 1); roster[0]["logical_path"] = Value::String(SECONDARY_IGNORED_PATH.to_owned()); - CodeIndexPublishedGenerationV1::decode_sealed(&reseal_outer_state(envelope)) + seal.restore(&reseal_manifest(envelope)) .expect_err("a self-consistent outer digest cannot forge roster state"); } @@ -310,7 +282,8 @@ fn sealed_json_requires_repository_parse_identity_without_a_default() { let generation = publish(request_with_ignored_sources(vec![admission( PRIMARY_IGNORED_PATH, )])); - let mut envelope = sealed_envelope(&generation); + let seal = PartitionedSealV1::of(&generation); + let mut envelope = seal.envelope(); assert!( envelope["generation"] .as_object_mut() @@ -319,7 +292,8 @@ fn sealed_json_requires_repository_parse_identity_without_a_default() { .is_some() ); - let error = CodeIndexPublishedGenerationV1::decode_sealed(&reseal_outer_state(envelope)) + let error = seal + .restore(&reseal_manifest(envelope)) .expect_err("sealed generations may not default missing repository parse identity"); assert!( error.to_string().contains("repository_parse_identity"), @@ -332,7 +306,8 @@ fn sealed_nonempty_roster_rejects_non_dirty_repository_identity_after_outer_rese let generation = publish(request_with_ignored_sources(vec![admission( PRIMARY_IGNORED_PATH, )])); - let envelope = sealed_envelope(&generation); + let seal = PartitionedSealV1::of(&generation); + let envelope = seal.envelope(); assert_eq!( envelope["generation"]["repository_parse_identity"]["dirty"], Value::String("dirty".to_owned()) @@ -342,7 +317,7 @@ fn sealed_nonempty_roster_rejects_non_dirty_repository_identity_after_outer_rese let mut forged = envelope.clone(); forged["generation"]["repository_parse_identity"]["dirty"] = Value::String(forged_dirty_state.to_owned()); - CodeIndexPublishedGenerationV1::decode_sealed(&reseal_outer_state(forged)) + seal.restore(&reseal_manifest(forged)) .expect_err("a nonempty ignored-source roster requires durable Dirty evidence"); } } @@ -352,7 +327,7 @@ fn sealed_nonempty_roster_rejects_forged_snapshot_source_revision_after_outer_re let mut pinned_request = request_with_ignored_sources(Vec::new()); pinned_request.snapshot.source_revision = Some(id::("commit.pinned-fixture")); let pinned = publish(pinned_request); - let pinned_envelope = sealed_envelope(&pinned); + let pinned_envelope = PartitionedSealV1::of(&pinned).envelope(); let valid_source_revision = pinned_envelope["generation"]["snapshot"]["source_revision"].clone(); assert!(!valid_source_revision.is_null()); @@ -360,10 +335,11 @@ fn sealed_nonempty_roster_rejects_forged_snapshot_source_revision_after_outer_re let generation = publish(request_with_ignored_sources(vec![admission( PRIMARY_IGNORED_PATH, )])); - let mut envelope = sealed_envelope(&generation); + let seal = PartitionedSealV1::of(&generation); + let mut envelope = seal.envelope(); envelope["generation"]["snapshot"]["source_revision"] = valid_source_revision; - CodeIndexPublishedGenerationV1::decode_sealed(&reseal_outer_state(envelope)) + seal.restore(&reseal_manifest(envelope)) .expect_err("a nonempty ignored-source roster cannot restore with a pinned snapshot"); } @@ -372,7 +348,8 @@ fn sealed_json_requires_ignored_source_roster_without_a_default() { let generation = publish(request_with_ignored_sources(vec![admission( PRIMARY_IGNORED_PATH, )])); - let mut envelope = sealed_envelope(&generation); + let seal = PartitionedSealV1::of(&generation); + let mut envelope = seal.envelope(); assert!( envelope["generation"] .as_object_mut() @@ -381,7 +358,8 @@ fn sealed_json_requires_ignored_source_roster_without_a_default() { .is_some() ); - let error = CodeIndexPublishedGenerationV1::decode_sealed(&reseal_outer_state(envelope)) + let error = seal + .restore(&reseal_manifest(envelope)) .expect_err("sealed generations may not default a missing ignored-source roster"); assert!( error.to_string().contains("ignored_source_admissions"), @@ -394,7 +372,8 @@ fn sealed_json_requires_ignored_source_admissions_digest_without_a_default() { let generation = publish(request_with_ignored_sources(vec![admission( PRIMARY_IGNORED_PATH, )])); - let mut envelope = sealed_envelope(&generation); + let seal = PartitionedSealV1::of(&generation); + let mut envelope = seal.envelope(); assert!( envelope["generation"] .as_object_mut() @@ -403,7 +382,8 @@ fn sealed_json_requires_ignored_source_admissions_digest_without_a_default() { .is_some() ); - let error = CodeIndexPublishedGenerationV1::decode_sealed(&reseal_outer_state(envelope)) + let error = seal + .restore(&reseal_manifest(envelope)) .expect_err("sealed generations may not default a missing ignored-source roster digest"); assert!( error @@ -418,7 +398,8 @@ fn sealed_json_rejects_legacy_ignored_sources_alias_after_outer_reseal() { let generation = publish(request_with_ignored_sources(vec![admission( PRIMARY_IGNORED_PATH, )])); - let mut envelope = sealed_envelope(&generation); + let seal = PartitionedSealV1::of(&generation); + let mut envelope = seal.envelope(); let generation = envelope["generation"] .as_object_mut() .expect("sealed generation object"); @@ -431,82 +412,22 @@ fn sealed_json_rejects_legacy_ignored_sources_alias_after_outer_reseal() { .is_none() ); - CodeIndexPublishedGenerationV1::decode_sealed(&reseal_outer_state(envelope)) + seal.restore(&reseal_manifest(envelope)) .expect_err("legacy ignored_sources alias must not restore"); } #[test] fn sealed_state_digest_changes_when_ignored_source_roster_changes() { - let one = sealed_envelope(&publish(request_with_ignored_sources(vec![admission( + let one = PartitionedSealV1::of(&publish(request_with_ignored_sources(vec![admission( PRIMARY_IGNORED_PATH, - )]))); - let two = sealed_envelope(&publish(request_with_ignored_sources(vec![ + )]))) + .envelope(); + let two = PartitionedSealV1::of(&publish(request_with_ignored_sources(vec![ admission(PRIMARY_IGNORED_PATH), admission(SECONDARY_IGNORED_PATH), - ]))); + ]))) + .envelope(); assert_eq!(one["generation"]["snapshot"], two["generation"]["snapshot"]); assert_ne!(one["state_digest"], two["state_digest"]); } - -#[test] -fn sealed_format_refuses_superseded_revisions_beside_the_partitioned_revision() { - let generation = publish(request_with_ignored_sources(vec![admission( - PRIMARY_IGNORED_PATH, - )])); - let sealed = generation - .encode_sealed() - .expect("current monolithic generation seals"); - assert!( - CodeIndexPublishedGenerationV1::sealed_format_is_compatible(&sealed) - .expect("current monolithic compatibility probe") - ); - - let mut superseded = sealed_envelope(&generation); - superseded["generation"]["format_revision"] = Value::from(5); - // No reseal: the revision gate fires ahead of any digest rule, so a - // superseded envelope is refused whatever its state digest says. - let superseded = serde_json::to_vec(&superseded).expect("superseded sealed-generation JSON"); - assert!( - !CodeIndexPublishedGenerationV1::sealed_format_is_compatible(&superseded) - .expect("revision-five compatibility probe") - ); - let error = CodeIndexPublishedGenerationV1::decode_sealed(&superseded) - .expect_err("a superseded revision must be refused, never migrated"); - assert!( - error.to_string().contains("will be rebuilt from source"), - "superseded revision reached the wrong rejection: {error}" - ); - - for incompatible_revision in [4, 11, 13] { - let mut incompatible = sealed_envelope(&generation); - incompatible["generation"]["format_revision"] = Value::from(incompatible_revision); - let incompatible = - serde_json::to_vec(&incompatible).expect("incompatible sealed-generation JSON"); - assert!( - !CodeIndexPublishedGenerationV1::sealed_format_is_compatible(&incompatible) - .expect("incompatible revision compatibility probe") - ); - CodeIndexPublishedGenerationV1::decode_sealed(&incompatible) - .expect_err("adjacent sealed-generation revisions are incompatible"); - } - - let mut partitioned = sealed_envelope(&generation); - partitioned["generation"]["format_revision"] = - Value::from(SEALED_GENERATION_FORMAT_REVISION_V1); - let partitioned = - serde_json::to_vec(&partitioned).expect("partitioned-format generation manifest"); - assert!( - CodeIndexPublishedGenerationV1::decode_sealed_if_compatible(&partitioned) - .expect("partitioned revision classification") - .is_none() - ); - - let mut retired = sealed_envelope(&generation); - retired["generation"]["format_revision"] = - Value::from(MINIMUM_SEALED_GENERATION_FORMAT_REVISION - 1); - let retired = serde_json::to_vec(&retired).expect("retired generation manifest"); - let error = CodeIndexPublishedGenerationV1::decode_sealed_if_compatible(&retired) - .expect_err("pre-clone manifest must be rebuilt"); - assert!(error.to_string().contains("will be rebuilt from source")); -} diff --git a/crates/tracedecay-code-index/tests/code_index_suite/import_evidence.rs b/crates/tracedecay-code-index/tests/code_index_suite/import_evidence.rs index 44ca61804b..4a850b4a53 100644 --- a/crates/tracedecay-code-index/tests/code_index_suite/import_evidence.rs +++ b/crates/tracedecay-code-index/tests/code_index_suite/import_evidence.rs @@ -3,27 +3,24 @@ use std::sync::Arc; use serde_json::Value; use tracedecay_code_extraction::{ImportModuleKindV1, ImportNamespaceV1}; use tracedecay_code_index::{ - chunks::{ - ChunkingFailureV1, CodeFileIndexArtifactsV1, CodeIndexImportEvidenceV1, content_digest, - }, - noncanonical::{NonCanonicalCauseV1, NonCanonicalReasonCodeV1}, + chunks::{CodeIndexImportEvidenceV1, content_digest}, production::{ CodeIndexBuildRequestV1, CodeIndexCapturedFileV1, CodeIndexProductionErrorV1, CodeIndexProductionOwnerV1, CodeIndexPublishedGenerationV1, - sealed_generation_payload_digest, + SEALED_GENERATION_FORMAT_REVISION_V1, }, }; use tracedecay_domain::{ EdgeAuthorityV1, FileOccurrenceId, LanguageId, RelationEdgeKindV1, SanitizationReceiptId, SanitizedCodeFileV1, SensitivityLevelV1, SnapshotFileDispositionV1, SourceSpan, - SymbolOccurrenceId, canonical_sha256, + SymbolOccurrenceId, }; use crate::{ production_orchestration::{ ActiveControl, ApplyingProjectionSink, SharedPublicationStore, config, request_with_source, }, - support::id, + support::{PartitionedSealV1, id}, }; const FIRST_SOURCE: &str = concat!( @@ -787,78 +784,41 @@ fn rust_parent_glob_does_not_override_a_local_type_binding() { ); } -fn sealed_envelope(generation: &CodeIndexPublishedGenerationV1) -> Value { - serde_json::from_slice(&generation.encode_sealed().expect("import generation seals")) - .expect("sealed generation JSON") -} - -fn file_artifact(envelope: &Value, index: usize) -> CodeFileIndexArtifactsV1 { - serde_json::from_value(envelope["generation"]["files"][index]["artifacts"].clone()) - .expect("file artifact JSON") -} - -fn import_rows_mut(envelope: &mut Value, file_index: usize) -> &mut Vec { - envelope["generation"]["files"][file_index]["artifacts"]["imports"] +fn import_rows_mut(file: &mut Value) -> &mut Vec { + file["artifacts"]["imports"] .as_array_mut() .expect("sealed file imports") } -fn assert_serialized_artifact_has_no_self_digest(envelope: &Value, file_index: usize) { - let imports = serde_json::from_value::>( - envelope["generation"]["files"][file_index]["artifacts"]["imports"].clone(), - ) - .expect("forged canonical import rows"); - let recomputed = canonical_sha256(&("attacker-controlled-import-rows", imports.as_slice())) - .expect("forged canonical import-row digest"); - assert!(recomputed.as_str().starts_with("sha256:")); - assert!( - envelope["generation"]["files"][file_index]["artifacts"] - .get("import_rows_digest") - .is_none(), - "sealed file artifacts must not contain a recomputable self-digest authority" - ); -} - -fn reseal_import_envelope(mut envelope: Value) -> Vec { - let format_revision = u32::try_from( - envelope["generation"]["format_revision"] - .as_u64() - .expect("forged payload format revision"), - ) - .expect("format revision fits u32"); - let state_digest = sealed_generation_payload_digest(format_revision, &envelope["generation"]) - .expect("forged payload state digest"); - envelope["state_digest"] = Value::String(state_digest.as_str().to_owned()); - serde_json::to_vec(&envelope).expect("forged sealed generation JSON") -} - -fn resealed_import_payload_error(envelope: Value, mutation: &str) -> CodeIndexProductionErrorV1 { - let bytes = reseal_import_envelope(envelope); - - match CodeIndexPublishedGenerationV1::decode_sealed(&bytes) { - Ok(_) => panic!("{mutation} restored after the outer state digest was recomputed"), +/// Tamper the first file's sealed import rows, re-address the segment, and +/// reseal the manifest, so the refusal comes from the restored file payload. +fn tampered_import_payload_error( + sealed: &PartitionedSealV1, + mutation: &str, + mutate: impl FnOnce(&mut Vec), +) -> CodeIndexProductionErrorV1 { + let tampered = sealed.with_tampered_file_segment(0, |file| mutate(import_rows_mut(file))); + match tampered.restore(&tampered.manifest) { + Ok(_) => panic!("{mutation} restored after the segment and manifest were re-addressed"), Err(error) => error, } } -fn assert_resealed_import_payload_is_rejected(envelope: Value, mutation: &str) { - let _ = resealed_import_payload_error(envelope, mutation); -} - -fn assert_sealed_envelope_restores(envelope: &Value) { - CodeIndexPublishedGenerationV1::decode_sealed(&reseal_import_envelope(envelope.clone())) - .expect("baseline generation restores"); +fn file_imports(generation: &CodeIndexPublishedGenerationV1) -> Vec { + generation + .imports() + .iter() + .filter(|row| row.logical_path == "src/a.ts") + .cloned() + .collect() } #[test] fn file_import_artifacts_require_nondefault_canonical_rows() { let generation = published_import_generation(); - let envelope = sealed_envelope(&generation); - let artifacts = file_artifact(&envelope, 0); assert_eq!( - artifacts - .imports + file_imports(&generation) .iter() .map(|row| ( row.logical_path.as_str(), @@ -906,61 +866,47 @@ fn file_import_artifacts_require_nondefault_canonical_rows() { ), ] ); - artifacts.validate().expect("canonical import rows"); - let mut missing = envelope["generation"]["files"][0]["artifacts"].clone(); - assert!( - missing - .as_object_mut() - .expect("file artifact object") - .remove("imports") - .is_some(), - "the serialized artifact must carry its required imports field" - ); - let error = serde_json::from_value::(missing) + let sealed = PartitionedSealV1::of(&generation); + let missing = sealed.with_tampered_file_segment(0, |file| { + assert!( + file["artifacts"] + .as_object_mut() + .expect("file artifact object") + .remove("imports") + .is_some(), + "the sealed artifact must carry its required imports field" + ); + }); + let error = missing + .restore(&missing.manifest) .expect_err("imports must be a required field without a serde default"); assert!( error.to_string().contains("missing field `imports`"), "unexpected missing-imports error: {error}" ); - - let mut reordered = artifacts.clone(); - reordered.imports.swap(0, 1); - let error = reordered - .validate() - .expect_err("source-order reversal must be rejected"); - assert!(matches!(error, ChunkingFailureV1::NonCanonicalIdentity(_))); - - let mut duplicated = artifacts; - let duplicate = duplicated.imports[0].clone(); - duplicated.imports.insert(1, duplicate); - let error = duplicated - .validate() - .expect_err("duplicate import rows must be rejected"); - assert!(matches!(error, ChunkingFailureV1::NonCanonicalIdentity(_))); } #[test] fn file_import_artifacts_bind_file_consistent_path_and_nonempty_span_to_indexed_extent() { let generation = published_import_generation(); - let artifacts = file_artifact(&sealed_envelope(&generation), 0); - let indexed_end = artifacts - .chunks - .chunks + let imports = file_imports(&generation); + let indexed_end = generation + .chunks() + .chunks() .iter() + .filter(|chunk| chunk.anchor.file_occurrence_id.as_str() == "file.import.a") .map(|chunk| chunk.anchor.source_span.end_byte) .max() .expect("complete file has indexed chunks"); - assert!(artifacts.imports.iter().all(|row| { - row.logical_path == "src/a.ts" - && row.file_occurrence_id == artifacts.chunks.document.file_occurrence_id + assert!(imports.iter().all(|row| { + row.file_occurrence_id.as_str() == "file.import.a" && !row.span.is_empty() && row.span.end_byte <= indexed_end })); assert_eq!( - artifacts - .imports + imports .iter() .map(|row| { &FIRST_SOURCE[usize::try_from(row.span.start_byte).expect("span start") @@ -969,42 +915,6 @@ fn file_import_artifacts_bind_file_consistent_path_and_nonempty_span_to_indexed_ .collect::>(), vec!["Foo", "Bar as Baz"] ); - - let mut wrong_file = artifacts.clone(); - for row in &mut wrong_file.imports { - row.file_occurrence_id = id("file.foreign"); - } - assert_eq!( - wrong_file.validate(), - Err(ChunkingFailureV1::GenerationMismatch) - ); - - let mut inconsistent_path = artifacts.clone(); - inconsistent_path.imports[1].logical_path = "src/foreign.ts".to_owned(); - assert_eq!( - inconsistent_path.validate(), - Err(ChunkingFailureV1::NonCanonicalIdentity( - NonCanonicalCauseV1::new(NonCanonicalReasonCodeV1::ImportMultiFile) - )) - ); - - let mut empty_span = artifacts.clone(); - empty_span.imports[0].span.end_byte = empty_span.imports[0].span.start_byte; - assert_eq!( - empty_span.validate(), - Err(ChunkingFailureV1::NonCanonicalIdentity( - NonCanonicalCauseV1::new(NonCanonicalReasonCodeV1::ImportEmptySourceSpan) - )) - ); - - let mut out_of_bounds = artifacts; - out_of_bounds.imports[0].span.end_byte = indexed_end + 1; - assert_eq!( - out_of_bounds.validate(), - Err(ChunkingFailureV1::NonCanonicalIdentity( - NonCanonicalCauseV1::new(NonCanonicalReasonCodeV1::ImportExceedsFileExtent) - )) - ); } #[test] @@ -1026,105 +936,77 @@ fn raw_use_and_imported_bindings_never_become_canonical_symbols() { } #[test] -fn sealed_revision_nine_import_generation_round_trips_to_identical_bytes() { +fn sealed_import_generation_round_trips_to_identical_bytes() { let first = published_import_generation(); - let first_sealed = first.encode_sealed().expect("first generation seals"); - let second_sealed = published_import_generation() - .encode_sealed() - .expect("identical generation seals"); - assert_eq!(first_sealed, second_sealed); + let first_sealed = PartitionedSealV1::of(&first); + let second_sealed = PartitionedSealV1::of(&published_import_generation()); + assert_eq!(first_sealed.manifest, second_sealed.manifest); + assert_eq!(first_sealed.segments, second_sealed.segments); - let envelope: Value = serde_json::from_slice(&first_sealed).expect("sealed generation JSON"); - assert_eq!(envelope["generation"]["format_revision"], 9); - let restored = - CodeIndexPublishedGenerationV1::decode_sealed(&first_sealed).expect("rev9 restores"); + assert_eq!( + first_sealed.envelope()["generation"]["format_revision"], + SEALED_GENERATION_FORMAT_REVISION_V1 + ); + let restored = first_sealed.restored(); assert_eq!(restored.imports(), first.imports()); assert_eq!( - restored.encode_sealed().expect("restored generation seals"), - first_sealed + PartitionedSealV1::of(&restored).manifest, + first_sealed.manifest ); } #[test] -fn sealed_import_generation_rejects_semantic_tampering_after_outer_digest_recompute() { - let generation = published_import_generation(); - let mut envelope = sealed_envelope(&generation); - assert_sealed_envelope_restores(&envelope); - - import_rows_mut(&mut envelope, 0)[0]["imported_name"] = Value::String("Forged".to_owned()); - file_artifact(&envelope, 0) - .validate() - .expect("binding-name tamper remains structurally canonical"); - let error = resealed_import_payload_error(envelope, "binding-name tamper"); +fn sealed_import_generation_rejects_semantic_tampering_after_segment_readdress() { + let sealed = PartitionedSealV1::of(&published_import_generation()); + sealed.restored(); assert!( - error.to_string().contains("import_authority_mismatch"), - "semantic tamper reached the wrong authority rejection: {error}" + sealed.file_segment_payload(0)["artifacts"] + .get("import_rows_digest") + .is_none(), + "sealed file artifacts must not contain a recomputable self-digest authority" ); -} - -#[test] -fn sealed_import_generation_rejects_semantic_tampering_after_import_and_outer_digest_recompute() { - let generation = published_import_generation(); - let mut envelope = sealed_envelope(&generation); - assert_sealed_envelope_restores(&envelope); - - import_rows_mut(&mut envelope, 0)[0]["imported_name"] = Value::String("Forged".to_owned()); - assert_serialized_artifact_has_no_self_digest(&envelope, 0); - file_artifact(&envelope, 0) - .validate() - .expect("self-consistent import digest remains structurally canonical"); - let error = resealed_import_payload_error( - envelope, - "binding-name tamper with recomputed import-row digest", - ); + let error = tampered_import_payload_error(&sealed, "binding-name tamper", |rows| { + rows[0]["imported_name"] = Value::String("Forged".to_owned()); + }); assert!( error.to_string().contains("import_authority_mismatch"), - "self-consistent import forgery reached the wrong authority rejection: {error}" + "semantic tamper reached the wrong authority rejection: {error}" ); } #[test] -fn sealed_import_generation_rejects_reorder_and_duplicate_after_outer_digest_recompute() { - let generation = published_import_generation(); - let envelope = sealed_envelope(&generation); - assert_sealed_envelope_restores(&envelope); - - let mut reordered = envelope.clone(); - import_rows_mut(&mut reordered, 0).swap(0, 1); - assert_resealed_import_payload_is_rejected(reordered, "row reorder"); - - let mut duplicated = envelope; - let duplicate = import_rows_mut(&mut duplicated, 0)[0].clone(); - import_rows_mut(&mut duplicated, 0).insert(1, duplicate); - assert_resealed_import_payload_is_rejected(duplicated, "duplicate row"); +fn sealed_import_generation_rejects_reorder_and_duplicate_after_segment_readdress() { + let sealed = PartitionedSealV1::of(&published_import_generation()); + sealed.restored(); + + tampered_import_payload_error(&sealed, "row reorder", |rows| rows.swap(0, 1)); + tampered_import_payload_error(&sealed, "duplicate row", |rows| { + let duplicate = rows[0].clone(); + rows.insert(1, duplicate); + }); } #[test] -fn sealed_import_generation_rejects_wrong_file_path_and_span_after_outer_digest_recompute() { - let generation = published_import_generation(); - let envelope = sealed_envelope(&generation); - assert_sealed_envelope_restores(&envelope); - - let mut wrong_file = envelope.clone(); - for row in import_rows_mut(&mut wrong_file, 0) { - row["file_occurrence_id"] = Value::String("file.foreign".to_owned()); - } - assert_resealed_import_payload_is_rejected(wrong_file, "foreign file occurrence"); - - let mut wrong_path = envelope.clone(); - for row in import_rows_mut(&mut wrong_path, 0) { - row["logical_path"] = Value::String("src/foreign.ts".to_owned()); - } - assert_resealed_import_payload_is_rejected(wrong_path, "foreign logical path"); - - let mut empty_span = envelope.clone(); - let start = import_rows_mut(&mut empty_span, 0)[0]["span"]["start_byte"].clone(); - import_rows_mut(&mut empty_span, 0)[0]["span"]["end_byte"] = start; - assert_resealed_import_payload_is_rejected(empty_span, "empty source span"); - - let mut out_of_bounds = envelope; - import_rows_mut(&mut out_of_bounds, 0)[0]["span"]["end_byte"] = - Value::from(FIRST_SOURCE.len() as u64 + 1); - assert_resealed_import_payload_is_rejected(out_of_bounds, "out-of-bounds source span"); +fn sealed_import_generation_rejects_wrong_file_path_and_span_after_segment_readdress() { + let sealed = PartitionedSealV1::of(&published_import_generation()); + sealed.restored(); + + tampered_import_payload_error(&sealed, "foreign file occurrence", |rows| { + for row in rows { + row["file_occurrence_id"] = Value::String("file.foreign".to_owned()); + } + }); + tampered_import_payload_error(&sealed, "foreign logical path", |rows| { + for row in rows { + row["logical_path"] = Value::String("src/foreign.ts".to_owned()); + } + }); + tampered_import_payload_error(&sealed, "empty source span", |rows| { + let start = rows[0]["span"]["start_byte"].clone(); + rows[0]["span"]["end_byte"] = start; + }); + tampered_import_payload_error(&sealed, "out-of-bounds source span", |rows| { + rows[0]["span"]["end_byte"] = Value::from(FIRST_SOURCE.len() as u64 + 1); + }); } diff --git a/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration.rs b/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration.rs index 55fae5a265..068d8e791f 100644 --- a/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration.rs +++ b/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration.rs @@ -1,7 +1,6 @@ use std::{ cell::Cell, collections::{BTreeMap, BTreeSet}, - io::Cursor, sync::atomic::{AtomicBool, AtomicUsize, Ordering}, sync::{Arc, Mutex}, time::Duration, @@ -26,7 +25,6 @@ use tracedecay_code_index::{ SealedGenerationSegmentPublicationV1, SealedGenerationSegmentReadV1, SharedPhysicalCodeArtifactPoolV1, VerifiedSealedLexicalPageReadV1, VerifiedSealedLexicalPageSourceV1, VerifiedSealedLexicalPageV1, - sealed_generation_payload_digest, }, projection::{ ChunkProjectionDecisionV1, CodeChunkProjectionSink, ProjectionReceiptBuilderV1, @@ -47,7 +45,7 @@ use tracedecay_domain::{ }; use tracedecay_graph_db::{GraphDbError, GraphNamespace, GraphProjectorRevision}; -use crate::support::{RUST_SOURCE, id}; +use crate::support::{PartitionedSealV1, RUST_SOURCE, id, reseal_manifest}; mod parallel_equivalence; @@ -795,8 +793,8 @@ fn physical_artifact_reuse_preserves_byte_exact_sealed_generation() { "rematerialize must rebind shared chunks onto the requesting occurrence" ); assert_ne!( - foreign.encode_sealed().expect("foreign generation seals"), - reused.encode_sealed().expect("reused generation seals"), + PartitionedSealV1::of(&foreign).manifest, + PartitionedSealV1::of(&reused).manifest, "rebound occurrence identity must change the sealed corpus" ); @@ -850,8 +848,8 @@ fn physical_artifact_reuse_preserves_byte_exact_sealed_generation() { "projection mismatch" ); assert_eq!( - reused.encode_sealed().expect("reused generation seals"), - cold.encode_sealed().expect("cold generation seals"), + PartitionedSealV1::of(&reused), + PartitionedSealV1::of(&cold), "sharing the physical allocation must preserve every durable byte and digest" ); drop(source); @@ -950,8 +948,8 @@ fn resumed_parse_quanta_publish_the_same_complete_generation() { .build_and_publish(cold_request, &ActiveControl) .expect("cold generation"); assert_eq!( - generation.encode_sealed().expect("resumed seal"), - cold.encode_sealed().expect("cold seal") + PartitionedSealV1::of(&generation), + PartitionedSealV1::of(&cold) ); } @@ -1083,7 +1081,34 @@ fn published_generation_serves_current_conservative_test_attribution() { join.test_watermark.snapshot_digest, generation.manifest().snapshot_digest ); - assert!(!join.records.is_empty()); + // Every callable in a test file is a test; neither fixture callable calls + // another fixture symbol, so each covers only itself. + let occurrence_of = |qualified_name: &str| { + generation + .symbols() + .symbols + .iter() + .find(|symbol| symbol.qualified_name == qualified_name) + .unwrap_or_else(|| panic!("fixture symbol {qualified_name}")) + .occurrence + .clone() + }; + let alpha = occurrence_of("tests/production.rs::alpha"); + let get = occurrence_of("tests/production.rs::Holder::get"); + let attributed = join + .records + .iter() + .map(|record| { + ( + record.attribution.test_occurrence.clone(), + record.attribution.covered_occurrences.clone(), + ) + }) + .collect::>(); + assert_eq!( + attributed, + BTreeMap::from([(alpha.clone(), vec![alpha]), (get.clone(), vec![get])]) + ); assert!(join.records.iter().all(|record| { record.attribution.evidence_class == TestAttributionEvidenceClassV1::ConservativeDependencyCandidates @@ -1357,8 +1382,8 @@ fn sealed_store_drops_whitespace_only_window_chunks() { chunks.len() < FUNCTIONS * 3, "sealed chunk count must drop below the three-per-function baseline" ); - let sealed = generation.encode_sealed().expect("generation seals"); - assert!(!sealed.is_empty(), "sealed store must carry bytes"); + let sealed = PartitionedSealV1::of(&generation); + assert!(!sealed.manifest.is_empty(), "sealed store must carry bytes"); } #[test] @@ -1420,9 +1445,7 @@ fn published_graph_manifest_projects_files_chunks_symbols_and_replays_byte_ident "the graph must not scale with the chunk count" ); - let sealed = generation.encode_sealed().expect("generation seals"); - let restored = - CodeIndexPublishedGenerationV1::decode_sealed(&sealed).expect("generation restores"); + let restored = PartitionedSealV1::of(&generation).restored(); let replayed = build_published_code_graph_manifest_checked( projection, &restored, @@ -1624,42 +1647,23 @@ fn sealed_generation_validation_is_memoized_but_decode_stays_fail_closed() { let generation = owner .build_and_publish(request("file.project-binding", 1_200_000), &ActiveControl) .expect("valid generation publishes"); - let sealed = generation.encode_sealed().expect("valid generation seals"); - assert_eq!( - generation - .encode_sealed() - .expect("memoized generation seals again"), - sealed - ); + let sealed = PartitionedSealV1::of(&generation); + assert_eq!(PartitionedSealV1::of(&generation).manifest, sealed.manifest); - let restored = - CodeIndexPublishedGenerationV1::decode_sealed(&sealed).expect("valid generation restores"); + let restored = sealed.restored(); assert_eq!(restored.manifest().project_id, config().project_id); - assert_eq!( - restored - .encode_sealed() - .expect("restored generation retains successful validation"), - sealed - ); - - let mut envelope: serde_json::Value = - serde_json::from_slice(&sealed).expect("sealed generation JSON"); - envelope["generation"]["files"][0]["authority"]["project_id"] = - serde_json::Value::String("project.foreign".to_owned()); - let state_digest = sealed_generation_payload_digest( - SEALED_GENERATION_FORMAT_REVISION_V1, - &envelope["generation"], - ) - .expect("forged payload has a state digest"); - envelope["state_digest"] = serde_json::Value::String(state_digest.as_str().to_owned()); - let forged = serde_json::to_vec(&envelope).expect("forged sealed generation JSON"); + assert_eq!(PartitionedSealV1::of(&restored).manifest, sealed.manifest); - let error = CodeIndexPublishedGenerationV1::decode_sealed(&forged) + // A segment's project comes from the manifest that names it, so a segment + // that tries to carry its own is refused rather than rebound. + let forged = sealed.with_tampered_file_segment(0, |file| { + file["authority"]["project_id"] = serde_json::Value::String("project.foreign".to_owned()); + }); + let error = forged + .restore(&forged.manifest) .expect_err("foreign file authority must fail sealed restoration"); assert!( - error - .to_string() - .contains("file authority project does not match the generation manifest"), + error.to_string().contains("unknown field `project_id`"), "unexpected project mismatch error: {error}" ); } @@ -1681,24 +1685,12 @@ fn verified_sealed_lexical_pages_are_bounded_exact_and_resumable_after_cancellat &ActiveControl, ) .expect("generation publishes"); - let sealed = generation.encode_sealed().expect("generation seals"); - let envelope: serde_json::Value = - serde_json::from_slice(&sealed).expect("sealed generation JSON"); - let expected_state_digest = id::( - envelope["state_digest"] - .as_str() - .expect("state digest string"), - ); + let sealed = PartitionedSealV1::of(&generation); + let expected_state_digest = sealed.state_digest(); let control = MutableCancellationControl::default(); - let mut source = VerifiedSealedLexicalPageSourceV1::open( - Cursor::new(sealed.clone()), - u64::try_from(sealed.len()).expect("sealed length"), - expected_state_digest.clone(), - 1, - 1024 * 1024, - &control, - ) - .expect("verified page source opens"); + let mut source = sealed + .lexical_source(1, 1024 * 1024) + .expect("verified page source opens"); let first = match source.next_page(&control).expect("first page") { VerifiedSealedLexicalPageReadV1::Page(page) => page, @@ -1766,7 +1758,10 @@ fn verified_sealed_lexical_pages_are_bounded_exact_and_resumable_after_cancellat assert_eq!(observed_clone_bodies, 3); assert_eq!(receipt.cumulative_digest(), &final_page_digest); assert_eq!(receipt.source_state_digest(), &expected_state_digest); - assert_eq!(receipt.format_revision(), 9); + assert_eq!( + receipt.format_revision(), + SEALED_GENERATION_FORMAT_REVISION_V1 + ); } #[test] @@ -1811,23 +1806,15 @@ fn verified_content_addressed_lexical_source_resumes_from_a_persisted_cursor() { let generation = owner .build_and_publish(request, &ActiveControl) .expect("generation publishes"); - let sealed = generation.encode_sealed().expect("generation seals"); - let file_digest = - id::(&format!("sha256:{}", hex::encode(Sha256::digest(&sealed)))); - - let mut initial = VerifiedSealedLexicalPageSourceV1::open_content_addressed( - Cursor::new(sealed.clone()), - u64::try_from(sealed.len()).expect("sealed length"), - file_digest.clone(), - 64, - 1024 * 1024, - &ActiveControl, - ) - .expect("content-addressed source opens"); + let sealed = PartitionedSealV1::of(&generation); + + let mut initial = sealed + .lexical_source(64, 1024 * 1024) + .expect("content-addressed source opens"); let retained_layout_bytes = initial.retained_layout_bytes(); assert!( retained_layout_bytes > std::mem::size_of::() * 4 - && retained_layout_bytes < sealed.len() / 8, + && retained_layout_bytes < sealed.byte_len() / 8, "source layout must count retained file positions while staying compact" ); let first = match initial.next_page(&ActiveControl).expect("first page") { @@ -1849,16 +1836,12 @@ fn verified_content_addressed_lexical_source_resumes_from_a_persisted_cursor() { ) .expect("persisted cursor restores"); - let mut resumed = VerifiedSealedLexicalPageSourceV1::open_content_addressed_at( - Cursor::new(sealed.clone()), - u64::try_from(sealed.len()).expect("sealed length"), - file_digest, - cursor.clone(), - 64, - 1024 * 1024, - &ActiveControl, - ) - .expect("persisted cursor reopens the source"); + let mut resumed = sealed + .lexical_source(64, 1024 * 1024) + .expect("source reopens"); + resumed + .restore_cursor(&cursor, &ActiveControl) + .expect("persisted cursor reopens the source"); let resumed_page = match resumed.next_page(&ActiveControl).expect("resumed page") { VerifiedSealedLexicalPageReadV1::Page(page) => page, VerifiedSealedLexicalPageReadV1::Complete(_) => panic!("fixture must emit a resumed page"), @@ -1896,35 +1879,19 @@ fn verified_content_addressed_lexical_source_resumes_from_a_persisted_cursor() { ), &ActiveControl, ) - .expect("foreign generation publishes") - .encode_sealed() - .expect("foreign generation seals"); - let foreign_digest = - id::(&format!("sha256:{}", hex::encode(Sha256::digest(&foreign)))); - let foreign_source = VerifiedSealedLexicalPageSourceV1::open_content_addressed( - Cursor::new(foreign.clone()), - u64::try_from(foreign.len()).expect("foreign sealed length"), - foreign_digest.clone(), - 64, - 1024 * 1024, - &ActiveControl, - ) - .expect("one-file content-addressed source opens"); + .expect("foreign generation publishes"); + let foreign = PartitionedSealV1::of(&foreign); + let mut foreign_source = foreign + .lexical_source(64, 1024 * 1024) + .expect("one-file content-addressed source opens"); assert!( foreign_source.retained_layout_bytes() > std::mem::size_of::() * 4 && foreign_source.retained_layout_bytes() <= retained_layout_bytes, "the one-file source must account for its positions within the two-file allocation bound" ); - let error = VerifiedSealedLexicalPageSourceV1::open_content_addressed_at( - Cursor::new(foreign.clone()), - u64::try_from(foreign.len()).expect("foreign sealed length"), - foreign_digest, - cursor, - 64, - 1024 * 1024, - &ActiveControl, - ) - .expect_err("a cursor minted for another source must fail closed"); + let error = foreign_source + .restore_cursor(&cursor, &ActiveControl) + .expect_err("a cursor minted for another source must fail closed"); assert!( error.to_string().contains("cursor") && error.to_string().contains("source"), "unexpected foreign cursor error: {error}" @@ -1959,22 +1926,30 @@ fn verified_lexical_source_pages_a_large_file_and_resumes_after_cancellation() { .expect("published generation retains exact chunks") .len() as u64; assert!(expected_chunks > 64, "fixture must require multiple pages"); - let sealed = generation.encode_sealed().expect("large generation seals"); - let file_digest = - id::(&format!("sha256:{}", hex::encode(Sha256::digest(&sealed)))); + let sealed = PartitionedSealV1::of(&generation); let control = MutableCancellationControl::default(); - let mut source = VerifiedSealedLexicalPageSourceV1::open_content_addressed( - Cursor::new(sealed.clone()), - u64::try_from(sealed.len()).expect("sealed length"), - file_digest.clone(), - 32, - 64 * 1024, - &control, - ) - .expect("a valid large file must not be rejected by its page bound"); + let mut source = sealed + .lexical_source(32, 64 * 1024) + .expect("a valid large file must not be rejected by its page bound"); + let largest_file_segment = sealed.envelope()["generation"]["file_segments"] + .as_array() + .expect("file segment descriptors") + .iter() + .map(|segment| { + segment["decoded_size_bytes"] + .as_u64() + .expect("decoded size") + }) + .max() + .expect("one file segment"); + assert!( + largest_file_segment > 64 * 1024, + "the one-file segment must exceed the page byte bound" + ); assert!( - source.staging_window_bytes() > 4 * 1024 * 1024, - "the authenticated one-file artifact must exceed four MiB" + source.staging_window_bytes() + >= usize::try_from(largest_file_segment).expect("segment size fits usize"), + "the staging window must admit the whole authenticated file segment" ); let mut emitted_chunks = 0_u64; @@ -2011,16 +1986,12 @@ fn verified_lexical_source_pages_a_large_file_and_resumes_after_cancellation() { &persisted, ) .expect("progress cursor restores"); - let mut resumed = VerifiedSealedLexicalPageSourceV1::open_content_addressed_at( - Cursor::new(sealed.clone()), - u64::try_from(sealed.len()).expect("sealed length"), - file_digest, - cursor, - 32, - 64 * 1024, - &control, - ) - .expect("resumed large source opens"); + let mut resumed = sealed + .lexical_source(32, 64 * 1024) + .expect("resumed large source opens"); + resumed + .restore_cursor(&cursor, &control) + .expect("progress cursor restores onto the source"); let receipt = loop { match resumed.next_page(&control).expect("resumed bounded page") { VerifiedSealedLexicalPageReadV1::Page(page) => { @@ -2037,32 +2008,6 @@ fn verified_lexical_source_pages_a_large_file_and_resumes_after_cancellation() { assert_eq!(receipt.page_count(), emitted_pages); } -#[test] -fn verified_sealed_lexical_source_refuses_a_foreign_state_digest() { - let store = SharedPublicationStore::default(); - let mut owner = CodeIndexProductionOwnerV1::new(config(), store, ApplyingProjectionSink) - .expect("production owner"); - let generation = owner - .build_and_publish(request("file.lexical-digest", 1_260_000), &ActiveControl) - .expect("generation publishes"); - let sealed = generation.encode_sealed().expect("generation seals"); - let error = VerifiedSealedLexicalPageSourceV1::open( - Cursor::new(sealed.clone()), - u64::try_from(sealed.len()).expect("sealed length"), - id::(&format!("sha256:{}", "0".repeat(64))), - 16, - 1024 * 1024, - &ActiveControl, - ) - .expect_err("a foreign durable state digest must not authorize lexical bytes"); - assert!( - error - .to_string() - .contains("state digest does not match the admitted source"), - "unexpected digest error: {error}" - ); -} - #[test] fn verified_sealed_lexical_imports_are_exact_once_and_page_boundary_independent() { let store = SharedPublicationStore::default(); @@ -2090,25 +2035,12 @@ fn verified_sealed_lexical_imports_are_exact_once_and_page_boundary_independent( !generation.imports().is_empty(), "the fixture must contain parser-backed import evidence" ); - let sealed = generation.encode_sealed().expect("generation seals"); - let envelope: serde_json::Value = - serde_json::from_slice(&sealed).expect("sealed generation JSON"); - let expected_state_digest = id::( - envelope["state_digest"] - .as_str() - .expect("state digest string"), - ); + let sealed = PartitionedSealV1::of(&generation); let read = |maximum_page_chunks| { - let mut source = VerifiedSealedLexicalPageSourceV1::open( - Cursor::new(sealed.clone()), - u64::try_from(sealed.len()).expect("sealed length"), - expected_state_digest.clone(), - maximum_page_chunks, - 1024 * 1024, - &ActiveControl, - ) - .expect("verified import page source opens"); + let mut source = sealed + .lexical_source(maximum_page_chunks, 1024 * 1024) + .expect("verified import page source opens"); let mut imports = Vec::new(); let receipt = loop { match source @@ -2205,23 +2137,10 @@ fn verified_sealed_lexical_page_transition_is_canonical_across_importing_files() "both files must contribute parser-backed import evidence" ); - let sealed = generation.encode_sealed().expect("generation seals"); - let envelope: serde_json::Value = - serde_json::from_slice(&sealed).expect("sealed generation JSON"); - let state_digest = id::( - envelope["state_digest"] - .as_str() - .expect("state digest string"), - ); - let mut source = VerifiedSealedLexicalPageSourceV1::open( - Cursor::new(sealed.clone()), - u64::try_from(sealed.len()).expect("sealed length"), - state_digest, - usize::MAX, - 1024 * 1024, - &ActiveControl, - ) - .expect("verified page source opens"); + let sealed = PartitionedSealV1::of(&generation); + let mut source = sealed + .lexical_source(usize::MAX, 1024 * 1024) + .expect("verified page source opens"); let mut previous_cursor = None; let mut observed_chunks = Vec::new(); let mut observed_imports = Vec::new(); @@ -2272,23 +2191,10 @@ fn rejected_sealed_lexical_page_admission_does_not_advance_the_source() { let generation = owner .build_and_publish(request("file.lexical-admission", 1_266_500), &ActiveControl) .expect("generation publishes"); - let sealed = generation.encode_sealed().expect("generation seals"); - let envelope: serde_json::Value = - serde_json::from_slice(&sealed).expect("sealed generation JSON"); - let state_digest = id::( - envelope["state_digest"] - .as_str() - .expect("state digest string"), - ); - let mut source = VerifiedSealedLexicalPageSourceV1::open( - Cursor::new(sealed.clone()), - u64::try_from(sealed.len()).expect("sealed length"), - state_digest, - usize::MAX, - 1024 * 1024, - &ActiveControl, - ) - .expect("verified page source opens"); + let sealed = PartitionedSealV1::of(&generation); + let mut source = sealed + .lexical_source(usize::MAX, 1024 * 1024) + .expect("verified page source opens"); let cursor_before = source .cursor() .persisted_bytes() @@ -2384,29 +2290,31 @@ fn verified_sealed_lexical_page_retained_bytes_include_real_owned_capacities() { let generation = owner .build_and_publish(request, &ActiveControl) .expect("generation publishes"); - let sealed = generation.encode_sealed().expect("generation seals"); - let envelope: serde_json::Value = - serde_json::from_slice(&sealed).expect("sealed generation JSON"); - let state_digest = id::( - envelope["state_digest"] - .as_str() - .expect("state digest string"), - ); - let mut source = VerifiedSealedLexicalPageSourceV1::open( - Cursor::new(sealed.clone()), - u64::try_from(sealed.len()).expect("sealed length"), - state_digest, - usize::MAX, - 1024 * 1024, - &ActiveControl, - ) - .expect("verified page source opens"); + let sealed = PartitionedSealV1::of(&generation); + let mut source = sealed + .lexical_source(usize::MAX, 1024 * 1024) + .expect("verified page source opens"); let page = match source.next_page(&ActiveControl).expect("verified page") { VerifiedSealedLexicalPageReadV1::Page(page) => page, VerifiedSealedLexicalPageReadV1::Complete(_) => panic!("fixture must emit a page"), }; - assert!(!page.chunks().is_empty()); - assert!(!page.imports().is_empty()); + assert!( + page.chunks() + .iter() + .all(|chunk| chunk.chunk().anchor.file_occurrence_id.as_str() + == "file.lexical-import-capacity"), + "every chunk belongs to the fixture file" + ); + assert!( + page.symbol_displays() + .iter() + .flatten() + .any(|display| display.qualified_name() == "src/imports.ts::render"), + "the page carries the fixture function under its logical path" + ); + assert_eq!(page.imports().len(), 1); + assert_eq!(page.imports()[0].module_specifier, "widget-kit"); + assert_eq!(page.imports()[0].logical_path, "src/imports.ts"); let vector_and_module_capacity_floor = page .chunk_capacity() @@ -2441,29 +2349,24 @@ fn published_generation_validation_is_amortized_per_loaded_generation() { generation.is_validated(), "publishing a generation must run its integrity gate" ); - let sealed = generation.encode_sealed().expect("valid generation seals"); - let resealed = generation - .encode_sealed() - .expect("an already-verified generation reseals"); + let sealed = PartitionedSealV1::of(&generation); + let resealed = PartitionedSealV1::of(&generation); assert_eq!( - sealed, resealed, + sealed.manifest, resealed.manifest, "the memoized gate must reach the same verdict and payload as the first check" ); // Restoring re-reads bytes from the sealed store, so it must verify fresh // rather than trust any carried mark. - let restored = - CodeIndexPublishedGenerationV1::decode_sealed(&sealed).expect("valid generation restores"); + let restored = sealed.restored(); assert!( restored.is_validated(), "a restored generation must be fully verified before it can serve" ); assert_eq!(restored.manifest(), generation.manifest()); assert_eq!( - restored - .encode_sealed() - .expect("restored generation reseals"), - sealed, + PartitionedSealV1::of(&restored).manifest, + sealed.manifest, "a restored generation must reseal to identical bytes" ); @@ -2516,22 +2419,15 @@ fn sealed_manifest_authenticates_source_commitments_and_refuses_missing_history( &ActiveControl, ) .expect("valid generation publishes"); - let sealed = generation.encode_sealed().expect("valid generation seals"); - let envelope: serde_json::Value = - serde_json::from_slice(&sealed).expect("sealed generation JSON"); + let sealed = PartitionedSealV1::of(&generation); + let envelope = sealed.envelope(); let mut tampered = envelope.clone(); tampered["generation"]["manifest"]["source_commitments"]["full_replay_digest"] = serde_json::json!(format!("sha256:{}", "f".repeat(64))); - let state_digest = sealed_generation_payload_digest( - SEALED_GENERATION_FORMAT_REVISION_V1, - &tampered["generation"], - ) - .expect("tampered payload has an outer digest"); - tampered["state_digest"] = serde_json::json!(state_digest.as_str()); - let tampered = serde_json::to_vec(&tampered).expect("tampered sealed generation"); assert!( - CodeIndexPublishedGenerationV1::decode_sealed(&tampered) + sealed + .restore(&reseal_manifest(tampered)) .expect_err("the authenticated source commitment must reject tampering") .to_string() .contains("seal") @@ -2543,15 +2439,8 @@ fn sealed_manifest_authenticates_source_commitments_and_refuses_missing_history( .expect("generation manifest") .remove("source_commitments") .expect("current manifest carries source commitments"); - let state_digest = sealed_generation_payload_digest( - SEALED_GENERATION_FORMAT_REVISION_V1, - &historical["generation"], - ) - .expect("historical payload has an outer digest"); - historical["state_digest"] = serde_json::json!(state_digest.as_str()); - let historical = serde_json::to_vec(&historical).expect("historical sealed generation"); assert!(matches!( - CodeIndexPublishedGenerationV1::decode_sealed(&historical), + sealed.restore(&reseal_manifest(historical)), Err(CodeIndexProductionErrorV1::SourceCommitmentsUnavailable) )); } @@ -2570,28 +2459,20 @@ fn corrupted_chunk_evidence_fails_the_first_validation_of_a_restored_generation( &ActiveControl, ) .expect("valid generation publishes"); - let sealed = generation.encode_sealed().expect("valid generation seals"); - - let mut envelope: serde_json::Value = - serde_json::from_slice(&sealed).expect("sealed generation JSON"); - let chunk = &mut envelope["generation"]["files"][0]["artifacts"]["chunks"]["chunks"][0]; - assert!( - !chunk.is_null(), - "the fixture generation must contain at least one chunk" - ); // Break the chunk's canonical identity so it no longer matches the document - // membership its file artifact claims. - chunk["id"] = serde_json::Value::String("chunk.tampered".to_owned()); - // Re-seal the envelope so the outer state digest cannot be what rejects it. - let state_digest = sealed_generation_payload_digest( - SEALED_GENERATION_FORMAT_REVISION_V1, - &envelope["generation"], - ) - .expect("forged payload has a state digest"); - envelope["state_digest"] = serde_json::Value::String(state_digest.as_str().to_owned()); - let forged = serde_json::to_vec(&envelope).expect("forged sealed generation JSON"); + // membership its file artifact claims, then re-address the segment and + // reseal the manifest so neither digest can be what rejects it. + let forged = PartitionedSealV1::of(&generation).with_tampered_file_segment(0, |file| { + let chunk = &mut file["artifacts"]["chunks"]["chunks"][0]; + assert!( + !chunk.is_null(), + "the fixture generation must contain at least one chunk" + ); + chunk["id"] = serde_json::Value::String("chunk.tampered".to_owned()); + }); - let error = CodeIndexPublishedGenerationV1::decode_sealed(&forged) + let error = forged + .restore(&forged.manifest) .expect_err("corrupted chunk evidence must fail the first validation"); let message = error.to_string(); assert!( @@ -2977,10 +2858,7 @@ fn code_shard_slot_key_is_the_sealed_branch_label_not_a_generation_id() { // The sealed branch label is durable through the sealed codec, so a // restored generation still names the exact slot it was sealed for. - let restored = CodeIndexPublishedGenerationV1::decode_sealed( - &pr.encode_sealed().expect("pr generation seals"), - ) - .expect("pr generation restores"); + let restored = PartitionedSealV1::of(&pr).restored(); assert_eq!(restored.sealed_scope(), pr.sealed_scope()); } @@ -3244,23 +3122,23 @@ fn partitioned_codec_fixture() -> ( } const PARTITIONED_FORMAT_STATE_DIGEST: &str = - "sha256:a4b25af0fc2d33060b2d7bc65077e24266ab4a97f3be0bdc770449abc68150d8"; + "sha256:ce70aa9c7cfe6357b9f56d8f31ad14c22fff1210cc6cb49d35a0be29bef9289a"; const PARTITIONED_FORMAT_SEGMENTS: &[(&str, u64)] = &[ ( - "sha256:f6ef37eeffb1395c6597060871a9bfdd28ee269714c9679550514fc586c5fb30", - 11_071, + "sha256:d459a8147cff5a99f10ce10fb144bdddfc443318c9b4eb36c2e03a3a7fcb7897", + 2_227, ), ( - "sha256:5405936e448505980327bf2dabc10367ea37a1a2edc4194be7305dd0f0d3ff28", - 5_171, + "sha256:bb00f9a412e136c877257955f5747bcd83ab98ea3635173e58905d656099abd4", + 1_430, ), ( - "sha256:cdb52cd810f0545b79555179179fd137dd17cd58bd226d6ea927321ea4ad9b57", - 6_279, + "sha256:8250f37e248f46fc1bdc10aa93b2a940f9de87e0f139fa71505444148a18dde3", + 1_477, ), ( - "sha256:0309a86f6ab77fe91e584ba4419fdc2e39807afda2fe7683ac792ed44a8a5801", - 6_837, + "sha256:51b7d16817def8c21c1ed0b154cce1ad5cc49c6966899ea7cc28d14a6d5005d9", + 2_837, ), ]; @@ -3274,8 +3152,7 @@ fn partitioned_codec_has_stable_bytes_and_round_trips() { "the canonical manifest payload bytes changed" ); let identities = CodeIndexPublishedGenerationV1::partitioned_segment_identities(&manifest) - .expect("partitioned segment identities parse") - .expect("current partitioned manifest"); + .expect("partitioned segment identities parse"); assert_eq!( identities .iter() @@ -3287,9 +3164,8 @@ fn partitioned_codec_has_stable_bytes_and_round_trips() { assert_eq!( CodeIndexPublishedGenerationV1::partitioned_text_metadata(&manifest) .expect("partitioned text metadata parses") - .expect("current partitioned manifest") .generation_statistics(), - Some(&expected.generation_statistics().expect("fixture census")), + &expected.generation_statistics().expect("fixture census"), "a sealed manifest carries the generation's own census" ); @@ -3381,8 +3257,7 @@ fn partitioned_codec_has_stable_bytes_and_round_trips() { segment_reads.set(segment_reads.get() + 1); Ok(()) }) - .expect("partitioned bytes decode") - .expect("current partitioned manifest"); + .expect("partitioned bytes decode"); assert_eq!(segment_reads.get(), PARTITIONED_FORMAT_SEGMENTS.len()); let largest_file_segment = largest_file_segment.get(); assert_eq!( @@ -3406,27 +3281,13 @@ fn partitioned_codec_has_stable_bytes_and_round_trips() { && evidence_buffer_capacity.get() <= largest_evidence_page.get().next_power_of_two(), "the evidence allocation must be bounded by the largest evidence page" ); - let restored_seal = restored.encode_sealed().expect("restored generation seals"); - let expected_seal = expected.encode_sealed().expect("expected generation seals"); - let restored_json: serde_json::Value = - serde_json::from_slice(&restored_seal).expect("restored sealed JSON"); - let expected_json: serde_json::Value = - serde_json::from_slice(&expected_seal).expect("expected sealed JSON"); assert_eq!( - restored_json["generation"]["files"] - .as_array() - .expect("restored files") - .iter() - .map(|file| &file["artifacts"]["clone_bodies"]) - .collect::>(), - expected_json["generation"]["files"] - .as_array() - .expect("expected files") - .iter() - .map(|file| &file["artifacts"]["clone_bodies"]) - .collect::>(), - "partitioned restore must preserve clone rows" + sealed_clone_bindings(&restored), + sealed_clone_bindings(&expected), + "decode must restore every file's clone bodies" ); + let restored_seal = PartitionedSealV1::of(&restored); + let expected_seal = PartitionedSealV1::of(&expected); assert_eq!( restored_seal, expected_seal, "decode must restore the same typed generation" @@ -3454,11 +3315,8 @@ fn partitioned_codec_has_stable_bytes_and_round_trips() { buffer.extend_from_slice(&bytes[start..end]); Ok(()) }; - assert!( - CodeIndexPublishedGenerationV1::verify_partitioned_sealed(&manifest, read) - .expect("intact partitioned segments authenticate"), - "the fixture's own segments must verify against its manifest" - ); + CodeIndexPublishedGenerationV1::verify_partitioned_sealed(&manifest, read) + .expect("the fixture's own segments must verify against its manifest"); for (corrupted, _) in PARTITIONED_FORMAT_SEGMENTS { let flip = |request: SealedGenerationSegmentReadV1<'_>, buffer: &mut Vec| { let hit = match &request { @@ -3523,8 +3381,7 @@ fn partitioned_codec_has_stable_bytes_and_round_trips() { fn partitioned_text_metadata_exposes_commitments_without_payload_reads() { let (expected, manifest, _) = partitioned_codec_fixture(); let metadata = CodeIndexPublishedGenerationV1::partitioned_text_metadata(&manifest) - .expect("authenticated text metadata") - .expect("current partitioned manifest"); + .expect("authenticated text metadata"); assert_eq!( metadata .source_commitments() @@ -3548,85 +3405,6 @@ fn partitioned_text_metadata_exposes_commitments_without_payload_reads() { ); } -#[test] -fn partitioned_codec_reads_pre_paging_evidence_descriptor() { - let (expected, manifest, segments) = partitioned_codec_fixture(); - let mut envelope: serde_json::Value = - serde_json::from_slice(&manifest).expect("partitioned manifest JSON"); - let evidence = envelope["generation"]["generation_evidence"] - .as_object_mut() - .expect("generation evidence descriptor"); - let evidence_digest = evidence["segment_digest"] - .as_str() - .expect("generation evidence digest") - .to_owned(); - evidence - .remove("pages") - .expect("current descriptor carries evidence pages"); - let state_digest = sealed_generation_payload_digest( - SEALED_GENERATION_FORMAT_REVISION_V1, - &envelope["generation"], - ) - .expect("legacy payload digest"); - envelope["state_digest"] = serde_json::Value::String(state_digest.as_str().to_owned()); - let legacy_manifest = - serde_json::to_vec(&envelope).expect("pre-paging partitioned manifest JSON"); - let whole_evidence_reads = Cell::new(0_usize); - let ranged_evidence_reads = Cell::new(0_usize); - let largest_evidence_read = Cell::new(0_u64); - - let restored = CodeIndexPublishedGenerationV1::decode_partitioned_sealed( - &legacy_manifest, - |request, buffer| { - let (digest, offset, length) = match request { - SealedGenerationSegmentReadV1::Whole { digest, size_bytes } => { - if digest.as_str() == evidence_digest { - whole_evidence_reads.set(whole_evidence_reads.get() + 1); - } - (digest, 0, size_bytes) - } - SealedGenerationSegmentReadV1::Range { - digest, - offset, - length, - .. - } => { - if digest.as_str() == evidence_digest { - ranged_evidence_reads.set(ranged_evidence_reads.get() + 1); - largest_evidence_read.set(largest_evidence_read.get().max(length)); - } - (digest, offset, length) - } - }; - let bytes = segments.get(digest.as_str()).ok_or_else(|| { - CodeIndexProductionErrorV1::Contract("legacy segment is missing".to_owned()) - })?; - let start = usize::try_from(offset).expect("legacy segment offset"); - let end = start + usize::try_from(length).expect("legacy segment length"); - buffer.clear(); - buffer.extend_from_slice(&bytes[start..end]); - Ok(()) - }, - ) - .expect("pre-paging partitioned bytes decode") - .expect("current partitioned manifest"); - - // A pre-paging segment carries no page table, but it is still read in - // bounded ranges: restoring it must never materialize the whole segment. - assert_eq!(whole_evidence_reads.get(), 0); - assert!(ranged_evidence_reads.get() > 0); - assert!( - largest_evidence_read.get() <= 256 * 1024, - "a pre-paging evidence read must stay within one page: {} bytes", - largest_evidence_read.get() - ); - assert_eq!( - restored.encode_sealed().expect("restored generation seals"), - expected.encode_sealed().expect("expected generation seals"), - "legacy evidence must restore the same typed generation" - ); -} - /// Bytes the unmodified pre-paging writer emitted (see the fixture README and /// `provenance.json`), sealed at the retired manifest revision seven. That /// revision named two payload shapes, a manifest with its census and one @@ -3842,18 +3620,13 @@ fn a_retired_parent_manifest_yields_no_reuse_instead_of_refusing_the_child() { buffer.extend_from_slice(&bytes[start..end]); Ok(()) }; - assert!( - CodeIndexPublishedGenerationV1::verify_partitioned_sealed(&manifest, read) - .expect("the child's own segments authenticate"), - "a child encoded without reuse must be self-sufficient" - ); + CodeIndexPublishedGenerationV1::verify_partitioned_sealed(&manifest, read) + .expect("a child encoded without reuse must be self-sufficient"); assert_eq!( CodeIndexPublishedGenerationV1::decode_partitioned_sealed(&manifest, read) - .expect("the child decodes from its own segments") - .expect("current partitioned manifest") - .encode_sealed() - .expect("restored child seals"), - child.encode_sealed().expect("child seals"), + .map(|restored| PartitionedSealV1::of(&restored)) + .expect("the child decodes from its own segments"), + PartitionedSealV1::of(&child), "no reuse must restore the same typed generation" ); } @@ -3935,16 +3708,15 @@ fn prior_partitioned_symbol_occurrence_revision_reaches_the_typed_refusal() { } /// Both public descriptor readers share one layout validator, so every -/// malformed descriptor mutation must be refused by both, while the supported -/// historical unpaged descriptor is accepted by both. Only the outer -/// authentication differs: the full reader verifies the state digest itself, -/// the retention projection leaves that to its caller. +/// malformed descriptor mutation, including a descriptor without a page +/// table, must be refused by both. Only the outer authentication differs: +/// the full reader verifies the state digest itself, the retention +/// projection leaves that to its caller. #[test] fn partitioned_descriptor_readers_share_validation_without_sharing_authentication() { let (_, manifest, _) = partitioned_codec_fixture(); let authenticated = CodeIndexPublishedGenerationV1::partitioned_segment_identities(&manifest) - .expect("authenticate current manifest") - .expect("supported partitioned format"); + .expect("authenticate current manifest"); assert_eq!( CodeIndexPublishedGenerationV1::partitioned_segment_identities_from_reader( manifest.as_slice(), @@ -3954,33 +3726,25 @@ fn partitioned_descriptor_readers_share_validation_without_sharing_authenticatio ); let original: serde_json::Value = serde_json::from_slice(&manifest).expect("fixture envelope"); - // A missing page table is the supported pre-paging descriptor: both - // readers accept it and project the same segment identities as the paged - // current descriptor, because the evidence segment itself is unchanged. + // A missing page table is the retired pre-paging descriptor; a current + // manifest revision carrying it is malformed for both readers. let mut historical = original.clone(); historical["generation"]["generation_evidence"] .as_object_mut() .unwrap() .remove("pages") .expect("current descriptor carries evidence pages"); - let digest = sealed_generation_payload_digest( - SEALED_GENERATION_FORMAT_REVISION_V1, - &historical["generation"], - ) - .expect("historical payload digest"); - historical["state_digest"] = serde_json::json!(digest.as_str()); - let historical_bytes = serde_json::to_vec(&historical).expect("historical envelope"); - assert_eq!( - CodeIndexPublishedGenerationV1::partitioned_segment_identities(&historical_bytes) - .expect("full reader accepts the historical unpaged descriptor"), - Some(authenticated.clone()), + let historical_bytes = reseal_manifest(historical); + assert!( + CodeIndexPublishedGenerationV1::partitioned_segment_identities(&historical_bytes).is_err(), + "the full reader refuses an unpaged descriptor" ); - assert_eq!( + assert!( CodeIndexPublishedGenerationV1::partitioned_segment_identities_from_reader( historical_bytes.as_slice(), ) - .expect("retention reader accepts the historical unpaged descriptor"), - Some(authenticated.clone()), + .is_err(), + "the retention reader refuses an unpaged descriptor" ); for mutation in [ @@ -4037,11 +3801,7 @@ fn partitioned_descriptor_readers_share_validation_without_sharing_authenticatio } // Authenticate the mutation so refusal exercises descriptors rather // than being masked by the full reader's outer digest check. - let digest = - sealed_generation_payload_digest(SEALED_GENERATION_FORMAT_REVISION_V1, generation) - .expect("mutated payload digest"); - envelope["state_digest"] = serde_json::json!(digest.as_str()); - let bytes = serde_json::to_vec(&envelope).expect("mutated envelope"); + let bytes = reseal_manifest(envelope); assert!( CodeIndexPublishedGenerationV1::partitioned_segment_identities(&bytes).is_err(), "full reader accepted {mutation}" @@ -4106,16 +3866,7 @@ fn partitioned_encode_rewrites_file_segments_across_extractor_revisions() { expected_seal_digest(&historical_manifest).expect("historical manifest seal"); parent_envelope["generation"]["manifest"] = serde_json::to_value(historical_manifest).expect("historical manifest JSON"); - parent_envelope["state_digest"] = serde_json::to_value( - sealed_generation_payload_digest( - SEALED_GENERATION_FORMAT_REVISION_V1, - &parent_envelope["generation"], - ) - .expect("historical envelope digest"), - ) - .expect("historical digest JSON"); - let historical_parent = - serde_json::to_vec(&parent_envelope).expect("historical parent encoding"); + let historical_parent = reseal_manifest(parent_envelope); let child = owner .build_and_publish(partitioned_codec_request(2, 1_200_000), &ActiveControl) @@ -4216,12 +3967,10 @@ fn partitioned_encode_publishes_only_the_edited_file_segment() { let parent_identities = CodeIndexPublishedGenerationV1::partitioned_segment_identities(&parent_manifest) - .expect("parent identities parse") - .expect("current partitioned manifest"); + .expect("parent identities parse"); let child_identities = CodeIndexPublishedGenerationV1::partitioned_segment_identities(&child_manifest) - .expect("child identities parse") - .expect("current partitioned manifest"); + .expect("child identities parse"); let carried = child_identities .iter() .filter(|identity| { @@ -4282,35 +4031,30 @@ fn increment_wedge_request(edited_value: u64, sealed_at: i64) -> CodeIndexBuildR } /// Every clone body occurrence per file path, in the order the file carries -/// them, read from the generation's sealed JSON. +/// them, read back through the generation's sealed lexical page source. fn sealed_clone_bindings( generation: &CodeIndexPublishedGenerationV1, ) -> BTreeMap> { - let sealed = generation.encode_sealed().expect("generation seals"); - let envelope: serde_json::Value = serde_json::from_slice(&sealed).expect("sealed JSON"); - envelope["generation"]["files"] - .as_array() - .expect("sealed files") - .iter() - .map(|file| { - let bodies = file["artifacts"]["clone_bodies"] - .as_array() - .expect("clone bodies") - .iter() - .map(|body| { - serde_json::from_value(body["occurrence"].clone()) - .expect("sealed clone body occurrence") - }) - .collect(); - ( - file["authority"]["logical_path"] - .as_str() - .expect("file logical path") - .to_owned(), - bodies, - ) - }) - .collect() + let mut source = PartitionedSealV1::of(generation) + .lexical_source(64, 1 << 20) + .expect("sealed generation opens as a lexical source"); + let mut bindings: BTreeMap> = BTreeMap::new(); + loop { + match source + .next_page(&ActiveControl) + .expect("sealed generation pages") + { + VerifiedSealedLexicalPageReadV1::Page(page) => { + for body in page.clone_bodies() { + bindings + .entry(body.occurrence.path.clone()) + .or_default() + .push(body.occurrence.clone()); + } + } + VerifiedSealedLexicalPageReadV1::Complete(_) => return bindings, + } + } } /// The daemon publishes a one-file increment with parent-segment reuse and @@ -4361,12 +4105,10 @@ fn carried_forward_clone_bodies_admit_through_the_reused_sealed_segment() { ); let parent_identities = CodeIndexPublishedGenerationV1::partitioned_segment_identities(&parent_manifest) - .expect("parent identities parse") - .expect("current partitioned manifest"); + .expect("parent identities parse"); let child_identities = CodeIndexPublishedGenerationV1::partitioned_segment_identities(&child_manifest) - .expect("child identities parse") - .expect("current partitioned manifest"); + .expect("child identities parse"); let carried_from_parent = child_identities .iter() .filter(|identity| { @@ -4411,7 +4153,6 @@ fn carried_forward_clone_bodies_admit_through_the_reused_sealed_segment() { ); let segments = Arc::new(segments); let mut source = VerifiedSealedLexicalPageSourceV1::open_partitioned_sealed( - Cursor::new(Vec::::new()), &child_manifest, state_digest, move |digest, _, buffer, _control| { @@ -4425,8 +4166,7 @@ fn carried_forward_clone_bodies_admit_through_the_reused_sealed_segment() { 64, 1 << 20, ) - .expect("child manifest opens through the daemon's text projection path") - .expect("current partitioned manifest"); + .expect("child manifest opens through the daemon's text projection path"); let mut restored: BTreeMap> = BTreeMap::new(); let receipt = loop { @@ -4470,442 +4210,3 @@ fn carried_forward_clone_bodies_admit_through_the_reused_sealed_segment() { "a carried-forward file keeps its parent clone binding across generations" ); } - -// --------------------------------------------------------------------------- -// Peak-RSS bound for the pre-paging (legacy) generation restore. -// -// The corpus is deliberately small so the check runs with the rest of the -// suite; the bound it asserts is a ratio, so a larger `TD_LEGACY_RSS_FILES` -// only widens the margin. Run it with `--nocapture` to read the numbers. -// --------------------------------------------------------------------------- - -fn rss_proc_kib(field: &str) -> Option { - let status = std::fs::read_to_string("/proc/self/status").ok()?; - let prefix = format!("{field}:"); - status - .lines() - .find_map(|line| line.strip_prefix(prefix.as_str())) - .and_then(|value| value.split_whitespace().next()) - .and_then(|value| value.parse().ok()) -} - -fn rss_reset_peak() -> bool { - std::fs::write("/proc/self/clear_refs", b"5\n").is_ok() -} - -fn rss_scaled_request(file_count: usize) -> CodeIndexBuildRequestV1 { - let mut identity = Sha256::new(); - let mut files = Vec::with_capacity(file_count); - let mut captured_files = Vec::with_capacity(file_count); - let mut receipts = Vec::with_capacity(file_count); - for index in 0..file_count { - let logical_path = format!("src/generated/module_{index:05}.rs"); - // Distinct bytes per file: identical bodies would collapse into shared - // content addresses and understate the corpus the restore must carry. - let source = format!("{RUST_SOURCE}\npub const MODULE_INDEX_{index}: u32 = {index};\n"); - identity.update(logical_path.as_bytes()); - identity.update([0]); - identity.update(source.as_bytes()); - let file_occurrence_id = id::(&format!("file.rss.{index:05}")); - files.push(SanitizedCodeFileV1 { - file_occurrence_id: file_occurrence_id.clone(), - logical_path, - language: Some(id::("rust")), - content_digest: content_digest(source.as_bytes()), - disposition: SnapshotFileDispositionV1::Present, - }); - captured_files.push(CodeIndexCapturedFileV1 { - file_occurrence_id, - sanitized_bytes: Arc::from(source.as_bytes()), - sensitivity_level: tracedecay_domain::SensitivityLevelV1::Public, - }); - receipts.push(id::(&format!( - "receipt.rss.{index:05}" - ))); - } - CodeIndexBuildRequestV1 { - snapshot: SanitizedCodeSnapshotV1 { - repository: id::("repository.production"), - worktree: None, - reference: None, - source_revision: None, - sanitizer_revision: id::("sanitizer.v1"), - sanitization_receipts: receipts, - content_identity: content_digest(&identity.finalize()), - captured_at: UtcMicros(1_000_000), - files, - }, - captured_files, - changed_files: BTreeSet::new(), - invalidations: BTreeSet::new(), - ignored_source_admissions: Vec::new(), - repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { - tree: None, - dirty: RepositoryDirtyStateV1::Dirty, - }, - sealed_at: UtcMicros(1_100_000), - target_projection_key: projection_key(), - } -} - -/// What one decode of a generation observed: the shape of the segment reads -/// the restore issued, and the peak RSS it grew over a freshly reset high -/// water mark. -struct RssDecodeProbeV1 { - hwm_delta_kib: Option, - evidence_reads: usize, - evidence_read_whole: bool, - largest_evidence_read: u64, - largest_evidence_buffer: usize, -} - -/// Decode `manifest`, serving every segment read from `segments`, and report -/// both the read shape and the peak RSS growth. -fn rss_measure_decode( - label: &str, - manifest: &[u8], - segments: &BTreeMap>, - evidence_digest: &str, - measure_rss: bool, -) -> RssDecodeProbeV1 { - let hwm_before = measure_rss.then(|| { - assert!(rss_reset_peak(), "reset VmHWM"); - rss_proc_kib("VmHWM").expect("VmHWM") - }); - let mut whole_reads = 0_usize; - let mut ranged_reads = 0_usize; - let mut probe = RssDecodeProbeV1 { - hwm_delta_kib: None, - evidence_reads: 0, - evidence_read_whole: false, - largest_evidence_read: 0, - largest_evidence_buffer: 0, - }; - let restored = - CodeIndexPublishedGenerationV1::decode_partitioned_sealed(manifest, |request, buffer| { - let (digest, offset, length, whole) = match request { - SealedGenerationSegmentReadV1::Whole { digest, size_bytes } => { - whole_reads += 1; - (digest, 0, size_bytes, true) - } - SealedGenerationSegmentReadV1::Range { - digest, - offset, - length, - .. - } => { - ranged_reads += 1; - (digest, offset, length, false) - } - }; - let evidence = digest.as_str() == evidence_digest; - if evidence { - probe.evidence_reads += 1; - probe.evidence_read_whole |= whole; - probe.largest_evidence_read = probe.largest_evidence_read.max(length); - } - let bytes = segments.get(digest.as_str()).ok_or_else(|| { - CodeIndexProductionErrorV1::Contract("measured segment is missing".to_owned()) - })?; - let start = usize::try_from(offset).expect("segment offset"); - let end = start + usize::try_from(length).expect("segment length"); - buffer.clear(); - buffer.extend_from_slice(&bytes[start..end]); - if evidence { - // The restore owns this buffer and reuses it across reads, so - // its capacity is the segment bytes the restore holds at once. - probe.largest_evidence_buffer = - probe.largest_evidence_buffer.max(buffer.capacity()); - } - Ok(()) - }) - .expect("measured manifest decodes") - .expect("measured manifest is the current partitioned revision"); - let hwm_after = hwm_before.map(|_| rss_proc_kib("VmHWM").expect("VmHWM")); - let file_count = restored.snapshot().files.len(); - drop(restored); - probe.hwm_delta_kib = hwm_before - .zip(hwm_after) - .map(|(before, after)| after.saturating_sub(before)); - println!( - "rss_probe form={label} files={file_count} whole_reads={whole_reads} \ -ranged_reads={ranged_reads} evidence_reads={} evidence_read_whole={} \ -largest_evidence_read={} largest_evidence_buffer={} hwm_before_kib={hwm_before:?} \ -hwm_after_kib={hwm_after:?} hwm_delta_kib={:?}", - probe.evidence_reads, - probe.evidence_read_whole, - probe.largest_evidence_read, - probe.largest_evidence_buffer, - probe.hwm_delta_kib, - ); - probe -} - -/// The same generation encoded both ways, with the segments both forms read. -struct LegacyRssFixtureV1 { - paged_manifest: Vec, - legacy_manifest: Vec, - segments: BTreeMap>, - evidence_digest: String, - evidence_bytes: usize, - generation_bytes: usize, - file_count: usize, -} - -/// Publish one generation, then rewrite its descriptor into the pre-paging -/// shape a historical writer emitted: one whole authenticated evidence -/// segment, no page table. -fn legacy_rss_fixture(file_count: usize) -> LegacyRssFixtureV1 { - let store = SharedPublicationStore::default(); - let mut owner = CodeIndexProductionOwnerV1::new(config(), store, ApplyingProjectionSink) - .expect("rss fixture owner"); - let generation = owner - .build_and_publish(rss_scaled_request(file_count), &ActiveControl) - .expect("rss fixture generation"); - - let mut segments: BTreeMap> = BTreeMap::new(); - let mut evidence_pack = Vec::new(); - let mut evidence_digest = String::new(); - let paged_manifest = generation - .encode_partitioned_sealed(|publication| { - match publication { - SealedGenerationSegmentPublicationV1::File { digest, bytes } => { - segments.insert(digest.as_str().to_owned(), bytes.to_vec()); - } - SealedGenerationSegmentPublicationV1::GenerationEvidencePage { bytes, .. } => { - evidence_pack.extend_from_slice(bytes); - } - SealedGenerationSegmentPublicationV1::GenerationEvidenceCommit { - segment_digest, - .. - } => { - evidence_digest = segment_digest.as_str().to_owned(); - segments.insert( - segment_digest.as_str().to_owned(), - std::mem::take(&mut evidence_pack), - ); - } - } - Ok(()) - }) - .expect("rss fixture encodes"); - - let mut envelope: serde_json::Value = - serde_json::from_slice(&paged_manifest).expect("manifest JSON"); - envelope["generation"]["generation_evidence"] - .as_object_mut() - .expect("evidence descriptor") - .remove("pages") - .expect("current descriptor carries evidence pages"); - let state_digest = sealed_generation_payload_digest( - SEALED_GENERATION_FORMAT_REVISION_V1, - &envelope["generation"], - ) - .expect("legacy payload digest"); - envelope["state_digest"] = serde_json::Value::String(state_digest.as_str().to_owned()); - let legacy_manifest = serde_json::to_vec(&envelope).expect("legacy manifest JSON"); - drop(envelope); - - let segment_bytes: usize = segments.values().map(Vec::len).sum(); - let evidence_bytes = segments - .get(evidence_digest.as_str()) - .map(Vec::len) - .expect("evidence segment"); - let generation_bytes = segment_bytes + paged_manifest.len(); - drop(generation); - drop(owner); - - LegacyRssFixtureV1 { - paged_manifest, - legacy_manifest, - segments, - evidence_digest, - evidence_bytes, - generation_bytes, - file_count, - } -} - -/// Restoring a pre-paging generation must not materialize its evidence -/// segment. -/// -/// The shipped restore read the whole segment, parsed a `serde_json::Value` -/// from it, rewrote identities in that tree and deserialized the tree again: -/// peak memory was 2.35x the on-disk generation and grew with the corpus. -/// -/// The proof is the read shape the restore asks the caller for, not its -/// memory footprint. A restore that materializes the segment has to hold it, -/// so it must ask for the whole segment in one read or for a range that grows -/// with the segment - the restore states its own peak segment residency in the -/// requests it issues. The paged form of the identical generation is the -/// reference: the pre-paging form must ask for the same bounded ranges into -/// the same bounded buffer. That is exact, needs no memory reading, and holds -/// on every platform. -/// -/// Peak RSS follows only as a loose ceiling on the whole restore, and it is -/// deliberately not the proof. VmHWM cannot see an allocation that fits inside -/// the heap the restored generation already made resident, so it does not -/// detect retention on its own: injecting a copy of every evidence page into a -/// buffer held for the decode moves it by a fifth of the segment or less, well -/// inside the clean band. What it does see is additive noise - a trimmed -/// allocator makes the next probe fault fresh pages, which on a loaded runner -/// cost more than the segment under test - so each form is measured over -/// several alternating rounds and the smallest growth is taken as its cost. -/// A restore that materializes pays that cost on every round, so the minimum -/// keeps whatever signal RSS carries and drops the noise that made a single -/// pair of probes flaky. -#[test] -fn legacy_generation_restore_does_not_materialize_its_evidence_segment() { - const RSS_CHILD: &str = "TD_LEGACY_RSS_CHILD"; - const RSS_TEST: &str = concat!( - "production_orchestration::", - "legacy_generation_restore_does_not_materialize_its_evidence_segment" - ); - /// Alternating paged/legacy rounds behind the discarded warm-up. - const RSS_ROUNDS: usize = 4; - - // VmHWM and `clear_refs` are Linux-only and mandatory there. The - // read-shape guard needs neither, so run it alone on other platforms. - let measure_rss = cfg!(target_os = "linux"); - - // VmHWM is process-wide, so the reading only means anything while nothing - // else is allocating: take it in a child that runs this test alone. - if measure_rss && std::env::var_os(RSS_CHILD).is_none() { - let status = std::process::Command::new(std::env::current_exe().expect("test binary")) - .args([RSS_TEST, "--exact", "--nocapture", "--test-threads=1"]) - .env(RSS_CHILD, "1") - .status() - .expect("run the peak-RSS measurement alone"); - assert!( - status.success(), - "the isolated restore measurement failed; its own failure is above" - ); - return; - } - - let file_count: usize = std::env::var("TD_LEGACY_RSS_FILES") - .ok() - .and_then(|value| value.parse().ok()) - .unwrap_or(300); - let fixture = legacy_rss_fixture(file_count); - - let paged = rss_measure_decode( - "paged", - &fixture.paged_manifest, - &fixture.segments, - &fixture.evidence_digest, - measure_rss, - ); - let legacy = rss_measure_decode( - "legacy", - &fixture.legacy_manifest, - &fixture.segments, - &fixture.evidence_digest, - measure_rss, - ); - - // --- Guard 1: the read shape, exact. ------------------------------------ - assert!( - !paged.evidence_read_whole && paged.evidence_reads > 1, - "the paged control must itself page the evidence segment: \ - evidence_reads={} evidence_read_whole={}", - paged.evidence_reads, - paged.evidence_read_whole - ); - assert!( - !legacy.evidence_read_whole, - "restoring a pre-paging generation asked for its whole \ - {}-byte evidence segment in one read: the segment is being materialized", - fixture.evidence_bytes - ); - assert!( - legacy.largest_evidence_read <= paged.largest_evidence_read, - "restoring a pre-paging generation read up to {} bytes of its \ - {}-byte evidence segment at once, beyond the {}-byte bound the paged \ - restore of the same generation holds to", - legacy.largest_evidence_read, - fixture.evidence_bytes, - paged.largest_evidence_read - ); - assert!( - legacy.largest_evidence_buffer <= paged.largest_evidence_buffer, - "restoring a pre-paging generation held a {}-byte evidence buffer, \ - beyond the {}-byte buffer the paged restore of the same generation \ - holds to: the segment is being materialized", - legacy.largest_evidence_buffer, - paged.largest_evidence_buffer - ); - assert_eq!( - legacy.evidence_reads, paged.evidence_reads, - "the pre-paging restore must read the evidence segment in the same \ - bounded chunks the page table would have named" - ); - - // --- Guard 2: peak RSS, noise-tolerant. --------------------------------- - if !measure_rss { - return; - } - // The first restore in a process pays a cold-start cost (the arena a - // restored generation needs) that has nothing to do with the form being - // restored; the two probes above spent it. Alternate from here so neither - // form is systematically the one that inherits a trimmed allocator. - let mut paged_hwm = u64::MAX; - let mut legacy_hwm = u64::MAX; - for _ in 0..RSS_ROUNDS { - paged_hwm = paged_hwm.min( - rss_measure_decode( - "paged", - &fixture.paged_manifest, - &fixture.segments, - &fixture.evidence_digest, - measure_rss, - ) - .hwm_delta_kib - .expect("Linux VmHWM measurement"), - ); - legacy_hwm = legacy_hwm.min( - rss_measure_decode( - "legacy", - &fixture.legacy_manifest, - &fixture.segments, - &fixture.evidence_digest, - measure_rss, - ) - .hwm_delta_kib - .expect("Linux VmHWM measurement"), - ); - } - // The paged decode of the same generation is the control, not a warm-up: - // both forms restore the identical generation, so only the difference is - // the pre-paging path's own cost. An absolute peak is not a usable - // measure, it is dominated by whether the allocator returned the control - // decode's pages to the OS between the two probes, which a developer box - // and a CI runner answer differently by more than the segment under test. - let legacy_extra_bytes = legacy_hwm.saturating_sub(paged_hwm) * 1024; - - println!( - "rss_summary files={} generation_on_disk_bytes={} \ -evidence_segment_bytes={} rounds={RSS_ROUNDS} paged_hwm_delta_kib={paged_hwm} \ -legacy_hwm_delta_kib={legacy_hwm} legacy_extra_bytes={legacy_extra_bytes} \ -legacy_extra_over_evidence={:.3}", - fixture.file_count, - fixture.generation_bytes, - fixture.evidence_bytes, - legacy_extra_bytes as f64 / fixture.evidence_bytes as f64, - ); - - // The read-shape guard above already refused a restore that holds the - // segment. This is the ceiling on the rest of the restore: the pre-paging - // path must not cost a segment's worth of anything over the paged path. - // The bound is the segment because that is the size the guard is about, - // not a threshold tuned to the noise - the minimum over the rounds is - // what removes the noise. - assert!( - legacy_extra_bytes < fixture.evidence_bytes as u64, - "restoring a pre-paging generation cost {legacy_extra_bytes} bytes of peak RSS beyond the \ - paged restore of the same {}-byte generation, which is not far below its \ - {}-byte evidence segment: the segment is being materialized", - fixture.generation_bytes, - fixture.evidence_bytes - ); -} diff --git a/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration/parallel_equivalence.rs b/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration/parallel_equivalence.rs index b86c406eaa..9712181d48 100644 --- a/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration/parallel_equivalence.rs +++ b/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration/parallel_equivalence.rs @@ -17,7 +17,7 @@ use tracedecay_domain::{ use super::{ ActiveControl, ApplyingProjectionSink, SharedPublicationStore, config, projection_key, }; -use crate::support::{RUST_SOURCE, id}; +use crate::support::{PartitionedSealV1, RUST_SOURCE, id}; /// One module's source: a body whose size varies with `index`, so per-file /// parse and chunk cost varies widely, plus a uniquely named helper and an @@ -115,10 +115,18 @@ fn at_width( read } -fn sealed_bytes_at_width(width: usize, file_count: usize) -> Vec { - at_width(width, file_count, |generation| { - generation.encode_sealed().expect("sealed encoding") - }) +/// Every sealed byte of `generation`: its partitioned manifest followed by +/// each segment in content-address order. +fn sealed_bytes(seal: &PartitionedSealV1) -> Vec { + let mut bytes = seal.manifest.clone(); + for segment in seal.segments.values() { + bytes.extend_from_slice(segment); + } + bytes +} + +fn seal_at_width(width: usize, file_count: usize) -> PartitionedSealV1 { + at_width(width, file_count, PartitionedSealV1::of) } /// Every cross-file edge sealing bound, in the generation's own edge order. @@ -171,8 +179,11 @@ pub(super) fn assert_cross_file_resolution_is_width_invariant() { pub(super) fn assert_parallel_and_sequential_generations_are_byte_identical() { const FILES: usize = 64; - let sequential = sealed_bytes_at_width(1, FILES); - let parallel = sealed_bytes_at_width(parallelism::indexing_worker_target(64), FILES); + let sequential = sealed_bytes(&seal_at_width(1, FILES)); + let parallel = sealed_bytes(&seal_at_width( + parallelism::indexing_worker_target(64), + FILES, + )); assert_eq!( sequential.len(), @@ -205,10 +216,9 @@ struct DecodedCensus { snapshot_files: usize, } -fn decode_at_width(width: usize, sealed: &[u8]) -> (Vec, DecodedCensus) { +fn decode_at_width(width: usize, sealed: &PartitionedSealV1) -> (Vec, DecodedCensus) { parallelism::force_indexing_workers_for_test(width); - let generation = - CodeIndexPublishedGenerationV1::decode_sealed(sealed).expect("sealed generation decodes"); + let generation = sealed.restored(); let census = DecodedCensus { generation_id: generation.manifest().generation_id.as_str().to_owned(), state_digest: generation @@ -227,7 +237,7 @@ fn decode_at_width(width: usize, sealed: &[u8]) -> (Vec, DecodedCensus) { // Re-encoding is canonical, so identical re-encoded bytes prove the whole // decoded state, every restored row, in order, is identical, not just // the fields the census names. - let reencoded = generation.encode_sealed().expect("sealed re-encoding"); + let reencoded = sealed_bytes(&PartitionedSealV1::of(&generation)); parallelism::clear_forced_indexing_workers_for_test(); (reencoded, census) } @@ -239,7 +249,7 @@ fn decode_at_width(width: usize, sealed: &[u8]) -> (Vec, DecodedCensus) { pub(super) fn assert_parallel_and_sequential_decodes_are_byte_identical() { const FILES: usize = 64; - let sealed = sealed_bytes_at_width(1, FILES); + let sealed = seal_at_width(1, FILES); let (sequential_bytes, sequential_census) = decode_at_width(1, &sealed); let (parallel_bytes, parallel_census) = @@ -256,7 +266,7 @@ pub(super) fn assert_parallel_and_sequential_decodes_are_byte_identical() { // A width-1 decode must reproduce the exact bytes it was handed, so the // sequential path is pinned to the seal itself and not merely to itself. assert!( - sequential_bytes == sealed, + sequential_bytes == sealed_bytes(&sealed), "width-1 decode did not round-trip the sealed bytes" ); assert_eq!( diff --git a/crates/tracedecay-code-index/tests/code_index_suite/retained_parse.rs b/crates/tracedecay-code-index/tests/code_index_suite/retained_parse.rs index 5faaaac070..78bffcf696 100644 --- a/crates/tracedecay-code-index/tests/code_index_suite/retained_parse.rs +++ b/crates/tracedecay-code-index/tests/code_index_suite/retained_parse.rs @@ -152,7 +152,7 @@ fn incremental_extraction_matches_cold_canonical_rows_and_visits_only_changed_ru assert!(incremental.metrics.visited_bytes < after.len()); assert_eq!( normalize_extraction(incremental.result), - normalize_extraction(extractor.extract("src/lib.rs", after)) + normalize_extraction(extractor.extract_artifact("src/lib.rs", after).result) ); assert_eq!( initial.disposition, @@ -211,7 +211,7 @@ fn composite_source_masking_preserves_incremental_astro_canonical_rows() { ); assert_eq!( normalize_extraction(incremental.result), - normalize_extraction(extractor.extract("src/page.astro", after)) + normalize_extraction(extractor.extract_artifact("src/page.astro", after).result) ); } diff --git a/crates/tracedecay-code-index/tests/code_index_suite/sealed_generation_restore.rs b/crates/tracedecay-code-index/tests/code_index_suite/sealed_generation_restore.rs index 46fe74fdcd..04d5c541c2 100644 --- a/crates/tracedecay-code-index/tests/code_index_suite/sealed_generation_restore.rs +++ b/crates/tracedecay-code-index/tests/code_index_suite/sealed_generation_restore.rs @@ -1,5 +1,5 @@ //! Sealed-generation restore contracts on a real multi-file corpus: decode -//! determinism across indexing widths, corrupt-payload rejection, and the +//! determinism across indexing widths, corrupt-manifest rejection, and the //! sealed format-revision gate. use std::sync::Arc; @@ -10,10 +10,8 @@ use tracedecay_code_index::parallelism::{ clear_forced_indexing_workers_for_test, force_indexing_workers_for_test, }; use tracedecay_code_index::production::{ - CodeIndexBuildRequestV1, CodeIndexCapturedFileV1, CodeIndexProductionOwnerV1, - CodeIndexPublishedGenerationV1, MINIMUM_SEALED_GENERATION_FORMAT_REVISION, - SEALED_GENERATION_FORMAT_REVISION_V1, UninterruptibleCodeIndexControlV1, - sealed_generation_payload_digest, + CodeIndexBuildRequestV1, CodeIndexCapturedFileV1, CodeIndexProductionErrorV1, + CodeIndexProductionOwnerV1, SEALED_GENERATION_FORMAT_REVISION_V1, }; use tracedecay_domain::{ FileOccurrenceId, LanguageId, SanitizedCodeFileV1, SensitivityLevelV1, @@ -23,7 +21,7 @@ use tracedecay_domain::{ use crate::production_orchestration::{ ActiveControl, ApplyingProjectionSink, SharedPublicationStore, config, request_with_source, }; -use crate::support::id; +use crate::support::{PartitionedSealV1, id, reseal_manifest}; fn add_present_typescript_file( request: &mut CodeIndexBuildRequestV1, @@ -48,7 +46,7 @@ fn add_present_typescript_file( request.changed_files.insert(logical_path.to_owned()); } -fn sealed_multi_file_generation() -> Vec { +fn sealed_multi_file_generation() -> PartitionedSealV1 { let mut request = request_with_source( "file.sealed-restore.root", 1_800_000, @@ -95,11 +93,11 @@ fn sealed_multi_file_generation() -> Vec { ApplyingProjectionSink, ) .expect("production owner"); - owner - .build_and_publish(request, &ActiveControl) - .expect("multi-file generation publishes") - .encode_sealed() - .expect("multi-file generation seals") + PartitionedSealV1::of( + &owner + .build_and_publish(request, &ActiveControl) + .expect("multi-file generation publishes"), + ) } /// Clears the forced width even when the guarded decode panics, so a failing @@ -121,151 +119,75 @@ impl Drop for ForcedSerialWidth { /// Restore fans per-file authority reconstruction across the indexing pool. /// Width is sizing policy, never semantics: a width-one and a full-width -/// restore of the same sealed bytes must re-encode to the identical envelope. +/// restore of the same sealed bytes must re-encode to the identical manifest. #[test] fn sealed_restore_reencodes_identically_at_serial_and_parallel_widths() { let sealed = sealed_multi_file_generation(); let serial = { let _width = ForcedSerialWidth::install(); - CodeIndexPublishedGenerationV1::decode_sealed(&sealed).expect("width-one restore") + sealed.restored() }; - let parallel = - CodeIndexPublishedGenerationV1::decode_sealed(&sealed).expect("full-width restore"); + let parallel = sealed.restored(); - assert_eq!( - serial - .encode_sealed() - .expect("width-one restored generation seals"), - sealed - ); - assert_eq!( - parallel - .encode_sealed() - .expect("full-width restored generation seals"), - sealed - ); + assert_eq!(PartitionedSealV1::of(&serial).manifest, sealed.manifest); + assert_eq!(PartitionedSealV1::of(¶llel).manifest, sealed.manifest); } -/// Streaming seat from a seekable reader must keep the same envelope bytes -/// as the in-memory decode: the digest proof and per-file restore cannot -/// change generation identity. #[test] -fn sealed_seek_reader_restore_reencodes_identically() { +fn sealed_restore_rejects_one_corrupt_manifest_byte() { let sealed = sealed_multi_file_generation(); - let admitted = u64::try_from(sealed.len()).expect("sealed length fits u64"); - let restored = CodeIndexPublishedGenerationV1::decode_sealed_seek_reader( - std::io::Cursor::new(sealed.as_slice()), - admitted, - None, - &UninterruptibleCodeIndexControlV1, - ) - .expect("seek restore") - .expect("compatible revision"); - assert_eq!( - restored - .encode_sealed() - .expect("seek-restored generation seals"), - sealed - ); -} - -/// `unresolved_references` was added to the per-file artifact after revision -/// six had already been persisted. Those earlier records mean exactly "no -/// retained cross-file reference candidates"; restoring them must preserve -/// that meaning so the scheduler can observe the old chunker revision and -/// build a current successor rather than retrying a decode failure forever. -#[test] -fn sealed_restore_defaults_absent_unresolved_references() { - let sealed = sealed_multi_file_generation(); - let mut envelope: Value = serde_json::from_slice(&sealed).expect("sealed envelope JSON"); - let files = envelope["generation"]["files"] - .as_array_mut() - .expect("sealed generation files"); - for file in files { - file["artifacts"] - .as_object_mut() - .expect("file artifacts") - .remove("unresolved_references"); - } - let state_digest = sealed_generation_payload_digest( - MINIMUM_SEALED_GENERATION_FORMAT_REVISION, - &envelope["generation"], - ) - .expect("compatible generation digest"); - envelope["state_digest"] = Value::String(state_digest.as_str().to_owned()); - let historical = serde_json::to_vec(&envelope).expect("historical sealed generation"); - - let restored = CodeIndexPublishedGenerationV1::decode_sealed(&historical) - .expect("current records without the additive field restore"); - let restored: Value = serde_json::from_slice( - &restored - .encode_sealed() - .expect("restored generation reseals"), - ) - .expect("restored envelope JSON"); - assert!( - restored["generation"]["files"] - .as_array() - .expect("restored files") - .iter() - .all(|file| file["artifacts"]["unresolved_references"] - .as_array() - .is_some_and(Vec::is_empty)), - "historical files must restore with an explicit empty unresolved-reference authority" - ); -} - -#[test] -fn sealed_restore_rejects_one_corrupt_payload_byte() { - let mut sealed = sealed_multi_file_generation(); - let position = sealed + let mut manifest = sealed.manifest.clone(); + let position = manifest .windows(5) .position(|window| window == b"gamma") - .expect("the sealed payload carries the fixture symbol"); - sealed[position] = b'q'; + .expect("the sealed manifest carries the fixture path"); + manifest[position] = b'q'; - let error = CodeIndexPublishedGenerationV1::decode_sealed(&sealed) - .expect_err("a corrupt payload byte must be rejected"); + let error = sealed + .restore(&manifest) + .expect_err("a corrupt manifest byte must be rejected"); assert!( error.to_string().contains("state digest does not match"), - "corrupt payload reached the wrong rejection: {error}" + "corrupt manifest reached the wrong rejection: {error}" ); } #[test] fn sealed_restore_refuses_superseded_and_adjacent_revisions() { let sealed = sealed_multi_file_generation(); - let envelope: Value = serde_json::from_slice(&sealed).expect("sealed envelope JSON"); - - // Below the minimum: refused with the typed rebuild error, so the daemon - // rebuilds the generation instead of decoding a retired envelope shape. - let mut superseded = envelope.clone(); - superseded["generation"]["format_revision"] = - Value::from(MINIMUM_SEALED_GENERATION_FORMAT_REVISION - 1); - let superseded = serde_json::to_vec(&superseded).expect("superseded sealed-generation JSON"); - for error in [ - CodeIndexPublishedGenerationV1::decode_sealed_if_compatible(&superseded).err(), - CodeIndexPublishedGenerationV1::decode_sealed(&superseded).err(), - ] { - let error = error.expect("a superseded revision must be refused"); + let envelope = sealed.envelope(); + + // Every retired revision, the monolithic envelope included, is refused + // with the typed rebuild error, so the daemon rebuilds the generation + // instead of decoding a retired shape. + for retired in [9, SEALED_GENERATION_FORMAT_REVISION_V1 - 1] { + let mut superseded = envelope.clone(); + superseded["generation"]["format_revision"] = Value::from(retired); + let error = sealed + .restore(&reseal_manifest(superseded)) + .expect_err("a superseded revision must be refused"); assert!( - error.to_string().contains("will be rebuilt from source"), + matches!( + error, + CodeIndexProductionErrorV1::SupersededSealedGenerationRevision(revision) + if revision == retired + ), "superseded revision reached the wrong rejection: {error}" ); + assert!(error.to_string().contains("will be rebuilt from source")); } - // Above every revision this build knows: abstain, then refuse. + // Above every revision this build knows: refused as incompatible. let mut incompatible = envelope; incompatible["generation"]["format_revision"] = Value::from(SEALED_GENERATION_FORMAT_REVISION_V1 + 1); - let incompatible = - serde_json::to_vec(&incompatible).expect("incompatible sealed-generation JSON"); - assert!(matches!( - CodeIndexPublishedGenerationV1::decode_sealed_if_compatible(&incompatible), - Ok(None) - )); - CodeIndexPublishedGenerationV1::decode_sealed(&incompatible) + let error = sealed + .restore(&reseal_manifest(incompatible)) .expect_err("adjacent sealed-generation revisions are incompatible"); + assert!( + error.to_string().contains("incompatible"), + "adjacent revision reached the wrong rejection: {error}" + ); } diff --git a/crates/tracedecay-code-index/tests/code_index_suite/search_chunks.rs b/crates/tracedecay-code-index/tests/code_index_suite/search_chunks.rs index 583784884b..aab5288a7c 100644 --- a/crates/tracedecay-code-index/tests/code_index_suite/search_chunks.rs +++ b/crates/tracedecay-code-index/tests/code_index_suite/search_chunks.rs @@ -4,7 +4,7 @@ use tracedecay_code_index::capabilities::{ BaseCapabilityEmitter, BaseCapabilityValidator, capability_manifest_digest, expected_seal_digest, }; -use tracedecay_code_index::chunks::{CodeChunker, DeterministicCodeChunker}; +use tracedecay_code_index::chunks::DeterministicCodeChunker; use tracedecay_code_index::extract::{ LanguageExtractor, MAX_EXTRACTION_SOURCE_BYTES, NeverCancelled, TreeSitterExtractor, }; @@ -12,7 +12,8 @@ use tracedecay_code_index::languages::LanguageRegistry; use tracedecay_domain::{ ChunkerRevision, CodeGenerationManifestV1, CodeSearchChunkGrainV1, ComponentVersion, CoverageSummaryV1, ExactTechnicalTermKindV1, GenerationSealV1, MAX_CHUNK_TEXT_BYTES, - PrivacyDomainId, RepositoryId, SanitizationReceiptId, SanitizerRevision, UtcMicros, + PrivacyDomainId, RepositoryId, SanitizationReceiptId, SanitizerRevision, SensitivityLevelV1, + UtcMicros, }; use crate::support::{RUST_SOURCE, digest, id, registry, rust_descriptor, validated_rust_file}; @@ -31,14 +32,27 @@ fn extraction_to_chunks_is_deterministic_and_covers_all_grains() { id::("sanitizer.v1"), id("policy.v1"), id::("chunker.v1"), - tracedecay_code_extraction::LanguageRegistry::new(), ); let first = chunker - .chunk_file(&file, batch.batch(), &descriptor, &NeverCancelled) + .index_file_with_authority_from_extraction( + &file, + &batch, + &descriptor, + SensitivityLevelV1::Public, + &NeverCancelled, + ) + .map(|(artifacts, _)| artifacts.chunks) .expect("chunk source"); let second = chunker - .chunk_file(&file, batch.batch(), &descriptor, &NeverCancelled) + .index_file_with_authority_from_extraction( + &file, + &batch, + &descriptor, + SensitivityLevelV1::Public, + &NeverCancelled, + ) + .map(|(artifacts, _)| artifacts.chunks) .expect("chunk source again"); assert_eq!(first, second); @@ -81,9 +95,15 @@ fn partial_extraction_never_chunks_unsupported_tail_bytes() { id::("sanitizer.v1"), id("policy.v1"), id::("chunker.v1"), - tracedecay_code_extraction::LanguageRegistry::new(), ) - .chunk_file(&file, batch.batch(), &descriptor, &NeverCancelled) + .index_file_with_authority_from_extraction( + &file, + &batch, + &descriptor, + SensitivityLevelV1::Public, + &NeverCancelled, + ) + .map(|(artifacts, _)| artifacts.chunks) .expect("chunk bounded evidence"); assert!(matches!( @@ -121,9 +141,15 @@ fn exact_term_kinds_cover_the_supported_search_contract() { id::("sanitizer.v1"), id("policy.v1"), id::("chunker.v1"), - tracedecay_code_extraction::LanguageRegistry::new(), ) - .chunk_file(&file, batch.batch(), &descriptor, &NeverCancelled) + .index_file_with_authority_from_extraction( + &file, + &batch, + &descriptor, + SensitivityLevelV1::Public, + &NeverCancelled, + ) + .map(|(artifacts, _)| artifacts.chunks) .expect("chunk exact-term fixture"); let kinds: BTreeSet<_> = result .chunks @@ -183,14 +209,27 @@ fn oversized_symbol_bodies_use_bounded_deterministic_fallback_windows() { id::("sanitizer.v1"), id("policy.v1"), id::("chunker.v1"), - tracedecay_code_extraction::LanguageRegistry::new(), ); let first = chunker - .chunk_file(&file, batch.batch(), &descriptor, &NeverCancelled) + .index_file_with_authority_from_extraction( + &file, + &batch, + &descriptor, + SensitivityLevelV1::Public, + &NeverCancelled, + ) + .map(|(artifacts, _)| artifacts.chunks) .expect("chunk oversized body"); let second = chunker - .chunk_file(&file, batch.batch(), &descriptor, &NeverCancelled) + .index_file_with_authority_from_extraction( + &file, + &batch, + &descriptor, + SensitivityLevelV1::Public, + &NeverCancelled, + ) + .map(|(artifacts, _)| artifacts.chunks) .expect("chunk oversized body again"); let bodies: Vec<_> = first .chunks @@ -226,14 +265,27 @@ fn multiple_file_windows_have_unique_stable_ids_and_ordinals() { id::("sanitizer.v1"), id("policy.v1"), id::("chunker.v1"), - tracedecay_code_extraction::LanguageRegistry::new(), ); let first = chunker - .chunk_file(&file, batch.batch(), &descriptor, &NeverCancelled) + .index_file_with_authority_from_extraction( + &file, + &batch, + &descriptor, + SensitivityLevelV1::Public, + &NeverCancelled, + ) + .map(|(artifacts, _)| artifacts.chunks) .expect("chunk multiple file windows"); let second = chunker - .chunk_file(&file, batch.batch(), &descriptor, &NeverCancelled) + .index_file_with_authority_from_extraction( + &file, + &batch, + &descriptor, + SensitivityLevelV1::Public, + &NeverCancelled, + ) + .map(|(artifacts, _)| artifacts.chunks) .expect("replay multiple file windows"); let first_windows: Vec<_> = first .chunks @@ -314,7 +366,10 @@ fn base_capability_manifest_is_deterministic_and_candidate_authorized() { let privacy_domain = id::("privacy.fixture"); let mut generation = CodeGenerationManifestV1 { project_id: id("project.fixture"), - generation_id: id("generation.v1.aaaaaaaa.00000001"), + generation_id: id(&format!( + "generation.v1.aaaaaaaa.00000001.{}", + "d".repeat(64) + )), snapshot_digest: digest('a'), invalidation_digest: digest('d'), registry_revision: registry.registry_revision(), @@ -332,9 +387,6 @@ fn base_capability_manifest_is_deterministic_and_candidate_authorized() { planner: id::("planner.v1"), }, }; - generation.invalidation_digest = generation - .expected_legacy_invalidation_digest() - .expect("legacy invalidation digest computes"); generation.seal.expected_digest = expected_seal_digest(&generation).expect("seal digest computes"); @@ -363,9 +415,6 @@ fn base_capability_manifest_is_deterministic_and_candidate_authorized() { let mut mixed_registry = generation.clone(); mixed_registry.registry_revision = id("registry.other.v1"); - mixed_registry.invalidation_digest = mixed_registry - .expected_legacy_invalidation_digest() - .expect("mixed invalidation digest computes"); mixed_registry.seal.expected_digest = expected_seal_digest(&mixed_registry).expect("mixed manifest still seals"); assert_eq!( diff --git a/crates/tracedecay-code-index/tests/code_index_suite/support.rs b/crates/tracedecay-code-index/tests/code_index_suite/support.rs index 8bb913b670..432aebc4c3 100644 --- a/crates/tracedecay-code-index/tests/code_index_suite/support.rs +++ b/crates/tracedecay-code-index/tests/code_index_suite/support.rs @@ -1,10 +1,25 @@ +use std::collections::BTreeMap; +use std::io::{Read as _, Write as _}; +use std::sync::Arc; + +use flate2::Compression; +use flate2::read::DeflateDecoder; +use flate2::write::DeflateEncoder; + +use serde_json::Value; +use sha2::{Digest, Sha256}; use tracedecay_code_index::chunks::content_digest; use tracedecay_code_index::intake::{CodeIndexIntake, ReceiptBoundCodeFileV1, SanitizedCodeIntake}; use tracedecay_code_index::languages::{LanguageRegistry, StaticLanguageRegistry}; +use tracedecay_code_index::production::{ + CodeIndexProductionErrorV1, CodeIndexPublishedGenerationV1, + SealedGenerationSegmentPublicationV1, SealedGenerationSegmentReadV1, + VerifiedSealedLexicalPageSourceV1, +}; use tracedecay_domain::{ - CodeGenerationId, FileOccurrenceId, LanguageDescriptorV1, LanguageId, ProjectId, RepositoryId, - SanitizationReceiptId, SanitizedCodeFileV1, SanitizedCodeSnapshotV1, SanitizerRevision, - SnapshotFileDispositionV1, UtcMicros, ValidatedCodeFileV1, + CodeGenerationId, FileOccurrenceId, LanguageDescriptorV1, LanguageId, ManifestDigest, + ProjectId, RepositoryId, SanitizationReceiptId, SanitizedCodeFileV1, SanitizedCodeSnapshotV1, + SanitizerRevision, SnapshotFileDispositionV1, UtcMicros, ValidatedCodeFileV1, }; pub const RUST_SOURCE: &str = "//! Module documentation.\n\nuse std::collections::HashMap;\n\n/// Increment a value.\npub fn alpha(value: u32) -> u32 {\n value + 1\n}\n\npub struct Holder {\n map: HashMap,\n}\n\nimpl Holder {\n pub fn get(&self, key: u32) -> Option {\n self.map.get(&key).copied()\n }\n}\n\n// trailing window text\n"; @@ -63,3 +78,177 @@ pub fn validated_rust_file(source: &[u8]) -> ReceiptBoundCodeFileV1 { ) .expect("receipt-bound rust source") } + +/// A partitioned sealed generation held in memory: the manifest plus every +/// published segment under its digest, evidence pages assembled into their +/// pack under the commit digest. +#[derive(Debug, PartialEq, Eq)] +pub struct PartitionedSealV1 { + pub manifest: Vec, + pub segments: BTreeMap>, +} + +impl PartitionedSealV1 { + pub fn of(generation: &CodeIndexPublishedGenerationV1) -> Self { + let mut segments = BTreeMap::new(); + let mut evidence_pack = Vec::new(); + let manifest = generation + .encode_partitioned_sealed(|publication| { + match publication { + SealedGenerationSegmentPublicationV1::File { digest, bytes } => { + segments.insert(digest.as_str().to_owned(), bytes.to_vec()); + } + SealedGenerationSegmentPublicationV1::GenerationEvidencePage { + bytes, .. + } => evidence_pack.extend_from_slice(bytes), + SealedGenerationSegmentPublicationV1::GenerationEvidenceCommit { + segment_digest, + .. + } => { + segments.insert( + segment_digest.as_str().to_owned(), + std::mem::take(&mut evidence_pack), + ); + } + } + Ok(()) + }) + .expect("generation seals partitioned"); + Self { manifest, segments } + } + + /// Restore `manifest`, which may be a tampered reseal of this seal's own + /// manifest, against this seal's segments. + pub fn restore( + &self, + manifest: &[u8], + ) -> Result { + CodeIndexPublishedGenerationV1::decode_partitioned_sealed(manifest, |request, buffer| { + let (digest, offset, length) = match request { + SealedGenerationSegmentReadV1::Whole { digest, size_bytes } => { + (digest, 0, size_bytes) + } + SealedGenerationSegmentReadV1::Range { + digest, + offset, + length, + .. + } => (digest, offset, length), + }; + let bytes = self.segments.get(digest.as_str()).ok_or_else(|| { + CodeIndexProductionErrorV1::Contract("fixture segment is missing".to_owned()) + })?; + let start = usize::try_from(offset).expect("segment offset fits usize"); + let end = start + usize::try_from(length).expect("segment length fits usize"); + buffer.clear(); + buffer.extend_from_slice(&bytes[start..end]); + Ok(()) + }) + } + + pub fn restored(&self) -> CodeIndexPublishedGenerationV1 { + self.restore(&self.manifest) + .expect("partitioned generation restores") + } + + pub fn envelope(&self) -> Value { + serde_json::from_slice(&self.manifest).expect("partitioned manifest envelope") + } + + /// The manifest's content address, the lexical source state digest. + pub fn state_digest(&self) -> ManifestDigest { + ManifestDigest::from_sha256_bytes(&Sha256::digest(&self.manifest)) + .expect("manifest digest is canonical") + } + + /// Every sealed byte: the manifest plus each segment. + pub fn byte_len(&self) -> usize { + self.segments.values().map(Vec::len).sum::() + self.manifest.len() + } + + pub fn lexical_source( + &self, + maximum_page_chunks: usize, + maximum_page_bytes: usize, + ) -> Result { + let segments = Arc::new(self.segments.clone()); + VerifiedSealedLexicalPageSourceV1::open_partitioned_sealed( + &self.manifest, + self.state_digest(), + move |digest, _, buffer, _control| { + let bytes = segments.get(digest.as_str()).ok_or_else(|| { + CodeIndexProductionErrorV1::Contract("fixture segment is missing".to_owned()) + })?; + buffer.clear(); + buffer.extend_from_slice(bytes); + Ok(()) + }, + maximum_page_chunks, + maximum_page_bytes, + ) + } + + /// A copy of this seal whose file segment at `file_index` carries + /// `mutate` applied to its `file` payload, re-addressed in the manifest + /// and resealed, so a refusal exercises the file payload rather than a + /// content-address or state-digest check. + pub fn with_tampered_file_segment( + &self, + file_index: usize, + mutate: impl FnOnce(&mut Value), + ) -> Self { + let mut envelope = self.envelope(); + let descriptor = &mut envelope["generation"]["file_segments"][file_index]; + let digest = descriptor["segment_digest"] + .as_str() + .expect("file segment digest") + .to_owned(); + let mut segment = inflate_segment(&self.segments[&digest]); + mutate(&mut segment["file"]); + let canonical = serde_json::to_vec(&segment).expect("tampered segment serializes"); + let mut encoder = DeflateEncoder::new(Vec::new(), Compression::default()); + encoder + .write_all(&canonical) + .expect("tampered segment compresses"); + let bytes = encoder.finish().expect("tampered segment compresses"); + let tampered = ManifestDigest::from_sha256_bytes(&Sha256::digest(&bytes)) + .expect("tampered segment digest"); + descriptor["segment_digest"] = Value::String(tampered.as_str().to_owned()); + descriptor["segment_size_bytes"] = Value::from(bytes.len() as u64); + descriptor["decoded_size_bytes"] = Value::from(canonical.len() as u64); + let mut segments = self.segments.clone(); + segments.insert(tampered.as_str().to_owned(), bytes); + Self { + manifest: reseal_manifest(envelope), + segments, + } + } + + /// The decoded JSON `file` payload of the segment at `file_index`. + pub fn file_segment_payload(&self, file_index: usize) -> Value { + let envelope = self.envelope(); + let digest = envelope["generation"]["file_segments"][file_index]["segment_digest"] + .as_str() + .expect("file segment digest"); + inflate_segment(&self.segments[digest])["file"].take() + } +} + +fn inflate_segment(bytes: &[u8]) -> Value { + let mut canonical = Vec::new(); + DeflateDecoder::new(bytes) + .read_to_end(&mut canonical) + .expect("file segment inflates"); + serde_json::from_slice(&canonical).expect("file segment JSON") +} + +/// Re-authenticate a tampered partitioned manifest envelope so its refusal +/// exercises the payload rather than the outer state digest. +pub fn reseal_manifest(mut envelope: Value) -> Vec { + let generation = + serde_json::to_vec(&envelope["generation"]).expect("manifest generation serializes"); + let state_digest = ManifestDigest::from_sha256_bytes(&Sha256::digest(generation)) + .expect("manifest digest is canonical"); + envelope["state_digest"] = Value::String(state_digest.as_str().to_owned()); + serde_json::to_vec(&envelope).expect("resealed manifest serializes") +} diff --git a/crates/tracedecay-code-index/tests/code_index_suite/symbol_span_digest.rs b/crates/tracedecay-code-index/tests/code_index_suite/symbol_span_digest.rs index 9a4593c17a..36a2588d8a 100644 --- a/crates/tracedecay-code-index/tests/code_index_suite/symbol_span_digest.rs +++ b/crates/tracedecay-code-index/tests/code_index_suite/symbol_span_digest.rs @@ -3,7 +3,8 @@ use std::collections::BTreeMap; use tracedecay_code_index::chunks::DeterministicCodeChunker; use tracedecay_code_index::extract::{LanguageExtractor, NeverCancelled, TreeSitterExtractor}; use tracedecay_domain::{ - ChunkerRevision, ContentDigest, RepositoryId, SanitizerRevision, SourceSpan, SymbolOccurrenceId, + ChunkerRevision, ContentDigest, RepositoryId, SanitizerRevision, SensitivityLevelV1, + SourceSpan, SymbolOccurrenceId, }; use tracedecay_privacy::{CodeSourceShapeV1, sanitize_code_source_bytes}; @@ -47,10 +48,16 @@ fn published_symbol_digests_cover_their_recorded_source_spans() { id::("sanitizer.v1"), id("policy.v1"), id::("chunker.v1"), - tracedecay_code_extraction::LanguageRegistry::new(), ) - .index_file(&file, extraction.batch(), &descriptor, &NeverCancelled) - .expect("index Rust source"); + .index_file_with_authority_from_extraction( + &file, + &extraction, + &descriptor, + SensitivityLevelV1::Public, + &NeverCancelled, + ) + .expect("index Rust source") + .0; let mut published_spans = BTreeMap::::new(); for chunk in &artifacts.chunks.chunks { diff --git a/crates/tracedecay-configuration/Cargo.toml b/crates/tracedecay-configuration/Cargo.toml index e337ff9731..cc7fd88f1b 100644 --- a/crates/tracedecay-configuration/Cargo.toml +++ b/crates/tracedecay-configuration/Cargo.toml @@ -7,8 +7,13 @@ license = "MIT" description = "Transport-neutral configuration control plane and runtime pin surfaces" repository = "https://github.com/ScriptedAlchemy/tracedecay" +[features] +# In-process pin cache and setting publication for tests that drive +# configuration-gated runtime paths (for example LCM summarizer executables) +# without the composition root's cache. Production code is never gated by it. +test-helpers = [] + [dependencies] -glob = "0.3" hotpath.workspace = true serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/crates/tracedecay-configuration/src/config/mod.rs b/crates/tracedecay-configuration/src/config/mod.rs index f93675f95a..c2974dec30 100644 --- a/crates/tracedecay-configuration/src/config/mod.rs +++ b/crates/tracedecay-configuration/src/config/mod.rs @@ -24,68 +24,48 @@ use tracedecay_domain::configuration::{ DIAGNOSTICS_PREWARM_SETTING_KEY, INDEX_EXCLUDE_SETTING_KEY, INDEX_EXTRACT_DOCSTRINGS_SETTING_KEY, INDEX_GIT_IGNORE_SETTING_KEY, INDEX_INCLUDE_SETTING_KEY, INDEX_MAX_FILE_SIZE_SETTING_KEY, INDEX_NATIVE_GRAPH_ACTIVATION_SETTING_KEY, - INDEX_TRACK_CALL_SITES_SETTING_KEY, SYNC_AUTO_TRACK_PR_BRANCHES_SETTING_KEY, - SYNC_AUTO_TRACK_PR_POLL_SECS_SETTING_KEY, SettingKey, TELEMETRY_TIMINGS_SETTING_KEY, + INDEX_TRACK_CALL_SITES_SETTING_KEY, LCM_SUMMARIZER_EXECUTABLES_SETTING_KEY, + LcmSummarizerExecutablesV1, SYNC_AUTO_INIT_SETTING_KEY, + SYNC_AUTO_TRACK_PR_BRANCHES_SETTING_KEY, SYNC_AUTO_TRACK_PR_POLL_SECS_SETTING_KEY, + SYNC_AUTO_WATCH_SETTING_KEY, SYNC_BACKSTOP_INTERVAL_MINS_SETTING_KEY, + SYNC_BRANCH_GC_DAYS_SETTING_KEY, SYNC_FULL_SYNC_ESCALATION_FILES_SETTING_KEY, + SYNC_MAX_CONCURRENT_SYNCS_SETTING_KEY, SYNC_READ_COOLDOWN_SECS_SETTING_KEY, + SYNC_READ_REFRESH_SETTING_KEY, SYNC_SESSION_START_STALE_THRESHOLD_SECS_SETTING_KEY, + SYNC_SESSION_START_SYNC_SETTING_KEY, SYNC_WATCH_DEBOUNCE_MS_SETTING_KEY, + SYNC_WATCH_LINKED_WORKTREES_SETTING_KEY, SYNC_WATCH_MAX_DELAY_MS_SETTING_KEY, + SYNC_WATCH_MAX_PROJECTS_SETTING_KEY, SettingKey, TELEMETRY_TIMINGS_SETTING_KEY, }; use tracedecay_domain::errors::{Result, TraceDecayError}; use tracedecay_global_db::RegisteredGlobalDbLeaseV1; use tracedecay_global_db::configuration::contracts::ConfigurationCurrentStateV1; +use model::{RetentionConfig, SyncConfig, TelemetryConfig}; + +/// Settings decoded from one resolved configuration snapshot. #[derive(Debug, Clone, PartialEq)] pub struct RuntimeTraceDecayConfig { + /// Glob patterns for paths to index despite the default hidden-directory, + /// generated-directory, and gitignore filters. pub include: Vec, + /// Glob patterns for files to exclude during indexing. pub exclude: Vec, + /// Maximum file size in bytes; larger files are skipped. pub max_file_size: u64, pub extract_docstrings: bool, pub track_call_sites: bool, pub git_ignore: bool, + /// A cold `tracedecay_diagnostics` call prewarms in the background instead + /// of blocking on the dependency build. pub diagnostics_prewarm: bool, + /// Whether the persistent native code graph may activate. Disabling it + /// leaves exact and lexical retrieval available. pub native_graph_activation: bool, - pub sync: RuntimeSyncConfig, - pub telemetry: RuntimeTelemetryConfig, -} - -impl Default for RuntimeTraceDecayConfig { - fn default() -> Self { - Self { - include: Vec::new(), - exclude: Vec::new(), - max_file_size: 1_048_576, - extract_docstrings: true, - track_call_sites: true, - git_ignore: true, - diagnostics_prewarm: false, - native_graph_activation: true, - sync: RuntimeSyncConfig::default(), - telemetry: RuntimeTelemetryConfig::default(), - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct RuntimeSyncConfig { - pub auto_track_pr_branches: bool, - pub auto_track_pr_poll_secs: u64, -} - -impl Default for RuntimeSyncConfig { - fn default() -> Self { - Self { - auto_track_pr_branches: false, - auto_track_pr_poll_secs: 300, - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct RuntimeTelemetryConfig { - pub timings: bool, -} - -impl Default for RuntimeTelemetryConfig { - fn default() -> Self { - Self { timings: true } - } + /// The host CLIs on-demand LCM summarization may launch. Every provider + /// is unconfigured until an operator names its executable; the daemon + /// never resolves one from `PATH` or its environment. + pub lcm_summarizers: LcmSummarizerExecutablesV1, + pub sync: SyncConfig, + pub telemetry: TelemetryConfig, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -186,6 +166,11 @@ pub trait PinnedRuntimeConfigurationCachePort: Send + Sync { fn publish(&self, configuration: PinnedRuntimeConfiguration) -> Result<()>; fn cached_for_root(&self, project_root: &Path) -> Result; + + /// The pin published for an already-authoritative registered project, + /// for daemon work (session shards, background convergence) that has a + /// project identity but no route root. + fn cached_for_project(&self, project_id: &ProjectId) -> Result; } static PINNED_RUNTIME_CONFIGURATION_CACHE: OnceLock> = @@ -219,6 +204,19 @@ pub fn cached_pinned_runtime_configuration( pinned_runtime_configuration_cache()?.cached_for_root(project_root) } +/// The summarizer executables the daemon published for one registered +/// project. A missing cache or pin is a typed configuration error, not an +/// unconfigured provider: the caller decides whether that means "pending". +pub fn lcm_summarizer_executables_for_project( + project_id: &ProjectId, +) -> Result { + Ok(pinned_runtime_configuration_cache()? + .cached_for_project(project_id)? + .config() + .lcm_summarizers + .clone()) +} + /// Converts a complete typed snapshot into the runtime settings every /// configuration consumer shares. There are no defaults, file reads, or /// environment reads: an absent or mistyped required setting is an error. @@ -241,7 +239,34 @@ fn runtime_config_from_snapshot( snapshot, INDEX_NATIVE_GRAPH_ACTIVATION_SETTING_KEY, )?, - sync: RuntimeSyncConfig { + lcm_summarizers: required_lcm_summarizer_executables(snapshot)?, + sync: SyncConfig { + auto_watch: required_bool(snapshot, SYNC_AUTO_WATCH_SETTING_KEY)?, + watch_linked_worktrees: required_bool( + snapshot, + SYNC_WATCH_LINKED_WORKTREES_SETTING_KEY, + )?, + watch_debounce_ms: required_unsigned(snapshot, SYNC_WATCH_DEBOUNCE_MS_SETTING_KEY)?, + watch_max_delay_ms: required_unsigned(snapshot, SYNC_WATCH_MAX_DELAY_MS_SETTING_KEY)?, + watch_max_projects: required_usize(snapshot, SYNC_WATCH_MAX_PROJECTS_SETTING_KEY)?, + read_refresh: required_bool(snapshot, SYNC_READ_REFRESH_SETTING_KEY)?, + read_cooldown_secs: required_unsigned(snapshot, SYNC_READ_COOLDOWN_SECS_SETTING_KEY)?, + session_start_sync: required_bool(snapshot, SYNC_SESSION_START_SYNC_SETTING_KEY)?, + session_start_stale_threshold_secs: required_unsigned( + snapshot, + SYNC_SESSION_START_STALE_THRESHOLD_SECS_SETTING_KEY, + )?, + backstop_interval_mins: required_unsigned( + snapshot, + SYNC_BACKSTOP_INTERVAL_MINS_SETTING_KEY, + )?, + full_sync_escalation_files: required_usize( + snapshot, + SYNC_FULL_SYNC_ESCALATION_FILES_SETTING_KEY, + )?, + max_concurrent_syncs: required_usize(snapshot, SYNC_MAX_CONCURRENT_SYNCS_SETTING_KEY)?, + branch_gc_days: required_unsigned(snapshot, SYNC_BRANCH_GC_DAYS_SETTING_KEY)?, + auto_init: required_bool(snapshot, SYNC_AUTO_INIT_SETTING_KEY)?, auto_track_pr_branches: required_bool( snapshot, SYNC_AUTO_TRACK_PR_BRANCHES_SETTING_KEY, @@ -250,8 +275,11 @@ fn runtime_config_from_snapshot( snapshot, SYNC_AUTO_TRACK_PR_POLL_SECS_SETTING_KEY, )?, + // Retention is not a registered setting, so a snapshot cannot + // carry retention policy. + retention: RetentionConfig::default(), }, - telemetry: RuntimeTelemetryConfig { + telemetry: TelemetryConfig { timings: required_bool(snapshot, TELEMETRY_TIMINGS_SETTING_KEY)?, }, }) @@ -320,6 +348,18 @@ pub fn required_string_list( } } +fn required_lcm_summarizer_executables( + snapshot: &ConfigurationSnapshotV1, +) -> Result { + match required_setting(snapshot, LCM_SUMMARIZER_EXECUTABLES_SETTING_KEY)? { + ConfigurationValueV1::LcmSummarizerExecutables(value) => Ok(value.clone()), + value => Err(config_error(format!( + "resolved configuration setting '{LCM_SUMMARIZER_EXECUTABLES_SETTING_KEY}' has wrong type: expected lcm summarizer executables, got {:?}", + value.kind() + ))), + } +} + fn config_error(message: impl Into) -> TraceDecayError { TraceDecayError::Config { message: message.into(), @@ -334,7 +374,8 @@ mod tests { use tracedecay_domain::ProjectId; use tracedecay_domain::configuration::{ ConfigurationLayerIdV1, ConfigurationRevisionId, ConfigurationSnapshotV1, - ConfigurationValueV1, INDEX_MAX_FILE_SIZE_SETTING_KEY, SettingKey, + ConfigurationValueV1, INDEX_MAX_FILE_SIZE_SETTING_KEY, SYNC_WATCH_DEBOUNCE_MS_SETTING_KEY, + SettingKey, }; use tracedecay_domain::errors::TraceDecayError; @@ -394,6 +435,33 @@ mod tests { ); } + #[test] + fn pin_decodes_daemon_sync_settings_and_rejects_their_absence() { + let complete = resolved(BTreeMap::new()); + let pinned = + PinnedRuntimeConfiguration::new(target(), revision(), complete.clone()).unwrap(); + let key = SettingKey::new(SYNC_WATCH_DEBOUNCE_MS_SETTING_KEY).unwrap(); + assert_eq!( + complete.effective_values.get(&key), + Some(&ConfigurationValueV1::Unsigned( + pinned.config().sync.watch_debounce_ms + )) + ); + + let mut values = complete.effective_values.clone(); + let mut provenance = complete.provenance.clone(); + values.remove(&key); + provenance.remove(&key); + let incomplete = ConfigurationSnapshotV1::new(values, provenance).unwrap(); + let message = config_message( + PinnedRuntimeConfiguration::new(target(), revision(), incomplete).unwrap_err(), + ); + assert!( + message.contains(SYNC_WATCH_DEBOUNCE_MS_SETTING_KEY) && message.contains("missing"), + "{message}" + ); + } + #[test] fn pin_rejects_a_required_setting_with_the_wrong_type() { let complete = resolved(BTreeMap::new()); diff --git a/crates/tracedecay-configuration/src/config/model.rs b/crates/tracedecay-configuration/src/config/model.rs index 79cd25a00b..d0be383f88 100644 --- a/crates/tracedecay-configuration/src/config/model.rs +++ b/crates/tracedecay-configuration/src/config/model.rs @@ -1,8 +1,6 @@ -//! Legacy `config.json` model, defaults, validation, and path policy. +//! Sync, telemetry, and retention policy shapes plus project path helpers. //! -//! Shared runtime pin settings stay in [`crate::config`]; this module owns the -//! serde/migration shape and the include/exclude/gitignore helpers every -//! caller imports directly. +//! Runtime values are decoded from a pinned snapshot in [`crate::config`]. use std::collections::BTreeMap; use std::ffi::OsString; @@ -10,38 +8,17 @@ use std::fs; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; -use glob::Pattern; use serde::{Deserialize, Serialize}; use tracedecay_contracts::storage::compaction::CompactionThresholdConfig; -use tracedecay_domain::configuration::ConfigurationSnapshotV1; -use tracedecay_domain::configuration::{ - SYNC_AUTO_INIT_SETTING_KEY, SYNC_AUTO_WATCH_SETTING_KEY, - SYNC_BACKSTOP_INTERVAL_MINS_SETTING_KEY, SYNC_BRANCH_GC_DAYS_SETTING_KEY, - SYNC_FULL_SYNC_ESCALATION_FILES_SETTING_KEY, SYNC_MAX_CONCURRENT_SYNCS_SETTING_KEY, - SYNC_ORPHAN_DB_GC_DAYS_SETTING_KEY, SYNC_READ_COOLDOWN_SECS_SETTING_KEY, - SYNC_READ_REFRESH_SETTING_KEY, SYNC_SESSION_START_STALE_THRESHOLD_SECS_SETTING_KEY, - SYNC_SESSION_START_SYNC_SETTING_KEY, SYNC_WATCH_DEBOUNCE_MS_SETTING_KEY, - SYNC_WATCH_LINKED_WORKTREES_SETTING_KEY, SYNC_WATCH_MAX_DELAY_MS_SETTING_KEY, - SYNC_WATCH_MAX_PROJECTS_SETTING_KEY, -}; use tracedecay_domain::errors::{Result, TraceDecayError}; pub use tracedecay_runtime_core::config::brand_env; use tracedecay_runtime_core::config::{ - GENERATED_DIR_SEGMENTS, active_data_dir_name, discover_project_root, get_tracedecay_dir, - is_generated_dir_segment, + active_data_dir_name, discover_project_root, is_generated_dir_segment, }; -use super::{PinnedRuntimeConfiguration, required_bool, required_unsigned, required_usize}; - -/// Name of the legacy configuration migration input stored inside the data -/// directory. It is not a runtime authority and production code must never -/// rewrite it. -pub const CONFIG_FILENAME: &str = "config.json"; - /// Returns `true` if any component of `path` is a generated/vendored /// directory segment, or `path` itself carries a minified-asset suffix -/// (`app.min.js`, `app.min.css`, ...). This mirrors the `**/*.min.*` default -/// exclude pattern built by `default_exclude_patterns`. +/// (`app.min.js`, `app.min.css`, ...). /// /// Path-level, including individual file paths, so callers can filter a flat /// list of file paths in one pass. @@ -55,136 +32,10 @@ fn has_minified_suffix(path: &str) -> bool { path.rfind(".min.").is_some_and(|idx| idx + 5 < path.len()) } -fn default_true() -> bool { - true -} - -fn default_false() -> bool { - false -} - fn default_thirty_day_retention() -> Option { Some(30) } -/// Default glob-pattern exclude list for [`TraceDecayConfig::default`]. -/// -/// Built from [`GENERATED_DIR_SEGMENTS`] (both the `segment/**` root form -/// and the `**/segment/**` nested form, since a generated directory can -/// appear at the project root or anywhere below it) plus site-local -/// additions that intentionally are *not* part of the shared segment set: -/// -/// - `.git/**`, `.tracedecay/**`. VCS and `TraceDecay`'s own metadata dirs; -/// these are tool/repo bookkeeping, not generated *code*, so they stay -/// local to the config's default patterns rather than joining -/// [`GENERATED_DIR_SEGMENTS`] (which migrate and scan call sites also -/// consult for non-config-driven decisions). -/// - `bin/**`. Historically excluded here by default, but not treated as -/// "generated" elsewhere: a `bin/` directory can hold real source in some -/// project layouts, so it isn't added to the shared segment list. -/// - `**/*.min.*`. Mirrors [`is_generated_path_segment`]'s suffix check. -fn default_exclude_patterns() -> Vec { - let mut patterns: Vec = vec![ - ".git/**".to_string(), - ".tracedecay/**".to_string(), - "bin/**".to_string(), - "**/*.min.*".to_string(), - ]; - for segment in GENERATED_DIR_SEGMENTS { - patterns.push(format!("{segment}/**")); - patterns.push(format!("**/{segment}/**")); - } - patterns -} - -/// Legacy `config.json` representation and the materialized shape used by an -/// already-pinned resolved configuration snapshot. -/// -/// `version` and `root_dir` are legacy migration metadata only. Every runtime -/// setting below is sourced from [`ConfigurationSnapshotV1`] before a project -/// opens; serializing this type is retained solely for migration fixtures and -/// backwards-compatible legacy input decoding. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[allow( - clippy::struct_excessive_bools, - reason = "Independent legacy configuration switches retain their serialized migration shape" -)] -pub struct TraceDecayConfig { - /// Schema version of the configuration. - pub version: u32, - /// Root directory of the project being indexed. - pub root_dir: String, - /// Glob patterns for files to exclude during indexing. - pub exclude: Vec, - /// Glob patterns for paths to include despite the default hidden-directory, - /// generated-directory, and gitignore filters. For example, - /// `[".github/**"]` indexes files under `.github/` that would otherwise be - /// skipped. - #[serde(default)] - pub include: Vec, - /// Maximum file size in bytes; files larger than this are skipped. - pub max_file_size: u64, - /// Whether to extract doc comments from source files. - pub extract_docstrings: bool, - /// Whether to track call-site locations for edges. - pub track_call_sites: bool, - /// Whether to respect `.gitignore` rules when scanning files. - #[serde(default = "default_true")] - pub git_ignore: bool, - /// Whether a cold `tracedecay_diagnostics` call prewarms in the background - /// (detached dependency build + immediate `warming` status) instead of - /// blocking for minutes. Environment precedence is resolved into the - /// pinned snapshot during legacy migration, never during a tool call. - #[serde(default)] - pub diagnostics_prewarm: bool, - /// Whether the persistent native code graph may activate for this project. - /// Disabling it leaves exact and lexical retrieval available and reports - /// graph capability as unavailable. - #[serde(default = "default_true")] - pub native_graph_activation: bool, - /// Index-freshness auto-sync settings (git-metadata watcher, serve-stale, - /// branch lifecycle). Absent in older `config.json` files, so defaulted. - #[serde(default)] - pub sync: SyncConfig, - /// Analytics telemetry settings. Absent in older `config.json` files, so - /// defaulted. - #[serde(default)] - pub telemetry: TelemetryConfig, -} - -fn default_sync_watch_debounce_ms() -> u64 { - 2000 -} -fn default_sync_watch_max_delay_ms() -> u64 { - 30000 -} -fn default_sync_watch_max_projects() -> usize { - 32 -} -fn default_sync_read_cooldown_secs() -> u64 { - 30 -} -fn default_sync_session_start_stale_threshold_secs() -> u64 { - 600 -} -fn default_sync_backstop_interval_mins() -> u64 { - 15 -} -fn default_sync_full_sync_escalation_files() -> usize { - 500 -} -fn default_sync_max_concurrent_syncs() -> usize { - 2 -} -fn default_sync_branch_gc_days() -> u64 { - 14 -} -fn default_sync_orphan_db_gc_days() -> u64 { - 7 -} -fn default_sync_auto_track_pr_poll_secs() -> u64 { - 300 -} fn default_retention_interval_hours() -> u64 { 24 } @@ -194,7 +45,7 @@ fn default_compaction_threshold() -> Option { } /// The daemon retention/compaction policy tree (Plan 38). Safe, bounded -/// maintenance is active by default for proven orphan stores, quarantined +/// maintenance is active by default for proven orphan stores, incident /// debris, redundant projection-durable session copies, and free-page bloat. /// Lossy session/evidence deletion remains disabled and soft budgets remain /// owner-configured findings only. @@ -210,10 +61,6 @@ pub struct RetentionConfig { /// the sweep; the Doctor surface still reports findings read-only. #[serde(default = "default_thirty_day_retention")] pub orphan_store_gc_days: Option, - /// Retention window for quarantined recovery/corruption artifacts (days). - /// `None` disables collection while Doctor continues surfacing debris. - #[serde(default = "default_thirty_day_retention")] - pub incident_debris_retention_days: Option, /// Incremental-vacuum compaction trigger. `None` disables compaction. #[serde(default = "default_compaction_threshold")] pub compaction: Option, @@ -233,7 +80,6 @@ impl Default for RetentionConfig { observation: tracedecay_global_db::observation::retention::ObservationRetentionConfig::default(), orphan_store_gc_days: default_thirty_day_retention(), - incident_debris_retention_days: default_thirty_day_retention(), compaction: default_compaction_threshold(), store_soft_budgets_bytes: BTreeMap::new(), interval_hours: default_retention_interval_hours(), @@ -259,43 +105,6 @@ impl RetentionConfig { .map_err(|error| config_error(error.to_string()))?; Ok(Some(budget)) } - - /// Validate collection windows and the compaction trigger. Immediate - /// collection and ratios outside the unit interval are rejected. - pub(crate) fn validate(&self) -> Result<()> { - if self.orphan_store_gc_days == Some(0) { - return Err(config_error( - "retention orphan_store_gc_days must be greater than zero", - )); - } - if self.incident_debris_retention_days == Some(0) { - return Err(config_error( - "retention incident_debris_retention_days must be greater than zero", - )); - } - if let Some(compaction) = &self.compaction - && (!compaction.free_page_ratio_threshold.is_finite() - || compaction.free_page_ratio_threshold <= 0.0 - || compaction.free_page_ratio_threshold > 1.0) - { - return Err(config_error( - "retention compaction free_page_ratio_threshold must be within (0.0, 1.0]", - )); - } - for (store, bytes) in &self.store_soft_budgets_bytes { - tracedecay_contracts::storage::StoreKeyV1::new(store.clone()).map_err(|_| { - config_error(format!( - "retention store soft budget key '{store}' is not a valid StoreKeyV1" - )) - })?; - if *bytes == 0 { - return Err(config_error(format!( - "retention store soft budget for '{store}' must be greater than zero" - ))); - } - } - Ok(()) - } } /// Floor for the PR-autotrack poll interval; polls faster than this hammer the @@ -303,94 +112,64 @@ impl RetentionConfig { /// clamped up to this. pub const MIN_AUTO_TRACK_PR_POLL_SECS: u64 = 60; -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq)] pub struct TelemetryConfig { - #[serde(default = "default_true")] pub timings: bool, } impl Default for TelemetryConfig { fn default() -> Self { - Self { - timings: default_true(), - } + Self { timings: true } } } -/// Auto-sync / index-freshness knobs in the legacy migration shape. +/// Auto-sync / index-freshness knobs. /// /// Runtime consumers receive these values only from a pinned resolved -/// configuration snapshot. `TRACEDECAY_SYNC_*` values are decoded as an -/// explicit legacy environment layer during migration, rather than being read -/// independently by each adapter. -/// -/// Every field carries a `#[serde(default = ...)]` so that a partial JSON -/// object (only some keys present) still deserializes, and a missing `sync` -/// key entirely falls back to [`SyncConfig::default`]. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +/// configuration snapshot; no environment layer overrides them. +#[derive(Debug, Clone, PartialEq)] #[allow( clippy::struct_excessive_bools, reason = "Independent sync admission switches are configuration choices, not mutually exclusive states" )] pub struct SyncConfig { /// Enable the daemon git-metadata watcher. - #[serde(default = "default_false")] pub auto_watch: bool, /// Admit linked worktrees into the daemon watcher without an explicit /// branch-indexing request. - #[serde(default = "default_false")] pub watch_linked_worktrees: bool, /// Per-project quiet-period debounce before a watcher-triggered sync (ms). - #[serde(default = "default_sync_watch_debounce_ms")] pub watch_debounce_ms: u64, /// Maximum time a watcher-triggered sync can be deferred by debounce (ms). - #[serde(default = "default_sync_watch_max_delay_ms")] pub watch_max_delay_ms: u64, /// Maximum number of recently-seen projects the watcher registers. - #[serde(default = "default_sync_watch_max_projects")] pub watch_max_projects: usize, /// Enable non-blocking sync-on-read for query tools. - #[serde(default = "default_true")] pub read_refresh: bool, /// Cooldown between read-triggered background refreshes (seconds). - #[serde(default = "default_sync_read_cooldown_secs")] pub read_cooldown_secs: u64, /// Fire a catch-up sync on session start. - #[serde(default = "default_true")] pub session_start_sync: bool, /// Staleness threshold above which session-start sync runs (seconds). - #[serde(default = "default_sync_session_start_stale_threshold_secs")] pub session_start_stale_threshold_secs: u64, /// Daemon backstop scheduler interval (minutes); 0 disables it. - #[serde(default = "default_sync_backstop_interval_mins")] pub backstop_interval_mins: u64, /// Diff-scoped syncs above this many changed files escalate to a full sync. - #[serde(default = "default_sync_full_sync_escalation_files")] pub full_sync_escalation_files: usize, /// Daemon-wide cap on concurrent syncs. - #[serde(default = "default_sync_max_concurrent_syncs")] pub max_concurrent_syncs: usize, /// Grace period before a dead tracked-branch store is GC'd (days). - #[serde(default = "default_sync_branch_gc_days")] pub branch_gc_days: u64, - /// Grace period before an orphan branch DB is GC'd (days). - #[serde(default = "default_sync_orphan_db_gc_days")] - pub orphan_db_gc_days: u64, /// Auto-initialise never-indexed repos on first contact. - #[serde(default = "default_true")] pub auto_init: bool, /// Enable the daemon PR-branch auto-tracking mode: when on, the daemon polls /// the repo's GitHub remote for open PRs and tracks/untracks each PR head - /// branch through the normal branch-tracking machinery. Off by default for - /// back-compat. - #[serde(default = "default_false")] + /// branch through the normal branch-tracking machinery. pub auto_track_pr_branches: bool, /// Poll cadence (seconds) for PR-branch auto-tracking discovery. Clamped up /// to [`MIN_AUTO_TRACK_PR_POLL_SECS`] at read time. - #[serde(default = "default_sync_auto_track_pr_poll_secs")] pub auto_track_pr_poll_secs: u64, /// Daemon retention/compaction policy tree (Plan 38). - #[serde(default)] pub retention: RetentionConfig, } @@ -406,323 +185,33 @@ impl SyncConfig { impl Default for SyncConfig { fn default() -> Self { Self { - auto_watch: default_false(), - watch_linked_worktrees: default_false(), - watch_debounce_ms: default_sync_watch_debounce_ms(), - watch_max_delay_ms: default_sync_watch_max_delay_ms(), - watch_max_projects: default_sync_watch_max_projects(), - read_refresh: default_true(), - read_cooldown_secs: default_sync_read_cooldown_secs(), - session_start_sync: default_true(), - session_start_stale_threshold_secs: default_sync_session_start_stale_threshold_secs(), - backstop_interval_mins: default_sync_backstop_interval_mins(), - full_sync_escalation_files: default_sync_full_sync_escalation_files(), - max_concurrent_syncs: default_sync_max_concurrent_syncs(), - branch_gc_days: default_sync_branch_gc_days(), - orphan_db_gc_days: default_sync_orphan_db_gc_days(), - auto_init: default_true(), - auto_track_pr_branches: default_false(), - auto_track_pr_poll_secs: default_sync_auto_track_pr_poll_secs(), + auto_watch: false, + watch_linked_worktrees: false, + watch_debounce_ms: 2000, + watch_max_delay_ms: 30000, + watch_max_projects: 32, + read_refresh: true, + read_cooldown_secs: 30, + session_start_sync: true, + session_start_stale_threshold_secs: 600, + backstop_interval_mins: 15, + full_sync_escalation_files: 500, + max_concurrent_syncs: 2, + branch_gc_days: 14, + auto_init: true, + auto_track_pr_branches: false, + auto_track_pr_poll_secs: 300, retention: RetentionConfig::default(), } } } -/// Parses a boolean env value. Truthy spellings (`1`/`true`/`yes`/`on`) share -/// [`tracedecay_global_db::env_value_truthy`]; `0`/`false` are false. Any -/// other value is ignored (returns `None`) so an override is not applied. -pub(crate) fn parse_env_bool(raw: &str) -> Option { - if tracedecay_global_db::env_value_truthy(raw) { - return Some(true); - } - match raw.trim().to_ascii_lowercase().as_str() { - "0" | "false" => Some(false), - _ => None, - } -} - -/// Reads a `TRACEDECAY_` env var and parses it as a bool. -pub(crate) fn env_bool(suffix: &str) -> Option { - brand_env(suffix).as_deref().and_then(parse_env_bool) -} - -/// Reads a `TRACEDECAY_` env var and parses it as an integer of the -/// caller's choosing. -fn env_parse(suffix: &str) -> Option { - brand_env(suffix) - .as_deref() - .and_then(|raw| raw.trim().parse::().ok()) -} - -impl SyncConfig { - /// Applies legacy `TRACEDECAY_SYNC_*` environment overrides on top of - /// `self`. This remains for pre-store/bootstrap compatibility only; live - /// runtime adapters must consume [`PinnedRuntimeConfiguration`] instead. - #[must_use] - pub fn with_env_overrides(mut self) -> Self { - if let Some(value) = env_bool("SYNC_AUTO_WATCH") { - self.auto_watch = value; - } - if let Some(value) = env_bool("SYNC_WATCH_LINKED_WORKTREES") { - self.watch_linked_worktrees = value; - } - if let Some(value) = env_parse("SYNC_WATCH_DEBOUNCE_MS") { - self.watch_debounce_ms = value; - } - if let Some(value) = env_parse("SYNC_WATCH_MAX_DELAY_MS") { - self.watch_max_delay_ms = value; - } - if let Some(value) = env_parse("SYNC_WATCH_MAX_PROJECTS") { - self.watch_max_projects = value; - } - if let Some(value) = env_bool("SYNC_READ_REFRESH") { - self.read_refresh = value; - } - if let Some(value) = env_parse("SYNC_READ_COOLDOWN_SECS") { - self.read_cooldown_secs = value; - } - if let Some(value) = env_bool("SYNC_SESSION_START_SYNC") { - self.session_start_sync = value; - } - if let Some(value) = env_parse("SYNC_SESSION_START_STALE_THRESHOLD_SECS") { - self.session_start_stale_threshold_secs = value; - } - if let Some(value) = env_parse("SYNC_BACKSTOP_INTERVAL_MINS") { - self.backstop_interval_mins = value; - } - if let Some(value) = env_parse("SYNC_FULL_SYNC_ESCALATION_FILES") { - self.full_sync_escalation_files = value; - } - if let Some(value) = env_parse("SYNC_MAX_CONCURRENT_SYNCS") { - self.max_concurrent_syncs = value; - } - if let Some(value) = env_parse("SYNC_BRANCH_GC_DAYS") { - self.branch_gc_days = value; - } - if let Some(value) = env_parse("SYNC_ORPHAN_DB_GC_DAYS") { - self.orphan_db_gc_days = value; - } - if let Some(value) = env_bool("SYNC_AUTO_INIT") { - self.auto_init = value; - } - if let Some(value) = env_bool("SYNC_AUTO_TRACK_PR_BRANCHES") { - self.auto_track_pr_branches = value; - } - if let Some(value) = env_parse("SYNC_AUTO_TRACK_PR_POLL_SECS") { - self.auto_track_pr_poll_secs = value; - } - self - } -} - -impl Default for TraceDecayConfig { - fn default() -> Self { - Self { - version: 1, - root_dir: String::new(), - exclude: default_exclude_patterns(), - include: Vec::new(), - max_file_size: 1_048_576, - extract_docstrings: true, - track_call_sites: true, - git_ignore: default_true(), - diagnostics_prewarm: false, - native_graph_activation: default_true(), - sync: SyncConfig::default(), - telemetry: TelemetryConfig::default(), - } - } -} - -impl TraceDecayConfig { - /// Layers the daemon-only policy over the shared runtime settings of an - /// already validated pin. The shared settings are copied from the pin, so - /// they agree with every other consumer by construction; only the - /// daemon-only sync and legacy metadata fields are decoded here, from the - /// same snapshot, without defaults, file reads, or environment reads. - /// - /// Retention is not a registered configuration setting. An unregistered - /// text blob is not parsed into policy; the admitted value is - /// [`RetentionConfig::default`]. - #[hotpath::measure(label = "daemon.config.parse")] - pub fn from_runtime(runtime: &PinnedRuntimeConfiguration) -> Result { - let shared = runtime.config(); - let snapshot = runtime.snapshot(); - Ok(Self { - version: 1, - root_dir: runtime.target().project_root.to_string_lossy().to_string(), - exclude: shared.exclude.clone(), - include: shared.include.clone(), - max_file_size: shared.max_file_size, - extract_docstrings: shared.extract_docstrings, - track_call_sites: shared.track_call_sites, - git_ignore: shared.git_ignore, - diagnostics_prewarm: shared.diagnostics_prewarm, - native_graph_activation: shared.native_graph_activation, - sync: SyncConfig { - auto_watch: required_bool(snapshot, SYNC_AUTO_WATCH_SETTING_KEY)?, - watch_linked_worktrees: required_bool( - snapshot, - SYNC_WATCH_LINKED_WORKTREES_SETTING_KEY, - )?, - watch_debounce_ms: required_unsigned(snapshot, SYNC_WATCH_DEBOUNCE_MS_SETTING_KEY)?, - watch_max_delay_ms: required_unsigned( - snapshot, - SYNC_WATCH_MAX_DELAY_MS_SETTING_KEY, - )?, - watch_max_projects: required_usize(snapshot, SYNC_WATCH_MAX_PROJECTS_SETTING_KEY)?, - read_refresh: required_bool(snapshot, SYNC_READ_REFRESH_SETTING_KEY)?, - read_cooldown_secs: required_unsigned( - snapshot, - SYNC_READ_COOLDOWN_SECS_SETTING_KEY, - )?, - session_start_sync: required_bool(snapshot, SYNC_SESSION_START_SYNC_SETTING_KEY)?, - session_start_stale_threshold_secs: required_unsigned( - snapshot, - SYNC_SESSION_START_STALE_THRESHOLD_SECS_SETTING_KEY, - )?, - backstop_interval_mins: required_unsigned( - snapshot, - SYNC_BACKSTOP_INTERVAL_MINS_SETTING_KEY, - )?, - full_sync_escalation_files: required_usize( - snapshot, - SYNC_FULL_SYNC_ESCALATION_FILES_SETTING_KEY, - )?, - max_concurrent_syncs: required_usize( - snapshot, - SYNC_MAX_CONCURRENT_SYNCS_SETTING_KEY, - )?, - branch_gc_days: required_unsigned(snapshot, SYNC_BRANCH_GC_DAYS_SETTING_KEY)?, - orphan_db_gc_days: required_unsigned(snapshot, SYNC_ORPHAN_DB_GC_DAYS_SETTING_KEY)?, - auto_init: required_bool(snapshot, SYNC_AUTO_INIT_SETTING_KEY)?, - auto_track_pr_branches: shared.sync.auto_track_pr_branches, - auto_track_pr_poll_secs: shared.sync.auto_track_pr_poll_secs, - retention: admitted_retention_config(snapshot), - }, - telemetry: TelemetryConfig { - timings: shared.telemetry.timings, - }, - }) - } -} - -/// Retention is not in the closed configuration key inventory. A snapshot may -/// still carry an invented text key; that private JSON is not policy. -fn admitted_retention_config(_snapshot: &ConfigurationSnapshotV1) -> RetentionConfig { - RetentionConfig::default() -} - fn config_error(message: impl Into) -> TraceDecayError { TraceDecayError::Config { message: message.into(), } } -/// Returns the path to the configuration file (`config.json`) within the -/// resolved data directory. -pub fn get_config_path(project_root: &Path) -> PathBuf { - if let Ok(layout) = - tracedecay_runtime_core::storage::resolve_layout_for_current_profile(project_root) - { - return layout.config_path; - } - get_tracedecay_dir(project_root).join(CONFIG_FILENAME) -} - -/// Loads a legacy configuration input from disk. -/// -/// This compatibility reader is for migration and read-only diagnostics only; -/// runtime consumers must use a pinned resolved snapshot. If the file does -/// not exist, it returns the legacy defaults with `root_dir` set to the given -/// project root. -pub fn load_config(project_root: &Path) -> Result { - let config_path = get_config_path(project_root); - load_config_from_path(project_root, &config_path) -} - -/// Loads configuration from an explicit config path while preserving the -/// project root used for default config values. -pub fn load_config_from_path(project_root: &Path, config_path: &Path) -> Result { - if !config_path.exists() { - return Ok(TraceDecayConfig { - root_dir: project_root.to_string_lossy().to_string(), - ..TraceDecayConfig::default() - }); - } - - let contents = fs::read_to_string(config_path).map_err(|e| TraceDecayError::Config { - message: format!( - "failed to read config file '{}': {}", - config_path.display(), - e - ), - })?; - - let config: TraceDecayConfig = - serde_json::from_str(&contents).map_err(|e| TraceDecayError::Config { - message: format!( - "failed to parse config file '{}': {}", - config_path.display(), - e - ), - })?; - config.sync.retention.validate()?; - - Ok(config) -} - -/// Writes a legacy configuration fixture to an explicit path using an atomic -/// write. -/// -/// Production runtime code must use the daemon control plane instead of this -/// compatibility helper. It remains for fixtures and legacy-input tests while -/// callers complete their migration. -pub fn save_config_to_path(config_path: &Path, config: &TraceDecayConfig) -> Result<()> { - let data_dir = config_path - .parent() - .ok_or_else(|| TraceDecayError::Config { - message: format!( - "configuration path '{}' has no parent directory", - config_path.display() - ), - })?; - tracedecay_runtime_core::storage::PrivateStoreIo::create_dir_all(data_dir).map_err(|e| { - TraceDecayError::Config { - message: format!( - "failed to create tracedecay directory '{}': {}", - data_dir.display(), - e - ), - } - })?; - - let tmp_path = config_path.with_extension("tmp"); - - let json = serde_json::to_string_pretty(config).map_err(|e| TraceDecayError::Config { - message: format!("failed to serialize config: {e}"), - })?; - - fs::write(&tmp_path, &json).map_err(|e| TraceDecayError::Config { - message: format!( - "failed to write temporary config file '{}': {}", - tmp_path.display(), - e - ), - })?; - - fs::rename(&tmp_path, config_path).map_err(|e| TraceDecayError::Config { - message: format!( - "failed to rename temporary config file '{}' to '{}': {}", - tmp_path.display(), - config_path.display(), - e - ), - })?; - - Ok(()) -} - /// Returns `true` if the project marker dir (`.tracedecay`) is ignored by Git /// for this project. /// @@ -865,56 +354,6 @@ pub fn resolve_path_with_discovery(path: Option) -> PathBuf { } } -/// Returns `true` if the path matches any of the configured `include` patterns. -/// -/// This is used to allow hidden (dot-prefixed) directories that would -/// otherwise be skipped by the file walker. -pub fn is_included(path: &str, config: &TraceDecayConfig) -> bool { - any_pattern_matches(&config.include, &[path]) -} - -/// Returns `true` if a directory should be pruned during scanning. -/// -/// Matches `dir/_` against exclude patterns (for `dir/**`-style globs) and -/// also matches `dir` itself (for bare `**/dirname`-style globs). This -/// ensures that patterns like `**/node_modules` and `**/node_modules/**` -/// both trigger directory pruning in `scan_files_walkdir`. -pub fn is_excluded_dir(dir_path: &str, config: &TraceDecayConfig) -> bool { - // Try both the dummy-file probe (catches `dir/**`) and the bare directory - // path (catches `**/dirname`). - let descendant_probe = format!("{dir_path}/_"); - any_pattern_matches(&config.exclude, &[&descendant_probe, dir_path]) -} - -/// Returns `true` if the file matches any of the configured exclude patterns. -pub fn is_excluded(file_path: &str, config: &TraceDecayConfig) -> bool { - any_pattern_matches(&config.exclude, &[file_path]) -} - -/// Glob semantics shared by every include/exclude test. Kept in one place so -/// the four entry points cannot drift apart on case or separator handling. -const PATTERN_MATCH_OPTIONS: glob::MatchOptions = glob::MatchOptions { - case_sensitive: true, - require_literal_separator: false, - require_literal_leading_dot: false, -}; - -/// True when any of `patterns` matches any of `candidates`. Unparseable -/// patterns are skipped rather than failing the whole test, matching the -/// long-standing behaviour of the include/exclude entry points. -/// -/// Callers pass every candidate string they want probed, built once per call: -/// the directory variants used to format their `dir/_` probe once per pattern. -fn any_pattern_matches(patterns: &[String], candidates: &[&str]) -> bool { - patterns.iter().any(|pattern_str| { - Pattern::new(pattern_str).is_ok_and(|pattern| { - candidates - .iter() - .any(|candidate| pattern.matches_with(candidate, PATTERN_MATCH_OPTIONS)) - }) - }) -} - #[cfg(test)] #[allow(clippy::unwrap_used, clippy::expect_used)] mod tests; diff --git a/crates/tracedecay-configuration/src/config/model/tests.rs b/crates/tracedecay-configuration/src/config/model/tests.rs index 80d8b8beca..ec4444c754 100644 --- a/crates/tracedecay-configuration/src/config/model/tests.rs +++ b/crates/tracedecay-configuration/src/config/model/tests.rs @@ -1,16 +1,12 @@ -use super::{ - TraceDecayConfig, is_excluded, is_excluded_dir, is_generated_path_segment, - is_ignored_by_explicit_global_excludes, is_ignored_by_git, is_included, parse_env_bool, -}; +use super::{is_generated_path_segment, is_ignored_by_explicit_global_excludes, is_ignored_by_git}; use std::ffi::OsString; use std::fs; use std::path::PathBuf; use std::process::Command; use tempfile::TempDir; use tracedecay_runtime_core::config::{ - GENERATED_DIR_SEGMENTS, PinnedUserDataDir, USER_DATA_DIR_ENV, db_filename, - discover_project_root, get_project_db_path, get_tracedecay_dir, is_ambient_project_root, - is_generated_dir_segment, lock_user_data_dir_test_env, user_data_dir, + PinnedUserDataDir, USER_DATA_DIR_ENV, db_filename, discover_project_root, get_tracedecay_dir, + is_ambient_project_root, is_generated_dir_segment, lock_user_data_dir_test_env, user_data_dir, }; struct EnvRestore { @@ -46,10 +42,6 @@ fn test_data_dir_defaults_to_tracedecay_for_new_installs() { get_tracedecay_dir(root.path()), root.path().join(".tracedecay") ); - assert_eq!( - get_project_db_path(root.path()), - root.path().join(".tracedecay/tracedecay.db") - ); } #[test] @@ -147,78 +139,6 @@ fn test_db_filename_tracks_dir_brand() { ); } -#[test] -fn test_is_included_matches_glob() { - let config = TraceDecayConfig { - include: vec![".github/**".to_string()], - ..TraceDecayConfig::default() - }; - assert!(is_included(".github/workflows/ci.yml", &config)); - assert!(is_included(".github/scripts/build.sh", &config)); - assert!(!is_included(".vscode/settings.json", &config)); - assert!(!is_included("src/main.rs", &config)); -} - -#[test] -fn test_is_included_empty_matches_nothing() { - let config = TraceDecayConfig::default(); - assert!(!is_included(".github/workflows/ci.yml", &config)); -} - -#[test] -fn test_include_records_explicit_override_even_when_excluded() { - let config = TraceDecayConfig { - include: vec![".config/**".to_string()], - exclude: vec![".config/secret/**".to_string()], - ..TraceDecayConfig::default() - }; - assert!(is_included(".config/secret/key.rs", &config)); - assert!(is_excluded(".config/secret/key.rs", &config)); -} - -#[test] -fn test_default_excludes_nested_node_modules() { - let config = TraceDecayConfig::default(); - // Top-level node_modules should be excluded - assert!(is_excluded("node_modules/express/index.js", &config)); - // Nested node_modules inside a sub-project must also be excluded - assert!(is_excluded( - "projectA/node_modules/express/index.js", - &config - )); - assert!(is_excluded( - "packages/web/node_modules/react/index.js", - &config - )); - assert!(is_excluded("dist/main.js", &config)); - assert!(is_excluded("packages/web/dist/main.js", &config)); - assert!(is_excluded("coverage/lcov.js", &config)); - assert!(is_excluded("packages/web/.next/server/app.js", &config)); -} - -#[test] -fn test_dir_pruning_pattern_matches_nested_dirs() { - // scan_files_walkdir checks is_excluded("{dir}/_") for directory pruning. - // Patterns like **/node_modules/** must match the dummy-file probe. - let config = TraceDecayConfig::default(); - assert!(is_excluded("node_modules/_", &config)); - assert!(is_excluded("projectA/node_modules/_", &config)); -} - -#[test] -fn test_is_excluded_dir_bare_pattern() { - // Users may write "**/node_modules" (no trailing /**). - // is_excluded_dir should match both bare and /**-suffixed patterns. - let config = TraceDecayConfig { - exclude: vec!["**/dist".to_string()], - ..TraceDecayConfig::default() - }; - assert!(is_excluded_dir("dist", &config)); - assert!(is_excluded_dir("packages/web/dist", &config)); - // Files inside dist should still be caught by accept_file's is_excluded - // but dir pruning prevents even walking into the directory. -} - #[test] fn test_is_in_gitignore_respects_global_excludes_file() { let sandbox = TempDir::new().unwrap(); @@ -275,193 +195,17 @@ fn test_explicit_global_excludes_ignores_comments_and_blank_lines() { assert_eq!(ignored, Some(true)); } -#[test] -fn telemetry_timing_defaults_on_and_round_trips() { - let config = TraceDecayConfig::default(); - assert!(config.telemetry.timings); - let json = serde_json::to_string(&config).unwrap(); - let parsed: TraceDecayConfig = serde_json::from_str(&json).unwrap(); - assert_eq!(parsed.telemetry, super::TelemetryConfig::default()); - - let legacy = r#"{ - "version": 1, - "root_dir": "/tmp/proj", - "exclude": [], - "max_file_size": 1048576, - "extract_docstrings": true, - "track_call_sites": true - }"#; - let parsed: TraceDecayConfig = serde_json::from_str(legacy).unwrap(); - assert!(parsed.telemetry.timings); - - let disabled = r#"{ - "version": 1, - "root_dir": "/tmp/proj", - "exclude": [], - "max_file_size": 1048576, - "extract_docstrings": true, - "track_call_sites": true, - "telemetry": { "timings": false } - }"#; - let parsed: TraceDecayConfig = serde_json::from_str(disabled).unwrap(); - assert!(!parsed.telemetry.timings); -} - -#[test] -fn diagnostics_prewarm_round_trips_and_defaults_off() { - let config = TraceDecayConfig::default(); - assert!(!config.diagnostics_prewarm, "prewarm must default off"); - let json = serde_json::to_string(&config).unwrap(); - let parsed: TraceDecayConfig = serde_json::from_str(&json).unwrap(); - assert!(!parsed.diagnostics_prewarm); - - // Explicit true round-trips, and old configs without the key default. - let mut on = config.clone(); - on.diagnostics_prewarm = true; - let parsed: TraceDecayConfig = - serde_json::from_str(&serde_json::to_string(&on).unwrap()).unwrap(); - assert!(parsed.diagnostics_prewarm); - let legacy = r#"{ - "version": 1, - "root_dir": "/tmp/proj", - "exclude": [], - "max_file_size": 1048576, - "extract_docstrings": true, - "track_call_sites": true - }"#; - let parsed: TraceDecayConfig = serde_json::from_str(legacy).unwrap(); - assert!(!parsed.diagnostics_prewarm); -} - -#[test] -fn config_without_sync_key_deserializes_to_default_sync() { - // Old config.json files predate the `sync` table; the field-level - // `#[serde(default)]` must fill it in. - let json = r#"{ - "version": 1, - "root_dir": "/tmp/proj", - "exclude": [], - "max_file_size": 1048576, - "extract_docstrings": true, - "track_call_sites": true - }"#; - let parsed: TraceDecayConfig = serde_json::from_str(json).unwrap(); - assert_eq!(parsed.sync, crate::SyncConfig::default()); -} - -#[test] -fn partial_sync_table_fills_missing_fields_with_defaults() { - // Only two sync keys present; every other field must default. - let json = r#"{ - "version": 1, - "root_dir": "/tmp/proj", - "exclude": [], - "max_file_size": 1048576, - "extract_docstrings": true, - "track_call_sites": true, - "sync": { "auto_watch": false, "backstop_interval_mins": 99 } - }"#; - let parsed: TraceDecayConfig = serde_json::from_str(json).unwrap(); - assert!(!parsed.sync.auto_watch); - assert!(!parsed.sync.watch_linked_worktrees); - assert_eq!(parsed.sync.backstop_interval_mins, 99); - // Untouched fields keep their defaults. - assert_eq!(parsed.sync.watch_debounce_ms, 2000); - assert_eq!(parsed.sync.max_concurrent_syncs, 2); - assert!(parsed.sync.read_refresh); -} - -#[test] -fn pr_autotrack_defaults_off_and_survives_missing_keys() { - // Back-compat: a config predating the PR-autotrack keys must default the - // feature OFF and to the 300s poll cadence. - let json = r#"{ - "version": 1, - "root_dir": "/tmp/proj", - "exclude": [], - "max_file_size": 1048576, - "extract_docstrings": true, - "track_call_sites": true, - "sync": { "auto_watch": true } - }"#; - let parsed: TraceDecayConfig = serde_json::from_str(json).unwrap(); - assert!(!parsed.sync.auto_track_pr_branches); - assert_eq!(parsed.sync.auto_track_pr_poll_secs, 300); - assert_eq!(parsed.sync.effective_auto_track_pr_poll_secs(), 300); -} - -#[test] -fn pr_autotrack_round_trips_and_clamps_poll_floor() { - let json = r#"{ - "version": 1, - "root_dir": "/tmp/proj", - "exclude": [], - "max_file_size": 1048576, - "extract_docstrings": true, - "track_call_sites": true, - "sync": { "auto_track_pr_branches": true, "auto_track_pr_poll_secs": 5 } - }"#; - let parsed: TraceDecayConfig = serde_json::from_str(json).unwrap(); - assert!(parsed.sync.auto_track_pr_branches); - assert_eq!(parsed.sync.auto_track_pr_poll_secs, 5); - // A too-small interval is clamped up to the safety floor. - assert_eq!( - parsed.sync.effective_auto_track_pr_poll_secs(), - crate::MIN_AUTO_TRACK_PR_POLL_SECS - ); - - // Serialize → deserialize preserves the raw values. - let round = serde_json::to_string(&parsed).unwrap(); - let reparsed: TraceDecayConfig = serde_json::from_str(&round).unwrap(); - assert_eq!(reparsed.sync, parsed.sync); -} - -#[test] -fn parse_env_bool_shares_canonical_truthy_spellings() { - for raw in ["1", "true", "TRUE", "yes", "on", " YES "] { - assert_eq!(parse_env_bool(raw), Some(true), "{raw}"); - } - for raw in ["0", "false", "FALSE"] { - assert_eq!(parse_env_bool(raw), Some(false), "{raw}"); - } - assert_eq!(parse_env_bool("maybe"), None); -} - -#[test] -fn pr_autotrack_env_overrides() { - let _lock = lock_user_data_dir_test_env(); - let _enable = EnvRestore::set("TRACEDECAY_SYNC_AUTO_TRACK_PR_BRANCHES", "true"); - let _poll = EnvRestore::set("TRACEDECAY_SYNC_AUTO_TRACK_PR_POLL_SECS", "120"); - - let overridden = crate::SyncConfig::default().with_env_overrides(); - assert!(overridden.auto_track_pr_branches); - assert_eq!(overridden.auto_track_pr_poll_secs, 120); -} - -#[test] -fn sync_config_env_overrides_bool_and_int() { - let _lock = lock_user_data_dir_test_env(); - let _watch = EnvRestore::set("TRACEDECAY_SYNC_AUTO_WATCH", "false"); - let _linked = EnvRestore::set("TRACEDECAY_SYNC_WATCH_LINKED_WORKTREES", "true"); - let _debounce = EnvRestore::set("TRACEDECAY_SYNC_WATCH_DEBOUNCE_MS", "5000"); - // Unparsable ints/bools are ignored (field keeps its base value). - let _bad = EnvRestore::set("TRACEDECAY_SYNC_MAX_CONCURRENT_SYNCS", "not-a-number"); - - let overridden = crate::SyncConfig::default().with_env_overrides(); - assert!(!overridden.auto_watch); - assert!(overridden.watch_linked_worktrees); - assert_eq!(overridden.watch_debounce_ms, 5000); - assert_eq!( - overridden.max_concurrent_syncs, - crate::SyncConfig::default().max_concurrent_syncs - ); -} - #[test] fn implicit_discovery_never_selects_the_user_profile_root() { let _profile = PinnedUserDataDir::new(); let home = PathBuf::from(std::env::var_os("HOME").expect("pinned HOME")); - fs::write(get_project_db_path(&home), b"").expect("ambient project marker"); + let home_store = tracedecay_runtime_core::storage::default_profile_sharded_layout( + &home, + &user_data_dir().expect("pinned profile"), + ) + .expect("home store layout"); + fs::create_dir_all(&home_store.data_root).expect("home store root"); + fs::write(&home_store.graph_db_path, b"").expect("ambient project marker"); let nested = home.join("unrelated/nested"); fs::create_dir_all(&nested).expect("nested directory"); @@ -472,43 +216,10 @@ fn implicit_discovery_never_selects_the_user_profile_root() { // --------------------------------------------------------------------------- // Shared generated/vendored segment list // -// GENERATED_DIR_SEGMENTS is the one list shared by this module's -// DEFAULT_EXCLUDE_PATTERNS, scan, and migrate inventory paths. +// GENERATED_DIR_SEGMENTS is the one list shared by the registry's default +// excludes, scan, and migrate inventory paths. // --------------------------------------------------------------------------- -#[test] -fn generated_dir_segments_cover_the_union_all_call_sites_need() { - // Formerly scan.rs-only (its HINTABLE_DIRS list). - for segment in [ - "node_modules", - "vendor", - "build", - "dist", - "out", - "coverage", - ".cache", - ".next", - ".turbo", - ".gradle", - ".venv", - "venv", - "__pycache__", - ] { - assert!( - GENERATED_DIR_SEGMENTS.contains(&segment), - "{segment} (from scan.rs's old list) missing from GENERATED_DIR_SEGMENTS" - ); - } - // Formerly migrate::inventory-only addition beyond the scan.rs set. - assert!(GENERATED_DIR_SEGMENTS.contains(&"target")); - // Worktree build directories are generated paths too. - assert!(GENERATED_DIR_SEGMENTS.contains(&".worktrees")); - // `.git` is intentionally NOT part of the shared list, it stays a - // site-local addition in migrate::inventory::should_prune_dir (see its - // doc comment) because it's VCS metadata, not generated/vendored code. - assert!(!GENERATED_DIR_SEGMENTS.contains(&".git")); -} - #[test] fn is_generated_dir_segment_delegates_for_segments_unique_to_one_former_list() { // Every one of these previously lived in only one of the four lists; @@ -526,6 +237,7 @@ fn is_generated_dir_segment_delegates_for_segments_unique_to_one_former_list() { #[test] fn is_generated_path_segment_matches_segments_and_minified_suffix() { assert!(is_generated_path_segment("packages/web/target/debug/x")); + assert!(is_generated_path_segment("web/node_modules/react/index.js")); assert!(is_generated_path_segment(".worktrees/feature/src/lib.rs")); assert!(is_generated_path_segment("assets/app.min.js")); assert!(is_generated_path_segment("assets/app.min.css")); @@ -533,117 +245,13 @@ fn is_generated_path_segment_matches_segments_and_minified_suffix() { assert!(!is_generated_path_segment("builder/mod.rs")); } -#[test] -fn default_excludes_still_catch_target_and_worktrees() { - // Regression guard for the DEFAULT_EXCLUDE_PATTERNS rebuild: target/** - // previously had no **/target/** nested form (a real drift bug this - // unification fixes), and .worktrees was never excluded by default at - // all. - let config = TraceDecayConfig::default(); - assert!(is_excluded("target/debug/build", &config)); - assert!(is_excluded("crates/sub/target/debug/build", &config)); - assert!(is_excluded(".worktrees/feature/src/lib.rs", &config)); - // Site-local additions (not part of GENERATED_DIR_SEGMENTS) still work. - assert!(is_excluded(".git/HEAD", &config)); - assert!(is_excluded(".tracedecay/tracedecay.db", &config)); - assert!(is_excluded("bin/cli.js", &config)); -} - #[cfg(test)] #[allow(clippy::unwrap_used)] mod retention_config_tests { - use std::collections::BTreeMap; - - use crate::{RetentionConfig, SyncConfig, TraceDecayConfig}; - use tracedecay_contracts::storage::compaction::CompactionThresholdConfig; - use tracedecay_domain::configuration::{ - CandidateDispositionV1, ConfigurationCandidateV1, ConfigurationLayerIdV1, - ConfigurationRevisionId, ConfigurationSnapshotV1, ConfigurationValueV1, SettingKey, - }; - - #[test] - fn unregistered_retention_text_is_not_policy() { - let key = SettingKey::new("sync.retention.v1").unwrap(); - let mut values = BTreeMap::new(); - values.insert( - key.clone(), - ConfigurationValueV1::Text(r#"{"orphan_store_gc_days":7}"#.to_owned()), - ); - let mut provenance = BTreeMap::new(); - provenance.insert( - key, - vec![ConfigurationCandidateV1 { - layer: ConfigurationLayerIdV1::Default, - revision_id: ConfigurationRevisionId::new("configuration.revision.fixture") - .unwrap(), - disposition: CandidateDispositionV1::Winning, - safe_reason: None, - }], - ); - let snapshot = ConfigurationSnapshotV1::new(values, provenance).unwrap(); - let retention = super::super::admitted_retention_config(&snapshot); - assert_eq!(retention, RetentionConfig::default()); - assert_ne!(retention.orphan_store_gc_days, Some(7)); - } - - #[test] - fn legacy_config_file_rejects_zero_retention_window() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("config.json"); - let mut config = TraceDecayConfig::default(); - config.sync.retention.orphan_store_gc_days = Some(0); - std::fs::write( - &path, - serde_json::to_string(&config).expect("serialize legacy config"), - ) - .unwrap(); - let error = super::super::load_config_from_path(dir.path(), &path).unwrap_err(); - assert!( - error.to_string().contains("orphan_store_gc_days"), - "{error}" - ); - } - - #[test] - fn default_retention_runs_only_safe_bounded_maintenance() { - let retention = RetentionConfig::default(); - assert!( - retention.session_lcm.enabled, - "projection-durable session dedupe enabled by default" - ); - assert_eq!(retention.session_lcm.offload_after_days, Some(30)); - assert_eq!(retention.session_lcm.drop_after_days, Some(180)); - assert_eq!(retention.session_lcm.dedupe_projected_after_days, Some(30)); - assert_eq!(retention.session_lcm.max_batch_size, 500); - assert!( - retention.observation.enabled, - "released observation evidence maintenance is active by default" - ); - assert_eq!(retention.observation.anchor_release_after_days, Some(30)); - assert_eq!( - retention.observation.observation_release_after_days, - Some(30) - ); - assert_eq!( - retention.observation.provenance_release_after_days, - Some(30) - ); - assert_eq!(retention.orphan_store_gc_days, Some(30)); - assert_eq!(retention.incident_debris_retention_days, Some(30)); - let compaction = retention.compaction.expect("compaction enabled"); - assert!((compaction.free_page_ratio_threshold - 0.25).abs() < f64::EPSILON); - assert_eq!(compaction.minimum_reclaimable_bytes, 64 * 1024 * 1024); - assert_eq!(compaction.max_pages_per_tick, 1024); - assert_eq!(compaction, CompactionThresholdConfig::default()); - assert!(retention.store_soft_budgets_bytes.is_empty()); - // A default SyncConfig carries the same bounded retention tree. - assert_eq!(SyncConfig::default().retention, retention); - } + use crate::RetentionConfig; #[test] fn empty_json_object_deserializes_to_safe_defaults() { - // A serde-compat empty object (older config with no retention block) - // must resolve the same safe maintenance policy. let retention: RetentionConfig = serde_json::from_str("{}").unwrap(); assert_eq!(retention, RetentionConfig::default()); @@ -653,36 +261,12 @@ mod retention_config_tests { assert!(nested.observation.reclaim_superseded_cursor_advances); } - #[test] - fn retention_rejects_immediate_collection_and_invalid_compaction_ratio() { - let retention = RetentionConfig { - orphan_store_gc_days: Some(0), - ..RetentionConfig::default() - }; - assert!(retention.validate().is_err()); - - let retention = RetentionConfig { - incident_debris_retention_days: Some(0), - ..RetentionConfig::default() - }; - assert!(retention.validate().is_err()); - - let mut retention = RetentionConfig::default(); - retention - .compaction - .as_mut() - .expect("default compaction") - .free_page_ratio_threshold = 1.01; - assert!(retention.validate().is_err()); - } - #[test] fn retention_config_json_round_trips_with_windows_set() { let json = r#"{ "session_lcm": { "enabled": true, "drop_after_days": 30 }, "observation": { "enabled": true, "anchor_release_after_days": 45 }, "orphan_store_gc_days": 14, - "incident_debris_retention_days": 21, "compaction": { "free_page_ratio_threshold": 0.25, "minimum_reclaimable_bytes": 1000000 }, "store_soft_budgets_bytes": { "sessions.db": 2000000000 }, "interval_hours": 12 @@ -693,7 +277,6 @@ mod retention_config_tests { assert!(retention.observation.enabled); assert_eq!(retention.observation.anchor_release_after_days, Some(45)); assert_eq!(retention.orphan_store_gc_days, Some(14)); - assert_eq!(retention.incident_debris_retention_days, Some(21)); assert_eq!(retention.interval_hours, 12); let compaction = retention.compaction.expect("compaction configured"); assert!((compaction.free_page_ratio_threshold - 0.25).abs() < f64::EPSILON); diff --git a/crates/tracedecay-configuration/src/configuration/operations.rs b/crates/tracedecay-configuration/src/configuration/operations.rs index 7f762ab4fb..8e104f712e 100644 --- a/crates/tracedecay-configuration/src/configuration/operations.rs +++ b/crates/tracedecay-configuration/src/configuration/operations.rs @@ -13,7 +13,7 @@ use crate::config::scope_control::{ ProtectedChangePlanDraftV1, plan_protected_change, validate_apply_binding, }; use tracedecay_global_db::configuration::contracts::ports::{ - ConfigurationClock, ConfigurationControlStore, ConfigurationMutationAuthorizationPort, + ConfigurationControlStore, ConfigurationMutationAuthorizationPort, ConfigurationOperationFuture, CurrentConfigurationMutationAuthorizationV1, ScopeResolutionPort, ScopeRevalidationEvidenceV1, }; @@ -80,23 +80,23 @@ pub trait ConfigurationControlPlane: Sync { ) -> ConfigurationOperationFuture<'_, ConfigurationAuditPage>; } -pub struct ConfigurationControlPlaneOperations<'a, Store, Scopes, Authorization, Clock> { +pub struct ConfigurationControlPlaneOperations<'a, Store, Scopes, Authorization> { registry: &'a ConfigurationRegistry, store: &'a Store, scopes: &'a Scopes, authorization: &'a Authorization, - clock: &'a Clock, + clock: fn() -> UtcMicros, } -impl<'a, Store, Scopes, Authorization, Clock> - ConfigurationControlPlaneOperations<'a, Store, Scopes, Authorization, Clock> +impl<'a, Store, Scopes, Authorization> + ConfigurationControlPlaneOperations<'a, Store, Scopes, Authorization> { pub fn new( registry: &'a ConfigurationRegistry, store: &'a Store, scopes: &'a Scopes, authorization: &'a Authorization, - clock: &'a Clock, + clock: fn() -> UtcMicros, ) -> Self { Self { registry, @@ -108,13 +108,12 @@ impl<'a, Store, Scopes, Authorization, Clock> } } -impl ConfigurationControlPlane - for ConfigurationControlPlaneOperations<'_, Store, Scopes, Authorization, Clock> +impl ConfigurationControlPlane + for ConfigurationControlPlaneOperations<'_, Store, Scopes, Authorization> where Store: ConfigurationControlStore, Scopes: ScopeResolutionPort, Authorization: ConfigurationMutationAuthorizationPort, - Clock: ConfigurationClock, { fn list( &self, @@ -250,7 +249,7 @@ where .resolve_protected_change(&actor, &change) .await?; validate_authorization_evidence(¤t_authorization, &evidence)?; - let now = self.clock.now(); + let now = (self.clock)(); let operation_digest = change .compute_digest() .map_err(ConfigurationError::validation)?; @@ -347,7 +346,7 @@ where ) .await?; self.store - .dry_run_rollback(&authority, &rollback, self.clock.now()) + .dry_run_rollback(&authority, &rollback, (self.clock)()) .await }) } @@ -406,13 +405,12 @@ where } } -impl - ConfigurationControlPlaneOperations<'_, Store, Scopes, Authorization, Clock> +impl + ConfigurationControlPlaneOperations<'_, Store, Scopes, Authorization> where Store: ConfigurationControlStore, Scopes: ScopeResolutionPort, Authorization: ConfigurationMutationAuthorizationPort, - Clock: ConfigurationClock, { fn apply_plan( &self, @@ -447,7 +445,7 @@ where { return Ok(receipt); } - let now = self.clock.now(); + let now = (self.clock)(); if plan.is_expired_at(now) { return Err(ConfigurationError::PlanExpired); } @@ -480,7 +478,7 @@ where effect: ConfigurationMutationEffectV1, ) -> Result { authority.validate_integrity()?; - let now = self.clock.now(); + let now = (self.clock)(); let current = self .authorization .recheck( @@ -803,20 +801,8 @@ mod tests { } } - struct Clock; - - impl ConfigurationClock for Clock { - fn now(&self) -> UtcMicros { - UtcMicros(10) - } - } - - struct AdvancedClock(UtcMicros); - - impl ConfigurationClock for AdvancedClock { - fn now(&self) -> UtcMicros { - self.0 - } + fn clock() -> UtcMicros { + UtcMicros(10) } #[test] @@ -898,13 +884,12 @@ mod tests { policy_epoch: 7, }, }; - let clock = Clock; let operations = ConfigurationControlPlaneOperations::new( ®istry, &store, &scope, &authorization, - &clock, + clock, ); let key = @@ -987,13 +972,12 @@ mod tests { ); let registry = ConfigurationRegistry::core().unwrap(); let scope = Scope { evidence }; - let clock = Clock; let operations = ConfigurationControlPlaneOperations::new( ®istry, &store, &scope, &authorization, - &clock, + clock, ); let plan = operations @@ -1116,14 +1100,13 @@ mod tests { }; let registry = ConfigurationRegistry::core().unwrap(); let scope = Scope { evidence }; - let clock = AdvancedClock(UtcMicros(10)); let restarted = ConfigurationControlPlaneOperations::new( ®istry, &store, &scope, &authorization, - &clock, + clock, ); assert_eq!( diff --git a/crates/tracedecay-configuration/src/configuration/runtime.rs b/crates/tracedecay-configuration/src/configuration/runtime.rs index f62f414847..706e06d64c 100644 --- a/crates/tracedecay-configuration/src/configuration/runtime.rs +++ b/crates/tracedecay-configuration/src/configuration/runtime.rs @@ -20,9 +20,9 @@ use tracedecay_domain::errors::{Result, TraceDecayError}; use tracedecay_global_db::RegisteredGlobalDbLeaseV1; use tracedecay_global_db::configuration::OwnedGlobalDbConfigurationControlStore; use tracedecay_global_db::configuration::contracts::ports::{ - ConfigurationClock, ConfigurationControlStore, ConfigurationCurrentStateV1, - ConfigurationMutationAuthorizationPort, ConfigurationOperationFuture, - CurrentConfigurationMutationAuthorizationV1, ScopeResolutionPort, ScopeRevalidationEvidenceV1, + ConfigurationControlStore, ConfigurationCurrentStateV1, ConfigurationMutationAuthorizationPort, + ConfigurationOperationFuture, CurrentConfigurationMutationAuthorizationV1, ScopeResolutionPort, + ScopeRevalidationEvidenceV1, }; use tracedecay_global_db::configuration::contracts::types::{ AuthorizedActor, ComponentConfigurationState, ConfigurationAuditPage, ConfigurationAuditQuery, @@ -72,7 +72,6 @@ impl ProjectConfigurationRuntime { store: store.clone(), scopes: SharedScopeResolution(Arc::clone(&authorities)), authorization: SharedMutationAuthorization(Arc::clone(&authorities)), - clock: SystemConfigurationClock, }); let client = Arc::new(ProductionConfigurationDaemonClient { target: configuration.target().clone(), @@ -249,7 +248,6 @@ struct RetainedConfigurationControlPlane { store: OwnedGlobalDbConfigurationControlStore, scopes: SharedScopeResolution, authorization: SharedMutationAuthorization, - clock: SystemConfigurationClock, } impl ConfigurationControlPlane for RetainedConfigurationControlPlane { @@ -263,7 +261,7 @@ impl ConfigurationControlPlane for RetainedConfigurationControlPlane { &self.store, &self.scopes, &self.authorization, - &self.clock, + now_micros, ) .list(actor) .await @@ -281,7 +279,7 @@ impl ConfigurationControlPlane for RetainedConfigurationControlPlane { &self.store, &self.scopes, &self.authorization, - &self.clock, + now_micros, ) .get(actor, key) .await @@ -300,7 +298,7 @@ impl ConfigurationControlPlane for RetainedConfigurationControlPlane { &self.store, &self.scopes, &self.authorization, - &self.clock, + now_micros, ) .mutate_direct(authority, mutation, expected_revision) .await @@ -317,7 +315,7 @@ impl ConfigurationControlPlane for RetainedConfigurationControlPlane { &self.store, &self.scopes, &self.authorization, - &self.clock, + now_micros, ) .observed_state(actor) .await @@ -336,7 +334,7 @@ impl ConfigurationControlPlane for RetainedConfigurationControlPlane { &self.store, &self.scopes, &self.authorization, - &self.clock, + now_micros, ) .dry_run_protected_change(authority, change, expected_revision) .await @@ -354,7 +352,7 @@ impl ConfigurationControlPlane for RetainedConfigurationControlPlane { &self.store, &self.scopes, &self.authorization, - &self.clock, + now_micros, ) .apply_protected_change(authority, request) .await @@ -372,7 +370,7 @@ impl ConfigurationControlPlane for RetainedConfigurationControlPlane { &self.store, &self.scopes, &self.authorization, - &self.clock, + now_micros, ) .dry_run_rollback(authority, rollback) .await @@ -390,7 +388,7 @@ impl ConfigurationControlPlane for RetainedConfigurationControlPlane { &self.store, &self.scopes, &self.authorization, - &self.clock, + now_micros, ) .apply_rollback(authority, request) .await @@ -408,7 +406,7 @@ impl ConfigurationControlPlane for RetainedConfigurationControlPlane { &self.store, &self.scopes, &self.authorization, - &self.clock, + now_micros, ) .audit(actor, query) .await @@ -514,14 +512,6 @@ impl ConfigurationMutationAuthorizationPort for SharedMutationAuthorization { } } -struct SystemConfigurationClock; - -impl ConfigurationClock for SystemConfigurationClock { - fn now(&self) -> UtcMicros { - now_micros() - } -} - #[cfg(test)] mod tests { use tracedecay_global_db::configuration::contracts::ports::CurrentConfigurationMutationAuthorizationV1; diff --git a/crates/tracedecay-configuration/src/configuration/user_settings.rs b/crates/tracedecay-configuration/src/configuration/user_settings.rs index be6f184d6b..d7beadfd0b 100644 --- a/crates/tracedecay-configuration/src/configuration/user_settings.rs +++ b/crates/tracedecay-configuration/src/configuration/user_settings.rs @@ -1,9 +1,9 @@ //! User-profile projection over the canonical configuration control plane. //! //! Editable values come only from the daemon-owned resolved snapshot. The -//! legacy `config.toml` remains a read-only metadata source for fields that are -//! not settings (installed agents, cached version state, and automation -//! discovery); transports cannot obtain a write capability for it. +//! profile `config.toml` is read only for fields that are not settings +//! (installed agents, cached version state, and automation discovery); +//! transports cannot obtain a write capability for it. use std::future::Future; use std::pin::Pin; @@ -26,7 +26,6 @@ pub type UserSettingsFuture<'a, T> = #[derive(Clone, Debug)] pub struct UserSettingsSnapshotV1 { - pub legacy_config_path: String, pub configuration_snapshot_id: String, pub configuration_revision_id: String, pub upload_enabled: bool, @@ -104,7 +103,7 @@ impl UserSettingsDaemonClient for ProductionUserSettingsDaemonClient { .current() .await .map_err(|error| unavailable(format!("resolved configuration: {error}")))?; - let metadata = tokio::task::spawn_blocking(read_legacy_user_metadata) + let metadata = tokio::task::spawn_blocking(read_user_metadata) .await .map_err(|error| unavailable(format!("user settings metadata task: {error}")))??; user_settings_snapshot(¤t, &profile_id, metadata) @@ -112,20 +111,16 @@ impl UserSettingsDaemonClient for ProductionUserSettingsDaemonClient { } } -struct LegacyUserMetadata { - path: String, +struct UserMetadata { installed_agents: Vec, cached_latest_version: String, automation: AutomationConfig, } -fn read_legacy_user_metadata() -> Result { - let path = tracedecay_session_memory::user_config::config_path() - .ok_or_else(|| unavailable("legacy user configuration path"))?; +fn read_user_metadata() -> Result { let config = UserConfig::load_strict() - .map_err(|error| unavailable(format!("legacy user configuration metadata: {error}")))?; - Ok(LegacyUserMetadata { - path: path.display().to_string(), + .map_err(|error| unavailable(format!("user configuration metadata: {error}")))?; + Ok(UserMetadata { installed_agents: config.installed_agents, cached_latest_version: config.cached_latest_version, automation: config.automation, @@ -135,7 +130,7 @@ fn read_legacy_user_metadata() -> Result Result { validate_profile_provenance(current, profile_id)?; let upload_enabled = required_bool(current, USER_UPLOAD_ENABLED_SETTING_KEY)?; @@ -143,7 +138,6 @@ fn user_settings_snapshot( let extraction_timeout_secs = required_unsigned(current, USER_EXTRACTION_TIMEOUT_SECS_SETTING_KEY)?; Ok(UserSettingsSnapshotV1 { - legacy_config_path: metadata.path, configuration_snapshot_id: current.snapshot().snapshot_id.as_str().to_owned(), configuration_revision_id: current.revision_id().as_str().to_owned(), upload_enabled, @@ -306,7 +300,6 @@ mod tests { fn snapshot() -> UserSettingsSnapshotV1 { UserSettingsSnapshotV1 { - legacy_config_path: "/profile/config.toml".to_owned(), configuration_snapshot_id: "configuration.snapshot.fixture".to_owned(), configuration_revision_id: "configuration.revision.fixture".to_owned(), upload_enabled: false, diff --git a/crates/tracedecay-configuration/src/lib.rs b/crates/tracedecay-configuration/src/lib.rs index 23b62fa17d..414ea9b75f 100644 --- a/crates/tracedecay-configuration/src/lib.rs +++ b/crates/tracedecay-configuration/src/lib.rs @@ -6,17 +6,19 @@ pub mod config; pub mod configuration; +#[cfg(any(test, feature = "test-helpers"))] +#[doc(hidden)] +pub mod test_support; pub use config::model::{ - CONFIG_FILENAME, MIN_AUTO_TRACK_PR_POLL_SECS, RetentionConfig, SyncConfig, TelemetryConfig, - TraceDecayConfig, brand_env, get_config_path, is_excluded, is_excluded_dir, - is_generated_path_segment, is_in_gitignore, is_included, load_config, load_config_from_path, - resolve_path, resolve_path_with_discovery, save_config_to_path, + MIN_AUTO_TRACK_PR_POLL_SECS, RetentionConfig, SyncConfig, TelemetryConfig, brand_env, + is_generated_path_segment, is_in_gitignore, resolve_path, resolve_path_with_discovery, }; pub use config::{ OpenedRuntimeConfiguration, PinnedRuntimeConfiguration, PinnedRuntimeConfigurationCachePort, RuntimeConfigurationTarget, cached_pinned_runtime_configuration, - install_pinned_runtime_configuration_cache, publish_pinned_runtime_configuration, + install_pinned_runtime_configuration_cache, lcm_summarizer_executables_for_project, + publish_pinned_runtime_configuration, }; pub use configuration::{ ConfigurationControlPlane, ConfigurationControlPlaneOperations, diff --git a/crates/tracedecay-configuration/src/test_support.rs b/crates/tracedecay-configuration/src/test_support.rs new file mode 100644 index 0000000000..865c654aea --- /dev/null +++ b/crates/tracedecay-configuration/src/test_support.rs @@ -0,0 +1,121 @@ +//! Test-only publication of pinned runtime configuration. +//! +//! Tests that exercise configuration-gated runtime behaviour (the LCM +//! summarizer executables, for instance) publish the setting through the same +//! pin cache production reads, instead of reaching for an environment or +//! `PATH` side channel the runtime no longer consults. + +use std::collections::BTreeMap; +use std::path::Path; +use std::sync::{Arc, RwLock}; + +use tracedecay_domain::ProjectId; +use tracedecay_domain::configuration::{ + ConfigurationLayerIdV1, ConfigurationRevisionId, ConfigurationValueV1, + LCM_SUMMARIZER_EXECUTABLES_SETTING_KEY, LcmSummarizerExecutablesV1, SettingKey, +}; +use tracedecay_domain::errors::{Result, TraceDecayError}; +use tracedecay_global_db::configuration::registry::ConfigurationRegistry; +use tracedecay_global_db::configuration::resolver::{ConfigurationLayerV1, resolve_configuration}; + +use crate::config::{ + PinnedRuntimeConfiguration, PinnedRuntimeConfigurationCachePort, RuntimeConfigurationTarget, + install_pinned_runtime_configuration_cache, publish_pinned_runtime_configuration, +}; + +/// Minimal in-process pin cache keyed by project id and root. +#[derive(Default)] +struct TestPinnedRuntimeConfigurationCache { + pins: RwLock>, +} + +impl PinnedRuntimeConfigurationCachePort for TestPinnedRuntimeConfigurationCache { + fn publish(&self, configuration: PinnedRuntimeConfiguration) -> Result<()> { + let mut pins = self + .pins + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner); + pins.retain(|pin| pin.target().project_id != configuration.target().project_id); + pins.push(configuration); + Ok(()) + } + + fn cached_for_root(&self, project_root: &Path) -> Result { + self.pins + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .iter() + .find(|pin| pin.target().project_root == project_root) + .cloned() + .ok_or_else(|| TraceDecayError::Config { + message: format!("no test pin published for root {}", project_root.display()), + }) + } + + fn cached_for_project(&self, project_id: &ProjectId) -> Result { + self.pins + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .iter() + .find(|pin| &pin.target().project_id == project_id) + .cloned() + .ok_or_else(|| TraceDecayError::Config { + message: format!("no test pin published for project {}", project_id.as_str()), + }) + } +} + +/// Publishes a pin for `project_id` whose only non-default setting is the +/// LCM summarizer binding. Installs the in-process test cache on first use; +/// when the composition root already installed its cache, the pin is +/// published through that one instead. +pub fn pin_lcm_summarizer_executables( + project_id: ProjectId, + project_root: &Path, + executables: LcmSummarizerExecutablesV1, +) -> Result { + let _ = install_pinned_runtime_configuration_cache(Arc::new( + TestPinnedRuntimeConfigurationCache::default(), + )); + let registry = ConfigurationRegistry::core().map_err(|error| TraceDecayError::Config { + message: format!("configuration registry unavailable: {error}"), + })?; + let revision_id = ConfigurationRevisionId::new(format!( + "configuration.test.lcm-summarizers.{}", + project_id.as_str().replace(['.', '/'], "-") + )) + .map_err(|error| TraceDecayError::Config { + message: format!("test revision id: {error}"), + })?; + let key = SettingKey::new(LCM_SUMMARIZER_EXECUTABLES_SETTING_KEY).map_err(|error| { + TraceDecayError::Config { + message: format!("summarizer setting key: {error}"), + } + })?; + let resolution = resolve_configuration( + ®istry, + &[ConfigurationLayerV1 { + layer: ConfigurationLayerIdV1::Project { + project_id: project_id.clone(), + }, + revision_id: revision_id.clone(), + entries: BTreeMap::from([( + key, + ConfigurationValueV1::LcmSummarizerExecutables(executables), + )]), + }], + ) + .map_err(|error| TraceDecayError::Config { + message: format!("resolve test configuration: {error}"), + })?; + let pinned = PinnedRuntimeConfiguration::new( + RuntimeConfigurationTarget { + project_id, + project_root: project_root.to_path_buf(), + }, + revision_id, + resolution.snapshot, + )?; + publish_pinned_runtime_configuration(pinned.clone())?; + Ok(pinned) +} diff --git a/crates/tracedecay-contracts/Cargo.toml b/crates/tracedecay-contracts/Cargo.toml index a0904ed58d..d7909c9057 100644 --- a/crates/tracedecay-contracts/Cargo.toml +++ b/crates/tracedecay-contracts/Cargo.toml @@ -16,7 +16,7 @@ hotpath = ["hotpath/hotpath"] [dependencies] hotpath.workspace = true -schemars = "1.2.1" +schemars.workspace = true getrandom = "0.2" hex = "0.4" serde = { version = "1", features = ["derive"] } diff --git a/crates/tracedecay-contracts/src/advisory.rs b/crates/tracedecay-contracts/src/advisory.rs index 8f67be24cf..e3d0379659 100644 --- a/crates/tracedecay-contracts/src/advisory.rs +++ b/crates/tracedecay-contracts/src/advisory.rs @@ -29,7 +29,7 @@ pub use tracedecay_domain::feedback::{ ProximityBranchWorktreeIncompatibilityV1, ProximityContributionIdV1, ProximityContributionV1, ProximityCoverageV1, ProximityInclusionV1, ProximityObservationIdV1, ProximityRelationPathKindV1, ProximityRelationPathV1, ProximityRelationStrengthV1, - ProximityRiskInputsV1, ProximityTierV1, ProximityWarningClassV1, ProximityWarningIdV1, + ProximityRiskInputsV1, ProximityTierV1, ProximityWarningClassV1, }; #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -209,7 +209,7 @@ impl AdvisoryFindingContributorV1 for ProximityContributionV1 { .ok_or_else(|| inconsistent("proximity contribution anchor"))?; let finding_id = tracedecay_domain::feedback::FeedbackFindingId::new(format!( "finding.proximity.{}", - self.warning_id.as_str() + self.contribution_id.as_str() )) .map_err(|_| inconsistent("proximity finding id"))?; validated_batch( diff --git a/crates/tracedecay-contracts/src/application_catalog_projection.rs b/crates/tracedecay-contracts/src/application_catalog_projection.rs index 1f4e1ebc87..9c9fbb2b55 100644 --- a/crates/tracedecay-contracts/src/application_catalog_projection.rs +++ b/crates/tracedecay-contracts/src/application_catalog_projection.rs @@ -1,7 +1,7 @@ use tracedecay_tool_catalog::{ - BindingStatus, BindingSurface, CatalogContributionV1, CodecBindingKey, - ExecutableBindingAvailabilityV1, ExecutableBindingRegistryV1, ExecutableBindingV1, - ExecutableUnavailableDispositionV1, OperationId, RouteExposureV1, ServiceId, SurfaceBindingV1, + BindingSurface, CatalogContributionV1, CodecBindingKey, ExecutableBindingAvailabilityV1, + ExecutableBindingRegistryV1, ExecutableBindingV1, ExecutableUnavailableDispositionV1, + OperationId, RouteExposureV1, ServiceId, SurfaceBindingV1, }; use crate::{ @@ -31,11 +31,11 @@ pub(crate) fn project_application_executable_bindings( let handlers = application_handler_descriptors()?; let mut bindings = Vec::new(); for contribution in application_catalog_contributions()? { - for surface in contribution.bindings().iter().filter(|binding| { - binding.surface() == projection.surface() - && matches!(binding.status(), BindingStatus::Current) - && !binding.is_alias() - }) { + for surface in contribution + .bindings() + .iter() + .filter(|binding| binding.surface() == projection.surface()) + { if let Some(binding) = project_availability(projection, &contribution, &handlers, surface, &exposure)? { diff --git a/crates/tracedecay-contracts/src/authorization/mod.rs b/crates/tracedecay-contracts/src/authorization/mod.rs index d66a89b5ba..e351a506d9 100644 --- a/crates/tracedecay-contracts/src/authorization/mod.rs +++ b/crates/tracedecay-contracts/src/authorization/mod.rs @@ -1,10 +1,12 @@ -mod non_disclosure; -mod ports; -mod service; +use tracedecay_domain::UtcMicros; -pub use non_disclosure::{ConcealedResourceCause, NonDisclosureHooks}; -pub use ports::{ - AuthorizationPhase, AuthorizationPort, AuthorizationPortOutcome, AuthorizationRequest, - SourceAuthorizationSnapshot, -}; -pub use service::{AuthorizationAdmission, AuthorizationService}; +use crate::context::RequestContext; +use crate::handlers::ApplicationOperation; + +/// Typed authorization input. It carries no transport-origin authority. +#[derive(Clone, Copy, Debug)] +pub struct AuthorizationRequest<'a> { + pub context: &'a RequestContext, + pub operation: &'a ApplicationOperation, + pub observed_at: UtcMicros, +} diff --git a/crates/tracedecay-contracts/src/authorization/non_disclosure.rs b/crates/tracedecay-contracts/src/authorization/non_disclosure.rs deleted file mode 100644 index d9b5837d1d..0000000000 --- a/crates/tracedecay-contracts/src/authorization/non_disclosure.rs +++ /dev/null @@ -1,85 +0,0 @@ -use tracedecay_policy::authorization::PublicSourceResultShapeV1; - -use crate::result::{ApplicationProblem, RetryDirective, SafeDiagnostic}; - -/// Internal causes intentionally collapsed before any application response is -/// constructed for a resource-addressed operation. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum ConcealedResourceCause { - Absent, - OutsideScope, - PolicyHidden, -} - -/// Central non-disclosure hooks for resource lookup, cursor resume, and anchor -/// expansion. All exposed paths preserve the same public problem shape. -#[derive(Clone, Copy, Debug, Default)] -pub struct NonDisclosureHooks; - -impl NonDisclosureHooks { - pub fn resource_problem( - &self, - _cause: ConcealedResourceCause, - retry: RetryDirective, - ) -> ApplicationProblem { - ApplicationProblem::not_found_or_not_authorized(retry) - } - - pub fn cursor_problem(&self, retry: RetryDirective) -> ApplicationProblem { - ApplicationProblem::not_found_or_not_authorized(retry) - } - - pub fn anchor_problem(&self, retry: RetryDirective) -> ApplicationProblem { - ApplicationProblem::not_found_or_not_authorized(retry) - } - - /// Convert a policy public shape into the application problem permitted at - /// an authorization boundary. `Live` and `Partial` only reach this hook - /// when a proof could not be verified, so they remain concealed too. - pub fn source_problem(&self, shape: PublicSourceResultShapeV1) -> ApplicationProblem { - match shape { - PublicSourceResultShapeV1::NotFoundOrNotAuthorized - | PublicSourceResultShapeV1::Live - | PublicSourceResultShapeV1::Partial - | PublicSourceResultShapeV1::AuthoritativeDeleted => { - self.resource_problem(ConcealedResourceCause::PolicyHidden, RetryDirective::Never) - } - PublicSourceResultShapeV1::PolicyExcluded => ApplicationProblem::Unsupported { - diagnostic: SafeDiagnostic::new( - "application.authorization.policy-excluded", - "The requested operation is not available.", - ) - .expect("static safe diagnostic is valid"), - retry: RetryDirective::Never, - legal_actions: Vec::new(), - }, - PublicSourceResultShapeV1::TemporarilyUnavailable => ApplicationProblem::unavailable( - SafeDiagnostic::new( - "application.authorization.source-unavailable", - "The requested resource is temporarily unavailable.", - ) - .expect("static safe diagnostic is valid"), - ), - } - } - - pub fn stale_policy_problem(&self) -> ApplicationProblem { - ApplicationProblem::stale( - SafeDiagnostic::new( - "application.authorization.policy-stale", - "Authorization information must be refreshed.", - ) - .expect("static safe diagnostic is valid"), - ) - } - - pub fn proof_problem(&self) -> ApplicationProblem { - ApplicationProblem::unavailable( - SafeDiagnostic::new( - "application.authorization.proof-invalid", - "The authorization proof could not be verified.", - ) - .expect("static safe diagnostic is valid"), - ) - } -} diff --git a/crates/tracedecay-contracts/src/authorization/ports.rs b/crates/tracedecay-contracts/src/authorization/ports.rs deleted file mode 100644 index ff3c367997..0000000000 --- a/crates/tracedecay-contracts/src/authorization/ports.rs +++ /dev/null @@ -1,76 +0,0 @@ -use tracedecay_domain::UtcMicros; -use tracedecay_policy::authorization::SourceAuthorizationInputV1; - -use crate::context::RequestContext; -use crate::handlers::ApplicationOperation; -use crate::result::SafeDiagnostic; - -/// Operation boundary at which authorization is evaluated or rechecked. -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub enum AuthorizationPhase { - Admission, - PageExpansion, - Hydration, - Publication, - Effect, -} - -/// Typed authorization input. Ports receive no transport-origin authority. -#[derive(Clone, Copy, Debug)] -pub struct AuthorizationRequest<'a> { - pub context: &'a RequestContext, - pub operation: &'a ApplicationOperation, - pub phase: AuthorizationPhase, - pub observed_at: UtcMicros, -} - -/// Immutable source-policy facts loaded by the application boundary. -/// -/// The source visibility bit is used only to apply the policy crate's public -/// non-disclosure projection. It is never treated as authorization. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct SourceAuthorizationSnapshot { - input: SourceAuthorizationInputV1, - source_visible: bool, -} - -impl SourceAuthorizationSnapshot { - pub fn new(input: SourceAuthorizationInputV1, source_visible: bool) -> Self { - Self { - input, - source_visible, - } - } - - pub fn input(&self) -> &SourceAuthorizationInputV1 { - &self.input - } - - #[hotpath::skip] - pub const fn source_visible(&self) -> bool { - self.source_visible - } -} - -/// Snapshot-loading result supplied by a policy/configuration authority. -/// -/// A port supplies immutable facts only. It never returns a policy decision, -/// receipt, or proof, so application code cannot reconstruct authority from a -/// [`crate::result::PolicyDecisionRef`]. -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum AuthorizationPortOutcome { - Snapshot(Box), - Absent, - Unavailable(SafeDiagnostic), - Stale(SafeDiagnostic), -} - -/// Narrow port for current policy/configuration snapshots. The approved -/// [`tracedecay_policy::authorization::SourceAuthorizationEvaluator`] evaluates -/// the returned input inside [`super::AuthorizationService`]. -pub trait AuthorizationPort { - fn source_authorization_snapshot( - &self, - request: &AuthorizationRequest<'_>, - ) -> AuthorizationPortOutcome; -} diff --git a/crates/tracedecay-contracts/src/authorization/service.rs b/crates/tracedecay-contracts/src/authorization/service.rs deleted file mode 100644 index 6c4e11754a..0000000000 --- a/crates/tracedecay-contracts/src/authorization/service.rs +++ /dev/null @@ -1,266 +0,0 @@ -use tracedecay_domain::{ComponentVersion, UtcMicros}; -use tracedecay_policy::authorization::{ - AuthorizationSnapshotStateV1, SinkAdmissionProofV1, SourceAuthorizationDecisionV1, - SourceAuthorizationDispositionV1, SourceAuthorizationEvaluator, SourceAuthorizationProofV1, - issue_source_authorization_proof, public_source_result_shape, recheck_sink_admission, -}; - -use crate::context::{RequestAdmission, RequestContext}; -use crate::handlers::ApplicationOperation; -use crate::result::{ApplicationProblem, AuthorityReceipt, PolicyDecisionRef, RetryDirective}; - -use super::{ - AuthorizationPhase, AuthorizationPort, AuthorizationPortOutcome, AuthorizationRequest, - ConcealedResourceCause, NonDisclosureHooks, SourceAuthorizationSnapshot, -}; - -/// One admitted source authorization. The opaque source proof is retained only -/// for fresh post-read publication or pre-effect rechecks. -#[derive(Clone, Debug)] -pub struct AuthorizationAdmission { - receipt: AuthorityReceipt, - source_proof: SourceAuthorizationProofV1, -} - -impl AuthorizationAdmission { - pub fn receipt(&self) -> &AuthorityReceipt { - &self.receipt - } - - pub fn source_proof(&self) -> &SourceAuthorizationProofV1 { - &self.source_proof - } -} - -/// Application-owned authorization boundary. It validates immutable context -/// inputs, loads a current snapshot through one narrow port, evaluates it with -/// the approved evaluator, and normalizes public disclosure. -pub struct AuthorizationService { - port: P, - evaluator: E, - non_disclosure: NonDisclosureHooks, -} - -impl AuthorizationService -where - P: AuthorizationPort, - E: SourceAuthorizationEvaluator, -{ - pub fn new(port: P, evaluator: E) -> Self { - Self { - port, - evaluator, - non_disclosure: NonDisclosureHooks, - } - } - - pub fn admit( - &self, - context: &RequestContext, - operation: &ApplicationOperation, - observed_at: UtcMicros, - ) -> Result { - let request = self.checked_request( - context, - operation, - AuthorizationPhase::Admission, - observed_at, - )?; - let snapshot = self.load_snapshot(&request)?; - self.authorize_snapshot(&request, snapshot) - } - - /// Revalidate current authority, scope, policy, and configuration after a - /// read and immediately before any retrieved evidence is published. - pub fn recheck_publication( - &self, - context: &RequestContext, - operation: &ApplicationOperation, - admission: &AuthorizationAdmission, - observed_at: UtcMicros, - ) -> Result { - let request = self.checked_request( - context, - operation, - AuthorizationPhase::Publication, - observed_at, - )?; - let snapshot = self.load_snapshot(&request)?; - let decision = self.evaluator.evaluate(snapshot.input()); - if snapshot.input().snapshot_state == AuthorizationSnapshotStateV1::Stale { - return Err(self.non_disclosure.stale_policy_problem()); - } - if recheck_sink_admission(&self.evaluator, admission.source_proof(), snapshot.input()) - .admission_proof() - .is_none() - { - return Err(self.public_problem(operation, &snapshot, &decision)); - } - - let policy = self.policy_reference(&decision)?; - AuthorityReceipt::from_context(context, policy, observed_at).map_err(|_| { - ApplicationProblem::invalid_request_without_action( - "application.authorization.invalid-context", - "The request context is invalid.", - ) - }) - } - - /// Re-run current policy and issue a sink admission proof immediately - /// before an effect. A receipt's [`PolicyDecisionRef`] is audit metadata; - /// it is never accepted in place of the retained source proof. - pub fn recheck_effect( - &self, - context: &RequestContext, - operation: &ApplicationOperation, - admission: &AuthorizationAdmission, - observed_at: UtcMicros, - ) -> Result { - let request = - self.checked_request(context, operation, AuthorizationPhase::Effect, observed_at)?; - let snapshot = self.load_snapshot(&request)?; - let decision = self.evaluator.evaluate(snapshot.input()); - if snapshot.input().snapshot_state == AuthorizationSnapshotStateV1::Stale { - return Err(self.non_disclosure.stale_policy_problem()); - } - - let recheck = - recheck_sink_admission(&self.evaluator, admission.source_proof(), snapshot.input()); - recheck - .admission_proof() - .cloned() - .ok_or_else(|| self.public_problem(operation, &snapshot, &decision)) - } - - fn checked_request<'a>( - &self, - context: &'a RequestContext, - operation: &'a ApplicationOperation, - phase: AuthorizationPhase, - observed_at: UtcMicros, - ) -> Result, ApplicationProblem> { - match context.admission_at(observed_at) { - RequestAdmission::Cancelled => { - return Err(ApplicationProblem::cancelled_before_admission()); - } - RequestAdmission::TimedOut => { - return Err(ApplicationProblem::timed_out_before_admission()); - } - RequestAdmission::Admitted => {} - } - if context.validate().is_err() - || !context.allows(operation.capability_id(), operation.use_case_id()) - { - return Err(self.denied(operation, ConcealedResourceCause::OutsideScope)); - } - - Ok(AuthorizationRequest { - context, - operation, - phase, - observed_at, - }) - } - - fn load_snapshot( - &self, - request: &AuthorizationRequest<'_>, - ) -> Result { - match self.port.source_authorization_snapshot(request) { - AuthorizationPortOutcome::Snapshot(snapshot) => Ok(*snapshot), - AuthorizationPortOutcome::Absent => { - Err(self.denied(request.operation, ConcealedResourceCause::Absent)) - } - AuthorizationPortOutcome::Unavailable(diagnostic) => { - Err(ApplicationProblem::unavailable(diagnostic)) - } - AuthorizationPortOutcome::Stale(diagnostic) => { - Err(ApplicationProblem::stale(diagnostic)) - } - } - } - - fn authorize_snapshot( - &self, - request: &AuthorizationRequest<'_>, - snapshot: SourceAuthorizationSnapshot, - ) -> Result { - let decision = self.evaluator.evaluate(snapshot.input()); - if snapshot.input().snapshot_state == AuthorizationSnapshotStateV1::Stale { - return Err(self.non_disclosure.stale_policy_problem()); - } - if !decision.is_authorized() - || decision.disposition != SourceAuthorizationDispositionV1::Allow - { - return Err(self.public_problem(request.operation, &snapshot, &decision)); - } - - let source_proof = - issue_source_authorization_proof(&self.evaluator, snapshot.input(), &decision) - .ok_or_else(|| self.non_disclosure.proof_problem())?; - let policy = self.policy_reference(&decision)?; - let receipt = AuthorityReceipt::from_context(request.context, policy, request.observed_at) - .map_err(|_| { - ApplicationProblem::invalid_request_without_action( - "application.authorization.invalid-context", - "The request context is invalid.", - ) - })?; - - Ok(AuthorizationAdmission { - receipt, - source_proof, - }) - } - - fn policy_reference( - &self, - decision: &SourceAuthorizationDecisionV1, - ) -> Result { - let evaluator_revision = ComponentVersion::new(format!( - "{}.{}", - decision.evaluator_version.evaluator_id.as_str(), - decision.evaluator_version.evaluator_revision - )) - .map_err(|_| self.non_disclosure.proof_problem())?; - PolicyDecisionRef::new( - format!( - "source-authorization.{}", - decision.evaluator_version.evaluator_id.as_str() - ), - decision.policy_revision, - decision.decision_digest.clone(), - evaluator_revision, - ) - .map_err(|_| self.non_disclosure.proof_problem()) - } - - fn public_problem( - &self, - operation: &ApplicationOperation, - snapshot: &SourceAuthorizationSnapshot, - decision: &SourceAuthorizationDecisionV1, - ) -> ApplicationProblem { - if !operation.resource_addressed() { - return ApplicationProblem::not_found_or_not_authorized(RetryDirective::Never); - } - self.non_disclosure - .source_problem(public_source_result_shape( - decision, - snapshot.source_visible(), - )) - } - - fn denied( - &self, - operation: &ApplicationOperation, - cause: ConcealedResourceCause, - ) -> ApplicationProblem { - if operation.resource_addressed() { - self.non_disclosure - .resource_problem(cause, RetryDirective::Never) - } else { - ApplicationProblem::not_found_or_not_authorized(RetryDirective::Never) - } - } -} diff --git a/crates/tracedecay-contracts/src/catalog_composition.rs b/crates/tracedecay-contracts/src/catalog_composition.rs index f2d8c64e19..6dd86fd04c 100644 --- a/crates/tracedecay-contracts/src/catalog_composition.rs +++ b/crates/tracedecay-contracts/src/catalog_composition.rs @@ -267,20 +267,14 @@ mod tests { use super::*; use crate::handlers::CanonicalApplicationDispatcher; use crate::{ApplicationOperation, ApplicationProblem, RetryDirective, SafeDiagnostic}; - use tracedecay_tool_catalog::{ - BindingStatus, CapabilityId, SurfaceBindingV1, SurfaceOperationName, - }; + use tracedecay_tool_catalog::{CapabilityId, SurfaceBindingV1, SurfaceOperationName}; fn current_bindings_on(surface: BindingSurface) -> Vec { application_catalog_contributions() .expect("application contributions") .into_iter() .flat_map(|contribution| contribution.bindings().to_vec()) - .filter(|binding| { - binding.surface() == surface - && matches!(binding.status(), BindingStatus::Current) - && !binding.is_alias() - }) + .filter(|binding| binding.surface() == surface) .collect() } diff --git a/crates/tracedecay-contracts/src/code_index_freshness.rs b/crates/tracedecay-contracts/src/code_index_freshness.rs index 60959d5f70..c370a4079a 100644 --- a/crates/tracedecay-contracts/src/code_index_freshness.rs +++ b/crates/tracedecay-contracts/src/code_index_freshness.rs @@ -221,10 +221,6 @@ pub struct CodeCloneIndexCoverageV1 { pub rename_partial_bodies: Option, /// Eligible bodies whose language has no rename normalization. pub rename_unsupported_bodies: Option, - /// Sealed source pages committed to clone indexing. - pub completed_source_pages: u64, - /// Sealed source pages in the generation. - pub total_source_pages: u64, } /// Fixed per-request clone candidate and verification budgets. @@ -242,7 +238,7 @@ pub struct CodeCloneIndexBudgetsV1 { /// Measured resources and update accounting for one clone-index generation. #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] pub struct CodeCloneIndexResourcesV1 { - /// Current durable artifact or staging-file bytes. + /// Durable artifact bytes. pub bytes_on_disk: Option, /// Largest measured clone-row serialization scratch during the build. pub peak_scratch_memory_bytes: Option, @@ -271,10 +267,6 @@ pub struct CodeCloneIndexObservationV1 { pub enum CodeCloneIndexStatusV1 { /// The sealed lexical artifact or its clone rows cannot be read. Unavailable { reason: String }, - /// A restartable clone successor is consuming sealed source pages. - Backfilling { - observation: CodeCloneIndexObservationV1, - }, /// Some clone evidence is readable, but required postings are missing. Partial { observation: CodeCloneIndexObservationV1, diff --git a/crates/tracedecay-contracts/src/code_index_freshness/clone_status_tests.rs b/crates/tracedecay-contracts/src/code_index_freshness/clone_status_tests.rs index 07486b98f5..74a3a93c22 100644 --- a/crates/tracedecay-contracts/src/code_index_freshness/clone_status_tests.rs +++ b/crates/tracedecay-contracts/src/code_index_freshness/clone_status_tests.rs @@ -7,7 +7,7 @@ fn observation() -> CodeCloneIndexObservationV1 { CodeCloneIndexObservationV1 { generation_id: "generation.clone.1".to_owned(), source_revision: Some("commit.clone.1".to_owned()), - artifact_format_revision: Some(16), + artifact_format_revision: Some(23), conservative_normalization_revision: 1, rename_normalization_revision: 1, coverage: CodeCloneIndexCoverageV1 { @@ -40,12 +40,9 @@ fn clone_readiness_preserves_all_states_and_a_complete_zero() { CodeCloneIndexStatusV1::Unavailable { reason: "artifact unreadable".to_owned(), }, - CodeCloneIndexStatusV1::Backfilling { - observation: sample.clone(), - }, CodeCloneIndexStatusV1::Partial { observation: sample.clone(), - omission_reasons: vec!["fingerprint successor missing".to_owned()], + omission_reasons: vec!["positional fingerprints missing".to_owned()], }, CodeCloneIndexStatusV1::Ready { observation: sample.clone(), @@ -65,10 +62,7 @@ fn clone_readiness_preserves_all_states_and_a_complete_zero() { }) .collect::>(); - assert_eq!( - states, - ["unavailable", "backfilling", "partial", "ready", "stale"] - ); + assert_eq!(states, ["unavailable", "partial", "ready", "stale"]); let ready = serde_json::to_value(CodeCloneIndexStatusV1::Ready { observation: observation(), }) diff --git a/crates/tracedecay-contracts/src/configuration.rs b/crates/tracedecay-contracts/src/configuration.rs index 45f4e79d72..611e0066fb 100644 --- a/crates/tracedecay-contracts/src/configuration.rs +++ b/crates/tracedecay-contracts/src/configuration.rs @@ -119,8 +119,6 @@ pub struct ConfigurationRollbackPreviewRequestV1 { pub mode: RollbackModeV1, } -pub type ConfigurationRollbackApplyRequestV1 = ConfigurationProtectedApplyRequestV1; - #[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct ConfigurationAuditRequestV1 { @@ -199,7 +197,7 @@ pub enum ConfigurationWireRequestV1 { ProtectedPreview(ConfigurationProtectedPreviewRequestV1), ProtectedApply(ConfigurationProtectedApplyRequestV1), RollbackPreview(ConfigurationRollbackPreviewRequestV1), - RollbackApply(ConfigurationRollbackApplyRequestV1), + RollbackApply(ConfigurationProtectedApplyRequestV1), Audit(ConfigurationAuditRequestV1), } @@ -515,7 +513,7 @@ fn configuration_executable_schemas( ); add!( "configuration_rollback_apply", - ConfigurationRollbackApplyRequestV1, + ConfigurationProtectedApplyRequestV1, ConfigurationMutationReceipt ); add!( diff --git a/crates/tracedecay-contracts/src/doctor/types.rs b/crates/tracedecay-contracts/src/doctor/types.rs index 3b9c9c5292..8cabf84986 100644 --- a/crates/tracedecay-contracts/src/doctor/types.rs +++ b/crates/tracedecay-contracts/src/doctor/types.rs @@ -37,8 +37,8 @@ pub enum DoctorFindingFamilyV1 { StorageRuntime, /// Storage retention, size, and efficiency over canonical observability /// read models. Distinct from [`Self::StorageRuntime`] health: this - /// family surfaces over-budget stores, identity-drift orphans, quarantined - /// incident debris, and retention backlog. The typed + /// family surfaces over-budget stores, identity-drift orphans, incident + /// debris, and retention backlog. The typed /// subclass vocabulary is [`DoctorStorageFindingKindV1`]. Storage, /// Language-server / analyzer engine status from the LSP gateway's @@ -68,8 +68,8 @@ pub enum DoctorStorageFindingKindV1 { /// A store whose project identity no longer resolves to a live repository /// root (identity-drift orphan), reported with age and size. OrphanStore, - /// Quarantined recovery/corruption artifacts are present and awaiting - /// collection. + /// Recovery/corruption artifacts are present beside a store and awaiting + /// deletion. IncidentDebrisPresent, /// Retention-eligible rows or stores are past their window and awaiting /// offload/collection. diff --git a/crates/tracedecay-contracts/src/feedback/catalog.rs b/crates/tracedecay-contracts/src/feedback/catalog.rs index 0ada961a9b..4a448e3add 100644 --- a/crates/tracedecay-contracts/src/feedback/catalog.rs +++ b/crates/tracedecay-contracts/src/feedback/catalog.rs @@ -489,38 +489,58 @@ fn schema(id: &str) -> Result { #[cfg(test)] mod tests { use super::*; + use serde_json::Value; - #[test] - fn catalog_advertises_every_transport_exposed_feedback_operation() { - let contribution = feedback_surface_catalog_contribution().expect("contribution"); - let mut names: Vec<_> = contribution - .bindings() - .iter() - .map(|binding| binding.operation().as_str().to_owned()) - .collect(); - names.sort(); - names.dedup(); - let mut expected = FEEDBACK_SPECS - .iter() - .filter(|spec| !spec.surfaces.is_empty()) - .map(|spec| spec.operation.to_owned()) - .collect::>(); - expected.sort(); - assert_eq!(names, expected); + fn property_keys(body: &Value) -> Vec<&str> { + let mut keys = body["properties"] + .as_object() + .map(|properties| properties.keys().map(String::as_str).collect::>()) + .unwrap_or_default(); + keys.sort_unstable(); + keys } #[test] fn mounted_test_results_and_advisory_cycle_have_exact_executable_schemas() { let contribution = feedback_surface_catalog_contribution().expect("contribution"); - for capability in [ - "capability.application.feedback.test-results", - ADVISORY_CYCLE_CAPABILITY_ID_V1, + for (capability, request_title, request_keys, result_title, result_keys) in [ + ( + "capability.application.feedback.test-results", + "TestResultsSurfaceRequestV1", + vec![], + "TestResultsResultV1", + vec![ + "available_results", + "code_generation_id", + "completed", + "generation", + "head_commit_id", + "operation_id", + "receipt", + "result_offset", + "results", + "termination", + "total", + ], + ), + ( + ADVISORY_CYCLE_CAPABILITY_ID_V1, + "FeedbackAdvisoryCycleSurfaceRequestV1", + vec!["document_uri"], + "FeedbackAdvisoryCycleSurfaceResultV1", + vec!["cycle", "finding_handles", "read_handles"], + ), ] { let capability = CapabilityId::new(capability).expect("capability ID"); - assert!( - contribution.executable_schema(&capability).is_some(), - "{capability} requires the exact mounted wire schema" - ); + let schema = contribution + .executable_schema(&capability) + .unwrap_or_else(|| panic!("{capability} requires the exact mounted wire schema")); + let request = schema.request_schema().body(); + let result = schema.result_schema().body(); + assert_eq!(request["title"], request_title, "{capability}"); + assert_eq!(property_keys(request), request_keys, "{capability}"); + assert_eq!(result["title"], result_title, "{capability}"); + assert_eq!(property_keys(result), result_keys, "{capability}"); } } } diff --git a/crates/tracedecay-contracts/src/feedback/mod.rs b/crates/tracedecay-contracts/src/feedback/mod.rs index 8ab6aca59f..cd69b7ea18 100644 --- a/crates/tracedecay-contracts/src/feedback/mod.rs +++ b/crates/tracedecay-contracts/src/feedback/mod.rs @@ -39,8 +39,7 @@ pub use ports::{ FeedbackDiagnosticsRequest, FeedbackImpactPort, FeedbackImpactPortOutcome, FeedbackImpactRequest, FeedbackObservationPort, FeedbackPortFuture, FeedbackPublicationReadPort, FeedbackPublicationRecordState, FeedbackPublicationV1, - FeedbackRouteAdmission, FeedbackRouteAuthorizationPort, FeedbackRuntimeStatePort, - FeedbackRuntimeStateV1, + FeedbackRouteAuthorizationPort, FeedbackRuntimeStatePort, FeedbackRuntimeStateV1, }; pub use proximity_read::{ FeedbackProximityAccessKindV1, FeedbackProximityCloneHandleV1, diff --git a/crates/tracedecay-contracts/src/feedback/ports.rs b/crates/tracedecay-contracts/src/feedback/ports.rs index b0cec51162..975c26f5d1 100644 --- a/crates/tracedecay-contracts/src/feedback/ports.rs +++ b/crates/tracedecay-contracts/src/feedback/ports.rs @@ -1,5 +1,6 @@ use std::future::Future; use std::pin::Pin; +use std::sync::Arc; use serde::{Deserialize, Serialize}; use tracedecay_domain::feedback::{ @@ -8,9 +9,7 @@ use tracedecay_domain::feedback::{ FeedbackDiagnosticV1, FeedbackDurabilityV1, FeedbackEvaluationInputV1, FeedbackImpactV1, }; use tracedecay_domain::{CodeGenerationId, UtcMicros}; -use tracedecay_policy::authorization::SourceAuthorizationEvaluator; -use crate::authorization::{AuthorizationAdmission, AuthorizationPort, AuthorizationService}; use crate::context::{RequestContext, ResolvedScope}; use crate::diagnostics::{DiagnosticProviderIdentity, DiagnosticProviderResult}; use crate::error::ApplicationContractError; @@ -20,72 +19,24 @@ use crate::result::{ApplicationProblem, AuthorityReceipt}; pub type FeedbackPortFuture<'a, T> = Pin + Send + 'a>>; /// One daemon-route authorization decision shared by feedback reads and the -/// one-shot cycle. The route owner retains the opaque admission proof and -/// reloads current authority immediately before publication; the feedback -/// service never invents or reconstructs that proof. -#[derive(Clone, Debug)] -pub enum FeedbackRouteAdmission { - /// Boxed: the full admission proof is ~3x the receipt variant, and this - /// enum travels through async port futures by value. - Source(Box), - Routed(AuthorityReceipt), -} - -impl FeedbackRouteAdmission { - pub fn receipt(&self) -> &AuthorityReceipt { - match self { - Self::Source(admission) => admission.receipt(), - Self::Routed(receipt) => receipt, - } - } -} - +/// one-shot cycle. The route owner retains its admission receipt and reloads +/// current authority immediately before publication; the feedback service +/// never invents or reconstructs that receipt. pub trait FeedbackRouteAuthorizationPort { fn admit( &self, context: &RequestContext, operation: &ApplicationOperation, observed_at: UtcMicros, - ) -> Result; - - fn recheck_publication( - &self, - context: &RequestContext, - operation: &ApplicationOperation, - admission: &FeedbackRouteAdmission, - observed_at: UtcMicros, ) -> Result; -} - -impl FeedbackRouteAuthorizationPort for AuthorizationService -where - P: AuthorizationPort, - E: SourceAuthorizationEvaluator, -{ - fn admit( - &self, - context: &RequestContext, - operation: &ApplicationOperation, - observed_at: UtcMicros, - ) -> Result { - AuthorizationService::admit(self, context, operation, observed_at) - .map(|admission| FeedbackRouteAdmission::Source(Box::new(admission))) - } fn recheck_publication( &self, context: &RequestContext, operation: &ApplicationOperation, - admission: &FeedbackRouteAdmission, + admission: &AuthorityReceipt, observed_at: UtcMicros, - ) -> Result { - let FeedbackRouteAdmission::Source(admission) = admission else { - return Err(ApplicationProblem::not_found_or_not_authorized( - crate::RetryDirective::Never, - )); - }; - AuthorizationService::recheck_publication(self, context, operation, admission, observed_at) - } + ) -> Result; } /// Runtime state resolved by a daemon-owned authority. The current clean @@ -170,17 +121,13 @@ pub trait FeedbackRuntimeStatePort { ) -> FeedbackPortFuture<'a, Option>; } -impl FeedbackRuntimeStatePort for F -where - F: Fn(&RequestContext, &FeedbackEvaluationInputV1) -> Option, -{ +impl FeedbackRuntimeStatePort for Arc { fn resolve<'a>( &'a self, context: &'a RequestContext, input: &'a FeedbackEvaluationInputV1, ) -> FeedbackPortFuture<'a, Option> { - let runtime = self(context, input); - Box::pin(async move { runtime }) + (**self).resolve(context, input) } } @@ -408,3 +355,9 @@ pub trait FeedbackPublicationReadPort { pub trait FeedbackObservationPort { fn observe(&self, input: &FeedbackEvaluationInputV1, observation: FeedbackCycleObservationV1); } + +impl FeedbackObservationPort for Arc { + fn observe(&self, input: &FeedbackEvaluationInputV1, observation: FeedbackCycleObservationV1) { + (**self).observe(input, observation); + } +} diff --git a/crates/tracedecay-contracts/src/feedback/service.rs b/crates/tracedecay-contracts/src/feedback/service.rs index 530cb7a86f..e597f70d62 100644 --- a/crates/tracedecay-contracts/src/feedback/service.rs +++ b/crates/tracedecay-contracts/src/feedback/service.rs @@ -27,8 +27,8 @@ use super::ports::{ FeedbackCycleDedupePort, FeedbackCycleDedupeState, FeedbackDiagnosticsPort, FeedbackDiagnosticsRequest, FeedbackImpactPort, FeedbackImpactPortOutcome, FeedbackImpactRequest, FeedbackObservationPort, FeedbackPublicationRecordState, - FeedbackPublicationV1, FeedbackRouteAdmission, FeedbackRouteAuthorizationPort, - FeedbackRuntimeStatePort, FeedbackRuntimeStateV1, + FeedbackPublicationV1, FeedbackRouteAuthorizationPort, FeedbackRuntimeStatePort, + FeedbackRuntimeStateV1, }; use super::problem_terminal::terminal_for_problem; @@ -187,7 +187,7 @@ impl FeedbackCycleExecutionRequest { /// Accumulated state carried between typed feedback-cycle stages. struct FeedbackCycleProgress { - admission: FeedbackRouteAdmission, + admission: AuthorityReceipt, runtime: Option, completed_stages: Vec, baselines: Vec, @@ -1159,7 +1159,7 @@ where &self, context: &RequestContext, request: &FeedbackCycleExecutionRequest, - admission: &FeedbackRouteAdmission, + admission: &AuthorityReceipt, initial_runtime: Option<&FeedbackRuntimeStateV1>, dedupe_key: Option, termination: FeedbackCycleTerminationV1, @@ -1851,7 +1851,6 @@ fn diagnostic_matches_input( fn finding_lifecycle(diagnostic: &GenerationDiagnosticV1) -> FeedbackFindingLifecycleV1 { match &diagnostic.state { DiagnosticRecordStateV1::Current => FeedbackFindingLifecycleV1::Active, - DiagnosticRecordStateV1::Superseded { .. } => FeedbackFindingLifecycleV1::Superseded, DiagnosticRecordStateV1::Cleared { .. } => FeedbackFindingLifecycleV1::Cleared, } } diff --git a/crates/tracedecay-contracts/src/graph_tool.rs b/crates/tracedecay-contracts/src/graph_tool.rs new file mode 100644 index 0000000000..ac52d9e540 --- /dev/null +++ b/crates/tracedecay-contracts/src/graph_tool.rs @@ -0,0 +1,81 @@ +//! Typed terminals for the graph and port reads whose results are their +//! catalog result schemas, plus the files they report beside the result. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::InvocationAnalyticsV1; +use crate::retrieval::{ + ContextResultV1, ImpactResultV1, NodeResultV1, PortOrderResultV1, PortStatusResultV1, + RedundancyResultV1, RenamePreviewPrimitiveOutcomeV1, SimilarResultV1, TodosResultV1, +}; + +/// One graph read's typed result, tagged by its operation. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(tag = "operation", content = "result", rename_all = "snake_case")] +pub enum GraphToolResultV1 { + Context(Box), + Node(NodeResultV1), + Impact(ImpactResultV1), + Similar(SimilarResultV1), + Redundancy(RedundancyResultV1), + RenamePreview(RenamePreviewPrimitiveOutcomeV1), + PortStatus(PortStatusResultV1), + PortOrder(PortOrderResultV1), + Todos(TodosResultV1), +} + +impl GraphToolResultV1 { + /// Decodes `operation`'s catalog result body. + pub fn from_result_value( + operation: tracedecay_tool_catalog::ApplicationSurfaceOperation, + value: serde_json::Value, + ) -> serde_json::Result { + use tracedecay_tool_catalog::ApplicationSurfaceOperation as Operation; + Ok(match operation { + Operation::Context => Self::Context(serde_json::from_value(value)?), + Operation::Node => Self::Node(serde_json::from_value(value)?), + Operation::Impact => Self::Impact(serde_json::from_value(value)?), + Operation::Similar => Self::Similar(serde_json::from_value(value)?), + Operation::Redundancy => Self::Redundancy(serde_json::from_value(value)?), + Operation::RenamePreview => Self::RenamePreview(serde_json::from_value(value)?), + Operation::PortStatus => Self::PortStatus(serde_json::from_value(value)?), + Operation::PortOrder => Self::PortOrder(serde_json::from_value(value)?), + Operation::Todos => Self::Todos(serde_json::from_value(value)?), + operation => { + return Err(serde::de::Error::custom(format!( + "{} is not a graph-tool operation", + operation.as_str() + ))); + } + }) + } + + /// The result body alone, the shape its catalog result schema names. + pub fn result_value(&self) -> serde_json::Result { + match self { + Self::Context(result) => serde_json::to_value(result), + Self::Node(result) => serde_json::to_value(result), + Self::Impact(result) => serde_json::to_value(result), + Self::Similar(result) => serde_json::to_value(result), + Self::Redundancy(result) => serde_json::to_value(result), + Self::RenamePreview(result) => serde_json::to_value(result), + Self::PortStatus(result) => serde_json::to_value(result), + Self::PortOrder(result) => serde_json::to_value(result), + Self::Todos(result) => serde_json::to_value(result), + } + } +} + +/// A completed graph read: the typed result and what it reports beside it. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct GraphToolCompletionV1 { + pub result: GraphToolResultV1, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub touched_files: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub code_graph: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub analytics: Option, +} diff --git a/crates/tracedecay-contracts/src/lib.rs b/crates/tracedecay-contracts/src/lib.rs index 7f91f67774..ff1e80cd5a 100644 --- a/crates/tracedecay-contracts/src/lib.rs +++ b/crates/tracedecay-contracts/src/lib.rs @@ -32,6 +32,7 @@ pub mod execution_topology_metrics; pub mod external_source; pub mod feedback; pub mod git; +pub mod graph_tool; pub mod handlers; pub mod handoff; pub mod handoff_catalog; @@ -95,9 +96,7 @@ mod error; mod surface_binding; pub mod surface_contracts; -pub(crate) use surface_binding::{ - current_application_bindings, current_bindings, current_bindings_with_slug, surface_name, -}; +pub(crate) use surface_binding::{current_application_bindings, current_bindings, surface_name}; pub use advisory::{ AdvisoryFindingContributionBatchV1, AdvisoryFindingContributorV1, @@ -116,25 +115,21 @@ pub use advisory::{ ProximityBranchWorktreeIncompatibilityV1, ProximityContributionIdV1, ProximityContributionV1, ProximityCoverageV1, ProximityInclusionV1, ProximityObservationIdV1, ProximityRelationPathKindV1, ProximityRelationPathV1, ProximityRelationStrengthV1, - ProximityRiskInputsV1, ProximityTierV1, ProximityWarningClassV1, ProximityWarningIdV1, -}; -pub use authorization::{ - AuthorizationAdmission, AuthorizationPhase, AuthorizationPort, AuthorizationPortOutcome, - AuthorizationRequest, AuthorizationService, ConcealedResourceCause, NonDisclosureHooks, - SourceAuthorizationSnapshot, + ProximityRiskInputsV1, ProximityTierV1, ProximityWarningClassV1, }; +pub use authorization::AuthorizationRequest; pub use clock::{ClockError, now_micros, try_now_micros}; pub use configuration::{ ActivationDriftV1, ComponentConfigurationState, ConfigurationAuditPage, ConfigurationAuditRequestV1, ConfigurationBatchRequestV1, ConfigurationDirectMutationRequestV1, ConfigurationGetRequestV1, ConfigurationListRequestV1, ConfigurationMutationReceipt, ConfigurationObservedStateRequestV1, ConfigurationProtectedApplyRequestV1, - ConfigurationProtectedPreviewRequestV1, ConfigurationRollbackApplyRequestV1, - ConfigurationRollbackPreviewRequestV1, ConfigurationSetRequestV1, ConfigurationUnsetRequestV1, - ConfigurationWireRequestV1, ResolvedSetting, SettingSummary, - configuration_surface_catalog_contribution, configuration_surface_handler_descriptors, - configuration_surface_operation, configuration_surface_request_schema, - configuration_surface_result_schema, configuration_wire_request_from_invocation_payload, + ConfigurationProtectedPreviewRequestV1, ConfigurationRollbackPreviewRequestV1, + ConfigurationSetRequestV1, ConfigurationUnsetRequestV1, ConfigurationWireRequestV1, + ResolvedSetting, SettingSummary, configuration_surface_catalog_contribution, + configuration_surface_handler_descriptors, configuration_surface_operation, + configuration_surface_request_schema, configuration_surface_result_schema, + configuration_wire_request_from_invocation_payload, }; pub use context::{ APPLICATION_REQUEST_ID_HEADER, ApplicationRequestControlV1, CancellationContext, @@ -338,17 +333,17 @@ pub use result::{ APPLICATION_PROBLEM_REVISION, ApplicationEnvelope, ApplicationExecutionFailureClassV1, ApplicationOutcome, ApplicationProblem, ApplicationProblemEnvelope, ApplicationProblemKind, ApplicationProblemRecord, ApplicationResult, ApplicationUnavailableClassV1, AuthorityReceipt, - BudgetClass, CancellationObservation, CancellationStage, CoverageCompleteness, - CoverageDomainState, EffectId, EffectReceipt, EffectResult, EffectTermination, - EvidenceAuthority, EvidenceCoverage, EvidenceDomain, EvidenceIdentity, EvidencePacket, - EvidenceScore, EvidenceScoreKind, EvidenceScoreValue, FreshnessState, IdempotencyKey, - LegalAction, Omission, OmissionReason, OpaqueCursor, OperationBudgetUsage, OperationReceipt, - OperationTermination, PageCursor, PageState, PolicyDecisionRef, PreviewId, PreviewResult, - ProblemOwningLayer, ProblemTerminality, RUNTIME_MOUNTING_REASON_CODE, ReconciliationState, - ResultContractRef, ResumeToken, RetrievalEvidence, RetrieverContribution, - RetrieverContributionState, RetryDirective, RetryScope, SafeDiagnostic, ScoreId, StreamEvent, - StreamEventKind, StreamFrontier, StreamGap, StreamTermination, StreamValidationError, - TemporalState, validate_stream, + BudgetClass, CancellationObservation, CancellationStage, ContextMemoryAnalyticsV1, + CoverageCompleteness, CoverageDomainState, EffectId, EffectReceipt, EffectResult, + EffectTermination, EvidenceAuthority, EvidenceCoverage, EvidenceDomain, EvidenceIdentity, + EvidencePacket, EvidenceScore, EvidenceScoreKind, EvidenceScoreValue, FreshnessState, + IdempotencyKey, InvocationAnalyticsV1, LegalAction, Omission, OmissionReason, OpaqueCursor, + OperationBudgetUsage, OperationReceipt, OperationTermination, PageCursor, PageState, + PolicyDecisionRef, PreviewId, PreviewResult, ProblemOwningLayer, ProblemTerminality, + RUNTIME_MOUNTING_REASON_CODE, ReconciliationState, ResultContractRef, ResumeToken, + RetrievalEvidence, RetrieverContribution, RetrieverContributionState, RetryDirective, + RetryScope, SafeDiagnostic, ScoreId, StreamEvent, StreamEventKind, StreamFrontier, StreamGap, + StreamTermination, StreamValidationError, TemporalState, validate_stream, }; pub use retained_receipts::{ PreparedRetainedEffect, authority_receipt, effective_memory_deadline, evidence_outcome, @@ -361,10 +356,9 @@ pub use retained_surfaces::{ RetainedSurfaceExecutionContextV1, RetainedSurfaceExecutionErrorV1, RetainedSurfaceExecutionFutureV1, RetainedSurfaceOperation, RetainedSurfacePortsV1, RetainedSurfaceServiceV1, retained_surface_application_operation, - retained_surface_catalog_contribution, retained_surface_executable_binding_registry, - retained_surface_execution_problem, retained_surface_handler_descriptors, - retained_surface_operation_is_effect, retained_surface_outcome_matches_terminal, - retained_surface_problem_matches_terminal, + retained_surface_catalog_contribution, retained_surface_execution_problem, + retained_surface_handler_descriptors, retained_surface_operation_is_effect, + retained_surface_outcome_matches_terminal, retained_surface_problem_matches_terminal, }; pub use retrieval::catalog::{ APPLICATION_ADMINISTRATIVE_PROFILE_ID, APPLICATION_COMPACT_PROFILE_ID, @@ -373,24 +367,23 @@ pub use retrieval::catalog::{ }; pub use retrieval::{ AffectedTestsRequest, AffectedTestsRetrievalPort, AnchorExpandRequest, AnchorExpandResult, - CALLABLE_CODE_OPERATION_COUNT, CallableCodeAuthorizationAdmission, - CallableCodeAuthorizationFuture, CallableCodeAuthorizationPort, CallableCodeOperationKind, - CallableCodeOperations, CallableCodeQueryFuture, CallableCodeQueryPort, - CallableCodeQueryService, CodeFacetDimension, CodeFacetRecord, CodeFacetRequest, - CodeHierarchyRequest, CodeImpactRequest, CodeImplementationsRequest, CodeLexicalField, - CodeLexicalFieldFilter, CodeNavigationRequest, CodeOccurrenceRecord, CodeQueryPage, - CodeQueryScope, CodeRelationRequest, CodeSignatureRequest, CodeSymbolSearchRequest, - CodeTimelineRecord, CodeTimelineRequest, ExactOccurrenceRecord, ExactOccurrenceRequest, - GraphImpactResult, HealthDeltaCoverageV1, HealthDeltaCurrentnessV1, HealthDeltaPointV1, - HealthDeltaRequest, HealthDeltaResult, HealthDeltaScopeV1, HealthDimensionDeltaV1, - HealthDimensionPointV1, HealthReadRequest, LexicalOccurrenceRecord, MAX_APPLICATION_PAGE_SIZE, - ModuleApiRequest, OperationalRetrievalPort, PageRequest, PhraseSearchRequest, - QualifiedNameRequest, ResultProjection, RetrievalOrder, RetrievalPortContext, - RetrievalPortOutcome, RetrievalRequestMeta, SessionLookupRequest, SourceLinesRequest, - SourceLinesResult, SourceMetadataRecord, SourceMetadataRequest, SourceRetrievalPort, - TemporalRetrievalPort, UNPINNED_LATEST_GENERATION_SENTINEL, callable_code_catalog_contribution, - callable_code_handler_descriptors, callable_code_operation, callable_code_operations, - callable_code_request_schema, callable_code_result_schema, + CALLABLE_CODE_OPERATION_COUNT, CallableCodeAuthorizationFuture, CallableCodeAuthorizationPort, + CallableCodeOperationKind, CallableCodeOperations, CallableCodeQueryFuture, + CallableCodeQueryPort, CallableCodeQueryService, CodeFacetDimension, CodeFacetRecord, + CodeFacetRequest, CodeHierarchyRequest, CodeImpactRequest, CodeImplementationsRequest, + CodeLexicalField, CodeLexicalFieldFilter, CodeNavigationRequest, CodeOccurrenceRecord, + CodeQueryPage, CodeQueryScope, CodeRelationRequest, CodeSignatureRequest, + CodeSymbolSearchRequest, CodeTimelineRecord, CodeTimelineRequest, ExactOccurrenceRecord, + ExactOccurrenceRequest, GraphImpactResult, HealthDeltaCoverageV1, HealthDeltaCurrentnessV1, + HealthDeltaPointV1, HealthDeltaRequest, HealthDeltaResult, HealthDeltaScopeV1, + HealthDimensionDeltaV1, HealthDimensionPointV1, HealthReadRequest, LexicalOccurrenceRecord, + MAX_APPLICATION_PAGE_SIZE, ModuleApiRequest, OperationalRetrievalPort, PageRequest, + PhraseSearchRequest, QualifiedNameRequest, ResultProjection, RetrievalOrder, + RetrievalPortContext, RetrievalPortOutcome, RetrievalRequestMeta, SessionLookupRequest, + SourceLinesRequest, SourceLinesResult, SourceMetadataRecord, SourceMetadataRequest, + SourceRetrievalPort, TemporalRetrievalPort, UNPINNED_LATEST_GENERATION_SENTINEL, + callable_code_catalog_contribution, callable_code_handler_descriptors, callable_code_operation, + callable_code_operations, callable_code_request_schema, callable_code_result_schema, }; pub use sdk_catalog::{ application_http_executable_binding_registry, application_http_route_path, @@ -421,11 +414,10 @@ pub use source_edit_rollback::{SourceEditRollbackRequestV1, source_edit_rollback pub use storage::{ CompactionDecisionV1, CompactionPlacementV1, CompactionTriggerPolicyV1, FreePageRatioV1, IncidentDebrisArtifactV1, IncidentDebrisKindV1, IncidentDebrisScanV1, OrphanStoreRecordV1, - QuarantineContractV1, QuarantineLocationV1, QuarantinedArtifactV1, RelativeArtifactPathV1, - RetentionBacklogRecordV1, StorageByteSizeV1, StorageTelemetryFuture, StorageTelemetryReadV1, - StoreBudgetEvaluationV1, StoreKeyV1, StoreSizeBudgetV1, StoreSizeSampleV1, - StoreSizeTelemetryPort, TableGrowthSampleV1, TableNameV1, incident_debris_finding, - orphan_store_finding, over_budget_finding, retention_backlog_finding, + RelativeArtifactPathV1, RetentionBacklogRecordV1, StorageByteSizeV1, StorageTelemetryFuture, + StorageTelemetryReadV1, StoreBudgetEvaluationV1, StoreKeyV1, StoreSizeBudgetV1, + StoreSizeSampleV1, StoreSizeTelemetryPort, TableGrowthSampleV1, TableNameV1, + incident_debris_finding, orphan_store_finding, over_budget_finding, retention_backlog_finding, }; pub use surface_contracts::{ CallableCodeSurfaceMeta, CallableCodeSurfaceRequest, CodeCalleesSurfaceRequest, @@ -435,10 +427,7 @@ pub use surface_contracts::{ CodeSymbolSearchSurfaceRequest, CodeTimelineSurfaceRequest, CodeTypeHierarchySurfaceRequest, NativeIntegrationSurfaceRequest, PrimitiveCodeSurfaceRequest, primitive_code_into_primitive, }; -pub use work::{ - ReviewProposalDispositionV1, WorkRoutingSnapshotErrorV1, WorkRoutingSnapshotPortV1, - WorkRoutingSnapshotV1, -}; +pub use work::{WorkRoutingSnapshotErrorV1, WorkRoutingSnapshotPortV1, WorkRoutingSnapshotV1}; pub use work_artifact_hydration::{ WorkArtifactHydrationRequestV1, WorkArtifactHydrationService, WorkArtifactHydrationV1, WorkAttemptArtifactsV1, WorkAttemptEvidencePageV1, WorkAttemptEvidenceReadPort, @@ -539,15 +528,16 @@ pub use work_product::{ WorkGraphTimelineV1, WorkGraphVersionEntryV1, WorkHistoryCoverageV1, WorkHistoryReadPortV1, WorkHistoryRequestV1, WorkHistoryServiceV1, WorkHistoryV1, WorkProductApplicationErrorV1, WorkProductAttemptAdmissionErrorV1, WorkProductAttemptAdmissionOutcomeV1, - WorkProductAttemptAdmissionPortV1, WorkProductAttemptAdmissionV1, WorkProductBindingV1, - WorkProductChangeDraftV1, WorkProductEventCommitOutcomeV1, WorkProductEventCommitV1, - WorkProductEventDraftV1, WorkProductEventPortErrorV1, WorkProductEventPortV1, - WorkProductEvidenceServiceV1, WorkProductExpectedAuthorityV1, WorkProductMutationIdentityV1, - WorkProductMutationReceiptV1, WorkProductMutationRequestV1, WorkProductMutationServiceV1, + WorkProductAttemptAdmissionPortV1, WorkProductAttemptAdmissionV1, + WorkProductAuthorizedRelationScopeV1, WorkProductBindingV1, WorkProductChangeDraftV1, + WorkProductEventCommitOutcomeV1, WorkProductEventCommitV1, WorkProductEventDraftV1, + WorkProductEventPortErrorV1, WorkProductEventPortV1, WorkProductEvidenceServiceV1, + WorkProductExpectedAuthorityV1, WorkProductMutationIdentityV1, WorkProductMutationReceiptV1, + WorkProductMutationRequestV1, WorkProductMutationServiceV1, WorkProductOwnerAuthorizationErrorV1, WorkProductOwnerAuthorizationPortV1, WorkProductPortContextV1, WorkProductReadServiceV1, WorkProductRetryAdmissionV1, WorkProductRevisionPinsV1, WorkProductSelectionScopeV1, WorkProductSynthesisAdmissionV1, - WorkRelationScopeV1, work_product_projection_generation, + work_product_projection_generation, }; pub use work_retry::{ RetryWorkAttemptCommandV1, RuntimeWorkRetryEvidenceV1, VerifiedWorkRetryFailureV1, diff --git a/crates/tracedecay-contracts/src/lsp_context_catalog.rs b/crates/tracedecay-contracts/src/lsp_context_catalog.rs index 79b13a16e9..1c1890c38c 100644 --- a/crates/tracedecay-contracts/src/lsp_context_catalog.rs +++ b/crates/tracedecay-contracts/src/lsp_context_catalog.rs @@ -1,11 +1,11 @@ use tracedecay_tool_catalog::{ - AvailabilityContract, BindingId, BindingStatus, BindingSurface, CancellationContract, - CancellationPoint, CapabilityId, CatalogContributionInputV1, CatalogContributionV1, - ContributionId, DeadlineBehavior, DeadlineContract, DeniedDisclosurePolicy, EffectClass, - FeatureId, LifecycleClass, PaginationContract, PrivacyClass, ProtocolRevisionRange, - RevalidationContract, RevalidationPoint, RoutingContractV1, SchemaId, SchemaRef, - ScopeDimension, ScopeRequirement, StreamingContract, SurfaceBindingInputV1, SurfaceBindingV1, - SurfaceOperationName, TerminalState, TerminalStateContract, UseCaseId, + AvailabilityContract, BindingId, BindingSurface, CancellationContract, CancellationPoint, + CapabilityId, CatalogContributionInputV1, CatalogContributionV1, ContributionId, + DeadlineBehavior, DeadlineContract, DeniedDisclosurePolicy, EffectClass, FeatureId, + LifecycleClass, PaginationContract, PrivacyClass, ProtocolRevisionRange, RevalidationContract, + RevalidationPoint, RoutingContractV1, SchemaId, SchemaRef, ScopeDimension, ScopeRequirement, + StreamingContract, SurfaceBindingInputV1, SurfaceBindingV1, SurfaceOperationName, + TerminalState, TerminalStateContract, UseCaseId, }; use crate::capability_manifest::{ @@ -66,8 +66,6 @@ pub fn lsp_context_catalog_contribution() -> Result impl Future, Self::Error>> + Send; + ) -> impl Future, Self::Error>> + Send; } pub struct MemoryApplication

{ @@ -613,7 +613,7 @@ impl MemoryApplication

{ pub async fn get_retrieval_anchor( &self, query: MemoryRetrievalAnchorQuery, - ) -> Result, MemoryUseCaseError> { + ) -> Result, MemoryUseCaseError> { let MemoryRetrievalAnchorQuery { owner, anchor_id, diff --git a/crates/tracedecay-contracts/src/remote/query.rs b/crates/tracedecay-contracts/src/remote/query.rs index 3e18d12a1c..44ca102b59 100644 --- a/crates/tracedecay-contracts/src/remote/query.rs +++ b/crates/tracedecay-contracts/src/remote/query.rs @@ -14,7 +14,7 @@ use tracedecay_domain::{ CanonicalObservationIdV1, CurrentRemoteAuthorityStateV1, DurableObservationV1, EvidenceAvailabilityV1, GenerationBoundRepositoryProvenanceV1, ObservationScopeV1, ObservationSourceCursorV1, ProjectionGenerationId, RemoteCapabilityV1, RemoteRepositoryScopeV1, - RemoteWriterFenceV1, RetrievalAnchorRecordV2, UtcMicros, + RemoteWriterFenceV1, RetrievalAnchorRecord, UtcMicros, }; use tracedecay_tool_catalog::SchemaId; @@ -188,10 +188,10 @@ pub struct RemoteSanitizedObservationV1 { pub sequence: u64, pub observation: DurableObservationV1, pub committed_cursor: ObservationSourceCursorV1, - pub retrieval_anchor: RetrievalAnchorRecordV2, + pub retrieval_anchor: RetrievalAnchorRecord, pub projection_generation: ProjectionGenerationId, pub repository_provenance: EvidenceAvailabilityV1, - pub repository_anchor: Option, + pub repository_anchor: Option, pub projection_queued: bool, } @@ -603,7 +603,7 @@ impl RemoteExactObservationQueryServiceV1 { &row.repository_provenance, row.repository_anchor .as_ref() - .map(RetrievalAnchorRecordV2::projection_generation), + .map(RetrievalAnchorRecord::projection_generation), request.body.observation_id(), &command.expected_authority.generation_id, &request.body.scope, @@ -779,7 +779,9 @@ fn query_payload( ) -> Option<&RemoteQueryResultV1> { match &envelope.outcome { ApplicationOutcome::Evidence(packet) => packet.payload.as_ref(), - ApplicationOutcome::Preview(_) | ApplicationOutcome::Effect(_) => None, + ApplicationOutcome::Preview(_) + | ApplicationOutcome::Effect(_) + | ApplicationOutcome::Result(_) => None, } } diff --git a/crates/tracedecay-contracts/src/result/analytics.rs b/crates/tracedecay-contracts/src/result/analytics.rs new file mode 100644 index 0000000000..ce7d844215 --- /dev/null +++ b/crates/tracedecay-contracts/src/result/analytics.rs @@ -0,0 +1,50 @@ +//! Analytics an operation reports beside its result for the invocation +//! ledger. Surfaces record them; they never render into the result a client +//! reads. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; + +#[derive(Clone, Debug, Default, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct InvocationAnalyticsV1 { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub context_memory: Option, +} + +impl InvocationAnalyticsV1 { + /// The metadata object the invocation ledger records for this call. + pub fn ledger_value(&self) -> Value { + let mut value = json!({}); + if let Some(context_memory) = &self.context_memory { + value["context_memory"] = context_memory.ledger_value(); + } + value + } +} + +/// How a context call searched project memory and which facts it surfaced. +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct ContextMemoryAnalyticsV1 { + pub include_memory: bool, + pub limit: u32, + pub min_trust_millionths: u32, + pub fact_ids: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +impl ContextMemoryAnalyticsV1 { + fn ledger_value(&self) -> Value { + json!({ + "include_memory": self.include_memory, + "limit": self.limit, + "min_trust": f64::from(self.min_trust_millionths) / 1_000_000.0, + "match_count": self.fact_ids.len(), + "fact_ids": self.fact_ids, + "error": self.error, + }) + } +} diff --git a/crates/tracedecay-contracts/src/result/envelope.rs b/crates/tracedecay-contracts/src/result/envelope.rs index 65faa8cf5d..f0e9922797 100644 --- a/crates/tracedecay-contracts/src/result/envelope.rs +++ b/crates/tracedecay-contracts/src/result/envelope.rs @@ -91,6 +91,9 @@ pub enum ApplicationOutcome { Evidence(EvidencePacket), Preview(PreviewResult), Effect(EffectResult), + /// An operation-owned typed result that carries its own receipts, such as + /// a source-edit result with its durable effect, or a rendered document. + Result(T), } impl ApplicationOutcome { @@ -99,6 +102,7 @@ impl ApplicationOutcome { Self::Evidence(result) => result.payload.as_ref(), Self::Preview(result) => result.payload.as_ref(), Self::Effect(result) => result.payload.as_ref(), + Self::Result(result) => Some(result), } } } @@ -111,6 +115,16 @@ pub struct ApplicationEnvelope { pub request_id: RequestId, pub scope: ResolvedScope, pub outcome: ApplicationOutcome, + /// Project-relative files the operation read, reported for session + /// activity beside the result rather than inside it. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub touched_files: Vec, + /// The code-graph generation a graph-backed operation served, so every + /// surface can report a stale seat beside the result. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub code_graph: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub analytics: Option, } impl ApplicationEnvelope { @@ -125,6 +139,9 @@ impl ApplicationEnvelope { request_id, scope, outcome: ApplicationOutcome::Evidence(packet), + touched_files: Vec::new(), + code_graph: None, + analytics: None, } } @@ -139,6 +156,9 @@ impl ApplicationEnvelope { request_id, scope, outcome: ApplicationOutcome::Preview(preview), + touched_files: Vec::new(), + code_graph: None, + analytics: None, } } @@ -153,6 +173,9 @@ impl ApplicationEnvelope { request_id, scope, outcome: ApplicationOutcome::Effect(effect), + touched_files: Vec::new(), + code_graph: None, + analytics: None, } } } diff --git a/crates/tracedecay-contracts/src/result/mod.rs b/crates/tracedecay-contracts/src/result/mod.rs index d59a83277c..777e09fc61 100644 --- a/crates/tracedecay-contracts/src/result/mod.rs +++ b/crates/tracedecay-contracts/src/result/mod.rs @@ -1,9 +1,11 @@ +mod analytics; mod envelope; mod evidence; mod problem; mod receipt; mod stream; +pub use analytics::{ContextMemoryAnalyticsV1, InvocationAnalyticsV1}; pub use envelope::{ APPLICATION_PROBLEM_REVISION, ApplicationEnvelope, ApplicationOutcome, ApplicationProblemEnvelope, ApplicationProblemRecord, ApplicationResult, MAX_PROBLEM_DETAILS, diff --git a/crates/tracedecay-contracts/src/retained_receipts.rs b/crates/tracedecay-contracts/src/retained_receipts.rs index afd31411ab..978ea354c5 100644 --- a/crates/tracedecay-contracts/src/retained_receipts.rs +++ b/crates/tracedecay-contracts/src/retained_receipts.rs @@ -9,7 +9,7 @@ use tracedecay_tool_catalog::{EffectClass, SortContractId}; use crate::retained_surfaces::{ RetainedSurfaceEvidenceFactsV1, RetainedSurfaceEvidenceTerminalV1, RetainedSurfaceOperation, - RetainedSurfaceResultV1, RetainedSurfaceTemporalRequestV1, SessionCoverageModeV1, + RetainedSurfaceResultV1, RetainedSurfaceTemporalRequestV1, }; use crate::{ ApplicationOutcome, AuthorityReceipt, CancellationStage, CoverageDomainState, Deadline, @@ -600,14 +600,7 @@ fn temporal_request_mode( "the temporal coverage requests disagreed on their mode", )); } - Ok(match first.mode { - SessionCoverageModeV1::Current => TemporalModeV1::Current, - SessionCoverageModeV1::AsOf { cutoff } => TemporalModeV1::AsOf { - cutoff: tracedecay_domain::UtcMicros(cutoff), - }, - SessionCoverageModeV1::Evolution => TemporalModeV1::Evolution, - SessionCoverageModeV1::Forensic => TemporalModeV1::Forensic, - }) + Ok(first.mode) } struct CountingSink { diff --git a/crates/tracedecay-contracts/src/retained_surfaces.rs b/crates/tracedecay-contracts/src/retained_surfaces.rs index 4ac163e660..35f35620bb 100644 --- a/crates/tracedecay-contracts/src/retained_surfaces.rs +++ b/crates/tracedecay-contracts/src/retained_surfaces.rs @@ -7,16 +7,14 @@ use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use tracedecay_tool_catalog::{ - AvailabilityContract, BindingId, BindingStatus, BindingSurface, CancellationContract, - CancellationPoint, CapabilityId, CapabilityManifestV1, CatalogContributionInputV1, - CatalogContributionV1, CodecBindingKey, ContributionId, DeadlineBehavior, DeadlineContract, - DeniedDisclosurePolicy, EffectClass, ExecutableBindingAvailabilityV1, - ExecutableBindingRegistryV1, ExecutableBindingV1, ExecutableSchemaAuthority, LifecycleClass, - OperationId, PaginationContract, PrivacyClass, ProfileId, ProtocolRevisionRange, - RevalidationContract, RevalidationPoint, RouteExposureV1, RoutingContractV1, SchemaId, - SchemaRef, ScopeDimension, ScopeRequirement, ServiceId, StreamingContract, - SurfaceBindingInputV1, SurfaceBindingV1, SurfaceOperationName, TerminalState, - TerminalStateContract, UseCaseId, + ApplicationSurfaceOperation, AvailabilityContract, BindingId, BindingSurface, + CancellationContract, CancellationPoint, CapabilityId, CapabilityManifestV1, + CatalogContributionInputV1, CatalogContributionV1, ContributionId, DeadlineBehavior, + DeadlineContract, DeniedDisclosurePolicy, EffectClass, ExecutableSchemaAuthority, + LifecycleClass, PaginationContract, PrivacyClass, ProfileId, ProtocolRevisionRange, + RevalidationContract, RevalidationPoint, RoutingContractV1, SchemaId, SchemaRef, + ScopeDimension, ScopeRequirement, StreamingContract, SurfaceBindingInputV1, SurfaceBindingV1, + SurfaceOperationName, TerminalState, TerminalStateContract, UseCaseId, }; use crate::capability_manifest::{ @@ -155,6 +153,13 @@ impl RetainedSurfaceOperation { } } + /// The retained operation an application-surface operation names. + pub fn from_application(operation: ApplicationSurfaceOperation) -> Option { + Self::ALL + .into_iter() + .find(|candidate| candidate.as_str() == operation.as_str()) + } + #[hotpath::skip] pub const fn as_str(self) -> &'static str { match self { @@ -265,8 +270,6 @@ pub fn retained_surface_catalog_contribution() operation: SurfaceOperationName::new(spec.operation.as_str())?, protocol_revisions: ProtocolRevisionRange::new(1, 1)?, required_features: Vec::new(), - status: BindingStatus::Current, - alias_of: None, })?); binding_ids.push(binding_id); } @@ -282,58 +285,6 @@ pub fn retained_surface_catalog_contribution() Ok(contribution.with_executable_schemas(schemas)?) } -/// Daemon-owned public HTTP bindings for retained V2 operations with a -/// project-opened execution port and exact raw-handler proof. -pub fn retained_surface_executable_binding_registry() --> Result { - let contribution = retained_surface_catalog_contribution()?; - let service_id = ServiceId::new("service.application.retained")?; - let mut bindings = Vec::with_capacity(RetainedSurfaceOperation::SDK_EXECUTABLE.len()); - for operation in RetainedSurfaceOperation::SDK_EXECUTABLE { - let capability_id = CapabilityId::new(capability_id(operation))?; - let manifest = contribution - .capabilities() - .iter() - .find(|manifest| manifest.capability_id() == &capability_id) - .ok_or(ApplicationContractError::Inconsistent { - field: "retained executable capability", - })?; - let schema = contribution.executable_schema(&capability_id).ok_or( - ApplicationContractError::Inconsistent { - field: "retained executable schema", - }, - )?; - let http_binding = contribution - .bindings() - .iter() - .find(|binding| { - binding.capability_id() == &capability_id - && binding.surface() == BindingSurface::Http - }) - .ok_or(ApplicationContractError::Inconsistent { - field: "retained HTTP binding", - })?; - bindings.push(ExecutableBindingAvailabilityV1::available( - ExecutableBindingV1::daemon_owned( - manifest, - OperationId::new(format!("operation.application.{}", operation.as_str()))?, - service_id.clone(), - schema.request_schema().clone(), - schema.result_schema().clone(), - CodecBindingKey::new(format!( - "codec.application.retained.{}.json.v1", - operation.as_str() - ))?, - RouteExposureV1::Public { - binding_id: http_binding.binding_id().clone(), - route_path: format!("/application/retained/{}", operation.as_str()), - }, - )?, - )); - } - Ok(ExecutableBindingRegistryV1::new(bindings)?) -} - fn retained_surface_executable_schemas( contribution: &CatalogContributionV1, ) -> Result, ApplicationContractError> { @@ -594,7 +545,7 @@ pub fn retained_surface_outcome_matches_terminal( && effect.receipt.operation == *application_operation.use_case_id() && effect.receipt.scope == *scope } - crate::ApplicationOutcome::Preview(_) => false, + crate::ApplicationOutcome::Preview(_) | crate::ApplicationOutcome::Result(_) => false, } } @@ -724,7 +675,9 @@ fn capability( fn handler_descriptor( spec: &RetainedSurfaceSpec, ) -> Result { - ApplicationHandlerDescriptor::new( + ApplicationHandlerDescriptor::for_catalog_operation( + spec.operation.as_str(), + "service.application.retained", application_operation(spec)?, schema(spec.operation, "request")?, schema(spec.operation, "result")?, @@ -772,6 +725,8 @@ fn use_case_id(operation: RetainedSurfaceOperation) -> String { #[cfg(test)] mod tests { + use tracedecay_tool_catalog::{ExecutableBindingAvailabilityV1, RouteExposureV1}; + use super::*; #[test] @@ -866,7 +821,7 @@ mod tests { #[test] fn fact_store_curate_is_the_only_public_automation_launcher() { let contribution = retained_surface_catalog_contribution().expect("contribution"); - let registry = retained_surface_executable_binding_registry().expect("registry"); + let registry = crate::application_http_executable_binding_registry().expect("registry"); let operation = RetainedSurfaceOperation::FactStoreCurate; let capability = CapabilityId::new(capability_id(operation)).expect("capability id"); let request_type = "tracedecay_contracts::retained_surfaces::FactStoreCurateRequestV1"; @@ -983,11 +938,7 @@ mod tests { #[test] fn every_mounted_retained_action_is_sdk_executable() { - let registry = retained_surface_executable_binding_registry().expect("registry"); - assert_eq!( - registry.iter().count(), - RetainedSurfaceOperation::SDK_EXECUTABLE.len() - ); + let registry = crate::application_http_executable_binding_registry().expect("registry"); for operation in RetainedSurfaceOperation::SDK_EXECUTABLE { let operation_id = format!("operation.application.{}", operation.as_str()); let binding = registry diff --git a/crates/tracedecay-contracts/src/retained_surfaces/evidence.rs b/crates/tracedecay-contracts/src/retained_surfaces/evidence.rs index 1c0d557058..0503c85c0e 100644 --- a/crates/tracedecay-contracts/src/retained_surfaces/evidence.rs +++ b/crates/tracedecay-contracts/src/retained_surfaces/evidence.rs @@ -5,15 +5,16 @@ //! use it to build the common application envelope without upgrading a //! bounded result into fabricated complete evidence. +use tracedecay_domain::TemporalModeV1; + use crate::{ CoverageCompleteness, EvidenceDomain, FreshnessState, OmissionReason, OpaqueCursor, PageCursor, }; use super::{ HydrationStateResultV1, LcmRetrievalOutcomeV1, LcmTemporalFieldsV1, RetainedOutcomeStatusV1, - RetainedSurfaceResultV1, SessionCoverageModeV1, SessionRefreshStatusResultV1, - SessionRefreshTerminalStateResultV1, SessionSourceCoverageV1, TemporalFreshnessV1, - TemporalMetadataV1, TemporalWatermarksV1, + RetainedSurfaceResultV1, SessionRefreshStatusResultV1, SessionRefreshTerminalStateResultV1, + SessionSourceCoverageV1, TemporalFreshnessV1, TemporalMetadataV1, TemporalWatermarksV1, }; #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -41,7 +42,7 @@ pub struct RetainedSurfaceEvidenceOmissionV1 { #[derive(Clone, Debug, PartialEq, Eq)] pub struct RetainedSurfaceTemporalRequestV1 { pub source_id: String, - pub mode: SessionCoverageModeV1, + pub mode: TemporalModeV1, } /// Exact temporal authority carried by retained session results. @@ -571,7 +572,7 @@ const fn omission_reason(value: HydrationStateResultV1) -> Option Some(OmissionReason::Unavailable), + | HydrationStateResultV1::Unverifiable => Some(OmissionReason::Unavailable), } } @@ -612,10 +613,7 @@ mod tests { results: Option>, ) -> MessageSearchResultV1 { MessageSearchResultV1 { - catch_up: false, - catch_up_failures: Vec::new(), - catch_up_performed: false, - catch_up_provider: "all".to_owned(), + require_fresh: false, count, goals: false, include_subagents: true, @@ -638,14 +636,7 @@ mod tests { git_filter_applied: None, message: None, omitted: None, - project_scope: None, - registry_truncated: None, - roots: None, - searched_project_count: None, - selected_project_root: None, service_status: None::, - skipped: None, - skipped_project_count: None, store_scope: None, temporal: None, workflow_agent: None, diff --git a/crates/tracedecay-contracts/src/retained_surfaces/sdk.rs b/crates/tracedecay-contracts/src/retained_surfaces/sdk.rs index 19119ec77a..5d5960bee0 100644 --- a/crates/tracedecay-contracts/src/retained_surfaces/sdk.rs +++ b/crates/tracedecay-contracts/src/retained_surfaces/sdk.rs @@ -62,10 +62,9 @@ pub use results::{ MemoryAutomationFactReceiptV1, MemoryAutomationFactRequestV1, MemoryAutomationFactStateV1, MemoryAutomationFactTargetV1, MemoryAutomationFactValidationStatusV1, MemoryAutomationFactValidationV1, MemoryFeedbackFunnelV1, MemoryStatusResultV1, MemoryStatusV1, - MessageSearchFreshnessV1, MessageSearchHitV1, MessageSearchResultV1, MessageSearchRootV1, - MessageSearchSkipV1, RetainedErrorV1, RetainedNextActionV1, RetainedOutcomeStatusV1, - RetainedSurfaceResultV1, RetrievalWorkerStatusV1, SessionCorrelationHitV1, - SessionCoverageIntervalV1, SessionCoverageModeV1, SessionCoverageReasonV1, + MessageSearchHitV1, MessageSearchResultV1, RetainedErrorV1, RetainedNextActionV1, + RetainedOutcomeStatusV1, RetainedSurfaceResultV1, RetrievalWorkerStatusV1, + SessionCorrelationHitV1, SessionCoverageIntervalV1, SessionCoverageReasonV1, SessionCoverageRequestV1, SessionCoverageStateV1, SessionMessageV1, SessionRecordV1, SessionRefreshBeginResultV1, SessionRefreshCancelResultV1, SessionRefreshFrontierResultV1, SessionRefreshProgressV1, SessionRefreshReceiptV1, SessionRefreshStatusResultV1, @@ -78,20 +77,10 @@ pub use results::{ use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -use tracedecay_domain::{FactEventId, FactId, ProjectId}; +use tracedecay_domain::{FactEventId, FactId, ProjectId, TemporalModeV1}; use super::RetainedSurfaceOperation; -/// Output formatting accepted by legacy MCP calls. SDK and HTTP callers use -/// JSON, but accepting this field keeps the schema aligned with the mounted -/// MCP request form while the transport discards presentation-only controls. -#[derive(Clone, Copy, Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum RetainedOutputFormatV1 { - Markdown, - Json, -} - /// Exact registered-project selector shared by retained reads. /// /// Inlined so every request schema advertises the closed selector contract @@ -316,26 +305,22 @@ pub struct MessageSearchRequestV1 { pub provider: Option, pub project_key: Option, pub include_subagents: Option, - pub catch_up: Option, + /// Freshness precondition: stale or partial coverage returns + /// `refresh_required` instead of stored evidence. The read never refreshes. + pub require_fresh: Option, pub cursor: Option, pub parent_session_id: Option, pub since: Option, pub until: Option, - pub time_from: Option, - pub time_to: Option, pub scope: Option, pub message_type: Option, pub limit: Option, pub project_selector: Option, - pub project_id: Option, - pub project_path: Option, - pub project_scope: Option, pub branch: Option, pub worktree: Option, pub commit: Option, pub workflow_run: Option, pub workflow_agent: Option, - pub format: Option, } #[derive(Clone, Copy, Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] @@ -363,7 +348,6 @@ pub struct SessionsForRequestV1 { pub until: Option, pub relation: Option, pub limit: Option, - pub format: Option, } #[derive(Clone, Debug, Default, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] @@ -376,7 +360,6 @@ pub struct WorkflowsRequestV1 { pub worktree: Option, pub commit: Option, pub limit: Option, - pub format: Option, } #[derive(Clone, Debug, Default, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] @@ -388,8 +371,6 @@ pub struct LcmStatusRequestV1 { pub session_id: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub deep: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub format: Option, } #[cfg(test)] @@ -404,7 +385,6 @@ mod lcm_status_request_tests { provider: None, session_id: Some("stock-check-session".to_owned()), deep: None, - format: None, }; assert_eq!( @@ -418,23 +398,13 @@ mod lcm_status_request_tests { #[serde(deny_unknown_fields)] pub struct LcmDoctorRequestV1 {} -#[derive(Clone, Copy, Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum LcmTemporalModeV1 { - Current, - AsOf, - Evolution, - Forensic, -} - #[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct LcmLoadSessionRequestV1 { pub provider: Option, pub session_id: String, pub cursor: Option, - pub temporal_mode: Option, - pub as_of_micros: Option, + pub temporal_mode: Option, pub limit: Option, pub role: Option, pub roles: Option>, @@ -442,7 +412,6 @@ pub struct LcmLoadSessionRequestV1 { pub end_time: Option, pub content_offset: Option, pub content_limit: Option, - pub format: Option, } #[derive(Clone, Copy, Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] @@ -552,12 +521,10 @@ pub struct LcmGrepRequestV1 { pub until: Option, pub limit: Option, pub cursor: Option, - pub temporal_mode: Option, - pub as_of_micros: Option, + pub temporal_mode: Option, pub branch: Option, pub worktree: Option, pub commit: Option, - pub format: Option, } #[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] @@ -574,7 +541,6 @@ pub struct LcmDescribeRequestV1 { pub provider: String, pub session_id: String, pub target: Option, - pub format: Option, } #[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] @@ -596,7 +562,6 @@ pub struct LcmExpandRequestV1 { pub content_limit: Option, pub source_limit: Option, pub cursor: Option, - pub format: Option, } #[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] @@ -618,7 +583,6 @@ pub struct LcmExpandQueryRequestV1 { pub max_tokens: Option, pub context_max_tokens: Option, pub cursor: Option, - pub format: Option, } #[derive(Clone, Copy, Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] @@ -664,15 +628,6 @@ pub struct SessionRefreshSourceV1 { pub scope: String, } -#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] -#[serde(tag = "kind", rename_all = "snake_case")] -pub enum SessionRefreshTemporalModeV1 { - Current, - AsOf { cutoff: u64 }, - Evolution, - Forensic, -} - #[derive(Clone, Copy, Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] #[serde(rename_all = "snake_case")] pub enum SessionRefreshGrainV1 { @@ -695,7 +650,7 @@ pub struct SessionRefreshFrontierV1 { #[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct SessionRefreshTargetV1 { - pub temporal_mode: SessionRefreshTemporalModeV1, + pub temporal_mode: TemporalModeV1, pub grain: SessionRefreshGrainV1, pub frontier: SessionRefreshFrontierV1, } @@ -713,7 +668,6 @@ pub struct SessionRefreshActionRequestV1 { pub source: SessionRefreshSourceV1, pub target: SessionRefreshTargetV1, pub handle: Option, - pub format: Option, } /// Operation-selected request used by the canonical application owner. @@ -762,8 +716,7 @@ mod session_refresh_request_tests { "grain": "session", "frontier": { "observed_through": 0, "committed_through": 0 } }, - "handle": null, - "format": "json" + "handle": null }) } @@ -832,10 +785,12 @@ mod session_refresh_request_tests { body["target"]["temporal_mode"] = json!({ "kind": "as_of", "cutoff": 42 }); let request = serde_json::from_value::(body) .expect("canonical as-of request"); - assert!(matches!( + assert_eq!( request.request.target.temporal_mode, - super::SessionRefreshTemporalModeV1::AsOf { cutoff: 42 } - )); + tracedecay_domain::TemporalModeV1::AsOf { + cutoff: tracedecay_domain::UtcMicros(42) + } + ); } } diff --git a/crates/tracedecay-contracts/src/retained_surfaces/sdk/results/automation/admission_binding.rs b/crates/tracedecay-contracts/src/retained_surfaces/sdk/results/automation/admission_binding.rs index 7aa2cd9318..141a143176 100644 --- a/crates/tracedecay-contracts/src/retained_surfaces/sdk/results/automation/admission_binding.rs +++ b/crates/tracedecay-contracts/src/retained_surfaces/sdk/results/automation/admission_binding.rs @@ -17,7 +17,12 @@ fn zero_effect_completion_and_skip_are_typed_without_partial_receipts() { #[test] fn unknown_skill_skip_reasons_fail_closed() { - for reason in ["skill_writer_evidence_unavailable", "skill_writer_not_due"] { + for reason in [ + "skill_writer_evidence_unavailable", + "skill_writer_not_due", + "no_skill_writer_evidence", + "session_cursor_manifest_participants_limit_exceeded", + ] { assert!(AutomationSkipReasonV1::from_ledger_reason(reason).is_none()); let mut terminal = zero_terminal("skipped"); terminal["terminal"]["reason"] = json!(reason); @@ -82,7 +87,7 @@ fn budget_backoff_suppression_is_a_typed_session_evidence_skip() { #[test] fn skill_writer_empty_evidence_is_a_typed_session_evidence_skip() { - let reason = AutomationSkipReasonV1::from_ledger_reason("no_skill_writer_evidence") + let reason = AutomationSkipReasonV1::from_ledger_reason("no_session_evidence") .expect("skill-writer empty evidence is a registered skip"); assert_eq!(reason, AutomationSkipReasonV1::NoSessionEvidence); assert!(reason.matches_task(AutomationTaskV1::SkillWriter)); diff --git a/crates/tracedecay-contracts/src/retained_surfaces/sdk/results/automation/terminal.rs b/crates/tracedecay-contracts/src/retained_surfaces/sdk/results/automation/terminal.rs index 8917576646..996745f780 100644 --- a/crates/tracedecay-contracts/src/retained_surfaces/sdk/results/automation/terminal.rs +++ b/crates/tracedecay-contracts/src/retained_surfaces/sdk/results/automation/terminal.rs @@ -2,7 +2,6 @@ use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use crate::retained_surfaces::AutomationTaskV1; -use crate::retrieval::SessionRetrievalBudgetStageV1; const MAX_AUTOMATION_TERMINAL_COUNT: u64 = 1_000_000; @@ -28,7 +27,9 @@ macro_rules! automation_skip_reasons { } } - fn from_canonical_token(reason: &str) -> Option { + /// Projects a persisted ledger label into the closed terminal. + /// Unknown labels cannot become durable skipped outcomes. + pub fn from_ledger_reason(reason: &str) -> Option { match reason { $($token => Some(Self::$variant),)+ _ => None, @@ -79,7 +80,6 @@ automation_skip_reasons! { (SessionEvidenceTimedOut, "session_evidence_timed_out"), (SessionEvidenceCancelled, "session_evidence_cancelled"), (NoSessionEvidence, "no_session_evidence"), - (ShippedFactProposalHistoryRetired, "shipped_fact_proposal_history_retired"), } /// How a skip affects the next cadence decision. @@ -94,40 +94,6 @@ pub enum AutomationSkipCadenceEffectV1 { } impl AutomationSkipReasonV1 { - /// Projects a persisted ledger label into the closed terminal. - /// - /// Canonical tokens come from [`Self::as_str`]. Historical aliases that - /// collapse into one variant (manifest-limit kinds, budget stages, the - /// skill-writer empty-evidence label) are parsed only at this boundary. - /// Unknown labels cannot become durable skipped outcomes. - pub fn from_ledger_reason(reason: &str) -> Option { - if let Some(reason) = Self::from_canonical_token(reason) { - return Some(reason); - } - Some(match reason { - "session_cursor_manifest_participants_limit_exceeded" - | "session_cursor_manifest_canonical_bytes_limit_exceeded" => { - Self::SessionCursorManifestLimitExceeded - } - reason - if reason - .strip_prefix("session_evidence_budget_exhausted_") - .is_some_and(|stage| { - SessionRetrievalBudgetStageV1::deserialize( - serde::de::value::StrDeserializer::::new( - stage, - ), - ) - .is_ok() - }) => - { - Self::SessionEvidenceBudgetExhausted - } - "no_skill_writer_evidence" => Self::NoSessionEvidence, - _ => return None, - }) - } - /// Whether this skip moved cadence and, if it did, whether it is a /// transient retrieval timeout. Admission diagnostics must not postpone /// the next attempt. A new variant fails compilation until it is classified. @@ -172,8 +138,7 @@ impl AutomationSkipReasonV1 { | Self::SessionCursorManifestLimitExceeded | Self::SessionEvidenceBudgetExhausted | Self::SessionEvidenceCancelled - | Self::NoSessionEvidence - | Self::ShippedFactProposalHistoryRetired => AutomationSkipCadenceEffectV1::Effectful, + | Self::NoSessionEvidence => AutomationSkipCadenceEffectV1::Effectful, } } @@ -197,9 +162,9 @@ impl AutomationSkipReasonV1 { | Self::SimilarityAuthorityUnavailable | Self::PartialCoverageNoCandidates | Self::NothingToReview => task == AutomationTaskV1::MemoryCurator, - Self::SessionReflectorDisabled - | Self::NoNewSessionActivity - | Self::ShippedFactProposalHistoryRetired => task == AutomationTaskV1::SessionReflector, + Self::SessionReflectorDisabled | Self::NoNewSessionActivity => { + task == AutomationTaskV1::SessionReflector + } // Skill writer and combined review retrieve the same session // evidence surface as the reflector. A typed evidence skip must // remain a skip for those tasks instead of failing settlement. @@ -288,15 +253,11 @@ mod tests { }; #[test] - fn budget_stage_skips_accept_known_stages_only() { + fn budget_exhaustion_has_one_ledger_token() { assert_eq!( AutomationSkipReasonV1::from_ledger_reason( "session_evidence_budget_exhausted_execution_work_exhausted" ), - Some(AutomationSkipReasonV1::SessionEvidenceBudgetExhausted), - ); - assert_eq!( - AutomationSkipReasonV1::from_ledger_reason("session_evidence_budget_exhausted_unknown"), None, ); } diff --git a/crates/tracedecay-contracts/src/retained_surfaces/sdk/results/lcm.rs b/crates/tracedecay-contracts/src/retained_surfaces/sdk/results/lcm.rs index e5056d83e9..54ddd9c8c4 100644 --- a/crates/tracedecay-contracts/src/retained_surfaces/sdk/results/lcm.rs +++ b/crates/tracedecay-contracts/src/retained_surfaces/sdk/results/lcm.rs @@ -152,7 +152,6 @@ pub struct LcmLifecycleStatusV1 { pub struct LcmRedactionStatusV1 { pub enabled: bool, pub lossy_records: i64, - pub legacy_truncated_count: i64, } #[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] @@ -237,7 +236,7 @@ pub enum LcmDoctorProjectionStateV1 { /// The temporal projection's serving state at diagnosis time. A store whose /// schema is healthy can still have nothing to serve while history is being -/// re-derived (for example after a scoped observation reset); this is where +/// re-derived (for example on a fresh profile); this is where /// that state is named instead of being read as absent data. #[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] #[serde(deny_unknown_fields)] @@ -299,8 +298,6 @@ pub struct LcmMessageV1 { pub content_hash: Option, pub storage_kind: LcmStorageKindV1, pub payload_ref: Option, - pub legacy_source: bool, - pub legacy_truncated: bool, pub metadata_json: Option, } @@ -443,8 +440,6 @@ pub struct LcmRawMessageV1 { pub content_hash: String, pub storage_kind: LcmStorageKindV1, pub payload_ref: Option, - pub legacy_source: bool, - pub legacy_truncated: bool, pub metadata_json: Option, } @@ -461,8 +456,6 @@ pub struct LcmRawMessageMetadataV1 { pub content_hash: String, pub storage_kind: LcmStorageKindV1, pub payload_ref: Option, - pub legacy_source: bool, - pub legacy_truncated: bool, pub metadata_json: Option, } @@ -505,8 +498,6 @@ pub struct LcmExpansionV1 { #[serde(default, skip_serializing_if = "Option::is_none")] pub from_current_session: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub externalized_note: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] pub source_pagination: Option, } diff --git a/crates/tracedecay-contracts/src/retained_surfaces/sdk/results/mod.rs b/crates/tracedecay-contracts/src/retained_surfaces/sdk/results/mod.rs index 96be80c5f6..259b48c11b 100644 --- a/crates/tracedecay-contracts/src/retained_surfaces/sdk/results/mod.rs +++ b/crates/tracedecay-contracts/src/retained_surfaces/sdk/results/mod.rs @@ -52,9 +52,8 @@ pub use memory::{ }; pub use session::{ ClosedUtcIntervalV1, CorrelationIndexCountModeV1, CorrelationIndexV1, GitScopeV1, - HydrationStateResultV1, MessageSearchFreshnessV1, MessageSearchHitV1, MessageSearchResultV1, - MessageSearchRootV1, MessageSearchSkipV1, RetainedNextActionV1, RetrievalWorkerStatusV1, - SessionCorrelationHitV1, SessionCoverageIntervalV1, SessionCoverageModeV1, + HydrationStateResultV1, MessageSearchHitV1, MessageSearchResultV1, RetainedNextActionV1, + RetrievalWorkerStatusV1, SessionCorrelationHitV1, SessionCoverageIntervalV1, SessionCoverageReasonV1, SessionCoverageRequestV1, SessionCoverageStateV1, SessionMessageV1, SessionRecordV1, SessionRefreshBeginResultV1, SessionRefreshCancelResultV1, SessionRefreshFrontierResultV1, SessionRefreshProgressV1, SessionRefreshReceiptV1, @@ -142,7 +141,7 @@ pub enum RetainedSurfaceResultV1 { LcmDoctor(LcmDoctorResultV1), LcmLoadSession(LcmLoadSessionResultV1), LcmGrep(LcmGrepResultV1), - LcmDescribe(LcmDescribeResultV1), + LcmDescribe(Box), LcmExpand(Box), LcmExpandQuery(LcmExpandQueryResultV1), } diff --git a/crates/tracedecay-contracts/src/retained_surfaces/sdk/results/session.rs b/crates/tracedecay-contracts/src/retained_surfaces/sdk/results/session.rs index 0b5e131812..dd823b5df2 100644 --- a/crates/tracedecay-contracts/src/retained_surfaces/sdk/results/session.rs +++ b/crates/tracedecay-contracts/src/retained_surfaces/sdk/results/session.rs @@ -98,7 +98,7 @@ pub enum HydrationStateResultV1 { RetentionExpired, Unauthorized, Locked, - UnverifiableLegacy, + Unverifiable, } impl From for HydrationStateResultV1 { @@ -111,7 +111,7 @@ impl From for HydrationStateResultV1 { HydrationStateV1::RetentionExpired => Self::RetentionExpired, HydrationStateV1::Unauthorized => Self::Unauthorized, HydrationStateV1::Locked => Self::Locked, - HydrationStateV1::UnverifiableLegacy => Self::UnverifiableLegacy, + HydrationStateV1::Unverifiable => Self::Unverifiable, } } } @@ -157,27 +157,7 @@ pub struct SessionSourceCoverageV1 { #[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct SessionCoverageRequestV1 { - pub mode: SessionCoverageModeV1, -} - -#[derive(Clone, Copy, Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] -#[serde(tag = "kind", rename_all = "snake_case")] -pub enum SessionCoverageModeV1 { - Current, - AsOf { cutoff: i64 }, - Evolution, - Forensic, -} - -impl From for SessionCoverageModeV1 { - fn from(value: TemporalModeV1) -> Self { - match value { - TemporalModeV1::Current => Self::Current, - TemporalModeV1::AsOf { cutoff } => Self::AsOf { cutoff: cutoff.0 }, - TemporalModeV1::Evolution => Self::Evolution, - TemporalModeV1::Forensic => Self::Forensic, - } - } + pub mode: TemporalModeV1, } #[derive(Clone, Copy, Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] @@ -316,44 +296,10 @@ pub struct RetrievalWorkerStatusV1 { pub retry_class: Option, } -#[derive(Clone, Copy, Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum MessageSearchFreshnessV1 { - Fresh, - Stored, - Partial, -} - -#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct MessageSearchRootV1 { - pub project_id: String, - pub root: String, - pub status: RetainedOutcomeStatusV1, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub count: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub freshness: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub omitted: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub reason: Option, -} - -#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct MessageSearchSkipV1 { - pub project_id: String, - pub reason: String, -} - #[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq)] #[serde(deny_unknown_fields)] pub struct MessageSearchResultV1 { - pub catch_up: bool, - pub catch_up_failures: Vec, - pub catch_up_performed: bool, - pub catch_up_provider: String, + pub require_fresh: bool, pub count: Option, pub goals: bool, pub include_subagents: bool, @@ -382,22 +328,8 @@ pub struct MessageSearchResultV1 { #[serde(default, skip_serializing_if = "Option::is_none")] pub omitted: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub project_scope: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub registry_truncated: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub roots: Option>, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub searched_project_count: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub selected_project_root: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] pub service_status: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub skipped: Option>, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub skipped_project_count: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] pub store_scope: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub temporal: Option, @@ -672,21 +604,20 @@ mod coverage_projection_tests { }; use super::{ - HydrationStateResultV1, SessionCoverageModeV1, SessionCoverageReasonV1, + HydrationStateResultV1, SessionCoverageReasonV1, SessionCoverageRequestV1, SessionCoverageStateV1, }; #[test] fn coverage_projection_keeps_mode_cutoff_and_status_labels() { assert_eq!( - SessionCoverageModeV1::from(TemporalModeV1::AsOf { - cutoff: UtcMicros(7), - }), - SessionCoverageModeV1::AsOf { cutoff: 7 } - ); - assert_eq!( - SessionCoverageModeV1::from(TemporalModeV1::Forensic), - SessionCoverageModeV1::Forensic + serde_json::to_value(SessionCoverageRequestV1 { + mode: TemporalModeV1::AsOf { + cutoff: UtcMicros(7), + }, + }) + .expect("coverage request serializes"), + serde_json::json!({"mode": {"kind": "as_of", "cutoff": 7}}) ); assert_eq!( SessionCoverageStateV1::from(SessionSourceCoverageStateV1::RetentionWithheld), @@ -699,8 +630,8 @@ mod coverage_projection_tests { SessionCoverageReasonV1::ProjectionBehindSource { lag: 4 } ); assert_eq!( - HydrationStateResultV1::from(HydrationStateV1::UnverifiableLegacy), - HydrationStateResultV1::UnverifiableLegacy + HydrationStateResultV1::from(HydrationStateV1::Unverifiable), + HydrationStateResultV1::Unverifiable ); } } diff --git a/crates/tracedecay-contracts/src/retained_surfaces/service.rs b/crates/tracedecay-contracts/src/retained_surfaces/service.rs index f2fbdf1463..c62d2a1b8a 100644 --- a/crates/tracedecay-contracts/src/retained_surfaces/service.rs +++ b/crates/tracedecay-contracts/src/retained_surfaces/service.rs @@ -407,7 +407,7 @@ pub(super) fn outcome_matches_operation( let result = match outcome { ApplicationOutcome::Evidence(packet) => packet.payload.as_ref(), ApplicationOutcome::Effect(effect) => effect.payload.as_ref(), - ApplicationOutcome::Preview(_) => None, + ApplicationOutcome::Preview(_) | ApplicationOutcome::Result(_) => None, }; if let Some(RetainedSurfaceResultV1::FactStoreCurate(result)) = result { return operation == RetainedSurfaceOperation::FactStoreCurate diff --git a/crates/tracedecay-contracts/src/retrieval/callable_code_catalog.rs b/crates/tracedecay-contracts/src/retrieval/callable_code_catalog.rs index f016d5661a..f22e133e37 100644 --- a/crates/tracedecay-contracts/src/retrieval/callable_code_catalog.rs +++ b/crates/tracedecay-contracts/src/retrieval/callable_code_catalog.rs @@ -1,6 +1,6 @@ use schemars::JsonSchema; use tracedecay_tool_catalog::{ - ApplicationSurfaceOperation, AvailabilityContract, BindingId, BindingStatus, BindingSurface, + ApplicationSurfaceOperation, AvailabilityContract, BindingId, BindingSurface, CancellationContract, CancellationPoint, CapabilityId, CapabilityManifestV1, CatalogContributionInputV1, CatalogContributionV1, ContributionId, DeadlineBehavior, DeadlineContract, DeniedDisclosurePolicy, EffectClass, ExecutableSchemaAuthority, @@ -140,8 +140,6 @@ pub fn callable_code_catalog_contribution() operation: SurfaceOperationName::new(*method)?, protocol_revisions: ProtocolRevisionRange::new(1, 1)?, required_features: Vec::new(), - status: BindingStatus::Current, - alias_of: None, })?); binding_ids.push(binding_id); } @@ -302,6 +300,10 @@ fn lsp_methods(kind: CallableCodeOperationKind) -> &'static [&'static str] { } } +/// The page every callable-code query serves when the caller omits one; +/// `meta.cursor` continues past it through the whole result set. +pub(crate) const CALLABLE_CODE_DEFAULT_PAGE_SIZE: u32 = 10; + fn code_query_capability_id( kind: CallableCodeOperationKind, ) -> Result { @@ -327,9 +329,12 @@ fn code_query_capability( routing: RoutingContractV1::new( 1, format!("Query {readable_name}"), - format!( - "Invoke the generation-bound query {readable_name} query without replacing its owning kernel." - ), + match kind { + CallableCodeOperationKind::Callees => "What does this call: outgoing calls of a known symbol node ID up to `maximum_depth` (default 3). A callee that is a trait method also returns the concrete impl methods reachable through the trait, tagged `dispatch_via_trait` with `dispatch_from`; set `resolve_trait_dispatch: false` for direct call edges only.".to_owned(), + _ => format!( + "Invoke the generation-bound query {readable_name} query without replacing its owning kernel." + ), + }, // Keep examples distinct from primitive-read fixtures ("Read …"). vec![format!("Query indexed {readable_name}")], )?, @@ -347,7 +352,11 @@ fn code_query_capability( CancellationPoint::DuringRead, ])?, deadline: DeadlineContract::new(10_000, DeadlineBehavior::ReturnOperationReceipt)?, - pagination: Some(PaginationContract::new(10, 1_000, 15 * 60 * 1_000)?), + pagination: Some(PaginationContract::new( + CALLABLE_CODE_DEFAULT_PAGE_SIZE, + 1_000, + 15 * 60 * 1_000, + )?), inverse: None, authority_revalidation: RevalidationContract::required(vec![ RevalidationPoint::Authority, diff --git a/crates/tracedecay-contracts/src/retrieval/callable_code_service.rs b/crates/tracedecay-contracts/src/retrieval/callable_code_service.rs index 8af708668d..a552885860 100644 --- a/crates/tracedecay-contracts/src/retrieval/callable_code_service.rs +++ b/crates/tracedecay-contracts/src/retrieval/callable_code_service.rs @@ -8,15 +8,12 @@ use std::pin::Pin; use std::sync::Arc; use tracedecay_domain::{CodeGenerationId, TemporalModeV1, UtcMicros}; -use tracedecay_policy::authorization::SourceAuthorizationEvaluator; -use crate::authorization::{AuthorizationAdmission, AuthorizationPort, AuthorizationService}; use crate::context::RequestContext; use crate::error::ApplicationContractError; use crate::handlers::ApplicationOperation; use crate::result::{ - ApplicationProblem, ApplicationResult, AuthorityReceipt, PageCursor, RetryDirective, - SafeDiagnostic, + ApplicationProblem, ApplicationResult, AuthorityReceipt, PageCursor, SafeDiagnostic, }; use super::callable_code::{ @@ -154,43 +151,22 @@ pub trait CallableCodeQueryPort: Send + Sync { ) -> CallableCodeQueryFuture<'a, SymbolRelationRecord>; } -/// Opaque authorization admission retained across one callable-code read. -/// -/// Canonical source authorization keeps its full proof. A production route -/// that already resolved exact project/source access may instead retain its -/// route-owned receipt without reconstructing policy inputs. -#[derive(Clone, Debug)] -pub enum CallableCodeAuthorizationAdmission { - Source(Box), - Routed(AuthorityReceipt), -} - -impl CallableCodeAuthorizationAdmission { - pub fn receipt(&self) -> &AuthorityReceipt { - match self { - Self::Source(admission) => admission.receipt(), - Self::Routed(receipt) => receipt, - } - } -} - -/// Authorization boundary for callable-code application reads. +/// Authorization boundary for callable-code application reads. The route +/// owner retains its admission receipt across one read and revalidates it +/// immediately before publication. pub trait CallableCodeAuthorizationPort: Send + Sync { fn admit<'a>( &'a self, context: &'a RequestContext, operation: &'a ApplicationOperation, observed_at: UtcMicros, - ) -> CallableCodeAuthorizationFuture< - 'a, - Result, - >; + ) -> CallableCodeAuthorizationFuture<'a, Result>; fn recheck_publication<'a>( &'a self, context: &'a RequestContext, operation: &'a ApplicationOperation, - admission: &'a CallableCodeAuthorizationAdmission, + admission: &'a AuthorityReceipt, observed_at: UtcMicros, ) -> CallableCodeAuthorizationFuture<'a, Result>; } @@ -201,10 +177,7 @@ impl CallableCodeAuthorizationPort for Arc { context: &'a RequestContext, operation: &'a ApplicationOperation, observed_at: UtcMicros, - ) -> CallableCodeAuthorizationFuture< - 'a, - Result, - > { + ) -> CallableCodeAuthorizationFuture<'a, Result> { (**self).admit(context, operation, observed_at) } @@ -212,57 +185,13 @@ impl CallableCodeAuthorizationPort for Arc { &'a self, context: &'a RequestContext, operation: &'a ApplicationOperation, - admission: &'a CallableCodeAuthorizationAdmission, + admission: &'a AuthorityReceipt, observed_at: UtcMicros, ) -> CallableCodeAuthorizationFuture<'a, Result> { (**self).recheck_publication(context, operation, admission, observed_at) } } -impl CallableCodeAuthorizationPort for AuthorizationService -where - P: AuthorizationPort + Send + Sync, - E: SourceAuthorizationEvaluator + Send + Sync, -{ - fn admit<'a>( - &'a self, - context: &'a RequestContext, - operation: &'a ApplicationOperation, - observed_at: UtcMicros, - ) -> CallableCodeAuthorizationFuture< - 'a, - Result, - > { - Box::pin(async move { - AuthorizationService::admit(self, context, operation, observed_at) - .map(|admission| CallableCodeAuthorizationAdmission::Source(Box::new(admission))) - }) - } - - fn recheck_publication<'a>( - &'a self, - context: &'a RequestContext, - operation: &'a ApplicationOperation, - admission: &'a CallableCodeAuthorizationAdmission, - observed_at: UtcMicros, - ) -> CallableCodeAuthorizationFuture<'a, Result> { - Box::pin(async move { - let CallableCodeAuthorizationAdmission::Source(admission) = admission else { - return Err(ApplicationProblem::not_found_or_not_authorized( - RetryDirective::Never, - )); - }; - AuthorizationService::recheck_publication( - self, - context, - operation, - admission, - observed_at, - ) - }) - } -} - pub struct CallableCodeQueryService { port: P, authorization: A, @@ -310,7 +239,7 @@ macro_rules! callable_code_service_method { evidence_envelope_with_async_publication_recheck( context, operation, - admission.receipt(), + &admission, outcome, observed_at, |finished_at| { diff --git a/crates/tracedecay-contracts/src/retrieval/catalog.rs b/crates/tracedecay-contracts/src/retrieval/catalog.rs index 1e8b0c1f49..e8921fed40 100644 --- a/crates/tracedecay-contracts/src/retrieval/catalog.rs +++ b/crates/tracedecay-contracts/src/retrieval/catalog.rs @@ -1,6 +1,6 @@ use schemars::JsonSchema; use tracedecay_tool_catalog::{ - ApplicationSurfaceOperation, AvailabilityContract, BindingId, BindingStatus, BindingSurface, + ApplicationSurfaceOperation, AvailabilityContract, BindingId, BindingSurface, CancellationContract, CancellationPoint, CapabilityId, CatalogContributionInputV1, CatalogContributionV1, ContributionContractRef, ContributionId, CoverageContractRef, DeadlineBehavior, DeadlineContract, DeniedDisclosurePolicy, EffectClass, @@ -19,12 +19,13 @@ use crate::capability_manifest::{ use crate::error::ApplicationContractError; use crate::handlers::{ApplicationHandlerDescriptor, ApplicationOperation}; use crate::result::ResultContractRef; +use crate::retrieval::callable_code_catalog::CALLABLE_CODE_DEFAULT_PAGE_SIZE; use crate::retrieval::primitive_surface::{ - CalleesResultV1, CalleesSurfaceRequestV1, ContextResultV1, ContextSurfaceRequestV1, - ImpactResultV1, ImpactSurfaceRequestV1, NodeResultV1, NodeSurfaceRequestV1, PortOrderResultV1, - PortOrderSurfaceRequestV1, PortStatusResultV1, PortStatusSurfaceRequestV1, RedundancyResultV1, - RedundancySurfaceRequestV1, RenamePreviewPrimitiveOutcomeV1, RenamePreviewPrimitiveRequestV1, - SimilarResultV1, SimilarSurfaceRequestV1, TodosResultV1, TodosSurfaceRequestV1, + ContextResultV1, ContextSurfaceRequestV1, ImpactResultV1, NodeDepthSurfaceRequestV1, + NodeResultV1, NodeSurfaceRequestV1, PortOrderResultV1, PortOrderSurfaceRequestV1, + PortStatusResultV1, PortStatusSurfaceRequestV1, RedundancyResultV1, RedundancySurfaceRequestV1, + RenamePreviewPrimitiveOutcomeV1, RenamePreviewPrimitiveRequestV1, SimilarResultV1, + SimilarSurfaceRequestV1, TodosResultV1, TodosSurfaceRequestV1, }; use crate::retrieval::requests::{ CallChainPrimitiveRequest, CallChainPrimitiveResult, DiagnosticsPrimitiveRequest, @@ -37,7 +38,8 @@ use crate::retrieval::requests::{ StorageStatusPrimitiveResult, }; use crate::retrieval::symbol_graph::{ - SymbolGraphPage, SymbolPrimitiveRecord, SymbolRelationRecord, TypeHierarchyRecord, + ImplementationRecord, SymbolGraphPage, SymbolPrimitiveRecord, SymbolRelationRecord, + TypeHierarchyRecord, }; use crate::surface_contracts::{ CodeCallersSurfaceRequest, CodeImplementationsSurfaceRequest, @@ -87,13 +89,16 @@ pub fn application_catalog_contributions() /// Resolves the page size an omitted transport control receives from the /// canonical primitive descriptor. /// -/// Operations outside this primitive family retain the inert page envelope's -/// established value of 10. +/// Operations outside this primitive family, the callable-code queries +/// included, retain the inert page envelope's established value of +/// [`CALLABLE_CODE_DEFAULT_PAGE_SIZE`]. pub fn application_operation_default_page_size(operation: ApplicationSurfaceOperation) -> u32 { PRIMITIVE_READ_SPECS .iter() .find(|spec| spec.operation == operation.as_str()) - .map_or(10, |spec| spec.default_page_size) + .map_or(CALLABLE_CODE_DEFAULT_PAGE_SIZE, |spec| { + spec.default_page_size + }) } struct PrimitiveReadSpec { @@ -147,13 +152,14 @@ fn primitive_lsp_methods(operation: &str) -> &'static [&'static str] { } const PRIMITIVE_READ_SPECS: &[PrimitiveReadSpec] = &[ - primitive_spec("code_signature_search"), - primitive_spec("code_implementations"), - primitive_spec("code_type_hierarchy"), - primitive_spec("code_callers"), + // MCP and CLI callers cannot choose a page size, so these navigation reads + // default to a page that holds a typical answer; `meta.cursor` continues. + primitive_spec_with_default_page_size("code_signature_search", 50), + primitive_spec_with_default_page_size("code_implementations", 20), + primitive_spec_with_default_page_size("code_type_hierarchy", 100), + primitive_spec_with_default_page_size("code_callers", 100), primitive_spec("context"), primitive_spec("node"), - primitive_spec("callees"), primitive_spec("impact"), primitive_spec("similar"), primitive_spec("redundancy"), @@ -192,10 +198,9 @@ const DASHBOARD_PRIMITIVE_SURFACES: [BindingSurface; 4] = [ fn primitive_read_surfaces(spec: &PrimitiveReadSpec) -> &'static [BindingSurface] { match spec.operation { - // These established tool handlers retain their current wire schemas - // and rendering across the generic CLI fallback and MCP, while using - // this operation identity for canonical code-graph read admission. - "context" | "node" | "callees" | "impact" | "similar" | "redundancy" | "rename_preview" + // The project's graph-tool owner answers these for the tool surfaces + // only; their typed results render as the established tool output. + "context" | "node" | "impact" | "similar" | "redundancy" | "rename_preview" | "port_status" | "port_order" | "todos" => &CLI_MCP_PRIMITIVE_SURFACES, "health_read" | "storage_status" | "diagnostics_read" => &DASHBOARD_PRIMITIVE_SURFACES, _ => &PRE_DASHBOARD_PRIMITIVE_SURFACES, @@ -229,8 +234,6 @@ fn clone_family_surface_bindings( operation: SurfaceOperationName::new(operation)?, protocol_revisions: ProtocolRevisionRange::new(1, 1)?, required_features: Vec::new(), - status: BindingStatus::Current, - alias_of: None, })?); binding_ids.push(binding_id); } @@ -240,16 +243,16 @@ fn clone_family_surface_bindings( fn primitive_read_description(operation: &str) -> &'static str { match operation { "code_signature_search" => { - "Find functions and methods by return type, parameter substrings, or async status. Use code_symbol_search for name or concept searches; this tool requires at least one signature filter." + "Find functions and methods by signature shape: `returns` (return-type substring), `params` (substrings that must all appear in the parameter list), or `is_async`; narrow with `scope.path_prefix`. At least one filter is required. Use symbol search for name or concept searches." } "code_implementations" => { - "Find types implementing a named trait, or functions and methods with a selected method name. Use code_type_hierarchy to traverse extends and implements relationships from a known node ID." + "Find every type implementing a trait (`selector: {\"selector\": \"trait\", \"name\": ...}`) or every function or method with a name (`selector: {\"selector\": \"method\", \"name\": ...}`). Each match carries its exact source body. Use type_hierarchy to traverse extends and implements relationships from a known node ID." } "code_type_hierarchy" => { - "Traverse extends and implements relationships from a symbol node ID returned by code_symbol_search or another graph read. Use code_implementations when starting from a trait or method name." + "Use for trait, interface, or class hierarchy questions before grepping `impl X for` or `extends X`: traverses the implementors and extenders of a type node ID up to `maximum_depth` (default 5). Use implementations when starting from a trait or method name." } "code_callers" => { - "Find symbols that call a known symbol node ID, up to the requested depth. Use call_chain when you need the shortest call path between two known node IDs." + "Who calls this: find references, usages, and call sites of a known symbol node ID up to `maximum_depth` (default 3). Coverage is partial when a call target cannot be resolved exactly. Use call_chain for the shortest call path between two known node IDs." } "redundancy" => { "Report bounded, token-verified exact and rename-normalized implementation families in the admitted repository. Results rank review candidates by repeated source bytes." @@ -408,8 +411,6 @@ pub fn primitive_read_contribution() -> Result + SymbolGraphPage ); add!( "code_type_hierarchy", @@ -578,8 +579,7 @@ fn primitive_executable_schemas( SymbolGraphPage ); add!("context", ContextSurfaceRequestV1, ContextResultV1); - add!("callees", CalleesSurfaceRequestV1, CalleesResultV1); - add!("impact", ImpactSurfaceRequestV1, ImpactResultV1); + add!("impact", NodeDepthSurfaceRequestV1, ImpactResultV1); add!("node", NodeSurfaceRequestV1, NodeResultV1); add!("similar", SimilarSurfaceRequestV1, SimilarResultV1); add!("redundancy", RedundancySurfaceRequestV1, RedundancyResultV1); @@ -688,8 +688,6 @@ pub fn symbol_search_contribution() -> Result Result { mod tests { use super::*; - const ESTABLISHED_TOOL_PRIMITIVES: [&str; 10] = [ + const ESTABLISHED_TOOL_PRIMITIVES: [&str; 9] = [ "context", "node", - "callees", "impact", "similar", "redundancy", diff --git a/crates/tracedecay-contracts/src/retrieval/git_topology_anchor.rs b/crates/tracedecay-contracts/src/retrieval/git_topology_anchor.rs index 0de5b8f771..df36c5d0fc 100644 --- a/crates/tracedecay-contracts/src/retrieval/git_topology_anchor.rs +++ b/crates/tracedecay-contracts/src/retrieval/git_topology_anchor.rs @@ -1,52 +1,52 @@ -//! Canonical V2 persistence port for Git topology retrieval anchors. +//! Canonical persistence port for Git topology retrieval anchors. use std::collections::BTreeSet; use std::future::Future; use std::pin::Pin; use tracedecay_domain::{ - ObservationScopeV1, RetrievalAnchorId, RetrievalAnchorRecordV2, RetrievalAnchorTargetV2, + ObservationScopeV1, RetrievalAnchorId, RetrievalAnchorRecord, RetrievalAnchorTarget, }; -pub const MAX_GIT_TOPOLOGY_ANCHORS_PER_PUBLICATION_V2: usize = 4_096; +pub const MAX_GIT_TOPOLOGY_ANCHORS_PER_PUBLICATION: usize = 4_096; #[derive(Clone, Debug, PartialEq, Eq)] -pub struct GitTopologyAnchorPublicationV2 { +pub struct GitTopologyAnchorPublication { owner: ObservationScopeV1, - records: Vec, + records: Vec, } -impl GitTopologyAnchorPublicationV2 { +impl GitTopologyAnchorPublication { pub fn new( owner: ObservationScopeV1, - records: Vec, - ) -> Result { + records: Vec, + ) -> Result { owner .validate() - .map_err(|_| GitTopologyAnchorAuthorityErrorV2::Conflict)?; - if records.is_empty() || records.len() > MAX_GIT_TOPOLOGY_ANCHORS_PER_PUBLICATION_V2 { - return Err(GitTopologyAnchorAuthorityErrorV2::Conflict); + .map_err(|_| GitTopologyAnchorAuthorityError::Conflict)?; + if records.is_empty() || records.len() > MAX_GIT_TOPOLOGY_ANCHORS_PER_PUBLICATION { + return Err(GitTopologyAnchorAuthorityError::Conflict); } let mut has_topology = false; let mut anchor_ids = BTreeSet::new(); for record in &records { record .validate() - .map_err(|_| GitTopologyAnchorAuthorityErrorV2::Conflict)?; + .map_err(|_| GitTopologyAnchorAuthorityError::Conflict)?; if record.owner() != &owner || !record.aliases().is_empty() { - return Err(GitTopologyAnchorAuthorityErrorV2::Conflict); + return Err(GitTopologyAnchorAuthorityError::Conflict); } if !anchor_ids.insert(record.anchor_id().clone()) { - return Err(GitTopologyAnchorAuthorityErrorV2::Conflict); + return Err(GitTopologyAnchorAuthorityError::Conflict); } match record.target() { - RetrievalAnchorTargetV2::GitTopology(_) => has_topology = true, - RetrievalAnchorTargetV2::ExactRepositoryCommit { .. } => {} - _ => return Err(GitTopologyAnchorAuthorityErrorV2::Conflict), + RetrievalAnchorTarget::GitTopology(_) => has_topology = true, + RetrievalAnchorTarget::ExactRepositoryCommit { .. } => {} + _ => return Err(GitTopologyAnchorAuthorityError::Conflict), } } if !has_topology { - return Err(GitTopologyAnchorAuthorityErrorV2::Conflict); + return Err(GitTopologyAnchorAuthorityError::Conflict); } if records.iter().any(|record| { record @@ -54,7 +54,7 @@ impl GitTopologyAnchorPublicationV2 { .iter() .any(|source| !anchor_ids.contains(source.anchor_id())) }) { - return Err(GitTopologyAnchorAuthorityErrorV2::Conflict); + return Err(GitTopologyAnchorAuthorityError::Conflict); } Ok(Self { owner, records }) } @@ -63,66 +63,66 @@ impl GitTopologyAnchorPublicationV2 { &self.owner } - pub fn records(&self) -> &[RetrievalAnchorRecordV2] { + pub fn records(&self) -> &[RetrievalAnchorRecord] { &self.records } - pub fn into_records(self) -> Vec { + pub fn into_records(self) -> Vec { self.records } } #[derive(Clone, Debug, PartialEq, Eq)] -pub struct GitTopologyAnchorResolutionV2 { +pub struct GitTopologyAnchorResolution { pub owner: ObservationScopeV1, pub anchor_id: RetrievalAnchorId, } -impl GitTopologyAnchorResolutionV2 { +impl GitTopologyAnchorResolution { pub fn new( owner: ObservationScopeV1, anchor_id: RetrievalAnchorId, - ) -> Result { + ) -> Result { owner .validate() - .map_err(|_| GitTopologyAnchorAuthorityErrorV2::Conflict)?; + .map_err(|_| GitTopologyAnchorAuthorityError::Conflict)?; anchor_id .validate() - .map_err(|_| GitTopologyAnchorAuthorityErrorV2::Conflict)?; + .map_err(|_| GitTopologyAnchorAuthorityError::Conflict)?; Ok(Self { owner, anchor_id }) } } #[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum GitTopologyAnchorPublicationOutcomeV2 { +pub enum GitTopologyAnchorPublicationOutcome { Published, Replayed, } #[derive(Clone, Debug, PartialEq, Eq)] -pub enum GitTopologyAnchorResolutionOutcomeV2 { - Resolved(Box), +pub enum GitTopologyAnchorResolutionOutcome { + Resolved(Box), Unavailable, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum GitTopologyAnchorAuthorityErrorV2 { +pub enum GitTopologyAnchorAuthorityError { Unavailable, ResetRequired, Conflict, } -pub type GitTopologyAnchorFutureV2<'a, T> = - Pin> + Send + 'a>>; +pub type GitTopologyAnchorFuture<'a, T> = + Pin> + Send + 'a>>; -pub trait GitTopologyAnchorAuthorityV2: Send + Sync { +pub trait GitTopologyAnchorAuthority: Send + Sync { fn publish<'a>( &'a self, - publication: GitTopologyAnchorPublicationV2, - ) -> GitTopologyAnchorFutureV2<'a, GitTopologyAnchorPublicationOutcomeV2>; + publication: GitTopologyAnchorPublication, + ) -> GitTopologyAnchorFuture<'a, GitTopologyAnchorPublicationOutcome>; fn resolve<'a>( &'a self, - resolution: GitTopologyAnchorResolutionV2, - ) -> GitTopologyAnchorFutureV2<'a, GitTopologyAnchorResolutionOutcomeV2>; + resolution: GitTopologyAnchorResolution, + ) -> GitTopologyAnchorFuture<'a, GitTopologyAnchorResolutionOutcome>; } diff --git a/crates/tracedecay-contracts/src/retrieval/mod.rs b/crates/tracedecay-contracts/src/retrieval/mod.rs index d2f663eab1..1ddf895d82 100644 --- a/crates/tracedecay-contracts/src/retrieval/mod.rs +++ b/crates/tracedecay-contracts/src/retrieval/mod.rs @@ -77,15 +77,13 @@ pub use callable_code_catalog::{ callable_code_operations, callable_code_request_schema, callable_code_result_schema, }; pub use callable_code_service::{ - CallableCodeAuthorizationAdmission, CallableCodeAuthorizationFuture, - CallableCodeAuthorizationPort, CallableCodeQueryFuture, CallableCodeQueryPort, - CallableCodeQueryService, UNPINNED_LATEST_GENERATION_SENTINEL, + CallableCodeAuthorizationFuture, CallableCodeAuthorizationPort, CallableCodeQueryFuture, + CallableCodeQueryPort, CallableCodeQueryService, UNPINNED_LATEST_GENERATION_SENTINEL, }; pub use git_topology_anchor::{ - GitTopologyAnchorAuthorityErrorV2, GitTopologyAnchorAuthorityV2, GitTopologyAnchorFutureV2, - GitTopologyAnchorPublicationOutcomeV2, GitTopologyAnchorPublicationV2, - GitTopologyAnchorResolutionOutcomeV2, GitTopologyAnchorResolutionV2, - MAX_GIT_TOPOLOGY_ANCHORS_PER_PUBLICATION_V2, + GitTopologyAnchorAuthority, GitTopologyAnchorAuthorityError, GitTopologyAnchorFuture, + GitTopologyAnchorPublication, GitTopologyAnchorPublicationOutcome, GitTopologyAnchorResolution, + GitTopologyAnchorResolutionOutcome, MAX_GIT_TOPOLOGY_ANCHORS_PER_PUBLICATION, }; pub use ports::{ AffectedTestsRetrievalPort, OperationalRetrievalPort, RetrievalPortContext, @@ -94,16 +92,16 @@ pub use ports::{ TemporalRetrievalFailure, TemporalRetrievalFuture, TemporalRetrievalPort, }; pub use primitive_surface::{ - CalleeV1, CalleesResultV1, CalleesSurfaceRequestV1, ContextCodeBlockV1, ContextModeV1, - ContextResultV1, ContextSearchMatchV1, ContextSurfaceRequestV1, ImpactNodeV1, ImpactResultV1, - ImpactSurfaceRequestV1, MAX_REDUNDANCY_FAMILIES_V1, MAX_REDUNDANCY_PULL_REQUEST_PATHS_V1, - MAX_REDUNDANCY_WORK_V1, NodeDepthSurfaceRequestV1, NodeDetailsV1, NodeExpansionCostV1, - NodeResultV1, NodeSurfaceRequestV1, PortCycleAnchorV1, PortCycleFileV1, PortCycleSymbolV1, - PortCycleV1, PortMatchedSymbolV1, PortOrderLevelV1, PortOrderResultV1, - PortOrderSurfaceRequestV1, PortOrderSymbolV1, PortStatusResultV1, PortStatusSurfaceRequestV1, - PortTargetOnlySymbolV1, PortUnmatchedSymbolV1, PrimitiveFreshnessStateV1, - PrimitiveIndexingStateV1, PrimitiveLaneCompleteV1, PrimitiveLaneStateV1, PrimitiveLaneStatusV1, - PrimitiveNotFoundV1, PrimitiveRecallV1, PrimitiveSearchCoverageV1, PrimitiveSearchFreshnessV1, + ContextCodeBlockV1, ContextExtensionPointV1, ContextModeV1, ContextPlanV1, ContextResultV1, + ContextSearchMatchV1, ContextSurfaceRequestV1, ImpactNodeV1, ImpactResultV1, + MAX_REDUNDANCY_FAMILIES_V1, MAX_REDUNDANCY_PULL_REQUEST_PATHS_V1, MAX_REDUNDANCY_WORK_V1, + NodeDepthSurfaceRequestV1, NodeDetailsV1, NodeExpansionCostV1, NodeResultV1, + NodeSurfaceRequestV1, PortCycleAnchorV1, PortCycleFileV1, PortCycleSymbolV1, PortCycleV1, + PortMatchedSymbolV1, PortOrderLevelV1, PortOrderResultV1, PortOrderSurfaceRequestV1, + PortOrderSymbolV1, PortStatusResultV1, PortStatusSurfaceRequestV1, PortTargetOnlySymbolV1, + PortUnmatchedSymbolV1, PrimitiveFreshnessStateV1, PrimitiveIndexingStateV1, + PrimitiveLaneCompleteV1, PrimitiveLaneStateV1, PrimitiveLaneStatusV1, PrimitiveNotFoundV1, + PrimitiveRecallV1, PrimitiveSearchCoverageV1, PrimitiveSearchFreshnessV1, PrimitiveSymbolLocationV1, PrimitiveUnavailableEvidenceV1, PrimitiveUnavailableStatusV1, RedundancyCoverageV1, RedundancyFamilyV1, RedundancyPartialReasonV1, RedundancyRankingV1, RedundancyResultV1, RedundancyScopeV1, RedundancySurfaceRequestV1, RenamePreviewNodeV1, @@ -134,12 +132,12 @@ pub use source_read::{ }; pub use symbol_graph::{ CodeGraphReadFreshnessV1, ExactSymbolRequest, GraphImpactPrimitiveRequest, - GraphRelationRequest, ImplementationSelector, ImplementationsRequest, MAX_SYMBOL_GRAPH_DEPTH, - MAX_SYMBOL_GRAPH_FILTERS, MAX_SYMBOL_GRAPH_QUERY_BYTES, PrimitiveFailure, PrimitiveFailureKind, - PrimitiveSupportGap, SignatureSearchRequest, SymbolGraphPage, SymbolGraphPortContext, - SymbolGraphPortFuture, SymbolGraphPortOutcome, SymbolGraphPrimitivePort, SymbolGraphScope, - SymbolPrimitiveRecord, SymbolRelationRecord, SymbolSearchPrimitiveRequest, TypeHierarchyRecord, - TypeHierarchyRequest, + GraphRelationRequest, ImplementationRecord, ImplementationSelector, ImplementationsRequest, + MAX_SYMBOL_GRAPH_DEPTH, MAX_SYMBOL_GRAPH_FILTERS, MAX_SYMBOL_GRAPH_QUERY_BYTES, + PrimitiveFailure, PrimitiveFailureKind, PrimitiveSupportGap, ServedCodeGraphGenerationV1, + SignatureSearchRequest, SymbolGraphPage, SymbolGraphPortContext, SymbolGraphPortFuture, + SymbolGraphPortOutcome, SymbolGraphPrimitivePort, SymbolGraphScope, SymbolPrimitiveRecord, + SymbolRelationRecord, SymbolSearchPrimitiveRequest, TypeHierarchyRecord, TypeHierarchyRequest, }; pub use test_attribution::{ AffectedFileTestsPrimitiveRequest, AffectedFileTestsPrimitiveResultV1, MAX_TEST_FILTER_BYTES, diff --git a/crates/tracedecay-contracts/src/retrieval/primitive_surface.rs b/crates/tracedecay-contracts/src/retrieval/primitive_surface.rs index ad147d1c37..d2a79dc2db 100644 --- a/crates/tracedecay-contracts/src/retrieval/primitive_surface.rs +++ b/crates/tracedecay-contracts/src/retrieval/primitive_surface.rs @@ -115,16 +115,6 @@ pub struct NodeDepthSurfaceRequestV1 { pub max_depth: Option, } -pub type ImpactSurfaceRequestV1 = NodeDepthSurfaceRequestV1; - -#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Eq, Serialize)] -#[serde(deny_unknown_fields)] -pub struct CalleesSurfaceRequestV1 { - pub node_id: String, - pub max_depth: Option, - pub resolve_dispatch: Option, -} - #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Eq, Serialize)] #[serde(deny_unknown_fields)] pub struct NodeSurfaceRequestV1 { @@ -318,6 +308,29 @@ pub struct PrimitiveSearchCoverageV1 { pub recall: PrimitiveRecallV1, } +/// A public trait or interface among the context's selected symbols. +#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Eq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ContextExtensionPointV1 { + pub name: String, + pub kind: String, + pub file: String, + pub line: u32, + pub implementor_count: usize, +} + +/// Plan-mode enrichment: where the selected code can be extended and which +/// test files reach it. +#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Eq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ContextPlanV1 { + pub extension_points: Vec, + /// Test files calling the selected symbols within two hops; absent when + /// no symbol was selected to trace from. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub test_files: Option>, +} + #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] #[serde(deny_unknown_fields)] pub struct ContextResultV1 { @@ -341,6 +354,9 @@ pub struct ContextResultV1 { pub memory_matches_error: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub verified_graph_evidence: Option, + /// Present in plan mode when the verified graph answered. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub plan: Option, } impl ContextResultV1 { @@ -357,24 +373,6 @@ impl ContextResultV1 { } } -#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Eq, Serialize)] -#[serde(deny_unknown_fields)] -pub struct CalleeV1 { - pub node_id: String, - pub name: String, - pub kind: String, - pub file: String, - pub line: u32, - pub edge_kind: String, - pub dispatch_via_trait: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub depth: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub dispatch_from: Option, -} - -pub type CalleesResultV1 = Vec; - #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Eq, Serialize)] #[serde(deny_unknown_fields)] pub struct ImpactNodeV1 { @@ -758,6 +756,7 @@ mod tests { memory_graph_coverage: None, memory_matches_error: None, verified_graph_evidence: None, + plan: None, } } diff --git a/crates/tracedecay-contracts/src/retrieval/requests.rs b/crates/tracedecay-contracts/src/retrieval/requests.rs index 22e845f864..0575a5e13a 100644 --- a/crates/tracedecay-contracts/src/retrieval/requests.rs +++ b/crates/tracedecay-contracts/src/retrieval/requests.rs @@ -152,16 +152,42 @@ pub struct AffectedTestAttributionV1 { pub evidence_class: TestAttributionEvidenceClassV1, } +impl AffectedTestAttributionV1 { + /// Stale and unknown attributions are reported, but never count as + /// affected tests. + pub const fn is_current_candidate(&self) -> bool { + match self.evidence_class { + TestAttributionEvidenceClassV1::ConservativeDependencyCandidates + | TestAttributionEvidenceClassV1::ObservedCoverageCandidates + | TestAttributionEvidenceClassV1::PredictiveRankedCandidates => true, + TestAttributionEvidenceClassV1::StaleEvidence + | TestAttributionEvidenceClassV1::UnknownUnsupported => false, + } + } +} + #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct AffectedTestsResult { - pub tests: Vec, /// Exact class reported by the generation-bound attribution authority. - /// `tests` remains the compatibility projection of current candidates. - #[serde(default)] pub attributions: Vec, } +impl AffectedTestsResult { + /// Distinct current-candidate tests in identity order. + pub fn current_tests(&self) -> Vec { + let mut tests: Vec<_> = self + .attributions + .iter() + .filter(|attribution| attribution.is_current_candidate()) + .map(|attribution| attribution.test.clone()) + .collect(); + tests.sort(); + tests.dedup(); + tests + } +} + #[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct SessionLookupRequest { diff --git a/crates/tracedecay-contracts/src/retrieval/symbol_graph.rs b/crates/tracedecay-contracts/src/retrieval/symbol_graph.rs index 34342c5e46..c6740e6cab 100644 --- a/crates/tracedecay-contracts/src/retrieval/symbol_graph.rs +++ b/crates/tracedecay-contracts/src/retrieval/symbol_graph.rs @@ -33,6 +33,16 @@ impl CodeGraphReadFreshnessV1 { } } +/// The code-graph generation an operation read and that generation's +/// freshness. A stale seat answers soundly for its generation but may trail +/// the live worktree. +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct ServedCodeGraphGenerationV1 { + pub generation: String, + pub freshness: CodeGraphReadFreshnessV1, +} + /// Optional narrowing inside the immutable project/repository/worktree scope /// carried by [`RequestContext`]. A path prefix never establishes identity or /// authorization. @@ -82,6 +92,19 @@ pub struct SymbolRelationRecord { pub depth: Option, } +/// One implementation match with its exact source: the implementing +/// impl/class block (methods included) for a trait selector, or the function +/// body for a method selector. +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct ImplementationRecord { + pub symbol: SymbolPrimitiveRecord, + pub edge_kind: String, + /// Trait or interface node a trait-selector match was reached through. + pub dispatch_from: Option, + pub body: String, +} + #[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq)] #[serde(deny_unknown_fields)] pub struct TypeHierarchyRecord { @@ -479,7 +502,7 @@ pub trait SymbolGraphPrimitivePort { &'a self, context: SymbolGraphPortContext<'a>, request: &'a ImplementationsRequest, - ) -> SymbolGraphPortFuture<'a, SymbolRelationRecord>; + ) -> SymbolGraphPortFuture<'a, ImplementationRecord>; fn type_hierarchy<'a>( &'a self, diff --git a/crates/tracedecay-contracts/src/sdk_catalog.rs b/crates/tracedecay-contracts/src/sdk_catalog.rs index 00bd3f1c89..ec7f7537ce 100644 --- a/crates/tracedecay-contracts/src/sdk_catalog.rs +++ b/crates/tracedecay-contracts/src/sdk_catalog.rs @@ -9,7 +9,7 @@ use std::collections::BTreeSet; use std::sync::LazyLock; use tracedecay_tool_catalog::{ - ApplicationSurfaceOperation, BindingStatus, BindingSurface, CatalogValidationError, + ApplicationSurfaceOperation, BindingSurface, CatalogValidationError, ExecutableBindingAvailabilityV1, ExecutableBindingRegistryV1, ExecutableUnavailableDispositionV1, OperationId, RouteExposureV1, SdkExecutableBindingAvailabilityV1, SdkExecutableBindingRegistryV1, SdkExecutableBindingV1, @@ -22,8 +22,7 @@ use crate::application_catalog_projection::{ use crate::{ ApplicationContractError, application_catalog_contributions, handoff_executable_binding_registry, multi_root::multi_root_executable_binding_registry, - retained_surface_executable_binding_registry, work_executable_binding_registry, - workflow_executable_binding_registry, + work_executable_binding_registry, workflow_executable_binding_registry, }; /// Canonical executable HTTP projection for every application-surface handler. @@ -109,7 +108,16 @@ pub fn application_http_route_path(operation: ApplicationSurfaceOperation) -> St | ApplicationSurfaceOperation::HealthRead | ApplicationSurfaceOperation::HealthDelta | ApplicationSurfaceOperation::StorageStatus - | ApplicationSurfaceOperation::DiagnosticsRead) => { + | ApplicationSurfaceOperation::DiagnosticsRead + | ApplicationSurfaceOperation::Context + | ApplicationSurfaceOperation::Node + | ApplicationSurfaceOperation::Impact + | ApplicationSurfaceOperation::Similar + | ApplicationSurfaceOperation::Redundancy + | ApplicationSurfaceOperation::RenamePreview + | ApplicationSurfaceOperation::PortStatus + | ApplicationSurfaceOperation::PortOrder + | ApplicationSurfaceOperation::Todos) => { format!("/primitives/{}", operation.as_str()) } operation @ (ApplicationSurfaceOperation::ConfigurationList @@ -139,21 +147,61 @@ pub fn application_http_route_path(operation: ApplicationSurfaceOperation) -> St | ApplicationSurfaceOperation::ContextScoutFeedback) => { format!("/context-scout/{}", operation.as_str()) } + operation @ (ApplicationSurfaceOperation::StrReplace + | ApplicationSurfaceOperation::MultiStrReplace + | ApplicationSurfaceOperation::InsertAt + | ApplicationSurfaceOperation::AstGrepRewrite + | ApplicationSurfaceOperation::ReplaceSymbol + | ApplicationSurfaceOperation::InsertAtSymbol + | ApplicationSurfaceOperation::MoveSymbol + | ApplicationSurfaceOperation::RenameSymbol + | ApplicationSurfaceOperation::SourceEditReconcile + | ApplicationSurfaceOperation::SourceEditRollback) => { + format!("/source-edit/{}", operation.as_str()) + } + operation @ (ApplicationSurfaceOperation::FactStoreCurate + | ApplicationSurfaceOperation::FactStoreAdd + | ApplicationSurfaceOperation::FactStoreSearch + | ApplicationSurfaceOperation::FactStoreProbe + | ApplicationSurfaceOperation::FactStoreRelated + | ApplicationSurfaceOperation::FactStoreReason + | ApplicationSurfaceOperation::FactStoreContradict + | ApplicationSurfaceOperation::FactStoreGet + | ApplicationSurfaceOperation::FactStoreUpdate + | ApplicationSurfaceOperation::FactStoreRemove + | ApplicationSurfaceOperation::FactStoreSupersede + | ApplicationSurfaceOperation::FactStoreList + | ApplicationSurfaceOperation::FactFeedback + | ApplicationSurfaceOperation::MemoryStatus + | ApplicationSurfaceOperation::SessionRefreshStatus + | ApplicationSurfaceOperation::SessionRefreshCancel + | ApplicationSurfaceOperation::SessionRefreshBegin + | ApplicationSurfaceOperation::MessageSearch + | ApplicationSurfaceOperation::SessionsFor + | ApplicationSurfaceOperation::Workflows + | ApplicationSurfaceOperation::LcmStatus + | ApplicationSurfaceOperation::LcmDoctor + | ApplicationSurfaceOperation::LcmLoadSession + | ApplicationSurfaceOperation::LcmGrep + | ApplicationSurfaceOperation::LcmDescribe + | ApplicationSurfaceOperation::LcmExpand + | ApplicationSurfaceOperation::LcmExpandQuery) => { + format!("/retained/{}", operation.as_str()) + } } } /// Mounted executable authorities outside the canonical application surface. /// /// Application operations project as one registry above. Work, Workflow, -/// retained, handoff, and multi-root keep separate entries because they have -/// distinct operation identities and runtime owners. +/// handoff, and multi-root keep separate entries because they have distinct +/// operation identities and runtime owners. fn mounted_executable_binding_registries() -> Result>, ApplicationContractError> { Ok(vec![ Cow::Borrowed(application_http_executable_binding_registry()?), Cow::Borrowed(work_executable_binding_registry()?), Cow::Borrowed(workflow_executable_binding_registry()?), - Cow::Owned(retained_surface_executable_binding_registry()?), Cow::Owned(handoff_executable_binding_registry()?), Cow::Owned(multi_root_executable_binding_registry()?), ]) @@ -186,11 +234,7 @@ pub fn sdk_executable_binding_registry() contribution .bindings() .iter() - .filter(|binding| { - binding.surface() == BindingSurface::Mcp - && matches!(binding.status(), BindingStatus::Current) - && !binding.is_alias() - }) + .filter(|binding| binding.surface() == BindingSurface::Mcp) .map(|binding| project_mcp_availability(mcp_registry, binding)) .collect::, _>>()? .into_iter() @@ -456,11 +500,6 @@ mod tests { .flat_map(|contribution| contribution.bindings().to_vec()) .filter(|binding| { binding.surface() == BindingSurface::Http - && matches!( - binding.status(), - tracedecay_tool_catalog::BindingStatus::Current - ) - && !binding.is_alias() && binding.operation().as_str().starts_with("code_") }) .map(|binding| { @@ -659,14 +698,7 @@ mod tests { let mcp_bindings = contribution .bindings() .iter() - .filter(|surface| { - surface.surface() == BindingSurface::Mcp - && matches!( - surface.status(), - tracedecay_tool_catalog::BindingStatus::Current - ) - && !surface.is_alias() - }) + .filter(|surface| surface.surface() == BindingSurface::Mcp) .collect::>(); assert!( !mcp_bindings.is_empty(), @@ -701,14 +733,7 @@ mod tests { let expected = contributions .iter() .flat_map(|contribution| contribution.bindings()) - .filter(|binding| { - binding.surface() == BindingSurface::Mcp - && matches!( - binding.status(), - tracedecay_tool_catalog::BindingStatus::Current - ) - && !binding.is_alias() - }) + .filter(|binding| binding.surface() == BindingSurface::Mcp) .map(|binding| { let operation = ApplicationSurfaceOperation::from_tool_name(binding.operation().as_str()) @@ -732,14 +757,11 @@ mod tests { assert_eq!(actual, expected); for contribution in &contributions { - for surface in contribution.bindings().iter().filter(|binding| { - binding.surface() == BindingSurface::Mcp - && matches!( - binding.status(), - tracedecay_tool_catalog::BindingStatus::Current - ) - && !binding.is_alias() - }) { + for surface in contribution + .bindings() + .iter() + .filter(|binding| binding.surface() == BindingSurface::Mcp) + { let operation = ApplicationSurfaceOperation::from_tool_name(surface.operation().as_str()) .map_or_else( diff --git a/crates/tracedecay-contracts/src/source_edit.rs b/crates/tracedecay-contracts/src/source_edit.rs index 4b3540ff11..b1399e321b 100644 --- a/crates/tracedecay-contracts/src/source_edit.rs +++ b/crates/tracedecay-contracts/src/source_edit.rs @@ -12,12 +12,12 @@ use tracedecay_tool_catalog::{ use crate::capability_manifest::{ ApplicationCapabilityManifestInput, application_capability_manifest, }; +use crate::current_bindings; use crate::error::ApplicationContractError; use crate::handlers::{ApplicationHandlerDescriptor, ApplicationOperation}; use crate::result::ResultContractRef; use crate::retrieval::catalog::APPLICATION_DEFAULT_PROFILE_ID; use crate::source_edit_rollback::{source_edit_rollback_operation, source_edit_rollback_schema}; -use crate::{current_bindings, current_bindings_with_slug}; /// `serde` `skip_serializing_if` predicate for default-off flags. #[allow(clippy::trivially_copy_pass_by_ref)] @@ -397,6 +397,8 @@ const SOURCE_EDIT_KINDS: [SourceEditKind; 8] = [ const SOURCE_EDIT_SURFACES: [BindingSurface; 2] = [BindingSurface::Cli, BindingSurface::Mcp]; +const SOURCE_EDIT_SERVICE_ID: &str = "service.application.source-edit"; + pub fn source_edit_operation( kind: SourceEditKind, ) -> Result { @@ -420,19 +422,25 @@ pub fn source_edit_handler_descriptors() let mut descriptors = SOURCE_EDIT_KINDS .into_iter() .map(|kind| { - ApplicationHandlerDescriptor::new( + ApplicationHandlerDescriptor::for_catalog_operation( + kind.operation_name(), + SOURCE_EDIT_SERVICE_ID, source_edit_operation(kind)?, source_edit_schema(kind, "request")?, source_edit_schema(kind, "result")?, ) }) .collect::, _>>()?; - descriptors.push(ApplicationHandlerDescriptor::new( + descriptors.push(ApplicationHandlerDescriptor::for_catalog_operation( + "source_edit_reconcile", + SOURCE_EDIT_SERVICE_ID, source_edit_reconciliation_operation()?, source_edit_reconciliation_schema("request")?, source_edit_reconciliation_schema("result")?, )?); - descriptors.push(ApplicationHandlerDescriptor::new( + descriptors.push(ApplicationHandlerDescriptor::for_catalog_operation( + "source_edit_rollback", + SOURCE_EDIT_SERVICE_ID, source_edit_rollback_operation()?, source_edit_rollback_schema("request")?, source_edit_rollback_schema("result")?, @@ -521,10 +529,9 @@ pub fn source_edit_catalog_contribution() -> Result Result &'static str { match self { Self::RegisteredSchema => "registered_schema", - Self::RuntimeWriterLedger => "runtime_writer_ledger", } } } diff --git a/crates/tracedecay-contracts/src/storage/debris.rs b/crates/tracedecay-contracts/src/storage/debris.rs index 238fbb8e98..2426d4adf4 100644 --- a/crates/tracedecay-contracts/src/storage/debris.rs +++ b/crates/tracedecay-contracts/src/storage/debris.rs @@ -2,27 +2,24 @@ //! //! Recovery and corruption artifacts (`*.corrupt-*`, `*.corrupt`, //! `*.recovered*`, `recovery-*`) accumulate as loose siblings of live stores with no owner -//! surface. This module gives them a typed classifier, a single quarantine -//! location contract with metadata, and a scan read model that a Doctor producer -//! turns into an `IncidentDebrisPresent` finding. It performs no filesystem -//! effect: detection consumes already-listed file names, and quarantine is a -//! declarative record the owning storage operation later enacts. +//! surface. This module gives them a typed classifier and a scan read model +//! that a Doctor producer turns into an `IncidentDebrisPresent` finding. It +//! performs no filesystem effect: detection consumes already-listed file names, +//! and retention deletes classified debris directly. use serde::{Deserialize, Serialize}; use tracedecay_domain::UtcMicros; use crate::error::ApplicationContractError; -use super::identity::{ - QuarantineLocationV1, RelativeArtifactPathV1, StorageByteSizeV1, StoreKeyV1, -}; +use super::identity::{RelativeArtifactPathV1, StorageByteSizeV1, StoreKeyV1}; /// The class of incident artifact a debris file represents. #[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)] #[serde(rename_all = "snake_case")] pub enum IncidentDebrisKindV1 { - /// A `*.corrupt-*` or `*.corrupt` sibling: a store copied aside after - /// corruption detection. + /// A `*.corrupt-*` sibling: a store copied aside after corruption + /// detection. Corrupt, /// A `*.recovered*` sibling: the output of a recovery pass. Recovered, @@ -36,7 +33,7 @@ impl IncidentDebrisKindV1 { /// /// Matching is deliberately narrow so a live store (`sessions.db`, /// `sessions.db-wal`, `sessions.db-shm`) is never misclassified as debris. - /// The patterns mirror the measured evidence: `*.corrupt-*`, `*.corrupt`, + /// The patterns mirror the measured evidence: `*.corrupt-*`, /// `*.recovered*`, and `recovery-*`. #[must_use] pub fn classify(file_name: &str) -> Option { @@ -45,10 +42,7 @@ impl IncidentDebrisKindV1 { return Some(Self::RecoveryScratch); } // `*.corrupt-`: a `.corrupt-` segment somewhere in the name. - // The bare `*.corrupt` suffix is the same artifact from an older - // quarantine naming convention; profiles upgraded across that change - // still carry it, and no live store name ends in `.corrupt`. - if file_name.contains(".corrupt-") || file_name.ends_with(".corrupt") { + if file_name.contains(".corrupt-") { return Some(Self::Corrupt); } // `*.recovered*`: a `.recovered` segment somewhere in the name. @@ -93,75 +87,6 @@ impl IncidentDebrisArtifactV1 { } } -/// The single quarantine location debris is collected into, with metadata. -/// -/// Recovery/corruption artifacts must be written into one quarantined -/// location with metadata, surfaced by Doctor and collected by the -/// retention machinery, never left as loose siblings. This contract names -/// that location (store-relative) and the retention window after which -/// quarantined artifacts become collection-eligible. -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct QuarantineContractV1 { - /// The single store-relative directory debris is moved into. - pub location: QuarantineLocationV1, - /// Micros after which a quarantined artifact is collection-eligible. - pub retention_window_micros: i64, -} - -impl QuarantineContractV1 { - /// Validate the contract. The retention window must be positive; a - /// non-positive window would make every artifact instantly collectible, - /// defeating the owner-visible retention guarantee. - pub fn validate(&self) -> Result<(), ApplicationContractError> { - if self.retention_window_micros <= 0 { - return Err(ApplicationContractError::ZeroValue { - field: "quarantine retention window", - }); - } - Ok(()) - } - - /// Declare the quarantined placement for an artifact. This is a record, not - /// a move: the owning storage operation enacts the relocation and honors the - /// window. The eligibility time is `quarantined_at + retention_window`. - pub fn quarantine( - &self, - artifact: IncidentDebrisArtifactV1, - quarantined_at: UtcMicros, - ) -> Result { - self.validate()?; - Ok(QuarantinedArtifactV1 { - collection_eligible_at: UtcMicros( - quarantined_at - .0 - .saturating_add(self.retention_window_micros), - ), - location: self.location.clone(), - artifact, - quarantined_at, - }) - } -} - -/// An artifact declared into the quarantine location with its collection window. -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct QuarantinedArtifactV1 { - pub artifact: IncidentDebrisArtifactV1, - pub location: QuarantineLocationV1, - pub quarantined_at: UtcMicros, - pub collection_eligible_at: UtcMicros, -} - -impl QuarantinedArtifactV1 { - /// True when `now` has reached the collection-eligibility watermark. - #[must_use] - pub fn is_collection_eligible(&self, now: UtcMicros) -> bool { - now.0 >= self.collection_eligible_at.0 - } -} - /// The read model of one debris scan over a store's siblings. /// /// A scan is *complete* when every sibling was listed and classified; it is @@ -226,19 +151,6 @@ mod tests { ); } - /// The pre-timestamp quarantine naming an upgraded profile still carries. - #[test] - fn classifier_matches_bare_corrupt_suffix() { - assert_eq!( - IncidentDebrisKindV1::classify("tracedecay.db.corrupt"), - Some(IncidentDebrisKindV1::Corrupt) - ); - assert_eq!( - IncidentDebrisKindV1::classify("sessions.db.corrupt"), - Some(IncidentDebrisKindV1::Corrupt) - ); - } - #[test] fn classifier_never_flags_live_store_files() { for name in [ @@ -251,36 +163,6 @@ mod tests { } } - #[test] - fn quarantine_computes_eligibility_and_rejects_nonpositive_window() { - let location = QuarantineLocationV1::new("quarantine").expect("valid"); - let contract = QuarantineContractV1 { - location, - retention_window_micros: 1_000, - }; - let path = RelativeArtifactPathV1::new("sessions.db.corrupt-9").expect("valid"); - let artifact = IncidentDebrisArtifactV1::classify_path( - store(), - path, - StorageByteSizeV1(10), - UtcMicros(1), - ) - .expect("ok") - .expect("debris"); - let quarantined = contract - .quarantine(artifact, UtcMicros(500)) - .expect("quarantined"); - assert_eq!(quarantined.collection_eligible_at, UtcMicros(1_500)); - assert!(!quarantined.is_collection_eligible(UtcMicros(1_499))); - assert!(quarantined.is_collection_eligible(UtcMicros(1_500))); - - let bad = QuarantineContractV1 { - location: QuarantineLocationV1::new("quarantine").expect("valid"), - retention_window_micros: 0, - }; - assert!(bad.validate().is_err()); - } - #[test] fn scan_totals_bytes_and_reports_emptiness() { let path = RelativeArtifactPathV1::new("sessions.db.corrupt-9").expect("valid"); diff --git a/crates/tracedecay-contracts/src/storage/findings.rs b/crates/tracedecay-contracts/src/storage/findings.rs index 04a9fff88b..a4b31217b1 100644 --- a/crates/tracedecay-contracts/src/storage/findings.rs +++ b/crates/tracedecay-contracts/src/storage/findings.rs @@ -434,7 +434,7 @@ pub fn incident_debris_finding( scan.artifact_count(), scan.total_bytes().get() ), - "quarantine-eligible incident artifacts present beside a live store", + "incident artifacts awaiting deletion beside a live store", )? } else if scan.listing_complete { clean_finding( diff --git a/crates/tracedecay-contracts/src/storage/identity.rs b/crates/tracedecay-contracts/src/storage/identity.rs index f61ed09a11..38d130ae75 100644 --- a/crates/tracedecay-contracts/src/storage/identity.rs +++ b/crates/tracedecay-contracts/src/storage/identity.rs @@ -23,9 +23,6 @@ application_identifier!( /// A store-relative path to an incident-debris artifact (for example /// `sessions.db.corrupt-1721692800`). Store-relative, never absolute. RelativeArtifactPathV1 => ("storage relative artifact path", 512), - /// The single logical quarantine location debris is collected into. A - /// store-relative directory name, never an absolute path. - QuarantineLocationV1 => ("storage quarantine location", 256), ); /// A byte size measurement. A newtype keeps sizes from being confused with diff --git a/crates/tracedecay-contracts/src/storage/inventory.rs b/crates/tracedecay-contracts/src/storage/inventory.rs index 5e001b6694..1930bbf88b 100644 --- a/crates/tracedecay-contracts/src/storage/inventory.rs +++ b/crates/tracedecay-contracts/src/storage/inventory.rs @@ -76,34 +76,26 @@ pub struct CodeGenerationRetentionRecordV1 { /// Scope roots under the shared `code-index-v1/` parent that no live /// canonical project root names. Absent (zero) when the reporter could not /// prove the live-root set, which is also when nothing may be collected. - #[serde(default)] pub stranded_scope_count: u64, - #[serde(default = "zero_storage_bytes")] pub stranded_scope_bytes: StorageByteSizeV1, /// Sealed graph generation artifacts in the project graph store whose /// generation is no longer any projection's verified head. They are /// retired when a newer head installs; a count here means that /// retirement has not run since the last publication. - #[serde(default)] pub superseded_sealed_generation_count: u64, - #[serde(default = "zero_storage_bytes")] pub superseded_sealed_generation_bytes: StorageByteSizeV1, /// `.staging-*` directories a seal left under the sealed root: a build /// that never installed. Swept on the next store open. - #[serde(default)] pub abandoned_sealed_staging_count: u64, - #[serde(default = "zero_storage_bytes")] pub abandoned_sealed_staging_bytes: StorageByteSizeV1, /// Bytes of the sealed artifacts every projection's verified head serves /// from: the size the live staging container converges to once its /// duplicate and superseded rows are gone. - #[serde(default = "zero_storage_bytes")] pub sealed_head_generation_bytes: StorageByteSizeV1, /// On-disk bytes of the live staging container (`tracedecay.grafeo` and /// its WAL). Grafeo rewrites the container out of place on every /// checkpoint and truncates the dead generation, so this shrinks on its /// own once retired rows are deleted from the engine. - #[serde(default = "zero_storage_bytes")] pub live_graph_container_bytes: StorageByteSizeV1, /// Retirements the journal has already decided whose native rows are /// still in the live container: retirement tombstones awaiting their @@ -111,16 +103,9 @@ pub struct CodeGenerationRetentionRecordV1 { /// hibernated engine is never opened just to delete: opening a /// multi-gigabyte LPG container costs about twice its size in RAM, so /// these wait for the next publication, which holds the engine open. - #[serde(default)] pub deferred_native_retirement_count: u64, } -/// `serde(default)` needs a value, and `StorageByteSizeV1` deliberately has no -/// `Default` impl; zero bytes is the only meaningful absence here. -fn zero_storage_bytes() -> StorageByteSizeV1 { - StorageByteSizeV1::ZERO -} - impl CodeGenerationRetentionRecordV1 { pub fn validate(&self) -> Result<(), ApplicationContractError> { if self.collectable_generation_count > self.superseded_generation_count @@ -332,22 +317,4 @@ mod tests { assert!(!record.has_collectable_generations()); assert!(record.has_stranded_scopes()); } - - #[test] - fn stranded_scope_totals_default_to_zero_for_records_without_them() { - let record: CodeGenerationRetentionRecordV1 = serde_json::from_str( - r#"{ - "store": "code-index-v1", - "superseded_generation_count": 3, - "superseded_generation_bytes": 3000, - "collectable_generation_count": 1, - "collectable_generation_bytes": 1000 - }"#, - ) - .expect("records predating scope reconciliation stay readable"); - - assert_eq!(record.stranded_scope_count, 0); - assert_eq!(record.stranded_scope_bytes, StorageByteSizeV1(0)); - assert!(!record.has_stranded_scopes()); - } } diff --git a/crates/tracedecay-contracts/src/storage/mod.rs b/crates/tracedecay-contracts/src/storage/mod.rs index f00f7fcb58..7f5af6b452 100644 --- a/crates/tracedecay-contracts/src/storage/mod.rs +++ b/crates/tracedecay-contracts/src/storage/mod.rs @@ -10,8 +10,8 @@ //! - [`telemetry`] (§7): per-store size, per-table growth, free-page ratio, soft //! budgets, and the [`telemetry::StoreSizeTelemetryPort`] seam over //! `dbstat`/pragma sources. -//! - [`debris`] (§5): incident-artifact classification, the quarantine-location -//! contract, and debris scan read models. +//! - [`debris`] (§5): incident-artifact classification and debris scan read +//! models. //! - [`compaction`] (§6): the free-page-ratio compaction trigger policy, off the //! hot path by construction. //! - [`inventory`]: orphan / retention-backlog read models. @@ -34,18 +34,14 @@ pub use convergence::{ SchemaConvergenceFindingV1, SchemaConvergenceProgressV1, SchemaConvergenceStageV1, SchemaConvergenceStateV1, }; -pub use debris::{ - IncidentDebrisArtifactV1, IncidentDebrisKindV1, IncidentDebrisScanV1, QuarantineContractV1, - QuarantinedArtifactV1, -}; +pub use debris::{IncidentDebrisArtifactV1, IncidentDebrisKindV1, IncidentDebrisScanV1}; pub use findings::{ code_generation_retention_finding, incident_debris_finding, orphan_store_finding, over_budget_finding, pending_schema_migration_finding, retention_backlog_finding, table_growth_finding, }; pub use identity::{ - FreePageRatioV1, QuarantineLocationV1, RelativeArtifactPathV1, StorageByteSizeV1, StoreKeyV1, - TableNameV1, + FreePageRatioV1, RelativeArtifactPathV1, StorageByteSizeV1, StoreKeyV1, TableNameV1, }; pub use inventory::{ CodeGenerationRetentionRecordV1, OrphanStoreRecordV1, RetentionBacklogRecordV1, diff --git a/crates/tracedecay-contracts/src/surface_binding.rs b/crates/tracedecay-contracts/src/surface_binding.rs index 034d8d5aa9..a14060bb92 100644 --- a/crates/tracedecay-contracts/src/surface_binding.rs +++ b/crates/tracedecay-contracts/src/surface_binding.rs @@ -5,8 +5,8 @@ //! and the default binding shape in its own per-surface loop. use tracedecay_tool_catalog::{ - ApplicationSurfaceOperation, BindingId, BindingStatus, BindingSurface, CapabilityId, - ProtocolRevisionRange, SurfaceBindingInputV1, SurfaceBindingV1, SurfaceOperationName, + ApplicationSurfaceOperation, BindingId, BindingSurface, CapabilityId, ProtocolRevisionRange, + SurfaceBindingInputV1, SurfaceBindingV1, SurfaceOperationName, }; use crate::error::ApplicationContractError; @@ -34,24 +34,14 @@ pub(crate) fn current_bindings( capability_id: &CapabilityId, operation: &str, surfaces: impl IntoIterator, -) -> Result<(Vec, Vec), ApplicationContractError> { - current_bindings_with_slug(capability_id, operation, operation, surfaces) -} - -/// [`current_bindings`] for the operations whose binding-id slug differs from -/// their wire operation name. -pub(crate) fn current_bindings_with_slug( - capability_id: &CapabilityId, - operation: &str, - slug: &str, - surfaces: impl IntoIterator, ) -> Result<(Vec, Vec), ApplicationContractError> { let surfaces = surfaces.into_iter(); let expected = surfaces.size_hint().0; let mut bindings = Vec::with_capacity(expected); let mut binding_ids = Vec::with_capacity(expected); for surface in surfaces { - let binding_id = BindingId::new(format!("binding.{}.{slug}.v1", surface_name(surface)))?; + let binding_id = + BindingId::new(format!("binding.{}.{operation}.v1", surface_name(surface)))?; bindings.push(SurfaceBindingV1::new(SurfaceBindingInputV1 { binding_id: binding_id.clone(), capability_id: capability_id.clone(), @@ -59,8 +49,6 @@ pub(crate) fn current_bindings_with_slug( operation: SurfaceOperationName::new(operation)?, protocol_revisions: ProtocolRevisionRange::new(1, 1)?, required_features: Vec::new(), - status: BindingStatus::Current, - alias_of: None, })?); binding_ids.push(binding_id); } @@ -94,8 +82,6 @@ pub(crate) fn current_application_bindings( operation: SurfaceOperationName::new(operation.name_for_surface(surface))?, protocol_revisions: ProtocolRevisionRange::new(1, 1)?, required_features: Vec::new(), - status: BindingStatus::Current, - alias_of: None, })?); binding_ids.push(binding_id); } diff --git a/crates/tracedecay-contracts/src/surface_contracts/callable_code.rs b/crates/tracedecay-contracts/src/surface_contracts/callable_code.rs index e01b3e3c50..af60d97a9a 100644 --- a/crates/tracedecay-contracts/src/surface_contracts/callable_code.rs +++ b/crates/tracedecay-contracts/src/surface_contracts/callable_code.rs @@ -2,7 +2,9 @@ use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -use tracedecay_domain::{ExactTechnicalTermKindV1, QueryNormalizationRevision, SanitizerRevision}; +use tracedecay_domain::{ + CodeGenerationId, ExactTechnicalTermKindV1, QueryNormalizationRevision, SanitizerRevision, +}; use crate::error::ApplicationContractError; use crate::result::OpaqueCursor; @@ -12,6 +14,7 @@ use crate::retrieval::{ GraphRelationRequest, ImplementationSelector, ImplementationsRequest, PageRequest, PhraseSearchRequest, PrimitiveRequest, ResultProjection, RetrievalOrder, RetrievalRequestMeta, SignatureSearchRequest, SymbolGraphScope, TypeHierarchyRequest, + UNPINNED_LATEST_GENERATION_SENTINEL, }; /// Surface-owned query semantics. Page size remains an invocation control, but @@ -27,6 +30,38 @@ pub struct CallableCodeSurfaceMeta { pub cursor: Option, } +/// First page of full evidence in source order: the navigation reads that +/// default their meta ignore relevance ranking. +impl Default for CallableCodeSurfaceMeta { + fn default() -> Self { + Self { + projection: ResultProjection::Evidence, + order: RetrievalOrder::SourcePosition, + cursor: None, + } + } +} + +const fn default_call_relation_depth() -> u32 { + 3 +} + +const fn default_type_hierarchy_depth() -> u32 { + 5 +} + +const fn default_resolve_trait_dispatch() -> bool { + true +} + +fn unpinned_latest_code_query_scope() -> CodeQueryScope { + CodeQueryScope { + generation: CodeGenerationId::new(UNPINNED_LATEST_GENERATION_SENTINEL) + .unwrap_or_else(|_| panic!("static unpinned-latest generation sentinel is valid")), + path_prefix: None, + } +} + impl CallableCodeSurfaceMeta { pub fn into_application(self, page: PageRequest) -> RetrievalRequestMeta { let Self { @@ -122,9 +157,12 @@ pub struct CodeSymbolSearchSurfaceRequest { #[serde(deny_unknown_fields)] pub struct CodeSignatureSearchSurfaceRequest { pub returns: Option, + #[serde(default)] pub params: Vec, pub is_async: Option, + #[serde(default)] pub scope: SymbolGraphScope, + #[serde(default)] pub meta: CallableCodeSurfaceMeta, } @@ -132,7 +170,9 @@ pub struct CodeSignatureSearchSurfaceRequest { #[serde(deny_unknown_fields)] pub struct CodeImplementationsSurfaceRequest { pub selector: ImplementationSelector, + #[serde(default)] pub scope: SymbolGraphScope, + #[serde(default)] pub meta: CallableCodeSurfaceMeta, } @@ -140,8 +180,11 @@ pub struct CodeImplementationsSurfaceRequest { #[serde(deny_unknown_fields)] pub struct CodeTypeHierarchySurfaceRequest { pub node_id: String, + #[serde(default = "default_type_hierarchy_depth")] pub maximum_depth: u32, + #[serde(default)] pub scope: SymbolGraphScope, + #[serde(default)] pub meta: CallableCodeSurfaceMeta, } @@ -149,8 +192,11 @@ pub struct CodeTypeHierarchySurfaceRequest { #[serde(deny_unknown_fields)] pub struct CodeCallersSurfaceRequest { pub node_id: String, + #[serde(default = "default_call_relation_depth")] pub maximum_depth: u32, + #[serde(default)] pub scope: SymbolGraphScope, + #[serde(default)] pub meta: CallableCodeSurfaceMeta, } @@ -234,9 +280,13 @@ impl CodeSymbolSearchSurfaceRequest { #[serde(deny_unknown_fields)] pub struct CodeCalleesSurfaceRequest { pub node_id: String, + #[serde(default = "default_call_relation_depth")] pub maximum_depth: u32, + #[serde(default = "default_resolve_trait_dispatch")] pub resolve_trait_dispatch: bool, + #[serde(default = "unpinned_latest_code_query_scope")] pub scope: CodeQueryScope, + #[serde(default)] pub meta: CallableCodeSurfaceMeta, } diff --git a/crates/tracedecay-contracts/src/work.rs b/crates/tracedecay-contracts/src/work.rs index 4e75df764b..5a14dd50b1 100644 --- a/crates/tracedecay-contracts/src/work.rs +++ b/crates/tracedecay-contracts/src/work.rs @@ -19,13 +19,6 @@ pub enum WorkRoutingSnapshotErrorV1 { Unavailable, } -#[derive(Clone, Copy, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum ReviewProposalDispositionV1 { - Rejected, - Superseded, -} - #[derive(Clone, Debug, Default, Deserialize, Serialize, JsonSchema, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct WorkRoutingSnapshotV1 { diff --git a/crates/tracedecay-contracts/src/work_attempt/product_admission.rs b/crates/tracedecay-contracts/src/work_attempt/product_admission.rs index 65190db05f..c7fd9c3397 100644 --- a/crates/tracedecay-contracts/src/work_attempt/product_admission.rs +++ b/crates/tracedecay-contracts/src/work_attempt/product_admission.rs @@ -14,10 +14,10 @@ use crate::{ ApplicationProblem, RequestContext, WorkGraphReadPortV1, WorkGraphReadRequestV1, WorkGraphReadV1, WorkProductApplicationErrorV1, WorkProductAttemptAdmissionErrorV1, WorkProductAttemptAdmissionOutcomeV1, WorkProductAttemptAdmissionPortV1, - WorkProductAttemptAdmissionV1, WorkProductBindingV1, WorkProductEventDraftV1, - WorkProductOwnerAuthorizationErrorV1, WorkProductOwnerAuthorizationPortV1, - WorkProductPortContextV1, WorkProductRevisionPinsV1, WorkProductSelectionScopeV1, - WorkRelationScopeV1, + WorkProductAttemptAdmissionV1, WorkProductAuthorizedRelationScopeV1, WorkProductBindingV1, + WorkProductEventDraftV1, WorkProductOwnerAuthorizationErrorV1, + WorkProductOwnerAuthorizationPortV1, WorkProductPortContextV1, WorkProductRevisionPinsV1, + WorkProductSelectionScopeV1, }; use super::{ @@ -90,12 +90,13 @@ where S: WorkGraphReadPortV1 + WorkProductOwnerAuthorizationPortV1, { admit_product_attempt_request(context, binding, observed_at)?; - let selection = - WorkProductSelectionScopeV1::relations(BTreeSet::from([WorkRelationScopeV1::Repository { + let selection = WorkProductSelectionScopeV1::relations(BTreeSet::from([ + WorkProductAuthorizedRelationScopeV1::Repository { project_id: context.scope().project_id.clone(), repository_id: context.scope().repository_id.clone(), - }])) - .map_err(|_| invalid_start_problem())?; + }, + ])) + .map_err(|_| invalid_start_problem())?; let authorized_scope = storage .authorize_scope(context, &selection, observed_at) .map_err(owner_problem)?; diff --git a/crates/tracedecay-contracts/src/work_catalog.rs b/crates/tracedecay-contracts/src/work_catalog.rs index ca4e292893..4ac45e9492 100644 --- a/crates/tracedecay-contracts/src/work_catalog.rs +++ b/crates/tracedecay-contracts/src/work_catalog.rs @@ -635,28 +635,15 @@ fn schema_ref(id: String) -> Result { mod tests { use tracedecay_tool_catalog::{CancellationPoint, RouteExposureV1}; - use super::{ - WORK_APPLICATION_OPERATION_IDS_V1, work_executable_binding, - work_executable_binding_registry, - }; + use super::{work_executable_binding, work_executable_binding_registry}; #[test] - fn work_registry_advertises_only_mounted_application_operations() { + fn every_work_binding_is_publicly_routed_and_cancellable_before_admission() { let registry = work_executable_binding_registry().unwrap(); - let advertised = registry + for binding in registry .iter() .filter_map(|availability| availability.binding()) - .collect::>(); - let expected = WORK_APPLICATION_OPERATION_IDS_V1 - .iter() - .map(|(operation, _, _)| format!("operation.work.{operation}")) - .collect::>(); - let actual = advertised - .iter() - .map(|binding| binding.operation_id().as_str().to_owned()) - .collect::>(); - assert_eq!(actual, expected); - for binding in advertised { + { let RouteExposureV1::Public { route_path, .. } = binding.exposure() else { panic!("available Work binding must have a public route"); }; @@ -666,23 +653,6 @@ mod tests { .cancellation() .observes(CancellationPoint::BeforeAdmission) ); - assert_ne!( - binding.request_schema().body()["title"], - serde_json::Value::String("Value".to_owned()) - ); - } - for retired in [ - "operation.work.snapshot", - "operation.work.delta", - "operation.work.replan_dependencies", - "operation.work.accept_task", - ] { - assert!( - registry - .get(&tracedecay_tool_catalog::OperationId::new(retired).unwrap()) - .is_none(), - "retired operation {retired} must not be advertised" - ); } } diff --git a/crates/tracedecay-contracts/src/work_evidence.rs b/crates/tracedecay-contracts/src/work_evidence.rs index 45f4d9514d..63c55474b3 100644 --- a/crates/tracedecay-contracts/src/work_evidence.rs +++ b/crates/tracedecay-contracts/src/work_evidence.rs @@ -241,7 +241,7 @@ pub enum WorkTaskSessionHydrationStateV1 { RetentionExpired, Unauthorized, Locked, - UnverifiableLegacy, + Unverifiable, } #[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] diff --git a/crates/tracedecay-contracts/src/work_product/read.rs b/crates/tracedecay-contracts/src/work_product/read.rs index a2962053e5..6009a40c5a 100644 --- a/crates/tracedecay-contracts/src/work_product/read.rs +++ b/crates/tracedecay-contracts/src/work_product/read.rs @@ -10,7 +10,7 @@ use tracedecay_domain::{ use crate::{ OpaqueCursor, RequestAdmission, RequestContext, WorkAttemptTopologyBindingV1, - WorkAttemptTopologyStateV1, WorkRelationScopeV1, + WorkAttemptTopologyStateV1, WorkProductAuthorizedRelationScopeV1, }; use super::{ @@ -642,7 +642,7 @@ where observed_at: UtcMicros, ) -> Result { let selection = WorkProductSelectionScopeV1::relations(BTreeSet::from([ - WorkRelationScopeV1::Repository { + WorkProductAuthorizedRelationScopeV1::Repository { project_id: context.scope().project_id.clone(), repository_id: context.scope().repository_id.clone(), }, diff --git a/crates/tracedecay-contracts/src/work_product/types.rs b/crates/tracedecay-contracts/src/work_product/types.rs index 4548169fa6..b26d534f81 100644 --- a/crates/tracedecay-contracts/src/work_product/types.rs +++ b/crates/tracedecay-contracts/src/work_product/types.rs @@ -57,7 +57,7 @@ pub enum WorkProductApplicationErrorV1 { ProposalAuthorityUnavailable, } -pub use tracedecay_domain::WorkProductAuthorizedRelationScopeV1 as WorkRelationScopeV1; +pub use tracedecay_domain::WorkProductAuthorizedRelationScopeV1; pub use tracedecay_domain::WorkProductSelectionScopeV1; /// Owner identity resolved by the registered profile authority. It is never diff --git a/crates/tracedecay-contracts/tests/contracts_suite/authorization_non_disclosure.rs b/crates/tracedecay-contracts/tests/contracts_suite/authorization_non_disclosure.rs deleted file mode 100644 index e3b2123219..0000000000 --- a/crates/tracedecay-contracts/tests/contracts_suite/authorization_non_disclosure.rs +++ /dev/null @@ -1,66 +0,0 @@ -use crate::common; - -use tracedecay_contracts::{ - ApplicationProblem, AuthorizationPortOutcome, AuthorizationService, ConcealedResourceCause, - NonDisclosureHooks, RetryDirective, -}; -use tracedecay_domain::UtcMicros; -use tracedecay_policy::authorization::SourceAuthorizationEvaluatorV1; - -#[test] -fn absent_out_of_scope_and_policy_hidden_resources_share_one_public_problem() { - let hooks = NonDisclosureHooks; - let public_shapes = [ - ConcealedResourceCause::Absent, - ConcealedResourceCause::OutsideScope, - ConcealedResourceCause::PolicyHidden, - ] - .map(|cause| { - serde_json::to_value(hooks.resource_problem(cause, RetryDirective::Never)).unwrap() - }); - - assert_eq!(public_shapes[0], public_shapes[1]); - assert_eq!(public_shapes[1], public_shapes[2]); - assert_eq!(public_shapes[0]["kind"], "not_found_or_not_authorized"); - assert!(public_shapes[0].get("detail").is_none()); - assert!(public_shapes[0].get("count").is_none()); - assert!(public_shapes[0].get("timing").is_none()); -} - -#[test] -fn cursor_and_anchor_rejections_use_the_same_non_disclosing_shape() { - let hooks = NonDisclosureHooks; - let cursor = hooks.cursor_problem(RetryDirective::AfterRevalidate); - let anchor = hooks.anchor_problem(RetryDirective::AfterRevalidate); - - assert_eq!(cursor, anchor); - assert_eq!( - cursor, - ApplicationProblem::not_found_or_not_authorized(RetryDirective::AfterRevalidate) - ); -} - -#[test] -fn denied_and_absent_sources_are_indistinguishable_after_policy_evaluation() { - let operation = common::operation(); - let context = common::context(&operation); - let denied = AuthorizationService::new( - common::StaticAuthorizationPort::new(AuthorizationPortOutcome::Snapshot(Box::new( - common::source_snapshot(common::source_authorization_input("project_owner_mismatch")), - ))), - SourceAuthorizationEvaluatorV1::default(), - ) - .admit(&context, &operation, UtcMicros(10)) - .unwrap_err(); - let absent = AuthorizationService::new( - common::StaticAuthorizationPort::new(AuthorizationPortOutcome::Absent), - SourceAuthorizationEvaluatorV1::default(), - ) - .admit(&context, &operation, UtcMicros(10)) - .unwrap_err(); - - assert_eq!( - serde_json::to_value(denied).expect("problem serializes"), - serde_json::to_value(absent).expect("problem serializes") - ); -} diff --git a/crates/tracedecay-contracts/tests/contracts_suite/authorization_recheck.rs b/crates/tracedecay-contracts/tests/contracts_suite/authorization_recheck.rs deleted file mode 100644 index d2c9f9e66a..0000000000 --- a/crates/tracedecay-contracts/tests/contracts_suite/authorization_recheck.rs +++ /dev/null @@ -1,175 +0,0 @@ -use crate::common; - -use std::cell::Cell; - -use tracedecay_contracts::{ApplicationProblemKind, AuthorizationService}; -use tracedecay_domain::UtcMicros; -use tracedecay_policy::authorization::{ - AuthorizationSnapshotStateV1, ExternalContentStatusV1, PolicyEvaluatorVersionV1, - SinkAdmissionProofV1, SourceAuthorizationDecisionV1, SourceAuthorizationEvaluator, - SourceAuthorizationEvaluatorV1, SourceAuthorizationInputV1, -}; - -fn requires_sink_admission(_proof: &SinkAdmissionProofV1) {} - -#[test] -fn admission_preserves_source_proof_until_the_effect_recheck() { - let operation = common::operation(); - let context = common::context(&operation); - let initial = common::authorized_source_input(); - let mut current = initial.clone(); - current.evaluated_at.0 += 1; - let service = AuthorizationService::new( - common::SequencedAuthorizationPort::snapshots([ - common::source_snapshot(initial.clone()), - common::source_snapshot(current), - ]), - SourceAuthorizationEvaluatorV1::default(), - ); - - let admission = service - .admit(&context, &operation, UtcMicros(10)) - .expect("live input admits with an opaque source proof"); - assert_eq!( - admission.source_proof().effective_grant().budgets, - initial.requested_access.budget - ); - - let sink_proof = service - .recheck_effect(&context, &operation, &admission, UtcMicros(11)) - .expect("unchanged authority admits immediately before the effect"); - requires_sink_admission(&sink_proof); - assert_eq!( - sink_proof.effective_grant().budgets, - initial.requested_access.budget - ); -} - -#[test] -fn stale_policy_at_effect_recheck_returns_stale_without_sink_proof() { - let operation = common::operation(); - let context = common::context(&operation); - let initial = common::authorized_source_input(); - let mut stale = initial.clone(); - stale.snapshot_state = AuthorizationSnapshotStateV1::Stale; - stale.evaluated_at.0 += 1; - let service = AuthorizationService::new( - common::SequencedAuthorizationPort::snapshots([ - common::source_snapshot(initial), - common::source_snapshot(stale), - ]), - SourceAuthorizationEvaluatorV1::default(), - ); - - let admission = service.admit(&context, &operation, UtcMicros(10)).unwrap(); - let problem = service - .recheck_effect(&context, &operation, &admission, UtcMicros(11)) - .unwrap_err(); - - assert_eq!(problem.kind(), ApplicationProblemKind::Stale); -} - -#[test] -fn deletion_after_admission_cannot_reach_an_effect_sink() { - let operation = common::operation(); - let context = common::context(&operation); - let initial = common::authorized_source_input(); - let mut deleted = initial.clone(); - deleted.content_status = ExternalContentStatusV1::AuthoritativeDeleted; - deleted.evaluated_at.0 += 1; - let service = AuthorizationService::new( - common::SequencedAuthorizationPort::snapshots([ - common::source_snapshot(initial), - common::source_snapshot(deleted), - ]), - SourceAuthorizationEvaluatorV1::default(), - ); - - let admission = service.admit(&context, &operation, UtcMicros(10)).unwrap(); - let problem = service - .recheck_effect(&context, &operation, &admission, UtcMicros(11)) - .unwrap_err(); - - assert_eq!( - problem.kind(), - ApplicationProblemKind::NotFoundOrNotAuthorized - ); -} - -#[test] -fn narrowing_budget_after_admission_cannot_widen_effect_authority() { - let operation = common::operation(); - let context = common::context(&operation); - let initial = common::authorized_source_input(); - let mut narrowed = initial.clone(); - narrowed.requester_grant.budgets.bytes = 999; - narrowed.evaluated_at.0 += 1; - let service = AuthorizationService::new( - common::SequencedAuthorizationPort::snapshots([ - common::source_snapshot(initial), - common::source_snapshot(narrowed), - ]), - SourceAuthorizationEvaluatorV1::default(), - ); - - let admission = service.admit(&context, &operation, UtcMicros(10)).unwrap(); - let problem = service - .recheck_effect(&context, &operation, &admission, UtcMicros(11)) - .unwrap_err(); - - assert_eq!( - problem.kind(), - ApplicationProblemKind::NotFoundOrNotAuthorized - ); -} - -struct TamperingEvaluator { - inner: SourceAuthorizationEvaluatorV1, - evaluations: Cell, -} - -impl TamperingEvaluator { - fn new() -> Self { - Self { - inner: SourceAuthorizationEvaluatorV1::default(), - evaluations: Cell::new(0), - } - } -} - -impl SourceAuthorizationEvaluator for TamperingEvaluator { - fn evaluator_version(&self) -> &PolicyEvaluatorVersionV1 { - self.inner.evaluator_version() - } - - fn evaluate(&self, input: &SourceAuthorizationInputV1) -> SourceAuthorizationDecisionV1 { - let evaluation = self.evaluations.get(); - self.evaluations.set(evaluation + 1); - let mut decision = self.inner.evaluate(input); - if evaluation > 0 { - decision - .effective_grant - .as_mut() - .expect("fixture input is initially authorized") - .budgets - .requests += 1; - } - decision - } -} - -#[test] -fn tampered_evaluation_cannot_mint_a_source_proof_or_policy_receipt() { - let operation = common::operation(); - let context = common::context(&operation); - let service = AuthorizationService::new( - common::StaticAuthorizationPort::authorized(), - TamperingEvaluator::new(), - ); - - let problem = service - .admit(&context, &operation, UtcMicros(10)) - .unwrap_err(); - - assert_eq!(problem.kind(), ApplicationProblemKind::Unavailable); -} diff --git a/crates/tracedecay-contracts/tests/contracts_suite/callable_code_queries.rs b/crates/tracedecay-contracts/tests/contracts_suite/callable_code_queries.rs index c6c8b507ac..8279456448 100644 --- a/crates/tracedecay-contracts/tests/contracts_suite/callable_code_queries.rs +++ b/crates/tracedecay-contracts/tests/contracts_suite/callable_code_queries.rs @@ -1,5 +1,6 @@ use crate::common; +use std::collections::BTreeSet; use std::future::Future; use std::task::{Context, Poll, Waker}; @@ -16,27 +17,23 @@ use tracedecay_contracts::surface_contracts::{ }; use tracedecay_contracts::{ ApplicationOperation, ApplicationOutcome, ApplicationProblem, ApplicationProblemKind, - AuthorityReceipt, AuthorizationService, CALLABLE_CODE_OPERATION_COUNT, - CallableCodeAuthorizationAdmission, CallableCodeAuthorizationFuture, - CallableCodeAuthorizationPort, CallableCodeOperationKind, CallableCodeQueryFuture, - CallableCodeQueryPort, CallableCodeQueryService, CodeHierarchyRequest, CodeImpactRequest, - CodeImplementationsRequest, CodeQueryPage, CodeQueryScope, CodeRelationRequest, - CodeSignatureRequest, CodeSymbolSearchRequest, CoverageCompleteness, ExactOccurrenceRecord, - ExactOccurrenceRequest, LexicalOccurrenceRecord, ModuleApiRequest, OpaqueCursor, PageCursor, - PageRequest, PhraseSearchRequest, QualifiedNameRequest, RequestContext, ResultProjection, - RetrievalOrder, RetrievalPortContext, RetrievalPortOutcome, RetrievalRequestMeta, - SourceMetadataRecord, SourceMetadataRequest, callable_code_catalog_contribution, - callable_code_handler_descriptors, callable_code_operations, + AuthorityReceipt, CallableCodeAuthorizationFuture, CallableCodeAuthorizationPort, + CallableCodeOperationKind, CallableCodeQueryFuture, CallableCodeQueryPort, + CallableCodeQueryService, CodeHierarchyRequest, CodeImpactRequest, CodeImplementationsRequest, + CodeQueryPage, CodeQueryScope, CodeRelationRequest, CodeSignatureRequest, + CodeSymbolSearchRequest, CoverageCompleteness, ExactOccurrenceRecord, ExactOccurrenceRequest, + LexicalOccurrenceRecord, ModuleApiRequest, OpaqueCursor, PageCursor, PageRequest, + PhraseSearchRequest, QualifiedNameRequest, RequestContext, ResultProjection, RetrievalOrder, + RetrievalPortContext, RetrievalPortOutcome, RetrievalRequestMeta, SourceMetadataRecord, + SourceMetadataRequest, callable_code_catalog_contribution, callable_code_handler_descriptors, + callable_code_operations, }; use tracedecay_domain::{ CodeGenerationId, EphemeralSanitizedQueryViewV1, FactId, PublicRetrieverStatus, QueryFallbackSubpayload, QueryNormalizationRevision, RetrieverKind, SanitizerRevision, TemporalModeV1, UtcMicros, }; -use tracedecay_policy::authorization::SourceAuthorizationEvaluatorV1; -use tracedecay_tool_catalog::{ - AuthorityRequirement, BindingStatus, BindingSurface, LifecycleClass, -}; +use tracedecay_tool_catalog::{AuthorityRequirement, BindingSurface, LifecycleClass}; fn meta() -> RetrievalRequestMeta { RetrievalRequestMeta::current( @@ -243,28 +240,18 @@ impl CallableCodeAuthorizationPort for RoutedAuthorization { context: &'a RequestContext, _operation: &'a ApplicationOperation, _observed_at: UtcMicros, - ) -> CallableCodeAuthorizationFuture< - 'a, - Result, - > { - Box::pin(async move { - Ok(CallableCodeAuthorizationAdmission::Routed( - common::authority(context), - )) - }) + ) -> CallableCodeAuthorizationFuture<'a, Result> { + Box::pin(async move { Ok(common::authority(context)) }) } fn recheck_publication<'a>( &'a self, context: &'a RequestContext, _operation: &'a ApplicationOperation, - admission: &'a CallableCodeAuthorizationAdmission, + admission: &'a AuthorityReceipt, observed_at: UtcMicros, ) -> CallableCodeAuthorizationFuture<'a, Result> { Box::pin(async move { - let CallableCodeAuthorizationAdmission::Routed(admission) = admission else { - panic!("routed authorization admission remains opaque"); - }; let mut current = common::authority(context); assert_eq!(admission.policy, current.policy); current.revalidated_at = observed_at; @@ -285,14 +272,8 @@ fn execute_exact_in_scope( ) -> tracedecay_contracts::ApplicationResult> { let operations = callable_code_operations().unwrap(); let context = common::context(operations.get(CallableCodeOperationKind::ExactOccurrence)); - let service = CallableCodeQueryService::new( - ExactOnlyPort { scenario }, - AuthorizationService::new( - common::StaticAuthorizationPort::authorized(), - SourceAuthorizationEvaluatorV1::default(), - ), - operations, - ); + let service = + CallableCodeQueryService::new(ExactOnlyPort { scenario }, RoutedAuthorization, operations); block_on(service.exact_occurrence( &context, ExactOccurrenceRequest::new("ApplicationOperation", None, scope, meta()).unwrap(), @@ -639,37 +620,17 @@ fn callable_code_catalog_exposes_only_production_owned_transport_bindings() { let operations = callable_code_operations().unwrap(); assert_eq!( - CallableCodeOperationKind::ALL.len(), - CALLABLE_CODE_OPERATION_COUNT + descriptors + .iter() + .map(|descriptor| descriptor.operation().capability_id()) + .collect::>(), + contribution + .capabilities() + .iter() + .map(|capability| capability.capability_id()) + .collect::>(), + "advertised callable capabilities and their handlers are the same set" ); - let canonical_equivalents = [ - CallableCodeOperationKind::SymbolSearch, - CallableCodeOperationKind::QualifiedName, - CallableCodeOperationKind::SignatureSearch, - CallableCodeOperationKind::Implementations, - CallableCodeOperationKind::TypeHierarchy, - CallableCodeOperationKind::Callers, - CallableCodeOperationKind::Impact, - CallableCodeOperationKind::ModuleApi, - CallableCodeOperationKind::SourceMetadata, - ]; - let callable_catalog_count = CALLABLE_CODE_OPERATION_COUNT - canonical_equivalents.len(); - assert_eq!(contribution.capabilities().len(), callable_catalog_count); - assert_eq!(descriptors.len(), callable_catalog_count); - assert_eq!(operations.iter().count(), CALLABLE_CODE_OPERATION_COUNT); - for kind in canonical_equivalents { - let capability_id = format!( - "capability.application.code-query.{}", - kind.as_str().replace('_', "-") - ); - assert!( - contribution - .capabilities() - .iter() - .all(|capability| capability.capability_id().as_str() != capability_id), - "{kind:?} is owned by its canonical application surface" - ); - } let reachable = [ ("exact_occurrence", "code_exact_occurrence"), ("phrase_search", "code_phrase_search"), @@ -680,23 +641,25 @@ fn callable_code_catalog_exposes_only_production_owned_transport_bindings() { ("type_definition", "code_type_definition"), ("references", "code_references"), ]; - let expected_lsp_bindings = 3; - assert_eq!( - contribution.bindings().len(), - reachable.len() * 3 + expected_lsp_bindings - ); + for (operation, _) in reachable { + let capability_id = format!( + "capability.application.code-query.{}", + operation.replace('_', "-") + ); + assert!( + contribution + .capabilities() + .iter() + .any(|capability| capability.capability_id().as_str() == capability_id), + "{capability_id} is advertised" + ); + } for capability in contribution.capabilities() { assert_eq!( capability.authority(), AuthorityRequirement::CapabilityGrantWithRevalidation ); assert_eq!(capability.lifecycle(), LifecycleClass::Resumable); - let pagination = capability - .pagination() - .expect("direct callable code query is resumable"); - assert_eq!(pagination.default_page_size(), 10); - assert_eq!(pagination.maximum_page_size(), 1_000); - assert_eq!(pagination.cursor_ttl_millis(), 15 * 60 * 1_000); let kind = CallableCodeOperationKind::ALL .into_iter() .find(|kind| { @@ -707,6 +670,12 @@ fn callable_code_catalog_exposes_only_production_owned_transport_bindings() { ) }) .expect("capability maps to one callable-code operation"); + let pagination = capability + .pagination() + .expect("direct callable code query is resumable"); + assert_eq!(pagination.default_page_size(), 10, "{kind:?}"); + assert_eq!(pagination.maximum_page_size(), 1_000); + assert_eq!(pagination.cursor_ttl_millis(), 15 * 60 * 1_000); let Some((_, surface_operation)) = reachable .iter() .find(|(operation, _)| *operation == kind.as_str()) @@ -748,12 +717,16 @@ fn callable_code_catalog_exposes_only_production_owned_transport_bindings() { binding.binding_id().as_str(), format!("binding.{surface_name}.{surface_operation}.v1") ); - assert_eq!(binding.operation().as_str(), *surface_operation); - assert_eq!(binding.status(), &BindingStatus::Current); + let expected_operation = match (kind, surface) { + (CallableCodeOperationKind::Callees, BindingSurface::Cli | BindingSurface::Mcp) => { + "callees" + } + _ => *surface_operation, + }; + assert_eq!(binding.operation().as_str(), expected_operation); assert!(binding.protocol_revisions().contains(1)); assert!(!binding.protocol_revisions().contains(2)); assert!(binding.required_features().is_empty()); - assert!(!binding.is_alias()); assert!(capability.binding_ids().contains(binding.binding_id())); } } diff --git a/crates/tracedecay-contracts/tests/contracts_suite/catalog_contributions.rs b/crates/tracedecay-contracts/tests/contracts_suite/catalog_contributions.rs index c1af574a6e..950f51e5be 100644 --- a/crates/tracedecay-contracts/tests/contracts_suite/catalog_contributions.rs +++ b/crates/tracedecay-contracts/tests/contracts_suite/catalog_contributions.rs @@ -1,3 +1,4 @@ +use tracedecay_contracts::catalog_composition::build_application_catalog_snapshot; use tracedecay_contracts::feedback::{ CI_FAILURE_LOCALIZE_CAPABILITY_ID_V1, GITHUB_REVIEW_INGEST_CAPABILITY_ID_V1, }; @@ -120,32 +121,6 @@ fn application_contribution_set_uses_registered_feedback_handlers() { ); } -#[test] -fn application_composition_excludes_planner_and_store_owned_surfaces() { - // Cargo.toml already keeps this crate free of store/transport deps; this - // composition check proves the public catalog API likewise exposes no - // planner/model-runtime ownership. - let contributions = application_catalog_contributions().unwrap(); - assert!(!contributions.is_empty()); - for capability in contributions - .iter() - .flat_map(|contribution| contribution.capabilities()) - { - let capability_id = capability.capability_id().as_str(); - assert!( - !capability_id.contains("planner") - && !capability_id.contains("model-runtime") - && !capability_id.contains("universal-retrieval"), - "application catalog must not own {capability_id}" - ); - let use_case = capability.use_case_id().as_str(); - assert!( - !use_case.contains("planner") && !use_case.contains("dispatcher"), - "application use cases must not own {use_case}" - ); - } -} - #[test] fn verified_graph_mcp_reads_have_application_primitive_admission_identity() { let contribution = primitive_read_contribution().unwrap(); @@ -153,7 +128,6 @@ fn verified_graph_mcp_reads_have_application_primitive_admission_identity() { for operation_name in [ "context", "node", - "callees", "impact", "similar", "rename_preview", @@ -186,8 +160,6 @@ fn verified_graph_mcp_reads_have_application_primitive_admission_identity() { #[test] fn similar_and_redundancy_use_the_current_protocol_revision_only() { - use tracedecay_tool_catalog::BindingStatus; - let contribution = primitive_read_contribution().unwrap(); for operation in ["similar", "redundancy"] { let mcp_bindings: Vec<_> = contribution @@ -204,8 +176,6 @@ fn similar_and_redundancy_use_the_current_protocol_revision_only() { "{operation} must keep one MCP (surface, operation) binding" ); let binding = mcp_bindings[0]; - assert!(matches!(binding.status(), BindingStatus::Current)); - assert_eq!(binding.alias_of(), None); assert!( binding.protocol_revisions().contains(1), "{operation} must accept the current protocol revision" @@ -221,6 +191,26 @@ fn similar_and_redundancy_use_the_current_protocol_revision_only() { #[test] fn application_catalog_snapshot_admits_one_similar_redundancy_binding() { - tracedecay_contracts::catalog_composition::build_application_catalog_snapshot() + let snapshot = build_application_catalog_snapshot() .expect("catalog construction must succeed with one binding per surface-operation"); + for operation in ["similar", "redundancy"] { + let capability_id = primitive_read_operation(operation) + .unwrap() + .unwrap_or_else(|| panic!("{operation} primitive operation")) + .capability_id() + .clone(); + let capability = snapshot + .capability(&capability_id) + .unwrap_or_else(|| panic!("{operation} is in the composed snapshot")); + let mcp_bindings: Vec<_> = capability + .binding_ids() + .iter() + .filter_map(|binding_id| snapshot.binding(binding_id)) + .filter(|binding| binding.surface() == BindingSurface::Mcp) + .collect(); + assert_eq!(mcp_bindings.len(), 1, "{operation}"); + assert_eq!(mcp_bindings[0].operation().as_str(), operation); + assert_eq!(mcp_bindings[0].protocol_revisions().minimum(), 1); + assert_eq!(mcp_bindings[0].protocol_revisions().maximum(), 1); + } } diff --git a/crates/tracedecay-contracts/tests/contracts_suite/common/mod.rs b/crates/tracedecay-contracts/tests/contracts_suite/common/mod.rs index af2fa312d3..0c78659b0a 100644 --- a/crates/tracedecay-contracts/tests/contracts_suite/common/mod.rs +++ b/crates/tracedecay-contracts/tests/contracts_suite/common/mod.rs @@ -2,27 +2,24 @@ mod work_product_attempt_support; -use std::cell::RefCell; -use std::collections::{BTreeMap, BTreeSet, VecDeque}; +use std::collections::{BTreeMap, BTreeSet}; use std::sync::{Arc, Mutex}; use tracedecay_contracts::{ - ApplicationOperation, AuthorityReceipt, AuthorizationPort, AuthorizationPortOutcome, - AuthorizationRequest, AuthorizedWorkProductScopeV1, CancellationContext, + ApplicationOperation, AuthorityReceipt, AuthorizedWorkProductScopeV1, CancellationContext, CapabilityGrantSnapshot, Deadline, DisclosureClass, EvidenceCoverage, EvidenceDomain, PageState, PolicyDecisionRef, RequestContext, RequestId, ResolvedScope, ResultContractRef, - RetrievalEvidence, SourceAuthorizationSnapshot, StartWorkAttemptCommand, TemporalState, - VerifiedWorkGraphVersionV1, WorkAttemptAdmissionKind, WorkAttemptCapacityV1, - WorkAttemptCapacityVerdictV1, WorkAttemptEvidenceRecordV1, WorkAttemptInsertOutcome, - WorkAttemptListPageV1, WorkAttemptStorageError, WorkAttemptStoragePort, - WorkGraphReadPortErrorV1, WorkGraphReadPortV1, WorkGraphReadRequestV1, WorkGraphReadV1, - WorkGraphVersionEntryV1, WorkProductAttemptAdmissionErrorV1, - WorkProductAttemptAdmissionOutcomeV1, WorkProductAttemptAdmissionPortV1, - WorkProductAttemptAdmissionV1, WorkProductBindingV1, WorkProductEventCommitV1, + RetrievalEvidence, StartWorkAttemptCommand, TemporalState, VerifiedWorkGraphVersionV1, + WorkAttemptAdmissionKind, WorkAttemptCapacityV1, WorkAttemptCapacityVerdictV1, + WorkAttemptEvidenceRecordV1, WorkAttemptInsertOutcome, WorkAttemptListPageV1, + WorkAttemptStorageError, WorkAttemptStoragePort, WorkGraphReadPortErrorV1, WorkGraphReadPortV1, + WorkGraphReadRequestV1, WorkGraphReadV1, WorkGraphVersionEntryV1, + WorkProductAttemptAdmissionErrorV1, WorkProductAttemptAdmissionOutcomeV1, + WorkProductAttemptAdmissionPortV1, WorkProductAttemptAdmissionV1, + WorkProductAuthorizedRelationScopeV1, WorkProductBindingV1, WorkProductEventCommitV1, WorkProductOwnerAuthorizationErrorV1, WorkProductOwnerAuthorizationPortV1, WorkProductPortContextV1, WorkProductRevisionPinsV1, WorkProductSelectionScopeV1, - WorkRelationScopeV1, WorkSynthesisAdmissionRecordV1, WorkSynthesisAdmissionStoragePort, - WorkSynthesisInsertOutcome, + WorkSynthesisAdmissionRecordV1, WorkSynthesisAdmissionStoragePort, WorkSynthesisInsertOutcome, }; use tracedecay_domain::configuration::TopologyConcurrencyPolicyV1; use tracedecay_domain::{ @@ -38,9 +35,6 @@ use tracedecay_domain::{ WorkRecoveryStateV1, WorkRouteDecisionV1, WorkRuntimeProjectionCoverageV1, WorkRuntimeProjectionV1, WorkScoreKindV1, WorkShapeAssessmentV1, WorkSizingV1, WorktreeId, }; -use tracedecay_policy::authorization::{ - SourceAuthorizationInputV1, SourceAuthorizationTruthTableV1, -}; use tracedecay_tool_catalog::{CapabilityId, SchemaId, SortContractId, UseCaseId}; use work_product_attempt_support::{ @@ -53,8 +47,6 @@ pub const SHA256_A: &str = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; pub const SHA256_B: &str = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; -const SOURCE_AUTHORIZATION_TRUTH_TABLES: &str = - include_str!("../../../../tracedecay-policy/tests/fixtures/source_authorization/core.json"); pub use tracedecay_domain::test_fixtures::id; @@ -172,77 +164,6 @@ pub fn authority(context: &RequestContext) -> AuthorityReceipt { .unwrap() } -pub fn source_authorization_input(name: &str) -> SourceAuthorizationInputV1 { - serde_json::from_str::>(SOURCE_AUTHORIZATION_TRUTH_TABLES) - .expect("checked-in source authorization truth tables deserialize") - .into_iter() - .find(|row| row.name == name) - .unwrap_or_else(|| panic!("source authorization fixture {name} exists")) - .input -} - -pub fn authorized_source_input() -> SourceAuthorizationInputV1 { - source_authorization_input("project_authorized_live") -} - -pub fn source_snapshot(input: SourceAuthorizationInputV1) -> SourceAuthorizationSnapshot { - SourceAuthorizationSnapshot::new(input, true) -} - -pub struct StaticAuthorizationPort { - outcome: AuthorizationPortOutcome, -} - -impl StaticAuthorizationPort { - pub fn authorized() -> Self { - Self::new(AuthorizationPortOutcome::Snapshot(Box::new( - source_snapshot(authorized_source_input()), - ))) - } - - pub fn new(outcome: AuthorizationPortOutcome) -> Self { - Self { outcome } - } -} - -impl AuthorizationPort for StaticAuthorizationPort { - fn source_authorization_snapshot( - &self, - _request: &AuthorizationRequest<'_>, - ) -> AuthorizationPortOutcome { - self.outcome.clone() - } -} - -pub struct SequencedAuthorizationPort { - outcomes: RefCell>, -} - -impl SequencedAuthorizationPort { - pub fn snapshots(snapshots: impl IntoIterator) -> Self { - Self { - outcomes: RefCell::new( - snapshots - .into_iter() - .map(|snapshot| AuthorizationPortOutcome::Snapshot(Box::new(snapshot))) - .collect(), - ), - } - } -} - -impl AuthorizationPort for SequencedAuthorizationPort { - fn source_authorization_snapshot( - &self, - _request: &AuthorizationRequest<'_>, - ) -> AuthorizationPortOutcome { - self.outcomes - .borrow_mut() - .pop_front() - .expect("authorization snapshot sequence is not exhausted") - } -} - pub fn evidence(payload: T) -> RetrievalEvidence { RetrievalEvidence { payload: Some(payload), @@ -267,10 +188,12 @@ pub fn evidence(payload: T) -> RetrievalEvidence { /// Canonical repository relation selected by Work-product attempt admission. pub fn work_product_selection(context: &RequestContext) -> WorkProductSelectionScopeV1 { - WorkProductSelectionScopeV1::relations(BTreeSet::from([WorkRelationScopeV1::Repository { - project_id: context.scope().project_id.clone(), - repository_id: context.scope().repository_id.clone(), - }])) + WorkProductSelectionScopeV1::relations(BTreeSet::from([ + WorkProductAuthorizedRelationScopeV1::Repository { + project_id: context.scope().project_id.clone(), + repository_id: context.scope().repository_id.clone(), + }, + ])) .expect("request scope produces a canonical Work product selection") } @@ -518,10 +441,10 @@ impl WorkProductOwnerAuthorizationPortV1 for WorkProductAttemptStore { WorkProductSelectionScopeV1::ProfileOwnedNoGit => true, WorkProductSelectionScopeV1::Relations { relation_scopes } => { relation_scopes.iter().all(|relation| match relation { - WorkRelationScopeV1::Project { project_id } => { + WorkProductAuthorizedRelationScopeV1::Project { project_id } => { project_id == &context.scope().project_id } - WorkRelationScopeV1::Repository { + WorkProductAuthorizedRelationScopeV1::Repository { project_id, repository_id, } => { diff --git a/crates/tracedecay-contracts/tests/contracts_suite/feedback_advisory_cycle.rs b/crates/tracedecay-contracts/tests/contracts_suite/feedback_advisory_cycle.rs index 53073e00bb..f5197c9036 100644 --- a/crates/tracedecay-contracts/tests/contracts_suite/feedback_advisory_cycle.rs +++ b/crates/tracedecay-contracts/tests/contracts_suite/feedback_advisory_cycle.rs @@ -282,7 +282,6 @@ fn feedback_sources_share_one_cycle_result_and_canonical_anchors() { let proximity = ProximityContributionV1 { contribution_id: ProximityContributionIdV1::new("proximity-contribution.1").unwrap(), - warning_id: ProximityWarningIdV1::new("proximity-warning.1").unwrap(), warning_class: ProximityWarningClassV1::SameSymbol, source_observation_ids: vec![ ProximityObservationIdV1::new("proximity-observation.1").unwrap(), diff --git a/crates/tracedecay-contracts/tests/contracts_suite/feedback_cycle.rs b/crates/tracedecay-contracts/tests/contracts_suite/feedback_cycle.rs index 799a839233..cb4b6c8d74 100644 --- a/crates/tracedecay-contracts/tests/contracts_suite/feedback_cycle.rs +++ b/crates/tracedecay-contracts/tests/contracts_suite/feedback_cycle.rs @@ -13,17 +13,18 @@ use tracedecay_contracts::feedback::{ FeedbackCycleExecutionRequest, FeedbackCycleExecutionResult, FeedbackCycleService, FeedbackDiagnosticsPort, FeedbackDiagnosticsRequest, FeedbackImpactPort, FeedbackImpactPortOutcome, FeedbackImpactRequest, FeedbackObservationPort, - FeedbackPublicationRecordState, FeedbackPublicationV1, FeedbackRuntimeStatePort, - FeedbackRuntimeStateV1, GenerationBoundFeedbackDiagnosticsAdapter, + FeedbackPublicationRecordState, FeedbackPublicationV1, FeedbackRouteAuthorizationPort, + FeedbackRuntimeStatePort, FeedbackRuntimeStateV1, GenerationBoundFeedbackDiagnosticsAdapter, }; use tracedecay_contracts::{ - AnalyzerAdmittedDiagnosticProviderV1, AuthorizationService, CancellationContext, - CurrentDiagnosticsRequest, Deadline, DiagnosticProviderDescriptor, DiagnosticProviderIdentity, - DiagnosticProviderIdentityParts, DiagnosticProviderPort, DiagnosticProviderResult, - DiagnosticProviderState, FreshnessState, GenerationDiagnosticHistoryPort, - GenerationDiagnosticHistoryRequest, ProviderCoverage, ProviderDocumentIdentity, - ProviderFreshness, ProviderOrigin, ProviderProvenance, ProviderSourceIdentity, RequestContext, - RevisionDigest, + AnalyzerAdmittedDiagnosticProviderV1, ApplicationOperation, ApplicationProblem, + AuthorityReceipt, CancellationContext, CurrentDiagnosticsRequest, Deadline, + DiagnosticProviderDescriptor, DiagnosticProviderIdentity, DiagnosticProviderIdentityParts, + DiagnosticProviderPort, DiagnosticProviderResult, DiagnosticProviderState, FreshnessState, + GenerationDiagnosticHistoryPort, GenerationDiagnosticHistoryRequest, ProviderCoverage, + ProviderDocumentIdentity, ProviderFreshness, ProviderOrigin, ProviderProvenance, + ProviderSourceIdentity, RequestAdmission, RequestContext, RetryDirective, RevisionDigest, + SafeDiagnostic, }; use tracedecay_domain::configuration::{ AnalyzerExecutableId, AnalyzerExecutableReferenceV1, AnalyzerLanguageId, @@ -51,7 +52,6 @@ use tracedecay_policy::analyzer::{ AnalyzerAdmissionEvaluatorV1, AnalyzerAdmissionInputV1, AnalyzerAvailabilityV1, AnalyzerCandidateV1, AnalyzerExecutionLocationV1, }; -use tracedecay_policy::authorization::SourceAuthorizationEvaluatorV1; use tracedecay_tool_catalog::CapabilityId; const GENERATION: &str = "generation.v1.fixture.00000001"; @@ -294,9 +294,9 @@ impl FeedbackCycleDedupePort for SerializedRaceDedupeFixture { } #[derive(Clone)] -struct ConcurrentRuntimeFixture(FeedbackRuntimeStateV1); +struct RuntimeStateFixture(FeedbackRuntimeStateV1); -impl FeedbackRuntimeStatePort for ConcurrentRuntimeFixture { +impl FeedbackRuntimeStatePort for RuntimeStateFixture { fn resolve<'a>( &'a self, _context: &'a RequestContext, @@ -308,6 +308,29 @@ impl FeedbackRuntimeStatePort for ConcurrentRuntimeFixture { } } +/// Runtime authority answering each resolution from a fixed sequence. +struct SequencedRuntimeFixture { + states: RefCell>>, + calls: Rc>, +} + +impl FeedbackRuntimeStatePort for SequencedRuntimeFixture { + fn resolve<'a>( + &'a self, + _context: &'a RequestContext, + _input: &'a FeedbackEvaluationInputV1, + ) -> tracedecay_contracts::feedback::FeedbackPortFuture<'a, Option> + { + self.calls.set(self.calls.get() + 1); + let runtime = self + .states + .borrow_mut() + .pop_front() + .expect("runtime-state sequence is not exhausted"); + Box::pin(async move { runtime }) + } +} + #[derive(Clone)] struct ConcurrentDiagnosticsFixture { results: Vec>>, @@ -377,6 +400,63 @@ impl FeedbackObservationPort for ObservationFixture { } } +/// Route-owned authorization that, like the production route owner, applies +/// the request's cancellation, deadline, and grant at admission and at every +/// publication recheck. A revoked route admits, then reports its source +/// unavailable before publication. +#[derive(Clone, Copy, Default)] +struct RouteAuthorizationFixture { + revoked_before_publication: bool, +} + +impl FeedbackRouteAuthorizationPort for RouteAuthorizationFixture { + fn admit( + &self, + context: &RequestContext, + operation: &ApplicationOperation, + observed_at: UtcMicros, + ) -> Result { + match context.admission_at(observed_at) { + RequestAdmission::Cancelled => { + return Err(ApplicationProblem::cancelled_before_admission()); + } + RequestAdmission::TimedOut => { + return Err(ApplicationProblem::timed_out_before_admission()); + } + RequestAdmission::Admitted => {} + } + if !context.allows(operation.capability_id(), operation.use_case_id()) { + return Err(ApplicationProblem::not_found_or_not_authorized( + RetryDirective::Never, + )); + } + let mut receipt = common::authority(context); + receipt.revalidated_at = observed_at; + Ok(receipt) + } + + fn recheck_publication( + &self, + context: &RequestContext, + operation: &ApplicationOperation, + admission: &AuthorityReceipt, + observed_at: UtcMicros, + ) -> Result { + if self.revoked_before_publication { + return Err(ApplicationProblem::unavailable( + SafeDiagnostic::new( + "fixture.authorization.source-unavailable", + "The feedback source became unavailable after admission.", + ) + .unwrap(), + )); + } + let current = self.admit(context, operation, observed_at)?; + assert_eq!(admission.policy, current.policy); + Ok(current) + } +} + fn scope() -> FeedbackScopeV1 { FeedbackScopeV1 { project_id: common::scope().project_id, @@ -996,25 +1076,17 @@ fn runtime_state(input: &FeedbackEvaluationInputV1) -> FeedbackRuntimeStateV1 { .unwrap() } -fn runtime_port( - input: &FeedbackEvaluationInputV1, -) -> impl Fn(&RequestContext, &FeedbackEvaluationInputV1) -> Option + use<> -{ - let state = runtime_state(input); - move |_context, _input| Some(state.clone()) +fn runtime_port(input: &FeedbackEvaluationInputV1) -> RuntimeStateFixture { + RuntimeStateFixture(runtime_state(input)) } fn sequenced_runtime( states: Vec>, calls: Rc>, -) -> impl Fn(&RequestContext, &FeedbackEvaluationInputV1) -> Option { - let states = Rc::new(RefCell::new(states.into_iter().collect::>())); - move |_context, _input| { - calls.set(calls.get() + 1); - states - .borrow_mut() - .pop_front() - .expect("runtime-state sequence is not exhausted") +) -> SequencedRuntimeFixture { + SequencedRuntimeFixture { + states: RefCell::new(states.into()), + calls, } } @@ -1040,7 +1112,7 @@ fn execute_concurrent_cycle( provider: DiagnosticProviderIdentity, dedupe: SerializedRaceDedupeFixture, ) -> FeedbackCycleExecutionResult { - let runtime = ConcurrentRuntimeFixture(runtime_state(&input)); + let runtime = RuntimeStateFixture(runtime_state(&input)); let diagnostics = ConcurrentDiagnosticsFixture { results: vec![complete_result(provider.clone(), Vec::new())], }; @@ -1053,10 +1125,7 @@ fn execute_concurrent_cycle( impact, dedupe, NoopObservationFixture, - AuthorizationService::new( - common::StaticAuthorizationPort::authorized(), - SourceAuthorizationEvaluatorV1::default(), - ), + RouteAuthorizationFixture::default(), operation, ); block_on(service.execute(&context, execution_request(input, provider))).unwrap() @@ -1083,10 +1152,7 @@ fn execute_before_provider_work( }, DedupeFixture(dedupe_state), ObservationFixture::default(), - AuthorizationService::new( - common::StaticAuthorizationPort::authorized(), - SourceAuthorizationEvaluatorV1::default(), - ), + RouteAuthorizationFixture::default(), common::operation(), ); let mut request = execution_request(input, provider); @@ -1118,10 +1184,7 @@ fn cycle_runs_diagnostics_impact_and_tests_once_with_anchored_new_findings() { }, DedupeFixture(FeedbackCycleDedupeState::Unique), observations.clone(), - AuthorizationService::new( - common::StaticAuthorizationPort::authorized(), - SourceAuthorizationEvaluatorV1::default(), - ), + RouteAuthorizationFixture::default(), common::operation(), ); @@ -1260,10 +1323,7 @@ fn authoritative_history_identity_drives_pre_existing_and_stale_classification() }, DedupeFixture(FeedbackCycleDedupeState::Unique), ObservationFixture::default(), - AuthorizationService::new( - common::StaticAuthorizationPort::authorized(), - SourceAuthorizationEvaluatorV1::default(), - ), + RouteAuthorizationFixture::default(), common::operation(), ); let result = block_on(service.execute( @@ -1297,10 +1357,7 @@ fn authoritative_history_identity_drives_pre_existing_and_stale_classification() }, DedupeFixture(FeedbackCycleDedupeState::Unique), ObservationFixture::default(), - AuthorizationService::new( - common::StaticAuthorizationPort::authorized(), - SourceAuthorizationEvaluatorV1::default(), - ), + RouteAuthorizationFixture::default(), common::operation(), ); let stale = block_on(stale_service.execute( @@ -1343,10 +1400,7 @@ fn dedupe_key_changes_when_authoritative_evidence_changes() { keys: keys.clone(), }, ObservationFixture::default(), - AuthorizationService::new( - common::StaticAuthorizationPort::authorized(), - SourceAuthorizationEvaluatorV1::default(), - ), + RouteAuthorizationFixture::default(), common::operation(), ); block_on(service.execute( @@ -1379,10 +1433,7 @@ fn unavailable_authoritative_baseline_cannot_produce_clean() { }, DedupeFixture(FeedbackCycleDedupeState::Unique), ObservationFixture::default(), - AuthorizationService::new( - common::StaticAuthorizationPort::authorized(), - SourceAuthorizationEvaluatorV1::default(), - ), + RouteAuthorizationFixture::default(), common::operation(), ); let result = block_on(service.execute( @@ -1413,9 +1464,7 @@ fn authoritative_no_prior_baseline_is_explicit_and_never_invented() { let mut no_prior_runtime = runtime_state(&input); no_prior_runtime.authoritative.baseline_horizon = None; let service = FeedbackCycleService::new( - move |_context: &RequestContext, _input: &FeedbackEvaluationInputV1| { - Some(no_prior_runtime.clone()) - }, + RuntimeStateFixture(no_prior_runtime), HistoryDiagnosticsFixture { calls: Rc::new(Cell::new(0)), history_calls: history_calls.clone(), @@ -1428,10 +1477,7 @@ fn authoritative_no_prior_baseline_is_explicit_and_never_invented() { }, DedupeFixture(FeedbackCycleDedupeState::Unique), ObservationFixture::default(), - AuthorizationService::new( - common::StaticAuthorizationPort::authorized(), - SourceAuthorizationEvaluatorV1::default(), - ), + RouteAuthorizationFixture::default(), common::operation(), ); @@ -1465,10 +1511,7 @@ fn complete_zero_diagnostics_and_impact_are_clean() { }, DedupeFixture(FeedbackCycleDedupeState::Unique), ObservationFixture::default(), - AuthorizationService::new( - common::StaticAuthorizationPort::authorized(), - SourceAuthorizationEvaluatorV1::default(), - ), + RouteAuthorizationFixture::default(), common::operation(), ); let result = block_on(service.execute( @@ -1512,10 +1555,7 @@ fn duplicate_noop_is_decided_after_authoritative_evidence_is_read() { }, DedupeFixture(FeedbackCycleDedupeState::Duplicate), observations.clone(), - AuthorizationService::new( - common::StaticAuthorizationPort::authorized(), - SourceAuthorizationEvaluatorV1::default(), - ), + RouteAuthorizationFixture::default(), common::operation(), ); @@ -1616,10 +1656,7 @@ fn duplicate_provider_diagnostics_collapse_to_one_finding() { }, DedupeFixture(FeedbackCycleDedupeState::Unique), ObservationFixture::default(), - AuthorizationService::new( - common::StaticAuthorizationPort::authorized(), - SourceAuthorizationEvaluatorV1::default(), - ), + RouteAuthorizationFixture::default(), common::operation(), ); @@ -1651,10 +1688,7 @@ fn mismatched_diagnostic_address_is_failed_not_current_truth() { }, DedupeFixture(FeedbackCycleDedupeState::Unique), ObservationFixture::default(), - AuthorizationService::new( - common::StaticAuthorizationPort::authorized(), - SourceAuthorizationEvaluatorV1::default(), - ), + RouteAuthorizationFixture::default(), common::operation(), ); @@ -1694,10 +1728,7 @@ fn bounded_preview_respects_its_byte_limit_for_unicode() { }, DedupeFixture(FeedbackCycleDedupeState::Unique), ObservationFixture::default(), - AuthorizationService::new( - common::StaticAuthorizationPort::authorized(), - SourceAuthorizationEvaluatorV1::default(), - ), + RouteAuthorizationFixture::default(), common::operation(), ); @@ -1748,10 +1779,7 @@ fn overlay_cycle_returns_session_only_truth_without_observations() { keys: dedupe_keys.clone(), }, observations.clone(), - AuthorizationService::new( - common::StaticAuthorizationPort::authorized(), - SourceAuthorizationEvaluatorV1::default(), - ), + RouteAuthorizationFixture::default(), common::operation(), ); @@ -1809,10 +1837,7 @@ fn overlay_provider_client_must_match_the_authenticated_owner_binding() { }, DedupeFixture(FeedbackCycleDedupeState::Unique), ObservationFixture::default(), - AuthorizationService::new( - common::StaticAuthorizationPort::authorized(), - SourceAuthorizationEvaluatorV1::default(), - ), + RouteAuthorizationFixture::default(), common::operation(), ); @@ -1866,10 +1891,7 @@ fn every_post_port_runtime_drift_suppresses_evidence_and_later_reads() { keys: dedupe_keys.clone(), }, observations.clone(), - AuthorizationService::new( - common::StaticAuthorizationPort::authorized(), - SourceAuthorizationEvaluatorV1::default(), - ), + RouteAuthorizationFixture::default(), common::operation(), ); @@ -1972,10 +1994,7 @@ fn partial_and_unavailable_impact_truth_never_becomes_clean() { }, DedupeFixture(FeedbackCycleDedupeState::Unique), ObservationFixture::default(), - AuthorizationService::new( - common::StaticAuthorizationPort::authorized(), - SourceAuthorizationEvaluatorV1::default(), - ), + RouteAuthorizationFixture::default(), common::operation(), ); @@ -2023,10 +2042,7 @@ fn partial_affected_test_coverage_never_becomes_clean() { }, DedupeFixture(FeedbackCycleDedupeState::Unique), ObservationFixture::default(), - AuthorizationService::new( - common::StaticAuthorizationPort::authorized(), - SourceAuthorizationEvaluatorV1::default(), - ), + RouteAuthorizationFixture::default(), common::operation(), ); let result = block_on(service.execute( @@ -2104,10 +2120,7 @@ fn every_terminal_reason_is_exact_and_one_shot() { }, DedupeFixture(FeedbackCycleDedupeState::Unavailable), ObservationFixture::default(), - AuthorizationService::new( - common::StaticAuthorizationPort::authorized(), - SourceAuthorizationEvaluatorV1::default(), - ), + RouteAuthorizationFixture::default(), common::operation(), ); let unavailable = block_on(unavailable_service.execute( @@ -2159,15 +2172,9 @@ fn post_read_authorization_is_rechecked_before_findings_publish() { }, DedupeFixture(FeedbackCycleDedupeState::Unique), ObservationFixture::default(), - AuthorizationService::new( - common::SequencedAuthorizationPort::snapshots([ - common::source_snapshot(common::authorized_source_input()), - common::source_snapshot(common::source_authorization_input( - "temporarily_unavailable_is_not_deletion", - )), - ]), - SourceAuthorizationEvaluatorV1::default(), - ), + RouteAuthorizationFixture { + revoked_before_publication: true, + }, common::operation(), ); @@ -2211,15 +2218,9 @@ fn authorization_revocation_overrides_early_and_duplicate_terminal_outcomes() { }, DedupeFixture(FeedbackCycleDedupeState::Unique), early_observations.clone(), - AuthorizationService::new( - common::SequencedAuthorizationPort::snapshots([ - common::source_snapshot(common::authorized_source_input()), - common::source_snapshot(common::source_authorization_input( - "temporarily_unavailable_is_not_deletion", - )), - ]), - SourceAuthorizationEvaluatorV1::default(), - ), + RouteAuthorizationFixture { + revoked_before_publication: true, + }, operation.clone(), ); let mut early_request = execution_request(early_input, early_provider); @@ -2260,15 +2261,9 @@ fn authorization_revocation_overrides_early_and_duplicate_terminal_outcomes() { }, DedupeFixture(FeedbackCycleDedupeState::Duplicate), duplicate_observations.clone(), - AuthorizationService::new( - common::SequencedAuthorizationPort::snapshots([ - common::source_snapshot(common::authorized_source_input()), - common::source_snapshot(common::source_authorization_input( - "temporarily_unavailable_is_not_deletion", - )), - ]), - SourceAuthorizationEvaluatorV1::default(), - ), + RouteAuthorizationFixture { + revoked_before_publication: true, + }, operation, ); let duplicate = block_on(duplicate_service.execute( @@ -2315,10 +2310,7 @@ fn cancellation_suppresses_findings_from_other_completed_providers() { }, DedupeFixture(FeedbackCycleDedupeState::Unique), ObservationFixture::default(), - AuthorizationService::new( - common::StaticAuthorizationPort::authorized(), - SourceAuthorizationEvaluatorV1::default(), - ), + RouteAuthorizationFixture::default(), common::operation(), ); let mut request = execution_request(input, provider); diff --git a/crates/tracedecay-contracts/tests/contracts_suite/github_stack_signal_expand_catalog.rs b/crates/tracedecay-contracts/tests/contracts_suite/github_stack_signal_expand_catalog.rs index b7f09e313f..ec3475d228 100644 --- a/crates/tracedecay-contracts/tests/contracts_suite/github_stack_signal_expand_catalog.rs +++ b/crates/tracedecay-contracts/tests/contracts_suite/github_stack_signal_expand_catalog.rs @@ -1,3 +1,5 @@ +use std::collections::BTreeSet; + use schemars::schema_for; use tracedecay_contracts::git::{ GitHubStackSignalExpandSurfaceRequest, GitHubStackSignalExpandSurfaceResultV1, @@ -71,20 +73,24 @@ fn github_stack_signal_expand_is_schema_backed_and_publicly_mounted() { fn github_stack_signal_expand_result_schema_stays_bounded() { let schema = serde_json::to_value(schema_for!(GitHubStackSignalExpandSurfaceResultV1)) .expect("result schema JSON"); - let rendered = schema.to_string(); - for field in [ - "signal_id", - "watermark_id", - "stack_revision_digest", - "state_digest", - "observed_at", - ] { - assert!( - rendered.contains(&format!("\"{field}\"")), - "missing {field}" - ); - } - assert!(!rendered.contains("repository_path")); - assert!(!rendered.contains("pull_request_body")); - assert!(!rendered.contains("commit_message")); + let evidence_fields = schema["$defs"]["GitHubStackSignalEvidenceRefV1"]["properties"] + .as_object() + .expect("evidence reference properties") + .keys() + .map(String::as_str) + .collect::>(); + assert_eq!( + evidence_fields, + BTreeSet::from([ + "github_stack_digest", + "kind", + "native_source", + "observed_at", + "signal_id", + "stack_revision_digest", + "stack_revision_id", + "state_digest", + "watermark_id", + ]) + ); } diff --git a/crates/tracedecay-contracts/tests/contracts_suite/main.rs b/crates/tracedecay-contracts/tests/contracts_suite/main.rs index 270dca1fbe..2feb72216b 100644 --- a/crates/tracedecay-contracts/tests/contracts_suite/main.rs +++ b/crates/tracedecay-contracts/tests/contracts_suite/main.rs @@ -9,8 +9,6 @@ mod common; mod advisory_requests; -mod authorization_non_disclosure; -mod authorization_recheck; mod callable_code_queries; mod catalog_contributions; mod diagnostic_provider_identity; diff --git a/crates/tracedecay-contracts/tests/contracts_suite/observability_share_contract.rs b/crates/tracedecay-contracts/tests/contracts_suite/observability_share_contract.rs index 53ae226ebf..d36a795b43 100644 --- a/crates/tracedecay-contracts/tests/contracts_suite/observability_share_contract.rs +++ b/crates/tracedecay-contracts/tests/contracts_suite/observability_share_contract.rs @@ -1,3 +1,4 @@ +use serde_json::json; use tracedecay_contracts::{ AggregateCapabilityV1, AggregateShareCellV1, AggregateShareDimensionV1, AggregateShareExportRequestV1, AggregateShareMetricV1, AggregateSharePacketV1, @@ -39,20 +40,30 @@ fn aggregate_share_packet_is_identity_free_and_bounded() { }; packet.validate().expect("valid aggregate packet"); - let json = serde_json::to_value(packet).expect("serialize packet"); - let object = json.as_object().expect("packet object"); - for prohibited in [ - "scope_ref", - "trace_id", - "event_id", - "project_id", - "repository", - "session_id", - "task_id", - ] { - assert!(!object.contains_key(prohibited)); - assert!(!json.to_string().contains(prohibited)); - } + assert_eq!( + serde_json::to_value(packet).expect("serialize packet"), + json!({ + "schema_revision": 1, + "descriptor_revision": "aggregate-share.v1", + "horizon": { "since_micros": 10, "until_micros": 20 }, + "generated_at_micros": 20, + "cells": [{ + "metric": "retrieval_queries", + "unit": "events", + "dimensions": [{ "kind": "capability", "value": "retrieval" }], + "eligible": 100, + "observed": 100, + "completed": 96, + "censored": 2, + "unknown": 2, + "value": 100.0, + "coverage": "partial", + "contribution_windows": 100 + }], + "suppressed_cell_count": 0, + "capped_cell_count": 0 + }) + ); } #[test] diff --git a/crates/tracedecay-contracts/tests/contracts_suite/primitive_sdk_catalog.rs b/crates/tracedecay-contracts/tests/contracts_suite/primitive_sdk_catalog.rs index 515cd1a676..2956f2fe68 100644 --- a/crates/tracedecay-contracts/tests/contracts_suite/primitive_sdk_catalog.rs +++ b/crates/tracedecay-contracts/tests/contracts_suite/primitive_sdk_catalog.rs @@ -1,45 +1,46 @@ -use std::collections::BTreeSet; - +use serde_json::json; use tracedecay_contracts::sdk_executable_binding_registry; use tracedecay_tool_catalog::{OperationId, SdkTransportBindingV1}; -const TYPED_PRIMITIVE_OPERATIONS: [&str; 9] = [ - "callees", - "context", - "impact", - "node", - "port_order", - "port_status", - "rename_preview", - "similar", - "todos", -]; - #[test] fn established_primitive_tools_are_typed_sdk_operations() { let registry = sdk_executable_binding_registry().expect("canonical SDK registry"); - let expected = TYPED_PRIMITIVE_OPERATIONS - .iter() - .map(|operation| format!("operation.application.{operation}")) - .collect::>(); - for operation_id in expected { + for (operation, required) in [ + ("context", json!(["task"])), + ("impact", json!(["node_id"])), + ("node", json!(["node_id"])), + ("port_order", json!(["source_dir"])), + ("port_status", json!(["source_dir", "target_dir"])), + ("rename_preview", json!(["node_id"])), + ( + "similar", + json!([ + "project_id", + "repository_id", + "target", + "match_classes", + "result_limit", + "work_limit" + ]), + ), + ("todos", json!([])), + ] { + let operation_id = format!("operation.application.{operation}"); let binding = registry .get(&OperationId::new(operation_id.clone()).expect("operation ID")) .and_then(|availability| availability.binding()) .unwrap_or_else(|| panic!("{operation_id} must be executable")); - assert!(matches!( - binding.transport(), - SdkTransportBindingV1::McpTool { tool_name } - if tool_name == &format!( - "tracedecay_{}", - operation_id.trim_start_matches("operation.application.") - ) - )); - assert_eq!(binding.request_schema().body()["type"], "object"); - assert_ne!( - binding.result_schema().body(), - &serde_json::Value::Bool(true) + assert!( + matches!( + binding.transport(), + SdkTransportBindingV1::McpTool { tool_name } + if tool_name == &format!("tracedecay_{operation}") + ), + "{operation}" ); + let request = binding.request_schema().body(); + let observed_required = request.get("required").cloned().unwrap_or(json!([])); + assert_eq!(observed_required, required, "{operation}"); } } diff --git a/crates/tracedecay-contracts/tests/contracts_suite/source_edit_sdk_catalog.rs b/crates/tracedecay-contracts/tests/contracts_suite/source_edit_sdk_catalog.rs index 5584c39c01..597a55aada 100644 --- a/crates/tracedecay-contracts/tests/contracts_suite/source_edit_sdk_catalog.rs +++ b/crates/tracedecay-contracts/tests/contracts_suite/source_edit_sdk_catalog.rs @@ -1,7 +1,7 @@ use std::collections::BTreeSet; use tracedecay_contracts::{sdk_executable_binding_registry, source_edit_catalog_contribution}; -use tracedecay_tool_catalog::{BindingStatus, BindingSurface, OperationId, SdkTransportBindingV1}; +use tracedecay_tool_catalog::{BindingSurface, OperationId, SdkTransportBindingV1}; #[test] fn sdk_registry_projects_source_edit_with_its_exact_mcp_schemas() { @@ -9,11 +9,11 @@ fn sdk_registry_projects_source_edit_with_its_exact_mcp_schemas() { let registry = sdk_executable_binding_registry().expect("SDK registry"); let mut projected_capabilities = BTreeSet::new(); - for surface in contribution.bindings().iter().filter(|binding| { - binding.surface() == BindingSurface::Mcp - && matches!(binding.status(), BindingStatus::Current) - && !binding.is_alias() - }) { + for surface in contribution + .bindings() + .iter() + .filter(|binding| binding.surface() == BindingSurface::Mcp) + { let operation_id = OperationId::new(format!( "operation.application.{}", surface.operation().as_str() diff --git a/crates/tracedecay-contracts/tests/contracts_suite/surface_binding_parity.rs b/crates/tracedecay-contracts/tests/contracts_suite/surface_binding_parity.rs index a419b2591c..0a544ddb5f 100644 --- a/crates/tracedecay-contracts/tests/contracts_suite/surface_binding_parity.rs +++ b/crates/tracedecay-contracts/tests/contracts_suite/surface_binding_parity.rs @@ -128,7 +128,6 @@ fn assert_surface_contract_parity( assert_eq!(binding.operation(), operation); assert!(capability.binding_ids().contains(binding.binding_id())); assert!(binding.required_features().is_empty()); - assert!(!binding.is_alias()); } } } diff --git a/crates/tracedecay-contracts/tests/contracts_suite/work_product_application.rs b/crates/tracedecay-contracts/tests/contracts_suite/work_product_application.rs index 674b40c01b..c5e00eac7d 100644 --- a/crates/tracedecay-contracts/tests/contracts_suite/work_product_application.rs +++ b/crates/tracedecay-contracts/tests/contracts_suite/work_product_application.rs @@ -14,12 +14,12 @@ use tracedecay_contracts::{ WorkGraphReadPortV1, WorkGraphReadRequestV1, WorkGraphReadV1, WorkGraphSelectionCoverageV1, WorkGraphTimelineV1, WorkGraphVersionEntryV1, WorkHistoryCoverageV1, WorkHistoryReadPortV1, WorkHistoryRequestV1, WorkHistoryServiceV1, WorkHistoryV1, WorkProductApplicationErrorV1, - WorkProductBindingV1, WorkProductEventCommitOutcomeV1, WorkProductEventCommitV1, - WorkProductEventDraftV1, WorkProductEventPortErrorV1, WorkProductEventPortV1, - WorkProductEvidenceServiceV1, WorkProductExpectedAuthorityV1, WorkProductMutationIdentityV1, - WorkProductMutationServiceV1, WorkProductOwnerAuthorizationErrorV1, - WorkProductOwnerAuthorizationPortV1, WorkProductReadServiceV1, WorkProductRevisionPinsV1, - WorkProductSelectionScopeV1, WorkRelationScopeV1, + WorkProductAuthorizedRelationScopeV1, WorkProductBindingV1, WorkProductEventCommitOutcomeV1, + WorkProductEventCommitV1, WorkProductEventDraftV1, WorkProductEventPortErrorV1, + WorkProductEventPortV1, WorkProductEvidenceServiceV1, WorkProductExpectedAuthorityV1, + WorkProductMutationIdentityV1, WorkProductMutationServiceV1, + WorkProductOwnerAuthorizationErrorV1, WorkProductOwnerAuthorizationPortV1, + WorkProductReadServiceV1, WorkProductRevisionPinsV1, WorkProductSelectionScopeV1, }; use tracedecay_domain::{ ActorId, BrainId, CatalogGenerationId, ConfigurationRevisionId, InitiativeId, ManifestDigest, @@ -47,10 +47,12 @@ fn binding() -> WorkProductBindingV1 { } fn repository_selection() -> WorkProductSelectionScopeV1 { - WorkProductSelectionScopeV1::relations(BTreeSet::from([WorkRelationScopeV1::Repository { - project_id: id("project.work.fixture"), - repository_id: id("repository.work.fixture"), - }])) + WorkProductSelectionScopeV1::relations(BTreeSet::from([ + WorkProductAuthorizedRelationScopeV1::Repository { + project_id: id("project.work.fixture"), + repository_id: id("repository.work.fixture"), + }, + ])) .unwrap() } @@ -114,10 +116,10 @@ impl WorkProductOwnerAuthorizationPortV1 for RegisteredOwner { WorkProductSelectionScopeV1::ProfileOwnedNoGit => true, WorkProductSelectionScopeV1::Relations { relation_scopes } => { relation_scopes.iter().all(|relation| match relation { - WorkRelationScopeV1::Project { project_id } => { + WorkProductAuthorizedRelationScopeV1::Project { project_id } => { project_id == &context.scope().project_id } - WorkRelationScopeV1::Repository { + WorkProductAuthorizedRelationScopeV1::Repository { project_id, repository_id, } => { diff --git a/crates/tracedecay-contracts/tests/contracts_suite/work_topology_view.rs b/crates/tracedecay-contracts/tests/contracts_suite/work_topology_view.rs index c47efc2084..8069b128b3 100644 --- a/crates/tracedecay-contracts/tests/contracts_suite/work_topology_view.rs +++ b/crates/tracedecay-contracts/tests/contracts_suite/work_topology_view.rs @@ -16,7 +16,7 @@ use tracedecay_contracts::{ WorkAttemptService, WorkAttemptTopologyBindingV1, WorkAttemptTopologyStateV1, WorkIntelligenceServiceV1, WorkPlacementReadingV1, WorkPlacementService, WorkPlacementStorageError, WorkPlacementStoragePort, WorkProductAttemptServiceV1, - WorkProductSelectionScopeV1, WorkRelationScopeV1, WorkRoutingSnapshotErrorV1, + WorkProductAuthorizedRelationScopeV1, WorkProductSelectionScopeV1, WorkRoutingSnapshotErrorV1, WorkRoutingSnapshotPortV1, WorkRoutingSnapshotV1, WorkTopologyViewRequestV1, execution_topology_view, }; @@ -178,10 +178,12 @@ fn execution_snapshot(topology: tracedecay_domain::WorkTopologyPolicyV1) -> Work } fn selected_product_scope(context: &RequestContext) -> WorkProductSelectionScopeV1 { - WorkProductSelectionScopeV1::relations(BTreeSet::from([WorkRelationScopeV1::Repository { - project_id: context.scope().project_id.clone(), - repository_id: context.scope().repository_id.clone(), - }])) + WorkProductSelectionScopeV1::relations(BTreeSet::from([ + WorkProductAuthorizedRelationScopeV1::Repository { + project_id: context.scope().project_id.clone(), + repository_id: context.scope().repository_id.clone(), + }, + ])) .unwrap() } diff --git a/crates/tracedecay-daemon-control/src/service.rs b/crates/tracedecay-daemon-control/src/service.rs index bb3b28d1d2..8f6e7b878f 100644 --- a/crates/tracedecay-daemon-control/src/service.rs +++ b/crates/tracedecay-daemon-control/src/service.rs @@ -1072,25 +1072,7 @@ fn refresh_installed_service_with_state_and_runner( refreshed_spec.data_dir_override = windows_task::profile_root_from_task_xml(&unit); } if let Some(socket_path) = socket_path_from_unit_text(&unit) { - #[cfg(unix)] - { - let profile_root = refreshed_spec - .data_dir_override - .clone() - .map_or_else(tracedecay_data_dir, Ok)?; - let legacy_generated_socket = profile_root.join("daemon.sock"); - if socket_path != legacy_generated_socket - || tracedecay_daemon_protocol::unix_socket_path_within_limit(&socket_path) - { - refreshed_spec.socket_path = socket_path; - } else { - refreshed_spec.socket_path = default_socket_path_for_profile(&profile_root); - } - } - #[cfg(not(unix))] - { - refreshed_spec.socket_path = socket_path; - } + refreshed_spec.socket_path = socket_path; } let previous_state = match previous_state { Some(state) => state, diff --git a/crates/tracedecay-daemon-control/src/service/probe.rs b/crates/tracedecay-daemon-control/src/service/probe.rs index 307e5f11df..ae24fead9f 100644 --- a/crates/tracedecay-daemon-control/src/service/probe.rs +++ b/crates/tracedecay-daemon-control/src/service/probe.rs @@ -295,12 +295,8 @@ pub(super) fn daemon_readiness_probe( } }; let deadline = std::time::Instant::now() + timeout; - let identity = query_daemon_identity_stream( - stream, - connection.auth_token.as_deref(), - expected_version, - deadline, - ); + let identity = + query_daemon_identity_stream(stream, connection.auth_token(), expected_version, deadline); ( DaemonSocketState::Connectable, classify_daemon_protocol_identity(identity, expected_version), @@ -355,8 +351,7 @@ pub(super) fn daemon_readiness_probe( ); } }; - let identity = - query_daemon_identity_stream(stream, Some(&auth_token), expected_version, deadline); + let identity = query_daemon_identity_stream(stream, &auth_token, expected_version, deadline); ( DaemonSocketState::Connectable, classify_daemon_protocol_identity(identity, expected_version), @@ -365,7 +360,7 @@ pub(super) fn daemon_readiness_probe( fn query_daemon_identity_stream( mut stream: impl ProbeStream, - auth_token: Option<&str>, + auth_token: &str, client_version: &str, deadline: std::time::Instant, ) -> Result<(Option, Option)> { @@ -376,12 +371,8 @@ fn query_daemon_identity_stream( "id": REQUEST_ID, "method": "initialize" }); - let mut preamble = String::new(); - if let Some(auth_token) = auth_token { - preamble - .push_str(&tracedecay_daemon_protocol::DaemonAuthPreface::new(auth_token).to_line()?); - preamble.push('\n'); - } + let mut preamble = tracedecay_daemon_protocol::DaemonAuthPreface::new(auth_token).to_line()?; + preamble.push('\n'); preamble.push_str(&handshake.to_line()?); preamble.push('\n'); preamble.push_str(&request.to_string()); @@ -794,7 +785,7 @@ mod timeout_classification_tests { }; let error = query_daemon_identity_stream( stream, - Some("token"), + "token", "0.1.0-test+service-probe", Instant::now() + Duration::from_secs(1), ) @@ -810,7 +801,7 @@ mod timeout_classification_tests { let _profile = PinnedUserDataDir::new(); let error = query_daemon_identity_stream( UnusedStream, - Some("token"), + "token", "0.1.0-test+service-probe", Instant::now() .checked_sub(Duration::from_secs(1)) diff --git a/crates/tracedecay-daemon-control/src/service/runner.rs b/crates/tracedecay-daemon-control/src/service/runner.rs index 5f47cc662e..9e5b1d5a95 100644 --- a/crates/tracedecay-daemon-control/src/service/runner.rs +++ b/crates/tracedecay-daemon-control/src/service/runner.rs @@ -143,22 +143,13 @@ impl ServiceRunner { pub(super) fn service_state(&self, socket_path: &Path) -> Result { match self { Self::Systemd { systemctl } => { - let running = Command::new(systemctl) - .args(["--user", "is-active", "--quiet", crate::SERVICE_NAME]) - .status() - .map_err(|error| { - service_program_spawn_error("systemctl", "systemd service state", &error) - })? - .success(); - let enablement = Command::new(systemctl) - .args(["--user", "is-enabled", crate::SERVICE_NAME]) - .output() - .map_err(|error| { - service_program_spawn_error("systemctl", "systemd service state", &error) - })?; - let enablement = String::from_utf8_lossy(&enablement.stdout) - .trim() - .to_string(); + let activity = systemctl_unit_query(systemctl, "is-active")?; + let running = match activity.as_str() { + "active" | "reloading" | "refreshing" => true, + "inactive" | "failed" | "activating" | "deactivating" | "maintenance" => false, + _ => return Err(systemctl_unknown_state("is-active", &activity)), + }; + let enablement = systemctl_unit_query(systemctl, "is-enabled")?; if enablement.starts_with("masked") { Ok(DaemonServiceState::Masked) } else if running && enablement.starts_with("enabled") { @@ -466,6 +457,39 @@ fn service_program_is_executable(_metadata: &std::fs::Metadata) -> bool { true } +/// `systemctl --user is-active`/`is-enabled` exit non-zero both for a stopped +/// or disabled unit and when the user manager is unreachable; only the printed +/// state tells them apart, so an empty answer is an error, not "stopped". +fn systemctl_unit_query(systemctl: &Path, verb: &str) -> Result { + let output = Command::new(systemctl) + .args(["--user", verb, crate::SERVICE_NAME]) + .output() + .map_err(|error| { + service_program_spawn_error("systemctl", "systemd service state", &error) + })?; + let state = String::from_utf8_lossy(&output.stdout).trim().to_string(); + if state.is_empty() { + return Err(TraceDecayError::Config { + message: format!( + "systemctl --user {verb} {} reported no unit state ({}): {}; the systemd user manager may be unreachable from this environment (check XDG_RUNTIME_DIR and DBUS_SESSION_BUS_ADDRESS)", + crate::SERVICE_NAME, + output.status, + String::from_utf8_lossy(&output.stderr).trim() + ), + }); + } + Ok(state) +} + +fn systemctl_unknown_state(verb: &str, state: &str) -> TraceDecayError { + TraceDecayError::Config { + message: format!( + "systemctl --user {verb} {} reported unrecognized unit state `{state}`", + crate::SERVICE_NAME + ), + } +} + fn service_program_spawn_error( program: &str, lifecycle: &str, diff --git a/crates/tracedecay-daemon-control/src/service/tests.rs b/crates/tracedecay-daemon-control/src/service/tests.rs index 82f47fc72b..a45d3da5aa 100644 --- a/crates/tracedecay-daemon-control/src/service/tests.rs +++ b/crates/tracedecay-daemon-control/src/service/tests.rs @@ -169,7 +169,7 @@ impl FailingRestoreFixture { let log = dir.path().join("systemctl.log"); std::fs::write( &systemctl, - "#!/bin/sh\nprintf '%s\\n' \"$*\" >> \"$TRACEDECAY_SYSTEMCTL_LOG\"\n[ \"$2\" = start ] && exit 7\n[ \"$2\" = is-enabled ] && echo enabled\nexit 0\n", + "#!/bin/sh\nprintf '%s\\n' \"$*\" >> \"$TRACEDECAY_SYSTEMCTL_LOG\"\n[ \"$2\" = start ] && exit 7\n[ \"$2\" = is-enabled ] && echo enabled\n[ \"$2\" = is-active ] && echo active\nexit 0\n", ) .expect("fake systemctl"); std::fs::set_permissions(&systemctl, std::fs::Permissions::from_mode(0o755)) @@ -335,25 +335,45 @@ fn strict_restoration_requires_readiness_only_for_running_state() { )); } +/// Publishes the authority record beside `socket` that probes resolve it +/// through. Hold the authority for as long as the socket should be served. #[cfg(unix)] -fn serve_probe_response( +pub(super) fn seed_socket_authority( + socket: &std::path::Path, +) -> tracedecay_daemon_identity::authority::DaemonAuthority { + tracedecay_daemon_identity::authority::DaemonAuthority::acquire( + socket.parent().expect("socket parent"), + &tracedecay_daemon_protocol::DaemonEndpoint::Unix(socket.to_path_buf()), + TEST_BUILD_VERSION, + ) + .expect("seed daemon authority") +} + +#[cfg(unix)] +fn read_auth_preface(reader: &mut impl BufRead, expected_auth_token: &str) -> usize { + let mut line = String::new(); + let read = reader.read_line(&mut line).expect("read auth preface"); + if read > 0 { + let preface = tracedecay_daemon_protocol::DaemonAuthPreface::from_line(line.trim()) + .expect("parse auth preface"); + assert!(preface.authenticate(expected_auth_token)); + } + read +} + +#[cfg(unix)] +pub(super) fn serve_probe_response( listener: UnixListener, name: &'static str, version: &'static str, - expected_auth_token: Option, + expected_auth_token: String, ) -> std::thread::JoinHandle<()> { std::thread::spawn(move || { let (mut stream, _) = listener.accept().expect("accept readiness probe"); let mut reader = std::io::BufReader::new(stream.try_clone().expect("clone readiness stream")); + read_auth_preface(&mut reader, &expected_auth_token); let mut line = String::new(); - if let Some(expected_auth_token) = expected_auth_token { - reader.read_line(&mut line).expect("read auth preface"); - let preface = tracedecay_daemon_protocol::DaemonAuthPreface::from_line(line.trim()) - .expect("parse auth preface"); - assert!(preface.authenticate(&expected_auth_token)); - line.clear(); - } reader.read_line(&mut line).expect("read handshake"); line.clear(); reader.read_line(&mut line).expect("read initialize"); @@ -431,6 +451,7 @@ fn serve_counted_authenticated_probe( fn serve_identity_probes( listener: UnixListener, versions: Vec<&'static str>, + expected_auth_token: String, ) -> (Arc, std::sync::mpsc::Receiver) { let served = Arc::new(AtomicUsize::new(0)); let count = Arc::clone(&served); @@ -444,6 +465,11 @@ fn serve_identity_probes( continue; }; let mut reader = std::io::BufReader::new(clone); + assert_ne!( + read_auth_preface(&mut reader, &expected_auth_token), + 0, + "readiness connection closed without an auth preface" + ); let mut line = String::new(); assert_ne!( reader @@ -488,8 +514,14 @@ fn daemon_protocol_probe_requires_current_tracedecay_identity() { let _data_dir_guard = EnvVarGuard::set(USER_DATA_DIR_ENV, profile.path()); let ready_socket = profile.path().join("ready.sock"); + let mut authority = seed_socket_authority(&ready_socket); let ready_listener = UnixListener::bind(&ready_socket).expect("bind ready socket"); - let ready_server = serve_probe_response(ready_listener, "tracedecay", TEST_BUILD_VERSION, None); + let ready_server = serve_probe_response( + ready_listener, + "tracedecay", + TEST_BUILD_VERSION, + authority.auth_token().to_owned(), + ); assert_eq!( super::probe::daemon_readiness_probe( &ready_socket, @@ -502,8 +534,18 @@ fn daemon_protocol_probe_requires_current_tracedecay_identity() { ready_server.join().expect("join ready server"); let stale_socket = profile.path().join("stale.sock"); + authority + .publish_endpoint(&tracedecay_daemon_protocol::DaemonEndpoint::Unix( + stale_socket.clone(), + )) + .expect("publish stale endpoint"); let stale_listener = UnixListener::bind(&stale_socket).expect("bind stale socket"); - let stale_server = serve_probe_response(stale_listener, "tracedecay", "0.0.0-stale", None); + let stale_server = serve_probe_response( + stale_listener, + "tracedecay", + "0.0.0-stale", + authority.auth_token().to_owned(), + ); assert_eq!( super::probe::daemon_readiness_probe( &stale_socket, @@ -552,6 +594,7 @@ fn daemon_readiness_probe_classifies_connect_and_protocol_failures() { )); let connectable_socket = profile.path().join("connectable.sock"); + let _authority = seed_socket_authority(&connectable_socket); let _listener = UnixListener::bind(&connectable_socket).expect("bind connectable socket"); let connectable = super::probe::daemon_readiness_probe( &connectable_socket, @@ -573,6 +616,16 @@ fn connectable_socket_is_not_a_live_daemon_until_initialize_answers() { let _env_lock = lock_user_data_dir_test_env(); let profile = TempDir::new().expect("profile temp dir"); let silent_socket = profile.path().join("silent.sock"); + let mut authority = seed_socket_authority(&silent_socket); + let token = authority.auth_token().to_owned(); + let mut serve_next = |socket: &std::path::Path| { + authority + .publish_endpoint(&tracedecay_daemon_protocol::DaemonEndpoint::Unix( + socket.to_path_buf(), + )) + .expect("publish probe endpoint"); + UnixListener::bind(socket).expect("bind probe socket") + }; let _listener = UnixListener::bind(&silent_socket).expect("bind silent socket"); let silent = super::probe::probe_daemon_process_with_timeout( &silent_socket, @@ -586,12 +639,11 @@ fn connectable_socket_is_not_a_live_daemon_until_initialize_answers() { assert!(!silent.names_tracedecay()); let ready_socket = profile.path().join("ready.sock"); - let ready_listener = UnixListener::bind(&ready_socket).expect("bind ready socket"); let server = serve_probe_response( - ready_listener, + serve_next(&ready_socket), "tracedecay", env!("CARGO_PKG_VERSION"), - None, + token.clone(), ); let ready = super::probe::probe_daemon_process_with_timeout( &ready_socket, @@ -604,8 +656,12 @@ fn connectable_socket_is_not_a_live_daemon_until_initialize_answers() { assert!(ready.version_matches()); let stale_socket = profile.path().join("stale-version.sock"); - let stale_listener = UnixListener::bind(&stale_socket).expect("bind stale socket"); - let stale_server = serve_probe_response(stale_listener, "tracedecay", "0.0.0-old", None); + let stale_server = serve_probe_response( + serve_next(&stale_socket), + "tracedecay", + "0.0.0-old", + token.clone(), + ); let stale = super::probe::probe_daemon_process_with_timeout( &stale_socket, env!("CARGO_PKG_VERSION"), @@ -618,8 +674,12 @@ fn connectable_socket_is_not_a_live_daemon_until_initialize_answers() { ); let foreign_socket = profile.path().join("foreign.sock"); - let foreign_listener = UnixListener::bind(&foreign_socket).expect("bind foreign socket"); - let foreign_server = serve_probe_response(foreign_listener, "not-tracedecay", "0.1.0", None); + let foreign_server = serve_probe_response( + serve_next(&foreign_socket), + "not-tracedecay", + "0.1.0", + token, + ); let foreign = super::probe::probe_daemon_process_with_timeout( &foreign_socket, env!("CARGO_PKG_VERSION"), @@ -646,8 +706,14 @@ fn daemon_reachable_requires_an_initialize_answer() { drop(missing_guard); let ready_socket = profile.path().join("ready.sock"); + let authority = seed_socket_authority(&ready_socket); let listener = UnixListener::bind(&ready_socket).expect("bind ready socket"); - let server = serve_probe_response(listener, "tracedecay", env!("CARGO_PKG_VERSION"), None); + let server = serve_probe_response( + listener, + "tracedecay", + env!("CARGO_PKG_VERSION"), + authority.auth_token().to_owned(), + ); let ready_guard = EnvVarGuard::set(SOCKET_ENV, &ready_socket); assert!( super::daemon_reachable(), @@ -683,6 +749,7 @@ fn daemon_socket_connectable_separates_a_slow_daemon_from_no_daemon() { // The cold-start case: a daemon is accepting but has not answered // initialize inside the one-second reachability probe. let silent = profile.path().join("silent.sock"); + let _authority = seed_socket_authority(&silent); let _listener = UnixListener::bind(&silent).expect("bind silent socket"); let silent_guard = EnvVarGuard::set(SOCKET_ENV, &silent); assert!( @@ -702,8 +769,14 @@ fn daemon_status_reports_the_initialize_proof_not_only_the_socket() { let _env_lock = lock_user_data_dir_test_env(); let profile = TempDir::new().expect("profile temp dir"); let socket = profile.path().join("status.sock"); + let authority = seed_socket_authority(&socket); let listener = UnixListener::bind(&socket).expect("bind status socket"); - let server = serve_probe_response(listener, "tracedecay", env!("CARGO_PKG_VERSION"), None); + let server = serve_probe_response( + listener, + "tracedecay", + env!("CARGO_PKG_VERSION"), + authority.auth_token().to_owned(), + ); let status = super::service_status(&socket, env!("CARGO_PKG_VERSION")); server.join().expect("join status server"); assert!( @@ -769,6 +842,29 @@ fn daemon_readiness_probe_classifies_authentication_denial() { } #[cfg(target_os = "linux")] +#[cfg(target_os = "linux")] +#[test] +fn unreachable_systemd_user_manager_is_an_error_not_a_stopped_unit() { + let dir = TempDir::new().expect("temp dir"); + let systemctl = dir.path().join("systemctl"); + std::fs::write( + &systemctl, + "#!/bin/sh\necho 'Failed to connect to bus: No medium found' >&2\nexit 1\n", + ) + .expect("fake systemctl"); + std::fs::set_permissions(&systemctl, std::fs::Permissions::from_mode(0o755)) + .expect("systemctl permissions"); + let runner = ServiceRunner::systemd(&systemctl).expect("fixture systemd runner"); + + let error = runner + .service_state(&dir.path().join("daemon.sock")) + .expect_err("an unreachable user manager has no unit state"); + let message = error.to_string(); + assert!(message.contains("reported no unit state"), "{message}"); + assert!(message.contains("Failed to connect to bus"), "{message}"); +} + +#[cfg(unix)] #[test] fn running_service_snapshot_uses_one_authenticated_connection() { let _env_lock = lock_user_data_dir_test_env(); @@ -1431,7 +1527,7 @@ fn refresh_installed_service_preserves_existing_socket_path() { let stopped = dir.path().join("systemctl.stopped"); std::fs::write( &systemctl, - "#!/bin/sh\nprintf '%s\\n' \"$*\" >> \"$TRACEDECAY_SYSTEMCTL_LOG\"\n[ \"$2\" = is-enabled ] && echo enabled\n[ \"$2\" = is-active ] && [ -f \"$TRACEDECAY_SYSTEMCTL_STOPPED\" ] && exit 3\n[ \"$2\" = stop ] && touch \"$TRACEDECAY_SYSTEMCTL_STOPPED\"\n[ \"$2\" = start ] && rm -f \"$TRACEDECAY_SYSTEMCTL_STOPPED\"\nexit 0\n", + "#!/bin/sh\nprintf '%s\\n' \"$*\" >> \"$TRACEDECAY_SYSTEMCTL_LOG\"\n[ \"$2\" = is-enabled ] && echo enabled\n[ \"$2\" = is-active ] && [ -f \"$TRACEDECAY_SYSTEMCTL_STOPPED\" ] && { echo inactive; exit 3; }\n[ \"$2\" = stop ] && touch \"$TRACEDECAY_SYSTEMCTL_STOPPED\"\n[ \"$2\" = start ] && rm -f \"$TRACEDECAY_SYSTEMCTL_STOPPED\"\n[ \"$2\" = is-active ] && echo active\nexit 0\n", ) .expect("fake systemctl"); std::fs::set_permissions(&systemctl, std::fs::Permissions::from_mode(0o755)) @@ -1477,8 +1573,13 @@ fn refresh_installed_service_preserves_existing_socket_path() { TEST_BUILD_VERSION, ) .expect("refresh service"); + let authority = seed_socket_authority(&custom_socket); let listener = UnixListener::bind(&custom_socket).expect("bind managed daemon socket"); - let (_served, _) = serve_identity_probes(listener, vec![TEST_BUILD_VERSION]); + let (_served, _) = serve_identity_probes( + listener, + vec![TEST_BUILD_VERSION], + authority.auth_token().to_owned(), + ); super::restore_installed_service_after_update_with_runner( &runner, previous_state, @@ -1501,7 +1602,7 @@ fn refresh_installed_service_preserves_existing_socket_path() { systemctl_log_contains_sequence( &commands, &[ - "--user is-active --quiet tracedecay.service", + "--user is-active tracedecay.service", "--user is-enabled tracedecay.service", "--user stop tracedecay.service", "--user daemon-reload", @@ -1514,67 +1615,6 @@ fn refresh_installed_service_preserves_existing_socket_path() { ); } -#[cfg(target_os = "linux")] -#[test] -fn refresh_installed_service_migrates_overlong_generated_socket_path() { - let _env_lock = lock_user_data_dir_test_env(); - let dir = TempDir::new().expect("temp dir"); - let config_home = dir.path().join("config"); - let fake_bin = dir.path().join("bin"); - let home = dir.path().join("home"); - let profile = dir.path().join("p".repeat(120)).join(".tracedecay"); - std::fs::create_dir_all(&fake_bin).expect("fake bin dir"); - std::fs::create_dir_all(&home).expect("home dir"); - - let systemctl = fake_bin.join("systemctl"); - std::fs::write( - &systemctl, - "#!/bin/sh\n[ \"$2\" = is-enabled ] && echo enabled\nexit 0\n", - ) - .expect("fake systemctl"); - std::fs::set_permissions(&systemctl, std::fs::Permissions::from_mode(0o755)) - .expect("systemctl permissions"); - let runner = ServiceRunner::systemd(&systemctl).expect("fixture systemd runner"); - - let _config_guard = EnvVarGuard::set("XDG_CONFIG_HOME", &config_home); - let _home_guard = EnvVarGuard::set("HOME", &home); - let _data_guard = EnvVarGuard::set(USER_DATA_DIR_ENV, &profile); - - let legacy_socket = profile.join("daemon.sock"); - let expected_socket = super::default_socket_path().expect("short default socket"); - assert_ne!(legacy_socket, expected_socket); - - let service_path = config_home.join("systemd/user").join(crate::SERVICE_NAME); - std::fs::create_dir_all(service_path.parent().expect("service parent")).expect("service dir"); - std::fs::write( - &service_path, - format!( - "[Unit]\nDescription=TraceDecay daemon\n\n[Service]\nExecStart=/old/tracedecay daemon run --socket {}\n", - legacy_socket.display() - ), - ) - .expect("existing service unit"); - - let spec = DaemonServiceSpec { - tracedecay_bin: PathBuf::from("/opt/tracedecay/bin/tracedecay"), - socket_path: expected_socket.clone(), - data_dir_override: Some(profile), - remote_tls: None, - }; - let outcome = super::refresh_installed_service_with_state_and_runner( - &runner, - &spec, - Some(DaemonServiceState::StoppedEnabled), - TEST_BUILD_VERSION, - ) - .expect("refresh service"); - - assert_eq!(outcome, Some(service_path.clone())); - let unit = std::fs::read_to_string(service_path).expect("service unit"); - assert!(unit.contains(&format!("--socket {}", expected_socket.display()))); - assert!(!unit.contains(&legacy_socket.display().to_string())); -} - #[cfg(target_os = "linux")] #[test] fn restore_quiesced_service_starts_existing_unit_without_rewriting_it() { @@ -1590,7 +1630,7 @@ fn restore_quiesced_service_starts_existing_unit_without_rewriting_it() { let log = dir.path().join("systemctl.log"); std::fs::write( &systemctl, - "#!/bin/sh\nprintf '%s\\n' \"$*\" >> \"$TRACEDECAY_SYSTEMCTL_LOG\"\n[ \"$2\" = is-enabled ] && echo enabled\nexit 0\n", + "#!/bin/sh\nprintf '%s\\n' \"$*\" >> \"$TRACEDECAY_SYSTEMCTL_LOG\"\n[ \"$2\" = is-enabled ] && echo enabled\n[ \"$2\" = is-active ] && echo active\nexit 0\n", ) .expect("fake systemctl"); std::fs::set_permissions(&systemctl, std::fs::Permissions::from_mode(0o755)) @@ -1609,8 +1649,13 @@ fn restore_quiesced_service_starts_existing_unit_without_rewriting_it() { custom_socket.display() ); std::fs::write(&service_path, &original_unit).expect("existing service unit"); + let authority = seed_socket_authority(&custom_socket); let listener = UnixListener::bind(&custom_socket).expect("bind managed daemon socket"); - let (served, acknowledged) = serve_identity_probes(listener, vec![TEST_BUILD_VERSION]); + let (served, acknowledged) = serve_identity_probes( + listener, + vec![TEST_BUILD_VERSION], + authority.auth_token().to_owned(), + ); super::restore_installed_service_after_update_with_runner( &runner, @@ -1659,7 +1704,7 @@ fn restore_after_update_does_not_activate_a_held_stopped_unit() { let log = dir.path().join("systemctl.log"); std::fs::write( &systemctl, - "#!/bin/sh\nprintf '%s\\n' \"$*\" >> \"$TRACEDECAY_SYSTEMCTL_LOG\"\n[ \"$2\" = is-enabled ] && echo enabled\nexit 0\n", + "#!/bin/sh\nprintf '%s\\n' \"$*\" >> \"$TRACEDECAY_SYSTEMCTL_LOG\"\n[ \"$2\" = is-enabled ] && echo enabled\n[ \"$2\" = is-active ] && echo active\nexit 0\n", ) .expect("fake systemctl"); std::fs::set_permissions(&systemctl, std::fs::Permissions::from_mode(0o755)) @@ -1777,7 +1822,7 @@ fn restore_after_update_waits_for_authenticated_daemon_identity() { let systemctl = fake_bin.join("systemctl"); std::fs::write( &systemctl, - "#!/bin/sh\n[ \"$2\" = is-enabled ] && echo enabled\nexit 0\n", + "#!/bin/sh\n[ \"$2\" = is-enabled ] && echo enabled\n[ \"$2\" = is-active ] && echo active\nexit 0\n", ) .expect("fake systemctl"); std::fs::set_permissions(&systemctl, std::fs::Permissions::from_mode(0o755)) @@ -1797,12 +1842,16 @@ fn restore_after_update_waits_for_authenticated_daemon_identity() { ), ) .expect("existing service unit"); + let authority = seed_socket_authority(&socket_path); let listener = UnixListener::bind(&socket_path).expect("bind managed daemon socket"); // The first identity answer is a stale daemon; restore must keep polling // until the expected version answers instead of trusting the systemctl // exit status. - let (served, acknowledged) = - serve_identity_probes(listener, vec!["0.0.0-stale", TEST_BUILD_VERSION]); + let (served, acknowledged) = serve_identity_probes( + listener, + vec!["0.0.0-stale", TEST_BUILD_VERSION], + authority.auth_token().to_owned(), + ); super::restore_installed_service_after_update( DaemonServiceState::RunningEnabled, @@ -1841,7 +1890,7 @@ fn start_service_reloads_units_and_requires_authenticated_identity() { let started = dir.path().join("systemctl.started"); std::fs::write( &systemctl, - "#!/bin/sh\nprintf '%s\\n' \"$*\" >> \"$TRACEDECAY_SYSTEMCTL_LOG\"\n[ \"$2\" = is-enabled ] && echo enabled\n[ \"$2\" = is-active ] && [ ! -f \"$TRACEDECAY_SYSTEMCTL_STARTED\" ] && exit 3\n[ \"$2\" = start ] && : > \"$TRACEDECAY_SYSTEMCTL_STARTED\"\nexit 0\n", + "#!/bin/sh\nprintf '%s\\n' \"$*\" >> \"$TRACEDECAY_SYSTEMCTL_LOG\"\n[ \"$2\" = is-enabled ] && echo enabled\n[ \"$2\" = is-active ] && [ ! -f \"$TRACEDECAY_SYSTEMCTL_STARTED\" ] && { echo inactive; exit 3; }\n[ \"$2\" = start ] && : > \"$TRACEDECAY_SYSTEMCTL_STARTED\"\n[ \"$2\" = is-active ] && echo active\nexit 0\n", ) .expect("fake systemctl"); std::fs::set_permissions(&systemctl, std::fs::Permissions::from_mode(0o755)) @@ -1863,8 +1912,13 @@ fn start_service_reloads_units_and_requires_authenticated_identity() { ), ) .expect("existing service unit"); + let authority = seed_socket_authority(&socket_path); let listener = UnixListener::bind(&socket_path).expect("bind managed daemon socket"); - let (served, acknowledged) = serve_identity_probes(listener, vec![TEST_BUILD_VERSION]); + let (served, acknowledged) = serve_identity_probes( + listener, + vec![TEST_BUILD_VERSION], + authority.auth_token().to_owned(), + ); super::start_service(TEST_BUILD_VERSION).expect("start service"); @@ -1898,7 +1952,7 @@ fn wait_for_installed_service_state_rejects_identity_mismatch_at_the_deadline() let systemctl = fake_bin.join("systemctl"); std::fs::write( &systemctl, - "#!/bin/sh\n[ \"$2\" = is-enabled ] && echo enabled\nexit 0\n", + "#!/bin/sh\n[ \"$2\" = is-enabled ] && echo enabled\n[ \"$2\" = is-active ] && echo active\nexit 0\n", ) .expect("fake systemctl"); std::fs::set_permissions(&systemctl, std::fs::Permissions::from_mode(0o755)) @@ -1918,8 +1972,13 @@ fn wait_for_installed_service_state_rejects_identity_mismatch_at_the_deadline() ), ) .expect("existing service unit"); + let authority = seed_socket_authority(&socket_path); let listener = UnixListener::bind(&socket_path).expect("bind managed daemon socket"); - let (_served, _) = serve_identity_probes(listener, vec!["0.0.0-stale"]); + let (_served, _) = serve_identity_probes( + listener, + vec!["0.0.0-stale"], + authority.auth_token().to_owned(), + ); let error = super::wait_for_installed_service_state_with( &runner, @@ -1952,7 +2011,7 @@ fn wait_for_installed_service_state_rejects_unresponsive_socket_at_the_deadline( let systemctl = fake_bin.join("systemctl"); std::fs::write( &systemctl, - "#!/bin/sh\n[ \"$2\" = is-enabled ] && echo enabled\nexit 0\n", + "#!/bin/sh\n[ \"$2\" = is-enabled ] && echo enabled\n[ \"$2\" = is-active ] && echo active\nexit 0\n", ) .expect("fake systemctl"); std::fs::set_permissions(&systemctl, std::fs::Permissions::from_mode(0o755)) @@ -2069,7 +2128,7 @@ fn refresh_installed_service_preserves_stopped_state() { let log = dir.path().join("systemctl.log"); std::fs::write( &systemctl, - "#!/bin/sh\nprintf '%s\\n' \"$*\" >> \"$TRACEDECAY_SYSTEMCTL_LOG\"\n[ \"$2\" = is-active ] && exit 3\n[ \"$2\" = is-enabled ] && echo enabled\nexit 0\n", + "#!/bin/sh\nprintf '%s\\n' \"$*\" >> \"$TRACEDECAY_SYSTEMCTL_LOG\"\n[ \"$2\" = is-active ] && { echo inactive; exit 3; }\n[ \"$2\" = is-enabled ] && echo enabled\n[ \"$2\" = is-active ] && echo active\nexit 0\n", ) .expect("fake systemctl"); std::fs::set_permissions(&systemctl, std::fs::Permissions::from_mode(0o755)) @@ -2109,7 +2168,7 @@ fn refresh_installed_service_preserves_stopped_state() { .expect("refresh service"); let commands = std::fs::read_to_string(log).expect("systemctl log"); - assert!(commands.contains("--user is-active --quiet tracedecay.service")); + assert!(commands.contains("--user is-active tracedecay.service")); assert!( !commands.contains("enable tracedecay.service"), "a stopped-enabled service is already enabled; refresh must not mutate its lifecycle" @@ -2127,7 +2186,7 @@ fn systemd_service_state_detects_runtime_mask() { let systemctl = fake_bin.join("systemctl"); std::fs::write( &systemctl, - "#!/bin/sh\n[ \"$2\" = is-active ] && exit 3\n[ \"$2\" = is-enabled ] && { echo masked-runtime; exit 1; }\nexit 0\n", + "#!/bin/sh\n[ \"$2\" = is-active ] && { echo inactive; exit 3; }\n[ \"$2\" = is-enabled ] && { echo masked-runtime; exit 1; }\n[ \"$2\" = is-active ] && echo active\nexit 0\n", ) .expect("fake systemctl"); std::fs::set_permissions(&systemctl, std::fs::Permissions::from_mode(0o755)) diff --git a/crates/tracedecay-daemon-control/src/service/update_restore_tests.rs b/crates/tracedecay-daemon-control/src/service/update_restore_tests.rs index 6fd2610130..462341d824 100644 --- a/crates/tracedecay-daemon-control/src/service/update_restore_tests.rs +++ b/crates/tracedecay-daemon-control/src/service/update_restore_tests.rs @@ -7,8 +7,6 @@ //! and the readiness probe keeps failing closed with a typed identity //! mismatch when the daemon that answers is not the expected version. -#[cfg(unix)] -use std::io::{BufRead, Write}; #[cfg(unix)] use std::os::unix::net::UnixListener; @@ -42,33 +40,6 @@ fn quiesced_guard() -> QuiescedDaemonLifecycle { #[cfg(unix)] use super::isolated_profile::EnvVarGuard; -#[cfg(unix)] -fn serve_initialize_identity( - listener: UnixListener, - name: &'static str, - version: &'static str, -) -> std::thread::JoinHandle<()> { - std::thread::spawn(move || { - let (mut stream, _) = listener.accept().expect("accept readiness probe"); - let mut reader = - std::io::BufReader::new(stream.try_clone().expect("clone readiness stream")); - let mut line = String::new(); - reader.read_line(&mut line).expect("read handshake"); - line.clear(); - reader.read_line(&mut line).expect("read initialize"); - let request: serde_json::Value = - serde_json::from_str(line.trim()).expect("initialize json"); - let response = serde_json::json!({ - "jsonrpc": "2.0", - "id": request["id"], - "result": { - "serverInfo": {"name": name, "version": version} - } - }); - writeln!(stream, "{response}").expect("write initialize response"); - }) -} - /// Version skew must keep failing closed: when the daemon that answers after /// an upgrade is still the OLD binary (restart raced or was lost), readiness /// against the installed version reports a typed identity mismatch instead of @@ -87,8 +58,14 @@ fn restore_readiness_rejects_a_stale_daemon_after_an_upgrade() { }); let socket_path = profile.path().join("stale.sock"); + let authority = super::tests::seed_socket_authority(&socket_path); let listener = UnixListener::bind(&socket_path).expect("bind stale daemon socket"); - let server = serve_initialize_identity(listener, "tracedecay", QUIESCED_VERSION); + let server = super::tests::serve_probe_response( + listener, + "tracedecay", + QUIESCED_VERSION, + authority.auth_token().to_owned(), + ); assert_eq!( super::probe::daemon_protocol_state_with_timeout( diff --git a/crates/tracedecay-daemon-identity/src/authority.rs b/crates/tracedecay-daemon-identity/src/authority.rs index 043dc2edff..1055567590 100644 --- a/crates/tracedecay-daemon-identity/src/authority.rs +++ b/crates/tracedecay-daemon-identity/src/authority.rs @@ -1,4 +1,4 @@ -use serde::{Deserialize, Deserializer, Serialize}; +use serde::{Deserialize, Serialize}; use std::fmt; use std::fs::File; #[cfg(not(windows))] @@ -7,6 +7,7 @@ use std::io::{Read, Seek, SeekFrom, Write}; use std::net::SocketAddr; use std::path::{Path, PathBuf}; use tracedecay_domain::{BrainId, UserProfileId}; +use tracedecay_private_fs::FileLease; use tracedecay_runtime_core::path_safety::{ canonicalize_existing_prefix, collapse_relative_components, }; @@ -24,35 +25,6 @@ mod windows_acl; const RECORD_FILE: &str = "daemon-authority.json"; -fn deserialize_endpoint<'de, D>(deserializer: D) -> std::result::Result -where - D: Deserializer<'de>, -{ - #[derive(Deserialize)] - #[serde(untagged)] - enum EndpointRecord { - Current(DaemonEndpoint), - Legacy(PathBuf), - } - - match EndpointRecord::deserialize(deserializer)? { - EndpointRecord::Current(endpoint) => Ok(endpoint), - EndpointRecord::Legacy(path) => { - #[cfg(unix)] - { - Ok(DaemonEndpoint::Unix(path)) - } - #[cfg(not(unix))] - { - Err(serde::de::Error::custom(format!( - "legacy Unix daemon endpoint '{}' is unsupported on this platform", - path.display() - ))) - } - } - } -} - #[derive(Clone, Deserialize, PartialEq, Eq, Serialize)] pub struct DaemonAuthorityRecord { pub pid: u32, @@ -60,7 +32,6 @@ pub struct DaemonAuthorityRecord { pub started_at_unix_secs: i64, pub epoch: u64, pub version: String, - #[serde(alias = "socket_path", deserialize_with = "deserialize_endpoint")] pub endpoint: DaemonEndpoint, #[serde(default, skip_serializing_if = "Option::is_none")] pub http_application_endpoint: Option, @@ -98,7 +69,7 @@ impl fmt::Debug for DaemonAuthorityRecord { #[derive(Debug)] pub struct DaemonAuthority { - _lock: File, + _lock: FileLease, record_path: PathBuf, record: DaemonAuthorityRecord, profile_identity: LocalProfileIdentityAuthorityV1, @@ -120,7 +91,7 @@ impl DaemonAuthority { restrict_directory(&authority_root)?; let lock_path = authority_root.join(LOCK_FILE); - let mut lock = open_private_lock(&lock_path)?; + let lock = open_private_lock(&lock_path)?; if let Err(error) = lock.try_lock().map_err(std::io::Error::from) { if !tracedecay_private_fs::is_lock_contended(&error) { return Err(config_io("lock", &lock_path, &error)); @@ -142,6 +113,7 @@ impl DaemonAuthority { ), }); } + let mut lock = FileLease::held(lock, "daemon_identity.authority"); let record_path = authority_root.join(RECORD_FILE); let prior_record = read_record_if_present(&record_path)?; @@ -316,6 +288,11 @@ pub fn current_record(profile_root: &Path) -> Result Result { + Ok(authority_state_root(&canonical_identity_path(profile_root)?).join(RECORD_FILE)) +} + #[cfg(windows)] fn validate_existing_profile_root(path: &Path) -> Result { match windows_acl::validate_directory_path(path) { @@ -789,42 +766,32 @@ mod tests { assert!(authority.ensure_current().is_ok()); } - #[cfg(unix)] #[test] - fn legacy_socket_path_record_is_accepted_by_current_reader() { - #[derive(Serialize)] - struct LegacySocketRecord { - pid: u32, - process_run_id: String, - started_at_unix_secs: i64, - epoch: u64, - version: String, - socket_path: PathBuf, - auth_token: String, - profile_root: PathBuf, - } - + fn legacy_endpoint_record_shapes_are_refused() { let temp = tempfile::tempdir().unwrap(); - let profile_root = temp.path().join("profile"); - std::fs::create_dir_all(&profile_root).unwrap(); - let record_path = profile_root.join(RECORD_FILE); - let socket_path = profile_root.join("daemon.sock"); - let legacy = LegacySocketRecord { - pid: 42, - process_run_id: "legacy-run".to_string(), - started_at_unix_secs: 1, - epoch: 3, - version: "legacy".to_string(), - socket_path: socket_path.clone(), - auth_token: "a".repeat(64), - profile_root: profile_root.clone(), - }; - std::fs::write(&record_path, serde_json::to_vec(&legacy).unwrap()).unwrap(); - restrict_file(&record_path).unwrap(); - - let decoded = read_record_if_present(&record_path).unwrap().unwrap(); - assert_eq!(decoded.endpoint, DaemonEndpoint::Unix(socket_path)); - assert_eq!(decoded.auth_token, "a".repeat(64)); + let profile = temp.path().join("profile"); + let endpoint = test_endpoint(&profile); + let authority = DaemonAuthority::acquire(&profile, &endpoint, "test").unwrap(); + let current = serde_json::to_value(authority.record()).unwrap(); + let mut socket_path_key = current.clone(); + let fields = socket_path_key.as_object_mut().unwrap(); + let tagged_endpoint = fields.remove("endpoint").unwrap(); + fields.insert("socket_path".to_string(), tagged_endpoint); + let mut bare_path_endpoint = current; + bare_path_endpoint["endpoint"] = serde_json::json!(profile.join("daemon.sock")); + + for legacy in [socket_path_key, bare_path_endpoint] { + std::fs::write(&authority.record_path, serde_json::to_vec(&legacy).unwrap()).unwrap(); + restrict_file(&authority.record_path).unwrap(); + + let TraceDecayError::Config { message } = current_record(&profile).unwrap_err() else { + panic!("a legacy authority record must be refused as a configuration error"); + }; + assert!( + message.contains("invalid daemon authority record"), + "{message}" + ); + } } #[test] @@ -887,29 +854,6 @@ mod tests { assert!(error.to_string().contains("not private"), "{error}"); } - #[test] - fn current_endpoint_record_fails_closed_for_legacy_reader() { - #[allow(dead_code)] - #[derive(Deserialize)] - struct LegacySocketRecord { - pid: u32, - process_run_id: String, - started_at_unix_secs: i64, - epoch: u64, - version: String, - socket_path: PathBuf, - profile_root: PathBuf, - } - - let temp = tempfile::tempdir().unwrap(); - let profile = temp.path().join("profile"); - let endpoint = test_endpoint(&profile); - let authority = DaemonAuthority::acquire(&profile, &endpoint, "current").unwrap(); - let encoded = serde_json::to_string(authority.record()).unwrap(); - - assert!(serde_json::from_str::(&encoded).is_err()); - } - #[test] fn stale_endpoint_or_token_is_rejected_by_the_elected_owner() { let temp = tempfile::tempdir().unwrap(); diff --git a/crates/tracedecay-daemon-identity/src/connection.rs b/crates/tracedecay-daemon-identity/src/connection.rs index 122208fe44..41490e6a52 100644 --- a/crates/tracedecay-daemon-identity/src/connection.rs +++ b/crates/tracedecay-daemon-identity/src/connection.rs @@ -9,6 +9,8 @@ use std::net::SocketAddr; use std::path::Path; +#[cfg(unix)] +use std::path::PathBuf; use std::sync::Arc; use tracedecay_daemon_protocol::{DaemonEndpoint, DaemonLivenessProbe}; @@ -16,45 +18,52 @@ use tracedecay_domain::errors::{Result, TraceDecayError}; use crate::authority; -/// A discovered daemon endpoint plus its credential and private authority -/// provenance. Distinct from the protocol crate's transport +/// Typed reason code for a daemon endpoint no readable authority record names. +/// +/// Retryable only while the record is absent: a starting daemon writes its +/// record before it binds, so absence resolves itself. A record naming a +/// different endpoint does not. +pub const DAEMON_AUTHORITY_UNAVAILABLE: &str = "daemon_authority_unavailable"; + +/// A daemon endpoint plus its credential, both read from the authority record +/// that names it. Distinct from the protocol crate's transport /// [`tracedecay_daemon_protocol::DaemonConnection`]; convert with /// [`Self::into_protocol`]. -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct ResolvedDaemonConnection { - pub endpoint: DaemonEndpoint, - pub auth_token: Option, - authority_record: Option, + record: authority::DaemonAuthorityRecord, } impl ResolvedDaemonConnection { + pub fn endpoint(&self) -> &DaemonEndpoint { + &self.record.endpoint + } + + pub fn auth_token(&self) -> &str { + &self.record.auth_token + } + /// The loopback HTTP application endpoint published by this connection's /// authority, when one is available. pub fn http_application_endpoint(&self) -> Option { - self.authority_record - .as_ref() - .and_then(|record| record.http_application_endpoint) + self.record.http_application_endpoint } pub fn into_protocol(self) -> tracedecay_daemon_protocol::DaemonConnection { - let connection = - tracedecay_daemon_protocol::DaemonConnection::new(self.endpoint, self.auth_token); - match self.authority_record { - Some(record) => connection - .with_daemon_version(record.version.clone()) - .with_liveness(Arc::new(AuthorityLivenessProbe { record })), - None => connection, - } + tracedecay_daemon_protocol::DaemonConnection::new( + self.record.endpoint.clone(), + self.record.auth_token.clone(), + ) + .with_daemon_version(self.record.version.clone()) + .with_liveness(Arc::new(AuthorityLivenessProbe { + record: self.record, + })) } /// Fails when the authority record that named this endpoint is no longer /// current (the daemon restarted or its authority disappeared). - /// Connections without a discovered record have nothing to check. pub fn ensure_authority_current(&self, request_label: &str) -> Result<()> { - match self.authority_record.as_ref() { - Some(record) => ensure_record_current(record, request_label), - None => Ok(()), - } + ensure_record_current(&self.record, request_label) } } @@ -68,21 +77,18 @@ impl DaemonLivenessProbe for AuthorityLivenessProbe { } } -fn connection_from_record(record: authority::DaemonAuthorityRecord) -> ResolvedDaemonConnection { - ResolvedDaemonConnection { - endpoint: record.endpoint.clone(), - auth_token: Some(record.auth_token.clone()), - authority_record: Some(record), - } -} - #[cfg(unix)] fn unix_endpoint_matches_socket(endpoint: &DaemonEndpoint, socket_path: &Path) -> bool { let DaemonEndpoint::Unix(authority_path) = endpoint else { return false; }; - authority::canonical_identity_path(authority_path).ok() - == authority::canonical_identity_path(socket_path).ok() + matches!( + ( + authority::canonical_identity_path(authority_path), + authority::canonical_identity_path(socket_path), + ), + (Ok(recorded), Ok(requested)) if recorded == requested + ) } fn ensure_record_current( @@ -124,48 +130,79 @@ pub fn current_daemon_connection() -> Result { message: "could not determine TraceDecay user data directory".to_string(), } })?; - let record = - authority::current_record(&profile_root)?.ok_or_else(|| TraceDecayError::Config { - message: - "TraceDecay daemon authority record is not available. Start or restart the daemon." - .to_string(), - })?; - Ok(connection_from_record(record)) + match authority::current_record(&profile_root)? { + Some(record) => Ok(ResolvedDaemonConnection { record }), + None => Err(TraceDecayError::project_route( + DAEMON_AUTHORITY_UNAVAILABLE, + true, + format!( + "no TraceDecay daemon authority record at '{}'. Start or restart the daemon.", + authority::record_path(&profile_root)?.display() + ), + )), + } } +/// The connection for the daemon serving `socket_path`, read from the record +/// that names it: the user profile's record, else the record beside the socket +/// (a daemon whose profile root holds its socket). #[cfg(unix)] -pub fn connection_for_socket_path(socket_path: &Path) -> ResolvedDaemonConnection { - if let Ok(connection) = current_daemon_connection() - && connection - .authority_record - .as_ref() - .is_some_and(|record| unix_endpoint_matches_socket(&record.endpoint, socket_path)) - { - return connection; - } - if let Some(profile_root) = socket_path.parent() - && let Ok(Some(record)) = authority::current_record(profile_root) - && unix_endpoint_matches_socket(&record.endpoint, socket_path) - { - return connection_from_record(record); - } - // Explicit paths are retained for test harnesses and legacy one-shot - // callers without a discoverable authority record. Default production - // routing always uses the authority record. - ResolvedDaemonConnection { - endpoint: DaemonEndpoint::Unix(socket_path.to_path_buf()), - auth_token: None, - authority_record: None, +fn connection_for_socket_path(socket_path: &Path) -> Result { + let user_profile = tracedecay_runtime_core::config::user_data_dir(); + connection_for_socket_in(user_profile.as_deref(), socket_path) +} + +#[cfg(unix)] +fn connection_for_socket_in( + user_profile: Option<&Path>, + socket_path: &Path, +) -> Result { + let mut checked: Vec = Vec::new(); + let mut named_elsewhere = Vec::new(); + for profile_root in user_profile.into_iter().chain(socket_path.parent()) { + let record_path = authority::record_path(profile_root)?; + if checked.contains(&record_path) { + continue; + } + checked.push(record_path); + if let Some(record) = authority::current_record(profile_root)? { + if unix_endpoint_matches_socket(&record.endpoint, socket_path) { + return Ok(ResolvedDaemonConnection { record }); + } + named_elsewhere.push(record.endpoint.to_string()); + } } + let records = checked + .iter() + .map(|path| format!("'{}'", path.display())) + .collect::>() + .join(", "); + let socket = socket_path.display(); + Err(if named_elsewhere.is_empty() { + TraceDecayError::project_route( + DAEMON_AUTHORITY_UNAVAILABLE, + true, + format!( + "no TraceDecay daemon authority record names socket '{socket}' (checked {records}). Start or restart the daemon." + ), + ) + } else { + TraceDecayError::project_route( + DAEMON_AUTHORITY_UNAVAILABLE, + false, + format!( + "TraceDecay daemon authority records {records} name {} instead of socket '{socket}'. Point {} at the running daemon or restart it.", + named_elsewhere.join(", "), + tracedecay_daemon_protocol::SOCKET_ENV, + ), + ) + }) } -// Windows discovers the current daemon through a fallible endpoint lookup; -// Unix keeps the same cross-platform contract even though its path is infallible. -#[allow(clippy::unnecessary_wraps)] pub fn client_connection(socket_path: &Path) -> Result { #[cfg(unix)] { - Ok(connection_for_socket_path(socket_path)) + connection_for_socket_path(socket_path) } #[cfg(not(unix))] { @@ -173,3 +210,59 @@ pub fn client_connection(socket_path: &Path) -> Result current_daemon_connection() } } + +#[cfg(all(test, unix))] +mod tests { + use std::os::unix::fs::PermissionsExt; + + use super::*; + + fn route(error: &TraceDecayError) -> Option<(&str, bool)> { + error + .project_route_context() + .map(|(code, retryable, _)| (code, retryable)) + } + + #[test] + fn socket_without_an_authority_record_is_a_typed_retryable_absence() { + let temp = tempfile::tempdir().unwrap(); + let socket = temp.path().join("daemon.sock"); + + let error = connection_for_socket_in(None, &socket) + .expect_err("a socket no record names has no credential"); + + assert_eq!(route(&error), Some((DAEMON_AUTHORITY_UNAVAILABLE, true))); + let record = authority::record_path(temp.path()).unwrap(); + assert!( + error.to_string().contains(&record.display().to_string()), + "{error}" + ); + } + + #[test] + fn socket_resolves_only_from_a_readable_record_that_names_it() { + let temp = tempfile::tempdir().unwrap(); + let socket = temp.path().join("daemon.sock"); + let authority = authority::DaemonAuthority::acquire( + temp.path(), + &DaemonEndpoint::Unix(socket.clone()), + "test", + ) + .unwrap(); + + let connection = connection_for_socket_in(None, &socket).unwrap(); + assert_eq!(connection.auth_token(), authority.auth_token()); + + let other = temp.path().join("other.sock"); + let error = connection_for_socket_in(None, &other) + .expect_err("a record naming another socket is not this daemon's credential"); + assert_eq!(route(&error), Some((DAEMON_AUTHORITY_UNAVAILABLE, false))); + + let record = authority::record_path(temp.path()).unwrap(); + std::fs::set_permissions(&record, std::fs::Permissions::from_mode(0o644)).unwrap(); + let error = connection_for_socket_in(None, &socket) + .expect_err("an unreadable record must not be skipped"); + assert_eq!(route(&error), None); + assert!(error.to_string().contains("not private"), "{error}"); + } +} diff --git a/crates/tracedecay-daemon-identity/src/lib.rs b/crates/tracedecay-daemon-identity/src/lib.rs index 306ebea1cd..dc86549a68 100644 --- a/crates/tracedecay-daemon-identity/src/lib.rs +++ b/crates/tracedecay-daemon-identity/src/lib.rs @@ -16,9 +16,6 @@ mod connection; pub mod profile_identity; pub use connection::{ - ResolvedDaemonConnection, client_connection, current_daemon_connection, - invocation_client_for_current, + DAEMON_AUTHORITY_UNAVAILABLE, ResolvedDaemonConnection, client_connection, + current_daemon_connection, invocation_client_for_current, }; - -#[cfg(unix)] -pub use connection::connection_for_socket_path; diff --git a/crates/tracedecay-daemon-protocol/Cargo.toml b/crates/tracedecay-daemon-protocol/Cargo.toml index 331de74950..e6358d3ff8 100644 --- a/crates/tracedecay-daemon-protocol/Cargo.toml +++ b/crates/tracedecay-daemon-protocol/Cargo.toml @@ -21,6 +21,7 @@ hotpath.workspace = true semver = "1" serde = { version = "1", features = ["derive"] } serde_json = "1" +serde_path_to_error = "0.1" thiserror = "2" tokio = { version = "1", features = ["io-util", "macros", "net", "rt", "sync", "time"] } tracedecay-contracts = { path = "../tracedecay-contracts", version = "0.1.0" } diff --git a/crates/tracedecay-daemon-protocol/src/application_surface.rs b/crates/tracedecay-daemon-protocol/src/application_surface.rs index f637e1a469..953ddbacf8 100644 --- a/crates/tracedecay-daemon-protocol/src/application_surface.rs +++ b/crates/tracedecay-daemon-protocol/src/application_surface.rs @@ -7,6 +7,19 @@ //! without depending on daemon-service. Execution stays in daemon-service. mod git; +mod invocation; +mod retained; +mod source_edit; + +pub use retained::decode_retained_request; +pub use source_edit::{is_source_edit_operation, parse_source_edit_arguments}; + +pub use invocation::{ + application_delivery_route, application_outcome_value, application_response, + application_surface_cancellation_policy, application_surface_feedback_is_observable, + application_surface_feedback_operation, invoke_application_surface, + parse_application_surface_invocation_payload, +}; use serde::{Deserialize, Serialize}; use serde_json::Value; @@ -33,6 +46,7 @@ use tracedecay_contracts::{ CodeSymbolSearchSurfaceRequest, CodeTimelineSurfaceRequest, CodeTypeHierarchySurfaceRequest, ConfigurationWireRequestV1, HealthReadRequest, NativeIntegrationSurfaceRequest, ObservatoryReadRequestV1, PrimitiveCodeSurfaceRequest, SessionLookupRequest, + SourceEditInvocationV1, SourceEditReconciliationInvocationV1, SourceEditRollbackInvocationV1, SourceLinesRequest, configuration_wire_request_from_invocation_payload, }; use tracedecay_tool_catalog::{ @@ -42,6 +56,7 @@ use tracedecay_tool_catalog::{ use crate::output_format::{RequestedOutputFormat, requested_output_format}; use crate::surface::GitReadSurfaceRequest; use tracedecay_contracts::context_scout::ContextScoutSurfaceRequestV1; +use tracedecay_contracts::retained_surfaces::{RetainedSurfaceOperation, RetainedSurfaceRequestV1}; #[derive(Debug, Error)] pub enum ApplicationSurfaceAdapterError { @@ -200,7 +215,13 @@ pub enum ApplicationSurfaceRequest { ObservatoryRead(ObservatoryReadRequestV1), Configuration(ConfigurationWireRequestV1), ContextScout(ContextScoutSurfaceRequestV1), - Retained(tracedecay_contracts::retained_surfaces::RetainedSurfaceRequestV1), + SourceEdit(SourceEditInvocationV1), + SourceEditReconcile(SourceEditReconciliationInvocationV1), + SourceEditRollback(SourceEditRollbackInvocationV1), + Retained(RetainedSurfaceRequestV1), + /// A graph or port read's argument object. Its owning handler decodes the + /// typed request so argument diagnostics stay the handler's own. + GraphTool(serde_json::Map), } pub struct ApplicationSurfaceInvocationResult { @@ -212,9 +233,15 @@ pub struct ApplicationSurfaceInvocationResult { impl ApplicationSurfaceRequest { pub fn matches(&self, operation: ApplicationSurfaceOperation) -> bool { + if let Self::SourceEdit(invocation) = self { + return source_edit::source_edit_kind(operation) == Some(invocation.edit.kind()); + } if let Self::Retained(request) = self { return request.operation().as_str() == operation.as_str(); } + if let Self::GraphTool(_) = self { + return operation.is_graph_tool(); + } matches!( (self, operation), ( @@ -456,6 +483,14 @@ impl ApplicationSurfaceRequest { Self::ContextScout(ContextScoutSurfaceRequestV1::Feedback(_)), ApplicationSurfaceOperation::ContextScoutFeedback ) + | ( + Self::SourceEditReconcile(_), + ApplicationSurfaceOperation::SourceEditReconcile + ) + | ( + Self::SourceEditRollback(_), + ApplicationSurfaceOperation::SourceEditRollback + ) ) } } @@ -802,6 +837,69 @@ pub fn parse_application_surface_request( ApplicationSurfaceOperation::FeedbackProximity => serde_json::from_value(value) .map(ApplicationSurfaceRequest::FeedbackProximity) .map_err(ApplicationSurfaceAdapterError::invalid_request), + ApplicationSurfaceOperation::StrReplace + | ApplicationSurfaceOperation::MultiStrReplace + | ApplicationSurfaceOperation::InsertAt + | ApplicationSurfaceOperation::AstGrepRewrite + | ApplicationSurfaceOperation::ReplaceSymbol + | ApplicationSurfaceOperation::InsertAtSymbol + | ApplicationSurfaceOperation::MoveSymbol + | ApplicationSurfaceOperation::RenameSymbol + | ApplicationSurfaceOperation::SourceEditReconcile + | ApplicationSurfaceOperation::SourceEditRollback => { + parse_source_edit_arguments(operation, &value) + .map_err(ApplicationSurfaceAdapterError::invalid_request) + } + ApplicationSurfaceOperation::FactStoreCurate + | ApplicationSurfaceOperation::FactStoreAdd + | ApplicationSurfaceOperation::FactStoreSearch + | ApplicationSurfaceOperation::FactStoreProbe + | ApplicationSurfaceOperation::FactStoreRelated + | ApplicationSurfaceOperation::FactStoreReason + | ApplicationSurfaceOperation::FactStoreContradict + | ApplicationSurfaceOperation::FactStoreGet + | ApplicationSurfaceOperation::FactStoreUpdate + | ApplicationSurfaceOperation::FactStoreRemove + | ApplicationSurfaceOperation::FactStoreSupersede + | ApplicationSurfaceOperation::FactStoreList + | ApplicationSurfaceOperation::FactFeedback + | ApplicationSurfaceOperation::MemoryStatus + | ApplicationSurfaceOperation::SessionRefreshStatus + | ApplicationSurfaceOperation::SessionRefreshCancel + | ApplicationSurfaceOperation::SessionRefreshBegin + | ApplicationSurfaceOperation::MessageSearch + | ApplicationSurfaceOperation::SessionsFor + | ApplicationSurfaceOperation::Workflows + | ApplicationSurfaceOperation::LcmStatus + | ApplicationSurfaceOperation::LcmDoctor + | ApplicationSurfaceOperation::LcmLoadSession + | ApplicationSurfaceOperation::LcmGrep + | ApplicationSurfaceOperation::LcmDescribe + | ApplicationSurfaceOperation::LcmExpand + | ApplicationSurfaceOperation::LcmExpandQuery => { + let retained = + RetainedSurfaceOperation::from_application(operation).ok_or_else(|| { + ApplicationSurfaceAdapterError::invalid_request("operation is not retained") + })?; + decode_retained_request(retained, value) + .map(ApplicationSurfaceRequest::Retained) + .map_err(ApplicationSurfaceAdapterError::invalid_request) + } + ApplicationSurfaceOperation::Context + | ApplicationSurfaceOperation::Node + | ApplicationSurfaceOperation::Impact + | ApplicationSurfaceOperation::Similar + | ApplicationSurfaceOperation::Redundancy + | ApplicationSurfaceOperation::RenamePreview + | ApplicationSurfaceOperation::PortStatus + | ApplicationSurfaceOperation::PortOrder + | ApplicationSurfaceOperation::Todos => match value { + Value::Object(arguments) => Ok(ApplicationSurfaceRequest::GraphTool(arguments)), + _ => Err(ApplicationSurfaceAdapterError::invalid_request(format!( + "invalid arguments: {} expects a JSON object", + operation.mcp_tool_name() + ))), + }, } } diff --git a/crates/tracedecay-daemon-protocol/src/application_surface/invocation.rs b/crates/tracedecay-daemon-protocol/src/application_surface/invocation.rs new file mode 100644 index 0000000000..a0aa8aeb74 --- /dev/null +++ b/crates/tracedecay-daemon-protocol/src/application_surface/invocation.rs @@ -0,0 +1,1045 @@ +//! Execution of one catalog-bound application-surface invocation. +//! +//! Surface adapters hand the executor the operation's request body. Socket +//! clients and daemon-local project servers both decode it here into the +//! closed daemon invocation, so cancellation policy, scope admission, +//! transport-failure observation, and result assembly have one owner. + +use serde::Serialize; +use serde_json::Value; +use tracedecay_contracts::feedback::observations::{ + FeedbackDeliveryRouteV1, FeedbackOperationV1, FeedbackOutcomeV1, FeedbackSourceEventV1, +}; +use tracedecay_contracts::retained_surfaces::RetainedSurfaceOperation; +use tracedecay_contracts::retrieval::PrimitiveRequest; +use tracedecay_contracts::{ + ApplicationEnvelope, ApplicationInvocationBinding, ApplicationInvocationContext, + ApplicationOutcome, ApplicationProblem, ApplicationResponse, CallableCodeSurfaceRequest, + InvocationError, NativeIntegrationSurfaceRequest, PageRequest, PrimitiveCodeSurfaceRequest, + RequestId, ResultContractRef, RetryDirective, SafeDiagnostic, now_micros, + retained_surface_operation_is_effect, retained_surface_outcome_matches_terminal, + retained_surface_problem_matches_terminal, try_now_micros, +}; +use tracedecay_domain::canonical_sha256; +use tracedecay_tool_catalog::{ApplicationSurfaceOperation, BindingSurface}; + +use super::{ + ApplicationSurfaceAdapterError, ApplicationSurfaceRequest, parse_application_surface_request, +}; +use crate::client::{ + DaemonInvocationError, DaemonInvocationExecutor, InvocationCancellationPolicy, +}; +use crate::contract::{ + DAEMON_INVOCATION_PROTOCOL, DAEMON_INVOCATION_REVISION, DaemonInvocationOutcome, + DaemonInvocationProblem, DaemonInvocationRequest, DaemonInvocationResponse, +}; + +impl ApplicationSurfaceRequest { + /// The operation's request body, the shape + /// [`parse_application_surface_invocation_payload`] decodes. + /// + /// Adjacently tagged request families carry the body under `request`; + /// Git reads carry their decoded bounds because their surface body is + /// lossy (defaults and scope names are resolved at parse time). + pub fn into_invocation_payload(self) -> Result { + fn body(request: impl Serialize) -> Result { + serde_json::to_value(request).map_err(ApplicationSurfaceAdapterError::invalid_request) + } + fn tagged_body(request: impl Serialize) -> Result { + body(request)? + .get_mut("request") + .map(Value::take) + .ok_or_else(|| { + ApplicationSurfaceAdapterError::invalid_request( + "tagged surface request has no body", + ) + }) + } + match self { + Self::GitRead(request) => body(request), + Self::GitPreview(request) => body(request), + Self::GitApply(request) => body(request), + Self::GitHubStackSignalExpand(request) => body(request), + Self::NativeIntegration(request) => match request { + NativeIntegrationSurfaceRequest::StackSnapshot(request) => body(request), + NativeIntegrationSurfaceRequest::Preflight(request) => body(request), + NativeIntegrationSurfaceRequest::Approve(request) => body(request), + NativeIntegrationSurfaceRequest::Apply(request) => body(request), + NativeIntegrationSurfaceRequest::Status(request) => body(request), + NativeIntegrationSurfaceRequest::Cancel(request) => body(request), + NativeIntegrationSurfaceRequest::Worktree(request) => tagged_body(request), + }, + Self::Feedback(request) => body(request), + Self::FeedbackAdvisoryCycle(request) => body(request), + Self::FeedbackProximity(request) => body(request), + Self::TestResults(request) => body(request), + Self::CallableCode(request) => match request { + CallableCodeSurfaceRequest::ExactOccurrence(request) => body(request), + CallableCodeSurfaceRequest::PhraseSearch(request) => body(request), + CallableCodeSurfaceRequest::Callees(request) => body(request), + CallableCodeSurfaceRequest::Facets(request) => body(request), + CallableCodeSurfaceRequest::Timeline(request) => body(request), + CallableCodeSurfaceRequest::Declaration(request) + | CallableCodeSurfaceRequest::TypeDefinition(request) + | CallableCodeSurfaceRequest::References(request) => body(request), + }, + Self::PrimitiveCode(request) => match request { + PrimitiveCodeSurfaceRequest::SymbolSearch(request) => body(request), + PrimitiveCodeSurfaceRequest::SignatureSearch(request) => body(request), + PrimitiveCodeSurfaceRequest::Implementations(request) => body(request), + PrimitiveCodeSurfaceRequest::TypeHierarchy(request) => body(request), + PrimitiveCodeSurfaceRequest::Callers(request) => body(request), + }, + Self::Primitive(request) => tagged_body(request), + Self::ObservatoryRead(request) => body(request), + Self::Configuration(request) => tagged_body(request), + Self::ContextScout(request) => tagged_body(request), + Self::SourceEdit(request) => body(request), + Self::SourceEditReconcile(request) => body(request), + Self::SourceEditRollback(request) => body(request), + Self::Retained(request) => body(request), + Self::GraphTool(arguments) => Ok(Value::Object(arguments)), + } + } +} + +/// Decode an invocation payload produced by +/// [`ApplicationSurfaceRequest::into_invocation_payload`]. +/// +/// Source edits carry their already-validated invocation, because their +/// public argument decoding resolves defaults and effect identities. +pub fn parse_application_surface_invocation_payload( + operation: ApplicationSurfaceOperation, + payload: Value, +) -> Result { + match operation { + ApplicationSurfaceOperation::GitStatus + | ApplicationSurfaceOperation::GitDiff + | ApplicationSurfaceOperation::GitHistory + | ApplicationSurfaceOperation::GitBlame + | ApplicationSurfaceOperation::GitHunks => serde_json::from_value(payload) + .map(ApplicationSurfaceRequest::GitRead) + .map_err(ApplicationSurfaceAdapterError::invalid_request), + ApplicationSurfaceOperation::SourceEditReconcile => serde_json::from_value(payload) + .map(ApplicationSurfaceRequest::SourceEditReconcile) + .map_err(ApplicationSurfaceAdapterError::invalid_request), + ApplicationSurfaceOperation::SourceEditRollback => serde_json::from_value(payload) + .map(ApplicationSurfaceRequest::SourceEditRollback) + .map_err(ApplicationSurfaceAdapterError::invalid_request), + operation if super::is_source_edit_operation(operation) => serde_json::from_value(payload) + .map(ApplicationSurfaceRequest::SourceEdit) + .map_err(ApplicationSurfaceAdapterError::invalid_request), + operation if RetainedSurfaceOperation::from_application(operation).is_some() => { + serde_json::from_value(payload) + .map(ApplicationSurfaceRequest::Retained) + .map_err(ApplicationSurfaceAdapterError::invalid_request) + } + _ => parse_application_surface_request(operation, payload), + } +} + +/// Effects past their commit point settle authoritatively; everything else +/// is abandoned on cancellation. +pub fn application_surface_cancellation_policy( + operation: ApplicationSurfaceOperation, +) -> InvocationCancellationPolicy { + if let Some(retained) = RetainedSurfaceOperation::from_application(operation) { + return if retained_surface_operation_is_effect(retained) { + InvocationCancellationPolicy::AuthoritativeEffect + } else { + InvocationCancellationPolicy::ReadOnly + }; + } + match operation { + ApplicationSurfaceOperation::ConfigurationSet + | ApplicationSurfaceOperation::ConfigurationUnset + | ApplicationSurfaceOperation::ConfigurationBatch + | ApplicationSurfaceOperation::ConfigurationProtectedApply + | ApplicationSurfaceOperation::ConfigurationRollbackApply + | ApplicationSurfaceOperation::GitApply + | ApplicationSurfaceOperation::NativeIntegrationApprove + | ApplicationSurfaceOperation::NativeIntegrationApply + | ApplicationSurfaceOperation::NativeIntegrationCancel + | ApplicationSurfaceOperation::ContextScoutPause + | ApplicationSurfaceOperation::ContextScoutResume + | ApplicationSurfaceOperation::ContextScoutCancel + | ApplicationSurfaceOperation::ContextScoutClaim + | ApplicationSurfaceOperation::ContextScoutDelivery + | ApplicationSurfaceOperation::ContextScoutFeedback + | ApplicationSurfaceOperation::StrReplace + | ApplicationSurfaceOperation::MultiStrReplace + | ApplicationSurfaceOperation::InsertAt + | ApplicationSurfaceOperation::AstGrepRewrite + | ApplicationSurfaceOperation::ReplaceSymbol + | ApplicationSurfaceOperation::InsertAtSymbol + | ApplicationSurfaceOperation::MoveSymbol + | ApplicationSurfaceOperation::RenameSymbol + | ApplicationSurfaceOperation::SourceEditReconcile + | ApplicationSurfaceOperation::SourceEditRollback => { + InvocationCancellationPolicy::AuthoritativeEffect + } + _ => InvocationCancellationPolicy::ReadOnly, + } +} + +fn daemon_invocation_request( + request_id: &RequestId, + operation: ApplicationSurfaceOperation, + request: ApplicationSurfaceRequest, + page: PageRequest, + deadline: tracedecay_contracts::Deadline, + cancellation: tracedecay_contracts::CancellationContext, +) -> DaemonInvocationRequest { + let request_id = request_id.as_str(); + let observed_at = now_micros(); + match request { + ApplicationSurfaceRequest::GitRead(request) => DaemonInvocationRequest::git_read( + request_id, + operation, + request, + observed_at, + deadline, + cancellation, + ), + ApplicationSurfaceRequest::GitPreview(request) => DaemonInvocationRequest::git_preview( + request_id, + request, + observed_at, + deadline, + cancellation, + ), + ApplicationSurfaceRequest::GitApply(request) => DaemonInvocationRequest::git_apply( + request_id, + request, + observed_at, + deadline, + cancellation, + ), + ApplicationSurfaceRequest::GitHubStackSignalExpand(request) => { + DaemonInvocationRequest::github_stack_signal_expand( + request_id, + request, + observed_at, + deadline, + cancellation, + ) + } + ApplicationSurfaceRequest::NativeIntegration(request) => { + DaemonInvocationRequest::native_integration( + request_id, + operation, + request, + observed_at, + deadline, + cancellation, + ) + } + ApplicationSurfaceRequest::Feedback(request) => DaemonInvocationRequest::feedback( + request_id, + operation, + request.request_handle, + observed_at, + deadline, + cancellation, + ), + ApplicationSurfaceRequest::FeedbackAdvisoryCycle(request) => { + DaemonInvocationRequest::feedback_advisory_cycle( + request_id, + request.document_uri, + observed_at, + deadline, + cancellation, + ) + } + ApplicationSurfaceRequest::FeedbackProximity(request) => { + DaemonInvocationRequest::feedback_proximity(request_id, request, deadline, cancellation) + } + ApplicationSurfaceRequest::TestResults(_) => DaemonInvocationRequest::primitive( + request_id, + operation, + PrimitiveRequest::RecentTestResults(page), + observed_at, + deadline, + cancellation, + ), + ApplicationSurfaceRequest::CallableCode(request) => DaemonInvocationRequest::callable_code( + request_id, + operation, + request, + page, + observed_at, + deadline, + cancellation, + ), + ApplicationSurfaceRequest::PrimitiveCode(request) => { + DaemonInvocationRequest::primitive_code( + request_id, + operation, + request, + page, + observed_at, + deadline, + cancellation, + ) + } + ApplicationSurfaceRequest::Primitive(request) => DaemonInvocationRequest::primitive( + request_id, + operation, + request, + observed_at, + deadline, + cancellation, + ), + ApplicationSurfaceRequest::ObservatoryRead(request) => { + DaemonInvocationRequest::observatory_read( + request_id, + request, + observed_at, + deadline, + cancellation, + ) + } + ApplicationSurfaceRequest::Configuration(request) => { + DaemonInvocationRequest::configuration( + request_id, + operation, + request, + observed_at, + deadline, + cancellation, + ) + } + ApplicationSurfaceRequest::ContextScout(request) => DaemonInvocationRequest::context_scout( + request_id, + operation, + request, + observed_at, + deadline, + cancellation, + ), + ApplicationSurfaceRequest::SourceEdit(request) => DaemonInvocationRequest::source_edit( + request_id, + request, + observed_at, + deadline, + cancellation, + ), + ApplicationSurfaceRequest::SourceEditReconcile(request) => { + DaemonInvocationRequest::source_edit_reconcile( + request_id, + request, + observed_at, + deadline, + cancellation, + ) + } + ApplicationSurfaceRequest::SourceEditRollback(request) => { + DaemonInvocationRequest::source_edit_rollback( + request_id, + request, + observed_at, + deadline, + cancellation, + ) + } + ApplicationSurfaceRequest::Retained(request) => { + DaemonInvocationRequest::retained_application( + request_id, + request, + observed_at, + deadline, + cancellation, + ) + } + ApplicationSurfaceRequest::GraphTool(arguments) => DaemonInvocationRequest::graph_tool( + request_id, + operation, + arguments, + observed_at, + deadline, + cancellation, + ), + } +} + +/// Execute one surface invocation through `executor`'s daemon transport. +/// +/// An unreachable daemon never saw the request, so it stays a dispatch +/// failure; every other transport failure keeps its exact stage-bearing +/// problem and is reported to the feedback ledger for observable reads. +#[hotpath::measure(label = "application_surface.invoke", future = true)] +pub async fn invoke_application_surface( + executor: &E, + context: ApplicationInvocationContext, + binding: ApplicationInvocationBinding, + payload: Value, +) -> Result { + let (request_id, target, deadline, cancellation) = context.into_parts(); + let (_binding_id, surface, operation, result_contract, page) = binding.into_parts(); + let operation = ApplicationSurfaceOperation::from_surface_name(surface, operation.as_str()) + .ok_or(InvocationError::InvalidRequest)?; + let request = parse_application_surface_invocation_payload(operation, payload) + .map_err(|_| InvocationError::InvalidRequest)?; + if !request.matches(operation) { + return Err(InvocationError::InvalidRequest); + } + let route = application_delivery_route(surface); + let request = daemon_invocation_request( + &request_id, + operation, + request, + page, + deadline.clone(), + cancellation.context(), + ) + .with_resolved_scope(target.resolved().cloned()) + .map_err(|_| InvocationError::InvalidRequest)? + .with_delivery_route(route); + let policy = application_surface_cancellation_policy(operation); + match executor + .invoke_controlled(request, deadline, cancellation, policy) + .await + { + Ok(response) => match RetainedSurfaceOperation::from_application(operation) { + Some(retained) => { + retained_application_response(retained, request_id, result_contract, response) + } + None => application_response(request_id, result_contract, response.outcome), + }, + Err(DaemonInvocationError::Unreachable { + reason_code, + detail, + }) => Err(InvocationError::Unreachable { + reason_code, + detail, + }), + Err(error) => { + observe_transport_failure(executor, &request_id, operation, route, &error).await; + Err(InvocationError::Problem(Box::new( + error.into_application_problem(), + ))) + } + } +} + +async fn observe_transport_failure( + executor: &E, + request_id: &RequestId, + operation: ApplicationSurfaceOperation, + route: FeedbackDeliveryRouteV1, + error: &DaemonInvocationError, +) { + if !application_surface_feedback_is_observable(operation) { + return; + } + let (Ok(subject_digest), Ok(observed_at)) = ( + canonical_sha256(&( + "tracedecay.feedback.transport-observation.v1", + request_id.as_str(), + operation.as_str(), + route, + )), + try_now_micros(), + ) else { + return; + }; + let feedback_operation = application_surface_feedback_operation(operation); + let event = match error { + DaemonInvocationError::Cancelled { .. } => FeedbackSourceEventV1::Cancellation { + operation: feedback_operation, + outcome: FeedbackOutcomeV1::Cancelled, + }, + DaemonInvocationError::TimedOut { .. } => FeedbackSourceEventV1::Cancellation { + operation: feedback_operation, + outcome: FeedbackOutcomeV1::TimedOut, + }, + DaemonInvocationError::Unavailable | DaemonInvocationError::Unreachable { .. } => { + FeedbackSourceEventV1::Delivery { + operation: feedback_operation, + route, + outcome: FeedbackOutcomeV1::Unavailable, + item_count: 0, + duration_micros: None, + } + } + }; + let _ = executor + .observe_feedback(subject_digest, observed_at, event) + .await; +} + +pub fn application_delivery_route(surface: BindingSurface) -> FeedbackDeliveryRouteV1 { + match surface { + BindingSurface::Cli => FeedbackDeliveryRouteV1::Cli, + BindingSurface::Mcp => FeedbackDeliveryRouteV1::Mcp, + BindingSurface::Http | BindingSurface::Dashboard => FeedbackDeliveryRouteV1::Http, + BindingSurface::Lsp => FeedbackDeliveryRouteV1::Lsp, + } +} + +/// Assemble the daemon's answer to a surface invocation. +pub fn application_response( + request_id: RequestId, + result_contract: ResultContractRef, + outcome: DaemonInvocationOutcome, +) -> Result { + let envelope = match outcome { + DaemonInvocationOutcome::GitRead { scope, result } + | DaemonInvocationOutcome::Feedback { scope, result } + | DaemonInvocationOutcome::Primitive { scope, result } + | DaemonInvocationOutcome::CallableCode { scope, result } + | DaemonInvocationOutcome::ObservatoryRead { scope, result } => { + ApplicationEnvelope::evidence( + result_contract, + request_id, + scope, + result.into_application(), + ) + } + DaemonInvocationOutcome::GitPreview { scope, preview } => ApplicationEnvelope::preview( + result_contract, + request_id, + scope, + preview + .into_application_result() + .map_err(|_| InvocationError::Unavailable)?, + ), + DaemonInvocationOutcome::GitApply { scope, effect } => ApplicationEnvelope::effect( + result_contract, + request_id, + scope, + effect + .into_application_result() + .map_err(|_| InvocationError::Unavailable)?, + ), + DaemonInvocationOutcome::Configuration { scope, outcome } + | DaemonInvocationOutcome::GitHubStackSignalExpand { scope, outcome } + | DaemonInvocationOutcome::NativeIntegration { scope, outcome } + | DaemonInvocationOutcome::ContextScout { scope, outcome } => ApplicationEnvelope { + contract: result_contract, + request_id, + scope, + outcome, + touched_files: Vec::new(), + code_graph: None, + analytics: None, + }, + DaemonInvocationOutcome::SourceEdit { scope, result } => ApplicationEnvelope { + contract: result_contract, + request_id, + scope, + outcome: ApplicationOutcome::Result( + serde_json::to_value(result).map_err(|_| InvocationError::Unavailable)?, + ), + touched_files: Vec::new(), + code_graph: None, + analytics: None, + }, + DaemonInvocationOutcome::GraphTool { scope, completion } => ApplicationEnvelope { + contract: result_contract, + request_id, + scope, + outcome: ApplicationOutcome::Result( + completion + .result + .result_value() + .map_err(|_| InvocationError::Unavailable)?, + ), + touched_files: completion.touched_files, + code_graph: completion.code_graph, + analytics: completion.analytics, + }, + // The daemon already resolved this invocation to a typed problem + // (e.g. `configuration.conflict`); carry it whole so surface adapters + // republish that diagnostic instead of refabricating a generic one. + DaemonInvocationOutcome::ApplicationProblem { problem } => { + return Err(InvocationError::Problem(Box::new(problem))); + } + DaemonInvocationOutcome::Problem { problem } => { + return Err(InvocationError::Problem(Box::new( + daemon_problem_into_application(problem), + ))); + } + _ => return Err(InvocationError::Unavailable), + }; + Ok(ApplicationResponse::unary(envelope)) +} + +/// Assemble a retained terminal only when it still belongs to the selected +/// operation, request, and authenticated scope. +fn retained_application_response( + operation: RetainedSurfaceOperation, + request_id: RequestId, + result_contract: ResultContractRef, + response: DaemonInvocationResponse, +) -> Result { + let unavailable = |message: &str| { + InvocationError::Problem(Box::new(ApplicationProblem::unavailable(SafeDiagnostic { + code: "application.surface.invalid_response".to_owned(), + message: message.to_owned(), + }))) + }; + if response.protocol != DAEMON_INVOCATION_PROTOCOL + || response.revision != DAEMON_INVOCATION_REVISION + || response.request_id != request_id.as_str() + { + return Err(unavailable( + "The daemon returned an invalid retained application envelope", + )); + } + let invalid = || unavailable("The daemon returned an invalid retained application response"); + match response.outcome { + DaemonInvocationOutcome::RetainedApplication { scope, outcome } + if retained_surface_outcome_matches_terminal( + operation, + &request_id, + &scope, + &outcome, + ) => + { + Ok(ApplicationResponse::unary(ApplicationEnvelope { + contract: result_contract, + request_id, + scope, + outcome: application_outcome_value(outcome).map_err(|_| invalid())?, + touched_files: Vec::new(), + code_graph: None, + analytics: None, + })) + } + DaemonInvocationOutcome::RetainedApplicationProblem { scope, problem } + if retained_surface_problem_matches_terminal( + operation, + &request_id, + Some(&scope), + &problem, + ) => + { + Err(InvocationError::Problem(Box::new(problem))) + } + DaemonInvocationOutcome::ApplicationProblem { problem } + if retained_surface_problem_matches_terminal( + operation, + &request_id, + None, + &problem, + ) => + { + Err(InvocationError::Problem(Box::new(problem))) + } + DaemonInvocationOutcome::Problem { problem } => Err(InvocationError::Problem(Box::new( + retained_daemon_problem(problem), + ))), + _ => Err(invalid()), + } +} + +fn retained_daemon_problem(problem: DaemonInvocationProblem) -> ApplicationProblem { + let diagnostic = |code: &str, message: &str| SafeDiagnostic { + code: code.to_owned(), + message: message.to_owned(), + }; + match problem { + DaemonInvocationProblem::InvalidRequest | DaemonInvocationProblem::UnsupportedRevision => { + ApplicationProblem::invalid_request_without_action( + "application.surface.invalid_request", + "The daemon rejected the retained application request", + ) + } + DaemonInvocationProblem::NotFoundOrNotAuthorized => { + ApplicationProblem::not_found_or_not_authorized(RetryDirective::Never) + } + DaemonInvocationProblem::ResetRequired => ApplicationProblem::reset_required(diagnostic( + "application.surface.reset_required", + "The retained application store requires an explicit reset", + )), + DaemonInvocationProblem::ApplicationContractViolation => { + ApplicationProblem::unavailable(diagnostic( + "application.surface.contract_violation", + "The retained application result violated its canonical contract", + )) + } + DaemonInvocationProblem::Unavailable => ApplicationProblem::unavailable(diagnostic( + "application.surface.unavailable", + "The retained application service is unavailable", + )), + } +} + +/// Re-encode an outcome's typed payload as its JSON carrier. +pub fn application_outcome_value( + outcome: ApplicationOutcome, +) -> Result, serde_json::Error> { + fn payload(payload: Option) -> Result, serde_json::Error> { + payload.map(serde_json::to_value).transpose() + } + Ok(match outcome { + ApplicationOutcome::Evidence(packet) => { + ApplicationOutcome::Evidence(tracedecay_contracts::EvidencePacket { + temporal: packet.temporal, + authority: packet.authority, + evidence_authorities: packet.evidence_authorities, + coverage: packet.coverage, + omissions: packet.omissions, + scores: packet.scores, + contributions: packet.contributions, + page: packet.page, + execution: packet.execution, + payload: payload(packet.payload)?, + }) + } + ApplicationOutcome::Preview(preview) => { + ApplicationOutcome::Preview(tracedecay_contracts::PreviewResult { + preview_id: preview.preview_id, + preview_digest: preview.preview_digest, + effect_class: preview.effect_class, + authority: preview.authority, + expected_state: preview.expected_state, + execution: preview.execution, + payload: payload(preview.payload)?, + }) + } + ApplicationOutcome::Effect(effect) => { + ApplicationOutcome::Effect(tracedecay_contracts::EffectResult { + effect_id: effect.effect_id, + effect_class: effect.effect_class, + idempotency_key: effect.idempotency_key, + authority: effect.authority, + expected_state: effect.expected_state, + execution: effect.execution, + reconciliation: effect.reconciliation, + receipt: effect.receipt, + payload: payload(effect.payload)?, + }) + } + ApplicationOutcome::Result(result) => { + ApplicationOutcome::Result(serde_json::to_value(result)?) + } + }) +} + +fn daemon_problem_into_application(problem: DaemonInvocationProblem) -> ApplicationProblem { + let diagnostic = |code: &str, message: &str| SafeDiagnostic { + code: code.to_owned(), + message: message.to_owned(), + }; + match problem { + DaemonInvocationProblem::InvalidRequest | DaemonInvocationProblem::UnsupportedRevision => { + ApplicationProblem::invalid_request_without_action( + "application.surface.invalid_request", + "The daemon rejected the application request", + ) + } + DaemonInvocationProblem::NotFoundOrNotAuthorized => { + ApplicationProblem::not_found_or_not_authorized(RetryDirective::Never) + } + DaemonInvocationProblem::ResetRequired => ApplicationProblem::reset_required(diagnostic( + "application.surface.reset_required", + "The application store requires an explicit reset", + )), + DaemonInvocationProblem::ApplicationContractViolation => { + ApplicationProblem::unavailable(diagnostic( + "application.surface.contract_violation", + "The application result violated its canonical contract", + )) + } + DaemonInvocationProblem::Unavailable => ApplicationProblem::unavailable(diagnostic( + "application.surface.unavailable", + "The application service for this operation is unavailable", + )), + } +} + +/// The feedback-ledger operation an application surface reports under. +pub fn application_surface_feedback_operation( + operation: ApplicationSurfaceOperation, +) -> FeedbackOperationV1 { + match operation { + ApplicationSurfaceOperation::FeedbackDiagnostics => { + FeedbackOperationV1::FeedbackDiagnostics + } + ApplicationSurfaceOperation::FeedbackGet => FeedbackOperationV1::FeedbackGet, + ApplicationSurfaceOperation::FeedbackExpand => FeedbackOperationV1::FeedbackExpand, + ApplicationSurfaceOperation::FeedbackList => FeedbackOperationV1::FeedbackList, + ApplicationSurfaceOperation::FeedbackAdvisoryCycle => FeedbackOperationV1::FeedbackCycle, + ApplicationSurfaceOperation::FeedbackProximity => FeedbackOperationV1::Proximity, + ApplicationSurfaceOperation::FeedbackImpact => FeedbackOperationV1::PrimitiveImpact, + ApplicationSurfaceOperation::AffectedTests => FeedbackOperationV1::PrimitiveAffectedTests, + ApplicationSurfaceOperation::TestResults => FeedbackOperationV1::PrimitiveTestResults, + ApplicationSurfaceOperation::GitStatus + | ApplicationSurfaceOperation::GitDiff + | ApplicationSurfaceOperation::GitHistory + | ApplicationSurfaceOperation::GitBlame + | ApplicationSurfaceOperation::GitHunks + | ApplicationSurfaceOperation::GitPreview + | ApplicationSurfaceOperation::GitApply + | ApplicationSurfaceOperation::GitHubStackSignalExpand + | ApplicationSurfaceOperation::NativeIntegrationStackSnapshot + | ApplicationSurfaceOperation::NativeIntegrationPreflight + | ApplicationSurfaceOperation::NativeIntegrationApprove + | ApplicationSurfaceOperation::NativeIntegrationApply + | ApplicationSurfaceOperation::NativeIntegrationStatus + | ApplicationSurfaceOperation::NativeIntegrationCancel + | ApplicationSurfaceOperation::NativeIntegrationWorktreeInventory + | ApplicationSurfaceOperation::NativeIntegrationWorktreeInspect + | ApplicationSurfaceOperation::NativeIntegrationWorktreeConfirm + | ApplicationSurfaceOperation::NativeIntegrationWorktreeRemove + | ApplicationSurfaceOperation::NativeIntegrationWorktreeReconcile + | ApplicationSurfaceOperation::CodeExactOccurrence + | ApplicationSurfaceOperation::CodePhraseSearch + | ApplicationSurfaceOperation::CodeSymbolSearch + | ApplicationSurfaceOperation::CodeSignatureSearch + | ApplicationSurfaceOperation::CodeImplementations + | ApplicationSurfaceOperation::CodeTypeHierarchy + | ApplicationSurfaceOperation::CodeCallers + | ApplicationSurfaceOperation::CodeCallees + | ApplicationSurfaceOperation::CodeFacets + | ApplicationSurfaceOperation::CodeTimeline + | ApplicationSurfaceOperation::CodeDeclaration + | ApplicationSurfaceOperation::CodeTypeDefinition + | ApplicationSurfaceOperation::CodeReferences + | ApplicationSurfaceOperation::SessionLookup + | ApplicationSurfaceOperation::QualifiedName + | ApplicationSurfaceOperation::CallChain + | ApplicationSurfaceOperation::FileDependents + | ApplicationSurfaceOperation::SourceLines + | ApplicationSurfaceOperation::SourceBody + | ApplicationSurfaceOperation::SourceOutline + | ApplicationSurfaceOperation::ModuleApi + | ApplicationSurfaceOperation::Context + | ApplicationSurfaceOperation::Node + | ApplicationSurfaceOperation::Impact + | ApplicationSurfaceOperation::Similar + | ApplicationSurfaceOperation::Redundancy + | ApplicationSurfaceOperation::RenamePreview + | ApplicationSurfaceOperation::PortStatus + | ApplicationSurfaceOperation::PortOrder + | ApplicationSurfaceOperation::Todos + | ApplicationSurfaceOperation::HealthRead + | ApplicationSurfaceOperation::HealthDelta + | ApplicationSurfaceOperation::StorageStatus + | ApplicationSurfaceOperation::DiagnosticsRead + | ApplicationSurfaceOperation::ObservatoryRead + | ApplicationSurfaceOperation::ConfigurationList + | ApplicationSurfaceOperation::ConfigurationGet + | ApplicationSurfaceOperation::ConfigurationSet + | ApplicationSurfaceOperation::ConfigurationUnset + | ApplicationSurfaceOperation::ConfigurationBatch + | ApplicationSurfaceOperation::ConfigurationObservedState + | ApplicationSurfaceOperation::ConfigurationProtectedPreview + | ApplicationSurfaceOperation::ConfigurationProtectedApply + | ApplicationSurfaceOperation::ConfigurationRollbackPreview + | ApplicationSurfaceOperation::ConfigurationRollbackApply + | ApplicationSurfaceOperation::ConfigurationAudit + | ApplicationSurfaceOperation::ContextScoutStatus + | ApplicationSurfaceOperation::ContextScoutRecent + | ApplicationSurfaceOperation::ContextScoutExplain + | ApplicationSurfaceOperation::ContextScoutCapability + | ApplicationSurfaceOperation::ContextScoutBudget + | ApplicationSurfaceOperation::ContextScoutPause + | ApplicationSurfaceOperation::ContextScoutResume + | ApplicationSurfaceOperation::ContextScoutCancel + | ApplicationSurfaceOperation::ContextScoutClaim + | ApplicationSurfaceOperation::ContextScoutDelivery + | ApplicationSurfaceOperation::ContextScoutFeedback + | ApplicationSurfaceOperation::StrReplace + | ApplicationSurfaceOperation::MultiStrReplace + | ApplicationSurfaceOperation::InsertAt + | ApplicationSurfaceOperation::AstGrepRewrite + | ApplicationSurfaceOperation::ReplaceSymbol + | ApplicationSurfaceOperation::InsertAtSymbol + | ApplicationSurfaceOperation::MoveSymbol + | ApplicationSurfaceOperation::RenameSymbol + | ApplicationSurfaceOperation::SourceEditReconcile + | ApplicationSurfaceOperation::SourceEditRollback + | ApplicationSurfaceOperation::FactStoreCurate + | ApplicationSurfaceOperation::FactStoreAdd + | ApplicationSurfaceOperation::FactStoreSearch + | ApplicationSurfaceOperation::FactStoreProbe + | ApplicationSurfaceOperation::FactStoreRelated + | ApplicationSurfaceOperation::FactStoreReason + | ApplicationSurfaceOperation::FactStoreContradict + | ApplicationSurfaceOperation::FactStoreGet + | ApplicationSurfaceOperation::FactStoreUpdate + | ApplicationSurfaceOperation::FactStoreRemove + | ApplicationSurfaceOperation::FactStoreSupersede + | ApplicationSurfaceOperation::FactStoreList + | ApplicationSurfaceOperation::FactFeedback + | ApplicationSurfaceOperation::MemoryStatus + | ApplicationSurfaceOperation::SessionRefreshStatus + | ApplicationSurfaceOperation::SessionRefreshCancel + | ApplicationSurfaceOperation::SessionRefreshBegin + | ApplicationSurfaceOperation::MessageSearch + | ApplicationSurfaceOperation::SessionsFor + | ApplicationSurfaceOperation::Workflows + | ApplicationSurfaceOperation::LcmStatus + | ApplicationSurfaceOperation::LcmDoctor + | ApplicationSurfaceOperation::LcmLoadSession + | ApplicationSurfaceOperation::LcmGrep + | ApplicationSurfaceOperation::LcmDescribe + | ApplicationSurfaceOperation::LcmExpand + | ApplicationSurfaceOperation::LcmExpandQuery => FeedbackOperationV1::FeedbackCycle, + } +} + +/// Surfaces whose rejections and transport failures feed the feedback ledger. +pub fn application_surface_feedback_is_observable(operation: ApplicationSurfaceOperation) -> bool { + matches!( + operation, + ApplicationSurfaceOperation::FeedbackDiagnostics + | ApplicationSurfaceOperation::FeedbackGet + | ApplicationSurfaceOperation::FeedbackExpand + | ApplicationSurfaceOperation::FeedbackList + | ApplicationSurfaceOperation::FeedbackAdvisoryCycle + | ApplicationSurfaceOperation::FeedbackProximity + | ApplicationSurfaceOperation::FeedbackImpact + | ApplicationSurfaceOperation::AffectedTests + | ApplicationSurfaceOperation::TestResults + | ApplicationSurfaceOperation::SessionLookup + | ApplicationSurfaceOperation::QualifiedName + | ApplicationSurfaceOperation::CallChain + | ApplicationSurfaceOperation::FileDependents + | ApplicationSurfaceOperation::SourceLines + | ApplicationSurfaceOperation::SourceBody + | ApplicationSurfaceOperation::SourceOutline + | ApplicationSurfaceOperation::ModuleApi + | ApplicationSurfaceOperation::HealthRead + | ApplicationSurfaceOperation::HealthDelta + | ApplicationSurfaceOperation::StorageStatus + | ApplicationSurfaceOperation::DiagnosticsRead + ) +} + +#[cfg(test)] +mod tests { + use serde_json::json; + use tracedecay_contracts::{ + ApplicationProblem, InvocationError, LegalAction, RequestId, ResultContractRef, + RetryDirective, + }; + use tracedecay_tool_catalog::{ApplicationSurfaceOperation, SchemaId}; + + use super::{application_response, parse_application_surface_invocation_payload}; + use crate::application_surface::parse_application_surface_request; + use crate::contract::{DaemonInvocationOutcome, DaemonInvocationProblem}; + + #[test] + fn caller_bodies_encode_to_literal_payloads_the_executor_accepts() { + for (operation, body, payload) in [ + ( + ApplicationSurfaceOperation::ConfigurationList, + json!({}), + json!({}), + ), + ( + ApplicationSurfaceOperation::FeedbackList, + json!({"request_handle": "feedback.handle.v1"}), + json!({"request_handle": "feedback.handle.v1"}), + ), + ( + ApplicationSurfaceOperation::GitStatus, + json!({}), + json!({ + "max_entries": 1000, + "max_bytes": 4_194_304, + "request": {"query": "status"} + }), + ), + ( + ApplicationSurfaceOperation::GitHistory, + json!({"count": 5, "path": "src/lib.rs"}), + json!({ + "max_entries": 1000, + "max_bytes": 4_194_304, + "request": { + "query": "history", + "max_count": 5, + "path": "src/lib.rs", + "follow": false, + "first_parent": false + } + }), + ), + ( + ApplicationSurfaceOperation::StorageStatus, + json!({"include_details": false}), + json!({"include_details": false}), + ), + ( + ApplicationSurfaceOperation::ObservatoryRead, + json!({}), + json!({"window_days": 14}), + ), + ] { + let encoded = parse_application_surface_request(operation, body) + .unwrap_or_else(|error| panic!("{operation:?} body: {error}")) + .into_invocation_payload() + .expect("payload"); + assert_eq!(encoded, payload, "{operation:?}"); + let decoded = parse_application_surface_invocation_payload(operation, payload) + .unwrap_or_else(|error| panic!("{operation:?} payload: {error}")); + assert!(decoded.matches(operation), "{operation:?}"); + } + } + + #[test] + fn configuration_payload_is_the_envelope_stripped_body() { + let payload = parse_application_surface_request( + ApplicationSurfaceOperation::ConfigurationGet, + json!({"key": "mcp.tool_timings"}), + ) + .expect("get") + .into_invocation_payload() + .expect("payload"); + assert_eq!(payload, json!({"key": "mcp.tool_timings"})); + assert!( + parse_application_surface_invocation_payload( + ApplicationSurfaceOperation::ConfigurationGet, + json!({"operation": "get", "request": {"key": "mcp.tool_timings"}}), + ) + .is_err(), + "the tagged envelope is not an invocation payload" + ); + } + + #[test] + fn feedback_payloads_validate_handles_at_the_executor() { + assert!( + parse_application_surface_invocation_payload( + ApplicationSurfaceOperation::FeedbackGet, + json!({"request_handle": " leading"}), + ) + .is_err() + ); + } + + #[test] + fn daemon_reset_response_remains_an_authoritative_typed_problem() { + let error = application_response( + RequestId::new("request.daemon-client.reset").expect("request"), + ResultContractRef::new( + SchemaId::new("schema.test.daemon-client-reset-result").expect("schema"), + 1, + ) + .expect("contract"), + DaemonInvocationOutcome::Problem { + problem: DaemonInvocationProblem::ResetRequired, + }, + ) + .expect_err("reset-required must not become a successful response"); + + let InvocationError::Problem(problem) = error else { + panic!("reset-required must remain an authoritative typed problem"); + }; + let ApplicationProblem::ResetRequired { + retry, + legal_actions, + .. + } = *problem + else { + panic!("reset-required must keep its terminal kind"); + }; + assert_eq!(retry, RetryDirective::Never); + assert_eq!(legal_actions, vec![LegalAction::Reset]); + } +} diff --git a/crates/tracedecay-daemon-protocol/src/application_surface/retained.rs b/crates/tracedecay-daemon-protocol/src/application_surface/retained.rs new file mode 100644 index 0000000000..57e0638338 --- /dev/null +++ b/crates/tracedecay-daemon-protocol/src/application_surface/retained.rs @@ -0,0 +1,162 @@ +//! Retained memory, session, and workflow request bodies. + +use tracedecay_contracts::retained_surfaces::{ + FactFeedbackRequestV1, FactStoreAddRequestV1, FactStoreContradictRequestV1, + FactStoreCurateRequestV1, FactStoreGetRequestV1, FactStoreListRequestV1, + FactStoreProbeRequestV1, FactStoreReasonRequestV1, FactStoreRelatedRequestV1, + FactStoreRemoveRequestV1, FactStoreSearchRequestV1, FactStoreSupersedeRequestV1, + FactStoreUpdateRequestV1, LcmDescribeRequestV1, LcmDoctorRequestV1, LcmExpandQueryRequestV1, + LcmExpandRequestV1, LcmGrepRequestV1, LcmLoadSessionRequestV1, LcmStatusRequestV1, + MemoryStatusRequestV1, MessageSearchRequestV1, RetainedSurfaceOperation, + RetainedSurfaceRequestV1, SessionRefreshActionRequestV1, SessionRefreshActionV1, + SessionRefreshRequestV1, SessionsForRequestV1, WorkflowsRequestV1, +}; + +/// Decode one retained operation body into its typed request. +/// +/// HTTP decodes the route body directly; MCP and the `tracedecay tool` CLI +/// decode the transport-normalized arguments through the same function, so +/// every surface lands on one canonical request. The returned error carries +/// the exact serde diagnostic (unknown field, unknown enum variant with the +/// admitted values, wrong type) so every dispatch surface can hand the caller +/// a corrective message instead of a blank "invalid request". +#[hotpath::measure(label = "application_surface.retained.decode")] +pub fn decode_retained_request( + operation: RetainedSurfaceOperation, + body: serde_json::Value, +) -> Result { + macro_rules! decode { + ($request:ty, $variant:ident) => { + serde_path_to_error::deserialize::<_, $request>(body) + .map(RetainedSurfaceRequestV1::$variant) + .map_err(named_argument_error) + }; + } + match operation { + RetainedSurfaceOperation::FactStoreCurate => { + decode!(FactStoreCurateRequestV1, FactStoreCurate) + } + RetainedSurfaceOperation::FactStoreAdd => { + decode!(FactStoreAddRequestV1, FactStoreAdd) + } + RetainedSurfaceOperation::FactStoreSearch => { + decode!(FactStoreSearchRequestV1, FactStoreSearch) + } + RetainedSurfaceOperation::FactStoreProbe => { + decode!(FactStoreProbeRequestV1, FactStoreProbe) + } + RetainedSurfaceOperation::FactStoreRelated => { + decode!(FactStoreRelatedRequestV1, FactStoreRelated) + } + RetainedSurfaceOperation::FactStoreReason => { + decode!(FactStoreReasonRequestV1, FactStoreReason) + } + RetainedSurfaceOperation::FactStoreContradict => { + decode!(FactStoreContradictRequestV1, FactStoreContradict) + } + RetainedSurfaceOperation::FactStoreGet => { + decode!(FactStoreGetRequestV1, FactStoreGet) + } + RetainedSurfaceOperation::FactStoreUpdate => { + decode!(FactStoreUpdateRequestV1, FactStoreUpdate) + } + RetainedSurfaceOperation::FactStoreRemove => { + decode!(FactStoreRemoveRequestV1, FactStoreRemove) + } + RetainedSurfaceOperation::FactStoreSupersede => { + decode!(FactStoreSupersedeRequestV1, FactStoreSupersede) + } + RetainedSurfaceOperation::FactStoreList => { + decode!(FactStoreListRequestV1, FactStoreList) + } + RetainedSurfaceOperation::FactFeedback => decode!(FactFeedbackRequestV1, FactFeedback), + RetainedSurfaceOperation::MemoryStatus => decode!(MemoryStatusRequestV1, MemoryStatus), + RetainedSurfaceOperation::SessionRefreshStatus => { + decode_session_refresh(body, SessionRefreshActionV1::Status) + } + RetainedSurfaceOperation::SessionRefreshCancel => { + decode_session_refresh(body, SessionRefreshActionV1::Cancel) + } + RetainedSurfaceOperation::SessionRefreshBegin => { + decode_session_refresh(body, SessionRefreshActionV1::Begin) + } + RetainedSurfaceOperation::MessageSearch => decode!(MessageSearchRequestV1, MessageSearch), + RetainedSurfaceOperation::SessionsFor => decode!(SessionsForRequestV1, SessionsFor), + RetainedSurfaceOperation::Workflows => decode!(WorkflowsRequestV1, Workflows), + RetainedSurfaceOperation::LcmStatus => decode!(LcmStatusRequestV1, LcmStatus), + RetainedSurfaceOperation::LcmDoctor => decode!(LcmDoctorRequestV1, LcmDoctor), + RetainedSurfaceOperation::LcmLoadSession => { + decode!(LcmLoadSessionRequestV1, LcmLoadSession) + } + RetainedSurfaceOperation::LcmGrep => decode!(LcmGrepRequestV1, LcmGrep), + RetainedSurfaceOperation::LcmDescribe => decode!(LcmDescribeRequestV1, LcmDescribe), + RetainedSurfaceOperation::LcmExpand => decode!(LcmExpandRequestV1, LcmExpand), + RetainedSurfaceOperation::LcmExpandQuery => { + decode!(LcmExpandQueryRequestV1, LcmExpandQuery) + } + } +} + +fn decode_session_refresh( + body: serde_json::Value, + action: SessionRefreshActionV1, +) -> Result { + let request = serde_path_to_error::deserialize::<_, SessionRefreshActionRequestV1>(body) + .map_err(named_argument_error)?; + Ok(RetainedSurfaceRequestV1::SessionRefresh( + SessionRefreshRequestV1::with_action(action, request), + )) +} + +/// Prefix the serde diagnostic with the offending argument path, so the +/// corrective message names the argument even for wrong-type errors, which +/// serde alone reports without the field. +fn named_argument_error(error: serde_path_to_error::Error) -> serde_json::Error { + let path = error.path().to_string(); + let inner = error.into_inner(); + if path == "." { + inner + } else { + serde::de::Error::custom(format!("{path}: {inner}")) + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn route_selected_session_refresh_rejects_embedded_action() { + assert!( + decode_retained_request( + RetainedSurfaceOperation::SessionRefreshStatus, + json!({ "action": "status" }), + ) + .is_err() + ); + } + + #[test] + fn fact_store_curate_rejects_caller_owned_authority() { + for forbidden in [ + "operations", + "proposal_id", + "approve", + "apply", + "run_id", + "task", + ] { + let mut value = serde_json::Map::new(); + value.insert(forbidden.to_owned(), serde_json::Value::Bool(true)); + assert!( + decode_retained_request( + RetainedSurfaceOperation::FactStoreCurate, + serde_json::Value::Object(value), + ) + .is_err() + ); + } + } +} diff --git a/crates/tracedecay-daemon-protocol/src/application_surface/source_edit.rs b/crates/tracedecay-daemon-protocol/src/application_surface/source_edit.rs new file mode 100644 index 0000000000..aa1fef7789 --- /dev/null +++ b/crates/tracedecay-daemon-protocol/src/application_surface/source_edit.rs @@ -0,0 +1,366 @@ +//! Source-edit tool arguments decoded into their typed daemon invocations. +//! +//! The messages are the public argument diagnostics every transport reports. + +use serde_json::{Value, json}; +use tracedecay_contracts::{ + EffectId, IdempotencyKey, RenameSymbolBindingV1, RenameSymbolSurfaceRequestV1, + SourceEditInvocationV1, SourceEditKind, SourceEditReconciliationDispositionV1, + SourceEditReconciliationInvocationV1, SourceEditRequest, SourceEditRollbackInvocationV1, +}; +use tracedecay_domain::ManifestDigest; +use tracedecay_tool_catalog::ApplicationSurfaceOperation; + +use super::ApplicationSurfaceRequest; + +fn missing_required_param(name: &str) -> String { + format!("missing required parameter: {name}") +} + +fn required_str<'a>(args: &'a Value, name: &str) -> Result<&'a str, String> { + args.get(name) + .and_then(Value::as_str) + .ok_or_else(|| missing_required_param(name)) +} + +fn required_string(args: &Value, name: &str) -> Result { + required_str(args, name).map(str::to_owned) +} + +fn identity_error(error: impl std::fmt::Display) -> String { + format!("invalid source edit effect identity: {error}") +} + +fn flag(args: &Value, name: &str, default: bool) -> bool { + args.get(name).and_then(Value::as_bool).unwrap_or(default) +} + +fn optional_idempotency_key(args: &Value) -> Result, String> { + args.get("idempotency_key") + .map(|value| { + let value = value + .as_str() + .ok_or_else(|| missing_required_param("idempotency_key"))?; + IdempotencyKey::new(value).map_err(|error| format!("invalid idempotency_key: {error}")) + }) + .transpose() +} + +fn optional_expected_state(args: &Value) -> Result, String> { + args.get("expected_state") + .map(|value| { + let value = value + .as_str() + .ok_or_else(|| missing_required_param("expected_state"))?; + ManifestDigest::new(value).map_err(|error| format!("invalid expected_state: {error}")) + }) + .transpose() +} + +fn edit_request( + operation: ApplicationSurfaceOperation, + args: &Value, +) -> Result { + let dry_run = flag(args, "dry_run", false); + let verify = flag(args, "verify", false); + Ok(match operation { + ApplicationSurfaceOperation::StrReplace => SourceEditRequest::StrReplace { + path: required_string(args, "path")?, + old_str: required_string(args, "old_str")?, + new_str: required_string(args, "new_str")?, + dry_run, + verify, + }, + ApplicationSurfaceOperation::MultiStrReplace => { + let path = required_string(args, "path")?; + let replacements = args + .get("replacements") + .and_then(Value::as_array) + .ok_or_else(|| missing_required_param("replacements"))?; + let parsed: Vec<(String, String)> = replacements + .iter() + .filter_map(|pair| match pair.as_array()?.as_slice() { + [old, new] => Some((old.as_str()?.to_owned(), new.as_str()?.to_owned())), + _ => None, + }) + .collect(); + if parsed.len() != replacements.len() { + return Err("each replacement must be an array of exactly 2 strings".to_owned()); + } + SourceEditRequest::MultiStrReplace { + path, + replacements: parsed, + dry_run, + verify, + } + } + ApplicationSurfaceOperation::InsertAt => SourceEditRequest::InsertAt { + path: required_string(args, "path")?, + anchor: required_string(args, "anchor")?, + content: required_string(args, "content")?, + before: flag(args, "before", false), + dry_run, + verify, + }, + ApplicationSurfaceOperation::AstGrepRewrite => SourceEditRequest::AstGrepRewrite { + path: required_string(args, "path")?, + pattern: required_string(args, "pattern")?, + rewrite: required_string(args, "rewrite")?, + dry_run, + verify, + }, + ApplicationSurfaceOperation::ReplaceSymbol => SourceEditRequest::ReplaceSymbol { + symbol: required_string(args, "symbol")?, + new_source: required_string(args, "new_source")?, + dry_run, + verify, + }, + ApplicationSurfaceOperation::InsertAtSymbol => SourceEditRequest::InsertAtSymbol { + symbol: required_string(args, "symbol")?, + content: required_string(args, "content")?, + position: args + .get("position") + .and_then(Value::as_str) + .unwrap_or("after") + .to_owned(), + dry_run, + verify, + }, + // The impact report is the product; applying is opt-in. + ApplicationSurfaceOperation::MoveSymbol => SourceEditRequest::MoveSymbol { + symbol: required_string(args, "symbol")?, + dest_file: required_string(args, "dest_file")?, + dry_run: flag(args, "dry_run", true), + update_references: flag(args, "update_references", false), + }, + ApplicationSurfaceOperation::RenameSymbol => { + let mut input = args.clone(); + if let Some(object) = input.as_object_mut() { + object.remove("format"); + object.remove("__mcp_request_id"); + } + let request: RenameSymbolSurfaceRequestV1 = serde_json::from_value(input) + .map_err(|error| format!("invalid source edit request: {error}"))?; + SourceEditRequest::RenameSymbol { + binding: RenameSymbolBindingV1 { + node_id: request.node_id, + qualified_name: request.qualified_name, + kind: request.kind, + file: request.file, + old_name: request.old_name, + accepted_preview: request.accepted_preview, + }, + new_name: request.new_name, + dry_run: request.dry_run, + verify: request.verify, + } + } + _ => return Err(format!("{} is not a source edit", operation.as_str())), + }) +} + +fn rollback(args: &Value) -> Result { + if args.get("confirm").and_then(Value::as_bool) != Some(true) { + return Err("source edit rollback requires confirm=true from the caller after it checks the receipt; do not pause for a human".to_owned()); + } + let effect_id = EffectId::new(required_str(args, "effect_id")?).map_err(identity_error)?; + let original_idempotency_key = + IdempotencyKey::new(required_str(args, "original_idempotency_key")?) + .map_err(identity_error)?; + let idempotency_key = + IdempotencyKey::new(required_str(args, "idempotency_key")?).map_err(identity_error)?; + if idempotency_key == original_idempotency_key { + return Err("rollback idempotency key must differ from the original edit key".to_owned()); + } + let original_input_digest = ManifestDigest::new(required_str(args, "original_input_digest")?) + .map_err(identity_error)?; + let expected_state = + ManifestDigest::new(required_str(args, "expected_state")?).map_err(identity_error)?; + Ok(SourceEditRollbackInvocationV1 { + effect_id, + original_idempotency_key, + idempotency_key, + original_input_digest, + expected_state, + }) +} + +fn reconcile(args: &Value) -> Result { + if args.get("confirm").and_then(Value::as_bool) != Some(true) { + return Err("source edit reconciliation requires confirm=true from the caller after it inspects the file; do not pause for a human".to_owned()); + } + let kind = serde_json::from_value::(json!(required_str(args, "kind")?)) + .map_err(|error| format!("invalid source edit kind: {error}"))?; + let effect_id = EffectId::new(required_str(args, "effect_id")?).map_err(identity_error)?; + let idempotency_key = + IdempotencyKey::new(required_str(args, "idempotency_key")?).map_err(identity_error)?; + let attempt_idempotency_key = + IdempotencyKey::new(required_str(args, "attempt_idempotency_key")?) + .map_err(identity_error)?; + if attempt_idempotency_key == idempotency_key { + return Err( + "reconciliation attempt idempotency key must differ from the original edit key" + .to_owned(), + ); + } + let input_digest = + ManifestDigest::new(required_str(args, "input_digest")?).map_err(identity_error)?; + let disposition = match required_str(args, "disposition")? { + "confirm_committed" => SourceEditReconciliationDispositionV1::ConfirmCommitted { + committed_state: ManifestDigest::new(required_str(args, "committed_state")?) + .map_err(identity_error)?, + }, + "confirm_rolled_back" => { + if args.get("committed_state").is_some() { + return Err( + "committed_state is only valid when disposition is confirm_committed" + .to_owned(), + ); + } + SourceEditReconciliationDispositionV1::ConfirmRolledBack + } + value => { + return Err(format!( + "invalid source edit reconciliation disposition: {value}" + )); + } + }; + Ok(SourceEditReconciliationInvocationV1 { + kind, + effect_id, + idempotency_key, + attempt_idempotency_key, + input_digest, + disposition, + }) +} + +/// Whether `operation` is one of the source-edit tools. +pub fn is_source_edit_operation(operation: ApplicationSurfaceOperation) -> bool { + matches!( + operation, + ApplicationSurfaceOperation::StrReplace + | ApplicationSurfaceOperation::MultiStrReplace + | ApplicationSurfaceOperation::InsertAt + | ApplicationSurfaceOperation::AstGrepRewrite + | ApplicationSurfaceOperation::ReplaceSymbol + | ApplicationSurfaceOperation::InsertAtSymbol + | ApplicationSurfaceOperation::MoveSymbol + | ApplicationSurfaceOperation::RenameSymbol + | ApplicationSurfaceOperation::SourceEditReconcile + | ApplicationSurfaceOperation::SourceEditRollback + ) +} + +/// Decode one source-edit tool's arguments; `Err` is the public diagnostic. +pub fn parse_source_edit_arguments( + operation: ApplicationSurfaceOperation, + args: &Value, +) -> Result { + match operation { + ApplicationSurfaceOperation::SourceEditRollback => { + rollback(args).map(ApplicationSurfaceRequest::SourceEditRollback) + } + ApplicationSurfaceOperation::SourceEditReconcile => { + reconcile(args).map(ApplicationSurfaceRequest::SourceEditReconcile) + } + _ => { + let edit = edit_request(operation, args)?; + let idempotency_key = optional_idempotency_key(args)?; + let expected_state = optional_expected_state(args)?; + if !edit.dry_run() && (idempotency_key.is_none() || expected_state.is_none()) { + return Err("source edit apply requires a fresh idempotency_key and the expected_state returned by a preview".to_owned()); + } + Ok(ApplicationSurfaceRequest::SourceEdit( + SourceEditInvocationV1 { + edit, + idempotency_key, + expected_state, + }, + )) + } + } +} + +/// The source-edit kind a surface operation names, when it names one. +pub(super) const fn source_edit_kind( + operation: ApplicationSurfaceOperation, +) -> Option { + Some(match operation { + ApplicationSurfaceOperation::StrReplace => SourceEditKind::StrReplace, + ApplicationSurfaceOperation::MultiStrReplace => SourceEditKind::MultiStrReplace, + ApplicationSurfaceOperation::InsertAt => SourceEditKind::InsertAt, + ApplicationSurfaceOperation::AstGrepRewrite => SourceEditKind::AstGrepRewrite, + ApplicationSurfaceOperation::ReplaceSymbol => SourceEditKind::ReplaceSymbol, + ApplicationSurfaceOperation::InsertAtSymbol => SourceEditKind::InsertAtSymbol, + ApplicationSurfaceOperation::MoveSymbol => SourceEditKind::MoveSymbol, + ApplicationSurfaceOperation::RenameSymbol => SourceEditKind::RenameSymbol, + _ => return None, + }) +} + +#[cfg(test)] +mod tests { + use serde_json::json; + use tracedecay_tool_catalog::ApplicationSurfaceOperation; + + use super::parse_source_edit_arguments; + use crate::application_surface::ApplicationSurfaceRequest; + + #[test] + fn apply_without_preview_identity_is_refused_with_the_public_message() { + let error = parse_source_edit_arguments( + ApplicationSurfaceOperation::StrReplace, + &json!({"path": "src/lib.rs", "old_str": "a", "new_str": "b"}), + ) + .expect_err("apply requires preview identity"); + assert_eq!( + error, + "source edit apply requires a fresh idempotency_key and the expected_state returned by a preview" + ); + } + + #[test] + fn move_symbol_defaults_to_preview_and_keeps_update_references() { + let request = parse_source_edit_arguments( + ApplicationSurfaceOperation::MoveSymbol, + &json!({"symbol": "a", "dest_file": "src/b.rs", "update_references": true}), + ) + .expect("move preview"); + let ApplicationSurfaceRequest::SourceEdit(invocation) = request else { + panic!("move is a source edit"); + }; + assert!(invocation.edit.dry_run()); + assert!(matches!( + invocation.edit, + tracedecay_contracts::SourceEditRequest::MoveSymbol { + update_references: true, + .. + } + )); + } + + #[test] + fn rollback_requires_explicit_confirmation() { + assert_eq!( + parse_source_edit_arguments( + ApplicationSurfaceOperation::SourceEditRollback, + &json!({}) + ) + .expect_err("confirm is required"), + "source edit rollback requires confirm=true from the caller after it checks the receipt; do not pause for a human" + ); + } + + #[test] + fn missing_replacement_pair_names_the_exact_shape() { + assert_eq!( + parse_source_edit_arguments( + ApplicationSurfaceOperation::MultiStrReplace, + &json!({"path": "src/lib.rs", "replacements": [["only"]], "dry_run": true}), + ) + .expect_err("pairs are exact"), + "each replacement must be an array of exactly 2 strings" + ); + } +} diff --git a/crates/tracedecay-daemon-protocol/src/client.rs b/crates/tracedecay-daemon-protocol/src/client.rs index 3f26345c22..02f1489adf 100644 --- a/crates/tracedecay-daemon-protocol/src/client.rs +++ b/crates/tracedecay-daemon-protocol/src/client.rs @@ -8,26 +8,24 @@ use std::future::Future; use std::pin::Pin; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use std::time::Duration; use tokio::io::{AsyncWriteExt, BufReader, ReadHalf, WriteHalf}; use tokio::sync::{OwnedSemaphorePermit, Semaphore}; use tracedecay_contracts::{ - ApplicationEnvelope, ApplicationInvocation, ApplicationInvocationExecutor, - ApplicationInvocationFuture, ApplicationProblem, ApplicationProblemKind, ApplicationRequest, - ApplicationResponse, CancellationSignal, CancellationStage, Deadline, InvocationError, - InvocationTarget, PageRequest, RequestId, SafeDiagnostic, + ApplicationInvocation, ApplicationInvocationExecutor, ApplicationInvocationFuture, + ApplicationProblem, ApplicationProblemKind, ApplicationRequest, ApplicationResponse, + CancellationSignal, CancellationStage, Deadline, InvocationError, InvocationTarget, + PageRequest, RequestId, SafeDiagnostic, try_now_micros, }; use tracedecay_domain::{ManifestDigest, UtcMicros}; use tracedecay_tool_catalog::{ - ApplicationSurfaceOperation, BindingId, BindingSurface, CatalogSnapshotV1, FeatureId, - ProfileId, SchemaRef, SurfaceOperationName, + BindingId, BindingSurface, CatalogSnapshotV1, FeatureId, ProfileId, SchemaRef, + SurfaceOperationName, }; -use tracedecay_contracts::feedback::observations::{ - FeedbackDeliveryRouteV1, FeedbackSourceEventV1, -}; +use tracedecay_contracts::feedback::observations::FeedbackSourceEventV1; use tracedecay_contracts::request_identity::{GlobalRequestSurface, mint_global_request_id}; pub type ScopeSelector = InvocationTarget; @@ -1005,31 +1003,6 @@ impl DaemonInvocationExecutor for DaemonInvocationClient { } } -/// Decode the envelope-stripped configuration body selected by `operation`. -/// -/// This is the socket-client dispatch arm: producers strip the tagged -/// `ConfigurationWireRequestV1` envelope before admission, so the client -/// deserializes the inner request and wraps the operation-selected variant. -fn configuration_request_from_surface_payload( - operation: ApplicationSurfaceOperation, - payload: serde_json::Value, -) -> Result { - tracedecay_contracts::configuration_wire_request_from_invocation_payload( - operation.as_str(), - payload, - ) - .map_err(|_| InvocationError::InvalidRequest) -} - -fn feedback_handle_from_surface_payload( - payload: serde_json::Value, -) -> Result { - let request: tracedecay_contracts::feedback::FeedbackHandleRequestV1 = - serde_json::from_value(payload).map_err(|_| InvocationError::InvalidRequest)?; - tracedecay_contracts::feedback::FeedbackHandleRequestV1::new(request.request_handle) - .map_err(|_| InvocationError::InvalidRequest) -} - impl ApplicationInvocationExecutor for DaemonInvocationClient { fn invoke( &self, @@ -1037,79 +1010,12 @@ impl ApplicationInvocationExecutor for DaemonInvocationClient { ) -> ApplicationInvocationFuture<'_, Result> { Box::pin(async move { let (context, request) = invocation.into_parts(); - let (request_id, target, deadline, cancellation) = context.into_parts(); match request { ApplicationRequest::Surface { binding, payload } => { - let (_binding_id, surface, operation, result_contract, _page) = - binding.into_parts(); - let operation = - ApplicationSurfaceOperation::from_surface_name(surface, operation.as_str()) - .ok_or(InvocationError::InvalidRequest)?; - let observed_at = invocation_now_micros(); - let cancellation_context = cancellation.context(); - let scope = match target { - InvocationTarget::CurrentProject => None, - InvocationTarget::Resolved(scope) => Some(scope), - }; - let policy = if matches!( - operation, - ApplicationSurfaceOperation::ConfigurationSet - | ApplicationSurfaceOperation::ConfigurationUnset - | ApplicationSurfaceOperation::ConfigurationBatch - ) { - InvocationCancellationPolicy::AuthoritativeEffect - } else { - InvocationCancellationPolicy::ReadOnly - }; - let request = match operation { - ApplicationSurfaceOperation::ConfigurationGet - | ApplicationSurfaceOperation::ConfigurationSet - | ApplicationSurfaceOperation::ConfigurationUnset - | ApplicationSurfaceOperation::ConfigurationBatch => { - let request = - configuration_request_from_surface_payload(operation, payload)?; - crate::contract::DaemonInvocationRequest::configuration( - request_id.as_str(), - operation, - request, - observed_at, - deadline.clone(), - cancellation_context, - ) - .with_resolved_scope(scope) - .map_err(|_| InvocationError::InvalidRequest)? - } - ApplicationSurfaceOperation::FeedbackGet => { - let request = feedback_handle_from_surface_payload(payload)?; - crate::contract::DaemonInvocationRequest::feedback( - request_id.as_str(), - operation, - request.request_handle, - observed_at, - deadline.clone(), - cancellation_context, - ) - .with_resolved_scope(scope) - .map_err(|_| InvocationError::InvalidRequest)? - } - ApplicationSurfaceOperation::FeedbackProximity => { - let request = serde_json::from_value(payload) - .map_err(|_| InvocationError::InvalidRequest)?; - crate::contract::DaemonInvocationRequest::feedback_proximity( - request_id.as_str(), - request, - deadline.clone(), - cancellation_context, - ) - } - _ => return Err(InvocationError::InvalidRequest), - } - .with_delivery_route(application_delivery_route(surface)); - let response = self - .invoke_controlled(request, deadline, cancellation, policy) - .await - .map_err(map_invocation_error)?; - application_response(request_id, result_contract, response.outcome) + crate::application_surface::invoke_application_surface( + self, context, binding, payload, + ) + .await } ApplicationRequest::FeedbackObservation { configuration_digest, @@ -1130,23 +1036,6 @@ impl ApplicationInvocationExecutor for DaemonInvocationClient { } } -/// Retained name for its call sites across the daemon, application surface, -/// and CLI commands (the bin target is a separate crate, so `pub(crate)` -/// would hide it from `src/commands`); the saturating clamp is the one -/// shared definition. -pub fn invocation_now_micros() -> UtcMicros { - tracedecay_contracts::clock::now_micros() -} - -pub fn application_delivery_route(surface: BindingSurface) -> FeedbackDeliveryRouteV1 { - match surface { - BindingSurface::Cli => FeedbackDeliveryRouteV1::Cli, - BindingSurface::Mcp => FeedbackDeliveryRouteV1::Mcp, - BindingSurface::Http | BindingSurface::Dashboard => FeedbackDeliveryRouteV1::Http, - BindingSurface::Lsp => FeedbackDeliveryRouteV1::Lsp, - } -} - pub fn map_invocation_error(error: DaemonInvocationError) -> InvocationError { match error { DaemonInvocationError::Cancelled { .. } => InvocationError::Cancelled, @@ -1162,66 +1051,6 @@ pub fn map_invocation_error(error: DaemonInvocationError) -> InvocationError { } } -pub fn application_response( - request_id: RequestId, - result_contract: tracedecay_contracts::ResultContractRef, - outcome: crate::contract::DaemonInvocationOutcome, -) -> Result { - let envelope = match outcome { - crate::contract::DaemonInvocationOutcome::Feedback { scope, result } => { - ApplicationEnvelope::evidence( - result_contract, - request_id, - scope, - result.into_application(), - ) - } - crate::contract::DaemonInvocationOutcome::Configuration { scope, outcome } => { - ApplicationEnvelope { - contract: result_contract, - request_id, - scope, - outcome, - } - } - crate::contract::DaemonInvocationOutcome::ApplicationProblem { problem } => { - // The daemon already resolved this invocation to a typed problem - // (e.g. `configuration.conflict`); carry it whole so surface - // adapters republish that diagnostic instead of refabricating a - // generic one. - return Err(InvocationError::Problem(Box::new(problem))); - } - crate::contract::DaemonInvocationOutcome::Problem { problem } => { - return Err(match problem { - crate::contract::DaemonInvocationProblem::InvalidRequest - | crate::contract::DaemonInvocationProblem::UnsupportedRevision => { - InvocationError::InvalidRequest - } - crate::contract::DaemonInvocationProblem::NotFoundOrNotAuthorized => { - InvocationError::Denied - } - crate::contract::DaemonInvocationProblem::ResetRequired => { - InvocationError::Problem(Box::new(ApplicationProblem::reset_required( - SafeDiagnostic { - code: "daemon.reset_required".to_owned(), - message: "The owning daemon authority requires an explicit reset" - .to_owned(), - }, - ))) - } - crate::contract::DaemonInvocationProblem::ApplicationContractViolation => { - InvocationError::Unavailable - } - crate::contract::DaemonInvocationProblem::Unavailable => { - InvocationError::Unavailable - } - }); - } - _ => return Err(InvocationError::Unavailable), - }; - Ok(ApplicationResponse::unary(envelope)) -} - fn invocation_error_from_problem(problem: &ApplicationProblem) -> InvocationError { match problem.kind() { ApplicationProblemKind::NotFoundOrNotAuthorized => InvocationError::Denied, @@ -1321,19 +1150,11 @@ fn with_daemon_version_skew_context( } pub fn deadline_remaining(deadline: &Deadline) -> Option { - let now = current_system_micros().map_or(i64::MAX, |now| now.0); + let now = try_now_micros().map_or(i64::MAX, |now| now.0); let remaining = deadline.expires_at.0.checked_sub(now)?; (remaining > 0).then(|| Duration::from_micros(remaining as u64)) } -fn current_system_micros() -> Option { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .ok() - .and_then(|duration| i64::try_from(duration.as_micros()).ok()) - .map(UtcMicros) -} - mod lsp_session; pub use lsp_session::DaemonLspSessionClient; @@ -1348,17 +1169,12 @@ mod controlled_invocation_tests; #[cfg(test)] mod tests { - use super::{ - DaemonInvocationError, application_response, configuration_request_from_surface_payload, - feedback_handle_from_surface_payload, invocation_now_micros, - }; + use super::DaemonInvocationError; use tracedecay_contracts::{ - ApplicationProblemKind, CancellationContext, CancellationStage, ConfigurationWireRequestV1, - Deadline, InvocationError, RequestId, ResultContractRef, + ApplicationProblemKind, CancellationContext, CancellationStage, Deadline, now_micros, }; use tracedecay_domain::UtcMicros; use tracedecay_tool_catalog::ApplicationSurfaceOperation; - use tracedecay_tool_catalog::SchemaId; #[test] fn daemon_invocation_errors_keep_canonical_problem_categories() { @@ -1384,27 +1200,6 @@ mod tests { } } - #[test] - fn daemon_reset_response_remains_an_authoritative_typed_problem() { - let error = application_response( - RequestId::new("request.daemon-client.reset").expect("request"), - ResultContractRef::new( - SchemaId::new("schema.test.daemon-client-reset-result").expect("schema"), - 1, - ) - .expect("contract"), - crate::contract::DaemonInvocationOutcome::Problem { - problem: crate::contract::DaemonInvocationProblem::ResetRequired, - }, - ) - .expect_err("reset-required must not become a successful response"); - - let InvocationError::Problem(problem) = error else { - panic!("reset-required must remain an authoritative typed problem"); - }; - assert_eq!(problem.kind(), ApplicationProblemKind::ResetRequired); - } - fn unused_test_endpoint() -> crate::transport::DaemonEndpoint { crate::transport::DaemonEndpoint::loopback(std::net::SocketAddr::from(([127, 0, 0, 1], 0))) .expect("loopback endpoint") @@ -1412,8 +1207,11 @@ mod tests { #[test] fn transport_failures_name_version_skew_when_the_authority_daemon_differs() { - let connection = crate::connection::DaemonConnection::new(unused_test_endpoint(), None) - .with_daemon_version("0.1.0-beta.36+aaaa"); + let connection = crate::connection::DaemonConnection::new( + unused_test_endpoint(), + "unused-token".to_owned(), + ) + .with_daemon_version("0.1.0-beta.36+aaaa"); let mut handshake = test_skew_handshake(); handshake.client_version = "0.1.0-beta.37+bbbb".to_owned(); @@ -1442,9 +1240,15 @@ mod tests { #[test] fn transport_failures_stay_untouched_without_version_skew() { - let matching = crate::connection::DaemonConnection::new(unused_test_endpoint(), None) - .with_daemon_version("0.1.0-beta.37+cccc"); - let unknown = crate::connection::DaemonConnection::new(unused_test_endpoint(), None); + let matching = crate::connection::DaemonConnection::new( + unused_test_endpoint(), + "unused-token".to_owned(), + ) + .with_daemon_version("0.1.0-beta.37+cccc"); + let unknown = crate::connection::DaemonConnection::new( + unused_test_endpoint(), + "unused-token".to_owned(), + ); let mut handshake = test_skew_handshake(); handshake.client_version = "0.1.0-beta.37+cccc".to_owned(); @@ -1490,8 +1294,11 @@ mod tests { // The skew decorator must not relabel the daemon's definitive answer, // even when the authority record names a different daemon version. - let connection = crate::connection::DaemonConnection::new(unused_test_endpoint(), None) - .with_daemon_version("0.1.0-beta.36+dddd"); + let connection = crate::connection::DaemonConnection::new( + unused_test_endpoint(), + "unused-token".to_owned(), + ) + .with_daemon_version("0.1.0-beta.36+dddd"); let decorated = super::with_daemon_version_skew_context(error, &connection, &handshake); let (code, _, _) = decorated .project_route_context() @@ -1517,7 +1324,8 @@ mod tests { let (stream, _) = listener.accept().await.expect("accept client"); let (reader, mut writer) = stream.into_split(); let mut lines = tokio::io::BufReader::new(reader).lines(); - // Handshake line, then the pipelined request line. + // Auth preface and handshake lines, then the pipelined request line. + let _ = lines.next_line().await.expect("read auth preface"); let _ = lines.next_line().await.expect("read handshake"); let _ = lines.next_line().await.expect("read request"); writer @@ -1532,13 +1340,13 @@ mod tests { let client = super::DaemonInvocationClient::new( crate::connection::DaemonConnection::new( crate::transport::DaemonEndpoint::Unix(socket), - None, + "refused-handshake-token".to_owned(), ) .with_daemon_version("0.1.0-beta.36+ffff"), handshake, ); - let observed_at = invocation_now_micros(); + let observed_at = now_micros(); let error = client .invoke(crate::contract::DaemonInvocationRequest::feedback( "request.refused-handshake", @@ -1579,59 +1387,4 @@ mod tests { moved_store_adoption: crate::handshake::MovedStoreAdoption::Never, } } - - #[test] - fn configuration_dispatch_accepts_envelope_stripped_get_and_set_payloads() { - let get = configuration_request_from_surface_payload( - ApplicationSurfaceOperation::ConfigurationGet, - serde_json::json!({"key": "mcp.tool_timings"}), - ) - .expect("stripped get payload"); - assert!(matches!( - get, - ConfigurationWireRequestV1::Get(request) if request.key.as_str() == "mcp.tool_timings" - )); - - let set = configuration_request_from_surface_payload( - ApplicationSurfaceOperation::ConfigurationSet, - serde_json::json!({ - "layer": {"kind": "default"}, - "key": "mcp.tool_timings", - "value": {"kind": "boolean", "value": true}, - "expected_revision": "revision.test-configuration-set", - "idempotency_key": "configuration.idempotency.test-set" - }), - ) - .expect("stripped set payload"); - assert!(matches!(set, ConfigurationWireRequestV1::Set(_))); - } - - #[test] - fn configuration_dispatch_rejects_the_tagged_envelope() { - assert!(matches!( - configuration_request_from_surface_payload( - ApplicationSurfaceOperation::ConfigurationGet, - serde_json::json!({ - "operation": "get", - "request": {"key": "mcp.tool_timings"} - }), - ), - Err(InvocationError::InvalidRequest) - )); - } - - #[test] - fn feedback_get_dispatch_validates_handles_client_side() { - let accepted = feedback_handle_from_surface_payload(serde_json::json!({ - "request_handle": "feedback.handle.v1" - })) - .expect("valid handle"); - assert_eq!(accepted.request_handle, "feedback.handle.v1"); - assert_eq!( - feedback_handle_from_surface_payload(serde_json::json!({ - "request_handle": " leading" - })), - Err(InvocationError::InvalidRequest) - ); - } } diff --git a/crates/tracedecay-daemon-protocol/src/client/controlled_invocation_tests.rs b/crates/tracedecay-daemon-protocol/src/client/controlled_invocation_tests.rs index 2e8497b943..f04dede570 100644 --- a/crates/tracedecay-daemon-protocol/src/client/controlled_invocation_tests.rs +++ b/crates/tracedecay-daemon-protocol/src/client/controlled_invocation_tests.rs @@ -22,6 +22,26 @@ use tracedecay_contracts::{ use tracedecay_domain::UtcMicros; use tracedecay_tool_catalog::ApplicationSurfaceOperation; +const TEST_AUTH_TOKEN: &str = "controlled-invocation-test-token"; + +/// Wraps a fake daemon's reader after consuming the auth preface every client +/// writes first. A peer that closes before writing one (a liveness probe) +/// yields lines already at EOF. +async fn authenticated_lines( + reader: R, +) -> tokio::io::Lines> { + let mut lines = BufReader::new(reader).lines(); + if let Ok(Some(preface)) = lines.next_line().await { + assert!( + crate::transport::DaemonAuthPreface::from_line(&preface) + .expect("auth preface") + .authenticate(TEST_AUTH_TOKEN), + "client must present the connection token" + ); + } + lines +} + fn now_micros() -> UtcMicros { let micros = SystemTime::now() .duration_since(UNIX_EPOCH) @@ -67,7 +87,7 @@ fn invocation_client( instance_id: &str, ) -> DaemonInvocationClient { invocation_client_for( - DaemonConnection::unauthenticated_for_test(endpoint), + DaemonConnection::new(endpoint, TEST_AUTH_TOKEN.to_owned()), instance_id, ) } @@ -150,7 +170,7 @@ fn rotating_authority_client( }); let probe: Arc = authority.clone(); let client = invocation_client_for( - DaemonConnection::unauthenticated_for_test(endpoint).with_liveness(probe), + DaemonConnection::new(endpoint, TEST_AUTH_TOKEN.to_owned()).with_liveness(probe), instance_id, ); (client, authority) @@ -197,7 +217,7 @@ async fn measure_parallel_invocation_workload( let delayed_admitted = Arc::clone(&delayed_admitted); tokio::spawn(async move { let (reader, mut writer) = stream.into_split(); - let mut lines = BufReader::new(reader).lines(); + let mut lines = authenticated_lines(reader).await; lines .next_line() .await @@ -307,7 +327,7 @@ async fn delayed_response_opens_no_periodic_probe_connections() { server_accepts.fetch_add(1, Ordering::SeqCst); tokio::spawn(async move { let (reader, mut writer) = stream.into_split(); - let mut lines = BufReader::new(reader).lines(); + let mut lines = authenticated_lines(reader).await; let Some(_handshake) = lines.next_line().await.expect("read handshake") else { return; }; @@ -349,7 +369,7 @@ async fn work_delivery_ack_uses_the_response_connection() { let server = tokio::spawn(async move { let stream = listener.accept().await.expect("accept Work invocation"); let (reader, mut writer) = stream.into_split(); - let mut lines = BufReader::new(reader).lines(); + let mut lines = authenticated_lines(reader).await; lines .next_line() .await @@ -414,7 +434,7 @@ async fn dropping_unacknowledged_work_delivery_closes_its_connection() { let server = tokio::spawn(async move { let stream = listener.accept().await.expect("accept Work invocation"); let (reader, mut writer) = stream.into_split(); - let mut lines = BufReader::new(reader).lines(); + let mut lines = authenticated_lines(reader).await; lines .next_line() .await @@ -464,7 +484,7 @@ async fn lsp_session_pins_one_connection_through_detach() { let stream = listener.accept().await.expect("accept LSP session"); server_accepts.fetch_add(1, Ordering::SeqCst); let (reader, mut writer) = stream.into_split(); - let mut lines = BufReader::new(reader).lines(); + let mut lines = authenticated_lines(reader).await; lines .next_line() .await @@ -588,7 +608,7 @@ async fn two_hundred_invocations_use_at_most_eight_connections_without_leaks() { server_accepts.fetch_add(1, Ordering::SeqCst); handlers.spawn(async move { let (reader, mut writer) = stream.into_split(); - let mut lines = BufReader::new(reader).lines(); + let mut lines = authenticated_lines(reader).await; lines .next_line() .await @@ -664,7 +684,7 @@ async fn daemon_restart_with_rotated_authority_purges_idle_connections_before_re .expect("accept first warm connection"); server_accepts.fetch_add(1, Ordering::SeqCst); let (first_reader, mut first_writer) = first_stream.into_split(); - let mut first_lines = BufReader::new(first_reader).lines(); + let mut first_lines = authenticated_lines(first_reader).await; first_lines .next_line() .await @@ -682,7 +702,7 @@ async fn daemon_restart_with_rotated_authority_purges_idle_connections_before_re .expect("accept second warm connection"); server_accepts.fetch_add(1, Ordering::SeqCst); let (second_reader, mut second_writer) = second_stream.into_split(); - let mut second_lines = BufReader::new(second_reader).lines(); + let mut second_lines = authenticated_lines(second_reader).await; second_lines .next_line() .await @@ -716,7 +736,7 @@ async fn daemon_restart_with_rotated_authority_purges_idle_connections_before_re .expect("accept recovered connection"); server_accepts.fetch_add(1, Ordering::SeqCst); let (recovered_reader, mut recovered_writer) = recovered_stream.into_split(); - let mut recovered_lines = BufReader::new(recovered_reader).lines(); + let mut recovered_lines = authenticated_lines(recovered_reader).await; recovered_lines .next_line() .await @@ -802,7 +822,7 @@ async fn concurrent_invocations_report_parallel_activity() { .await .expect("accept invocation connection"); let (reader, mut writer) = stream.into_split(); - let mut lines = BufReader::new(reader).lines(); + let mut lines = authenticated_lines(reader).await; lines .next_line() .await @@ -825,7 +845,7 @@ async fn concurrent_invocations_report_parallel_activity() { .await .expect("accept second invocation connection"); let (second_reader, mut second_writer) = second_stream.into_split(); - let mut second_lines = BufReader::new(second_reader).lines(); + let mut second_lines = authenticated_lines(second_reader).await; second_lines .next_line() .await @@ -899,7 +919,7 @@ async fn controlled_client( let server = tokio::spawn(async move { let invocation_stream = listener.accept().await.expect("accept invocation"); let (invocation_reader, mut invocation_writer) = invocation_stream.into_split(); - let mut invocation_lines = BufReader::new(invocation_reader).lines(); + let mut invocation_lines = authenticated_lines(invocation_reader).await; invocation_lines .next_line() .await @@ -917,7 +937,7 @@ async fn controlled_client( let control_stream = listener.accept().await.expect("accept cancellation"); let (control_reader, _control_writer) = control_stream.into_split(); - let mut control_lines = BufReader::new(control_reader).lines(); + let mut control_lines = authenticated_lines(control_reader).await; control_lines .next_line() .await @@ -971,7 +991,7 @@ async fn controlled_client( }; ( DaemonInvocationClient::for_connection_for_test( - DaemonConnection::unauthenticated_for_test(endpoint), + DaemonConnection::new(endpoint, TEST_AUTH_TOKEN.to_owned()), handshake, ), admitted, @@ -1033,7 +1053,7 @@ async fn reset_then_reconnect_client( let server = tokio::spawn(async move { let first_stream = listener.accept().await.expect("accept first invocation"); let (first_reader, _first_writer) = first_stream.into_split(); - let mut first_lines = BufReader::new(first_reader).lines(); + let mut first_lines = authenticated_lines(first_reader).await; first_lines .next_line() .await @@ -1051,7 +1071,7 @@ async fn reset_then_reconnect_client( let control_stream = listener.accept().await.expect("accept cancellation"); let (control_reader, _control_writer) = control_stream.into_split(); - let mut control_lines = BufReader::new(control_reader).lines(); + let mut control_lines = authenticated_lines(control_reader).await; control_lines .next_line() .await @@ -1072,7 +1092,7 @@ async fn reset_then_reconnect_client( let (mut second_lines, mut second_writer) = loop { let second_stream = listener.accept().await.expect("accept second invocation"); let (second_reader, second_writer) = second_stream.into_split(); - let mut second_lines = BufReader::new(second_reader).lines(); + let mut second_lines = authenticated_lines(second_reader).await; if let Ok(Some(_handshake)) = second_lines.next_line().await { break (second_lines, second_writer); } @@ -1123,7 +1143,7 @@ async fn reset_then_reconnect_client( }; ( DaemonInvocationClient::for_connection_for_test( - DaemonConnection::unauthenticated_for_test(endpoint), + DaemonConnection::new(endpoint, TEST_AUTH_TOKEN.to_owned()), handshake, ), admitted, @@ -1150,7 +1170,7 @@ async fn unsettled_client(request_id: &'static str, control: UnsettledControl) - let server = tokio::spawn(async move { let invocation_stream = listener.accept().await.expect("accept invocation"); let (invocation_reader, invocation_writer) = invocation_stream.into_split(); - let mut invocation_lines = BufReader::new(invocation_reader).lines(); + let mut invocation_lines = authenticated_lines(invocation_reader).await; invocation_lines .next_line() .await @@ -1170,7 +1190,7 @@ async fn unsettled_client(request_id: &'static str, control: UnsettledControl) - let _ = request_admitted.send(()); let control_stream = listener.accept().await.expect("accept cancellation"); let (control_reader, control_writer) = control_stream.into_split(); - let mut control_lines = BufReader::new(control_reader).lines(); + let mut control_lines = authenticated_lines(control_reader).await; control_lines .next_line() .await @@ -1224,7 +1244,7 @@ async fn unsettled_client(request_id: &'static str, control: UnsettledControl) - }; UnsettledDaemon { client: DaemonInvocationClient::for_connection_for_test( - DaemonConnection::unauthenticated_for_test(endpoint), + DaemonConnection::new(endpoint, TEST_AUTH_TOKEN.to_owned()), handshake, ), admitted, @@ -1467,7 +1487,7 @@ fn spawn_pool_daemon( let warm_up = Arc::clone(&warm_up); tokio::spawn(async move { let (reader, mut writer) = stream.into_split(); - let mut lines = BufReader::new(reader).lines(); + let mut lines = authenticated_lines(reader).await; lines .next_line() .await diff --git a/crates/tracedecay-daemon-protocol/src/connection.rs b/crates/tracedecay-daemon-protocol/src/connection.rs index 0f1556d9f2..a41d509b62 100644 --- a/crates/tracedecay-daemon-protocol/src/connection.rs +++ b/crates/tracedecay-daemon-protocol/src/connection.rs @@ -47,7 +47,7 @@ pub trait DaemonLivenessProbe: Send + Sync { #[derive(Clone)] pub struct DaemonConnection { pub endpoint: DaemonEndpoint, - pub auth_token: Option, + pub auth_token: String, /// The daemon version advertised by the authority record that named this /// endpoint. Lets transport failures name version skew instead of hiding /// it behind a raw io error. @@ -56,7 +56,7 @@ pub struct DaemonConnection { } impl DaemonConnection { - pub fn new(endpoint: DaemonEndpoint, auth_token: Option) -> Self { + pub fn new(endpoint: DaemonEndpoint, auth_token: String) -> Self { Self { endpoint, auth_token, @@ -76,10 +76,6 @@ impl DaemonConnection { self.daemon_version = Some(daemon_version.into()); self } - - pub fn unauthenticated_for_test(endpoint: DaemonEndpoint) -> Self { - Self::new(endpoint, None) - } } /// The local read bound for a request whose caller deadline is `request_deadline`. @@ -231,7 +227,7 @@ where .await } -/// Writes the optional auth preface and the handshake line. +/// Writes the auth preface and the handshake line. /// /// Callers that already hold a [`DaemonConnection`] use /// [`write_daemon_preamble`]. The composition-root client uses this directly @@ -239,15 +235,13 @@ where /// connection. pub async fn write_daemon_handshake_preamble( writer: &mut (impl tokio::io::AsyncWrite + Unpin), - auth_token: Option<&str>, + auth_token: &str, handshake: &DaemonHandshake, ) -> Result<()> { - if let Some(token) = auth_token { - writer - .write_all(DaemonAuthPreface::new(token).to_line()?.as_bytes()) - .await?; - writer.write_all(b"\n").await?; - } + writer + .write_all(DaemonAuthPreface::new(auth_token).to_line()?.as_bytes()) + .await?; + writer.write_all(b"\n").await?; writer.write_all(handshake.to_line()?.as_bytes()).await?; writer.write_all(b"\n").await?; Ok(()) @@ -259,7 +253,7 @@ pub async fn write_daemon_preamble( connection: &DaemonConnection, handshake: &DaemonHandshake, ) -> Result<()> { - write_daemon_handshake_preamble(writer, connection.auth_token.as_deref(), handshake).await + write_daemon_handshake_preamble(writer, &connection.auth_token, handshake).await } pub fn is_transient_daemon_connect_error(kind: std::io::ErrorKind) -> bool { diff --git a/crates/tracedecay-daemon-protocol/src/contract/mod.rs b/crates/tracedecay-daemon-protocol/src/contract/mod.rs index c36fd9a229..decf130d98 100644 --- a/crates/tracedecay-daemon-protocol/src/contract/mod.rs +++ b/crates/tracedecay-daemon-protocol/src/contract/mod.rs @@ -426,6 +426,7 @@ pub enum DaemonInvocationOperation { SourceEdit, SourceEditReconcile, SourceEditRollback, + GraphTool, } impl DaemonInvocationOperation { @@ -491,6 +492,7 @@ impl DaemonInvocationOperation { Self::SourceEdit => "source_edit", Self::SourceEditReconcile => "source_edit_reconcile", Self::SourceEditRollback => "source_edit_rollback", + Self::GraphTool => "graph_tool", } } } @@ -618,6 +620,8 @@ pub enum DaemonInvocationPayload { PrimitiveRead { surface_operation: ApplicationSurfaceOperation, request: PrimitiveRequest, + #[serde(default, skip_serializing_if = "Option::is_none")] + resolved_scope: Option, observed_at: UtcMicros, deadline: Deadline, cancellation: CancellationContext, @@ -626,6 +630,8 @@ pub enum DaemonInvocationPayload { surface_operation: ApplicationSurfaceOperation, request: tracedecay_contracts::PrimitiveCodeSurfaceRequest, page: PageRequest, + #[serde(default, skip_serializing_if = "Option::is_none")] + resolved_scope: Option, observed_at: UtcMicros, deadline: Deadline, cancellation: CancellationContext, @@ -634,6 +640,8 @@ pub enum DaemonInvocationPayload { surface_operation: ApplicationSurfaceOperation, request: tracedecay_contracts::CallableCodeSurfaceRequest, page: PageRequest, + #[serde(default, skip_serializing_if = "Option::is_none")] + resolved_scope: Option, observed_at: UtcMicros, deadline: Deadline, cancellation: CancellationContext, @@ -755,9 +763,39 @@ pub enum DaemonInvocationPayload { deadline: Deadline, cancellation: CancellationContext, }, + GraphTool { + surface_operation: ApplicationSurfaceOperation, + arguments: serde_json::Map, + observed_at: UtcMicros, + deadline: Deadline, + cancellation: CancellationContext, + }, } impl DaemonInvocationRequest { + pub fn graph_tool( + request_id: impl Into, + surface_operation: ApplicationSurfaceOperation, + arguments: serde_json::Map, + observed_at: UtcMicros, + deadline: Deadline, + cancellation: CancellationContext, + ) -> Self { + Self { + protocol: DAEMON_INVOCATION_PROTOCOL.to_owned(), + revision: DAEMON_INVOCATION_REVISION, + request_id: request_id.into(), + delivery_route: None, + payload: DaemonInvocationPayload::GraphTool { + surface_operation, + arguments, + observed_at, + deadline, + cancellation, + }, + } + } + /// One typed constructor for the whole Plan 36 native-integration journey. /// /// The transport carries exact typed identity only; it contains no Git @@ -921,6 +959,58 @@ impl DaemonInvocationRequest { | ApplicationSurfaceOperation::ContextScoutFeedback => { unreachable!("Context Scout operations use their typed constructor") } + ApplicationSurfaceOperation::StrReplace + | ApplicationSurfaceOperation::MultiStrReplace + | ApplicationSurfaceOperation::InsertAt + | ApplicationSurfaceOperation::AstGrepRewrite + | ApplicationSurfaceOperation::ReplaceSymbol + | ApplicationSurfaceOperation::InsertAtSymbol + | ApplicationSurfaceOperation::MoveSymbol + | ApplicationSurfaceOperation::RenameSymbol + | ApplicationSurfaceOperation::SourceEditReconcile + | ApplicationSurfaceOperation::SourceEditRollback => { + unreachable!("source-edit operations use their typed constructors") + } + ApplicationSurfaceOperation::Context + | ApplicationSurfaceOperation::Node + | ApplicationSurfaceOperation::Impact + | ApplicationSurfaceOperation::Similar + | ApplicationSurfaceOperation::Redundancy + | ApplicationSurfaceOperation::RenamePreview + | ApplicationSurfaceOperation::PortStatus + | ApplicationSurfaceOperation::PortOrder + | ApplicationSurfaceOperation::Todos => { + unreachable!("graph-tool operations use their typed constructor") + } + ApplicationSurfaceOperation::FactStoreCurate + | ApplicationSurfaceOperation::FactStoreAdd + | ApplicationSurfaceOperation::FactStoreSearch + | ApplicationSurfaceOperation::FactStoreProbe + | ApplicationSurfaceOperation::FactStoreRelated + | ApplicationSurfaceOperation::FactStoreReason + | ApplicationSurfaceOperation::FactStoreContradict + | ApplicationSurfaceOperation::FactStoreGet + | ApplicationSurfaceOperation::FactStoreUpdate + | ApplicationSurfaceOperation::FactStoreRemove + | ApplicationSurfaceOperation::FactStoreSupersede + | ApplicationSurfaceOperation::FactStoreList + | ApplicationSurfaceOperation::FactFeedback + | ApplicationSurfaceOperation::MemoryStatus + | ApplicationSurfaceOperation::SessionRefreshStatus + | ApplicationSurfaceOperation::SessionRefreshCancel + | ApplicationSurfaceOperation::SessionRefreshBegin + | ApplicationSurfaceOperation::MessageSearch + | ApplicationSurfaceOperation::SessionsFor + | ApplicationSurfaceOperation::Workflows + | ApplicationSurfaceOperation::LcmStatus + | ApplicationSurfaceOperation::LcmDoctor + | ApplicationSurfaceOperation::LcmLoadSession + | ApplicationSurfaceOperation::LcmGrep + | ApplicationSurfaceOperation::LcmDescribe + | ApplicationSurfaceOperation::LcmExpand + | ApplicationSurfaceOperation::LcmExpandQuery => { + unreachable!("retained operations use their typed constructor") + } }; Self { protocol: DAEMON_INVOCATION_PROTOCOL.to_owned(), @@ -1075,6 +1165,7 @@ impl DaemonInvocationRequest { ) => DaemonInvocationPayload::PrimitiveRead { surface_operation, request, + resolved_scope: None, observed_at, deadline, cancellation, @@ -1352,6 +1443,7 @@ impl DaemonInvocationRequest { surface_operation, request, page, + resolved_scope: None, observed_at, deadline, cancellation, @@ -1377,6 +1469,7 @@ impl DaemonInvocationRequest { surface_operation, request, page, + resolved_scope: None, observed_at, deadline, cancellation, @@ -1524,7 +1617,10 @@ impl DaemonInvocationRequest { ( DaemonInvocationPayload::FeedbackGet { resolved_scope, .. } | DaemonInvocationPayload::Configuration { resolved_scope, .. } - | DaemonInvocationPayload::ObservatoryRead { resolved_scope, .. }, + | DaemonInvocationPayload::ObservatoryRead { resolved_scope, .. } + | DaemonInvocationPayload::PrimitiveRead { resolved_scope, .. } + | DaemonInvocationPayload::PrimitiveCode { resolved_scope, .. } + | DaemonInvocationPayload::CallableCode { resolved_scope, .. }, scope, ) => { *resolved_scope = scope; @@ -1734,6 +1830,7 @@ impl DaemonInvocationRequest { DaemonInvocationPayload::SourceEditRollback { .. } => { DaemonInvocationOperation::SourceEditRollback } + DaemonInvocationPayload::GraphTool { .. } => DaemonInvocationOperation::GraphTool, } } @@ -1794,6 +1891,7 @@ impl DaemonInvocationRequest { | DaemonInvocationOperation::SourceEdit | DaemonInvocationOperation::SourceEditReconcile | DaemonInvocationOperation::SourceEditRollback + | DaemonInvocationOperation::GraphTool ) } @@ -2027,6 +2125,19 @@ impl DaemonInvocationRequest { return Err(DaemonInvocationProblem::InvalidRequest); } } + DaemonInvocationPayload::GraphTool { + surface_operation, + observed_at, + deadline, + cancellation, + .. + } => { + if !valid_observation_window(observed_at, deadline, cancellation) + || !surface_operation.is_graph_tool() + { + return Err(DaemonInvocationProblem::InvalidRequest); + } + } DaemonInvocationPayload::ObservatoryRead { request, observed_at, @@ -2057,6 +2168,7 @@ impl DaemonInvocationRequest { observed_at, deadline, cancellation, + .. } => { if observed_at.0 <= 0 || deadline.expires_at.0 <= 0 @@ -2095,6 +2207,7 @@ impl DaemonInvocationRequest { observed_at, deadline, cancellation, + .. } => { if observed_at.0 <= 0 || deadline.expires_at.0 <= 0 @@ -2748,6 +2861,10 @@ pub enum DaemonInvocationOutcome { scope: ResolvedScope, result: tracedecay_contracts::source_edit::SourceEditSurfaceResultV1, }, + GraphTool { + scope: ResolvedScope, + completion: tracedecay_contracts::graph_tool::GraphToolCompletionV1, + }, Problem { problem: DaemonInvocationProblem, }, diff --git a/crates/tracedecay-daemon-protocol/src/lib.rs b/crates/tracedecay-daemon-protocol/src/lib.rs index eed99da37a..763ba2f539 100644 --- a/crates/tracedecay-daemon-protocol/src/lib.rs +++ b/crates/tracedecay-daemon-protocol/src/lib.rs @@ -61,7 +61,12 @@ pub mod transport; pub use application_surface::{ ApplicationSurfaceAdapterError, ApplicationSurfaceInvocationResult, ApplicationSurfaceRequest, ApplicationToolRequest, FeedbackSurfaceRequest, adapt_application_tool_request, - parse_application_surface_request, separate_application_tool_request, + application_delivery_route, application_outcome_value, application_response, + application_surface_cancellation_policy, application_surface_feedback_is_observable, + application_surface_feedback_operation, decode_retained_request, invoke_application_surface, + is_source_edit_operation, parse_application_surface_invocation_payload, + parse_application_surface_request, parse_source_edit_arguments, + separate_application_tool_request, }; pub use client::{ AdapterInvocation, BindingResolution, BindingResolver, BoundInvocation, CanonicalInvocation, @@ -69,9 +74,8 @@ pub use client::{ DaemonInvocationError, DaemonInvocationExecutor, DaemonInvocationExecutorFuture, DaemonInvocationResult, DaemonLspSessionClient, DispatchError, DispatchInput, DispatchedInvocation, InvocationCancellationPolicy, InvocationControls, ResolvedBinding, - ScopeSelector, application_delivery_route, application_response, deadline_remaining, - handshake_refusal_error, invocation_now_micros, map_invocation_error, resolve_dispatch, - wait_for_cancellation, + ScopeSelector, deadline_remaining, handshake_refusal_error, map_invocation_error, + resolve_dispatch, wait_for_cancellation, }; pub use client_identity::DaemonClientIdentity; pub use connection::{ diff --git a/crates/tracedecay-daemon-service/src/adoption_observation.rs b/crates/tracedecay-daemon-service/src/adoption_observation.rs index 44917258b7..9f734cc78b 100644 --- a/crates/tracedecay-daemon-service/src/adoption_observation.rs +++ b/crates/tracedecay-daemon-service/src/adoption_observation.rs @@ -147,17 +147,6 @@ mod tests { "capability.application.retained.", "capability.application.source-edit.", ]; - - /// Families the composed application catalog can truthfully census today. - const COMPOSED_FAMILIES: &[&str] = &[ - "retrieval", - "context_scout", - "feedback", - "git", - "lsp", - "analytics", - ]; - #[test] fn every_composed_capability_is_classified_or_deliberately_out_of_scope() { let contributions = application_catalog_contributions().expect("composed catalog"); @@ -179,43 +168,33 @@ mod tests { } #[test] - fn census_counts_hold_the_funnel_order_for_every_composed_family() { + fn census_counts_each_composed_family_and_omits_uncomposed_families() { let census = adoption_eligibility_census().expect("catalog census"); - assert!(!census.is_empty(), "the composed catalog census is empty"); - let by_family: BTreeMap<&str, &AdoptionEligibilityObservedV1> = census + let counts: Vec<(&str, u64, u64, u64)> = census .iter() - .map(|observation| (observation.capability.as_str(), observation)) + .map(|observation| { + ( + observation.capability.as_str(), + observation.eligible, + observation.enabled, + observation.available, + ) + }) .collect(); - for family in COMPOSED_FAMILIES { - let observation = by_family - .get(family) - .unwrap_or_else(|| panic!("{family} family missing from the catalog census")); - assert!( - observation.eligible > 0, - "{family} must census a non-zero eligible population" - ); - assert!(observation.enabled <= observation.eligible); - assert!(observation.available <= observation.enabled); - } - // The default profile serves callable retrieval capabilities, so the - // census must observe them as enabled and available, not merely - // composed. - assert!(by_family["retrieval"].available > 0); - // Families this catalog authority does not compose must be absent - // instead of claiming a Known-zero eligible population. - for family in [ - "automation", - "work", - "workflow", - "hooks", - "mcp", - "dashboard", - ] { - assert!( - !by_family.contains_key(family), - "{family} has no composed catalog capability and must not be emitted" - ); - } + // (family, eligible, enabled, available); families such as + // automation, work, and workflow compose no catalog capability and so + // must be absent rather than a Known-zero population. + assert_eq!( + counts, + [ + ("analytics", 1, 1, 1), + ("context_scout", 11, 11, 11), + ("feedback", 11, 9, 9), + ("git", 22, 19, 19), + ("lsp", 2, 0, 0), + ("retrieval", 34, 34, 34), + ] + ); } #[tokio::test] diff --git a/crates/tracedecay-daemon-service/src/application_surface.rs b/crates/tracedecay-daemon-service/src/application_surface.rs index 5f2d9327a5..4bc3d08e4d 100644 --- a/crates/tracedecay-daemon-service/src/application_surface.rs +++ b/crates/tracedecay-daemon-service/src/application_surface.rs @@ -334,7 +334,6 @@ pub fn assemble_http_application_router( let workflow_router = workflow_application_router_with_executor(Arc::clone(&executor))?; let handoff_router = handoff_application_router_with_executor(Arc::clone(&executor))?; let multi_root_router = multi_root_application_router_with_executor(Arc::clone(&executor))?; - let retained_router = retained::router_with_executor(Arc::clone(&executor))?; Ok( tracedecay_api::application_router(application_invoker_for_surface( executor, @@ -345,7 +344,6 @@ pub fn assemble_http_application_router( .merge(workflow_router) .merge(handoff_router) .merge(multi_root_router) - .merge(retained_router) .layer(axum::middleware::from_fn_with_state( Arc::clone(&cancellations), application_http_context, diff --git a/crates/tracedecay-daemon-service/src/application_surface/configuration_wire.rs b/crates/tracedecay-daemon-service/src/application_surface/configuration_wire.rs index 97a965f1cc..f676ac7e74 100644 --- a/crates/tracedecay-daemon-service/src/application_surface/configuration_wire.rs +++ b/crates/tracedecay-daemon-service/src/application_surface/configuration_wire.rs @@ -12,8 +12,6 @@ use tracedecay_tool_catalog::{ TerminalState, TerminalStateContract, }; -use tracedecay_daemon_protocol::ApplicationSurfaceAdapterError; - pub(super) const CONFIGURATION_WIRE_OPERATIONS: [ApplicationSurfaceOperation; 11] = [ ApplicationSurfaceOperation::ConfigurationList, ApplicationSurfaceOperation::ConfigurationGet, @@ -51,25 +49,6 @@ pub(super) fn configuration_binding_has_schema( }) } -/// The application invocation payload for a configuration operation is the -/// operation's own request body, the same shape -/// `parse_application_surface_request` accepts from every caller surface, not -/// the `operation`/`request` envelope `ConfigurationWireRequestV1` uses to -/// carry it across the daemon contract. Sending the envelope made the executor -/// re-parse a tagged wrapper against a `deny_unknown_fields` request struct, -/// so every configuration read and write routed through the daemon invocation -/// executor failed admission as `InvalidRequest`. -#[hotpath::measure(label = "application_surface.configuration.payload")] -pub(super) fn configuration_invocation_payload( - request: &tracedecay_contracts::ConfigurationWireRequestV1, -) -> Result { - let mut wire = - serde_json::to_value(request).map_err(ApplicationSurfaceAdapterError::invalid_request)?; - wire.get_mut("request").map(Value::take).ok_or_else(|| { - ApplicationSurfaceAdapterError::invalid_request("configuration wire request has no body") - }) -} - fn payload_decodes(payload: Option<&Value>) -> bool { payload.is_none_or(|value| serde_json::from_value::(value.clone()).is_ok()) } @@ -98,6 +77,7 @@ fn configuration_cancellation_is_legal( ApplicationOutcome::Evidence(packet) => packet.execution.cancellation.as_ref(), ApplicationOutcome::Preview(preview) => preview.execution.cancellation.as_ref(), ApplicationOutcome::Effect(effect) => effect.execution.cancellation.as_ref(), + ApplicationOutcome::Result(_) => return false, }; let Some(observation) = observation else { return true; @@ -116,6 +96,9 @@ fn configuration_cancellation_is_legal( /// Validate the transport serialization carrier against the concrete result /// DTO before an adapter can publish it. +/// +/// Only configuration and `feedback_get` results have a reviewed DTO here; +/// every other operation's outcome is published as the daemon assembled it. pub(super) fn validate_application_outcome( operation: ApplicationSurfaceOperation, outcome: &ApplicationOutcome, @@ -124,10 +107,16 @@ pub(super) fn validate_application_outcome( receipt: ReceiptContract, reconciliation: ReconciliationContract, ) -> bool { + if operation != ApplicationSurfaceOperation::FeedbackGet + && !is_configuration_operation(operation) + { + return true; + } let termination = match outcome { ApplicationOutcome::Evidence(packet) => packet.execution.termination, ApplicationOutcome::Preview(preview) => preview.execution.termination, ApplicationOutcome::Effect(effect) => effect.execution.termination, + ApplicationOutcome::Result(_) => return false, }; let lifecycle_shape_is_legal = matches!( (receipt, reconciliation, outcome), @@ -235,6 +224,27 @@ mod tests { } } + #[test] + fn configuration_cancellation_policy_follows_the_catalog_effect() { + let catalog = super::super::application_surface_catalog_ref().unwrap(); + for operation in CONFIGURATION_WIRE_OPERATIONS { + let application_operation = configuration_surface_operation(operation.as_str()) + .unwrap() + .unwrap(); + let is_effect = catalog + .capability(application_operation.capability_id()) + .unwrap() + .effect() + .is_effect(); + assert_eq!( + tracedecay_daemon_protocol::application_surface_cancellation_policy(operation) + == tracedecay_daemon_protocol::InvocationCancellationPolicy::AuthoritativeEffect, + is_effect, + "{operation:?}" + ); + } + } + #[test] fn configuration_terminals_are_checked_against_the_owning_manifest() { let catalog = super::super::application_surface_catalog_ref().unwrap(); diff --git a/crates/tracedecay-daemon-service/src/application_surface/dispatch.rs b/crates/tracedecay-daemon-service/src/application_surface/dispatch.rs index 16463c3caa..a22b1b8a18 100644 --- a/crates/tracedecay-daemon-service/src/application_surface/dispatch.rs +++ b/crates/tracedecay-daemon-service/src/application_surface/dispatch.rs @@ -7,20 +7,18 @@ use tracedecay_api::{ CanonicalInvocationResult, HttpApplicationInvocationFuture, HttpApplicationRequest, }; use tracedecay_contracts::catalog_composition::ApplicationCatalogComposition; -use tracedecay_contracts::feedback::observations::{FeedbackOutcomeV1, FeedbackSourceEventV1}; -use tracedecay_contracts::retrieval::PrimitiveRequest; use tracedecay_contracts::{ - APPLICATION_DEFAULT_PROFILE_ID, ApplicationContractError, ApplicationEnvelope, - ApplicationProblem, ApplicationProblemEnvelope, CancellationSignal, Deadline, PageRequest, - RequestId, ResultContractRef, SafeDiagnostic, + APPLICATION_DEFAULT_PROFILE_ID, ApplicationContractError, ApplicationProblem, + ApplicationProblemEnvelope, CancellationSignal, Deadline, PageRequest, RequestId, + ResultContractRef, SafeDiagnostic, }; use tracedecay_daemon_protocol::{ ApplicationSurfaceAdapterError, ApplicationSurfaceInvocationResult, ApplicationSurfaceRequest, - BindingResolution, CatalogBindingResolver, DaemonInvocationError, DispatchInput, - DispatchedInvocation, InvocationCancellationPolicy, InvocationControls, RequestedOutputFormat, - ScopeSelector, parse_application_surface_request, resolve_dispatch, + BindingResolution, CatalogBindingResolver, DispatchInput, DispatchedInvocation, + InvocationControls, RequestedOutputFormat, ScopeSelector, parse_application_surface_request, + resolve_dispatch, }; -use tracedecay_domain::{UtcMicros, canonical_sha256}; +use tracedecay_domain::UtcMicros; use tracedecay_tool_catalog::{ ApplicationSurfaceOperation, BindingSurface, CatalogSnapshotV1, ProfileId, SurfaceOperationName, }; @@ -29,20 +27,14 @@ use super::catalog::{ application_negotiated_features, application_surface_catalog_ref, resolve_application_binding, validate_current_application_binding, }; -use super::configuration_wire::{ - configuration_invocation_payload, is_configuration_operation, validate_application_outcome, -}; -use super::feedback_observation::{ - feedback_delivery_route, feedback_surface_is_observable, feedback_surface_operation, - observe_surface_argument_rejection, -}; +use super::configuration_wire::validate_application_outcome; +use super::feedback_observation::observe_surface_argument_rejection; use super::problems::{ - current_micros, http_adapter_problem, invocation_contract_problem, invocation_problem, - map_dispatch_error, + current_micros, http_adapter_problem, invocation_contract_problem, map_dispatch_error, }; use super::{ APPLICATION_PROTOCOL_REVISION, CatalogBoundHttpApplicationRequest, - HttpApplicationCatalogDispatcher, retained, + HttpApplicationCatalogDispatcher, }; pub fn application_surface_dispatch_input_with_controls( @@ -90,7 +82,6 @@ pub async fn execute_application_surface( let binding_id = dispatched.invocation.binding_id.clone(); let request_id = dispatched.request_id; let surface = dispatched.surface; - let delivery_route = feedback_delivery_route(dispatched.surface); let (invocation, requested_format) = dispatched.invocation.into_application_invocation(); let observed_at = current_micros()?; let ( @@ -99,7 +90,6 @@ pub async fn execute_application_surface( terminal_states, receipt_contract, reconciliation_contract, - catalog_effect, ) = hotpath::measure_block!("application_surface.execute.catalog", { let catalog = application_surface_catalog_ref()?; let capability = catalog @@ -114,7 +104,6 @@ pub async fn execute_application_surface( capability.terminal_states().clone(), capability.receipt(), capability.reconciliation(), - capability.effect().is_effect(), ) }); let maximum_deadline_at = UtcMicros(observed_at.0.saturating_add(deadline_ceiling_micros)); @@ -125,282 +114,7 @@ pub async fn execute_application_surface( .filter(|expires_at| *expires_at <= maximum_deadline_at) .unwrap_or(maximum_deadline_at); let deadline = Deadline::new(effective_deadline_at)?; - let cancellation = invocation.cancellation; - let cancellation_context = cancellation.context(); - let resolved_scope = match &invocation.scope { - tracedecay_contracts::InvocationTarget::CurrentProject => None, - tracedecay_contracts::InvocationTarget::Resolved(scope) => Some(scope.clone()), - }; - let request_deadline = deadline.clone(); - let migrated_payload = match (&operation, &invocation.request) { - ( - ApplicationSurfaceOperation::ConfigurationGet - | ApplicationSurfaceOperation::ConfigurationSet - | ApplicationSurfaceOperation::ConfigurationUnset - | ApplicationSurfaceOperation::ConfigurationBatch, - ApplicationSurfaceRequest::Configuration(request), - ) => Some(configuration_invocation_payload(request)?), - ( - ApplicationSurfaceOperation::FeedbackGet, - ApplicationSurfaceRequest::Feedback(request), - ) => Some( - serde_json::to_value(request) - .map_err(ApplicationSurfaceAdapterError::invalid_request)?, - ), - _ => None, - }; - if let Some(payload) = migrated_payload { - let Some(executor) = executor else { - return Ok(ApplicationSurfaceInvocationResult { - operation, - binding_id, - result: Err(ApplicationProblemEnvelope::new( - result_contract, - request_id, - ApplicationProblem::unavailable(SafeDiagnostic::new( - "application.transport.unavailable", - "The daemon application transport is unavailable", - )?), - )?), - requested_format, - }); - }; - let binding = tracedecay_contracts::ApplicationInvocationBinding::new( - binding_id.clone(), - surface, - SurfaceOperationName::new(operation.name_for_surface(surface))?, - result_contract.clone(), - invocation.page, - )?; - let context = tracedecay_contracts::ApplicationInvocationContext::new( - request_id.clone(), - invocation.scope, - deadline, - cancellation, - )?; - let request = tracedecay_contracts::ApplicationRequest::surface(binding, payload)?; - let invocation = tracedecay_contracts::ApplicationInvocation::new(context, request)?; - let result = match hotpath::future!( - tracedecay_contracts::ApplicationInvocationExecutor::invoke(executor, invocation), - label = "application_surface.execute.invoke" - ) - .await - { - Ok(response) => match response - .envelope() - .filter(|envelope| { - validate_application_outcome( - operation, - &envelope.outcome, - &cancellation_contract, - &terminal_states, - receipt_contract, - reconciliation_contract, - ) - }) - .cloned() - { - Some(envelope) => Ok(envelope), - None => Err(ApplicationProblemEnvelope::new( - result_contract.clone(), - request_id.clone(), - ApplicationProblem::unavailable(SafeDiagnostic { - code: "application.surface.invalid_response".to_owned(), - message: "The daemon returned an invalid application response".to_owned(), - }), - )?), - }, - // Same dispatch-failure contract as the non-migrated arm below: an - // unreachable daemon never saw the request, so it is an error, not - // a retryable problem envelope. - Err(tracedecay_contracts::InvocationError::Unreachable { - reason_code, - detail, - }) => { - return Err(ApplicationSurfaceAdapterError::DaemonUnreachable { - reason_code, - detail, - }); - } - Err(error) => Err(ApplicationProblemEnvelope::new( - result_contract, - request_id, - invocation_contract_problem(error)?, - )?), - }; - return Ok(ApplicationSurfaceInvocationResult { - operation, - binding_id, - result, - requested_format, - }); - } - let request = hotpath::measure_block!("application_surface.execute.request_build", { - match invocation.request { - ApplicationSurfaceRequest::GitRead(request) => { - tracedecay_daemon_protocol::DaemonInvocationRequest::git_read( - request_id.as_str(), - operation, - request, - observed_at, - deadline, - cancellation_context, - ) - } - ApplicationSurfaceRequest::GitPreview(request) => { - tracedecay_daemon_protocol::DaemonInvocationRequest::git_preview( - request_id.as_str(), - request, - observed_at, - deadline, - cancellation_context, - ) - } - ApplicationSurfaceRequest::GitApply(request) => { - tracedecay_daemon_protocol::DaemonInvocationRequest::git_apply( - request_id.as_str(), - request, - observed_at, - deadline, - cancellation_context, - ) - } - ApplicationSurfaceRequest::GitHubStackSignalExpand(request) => { - tracedecay_daemon_protocol::DaemonInvocationRequest::github_stack_signal_expand( - request_id.as_str(), - request, - observed_at, - deadline, - cancellation_context, - ) - } - ApplicationSurfaceRequest::NativeIntegration(request) => { - tracedecay_daemon_protocol::DaemonInvocationRequest::native_integration( - request_id.as_str(), - operation, - request, - observed_at, - deadline, - cancellation_context, - ) - } - ApplicationSurfaceRequest::Feedback(request) => { - tracedecay_daemon_protocol::DaemonInvocationRequest::feedback( - request_id.as_str(), - operation, - request.request_handle, - observed_at, - deadline, - cancellation_context, - ) - } - ApplicationSurfaceRequest::FeedbackAdvisoryCycle(request) => { - tracedecay_daemon_protocol::DaemonInvocationRequest::feedback_advisory_cycle( - request_id.as_str(), - request.document_uri, - observed_at, - deadline, - cancellation_context, - ) - } - ApplicationSurfaceRequest::FeedbackProximity(request) => { - tracedecay_daemon_protocol::DaemonInvocationRequest::feedback_proximity( - request_id.as_str(), - request, - deadline, - cancellation_context, - ) - } - ApplicationSurfaceRequest::TestResults(_) => { - tracedecay_daemon_protocol::DaemonInvocationRequest::primitive( - request_id.as_str(), - operation, - PrimitiveRequest::RecentTestResults(invocation.page), - observed_at, - deadline, - cancellation_context, - ) - } - ApplicationSurfaceRequest::CallableCode(request) => { - tracedecay_daemon_protocol::DaemonInvocationRequest::callable_code( - request_id.as_str(), - operation, - request, - invocation.page, - observed_at, - deadline, - cancellation_context, - ) - } - ApplicationSurfaceRequest::PrimitiveCode(request) => { - tracedecay_daemon_protocol::DaemonInvocationRequest::primitive_code( - request_id.as_str(), - operation, - request, - invocation.page, - observed_at, - deadline, - cancellation_context, - ) - } - ApplicationSurfaceRequest::Primitive(request) => { - tracedecay_daemon_protocol::DaemonInvocationRequest::primitive( - request_id.as_str(), - operation, - request, - observed_at, - deadline, - cancellation_context, - ) - } - ApplicationSurfaceRequest::ObservatoryRead(request) => { - tracedecay_daemon_protocol::DaemonInvocationRequest::observatory_read( - request_id.as_str(), - request, - observed_at, - deadline, - cancellation_context, - ) - } - ApplicationSurfaceRequest::Configuration(request) => { - tracedecay_daemon_protocol::DaemonInvocationRequest::configuration( - request_id.as_str(), - operation, - request, - observed_at, - deadline, - cancellation_context, - ) - } - ApplicationSurfaceRequest::ContextScout(request) => { - tracedecay_daemon_protocol::DaemonInvocationRequest::context_scout( - request_id.as_str(), - operation, - request, - observed_at, - deadline, - cancellation_context, - ) - } - ApplicationSurfaceRequest::Retained(request) => { - tracedecay_daemon_protocol::DaemonInvocationRequest::retained_application( - request_id.as_str(), - request, - observed_at, - deadline, - cancellation_context, - ) - } - } - }); - let request = request - .with_resolved_scope(resolved_scope) - .map_err(|problem| { - ApplicationSurfaceAdapterError::invalid_request(format!( - "resolved scope was refused: {problem:?}" - )) - })? - .with_delivery_route(delivery_route); + let payload = invocation.request.into_invocation_payload()?; let Some(executor) = executor else { return Ok(ApplicationSurfaceInvocationResult { operation, @@ -416,222 +130,70 @@ pub async fn execute_application_surface( requested_format, }); }; - let policy = if (is_configuration_operation(operation) && catalog_effect) - || matches!( - operation, - ApplicationSurfaceOperation::GitApply - | ApplicationSurfaceOperation::NativeIntegrationApprove - | ApplicationSurfaceOperation::NativeIntegrationApply - | ApplicationSurfaceOperation::NativeIntegrationCancel - | ApplicationSurfaceOperation::ContextScoutPause - | ApplicationSurfaceOperation::ContextScoutResume - | ApplicationSurfaceOperation::ContextScoutCancel - | ApplicationSurfaceOperation::ContextScoutClaim - | ApplicationSurfaceOperation::ContextScoutDelivery - | ApplicationSurfaceOperation::ContextScoutFeedback - ) { - InvocationCancellationPolicy::AuthoritativeEffect - } else { - InvocationCancellationPolicy::ReadOnly - }; - let response = hotpath::future!( - executor.invoke_controlled(request, request_deadline, cancellation, policy), + let binding = tracedecay_contracts::ApplicationInvocationBinding::new( + binding_id.clone(), + surface, + SurfaceOperationName::new(operation.name_for_surface(surface))?, + result_contract.clone(), + invocation.page, + )?; + let context = tracedecay_contracts::ApplicationInvocationContext::new( + request_id.clone(), + invocation.scope, + deadline, + invocation.cancellation, + )?; + let request = tracedecay_contracts::ApplicationRequest::surface(binding, payload)?; + let invocation = tracedecay_contracts::ApplicationInvocation::new(context, request)?; + let result = match hotpath::future!( + tracedecay_contracts::ApplicationInvocationExecutor::invoke(executor, invocation), label = "application_surface.execute.invoke" ) - .await; - let response = match response { - Ok(response) => response, - Err(error) => { - // An unreachable daemon is a dispatch failure, not an answer: - // wrapping it in a retryable problem envelope made every CLI - // surface re-dispatch (and re-pay the connect grace) until its - // deadline, 128 s against a dead socket, while sibling - // compatibility tools failed typed in one grace. The feedback - // observation below rides the same dead transport, so it is - // skipped too: it would pay one more full connect grace to - // observe that the daemon it reports to is down. - if let DaemonInvocationError::Unreachable { - reason_code, - detail, - } = error - { - return Err(ApplicationSurfaceAdapterError::DaemonUnreachable { - reason_code, - detail, - }); - } - if feedback_surface_is_observable(operation) - && let Ok(subject_digest) = canonical_sha256(&( - "tracedecay.feedback.transport-observation.v1", - request_id.as_str(), - operation.as_str(), - delivery_route, - )) - && let Ok(observed_at) = current_micros() - { - let event = match &error { - DaemonInvocationError::Cancelled { .. } => { - FeedbackSourceEventV1::Cancellation { - operation: feedback_surface_operation(operation), - outcome: FeedbackOutcomeV1::Cancelled, - } - } - DaemonInvocationError::TimedOut { .. } => FeedbackSourceEventV1::Cancellation { - operation: feedback_surface_operation(operation), - outcome: FeedbackOutcomeV1::TimedOut, - }, - DaemonInvocationError::Unavailable - | DaemonInvocationError::Unreachable { .. } => { - FeedbackSourceEventV1::Delivery { - operation: feedback_surface_operation(operation), - route: delivery_route, - outcome: FeedbackOutcomeV1::Unavailable, - item_count: 0, - duration_micros: None, - } - } - }; - let _ = executor - .observe_feedback(subject_digest, observed_at, event) - .await; - } - return Ok(ApplicationSurfaceInvocationResult { - operation, - binding_id, - result: Err(ApplicationProblemEnvelope::new( - result_contract, - request_id, - error.into_application_problem(), - )?), - requested_format, - }); - } - }; - let result = hotpath::measure_block!("application_surface.execute.assemble", { - match response.outcome { - tracedecay_daemon_protocol::DaemonInvocationOutcome::GitRead { scope, result } => { - Ok(ApplicationEnvelope::evidence( - result_contract.clone(), - request_id.clone(), - scope, - result.into_application(), - )) - } - tracedecay_daemon_protocol::DaemonInvocationOutcome::GitPreview { scope, preview } => { - Ok(ApplicationEnvelope::preview( - result_contract.clone(), - request_id.clone(), - scope, - preview.into_application_result()?, - )) - } - tracedecay_daemon_protocol::DaemonInvocationOutcome::GitApply { scope, effect } => { - Ok(ApplicationEnvelope::effect( - result_contract.clone(), - request_id.clone(), - scope, - effect.into_application_result()?, - )) - } - tracedecay_daemon_protocol::DaemonInvocationOutcome::Feedback { scope, result } - | tracedecay_daemon_protocol::DaemonInvocationOutcome::Primitive { scope, result } - | tracedecay_daemon_protocol::DaemonInvocationOutcome::ObservatoryRead { - scope, - result, - } => Ok(ApplicationEnvelope::evidence( - result_contract.clone(), - request_id.clone(), - scope, - result.into_application(), - )), - tracedecay_daemon_protocol::DaemonInvocationOutcome::CallableCode { scope, result } => { - Ok(ApplicationEnvelope::evidence( - result_contract.clone(), - request_id.clone(), - scope, - result.into_application(), - )) - } - tracedecay_daemon_protocol::DaemonInvocationOutcome::Configuration { - scope, - outcome, - } => { - if validate_application_outcome( + .await + { + Ok(response) => match response + .envelope() + .filter(|envelope| { + validate_application_outcome( operation, - &outcome, + &envelope.outcome, &cancellation_contract, &terminal_states, receipt_contract, reconciliation_contract, - ) { - Ok(ApplicationEnvelope { - contract: result_contract.clone(), - request_id: request_id.clone(), - scope, - outcome, - }) - } else { - Err(ApplicationProblemEnvelope::new( - result_contract.clone(), - request_id.clone(), - ApplicationProblem::unavailable(SafeDiagnostic::new( - "application.surface.invalid_configuration_response", - "The daemon returned a configuration result that did not match its wire contract", - )?), - )?) - } - } - tracedecay_daemon_protocol::DaemonInvocationOutcome::GitHubStackSignalExpand { - scope, - outcome, - } - | tracedecay_daemon_protocol::DaemonInvocationOutcome::NativeIntegration { - scope, - outcome, - } - | tracedecay_daemon_protocol::DaemonInvocationOutcome::ContextScout { - scope, - outcome, - } => Ok(ApplicationEnvelope { - contract: result_contract.clone(), - request_id: request_id.clone(), - scope, - outcome, - }), - tracedecay_daemon_protocol::DaemonInvocationOutcome::RetainedApplication { - scope, - outcome, - } => Ok(ApplicationEnvelope { - contract: result_contract.clone(), - request_id: request_id.clone(), - scope, - outcome: retained::outcome_value(outcome)?, - }), - tracedecay_daemon_protocol::DaemonInvocationOutcome::ApplicationProblem { problem } => { - Err(ApplicationProblemEnvelope::new( - result_contract.clone(), - request_id.clone(), - problem, - )?) - } - tracedecay_daemon_protocol::DaemonInvocationOutcome::Problem { problem } => { - Err(ApplicationProblemEnvelope::new( - result_contract.clone(), - request_id.clone(), - invocation_problem(problem)?, - )?) - } - _ => Err(ApplicationProblemEnvelope::new( + ) + }) + .cloned() + { + Some(envelope) => Ok(envelope), + None => Err(ApplicationProblemEnvelope::new( result_contract.clone(), request_id.clone(), - ApplicationProblem::unavailable(SafeDiagnostic::new( - "application.surface.invalid_response", - "The daemon returned an invalid application response", - )?), + ApplicationProblem::unavailable(SafeDiagnostic { + code: "application.surface.invalid_response".to_owned(), + message: "The daemon returned an invalid application response".to_owned(), + }), )?), + }, + // An unreachable daemon never saw the request: it is a dispatch + // failure, not a retryable problem envelope. Wrapping it made every + // CLI surface re-dispatch (and re-pay the connect grace) until its + // deadline, 128 s against a dead socket. + Err(tracedecay_contracts::InvocationError::Unreachable { + reason_code, + detail, + }) => { + return Err(ApplicationSurfaceAdapterError::DaemonUnreachable { + reason_code, + detail, + }); } - }); - + Err(error) => Err(ApplicationProblemEnvelope::new( + result_contract, + request_id, + invocation_contract_problem(error)?, + )?), + }; Ok(ApplicationSurfaceInvocationResult { operation, binding_id, diff --git a/crates/tracedecay-daemon-service/src/application_surface/feedback_observation.rs b/crates/tracedecay-daemon-service/src/application_surface/feedback_observation.rs index 49b221b265..367636db16 100644 --- a/crates/tracedecay-daemon-service/src/application_surface/feedback_observation.rs +++ b/crates/tracedecay-daemon-service/src/application_surface/feedback_observation.rs @@ -2,136 +2,18 @@ use tracedecay_contracts::RequestId; use tracedecay_contracts::feedback::observations::{ - FeedbackArgumentRejectionClassV1, FeedbackDeliveryRouteV1, FeedbackOperationV1, - FeedbackOutcomeV1, FeedbackRejectedArgumentV1, FeedbackSourceEventV1, + FeedbackArgumentRejectionClassV1, FeedbackOutcomeV1, FeedbackRejectedArgumentV1, + FeedbackSourceEventV1, +}; +use tracedecay_daemon_protocol::{ + ApplicationSurfaceAdapterError, application_delivery_route, + application_surface_feedback_is_observable, application_surface_feedback_operation, }; -use tracedecay_daemon_protocol::ApplicationSurfaceAdapterError; use tracedecay_domain::canonical_sha256; use tracedecay_tool_catalog::{ApplicationSurfaceOperation, BindingSurface}; use super::problems::current_micros; -pub(super) fn feedback_delivery_route(surface: BindingSurface) -> FeedbackDeliveryRouteV1 { - match surface { - BindingSurface::Cli => FeedbackDeliveryRouteV1::Cli, - BindingSurface::Mcp => FeedbackDeliveryRouteV1::Mcp, - BindingSurface::Http | BindingSurface::Dashboard => FeedbackDeliveryRouteV1::Http, - BindingSurface::Lsp => FeedbackDeliveryRouteV1::Lsp, - } -} - -pub(super) fn feedback_surface_operation( - operation: ApplicationSurfaceOperation, -) -> FeedbackOperationV1 { - match operation { - ApplicationSurfaceOperation::FeedbackDiagnostics => { - FeedbackOperationV1::FeedbackDiagnostics - } - ApplicationSurfaceOperation::FeedbackGet => FeedbackOperationV1::FeedbackGet, - ApplicationSurfaceOperation::FeedbackExpand => FeedbackOperationV1::FeedbackExpand, - ApplicationSurfaceOperation::FeedbackList => FeedbackOperationV1::FeedbackList, - ApplicationSurfaceOperation::FeedbackAdvisoryCycle => FeedbackOperationV1::FeedbackCycle, - ApplicationSurfaceOperation::FeedbackProximity => FeedbackOperationV1::Proximity, - ApplicationSurfaceOperation::FeedbackImpact => FeedbackOperationV1::PrimitiveImpact, - ApplicationSurfaceOperation::AffectedTests => FeedbackOperationV1::PrimitiveAffectedTests, - ApplicationSurfaceOperation::TestResults => FeedbackOperationV1::PrimitiveTestResults, - ApplicationSurfaceOperation::GitStatus - | ApplicationSurfaceOperation::GitDiff - | ApplicationSurfaceOperation::GitHistory - | ApplicationSurfaceOperation::GitBlame - | ApplicationSurfaceOperation::GitHunks - | ApplicationSurfaceOperation::GitPreview - | ApplicationSurfaceOperation::GitApply - | ApplicationSurfaceOperation::GitHubStackSignalExpand - | ApplicationSurfaceOperation::NativeIntegrationStackSnapshot - | ApplicationSurfaceOperation::NativeIntegrationPreflight - | ApplicationSurfaceOperation::NativeIntegrationApprove - | ApplicationSurfaceOperation::NativeIntegrationApply - | ApplicationSurfaceOperation::NativeIntegrationStatus - | ApplicationSurfaceOperation::NativeIntegrationCancel - | ApplicationSurfaceOperation::NativeIntegrationWorktreeInventory - | ApplicationSurfaceOperation::NativeIntegrationWorktreeInspect - | ApplicationSurfaceOperation::NativeIntegrationWorktreeConfirm - | ApplicationSurfaceOperation::NativeIntegrationWorktreeRemove - | ApplicationSurfaceOperation::NativeIntegrationWorktreeReconcile - | ApplicationSurfaceOperation::CodeExactOccurrence - | ApplicationSurfaceOperation::CodePhraseSearch - | ApplicationSurfaceOperation::CodeSymbolSearch - | ApplicationSurfaceOperation::CodeSignatureSearch - | ApplicationSurfaceOperation::CodeImplementations - | ApplicationSurfaceOperation::CodeTypeHierarchy - | ApplicationSurfaceOperation::CodeCallers - | ApplicationSurfaceOperation::CodeCallees - | ApplicationSurfaceOperation::CodeFacets - | ApplicationSurfaceOperation::CodeTimeline - | ApplicationSurfaceOperation::CodeDeclaration - | ApplicationSurfaceOperation::CodeTypeDefinition - | ApplicationSurfaceOperation::CodeReferences - | ApplicationSurfaceOperation::SessionLookup - | ApplicationSurfaceOperation::QualifiedName - | ApplicationSurfaceOperation::CallChain - | ApplicationSurfaceOperation::FileDependents - | ApplicationSurfaceOperation::SourceLines - | ApplicationSurfaceOperation::SourceBody - | ApplicationSurfaceOperation::SourceOutline - | ApplicationSurfaceOperation::ModuleApi - | ApplicationSurfaceOperation::HealthRead - | ApplicationSurfaceOperation::HealthDelta - | ApplicationSurfaceOperation::StorageStatus - | ApplicationSurfaceOperation::DiagnosticsRead - | ApplicationSurfaceOperation::ObservatoryRead - | ApplicationSurfaceOperation::ConfigurationList - | ApplicationSurfaceOperation::ConfigurationGet - | ApplicationSurfaceOperation::ConfigurationSet - | ApplicationSurfaceOperation::ConfigurationUnset - | ApplicationSurfaceOperation::ConfigurationBatch - | ApplicationSurfaceOperation::ConfigurationObservedState - | ApplicationSurfaceOperation::ConfigurationProtectedPreview - | ApplicationSurfaceOperation::ConfigurationProtectedApply - | ApplicationSurfaceOperation::ConfigurationRollbackPreview - | ApplicationSurfaceOperation::ConfigurationRollbackApply - | ApplicationSurfaceOperation::ConfigurationAudit - | ApplicationSurfaceOperation::ContextScoutStatus - | ApplicationSurfaceOperation::ContextScoutRecent - | ApplicationSurfaceOperation::ContextScoutExplain - | ApplicationSurfaceOperation::ContextScoutCapability - | ApplicationSurfaceOperation::ContextScoutBudget - | ApplicationSurfaceOperation::ContextScoutPause - | ApplicationSurfaceOperation::ContextScoutResume - | ApplicationSurfaceOperation::ContextScoutCancel - | ApplicationSurfaceOperation::ContextScoutClaim - | ApplicationSurfaceOperation::ContextScoutDelivery - | ApplicationSurfaceOperation::ContextScoutFeedback => FeedbackOperationV1::FeedbackCycle, - } -} - -pub(super) fn feedback_surface_is_observable(operation: ApplicationSurfaceOperation) -> bool { - matches!( - operation, - ApplicationSurfaceOperation::FeedbackDiagnostics - | ApplicationSurfaceOperation::FeedbackGet - | ApplicationSurfaceOperation::FeedbackExpand - | ApplicationSurfaceOperation::FeedbackList - | ApplicationSurfaceOperation::FeedbackAdvisoryCycle - | ApplicationSurfaceOperation::FeedbackProximity - | ApplicationSurfaceOperation::FeedbackImpact - | ApplicationSurfaceOperation::AffectedTests - | ApplicationSurfaceOperation::TestResults - | ApplicationSurfaceOperation::SessionLookup - | ApplicationSurfaceOperation::QualifiedName - | ApplicationSurfaceOperation::CallChain - | ApplicationSurfaceOperation::FileDependents - | ApplicationSurfaceOperation::SourceLines - | ApplicationSurfaceOperation::SourceBody - | ApplicationSurfaceOperation::SourceOutline - | ApplicationSurfaceOperation::ModuleApi - | ApplicationSurfaceOperation::HealthRead - | ApplicationSurfaceOperation::HealthDelta - | ApplicationSurfaceOperation::StorageStatus - | ApplicationSurfaceOperation::DiagnosticsRead - ) -} - pub async fn observe_surface_argument_rejection( executor: Option<&dyn tracedecay_daemon_protocol::DaemonInvocationExecutor>, surface: BindingSurface, @@ -139,7 +21,7 @@ pub async fn observe_surface_argument_rejection( request_id: &RequestId, error: &ApplicationSurfaceAdapterError, ) { - if !feedback_surface_is_observable(operation) { + if !application_surface_feedback_is_observable(operation) { return; } let Some((argument, rejection, outcome)) = surface_rejection_metadata(error) else { @@ -162,8 +44,8 @@ pub async fn observe_surface_argument_rejection( subject_digest, observed_at, FeedbackSourceEventV1::SurfaceArgumentRejected { - operation: feedback_surface_operation(operation), - route: Some(feedback_delivery_route(surface)), + operation: application_surface_feedback_operation(operation), + route: Some(application_delivery_route(surface)), argument, rejection, schema_revision: 1, diff --git a/crates/tracedecay-daemon-service/src/application_surface/handoff.rs b/crates/tracedecay-daemon-service/src/application_surface/handoff.rs index e8e450ca5a..a20518e365 100644 --- a/crates/tracedecay-daemon-service/src/application_surface/handoff.rs +++ b/crates/tracedecay-daemon-service/src/application_surface/handoff.rs @@ -68,7 +68,7 @@ async fn invoke_operation( tracedecay_daemon_protocol::DaemonInvocationRequest::handoff_application( request_id.as_str(), HandoffApplicationInvocationV1::IssueTaskHandoff(decoded), - tracedecay_daemon_protocol::invocation_now_micros(), + tracedecay_contracts::now_micros(), controls.deadline.clone(), controls.cancellation.context(), ); @@ -102,7 +102,7 @@ async fn invoke_operation( tracedecay_daemon_protocol::DaemonInvocationRequest::handoff_application( request_id.as_str(), HandoffApplicationInvocationV1::ListTaskHandoffs(decoded), - tracedecay_daemon_protocol::invocation_now_micros(), + tracedecay_contracts::now_micros(), controls.deadline.clone(), controls.cancellation.context(), ); @@ -140,7 +140,7 @@ async fn invoke_operation( tracedecay_daemon_protocol::DaemonInvocationRequest::handoff_application( request_id.as_str(), HandoffApplicationInvocationV1::OpenInvestigationHandoff(decoded), - tracedecay_daemon_protocol::invocation_now_micros(), + tracedecay_contracts::now_micros(), controls.deadline.clone(), controls.cancellation.context(), ); @@ -174,7 +174,7 @@ async fn invoke_operation( tracedecay_daemon_protocol::DaemonInvocationRequest::handoff_application( request_id.as_str(), HandoffApplicationInvocationV1::OpenTaskHandoff(decoded), - tracedecay_daemon_protocol::invocation_now_micros(), + tracedecay_contracts::now_micros(), controls.deadline.clone(), controls.cancellation.context(), ); diff --git a/crates/tracedecay-daemon-service/src/application_surface/multi_root_http.rs b/crates/tracedecay-daemon-service/src/application_surface/multi_root_http.rs index b8b2734801..b348013fc1 100644 --- a/crates/tracedecay-daemon-service/src/application_surface/multi_root_http.rs +++ b/crates/tracedecay-daemon-service/src/application_surface/multi_root_http.rs @@ -106,7 +106,7 @@ async fn invoke_operation( controls, body, } = request; - let observed_at = tracedecay_daemon_protocol::invocation_now_micros(); + let observed_at = tracedecay_contracts::now_micros(); match operation { MultiRootHttpOperation::ScopeSetRead => { let Ok(decoded) = serde_json::from_value::(body) else { diff --git a/crates/tracedecay-daemon-service/src/application_surface/operation_events.rs b/crates/tracedecay-daemon-service/src/application_surface/operation_events.rs index c945cff93a..ec02c3399a 100644 --- a/crates/tracedecay-daemon-service/src/application_surface/operation_events.rs +++ b/crates/tracedecay-daemon-service/src/application_surface/operation_events.rs @@ -51,12 +51,14 @@ impl Drop for SseDisconnectObserver { } let executor = Arc::clone(&self.executor); let subject = self.subject.clone(); - if let Ok(runtime) = tokio::runtime::Handle::try_current() { + if let (Ok(runtime), Ok(observed_at)) = + (tokio::runtime::Handle::try_current(), current_micros()) + { runtime.spawn(async move { let _ = executor .observe_feedback( subject, - current_micros().unwrap_or(UtcMicros(1)), + observed_at, FeedbackSourceEventV1::SseLifecycle { lifecycle: FeedbackSseLifecycleV1::Disconnected, sequence: None, @@ -363,13 +365,22 @@ pub(super) async fn http_operation_events( Query(query): Query, ) -> Response { let observation_subject = sse_observation_subject(&request_id, &operation_id); + let observed_at = match current_micros() { + Ok(observed_at) => observed_at, + Err(error) => { + return operation_event_problem( + &request_id, + OperationEventError::InvalidContext(error.to_string()), + ); + } + }; let operation_id = if let Ok(operation_id) = RequestId::new(operation_id) { OperationId::from_request(operation_id) } else { emit_http_feedback_observation( &state, observation_subject.as_ref(), - current_micros().unwrap_or(UtcMicros(1)), + observed_at, FeedbackSourceEventV1::SurfaceArgumentRejected { operation: FeedbackOperationV1::SseStream, route: Some(FeedbackDeliveryRouteV1::Http), @@ -386,15 +397,6 @@ pub(super) async fn http_operation_events( Ok(next_sequence) => next_sequence, Err(error) => return operation_event_problem(&request_id, error), }; - let observed_at = match current_micros() { - Ok(observed_at) => observed_at, - Err(error) => { - return operation_event_problem( - &request_id, - OperationEventError::InvalidContext(error.to_string()), - ); - } - }; // Same owner rule as cancellation: this authority answers for the // operations it began, and only an operation it does not own is delegated // to the daemon executor. A resume token is always redeemed locally, the @@ -536,11 +538,14 @@ pub(super) async fn http_operation_events( if is_terminal { observer.terminal.store(true, Ordering::Relaxed); } + let Ok(observed_at) = current_micros() else { + return event; + }; let _ = observer .executor .observe_feedback( observer.subject.clone(), - current_micros().unwrap_or(UtcMicros(1)), + observed_at, FeedbackSourceEventV1::SseLifecycle { lifecycle, sequence: Some(event.sequence), @@ -644,13 +649,22 @@ pub(super) async fn http_operation_cancel( Extension(controls): Extension, ) -> Response { let observation_subject = sse_observation_subject(&request_id, &operation_id); + let observed_at = match current_micros() { + Ok(observed_at) => observed_at, + Err(error) => { + return operation_event_problem( + &request_id, + OperationEventError::InvalidContext(error.to_string()), + ); + } + }; let operation_id = if let Ok(operation_id) = RequestId::new(operation_id) { OperationId::from_request(operation_id) } else { emit_http_feedback_observation( &state, observation_subject.as_ref(), - current_micros().unwrap_or(UtcMicros(1)), + observed_at, FeedbackSourceEventV1::SurfaceArgumentRejected { operation: FeedbackOperationV1::SseStream, route: Some(FeedbackDeliveryRouteV1::Http), @@ -663,15 +677,6 @@ pub(super) async fn http_operation_cancel( .await; return operation_event_problem(&request_id, OperationEventError::NotFoundOrNotAuthorized); }; - let observed_at = match current_micros() { - Ok(observed_at) => observed_at, - Err(error) => { - return operation_event_problem( - &request_id, - OperationEventError::InvalidContext(error.to_string()), - ); - } - }; // The canonical owner of an operation is whichever authority began it. The // daemon mounts these routes with its *own* process-global authority and an // invocation client pointed back at its own socket, so delegating first diff --git a/crates/tracedecay-daemon-service/src/application_surface/problems.rs b/crates/tracedecay-daemon-service/src/application_surface/problems.rs index 108ddadc58..4c2f97abd3 100644 --- a/crates/tracedecay-daemon-service/src/application_surface/problems.rs +++ b/crates/tracedecay-daemon-service/src/application_surface/problems.rs @@ -144,41 +144,6 @@ pub(crate) fn current_micros() -> Result Result { - Ok(match problem { - tracedecay_daemon_protocol::DaemonInvocationProblem::InvalidRequest - | tracedecay_daemon_protocol::DaemonInvocationProblem::UnsupportedRevision => { - ApplicationProblem::invalid_request_without_action( - "application.surface.invalid_request", - "The daemon rejected the application request", - ) - } - tracedecay_daemon_protocol::DaemonInvocationProblem::NotFoundOrNotAuthorized => { - ApplicationProblem::not_found_or_not_authorized(RetryDirective::Never) - } - tracedecay_daemon_protocol::DaemonInvocationProblem::ResetRequired => { - ApplicationProblem::reset_required(SafeDiagnostic::new( - "application.surface.reset_required", - "The application store requires an explicit reset", - )?) - } - tracedecay_daemon_protocol::DaemonInvocationProblem::ApplicationContractViolation => { - ApplicationProblem::unavailable(SafeDiagnostic::new( - "application.surface.contract_violation", - "The application result violated its canonical contract", - )?) - } - tracedecay_daemon_protocol::DaemonInvocationProblem::Unavailable => { - ApplicationProblem::unavailable(SafeDiagnostic::new( - "application.surface.unavailable", - "The application service for this operation is unavailable", - )?) - } - }) -} - pub(super) fn invocation_contract_problem( error: tracedecay_contracts::InvocationError, ) -> Result { diff --git a/crates/tracedecay-daemon-service/src/application_surface/registered_http.rs b/crates/tracedecay-daemon-service/src/application_surface/registered_http.rs index 31ad79be98..37b5dcf2ce 100644 --- a/crates/tracedecay-daemon-service/src/application_surface/registered_http.rs +++ b/crates/tracedecay-daemon-service/src/application_surface/registered_http.rs @@ -421,6 +421,9 @@ where request_id, scope, outcome, + touched_files: Vec::new(), + code_graph: None, + analytics: None, }), ) .into_http_response(); diff --git a/crates/tracedecay-daemon-service/src/application_surface/retained.rs b/crates/tracedecay-daemon-service/src/application_surface/retained.rs index 4566d3a7b1..095b38ae9f 100644 --- a/crates/tracedecay-daemon-service/src/application_surface/retained.rs +++ b/crates/tracedecay-daemon-service/src/application_surface/retained.rs @@ -1,52 +1,10 @@ -//! HTTP owner for the canonical retained application operations. - -use std::sync::Arc; +//! Retained application results and the HTTP replay-collision terminal. use axum::response::{IntoResponse, Response}; -use tracedecay_contracts::retained_surfaces::{ - FactFeedbackRequestV1, FactStoreAddRequestV1, FactStoreContradictRequestV1, - FactStoreCurateRequestV1, FactStoreGetRequestV1, FactStoreListRequestV1, - FactStoreProbeRequestV1, FactStoreReasonRequestV1, FactStoreRelatedRequestV1, - FactStoreRemoveRequestV1, FactStoreSearchRequestV1, FactStoreSupersedeRequestV1, - FactStoreUpdateRequestV1, LcmDescribeRequestV1, LcmDoctorRequestV1, LcmExpandQueryRequestV1, - LcmExpandRequestV1, LcmGrepRequestV1, LcmLoadSessionRequestV1, LcmStatusRequestV1, - MemoryStatusRequestV1, MessageSearchRequestV1, RetainedSurfaceOperation, - RetainedSurfaceRequestV1, RetainedSurfaceResultV1, SessionRefreshActionRequestV1, - SessionRefreshActionV1, SessionRefreshRequestV1, SessionsForRequestV1, WorkflowsRequestV1, -}; -use tracedecay_tool_catalog::RouteExposureV1; +use tracedecay_contracts::retained_surfaces::RetainedSurfaceResultV1; +use tracedecay_tool_catalog::{ApplicationSurfaceOperation, RouteExposureV1}; -use super::registered_http::{RegisteredHttpOperation, invoke_registered_http}; -use super::require_public_catalog_route; use tracedecay_daemon_protocol::ApplicationSurfaceAdapterError; -use tracedecay_daemon_protocol::DaemonInvocationExecutor; -use tracedecay_daemon_protocol::{DaemonInvocationOutcome, DaemonInvocationRequest}; - -pub(super) fn router_with_executor( - executor: Arc, -) -> Result { - validate_catalog_bindings()?; - Ok(tracedecay_api::retained_application_router( - RetainedExecutorOwner { executor }, - )) -} - -fn validate_catalog_bindings() -> Result<(), ApplicationSurfaceAdapterError> { - let registry = tracedecay_contracts::retained_surface_executable_binding_registry() - .map_err(ApplicationSurfaceAdapterError::Contract)?; - for operation in RetainedSurfaceOperation::CALLABLE { - let operation_id = tracedecay_tool_catalog::OperationId::new( - tracedecay_api::retained_operation_id(operation), - ) - .map_err(ApplicationSurfaceAdapterError::Identifier)?; - require_public_catalog_route( - ®istry, - &operation_id, - &tracedecay_api::retained_application_route_path(operation), - )?; - } - Ok(()) -} pub(super) fn active_request_conflict_response( request_id: tracedecay_contracts::RequestId, @@ -63,10 +21,13 @@ fn active_request_conflict( tracedecay_api::CanonicalInvocationResult, ApplicationSurfaceAdapterError, > { - let operation = RetainedSurfaceOperation::FactStoreCurate; - let registry = operation.registry()?; - let operation_id = tracedecay_tool_catalog::OperationId::new(operation.operation_id()) - .map_err(ApplicationSurfaceAdapterError::Identifier)?; + let operation = ApplicationSurfaceOperation::FactStoreCurate; + let registry = tracedecay_contracts::application_http_executable_binding_registry()?; + let operation_id = tracedecay_tool_catalog::OperationId::new(format!( + "operation.application.{}", + operation.as_str() + )) + .map_err(ApplicationSurfaceAdapterError::Identifier)?; let binding = registry .get(&operation_id) .and_then(|availability| availability.binding()) @@ -97,236 +58,6 @@ fn active_request_conflict( )) } -#[cfg(test)] -mod conflict_tests { - use super::active_request_conflict; - - #[test] - fn active_replay_collision_preserves_same_request_retry_authority() { - let request_id = - tracedecay_contracts::RequestId::new("request.sdk.curate").expect("request id"); - let envelope = serde_json::to_value( - active_request_conflict(request_id) - .expect("conflict") - .into_http_json(), - ) - .expect("wire envelope"); - assert_eq!(envelope["value"]["problem"]["retry"], "same_request"); - assert_eq!(envelope["value"]["problem"]["retry_scope"], "same_request"); - assert_eq!( - envelope["value"]["problem"]["legal_actions"], - serde_json::json!(["retry"]) - ); - } -} - -impl RegisteredHttpOperation for RetainedSurfaceOperation { - fn operation_id(self) -> String { - tracedecay_api::retained_operation_id(self) - } - - fn is_read_only(self) -> bool { - !tracedecay_contracts::retained_surfaces::retained_surface_operation_is_effect(self) - } - - fn problem_family(self) -> &'static str { - "retained" - } - - fn display_family(self) -> &'static str { - "retained" - } - - fn application_problem_is_bound( - self, - request_id: &tracedecay_contracts::RequestId, - scope: Option<&tracedecay_contracts::ResolvedScope>, - problem: &tracedecay_contracts::ApplicationProblem, - ) -> bool { - tracedecay_contracts::retained_surface_problem_matches_terminal( - self, request_id, scope, problem, - ) - } - - fn registry( - self, - ) -> Result< - std::borrow::Cow<'static, tracedecay_tool_catalog::ExecutableBindingRegistryV1>, - ApplicationSurfaceAdapterError, - > { - tracedecay_contracts::retained_surface_executable_binding_registry() - .map(std::borrow::Cow::Owned) - .map_err(ApplicationSurfaceAdapterError::Contract) - } -} - -#[derive(Clone)] -struct RetainedExecutorOwner { - executor: Arc, -} - -impl tracedecay_api::RetainedApplicationOwner for RetainedExecutorOwner { - fn invoke_retained( - &self, - request: tracedecay_api::RetainedHttpRequest, - ) -> tracedecay_api::RetainedInvocationFuture { - Box::pin(invoke_operation(Arc::clone(&self.executor), request)) - } -} - -#[hotpath::measure(label = "application_surface.retained.invoke", future = true)] -async fn invoke_operation( - executor: Arc, - request: tracedecay_api::RetainedHttpRequest, -) -> Response { - let tracedecay_api::RetainedHttpRequest { - operation, - request_id, - controls, - body, - } = request; - let Ok(request) = decode_request(operation, body) else { - return tracedecay_api::retained_invalid_request_response(request_id); - }; - let invocation = DaemonInvocationRequest::retained_application( - request_id.as_str(), - request, - tracedecay_daemon_protocol::invocation_now_micros(), - controls.deadline.clone(), - controls.cancellation.context(), - ); - let selected_request_id = request_id.clone(); - invoke_registered_http::( - executor.as_ref(), - operation, - request_id, - controls, - invocation, - move |outcome| match outcome { - DaemonInvocationOutcome::RetainedApplication { scope, outcome } => { - tracedecay_contracts::retained_surface_outcome_matches_terminal( - operation, - &selected_request_id, - &scope, - &outcome, - ) - .then_some((scope, outcome)) - } - _ => None, - }, - ) - .await -} - -/// Decode one retained operation body into its typed request. -/// -/// HTTP decodes the route body directly; MCP and the `tracedecay tool` CLI -/// decode the transport-normalized arguments through the same function, so -/// every surface lands on one canonical request. The returned error carries -/// the exact serde diagnostic (unknown field, unknown enum variant with the -/// admitted values, wrong type) so every dispatch surface can hand the caller -/// a corrective message instead of a blank "invalid request". -#[hotpath::measure(label = "application_surface.retained.decode")] -pub fn decode_request( - operation: RetainedSurfaceOperation, - body: serde_json::Value, -) -> Result { - macro_rules! decode { - ($request:ty, $variant:ident) => { - serde_path_to_error::deserialize::<_, $request>(body) - .map(RetainedSurfaceRequestV1::$variant) - .map_err(named_argument_error) - }; - } - match operation { - RetainedSurfaceOperation::FactStoreCurate => { - decode!(FactStoreCurateRequestV1, FactStoreCurate) - } - RetainedSurfaceOperation::FactStoreAdd => { - decode!(FactStoreAddRequestV1, FactStoreAdd) - } - RetainedSurfaceOperation::FactStoreSearch => { - decode!(FactStoreSearchRequestV1, FactStoreSearch) - } - RetainedSurfaceOperation::FactStoreProbe => { - decode!(FactStoreProbeRequestV1, FactStoreProbe) - } - RetainedSurfaceOperation::FactStoreRelated => { - decode!(FactStoreRelatedRequestV1, FactStoreRelated) - } - RetainedSurfaceOperation::FactStoreReason => { - decode!(FactStoreReasonRequestV1, FactStoreReason) - } - RetainedSurfaceOperation::FactStoreContradict => { - decode!(FactStoreContradictRequestV1, FactStoreContradict) - } - RetainedSurfaceOperation::FactStoreGet => { - decode!(FactStoreGetRequestV1, FactStoreGet) - } - RetainedSurfaceOperation::FactStoreUpdate => { - decode!(FactStoreUpdateRequestV1, FactStoreUpdate) - } - RetainedSurfaceOperation::FactStoreRemove => { - decode!(FactStoreRemoveRequestV1, FactStoreRemove) - } - RetainedSurfaceOperation::FactStoreSupersede => { - decode!(FactStoreSupersedeRequestV1, FactStoreSupersede) - } - RetainedSurfaceOperation::FactStoreList => { - decode!(FactStoreListRequestV1, FactStoreList) - } - RetainedSurfaceOperation::FactFeedback => decode!(FactFeedbackRequestV1, FactFeedback), - RetainedSurfaceOperation::MemoryStatus => decode!(MemoryStatusRequestV1, MemoryStatus), - RetainedSurfaceOperation::SessionRefreshStatus => { - decode_session_refresh(body, SessionRefreshActionV1::Status) - } - RetainedSurfaceOperation::SessionRefreshCancel => { - decode_session_refresh(body, SessionRefreshActionV1::Cancel) - } - RetainedSurfaceOperation::SessionRefreshBegin => { - decode_session_refresh(body, SessionRefreshActionV1::Begin) - } - RetainedSurfaceOperation::MessageSearch => decode!(MessageSearchRequestV1, MessageSearch), - RetainedSurfaceOperation::SessionsFor => decode!(SessionsForRequestV1, SessionsFor), - RetainedSurfaceOperation::Workflows => decode!(WorkflowsRequestV1, Workflows), - RetainedSurfaceOperation::LcmStatus => decode!(LcmStatusRequestV1, LcmStatus), - RetainedSurfaceOperation::LcmDoctor => decode!(LcmDoctorRequestV1, LcmDoctor), - RetainedSurfaceOperation::LcmLoadSession => { - decode!(LcmLoadSessionRequestV1, LcmLoadSession) - } - RetainedSurfaceOperation::LcmGrep => decode!(LcmGrepRequestV1, LcmGrep), - RetainedSurfaceOperation::LcmDescribe => decode!(LcmDescribeRequestV1, LcmDescribe), - RetainedSurfaceOperation::LcmExpand => decode!(LcmExpandRequestV1, LcmExpand), - RetainedSurfaceOperation::LcmExpandQuery => { - decode!(LcmExpandQueryRequestV1, LcmExpandQuery) - } - } -} - -fn decode_session_refresh( - body: serde_json::Value, - action: SessionRefreshActionV1, -) -> Result { - let request = serde_path_to_error::deserialize::<_, SessionRefreshActionRequestV1>(body) - .map_err(named_argument_error)?; - Ok(RetainedSurfaceRequestV1::SessionRefresh( - SessionRefreshRequestV1::with_action(action, request), - )) -} - -/// Prefix the serde diagnostic with the offending argument path, so the -/// corrective message names the argument even for wrong-type errors, which -/// serde alone reports without the field. -fn named_argument_error(error: serde_path_to_error::Error) -> serde_json::Error { - let path = error.path().to_string(); - let inner = error.into_inner(); - if path == "." { - inner - } else { - serde::de::Error::custom(format!("{path}: {inner}")) - } -} - pub fn result_value( result: tracedecay_contracts::ApplicationResult, ) -> Result< @@ -338,107 +69,35 @@ pub fn result_value( contract: envelope.contract, request_id: envelope.request_id, scope: envelope.scope, - outcome: outcome_value(envelope.outcome)?, + outcome: tracedecay_daemon_protocol::application_outcome_value(envelope.outcome) + .map_err(ApplicationSurfaceAdapterError::invalid_request)?, + touched_files: envelope.touched_files, + code_graph: envelope.code_graph, + analytics: envelope.analytics, })), Err(problem) => Ok(Err(problem)), } } -pub(super) fn outcome_value( - outcome: tracedecay_contracts::ApplicationOutcome, -) -> Result< - tracedecay_contracts::ApplicationOutcome, - ApplicationSurfaceAdapterError, -> { - use tracedecay_contracts::ApplicationOutcome; - - fn payload( - payload: Option, - ) -> Result, ApplicationSurfaceAdapterError> { - payload - .map(serde_json::to_value) - .transpose() - .map_err(ApplicationSurfaceAdapterError::invalid_request) - } - - Ok(match outcome { - ApplicationOutcome::Evidence(packet) => { - ApplicationOutcome::Evidence(tracedecay_contracts::EvidencePacket { - temporal: packet.temporal, - authority: packet.authority, - evidence_authorities: packet.evidence_authorities, - coverage: packet.coverage, - omissions: packet.omissions, - scores: packet.scores, - contributions: packet.contributions, - page: packet.page, - execution: packet.execution, - payload: payload(packet.payload)?, - }) - } - ApplicationOutcome::Preview(preview) => { - ApplicationOutcome::Preview(tracedecay_contracts::PreviewResult { - preview_id: preview.preview_id, - preview_digest: preview.preview_digest, - effect_class: preview.effect_class, - authority: preview.authority, - expected_state: preview.expected_state, - execution: preview.execution, - payload: payload(preview.payload)?, - }) - } - ApplicationOutcome::Effect(effect) => { - ApplicationOutcome::Effect(tracedecay_contracts::EffectResult { - effect_id: effect.effect_id, - effect_class: effect.effect_class, - idempotency_key: effect.idempotency_key, - authority: effect.authority, - expected_state: effect.expected_state, - execution: effect.execution, - reconciliation: effect.reconciliation, - receipt: effect.receipt, - payload: payload(effect.payload)?, - }) - } - }) -} - #[cfg(test)] -mod tests { - use serde_json::json; - - use super::*; +mod conflict_tests { + use super::active_request_conflict; #[test] - fn route_selected_session_refresh_rejects_embedded_action() { - assert!( - decode_request( - RetainedSurfaceOperation::SessionRefreshStatus, - json!({ "action": "status" }), - ) - .is_err() + fn active_replay_collision_preserves_same_request_retry_authority() { + let request_id = + tracedecay_contracts::RequestId::new("request.sdk.curate").expect("request id"); + let envelope = serde_json::to_value( + active_request_conflict(request_id) + .expect("conflict") + .into_http_json(), + ) + .expect("wire envelope"); + assert_eq!(envelope["value"]["problem"]["retry"], "same_request"); + assert_eq!(envelope["value"]["problem"]["retry_scope"], "same_request"); + assert_eq!( + envelope["value"]["problem"]["legal_actions"], + serde_json::json!(["retry"]) ); } - - #[test] - fn fact_store_curate_rejects_caller_owned_authority() { - for forbidden in [ - "operations", - "proposal_id", - "approve", - "apply", - "run_id", - "task", - ] { - let mut value = serde_json::Map::new(); - value.insert(forbidden.to_owned(), serde_json::Value::Bool(true)); - assert!( - decode_request( - RetainedSurfaceOperation::FactStoreCurate, - serde_json::Value::Object(value), - ) - .is_err() - ); - } - } } diff --git a/crates/tracedecay-daemon-service/src/application_surface/retained_http_identity_tests.rs b/crates/tracedecay-daemon-service/src/application_surface/retained_http_identity_tests.rs index 642ab177bf..a5462c745a 100644 --- a/crates/tracedecay-daemon-service/src/application_surface/retained_http_identity_tests.rs +++ b/crates/tracedecay-daemon-service/src/application_surface/retained_http_identity_tests.rs @@ -19,8 +19,7 @@ use tracedecay_domain::{ }; use tracedecay_tool_catalog::EffectClass; -use super::super::registered_http::{RegisteredHttpOperation, invoke_registered_http}; -use super::validated_daemon_outcome; +use super::super::registered_http::RegisteredHttpOperation; use tracedecay_api::WorkOperation; use tracedecay_domain::test_fixtures::digest; @@ -148,12 +147,20 @@ impl StaticDaemonResponseExecutor { impl tracedecay_contracts::ApplicationInvocationExecutor for StaticDaemonResponseExecutor { fn invoke( &self, - _invocation: tracedecay_contracts::ApplicationInvocation, + invocation: tracedecay_contracts::ApplicationInvocation, ) -> tracedecay_contracts::ApplicationInvocationFuture< '_, Result, > { - Box::pin(async { Err(tracedecay_contracts::InvocationError::Unavailable) }) + Box::pin(async move { + let (context, request) = invocation.into_parts(); + let tracedecay_contracts::ApplicationRequest::Surface { binding, payload } = request + else { + return Err(tracedecay_contracts::InvocationError::Unavailable); + }; + tracedecay_daemon_protocol::invoke_application_surface(self, context, binding, payload) + .await + }) } } @@ -202,56 +209,43 @@ async fn response_json(response: axum::response::Response) -> Value { .expect("problem JSON") } +/// Dispatch one retained HTTP operation through the application path against +/// a daemon that answers with `response`. async fn invoke_retained_http_with_response( operation: RetainedSurfaceOperation, request_id: RequestId, response: tracedecay_daemon_protocol::DaemonInvocationResponse, ) -> axum::response::Response { - let deadline = Deadline::new(UtcMicros(1_000)).expect("deadline"); - let cancellation = - CancellationSignal::active("cancellation.retained.http").expect("cancellation"); - let retained_request = super::super::retained::decode_request( + let application_operation = + tracedecay_tool_catalog::ApplicationSurfaceOperation::from_catalog_name(operation.as_str()) + .expect("retained application operation"); + let retained_request = tracedecay_daemon_protocol::decode_retained_request( operation, json!({"fact_id": "fact.retained.fixture"}), ) .expect("retained request"); - let invocation = tracedecay_daemon_protocol::DaemonInvocationRequest::retained_application( - request_id.as_str(), - retained_request, - UtcMicros(10), - deadline.clone(), - cancellation.context(), - ); - let executor = StaticDaemonResponseExecutor::new(response); - let selected_request_id = request_id.clone(); - invoke_registered_http::( - &executor, - operation, + let dispatched = super::super::resolve_application_surface_dispatch( + tracedecay_tool_catalog::BindingSurface::Http, + application_operation, request_id, - tracedecay_api::HttpApplicationControls { - deadline, - cancellation, - }, - invocation, - |outcome| match outcome { - tracedecay_daemon_protocol::DaemonInvocationOutcome::RetainedApplication { - scope, - outcome, - } => tracedecay_contracts::retained_surface_outcome_matches_terminal( - operation, - &selected_request_id, - &scope, - &outcome, - ) - .then_some((scope, outcome)), - _ => None, - }, + tracedecay_daemon_protocol::ApplicationSurfaceRequest::Retained(retained_request), + tracedecay_daemon_protocol::RequestedOutputFormat::Json, + ) + .expect("retained dispatch"); + let executor = StaticDaemonResponseExecutor::new(response); + let result = super::super::execute_application_surface( + application_operation, + dispatched, + Some(&executor), ) .await + .expect("retained invocation"); + tracedecay_api::CanonicalInvocationResult::new(result.binding_id, result.result) + .into_http_response() } -#[test] -fn rejects_each_untrusted_daemon_envelope_field_before_payload_selection() { +#[tokio::test] +async fn rejects_each_untrusted_daemon_envelope_field_before_payload_selection() { let operation = RetainedSurfaceOperation::FactStoreRemove; let caller_request_id = RequestId::new("request.retained.http.caller").expect("caller request id"); @@ -276,13 +270,17 @@ fn rejects_each_untrusted_daemon_envelope_field_before_payload_selection() { invalid_responses.push(invalid_request_id); for response in invalid_responses { - let problem = validated_daemon_outcome(operation, &caller_request_id, Ok(response)) - .expect_err("invalid daemon identity must be rejected before reading its payload"); - let ApplicationProblem::Unavailable { diagnostic, .. } = problem else { - panic!("invalid daemon identity must become a pre-admission unavailable problem"); - }; - assert_eq!(diagnostic.code, "retained.invalid_envelope"); - assert_ne!(diagnostic.code, "retained.fixture.partial_effect"); + let response = + invoke_retained_http_with_response(operation, caller_request_id.clone(), response) + .await; + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + let body = response_json(response).await; + assert_eq!(body["value"]["problem"]["kind"], "unavailable"); + assert_eq!( + body["value"]["problem"]["code"], + "application.surface.invalid_response" + ); + assert_eq!(body["value"]["problem"]["committed_receipt"], Value::Null); } } @@ -321,7 +319,7 @@ async fn registered_http_rejects_invalid_identity_without_exposing_its_receipt() assert_eq!(body["value"]["problem"]["kind"], "unavailable"); assert_eq!( body["value"]["problem"]["code"], - "retained.invalid_envelope" + "application.surface.invalid_response" ); assert_eq!(body["value"]["problem"]["committed_receipt"], Value::Null); } @@ -405,7 +403,7 @@ async fn registered_http_rejects_unbound_partial_effect_receipts_without_exposin let body = response_json(response).await; assert_eq!( body["value"]["problem"]["code"], - "retained.invalid_terminal" + "application.surface.invalid_response" ); assert_eq!(body["value"]["problem"]["committed_receipt"], Value::Null); } @@ -479,7 +477,7 @@ async fn registered_http_rejects_successes_with_the_wrong_payload_receipt_or_sco let body = response_json(response).await; assert_eq!( body["value"]["problem"]["code"], - "retained.protocol_unavailable" + "application.surface.invalid_response" ); } } diff --git a/crates/tracedecay-daemon-service/src/application_surface/tests.rs b/crates/tracedecay-daemon-service/src/application_surface/tests.rs index a9f50f7f2a..447e992dbb 100644 --- a/crates/tracedecay-daemon-service/src/application_surface/tests.rs +++ b/crates/tracedecay-daemon-service/src/application_surface/tests.rs @@ -31,7 +31,7 @@ use super::operation_events::{ HttpOperationEventState, http_operation_event_router, resolve_authenticated_http_request_context, }; -use super::problems::{current_micros, invocation_problem}; +use super::problems::current_micros; use super::request_control::{ ActiveHttpRequest, HttpCancellationRegistry, application_http_context, }; @@ -95,27 +95,6 @@ fn operation_context(project_id: &ProjectId) -> RequestContext { .expect("context") } -#[test] -fn daemon_reset_problem_preserves_reset_terminal_contract() { - let problem = - invocation_problem(tracedecay_daemon_protocol::DaemonInvocationProblem::ResetRequired) - .expect("canonical reset problem"); - let ApplicationProblem::ResetRequired { - retry, - legal_actions, - .. - } = problem - else { - panic!("application surface must preserve reset-required"); - }; - - assert_eq!(retry, tracedecay_contracts::RetryDirective::Never); - assert_eq!( - legal_actions, - vec![tracedecay_contracts::LegalAction::Reset] - ); -} - #[test] fn workflow_http_descriptors_match_every_executable_manifest_route() { validate_workflow_catalog_bindings().expect("every Workflow descriptor has one mounted route"); @@ -709,9 +688,9 @@ async fn http_git_read_routes_preserve_the_canonical_typed_request() { } #[test] -fn catalog_bound_compatibility_tools_resolve_before_retained_dispatch() { +fn every_callable_cli_and_mcp_binding_names_an_application_operation() { let catalog = super::application_surface_catalog().expect("application catalog"); - let mut compatibility_operations = std::collections::BTreeSet::new(); + let mut unmapped = std::collections::BTreeSet::new(); for capability in catalog.capabilities() { if !capability.availability().is_callable() { @@ -719,80 +698,19 @@ fn catalog_bound_compatibility_tools_resolve_before_retained_dispatch() { } for binding_id in capability.binding_ids() { let binding = catalog.binding(binding_id).expect("catalog binding"); - if !matches!( + if matches!( binding.surface(), tracedecay_tool_catalog::BindingSurface::Cli | tracedecay_tool_catalog::BindingSurface::Mcp - ) || ApplicationSurfaceOperation::from_tool_name(binding.operation().as_str()) - .is_some() + ) && ApplicationSurfaceOperation::from_tool_name(binding.operation().as_str()) + .is_none() { - continue; + unmapped.insert(binding.operation().as_str().to_owned()); } - - let tool_name = format!("tracedecay_{}", binding.operation().as_str()); - let resolved = super::resolve_catalog_tool_binding(binding.surface(), &tool_name) - .expect("compatibility binding resolution") - .unwrap_or_else(|| panic!("{tool_name} must resolve before retained dispatch")); - assert_eq!(resolved.binding_id, *binding_id); - compatibility_operations.insert(binding.operation().as_str().to_owned()); } } - assert_eq!( - compatibility_operations, - [ - "ast_grep_rewrite", - "callees", - "context", - "fact_feedback", - "fact_store_add", - "fact_store_contradict", - "fact_store_curate", - "fact_store_get", - "fact_store_list", - "fact_store_probe", - "fact_store_reason", - "fact_store_related", - "fact_store_remove", - "fact_store_search", - "fact_store_supersede", - "fact_store_update", - "impact", - "insert_at", - "insert_at_symbol", - "lcm_describe", - "lcm_doctor", - "lcm_expand", - "lcm_expand_query", - "lcm_grep", - "lcm_load_session", - "lcm_status", - "memory_status", - "message_search", - "move_symbol", - "multi_str_replace", - "node", - "port_order", - "port_status", - "redundancy", - "rename_preview", - "rename_symbol", - "replace_symbol", - "session_refresh_begin", - "session_refresh_cancel", - "session_refresh_status", - "sessions_for", - "similar", - "source_edit_reconcile", - "source_edit_rollback", - "str_replace", - "todos", - "workflows", - ] - .into_iter() - .map(str::to_owned) - .collect() - ); + assert_eq!(unmapped, std::collections::BTreeSet::new()); } #[test] @@ -1464,79 +1382,6 @@ fn callable_code_page_is_transport_owned() { )); } -#[test] -fn callable_code_operation_names_are_exact_and_not_primitive_aliases() { - for (operation, name) in [ - ( - ApplicationSurfaceOperation::CodeExactOccurrence, - "code_exact_occurrence", - ), - ( - ApplicationSurfaceOperation::CodePhraseSearch, - "code_phrase_search", - ), - ( - ApplicationSurfaceOperation::CodeSymbolSearch, - "code_symbol_search", - ), - ( - ApplicationSurfaceOperation::CodeSignatureSearch, - "code_signature_search", - ), - ( - ApplicationSurfaceOperation::CodeImplementations, - "code_implementations", - ), - ( - ApplicationSurfaceOperation::CodeTypeHierarchy, - "code_type_hierarchy", - ), - (ApplicationSurfaceOperation::CodeCallers, "code_callers"), - (ApplicationSurfaceOperation::CodeCallees, "code_callees"), - (ApplicationSurfaceOperation::CodeFacets, "code_facets"), - (ApplicationSurfaceOperation::CodeTimeline, "code_timeline"), - ( - ApplicationSurfaceOperation::CodeDeclaration, - "code_declaration", - ), - ( - ApplicationSurfaceOperation::CodeTypeDefinition, - "code_type_definition", - ), - ( - ApplicationSurfaceOperation::CodeReferences, - "code_references", - ), - ] { - assert_eq!(operation.as_str(), name); - assert_eq!( - ApplicationSurfaceOperation::from_tool_name(&format!("tracedecay_{name}")), - Some(operation) - ); - } - for primitive_alias in [ - "exact_occurrence", - "phrase_search", - "symbol_search", - "signature_search", - "implementations", - "type_hierarchy", - "callers", - "callees", - "facets", - "timeline", - "declaration", - "definition", - "type_definition", - "references", - ] { - assert_eq!( - ApplicationSurfaceOperation::from_tool_name(primitive_alias), - None - ); - } -} - #[test] fn dropped_http_request_unregisters_without_cancelling_work() { let request_id = RequestId::new("request.http.disconnect").expect("request"); diff --git a/crates/tracedecay-daemon-service/src/application_surface/work.rs b/crates/tracedecay-daemon-service/src/application_surface/work.rs index e3b3aa3754..3c8d7a9648 100644 --- a/crates/tracedecay-daemon-service/src/application_surface/work.rs +++ b/crates/tracedecay-daemon-service/src/application_surface/work.rs @@ -105,7 +105,7 @@ pub(crate) async fn invoke_work_operation( let invocation = tracedecay_daemon_protocol::DaemonInvocationRequest::work_application( request_id.as_str(), WorkApplicationInvocationV1::$variant(decoded), - tracedecay_daemon_protocol::invocation_now_micros(), + tracedecay_contracts::now_micros(), controls.deadline.clone(), controls.cancellation.context(), ); @@ -135,7 +135,7 @@ pub(crate) async fn invoke_work_operation( let invocation = tracedecay_daemon_protocol::DaemonInvocationRequest::work_application( request_id.as_str(), WorkApplicationInvocationV1::$variant(decoded), - tracedecay_daemon_protocol::invocation_now_micros(), + tracedecay_contracts::now_micros(), controls.deadline.clone(), controls.cancellation.context(), ); diff --git a/crates/tracedecay-daemon-service/src/application_surface/workflow.rs b/crates/tracedecay-daemon-service/src/application_surface/workflow.rs index fe796483d5..598f39fe08 100644 --- a/crates/tracedecay-daemon-service/src/application_surface/workflow.rs +++ b/crates/tracedecay-daemon-service/src/application_surface/workflow.rs @@ -324,7 +324,7 @@ where let invocation = tracedecay_daemon_protocol::DaemonInvocationRequest::workflow_application( request_id.as_str(), request, - tracedecay_daemon_protocol::invocation_now_micros(), + tracedecay_contracts::now_micros(), controls.deadline.clone(), controls.cancellation.context(), ); diff --git a/crates/tracedecay-daemon-service/src/automation_effect/journal/tests.rs b/crates/tracedecay-daemon-service/src/automation_effect/journal/tests.rs index a40c89af90..fafb49eff3 100644 --- a/crates/tracedecay-daemon-service/src/automation_effect/journal/tests.rs +++ b/crates/tracedecay-daemon-service/src/automation_effect/journal/tests.rs @@ -39,6 +39,10 @@ impl tracedecay_automation_runtime::automation::backend::AgentTaskBackend > { panic!("disabled retained automation must not invoke its backend") } + + fn executable(&self) -> Option<&std::path::Path> { + None + } } use tracedecay_domain::test_fixtures::digest; @@ -132,8 +136,6 @@ fn admission(run_id: &str, request_id: &str) -> DurableAutomationAdmission { project_id: scope.project_id.clone(), }, recovery_problem: reset_problem(&request_id, &scope, &request), - retirement: None, - reset_source_digest: None, }, }) } @@ -206,50 +208,6 @@ fn admission_for_recovery_project( admission.recovery = AutomationRecoveryBinding::Memory { owner, recovery_problem: reset_problem(&admission.request_id, &recovery_scope, &admission.request), - retirement: None, - reset_source_digest: None, - }; - seal_effect_authority(admission) -} - -fn retirement_admission_for_recovery_project( - cg: &tracedecay_project::project::TraceDecay, - run_id: &str, - request_id: &str, - binding: tracedecay_automation_runtime::automation::effect_runtime::retirement::RetirementBinding, -) -> DurableAutomationAdmission { - let owner = cg.project_memory_owner().expect("project memory owner"); - let FactOwnerV1::Project { project_id } = owner.clone() else { - panic!("automation retirement fixture requires a project owner") - }; - let recovery_scope = - tracedecay_code_index_runtime::resolved_scope_for_project(cg.project_root(), &project_id) - .expect("retirement recovery scope"); - let mut admission = admission(run_id, request_id); - admission.request.task = AutomationTaskRequestV1::SessionReflector( - tracedecay_contracts::retained_surfaces::SessionReflectorRunInputV1 { - provider: "cursor".to_owned(), - query: "retire exact shipped proposal history".to_owned(), - scope: tracedecay_contracts::retained_surfaces::LcmSearchScopeV1::Current, - session_id: None, - include_summaries: true, - evidence_limit: 5, - include_recent_sessions: false, - recent_sessions_limit: 1, - sort: tracedecay_contracts::retained_surfaces::LcmGrepSortV1::Recency, - source: None, - role: None, - start_time: None, - end_time: None, - }, - ); - admission.scope = recovery_scope.clone(); - admission.effect_receipt_template.scope = recovery_scope.clone(); - admission.recovery = AutomationRecoveryBinding::Memory { - owner, - recovery_problem: reset_problem(&admission.request_id, &recovery_scope, &admission.request), - retirement: Some(binding), - reset_source_digest: None, }; seal_effect_authority(admission) } @@ -523,19 +481,6 @@ fn exact_spool_file_count(dashboard_root: &std::path::Path) -> usize { exact_spool_files(dashboard_root).len() } -fn retirement_capture_count(dashboard_root: &std::path::Path) -> usize { - std::fs::read_dir(dashboard_root) - .expect("retirement capture inventory") - .filter_map(std::result::Result::ok) - .filter(|entry| { - entry - .file_name() - .to_string_lossy() - .starts_with(".fact_proposals.retirement-") - }) - .count() -} - fn partial_receipt_template(request_id: &RequestId, scope: &ResolvedScope) -> EffectReceipt { let operation = retained_surface_application_operation(RetainedSurfaceOperation::FactStoreCurate) @@ -597,27 +542,6 @@ fn success_terminal( ) } -fn retirement_terminal(admission: &DurableAutomationAdmission) -> AutomationSettledTerminal { - result_terminal( - admission, - admission.request.run_id.as_str(), - AutomationTaskV1::SessionReflector, - AutomationRunTerminalV1::Skipped { - reason: - tracedecay_contracts::retained_surfaces::AutomationSkipReasonV1::from_ledger_reason( - "shipped_fact_proposal_history_retired", - ) - .expect("retirement skip reason"), - summary: AutomationRunSummaryV1 { - reviewed_count: 0, - accepted_count: 0, - rejected_count: 0, - skipped_count: 1, - }, - }, - ) -} - fn result_terminal( admission: &DurableAutomationAdmission, result_run_id: &str, @@ -677,422 +601,6 @@ fn result_terminal( } } -#[tokio::test] -async fn terminal_retirement_recovery_keeps_pending_until_source_is_exactly_archived() { - let temp = tempfile::tempdir().expect("tempdir"); - let fixture_name = "terminal-retirement-recovery"; - let cg = retained_recovery_project(&temp, fixture_name).await; - let dashboard_root = cg.store_layout().dashboard_root.clone(); - let project_root = cg.project_root().to_path_buf(); - let profile_root = temp.path().join(format!("{fixture_name}-profile")); - let source_path = dashboard_root.join("fact_proposals.json"); - let source_bytes = br#"{"schema_version":1,"proposals":[]}"#.to_vec(); - write_private_test_file(&source_path, &source_bytes); - let plan = match tracedecay_automation_runtime::automation::effect_runtime::retirement::classify_for_task( - AutomationTaskV1::SessionReflector, - &dashboard_root, - ) - .await - .expect("classify exact retirement source") - { - tracedecay_automation_runtime::automation::effect_runtime::retirement::RetirementClassification::Terminal(plan) => plan, - _ => panic!("terminal shipped history must yield an exact retirement plan"), - }; - let binding = plan.binding.clone(); - let archive_path = dashboard_root - .join("fact_proposals.archive") - .join(&binding.archive_name); - let admission = retirement_admission_for_recovery_project( - &cg, - "run.terminal-retirement-recovery", - "request.terminal-retirement-recovery", - binding, - ); - let journal_path = canonical_journal_path(&dashboard_root, &admission.request.run_id); - - let (anchor, anchor_guard) = retained_disabled_user_job( - &dashboard_root, - "run.terminal-retirement-ledger-anchor", - "terminal-retirement-ledger-anchor", - ) - .await; - drop(anchor_guard); - let (anchor_publication, _) = - tracedecay_automation_runtime::automation::run_ledger::bind_staged_run_record_exact( - &dashboard_root, - &anchor.ledger_record, - |publication| Ok(publication.clone()), - ) - .expect("stage exact anchor ledger row"); - assert_eq!( - tracedecay_automation_runtime::automation::run_ledger::publish_staged_run_record_exact( - &dashboard_root, - &anchor.ledger_record.run_id, - &anchor_publication, - ) - .await - .expect("publish exact anchor ledger row"), - tracedecay_automation_runtime::automation::run_ledger::ExactRunPublishOutcome::Published - ); - tracedecay_automation_runtime::automation::run_ledger::discard_staged_run_record_exact( - &dashboard_root, - &anchor.ledger_record.run_id, - &anchor_publication, - ) - .await - .expect("retire exact anchor spool"); - - let claim = match reserve_or_replay_indexed_blocking( - &journal_path, - admission.clone(), - || recovery_index::add_pending_blocking(&dashboard_root, &journal_path, &admission), - || recovery_index::remove_pending_blocking(&dashboard_root, &journal_path), - ) - .expect("reserve indexed retirement") - { - ReservationResult::Execute { claim, retirement } => { - assert_eq!(retirement, admission.retirement().cloned()); - claim - } - _ => panic!("fresh retirement admission must execute"), - }; - let terminal = retirement_terminal(&admission); - persist_terminal_blocking(&journal_path, &admission, terminal.clone()) - .expect("persist exact retirement Terminal"); - drop(claim); - - let sidecar_path = terminal_sidecar_path(&journal_path).expect("terminal sidecar path"); - let ledger_path = - tracedecay_automation_runtime::automation::run_ledger::run_ledger_path(&dashboard_root); - let journal_bytes = std::fs::read(&journal_path).expect("terminal journal bytes"); - let sidecar_bytes = std::fs::read(&sidecar_path).expect("terminal sidecar bytes"); - let ledger_bytes = std::fs::read(&ledger_path).expect("anchor ledger bytes"); - assert_eq!( - recovery_index::indexed_journals_blocking(&dashboard_root, &admission.scope) - .expect("pending retirement index") - .len(), - 1 - ); - assert!(!archive_path.exists()); - - let corrupt_source = b"source changed after exact retirement admission"; - write_private_test_file(&source_path, corrupt_source); - let failed = recovery_composition::reconcile_reserved_automation_effects_for_project( - &cg, - &dashboard_root, - &tracedecay_contracts::CancellationSignal::active( - "cancellation.terminal-retirement-failure", - ) - .expect("failure cancellation"), - ) - .await - .expect("retirement finalization failure is deferred"); - assert_eq!(failed.inspected, 1); - assert_eq!(failed.deferred, 1); - assert_eq!( - std::fs::read(&source_path).expect("retained source"), - corrupt_source.to_vec() - ); - assert!(!archive_path.exists()); - assert_eq!( - recovery_index::indexed_journals_blocking(&dashboard_root, &admission.scope) - .expect("retained pending retirement") - .len(), - 1 - ); - assert_eq!( - std::fs::read(&journal_path).expect("unchanged journal"), - journal_bytes - ); - assert_eq!( - std::fs::read(&sidecar_path).expect("unchanged sidecar"), - sidecar_bytes - ); - assert_eq!( - std::fs::read(&ledger_path).expect("unchanged ledger"), - ledger_bytes - ); - - write_private_test_file(&source_path, &source_bytes); - let pending_retirement = - tracedecay_automation_runtime::automation::effect_runtime::retirement::finalize_after_terminal(&dashboard_root, &plan.binding, Some(&plan)) - .expect("finalize exact retirement through source capture"); - assert!(!source_path.exists()); - assert_eq!(retirement_capture_count(&dashboard_root), 1); - - std::fs::create_dir(&source_path).expect("nonregular replacement source"); - recovery_index::remove_pending_for_retirement_blocking( - &dashboard_root, - &journal_path, - &admission, - &pending_retirement, - ) - .expect("publish retirement transition before pending removal"); - assert_eq!(retirement_capture_count(&dashboard_root), 1); - assert!( - recovery_index::indexed_journals_blocking(&dashboard_root, &admission.scope) - .expect("retirement transition removes pending entry") - .is_empty() - ); - tracedecay_automation_runtime::automation::effect_runtime::retirement::complete_after_pending_removal(&pending_retirement) - .expect("complete retirement witness after pending removal"); - assert_eq!(retirement_capture_count(&dashboard_root), 0); - recovery_index::finish_retirement_transition_blocking( - &dashboard_root, - &journal_path, - &admission, - &pending_retirement, - ) - .expect("close durable retirement transition"); - recovery_index::reject_unbound_retirement_witness_if_index_empty(&dashboard_root) - .expect("completed retirement leaves no unbound witness"); - assert!(source_path.is_dir()); - assert_eq!(retirement_capture_count(&dashboard_root), 0); - assert!( - recovery_index::indexed_journals_blocking(&dashboard_root, &admission.scope) - .expect("pending-witness index closed") - .is_empty() - ); - - std::fs::remove_dir(&source_path).expect("remove nonregular replacement fixture"); - write_private_test_file(&source_path, &source_bytes); - recovery_index::add_pending_blocking(&dashboard_root, &journal_path, &admission) - .expect("re-index Terminal before pending-absent crash"); - let orphaned_retirement = - tracedecay_automation_runtime::automation::effect_runtime::retirement::finalize_after_terminal(&dashboard_root, &plan.binding, Some(&plan)) - .expect("capture exact source before pending-absent crash"); - assert!(!source_path.exists()); - assert_eq!(retirement_capture_count(&dashboard_root), 1); - recovery_index::remove_pending_for_retirement_blocking( - &dashboard_root, - &journal_path, - &admission, - &orphaned_retirement, - ) - .expect("durably hand off pending recovery before simulated crash"); - drop(orphaned_retirement); - assert!( - recovery_index::indexed_journals_blocking(&dashboard_root, &admission.scope) - .expect("crash-state pending index") - .is_empty() - ); - let replacement_source = - br#"{"schema_version":1,"proposals":[{"state":"pending_approval"}]}"#.to_vec(); - write_private_test_file(&source_path, &replacement_source); - let pending_index_path = dashboard_root - .join("automation_effects") - .join("pending-index.json"); - let exact_transition_index = std::fs::read(&pending_index_path).expect("transition index"); - let mut mismatched_transition: serde_json::Value = - serde_json::from_slice(&exact_transition_index).expect("transition index JSON"); - mismatched_transition["retirement_transitions"][0]["source_digest"] = - serde_json::Value::String(format!("sha256:{}", "f".repeat(64))); - write_private_test_file( - &pending_index_path, - &serde_json::to_vec_pretty(&mismatched_transition).expect("mismatched transition bytes"), - ); - cg.close(); - let reopened = tracedecay_project::project::TraceDecay::init_with_options_for_test( - &project_root, - tracedecay_project::project::TraceDecayOpenOptions { - profile_root: Some(profile_root.clone()), - global_db_path: Some(profile_root.join("global.db")), - }, - ) - .await - .expect("reopen retirement recovery project"); - let rejected = recovery_composition::reconcile_reserved_automation_effects_for_project( - &reopened, - &dashboard_root, - &tracedecay_contracts::CancellationSignal::active( - "cancellation.terminal-retirement-mismatch", - ) - .expect("mismatch cancellation"), - ) - .await - .expect("mismatched transition remains deferred"); - assert_eq!(rejected.inspected, 1); - assert_eq!(rejected.deferred, 1); - assert_eq!(retirement_capture_count(&dashboard_root), 1); - assert_eq!( - std::fs::read(&source_path).expect("replacement source retained across mismatch"), - replacement_source - ); - - write_private_test_file(&pending_index_path, &exact_transition_index); - let recovered = recovery_composition::reconcile_reserved_automation_effects_for_project( - &reopened, - &dashboard_root, - &tracedecay_contracts::CancellationSignal::active( - "cancellation.terminal-retirement-recovery", - ) - .expect("recovery cancellation"), - ) - .await - .expect("recover exact retirement Terminal"); - assert_eq!(recovered.inspected, 1); - assert_eq!(recovered.already_terminal, 1); - assert_eq!( - std::fs::read(&archive_path).expect("retirement archive"), - source_bytes - ); - assert_eq!( - std::fs::read(&source_path).expect("replacement source preserved"), - replacement_source - ); - assert_eq!(retirement_capture_count(&dashboard_root), 0); - assert!( - recovery_index::indexed_journals_blocking(&dashboard_root, &admission.scope) - .expect("closed retirement index") - .is_empty() - ); - - write_private_test_file(&source_path, &source_bytes); - recovery_index::add_pending_blocking(&dashboard_root, &journal_path, &admission) - .expect("re-index Terminal before entry-plus-marker restart"); - let entry_plus_marker_retirement = - tracedecay_automation_runtime::automation::effect_runtime::retirement::finalize_after_terminal(&dashboard_root, &plan.binding, Some(&plan)) - .expect("capture exact source before entry-plus-marker restart"); - recovery_index::remove_pending_for_retirement_blocking( - &dashboard_root, - &journal_path, - &admission, - &entry_plus_marker_retirement, - ) - .expect("publish exact transition before entry-plus-marker restart"); - recovery_index::add_pending_blocking(&dashboard_root, &journal_path, &admission) - .expect("simulate crash-visible entry plus exact marker"); - drop(entry_plus_marker_retirement); - write_private_test_file(&source_path, &replacement_source); - - let entry_plus_marker = - recovery_composition::reconcile_reserved_automation_effects_for_project( - &reopened, - &dashboard_root, - &tracedecay_contracts::CancellationSignal::active( - "cancellation.terminal-retirement-entry-plus-marker", - ) - .expect("entry-plus-marker cancellation"), - ) - .await - .expect("entry-plus-marker restart converges through its marker first"); - assert_eq!(entry_plus_marker.inspected, 1); - assert_eq!(entry_plus_marker.already_terminal, 1); - assert_eq!(retirement_capture_count(&dashboard_root), 0); - assert_eq!( - std::fs::read(&source_path).expect("entry-plus-marker replacement preserved"), - replacement_source - ); - assert!( - recovery_index::indexed_journals_blocking(&dashboard_root, &admission.scope) - .expect("entry-plus-marker index closed") - .is_empty() - ); - assert_eq!( - std::fs::read(&journal_path).expect("exact journal"), - journal_bytes - ); - assert_eq!( - std::fs::read(&sidecar_path).expect("exact sidecar"), - sidecar_bytes - ); - assert_eq!( - std::fs::read(&ledger_path).expect("exact ledger"), - ledger_bytes - ); - assert_eq!( - read_indexed_terminal_blocking(&journal_path).expect("exact terminal readback"), - Some(terminal.clone()) - ); - assert_eq!( - tracedecay_automation_runtime::automation::run_ledger::find_run_record_exact_bounded_blocking( - &dashboard_root, - &anchor.ledger_record.run_id, - ) - .expect("exact anchor lookup"), - Some(anchor.ledger_record) - ); - - recovery_index::add_pending_blocking(&dashboard_root, &journal_path, &admission) - .expect("re-index exact Terminal for idempotent retry"); - let replayed = recovery_composition::reconcile_reserved_automation_effects_for_project( - &reopened, - &dashboard_root, - &tracedecay_contracts::CancellationSignal::active( - "cancellation.terminal-retirement-idempotent", - ) - .expect("idempotent cancellation"), - ) - .await - .expect("idempotently replay retirement finalization"); - assert_eq!(replayed.inspected, 1); - assert_eq!(replayed.already_terminal, 1); - assert_eq!( - std::fs::read(&archive_path).expect("stable archive"), - source_bytes - ); - assert_eq!( - std::fs::read(&source_path).expect("stable replacement source"), - replacement_source - ); - assert!( - recovery_index::indexed_journals_blocking(&dashboard_root, &admission.scope) - .expect("idempotently closed index") - .is_empty() - ); - assert_eq!( - std::fs::read(&journal_path).expect("stable journal"), - journal_bytes - ); - assert_eq!( - std::fs::read(&sidecar_path).expect("stable sidecar"), - sidecar_bytes - ); - assert_eq!( - std::fs::read(&ledger_path).expect("stable ledger"), - ledger_bytes - ); - - write_private_test_file(&source_path, &source_bytes); - recovery_index::add_pending_blocking(&dashboard_root, &journal_path, &admission) - .expect("re-index exact Terminal with archive and live admitted source"); - let archive_and_live_source = - recovery_composition::reconcile_reserved_automation_effects_for_project( - &reopened, - &dashboard_root, - &tracedecay_contracts::CancellationSignal::active( - "cancellation.terminal-retirement-archive-live-source", - ) - .expect("archive-live-source cancellation"), - ) - .await - .expect("project recovery retires an exact live source despite an existing archive"); - assert_eq!(archive_and_live_source.inspected, 1); - assert_eq!(archive_and_live_source.already_terminal, 1); - assert!(!source_path.exists()); - assert_eq!( - std::fs::read(&archive_path).expect("archive remains exact"), - source_bytes - ); - assert!( - recovery_index::indexed_journals_blocking(&dashboard_root, &admission.scope) - .expect("archive-live-source index closed") - .is_empty() - ); - assert_eq!( - std::fs::read(&journal_path).expect("archive-live-source journal"), - journal_bytes - ); - assert_eq!( - std::fs::read(&sidecar_path).expect("archive-live-source sidecar"), - sidecar_bytes - ); - assert_eq!( - std::fs::read(&ledger_path).expect("archive-live-source ledger"), - ledger_bytes - ); -} - #[cfg(unix)] #[test] fn scheduler_stable_request_identity_reopens_the_same_terminal() { diff --git a/crates/tracedecay-daemon-service/src/automation_effect/recovery_composition.rs b/crates/tracedecay-daemon-service/src/automation_effect/recovery_composition.rs index 05b9a98f7b..38edc6a72b 100644 --- a/crates/tracedecay-daemon-service/src/automation_effect/recovery_composition.rs +++ b/crates/tracedecay-daemon-service/src/automation_effect/recovery_composition.rs @@ -22,8 +22,7 @@ pub async fn reconcile_reserved_automation_effects_for_project( dashboard_root: &Path, cancellation: &CancellationSignal, ) -> Result { - let preparation = - prepare_reserved_automation_effect_recovery(dashboard_root, cancellation).await?; + let preparation = prepare_reserved_automation_effect_recovery(dashboard_root).await?; let preparation = match preparation { AutomationEffectRecoveryPreparation::Complete(report) => return Ok(report), AutomationEffectRecoveryPreparation::Pending(preparation) => preparation, diff --git a/crates/tracedecay-daemon-service/src/callable_code_authorization.rs b/crates/tracedecay-daemon-service/src/callable_code_authorization.rs index 2b6b68b8a2..41161a1442 100644 --- a/crates/tracedecay-daemon-service/src/callable_code_authorization.rs +++ b/crates/tracedecay-daemon-service/src/callable_code_authorization.rs @@ -4,9 +4,8 @@ use std::time::Duration; use tracedecay_contracts::{ ApplicationContractError, ApplicationOperation, ApplicationProblem, ApplicationProblemKind, - AuthorityReceipt, CallableCodeAuthorizationAdmission, CallableCodeAuthorizationFuture, - CallableCodeAuthorizationPort, RequestAdmission, RequestContext, ResolvedScope, RetryDirective, - SafeDiagnostic, + AuthorityReceipt, CallableCodeAuthorizationFuture, CallableCodeAuthorizationPort, + RequestAdmission, RequestContext, ResolvedScope, RetryDirective, SafeDiagnostic, }; use tracedecay_domain::{ActorId, ComponentVersion, UtcMicros}; @@ -347,22 +346,15 @@ impl CallableCodeAuthorizationPort for DaemonCallableCodeAuthorization { context: &'a RequestContext, operation: &'a ApplicationOperation, observed_at: UtcMicros, - ) -> CallableCodeAuthorizationFuture< - 'a, - Result, - > { - Box::pin(async move { - self.route_receipt(context, operation, observed_at) - .await - .map(CallableCodeAuthorizationAdmission::Routed) - }) + ) -> CallableCodeAuthorizationFuture<'a, Result> { + Box::pin(self.route_receipt(context, operation, observed_at)) } fn recheck_publication<'a>( &'a self, context: &'a RequestContext, operation: &'a ApplicationOperation, - admission: &'a CallableCodeAuthorizationAdmission, + admission: &'a AuthorityReceipt, observed_at: UtcMicros, ) -> CallableCodeAuthorizationFuture<'a, Result> { Box::pin(async move { @@ -390,12 +382,9 @@ impl DaemonCallableCodeAuthorization { &self, context: &RequestContext, operation: &ApplicationOperation, - admission: &CallableCodeAuthorizationAdmission, + admission: &AuthorityReceipt, observed_at: UtcMicros, ) -> Result { - let CallableCodeAuthorizationAdmission::Routed(admission) = admission else { - return Err(concealed()); - }; let current = self.route_receipt(context, operation, observed_at).await?; if admission.grant_id != current.grant_id || admission.grant_revision != current.grant_revision diff --git a/crates/tracedecay-daemon-service/src/context_scout_lifecycle.rs b/crates/tracedecay-daemon-service/src/context_scout_lifecycle.rs index ae626ec358..d8c06124e5 100644 --- a/crates/tracedecay-daemon-service/src/context_scout_lifecycle.rs +++ b/crates/tracedecay-daemon-service/src/context_scout_lifecycle.rs @@ -9,7 +9,7 @@ use tracedecay_domain::{ }; use tracedecay_store::StoreShardScopeV1; -use tracedecay_agent_hosts::agents::context_scout::ports::ContextScoutLifecycleAddressV1; +use tracedecay_agent_hosts::agents::context_scout::address_registry::ContextScoutLifecycleAddressV1; use tracedecay_global_db::{RegisteredGlobalDb, RegisteredGlobalDbLeaseV1}; const MAX_CONTEXT_SCOUT_SESSION_OBSERVATIONS_V1: usize = 64; diff --git a/crates/tracedecay-daemon-service/src/context_scout_lifecycle/tests.rs b/crates/tracedecay-daemon-service/src/context_scout_lifecycle/tests.rs index 494dcc4b8c..96e1bca3be 100644 --- a/crates/tracedecay-daemon-service/src/context_scout_lifecycle/tests.rs +++ b/crates/tracedecay-daemon-service/src/context_scout_lifecycle/tests.rs @@ -11,7 +11,7 @@ use tracedecay_domain::{ }; use tracedecay_store::{ AnchoredObservationWrite, ObservationStore, ObservationWrite, - build_observation_resolution_authorization_v1, build_observation_retrieval_anchor_v2, + build_observation_resolution_authorization_v1, build_observation_retrieval_anchor, }; use super::*; @@ -97,7 +97,7 @@ fn durable_native_observation(project_id: &ProjectId) -> AnchoredObservationWrit let projection_generation = ProjectionGenerationId::new("projection.native-test.v1").unwrap(); let authorization = build_observation_resolution_authorization_v1(write.observation(), "native-test").unwrap(); - let anchor = build_observation_retrieval_anchor_v2( + let anchor = build_observation_retrieval_anchor( write.observation(), projection_generation.clone(), UtcMicros(1), diff --git a/crates/tracedecay-daemon-service/src/doctor_kernel.rs b/crates/tracedecay-daemon-service/src/doctor_kernel.rs index b260033bd5..c19fc6465e 100644 --- a/crates/tracedecay-daemon-service/src/doctor_kernel.rs +++ b/crates/tracedecay-daemon-service/src/doctor_kernel.rs @@ -13,6 +13,7 @@ use std::path::{Path, PathBuf}; use std::sync::Arc; use tracedecay_code_index_runtime::code_index_scheduler::CodeIndexSchedulerRegistryV1; use tracedecay_code_index_runtime::code_index_scheduler::identity::repository_id_for; +use tracedecay_configuration::config::PinnedRuntimeConfiguration; use tracedecay_contracts::doctor::{ AdvisoryFeedbackDoctorPort, AdvisoryFeedbackReadV1, CodeIndexMountDoctorPort, CodeIndexMountReadV1, CodeIndexMountStateV1, ConfigurationAuthorityDoctorPort, @@ -34,10 +35,10 @@ use tracedecay_contracts::{ }; use tracedecay_domain::CodeGenerationId; use tracedecay_global_db::{GlobalDbNativeIntegrationStore, RegisteredGlobalDb}; -use tracedecay_project::config::DaemonRuntimeConfiguration; use crate::DaemonFeedbackRuntimeRegistrar; use tracedecay_maintenance::telemetry::GuardedStoreTelemetryPort; +use tracedecay_session_temporal_store::SessionTemporalAccess; const DOCTOR_REPORT_CAPABILITY: &str = "capability.application.doctor.report"; const DOCTOR_REPORT_USE_CASE: &str = "use-case.application.doctor.report"; @@ -54,7 +55,7 @@ const DOCTOR_CONTEXT_HORIZON_MICROS: i64 = 30_000_000; /// a fabricated healthy result. #[must_use] pub fn configuration_read_from_pin( - resolved: &Result, + resolved: &Result, ) -> ConfigurationAuthorityReadV1 { match resolved { Ok(_) => ConfigurationAuthorityReadV1::Resolved { @@ -121,10 +122,10 @@ fn host_integration_read_from_report( }) { HostConformanceV1::ProtocolDrift } else if report.components.iter().any(|component| { - // `Drifted`, `OrphanedRegistration`, and `ActivationDeferred` are - // repairable conformance, not protocol drift: the component's ownership - // is intact and either the ordinary reinstall or the host's own - // activation converges it, so none may escalate to `ProtocolDrift`. + // `Drifted`, `OrphanedRegistration`, `ActivationDeferred`, and + // `ReinstallRequired` are repairable conformance, not protocol drift: + // an ordinary or adopting install, or the host's own activation, + // converges each, so none may escalate to `ProtocolDrift`. matches!( component.state, HostBundleComponentDoctorStateV1::Repairable @@ -132,6 +133,7 @@ fn host_integration_read_from_report( | HostBundleComponentDoctorStateV1::Drifted | HostBundleComponentDoctorStateV1::OrphanedRegistration | HostBundleComponentDoctorStateV1::ActivationDeferred + | HostBundleComponentDoctorStateV1::ReinstallRequired ) }) { HostConformanceV1::Drifted @@ -1005,6 +1007,7 @@ pub fn production_doctor_report_reader( }) }) }); + let project_temporal = SessionTemporalAccess::new(&*project_sessions); let ( quick_check, authority_audit_ok, @@ -1026,7 +1029,7 @@ pub fn production_doctor_report_reader( tokio::join!( graph.quick_check_report(), observation_authority_audit_ok(registry.as_ref()), - project_sessions.session_temporal_doctor_health(), + project_temporal.session_temporal_doctor_health(), profile_storage_reads, collect_over_budget_store_findings(&context, &telemetry_ports, &retention), tracedecay_maintenance::retention::diagnostics::collect_session_retention_findings( diff --git a/crates/tracedecay-daemon-service/src/doctor_kernel/tests.rs b/crates/tracedecay-daemon-service/src/doctor_kernel/tests.rs index 6f3ade68d8..c48f709172 100644 --- a/crates/tracedecay-daemon-service/src/doctor_kernel/tests.rs +++ b/crates/tracedecay-daemon-service/src/doctor_kernel/tests.rs @@ -18,7 +18,7 @@ use super::*; #[test] fn configuration_read_from_pin_absent_on_cold_cache() { - let missing: Result = + let missing: Result = Err("cold cache"); assert_eq!( configuration_read_from_pin(&missing), diff --git a/crates/tracedecay-daemon-service/src/invocation.rs b/crates/tracedecay-daemon-service/src/invocation.rs index a6efe15a23..b1c8399a70 100644 --- a/crates/tracedecay-daemon-service/src/invocation.rs +++ b/crates/tracedecay-daemon-service/src/invocation.rs @@ -78,7 +78,7 @@ use crate::project_runtime::{ ProjectRuntimeRegistryV1, RegisteredObservabilityProducerV1, StoreObservabilityMountErrorV1, StoreObservabilityMountV1, StoreObservabilityRegistryV1, }; -use tracedecay_agent_hosts::agents::context_scout::ports::{ +use tracedecay_agent_hosts::agents::context_scout::address_registry::{ AdmittedContextScoutHookV1, ContextScoutLifecycleAddressV1, ProjectContextScoutAddressRegistryV1, }; @@ -181,6 +181,7 @@ mod observability_producer; mod observatory; mod primitive; pub use primitive::callable_code_request_context; +mod graph_tool; mod recovery_schedule; mod registrars; mod retained; @@ -193,12 +194,16 @@ mod work_attempt_exec; mod work_blocked_interval_recovery; mod work_routing; -use clock::now_micros; -pub use clock::{current_micros, now_millis}; +pub use clock::now_millis; use configuration::*; use feedback::*; use git::*; use github_stack_signal::execute_github_stack_signal_expand; +use graph_tool::execute_graph_tool; +pub use graph_tool::{ + DaemonGraphToolOwnerRegistrationError, GraphToolFuture, GraphToolInvocationV1, + ProjectGraphToolPortV1, RegisteredGraphToolOwnerV1, +}; use handoff::*; use invocation_observability::{ emit_invocation_observation, feedback_observation_operation, invocation_observation_subject, @@ -217,6 +222,7 @@ use retained::*; use source_edit::{ execute_source_edit, execute_source_edit_reconcile, execute_source_edit_rollback, }; +use tracedecay_contracts::now_micros; use types::*; use work::*; pub use work_routing::DaemonWorkProposalRoutingAuthorityV1; diff --git a/crates/tracedecay-daemon-service/src/invocation/administrative_effect.rs b/crates/tracedecay-daemon-service/src/invocation/administrative_effect.rs index 5e608fb900..723711fe4d 100644 --- a/crates/tracedecay-daemon-service/src/invocation/administrative_effect.rs +++ b/crates/tracedecay-daemon-service/src/invocation/administrative_effect.rs @@ -16,7 +16,8 @@ use tracedecay_domain::{ }; use tracedecay_tool_catalog::{EffectClass, UseCaseId}; -use super::{RegisteredWorkRuntime, current_micros}; +use super::RegisteredWorkRuntime; +use tracedecay_contracts::now_micros; /// Policy-bound authority receipt and completed operation receipt for one /// admitted operation of the `family` request family. @@ -49,7 +50,7 @@ pub(super) fn administrative_authority( let authority = AuthorityReceipt::from_context(context, policy, observed_at)?; let execution = OperationReceipt::completed( observed_at, - current_micros(), + now_micros(), deadline, OperationBudgetUsage::default(), )?; diff --git a/crates/tracedecay-daemon-service/src/invocation/clock.rs b/crates/tracedecay-daemon-service/src/invocation/clock.rs index 492d15c2d2..6f1d521352 100644 --- a/crates/tracedecay-daemon-service/src/invocation/clock.rs +++ b/crates/tracedecay-daemon-service/src/invocation/clock.rs @@ -1,8 +1,5 @@ //! Shared wall-clock readings for daemon invocation admission and expiry. -pub use tracedecay_contracts::clock::now_micros; -pub use tracedecay_contracts::clock::now_micros as current_micros; - pub fn now_millis() -> u64 { tracedecay_runtime_core::tracedecay::unix_millis() } diff --git a/crates/tracedecay-daemon-service/src/invocation/configuration.rs b/crates/tracedecay-daemon-service/src/invocation/configuration.rs index 7e1313cd1e..3474a45c70 100644 --- a/crates/tracedecay-daemon-service/src/invocation/configuration.rs +++ b/crates/tracedecay-daemon-service/src/invocation/configuration.rs @@ -27,7 +27,7 @@ pub(super) async fn execute_configuration( ApplicationProblem::cancelled_before_admission(), ); } - if deadline.is_elapsed_at(observed_at) || deadline.is_elapsed_at(current_micros()) { + if deadline.is_elapsed_at(observed_at) || deadline.is_elapsed_at(now_micros()) { return application_problem( wire_request_id, ApplicationProblem::timed_out_before_admission(), @@ -660,7 +660,7 @@ pub(super) fn configuration_evidence( ) -> Result, ConfigurationError> { let execution = OperationReceipt::completed( observed_at, - current_micros(), + now_micros(), deadline, OperationBudgetUsage::default(), ) @@ -704,7 +704,7 @@ fn configuration_preview( .map_err(ConfigurationError::validation)?; let execution = OperationReceipt::completed( observed_at, - current_micros(), + now_micros(), deadline, OperationBudgetUsage::default(), ) diff --git a/crates/tracedecay-daemon-service/src/invocation/configuration/settlement.rs b/crates/tracedecay-daemon-service/src/invocation/configuration/settlement.rs index 097ac08ef1..6e8f384299 100644 --- a/crates/tracedecay-daemon-service/src/invocation/configuration/settlement.rs +++ b/crates/tracedecay-daemon-service/src/invocation/configuration/settlement.rs @@ -314,7 +314,7 @@ mod tests { .unwrap(); let binding = WorkExecutableBindingV1::new( executable, - std::path::PathBuf::from("/tmp/provider-configuration-restart-fixture"), + std::env::temp_dir().join("provider-configuration-restart-fixture"), vec![WorkExecutableCapabilityV1::CodexCliExecJson], Vec::new(), ) @@ -387,7 +387,7 @@ mod tests { digest('f'), ) .unwrap(), - std::path::PathBuf::from("/tmp/provider-configuration-mixed-fixture"), + std::env::temp_dir().join("provider-configuration-mixed-fixture"), vec![WorkExecutableCapabilityV1::CodexCliExecJson], Vec::new(), ) diff --git a/crates/tracedecay-daemon-service/src/invocation/dispatch.rs b/crates/tracedecay-daemon-service/src/invocation/dispatch.rs index 641a312b28..bd57dd6ca1 100644 --- a/crates/tracedecay-daemon-service/src/invocation/dispatch.rs +++ b/crates/tracedecay-daemon-service/src/invocation/dispatch.rs @@ -254,7 +254,7 @@ impl DaemonInvocationService { emit_invocation_observation( observations.as_ref(), observation_subject.as_ref(), - current_micros(), + now_micros(), FeedbackSourceEventV1::SurfaceArgumentRejected { operation: feedback_observation_operation(operation), route: delivery_route, @@ -300,7 +300,7 @@ impl DaemonInvocationService { DaemonInvocationProblem::Unavailable, ); } - let dispatched_at = current_micros(); + let dispatched_at = now_micros(); if is_observable_operation(operation) { emit_invocation_observation( observations.as_ref(), @@ -327,6 +327,7 @@ impl DaemonInvocationService { let retained_runtime = runtimes.retained; let lsp_owner = runtimes.lsp_owner; let source_edit_owner = runtimes.source_edit; + let graph_tool_owner = runtimes.graph_tool; let response = match request.payload { DaemonInvocationPayload::GitRead { @@ -603,6 +604,7 @@ impl DaemonInvocationService { request_id, ApplicationSurfaceOperation::FeedbackImpact, PrimitiveRequest::Impact(request), + None, observed_at, deadline, cancellation, @@ -622,6 +624,7 @@ impl DaemonInvocationService { request_id, ApplicationSurfaceOperation::AffectedTests, PrimitiveRequest::AffectedFileTests(request), + None, observed_at, deadline, cancellation, @@ -641,6 +644,7 @@ impl DaemonInvocationService { request_id, ApplicationSurfaceOperation::TestResults, PrimitiveRequest::RecentTestResults(page), + None, observed_at, deadline, cancellation, @@ -650,6 +654,7 @@ impl DaemonInvocationService { DaemonInvocationPayload::PrimitiveRead { surface_operation, request, + resolved_scope, observed_at, deadline, cancellation, @@ -661,6 +666,7 @@ impl DaemonInvocationService { request_id, surface_operation, request, + resolved_scope.as_ref(), observed_at, deadline, cancellation, @@ -671,6 +677,7 @@ impl DaemonInvocationService { surface_operation, request, page, + resolved_scope, observed_at, deadline, cancellation, @@ -698,6 +705,7 @@ impl DaemonInvocationService { request_id, surface_operation, request, + resolved_scope.as_ref(), observed_at, deadline, cancellation, @@ -708,6 +716,7 @@ impl DaemonInvocationService { surface_operation, request, page, + resolved_scope, observed_at, deadline, cancellation, @@ -720,6 +729,7 @@ impl DaemonInvocationService { surface_operation, request, page, + resolved_scope.as_ref(), observed_at, deadline, cancellation, @@ -800,6 +810,27 @@ impl DaemonInvocationService { )) .await } + DaemonInvocationPayload::GraphTool { + surface_operation, + arguments, + observed_at: _, + deadline, + cancellation, + } => { + let Some(owner) = graph_tool_owner else { + return missing_registered_owner_problem(publication, request_id); + }; + Box::pin(execute_graph_tool( + request_id, + owner, + surface_operation, + arguments, + deadline, + cancellation, + request_cancellation, + )) + .await + } DaemonInvocationPayload::RetainedApplication { request, observed_at, diff --git a/crates/tracedecay-daemon-service/src/invocation/feedback.rs b/crates/tracedecay-daemon-service/src/invocation/feedback.rs index 7c6fac4178..ac6141c5b7 100644 --- a/crates/tracedecay-daemon-service/src/invocation/feedback.rs +++ b/crates/tracedecay-daemon-service/src/invocation/feedback.rs @@ -265,7 +265,9 @@ fn feedback_invocation_result_with( let application = result.map_err(|problem| problem.problem.into_source())?; let evidence = match application.outcome { ApplicationOutcome::Evidence(packet) => packet, - ApplicationOutcome::Preview(_) | ApplicationOutcome::Effect(_) => { + ApplicationOutcome::Preview(_) + | ApplicationOutcome::Effect(_) + | ApplicationOutcome::Result(_) => { return Err(ApplicationProblem::unavailable(SafeDiagnostic { code: "feedback.invalid_owner_result".to_owned(), message: "The feedback read owner returned an invalid outcome".to_owned(), @@ -388,7 +390,7 @@ pub fn advisory_cycle_invocation_result( use tracedecay_application::advisory::AdvisoryCycleOutcome; use tracedecay_domain::feedback::FeedbackCycleTerminationV1; - let ended_at = current_micros(); + let ended_at = now_micros(); let policy_digest = canonical_sha256(&( "tracedecay.daemon.feedback-advisory-policy", context.scope(), @@ -616,7 +618,7 @@ pub fn feedback_proximity_invocation_result( result .validate() .map_err(|_| feedback_proximity_contract_problem())?; - let ended_at = current_micros(); + let ended_at = now_micros(); let (termination, completeness, returned, omission_reason) = match &result { FeedbackProximityReadResultV1::Complete { page } => ( OperationTermination::Completed, @@ -802,7 +804,7 @@ pub(super) async fn execute_feedback_advisory_cycle( ApplicationProblem::cancelled_before_admission(), ); } - if deadline.is_elapsed_at(observed_at) || deadline.is_elapsed_at(current_micros()) { + if deadline.is_elapsed_at(observed_at) || deadline.is_elapsed_at(now_micros()) { return application_problem( wire_request_id, ApplicationProblem::timed_out_before_admission(), diff --git a/crates/tracedecay-daemon-service/src/invocation/git.rs b/crates/tracedecay-daemon-service/src/invocation/git.rs index cd31e22e7a..35bd54b390 100644 --- a/crates/tracedecay-daemon-service/src/invocation/git.rs +++ b/crates/tracedecay-daemon-service/src/invocation/git.rs @@ -129,7 +129,7 @@ pub(super) fn git_read_evidence_packet( .collect(); let execution = OperationReceipt::completed( observed_at, - current_micros(), + now_micros(), deadline, OperationBudgetUsage::default(), ) @@ -231,7 +231,7 @@ pub(super) async fn execute_git_read( ApplicationProblem::cancelled_before_admission(), ); } - if deadline.is_elapsed_at(observed_at) || deadline.is_elapsed_at(current_micros()) { + if deadline.is_elapsed_at(observed_at) || deadline.is_elapsed_at(now_micros()) { return application_problem( wire_request_id, ApplicationProblem::timed_out_before_admission(), @@ -241,11 +241,7 @@ pub(super) async fn execute_git_read( Ok(authority) => authority, Err(_) => return concealed_application_problem(wire_request_id), }; - let remaining_micros = deadline - .expires_at - .0 - .saturating_sub(current_micros().0) - .max(0) as u64; + let remaining_micros = deadline.expires_at.0.saturating_sub(now_micros().0).max(0) as u64; let bounds = tracedecay_application::git_query::GitQueryBounds { max_entries: if matches!( &request.request, @@ -317,7 +313,7 @@ pub(super) async fn execute_git_read( ApplicationProblem::cancelled_before_admission(), ); } - if deadline.is_elapsed_at(current_micros()) { + if deadline.is_elapsed_at(now_micros()) { return application_problem( wire_request_id, ApplicationProblem::timed_out_before_admission(), @@ -372,7 +368,7 @@ pub(super) async fn execute_git_read( || initial.configuration_digest != terminal.configuration_digest || initial.catalog_digest != terminal.catalog_digest || initial.privacy_digest != terminal.privacy_digest - || current_micros() >= terminal.grant_expires_at + || now_micros() >= terminal.grant_expires_at { return concealed_application_problem(wire_request_id); } @@ -829,7 +825,7 @@ async fn publish_invocation_terminal( started_at: UtcMicros, effective_deadline: Deadline, ) { - let ended_at = current_micros(); + let ended_at = now_micros(); let ended_at = if ended_at < started_at { started_at } else { diff --git a/crates/tracedecay-daemon-service/src/invocation/github_stack_signal.rs b/crates/tracedecay-daemon-service/src/invocation/github_stack_signal.rs index 51941a46cd..db9651b610 100644 --- a/crates/tracedecay-daemon-service/src/invocation/github_stack_signal.rs +++ b/crates/tracedecay-daemon-service/src/invocation/github_stack_signal.rs @@ -166,7 +166,7 @@ fn github_stack_signal_evidence( ) -> Result, ApplicationProblem> { let execution = OperationReceipt::completed( observed_at, - current_micros(), + now_micros(), deadline, OperationBudgetUsage::default(), ) diff --git a/crates/tracedecay-daemon-service/src/invocation/graph_tool.rs b/crates/tracedecay-daemon-service/src/invocation/graph_tool.rs new file mode 100644 index 0000000000..a42dee7472 --- /dev/null +++ b/crates/tracedecay-daemon-service/src/invocation/graph_tool.rs @@ -0,0 +1,134 @@ +//! Per-project owner for the graph and port reads whose typed results are +//! computed by the project's handler authority. + +use std::future::Future; +use std::path::PathBuf; +use std::pin::Pin; +use std::sync::Arc; + +use serde_json::{Map, Value}; +use tracedecay_contracts::graph_tool::GraphToolCompletionV1; +use tracedecay_contracts::{ + ApplicationProblem, CancellationContext, CancellationSignal, CancellationState, Deadline, + RequestId, ResolvedScope, now_micros, +}; +use tracedecay_daemon_protocol::{ + DaemonInvocationOutcome, DaemonInvocationProblem, DaemonInvocationResponse, +}; +use tracedecay_tool_catalog::ApplicationSurfaceOperation; + +use super::DaemonInvocationService; +use crate::project_runtime::ProjectRuntimeRegistryError; + +/// One admitted graph-tool invocation. +pub struct GraphToolInvocationV1 { + pub operation: ApplicationSurfaceOperation, + pub arguments: Map, + pub request_id: RequestId, + pub deadline: Deadline, + pub cancellation: CancellationSignal, +} + +pub type GraphToolFuture<'a> = + Pin> + Send + 'a>>; + +/// The project's handler authority that computes graph-tool results. +pub trait ProjectGraphToolPortV1: Send + Sync { + fn execute(&self, invocation: GraphToolInvocationV1) -> GraphToolFuture<'_>; +} + +/// The registered owner: the authorized scope it answers for and its port. +#[derive(Clone)] +pub struct RegisteredGraphToolOwnerV1 { + scope: ResolvedScope, + port: Arc, +} + +impl RegisteredGraphToolOwnerV1 { + pub fn new(scope: ResolvedScope, port: Arc) -> Self { + Self { scope, port } + } +} + +/// Typed refusal from [`DaemonInvocationService::register_graph_tool_owner`]. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum DaemonGraphToolOwnerRegistrationError { + #[error(transparent)] + Registry(#[from] ProjectRuntimeRegistryError), + #[error("a graph-tool owner for a different authorized scope is already registered")] + ForeignAuthority, +} + +impl DaemonInvocationService { + /// Registers this project's graph-tool owner. A later server for the same + /// authorized scope (the full server replacing the core, a reopen) + /// replaces the port; a foreign scope is refused. + #[hotpath::skip] + pub async fn register_graph_tool_owner( + &self, + project_root: PathBuf, + owner: RegisteredGraphToolOwnerV1, + ) -> Result<(), DaemonGraphToolOwnerRegistrationError> { + self.project_runtimes + .register_or_reconcile( + project_root, + |incumbent: &mut RegisteredGraphToolOwnerV1| { + if incumbent.scope == owner.scope { + incumbent.port = Arc::clone(&owner.port); + Ok(()) + } else { + Err(DaemonGraphToolOwnerRegistrationError::ForeignAuthority) + } + }, + || async { Ok(owner.clone()) }, + ) + .await + } +} + +#[hotpath::measure(label = "daemon.service.graph_tool.execute", future = true)] +pub(super) async fn execute_graph_tool( + request_id: String, + owner: RegisteredGraphToolOwnerV1, + operation: ApplicationSurfaceOperation, + arguments: Map, + deadline: Deadline, + cancellation: CancellationContext, + request_cancellation: tracedecay_runtime_core::cancellation::CancellationToken, +) -> DaemonInvocationResponse { + let (Ok(typed_request_id), Ok(signal)) = ( + RequestId::new(request_id.clone()), + CancellationSignal::active(cancellation.token_id.as_str()), + ) else { + return DaemonInvocationResponse::problem( + request_id, + DaemonInvocationProblem::InvalidRequest, + ); + }; + if let CancellationState::Cancelled { requested_at } = &cancellation.state { + signal.cancel(*requested_at); + } + let execution = owner.port.execute(GraphToolInvocationV1 { + operation, + arguments, + request_id: typed_request_id, + deadline, + cancellation: signal.clone(), + }); + tokio::pin!(execution); + let outcome = tokio::select! { + outcome = &mut execution => outcome, + () = request_cancellation.cancelled() => { + signal.cancel(now_micros()); + execution.await + } + }; + let outcome = match outcome { + Ok(completion) => DaemonInvocationOutcome::GraphTool { + scope: owner.scope, + completion, + }, + Err(problem) => DaemonInvocationOutcome::ApplicationProblem { problem }, + }; + DaemonInvocationResponse::with_outcome(request_id, outcome) +} diff --git a/crates/tracedecay-daemon-service/src/invocation/handoff.rs b/crates/tracedecay-daemon-service/src/invocation/handoff.rs index d5091daf66..4bf4bf104d 100644 --- a/crates/tracedecay-daemon-service/src/invocation/handoff.rs +++ b/crates/tracedecay-daemon-service/src/invocation/handoff.rs @@ -41,7 +41,7 @@ impl HandoffOpenTargetPort for DaemonHandoffOpenTargets { } => { let selection = tracedecay_contracts::WorkProductSelectionScopeV1::relations( std::collections::BTreeSet::from([ - tracedecay_contracts::WorkRelationScopeV1::Repository { + tracedecay_contracts::WorkProductAuthorizedRelationScopeV1::Repository { project_id: context.scope().project_id.clone(), repository_id: context.scope().repository_id.clone(), }, @@ -161,9 +161,9 @@ fn current_feedback_finding( .payload .ok_or(HandoffOpenTargetError::Unavailable) .map(|result| Some(result.finding)), - ApplicationOutcome::Preview(_) | ApplicationOutcome::Effect(_) => { - Err(HandoffOpenTargetError::Unavailable) - } + ApplicationOutcome::Preview(_) + | ApplicationOutcome::Effect(_) + | ApplicationOutcome::Result(_) => Err(HandoffOpenTargetError::Unavailable), } } diff --git a/crates/tracedecay-daemon-service/src/invocation/invocation_observability.rs b/crates/tracedecay-daemon-service/src/invocation/invocation_observability.rs index c597ef7fa8..cd519d972f 100644 --- a/crates/tracedecay-daemon-service/src/invocation/invocation_observability.rs +++ b/crates/tracedecay-daemon-service/src/invocation/invocation_observability.rs @@ -100,7 +100,8 @@ pub(super) fn feedback_observation_operation( | DaemonInvocationOperation::NativeIntegrationWorktreeReconcile | DaemonInvocationOperation::SourceEdit | DaemonInvocationOperation::SourceEditReconcile - | DaemonInvocationOperation::SourceEditRollback => FeedbackOperationV1::FeedbackCycle, + | DaemonInvocationOperation::SourceEditRollback + | DaemonInvocationOperation::GraphTool => FeedbackOperationV1::FeedbackCycle, } } @@ -126,6 +127,7 @@ pub(super) fn invocation_response_outcome( | DaemonInvocationOutcome::Configuration { .. } | DaemonInvocationOutcome::ContextScout { .. } | DaemonInvocationOutcome::RetainedApplication { .. } + | DaemonInvocationOutcome::GraphTool { .. } | DaemonInvocationOutcome::GitHubStackSignalExpand { .. } | DaemonInvocationOutcome::MultiRootScopeSetRead { .. } | DaemonInvocationOutcome::MultiRootScopeSetCompareAndSwap { .. } @@ -245,7 +247,7 @@ pub(super) fn observe_invocation_response( started_at: UtcMicros, response: &DaemonInvocationResponse, ) { - let observed_at = current_micros(); + let observed_at = now_micros(); let outcome = invocation_response_outcome(response); let duration_micros = u64::try_from(observed_at.0.saturating_sub(started_at.0)).ok(); if let Some(route) = route { diff --git a/crates/tracedecay-daemon-service/src/invocation/lsp.rs b/crates/tracedecay-daemon-service/src/invocation/lsp.rs index 5621439003..836b09b5ea 100644 --- a/crates/tracedecay-daemon-service/src/invocation/lsp.rs +++ b/crates/tracedecay-daemon-service/src/invocation/lsp.rs @@ -28,7 +28,7 @@ pub(super) fn admit_lsp_control( ApplicationProblem::cancelled_before_admission(), ))); } - if deadline.is_elapsed_at(current_micros()) { + if deadline.is_elapsed_at(now_micros()) { return Err(Box::new(DaemonInvocationResponse::application_problem( request_id, ApplicationProblem::timed_out_before_admission(), @@ -234,7 +234,7 @@ impl DaemonInvocationService { ) .ok()?; if request_cancellation.is_some_and(CancellationToken::is_cancelled) - || deadline.is_elapsed_at(current_micros()) + || deadline.is_elapsed_at(now_micros()) { return None; } @@ -308,7 +308,7 @@ impl DaemonInvocationService { let authority = AuthorityReceipt::from_context(&context, policy, observed_at).ok()?; let execution = OperationReceipt::completed( observed_at, - current_micros(), + now_micros(), deadline, OperationBudgetUsage::default(), ) @@ -378,8 +378,10 @@ impl DaemonInvocationService { let project_runtimes_clean = self.project_runtimes.shut_down_all().await; step("project_runtimes_shut_down"); self.session_holder_databases.lock().await.clear(); - self.operation_events.expire_all().await; - step("operation_events_expired"); + // The operation-event authority is process-global, not owned by this + // composition: only frontiers without a live producer expire here. + self.operation_events.expire_idle().await; + step("idle_operation_events_expired"); let lease_shutdown_clean = lease_shutdown.is_ok(); if let Err(problem) = lease_shutdown { tracing::error!( @@ -677,7 +679,7 @@ impl DaemonInvocationService { &mut session.next_delivery_sequence, frame, access.session_id(), - current_micros(), + now_micros(), ); } let frame = outbound.and_then(|frame| String::from_utf8(frame).ok()); diff --git a/crates/tracedecay-daemon-service/src/invocation/lsp/federated_pairing_tests.rs b/crates/tracedecay-daemon-service/src/invocation/lsp/federated_pairing_tests.rs index e8da040a99..bc7851057d 100644 --- a/crates/tracedecay-daemon-service/src/invocation/lsp/federated_pairing_tests.rs +++ b/crates/tracedecay-daemon-service/src/invocation/lsp/federated_pairing_tests.rs @@ -16,7 +16,7 @@ //! not name. use std::collections::BTreeSet; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use tracedecay_contracts::{ CapabilityGrantId, DisclosureClass, RegisteredRootLocatorV1, ResolvedScope, @@ -82,13 +82,29 @@ fn grant(scope: &ResolvedScope, suffix: &str) -> CapabilityGrantSnapshot { .expect("scope grant") } +/// A never-created root that is absolute on the host, as a registered root +/// locator requires: `/name` has no drive, so Windows does not call it absolute. +fn fixture_root(name: &str) -> PathBuf { + if cfg!(windows) { + PathBuf::from(format!(r"C:\{name}")) + } else { + PathBuf::from(format!("/{name}")) + } +} + +fn fixture_root_uri(root: &Path) -> String { + url::Url::from_file_path(root) + .expect("absolute fixture root") + .to_string() +} + async fn install_root( service: &DaemonInvocationService, scope: &ResolvedScope, suffix: &str, ) -> (PathBuf, String, ResolvedScope, RegisteredRootLocatorV1) { - let project_root = PathBuf::from(format!("/federated-pairing-{suffix}")); - let uri = format!("file:///federated-pairing-{suffix}"); + let project_root = fixture_root(&format!("federated-pairing-{suffix}")); + let uri = fixture_root_uri(&project_root); service .install_lsp_owner( project_root.clone(), @@ -197,7 +213,7 @@ async fn federated_workspace_authority_pairs_by_scope_digest_not_list_position() // to pair with, so the whole workspace is refused rather than partially // admitted. let unnamed = AdmittedRoot::authorized( - "file:///federated-pairing-unnamed".to_owned(), + fixture_root_uri(&fixture_root("federated-pairing-unnamed")), scope_for(999).scope_digest, ); let mut widened = authorized.clone(); diff --git a/crates/tracedecay-daemon-service/src/invocation/lsp_delivery.rs b/crates/tracedecay-daemon-service/src/invocation/lsp_delivery.rs index 8cd5638879..0d448043ee 100644 --- a/crates/tracedecay-daemon-service/src/invocation/lsp_delivery.rs +++ b/crates/tracedecay-daemon-service/src/invocation/lsp_delivery.rs @@ -102,7 +102,7 @@ impl RuntimeLspSession { return LspDeliverySettlementAdmissionV1::RecorderUnavailable; }; let settlement = tracedecay_domain::DeliverySettlementV1 { - settled_at: current_micros().max(attempt.attempted_at), + settled_at: now_micros().max(attempt.attempted_at), attempt, outcome, drop_reason, diff --git a/crates/tracedecay-daemon-service/src/invocation/native_integration.rs b/crates/tracedecay-daemon-service/src/invocation/native_integration.rs index 70f695e4e6..a74bb173de 100644 --- a/crates/tracedecay-daemon-service/src/invocation/native_integration.rs +++ b/crates/tracedecay-daemon-service/src/invocation/native_integration.rs @@ -1005,7 +1005,7 @@ fn native_integration_evidence( let invalid = invalid_native_integration_request; let execution = OperationReceipt::completed( observed_at, - current_micros(), + now_micros(), deadline, OperationBudgetUsage::default(), ) diff --git a/crates/tracedecay-daemon-service/src/invocation/observatory.rs b/crates/tracedecay-daemon-service/src/invocation/observatory.rs index 22db6fa6ee..f8533bed5e 100644 --- a/crates/tracedecay-daemon-service/src/invocation/observatory.rs +++ b/crates/tracedecay-daemon-service/src/invocation/observatory.rs @@ -128,7 +128,7 @@ pub(super) async fn execute_observatory_read( ); } }; - let finished_at = current_micros(); + let finished_at = now_micros(); let authority = match authorization .recheck_publication(&context, &operation, &admission, finished_at) .await @@ -157,7 +157,7 @@ fn observatory_evidence( ) -> Result { let execution = OperationReceipt::completed( observed_at, - current_micros(), + now_micros(), deadline, OperationBudgetUsage::default(), ) diff --git a/crates/tracedecay-daemon-service/src/invocation/primitive.rs b/crates/tracedecay-daemon-service/src/invocation/primitive.rs index d4b66a3f19..9d4951ccdd 100644 --- a/crates/tracedecay-daemon-service/src/invocation/primitive.rs +++ b/crates/tracedecay-daemon-service/src/invocation/primitive.rs @@ -35,6 +35,7 @@ pub(super) async fn execute_primitive( wire_request_id: String, surface_operation: ApplicationSurfaceOperation, request: PrimitiveRequest, + resolved_scope: Option<&ResolvedScope>, observed_at: UtcMicros, deadline: Deadline, cancellation: CancellationContext, @@ -61,6 +62,12 @@ pub(super) async fn execute_primitive( let Some(registered) = registered else { return missing_registered_owner_problem(publication, wire_request_id); }; + // A cross-project selection executes only on the runtime registered for + // exactly that scope; anything else is concealed rather than served from + // whichever project this route reached. + if !resolved_scope.is_none_or(|scope| scope == ®istered.scope) { + return concealed_application_problem(wire_request_id); + } let access = match registered.authorization.current(observed_at).await { Ok(access) if access.scope == registered.scope => access, Ok(_) | Err(_) => return concealed_application_problem(wire_request_id), @@ -120,7 +127,7 @@ pub(super) async fn execute_primitive( } }; if result.is_ok() { - let finished_at = current_micros(); + let finished_at = now_micros(); let publication_authority = match authorization .recheck_publication(&context, &operation, &admission, finished_at) .await @@ -160,6 +167,7 @@ pub(super) async fn execute_callable_code( surface_operation: ApplicationSurfaceOperation, request: CallableCodeSurfaceRequest, page: PageRequest, + resolved_scope: Option<&ResolvedScope>, observed_at: UtcMicros, deadline: Deadline, cancellation: CancellationContext, @@ -176,6 +184,12 @@ pub(super) async fn execute_callable_code( // warming unless project-open already recorded a terminal failure. return missing_registered_owner_problem(publication, wire_request_id); }; + // A cross-project selection executes only on the runtime registered for + // exactly that scope; anything else is concealed rather than served from + // whichever project this route reached. + if !resolved_scope.is_none_or(|scope| scope == ®istered.scope) { + return concealed_application_problem(wire_request_id); + } let access = match registered.authorization.current(observed_at).await { Ok(access) => access, Err(problem) => return application_problem(wire_request_id, problem), @@ -347,7 +361,7 @@ pub fn callable_code_request_context( if cancellation.is_cancelled() { return Err(ApplicationProblem::cancelled_before_admission()); } - if deadline.is_elapsed_at(observed_at) || deadline.is_elapsed_at(current_micros()) { + if deadline.is_elapsed_at(observed_at) || deadline.is_elapsed_at(now_micros()) { return Err(ApplicationProblem::timed_out_before_admission()); } let expires_at = UtcMicros(deadline.expires_at.0.min(access.grant_expires_at.0)); @@ -477,7 +491,7 @@ pub(super) async fn execute_context_scout( } }; let Some(configuration) = - tracedecay_agent_hosts::agents::context_scout::ports::ContextScoutConfigurationPinV1::from_current(¤t) + tracedecay_agent_hosts::agents::context_scout::address_registry::ContextScoutConfigurationPinV1::from_current(¤t) else { return DaemonInvocationResponse::problem( wire_request_id, @@ -824,7 +838,7 @@ async fn execute_context_scout_mutation( }; let execution = match OperationReceipt::completed( observed_at, - current_micros(), + now_micros(), deadline, OperationBudgetUsage::default(), ) { @@ -1117,7 +1131,7 @@ async fn reconcile_context_scout_configuration( .map_err(|_| ContextScoutActivationReconciliationError::ConfigurationUnavailable)?; let current = current.into_current_state(); let refreshed = - tracedecay_agent_hosts::agents::context_scout::ports::ContextScoutConfigurationPinV1::from_current(¤t) + tracedecay_agent_hosts::agents::context_scout::address_registry::ContextScoutConfigurationPinV1::from_current(¤t) .ok_or(ContextScoutActivationReconciliationError::InvalidConfiguration)?; if !registry .advance_control_exact_address(address, scope, &refreshed) @@ -1162,7 +1176,7 @@ pub enum DaemonPrimitiveRuntimeRegistrationError { RegistryClosed, #[error("a concurrent primitive runtime build failed: {detail}")] ConcurrentBuildFailed { detail: String }, - #[error("the application primitive runtime could not be opened")] + #[error("the application primitive runtime could not be opened: {0}")] Open(#[from] ApplicationContractError), } diff --git a/crates/tracedecay-daemon-service/src/invocation/primitive/context_scout_registry.rs b/crates/tracedecay-daemon-service/src/invocation/primitive/context_scout_registry.rs index 8fb021888e..dbd903db50 100644 --- a/crates/tracedecay-daemon-service/src/invocation/primitive/context_scout_registry.rs +++ b/crates/tracedecay-daemon-service/src/invocation/primitive/context_scout_registry.rs @@ -5,7 +5,7 @@ use thiserror::Error; use tracedecay_domain::{ProjectId, UserProfileId}; use super::super::{DaemonInvocationService, InvocationProjectRuntimeIdentityV1}; -use tracedecay_agent_hosts::agents::context_scout::ports::ProjectContextScoutAddressRegistryV1; +use tracedecay_agent_hosts::agents::context_scout::address_registry::ProjectContextScoutAddressRegistryV1; use tracedecay_runtime_core::db::Database; #[derive(Debug, Error)] diff --git a/crates/tracedecay-daemon-service/src/invocation/registrars.rs b/crates/tracedecay-daemon-service/src/invocation/registrars.rs index d7d784566a..fe04b49a8d 100644 --- a/crates/tracedecay-daemon-service/src/invocation/registrars.rs +++ b/crates/tracedecay-daemon-service/src/invocation/registrars.rs @@ -842,7 +842,7 @@ impl DaemonConfigurationRuntimeRegistrar { selection, ) .map_err(tracedecay_configuration::map_profile_worker_configuration_error)?; - let observed_at = current_micros(); + let observed_at = now_micros(); let authority = registered .grants .issue_direct( @@ -931,7 +931,7 @@ impl DaemonConfigurationRuntimeRegistrar { } })?; runtime - .record_runtime_activation(Some(current.revision_id().clone()), None, current_micros()) + .record_runtime_activation(Some(current.revision_id().clone()), None, now_micros()) .await .map_err(|error| TraceDecayError::Config { message: format!("configuration runtime activation could not be recorded: {error}"), diff --git a/crates/tracedecay-daemon-service/src/invocation/tests/dispatch_tests.rs b/crates/tracedecay-daemon-service/src/invocation/tests/dispatch_tests.rs index f7f7181e78..e761e6cc05 100644 --- a/crates/tracedecay-daemon-service/src/invocation/tests/dispatch_tests.rs +++ b/crates/tracedecay-daemon-service/src/invocation/tests/dispatch_tests.rs @@ -70,7 +70,7 @@ fn only_explicit_protocol_frames_select_the_invocation_route() { async fn lsp_gateway_control_terminates_before_owner_lookup() { let service = DaemonInvocationService::default(); let registry = Arc::new(Mutex::new(LspSessionRegistry::default())); - let now = current_micros(); + let now = now_micros(); let requests = [ ( DaemonInvocationRequest::lsp_open( @@ -232,6 +232,7 @@ fn callable_code_validation_accepts_only_matching_operation_request_pairs() { surface_operation: *operation, request: request(*request_case), page: page.clone(), + resolved_scope: None, observed_at: UtcMicros(30), deadline: deadline.clone(), cancellation: cancellation.clone(), diff --git a/crates/tracedecay-daemon-service/src/invocation/tests/feedback_tests.rs b/crates/tracedecay-daemon-service/src/invocation/tests/feedback_tests.rs index 6ec8dd5829..b7916f3b69 100644 --- a/crates/tracedecay-daemon-service/src/invocation/tests/feedback_tests.rs +++ b/crates/tracedecay-daemon-service/src/invocation/tests/feedback_tests.rs @@ -69,7 +69,7 @@ impl DaemonAdvisoryCycleInvocationPort for MountedAdvisoryCycle { #[tokio::test] async fn advisory_cycle_dispatches_to_the_mounted_project_owner() { - let observed_at = current_micros(); + let observed_at = now_micros(); let project_id = ProjectId::new("project.feedback-cycle-mounted").expect("project id"); let owner = DaemonAdvisoryCycleInvocationOwner::new(project_id, Arc::new(MountedAdvisoryCycle)); let response = execute_feedback_advisory_cycle( @@ -112,7 +112,7 @@ fn proximity_wire_request_has_one_typed_body() { #[tokio::test] async fn proximity_dispatches_to_the_mounted_project_owner() { - let observed_at = current_micros(); + let observed_at = now_micros(); let project_id = ProjectId::new("project.feedback-proximity-mounted").expect("project id"); let owner = DaemonAdvisoryCycleInvocationOwner::new(project_id, Arc::new(MountedAdvisoryCycle)); let response = execute_feedback_proximity( diff --git a/crates/tracedecay-daemon-service/src/invocation/tests/project_admission_tests.rs b/crates/tracedecay-daemon-service/src/invocation/tests/project_admission_tests.rs index 75a01fe439..1dbb7aae5b 100644 --- a/crates/tracedecay-daemon-service/src/invocation/tests/project_admission_tests.rs +++ b/crates/tracedecay-daemon-service/src/invocation/tests/project_admission_tests.rs @@ -44,7 +44,7 @@ async fn project_quiescence_denies_git_cached_routes() { .await .expect("quiesce project runtime"); let registry = Arc::new(Mutex::new(LspSessionRegistry::default())); - let now = current_micros(); + let now = now_micros(); let deadline = Deadline::new(UtcMicros(now.0.saturating_add(30_000_000))).expect("deadline"); let request = DaemonInvocationRequest { protocol: tracedecay_daemon_protocol::DAEMON_INVOCATION_PROTOCOL.to_owned(), @@ -666,7 +666,7 @@ async fn missing_work_owner_stops_retrying_after_publication() { .mark_publication_ready(&publication) ); } - let now = current_micros(); + let now = now_micros(); let request = DaemonInvocationRequest::work_application( "request.work.unmounted", tracedecay_daemon_protocol::WorkApplicationInvocationV1::Topology( diff --git a/crates/tracedecay-daemon-service/src/invocation/tests/source_edit_tests.rs b/crates/tracedecay-daemon-service/src/invocation/tests/source_edit_tests.rs index 758a30aed0..922d4562fd 100644 --- a/crates/tracedecay-daemon-service/src/invocation/tests/source_edit_tests.rs +++ b/crates/tracedecay-daemon-service/src/invocation/tests/source_edit_tests.rs @@ -240,7 +240,7 @@ async fn source_edit_dispatch_refuses_an_expired_deadline_without_a_silent_succe let project_root = PathBuf::from("/projects/source-edit-expired"); admit_project_without_source_edit_owner(&service, project_root.clone()).await; let registry = Arc::new(Mutex::new(LspSessionRegistry::default())); - let now = current_micros(); + let now = now_micros(); let request = DaemonInvocationRequest::source_edit( "request.source-edit.expired", source_edit_invocation(), diff --git a/crates/tracedecay-daemon-service/src/invocation/types.rs b/crates/tracedecay-daemon-service/src/invocation/types.rs index 14cebaace2..eaf45dd83d 100644 --- a/crates/tracedecay-daemon-service/src/invocation/types.rs +++ b/crates/tracedecay-daemon-service/src/invocation/types.rs @@ -399,7 +399,8 @@ impl BoundedHookOrchestratorV1 { entry.cancellation.cancel(); } } - let deadline = tokio::time::Instant::now() + crate::TASK_ABORT_DEADLINE; + let deadline = + tokio::time::Instant::now() + tracedecay_runtime_core::DAEMON_TASK_ABORT_DEADLINE; for mut task in tasks { match tokio::time::timeout_at(deadline, &mut task).await { Ok(Ok(())) => {} diff --git a/crates/tracedecay-daemon-service/src/invocation/work/outcome.rs b/crates/tracedecay-daemon-service/src/invocation/work/outcome.rs index 8b9e86d7b2..390e568802 100644 --- a/crates/tracedecay-daemon-service/src/invocation/work/outcome.rs +++ b/crates/tracedecay-daemon-service/src/invocation/work/outcome.rs @@ -21,8 +21,8 @@ use tracedecay_daemon_protocol::{ }; use super::super::administrative_effect::administrative_command_effect; -use super::super::current_micros; use super::{RegisteredWorkRuntime, application_problem}; +use tracedecay_contracts::now_micros; pub(super) fn offer_work_blocked_interval_receipts( durable_write_signal: &super::WorkDurableWriteSignalV1, @@ -142,9 +142,7 @@ pub(crate) fn work_background_context( identity.attempt_id().as_str() ))?; let deadline = Deadline::new(UtcMicros( - current_micros() - .0 - .saturating_add(BACKGROUND_DEADLINE_MICROS), + now_micros().0.saturating_add(BACKGROUND_DEADLINE_MICROS), ))?; let cancellation = CancellationContext::active(format!( "work-attempt-exec-{}", @@ -170,9 +168,7 @@ pub(crate) fn work_blocked_interval_recovery_context( const BACKGROUND_DEADLINE_MICROS: i64 = 86_400_000_000; let request_id = RequestId::new("work-blocked-interval-recovery")?; let deadline = Deadline::new(UtcMicros( - current_micros() - .0 - .saturating_add(BACKGROUND_DEADLINE_MICROS), + now_micros().0.saturating_add(BACKGROUND_DEADLINE_MICROS), ))?; let cancellation = CancellationContext::active("cancel.work-blocked-interval-recovery")?; RequestContext::new( @@ -384,7 +380,7 @@ where })?; let execution = OperationReceipt::completed( observed_at, - current_micros(), + now_micros(), deadline, OperationBudgetUsage::default(), )?; diff --git a/crates/tracedecay-daemon-service/src/invocation/work/preparation.rs b/crates/tracedecay-daemon-service/src/invocation/work/preparation.rs index b923172791..9572ebacef 100644 --- a/crates/tracedecay-daemon-service/src/invocation/work/preparation.rs +++ b/crates/tracedecay-daemon-service/src/invocation/work/preparation.rs @@ -254,10 +254,12 @@ pub(super) fn current_work_product_snapshot( CapabilityId::new(capability).map_err(|_| work_product_authority_unavailable())?; let binding = tracedecay_contracts::WorkProductBindingV1::new(capability, use_case.clone()); let selection = tracedecay_contracts::WorkProductSelectionScopeV1::relations( - std::collections::BTreeSet::from([tracedecay_contracts::WorkRelationScopeV1::Repository { - project_id: context.scope().project_id.clone(), - repository_id: context.scope().repository_id.clone(), - }]), + std::collections::BTreeSet::from([ + tracedecay_contracts::WorkProductAuthorizedRelationScopeV1::Repository { + project_id: context.scope().project_id.clone(), + repository_id: context.scope().repository_id.clone(), + }, + ]), ) .map_err(|_| work_product_authority_unavailable())?; let read = diff --git a/crates/tracedecay-daemon-service/src/invocation/work/request_dispatch.rs b/crates/tracedecay-daemon-service/src/invocation/work/request_dispatch.rs index d6c951eb4d..468b12110f 100644 --- a/crates/tracedecay-daemon-service/src/invocation/work/request_dispatch.rs +++ b/crates/tracedecay-daemon-service/src/invocation/work/request_dispatch.rs @@ -176,19 +176,11 @@ pub(super) async fn dispatch_work_application( request.into(), ); if result.is_ok() { - let disposition = match disposition { - tracedecay_contracts::ReviewWorkProposalDispositionV1::Rejected => { - Some(tracedecay_contracts::ReviewProposalDispositionV1::Rejected) - } - tracedecay_contracts::ReviewWorkProposalDispositionV1::Superseded => { - Some(tracedecay_contracts::ReviewProposalDispositionV1::Superseded) - } - }; let _ = tracedecay_application::observability::record_reliance_decision( observability_producer.as_deref(), &proposal_ref, &command_ref, - disposition, + Some(disposition), occurred_at, ); } diff --git a/crates/tracedecay-daemon-service/src/invocation/work/workflow_dispatch.rs b/crates/tracedecay-daemon-service/src/invocation/work/workflow_dispatch.rs index 43d93df99b..d8f08a0c04 100644 --- a/crates/tracedecay-daemon-service/src/invocation/work/workflow_dispatch.rs +++ b/crates/tracedecay-daemon-service/src/invocation/work/workflow_dispatch.rs @@ -55,7 +55,7 @@ pub(crate) async fn execute_workflow_application( else { return DaemonInvocationResponse::problem(request_id, DaemonInvocationProblem::Unavailable); }; - let observed_at = tracedecay_daemon_protocol::invocation_now_micros(); + let observed_at = tracedecay_contracts::now_micros(); let operation_key = request.operation_key(); let Some((_, capability, use_case)) = tracedecay_contracts::WORKFLOW_APPLICATION_OPERATION_IDS .iter() diff --git a/crates/tracedecay-daemon-service/src/invocation/work/workflow_effect_journal.rs b/crates/tracedecay-daemon-service/src/invocation/work/workflow_effect_journal.rs index 40287d47f0..cdd9964af1 100644 --- a/crates/tracedecay-daemon-service/src/invocation/work/workflow_effect_journal.rs +++ b/crates/tracedecay-daemon-service/src/invocation/work/workflow_effect_journal.rs @@ -21,11 +21,11 @@ use tracedecay_daemon_protocol::{ }; use tracedecay_domain::errors::TraceDecayError; -use super::super::current_micros; use super::workflow_run_control::{ workflow_coordination_application_problem, workflow_coordination_problem, }; use super::{RegisteredWorkRuntime, work_command_effect, work_effect, work_evidence_packet}; +use tracedecay_contracts::now_micros; #[allow(clippy::too_many_arguments)] pub(super) fn complete_workflow_run_effect( @@ -207,7 +207,7 @@ pub(super) fn execute_journaled_workflow_effect( ); } }; - let prepared = if identity.deadline().is_elapsed_at(current_micros()) { + let prepared = if identity.deadline().is_elapsed_at(now_micros()) { WorkflowEffectPreparedV1::problem( identity.input_digest().clone(), WorkflowEffectProblemV1::TimedOut, @@ -215,7 +215,7 @@ pub(super) fn execute_journaled_workflow_effect( } else { prepared }; - let record = match authority.execute_effect(&identity, &prepared, current_micros()) { + let record = match authority.execute_effect(&identity, &prepared, now_micros()) { Ok(record) => record, Err(_) => { return DaemonInvocationResponse::problem( diff --git a/crates/tracedecay-daemon-service/src/invocation/work/workflow_fan_out.rs b/crates/tracedecay-daemon-service/src/invocation/work/workflow_fan_out.rs index 7bf1885de1..e4a7176e1e 100644 --- a/crates/tracedecay-daemon-service/src/invocation/work/workflow_fan_out.rs +++ b/crates/tracedecay-daemon-service/src/invocation/work/workflow_fan_out.rs @@ -7,9 +7,9 @@ use tracedecay_tool_catalog::{CapabilityId, UseCaseId}; use tracedecay_daemon_protocol::DaemonInvocationProblem; -use super::super::current_micros; use super::workflow_run_control::workflow_run_problem; use super::{RegisteredWorkRuntime, work_background_context}; +use tracedecay_contracts::now_micros; mod recovery; @@ -587,10 +587,12 @@ pub(crate) fn admit_workflow_child( occurred_at: UtcMicros, ) -> Result<(), DaemonInvocationProblem> { let selection = tracedecay_contracts::WorkProductSelectionScopeV1::relations( - [tracedecay_contracts::WorkRelationScopeV1::Repository { - project_id: context.scope().project_id.clone(), - repository_id: context.scope().repository_id.clone(), - }] + [ + tracedecay_contracts::WorkProductAuthorizedRelationScopeV1::Repository { + project_id: context.scope().project_id.clone(), + repository_id: context.scope().repository_id.clone(), + }, + ] .into_iter() .collect(), ) @@ -851,7 +853,7 @@ pub(crate) fn reconcile_workflow_fan_out_after_attempt( &services, &context, projection, - current_micros(), + now_micros(), attempt_processes, project_root, observability_producer, diff --git a/crates/tracedecay-daemon-service/src/invocation/work/workflow_fan_out/recovery.rs b/crates/tracedecay-daemon-service/src/invocation/work/workflow_fan_out/recovery.rs index 0d44ff9775..ab4f2e5512 100644 --- a/crates/tracedecay-daemon-service/src/invocation/work/workflow_fan_out/recovery.rs +++ b/crates/tracedecay-daemon-service/src/invocation/work/workflow_fan_out/recovery.rs @@ -10,10 +10,10 @@ use tracedecay_domain::UtcMicros; use tracedecay_daemon_protocol::DaemonInvocationProblem; -use super::super::super::current_micros; use super::super::workflow_run_control::workflow_run_storage_problem; use super::super::{RegisteredWorkRuntime, work_background_context}; use super::reconcile_workflow_fan_out; +use tracedecay_contracts::now_micros; const RECOVERY_RETRY_DELAY: std::time::Duration = std::time::Duration::from_secs(5); @@ -80,7 +80,7 @@ fn reconcile_active_workflow_fan_out_page( &services, &context, projection, - current_micros(), + now_micros(), Arc::clone(&attempt_processes), project_root, observability_producer.clone(), @@ -113,7 +113,7 @@ fn reconcile_active_workflow_fan_out_page( &services, &context, &projection, - current_micros(), + now_micros(), observability_producer.clone(), ); } @@ -141,7 +141,7 @@ fn resume_work_attempts_for_workflow_recovery( .resume( context, &tracedecay_contracts::ResumeWorkAttemptsCommand { - occurred_at: current_micros(), + occurred_at: now_micros(), }, ) .map_err(|error| { @@ -179,9 +179,7 @@ fn workflow_fan_out_recovery_context( registered.grant.clone(), RequestId::new("workflow-fan-out-startup-recovery")?, Deadline::new(UtcMicros( - current_micros() - .0 - .saturating_add(BACKGROUND_DEADLINE_MICROS), + now_micros().0.saturating_add(BACKGROUND_DEADLINE_MICROS), ))?, CancellationContext::active("cancel.workflow-fan-out-startup-recovery")?, ) diff --git a/crates/tracedecay-daemon-service/src/invocation/work_attempt_exec.rs b/crates/tracedecay-daemon-service/src/invocation/work_attempt_exec.rs index b4071b357b..f8aa24daae 100644 --- a/crates/tracedecay-daemon-service/src/invocation/work_attempt_exec.rs +++ b/crates/tracedecay-daemon-service/src/invocation/work_attempt_exec.rs @@ -72,7 +72,7 @@ use tracedecay_domain::{ WorkAttemptIdentityV1, WorkAttemptV1, WorkExecutableReference, WorkFallbackTopology, WorkProviderBackendV1, WorkProviderProtocol, WorkProviderRouteV1, WorktreeId, }; -use tracedecay_sessions::runtime::codex_app_server::{ +use tracedecay_sessions::runtime::hosts::codex_app_server::{ CodexAppServerCancellation, CodexAppServerLaunchReceipt, CodexAppServerSummaryConfig, CodexAppServerWorkExecution, run_work_with_codex_app_server, }; @@ -83,7 +83,8 @@ use tracedecay_configuration::config::work_executable_binding::{ use super::types::RegisteredWorkRuntime; use super::work::work_background_context; -use super::{Arc, RequestContext, current_micros}; +use super::{Arc, RequestContext}; +use tracedecay_contracts::now_micros; mod operation_resource; mod provider_output; @@ -427,7 +428,8 @@ impl WorkAttemptProcessRegistryV1 { process.cancellation.notify_waiters(); process.lifecycle.cancel(); } - let deadline = tokio::time::Instant::now() + crate::TASK_ABORT_DEADLINE; + let deadline = + tokio::time::Instant::now() + tracedecay_runtime_core::DAEMON_TASK_ABORT_DEADLINE; let mut clean = true; for process in processes.into_values() { let Some(mut handle) = process.handle else { @@ -789,7 +791,7 @@ fn settle_unstarted( ) where S: tracedecay_contracts::WorkAttemptStoragePort, { - let observed_at = current_micros(); + let observed_at = now_micros(); if let Err(problem) = attempts.mark_provider_unavailable(context, identity, observed_at) { tracing::warn!( task = identity.task_id().as_str(), @@ -841,7 +843,7 @@ where context, attempt.identity().clone(), attempt.execution().effect_state(), - current_micros(), + now_micros(), attempt.execution().deadline(), ) { Ok(WorkAttemptEffectDispatchOutcomeV1::Recorded(_)) => EffectDispatchAdmission::Recorded, @@ -879,8 +881,7 @@ where } else { WorkAttemptEffectResolutionV1::Unknown }; - if let Err(problem) = effects.settle(context, attempt.identity(), resolution, current_micros()) - { + if let Err(problem) = effects.settle(context, attempt.identity(), resolution, now_micros()) { tracing::warn!( task = attempt.identity().task_id().as_str(), ?problem, @@ -1025,7 +1026,7 @@ async fn execute_provider_with_environment( tokio::spawn(async move { read_capped(stderr, budget.max_stderr_bytes()).await }); let deadline_micros = - u64::try_from(envelope.deadline().0.saturating_sub(current_micros().0)).unwrap_or(0); + u64::try_from(envelope.deadline().0.saturating_sub(now_micros().0)).unwrap_or(0); let wall = std::time::Duration::from_micros(deadline_micros); let outcome = tokio::select! { @@ -1073,7 +1074,7 @@ async fn execute_provider_with_environment( stderr, provider_session, provider_fallback: selection.fallback.clone(), - observed_at: current_micros(), + observed_at: now_micros(), }; if !settle_effect_dispatch(attempt_effects, context, attempt, true) { return; @@ -1118,7 +1119,7 @@ struct AppServerSessionOutput { /// The transport itself is not reimplemented here: process spawn, the /// `initialize` handshake, ephemeral thread lifecycle, turn collection and /// process-tree cancellation all live in -/// `tracedecay_sessions::runtime::codex_app_server`, which the row-56 rework +/// `tracedecay_sessions::runtime::hosts::codex_app_server`, which the row-56 rework /// already built for exactly this call /// ([`run_work_with_codex_app_server`] takes the cwd, wall budget and /// cancellation handle a Work attempt needs and had no other caller). @@ -1182,7 +1183,7 @@ async fn execute_app_server( }; let attempt_started = std::time::Instant::now(); let deadline_micros = - u64::try_from(envelope.deadline().0.saturating_sub(current_micros().0)).unwrap_or(0); + u64::try_from(envelope.deadline().0.saturating_sub(now_micros().0)).unwrap_or(0); let wall = std::time::Duration::from_micros(deadline_micros); let cancellation = CodexAppServerCancellation::default(); let config = CodexAppServerSummaryConfig { @@ -1268,7 +1269,7 @@ async fn execute_app_server( // whole process tree, so the ladder acknowledges and stops there // rather than pretending an interrupt was survived. if let Err(problem) = - attempts.acknowledge_cancellation(context, &identity, current_micros()) + attempts.acknowledge_cancellation(context, &identity, now_micros()) { tracing::warn!( task = identity.task_id().as_str(), @@ -1337,7 +1338,7 @@ async fn execute_app_server( stderr: None, provider_session, provider_fallback: selection.fallback.clone(), - observed_at: current_micros(), + observed_at: now_micros(), }; if !settle_effect_dispatch( attempt_effects, @@ -1401,7 +1402,7 @@ fn offer_no_progress_observation( concurrency_policy_revision: topology_policy_digest.0.as_str(), configured_timeout_micros, elapsed_stall_micros, - observed_at: current_micros(), + observed_at: now_micros(), }, ); if result != WorkOwnerObservationResultV1::Enqueued { @@ -1425,7 +1426,7 @@ async fn cancel_ladder( where S: tracedecay_contracts::WorkAttemptStoragePort, { - if let Err(problem) = attempts.acknowledge_cancellation(context, identity, current_micros()) { + if let Err(problem) = attempts.acknowledge_cancellation(context, identity, now_micros()) { tracing::warn!( task = identity.task_id().as_str(), ?problem, @@ -1437,7 +1438,7 @@ where .await .is_err() { - if let Err(problem) = attempts.escalate_cancellation(context, identity, current_micros()) { + if let Err(problem) = attempts.escalate_cancellation(context, identity, now_micros()) { tracing::warn!( task = identity.task_id().as_str(), ?problem, diff --git a/crates/tracedecay-daemon-service/src/invocation/work_attempt_exec/tests.rs b/crates/tracedecay-daemon-service/src/invocation/work_attempt_exec/tests.rs index 8d0e48a8f0..f3599467a1 100644 --- a/crates/tracedecay-daemon-service/src/invocation/work_attempt_exec/tests.rs +++ b/crates/tracedecay-daemon-service/src/invocation/work_attempt_exec/tests.rs @@ -344,7 +344,7 @@ fn request_context() -> RequestContext { id::("actor.issuer"), UtcMicros(1), // Admission validates the grant window and the context deadline - // against real `current_micros()` timestamps (the cancellation path + // against real `now_micros()` timestamps (the cancellation path // observes wall-clock time), so both must sit in the real future. deadline_in(3_600), scope.clone(), @@ -393,7 +393,7 @@ fn pinned_protocol(backend: WorkProviderBackendV1) -> WorkProviderProtocol { /// Deadline far enough ahead that the wall-clock arm of the execution select /// never fires; every fixture below is expected to finish on its own terms. fn deadline_in(seconds: i64) -> UtcMicros { - UtcMicros(current_micros().0.saturating_add(seconds * 1_000_000)) + UtcMicros(now_micros().0.saturating_add(seconds * 1_000_000)) } impl WorkAttemptEffectStoragePortV1 for AttemptStore { @@ -1232,7 +1232,7 @@ async fn a_provider_that_ignores_interrupt_is_escalated_to_a_kill_on_the_record( run_id: identity.run_id().clone(), attempt_id: identity.attempt_id().clone(), request_id: id("cancellation.work-attempt-exec.1"), - occurred_at: current_micros(), + occurred_at: now_micros(), }, ) .unwrap(); @@ -1378,7 +1378,7 @@ async fn a_wall_exhausted_provider_seals_timed_out_and_emits_the_no_progress_ter event_kinds: vec!["operation.no_progress.terminal.v1".to_owned()], horizon: ObservabilityHorizonV1 { since_micros: 0, - until_micros: current_micros().0.saturating_add(1_000_000), + until_micros: now_micros().0.saturating_add(1_000_000), }, after_watermark: None, limit: 8, diff --git a/crates/tracedecay-daemon-service/src/lib.rs b/crates/tracedecay-daemon-service/src/lib.rs index 4d22027add..058fc11f73 100644 --- a/crates/tracedecay-daemon-service/src/lib.rs +++ b/crates/tracedecay-daemon-service/src/lib.rs @@ -50,12 +50,6 @@ #![allow(clippy::single_match_else)] #![allow(clippy::large_futures)] -/// Abort bound for in-flight invocation tasks during shutdown. -/// -/// Re-exported here as the daemon-service authority while the lower runtime -/// crate remains the cycle-free owner shared with code-index runtime. -pub use tracedecay_runtime_core::DAEMON_TASK_ABORT_DEADLINE as TASK_ABORT_DEADLINE; - pub mod adoption_observation; pub mod application_surface; pub mod automation_effect; @@ -87,7 +81,7 @@ pub use callable_code_authorization::{ pub use invocation::{ AuthorizedDaemonLspWorkspace, DaemonConfigurationRuntimeRegistrationPauseV1, DaemonFeedbackPublicationTestGate, InvocationProjectRuntimeIdentityV1, LspLeaseTaskRegistry, - RuntimeLspSession, WorkAttemptProcessRegistryV1, canonicalize_lsp_roots, current_micros, + RuntimeLspSession, WorkAttemptProcessRegistryV1, canonicalize_lsp_roots, execute_work_application, lsp_delivery_attempt, mounted_configuration_layers, now_millis, retain_lsp_delivery_attempt, }; @@ -100,15 +94,17 @@ pub use invocation::{ DaemonContextScoutRuntimeRegistrar, DaemonContextScoutRuntimeRegistrationError, DaemonFeedbackInvocationOwner, DaemonFeedbackProximityInvocationFuture, DaemonFeedbackProximityInvocationRequest, DaemonFeedbackRuntimeRegistrar, - DaemonFeedbackRuntimeRegistrationError, DaemonInvocationService, DaemonLspInvocationOwner, - DaemonLspOwnerRegistrar, DaemonPrimitiveRuntimeRegistrar, - DaemonPrimitiveRuntimeRegistrationError, DaemonRetainedRuntimeRegistrar, - DaemonSourceEditOwnerRegistrationError, DaemonWorkProposalRoutingAuthorityV1, - DaemonWorkRuntimeRegistrar, FeedbackCycleRuntimeBuilderV1, HookOrchestrationAdmissionV1, - HookOrchestrationRequestV1, HookOrchestrationTriggerV1, HookOrchestrationWorkOutcomeV1, - LSP_WORKSPACE_CAPABILITY_ID_V1, LSP_WORKSPACE_USE_CASE_ID_V1, LspDeliverySettlementAdmissionV1, - MAX_COALESCED_HOOK_COMPLETIONS, RegisteredCallableCodeRuntime, RegisteredConfigurationRuntime, - RegisteredFeedbackRuntime, RegisteredRetainedRequestContextError, RegisteredRetainedRuntime, + DaemonFeedbackRuntimeRegistrationError, DaemonGraphToolOwnerRegistrationError, + DaemonInvocationService, DaemonLspInvocationOwner, DaemonLspOwnerRegistrar, + DaemonPrimitiveRuntimeRegistrar, DaemonPrimitiveRuntimeRegistrationError, + DaemonRetainedRuntimeRegistrar, DaemonSourceEditOwnerRegistrationError, + DaemonWorkProposalRoutingAuthorityV1, DaemonWorkRuntimeRegistrar, + FeedbackCycleRuntimeBuilderV1, GraphToolFuture, GraphToolInvocationV1, + HookOrchestrationAdmissionV1, HookOrchestrationRequestV1, HookOrchestrationTriggerV1, + HookOrchestrationWorkOutcomeV1, LSP_WORKSPACE_CAPABILITY_ID_V1, LSP_WORKSPACE_USE_CASE_ID_V1, + LspDeliverySettlementAdmissionV1, MAX_COALESCED_HOOK_COMPLETIONS, ProjectGraphToolPortV1, + RegisteredCallableCodeRuntime, RegisteredConfigurationRuntime, RegisteredFeedbackRuntime, + RegisteredGraphToolOwnerV1, RegisteredRetainedRequestContextError, RegisteredRetainedRuntime, RegisteredWorkRuntime, SwitchableFeedbackCycleRuntimeV1, UnavailableFeedbackCycleRuntimeV1, admit_registered_hook_orchestration, advisory_cycle_invocation_result, callable_code_request_context, daemon_operation_event_authority, diff --git a/crates/tracedecay-daemon-service/src/mcp_workflow_index.rs b/crates/tracedecay-daemon-service/src/mcp_workflow_index.rs index 05ad4495c9..49ba5f13b6 100644 --- a/crates/tracedecay-daemon-service/src/mcp_workflow_index.rs +++ b/crates/tracedecay-daemon-service/src/mcp_workflow_index.rs @@ -14,6 +14,7 @@ use tracedecay_sessions::{ use tracedecay_global_db::GlobalDbWorkflowStore; use tracedecay_global_db::RegisteredGlobalDbLeaseV1; +use tracedecay_session_temporal_store::SessionTemporalAccess; use tracedecay_sessions::runtime::git_correlation::{GitCorrelationError, GitScopeFilter}; use tracedecay_sessions::runtime::workflow_index::{ MAX_WORKFLOW_LIMIT, RegisteredWorkflowIndexSnapshot, WorkflowIndexError, @@ -90,8 +91,7 @@ impl DaemonWorkflowIndexReadService { worktree: filter.worktree, commit: filter.commit, }; - let session_ids = match self - .database + let session_ids = match SessionTemporalAccess::new(&*self.database) .git_scope_session_ids_bounded(&filter, MAX_WORKFLOW_LIMIT + 1) { Ok(session_ids) => session_ids, diff --git a/crates/tracedecay-daemon-service/src/project_runtime.rs b/crates/tracedecay-daemon-service/src/project_runtime.rs index 761371efee..171e77a36b 100644 --- a/crates/tracedecay-daemon-service/src/project_runtime.rs +++ b/crates/tracedecay-daemon-service/src/project_runtime.rs @@ -185,6 +185,7 @@ pub struct ProjectRuntime { retained: Option, lsp_owner: Option, source_edit: Option>, + graph_tool: Option, #[cfg(any(test, feature = "test-helpers"))] test_marker: Option>, observability: Option, @@ -234,6 +235,7 @@ impl ProjectRuntime { || self.retained.is_some() || self.lsp_owner.is_some() || self.source_edit.is_some() + || self.graph_tool.is_some() || self.observability.is_some() || { #[cfg(any(test, feature = "test-helpers"))] @@ -313,6 +315,7 @@ project_runtime_components!( RegisteredRetainedRuntime => retained, DaemonLspInvocationOwner => lsp_owner, Arc => source_edit, + crate::invocation::RegisteredGraphToolOwnerV1 => graph_tool, RegisteredObservabilityProducerV1 => observability, ); diff --git a/crates/tracedecay-daemon-service/src/project_runtime/request_snapshot.rs b/crates/tracedecay-daemon-service/src/project_runtime/request_snapshot.rs index 8364e049da..7f19af3dc9 100644 --- a/crates/tracedecay-daemon-service/src/project_runtime/request_snapshot.rs +++ b/crates/tracedecay-daemon-service/src/project_runtime/request_snapshot.rs @@ -36,6 +36,7 @@ pub struct ProjectRequestRuntimesV1 { pub retained: Option, pub lsp_owner: Option, pub source_edit: Option>, + pub graph_tool: Option, } /// The owners of the registered runtime as they stood under the admission @@ -50,6 +51,7 @@ pub(super) struct AdmittedProjectRuntimeV1 { retained: Option, lsp_owner: Option, source_edit: Option>, + graph_tool: Option, } impl AdmittedProjectRuntimeV1 { @@ -65,6 +67,7 @@ impl AdmittedProjectRuntimeV1 { retained: runtime.retained.clone(), lsp_owner: runtime.lsp_owner.clone(), source_edit: runtime.source_edit.clone(), + graph_tool: runtime.graph_tool.clone(), } } } @@ -225,6 +228,7 @@ impl ProjectRequestRuntimesV1 { retained: admitted.retained.clone(), lsp_owner: admitted.lsp_owner.clone(), source_edit: admitted.source_edit.clone(), + graph_tool: admitted.graph_tool.clone(), _request_lease: Some(request_lease), } } diff --git a/crates/tracedecay-daemon-service/src/project_runtime/shutdown.rs b/crates/tracedecay-daemon-service/src/project_runtime/shutdown.rs index bf79ad8d46..cb628c21e9 100644 --- a/crates/tracedecay-daemon-service/src/project_runtime/shutdown.rs +++ b/crates/tracedecay-daemon-service/src/project_runtime/shutdown.rs @@ -46,36 +46,37 @@ impl ProjectRuntimeRegistryV1 { #[hotpath::skip] async fn drain_roots(&self, roots: &BTreeSet) -> bool { - let retired = tokio::time::timeout(crate::TASK_ABORT_DEADLINE, async { - loop { - let mut changed = self.reservation_changed.subscribe(); - let retired = { - let fences = self.lock_root_fences(); - let mut current = self.lock_runtimes(); - (fences.requests_drained(roots) - && roots.iter().all(|root| { - current - .get(root) - .is_none_or(|runtime| runtime.reservations.is_empty()) - })) - .then(|| { - roots - .iter() - .filter_map(|root| { - current.remove(root).map(|runtime| (root.clone(), runtime)) - }) - .collect::>() - }) - }; - if let Some(retired) = retired { - break retired; - } - if changed.changed().await.is_err() { - break BTreeMap::new(); + let retired = + tokio::time::timeout(tracedecay_runtime_core::DAEMON_TASK_ABORT_DEADLINE, async { + loop { + let mut changed = self.reservation_changed.subscribe(); + let retired = { + let fences = self.lock_root_fences(); + let mut current = self.lock_runtimes(); + (fences.requests_drained(roots) + && roots.iter().all(|root| { + current + .get(root) + .is_none_or(|runtime| runtime.reservations.is_empty()) + })) + .then(|| { + roots + .iter() + .filter_map(|root| { + current.remove(root).map(|runtime| (root.clone(), runtime)) + }) + .collect::>() + }) + }; + if let Some(retired) = retired { + break retired; + } + if changed.changed().await.is_err() { + break BTreeMap::new(); + } } - } - }) - .await; + }) + .await; match retired { Ok(mut runtimes) => { let mut clean = shut_down_advisory(&runtimes).await; diff --git a/crates/tracedecay-daemon-service/src/remote_protocol/observability.rs b/crates/tracedecay-daemon-service/src/remote_protocol/observability.rs index 6f18cbbcf2..716abc2eac 100644 --- a/crates/tracedecay-daemon-service/src/remote_protocol/observability.rs +++ b/crates/tracedecay-daemon-service/src/remote_protocol/observability.rs @@ -211,14 +211,14 @@ fn remote_query_response_observation( "remote_query_result_payload_unavailable", ), }, - ApplicationOutcome::Preview(_) | ApplicationOutcome::Effect(_) => { - unavailable_observation( - operation_ref, - expected_shards, - ObservedTernaryV1::Unknown, - "remote_query_result_kind_unavailable", - ) - } + ApplicationOutcome::Preview(_) + | ApplicationOutcome::Effect(_) + | ApplicationOutcome::Result(_) => unavailable_observation( + operation_ref, + expected_shards, + ObservedTernaryV1::Unknown, + "remote_query_result_kind_unavailable", + ), }, Err(problem) => unavailable_observation( operation_ref, diff --git a/crates/tracedecay-daemon-service/src/retained_owner/profile_refresh_journeys.rs b/crates/tracedecay-daemon-service/src/retained_owner/profile_refresh_journeys.rs index 5012399e74..31373353e0 100644 --- a/crates/tracedecay-daemon-service/src/retained_owner/profile_refresh_journeys.rs +++ b/crates/tracedecay-daemon-service/src/retained_owner/profile_refresh_journeys.rs @@ -4,14 +4,14 @@ use tracedecay_contracts::retained_surfaces::{ RetainedOutcomeStatusV1, RetainedSurfaceRequestV1, RetainedSurfaceResultV1, SessionRefreshActionRequestV1, SessionRefreshActionV1, SessionRefreshFrontierV1, SessionRefreshGrainV1, SessionRefreshRequestV1, SessionRefreshScopeV1, SessionRefreshSessionV1, - SessionRefreshSourceV1, SessionRefreshTargetV1, SessionRefreshTemporalModeV1, + SessionRefreshSourceV1, SessionRefreshTargetV1, }; use tracedecay_contracts::{ ApplicationOutcome, ApplicationProblemKind, ApplicationResult, CancellationSignal, Deadline, RequestId, now_micros, }; use tracedecay_daemon_identity::profile_identity; -use tracedecay_domain::UtcMicros; +use tracedecay_domain::{TemporalModeV1, UtcMicros}; use tracedecay_session_memory::context::ResolvedSessionIdentity; use tracedecay_session_runtime::retained::{ ProfileRetainedAuthoritiesV1, ProfileRetainedConnectionAuthorityV1, @@ -53,7 +53,7 @@ fn refresh_request( scope: "codex".to_owned(), }, target: SessionRefreshTargetV1 { - temporal_mode: SessionRefreshTemporalModeV1::Current, + temporal_mode: TemporalModeV1::Current, grain: SessionRefreshGrainV1::LogicalMessage, frontier: SessionRefreshFrontierV1 { observed_through: 0, @@ -61,7 +61,6 @@ fn refresh_request( }, }, handle, - format: None, }, )) } diff --git a/crates/tracedecay-daemon-service/src/retained_owner/retained_curator.rs b/crates/tracedecay-daemon-service/src/retained_owner/retained_curator.rs index f543659358..497fa4060c 100644 --- a/crates/tracedecay-daemon-service/src/retained_owner/retained_curator.rs +++ b/crates/tracedecay-daemon-service/src/retained_owner/retained_curator.rs @@ -51,7 +51,10 @@ pub async fn execute_retained_memory_curator( })?; let min_confidence = f64::from(request.min_confidence_millionths) / 1_000_000.0; config.timeout_secs = config.timeout_secs.min(MEMORY_CURATOR_REQUEST_TIMEOUT_SECS); - let backend = CodexAppServerBackend::from_automation_config(&config); + let backend = CodexAppServerBackend::from_automation_config( + &config, + &pinned.config().lcm_summarizers.codex, + ); let configuration_digest = tracedecay_automation_runtime::automation::effect_runtime::pinned_automation_configuration_digest( pinned.revision_id(), diff --git a/crates/tracedecay-daemon-service/src/retained_owner/session_retained_effect_tests.rs b/crates/tracedecay-daemon-service/src/retained_owner/session_retained_effect_tests.rs index aacd51d3ec..a371ec231b 100644 --- a/crates/tracedecay-daemon-service/src/retained_owner/session_retained_effect_tests.rs +++ b/crates/tracedecay-daemon-service/src/retained_owner/session_retained_effect_tests.rs @@ -7,7 +7,7 @@ use tracedecay_contracts::retained_surfaces::{ RetainedSurfaceOperation, RetainedSurfaceRequestV1, SessionRefreshActionRequestV1, SessionRefreshActionV1, SessionRefreshFrontierV1, SessionRefreshGrainV1, SessionRefreshRequestV1, SessionRefreshScopeV1, SessionRefreshSessionV1, - SessionRefreshSourceV1, SessionRefreshTargetV1, SessionRefreshTemporalModeV1, + SessionRefreshSourceV1, SessionRefreshTargetV1, }; use tracedecay_contracts::{ ApplicationProblem, ApplicationProblemKind, CancellationContext, CancellationSignal, @@ -18,7 +18,8 @@ use tracedecay_contracts::{ }; use tracedecay_domain::{ ActorId, ManifestDigest, ProjectId, RefId, RepositoryId, SessionId, - SessionRefreshOperationIdV1, UserProfileId, UtcMicros, WorktreeId, canonical_sha256, + SessionRefreshOperationIdV1, TemporalModeV1, UserProfileId, UtcMicros, WorktreeId, + canonical_sha256, }; use tracedecay_session_memory::context::{BranchId, ProfileId, SessionRootId, SessionStoreId}; use tracedecay_sessions::admission::HostAdmissionScope; @@ -176,7 +177,7 @@ impl RetiredRefreshFixture { scope: "cursor".to_owned(), }, target: SessionRefreshTargetV1 { - temporal_mode: SessionRefreshTemporalModeV1::Current, + temporal_mode: TemporalModeV1::Current, grain: SessionRefreshGrainV1::LogicalMessage, frontier: SessionRefreshFrontierV1 { observed_through: 0, @@ -184,7 +185,6 @@ impl RetiredRefreshFixture { }, }, handle, - format: None, }, ) } diff --git a/crates/tracedecay-daemon-service/src/shutdown/lifecycle.rs b/crates/tracedecay-daemon-service/src/shutdown/lifecycle.rs index 637f679b7e..301f7665b5 100644 --- a/crates/tracedecay-daemon-service/src/shutdown/lifecycle.rs +++ b/crates/tracedecay-daemon-service/src/shutdown/lifecycle.rs @@ -5,12 +5,11 @@ use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use tokio::time::Duration; +pub use tracedecay_runtime_core::DAEMON_TASK_ABORT_DEADLINE; pub use tracedecay_session_runtime::DAEMON_CLIENT_DRAIN_DEADLINE; use super::orchestration::{DaemonShutdownFailures, DaemonShutdownReceipt}; -pub const DAEMON_TASK_ABORT_DEADLINE: Duration = crate::TASK_ABORT_DEADLINE; - /// Per-phase shutdown budgets. /// /// A single global deadline shared by every phase lets one stuck phase spend diff --git a/crates/tracedecay-daemon-service/src/shutdown/watchdog.rs b/crates/tracedecay-daemon-service/src/shutdown/watchdog.rs index 723e28ccdd..e71866af54 100644 --- a/crates/tracedecay-daemon-service/src/shutdown/watchdog.rs +++ b/crates/tracedecay-daemon-service/src/shutdown/watchdog.rs @@ -313,7 +313,7 @@ mod tests { let report = temp.path().join("watchdog-hotpath.json"); let output = Command::new(std::env::current_exe().expect("current test binary")) .arg("--exact") - .arg("daemon::shutdown_watchdog::tests::watchdog_exit_child") + .arg("shutdown::watchdog::tests::watchdog_exit_child") .arg("--nocapture") .env("TRACEDECAY_WATCHDOG_EXIT_CHILD", "1") .env("HOTPATH_METRICS_SERVER_OFF", "true") diff --git a/crates/tracedecay-dashboard-api/Cargo.toml b/crates/tracedecay-dashboard-api/Cargo.toml index 042a8f1e64..76a7d35a52 100644 --- a/crates/tracedecay-dashboard-api/Cargo.toml +++ b/crates/tracedecay-dashboard-api/Cargo.toml @@ -12,7 +12,7 @@ axum = "0.8" clru = "0.6" futures-util = "0.3.33" hotpath.workspace = true -schemars = "1.2.1" +schemars.workspace = true semver = "1" serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/crates/tracedecay-dashboard-api/src/analytics_api.rs b/crates/tracedecay-dashboard-api/src/analytics_api.rs index 31f819e5f1..a09589ba93 100644 --- a/crates/tracedecay-dashboard-api/src/analytics_api.rs +++ b/crates/tracedecay-dashboard-api/src/analytics_api.rs @@ -1,8 +1,8 @@ //! Read-only durable analytics API for dashboard-level agent behavior. //! //! Durable `analytics_events` rows are preferred when available. Older session -//! stores still get session-message usage rollups, and hint lifecycle telemetry -//! falls back to the legacy `dashboard_hint_events` table when present. +//! stores still get session-message usage rollups; hint lifecycle telemetry +//! comes only from durable analytics events. use std::collections::BTreeMap; use std::path::Path; @@ -13,7 +13,7 @@ use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use tracedecay_contracts::ObservatoryReadModelV1; -use tracedecay_domain::CoverageStateV1; +use tracedecay_domain::{CoverageStateV1, ObservationScopeV1}; use tracedecay_automation::analytics::{ ToolUsageObservation, UsageKind, categorize_skill, infer_usage_events, @@ -25,10 +25,14 @@ use tracedecay_global_db::{ AnalyticsEventQuery, AnalyticsEventRecord, AnalyticsHintCounts, RegisteredGlobalDb, }; use tracedecay_runtime_core::db::engine::params; +use tracedecay_session_memory::provider_usage::{ + ProviderUsageAggregateV1, ProviderUsageCoverageV1, ProviderUsageSessionTotalsV1, + provider_usage_aggregate, provider_usage_by_session, +}; use super::DashboardState; use super::read_model::{DashboardCoverageV1, DashboardEnvelopeV1, scope_from_state}; -use super::util::{i64_field, query_i64, query_i64_result, query_rows, str_field}; +use super::util::{i64_field, query_i64_result, query_rows, str_field}; pub use tracedecay_application::analytics_bridge::{ AnalyticsDiagnosticsPayloadV1, AnalyticsDiagnosticsRatiosV1, AnalyticsEventKindCountV1, @@ -154,6 +158,12 @@ pub struct AnalyticsSubagentNodeV1 { /// Sessions below this one, transitively, excluding itself. pub descendants: i64, pub link: AnalyticsSubagentLinkV1, + /// Provider-reported billing usage the canonical provider-usage projection + /// attributes to this session. Absent means the projection holds no usage + /// event for it (or the whole read was unavailable, see the payload's + /// `usage_coverage`); nothing is estimated from message text. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub usage: Option, } /// The subagent tree: parent/child session edges, not a per-agent rollup. @@ -168,6 +178,11 @@ pub struct AnalyticsSubagentTreePayloadV1 { #[serde(default)] pub error: Option, pub nodes: Vec, + /// Coverage of the provider-usage read that populated each node's `usage`. + /// `unavailable` means no node may be read as "used no tokens"; absent + /// means the tree itself was not read. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub usage_coverage: Option, /// Sessions read for this project before any tree was built. The only /// honest denominator for the counts below. pub sessions_read: i64, @@ -645,15 +660,31 @@ fn build_subagent_tree(rows: Vec) -> Vec ProviderUsageCoverageV1 { + let mut by_session = provider_usage_by_session(aggregate); + for node in nodes { + node.usage = by_session.remove(&(node.provider.clone(), node.session_id.clone())); + } + aggregate.coverage +} + async fn subagent_tree_reading( host_io: &HostIo, db: Option<&RegisteredGlobalDb>, project_root: &Path, + usage_scope: Option<&ObservationScopeV1>, ) -> Result { let Some(db) = db else { return Ok(AnalyticsSubagentTreePayloadV1 { @@ -661,6 +692,7 @@ async fn subagent_tree_reading( source: "session_store_unavailable".to_owned(), error: None, nodes: Vec::new(), + usage_coverage: None, sessions_read: 0, root_count: 0, edge_count: 0, @@ -725,7 +757,16 @@ async fn subagent_tree_reading( }) .collect(); - let nodes = build_subagent_tree(session_rows); + let mut nodes = build_subagent_tree(session_rows); + // Without a resolved project scope there is no usage projection to read; + // that is the typed `unavailable`, not an empty aggregate. + let usage_coverage = match usage_scope { + Some(scope) => { + let aggregate = provider_usage_aggregate(db, scope, None, None).await; + attach_subagent_usage(&mut nodes, &aggregate) + } + None => ProviderUsageCoverageV1::Unavailable, + }; let count_link = |wanted: AnalyticsSubagentLinkV1| { nodes.iter().filter(|node| node.link == wanted).count() as i64 }; @@ -733,6 +774,7 @@ async fn subagent_tree_reading( available: true, source: "sessions".to_owned(), error: None, + usage_coverage: Some(usage_coverage), root_count: count_link(AnalyticsSubagentLinkV1::Root), edge_count: count_link(AnalyticsSubagentLinkV1::Linked), missing_parent_count: count_link(AnalyticsSubagentLinkV1::MissingParent), @@ -756,10 +798,18 @@ pub async fn subagent_tree( ) -> Json>> { hotpath::future!( async move { + let usage_scope = + state + .resolved_scope + .as_ref() + .map(|scope| ObservationScopeV1::Project { + project_id: scope.project_id.clone(), + }); match subagent_tree_reading( &state.host_io, state.lcm_db.as_deref(), &state.project_root, + usage_scope.as_ref(), ) .await { @@ -1157,81 +1207,16 @@ async fn hint_summary( return hint_summary_from_events(events); } - let Some(db) = db else { - return AnalyticsHintsPayloadV1 { - available: false, - source: "session_store_unavailable".to_owned(), - error: None, - by_category: empty_hint_categories(), - }; - }; - - let connection = db.read_connection(); - let has_table = query_i64( - &connection, - "SELECT COUNT(*) FROM sqlite_master - WHERE type IN ('table', 'view') AND name = 'dashboard_hint_events'", - (), - ) - .await - > 0; - if !has_table { - return AnalyticsHintsPayloadV1 { - available: false, - source: "dashboard_hint_events_missing".to_owned(), - error: None, - by_category: empty_hint_categories(), - }; - } - - let rows = match query_rows( - &connection, - "SELECT category, - SUM(CASE WHEN event_type = 'emitted' THEN 1 ELSE 0 END) AS emitted, - SUM(CASE WHEN event_type = 'followed' THEN 1 ELSE 0 END) AS followed, - SUM(CASE WHEN event_type = 'ignored' THEN 1 ELSE 0 END) AS ignored, - SUM(CASE WHEN event_type = 'suppressed' THEN 1 ELSE 0 END) AS suppressed - FROM dashboard_hint_events - GROUP BY category - ORDER BY category", - (), - ) - .await - { - Ok(rows) => rows, - Err(err) => { - return AnalyticsHintsPayloadV1 { - available: false, - source: "dashboard_hint_events_error".to_owned(), - error: Some(err), - by_category: empty_hint_categories(), - }; - } - }; - - let mut by_category: BTreeMap = empty_hint_categories() - .into_iter() - .map(|row| (row.category.clone(), row)) - .collect(); - for row in rows { - let category = str_field(&row, "category"); - by_category.insert( - category.to_owned(), - AnalyticsHintCategoryV1 { - category: category.to_owned(), - emitted: i64_field(&row, "emitted"), - followed: i64_field(&row, "followed"), - ignored: i64_field(&row, "ignored"), - suppressed: i64_field(&row, "suppressed"), - }, - ); - } - AnalyticsHintsPayloadV1 { - available: true, - source: "dashboard_hint_events".to_owned(), + available: false, + source: if db.is_some() { + "analytics_events_missing" + } else { + "session_store_unavailable" + } + .to_owned(), error: None, - by_category: by_category.into_values().collect(), + by_category: empty_hint_categories(), } } @@ -1245,9 +1230,9 @@ async fn session_message_rows( query_rows( &connection, "SELECT COALESCE(tool_names, '') AS tool_names, - COALESCE(text, '') AS text, + index_text AS text, COALESCE(metadata_json, '') AS metadata_json - FROM session_messages + FROM lcm_raw_messages ORDER BY timestamp, ordinal LIMIT 10000", (), @@ -1389,7 +1374,7 @@ fn usage_count_rows(counts: BTreeMap<(String, String), i64>) -> Vec) -> Result ProviderUsageDeltaV1 { + ProviderUsageDeltaV1 { + observation_id: format!("sha256:{sequence:064x}"), + receipt_id: format!("receipt:{sequence}"), + observation_sequence: sequence, + usage_ordinal: 0, + scope: ObservationScopeV1::Profile, + provider: "codex".to_owned(), + model: Some("openai/gpt-5.6-codex".to_owned()), + session_id: session_id.to_owned(), + turn_id: None, + message_id: None, + request_id: None, + native_kind: "token_count".to_owned(), + native_field: "fixture.usage".to_owned(), + native_timestamp: Some(1_700_000_000 + sequence as i64), + derivation: ProviderUsageDeltaDerivationV1::NativeDelta, + derived_from_sequence: None, + counters: AggregatedProviderUsageCountersV1 { + input_tokens: Some(input), + output_tokens: Some(output), + cache_read_tokens: None, + cache_write_tokens: None, + reasoning_tokens: Some(0), + total_tokens: Some(input + output), + }, + } + } + + #[test] + fn node_usage_is_the_projection_sum_per_session_and_absent_where_unobserved() { + let mut nodes = build_subagent_tree(vec![ + row("root", None), + row("child.billed", Some("root")), + row("child.silent", Some("root")), + row("child.broken", Some("root")), + ]); + let aggregate = ProviderUsageAggregateV1 { + coverage: ProviderUsageCoverageV1::Partial, + observations_seen: 4, + totals: AggregatedProviderUsageCountersV1::unknown(), + deltas: vec![ + usage_delta(1, "root", 1_000, 50), + usage_delta(2, "child.billed", 300, 25), + usage_delta(3, "root", 200, 10), + ], + issues: vec![ProviderUsageIssueV1 { + kind: ProviderUsageIssueKindV1::MalformedCounters, + observation_sequence: Some(4), + provider: Some("codex".to_owned()), + session_id: Some("child.broken".to_owned()), + }], + upper_observation_sequence: Some(4), + }; + + let coverage = attach_subagent_usage(&mut nodes, &aggregate); + assert_eq!(coverage, ProviderUsageCoverageV1::Partial); + + let usage = |id: &str| { + nodes + .iter() + .find(|node| node.session_id == id) + .unwrap() + .usage + .clone() + }; + let root = usage("root").expect("root usage"); + assert_eq!(root.usage_events, 2); + assert_eq!(root.counters.input_tokens, Some(1_200)); + assert_eq!(root.counters.output_tokens, Some(60)); + assert_eq!(root.counters.total_tokens, Some(1_260)); + assert_eq!(root.counters.cache_read_tokens, None); + assert!(root.complete); + + let billed = usage("child.billed").expect("billed child usage"); + assert_eq!(billed.usage_events, 1); + assert_eq!(billed.counters.input_tokens, Some(300)); + assert_eq!(billed.counters.output_tokens, Some(25)); + + // No usage event names this session: absent, never a zero. + assert_eq!(usage("child.silent"), None); + + // The provider wrote usage that could not be reduced: present, flagged, + // and still not zero-filled. + let broken = usage("child.broken").expect("broken child usage"); + assert_eq!(broken.usage_events, 0); + assert_eq!( + broken.counters, + AggregatedProviderUsageCountersV1::unknown() + ); + assert!(!broken.complete); + } + #[test] fn a_session_whose_parent_is_absent_is_a_cut_edge_not_a_root() { let nodes = build_subagent_tree(vec![ diff --git a/crates/tracedecay-dashboard-api/src/automation_config_api.rs b/crates/tracedecay-dashboard-api/src/automation_config_api.rs index da31aa2929..7cb43d2a2c 100644 --- a/crates/tracedecay-dashboard-api/src/automation_config_api.rs +++ b/crates/tracedecay-dashboard-api/src/automation_config_api.rs @@ -18,7 +18,7 @@ use tracedecay_contracts::ApplicationOutcome; use tracedecay_domain::ProjectId; use tracedecay_domain::configuration::{ AUTOMATION_SETTINGS_SETTING_KEY, ConfigurationIdempotencyKey, ConfigurationLayerIdV1, - ConfigurationRevisionId, ConfigurationValueV1, SettingKey, + ConfigurationRevisionId, ConfigurationValueV1, LcmSummarizerExecutableV1, SettingKey, }; use super::DashboardState; @@ -46,11 +46,12 @@ pub struct AutomationConfigMutationRequest { #[hotpath::measure(label = "dashboard_api.automation.get_config", future = true)] pub async fn get_config(State(state): State) -> ApiResult { - let (configuration_revision_id, effective) = effective_automation_config(&state) + let (configuration_revision_id, effective, codex) = effective_automation_config(&state) .map_err(|_| configuration_authority_unavailable_error())?; Ok(Json(config_payload( &configuration_revision_id, &effective, + &codex, None, )?)) } @@ -80,7 +81,7 @@ pub async fn patch_config( "message": "idempotency_key must be one non-empty canonical caller-stable value" }])) })?; - let (current_revision, current) = effective_automation_config(&state) + let (current_revision, current, _) = effective_automation_config(&state) .map_err(|_| configuration_authority_unavailable_error())?; if expected_revision != current_revision { return Err(configuration_revision_conflict_error( @@ -136,36 +137,47 @@ pub async fn patch_config( // The runtime refreshes the pinned snapshot as part of a settled // configuration effect. Re-read it instead of projecting the submitted // candidate, so a response never claims a setting that failed activation. - let (configuration_revision_id, effective) = effective_automation_config(&state) + let (configuration_revision_id, effective, codex) = effective_automation_config(&state) .map_err(|_| configuration_authority_unavailable_error())?; Ok(Json(config_payload( &configuration_revision_id, &effective, + &codex, application_outcome.as_ref(), )?)) } -/// Returns the one admitted runtime configuration for an automation caller. -/// The revision is returned with the value so consumers cannot accidentally -/// pair a status result with an unrelated configuration revision. +/// Returns the one admitted runtime configuration for an automation caller: +/// the automation settings and the `codex` executable the same pinned snapshot +/// binds. The revision is returned with the values so consumers cannot +/// accidentally pair a status result with an unrelated configuration revision. pub(crate) fn effective_automation_config( state: &DashboardState, -) -> tracedecay_domain::errors::Result<(ConfigurationRevisionId, AutomationConfig)> { +) -> tracedecay_domain::errors::Result<( + ConfigurationRevisionId, + AutomationConfig, + LcmSummarizerExecutableV1, +)> { let pinned = crate::config::cached_runtime_configuration(&state.project_root)?; let config = from_configuration_snapshot(pinned.snapshot())?; - Ok((pinned.revision_id().clone(), config)) + Ok(( + pinned.revision_id().clone(), + config, + pinned.config().lcm_summarizers.codex.clone(), + )) } fn config_payload( configuration_revision_id: &ConfigurationRevisionId, effective: &AutomationConfig, + codex: &LcmSummarizerExecutableV1, application_outcome: Option<&ApplicationOutcome>, ) -> std::result::Result { let mut payload = json!({ "configuration_revision_id": configuration_revision_id.as_str(), "source": "daemon_pinned_snapshot", "effective": effective, - "backend_availability": backend::backend_availability(effective), + "backend_availability": backend::backend_availability(effective, codex), }); if let Some(application_outcome) = application_outcome { payload["application_outcome"] = serde_json::to_value(application_outcome) diff --git a/crates/tracedecay-dashboard-api/src/automation_fact_receipts_api.rs b/crates/tracedecay-dashboard-api/src/automation_fact_receipts_api.rs index d86fccd5cb..a3411ef858 100644 --- a/crates/tracedecay-dashboard-api/src/automation_fact_receipts_api.rs +++ b/crates/tracedecay-dashboard-api/src/automation_fact_receipts_api.rs @@ -1,7 +1,8 @@ use axum::extract::{Path as AxumPath, State}; use axum::http::StatusCode; -use axum::response::Json; -use serde::Deserialize; +use axum::response::{IntoResponse, Json, Response}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use super::util::{JsonQuery, coerce_limit, json_error}; @@ -22,16 +23,26 @@ pub struct ListParams { limit: Option, } +/// `GET /api/automation/automatic-fact-receipts`, newest first under `limit`. +#[derive(Debug, Serialize, JsonSchema)] +pub(crate) struct AutomaticFactReceiptsPayloadV1 { + receipts: Vec, + count: usize, + limit: usize, +} + #[hotpath::measure(label = "dashboard_api.receipts.list", future = true)] pub async fn list( State(state): State, RequestControl(control): RequestControl, JsonQuery(params): JsonQuery, -) -> (StatusCode, Json) { +) -> Response { let receipt_state = match params.state.as_deref() { Some(value) => match AutomaticFactState::parse(value) { Ok(state) => Some(state), - Err(err) => return json_error(StatusCode::BAD_REQUEST, err.to_string()), + Err(err) => { + return json_error(StatusCode::BAD_REQUEST, err.to_string()).into_response(); + } }, None => None, }; @@ -42,31 +53,26 @@ pub async fn list( ) as usize; let memory = match open_receipt_memory(&state) { Ok(memory) => memory, - Err(error) => return error, + Err(error) => return error.into_response(), }; let result = list_automatic_fact_receipts(&memory, receipt_state, limit, &fact_read_control(&control)) .await; if let Some(state) = request_terminal_state(&control) { - return terminal_read_response(state); + return terminal_read_response(state).into_response(); } match result { - Ok(receipts) => { - let count = receipts.len(); - ( - StatusCode::OK, - Json(json!({ - "receipts": receipts, - "count": count, - "limit": limit, - "error": "", - })), - ) - } + Ok(receipts) => Json(AutomaticFactReceiptsPayloadV1 { + count: receipts.len(), + receipts, + limit, + }) + .into_response(), Err(err) => json_error( StatusCode::INTERNAL_SERVER_ERROR, format!("Failed to load automatic fact receipts: {err}"), - ), + ) + .into_response(), } } diff --git a/crates/tracedecay-dashboard-api/src/automation_jobs_api.rs b/crates/tracedecay-dashboard-api/src/automation_jobs_api.rs index e2696ced13..fde80ea4a4 100644 --- a/crates/tracedecay-dashboard-api/src/automation_jobs_api.rs +++ b/crates/tracedecay-dashboard-api/src/automation_jobs_api.rs @@ -12,7 +12,8 @@ use axum::Extension; use axum::Json; use axum::extract::{Path as AxumPath, State}; use axum::http::StatusCode; -use serde::Deserialize; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use super::DashboardState; @@ -86,12 +87,24 @@ where Option::::deserialize(deserializer).map(Some) } +/// `GET /api/automation/jobs`. +#[derive(Debug, Serialize, JsonSchema)] +pub(crate) struct AutomationJobsPayloadV1 { + jobs: Vec, + count: usize, +} + #[hotpath::measure(label = "dashboard_api.jobs.list", future = true)] -pub async fn list(State(state): State) -> ApiResult { +pub async fn list( + State(state): State, +) -> std::result::Result, JsonError> { let jobs = load_jobs(&state.dashboard_root) .await .map_err(|err| internal_error(&err))?; - Ok(Json(json!({ "jobs": jobs, "count": jobs.len() }))) + Ok(Json(AutomationJobsPayloadV1 { + count: jobs.len(), + jobs, + })) } #[hotpath::measure(label = "dashboard_api.jobs.create", future = true)] diff --git a/crates/tracedecay-dashboard-api/src/automation_outcomes_api.rs b/crates/tracedecay-dashboard-api/src/automation_outcomes_api.rs index c436016006..0787753239 100644 --- a/crates/tracedecay-dashboard-api/src/automation_outcomes_api.rs +++ b/crates/tracedecay-dashboard-api/src/automation_outcomes_api.rs @@ -3,9 +3,9 @@ //! facts. use axum::extract::State; -use axum::http::StatusCode; -use axum::response::Json; -use serde_json::{Value, json}; +use axum::response::{IntoResponse, Json, Response}; +use schemars::JsonSchema; +use serde::Serialize; use super::automation_authority_error_response; use super::exact_automation_authority; @@ -15,33 +15,54 @@ use crate::memory_api::control::{ }; use tracedecay_automation_runtime::automation::managed_skills::list_managed_skills; use tracedecay_automation_runtime::automation::outcomes::{ - AutomationOutcomesSnapshot, compute_fact_outcomes, compute_skill_outcomes, - load_outcomes_snapshot, + AutomationOutcomesSnapshot, FactOutcomeRecord, SkillOutcomeRecord, compute_fact_outcomes, + compute_skill_outcomes, load_outcomes_snapshot, }; use tracedecay_automation_runtime::automation::skill_usage::summarize_skill_usage; use tracedecay_domain::errors::Result; use tracedecay_runtime_core::tracedecay::current_timestamp; use tracedecay_store::FactReadControl; +/// Refresh watermarks of the persisted outcomes snapshot. `available` is +/// false when the snapshot could not be read, which differs from a snapshot +/// that was never refreshed. +#[derive(Debug, PartialEq, Eq, Serialize, JsonSchema)] +pub(crate) struct AutomationOutcomesSnapshotStatusV1 { + available: bool, + skills_refreshed_at: Option, + facts_refreshed_at: Option, +} + +/// `GET /api/automation/outcomes`. +#[derive(Debug, Serialize, JsonSchema)] +pub(crate) struct AutomationOutcomesPayloadV1 { + generated_at: i64, + skills: Vec, + facts: Vec, + snapshot: AutomationOutcomesSnapshotStatusV1, + /// Why the snapshot could not be read; empty when it was. + error: String, +} + #[hotpath::measure(label = "dashboard_api.outcomes.read", future = true)] pub async fn outcomes( State(state): State, RequestControl(control): RequestControl, -) -> (StatusCode, Json) { +) -> Response { let result = outcomes_payload(&state, &fact_read_control(&control)).await; if let Some(state) = request_terminal_state(&control) { - return terminal_read_response(state); + return terminal_read_response(state).into_response(); } match result { - Ok(payload) => (StatusCode::OK, Json(payload)), - Err(error) => automation_authority_error_response(error), + Ok(payload) => Json(payload).into_response(), + Err(error) => automation_authority_error_response(error).into_response(), } } async fn outcomes_payload( state: &DashboardState, read_control: &FactReadControl, -) -> std::result::Result { +) -> std::result::Result { let now = current_timestamp(); let authority = exact_automation_authority(state)?; let profile_root = authority.profile_root(); @@ -67,13 +88,13 @@ async fn outcomes_payload( .map_err(automation_failure)?; let (snapshot, error) = snapshot_fields(load_outcomes_snapshot(&state.dashboard_root).await); - Ok(json!({ - "generated_at": now, - "skills": skill_outcomes, - "facts": fact_outcomes, - "snapshot": snapshot, - "error": error, - })) + Ok(AutomationOutcomesPayloadV1 { + generated_at: now, + skills: skill_outcomes, + facts: fact_outcomes, + snapshot, + error, + }) } fn automation_failure(error: impl ToString) -> DashboardAutomationAuthorityErrorV1 { @@ -91,22 +112,24 @@ fn automation_failure(error: impl ToString) -> DashboardAutomationAuthorityError /// A snapshot that failed to load is not a snapshot that has never been /// refreshed: reporting the defaulted `None` watermarks with an empty `error` /// asserted that the read succeeded and found nothing. -fn snapshot_fields(loaded: Result) -> (Value, String) { +fn snapshot_fields( + loaded: Result, +) -> (AutomationOutcomesSnapshotStatusV1, String) { match loaded { Ok(snapshot) => ( - json!({ - "available": true, - "skills_refreshed_at": snapshot.skills_refreshed_at, - "facts_refreshed_at": snapshot.facts_refreshed_at, - }), + AutomationOutcomesSnapshotStatusV1 { + available: true, + skills_refreshed_at: snapshot.skills_refreshed_at, + facts_refreshed_at: snapshot.facts_refreshed_at, + }, String::new(), ), Err(error) => ( - json!({ - "available": false, - "skills_refreshed_at": Value::Null, - "facts_refreshed_at": Value::Null, - }), + AutomationOutcomesSnapshotStatusV1 { + available: false, + skills_refreshed_at: None, + facts_refreshed_at: None, + }, error.to_string(), ), } @@ -124,9 +147,14 @@ mod tests { message: "failed to parse automation outcomes snapshot '/x/outcomes.json'".to_owned(), })); - assert_eq!(snapshot["available"], json!(false)); - assert_eq!(snapshot["skills_refreshed_at"], Value::Null); - assert_eq!(snapshot["facts_refreshed_at"], Value::Null); + assert_eq!( + snapshot, + AutomationOutcomesSnapshotStatusV1 { + available: false, + skills_refreshed_at: None, + facts_refreshed_at: None, + } + ); assert!( error.contains("failed to parse automation outcomes snapshot"), "the failed read must be reported, not an empty error: {error}" @@ -137,8 +165,8 @@ mod tests { fn a_never_refreshed_snapshot_stays_distinct_from_a_failed_read() { let (snapshot, error) = snapshot_fields(Ok(AutomationOutcomesSnapshot::default())); - assert_eq!(snapshot["available"], json!(true)); - assert_eq!(snapshot["skills_refreshed_at"], Value::Null); + assert!(snapshot.available); + assert_eq!(snapshot.skills_refreshed_at, None); assert!(error.is_empty(), "a successful read reports no error"); } } diff --git a/crates/tracedecay-dashboard-api/src/automation_run_api.rs b/crates/tracedecay-dashboard-api/src/automation_run_api.rs index c50e2f4f32..241dac6963 100644 --- a/crates/tracedecay-dashboard-api/src/automation_run_api.rs +++ b/crates/tracedecay-dashboard-api/src/automation_run_api.rs @@ -1,16 +1,101 @@ use axum::extract::{Path as AxumPath, State}; use axum::http::StatusCode; -use axum::response::Json; -use serde::Deserialize; -use serde_json::{Value, json}; +use axum::response::{IntoResponse, Json, Response}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use tracedecay_automation::backend::{AgentTaskFailureClass, AgentTaskKind}; use super::DashboardState; use super::util::{internal_error, json_error}; use tracedecay_automation_runtime::automation::run_ledger::{ - AutomationRunArtifact, AutomationRunArtifactKind, AutomationRunLedgerRecord, find_run_record, - read_published_artifact_chain, read_run_artifact_payload, + AutomationRunArtifact, AutomationRunArtifactKind, AutomationRunLedgerRecord, + AutomationRunStatus, AutomationTrigger, find_run_record, read_published_artifact_chain, + read_run_artifact_payload, }; +/// One ledger record as the run-history row. `task_key` is the exact per-job +/// identity (`user_job:`); rows written before it existed carry `null` and +/// cannot be joined to a job. +#[derive(Debug, Serialize, JsonSchema)] +pub(crate) struct AutomationRunRowV1 { + run_id: String, + task: AgentTaskKind, + task_key: Option, + trigger: AutomationTrigger, + backend: String, + model: Option, + status: AutomationRunStatus, + reviewed_count: usize, + accepted_count: usize, + rejected_count: usize, + skipped_count: usize, + error: Option, + error_classification: Option, + error_retryable: Option, + backend_attempt_count: usize, + started_at: String, + completed_at: String, + artifact_kinds: Vec, +} + +/// `known` only when the ledger page holds every row and none was malformed. +#[derive(Debug, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub(crate) enum AutomationRunLedgerCompletenessV1 { + Known, + Partial, +} + +/// `GET /api/automation/runs`, newest first under `limit`. +#[derive(Debug, Serialize, JsonSchema)] +pub(crate) struct AutomationRunsPayloadV1 { + runs: Vec, + count: usize, + limit: usize, + has_more: bool, + malformed_row_count: usize, + completeness: AutomationRunLedgerCompletenessV1, +} + +/// Whether the ledger's artifact list matches the published artifact chain. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub(crate) enum AutomationRunArtifactIntegrityV1 { + Verified, + LedgerPublicationMismatch, + PublicationUnavailable, + VerificationFailed, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub(crate) struct AutomationRunArtifactChainV1 { + expected_kinds: Vec, + present_kinds: Vec, + metadata_complete: bool, + /// Every expected kind is present and the chain verified. + complete: bool, + integrity_status: AutomationRunArtifactIntegrityV1, +} + +/// `GET /api/automation/runs/{id}/artifacts`. +#[derive(Debug, Serialize, JsonSchema)] +pub(crate) struct AutomationRunArtifactsPayloadV1 { + run_id: String, + artifacts: Vec, + artifact_chain: AutomationRunArtifactChainV1, + count: usize, +} + +/// `GET /api/automation/runs/{id}/artifacts/{kind}`. The artifact kind owns its +/// payload shape, so it is served as opaque JSON. +#[derive(Debug, Serialize, JsonSchema)] +pub(crate) struct AutomationRunArtifactPayloadV1 { + run_id: String, + artifact: AutomationRunArtifact, + payload: Value, +} + #[derive(Debug, Default, Deserialize)] pub struct RunListParams { limit: Option, @@ -23,7 +108,7 @@ pub struct RunListParams { pub async fn run_list( State(state): State, axum::extract::Query(params): axum::extract::Query, -) -> (StatusCode, Json) { +) -> Response { let limit = super::util::coerce_limit(params.limit, 50, 200) as usize; // The locked ledger tail read is this route's only I/O; row projection // after it is linear in the (bounded) page. @@ -37,67 +122,59 @@ pub async fn run_list( .await { Ok(page) => { - let runs: Vec = page.records.iter().map(run_history_row).collect(); - let count = runs.len(); - let completeness = if page.is_complete() { - "known" - } else { - "partial" - }; - ( - StatusCode::OK, - Json(json!({ - "runs": runs, - "count": count, - "limit": limit, - "has_more": page.has_more, - "malformed_row_count": page.malformed_row_count, - "completeness": completeness, - "error": "", - })), - ) + let runs: Vec<_> = page.records.iter().map(run_history_row).collect(); + Json(AutomationRunsPayloadV1 { + count: runs.len(), + runs, + limit, + has_more: page.has_more, + malformed_row_count: page.malformed_row_count, + completeness: if page.is_complete() { + AutomationRunLedgerCompletenessV1::Known + } else { + AutomationRunLedgerCompletenessV1::Partial + }, + }) + .into_response() + } + Err(err) => { + internal_error(format!("Failed to read automation run ledger: {err}")).into_response() } - Err(err) => internal_error(format!("Failed to read automation run ledger: {err}")), } } -/// One ledger record as the run-history row: identity, outcome, review tallies, -/// typed failure class and backend attempt count, and which artifacts exist, -/// every field measured from the record itself. `task_key` is the exact -/// per-job identity (`user_job:`) the ledger writer recorded; rows written -/// before it existed carry `null` and cannot be joined to a job. -fn run_history_row(record: &AutomationRunLedgerRecord) -> Value { - json!({ - "run_id": record.run_id, - "task": record.task, - "task_key": record.task_key, - "trigger": record.trigger, - "backend": record.backend, - "model": record.model, - "status": record.status, - "reviewed_count": record.reviewed_count, - "accepted_count": record.accepted_count, - "rejected_count": record.rejected_count, - "skipped_count": record.skipped_count, - "error": record.error, - "error_classification": record.error_classification, - "error_retryable": record.error_retryable, - "backend_attempt_count": record.backend_attempt_count, - "started_at": record.started_at, - "completed_at": record.completed_at, - "artifact_kinds": record +fn run_history_row(record: &AutomationRunLedgerRecord) -> AutomationRunRowV1 { + AutomationRunRowV1 { + run_id: record.run_id.clone(), + task: record.task, + task_key: record.task_key.clone(), + trigger: record.trigger, + backend: record.backend.clone(), + model: record.model.clone(), + status: record.status, + reviewed_count: record.reviewed_count, + accepted_count: record.accepted_count, + rejected_count: record.rejected_count, + skipped_count: record.skipped_count, + error: record.error.clone(), + error_classification: record.error_classification, + error_retryable: record.error_retryable, + backend_attempt_count: record.backend_attempt_count, + started_at: record.started_at.clone(), + completed_at: record.completed_at.clone(), + artifact_kinds: record .artifacts .iter() .map(|artifact| artifact.kind.clone()) - .collect::>(), - }) + .collect(), + } } #[hotpath::measure(label = "dashboard_api.runs.artifacts", future = true)] pub async fn artifact_list( State(state): State, AxumPath(run_id): AxumPath, -) -> (StatusCode, Json) { +) -> Response { match find_run_record(&state.dashboard_root, &run_id).await { Ok(Some(record)) => { let count = record.artifacts.len(); @@ -108,32 +185,29 @@ pub async fn artifact_list( label = "dashboard_api.runs.chain_verify" ) .await; - let (integrity_status, integrity_verified) = match integrity { - Ok(Some(published)) if published == record.artifacts => ("verified", true), - Ok(Some(_)) => ("ledger_publication_mismatch", false), - Ok(None) => ("publication_unavailable", false), - Err(_) => ("verification_failed", false), + let integrity_status = match integrity { + Ok(Some(published)) if published == record.artifacts => { + AutomationRunArtifactIntegrityV1::Verified + } + Ok(Some(_)) => AutomationRunArtifactIntegrityV1::LedgerPublicationMismatch, + Ok(None) => AutomationRunArtifactIntegrityV1::PublicationUnavailable, + Err(_) => AutomationRunArtifactIntegrityV1::VerificationFailed, }; - ( - StatusCode::OK, - Json(json!({ - "run_id": run_id, - "artifacts": record.artifacts, - "artifact_chain": artifact_chain_summary( - &record.artifacts, - integrity_status, - integrity_verified, - ), - "count": count, - "error": "", - })), - ) + Json(AutomationRunArtifactsPayloadV1 { + artifact_chain: artifact_chain_summary(&record.artifacts, integrity_status), + run_id, + artifacts: record.artifacts, + count, + }) + .into_response() } Ok(None) => json_error( StatusCode::NOT_FOUND, format!("automation run '{run_id}' not found"), - ), - Err(err) => internal_error(format!("Failed to load automation run artifacts: {err}")), + ) + .into_response(), + Err(err) => internal_error(format!("Failed to load automation run artifacts: {err}")) + .into_response(), } } @@ -141,24 +215,27 @@ pub async fn artifact_list( pub async fn artifact_payload( State(state): State, AxumPath((run_id, kind)): AxumPath<(String, String)>, -) -> (StatusCode, Json) { +) -> Response { let record = match find_run_record(&state.dashboard_root, &run_id).await { Ok(Some(record)) => record, Ok(None) => { return json_error( StatusCode::NOT_FOUND, format!("automation run '{run_id}' not found"), - ); + ) + .into_response(); } Err(err) => { - return internal_error(format!("Failed to load automation run artifact: {err}")); + return internal_error(format!("Failed to load automation run artifact: {err}")) + .into_response(); } }; let Some(artifact) = find_artifact(&record.artifacts, &kind) else { return json_error( StatusCode::NOT_FOUND, format!("automation run artifact '{kind}' not found for run '{run_id}'"), - ); + ) + .into_response(); }; // Heavy per-run payloads (proposed/applied ops, validation reports) are // read and parsed here; this span scales with artifact size while the @@ -169,16 +246,15 @@ pub async fn artifact_payload( ) .await { - Ok(payload) => ( - StatusCode::OK, - Json(json!({ - "run_id": run_id, - "artifact": artifact, - "payload": payload, - "error": "", - })), - ), - Err(err) => internal_error(format!("Failed to read automation run artifact: {err}")), + Ok(payload) => Json(AutomationRunArtifactPayloadV1 { + run_id, + artifact: artifact.clone(), + payload, + }) + .into_response(), + Err(err) => { + internal_error(format!("Failed to read automation run artifact: {err}")).into_response() + } } } @@ -189,42 +265,43 @@ fn find_artifact<'a>( artifacts.iter().find(|artifact| artifact.kind == kind) } +const EXPECTED_ARTIFACT_CHAIN_KINDS: [AutomationRunArtifactKind; 6] = [ + AutomationRunArtifactKind::Traces, + AutomationRunArtifactKind::Feedback, + AutomationRunArtifactKind::GeneratedEvals, + AutomationRunArtifactKind::ValidationGate, + AutomationRunArtifactKind::OptimizerDiagnosis, + AutomationRunArtifactKind::CodexHandoff, +]; + fn artifact_chain_summary( artifacts: &[AutomationRunArtifact], - integrity_status: &str, - integrity_verified: bool, -) -> Value { - let expected_kinds = expected_artifact_chain_kinds(); - let present_kinds = artifacts - .iter() - .map(|artifact| artifact.kind.as_str()) - .collect::>(); - let complete = expected_kinds + integrity_status: AutomationRunArtifactIntegrityV1, +) -> AutomationRunArtifactChainV1 { + let present_kinds: Vec = artifacts .iter() - .all(|expected| present_kinds.iter().any(|present| present == expected)); - json!({ - "expected_kinds": expected_kinds, - "present_kinds": present_kinds, - "metadata_complete": complete, - "complete": complete && integrity_verified, - "integrity_status": integrity_status, - }) -} - -fn expected_artifact_chain_kinds() -> Vec<&'static str> { - vec![ - AutomationRunArtifactKind::Traces.as_str(), - AutomationRunArtifactKind::Feedback.as_str(), - AutomationRunArtifactKind::GeneratedEvals.as_str(), - AutomationRunArtifactKind::ValidationGate.as_str(), - AutomationRunArtifactKind::OptimizerDiagnosis.as_str(), - AutomationRunArtifactKind::CodexHandoff.as_str(), - ] + .map(|artifact| artifact.kind.clone()) + .collect(); + let metadata_complete = EXPECTED_ARTIFACT_CHAIN_KINDS.iter().all(|expected| { + present_kinds + .iter() + .any(|present| present == expected.as_str()) + }); + AutomationRunArtifactChainV1 { + expected_kinds: EXPECTED_ARTIFACT_CHAIN_KINDS.to_vec(), + present_kinds, + metadata_complete, + complete: metadata_complete + && integrity_status == AutomationRunArtifactIntegrityV1::Verified, + integrity_status, + } } #[cfg(test)] #[allow(clippy::unwrap_used)] mod tests { + use serde_json::json; + use super::*; fn record(value: Value) -> AutomationRunLedgerRecord { @@ -251,11 +328,14 @@ mod tests { "completed_at": "1754000031", }))); - assert_eq!(row["task_key"], "user_job:nightly"); - assert_eq!(row["error_classification"], "retryable"); - assert_eq!(row["error_retryable"], true); - assert_eq!(row["backend_attempt_count"], 2); - assert_eq!(row["artifact_kinds"], json!([])); + assert_eq!(row.task_key.as_deref(), Some("user_job:nightly")); + assert_eq!( + row.error_classification, + Some(AgentTaskFailureClass::Retryable) + ); + assert_eq!(row.error_retryable, Some(true)); + assert_eq!(row.backend_attempt_count, 2); + assert!(row.artifact_kinds.is_empty()); } #[test] @@ -275,9 +355,9 @@ mod tests { // A pre-`task_key` row must not be joined to any job, and an absent // failure classification is an absence rather than a default class. - assert_eq!(row["task_key"], Value::Null); - assert_eq!(row["error_classification"], Value::Null); - assert_eq!(row["error_retryable"], Value::Null); - assert_eq!(row["backend_attempt_count"], 0); + assert_eq!(row.task_key, None); + assert_eq!(row.error_classification, None); + assert_eq!(row.error_retryable, None); + assert_eq!(row.backend_attempt_count, 0); } } diff --git a/crates/tracedecay-dashboard-api/src/automation_scheduler_api.rs b/crates/tracedecay-dashboard-api/src/automation_scheduler_api.rs index 3b694c9980..53aaf1dc0a 100644 --- a/crates/tracedecay-dashboard-api/src/automation_scheduler_api.rs +++ b/crates/tracedecay-dashboard-api/src/automation_scheduler_api.rs @@ -4,7 +4,6 @@ use axum::Json; use axum::extract::State; use schemars::JsonSchema; use serde::Serialize; -use serde_json::Value; use super::DashboardState; use super::automation_config_api::effective_automation_config; @@ -12,13 +11,14 @@ use super::util::{JsonError, internal_error}; use tracedecay_automation_runtime::automation::backend::{AgentTaskKind, task_key}; use tracedecay_automation_runtime::automation::config::AutomationConfig; use tracedecay_automation_runtime::automation::run_ledger::{ - AutomationRunLedgerTaskSummary, load_run_ledger_task_summary, + AutomationRunLedgerRecord, AutomationRunLedgerTaskSummary, load_run_ledger_task_summary, }; use tracedecay_automation_runtime::automation::scheduler::{ AutomationSchedulerControl, SessionActivity, load_scheduler_control, load_session_activity, save_scheduler_control, schedule_decision, scheduler_control_path, }; use tracedecay_contracts::retained_surfaces::AutomationSkipReasonV1; +use tracedecay_domain::configuration::LcmSummarizerExecutableV1; use tracedecay_runtime_core::tracedecay::current_timestamp; type ApiResult = std::result::Result, JsonError>; @@ -63,7 +63,7 @@ pub(super) struct AutomationTaskStatusV1 { pub skip_reason: Option, /// The most recent scheduler-triggered ledger record. Its run artifacts /// remain the canonical detailed receipt surface. - pub last_scheduler_run: Option, + pub last_scheduler_run: Option, } #[hotpath::measure(label = "dashboard_api.scheduler.status", future = true)] @@ -96,7 +96,7 @@ async fn set_scheduler_paused( } async fn scheduler_status_payload(state: &DashboardState) -> ApiResult { - let (configuration_revision_id, effective) = + let (configuration_revision_id, effective, codex) = effective_automation_config(state).map_err(|err| internal_error(&err))?; let control = load_scheduler_control(&state.dashboard_root) .await @@ -141,57 +141,64 @@ async fn scheduler_status_payload(state: &DashboardState) -> ApiResult { tasks: vec![ task_status( &effective, + &codex, control.paused, &memory_summary, activity, now, AgentTaskKind::MemoryCurator, - )?, + ), task_status( &effective, + &codex, control.paused, &session_summary, activity, now, AgentTaskKind::SessionReflector, - )?, + ), task_status( &effective, + &codex, control.paused, &skill_summary, activity, now, AgentTaskKind::SkillWriter, - )?, + ), ], })) } fn task_status( config: &AutomationConfig, + codex: &LcmSummarizerExecutableV1, paused: bool, summary: &AutomationRunLedgerTaskSummary, activity: SessionActivity, now: i64, task: AgentTaskKind, -) -> std::result::Result { +) -> AutomationTaskStatusV1 { let decision = if paused { tracedecay_automation_runtime::automation::scheduler::AutomationScheduleDecision::skipped( AutomationSkipReasonV1::SchedulerPaused, ) } else { - schedule_decision(config, task, summary.records(), activity, now) + schedule_decision( + config, + codex.canonical_path(), + task, + summary.records(), + activity, + now, + ) }; - Ok(AutomationTaskStatusV1 { + AutomationTaskStatusV1 { task: task_key(task).to_string(), due: decision.is_due(), skip_reason: decision.skip_reason(), - last_scheduler_run: summary - .latest_scheduler_activity() - .map(serde_json::to_value) - .transpose() - .map_err(|error| internal_error(&error))?, - }) + last_scheduler_run: summary.latest_scheduler_activity().cloned(), + } } fn scheduler_status_label( @@ -313,19 +320,19 @@ mod tests { let status = task_status( &config, + &LcmSummarizerExecutableV1::Unconfigured, false, &summary, SessionActivity::none(), 150, AgentTaskKind::MemoryCurator, - ) - .unwrap(); + ); assert!(!status.due); assert_eq!( status.skip_reason, Some(AutomationSkipReasonV1::SchedulerNonRetryableFailure) ); - assert_eq!(status.last_scheduler_run.unwrap()["run_id"], "z-failure"); + assert_eq!(status.last_scheduler_run.unwrap().run_id, "z-failure"); } } diff --git a/crates/tracedecay-dashboard-api/src/automation_skills_api.rs b/crates/tracedecay-dashboard-api/src/automation_skills_api.rs index 53bc094bfa..188c49e257 100644 --- a/crates/tracedecay-dashboard-api/src/automation_skills_api.rs +++ b/crates/tracedecay-dashboard-api/src/automation_skills_api.rs @@ -3,7 +3,8 @@ use axum::Json; use axum::extract::{Path, State}; use axum::http::StatusCode; -use serde::Deserialize; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use super::util::{JsonError, internal_error, json_error}; @@ -17,8 +18,7 @@ use tracedecay_automation_runtime::automation::managed_skills::{ load_managed_skill, managed_skill_dir, managed_skill_root, }; use tracedecay_automation_runtime::automation::skill_usage::{ - skill_improvement_recommendations, stale_skill_recommendations, summarize_skill_usage, - summarize_skill_usage_for, + skill_improvement_recommendations, stale_skill_recommendations, summarize_skill_usage_for, }; use tracedecay_automation_runtime::automation::skill_writer::ManagedSkillDeploymentReceipt; use tracedecay_runtime_core::tracedecay::current_timestamp; @@ -52,32 +52,25 @@ pub struct ManagedSkillUpdateRequest { update: ManagedSkillUpdate, } +/// `GET /api/automation/skills`. +#[derive(Debug, Serialize, JsonSchema)] +pub(crate) struct AutomationSkillsPayloadV1 { + skills: Vec, + count: usize, +} + #[hotpath::measure(label = "dashboard_api.skills.list", future = true)] -pub async fn list(State(state): State) -> ApiResult { +pub async fn list( + State(state): State, +) -> std::result::Result, JsonError> { let profile_root = profile_root(&state)?; let skills = list_managed_skills(profile_root) .await .map_err(|err| internal_error(&err))?; - let skill_metadata = skills - .iter() - .map(|skill| skill.metadata.clone()) - .collect::>(); - let usage_summaries = summarize_skill_usage(profile_root, &skills) - .await - .map_err(|err| internal_error(&err))?; - let stale_recommendations = - stale_skill_recommendations(&usage_summaries, current_timestamp(), 60 * 60 * 24 * 90); - let improvement_recommendations = skill_improvement_recommendations(&usage_summaries); - Ok(Json(json!({ - "profile_root": profile_root.display().to_string(), - "skills_root": managed_skill_root(profile_root).display().to_string(), - "count": skills.len(), - "skills": skills, - "skill_metadata": skill_metadata, - "usage_summaries": usage_summaries, - "stale_recommendations": stale_recommendations, - "improvement_recommendations": improvement_recommendations, - }))) + Ok(Json(AutomationSkillsPayloadV1 { + count: skills.len(), + skills, + })) } #[hotpath::measure(label = "dashboard_api.skills.view", future = true)] diff --git a/crates/tracedecay-dashboard-api/src/contract_schema.rs b/crates/tracedecay-dashboard-api/src/contract_schema.rs index e78b09d625..7f3ab932b0 100644 --- a/crates/tracedecay-dashboard-api/src/contract_schema.rs +++ b/crates/tracedecay-dashboard-api/src/contract_schema.rs @@ -44,7 +44,14 @@ use super::analytics_api::{ AnalyticsOverviewPayloadV1, AnalyticsSubagentTreePayloadV1, AnalyticsUnderusedPayloadV1, AnalyticsUsageSummaryV1, }; +use super::automation_fact_receipts_api::AutomaticFactReceiptsPayloadV1; +use super::automation_jobs_api::AutomationJobsPayloadV1; +use super::automation_outcomes_api::AutomationOutcomesPayloadV1; +use super::automation_run_api::{ + AutomationRunArtifactPayloadV1, AutomationRunArtifactsPayloadV1, AutomationRunsPayloadV1, +}; use super::automation_scheduler_api::AutomationSchedulerStatusV1; +use super::automation_skills_api::AutomationSkillsPayloadV1; use super::code_read_api::RevisionPairUnionLayoutV1; use super::delivery_api::{DeliveryInboxV1, DeliveryOverviewV1}; use super::doctor_findings_api::DoctorFindingsPayloadV1; @@ -63,17 +70,18 @@ use super::lcm_api::{ use super::loom_api::LoomTemporalPayloadV1; use super::memory_api::{ MemoryFactDetailPayloadV1, MemoryOverviewPayloadV1, MemoryStatusPayloadV1, + MemoryTrustHistoryPayloadV1, +}; +use super::memory_service::{ + MemoryOplogPayloadV1, MemoryProjectionPayloadV1, MemorySimilarityPayloadV1, }; use super::projects::{ProjectContextPayloadV1, ProjectsPayloadV1}; use super::read_model::{DASHBOARD_SCHEMA_REVISION_V1, DashboardEnvelopeV1}; use super::remote_status_api::RemoteOperationalStatusPayloadV1; -use super::savings_api::{ - SavingsModelsPayloadV1, SavingsOverviewPayloadV1, SavingsSessionsPayloadV1, -}; +use super::savings_api::{SavingsModelsPayloadV1, SavingsOverviewPayloadV1}; use super::settings_api::{ CodeIndexWorkerSettingsPatch, ProjectSettingsPatch, SettingsPayloadV1, UserSettingsPatch, }; -use super::storage_findings_api::StorageFindingsPayloadV1; use super::storage_telemetry_api::StorageTelemetryPayloadV1; use super::work_api::registered_route_contracts as registered_work_route_contracts; use crate::application::feedback::observations::FeedbackObservationReadModelV1; @@ -84,7 +92,6 @@ use tracedecay_contracts::code_index_freshness::CodeIndexFreshnessPayloadV1; struct DashboardContractCatalogV1 { envelope: DashboardEnvelopeV1, storage_telemetry: StorageTelemetryPayloadV1, - storage_findings: StorageFindingsPayloadV1, doctor_findings: DoctorFindingsPayloadV1, remote_operational_status: RemoteOperationalStatusPayloadV1, explorer_query_run: ExplorerQueryRunV1, @@ -103,6 +110,10 @@ struct DashboardContractCatalogV1 { memory_overview: DashboardEnvelopeV1>, memory_status: DashboardEnvelopeV1>, memory_fact_detail: DashboardEnvelopeV1>, + memory_trust_history: MemoryTrustHistoryPayloadV1, + memory_projection: MemoryProjectionPayloadV1, + memory_similarity: MemorySimilarityPayloadV1, + memory_oplog: MemoryOplogPayloadV1, analytics_overview: DashboardEnvelopeV1>, analytics_usage: DashboardEnvelopeV1>, analytics_agents: DashboardEnvelopeV1>, @@ -117,7 +128,6 @@ struct DashboardContractCatalogV1 { analytics_underused: DashboardEnvelopeV1>, analytics_diagnostics: DashboardEnvelopeV1>, savings_overview: DashboardEnvelopeV1>, - savings_sessions: SavingsSessionsPayloadV1, savings_models: SavingsModelsPayloadV1, lcm_session: DashboardEnvelopeV1>, lcm_timeline: DashboardEnvelopeV1>, @@ -218,6 +228,13 @@ struct DashboardContractCatalogV1 { /// Served identically by `GET /api/automation/scheduler/status` and by the /// `pause`/`resume` controls, which re-read rather than acknowledge. automation_scheduler_status: AutomationSchedulerStatusV1, + automation_jobs: AutomationJobsPayloadV1, + automation_skills: AutomationSkillsPayloadV1, + automation_fact_receipts: AutomaticFactReceiptsPayloadV1, + automation_runs: AutomationRunsPayloadV1, + automation_run_artifacts: AutomationRunArtifactsPayloadV1, + automation_run_artifact: AutomationRunArtifactPayloadV1, + automation_outcomes: AutomationOutcomesPayloadV1, fact_store_curate_request: FactStoreCurateRequestV1, automation_run: AutomationRunResultV1, automation_problem: AutomationRunProblemV1, @@ -581,7 +598,6 @@ mod tests { "MemoryEntityRowV1", "AnalyticsOverviewPayloadV1", "SavingsOverviewPayloadV1", - "SavingsSessionsPayloadV1", "SavingsModelsPayloadV1", "SavingsProviderSpendV1", "SavingsProviderDayPointV1", diff --git a/crates/tracedecay-dashboard-api/src/doctor_findings_api.rs b/crates/tracedecay-dashboard-api/src/doctor_findings_api.rs index 68cdefb954..a1f24394b3 100644 --- a/crates/tracedecay-dashboard-api/src/doctor_findings_api.rs +++ b/crates/tracedecay-dashboard-api/src/doctor_findings_api.rs @@ -13,11 +13,12 @@ use schemars::JsonSchema; use serde::Serialize; use tracedecay_api::doctor::{ DOCTOR_REPORT_SOURCE_UNSUPPORTED_NOTE, DoctorFindingsQueryV1, DoctorReadPresentationV1, - KNOWN_DOCTOR_FINDING_FAMILIES, doctor_report_failure_note, parse_doctor_finding_family, - project_doctor_report, + doctor_report_failure_note, parse_doctor_finding_family, project_doctor_report, }; use tracedecay_contracts::doctor::{ - DoctorFindingFamilyV1, DoctorReportCoverageV1, DoctorReportEntryV1, + DOCTOR_FINDING_FAMILIES, DoctorCoverageCompletenessV1, DoctorEvidenceStateV1, + DoctorFamilyConsultationV1, DoctorFamilyCoverageV1, DoctorFamilyUnavailableReasonV1, + DoctorFindingFamilyV1, DoctorReportCoverageV1, DoctorReportEntryV1, DoctorStorageFindingKindV1, }; use tracedecay_contracts::storage::SchemaConvergenceFindingV1; @@ -26,6 +27,15 @@ use super::read_model::{ DashboardDomainStateV1, DashboardEnvelopeV1, DashboardScopeV1, scope_from_state, }; +const STORAGE_KINDS: [DoctorStorageFindingKindV1; 6] = [ + DoctorStorageFindingKindV1::OverBudgetStore, + DoctorStorageFindingKindV1::OrphanStore, + DoctorStorageFindingKindV1::IncidentDebrisPresent, + DoctorStorageFindingKindV1::RetentionBacklog, + DoctorStorageFindingKindV1::TableGrowth, + DoctorStorageFindingKindV1::PendingSchemaMigration, +]; + /// The canonical Doctor report projection for the read-only dashboard. #[derive(Clone, Debug, Serialize, JsonSchema)] pub struct DoctorFindingsPayloadV1 { @@ -34,9 +44,32 @@ pub struct DoctorFindingsPayloadV1 { pub report_coverage: Option, pub known_families: Vec, pub schema_convergences: Vec, + /// Source coverage for each typed storage finding producer. Empty when the + /// family filter excludes storage or the family filter was rejected. + pub storage_kind_statuses: Vec, pub note: String, } +/// Whether one storage finding producer had enough source evidence to report +/// a real result. This is source coverage, not a health grade: `Real` can +/// describe a clean observation or a problem finding. +#[derive(Clone, Copy, Debug, Serialize, JsonSchema, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum StorageFindingSourceStateV1 { + Real, + Partial, + Unsupported, +} + +/// Source-coverage status for one typed storage finding producer. +#[derive(Clone, Debug, Serialize, JsonSchema, PartialEq, Eq)] +pub struct StorageFindingKindStatusV1 { + pub kind: DoctorStorageFindingKindV1, + pub state: StorageFindingSourceStateV1, + pub observed_entries: usize, + pub reason: String, +} + /// `GET /api/doctor/findings` #[hotpath::measure(label = "dashboard_api.doctor.findings", future = true)] pub async fn findings( @@ -64,27 +97,6 @@ async fn findings_with_authorities( return envelope; } }; - findings_for_family_with_authorities(scope, family_filter, doctor_report_reader).await -} - -/// Project the admitted canonical Doctor report for one closed finding family. -/// -/// Compatibility routes such as `/api/storage/findings` call this seam instead -/// of evaluating health from dashboard-held database handles. -pub async fn findings_for_family( - state: DashboardState, - family_filter: Option, -) -> DashboardEnvelopeV1 { - let scope = scope_from_state(&state); - findings_for_family_with_authorities(scope, family_filter, state.doctor_report_reader.clone()) - .await -} - -async fn findings_for_family_with_authorities( - scope: DashboardScopeV1, - family_filter: Option, - doctor_report_reader: Option, -) -> DashboardEnvelopeV1 { let Some(reader) = doctor_report_reader.as_ref() else { return envelope( scope, @@ -94,8 +106,8 @@ async fn findings_for_family_with_authorities( }; // The admitted daemon composes the report across every finding producer; - // this single await is the expensive phase behind both `/api/doctor/*` - // and `/api/storage/findings`, and the span records failed reads too. + // this single await is the expensive phase behind `/api/doctor/*`, and + // the span records failed reads too. let admitted = match hotpath::future!(reader(), label = "dashboard_api.doctor.report_read").await { Ok(admitted) => admitted, @@ -126,8 +138,9 @@ async fn findings_for_family_with_authorities( family_filter, entries: projection.entries, report_coverage: Some(projection.report_coverage), - known_families: KNOWN_DOCTOR_FINDING_FAMILIES.to_vec(), + known_families: DOCTOR_FINDING_FAMILIES.to_vec(), schema_convergences: admitted.schema_convergences, + storage_kind_statuses: Vec::new(), note: projection.note, }, ) @@ -136,8 +149,17 @@ async fn findings_for_family_with_authorities( fn envelope( scope: DashboardScopeV1, presentation: DoctorReadPresentationV1, - payload: DoctorFindingsPayloadV1, + mut payload: DoctorFindingsPayloadV1, ) -> DashboardEnvelopeV1 { + if payload + .family_filter + .is_none_or(|family| family == DoctorFindingFamilyV1::Storage) + { + payload.storage_kind_statuses = STORAGE_KINDS + .into_iter() + .map(|kind| storage_kind_status(&payload, kind)) + .collect(); + } DashboardEnvelopeV1::new( scope, presentation.domain_state, @@ -156,12 +178,128 @@ fn unavailable_payload( family_filter, entries: Vec::new(), report_coverage: None, - known_families: KNOWN_DOCTOR_FINDING_FAMILIES.to_vec(), + known_families: DOCTOR_FINDING_FAMILIES.to_vec(), schema_convergences: Vec::new(), + storage_kind_statuses: Vec::new(), note: note.into(), } } +fn storage_kind_status( + payload: &DoctorFindingsPayloadV1, + kind: DoctorStorageFindingKindV1, +) -> StorageFindingKindStatusV1 { + let consultation = payload.report_coverage.as_ref().and_then(|coverage| { + coverage + .families() + .iter() + .find(|family| family.family() == DoctorFindingFamilyV1::Storage) + .map(DoctorFamilyCoverageV1::consultation) + }); + let matching = payload + .entries + .iter() + .filter(|entry| entry.storage_kind() == Some(kind)) + .collect::>(); + if !matching.is_empty() { + let complete_observations = consultation == Some(DoctorFamilyConsultationV1::Consulted) + && matching.iter().all(|entry| { + entry.finding().coverage().completeness() == DoctorCoverageCompletenessV1::Complete + && matches!( + entry.finding().state(), + DoctorEvidenceStateV1::Stale + | DoctorEvidenceStateV1::Degraded + | DoctorEvidenceStateV1::HealthyCompleteCoverage + ) + }); + let state = if complete_observations { + StorageFindingSourceStateV1::Real + } else { + StorageFindingSourceStateV1::Partial + }; + let reason = if complete_observations { + format!( + "canonical Doctor producer returned {} observed {}", + matching.len(), + if matching.len() == 1 { + "entry with complete coverage" + } else { + "entries with complete coverage" + } + ) + } else if let Some(DoctorFamilyConsultationV1::Unavailable { reason }) = consultation { + format!( + "canonical Doctor producer returned {} observed entries, but storage family coverage is incomplete ({})", + matching.len(), + unavailable_reason(reason) + ) + } else { + format!( + "canonical Doctor producer returned {} entries, but coverage or evidence state was incomplete", + matching.len() + ) + }; + return StorageFindingKindStatusV1 { + kind, + state, + observed_entries: matching.len(), + reason, + }; + } + + let (state, reason) = match consultation { + Some(DoctorFamilyConsultationV1::Consulted) => ( + StorageFindingSourceStateV1::Partial, + "the storage family was consulted, but the canonical report returned no typed entry for this producer; absence does not prove clean per-producer coverage" + .to_string(), + ), + Some(DoctorFamilyConsultationV1::Unavailable { + reason: + reason @ (DoctorFamilyUnavailableReasonV1::Unwired + | DoctorFamilyUnavailableReasonV1::Unsupported), + }) => ( + StorageFindingSourceStateV1::Unsupported, + format!( + "canonical Doctor storage source is unavailable ({})", + unavailable_reason(reason) + ), + ), + Some(DoctorFamilyConsultationV1::Unavailable { reason }) => ( + StorageFindingSourceStateV1::Partial, + format!( + "canonical Doctor storage source is unavailable ({}); no clean result is asserted", + unavailable_reason(reason) + ), + ), + None => ( + StorageFindingSourceStateV1::Unsupported, + format!( + "canonical Doctor storage source supplied no consultation record: {}", + payload.note + ), + ), + }; + StorageFindingKindStatusV1 { + kind, + state, + observed_entries: 0, + reason, + } +} + +const fn unavailable_reason(reason: DoctorFamilyUnavailableReasonV1) -> &'static str { + match reason { + DoctorFamilyUnavailableReasonV1::Unwired => "unwired", + DoctorFamilyUnavailableReasonV1::Unsupported => "unsupported", + DoctorFamilyUnavailableReasonV1::Absent => "absent", + DoctorFamilyUnavailableReasonV1::Denied => "denied", + DoctorFamilyUnavailableReasonV1::Unknown => "unknown", + DoctorFamilyUnavailableReasonV1::Unavailable => "unavailable", + DoctorFamilyUnavailableReasonV1::ResetRequired => "reset_required", + DoctorFamilyUnavailableReasonV1::Corrupt => "corrupt", + } +} + #[cfg(test)] #[allow(clippy::unwrap_used, clippy::expect_used)] mod tests { @@ -381,7 +519,7 @@ mod tests { assert!(envelope.payload.entries.is_empty()); assert_eq!( envelope.payload.known_families.len(), - KNOWN_DOCTOR_FINDING_FAMILIES.len() + DOCTOR_FINDING_FAMILIES.len() ); assert_eq!(envelope.payload.family_filter, None); assert_eq!(envelope.payload.note, DOCTOR_REPORT_SOURCE_UNSUPPORTED_NOTE); @@ -391,6 +529,68 @@ mod tests { ); } + #[tokio::test] + async fn storage_family_without_admitted_reader_projects_every_producer_as_unsupported() { + let envelope = findings_for_test( + DoctorFindingsQueryV1 { + family: Some("storage".to_string()), + }, + None, + ) + .await; + + assert_eq!( + envelope.payload.family_filter, + Some(DoctorFindingFamilyV1::Storage) + ); + assert_eq!(envelope.domain_state, DashboardDomainStateV1::Unsupported); + let statuses = &envelope.payload.storage_kind_statuses; + assert_eq!( + statuses + .iter() + .map(|status| status.kind) + .collect::>(), + STORAGE_KINDS + ); + assert!( + statuses.iter().all( + |status| status.state == StorageFindingSourceStateV1::Unsupported + && !status.reason.is_empty() + ), + "an unadmitted canonical source must not report any producer as real: {statuses:?}" + ); + + let other = findings_for_test( + DoctorFindingsQueryV1 { + family: Some("advisory".to_string()), + }, + None, + ) + .await; + assert!(other.payload.storage_kind_statuses.is_empty()); + } + + #[tokio::test] + async fn storage_family_consulted_without_entries_is_partial_not_clean() { + let report = compose_report(&DoctorTestSourcesV1::all_unknown()).await; + let envelope = findings_for_test( + DoctorFindingsQueryV1 { + family: Some("storage".to_string()), + }, + Some(report), + ) + .await; + + let statuses = &envelope.payload.storage_kind_statuses; + assert_eq!(statuses.len(), STORAGE_KINDS.len()); + assert!( + statuses + .iter() + .all(|status| status.state != StorageFindingSourceStateV1::Real), + "unknown storage evidence must never read as a real producer result: {statuses:?}" + ); + } + #[tokio::test] async fn findings_route_rejects_unknown_family_with_error_state() { let envelope = findings_for_test( @@ -430,7 +630,7 @@ mod tests { .unwrap() .families() .len(), - KNOWN_DOCTOR_FINDING_FAMILIES.len() + DOCTOR_FINDING_FAMILIES.len() ); } diff --git a/crates/tracedecay-dashboard-api/src/events_api.rs b/crates/tracedecay-dashboard-api/src/events_api.rs index cd89dbbb9d..98a5b46a22 100644 --- a/crates/tracedecay-dashboard-api/src/events_api.rs +++ b/crates/tracedecay-dashboard-api/src/events_api.rs @@ -1044,7 +1044,6 @@ pub(crate) async fn dashboard_state_fixture( pr_autotrack_reader: None, storage_mode: "profile_sharded".to_owned(), store_root, - config_path: project.path().join("config.json"), dashboard_root, retention_config: tracedecay_configuration::RetentionConfig::default(), user_settings: Arc::new(ProductionUserSettingsDaemonClient::default()), @@ -1360,28 +1359,19 @@ mod tests { } #[test] - fn activity_families_serialize_with_their_own_family_tags() { - for family in ActivityFamilyV1::ALL { - let kind = DashboardEventKindV1::activity(family, 1, 1, None); - let value = serde_json::to_value(&kind).unwrap(); - let tag = value["family"].as_str().expect("family tag").to_string(); - assert!( - tag.ends_with("_activity"), - "activity families are tagged as activity: {tag}" - ); - // The SSE event name must be the one the frontend subscribes to. - assert_eq!(kind.stream(), family.stream_name()); - } + fn tool_call_activity_serializes_its_family_tag_and_sse_stream() { + let kind = DashboardEventKindV1::activity( + ActivityFamilyV1::ToolCall, + 4, + 4, + Some("tracedecay_context".into()), + ); assert_eq!( - serde_json::to_value(DashboardEventKindV1::activity( - ActivityFamilyV1::ToolCall, - 4, - 4, - Some("tracedecay_context".into()), - )) - .unwrap()["family"], + serde_json::to_value(&kind).unwrap()["family"], "tool_call_activity" ); + // The SSE event name the frontend subscribes to. + assert_eq!(kind.stream(), "tool_call"); } #[test] diff --git a/crates/tracedecay-dashboard-api/src/explorer_api.rs b/crates/tracedecay-dashboard-api/src/explorer_api.rs index 5cda18704a..6f19d36547 100644 --- a/crates/tracedecay-dashboard-api/src/explorer_api.rs +++ b/crates/tracedecay-dashboard-api/src/explorer_api.rs @@ -26,6 +26,7 @@ use super::lcm_api::{ DashboardLcmCanonicalMessageV1, DashboardLcmCanonicalPageV1, DashboardLcmCanonicalStatsV1, DashboardLcmCanonicalSummaryV1, DashboardLcmReadOutcomeV1, DashboardLcmReadRequestV1, DashboardLcmReadStateV1, LcmMessageV1, LcmSummaryNodeV1, LcmTokenCountProvenanceV1, + message_tool_use_id, }; use super::read_model::{ DashboardCoverageV1, DashboardDomainStateV1, DashboardEnvelopeV1, DashboardFreshnessV1, @@ -34,7 +35,7 @@ use super::read_model::{ use super::util::json_error; use super::{DashboardHttpRequestControlV1, DashboardState, RequestControl, graph_service}; use crate::request_identity::{GlobalOpaqueIdentityKind, mint_global_opaque_id}; -use tracedecay_session_memory::context::CancellationToken; +use tracedecay_runtime_core::cancellation::CancellationToken; const SOURCE_IDS: [ExplorerSourceIdV1; 3] = [ ExplorerSourceIdV1::CodeGraph, @@ -1194,6 +1195,7 @@ fn explorer_lcm_message(message: DashboardLcmCanonicalMessageV1) -> LcmMessageV1 message_id: message.message_id, ordinal: Some(message.ordinal), storage_kind: Some("canonical_temporal".to_owned()), + tool_use_id: message_tool_use_id(message.metadata_json.as_deref()), metadata_json: message.metadata_json, tool_name: message.tool_names, pinned: None, diff --git a/crates/tracedecay-dashboard-api/src/explorer_api/knowledge.rs b/crates/tracedecay-dashboard-api/src/explorer_api/knowledge.rs index 7aff8e671d..a4e0d383b7 100644 --- a/crates/tracedecay-dashboard-api/src/explorer_api/knowledge.rs +++ b/crates/tracedecay-dashboard-api/src/explorer_api/knowledge.rs @@ -10,7 +10,7 @@ use super::{ExplorerQueryRequestV1, ExplorerSourceIdV1, ExplorerSourceProgressV1 use crate::memory_service::MemoryFactsCoverageV1; use crate::read_model::DashboardCoverageCompletenessV1; use crate::{DashboardHttpRequestControlV1, DashboardState, memory_service}; -use tracedecay_session_memory::context::CancellationToken; +use tracedecay_runtime_core::cancellation::CancellationToken; fn coverage_summary( coverage: &MemoryFactsCoverageV1, diff --git a/crates/tracedecay-dashboard-api/src/graph_structure_api.rs b/crates/tracedecay-dashboard-api/src/graph_structure_api.rs index 792f314eba..148b19f272 100644 --- a/crates/tracedecay-dashboard-api/src/graph_structure_api.rs +++ b/crates/tracedecay-dashboard-api/src/graph_structure_api.rs @@ -784,7 +784,7 @@ async fn node_tests( return graph_error_response::(&state, error); } }; - let qualification = if crate::tracedecay::is_test_file(&caller.file_path) { + let qualification = if tracedecay_code_index::is_test_file(&caller.file_path) { "test_file" } else if match has_test_annotation(&annotations) { Ok(has_test_annotation) => has_test_annotation, diff --git a/crates/tracedecay-dashboard-api/src/lcm_api.rs b/crates/tracedecay-dashboard-api/src/lcm_api.rs index d8c7ccb117..b789b5c02f 100644 --- a/crates/tracedecay-dashboard-api/src/lcm_api.rs +++ b/crates/tracedecay-dashboard-api/src/lcm_api.rs @@ -17,6 +17,7 @@ use super::read_model::{ }; use super::util::{JsonPath, JsonQuery}; use super::{DashboardHttpRequestControlV1, DashboardState, RequestControl}; +use tracedecay_store::TOOL_USE_ID_KEY; mod aggregates; @@ -184,12 +185,27 @@ pub(super) struct LcmMessageV1 { pub(super) storage_kind: Option, pub(super) metadata_json: Option, pub(super) tool_name: Option, + /// The host's own identifier of this message's tool invocation (Claude + /// `tool_use.id`, Codex `call_id`, Cursor composer `toolCallId`), the value + /// a child session's `parent_tool_use_id` names. Absent when the host + /// recorded none; never synthesized. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(super) tool_use_id: Option, pub(super) pinned: Option, pub(super) summary_node_ids: Vec, #[serde(default)] pub(super) snippet: Option, } +/// The typed `tool_use_id` a stored message row carries in its metadata. +pub(super) fn message_tool_use_id(metadata_json: Option<&str>) -> Option { + let metadata: serde_json::Value = serde_json::from_str(metadata_json?).ok()?; + metadata + .get(TOOL_USE_ID_KEY) + .and_then(serde_json::Value::as_str) + .map(str::to_owned) +} + #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] pub(super) struct LcmSummaryNodeV1 { pub(super) node_id: String, diff --git a/crates/tracedecay-dashboard-api/src/lcm_api/aggregates.rs b/crates/tracedecay-dashboard-api/src/lcm_api/aggregates.rs index 00dd40ee6d..2f15f7321a 100644 --- a/crates/tracedecay-dashboard-api/src/lcm_api/aggregates.rs +++ b/crates/tracedecay-dashboard-api/src/lcm_api/aggregates.rs @@ -8,7 +8,7 @@ use super::super::token_count::{ use super::{ DashboardLcmCanonicalMatchesV1, DashboardLcmCanonicalMessageV1, DashboardLcmCanonicalPageV1, DashboardLcmCanonicalSummaryV1, DashboardLcmReadRequestV1, DashboardLcmTimelineBucketV1, - LcmTokenCountProvenanceV1, + LcmTokenCountProvenanceV1, message_tool_use_id, }; pub(super) fn render_canonical_payload( @@ -566,6 +566,7 @@ fn message_json( "message_id": message.message_id, "ordinal": message.ordinal, "storage_kind": "canonical_temporal", + "tool_use_id": message_tool_use_id(message.metadata_json.as_deref()), "metadata_json": message.metadata_json, "tool_name": message.tool_names, "pinned": null, diff --git a/crates/tracedecay-dashboard-api/src/lib.rs b/crates/tracedecay-dashboard-api/src/lib.rs index 9b77f7c611..438f3950bb 100644 --- a/crates/tracedecay-dashboard-api/src/lib.rs +++ b/crates/tracedecay-dashboard-api/src/lib.rs @@ -73,11 +73,11 @@ pub(crate) mod test_support { Vec::new() } - fn write_text(path: &Path, contents: &str, _: Option<&Path>) -> Result<()> { + fn write_text(path: &Path, contents: &str) -> Result<()> { Ok(std::fs::write(path, contents)?) } - fn write_json(path: &Path, value: &serde_json::Value, _: Option<&Path>) -> Result<()> { + fn write_json(path: &Path, value: &serde_json::Value) -> Result<()> { Ok(std::fs::write(path, serde_json::to_vec_pretty(value)?)?) } @@ -180,7 +180,6 @@ pub use settings_api::{ DashboardCodeIndexWorkerSettingsFuture, DashboardProfileCodeIndexWorkerSettingsPort, PrAutoTrackManagedSummaryEntryV1, PrAutoTrackManagedSummaryReader, }; -mod storage_findings_api; mod storage_telemetry_api; mod token_count; mod util; @@ -477,8 +476,6 @@ pub struct DashboardState { pub storage_mode: String, /// Resolved active project store root. pub store_root: PathBuf, - /// Resolved `config.json` path for the active project store. - pub config_path: PathBuf, /// Resolved dashboard sidecar root inside the active project store. pub dashboard_root: PathBuf, /// Retention policy resolved with the owning runtime configuration. @@ -776,7 +773,6 @@ fn resolve_lcm_store_for_layout( pub fn storage_mode_label(mode: &StorageMode) -> &'static str { match mode { - StorageMode::ProjectLocal => "project_local", StorageMode::ProfileSharded => "profile_sharded", } } @@ -844,7 +840,6 @@ async fn build_state_inner( let lcm = resolve_lcm_store(cg, registered_project_session_db).await; let dashboard_root = cg.store_layout.dashboard_root.clone(); let store_root = cg.store_layout.data_root.clone(); - let config_path = cg.store_layout.config_path.clone(); let storage_mode = storage_mode_label(&cg.store_layout.storage_mode).to_string(); let code_diagnostics_authority = match ( code_diagnostics_broker, @@ -907,7 +902,6 @@ async fn build_state_inner( pr_autotrack_reader, storage_mode, store_root, - config_path, dashboard_root, retention_config: cg.retention_config.clone(), user_settings: Arc::clone(&cg.user_settings_client), @@ -1633,7 +1627,6 @@ fn project_api_router() -> Router { ) .route("/api/feedback/status", get(feedback_api::status)) // Holographic memory plugin API (mirrors holographic_plus plugin_api.py) - .route("/api/plugins/holographic/", get(memory_api::overview)) .route("/api/plugins/holographic", get(memory_api::overview)) .route("/api/plugins/holographic/status", get(memory_api::status)) .route( @@ -1788,10 +1781,7 @@ fn project_api_router() -> Router { // Savings & Cost API (savings ledger + session cost accounting) .route("/api/plugins/savings/overview", get(savings_api::overview)) .route("/api/costs", get(savings_api::costs)) - .route("/api/plugins/savings/ledger", get(savings_api::ledger)) - .route("/api/plugins/savings/sessions", get(savings_api::sessions)) .route("/api/plugins/savings/models", get(savings_api::models)) - .route("/api/plugins/savings/pricing", get(savings_api::pricing)) // Settings API (aggregated project/user config + read-only env gates) .route("/api/settings", get(settings_api::get_settings)) .route( @@ -1821,7 +1811,7 @@ fn project_api_router() -> Router { ) .route("/api/loom/temporal", get(loom_api::temporal)) // V2 read-model surfaces (DashboardEnvelope). Doctor finding - // family, storage telemetry/findings, code-index freshness, and + // family, storage telemetry, code-index freshness, and // the typed SSE stream. See `read_model` for the normative envelope. // Read-only Doctor/health paths come from the API-owned descriptors in // `tracedecay_api::doctor` so the mount cannot drift from them. @@ -1833,10 +1823,6 @@ fn project_api_router() -> Router { "/api/storage/telemetry", get(storage_telemetry_api::telemetry), ) - .route( - tracedecay_api::doctor::STORAGE_FINDINGS_ROUTE_PATH, - get(storage_findings_api::findings), - ) .route( "/api/code-index/freshness", get(code_index_freshness_api::freshness), @@ -2138,7 +2124,7 @@ async fn capabilities( let has_lcm = state.lcm_read_authority.is_some(); let automation = automation_config_api::effective_automation_config(&state); let (automation_configured, automation_mode, automation_payload) = match automation { - Ok((configuration_revision_id, config)) => { + Ok((configuration_revision_id, config, codex)) => { let backend_supported = matches!(config.backend, AutomationBackend::CodexAppServer); let configured = config.enabled && backend_supported; let mode = if !configured { @@ -2158,7 +2144,7 @@ async fn capabilities( "mode": mode, "backend": config.backend, "host_mode": config.host_mode, - "availability": backend::backend_availability(&config), + "availability": backend::backend_availability(&config, &codex), }), ) } @@ -2527,7 +2513,6 @@ mod authority_tests { pr_autotrack_reader: None, storage_mode: storage_mode_label(&layout.storage_mode).to_owned(), store_root: layout.data_root.clone(), - config_path: layout.config_path.clone(), dashboard_root: layout.dashboard_root.clone(), retention_config: tracedecay_configuration::RetentionConfig::default(), user_settings: Arc::new( @@ -2686,18 +2671,18 @@ mod authority_tests { let similarity_warm = memory_service::similarity_payload(&fixture.state, 0.5, 100, &control).await; - assert_eq!(projection_before["points"].as_array().unwrap().len(), 0); - assert_eq!(similarity_before["count"], 0); - assert_eq!(projection_after["scan"]["cache_state"], "miss"); - assert_eq!(projection_after["scan"]["vector_rows_read"], 1); - assert_eq!(projection_after["points"].as_array().unwrap().len(), 1); - assert_eq!(similarity_after["scan"]["cache_state"], "miss"); - assert_eq!(similarity_after["scan"]["vector_rows_read"], 1); - assert_eq!(similarity_after["count"], 1); - assert_eq!(projection_warm["scan"]["cache_state"], "hit"); - assert_eq!(projection_warm["scan"]["vector_rows_read"], 0); - assert_eq!(similarity_warm["scan"]["cache_state"], "hit"); - assert_eq!(similarity_warm["scan"]["vector_rows_read"], 0); + assert_eq!(projection_before.points.len(), 0); + assert_eq!(similarity_before.count, 0); + assert_eq!(projection_after.scan.as_ref().unwrap().cache_state, "miss"); + assert_eq!(projection_after.scan.as_ref().unwrap().vector_rows_read, 1); + assert_eq!(projection_after.points.len(), 1); + assert_eq!(similarity_after.scan.as_ref().unwrap().cache_state, "miss"); + assert_eq!(similarity_after.scan.as_ref().unwrap().vector_rows_read, 1); + assert_eq!(similarity_after.count, 1); + assert_eq!(projection_warm.scan.as_ref().unwrap().cache_state, "hit"); + assert_eq!(projection_warm.scan.as_ref().unwrap().vector_rows_read, 0); + assert_eq!(similarity_warm.scan.as_ref().unwrap().cache_state, "hit"); + assert_eq!(similarity_warm.scan.as_ref().unwrap().vector_rows_read, 0); } #[tokio::test] @@ -2710,10 +2695,10 @@ mod authority_tests { memory_service::projection_payload(&fixture.state, "", 2_000, &control).await; let similarity = memory_service::similarity_payload(&fixture.state, 0.5, 100, &control).await; - assert_eq!(projection["scan"]["cache_state"], "miss"); - assert_eq!(projection["points"].as_array().unwrap().len(), 3); - assert_eq!(similarity["scan"]["cache_state"], "miss"); - assert_eq!(similarity["count"], 3); + assert_eq!(projection.scan.as_ref().unwrap().cache_state, "miss"); + assert_eq!(projection.points.len(), 3); + assert_eq!(similarity.scan.as_ref().unwrap().cache_state, "miss"); + assert_eq!(similarity.count, 3); // The populated caches are owned by the state (and its clones), not by // the process: once the last handle to this store's dashboard state is @@ -2801,11 +2786,13 @@ mod authority_tests { .await; projection_cold.push(started.elapsed()); projection_cold_rows.push( - projection["scan"]["vector_rows_read"] - .as_u64() - .expect("projection cold row count"), + projection + .scan + .as_ref() + .expect("projection cold row count") + .vector_rows_read, ); - assert_eq!(projection["scan"]["cache_state"], "miss"); + assert_eq!(projection.scan.as_ref().unwrap().cache_state, "miss"); let started = Instant::now(); let (similarity, heartbeat) = if index == 0 { @@ -2827,11 +2814,13 @@ mod authority_tests { }; similarity_cold.push(started.elapsed()); similarity_cold_rows.push( - similarity["scan"]["vector_rows_read"] - .as_u64() - .expect("similarity cold row count"), + similarity + .scan + .as_ref() + .expect("similarity cold row count") + .vector_rows_read, ); - assert_eq!(similarity["scan"]["cache_state"], "miss"); + assert_eq!(similarity.scan.as_ref().unwrap().cache_state, "miss"); if heartbeat.is_some() { similarity_cold_heartbeat = heartbeat; } @@ -2854,11 +2843,13 @@ mod authority_tests { .await; projection_warm.push(started.elapsed()); projection_warm_rows.push( - projection["scan"]["vector_rows_read"] - .as_u64() - .expect("projection warm row count"), + projection + .scan + .as_ref() + .expect("projection warm row count") + .vector_rows_read, ); - assert_eq!(projection["scan"]["cache_state"], "hit"); + assert_eq!(projection.scan.as_ref().unwrap().cache_state, "hit"); } let ((similarity_warm, similarity_warm_rows), similarity_warm_heartbeat) = @@ -2879,11 +2870,13 @@ mod authority_tests { .await; timings.push(started.elapsed()); rows.push( - similarity["scan"]["vector_rows_read"] - .as_u64() - .expect("similarity warm row count"), + similarity + .scan + .as_ref() + .expect("similarity warm row count") + .vector_rows_read, ); - assert_eq!(similarity["scan"]["cache_state"], "hit"); + assert_eq!(similarity.scan.as_ref().unwrap().cache_state, "hit"); } (timings, rows) }) @@ -3439,7 +3432,6 @@ mod authority_tests { for tail in [ "doctor/findings", "storage/telemetry", - "storage/findings", "code-index/freshness", "feedback/status", ] { diff --git a/crates/tracedecay-dashboard-api/src/loom_api.rs b/crates/tracedecay-dashboard-api/src/loom_api.rs index a7bcbb31bf..6d44d12fde 100644 --- a/crates/tracedecay-dashboard-api/src/loom_api.rs +++ b/crates/tracedecay-dashboard-api/src/loom_api.rs @@ -1,7 +1,7 @@ //! Authorized Loom temporal projection over the retained project session store. //! //! The endpoint composes existing authorities; it does not collect new data. -//! `sessions`/`session_messages` provide thread bounds and +//! `sessions`/`lcm_raw_messages` provide thread bounds and //! `sessions.metadata_json` provides provider-native edited-file rollups. Git //! correlation is read through [`DashboardGitCorrelationReadPortV1`], the //! daemon-owned typed read over the verified session-git-evidence graph @@ -138,6 +138,15 @@ struct LoomSessionRowV1 { messages: i64, edited_files_recorded: bool, models: Vec, + /// The delegating session as the provider recorded it on `sessions`, the + /// same column the analytics subagent tree reads. Absent means no parent + /// was recorded; a branch is never inferred from time overlap. + #[serde(default, skip_serializing_if = "Option::is_none")] + parent_session_id: Option, + /// The tool invocation that spawned this session, when the provider + /// recorded one; it binds a fork to a specific call, not just a parent. + #[serde(default, skip_serializing_if = "Option::is_none")] + parent_tool_use_id: Option, } #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] @@ -161,6 +170,11 @@ struct LoomEditedFileV1 { path: String, change_type: Option, hunks: Option, + /// Unix microseconds of the edit when the provider rollup recorded + /// `edited_at_micros` as an integer. Absent is unrecorded; it is never + /// derived from the session bounds. + #[serde(default, skip_serializing_if = "Option::is_none")] + edited_at_micros: Option, } #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] @@ -293,13 +307,16 @@ async fn read_temporal( let total = query_count(conn, "SELECT COUNT(*) AS total FROM sessions", (), "total").await?; let session_sql = " SELECT s.provider, s.session_id, s.title, s.started_at, s.ended_at, - s.is_subagent, COUNT(m.message_id) AS messages, + s.is_subagent, + NULLIF(TRIM(s.parent_session_id), '') AS parent_session_id, + NULLIF(TRIM(s.parent_tool_use_id), '') AS parent_tool_use_id, + COUNT(m.message_id) AS messages, MAX(m.timestamp) AS last_message_at, CASE WHEN json_valid(s.metadata_json) AND json_type(s.metadata_json, '$.edited_files') = 'array' THEN 1 ELSE 0 END AS edited_files_recorded FROM sessions s - LEFT JOIN session_messages m + LEFT JOIN lcm_raw_messages m ON m.provider = s.provider AND m.session_id = s.session_id GROUP BY s.provider, s.session_id ORDER BY (s.started_at IS NULL), s.started_at DESC, s.rowid DESC @@ -310,7 +327,7 @@ async fn read_temporal( let model_sql = format!( "{PAGE_CTE} SELECT m.provider, m.session_id, m.model - FROM session_messages m + FROM lcm_raw_messages m JOIN page p ON p.provider = m.provider AND p.session_id = m.session_id WHERE m.model IS NOT NULL AND TRIM(m.model) != '' GROUP BY m.provider, m.session_id, m.model @@ -349,7 +366,10 @@ async fn read_temporal( SELECT p.provider, p.session_id, json_extract(file.value, '$.path') AS path, json_extract(file.value, '$.change_type') AS change_type, - json_extract(file.value, '$.hunks') AS hunks + json_extract(file.value, '$.hunks') AS hunks, + CASE WHEN json_type(file.value, '$.edited_at_micros') = 'integer' + THEN json_extract(file.value, '$.edited_at_micros') END + AS edited_at_micros FROM page p JOIN sessions s ON s.provider = p.provider AND s.session_id = p.session_id JOIN json_each( @@ -413,7 +433,9 @@ async fn read_temporal( rows: &edited_files, reason: Some( "edited-file coverage is provider-native metadata; sessions without an \ - edited_files array are omitted, never treated as no edits" + edited_files array are omitted, never treated as no edits; \ + edited_at_micros is served only where the rollup recorded it, \ + files without it have no known edit time" .to_string(), ), required_authority: None, diff --git a/crates/tracedecay-dashboard-api/src/memory_analysis.rs b/crates/tracedecay-dashboard-api/src/memory_analysis.rs index d2d4139bda..7e3d4abff5 100644 --- a/crates/tracedecay-dashboard-api/src/memory_analysis.rs +++ b/crates/tracedecay-dashboard-api/src/memory_analysis.rs @@ -5,7 +5,9 @@ mod pca; -use serde_json::{Value, json}; +use schemars::JsonSchema; +use serde::Serialize; +use serde_json::Value; use tracedecay_session_memory::memory::encoding::{HolographicEncoder, HolographicEncodingError}; use tracedecay_store::FactReadControl; @@ -129,6 +131,56 @@ fn round_bin_edge(edge: f64) -> f64 { (edge * 1e9).round() / 1e9 } +/// One fixed-width similarity histogram bin. +#[derive(Clone, Debug, Serialize, JsonSchema)] +pub struct MemoryScoreBinV1 { + pub start: f64, + pub end: f64, + pub count: u64, +} + +/// Similarity score distribution over every finite scored pair. Every +/// statistic is `None` when no finite pair was scored, never zero. +#[derive(Clone, Debug, Serialize, JsonSchema)] +pub struct MemoryScoreDistributionV1 { + pub bin_count: usize, + pub total_pairs: u64, + pub min_score: Option, + pub max_score: Option, + pub average_score: Option, + pub bins: Vec, +} + +impl MemoryScoreDistributionV1 { + fn observed( + min: f64, + max: f64, + total_pairs: u64, + sum: f64, + bins: Vec, + ) -> Self { + Self { + bin_count: bins.len(), + total_pairs, + min_score: Some(min), + max_score: Some(max), + average_score: Some(sum / total_pairs as f64), + bins, + } + } +} + +pub fn empty_score_distribution() -> MemoryScoreDistributionV1 { + MemoryScoreDistributionV1 { + bin_count: 0, + total_pairs: 0, + min_score: None, + max_score: None, + average_score: None, + bins: Vec::new(), + } +} + /// Fixed-width histogram over the observed `[min_score, max_score]` range of /// the computed pairs (adaptive, not a fixed `[-1, 1]` window, real HRR data /// clusters tightly and a fixed window collapses into one bin). A degenerate @@ -136,30 +188,17 @@ fn round_bin_edge(edge: f64) -> f64 { /// /// Two passes over the slice, no intermediate allocation: at n = 2000 facts /// the input is ~2M pairs, and a per-request copy would be ~16 MB. -pub fn empty_score_distribution() -> Value { - json!({ - "min": Value::Null, - "max": Value::Null, - "bin_count": 0, - "total_pairs": 0, - "min_score": Value::Null, - "max_score": Value::Null, - "average_score": Value::Null, - "bins": [], - }) -} - pub fn score_distribution( scored: &[(f64, usize, usize)], read_control: &FactReadControl, -) -> Result { +) -> Result { if read_control.interrupted() { return Err(MemoryAnalysisError::Interrupted); } let mut min_seen = f64::INFINITY; let mut max_seen = f64::NEG_INFINITY; let mut sum = 0.0_f64; - let mut total_pairs = 0_i64; + let mut total_pairs = 0_u64; for (score, _, _) in scored { if read_control.interrupted() { return Err(MemoryAnalysisError::Interrupted); @@ -179,19 +218,20 @@ pub fn score_distribution( let range = max_seen - min_seen; if range <= 0.0 { - return Ok(json!({ - "min": min_seen, - "max": max_seen, - "bin_count": 1, - "total_pairs": total_pairs, - "min_score": min_seen, - "max_score": max_seen, - "average_score": sum / total_pairs as f64, - "bins": [{ "start": min_seen, "end": max_seen, "count": total_pairs }], - })); + return Ok(MemoryScoreDistributionV1::observed( + min_seen, + max_seen, + total_pairs, + sum, + vec![MemoryScoreBinV1 { + start: min_seen, + end: max_seen, + count: total_pairs, + }], + )); } - let mut counts = vec![0_i64; SIMILARITY_DISTRIBUTION_BINS]; + let mut counts = vec![0_u64; SIMILARITY_DISTRIBUTION_BINS]; for (score, _, _) in scored { if read_control.interrupted() { return Err(MemoryAnalysisError::Interrupted); @@ -222,28 +262,23 @@ pub fn score_distribution( round_bin_edge(min_seen + idx as f64 * width) } }; - let bins: Vec = counts + let bins = counts .into_iter() .enumerate() - .map(|(idx, count)| { - json!({ - "start": edge(idx), - "end": edge(idx + 1), - "count": count, - }) + .map(|(idx, count)| MemoryScoreBinV1 { + start: edge(idx), + end: edge(idx + 1), + count, }) .collect(); - Ok(json!({ - "min": min_seen, - "max": max_seen, - "bin_count": SIMILARITY_DISTRIBUTION_BINS, - "total_pairs": total_pairs, - "min_score": min_seen, - "max_score": max_seen, - "average_score": sum / total_pairs as f64, - "bins": bins, - })) + Ok(MemoryScoreDistributionV1::observed( + min_seen, + max_seen, + total_pairs, + sum, + bins, + )) } /// One retained similarity pair with its lexical-overlap analysis, computed @@ -256,9 +291,6 @@ pub struct ScoredPair { /// Indices into [`SimilarityComputation::facts`]. pub a: usize, pub b: usize, - /// Lexical-overlap payload keys merged into the pair JSON - /// (`token_overlap`, `overlap_coefficient`, `shared_tokens`, …). - pub overlap: Value, pub classification: &'static str, } @@ -283,12 +315,11 @@ impl ScoredPair { .get("content") .and_then(Value::as_str) .ok_or(MemoryAnalysisError::MissingFactContent { index: b })?; - let (overlap, token_overlap, overlap_coefficient) = lexical_overlap(a_content, b_content); + let (_, token_overlap, overlap_coefficient) = lexical_overlap(a_content, b_content); Ok(Self { similarity, a, b, - overlap, classification: similarity_classification( similarity, token_overlap, @@ -318,7 +349,7 @@ pub struct SimilarityComputation { pub total_pairs: i64, /// [`score_distribution`] over all scored pairs, precomputed so requests /// never re-bin the full pair set. - pub distribution: Value, + pub distribution: MemoryScoreDistributionV1, } /// Finalizes a similarity computation from the full scored pair set: @@ -366,6 +397,7 @@ pub fn build_similarity_computation( #[cfg(test)] mod tests { use super::*; + use serde_json::json; use std::sync::Arc; use tracedecay_domain::{ FactId, FactIdentityMaterialV1, FactIdentitySourceV1, FactOwnerV1, ProvenanceId, @@ -451,23 +483,18 @@ mod tests { let scored = vec![(0.75, 0, 1), (0.0, 0, 2), (-0.25, 1, 2)]; let distribution = score_distribution(&scored, &read_control()) .expect("distribution must not be interrupted"); - assert_eq!(distribution["min"], -0.25); - assert_eq!(distribution["max"], 0.75); - assert_eq!(distribution["bin_count"], 20); - let bins = distribution["bins"] - .as_array() - .unwrap_or_else(|| panic!("expected distribution bins")); + assert_eq!(distribution.min_score, Some(-0.25)); + assert_eq!(distribution.max_score, Some(0.75)); + assert_eq!(distribution.bin_count, 20); + let bins = &distribution.bins; assert_eq!(bins.len(), 20); - assert_eq!(bins[0]["start"], -0.25); - assert_eq!(bins[19]["end"], 0.75); - assert_eq!(bins[0]["count"], 1, "min score lands in the first bin"); - assert_eq!(bins[19]["count"], 1, "max score lands in the last bin"); + assert_eq!(bins[0].start, -0.25); + assert_eq!(bins[19].end, 0.75); + assert_eq!(bins[0].count, 1, "min score lands in the first bin"); + assert_eq!(bins[19].count, 1, "max score lands in the last bin"); // Bin edges must be clean values, not float-accumulation noise. for bin in bins { - for key in ["start", "end"] { - let edge = bin[key] - .as_f64() - .unwrap_or_else(|| panic!("expected numeric bin edge")); + for edge in [bin.start, bin.end] { let rounded = (edge * 1e9).round() / 1e9; assert!( (edge - rounded).abs() < 1e-12, @@ -482,15 +509,13 @@ mod tests { let scored = vec![(0.5, 0, 1), (0.5, 0, 2), (0.5, 1, 2)]; let distribution = score_distribution(&scored, &read_control()) .expect("distribution must not be interrupted"); - assert_eq!(distribution["bin_count"], 1); - assert_eq!(distribution["total_pairs"], 3); - let bins = distribution["bins"] - .as_array() - .unwrap_or_else(|| panic!("expected distribution bins")); + assert_eq!(distribution.bin_count, 1); + assert_eq!(distribution.total_pairs, 3); + let bins = &distribution.bins; assert_eq!(bins.len(), 1); - assert_eq!(bins[0]["start"], 0.5); - assert_eq!(bins[0]["end"], 0.5); - assert_eq!(bins[0]["count"], 3); + assert_eq!(bins[0].start, 0.5); + assert_eq!(bins[0].end, 0.5); + assert_eq!(bins[0].count, 3); } #[test] @@ -539,7 +564,7 @@ mod tests { assert_eq!(computation.pairs.len(), cap); assert_eq!(computation.total_pairs, candidate_pairs as i64); - assert_eq!(computation.distribution["total_pairs"], candidate_pairs); + assert_eq!(computation.distribution.total_pairs, candidate_pairs as u64); assert!( computation .pairs diff --git a/crates/tracedecay-dashboard-api/src/memory_api.rs b/crates/tracedecay-dashboard-api/src/memory_api.rs index 06630f5004..0c3786eec8 100644 --- a/crates/tracedecay-dashboard-api/src/memory_api.rs +++ b/crates/tracedecay-dashboard-api/src/memory_api.rs @@ -7,14 +7,11 @@ use std::collections::BTreeMap; use axum::extract::State; use axum::http::StatusCode; -use axum::response::Json; +use axum::response::{IntoResponse, Json, Response}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -use serde_json::{Map, Value, json}; -use super::memory_analysis::{ - SIMILARITY_DEFAULT_THRESHOLD, SIMILARITY_PAIR_CAP, empty_score_distribution, -}; +use super::memory_analysis::{SIMILARITY_DEFAULT_THRESHOLD, SIMILARITY_PAIR_CAP}; use super::memory_service; use super::read_model::{ DashboardCoverageCompletenessV1, DashboardCoverageV1, DashboardDomainStateV1, @@ -116,6 +113,66 @@ pub(super) struct MemoryFactDetailPayloadV1 { error: String, } +#[derive(Clone, Copy, Debug, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +enum MemoryFeedbackActionV1 { + Helpful, + Unhelpful, +} + +/// How much of a feedback event this store can still account for. Redacted +/// detail was withheld; unknown detail was never recorded. +#[derive(Clone, Copy, Debug, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +enum MemoryFeedbackDetailsAvailabilityV1 { + Available, + Redacted, + Unknown, +} + +/// One append-only feedback event. `source` and `note` are absent when the +/// event carried none, which differs from an unknown value. +#[derive(Clone, Debug, Serialize, JsonSchema)] +struct MemoryTrustHistoryEventV1 { + event_id: String, + timestamp: i64, + action: MemoryFeedbackActionV1, + old_trust: f64, + new_trust: f64, + delta: f64, + details_availability: MemoryFeedbackDetailsAvailabilityV1, + #[serde(skip_serializing_if = "Option::is_none")] + source: Option, + #[serde(skip_serializing_if = "Option::is_none")] + note: Option, +} + +#[derive(Clone, Copy, Debug, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +enum MemoryTrustHistoryCompletenessV1 { + Complete, + Partial, +} + +#[derive(Clone, Debug, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +struct MemoryTrustHistoryCursorV1 { + occurred_at: i64, + event_id: String, +} + +/// `GET /api/plugins/holographic/fact/{fact_id}/trust-history`. `partial` +/// exactly when `next_after` names the continuation. +#[derive(Clone, Debug, Serialize, JsonSchema)] +pub(super) struct MemoryTrustHistoryPayloadV1 { + fact_id: String, + trust_history: Vec, + limit: usize, + completeness: MemoryTrustHistoryCompletenessV1, + next_after: Option, + error: String, +} + fn owned_fact_id(state: &DashboardState, raw: String) -> Result { let fact_id = FactId::new(raw).map_err(|error| error.to_string())?; fact_id @@ -214,7 +271,7 @@ async fn fact_trust_history_payload( state: &DashboardState, fact_id: FactId, read_control: &FactReadControl, -) -> Result, String> { +) -> Result, String> { let application = memory_application_for_db(state.memory_owner.clone(), &state.mem_db) .map_err(|error| error.to_string())?; let Some(_detail) = application @@ -229,62 +286,59 @@ async fn fact_trust_history_payload( .dashboard_feedback_history(fact_id.clone(), HISTORY_LIMIT, read_control) .await .map_err(|error| error.to_string())?; - let trust_history: Vec = history + let trust_history = history .events() .iter() - .map(|event| { - let action = match event.action() { - tracedecay_store::ProjectMemoryFactFeedbackActionV1::Helpful => "helpful", - tracedecay_store::ProjectMemoryFactFeedbackActionV1::Unhelpful => "unhelpful", - }; - let availability = match event.details_availability() { + .map(|event| MemoryTrustHistoryEventV1 { + event_id: event.event_id().as_str().to_owned(), + timestamp: event.occurred_at().0, + action: match event.action() { + tracedecay_store::ProjectMemoryFactFeedbackActionV1::Helpful => { + MemoryFeedbackActionV1::Helpful + } + tracedecay_store::ProjectMemoryFactFeedbackActionV1::Unhelpful => { + MemoryFeedbackActionV1::Unhelpful + } + }, + old_trust: event.old_trust().as_f64(), + new_trust: event.new_trust().as_f64(), + delta: event.new_trust().as_f64() - event.old_trust().as_f64(), + details_availability: match event.details_availability() { tracedecay_store::ProjectMemoryFactFeedbackDetailsAvailabilityV1::Available => { - "available" + MemoryFeedbackDetailsAvailabilityV1::Available } tracedecay_store::ProjectMemoryFactFeedbackDetailsAvailabilityV1::Redacted => { - "redacted" + MemoryFeedbackDetailsAvailabilityV1::Redacted } tracedecay_store::ProjectMemoryFactFeedbackDetailsAvailabilityV1::Unknown => { - "unknown" + MemoryFeedbackDetailsAvailabilityV1::Unknown } - }; - let mut row = Map::new(); - row.insert("event_id".into(), json!(event.event_id().as_str())); - row.insert("timestamp".into(), json!(event.occurred_at().0)); - row.insert("action".into(), json!(action)); - row.insert("old_trust".into(), json!(event.old_trust().as_f64())); - row.insert("new_trust".into(), json!(event.new_trust().as_f64())); - row.insert( - "delta".into(), - json!(event.new_trust().as_f64() - event.old_trust().as_f64()), - ); - row.insert("details_availability".into(), json!(availability)); - if let Some(source) = event.source() { - row.insert("source".into(), json!(source)); - } - if let Some(note) = event.note() { - row.insert("note".into(), json!(note)); - } - Value::Object(row) + }, + source: event.source().map(ToOwned::to_owned), + note: event.note().map(ToOwned::to_owned), }) .collect(); - let next_after = history.next_after().map(|cursor| { - json!({ - "occurred_at": cursor.occurred_at().0, - "event_id": cursor.event_id().as_str(), - }) - }); - Ok(Some(json!({ - "fact_id": fact_id.as_str(), - "trust_history": trust_history, - "limit": HISTORY_LIMIT, - "completeness": if next_after.is_some() { "partial" } else { "complete" }, - "next_after": next_after, - "error": "", - }))) + let next_after = history + .next_after() + .map(|cursor| MemoryTrustHistoryCursorV1 { + occurred_at: cursor.occurred_at().0, + event_id: cursor.event_id().as_str().to_owned(), + }); + Ok(Some(MemoryTrustHistoryPayloadV1 { + fact_id: fact_id.as_str().to_owned(), + trust_history, + limit: HISTORY_LIMIT, + completeness: if next_after.is_some() { + MemoryTrustHistoryCompletenessV1::Partial + } else { + MemoryTrustHistoryCompletenessV1::Complete + }, + next_after, + error: String::new(), + })) } -/// `GET /api/plugins/holographic/`, overview + facts + entities + graph. +/// `GET /api/plugins/holographic`, overview + facts + entities + graph. pub async fn overview( State(state): State, RequestControl(control): RequestControl, @@ -703,7 +757,7 @@ pub async fn fact_trust_history( State(state): State, RequestControl(control): RequestControl, JsonPath(fact_id): JsonPath, -) -> (StatusCode, Json) { +) -> Response { hotpath::future!( async move { let fact_id = match owned_fact_id(&state, fact_id) { @@ -712,27 +766,30 @@ pub async fn fact_trust_history( return ( StatusCode::BAD_REQUEST, Json(http_detail(&format!("invalid canonical fact id: {error}"))), - ); + ) + .into_response(); } }; let fact_id_label = fact_id.as_str().to_owned(); let result = fact_trust_history_payload(&state, fact_id, &fact_read_control(&control)).await; if let Some(state) = request_terminal_state(&control) { - return terminal_read_response(state); + return terminal_read_response(state).into_response(); } match result { - Ok(Some(payload)) => (StatusCode::OK, Json(payload)), + Ok(Some(payload)) => (StatusCode::OK, Json(payload)).into_response(), Ok(None) => ( StatusCode::NOT_FOUND, Json(http_detail(&format!("fact not found: {fact_id_label}"))), - ), + ) + .into_response(), Err(e) => ( StatusCode::INTERNAL_SERVER_ERROR, Json(http_detail(&format!( "Failed to load trust history for fact {fact_id_label}: {e}" ))), - ), + ) + .into_response(), } }, label = "dashboard_api.memory.trust_history" @@ -746,7 +803,7 @@ pub async fn projection( State(state): State, RequestControl(control): RequestControl, JsonQuery(params): JsonQuery, -) -> Json { +) -> Json { hotpath::future!( async move { let limit = coerce_limit(params.limit, 25, memory_service::projection_point_cap()); @@ -759,16 +816,11 @@ pub async fn projection( .await; if let Some(state) = request_terminal_state(&control) { let (code, error) = terminal_read_code(state); - return Json(json!({ - "exists": true, - "dim": 0, - "limit": limit, - "method": "none", - "points": [], - "state": state, - "code": code, - "error": error, - })); + return Json(memory_service::MemoryProjectionPayloadV1 { + state: Some(state), + code: Some(code.to_owned()), + ..memory_service::MemoryProjectionPayloadV1::empty(limit, error) + }); } Json(payload) }, @@ -783,7 +835,7 @@ pub async fn similarity( State(state): State, RequestControl(control): RequestControl, JsonQuery(params): JsonQuery, -) -> Json { +) -> Json { hotpath::future!( async move { let min_similarity = memory_service::coerce_similarity_score( @@ -800,19 +852,15 @@ pub async fn similarity( .await; if let Some(state) = request_terminal_state(&control) { let (code, error) = terminal_read_code(state); - return Json(json!({ - "exists": true, - "dim": 0, - "count": 0, - "limit": pair_cap, - "min_similarity": min_similarity, - "total_pairs": 0, - "score_distribution": empty_score_distribution(), - "pairs": [], - "state": state, - "code": code, - "error": error, - })); + return Json(memory_service::MemorySimilarityPayloadV1 { + state: Some(state), + code: Some(code.to_owned()), + ..memory_service::MemorySimilarityPayloadV1::empty( + pair_cap, + min_similarity, + error, + ) + }); } Json(payload) }, @@ -827,7 +875,7 @@ pub async fn oplog( State(state): State, RequestControl(control): RequestControl, JsonQuery(params): JsonQuery, -) -> Json { +) -> Json { hotpath::future!( async move { let limit = coerce_limit(params.limit, 50, 300); @@ -835,14 +883,11 @@ pub async fn oplog( memory_service::oplog_payload(&state, limit, &fact_read_control(&control)).await; if let Some(state) = request_terminal_state(&control) { let (code, error) = terminal_read_code(state); - return Json(json!({ - "events": [], - "count": 0, - "limit": limit, - "state": state, - "code": code, - "error": error, - })); + return Json(memory_service::MemoryOplogPayloadV1 { + state: Some(state), + code: Some(code.to_owned()), + ..memory_service::MemoryOplogPayloadV1::empty(limit, error) + }); } Json(payload) }, diff --git a/crates/tracedecay-dashboard-api/src/memory_service/mod.rs b/crates/tracedecay-dashboard-api/src/memory_service/mod.rs index 7420cfe4ab..1fcab0e781 100644 --- a/crates/tracedecay-dashboard-api/src/memory_service/mod.rs +++ b/crates/tracedecay-dashboard-api/src/memory_service/mod.rs @@ -11,7 +11,7 @@ pub use facts::{ fetch_facts, overview_payload, providers_payload, }; pub use graph::{MemoryGraphPayloadV1, graph_payload}; -pub use oplog::oplog_payload; +pub use oplog::{MemoryOplogPayloadV1, oplog_payload}; +pub use projection::{MemoryProjectionPayloadV1, projection_payload, projection_point_cap}; pub(crate) use projection::{ProjectionCacheRevision, ProjectionComputation}; -pub use projection::{projection_payload, projection_point_cap}; -pub use similarity::{coerce_similarity_score, similarity_payload}; +pub use similarity::{MemorySimilarityPayloadV1, coerce_similarity_score, similarity_payload}; diff --git a/crates/tracedecay-dashboard-api/src/memory_service/oplog.rs b/crates/tracedecay-dashboard-api/src/memory_service/oplog.rs index 4cdbd580d9..63049a071d 100644 --- a/crates/tracedecay-dashboard-api/src/memory_service/oplog.rs +++ b/crates/tracedecay-dashboard-api/src/memory_service/oplog.rs @@ -1,16 +1,55 @@ //! Memory oplog payload. -use serde_json::{Value, json}; +use schemars::JsonSchema; +use serde::Serialize; use super::super::DashboardState; +use crate::read_model::DashboardDomainStateV1; use crate::tracedecay::facts::memory_application_for_db; use tracedecay_store::FactReadControl; +/// One canonical lineage operation. Operations without a fact target carry no +/// `fact_id`; the route does not expose mutation detail. +#[derive(Clone, Debug, Serialize, JsonSchema)] +pub struct MemoryOplogEventV1 { + pub id: i64, + pub ts: i64, + pub op: String, + pub fact_id: Option, +} + +/// `GET /api/plugins/holographic/oplog`, newest first. +#[derive(Clone, Debug, Serialize, JsonSchema)] +pub struct MemoryOplogPayloadV1 { + pub events: Vec, + pub count: usize, + pub limit: i64, + /// Request lifecycle state when the read ended before a result. + #[serde(skip_serializing_if = "Option::is_none")] + pub state: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub code: Option, + pub error: String, +} + +impl MemoryOplogPayloadV1 { + pub fn empty(limit: i64, error: impl Into) -> Self { + Self { + events: Vec::new(), + count: 0, + limit, + state: None, + code: None, + error: error.into(), + } + } +} + pub async fn oplog_payload( state: &DashboardState, limit: i64, read_control: &FactReadControl, -) -> Value { +) -> MemoryOplogPayloadV1 { let bounded_limit = limit.clamp(1, 300) as usize; let result = match memory_application_for_db(state.memory_owner.clone(), &state.mem_db) { Ok(application) => application @@ -21,23 +60,24 @@ pub async fn oplog_payload( }; match result { Ok(entries) => { - let events: Vec = entries + let events: Vec<_> = entries .iter() - .map(|entry| { - json!({ - "id": entry.id, - "ts": entry.occurred_at.0, - "op": entry.operation, - "fact_id": entry - .fact - .as_ref() - .map(|fact| fact.fact_id().as_str()), - }) + .map(|entry| MemoryOplogEventV1 { + id: entry.id, + ts: entry.occurred_at.0, + op: entry.operation.clone(), + fact_id: entry + .fact + .as_ref() + .map(|fact| fact.fact_id().as_str().to_owned()), }) .collect(); - let count = events.len(); - json!({ "events": events, "count": count, "limit": limit, "error": "" }) + MemoryOplogPayloadV1 { + count: events.len(), + events, + ..MemoryOplogPayloadV1::empty(limit, "") + } } - Err(error) => json!({ "events": [], "count": 0, "limit": limit, "error": error }), + Err(error) => MemoryOplogPayloadV1::empty(limit, error), } } diff --git a/crates/tracedecay-dashboard-api/src/memory_service/projection.rs b/crates/tracedecay-dashboard-api/src/memory_service/projection.rs index fd5ed51cee..e32c776bd0 100644 --- a/crates/tracedecay-dashboard-api/src/memory_service/projection.rs +++ b/crates/tracedecay-dashboard-api/src/memory_service/projection.rs @@ -3,11 +3,15 @@ use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; -use serde_json::{Map, Value, json}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use tracedecay_domain::{FactId, PayloadAccessState}; use super::super::DashboardState; use super::super::memory_analysis::pca_scores; use super::facts::fact_summary_json; +use crate::read_model::DashboardDomainStateV1; use crate::snapshot_cache::DerivedSnapshotCacheState; use crate::tracedecay::facts::memory_application_for_db; use tracedecay_store::{ @@ -17,6 +21,128 @@ use tracedecay_store::{ pub(super) const PROJECTION_POINT_CAP: i64 = 2000; +/// Cache provenance of one derived (projection or similarity) read. +#[derive(Clone, Debug, Serialize, JsonSchema)] +pub struct MemoryDerivedScanV1 { + pub cache_scope: String, + pub cache_state: String, + /// Vector rows loaded for this response; zero on a cache hit. + pub vector_rows_read: usize, +} + +impl MemoryDerivedScanV1 { + pub(super) fn store_revision( + cache_state: DerivedSnapshotCacheState, + vector_rows_read: usize, + ) -> Self { + Self { + cache_scope: "store_revision".to_owned(), + cache_state: cache_state.as_str().to_owned(), + vector_rows_read, + } + } +} + +/// One eligible fact as the canonical dashboard fact summary projects it. +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct MemoryProjectedFactV1 { + pub fact_id: FactId, + pub payload_access: PayloadAccessState, + pub trust_score: f64, + pub retrieval_count: u64, + pub access_count: u64, + pub helpful_count: u64, + pub unhelpful_count: u64, + pub created_at: i64, + pub updated_at: i64, + pub projected_as_of: i64, + pub last_recalled_at: Option, + pub content: String, + pub category: String, + pub tags: Vec, + pub entities: Vec, + pub metadata: Value, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_label: Option, + pub entity_count: u64, +} + +/// One projected fact placed in the 2D phase projection. +#[derive(Clone, Debug, Serialize, JsonSchema)] +pub struct MemoryProjectionPointV1 { + #[serde(flatten)] + pub fact: MemoryProjectedFactV1, + pub x: f64, + pub y: f64, +} + +/// `pca` only when the decomposition succeeded over at least two equal-length +/// vectors; every other outcome is `none` and is not a semantic map. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum MemoryProjectionMethodV1 { + Pca, + None, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum MemoryProjectionCompletenessV1 { + Complete, + Bounded, + Unknown, +} + +#[derive(Clone, Debug, Serialize, JsonSchema)] +pub struct MemoryProjectionCoverageV1 { + pub completeness: MemoryProjectionCompletenessV1, + pub examined: usize, + pub limit: i64, + pub omission_reasons: Vec, +} + +/// `GET /api/plugins/holographic/projection`. +#[derive(Clone, Debug, Serialize, JsonSchema)] +pub struct MemoryProjectionPayloadV1 { + pub exists: bool, + pub dim: usize, + pub limit: i64, + pub method: MemoryProjectionMethodV1, + pub points: Vec, + pub coverage: MemoryProjectionCoverageV1, + #[serde(skip_serializing_if = "Option::is_none")] + pub scan: Option, + /// Request lifecycle state when the read ended before a result. + #[serde(skip_serializing_if = "Option::is_none")] + pub state: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub code: Option, + pub error: String, +} + +impl MemoryProjectionPayloadV1 { + pub fn empty(limit: i64, error: impl Into) -> Self { + Self { + exists: true, + dim: 0, + limit, + method: MemoryProjectionMethodV1::None, + points: Vec::new(), + coverage: MemoryProjectionCoverageV1 { + completeness: MemoryProjectionCompletenessV1::Unknown, + examined: 0, + limit, + omission_reasons: vec!["read_not_completed".to_owned()], + }, + scan: None, + state: None, + code: None, + error: error.into(), + } + } +} + pub fn projection_point_cap() -> i64 { PROJECTION_POINT_CAP } @@ -73,68 +199,24 @@ pub(super) fn vector_rows( /// One cached PCA projection of a store revision for a query/limit pair. pub(crate) struct ProjectionComputation { dim: usize, - method: &'static str, + method: MemoryProjectionMethodV1, error: &'static str, - points: Vec, + points: Vec, examined: usize, - point_limit: usize, coverage_complete: bool, } pub(crate) type ProjectionCacheRevision = (ProjectMemoryStoreRevisionV1, String, i64); -fn projection_point(meta: &Value, x: f64, y: f64) -> Result { - let mut point = meta.clone(); - let object = point - .as_object_mut() - .ok_or_else(|| "projection metadata was not an object".to_owned())?; - object - .get("fact_id") - .and_then(Value::as_str) - .ok_or_else(|| "projection metadata omitted its canonical fact ID".to_owned())?; - object - .get("payload_access") - .ok_or_else(|| "projection metadata omitted its payload-access state".to_owned())?; - object - .get("category") - .and_then(Value::as_str) - .ok_or_else(|| "projection metadata omitted its authoritative category".to_owned())?; - object - .get("trust_score") - .and_then(Value::as_f64) - .ok_or_else(|| "projection metadata omitted its authoritative trust score".to_owned())?; - object - .get("retrieval_count") - .and_then(Value::as_u64) - .ok_or_else(|| { - "projection metadata omitted its authoritative retrieval count".to_owned() - })?; - object - .get("created_at") - .and_then(Value::as_i64) - .ok_or_else(|| "projection metadata omitted its authoritative creation time".to_owned())?; - object - .get("updated_at") - .and_then(Value::as_i64) - .ok_or_else(|| "projection metadata omitted its authoritative update time".to_owned())?; - object - .get("metadata") - .ok_or_else(|| "projection metadata omitted authoritative fact metadata".to_owned())?; - object - .get("entity_count") - .and_then(Value::as_u64) - .ok_or_else(|| "projection metadata omitted its authoritative entity count".to_owned())?; - let content = object - .get("content") - .and_then(Value::as_str) - .ok_or_else(|| "projection metadata omitted authoritative fact content".to_owned())? - .chars() - .take(200) - .collect::(); - object.insert("content".into(), json!(content)); - object.insert("x".into(), json!((x * 1e6).round() / 1e6)); - object.insert("y".into(), json!((y * 1e6).round() / 1e6)); - Ok(point) +fn projection_point(meta: &Value, x: f64, y: f64) -> Result { + let mut fact = MemoryProjectedFactV1::deserialize(meta) + .map_err(|error| format!("projection metadata did not match its contract: {error}"))?; + fact.content = fact.content.chars().take(200).collect(); + Ok(MemoryProjectionPointV1 { + fact, + x: (x * 1e6).round() / 1e6, + y: (y * 1e6).round() / 1e6, + }) } fn compute_projection( @@ -159,11 +241,10 @@ fn compute_projection( .collect(); return Ok(ProjectionComputation { dim, - method: "none", + method: MemoryProjectionMethodV1::None, error: "", points, examined: rows.len(), - point_limit, coverage_complete: rows.len() < point_limit, }); } @@ -184,7 +265,7 @@ fn compute_projection( match pca_scores(&features, &read_control).map_err(|error| error.to_string())? { Some(scores) => Ok(ProjectionComputation { dim, - method: "pca", + method: MemoryProjectionMethodV1::Pca, error: "", points: rows .iter() @@ -192,16 +273,14 @@ fn compute_projection( .map(|((meta, _), s)| projection_point(meta, s[0], s[1])) .collect::, _>>()?, examined: rows.len(), - point_limit, coverage_complete: rows.len() < point_limit, }), None => Ok(ProjectionComputation { dim, - method: "none", + method: MemoryProjectionMethodV1::None, error: "projection failed", points: Vec::new(), examined: rows.len(), - point_limit, coverage_complete: rows.len() < point_limit, }), } @@ -212,48 +291,21 @@ pub async fn projection_payload( query: &str, limit: i64, read_control: &FactReadControl, -) -> Value { - let mut obj = Map::new(); - obj.insert("exists".into(), json!(true)); - obj.insert("dim".into(), json!(0)); - obj.insert("limit".into(), json!(limit)); - obj.insert("method".into(), json!("none")); - obj.insert("points".into(), json!([])); - obj.insert( - "coverage".into(), - json!({ - "completeness": "unknown", - "examined": 0, - "limit": limit, - "omission_reasons": ["read_not_completed"], - }), - ); - obj.insert("error".into(), json!("")); - +) -> MemoryProjectionPayloadV1 { if read_control.interrupted() { - obj.insert("error".into(), json!("memory projection interrupted")); - return Value::Object(obj); + return MemoryProjectionPayloadV1::empty(limit, "memory projection interrupted"); } let application = match memory_application_for_db(state.memory_owner.clone(), &state.mem_db) { Ok(application) => application, - Err(error) => { - obj.insert("error".into(), json!(error.to_string())); - return Value::Object(obj); - } + Err(error) => return MemoryProjectionPayloadV1::empty(limit, error.to_string()), }; let point_limit = match usize::try_from(limit.clamp(1, PROJECTION_POINT_CAP)) { Ok(limit) => limit, - Err(error) => { - obj.insert("error".into(), json!(error.to_string())); - return Value::Object(obj); - } + Err(error) => return MemoryProjectionPayloadV1::empty(limit, error.to_string()), }; let store_revision = match application.dashboard_store_revision(read_control).await { Ok(revision) => revision, - Err(error) => { - obj.insert("error".into(), json!(error.to_string())); - return Value::Object(obj); - } + Err(error) => return MemoryProjectionPayloadV1::empty(limit, error.to_string()), }; let normalized_query = query.trim().to_owned(); let revision = (store_revision, normalized_query.clone(), limit); @@ -288,53 +340,33 @@ pub async fn projection_payload( .await { Ok(cached) => cached, - Err(error) => { - obj.insert("error".into(), json!(error)); - return Value::Object(obj); - } + Err(error) => return MemoryProjectionPayloadV1::empty(limit, error), }; if read_control.interrupted() { - obj.insert("error".into(), json!("memory projection interrupted")); - return Value::Object(obj); + return MemoryProjectionPayloadV1::empty(limit, "memory projection interrupted"); + } + let (completeness, omission_reasons) = if computed.coverage_complete { + (MemoryProjectionCompletenessV1::Complete, Vec::new()) + } else { + ( + MemoryProjectionCompletenessV1::Bounded, + vec!["request_limit_reached".to_owned()], + ) + }; + MemoryProjectionPayloadV1 { + dim: computed.dim, + method: computed.method, + points: computed.points.clone(), + coverage: MemoryProjectionCoverageV1 { + completeness, + examined: computed.examined, + limit: limit.clamp(1, PROJECTION_POINT_CAP), + omission_reasons, + }, + scan: Some(MemoryDerivedScanV1::store_revision( + cache_state, + vector_rows_read.load(Ordering::Relaxed), + )), + ..MemoryProjectionPayloadV1::empty(limit, computed.error) } - projection_response( - &computed, - cache_state, - vector_rows_read.load(Ordering::Relaxed), - obj, - ) -} - -fn projection_response( - computation: &ProjectionComputation, - cache_state: DerivedSnapshotCacheState, - vector_rows_read: usize, - mut obj: Map, -) -> Value { - obj.insert( - "coverage".into(), - json!({ - "completeness": if computation.coverage_complete { "complete" } else { "bounded" }, - "examined": computation.examined, - "limit": computation.point_limit, - "omission_reasons": if computation.coverage_complete { - Vec::<&str>::new() - } else { - vec!["request_limit_reached"] - }, - }), - ); - obj.insert( - "scan".into(), - json!({ - "cache_scope": "store_revision", - "cache_state": cache_state.as_str(), - "vector_rows_read": vector_rows_read, - }), - ); - obj.insert("dim".into(), json!(computation.dim)); - obj.insert("method".into(), json!(computation.method)); - obj.insert("points".into(), json!(computation.points)); - obj.insert("error".into(), json!(computation.error)); - Value::Object(obj) } diff --git a/crates/tracedecay-dashboard-api/src/memory_service/similarity.rs b/crates/tracedecay-dashboard-api/src/memory_service/similarity.rs index 94df60c4f4..530f2d911c 100644 --- a/crates/tracedecay-dashboard-api/src/memory_service/similarity.rs +++ b/crates/tracedecay-dashboard-api/src/memory_service/similarity.rs @@ -3,19 +3,78 @@ use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; -use serde_json::{Map, Value, json}; +use schemars::JsonSchema; +use serde::Serialize; +use serde_json::Value; use tracedecay_store::FactReadControl; use super::super::DashboardState; use super::super::memory_analysis::{ - MemoryAnalysisError, SIMILARITY_FACT_CAP, SIMILARITY_PAIR_FLOOR, SIMILARITY_SCORE_MAX, - SIMILARITY_SCORE_MIN, SimilarityComputation, build_similarity_computation, - empty_score_distribution, score_similar_pairs, + MemoryAnalysisError, MemoryScoreDistributionV1, SIMILARITY_FACT_CAP, SIMILARITY_PAIR_FLOOR, + SIMILARITY_SCORE_MAX, SIMILARITY_SCORE_MIN, SimilarityComputation, + build_similarity_computation, empty_score_distribution, score_similar_pairs, }; -use super::projection::vector_rows; +use super::projection::{MemoryDerivedScanV1, vector_rows}; +use crate::read_model::DashboardDomainStateV1; use crate::snapshot_cache::DerivedSnapshotCacheState; use crate::tracedecay::facts::memory_application_for_db; +/// One scored fact pair above the requested similarity floor. +#[derive(Clone, Debug, Serialize, JsonSchema)] +pub struct MemorySimilarityPairV1 { + pub a_id: String, + pub b_id: String, + pub a_content: String, + pub b_content: String, + pub a_category: String, + pub b_category: String, + pub similarity: f64, + pub classification: String, +} + +/// `GET /api/plugins/holographic/similarity`. +/// +/// `count` is the number of vectored facts scored, `total_pairs` the finite +/// pairs scored before the floor and cap, and `pairs` what survived both. +#[derive(Clone, Debug, Serialize, JsonSchema)] +pub struct MemorySimilarityPayloadV1 { + pub exists: bool, + pub dim: usize, + pub count: usize, + pub limit: usize, + pub min_similarity: f64, + pub total_pairs: i64, + pub score_distribution: MemoryScoreDistributionV1, + pub pairs: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub scan: Option, + /// Request lifecycle state when the read ended before a result. + #[serde(skip_serializing_if = "Option::is_none")] + pub state: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub code: Option, + pub error: String, +} + +impl MemorySimilarityPayloadV1 { + pub fn empty(pair_cap: usize, min_similarity: f64, error: impl Into) -> Self { + Self { + exists: true, + dim: 0, + count: 0, + limit: pair_cap, + min_similarity, + total_pairs: 0, + score_distribution: empty_score_distribution(), + pairs: Vec::new(), + scan: None, + state: None, + code: None, + error: error.into(), + } + } +} + pub fn coerce_similarity_score(value: Option, default: f64) -> f64 { value .filter(|score| score.is_finite()) @@ -62,7 +121,7 @@ async fn similarity_computation( } else { score_similar_pairs(&decoded, SIMILARITY_PAIR_FLOOR, &blocking_control)? }; - let facts: Vec = decoded.into_iter().map(|(meta, _)| meta).collect(); + let facts = decoded.into_iter().map(|(meta, _)| meta).collect(); build_similarity_computation(dim, facts, scored, &blocking_control) }) }, @@ -89,43 +148,25 @@ pub async fn similarity_payload( min_similarity: f64, pair_cap: usize, read_control: &FactReadControl, -) -> Value { - let mut obj = Map::new(); - obj.insert("exists".into(), json!(true)); - obj.insert("dim".into(), json!(0)); - obj.insert("count".into(), json!(0)); - obj.insert("limit".into(), json!(pair_cap)); - obj.insert("min_similarity".into(), json!(min_similarity)); - obj.insert("total_pairs".into(), json!(0)); - obj.insert("score_distribution".into(), empty_score_distribution()); - obj.insert("pairs".into(), json!([])); - obj.insert("error".into(), json!("")); - +) -> MemorySimilarityPayloadV1 { let (computation, cache_state, vector_rows_read) = match similarity_computation(state, read_control).await { Ok(cached) => cached, - Err(e) => { - obj.insert("error".into(), json!(e)); - return Value::Object(obj); - } + Err(error) => return MemorySimilarityPayloadV1::empty(pair_cap, min_similarity, error), }; - obj.insert( - "scan".into(), - json!({ - "cache_scope": "store_revision", - "cache_state": cache_state.as_str(), - "vector_rows_read": vector_rows_read, - }), - ); - obj.insert("dim".into(), json!(computation.dim)); - obj.insert("count".into(), json!(computation.facts.len())); - obj.insert("total_pairs".into(), json!(computation.total_pairs)); - obj.insert( - "score_distribution".into(), - computation.distribution.clone(), - ); + let mut payload = MemorySimilarityPayloadV1 { + dim: computation.dim, + count: computation.facts.len(), + total_pairs: computation.total_pairs, + score_distribution: computation.distribution.clone(), + scan: Some(MemoryDerivedScanV1::store_revision( + cache_state, + vector_rows_read, + )), + ..MemorySimilarityPayloadV1::empty(pair_cap, min_similarity, "") + }; if computation.facts.len() < 2 || computation.dim == 0 { - return Value::Object(obj); + return payload; } let pairs = @@ -154,44 +195,30 @@ pub async fn similarity_payload( })?; let a_category = a .get("category") - .cloned() + .and_then(Value::as_str) .ok_or_else(|| "similarity left fact omitted its category".to_owned())?; let b_category = b .get("category") - .cloned() + .and_then(Value::as_str) .ok_or_else(|| "similarity right fact omitted its category".to_owned())?; - let mut pair = json!({ - "a_id": a_id, - "b_id": b_id, - "a_content": a_content.chars().take(200).collect::(), - "b_content": b_content.chars().take(200).collect::(), - "a_category": a_category, - "b_category": b_category, - "similarity": scored_pair.similarity, - "classification": scored_pair.classification, - }); - if let (Some(obj), Some(extra)) = - (pair.as_object_mut(), scored_pair.overlap.as_object()) - { - for (k, v) in extra { - obj.insert(k.clone(), v.clone()); - } - } - Ok::(pair) + Ok::<_, String>(MemorySimilarityPairV1 { + a_id: a_id.to_owned(), + b_id: b_id.to_owned(), + a_content: a_content.chars().take(200).collect(), + b_content: b_content.chars().take(200).collect(), + a_category: a_category.to_owned(), + b_category: b_category.to_owned(), + similarity: scored_pair.similarity, + classification: scored_pair.classification.to_owned(), + }) }) .collect::, _>>(); - let pairs = match pairs { - Ok(pairs) => pairs, - Err(error) => { - obj.insert("error".into(), json!(error)); - return Value::Object(obj); + match pairs { + Ok(_) if read_control.interrupted() => { + payload.error = "memory similarity interrupted".to_owned(); } - }; - if read_control.interrupted() { - obj.insert("pairs".into(), json!([])); - obj.insert("error".into(), json!("memory similarity interrupted")); - return Value::Object(obj); + Ok(pairs) => payload.pairs = pairs, + Err(error) => payload.error = error, } - obj.insert("pairs".into(), json!(pairs)); - Value::Object(obj) + payload } diff --git a/crates/tracedecay-dashboard-api/src/savings_api.rs b/crates/tracedecay-dashboard-api/src/savings_api.rs index ac4f022462..e43a9a0efc 100644 --- a/crates/tracedecay-dashboard-api/src/savings_api.rs +++ b/crates/tracedecay-dashboard-api/src/savings_api.rs @@ -4,12 +4,11 @@ //! //! - **Global accounting DB** (the registered profile store behind //! `tracedecay gain` / `tracedecay cost` / `tracedecay monitor`): the -//! `savings_ledger` and legacy lifetime savings counters. -//! Ledger aggregation reuses [`RegisteredGlobalDb::sum_savings`] / +//! `savings_ledger`. Ledger aggregation reuses [`RegisteredGlobalDb::sum_savings`] / //! [`RegisteredGlobalDb::savings_history`], the same queries `tracedecay gain` runs. //! - **Session store** (the resolved LCM store the dashboard already serves): //! canonical provider-usage observations plus `sessions` + -//! `session_messages`, whose content and model fields provide a separate +//! `lcm_raw_messages`, whose content and model fields provide a separate //! non-billing token-count overlay. //! //! Content token counts carry an explicit provenance label: @@ -25,8 +24,7 @@ //! Provider billing counters are exposed separately as provider-usage events; //! they are never treated as message counts. //! -//! Dollar costs and `/pricing` use one bundled, deterministic all-provider -//! authority. Unknown models keep their token counts but get no invented +//! Dollar costs use one bundled, deterministic all-provider authority. Unknown models keep their token counts but get no invented //! price. use std::collections::{BTreeMap, HashMap}; @@ -50,12 +48,10 @@ use super::read_model::{DashboardCoverageV1, DashboardEnvelopeV1, scope_from_sta use super::token_count::{ MESSAGE_TOKENS_CTE, MessageTokens, counting_available, encoder_for_model, }; -use super::util::{ - JsonQuery, coerce_limit, i64_field, query_i64, query_i64_result, query_rows, str_field, -}; +use super::util::{JsonQuery, i64_field, query_i64_result, query_rows, str_field}; use super::{DashboardState, savings_pricing, token_count}; use tracedecay_global_db::RegisteredGlobalDb; -use tracedecay_runtime_core::db::engine::{Value as DbValue, params, params_from_iter}; +use tracedecay_runtime_core::db::engine::params; /// Content-size aggregate shared by the per-session and per-model rollups. /// Provider billing usage is joined from the canonical observation projection, @@ -70,13 +66,6 @@ pub struct RangeParams { range: Option, } -#[derive(Deserialize)] -pub struct SessionsParams { - range: Option, - limit: Option, - offset: Option, -} - #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] struct SavingsSumV1 { saved_tokens: i64, @@ -91,21 +80,6 @@ struct SavingsLedgerSummaryV1 { all_time: SavingsSumV1, } -#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] -struct SavingsLifetimeProjectV1 { - path: Option, - tokens_saved: Option, -} - -#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] -struct SavingsLifetimeCountersV1 { - total_tokens_saved: i64, - project_total: i64, - projects_limit: i64, - projects_truncated: bool, - projects: Vec, -} - #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] struct SavingsAccountingSummaryV1 { available: bool, @@ -115,8 +89,6 @@ struct SavingsAccountingSummaryV1 { error: Option, #[serde(default)] ledger: Option, - #[serde(default)] - lifetime_counters: Option, } #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] @@ -213,49 +185,6 @@ fn provider_usage_scope(state: &DashboardState) -> Option { }) } -#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] -struct SavingsSessionModelV1 { - model: Option, - tokenizer: Option, - messages: i64, - provider_usage_events: i64, - tokenized_messages: i64, - estimated_messages: i64, - cost_basis: String, - provider_actual: Option, - tokenized: TokenPairV1, - estimated: TokenPairV1, -} - -#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] -struct SavingsSessionRowV1 { - provider: String, - session_id: String, - title: Option, - started_at: Option, - last_message_at: Option, - is_subagent: bool, - messages: i64, - provider_usage_events: i64, - tokenized_messages: i64, - estimated_messages: i64, - cost_basis: String, - models: Vec, -} - -#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] -pub(super) struct SavingsSessionsPayloadV1 { - available: bool, - db: String, - #[serde(default)] - scope: Option, - range: String, - #[serde(default)] - since: Option, - total: i64, - sessions: Vec, -} - /// One model-keyed content aggregate from the session store, joined to the /// exact provider usage recorded for that model. `model` is `None` for /// messages whose model was never recorded; that row keeps its token counts @@ -1029,14 +958,6 @@ pub async fn costs( .await } -// `costs_http` / `costs_export` are deleted with their last caller, for the -// same reason as their Observatory twins above `observatory_model`. They -// mounted `/api/plugins/savings/costs{,/export}` over the identical -// `costs_model` that `/api/costs`, the route `CanonicalCosts.tsx` reads, -// already serves. The savings family's OTHER routes (`overview`, `ledger`, -// `sessions`, `models`, `pricing`) are not aliases: each is the sole mount of -// its handler and has live consumers, so they stay. - async fn costs_model(state: &DashboardState) -> CostsReadModelV1 { let provider_scope = provider_usage_scope(state); match ( @@ -1060,7 +981,6 @@ async fn costs_model(state: &DashboardState) -> CostsReadModelV1 { } async fn savings_overview(gdb: &RegisteredGlobalDb, db_path: &str) -> Value { - const PROJECT_LIMIT: i64 = 25; let now = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap_or_default() @@ -1086,62 +1006,6 @@ async fn savings_overview(gdb: &RegisteredGlobalDb, db_path: &str) -> Value { } }; - // Legacy lifetime counters (`projects.tokens_saved`) predate the ledger - // and often carry history the event log does not, surface both. - let conn = gdb.read_connection(); - let lifetime_projects = match query_rows( - &conn, - "SELECT path, tokens_saved FROM projects - WHERE tokens_saved > 0 ORDER BY tokens_saved DESC LIMIT ?1", - params![PROJECT_LIMIT], - ) - .await - { - Ok(projects) => projects, - Err(error) => { - return json!({ - "available": false, - "db": db_path, - "recording": recording_block(), - "error": format!("failed to read lifetime project savings: {error}"), - }); - } - }; - let lifetime_total = match query_i64_result( - &conn, - "SELECT COALESCE(SUM(tokens_saved), 0) FROM projects", - (), - ) - .await - { - Ok(total) => total, - Err(error) => { - return json!({ - "available": false, - "db": db_path, - "recording": recording_block(), - "error": format!("failed to read lifetime savings total: {error}"), - }); - } - }; - let project_total = match query_i64_result( - &conn, - "SELECT COUNT(*) FROM projects WHERE tokens_saved > 0", - (), - ) - .await - { - Ok(total) => total, - Err(error) => { - return json!({ - "available": false, - "db": db_path, - "recording": recording_block(), - "error": format!("failed to read lifetime project count: {error}"), - }); - } - }; - let sum_json = |total: &tracedecay_global_db::SavingsTotal| json!({ "saved_tokens": total.saved_tokens, "calls": total.calls }); json!({ "available": true, @@ -1153,16 +1017,6 @@ async fn savings_overview(gdb: &RegisteredGlobalDb, db_path: &str) -> Value { "last_30d": sum_json(&month), "all_time": sum_json(&all_time), }, - "lifetime_counters": { - "total_tokens_saved": lifetime_total, - "project_total": project_total, - "projects_limit": PROJECT_LIMIT, - "projects_truncated": project_total > lifetime_projects.len() as i64, - "projects": lifetime_projects.iter().map(|row| json!({ - "path": str_field(row, "path"), - "tokens_saved": i64_field(row, "tokens_saved"), - })).collect::>(), - }, }) } @@ -1262,319 +1116,6 @@ fn provider_usage_overview(aggregate: &ProviderUsageAggregateV1) -> Value { }) } -/// GET `/api/plugins/savings/ledger?range=today|7d|30d|all` -pub async fn ledger( - State(state): State, - JsonQuery(params): JsonQuery, -) -> Json { - hotpath::future!( - async move { - let (range, since) = match range_since(params.range.as_deref()) { - Ok(range) => range, - Err(error) => return Json(read_failed_block(error)), - }; - let Some(gdb) = state.savings_db.as_deref() else { - return Json(json!({ - "available": false, - "db": state.savings_db_path, - "range": range, - })); - }; - - // The ledger route fails closed to its typed read_failed block: an - // unreadable ledger is not an empty ledger with zero totals. - let (total, history) = match async { - Ok::<_, String>(( - gdb.sum_savings(None, since).await?, - gdb.savings_history(None, since).await?, - )) - } - .await - { - Ok(read) => read, - Err(error) => { - return Json(merge( - json!({ "db": state.savings_db_path, "range": range, "since": since }), - read_failed_block(error), - )); - } - }; - let conn = gdb.read_connection(); - const SAVED_TOKENS_EXPR: &str = "COALESCE(SUM(CASE WHEN before_tokens > after_tokens THEN before_tokens - after_tokens ELSE 0 END), 0)"; - let by_tool = query_rows( - &conn, - &format!( - "SELECT tool_name, - {SAVED_TOKENS_EXPR} AS saved_tokens, - COUNT(*) AS calls - FROM savings_ledger WHERE ts >= ?1 - GROUP BY tool_name ORDER BY saved_tokens DESC LIMIT 50" - ), - params![since], - ) - .await - .unwrap_or_default(); - let by_project = query_rows( - &conn, - &format!( - "SELECT project_path, - {SAVED_TOKENS_EXPR} AS saved_tokens, - COUNT(*) AS calls - FROM savings_ledger WHERE ts >= ?1 - GROUP BY project_path ORDER BY saved_tokens DESC LIMIT 50" - ), - params![since], - ) - .await - .unwrap_or_default(); - - Json(json!({ - "available": true, - "db": state.savings_db_path, - "range": range, - "since": since, - "total": { "saved_tokens": total.saved_tokens, "calls": total.calls }, - "by_day": history.iter().map(|day| json!({ - "day": day.day, - "saved_tokens": day.saved_tokens, - "calls": day.calls, - })).collect::>(), - "by_tool": by_tool.iter().map(|row| json!({ - "tool": str_field(row, "tool_name"), - "saved_tokens": i64_field(row, "saved_tokens"), - "calls": i64_field(row, "calls"), - })).collect::>(), - "by_project": by_project.iter().map(|row| json!({ - "project": str_field(row, "project_path"), - "saved_tokens": i64_field(row, "saved_tokens"), - "calls": i64_field(row, "calls"), - })).collect::>(), - })) - - }, - label = "dashboard_api.savings.ledger" - ) - .await -} - -/// GET `/api/plugins/savings/sessions?range=&limit=&offset=` -/// -/// Sessions without any timestamp (neither `started_at` nor message -/// timestamps, true for Cursor hook ingests today) are only included in the -/// default `all` range, since they cannot be placed on a timeline. -pub async fn sessions( - State(state): State, - JsonQuery(params): JsonQuery, -) -> Response { - hotpath::future!( - async move { - let (range, since) = match range_since(params.range.as_deref()) { - Ok(range) => range, - Err(error) => { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - Json(read_failed_block(error)), - ) - .into_response(); - } - }; - let limit = coerce_limit(params.limit, 25, 100); - let offset = params.offset.unwrap_or(0).max(0); - let Some(db) = state.lcm_db.as_deref() else { - return match decode_contract::( - json!({ - "available": false, - "db": state.lcm_db_path, - "range": range, - "sessions": [], - "total": 0, - }), - "savings sessions", - ) { - Ok(payload) => Json(payload).into_response(), - Err(error) => ( - StatusCode::INTERNAL_SERVER_ERROR, - Json(json!({"status": "contract_invalid", "error": error})), - ) - .into_response(), - }; - }; - let conn = db.read_connection(); - - let page_sql = " - SELECT s.provider, s.session_id, s.title, s.started_at, s.ended_at, - s.is_subagent, - (SELECT MAX(m.timestamp) FROM session_messages m - WHERE m.provider = s.provider AND m.session_id = s.session_id) AS last_message_at - FROM sessions s - WHERE ?1 = 0 OR COALESCE(s.started_at, - (SELECT MAX(m.timestamp) FROM session_messages m - WHERE m.provider = s.provider AND m.session_id = s.session_id), 0) >= ?1 - ORDER BY (s.started_at IS NULL), s.started_at DESC, s.rowid DESC - LIMIT ?2 OFFSET ?3"; - let page = query_rows(&conn, page_sql, params![since, limit, offset]) - .await - .unwrap_or_default(); - let total = query_i64( - &conn, - "SELECT COUNT(*) FROM sessions s - WHERE ?1 = 0 OR COALESCE(s.started_at, - (SELECT MAX(m.timestamp) FROM session_messages m - WHERE m.provider = s.provider AND m.session_id = s.session_id), 0) >= ?1", - params![since], - ) - .await; - - let overlay = token_count::non_usage_message_tokens(&state).await; - let provider_scope = provider_usage_scope(&state); - let provider_usage = match (state.lcm_db.as_deref(), provider_scope.as_ref()) { - (Some(usage_db), Some(scope)) => { - Some(provider_usage_aggregate(usage_db, scope, None, None).await) - } - _ => None, - }; - let usage_deltas = provider_usage - .as_ref() - .filter(|usage| usage.coverage == ProviderUsageCoverageV1::Complete) - .map(|usage| usage.deltas.as_slice()); - let session_model_tiers = overlay.as_deref().map(|messages| { - fold_overlay(messages, |msg| { - Some(( - msg.provider.clone(), - msg.session_id.clone(), - msg.model.clone(), - )) - }) - }); - - // One grouped aggregate over the page's (provider, session_id) pairs, - // previously each page row ran its own aggregate query (N+1, up to 100 - // round-trips re-running the json_extract CTE per page render). The - // VALUES list joins as the outer loop so each pair stays an indexed - // probe of session_messages (a row-value `IN (VALUES …)` predicate does - // not get pushed into the index and full-scans instead). The global - // `messages DESC` order keeps each session's model rows descending after - // bucketing, matching the old per-session ORDER BY. - let mut model_rows_by_session: HashMap<(String, String), Vec> = HashMap::new(); - if !page.is_empty() { - let tuples = vec!["(?, ?)"; page.len()].join(", "); - let agg_sql = format!( - "SELECT provider, session_id, model, {TOKEN_AGG_COLUMNS} - FROM (VALUES {tuples}) pairs - JOIN ({MESSAGE_TOKENS_CTE}) ON provider = pairs.column1 - AND session_id = pairs.column2 - GROUP BY provider, session_id, model - ORDER BY messages DESC" - ); - let mut agg_params: Vec = Vec::with_capacity(page.len() * 2); - for row in &page { - agg_params.push(DbValue::Text(str_field(row, "provider").to_string())); - agg_params.push(DbValue::Text(str_field(row, "session_id").to_string())); - } - let rows = query_rows(&conn, &agg_sql, params_from_iter(agg_params)) - .await - .unwrap_or_default(); - for row in rows { - let key = ( - str_field(&row, "provider").to_string(), - str_field(&row, "session_id").to_string(), - ); - model_rows_by_session.entry(key).or_default().push(row); - } - } - - let mut sessions_json = Vec::with_capacity(page.len()); - for row in &page { - let provider = str_field(row, "provider"); - let session_id = str_field(row, "session_id"); - let model_rows = model_rows_by_session - .remove(&(provider.to_string(), session_id.to_string())) - .unwrap_or_default(); - - let mut messages = 0; - let mut provider_usage_events = 0; - let mut tokenized_messages = 0; - let mut estimated_messages = 0; - let models: Vec = model_rows - .iter() - .map(|model_row| { - let model = str_field(model_row, "model"); - let tiers = session_model_tiers.as_ref().and_then(|map| { - map.get(&( - provider.to_string(), - session_id.to_string(), - model.to_string(), - )) - }); - let mut block = token_block(model_row, tiers); - let (event_count, actual) = usage_deltas.map_or((0, None), |deltas| { - actual_for_deltas(deltas.iter().filter(|delta| { - delta.provider == provider - && delta.session_id == session_id - && delta.model.as_deref().unwrap_or_default() == model - && (since == 0 - || delta - .native_timestamp - .is_some_and(|timestamp| timestamp >= since)) - })) - }); - apply_provider_actual(&mut block, event_count, actual); - messages += i64_field(&block, "messages"); - provider_usage_events += i64_field(&block, "provider_usage_events"); - tokenized_messages += i64_field(&block, "tokenized_messages"); - estimated_messages += i64_field(&block, "estimated_messages"); - merge( - block, - json!({ - "model": model_value(model), - "tokenizer": tokenizer_block(model), - }), - ) - }) - .collect(); - - sessions_json.push(json!({ - "provider": provider, - "session_id": session_id, - "title": row.get("title").cloned().unwrap_or(Value::Null), - "started_at": row.get("started_at").cloned().unwrap_or(Value::Null), - "last_message_at": row.get("last_message_at").cloned().unwrap_or(Value::Null), - "is_subagent": i64_field(row, "is_subagent") != 0, - "messages": messages, - "provider_usage_events": provider_usage_events, - "tokenized_messages": tokenized_messages, - "estimated_messages": estimated_messages, - "cost_basis": basis_label(tokenized_messages, messages), - "models": models, - })); - } - - match decode_contract::( - json!({ - "available": true, - "db": state.lcm_db_path, - "scope": state.lcm_scope, - "range": range, - "since": since, - "total": total, - "sessions": sessions_json, - }), - "savings sessions", - ) { - Ok(payload) => Json(payload).into_response(), - Err(error) => ( - StatusCode::INTERNAL_SERVER_ERROR, - Json(json!({"status": "contract_invalid", "error": error})), - ) - .into_response(), - } - - }, - label = "dashboard_api.savings.sessions" - ) - .await -} - /// The typed `/models` failure body: the request could not be served and no /// row of it is invented. `range` echoes what was asked for. fn models_read_failed(range: Option<&str>, error: String) -> SavingsModelsPayloadV1 { @@ -1770,13 +1311,6 @@ pub async fn models( .into_response(), } } - -/// GET `/api/plugins/savings/pricing`, deterministic bundled all-provider -/// prices with content-addressed provenance. -pub async fn pricing() -> Json { - Json(savings_pricing::pricing_payload()) -} - #[cfg(test)] mod tests { use super::*; @@ -1975,8 +1509,6 @@ mod tests { fn tier_sums_attribute_roles_like_sql() { let mut sums = TierSums::default(); let msg = |role: &str, tokens: i64, tokenized: bool| MessageTokens { - provider: "cursor".into(), - session_id: "s".into(), model: "gpt-5".into(), role: role.into(), timestamp: None, diff --git a/crates/tracedecay-dashboard-api/src/settings_api.rs b/crates/tracedecay-dashboard-api/src/settings_api.rs index 4e64262c3c..0cbe4d3846 100644 --- a/crates/tracedecay-dashboard-api/src/settings_api.rs +++ b/crates/tracedecay-dashboard-api/src/settings_api.rs @@ -149,9 +149,6 @@ pub struct ProjectSettingsPatchResponseV1 { #[derive(Clone, Debug, JsonSchema, Serialize)] struct ProjectSettingsPayloadV1 { - config_path: String, - legacy_config_path: String, - legacy_config_read_only: bool, configuration_snapshot_id: String, configuration_revision_id: String, config: ProjectEditableSettingsV1, @@ -211,8 +208,6 @@ struct SyncSettingsV1 { #[derive(Clone, Debug, JsonSchema, Serialize)] struct UserSettingsPayloadV1 { - legacy_config_path: String, - legacy_config_read_only: bool, configuration_snapshot_id: String, configuration_revision_id: String, /// Independent ProfileSessions revision for the code-index worker @@ -604,7 +599,6 @@ async fn settings_envelope( { let project_configuration = crate::config::cached_runtime_configuration(&state.project_root) .map_err(|_| configuration_authority_unavailable_error())?; - let legacy_config_path = state.config_path.clone(); let user = state .user_settings .read() @@ -623,9 +617,6 @@ async fn settings_envelope( let automation = automation_settings_payload(&project_configuration); let payload = SettingsPayloadV1 { project: ProjectSettingsPayloadV1 { - config_path: legacy_config_path.display().to_string(), - legacy_config_path: legacy_config_path.display().to_string(), - legacy_config_read_only: true, configuration_snapshot_id: project_configuration .snapshot() .snapshot_id @@ -697,8 +688,6 @@ fn user_settings_payload( worker_configuration: &DashboardCodeIndexWorkerConfigurationV1, ) -> UserSettingsPayloadV1 { UserSettingsPayloadV1 { - legacy_config_path: user.legacy_config_path.clone(), - legacy_config_read_only: true, configuration_snapshot_id: user.configuration_snapshot_id.clone(), configuration_revision_id: user.configuration_revision_id.clone(), code_index_worker_configuration_snapshot_id: worker_configuration diff --git a/crates/tracedecay-dashboard-api/src/storage_findings_api.rs b/crates/tracedecay-dashboard-api/src/storage_findings_api.rs deleted file mode 100644 index 1dd19d9e23..0000000000 --- a/crates/tracedecay-dashboard-api/src/storage_findings_api.rs +++ /dev/null @@ -1,233 +0,0 @@ -//! `GET /api/storage/findings`, compatibility projection of the canonical -//! Doctor storage family. -//! -//! The admitted daemon Doctor reader owns finding production and health -//! composition. This route only selects the storage family and projects typed -//! producer status from that same report. It never invokes a finding producer, -//! consults a dashboard-held telemetry authority, or derives a health verdict. - -use axum::Json; -use axum::extract::State; -use schemars::JsonSchema; -use serde::Serialize; -use tracedecay_contracts::doctor::{ - DoctorCoverageCompletenessV1, DoctorEvidenceStateV1, DoctorFamilyConsultationV1, - DoctorFamilyUnavailableReasonV1, DoctorFindingFamilyV1, DoctorStorageFindingKindV1, -}; - -use super::DashboardState; -use super::doctor_findings_api::DoctorFindingsPayloadV1; -use super::read_model::DashboardEnvelopeV1; - -const STORAGE_KINDS: [DoctorStorageFindingKindV1; 6] = [ - DoctorStorageFindingKindV1::OverBudgetStore, - DoctorStorageFindingKindV1::OrphanStore, - DoctorStorageFindingKindV1::IncidentDebrisPresent, - DoctorStorageFindingKindV1::RetentionBacklog, - DoctorStorageFindingKindV1::TableGrowth, - DoctorStorageFindingKindV1::PendingSchemaMigration, -]; - -/// Whether one storage finding producer had enough source evidence to report -/// a real result. This is source coverage, not a health grade: `Real` can -/// describe a clean observation or a problem finding. -#[derive(Clone, Copy, Debug, Serialize, JsonSchema, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum StorageFindingSourceStateV1 { - Real, - Partial, - Unsupported, -} - -/// Source-coverage status for one typed storage finding producer. -#[derive(Clone, Debug, Serialize, JsonSchema, PartialEq, Eq)] -pub struct StorageFindingKindStatusV1 { - pub kind: DoctorStorageFindingKindV1, - pub state: StorageFindingSourceStateV1, - pub observed_entries: usize, - pub reason: String, -} - -/// Route-specific payload for `/api/storage/findings`. -/// -/// Storage producer coverage is required here rather than an optional field on -/// the general Doctor payload, so generated consumers cannot mistake one route -/// for the other. -#[derive(Clone, Debug, Serialize, JsonSchema)] -pub struct StorageFindingsPayloadV1 { - #[serde(flatten)] - pub findings: DoctorFindingsPayloadV1, - pub kind_statuses: Vec, -} - -/// `GET /api/storage/findings` -#[hotpath::measure(label = "dashboard_api.storage.findings", future = true)] -pub async fn findings( - State(state): State, -) -> Json> { - let envelope = super::doctor_findings_api::findings_for_family( - state, - Some(DoctorFindingFamilyV1::Storage), - ) - .await; - let kind_statuses = storage_kind_statuses(&envelope.payload); - Json(envelope.map_payload(|findings| StorageFindingsPayloadV1 { - findings, - kind_statuses, - })) -} - -fn storage_kind_statuses(payload: &DoctorFindingsPayloadV1) -> Vec { - STORAGE_KINDS - .into_iter() - .map(|kind| canonical_kind_status(payload, kind)) - .collect() -} - -fn canonical_kind_status( - payload: &DoctorFindingsPayloadV1, - kind: DoctorStorageFindingKindV1, -) -> StorageFindingKindStatusV1 { - let consultation = payload.report_coverage.as_ref().and_then(|coverage| { - coverage - .families() - .iter() - .find(|family| family.family() == DoctorFindingFamilyV1::Storage) - .map(tracedecay_contracts::DoctorFamilyCoverageV1::consultation) - }); - let matching = payload - .entries - .iter() - .filter(|entry| entry.storage_kind() == Some(kind)) - .collect::>(); - if !matching.is_empty() { - let complete_observations = consultation == Some(DoctorFamilyConsultationV1::Consulted) - && matching.iter().all(|entry| { - entry.finding().coverage().completeness() == DoctorCoverageCompletenessV1::Complete - && matches!( - entry.finding().state(), - DoctorEvidenceStateV1::Stale - | DoctorEvidenceStateV1::Degraded - | DoctorEvidenceStateV1::HealthyCompleteCoverage - ) - }); - let state = if complete_observations { - StorageFindingSourceStateV1::Real - } else { - StorageFindingSourceStateV1::Partial - }; - let reason = if complete_observations { - format!( - "canonical Doctor producer returned {} observed {}", - matching.len(), - if matching.len() == 1 { - "entry with complete coverage" - } else { - "entries with complete coverage" - } - ) - } else if let Some(DoctorFamilyConsultationV1::Unavailable { reason }) = consultation { - format!( - "canonical Doctor producer returned {} observed entries, but storage family coverage is incomplete ({})", - matching.len(), - unavailable_reason(reason) - ) - } else { - format!( - "canonical Doctor producer returned {} entries, but coverage or evidence state was incomplete", - matching.len() - ) - }; - return StorageFindingKindStatusV1 { - kind, - state, - observed_entries: matching.len(), - reason, - }; - } - - let (state, reason) = match consultation { - Some(DoctorFamilyConsultationV1::Consulted) => ( - StorageFindingSourceStateV1::Partial, - "the storage family was consulted, but the canonical report returned no typed entry for this producer; absence does not prove clean per-producer coverage" - .to_string(), - ), - Some(DoctorFamilyConsultationV1::Unavailable { - reason: - reason @ (DoctorFamilyUnavailableReasonV1::Unwired - | DoctorFamilyUnavailableReasonV1::Unsupported), - }) => ( - StorageFindingSourceStateV1::Unsupported, - format!( - "canonical Doctor storage source is unavailable ({})", - unavailable_reason(reason) - ), - ), - Some(DoctorFamilyConsultationV1::Unavailable { reason }) => ( - StorageFindingSourceStateV1::Partial, - format!( - "canonical Doctor storage source is unavailable ({}); no clean result is asserted", - unavailable_reason(reason) - ), - ), - None => ( - StorageFindingSourceStateV1::Unsupported, - format!( - "canonical Doctor storage source supplied no consultation record: {}", - payload.note - ), - ), - }; - StorageFindingKindStatusV1 { - kind, - state, - observed_entries: 0, - reason, - } -} - -const fn unavailable_reason(reason: DoctorFamilyUnavailableReasonV1) -> &'static str { - match reason { - DoctorFamilyUnavailableReasonV1::Unwired => "unwired", - DoctorFamilyUnavailableReasonV1::Unsupported => "unsupported", - DoctorFamilyUnavailableReasonV1::Absent => "absent", - DoctorFamilyUnavailableReasonV1::Denied => "denied", - DoctorFamilyUnavailableReasonV1::Unknown => "unknown", - DoctorFamilyUnavailableReasonV1::Unavailable => "unavailable", - DoctorFamilyUnavailableReasonV1::ResetRequired => "reset_required", - DoctorFamilyUnavailableReasonV1::Corrupt => "corrupt", - } -} - -#[cfg(test)] -#[allow(clippy::unwrap_used, clippy::expect_used)] -mod tests { - use super::*; - - #[tokio::test] - async fn route_without_admitted_reader_projects_all_kinds_as_unsupported() { - let _pin = tracedecay_runtime_core::config::PinnedUserDataDir::new(); - let (_project, state) = - crate::events_api::dashboard_state_fixture("project.dashboard-storage-findings").await; - - let Json(envelope) = findings(State(state)).await; - - assert_eq!( - envelope.payload.findings.family_filter, - Some(DoctorFindingFamilyV1::Storage) - ); - assert_eq!( - envelope.domain_state, - super::super::read_model::DashboardDomainStateV1::Unsupported - ); - assert!(envelope.payload.findings.entries.is_empty()); - let statuses = &envelope.payload.kind_statuses; - assert_eq!(statuses.len(), STORAGE_KINDS.len()); - assert!( - statuses - .iter() - .all(|status| status.state == StorageFindingSourceStateV1::Unsupported), - "dashboard-held telemetry must not override the canonical unavailable report" - ); - } -} diff --git a/crates/tracedecay-dashboard-api/src/token_count.rs b/crates/tracedecay-dashboard-api/src/token_count.rs index 7c63d00d0b..21ec2e580c 100644 --- a/crates/tracedecay-dashboard-api/src/token_count.rs +++ b/crates/tracedecay-dashboard-api/src/token_count.rs @@ -45,9 +45,9 @@ pub(super) const MESSAGE_TOKENS_CTE: &str = " role, timestamp, TRIM(COALESCE(model, '')) AS model, - LENGTH(COALESCE(text, '')) AS msg_len, - (LENGTH(COALESCE(text, '')) + 3) / 4 AS est_tokens - FROM session_messages + LENGTH(COALESCE(content, placeholder_text, '')) AS msg_len, + (LENGTH(COALESCE(content, placeholder_text, '')) + 3) / 4 AS est_tokens + FROM lcm_raw_messages WHERE kind IS NULL OR kind NOT IN ('summary', 'tool_event', 'hook_event', 'reasoning')"; /// Which BPE vocabulary a model id maps to, and whether the resulting count @@ -165,10 +165,10 @@ fn displayed_message_cache() -> DisplayedMessageCache { CLruCache::new(DISPLAYED_MESSAGE_CACHE_CAPACITY) } -/// Cached non-usage overlay plus the `session_messages` fingerprint it was +/// Cached non-usage overlay plus the `lcm_raw_messages` fingerprint it was /// built from. struct OverlayCache { - /// Cheap aggregate fingerprint of `session_messages` at build time: + /// Cheap aggregate fingerprint of `lcm_raw_messages` at build time: /// `(COUNT(*), MAX(rowid))`. Provider-accounting metadata is deliberately /// excluded because it is not content-token evidence. fingerprint: OverlayFingerprint, @@ -182,7 +182,7 @@ pub struct TokenCountCache { map: Mutex>, /// Last built non-usage overlay; `/overview`, `/sessions`, and `/models` /// all need it, so without this every Savings-tab interaction re-ran the - /// full `session_messages` scan + fold three times. + /// full `lcm_raw_messages` scan + fold three times. overlay: tokio::sync::Mutex>, /// Displayed-content counts for the LCM render path, keyed by provider /// then message id and guarded by a content fingerprint. Bounded LRU @@ -263,8 +263,6 @@ struct ComputedTokenCount { /// best-available token count. #[derive(Debug, Clone)] pub struct MessageTokens { - pub provider: String, - pub session_id: String, /// Normalized like the SQL CTE: `""` when no model id was recorded. pub model: String, pub role: String, @@ -280,7 +278,7 @@ pub struct MessageTokens { /// session store is being served (callers fall back to the SQL estimates). /// /// The result is cached on [`TokenCountCache`] keyed by a cheap -/// `(COUNT(*), MAX(rowid))` fingerprint of `session_messages`; the cache +/// `(COUNT(*), MAX(rowid))` fingerprint of `lcm_raw_messages`; the cache /// lock is held across a rebuild so the three savings endpoints firing /// concurrently share one scan instead of racing three. pub async fn non_usage_message_tokens(state: &DashboardState) -> Option>> { @@ -303,12 +301,12 @@ pub async fn non_usage_message_tokens(state: &DashboardState) -> Option Option { let rows = query_rows( conn, "SELECT COUNT(*) AS n, COALESCE(MAX(rowid), 0) AS max_rowid - FROM session_messages", + FROM lcm_raw_messages", (), ) .await @@ -326,7 +324,7 @@ async fn build_overlay( ) -> Option> { // Metadata only, text never leaves SQLite unless a count is missing. let sql = format!( - "SELECT provider, message_id, session_id, role, timestamp, model, msg_len + "SELECT provider, message_id, role, timestamp, model, msg_len FROM ({MESSAGE_TOKENS_CTE})" ); let rows = query_rows(conn, &sql, ()).await.ok()?; @@ -371,8 +369,6 @@ async fn build_overlay( .get(&(provider.to_owned(), message_id.to_owned())) .filter(|c| c.text_len == len); MessageTokens { - provider: provider.to_owned(), - session_id: str_field(row, "session_id").to_owned(), model: str_field(row, "model").to_owned(), role: str_field(row, "role").to_owned(), timestamp: row.get("timestamp").and_then(Value::as_i64), @@ -406,8 +402,8 @@ async fn count_and_store( { let placeholders = build_qmark_placeholders(chunk.len()); let sql = format!( - "SELECT provider, message_id, COALESCE(text, '') AS text - FROM session_messages WHERE provider = ? AND message_id IN ({placeholders})" + "SELECT provider, message_id, COALESCE(content, placeholder_text, '') AS text + FROM lcm_raw_messages WHERE provider = ? AND message_id IN ({placeholders})" ); let mut params: Vec = Vec::with_capacity(chunk.len() + 1); params.push(DbValue::Text(chunk[0].0.clone())); @@ -658,27 +654,28 @@ mod tests { let (_dir, conn) = test_conn(); if let Err(err) = conn .execute_batch( - "CREATE TABLE session_messages ( + "CREATE TABLE lcm_raw_messages ( provider TEXT NOT NULL, message_id TEXT NOT NULL, session_id TEXT NOT NULL, role TEXT NOT NULL, timestamp INTEGER, ordinal INTEGER NOT NULL, - text TEXT NOT NULL, + content TEXT, + placeholder_text TEXT, kind TEXT, model TEXT, metadata_json TEXT, - PRIMARY KEY(provider, message_id) + UNIQUE(provider, message_id) ); - INSERT INTO session_messages - (provider, message_id, session_id, role, timestamp, ordinal, text, kind, model, metadata_json) + INSERT INTO lcm_raw_messages + (provider, message_id, session_id, role, timestamp, ordinal, content, kind, model, metadata_json) VALUES ('codex', 'm1', 's1', 'assistant', 1, 1, 'hello', NULL, 'gpt-5', NULL);", ) .await { - panic!("failed to seed session_messages: {err}"); + panic!("failed to seed lcm_raw_messages: {err}"); } let Some(before) = overlay_fingerprint(&*conn).await else { @@ -686,7 +683,7 @@ mod tests { }; if let Err(err) = conn .execute( - "UPDATE session_messages + "UPDATE lcm_raw_messages SET metadata_json = '{\"usage\":{\"input_tokens\":1,\"output_tokens\":2}}' WHERE provider = 'codex' AND message_id = 'm1'", (), @@ -711,7 +708,7 @@ mod tests { }; if let Err(err) = conn .execute( - "UPDATE session_messages + "UPDATE lcm_raw_messages SET metadata_json = '{\"usage\":{\"input_tokens\":9,\"output_tokens\":8}}' WHERE provider = 'codex' AND message_id = 'm1'", (), @@ -761,21 +758,22 @@ mod tests { async fn derived_kinds_are_excluded_from_token_cte() { let (_dir, conn) = test_conn(); conn.execute_batch( - "CREATE TABLE session_messages ( + "CREATE TABLE lcm_raw_messages ( provider TEXT NOT NULL, message_id TEXT NOT NULL, session_id TEXT NOT NULL, role TEXT NOT NULL, timestamp INTEGER, ordinal INTEGER NOT NULL, - text TEXT NOT NULL, + content TEXT, + placeholder_text TEXT, kind TEXT, model TEXT, metadata_json TEXT, - PRIMARY KEY(provider, message_id) + UNIQUE(provider, message_id) ); - INSERT INTO session_messages - (provider, message_id, session_id, role, timestamp, ordinal, text, kind, model, metadata_json) + INSERT INTO lcm_raw_messages + (provider, message_id, session_id, role, timestamp, ordinal, content, kind, model, metadata_json) VALUES ('codex', 'm1', 's1', 'assistant', 1, 1, 'kept', NULL, 'gpt-5', NULL), ('codex', 'm2', 's1', 'assistant', 2, 2, 'sum', 'summary', 'gpt-5', NULL), @@ -784,7 +782,7 @@ mod tests { ('codex', 'm5', 's1', 'assistant', 5, 5, 're', 'reasoning', 'gpt-5', NULL);", ) .await - .expect("seed session_messages"); + .expect("seed lcm_raw_messages"); let sql = format!("SELECT COUNT(*) FROM ({MESSAGE_TOKENS_CTE})"); let mut rows = conn.query(&sql, ()).await.expect("run token CTE count"); diff --git a/crates/tracedecay-dashboard-api/src/tracedecay.rs b/crates/tracedecay-dashboard-api/src/tracedecay.rs index 282efa79d9..e7294d1b5c 100644 --- a/crates/tracedecay-dashboard-api/src/tracedecay.rs +++ b/crates/tracedecay-dashboard-api/src/tracedecay.rs @@ -4,7 +4,6 @@ use std::path::PathBuf; use std::sync::Arc; use tracedecay_automation_runtime::automation::host_io::HostIo; -pub use tracedecay_code_index::is_test_file; use tracedecay_configuration::UserSettingsDaemonClient; use tracedecay_runtime_core::db::Database; use tracedecay_runtime_core::storage::StoreLayout; diff --git a/crates/tracedecay-dashboard-api/src/util.rs b/crates/tracedecay-dashboard-api/src/util.rs index 01fd55f281..7379f18e52 100644 --- a/crates/tracedecay-dashboard-api/src/util.rs +++ b/crates/tracedecay-dashboard-api/src/util.rs @@ -73,28 +73,6 @@ pub async fn query_rows( .await } -/// Runs a scalar `SELECT COUNT(*)`-style query; errors and missing rows -/// collapse to 0 (these feed overview cards, not critical paths). -pub async fn query_i64( - conn: &(impl QueryExecutor + ?Sized), - sql: &str, - params: impl IntoParams, -) -> i64 { - hotpath::future!( - async move { - let Ok(mut rows) = conn.query(sql, params).await else { - return 0; - }; - match rows.next().await { - Ok(Some(row)) => row.get::(0).unwrap_or(0), - _ => 0, - } - }, - label = "dashboard_api.store.query_scalar" - ) - .await -} - /// Runs a scalar integer query while preserving SQL, row-iteration, empty-row, /// and conversion failures for read models where zero carries domain meaning. pub async fn query_i64_result( @@ -230,38 +208,6 @@ mod tests { assert!(err.unwrap_err().contains("missing_table")); } - #[tokio::test] - #[allow(clippy::unwrap_used)] - async fn query_i64_returns_scalar_and_collapses_failures_to_zero() { - let (_directory, conn) = test_conn(); - conn.execute_batch("CREATE TABLE c (v INTEGER)") - .await - .unwrap(); - conn.execute_batch("INSERT INTO c VALUES (7), (8)") - .await - .unwrap(); - - assert_eq!(query_i64(&conn, "SELECT COUNT(*) FROM c", ()).await, 2); - assert_eq!( - query_i64( - &conn, - "SELECT v FROM c WHERE v = ?1", - tracedecay_runtime_core::db::engine::params![7], - ) - .await, - 7 - ); - // Bad SQL and empty result sets both collapse to 0 (overview-card semantics). - assert_eq!( - query_i64(&conn, "SELECT COUNT(*) FROM missing", ()).await, - 0 - ); - assert_eq!( - query_i64(&conn, "SELECT v FROM c WHERE v = 999", ()).await, - 0 - ); - } - #[tokio::test] #[allow(clippy::unwrap_used)] async fn query_i64_result_preserves_scalar_read_failures() { diff --git a/crates/tracedecay-dashboard-api/src/work_api.rs b/crates/tracedecay-dashboard-api/src/work_api.rs index 61c3ae542e..6061641015 100644 --- a/crates/tracedecay-dashboard-api/src/work_api.rs +++ b/crates/tracedecay-dashboard-api/src/work_api.rs @@ -11,16 +11,10 @@ use std::borrow::Cow; use tracedecay_api::WorkOperation; -/// `operation_id` and `application_path` are the identity the contract test -/// checks this document against; schema generation itself needs only the -/// method, path, and schema names. #[derive(Clone, Copy)] -#[cfg_attr(not(test), allow(dead_code))] pub(super) struct RegisteredWorkRouteContractV1 { pub method: &'static str, - pub operation_id: &'static str, pub path: &'static str, - pub application_path: &'static str, pub request_schema_name: fn() -> Cow<'static, str>, pub response_schema_name: fn() -> Cow<'static, str>, } @@ -34,9 +28,7 @@ macro_rules! dashboard_work_routes { $( RegisteredWorkRouteContractV1 { method: "POST", - operation_id: WorkOperation::$variant.operation_id_str(), path: WorkOperation::$variant.dashboard_route_path(), - application_path: WorkOperation::$variant.application_route_path(), request_schema_name: || WorkOperation::$variant.request_schema_name(), response_schema_name: || WorkOperation::$variant.result_schema_name(), }, @@ -90,7 +82,7 @@ mod tests { use axum::http::{Method, Request, StatusCode}; use axum::response::IntoResponse; use tower::ServiceExt; - use tracedecay_api::{WorkHttpRequest, WorkOperation}; + use tracedecay_api::WorkHttpRequest; use tracedecay_contracts::{CancellationSignal, Deadline, RequestId}; use tracedecay_domain::UtcMicros; @@ -155,69 +147,4 @@ mod tests { ); } } - - #[test] - fn the_route_document_covers_every_canonical_core_work_binding() { - use std::collections::BTreeSet; - - let registry = tracedecay_contracts::work_executable_binding_registry() - .expect("canonical Work registry"); - let routes = super::registered_route_contracts(); - let actual_ids = routes - .iter() - .map(|route| route.operation_id) - .collect::>(); - let expected_ids = WorkOperation::ALL - .into_iter() - .filter(|operation| operation.is_dashboard_operation()) - .map(|operation| operation.operation_id_str()) - .collect::>(); - assert_eq!( - routes.len(), - expected_ids.len(), - "dashboard must expose each core Work operation exactly once" - ); - assert_eq!( - actual_ids, expected_ids, - "dashboard routes must cover every core Work operation exactly once" - ); - assert_eq!( - routes - .iter() - .map(|route| route.path) - .collect::>() - .len(), - routes.len(), - "dashboard Work paths must be unique" - ); - - for route in routes { - assert!(!route.path.is_empty(), "{}", route.operation_id); - let binding = registry - .get( - &tracedecay_tool_catalog::OperationId::new(route.operation_id) - .expect("operation id"), - ) - .and_then(|availability| availability.binding()) - .expect("available canonical Work binding"); - let tracedecay_tool_catalog::RouteExposureV1::Public { route_path, .. } = - binding.exposure() - else { - panic!("canonical Work binding must be public"); - }; - assert_eq!(route.application_path, route_path); - assert_eq!( - (route.request_schema_name)(), - binding.request_schema().body()["title"] - .as_str() - .expect("a titled request schema") - ); - assert_eq!( - (route.response_schema_name)(), - binding.result_schema().body()["title"] - .as_str() - .expect("a titled result schema") - ); - } - } } diff --git a/crates/tracedecay-domain/Cargo.toml b/crates/tracedecay-domain/Cargo.toml index b3712c0c8d..a531a75bd2 100644 --- a/crates/tracedecay-domain/Cargo.toml +++ b/crates/tracedecay-domain/Cargo.toml @@ -9,7 +9,7 @@ repository = "https://github.com/ScriptedAlchemy/tracedecay" [dependencies] hotpath.workspace = true -schemars = "1.2.1" +schemars.workspace = true serde = { version = "1", features = ["derive"] } serde_json = "1" sha2 = "0.11" diff --git a/crates/tracedecay-domain/src/code_intelligence/graph.rs b/crates/tracedecay-domain/src/code_intelligence/graph.rs index 30b9277c88..bab77c31c9 100644 --- a/crates/tracedecay-domain/src/code_intelligence/graph.rs +++ b/crates/tracedecay-domain/src/code_intelligence/graph.rs @@ -16,17 +16,16 @@ use sha2::{Digest, Sha256}; /// emits the enum variant, the `ALL` slot, the `as_str` arm, and the /// `from_str` arm, so a spelling cannot drift between them. `as_str` stays a /// direct exhaustive `match`, a new variant fails to compile until it is -/// declared here, and the node-ID hot path never scans the table. Extra -/// `| "alias"` spellings widen `from_str` only; `ALL` and `as_str` record what -/// is written. Serde representations come from the derives passed through on -/// the enum and are independent of these spellings. +/// declared here, and the node-ID hot path never scans the table. Serde +/// representations come from the derives passed through on the enum and are +/// independent of these spellings. macro_rules! wire_enum { ( $(#[$meta:meta])* $vis:vis enum $name:ident { $( $(#[$variant_meta:meta])* - $variant:ident => $wire:literal $(| $alias:literal)* + $variant:ident => $wire:literal ),+ $(,)? } ) => { @@ -38,8 +37,7 @@ macro_rules! wire_enum { #[allow(clippy::should_implement_trait)] impl $name { /// Every variant paired with the spelling [`Self::as_str`] emits and - /// [`Self::from_str`] accepts, in declaration order. Inbound-only - /// aliases are not listed: `ALL` records what is written. + /// [`Self::from_str`] accepts, in declaration order. pub const ALL: [($name, &'static str); wire_enum!(@count $($variant)+)] = [$((Self::$variant, $wire),)+]; @@ -51,7 +49,7 @@ macro_rules! wire_enum { pub fn from_str(s: &str) -> Option { match s { - $($wire $(| $alias)* => Some(Self::$variant),)+ + $($wire => Some(Self::$variant),)+ _ => None, } } @@ -167,7 +165,7 @@ impl NodeKind { } wire_enum! { - #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] + #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] pub enum EdgeKind { Contains => "contains", Calls => "calls", @@ -185,8 +183,7 @@ wire_enum! { wire_enum! { #[derive(Debug, Clone, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum Visibility { - // `"pub"` is accepted inbound only; `"public"` is what is written. - Pub => "public" | "pub", + Pub => "public", PubCrate => "pub_crate", PubSuper => "pub_super", #[default] @@ -546,10 +543,10 @@ mod empty_name_node_id_tests { mod wire_spelling_tests { use super::{EdgeKind, NodeKind, Visibility, generate_node_id}; - /// Spellings that do not follow from the variant name, the inbound-only - /// `"pub"` alias, and refusal of unknown spellings. + /// Spellings that do not follow from the variant name and refusal of + /// unknown spellings, including Rust's `pub` keyword. #[test] - fn representative_spellings_alias_and_refusal() { + fn representative_spellings_and_refusal() { assert_eq!(NodeKind::ScalaObject.as_str(), "object"); assert_eq!(NodeKind::ValField.as_str(), "val"); assert_eq!(NodeKind::VarField.as_str(), "var"); @@ -561,8 +558,8 @@ mod wire_spelling_tests { .starts_with("object:") ); - assert_eq!(Visibility::from_str("pub"), Some(Visibility::Pub)); - assert!(Visibility::ALL.iter().all(|(_, wire)| *wire != "pub")); + assert_eq!(Visibility::from_str("public"), Some(Visibility::Pub)); + assert!(Visibility::from_str("pub").is_none()); assert_eq!(Visibility::default(), Visibility::Private); assert!(NodeKind::from_str("unknown_kind").is_none()); diff --git a/crates/tracedecay-domain/src/code_intelligence/index.rs b/crates/tracedecay-domain/src/code_intelligence/index.rs index bed3d0a1bd..f10be4c912 100644 --- a/crates/tracedecay-domain/src/code_intelligence/index.rs +++ b/crates/tracedecay-domain/src/code_intelligence/index.rs @@ -11,14 +11,14 @@ use std::collections::BTreeSet; -use serde::{Deserialize, Deserializer, Serialize}; +use serde::{Deserialize, Serialize}; +use crate::research::DomainError; use crate::research::id::{ CommitId, ManifestDigest, PrivacyDomainId, ProjectId, RefId, RepositoryId, RetrievalAnchorId, SanitizationReceiptId, WorktreeId, }; use crate::research::time::UtcMicros; -use crate::research::{DomainError, canonical_sha256}; use super::identity::{ ChunkerRevision, CodeGenerationId, ContentDigest, ExtractorRevision, FileOccurrenceId, @@ -180,7 +180,7 @@ pub struct ValidatedCodeFileV1 { /// The sealed manifest of one immutable logical generation. Generations are /// planned, sealed, digested, and never mutated after publication. -#[derive(Clone, Debug, Serialize, PartialEq, Eq)] +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct CodeGenerationManifestV1 { pub project_id: ProjectId, @@ -205,70 +205,6 @@ pub struct CodeGenerationManifestV1 { pub seal: GenerationSealV1, } -const LEGACY_GENERATION_INVALIDATION_DIGEST_DOMAIN: &str = - "tracedecay.code-generation-legacy-v1-migration.v1"; - -#[derive(Deserialize)] -#[serde(deny_unknown_fields)] -struct CodeGenerationManifestWireV1 { - project_id: ProjectId, - generation_id: CodeGenerationId, - snapshot_digest: ManifestDigest, - #[serde(default)] - invalidation_digest: Option, - registry_revision: LanguageRegistryRevision, - grammar_revisions: Vec<(LanguageId, GrammarRevision)>, - extractor_revisions: Vec<(LanguageId, ExtractorRevision)>, - sanitizer_revision: SanitizerRevision, - chunker_revision: ChunkerRevision, - privacy_domain: PrivacyDomainId, - privacy_key_epoch: u64, - parent_generation: Option, - source_commitments: Option, - seal: GenerationSealV1, -} - -impl<'de> Deserialize<'de> for CodeGenerationManifestV1 { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let wire = CodeGenerationManifestWireV1::deserialize(deserializer)?; - let needs_legacy_migration = wire.invalidation_digest.is_none(); - let mut manifest = Self { - project_id: wire.project_id, - generation_id: wire.generation_id, - snapshot_digest: wire.snapshot_digest, - invalidation_digest: match wire.invalidation_digest { - Some(digest) => digest, - None => ManifestDigest::zero().map_err(serde::de::Error::custom)?, - }, - registry_revision: wire.registry_revision, - grammar_revisions: wire.grammar_revisions, - extractor_revisions: wire.extractor_revisions, - sanitizer_revision: wire.sanitizer_revision, - chunker_revision: wire.chunker_revision, - privacy_domain: wire.privacy_domain, - privacy_key_epoch: wire.privacy_key_epoch, - parent_generation: wire.parent_generation, - source_commitments: wire.source_commitments, - seal: wire.seal, - }; - if needs_legacy_migration { - if !manifest - .uses_legacy_v1_identity() - .map_err(serde::de::Error::custom)? - { - return Err(serde::de::Error::missing_field("invalidation_digest")); - } - manifest.invalidation_digest = manifest - .expected_legacy_invalidation_digest() - .map_err(serde::de::Error::custom)?; - } - Ok(manifest) - } -} - /// The seal applied before rows and the expected digest are handed to the /// store publication port. #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] @@ -371,30 +307,6 @@ pub enum TestAttributionEvidenceClassV1 { } impl CodeGenerationManifestV1 { - pub fn uses_legacy_v1_identity(&self) -> Result { - Ok(matches!( - generation_identity_kind(&self.generation_id)?, - GenerationIdentityKind::Legacy - )) - } - - pub fn expected_legacy_invalidation_digest(&self) -> Result { - canonical_sha256(&( - LEGACY_GENERATION_INVALIDATION_DIGEST_DOMAIN, - &self.project_id, - &self.generation_id, - &self.snapshot_digest, - &self.registry_revision, - &self.grammar_revisions, - &self.extractor_revisions, - &self.sanitizer_revision, - &self.chunker_revision, - &self.privacy_domain, - self.privacy_key_epoch, - &self.parent_generation, - )) - } - /// A manifest is single-generation: it names exactly one generation and /// at most one parent. Mixed-generation manifests are rejected before /// publication. @@ -412,28 +324,19 @@ impl CodeGenerationManifestV1 { } self.seal.expected_digest.validate()?; self.seal.planner.validate()?; - match generation_identity_kind(&self.generation_id)? { - GenerationIdentityKind::Legacy => { - if self.invalidation_digest != self.expected_legacy_invalidation_digest()? { - return Err(DomainError::DigestMismatch); - } - } - GenerationIdentityKind::Fingerprinted(fingerprint) => { - let expected = crate::canonical_text::sha256_hex_body( - self.invalidation_digest.as_str(), - "generation invalidation digest", - )?; - if fingerprint != expected { - return Err(DomainError::DigestMismatch); - } - } + let expected = crate::canonical_text::sha256_hex_body( + self.invalidation_digest.as_str(), + "generation invalidation digest", + )?; + if generation_identity_fingerprint(&self.generation_id)? != expected { + return Err(DomainError::DigestMismatch); } if self.parent_generation.as_ref() == Some(&self.generation_id) { return Err(DomainError::SelfSupersession); } if let Some(parent_generation) = &self.parent_generation { parent_generation.validate()?; - generation_identity_kind(parent_generation)?; + generation_identity_fingerprint(parent_generation)?; } validate_language_revisions( &self.grammar_revisions, @@ -462,14 +365,9 @@ impl CodeGenerationManifestV1 { } } -enum GenerationIdentityKind<'a> { - Legacy, - Fingerprinted(&'a str), -} - -fn generation_identity_kind( - generation_id: &CodeGenerationId, -) -> Result, DomainError> { +/// The invalidation fingerprint of a canonical +/// `generation.v1...` identity. +fn generation_identity_fingerprint(generation_id: &CodeGenerationId) -> Result<&str, DomainError> { let mut parts = generation_id.as_str().split('.'); let scheme = parts.next(); let version = parts.next(); @@ -489,11 +387,8 @@ fn generation_identity_kind( }); } match fingerprint { - None => Ok(GenerationIdentityKind::Legacy), - Some(value) if crate::canonical_text::is_lowercase_hex(value, 64) => { - Ok(GenerationIdentityKind::Fingerprinted(value)) - } - Some(_) => Err(DomainError::NonCanonical { + Some(value) if crate::canonical_text::is_lowercase_hex(value, 64) => Ok(value), + _ => Err(DomainError::NonCanonical { field: "code generation identity fingerprint", }), } @@ -552,9 +447,12 @@ mod tests { } fn generation_manifest() -> CodeGenerationManifestV1 { - let mut manifest = CodeGenerationManifestV1 { + CodeGenerationManifestV1 { project_id: id("project.fixture"), - generation_id: id("generation.v1.aaaaaaaa.00000002"), + generation_id: id(&format!( + "generation.v1.aaaaaaaa.00000002.{}", + "b".repeat(64) + )), snapshot_digest: id(&digest('a')), invalidation_digest: id(&digest('b')), registry_revision: id("registry.v1"), @@ -570,39 +468,63 @@ mod tests { chunker_revision: id("chunker.v1"), privacy_domain: id("privacy.fixture"), privacy_key_epoch: 1, - parent_generation: Some(id("generation.v1.aaaaaaaa.00000001")), + parent_generation: Some(id(&format!( + "generation.v1.aaaaaaaa.00000001.{}", + "c".repeat(64) + ))), source_commitments: None, seal: GenerationSealV1 { expected_digest: id(&digest('d')), sealed_at: UtcMicros(20), planner: id("planner.v1"), }, - }; - manifest.invalidation_digest = manifest - .expected_legacy_invalidation_digest() - .expect("legacy invalidation digest"); - manifest + } } #[test] - fn relation_edge_kind_as_str_matches_its_serde_spelling() { - assert_eq!(RelationEdgeKindV1::TypeOf.as_str(), "type_of"); - for kind in [ - RelationEdgeKindV1::Calls, - RelationEdgeKindV1::Uses, - RelationEdgeKindV1::TypeOf, - RelationEdgeKindV1::Contains, - RelationEdgeKindV1::Implements, - RelationEdgeKindV1::Extends, - RelationEdgeKindV1::Annotates, - RelationEdgeKindV1::Returns, - RelationEdgeKindV1::Receives, + fn generation_manifest_requires_fingerprinted_identities() { + let mut unfingerprinted = generation_manifest(); + unfingerprinted.generation_id = id("generation.v1.aaaaaaaa.00000002"); + assert!(unfingerprinted.validate().is_err()); + + let mut unfingerprinted_parent = generation_manifest(); + unfingerprinted_parent.parent_generation = Some(id("generation.v1.aaaaaaaa.00000001")); + assert!(unfingerprinted_parent.validate().is_err()); + + let mut mismatched = generation_manifest(); + mismatched.invalidation_digest = id(&digest('e')); + assert!(mismatched.validate().is_err()); + + let mut wire = serde_json::to_value(generation_manifest()).expect("serialize"); + wire.as_object_mut() + .expect("manifest object") + .remove("invalidation_digest"); + assert!(serde_json::from_value::(wire).is_err()); + } + + #[test] + fn relation_edge_kinds_have_literal_wire_spellings() { + for (kind, wire) in [ + (RelationEdgeKindV1::Calls, "calls"), + (RelationEdgeKindV1::Uses, "uses"), + (RelationEdgeKindV1::TypeOf, "type_of"), + (RelationEdgeKindV1::Contains, "contains"), + (RelationEdgeKindV1::Implements, "implements"), + (RelationEdgeKindV1::Extends, "extends"), + (RelationEdgeKindV1::Annotates, "annotates"), + (RelationEdgeKindV1::Returns, "returns"), + (RelationEdgeKindV1::Receives, "receives"), ] { assert_eq!( serde_json::to_value(kind).expect("serialize"), - serde_json::Value::String(kind.as_str().to_owned()), - "{kind:?} as_str diverged from its serde spelling" + serde_json::json!(wire) + ); + assert_eq!( + serde_json::from_value::(serde_json::json!(wire)) + .expect("deserialize"), + kind ); + assert_eq!(kind.as_str(), wire); } } diff --git a/crates/tracedecay-domain/src/configuration.rs b/crates/tracedecay-domain/src/configuration.rs index aa9c190e2a..cfbb83f2ff 100644 --- a/crates/tracedecay-domain/src/configuration.rs +++ b/crates/tracedecay-domain/src/configuration.rs @@ -18,10 +18,12 @@ use crate::research::{ ProjectId, UtcMicros, canonical_sha256, }; +mod lcm_summarizer_executables; pub mod topology; mod work_executable_bindings; mod work_expertise_consent; +pub use lcm_summarizer_executables::*; pub use topology::*; pub use work_executable_bindings::*; pub use work_expertise_consent::*; @@ -38,6 +40,7 @@ pub const WORK_EXECUTABLE_BINDINGS_SETTING_KEY: &str = "work.executable_bindings pub const PROJECT_WORK_EXPERTISE_CONSENT_SETTING_KEY: &str = "work.expertise_consent.v1"; pub const CONTEXT_SCOUT_SETTINGS_SETTING_KEY: &str = "context_scout.settings.v1"; pub const AUTOMATION_SETTINGS_SETTING_KEY: &str = "automation.settings.v1"; +pub const LCM_SUMMARIZER_EXECUTABLES_SETTING_KEY: &str = "lcm.summarizer_executables.v1"; /// Core setting keys that shipped in published betas and were then retired. /// A persisted snapshot carrying one converges by dropping it; its value has @@ -75,7 +78,6 @@ pub const SYNC_BACKSTOP_INTERVAL_MINS_SETTING_KEY: &str = "sync.backstop_interva pub const SYNC_FULL_SYNC_ESCALATION_FILES_SETTING_KEY: &str = "sync.full_sync_escalation_files.v1"; pub const SYNC_MAX_CONCURRENT_SYNCS_SETTING_KEY: &str = "sync.max_concurrent_syncs.v1"; pub const SYNC_BRANCH_GC_DAYS_SETTING_KEY: &str = "sync.branch_gc_days.v1"; -pub const SYNC_ORPHAN_DB_GC_DAYS_SETTING_KEY: &str = "sync.orphan_db_gc_days.v1"; pub const SYNC_AUTO_INIT_SETTING_KEY: &str = "sync.auto_init.v1"; pub const SYNC_AUTO_TRACK_PR_BRANCHES_SETTING_KEY: &str = "sync.auto_track_pr_branches.v1"; pub const SYNC_AUTO_TRACK_PR_POLL_SECS_SETTING_KEY: &str = "sync.auto_track_pr_poll_secs.v1"; @@ -92,6 +94,7 @@ pub const CONFIGURATION_SETTING_KEYS_V1: &[&str] = &[ PROJECT_WORK_EXPERTISE_CONSENT_SETTING_KEY, CONTEXT_SCOUT_SETTINGS_SETTING_KEY, AUTOMATION_SETTINGS_SETTING_KEY, + LCM_SUMMARIZER_EXECUTABLES_SETTING_KEY, crate::feedback::PROXIMITY_RISK_THRESHOLD_SETTING_KEY_V1, USER_UPLOAD_ENABLED_SETTING_KEY, USER_CODE_INDEX_WORKERS_SETTING_KEY, @@ -119,7 +122,6 @@ pub const CONFIGURATION_SETTING_KEYS_V1: &[&str] = &[ SYNC_FULL_SYNC_ESCALATION_FILES_SETTING_KEY, SYNC_MAX_CONCURRENT_SYNCS_SETTING_KEY, SYNC_BRANCH_GC_DAYS_SETTING_KEY, - SYNC_ORPHAN_DB_GC_DAYS_SETTING_KEY, SYNC_AUTO_INIT_SETTING_KEY, SYNC_AUTO_TRACK_PR_BRANCHES_SETTING_KEY, SYNC_AUTO_TRACK_PR_POLL_SECS_SETTING_KEY, @@ -526,6 +528,7 @@ pub enum ConfigurationValueKindV1 { WorkExpertiseConsent, ContextScoutSettings, AutomationSettings, + LcmSummarizerExecutables, } /// Profile-level worker-count intent for the process-wide code-index pool. @@ -1168,6 +1171,7 @@ pub enum ConfigurationValueV1 { WorkExpertiseConsent(WorkExpertiseConsentV1), ContextScoutSettings(ContextScoutSettingsV1), AutomationSettings(Box), + LcmSummarizerExecutables(LcmSummarizerExecutablesV1), } impl ConfigurationValueV1 { @@ -1186,6 +1190,7 @@ impl ConfigurationValueV1 { Self::WorkExpertiseConsent(_) => ConfigurationValueKindV1::WorkExpertiseConsent, Self::ContextScoutSettings(_) => ConfigurationValueKindV1::ContextScoutSettings, Self::AutomationSettings(_) => ConfigurationValueKindV1::AutomationSettings, + Self::LcmSummarizerExecutables(_) => ConfigurationValueKindV1::LcmSummarizerExecutables, } } @@ -1223,6 +1228,7 @@ impl ConfigurationValueV1 { Self::WorkExpertiseConsent(consent) => consent.validate(), Self::ContextScoutSettings(settings) => settings.validate(), Self::AutomationSettings(settings) => settings.validate(), + Self::LcmSummarizerExecutables(executables) => executables.validate(), } } } diff --git a/crates/tracedecay-domain/src/configuration/lcm_summarizer_executables.rs b/crates/tracedecay-domain/src/configuration/lcm_summarizer_executables.rs new file mode 100644 index 0000000000..c8d4a5ec41 --- /dev/null +++ b/crates/tracedecay-domain/src/configuration/lcm_summarizer_executables.rs @@ -0,0 +1,130 @@ +//! Explicit executables behind on-demand LCM summarization. +//! +//! The daemon asks a host CLI for a summary only when this setting names the +//! executable. There is no fallback: an unconfigured provider is a typed state +//! the compaction journey reports as pending, never a `PATH` or environment +//! lookup that could reach whatever binary the operator's shell resolves. + +use std::path::{Component, Path, PathBuf}; + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::DomainError; + +/// One provider's summarizer executable: absent by default, or the exact +/// absolute path the operator configured. +#[derive(Clone, Debug, Default, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] +#[serde(rename_all = "snake_case", tag = "state")] +pub enum LcmSummarizerExecutableV1 { + #[default] + Unconfigured, + Configured { + canonical_path: PathBuf, + }, +} + +impl LcmSummarizerExecutableV1 { + pub fn configured(canonical_path: PathBuf) -> Result { + let executable = Self::Configured { canonical_path }; + executable.validate()?; + Ok(executable) + } + + /// The configured path, or `None` while the provider is unconfigured. + pub fn canonical_path(&self) -> Option<&Path> { + match self { + Self::Unconfigured => None, + Self::Configured { canonical_path } => Some(canonical_path), + } + } + + pub fn validate(&self) -> Result<(), DomainError> { + match self { + Self::Unconfigured => Ok(()), + Self::Configured { canonical_path } => { + if !canonical_path.is_absolute() + || canonical_path.components().any(|component| { + matches!(component, Component::CurDir | Component::ParentDir) + }) + { + return Err(DomainError::NonCanonical { + field: "lcm summarizer executable path", + }); + } + Ok(()) + } + } + } +} + +/// The per-provider summarizer executables one configuration snapshot admits. +#[derive(Clone, Debug, Default, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct LcmSummarizerExecutablesV1 { + /// `cursor-agent`, asked for a summary of Cursor sessions. + #[serde(default)] + pub cursor_agent: LcmSummarizerExecutableV1, + /// `codex`, driven over app-server JSON-RPC for Codex sessions. + #[serde(default)] + pub codex: LcmSummarizerExecutableV1, +} + +impl LcmSummarizerExecutablesV1 { + /// Every provider unconfigured: the registry default. + pub const fn unconfigured() -> Self { + Self { + cursor_agent: LcmSummarizerExecutableV1::Unconfigured, + codex: LcmSummarizerExecutableV1::Unconfigured, + } + } + + pub fn validate(&self) -> Result<(), DomainError> { + self.cursor_agent.validate()?; + self.codex.validate() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn configured_executable_requires_an_absolute_clean_path() { + let absolute_base = std::env::temp_dir(); + let non_canonical = Err(DomainError::NonCanonical { + field: "lcm summarizer executable path", + }); + assert_eq!( + LcmSummarizerExecutableV1::configured(PathBuf::from("cursor-agent")), + non_canonical + ); + assert_eq!( + LcmSummarizerExecutableV1::configured(absolute_base.join("opt/../bin/cursor-agent")), + non_canonical + ); + let clean = absolute_base.join("bin").join("cursor-agent"); + let configured = LcmSummarizerExecutableV1::configured(clean.clone()).unwrap(); + assert_eq!(configured.canonical_path(), Some(clean.as_path())); + } + + #[test] + fn executables_decode_from_tagged_states_and_absent_providers_are_unconfigured() { + assert_eq!( + serde_json::to_value(LcmSummarizerExecutablesV1::unconfigured()).unwrap(), + serde_json::json!({ + "cursor_agent": {"state": "unconfigured"}, + "codex": {"state": "unconfigured"}, + }) + ); + let decoded: LcmSummarizerExecutablesV1 = serde_json::from_value(serde_json::json!({ + "cursor_agent": {"state": "configured", "canonical_path": "/opt/bin/cursor-agent"}, + })) + .unwrap(); + assert_eq!( + decoded.cursor_agent.canonical_path(), + Some(Path::new("/opt/bin/cursor-agent")) + ); + assert_eq!(decoded.codex, LcmSummarizerExecutableV1::Unconfigured); + } +} diff --git a/crates/tracedecay-domain/src/diagnostics.rs b/crates/tracedecay-domain/src/diagnostics.rs index 9dd29e8597..ecfe471408 100644 --- a/crates/tracedecay-domain/src/diagnostics.rs +++ b/crates/tracedecay-domain/src/diagnostics.rs @@ -122,8 +122,8 @@ impl DiagnosticProvenanceV1 { } /// Current-vs-stale typing for a durable diagnostic record. Publication is -/// version-monotone: a newer clean generation clears or supersedes the prior -/// publication deterministically, and stale findings cannot cross snapshots. +/// version-monotone: a newer clean generation clears the prior publication +/// deterministically, and stale findings cannot cross snapshots. /// Stale and historical records remain queryable through /// application APIs but are excluded from active publication. #[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] @@ -131,11 +131,6 @@ impl DiagnosticProvenanceV1 { pub enum DiagnosticRecordStateV1 { /// Current for exactly the clean generation named by the record. Current, - /// A successor clean generation republished the same logical finding - /// space; this record is historical. - Superseded { - successor_generation: CodeGenerationId, - }, /// A clean generation completed and deterministically removed this /// finding (resolution, deletion, source-revision drift, or content or /// generation change). @@ -152,15 +147,6 @@ impl DiagnosticRecordStateV1 { fn validate(&self, own_generation: &CodeGenerationId) -> Result<(), DomainError> { match self { Self::Current => Ok(()), - Self::Superseded { - successor_generation, - } => { - successor_generation.validate()?; - if successor_generation == own_generation { - return Err(DomainError::SelfSupersession); - } - Ok(()) - } Self::Cleared { cleared_in_generation, } => { @@ -269,26 +255,6 @@ impl GenerationDiagnosticV1 { self.state.is_current() } - /// Returns a copy marked superseded by `successor_generation`. A record - /// can only be superseded out of the current state, and a generation can - /// never supersede itself (version-monotone publication). - pub fn supersede(&self, successor_generation: CodeGenerationId) -> Result { - successor_generation.validate()?; - if successor_generation == self.generation_id { - return Err(DomainError::SelfSupersession); - } - if !self.state.is_current() { - return Err(DomainError::NonCanonical { - field: "diagnostic record state transition", - }); - } - let mut next = self.clone(); - next.state = DiagnosticRecordStateV1::Superseded { - successor_generation, - }; - Ok(next) - } - /// Returns a copy marked cleared by a clean generation that completed /// without this finding. A record can only be cleared out of the current /// state, and a generation can never clear itself. @@ -440,20 +406,6 @@ mod tests { )); } - #[test] - fn supersession_requires_a_distinct_generation() { - let record = fixture_record(); - assert!(matches!( - record.clone().supersede(record.generation_id.clone()), - Err(DomainError::SelfSupersession) - )); - let superseded = record - .supersede(id("generation.clean.2")) - .expect("distinct successor supersedes"); - assert!(!superseded.is_current()); - superseded.validate().expect("superseded record validates"); - } - #[test] fn clearing_requires_a_distinct_generation() { let record = fixture_record(); @@ -474,16 +426,15 @@ mod tests { #[test] fn stale_records_cannot_transition_again() { let record = fixture_record(); - let superseded = record.supersede(id("generation.clean.2")).unwrap(); - assert!(superseded.supersede(id("generation.clean.3")).is_err()); - assert!(superseded.clear(id("generation.clean.3")).is_err()); + let cleared = record.clear(id("generation.clean.2")).unwrap(); + assert!(cleared.clear(id("generation.clean.3")).is_err()); } #[test] fn state_rejects_self_referencing_generations() { let mut record = fixture_record(); - record.state = DiagnosticRecordStateV1::Superseded { - successor_generation: record.generation_id.clone(), + record.state = DiagnosticRecordStateV1::Cleared { + cleared_in_generation: record.generation_id.clone(), }; assert!(matches!( record.validate(), diff --git a/crates/tracedecay-domain/src/feedback/proximity.rs b/crates/tracedecay-domain/src/feedback/proximity.rs index b93d341cf9..d4ecf3fd23 100644 --- a/crates/tracedecay-domain/src/feedback/proximity.rs +++ b/crates/tracedecay-domain/src/feedback/proximity.rs @@ -20,7 +20,6 @@ crate::canonical_text::validated_string_newtype!( DomainError, super::validate_label; ProximityContributionIdV1 => "proximity contribution id", - ProximityWarningIdV1 => "proximity warning id", ProximityObservationIdV1 => "proximity observation id", ); @@ -166,7 +165,6 @@ impl ProximityRiskInputsV1 { #[serde(deny_unknown_fields)] pub struct ProximityContributionV1 { pub contribution_id: ProximityContributionIdV1, - pub warning_id: ProximityWarningIdV1, pub warning_class: ProximityWarningClassV1, pub source_observation_ids: Vec, pub retrieval_anchor_ids: Vec, @@ -193,7 +191,6 @@ impl ProximityContributionV1 { pub fn validate(&self) -> Result<(), DomainError> { self.contribution_id.validate()?; - self.warning_id.validate()?; let immediate_class = matches!( self.warning_class, ProximityWarningClassV1::SameFile @@ -399,7 +396,6 @@ mod tests { fn concealed_private_contribution() -> ProximityContributionV1 { ProximityContributionV1 { contribution_id: ProximityContributionIdV1::new("contribution.private").unwrap(), - warning_id: ProximityWarningIdV1::new("warning.private").unwrap(), warning_class: ProximityWarningClassV1::Neighborhood, source_observation_ids: Vec::new(), retrieval_anchor_ids: Vec::new(), diff --git a/crates/tracedecay-domain/src/lib.rs b/crates/tracedecay-domain/src/lib.rs index e7e360e155..e0ea27297a 100644 --- a/crates/tracedecay-domain/src/lib.rs +++ b/crates/tracedecay-domain/src/lib.rs @@ -98,6 +98,7 @@ pub use configuration::{ INDEX_EXCLUDE_SETTING_KEY, INDEX_EXTRACT_DOCSTRINGS_SETTING_KEY, INDEX_GIT_IGNORE_SETTING_KEY, INDEX_INCLUDE_SETTING_KEY, INDEX_MAX_FILE_SIZE_SETTING_KEY, INDEX_NATIVE_GRAPH_ACTIVATION_SETTING_KEY, INDEX_TRACK_CALL_SITES_SETTING_KEY, + LCM_SUMMARIZER_EXECUTABLES_SETTING_KEY, LcmSummarizerExecutableV1, LcmSummarizerExecutablesV1, MAX_WORK_EXPERTISE_CONSENT_LIFETIME_MICROS_V1, PROJECT_WORK_EXPERTISE_CONSENT_SETTING_KEY, ProtectedApplyRequest, ProtectedChange, ProtectedChangePlan, ProtectedChangeSnapshotError, ProtectedRefDispositionV1, ProtectedRefRuleV1, ProtectedRefSelectorV1, QueryCollectionId, @@ -108,23 +109,23 @@ pub use configuration::{ SYNC_AUTO_TRACK_PR_BRANCHES_SETTING_KEY, SYNC_AUTO_TRACK_PR_POLL_SECS_SETTING_KEY, SYNC_AUTO_WATCH_SETTING_KEY, SYNC_BACKSTOP_INTERVAL_MINS_SETTING_KEY, SYNC_BRANCH_GC_DAYS_SETTING_KEY, SYNC_FULL_SYNC_ESCALATION_FILES_SETTING_KEY, - SYNC_MAX_CONCURRENT_SYNCS_SETTING_KEY, SYNC_ORPHAN_DB_GC_DAYS_SETTING_KEY, - SYNC_READ_COOLDOWN_SECS_SETTING_KEY, SYNC_READ_REFRESH_SETTING_KEY, - SYNC_SESSION_START_STALE_THRESHOLD_SECS_SETTING_KEY, SYNC_SESSION_START_SYNC_SETTING_KEY, - SYNC_WATCH_DEBOUNCE_MS_SETTING_KEY, SYNC_WATCH_LINKED_WORKTREES_SETTING_KEY, - SYNC_WATCH_MAX_DELAY_MS_SETTING_KEY, SYNC_WATCH_MAX_PROJECTS_SETTING_KEY, ScopeAccessRule, - ScopeAccessSubjectV1, ScopeControlOperationV1, ScopeSourceBinding, - SensitiveFilesystemLocatorV1, SettingDefinitionV1, SettingKey, SettingScopeV1, - SettingSensitivityV1, SourceBindingId, SourceKindV1, TELEMETRY_TIMINGS_SETTING_KEY, - TopologyConcurrencyPolicyV1, TopologyEscalationPolicyV1, TopologyGatePolicyV1, - TopologyNotificationLevelV1, TopologyPolicyDigestV1, USER_CODE_INDEX_WORKERS_SETTING_KEY, - USER_EXTRACTION_TIMEOUT_SECS_SETTING_KEY, USER_UPLOAD_ENABLED_SETTING_KEY, - USER_WATCHER_DEBOUNCE_MS_SETTING_KEY, USER_WORK_EXPERTISE_CONSENT_SETTING_KEY, UserProfileId, - WORK_EXECUTABLE_BINDINGS_SETTING_KEY, WORK_TOPOLOGY_POLICY_SETTING_KEY, - WorkExecutableBindingV1, WorkExecutableCapabilityV1, WorkExpertiseCategoryV1, - WorkExpertiseConsentV1, WorkTopologyPolicyV1, WorktreeCleanlinessRequirementV1, - WorktreePlacementModeV1, WorktreePlacementRootId, WorktreeRetentionPolicyV1, - WorktreeRootPolicyV1, resolve_restrictive_capabilities, safe_work_topology_policy_v1, + SYNC_MAX_CONCURRENT_SYNCS_SETTING_KEY, SYNC_READ_COOLDOWN_SECS_SETTING_KEY, + SYNC_READ_REFRESH_SETTING_KEY, SYNC_SESSION_START_STALE_THRESHOLD_SECS_SETTING_KEY, + SYNC_SESSION_START_SYNC_SETTING_KEY, SYNC_WATCH_DEBOUNCE_MS_SETTING_KEY, + SYNC_WATCH_LINKED_WORKTREES_SETTING_KEY, SYNC_WATCH_MAX_DELAY_MS_SETTING_KEY, + SYNC_WATCH_MAX_PROJECTS_SETTING_KEY, ScopeAccessRule, ScopeAccessSubjectV1, + ScopeControlOperationV1, ScopeSourceBinding, SensitiveFilesystemLocatorV1, SettingDefinitionV1, + SettingKey, SettingScopeV1, SettingSensitivityV1, SourceBindingId, SourceKindV1, + TELEMETRY_TIMINGS_SETTING_KEY, TopologyConcurrencyPolicyV1, TopologyEscalationPolicyV1, + TopologyGatePolicyV1, TopologyNotificationLevelV1, TopologyPolicyDigestV1, + USER_CODE_INDEX_WORKERS_SETTING_KEY, USER_EXTRACTION_TIMEOUT_SECS_SETTING_KEY, + USER_UPLOAD_ENABLED_SETTING_KEY, USER_WATCHER_DEBOUNCE_MS_SETTING_KEY, + USER_WORK_EXPERTISE_CONSENT_SETTING_KEY, UserProfileId, WORK_EXECUTABLE_BINDINGS_SETTING_KEY, + WORK_TOPOLOGY_POLICY_SETTING_KEY, WorkExecutableBindingV1, WorkExecutableCapabilityV1, + WorkExpertiseCategoryV1, WorkExpertiseConsentV1, WorkTopologyPolicyV1, + WorktreeCleanlinessRequirementV1, WorktreePlacementModeV1, WorktreePlacementRootId, + WorktreeRetentionPolicyV1, WorktreeRootPolicyV1, resolve_restrictive_capabilities, + safe_work_topology_policy_v1, }; pub use diagnostics::{ DiagnosticEvidenceClassV1, DiagnosticProducerKindV1, DiagnosticProvenanceV1, @@ -174,7 +175,7 @@ pub use feedback::{ ProximityContributionIdV1, ProximityContributionV1, ProximityCoverageV1, ProximityInclusionV1, ProximityObservationIdV1, ProximityRelationPathKindV1, ProximityRelationPathV1, ProximityRelationStrengthV1, ProximityRiskInputsV1, ProximityTierV1, ProximityWarningClassV1, - ProximityWarningIdV1, derive_feedback_finding_id, derive_overlay_feedback_finding_id, + derive_feedback_finding_id, derive_overlay_feedback_finding_id, }; pub use framed_log::{CHECKSUM_BYTES, checksum, partial_tail_matches_prefix}; pub use git::{ @@ -261,12 +262,12 @@ pub use observation::{ CanonicalObservationFactV1, CanonicalObservationIdV1, CanonicalObservationRelationsV1, CanonicalReasoningVisibilityV1, CanonicalUnknownStateV1, CanonicalWorkflowEvidenceKindV1, CanonicalWorkflowSemanticKindV1, ClineNativeSourceTransition, ClineTranscriptStream, - DurableClaudeObservationV1, DurableObservationV1, MAX_CANONICAL_OBSERVATION_FACTS_V1, - MAX_OBSERVATION_RECORD_BYTES, MAX_OBSERVATION_STRUCTURE_DEPTH, - MAX_OBSERVATION_STRUCTURE_VALUES, ObservationCollisionOutcomeV1, ObservationContractError, - ObservationIdentityMaterialV1, ObservationOrderingDomainV1, ObservationPositionalOccurrenceV1, - ObservationScopeV1, ObservationSourceCursorV1, ObservationSourceGenerationV1, - ObservationSourceIdentityV1, ObservationSourceRangeV1, PayloadDigestV1, PayloadReferenceV1, + DurableObservationV1, MAX_CANONICAL_OBSERVATION_FACTS_V1, MAX_OBSERVATION_RECORD_BYTES, + MAX_OBSERVATION_STRUCTURE_DEPTH, MAX_OBSERVATION_STRUCTURE_VALUES, + ObservationCollisionOutcomeV1, ObservationContractError, ObservationIdentityMaterialV1, + ObservationOrderingDomainV1, ObservationPositionalOccurrenceV1, ObservationScopeV1, + ObservationSourceCursorV1, ObservationSourceGenerationV1, ObservationSourceIdentityV1, + ObservationSourceRangeV1, PayloadDigestV1, PayloadReferenceV1, ProviderUsageContractDimensionV1, ProviderUsageCounterSemanticsV1, ProviderUsageCountersV1, ProviderUsageCursorV1, ProviderUsageModelV1, ProviderUsageObservationV1, ProviderUsageReadV1, ProviderUsageScopeV1, SanitizationReceiptV1, SanitizerDispositionV1, SensitivityV1, @@ -287,28 +288,26 @@ pub use repository::{ RepositoryEvidenceV1, RepositoryProvenanceV1, RepositoryRemoteIdentityV1, }; pub use research::{ - AccessPolicyDigest, ActorId, AgentInstanceId, AnchorDurabilityClass, AnchorLineageRefV2, - AnchorLineageRefV3, AnchorOwnerBindingV1, AnchorProvenanceRelationV2, AnchorResolutionStateV2, - AnchorSourceGenerationV2, AnchorSourceGenerationV3, ApplyReceiptAnchorRefV1, AttemptId, - AuthorityEpoch, AuthorizedAnchorResolution, BlobId, BoundedVec, BrainId, BrainNodeId, - BrainNodeRoleV1, BranchStackEdgeV1, BranchStackId, BranchStackNodeV1, BranchStackRevisionId, - BranchStackRevisionV1, BranchStackSourceV1, CanonicalSourceOccurrenceSetIdV1, CapabilityId, - CatalogGenerationId, CatalogSnapshotRefV1, CheckSnapshotAnchorRefV1, CommitId, - ComponentVersion, Confidence, ConflictEvidenceAnchorRefV1, CoverageReportV1, - CoverageUniverseKnowledgeV1, DataVersionDigest, DomainError, EntityId, EntityKind, EntityRef, - EntityVersionId, EvidenceAssemblyPublicationReceiptIdV1, EvidenceClass, - EvidenceRetentionWatermark, EvidenceSpanProjectionReceiptIdV1, FactAssertionId, FactEventId, - FactEvidenceId, FactId, FrozenBranchStackSnapshotV1, FrozenIndependentBranchSelectionV1, - FrozenWatermarkResolutionV1, GitHubStackCapabilitySnapshotV1, GitHubStackCapabilityStateV1, - GitHubStackLayerSnapshotV1, GitHubStackSnapshotV1, GitTopologyAnchorTargetV1, - GitTopologyGenerationRefV1, GitTopologySourceRoleV1, HostInstanceId, - IntegrationReceiptAnchorRefV1, LocatorDigest, LogSafeText, ManifestDigest, - ManifestDigestHasher, MechanicalIntegrationModeV1, MessageId, NativeAliasKindV2, NativeAliasV2, - NativeGitObjectAnchorRefV1, NativeGitObjectKindV1, NativeIntegrationAnalysisAnchorV1, - NativeIntegrationAnalysisCoverageV1, NativeIntegrationAnalysisGapV1, - NativeIntegrationAnalysisLaneV1, NativeIntegrationAnalysisReportV1, - NativeIntegrationApprovalId, NativeIntegrationApprovalV1, NativeIntegrationDirectionV1, - NativeIntegrationGenerationBindingV1, NativeIntegrationPhaseV1, + AccessPolicyDigest, ActorId, AgentInstanceId, AnchorDurabilityClass, AnchorLineageRef, + AnchorOwnerBindingV1, AnchorProvenanceRelation, AnchorResolutionStateV2, + AnchorSourceGeneration, ApplyReceiptAnchorRefV1, AttemptId, AuthorityEpoch, + AuthorizedAnchorResolution, BlobId, BoundedVec, BrainId, BrainNodeId, BrainNodeRoleV1, + BranchStackEdgeV1, BranchStackId, BranchStackNodeV1, BranchStackRevisionId, + BranchStackRevisionV1, BranchStackSourceV1, CapabilityId, CatalogGenerationId, + CatalogSnapshotRefV1, CheckSnapshotAnchorRefV1, CommitId, ComponentVersion, Confidence, + ConflictEvidenceAnchorRefV1, CoverageReportV1, CoverageUniverseKnowledgeV1, DataVersionDigest, + DomainError, EntityId, EntityKind, EntityRef, EntityVersionId, EvidenceClass, + EvidenceRetentionWatermark, FactAssertionId, FactEventId, FactEvidenceId, FactId, + FrozenBranchStackSnapshotV1, FrozenIndependentBranchSelectionV1, FrozenWatermarkResolutionV1, + GitHubStackCapabilitySnapshotV1, GitHubStackCapabilityStateV1, GitHubStackLayerSnapshotV1, + GitHubStackSnapshotV1, GitTopologyAnchorTargetV1, GitTopologyGenerationRefV1, + GitTopologySourceRoleV1, HostInstanceId, IntegrationReceiptAnchorRefV1, LocatorDigest, + LogSafeText, ManifestDigest, ManifestDigestHasher, MechanicalIntegrationModeV1, MessageId, + NativeAlias, NativeAliasKind, NativeGitObjectAnchorRefV1, NativeGitObjectKindV1, + NativeIntegrationAnalysisAnchorV1, NativeIntegrationAnalysisCoverageV1, + NativeIntegrationAnalysisGapV1, NativeIntegrationAnalysisLaneV1, + NativeIntegrationAnalysisReportV1, NativeIntegrationApprovalId, NativeIntegrationApprovalV1, + NativeIntegrationDirectionV1, NativeIntegrationGenerationBindingV1, NativeIntegrationPhaseV1, NativeIntegrationPreviewDispositionV1, NativeIntegrationPreviewId, NativeIntegrationPreviewV1, NativeIntegrationReceiptV1, NativeIntegrationRepositorySnapshotV1, NativeIntegrationSelectionV1, NativeIntegrationSemanticConflictKindV1, @@ -323,22 +322,20 @@ pub use research::{ RefSnapshotAnchorRefV1, RefSnapshotKindV1, RegistryManifestDigest, RemoteCoverageV1, RemoteShardCoverageV1, RepositoryCaptureAnchorRefV1, RepositoryCaptureId, RepositoryId, ResolutionAuthorizationV1, RetentionClass, RetrievalAnchorId, RetrievalAnchorRecord, - RetrievalAnchorRecordV2, RetrievalAnchorRecordV2Parts, RetrievalAnchorRecordV3, - RetrievalAnchorRecordV3Parts, RetrievalAnchorTargetV2, RetrievalAnchorTargetV3, - RetrieverContributionIdV1, ReviewSnapshotAnchorRefV1, RunId, SanitizationProofV1, - SanitizationReceiptId, SanitizationReceiptRefV1, SanitizationReceiptResolverV1, - SanitizedTextRefV1, SanitizedTextV1, ScopeResolutionId, SessionId, ShardDispositionV1, ShardId, - ShardWatermark, SourceInstanceId, SourcePosition, SourceStoreId, StackDeliveryWatermarkId, - StackNodeId, StackSignalId, StoreAuthorityId, TaskId, ThreadId, TimeInterval, ToolInvocationId, - TreeId, TurnId, UseCaseId, UtcMicros, VectorWatermark, VerifiedCacheGrantSnapshotV1, - WatermarkDriftV1, WorkArtifactId, WorkCancellationRequestId, WorkCommandId, WorkLeaseId, - WorkProviderRouteId, WorkTopologyGenerationRefV1, WorkflowDefinitionId, WorkflowOperationRef, - WorkflowOutputName, WorkflowStepId, WorktreeCaptureAnchorRefV1, WorktreeId, - WorktreeInventoryEpoch, WorktreeInventorySnapshotId, canonical_json_bytes, + RetrievalAnchorRecordParts, RetrievalAnchorTarget, ReviewSnapshotAnchorRefV1, RunId, + SanitizationProofV1, SanitizationReceiptId, SanitizationReceiptRefV1, + SanitizationReceiptResolverV1, SanitizedTextRefV1, SanitizedTextV1, ScopeResolutionId, + SessionId, ShardDispositionV1, ShardId, ShardWatermark, SourceInstanceId, SourcePosition, + SourceStoreId, StackDeliveryWatermarkId, StackNodeId, StackSignalId, StoreAuthorityId, TaskId, + ThreadId, TimeInterval, ToolInvocationId, TreeId, TurnId, UseCaseId, UtcMicros, + VectorWatermark, VerifiedCacheGrantSnapshotV1, WatermarkDriftV1, WorkArtifactId, + WorkCancellationRequestId, WorkCommandId, WorkLeaseId, WorkProviderRouteId, + WorkTopologyGenerationRefV1, WorkflowDefinitionId, WorkflowOperationRef, WorkflowOutputName, + WorkflowStepId, WorktreeCaptureAnchorRefV1, WorktreeId, WorktreeInventoryEpoch, + WorktreeInventorySnapshotId, authority_access_policy_digest, canonical_json_bytes, canonical_json_bytes_and_sha256, canonical_json_value, canonical_sha256, decode_with_canonical_digest, derive_exact_observation_anchor_id, - derive_exact_source_occurrence_anchor_id, derive_git_topology_anchor_id, - validate_anchor_lineage_v3, + derive_git_topology_anchor_id, }; pub use resource_policy::host_cpu_target; pub use retrieval::{ diff --git a/crates/tracedecay-domain/src/observation.rs b/crates/tracedecay-domain/src/observation.rs index 4953f5ff27..c88f6995e2 100644 --- a/crates/tracedecay-domain/src/observation.rs +++ b/crates/tracedecay-domain/src/observation.rs @@ -787,22 +787,6 @@ pub struct ObservationSourceCursorV1 { } impl ObservationSourceCursorV1 { - /// Constructs the legacy-compatible file-byte cursor. - pub fn new( - source: ObservationSourceIdentityV1, - scope: ObservationScopeV1, - generation: ObservationSourceGenerationV1, - byte_offset: u64, - ) -> Result { - Self::for_ordering( - source, - scope, - generation, - ObservationOrderingDomainV1::FileBytes, - byte_offset, - ) - } - pub fn for_ordering( source: ObservationSourceIdentityV1, scope: ObservationScopeV1, @@ -1408,6 +1392,10 @@ pub struct CanonicalObservationRelationsV1 { agent_id: Option, #[serde(default, skip_serializing_if = "Option::is_none")] parent_agent_id: Option, + /// The host's id of the parent-session tool call that spawned this + /// session, as recorded by the host on the child side. + #[serde(default, skip_serializing_if = "Option::is_none")] + parent_tool_use_id: Option, } impl CanonicalObservationRelationsV1 { @@ -1421,6 +1409,7 @@ impl CanonicalObservationRelationsV1 { parent_message_id: None, agent_id: None, parent_agent_id: None, + parent_tool_use_id: None, } } @@ -1466,6 +1455,12 @@ impl CanonicalObservationRelationsV1 { self } + #[must_use] + pub fn with_parent_tool_use_id(mut self, parent_tool_use_id: ObservationId) -> Self { + self.parent_tool_use_id = Some(parent_tool_use_id); + self + } + pub fn session_id(&self) -> &SessionId { &self.session_id } @@ -1498,6 +1493,10 @@ impl CanonicalObservationRelationsV1 { self.parent_agent_id.as_ref() } + pub fn parent_tool_use_id(&self) -> Option<&ObservationId> { + self.parent_tool_use_id.as_ref() + } + fn validate(&self) -> Result<(), ObservationContractError> { self.session_id .validate() @@ -1514,6 +1513,7 @@ impl CanonicalObservationRelationsV1 { self.parent_message_id.as_ref(), self.agent_id.as_ref(), self.parent_agent_id.as_ref(), + self.parent_tool_use_id.as_ref(), ] .into_iter() .flatten() @@ -2684,8 +2684,6 @@ impl<'de> Deserialize<'de> for DurableObservationV1 { } } -pub type DurableClaudeObservationV1 = DurableObservationV1; - /// Relationship between an existing record and a candidate retry. #[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)] #[serde(rename_all = "snake_case")] diff --git a/crates/tracedecay-domain/src/research/anchor.rs b/crates/tracedecay-domain/src/research/anchor.rs index 02b8c72a26..a0a9c9d925 100644 --- a/crates/tracedecay-domain/src/research/anchor.rs +++ b/crates/tracedecay-domain/src/research/anchor.rs @@ -6,8 +6,6 @@ use crate::configuration::UserProfileId; use crate::observation::{ CanonicalObservationIdV1, ObservationScopeV1, ObservationSourceGenerationV1, }; -use crate::retrieval::SourceOccurrenceId; -use crate::session_derived::EvidenceSpanIdV1; use super::canonical::canonical_sha256; use super::coverage::{CoverageReportV1, RetentionClass}; @@ -16,7 +14,7 @@ use super::evidence::{EvidenceClass, SanitizationReceiptRefV1}; use super::git_topology::{GitTopologyAnchorTargetV1, GitTopologyGenerationRefV1}; use super::id::{ BlobId, CommitId, PrivacyDomainId, ProjectId, ProjectionGenerationId, RepositoryCaptureId, - RepositoryId, RetrievalAnchorId, RetrieverContributionIdV1, TreeId, + RepositoryId, RetrievalAnchorId, TreeId, }; use super::resolution::ResolutionAuthorizationV1; use super::retrieval::{ @@ -39,7 +37,7 @@ const MAX_ANCHOR_SOURCE_ANCHORS: usize = 256; /// owning stores. #[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)] #[serde(rename_all = "snake_case")] -pub enum NativeAliasKindV2 { +pub enum NativeAliasKind { ProviderRecord, LegacyIdentity, RepositoryRoot, @@ -50,14 +48,14 @@ pub enum NativeAliasKindV2 { #[derive(Clone, Debug, Serialize, PartialEq, Eq, PartialOrd, Ord, Hash)] #[serde(deny_unknown_fields)] -pub struct NativeAliasV2 { - kind: NativeAliasKindV2, +pub struct NativeAlias { + kind: NativeAliasKind, locator_digest: PrivacyDomainBoundLocatorDigest, } -impl NativeAliasV2 { +impl NativeAlias { pub fn new( - kind: NativeAliasKindV2, + kind: NativeAliasKind, locator_digest: PrivacyDomainBoundLocatorDigest, ) -> Result { locator_digest.validate()?; @@ -67,7 +65,7 @@ impl NativeAliasV2 { }) } - pub fn kind(&self) -> NativeAliasKindV2 { + pub fn kind(&self) -> NativeAliasKind { self.kind } @@ -80,7 +78,7 @@ impl NativeAliasV2 { } } -impl<'de> Deserialize<'de> for NativeAliasV2 { +impl<'de> Deserialize<'de> for NativeAlias { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, @@ -88,7 +86,7 @@ impl<'de> Deserialize<'de> for NativeAliasV2 { #[derive(Deserialize)] #[serde(deny_unknown_fields)] struct Wire { - kind: NativeAliasKindV2, + kind: NativeAliasKind, locator_digest: PrivacyDomainBoundLocatorDigest, } @@ -106,7 +104,7 @@ impl<'de> Deserialize<'de> for NativeAliasV2 { rename_all = "snake_case", deny_unknown_fields )] -pub enum RetrievalAnchorTargetV2 { +pub enum RetrievalAnchorTarget { ExactObservation(CanonicalObservationIdV1), Entity(EntityRef), ExactRepositoryCommit { @@ -129,7 +127,7 @@ pub enum RetrievalAnchorTargetV2 { GitTopology(Box), } -/// Exact profile/project and privacy owner for V3 anchors and lineage. +/// Exact profile/project and privacy owner binding. /// /// Ambient paths, labels, store filenames, host profiles, and process state /// cannot fill this identity. @@ -198,15 +196,6 @@ impl AnchorOwnerBindingV1 { } } - fn observation_scope(&self) -> ObservationScopeV1 { - match self { - Self::Profile { .. } => ObservationScopeV1::Profile, - Self::Project { project_id, .. } => ObservationScopeV1::Project { - project_id: project_id.clone(), - }, - } - } - pub fn validate(&self) -> Result<(), DomainError> { self.profile_id().validate()?; if let Some(project_id) = self.project_id() { @@ -258,187 +247,7 @@ impl<'de> Deserialize<'de> for AnchorOwnerBindingV1 { } } -/// Canonical V3 target type for authoritative retrieval anchors. -/// -/// Legacy variants intentionally keep their V2 wire representation. The V3 -/// evidence targets add immutable, payload-free references without changing -/// persisted V2 decoding. -#[derive(Clone, Debug, Serialize, PartialEq, Eq)] -#[serde( - tag = "kind", - content = "target", - rename_all = "snake_case", - deny_unknown_fields -)] -pub enum RetrievalAnchorTargetV3 { - ExactObservation(CanonicalObservationIdV1), - Entity(EntityRef), - ExactRepositoryCommit { - repository_id: RepositoryId, - commit_id: CommitId, - }, - ExactRepositoryTree { - repository_id: RepositoryId, - tree_id: TreeId, - }, - ExactRepositoryBlob { - repository_id: RepositoryId, - blob_id: BlobId, - }, - RepositoryCapture { - repository_id: RepositoryId, - capture_id: RepositoryCaptureId, - receipt: SanitizationReceiptRefV1, - }, - GitTopology(Box), - ExactSourceOccurrence(SourceOccurrenceId), - ExactEvidenceSpan(EvidenceSpanIdV1), - RetrieverContribution(RetrieverContributionIdV1), -} - -impl RetrievalAnchorTargetV3 { - pub fn validate(&self) -> Result<(), DomainError> { - if let Some(legacy) = self.as_v2() { - return legacy.validate(); - } - match self { - Self::ExactSourceOccurrence(occurrence_id) => { - occurrence_id - .validate() - .map_err(|_| DomainError::NonCanonical { - field: "source occurrence anchor target", - }) - } - Self::ExactEvidenceSpan(_) => Ok(()), - Self::RetrieverContribution(contribution_id) => contribution_id.validate(), - _ => unreachable!("legacy targets return before V3 evidence validation"), - } - } - - fn as_v2(&self) -> Option { - Some(match self { - Self::ExactObservation(observation_id) => { - RetrievalAnchorTargetV2::ExactObservation(observation_id.clone()) - } - Self::Entity(entity) => RetrievalAnchorTargetV2::Entity(entity.clone()), - Self::ExactRepositoryCommit { - repository_id, - commit_id, - } => RetrievalAnchorTargetV2::ExactRepositoryCommit { - repository_id: repository_id.clone(), - commit_id: commit_id.clone(), - }, - Self::ExactRepositoryTree { - repository_id, - tree_id, - } => RetrievalAnchorTargetV2::ExactRepositoryTree { - repository_id: repository_id.clone(), - tree_id: tree_id.clone(), - }, - Self::ExactRepositoryBlob { - repository_id, - blob_id, - } => RetrievalAnchorTargetV2::ExactRepositoryBlob { - repository_id: repository_id.clone(), - blob_id: blob_id.clone(), - }, - Self::RepositoryCapture { - repository_id, - capture_id, - receipt, - } => RetrievalAnchorTargetV2::RepositoryCapture { - repository_id: repository_id.clone(), - capture_id: capture_id.clone(), - receipt: receipt.clone(), - }, - Self::GitTopology(target) => RetrievalAnchorTargetV2::GitTopology(target.clone()), - Self::ExactSourceOccurrence(_) - | Self::ExactEvidenceSpan(_) - | Self::RetrieverContribution(_) => return None, - }) - } -} - -impl From for RetrievalAnchorTargetV3 { - fn from(target: RetrievalAnchorTargetV2) -> Self { - match target { - RetrievalAnchorTargetV2::ExactObservation(observation_id) => { - Self::ExactObservation(observation_id) - } - RetrievalAnchorTargetV2::Entity(entity) => Self::Entity(entity), - RetrievalAnchorTargetV2::ExactRepositoryCommit { - repository_id, - commit_id, - } => Self::ExactRepositoryCommit { - repository_id, - commit_id, - }, - RetrievalAnchorTargetV2::ExactRepositoryTree { - repository_id, - tree_id, - } => Self::ExactRepositoryTree { - repository_id, - tree_id, - }, - RetrievalAnchorTargetV2::ExactRepositoryBlob { - repository_id, - blob_id, - } => Self::ExactRepositoryBlob { - repository_id, - blob_id, - }, - RetrievalAnchorTargetV2::RepositoryCapture { - repository_id, - capture_id, - receipt, - } => Self::RepositoryCapture { - repository_id, - capture_id, - receipt, - }, - RetrievalAnchorTargetV2::GitTopology(target) => Self::GitTopology(target), - } - } -} - -impl<'de> Deserialize<'de> for RetrievalAnchorTargetV3 { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - #[derive(Deserialize)] - #[serde( - tag = "kind", - content = "target", - rename_all = "snake_case", - deny_unknown_fields - )] - enum EvidenceWire { - ExactSourceOccurrence(SourceOccurrenceId), - ExactEvidenceSpan(EvidenceSpanIdV1), - RetrieverContribution(RetrieverContributionIdV1), - } - - let value = serde_json::Value::deserialize(deserializer)?; - let target = if let Ok(legacy) = RetrievalAnchorTargetV2::deserialize(&value) { - legacy.into() - } else { - match EvidenceWire::deserialize(value).map_err(serde::de::Error::custom)? { - EvidenceWire::ExactSourceOccurrence(occurrence_id) => { - Self::ExactSourceOccurrence(occurrence_id) - } - EvidenceWire::ExactEvidenceSpan(span_id) => Self::ExactEvidenceSpan(span_id), - EvidenceWire::RetrieverContribution(contribution_id) => { - Self::RetrieverContribution(contribution_id) - } - } - }; - target.validate().map_err(serde::de::Error::custom)?; - Ok(target) - } -} - -impl RetrievalAnchorTargetV2 { +impl RetrievalAnchorTarget { pub fn validate(&self) -> Result<(), DomainError> { match self { Self::ExactObservation(_) => Ok(()), @@ -492,7 +301,7 @@ impl RetrievalAnchorTargetV2 { } } -impl<'de> Deserialize<'de> for RetrievalAnchorTargetV2 { +impl<'de> Deserialize<'de> for RetrievalAnchorTarget { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, @@ -577,7 +386,7 @@ impl<'de> Deserialize<'de> for RetrievalAnchorTargetV2 { rename_all = "snake_case", deny_unknown_fields )] -pub enum AnchorSourceGenerationV2 { +pub enum AnchorSourceGeneration { Observation(ObservationSourceGenerationV1), RepositoryCapture(RepositoryCaptureId), GitTopology(GitTopologyGenerationRefV1), @@ -585,26 +394,24 @@ pub enum AnchorSourceGenerationV2 { Unknown, } -pub type AnchorSourceGenerationV3 = AnchorSourceGenerationV2; - -impl AnchorSourceGenerationV2 { - fn validate_for_target(&self, target: &RetrievalAnchorTargetV2) -> Result<(), DomainError> { +impl AnchorSourceGeneration { + fn validate_for_target(&self, target: &RetrievalAnchorTarget) -> Result<(), DomainError> { let valid = match (self, target) { - (Self::Observation(_), RetrievalAnchorTargetV2::ExactObservation(_)) => true, + (Self::Observation(_), RetrievalAnchorTarget::ExactObservation(_)) => true, ( Self::RepositoryCapture(source), - RetrievalAnchorTargetV2::RepositoryCapture { capture_id, .. }, + RetrievalAnchorTarget::RepositoryCapture { capture_id, .. }, ) => source == capture_id, ( Self::RepositoryCapture(_) | Self::Unavailable | Self::Unknown, - RetrievalAnchorTargetV2::ExactRepositoryCommit { .. } - | RetrievalAnchorTargetV2::ExactRepositoryTree { .. } - | RetrievalAnchorTargetV2::ExactRepositoryBlob { .. }, + RetrievalAnchorTarget::ExactRepositoryCommit { .. } + | RetrievalAnchorTarget::ExactRepositoryTree { .. } + | RetrievalAnchorTarget::ExactRepositoryBlob { .. }, ) => true, - (Self::GitTopology(source), RetrievalAnchorTargetV2::GitTopology(target)) => { + (Self::GitTopology(source), RetrievalAnchorTarget::GitTopology(target)) => { source == &target.generation() } - (_, RetrievalAnchorTargetV2::Entity(_)) => true, + (_, RetrievalAnchorTarget::Entity(_)) => true, _ => false, }; if !valid { @@ -624,7 +431,7 @@ impl AnchorSourceGenerationV2 { #[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)] #[serde(rename_all = "snake_case")] -pub enum AnchorProvenanceRelationV2 { +pub enum AnchorProvenanceRelation { CapturedFrom, Produced, Observed, @@ -641,15 +448,15 @@ pub enum AnchorProvenanceRelationV2 { /// Owner-bound reference to an earlier anchor in the provenance graph. #[derive(Clone, Debug, Serialize, PartialEq, Eq, PartialOrd, Ord, Hash)] #[serde(deny_unknown_fields)] -pub struct AnchorLineageRefV2 { - relation: AnchorProvenanceRelationV2, +pub struct AnchorLineageRef { + relation: AnchorProvenanceRelation, anchor_id: RetrievalAnchorId, owner: ObservationScopeV1, } -impl AnchorLineageRefV2 { +impl AnchorLineageRef { pub fn new( - relation: AnchorProvenanceRelationV2, + relation: AnchorProvenanceRelation, anchor_id: RetrievalAnchorId, owner: ObservationScopeV1, ) -> Result { @@ -662,7 +469,7 @@ impl AnchorLineageRefV2 { }) } - pub fn relation(&self) -> AnchorProvenanceRelationV2 { + pub fn relation(&self) -> AnchorProvenanceRelation { self.relation } @@ -680,7 +487,7 @@ impl AnchorLineageRefV2 { } } -impl<'de> Deserialize<'de> for AnchorLineageRefV2 { +impl<'de> Deserialize<'de> for AnchorLineageRef { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, @@ -688,7 +495,7 @@ impl<'de> Deserialize<'de> for AnchorLineageRefV2 { #[derive(Deserialize)] #[serde(deny_unknown_fields)] struct Wire { - relation: AnchorProvenanceRelationV2, + relation: AnchorProvenanceRelation, anchor_id: RetrievalAnchorId, owner: ObservationScopeV1, } @@ -698,202 +505,115 @@ impl<'de> Deserialize<'de> for AnchorLineageRefV2 { } } -/// Ordered, owner- and privacy-bound lineage for V3 evidence assemblies. -/// -/// `source_ordinal` is assembly order, not chronology. Keeping it in the -/// immutable record prevents sorted V2 lineage from silently replacing -/// lossless cross-source order. -#[derive(Clone, Debug, Serialize, PartialEq, Eq, PartialOrd, Ord, Hash)] -#[serde(deny_unknown_fields)] -pub struct AnchorLineageRefV3 { - source_ordinal: u64, - relation: AnchorProvenanceRelationV2, - anchor_id: RetrievalAnchorId, - owner: AnchorOwnerBindingV1, -} - -impl AnchorLineageRefV3 { - pub fn new( - source_ordinal: u64, - relation: AnchorProvenanceRelationV2, - anchor_id: RetrievalAnchorId, - owner: AnchorOwnerBindingV1, - ) -> Result { - let lineage = Self { - source_ordinal, - relation, - anchor_id, - owner, - }; - lineage.validate()?; - Ok(lineage) - } - - pub const fn source_ordinal(&self) -> u64 { - self.source_ordinal - } - - pub fn anchor_id(&self) -> &RetrievalAnchorId { - &self.anchor_id - } - - pub fn owner(&self) -> &AnchorOwnerBindingV1 { - &self.owner - } - - pub fn validate(&self) -> Result<(), DomainError> { - self.anchor_id.validate()?; - self.owner.validate() - } -} - -impl<'de> Deserialize<'de> for AnchorLineageRefV3 { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - #[derive(Deserialize)] - #[serde(deny_unknown_fields)] - struct Wire { - source_ordinal: u64, - relation: AnchorProvenanceRelationV2, - anchor_id: RetrievalAnchorId, - owner: AnchorOwnerBindingV1, - } - - let wire = Wire::deserialize(deserializer)?; - Self::new( - wire.source_ordinal, - wire.relation, - wire.anchor_id, - wire.owner, - ) - .map_err(serde::de::Error::custom) - } -} - -/// Validate lossless V3 assembly order without inferring chronology. -pub fn validate_anchor_lineage_v3(lineage: &[AnchorLineageRefV3]) -> Result<(), DomainError> { - let mut seen = BTreeSet::new(); - for (expected_ordinal, source) in lineage.iter().enumerate() { - source.validate()?; - if source.source_ordinal - != u64::try_from(expected_ordinal).map_err(|_| DomainError::NonCanonical { - field: "retrieval anchor V3 source lineage order", - })? - { - return Err(DomainError::NonCanonical { - field: "retrieval anchor V3 source lineage order", - }); - } - if !seen.insert((source.anchor_id(), source.owner())) { - return Err(DomainError::DuplicateId { - field: "retrieval anchor V3 source lineage", - }); - } - } - Ok(()) -} - -/// Constructor material for a validated V2 record. `anchor_id` is omitted +/// Constructor material for a validated record. `anchor_id` is omitted /// because it is derived exclusively from the owner and immutable target. #[derive(Clone, Debug)] -pub struct RetrievalAnchorRecordV2Parts { - pub target: RetrievalAnchorTargetV2, +pub struct RetrievalAnchorRecordParts { + pub target: RetrievalAnchorTarget, pub owner: ObservationScopeV1, - pub aliases: Vec, + pub aliases: Vec, pub occurred_at: Option, pub ingested_at: UtcMicros, pub evidence_class: EvidenceClass, - pub source_generation: AnchorSourceGenerationV2, + pub source_generation: AnchorSourceGeneration, pub projection_generation: ProjectionGenerationId, pub projection_watermark: VectorWatermark, pub coverage: CoverageReportV1, pub source_observations: Vec, - pub source_anchors: Vec, + pub source_anchors: Vec, pub authorization: ResolutionAuthorizationV1, pub payload_access: PayloadAccessState, pub retention_class: RetentionClass, pub durability: AnchorDurabilityClass, } +/// The encoded record omits what it can re-derive: empty or default optional +/// fields, and the namespace-constant fields of an `authorization` that is +/// exactly its namespace's derivation. #[derive(Clone, Debug, Serialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] -pub struct RetrievalAnchorRecordV2 { +pub struct RetrievalAnchorRecord { anchor_id: RetrievalAnchorId, - target: RetrievalAnchorTargetV2, + target: RetrievalAnchorTarget, owner: ObservationScopeV1, - aliases: Vec, + #[serde(skip_serializing_if = "Vec::is_empty")] + aliases: Vec, + #[serde(skip_serializing_if = "Option::is_none")] occurred_at: Option, ingested_at: UtcMicros, evidence_class: EvidenceClass, - source_generation: AnchorSourceGenerationV2, + source_generation: AnchorSourceGeneration, projection_generation: ProjectionGenerationId, + #[serde(skip_serializing_if = "watermark_is_default")] projection_watermark: VectorWatermark, + #[serde(skip_serializing_if = "coverage_is_default")] coverage: CoverageReportV1, source_observations: Vec, - source_anchors: Vec, + #[serde(skip_serializing_if = "Vec::is_empty")] + source_anchors: Vec, + #[serde(serialize_with = "serialize_anchor_authorization")] authorization: ResolutionAuthorizationV1, payload_access: PayloadAccessState, retention_class: RetentionClass, durability: AnchorDurabilityClass, } -/// Constructor material for an owner- and privacy-bound V3 anchor record. -/// -/// Source lineage order is authoritative assembly order and is therefore not -/// canonicalized by sorting. -#[derive(Clone, Debug)] -pub struct RetrievalAnchorRecordV3Parts { - pub target: RetrievalAnchorTargetV3, - pub owner: AnchorOwnerBindingV1, - pub aliases: Vec, - pub occurred_at: Option, - pub ingested_at: UtcMicros, - pub evidence_class: EvidenceClass, - pub source_generation: AnchorSourceGenerationV3, - pub projection_generation: ProjectionGenerationId, - pub projection_watermark: VectorWatermark, - pub coverage: CoverageReportV1, - pub source_observations: Vec, - pub source_anchors: Vec, - pub authorization: ResolutionAuthorizationV1, - pub payload_access: PayloadAccessState, - pub retention_class: RetentionClass, - pub durability: AnchorDurabilityClass, +fn watermark_is_default(watermark: &VectorWatermark) -> bool { + *watermark == VectorWatermark::default() } -/// Authoritative V3 record for exact evidence and retriever provenance. -#[derive(Clone, Debug, Serialize, PartialEq, Eq)] +fn coverage_is_default(coverage: &CoverageReportV1) -> bool { + *coverage == CoverageReportV1::default() +} + +/// Stored form of an anchor's authorization: just the namespace and request +/// digest when the rest is that namespace's derivation, the full record +/// otherwise. +#[derive(Deserialize)] +#[serde(untagged)] +enum AnchorAuthorizationWire { + Derived(DerivedAuthorizationWire), + Explicit(ResolutionAuthorizationV1), +} + +#[derive(Serialize, Deserialize)] #[serde(deny_unknown_fields)] -pub struct RetrievalAnchorRecordV3 { - anchor_id: RetrievalAnchorId, - target: RetrievalAnchorTargetV3, - owner: AnchorOwnerBindingV1, - aliases: Vec, - occurred_at: Option, - ingested_at: UtcMicros, - evidence_class: EvidenceClass, - source_generation: AnchorSourceGenerationV3, - projection_generation: ProjectionGenerationId, - projection_watermark: VectorWatermark, - coverage: CoverageReportV1, - source_observations: Vec, - source_anchors: Vec, - authorization: ResolutionAuthorizationV1, - payload_access: PayloadAccessState, - retention_class: RetentionClass, - durability: AnchorDurabilityClass, +struct DerivedAuthorizationWire { + authority: String, + canonical_request_digest: PrivacyDomainBoundLocatorDigest, } -/// Canonical authoritative retrieval-anchor record. -/// -/// Existing product paths remain on the byte-compatible V2 record while V3 -/// evidence assemblies migrate through [`RetrievalAnchorRecordV3`]. -pub type RetrievalAnchorRecord = RetrievalAnchorRecordV2; +fn serialize_anchor_authorization( + authorization: &ResolutionAuthorizationV1, + serializer: S, +) -> Result +where + S: serde::Serializer, +{ + match authorization.derived_authority() { + Some(authority) => DerivedAuthorizationWire { + authority: authority.to_owned(), + canonical_request_digest: authorization.canonical_request_digest.clone(), + } + .serialize(serializer), + None => authorization.serialize(serializer), + } +} -impl RetrievalAnchorRecordV2 { - pub fn new(mut parts: RetrievalAnchorRecordV2Parts) -> Result { +impl AnchorAuthorizationWire { + fn into_authorization(self) -> Result { + match self { + Self::Derived(derived) => ResolutionAuthorizationV1::for_authority( + &derived.authority, + derived.canonical_request_digest, + ), + Self::Explicit(authorization) => Ok(authorization), + } + } +} + +impl RetrievalAnchorRecord { + pub fn new(mut parts: RetrievalAnchorRecordParts) -> Result { validate_collection_bounds(&parts)?; parts.aliases.sort_unstable_by(|left, right| { (left.locator_digest(), left.kind()).cmp(&(right.locator_digest(), right.kind())) @@ -928,7 +648,7 @@ impl RetrievalAnchorRecordV2 { &self.anchor_id } - pub fn target(&self) -> &RetrievalAnchorTargetV2 { + pub fn target(&self) -> &RetrievalAnchorTarget { &self.target } @@ -947,7 +667,7 @@ impl RetrievalAnchorRecordV2 { self.owner_column_json().ok().as_deref() == Some(stored) } - pub fn aliases(&self) -> &[NativeAliasV2] { + pub fn aliases(&self) -> &[NativeAlias] { &self.aliases } @@ -989,7 +709,7 @@ impl RetrievalAnchorRecordV2 { self.evidence_class } - pub fn source_generation(&self) -> &AnchorSourceGenerationV2 { + pub fn source_generation(&self) -> &AnchorSourceGeneration { &self.source_generation } @@ -1009,7 +729,7 @@ impl RetrievalAnchorRecordV2 { &self.source_observations } - pub fn source_anchors(&self) -> &[AnchorLineageRefV2] { + pub fn source_anchors(&self) -> &[AnchorLineageRef] { &self.source_anchors } @@ -1042,7 +762,7 @@ impl RetrievalAnchorRecordV2 { }); } if let ( - RetrievalAnchorTargetV2::GitTopology(target), + RetrievalAnchorTarget::GitTopology(target), ObservationScopeV1::Project { project_id }, ) = (&self.target, &self.owner) && target.project_id() != project_id @@ -1065,7 +785,7 @@ impl RetrievalAnchorRecordV2 { } ensure_unique_aliases(&self.aliases)?; ensure_unique_observations(&self.source_observations)?; - if let RetrievalAnchorTargetV2::ExactObservation(target) = &self.target + if let RetrievalAnchorTarget::ExactObservation(target) = &self.target && !self.source_observations.contains(target) { return Err(DomainError::UnknownReference { @@ -1073,7 +793,7 @@ impl RetrievalAnchorRecordV2 { }); } ensure_unique_lineage(&self.source_anchors)?; - if let RetrievalAnchorTargetV2::GitTopology(target) = &self.target { + if let RetrievalAnchorTarget::GitTopology(target) = &self.target { for expected in target.ordered_sources() { if !self .source_anchors @@ -1105,144 +825,6 @@ impl RetrievalAnchorRecordV2 { } } -impl RetrievalAnchorRecordV3 { - pub fn new(mut parts: RetrievalAnchorRecordV3Parts) -> Result { - validate_collection_bounds_v3(&parts)?; - parts.aliases.sort_unstable_by(|left, right| { - (left.locator_digest(), left.kind()).cmp(&(right.locator_digest(), right.kind())) - }); - parts.source_observations.sort_unstable(); - let anchor_id = derive_v3_anchor_id(&parts.owner, &parts.target)?; - let record = Self { - anchor_id, - target: parts.target, - owner: parts.owner, - aliases: parts.aliases, - occurred_at: parts.occurred_at, - ingested_at: parts.ingested_at, - evidence_class: parts.evidence_class, - source_generation: parts.source_generation, - projection_generation: parts.projection_generation, - projection_watermark: parts.projection_watermark, - coverage: parts.coverage, - source_observations: parts.source_observations, - source_anchors: parts.source_anchors, - authorization: parts.authorization, - payload_access: parts.payload_access, - retention_class: parts.retention_class, - durability: parts.durability, - }; - record.validate()?; - Ok(record) - } - - pub fn anchor_id(&self) -> &RetrievalAnchorId { - &self.anchor_id - } - - pub fn target(&self) -> &RetrievalAnchorTargetV3 { - &self.target - } - - pub fn owner(&self) -> &AnchorOwnerBindingV1 { - &self.owner - } - - pub fn projection_generation(&self) -> &ProjectionGenerationId { - &self.projection_generation - } - - pub fn source_anchors(&self) -> &[AnchorLineageRefV3] { - &self.source_anchors - } - - pub fn validate(&self) -> Result<(), DomainError> { - self.anchor_id.validate()?; - self.target.validate()?; - self.owner.validate()?; - validate_source_generation_v3(&self.source_generation, &self.target)?; - if let Some(legacy) = self.target.as_v2() { - if legacy.requires_project_owner() && self.owner.project_id().is_none() { - return Err(DomainError::UnknownReference { - field: "repository anchor V3 owner", - }); - } - if let RetrievalAnchorTargetV2::GitTopology(target) = legacy - && self.owner.project_id() != Some(target.project_id()) - { - return Err(DomainError::UnknownReference { - field: "git topology anchor V3 project owner", - }); - } - } - if let Some(occurred_at) = &self.occurred_at { - occurred_at.validate()?; - } - self.projection_generation.validate()?; - for shard in self.projection_watermark.components.keys() { - shard.validate()?; - } - self.coverage.validate()?; - self.authorization.validate()?; - if &self.authorization.privacy_domain_id != self.owner.privacy_domain_id() { - return Err(DomainError::UnknownReference { - field: "retrieval anchor V3 authorization owner", - }); - } - for alias in &self.aliases { - alias.validate()?; - } - ensure_unique_aliases(&self.aliases)?; - ensure_unique_observations(&self.source_observations)?; - if let RetrievalAnchorTargetV3::ExactObservation(target) = &self.target - && !self.source_observations.contains(target) - { - return Err(DomainError::UnknownReference { - field: "exact observation source lineage", - }); - } - validate_anchor_lineage_v3(&self.source_anchors)?; - if matches!( - self.target, - RetrievalAnchorTargetV3::ExactSourceOccurrence(_) - | RetrievalAnchorTargetV3::ExactEvidenceSpan(_) - | RetrievalAnchorTargetV3::RetrieverContribution(_) - ) && self.source_anchors.is_empty() - { - return Err(DomainError::UnknownReference { - field: "exact evidence source lineage", - }); - } - if let RetrievalAnchorTargetV3::GitTopology(target) = &self.target { - for expected in target.ordered_sources() { - if !self - .source_anchors - .iter() - .any(|source| source.anchor_id() == &expected.anchor_id) - { - return Err(DomainError::UnknownReference { - field: "git topology ordered source lineage", - }); - } - } - } - for source in &self.source_anchors { - if source.owner() != &self.owner { - return Err(DomainError::UnknownReference { - field: "retrieval anchor V3 lineage owner", - }); - } - if source.anchor_id() == &self.anchor_id { - return Err(DomainError::SelfSupersession); - } - } - if self.anchor_id != derive_v3_anchor_id(&self.owner, &self.target)? { - return Err(DomainError::DigestMismatch); - } - Ok(()) - } -} - /// Derive the canonical retrieval anchor for one durable observation. /// /// Projection generations and rebuild watermarks are deliberately excluded: @@ -1253,33 +835,22 @@ pub fn derive_exact_observation_anchor_id( ) -> Result { derive_anchor_id( owner, - &RetrievalAnchorTargetV2::ExactObservation(observation_id.clone()), + &RetrievalAnchorTarget::ExactObservation(observation_id.clone()), ) } -/// Derive the canonical V3 identity for one immutable Git-topology target. +/// Derive the canonical identity for one immutable Git-topology target. pub fn derive_git_topology_anchor_id( owner: &ObservationScopeV1, target: &GitTopologyAnchorTargetV1, ) -> Result { derive_anchor_id( owner, - &RetrievalAnchorTargetV2::GitTopology(Box::new(target.clone())), + &RetrievalAnchorTarget::GitTopology(Box::new(target.clone())), ) } -/// Derive the canonical public anchor for one exact source occurrence. -pub fn derive_exact_source_occurrence_anchor_id( - owner: &AnchorOwnerBindingV1, - occurrence_id: &SourceOccurrenceId, -) -> Result { - derive_v3_anchor_id( - owner, - &RetrievalAnchorTargetV3::ExactSourceOccurrence(occurrence_id.clone()), - ) -} - -impl<'de> Deserialize<'de> for RetrievalAnchorRecordV2 { +impl<'de> Deserialize<'de> for RetrievalAnchorRecord { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, @@ -1288,74 +859,24 @@ impl<'de> Deserialize<'de> for RetrievalAnchorRecordV2 { #[serde(deny_unknown_fields)] struct Wire { anchor_id: RetrievalAnchorId, - target: RetrievalAnchorTargetV2, + target: RetrievalAnchorTarget, owner: ObservationScopeV1, - aliases: Vec, - occurred_at: Option, - ingested_at: UtcMicros, - evidence_class: EvidenceClass, - source_generation: AnchorSourceGenerationV2, - projection_generation: ProjectionGenerationId, - projection_watermark: VectorWatermark, - coverage: CoverageReportV1, - source_observations: Vec, - source_anchors: Vec, - authorization: ResolutionAuthorizationV1, - payload_access: PayloadAccessState, - retention_class: RetentionClass, - durability: AnchorDurabilityClass, - } - - let wire = Wire::deserialize(deserializer)?; - let claimed_id = wire.anchor_id; - let record = Self::new(RetrievalAnchorRecordV2Parts { - target: wire.target, - owner: wire.owner, - aliases: wire.aliases, - occurred_at: wire.occurred_at, - ingested_at: wire.ingested_at, - evidence_class: wire.evidence_class, - source_generation: wire.source_generation, - projection_generation: wire.projection_generation, - projection_watermark: wire.projection_watermark, - coverage: wire.coverage, - source_observations: wire.source_observations, - source_anchors: wire.source_anchors, - authorization: wire.authorization, - payload_access: wire.payload_access, - retention_class: wire.retention_class, - durability: wire.durability, - }) - .map_err(serde::de::Error::custom)?; - if claimed_id != record.anchor_id { - return Err(serde::de::Error::custom(DomainError::DigestMismatch)); - } - Ok(record) - } -} - -impl<'de> Deserialize<'de> for RetrievalAnchorRecordV3 { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - #[derive(Deserialize)] - #[serde(deny_unknown_fields)] - struct Wire { - anchor_id: RetrievalAnchorId, - target: RetrievalAnchorTargetV3, - owner: AnchorOwnerBindingV1, - aliases: Vec, + #[serde(default)] + aliases: Vec, + #[serde(default)] occurred_at: Option, ingested_at: UtcMicros, evidence_class: EvidenceClass, - source_generation: AnchorSourceGenerationV3, + source_generation: AnchorSourceGeneration, projection_generation: ProjectionGenerationId, + #[serde(default)] projection_watermark: VectorWatermark, + #[serde(default)] coverage: CoverageReportV1, source_observations: Vec, - source_anchors: Vec, - authorization: ResolutionAuthorizationV1, + #[serde(default)] + source_anchors: Vec, + authorization: AnchorAuthorizationWire, payload_access: PayloadAccessState, retention_class: RetentionClass, durability: AnchorDurabilityClass, @@ -1363,7 +884,11 @@ impl<'de> Deserialize<'de> for RetrievalAnchorRecordV3 { let wire = Wire::deserialize(deserializer)?; let claimed_id = wire.anchor_id; - let record = Self::new(RetrievalAnchorRecordV3Parts { + let authorization = wire + .authorization + .into_authorization() + .map_err(serde::de::Error::custom)?; + let record = Self::new(RetrievalAnchorRecordParts { target: wire.target, owner: wire.owner, aliases: wire.aliases, @@ -1376,7 +901,7 @@ impl<'de> Deserialize<'de> for RetrievalAnchorRecordV3 { coverage: wire.coverage, source_observations: wire.source_observations, source_anchors: wire.source_anchors, - authorization: wire.authorization, + authorization, payload_access: wire.payload_access, retention_class: wire.retention_class, durability: wire.durability, @@ -1391,18 +916,18 @@ impl<'de> Deserialize<'de> for RetrievalAnchorRecordV3 { fn derive_anchor_id( owner: &ObservationScopeV1, - target: &RetrievalAnchorTargetV2, + target: &RetrievalAnchorTarget, ) -> Result { #[derive(Serialize)] struct Identity<'a> { domain: &'static str, owner: &'a ObservationScopeV1, - target: &'a RetrievalAnchorTargetV2, + target: &'a RetrievalAnchorTarget, } validate_owner(owner)?; target.validate()?; - let domain = if matches!(target, RetrievalAnchorTargetV2::GitTopology(_)) { + let domain = if matches!(target, RetrievalAnchorTarget::GitTopology(_)) { RETRIEVAL_ANCHOR_V3_ID_DOMAIN } else { RETRIEVAL_ANCHOR_V2_ID_DOMAIN @@ -1412,7 +937,7 @@ fn derive_anchor_id( owner, target, })?; - let version = if matches!(target, RetrievalAnchorTargetV2::GitTopology(_)) { + let version = if matches!(target, RetrievalAnchorTarget::GitTopology(_)) { "v3" } else { "v2" @@ -1420,32 +945,6 @@ fn derive_anchor_id( RetrievalAnchorId::new(format!("retrieval.{version}.{}", digest.as_str())) } -fn derive_v3_anchor_id( - owner: &AnchorOwnerBindingV1, - target: &RetrievalAnchorTargetV3, -) -> Result { - #[derive(Serialize)] - struct Identity<'a> { - domain: &'static str, - owner: &'a AnchorOwnerBindingV1, - target: &'a RetrievalAnchorTargetV3, - } - - owner.validate()?; - target.validate()?; - if !matches!(target, RetrievalAnchorTargetV3::GitTopology(_)) - && let Some(legacy) = target.as_v2() - { - return derive_anchor_id(&owner.observation_scope(), &legacy); - } - let digest = canonical_sha256(&Identity { - domain: RETRIEVAL_ANCHOR_V3_ID_DOMAIN, - owner, - target, - })?; - RetrievalAnchorId::new(format!("retrieval.v3.{}", digest.as_str())) -} - fn validate_owner(owner: &ObservationScopeV1) -> Result<(), DomainError> { owner.validate().map_err(|_| DomainError::UnknownReference { field: "retrieval anchor owner", @@ -1454,7 +953,7 @@ fn validate_owner(owner: &ObservationScopeV1) -> Result<(), DomainError> { use crate::canonical_text::validate_git_object_id; -fn ensure_unique_aliases(aliases: &[NativeAliasV2]) -> Result<(), DomainError> { +fn ensure_unique_aliases(aliases: &[NativeAlias]) -> Result<(), DomainError> { let mut seen = BTreeSet::new(); for alias in aliases { if !seen.insert(alias.locator_digest()) { @@ -1466,7 +965,7 @@ fn ensure_unique_aliases(aliases: &[NativeAliasV2]) -> Result<(), DomainError> { Ok(()) } -fn validate_collection_bounds(parts: &RetrievalAnchorRecordV2Parts) -> Result<(), DomainError> { +fn validate_collection_bounds(parts: &RetrievalAnchorRecordParts) -> Result<(), DomainError> { if parts.aliases.len() > MAX_ANCHOR_ALIASES { return Err(DomainError::NonCanonical { field: "retrieval anchor aliases", @@ -1485,41 +984,6 @@ fn validate_collection_bounds(parts: &RetrievalAnchorRecordV2Parts) -> Result<() Ok(()) } -fn validate_collection_bounds_v3(parts: &RetrievalAnchorRecordV3Parts) -> Result<(), DomainError> { - if parts.aliases.len() > MAX_ANCHOR_ALIASES { - return Err(DomainError::NonCanonical { - field: "retrieval anchor aliases", - }); - } - if parts.source_observations.len() > MAX_ANCHOR_SOURCE_OBSERVATIONS { - return Err(DomainError::NonCanonical { - field: "retrieval anchor source observations", - }); - } - if parts.source_anchors.len() > MAX_ANCHOR_SOURCE_ANCHORS { - return Err(DomainError::NonCanonical { - field: "retrieval anchor V3 source lineage", - }); - } - Ok(()) -} - -fn validate_source_generation_v3( - source: &AnchorSourceGenerationV3, - target: &RetrievalAnchorTargetV3, -) -> Result<(), DomainError> { - if let Some(legacy) = target.as_v2() { - return source.validate_for_target(&legacy); - } - match source { - AnchorSourceGenerationV3::RepositoryCapture(capture_id) => capture_id.validate(), - AnchorSourceGenerationV3::GitTopology(generation) => generation.validate(), - AnchorSourceGenerationV3::Observation(_) - | AnchorSourceGenerationV3::Unavailable - | AnchorSourceGenerationV3::Unknown => Ok(()), - } -} - fn ensure_unique_observations( observations: &[CanonicalObservationIdV1], ) -> Result<(), DomainError> { @@ -1534,7 +998,7 @@ fn ensure_unique_observations( Ok(()) } -fn ensure_unique_lineage(lineage: &[AnchorLineageRefV2]) -> Result<(), DomainError> { +fn ensure_unique_lineage(lineage: &[AnchorLineageRef]) -> Result<(), DomainError> { let mut seen = BTreeSet::new(); if lineage.iter().any(|source| !seen.insert(source)) { return Err(DomainError::DuplicateId { diff --git a/crates/tracedecay-domain/src/research/anchor_test.rs b/crates/tracedecay-domain/src/research/anchor_test.rs index f45c8f4d3b..346e4f6b71 100644 --- a/crates/tracedecay-domain/src/research/anchor_test.rs +++ b/crates/tracedecay-domain/src/research/anchor_test.rs @@ -1,13 +1,10 @@ use serde_json::json; use super::*; -use crate::configuration::UserProfileId; use crate::research::{ AccessPolicyDigest, ComponentVersion, EntityId, EntityKind, PrivacyDomainId, ProjectId, - RetrieverContributionIdV1, SanitizationReceiptId, ScopeResolutionId, + SanitizationReceiptId, ScopeResolutionId, }; -use crate::retrieval::SourceOccurrenceId; -use crate::session_derived::EvidenceSpanIdV1; const DIGEST_A: &str = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; const DIGEST_B: &str = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; @@ -26,15 +23,6 @@ fn owner(project: &str) -> ObservationScopeV1 { } } -fn v3_owner(project: &str, privacy: &str) -> AnchorOwnerBindingV1 { - AnchorOwnerBindingV1::for_project( - UserProfileId::new("profile.fixture").unwrap(), - ProjectId::new(project).unwrap(), - PrivacyDomainId::new(privacy).unwrap(), - ) - .unwrap() -} - fn authorization() -> ResolutionAuthorizationV1 { ResolutionAuthorizationV1 { resolved_scope_id: ScopeResolutionId::new("scope.fixture").unwrap(), @@ -46,14 +34,14 @@ fn authorization() -> ResolutionAuthorizationV1 { } fn record_parts( - target: RetrievalAnchorTargetV2, + target: RetrievalAnchorTarget, owner: ObservationScopeV1, -) -> RetrievalAnchorRecordV2Parts { +) -> RetrievalAnchorRecordParts { let source_observations = match &target { - RetrievalAnchorTargetV2::ExactObservation(id) => vec![id.clone()], + RetrievalAnchorTarget::ExactObservation(id) => vec![id.clone()], _ => vec![observation('c')], }; - RetrievalAnchorRecordV2Parts { + RetrievalAnchorRecordParts { target, owner, aliases: vec![], @@ -63,7 +51,7 @@ fn record_parts( }), ingested_at: UtcMicros(3), evidence_class: EvidenceClass::Observed, - source_generation: AnchorSourceGenerationV2::Observation( + source_generation: AnchorSourceGeneration::Observation( ObservationSourceGenerationV1::new(7).unwrap(), ), projection_generation: ProjectionGenerationId::new("projection.fixture").unwrap(), @@ -78,8 +66,8 @@ fn record_parts( } } -fn entity_target(id: &str) -> RetrievalAnchorTargetV2 { - RetrievalAnchorTargetV2::Entity(EntityRef { +fn entity_target(id: &str) -> RetrievalAnchorTarget { + RetrievalAnchorTarget::Entity(EntityRef { id: EntityId::new(id).unwrap(), kind: EntityKind::Document, }) @@ -88,14 +76,14 @@ fn entity_target(id: &str) -> RetrievalAnchorTargetV2 { #[test] fn assertion_provenance_relations_have_stable_snake_case_wire_values() { for (relation, expected) in [ - (AnchorProvenanceRelationV2::Corrects, "corrects"), - (AnchorProvenanceRelationV2::Contradicts, "contradicts"), - (AnchorProvenanceRelationV2::Supersedes, "supersedes"), - (AnchorProvenanceRelationV2::Supports, "supports"), + (AnchorProvenanceRelation::Corrects, "corrects"), + (AnchorProvenanceRelation::Contradicts, "contradicts"), + (AnchorProvenanceRelation::Supersedes, "supersedes"), + (AnchorProvenanceRelation::Supports, "supports"), ] { assert_eq!(serde_json::to_value(relation).unwrap(), json!(expected)); assert_eq!( - serde_json::from_value::(json!(expected)).unwrap(), + serde_json::from_value::(json!(expected)).unwrap(), relation ); } @@ -103,7 +91,7 @@ fn assertion_provenance_relations_have_stable_snake_case_wire_values() { #[test] fn replay_derives_the_same_anchor_identity() { - let first = RetrievalAnchorRecordV2::new(record_parts( + let first = RetrievalAnchorRecord::new(record_parts( entity_target("document.fixture"), owner("project.fixture"), )) @@ -112,13 +100,13 @@ fn replay_derives_the_same_anchor_identity() { record_parts(entity_target("document.fixture"), owner("project.fixture")); replay_parts.ingested_at = UtcMicros(999); replay_parts.aliases = vec![ - NativeAliasV2::new( - NativeAliasKindV2::Path, + NativeAlias::new( + NativeAliasKind::Path, PrivacyDomainBoundLocatorDigest::new(DIGEST_A).unwrap(), ) .unwrap(), ]; - let replay = RetrievalAnchorRecordV2::new(replay_parts).unwrap(); + let replay = RetrievalAnchorRecord::new(replay_parts).unwrap(); assert_eq!(first.anchor_id(), replay.anchor_id()); } @@ -129,13 +117,13 @@ fn exact_observation_anchor_identity_ignores_projection_generation() { let owner = owner("project.fixture"); let expected = derive_exact_observation_anchor_id(&owner, &observation_id).unwrap(); let mut parts = record_parts( - RetrievalAnchorTargetV2::ExactObservation(observation_id.clone()), + RetrievalAnchorTarget::ExactObservation(observation_id.clone()), owner.clone(), ); parts.source_observations = vec![observation_id]; - let first = RetrievalAnchorRecordV2::new(parts.clone()).unwrap(); + let first = RetrievalAnchorRecord::new(parts.clone()).unwrap(); parts.projection_generation = ProjectionGenerationId::new("projection.rebuilt").unwrap(); - let rebuilt = RetrievalAnchorRecordV2::new(parts).unwrap(); + let rebuilt = RetrievalAnchorRecord::new(parts).unwrap(); assert_eq!(first.anchor_id(), &expected); assert_eq!(rebuilt.anchor_id(), &expected); @@ -143,12 +131,12 @@ fn exact_observation_anchor_identity_ignores_projection_generation() { #[test] fn owner_is_part_of_anchor_identity() { - let first = RetrievalAnchorRecordV2::new(record_parts( + let first = RetrievalAnchorRecord::new(record_parts( entity_target("document.fixture"), owner("project.one"), )) .unwrap(); - let second = RetrievalAnchorRecordV2::new(record_parts( + let second = RetrievalAnchorRecord::new(record_parts( entity_target("document.fixture"), owner("project.two"), )) @@ -157,161 +145,24 @@ fn owner_is_part_of_anchor_identity() { assert_ne!(first.anchor_id(), second.anchor_id()); } -#[test] -fn v3_targets_exact_occurrences_spans_and_contributions() { - let occurrence = SourceOccurrenceId::new("occurrence.fixture").unwrap(); - let span = EvidenceSpanIdV1::new(format!("sha256:{}", "12".repeat(32))).unwrap(); - let contribution = RetrieverContributionIdV1::new("contribution.fixture").unwrap(); - - for (target, expected_kind) in [ - ( - RetrievalAnchorTargetV3::ExactSourceOccurrence(occurrence), - "exact_source_occurrence", - ), - ( - RetrievalAnchorTargetV3::ExactEvidenceSpan(span), - "exact_evidence_span", - ), - ( - RetrievalAnchorTargetV3::RetrieverContribution(contribution), - "retriever_contribution", - ), - ] { - target.validate().unwrap(); - let wire = serde_json::to_value(&target).unwrap(); - assert_eq!(wire["kind"], json!(expected_kind)); - assert_eq!( - serde_json::from_value::(wire).unwrap(), - target - ); - } -} - -#[test] -fn v3_target_decodes_existing_v2_wire_unchanged() { - let v2 = entity_target("document.fixture"); - let v2_wire = serde_json::to_value(&v2).unwrap(); - let v3 = serde_json::from_value::(v2_wire.clone()).unwrap(); - - assert_eq!(serde_json::to_value(&v3).unwrap(), v2_wire); -} - -#[test] -fn v3_exact_evidence_anchor_identity_is_owner_bound() { - let occurrence = SourceOccurrenceId::new("occurrence.fixture").unwrap(); - let first = derive_exact_source_occurrence_anchor_id( - &v3_owner("project.one", "privacy.one"), - &occurrence, - ) - .unwrap(); - let replay = derive_exact_source_occurrence_anchor_id( - &v3_owner("project.one", "privacy.one"), - &occurrence, - ) - .unwrap(); - let other_owner = derive_exact_source_occurrence_anchor_id( - &v3_owner("project.two", "privacy.one"), - &occurrence, - ) - .unwrap(); - let other_privacy = derive_exact_source_occurrence_anchor_id( - &v3_owner("project.one", "privacy.two"), - &occurrence, - ) - .unwrap(); - - assert_eq!(first, replay); - assert_ne!(first, other_owner); - assert_ne!(first, other_privacy); - assert!(first.as_str().starts_with("retrieval.v3.")); -} - -#[test] -fn v3_lineage_preserves_source_order_and_privacy_binding() { - let owner = v3_owner("project.fixture", "privacy.fixture"); - let first = AnchorLineageRefV3::new( - 0, - AnchorProvenanceRelationV2::DerivedFrom, - RetrievalAnchorId::new("retrieval.source.first").unwrap(), - owner.clone(), - ) - .unwrap(); - let second = AnchorLineageRefV3::new( - 1, - AnchorProvenanceRelationV2::DerivedFrom, - RetrievalAnchorId::new("retrieval.source.second").unwrap(), - owner, - ) - .unwrap(); - - validate_anchor_lineage_v3(&[first.clone(), second.clone()]).unwrap(); - assert_eq!(first.source_ordinal(), 0); - assert_eq!(second.source_ordinal(), 1); - assert_eq!( - validate_anchor_lineage_v3(&[second, first]).unwrap_err(), - DomainError::NonCanonical { - field: "retrieval anchor V3 source lineage order" - } - ); -} - -#[test] -fn v3_record_rejects_cross_privacy_authorization() { - let owner = v3_owner("project.fixture", "privacy.other"); - let source = AnchorLineageRefV3::new( - 0, - AnchorProvenanceRelationV2::DerivedFrom, - RetrievalAnchorId::new("retrieval.source.fixture").unwrap(), - owner.clone(), - ) - .unwrap(); - let parts = RetrievalAnchorRecordV3Parts { - target: RetrievalAnchorTargetV3::ExactSourceOccurrence( - SourceOccurrenceId::new("occurrence.fixture").unwrap(), - ), - owner, - aliases: vec![], - occurred_at: None, - ingested_at: UtcMicros(1), - evidence_class: EvidenceClass::Observed, - source_generation: AnchorSourceGenerationV3::Unknown, - projection_generation: ProjectionGenerationId::new("projection.fixture").unwrap(), - projection_watermark: VectorWatermark::default(), - coverage: CoverageReportV1::default(), - source_observations: vec![], - source_anchors: vec![source], - authorization: authorization(), - payload_access: PayloadAccessState::Eligible, - retention_class: RetentionClass::new("retention.fixture").unwrap(), - durability: AnchorDurabilityClass::DurableEvidence, - }; - - assert_eq!( - RetrievalAnchorRecordV3::new(parts).unwrap_err(), - DomainError::UnknownReference { - field: "retrieval anchor V3 authorization owner" - } - ); -} - #[test] fn rejects_alias_digest_collisions_across_alias_kinds() { let mut parts = record_parts(entity_target("document.fixture"), owner("project.fixture")); parts.aliases = vec![ - NativeAliasV2::new( - NativeAliasKindV2::Path, + NativeAlias::new( + NativeAliasKind::Path, PrivacyDomainBoundLocatorDigest::new(DIGEST_A).unwrap(), ) .unwrap(), - NativeAliasV2::new( - NativeAliasKindV2::Ref, + NativeAlias::new( + NativeAliasKind::Ref, PrivacyDomainBoundLocatorDigest::new(DIGEST_A).unwrap(), ) .unwrap(), ]; assert_eq!( - RetrievalAnchorRecordV2::new(parts).unwrap_err(), + RetrievalAnchorRecord::new(parts).unwrap_err(), DomainError::DuplicateId { field: "retrieval anchor aliases" } @@ -320,33 +171,33 @@ fn rejects_alias_digest_collisions_across_alias_kinds() { #[test] fn copied_lineage_does_not_reuse_source_anchor_identity() { - let source = RetrievalAnchorRecordV2::new(record_parts( + let source = RetrievalAnchorRecord::new(record_parts( entity_target("document.source"), owner("project.fixture"), )) .unwrap(); let mut copied_parts = record_parts(entity_target("document.copy"), owner("project.fixture")); copied_parts.source_anchors = vec![ - AnchorLineageRefV2::new( - AnchorProvenanceRelationV2::CopiedFrom, + AnchorLineageRef::new( + AnchorProvenanceRelation::CopiedFrom, source.anchor_id().clone(), owner("project.fixture"), ) .unwrap(), ]; - let copied = RetrievalAnchorRecordV2::new(copied_parts).unwrap(); + let copied = RetrievalAnchorRecord::new(copied_parts).unwrap(); assert_ne!(source.anchor_id(), copied.anchor_id()); assert_eq!( copied.source_anchors()[0].relation(), - AnchorProvenanceRelationV2::CopiedFrom + AnchorProvenanceRelation::CopiedFrom ); } #[test] fn repository_capture_requires_a_project_owner() { let capture_id = RepositoryCaptureId::new("capture.fixture").unwrap(); - let target = RetrievalAnchorTargetV2::RepositoryCapture { + let target = RetrievalAnchorTarget::RepositoryCapture { repository_id: RepositoryId::new("repository.fixture").unwrap(), capture_id: capture_id.clone(), receipt: SanitizationReceiptRefV1::new( @@ -356,24 +207,24 @@ fn repository_capture_requires_a_project_owner() { .unwrap(), }; let mut parts = record_parts(target, ObservationScopeV1::Profile); - parts.source_generation = AnchorSourceGenerationV2::RepositoryCapture(capture_id); + parts.source_generation = AnchorSourceGeneration::RepositoryCapture(capture_id); - assert!(RetrievalAnchorRecordV2::new(parts).is_err()); + assert!(RetrievalAnchorRecord::new(parts).is_err()); } #[test] fn exact_git_targets_require_canonical_object_ids() { let mut parts = record_parts( - RetrievalAnchorTargetV2::ExactRepositoryCommit { + RetrievalAnchorTarget::ExactRepositoryCommit { repository_id: RepositoryId::new("repository.fixture").unwrap(), commit_id: CommitId::new("main").unwrap(), }, owner("project.fixture"), ); - parts.source_generation = AnchorSourceGenerationV2::Unknown; + parts.source_generation = AnchorSourceGeneration::Unknown; assert_eq!( - RetrievalAnchorRecordV2::new(parts).unwrap_err(), + RetrievalAnchorRecord::new(parts).unwrap_err(), DomainError::NonCanonical { field: "retrieval anchor commit" } @@ -390,30 +241,30 @@ fn standalone_target_deserialization_enforces_git_identity() { } }); - assert!(serde_json::from_value::(wire).is_err()); + assert!(serde_json::from_value::(wire).is_err()); } #[test] fn record_canonicalizes_and_bounds_source_collections() { let owner = owner("project.fixture"); - let alias_a = NativeAliasV2::new( - NativeAliasKindV2::Path, + let alias_a = NativeAlias::new( + NativeAliasKind::Path, PrivacyDomainBoundLocatorDigest::new(DIGEST_A).unwrap(), ) .unwrap(); - let alias_b = NativeAliasV2::new( - NativeAliasKindV2::Ref, + let alias_b = NativeAlias::new( + NativeAliasKind::Ref, PrivacyDomainBoundLocatorDigest::new(DIGEST_B).unwrap(), ) .unwrap(); - let source_a = AnchorLineageRefV2::new( - AnchorProvenanceRelationV2::Observed, + let source_a = AnchorLineageRef::new( + AnchorProvenanceRelation::Observed, RetrievalAnchorId::new("retrieval.a").unwrap(), owner.clone(), ) .unwrap(); - let source_b = AnchorLineageRefV2::new( - AnchorProvenanceRelationV2::Observed, + let source_b = AnchorLineageRef::new( + AnchorProvenanceRelation::Observed, RetrievalAnchorId::new("retrieval.b").unwrap(), owner.clone(), ) @@ -422,7 +273,7 @@ fn record_canonicalizes_and_bounds_source_collections() { parts.aliases = vec![alias_b.clone(), alias_a.clone()]; parts.source_observations = vec![observation('b'), observation('a')]; parts.source_anchors = vec![source_b.clone(), source_a.clone()]; - let record = RetrievalAnchorRecordV2::new(parts).unwrap(); + let record = RetrievalAnchorRecord::new(parts).unwrap(); assert_eq!(record.aliases(), &[alias_a.clone(), alias_b]); assert_eq!( @@ -434,7 +285,7 @@ fn record_canonicalizes_and_bounds_source_collections() { let mut aliases = record_parts(entity_target("document.aliases"), owner.clone()); aliases.aliases = vec![alias_a; MAX_ANCHOR_ALIASES + 1]; assert!(matches!( - RetrievalAnchorRecordV2::new(aliases), + RetrievalAnchorRecord::new(aliases), Err(DomainError::NonCanonical { field: "retrieval anchor aliases" }) @@ -443,7 +294,7 @@ fn record_canonicalizes_and_bounds_source_collections() { let mut observations = record_parts(entity_target("document.observations"), owner.clone()); observations.source_observations = vec![observation('a'); MAX_ANCHOR_SOURCE_OBSERVATIONS + 1]; assert!(matches!( - RetrievalAnchorRecordV2::new(observations), + RetrievalAnchorRecord::new(observations), Err(DomainError::NonCanonical { field: "retrieval anchor source observations" }) @@ -452,7 +303,7 @@ fn record_canonicalizes_and_bounds_source_collections() { let mut lineage = record_parts(entity_target("document.lineage"), owner); lineage.source_anchors = vec![source_b; MAX_ANCHOR_SOURCE_ANCHORS + 1]; assert!(matches!( - RetrievalAnchorRecordV2::new(lineage), + RetrievalAnchorRecord::new(lineage), Err(DomainError::NonCanonical { field: "retrieval anchor source lineage" }) @@ -461,7 +312,7 @@ fn record_canonicalizes_and_bounds_source_collections() { #[test] fn repository_capture_requires_the_matching_source_generation() { - let target = RetrievalAnchorTargetV2::RepositoryCapture { + let target = RetrievalAnchorTarget::RepositoryCapture { repository_id: RepositoryId::new("repository.fixture").unwrap(), capture_id: RepositoryCaptureId::new("capture.target").unwrap(), receipt: SanitizationReceiptRefV1::new( @@ -471,12 +322,12 @@ fn repository_capture_requires_the_matching_source_generation() { .unwrap(), }; let mut parts = record_parts(target, owner("project.fixture")); - parts.source_generation = AnchorSourceGenerationV2::RepositoryCapture( + parts.source_generation = AnchorSourceGeneration::RepositoryCapture( RepositoryCaptureId::new("capture.other").unwrap(), ); assert_eq!( - RetrievalAnchorRecordV2::new(parts).unwrap_err(), + RetrievalAnchorRecord::new(parts).unwrap_err(), DomainError::UnknownReference { field: "retrieval anchor source generation" } @@ -485,7 +336,7 @@ fn repository_capture_requires_the_matching_source_generation() { #[test] fn deserialization_rejects_a_tampered_anchor_identity() { - let record = RetrievalAnchorRecordV2::new(record_parts( + let record = RetrievalAnchorRecord::new(record_parts( entity_target("document.fixture"), owner("project.fixture"), )) @@ -493,5 +344,53 @@ fn deserialization_rejects_a_tampered_anchor_identity() { let mut wire = serde_json::to_value(record).unwrap(); wire["anchor_id"] = json!("retrieval.v2.tampered"); - assert!(serde_json::from_value::(wire).is_err()); + assert!(serde_json::from_value::(wire).is_err()); +} + +#[test] +fn stored_anchor_omits_derivable_authorization_and_default_coverage() { + let mut parts = record_parts( + RetrievalAnchorTarget::ExactObservation(observation('a')), + owner("project.fixture"), + ); + parts.authorization = ResolutionAuthorizationV1::for_authority( + "observation-capture.v1", + PrivacyDomainBoundLocatorDigest::new(DIGEST_B).unwrap(), + ) + .unwrap(); + let derived = RetrievalAnchorRecord::new(parts).unwrap(); + let encoded = serde_json::to_value(&derived).unwrap(); + for omitted in [ + "coverage", + "aliases", + "projection_watermark", + "source_anchors", + ] { + assert!(encoded.get(omitted).is_none(), "{omitted}: {encoded}"); + } + assert_eq!( + encoded["authorization"], + json!({"authority": "observation-capture.v1", "canonical_request_digest": DIGEST_B}) + ); + assert_eq!( + serde_json::from_value::(encoded).unwrap(), + derived + ); + + // A fixture authorization is not its namespace's derivation, so it keeps + // every field. + let explicit = RetrievalAnchorRecord::new(record_parts( + RetrievalAnchorTarget::ExactObservation(observation('a')), + owner("project.fixture"), + )) + .unwrap(); + let encoded = serde_json::to_value(&explicit).unwrap(); + assert_eq!( + encoded["authorization"], + serde_json::to_value(authorization()).unwrap() + ); + assert_eq!( + serde_json::from_value::(encoded).unwrap(), + explicit + ); } diff --git a/crates/tracedecay-domain/src/research/id.rs b/crates/tracedecay-domain/src/research/id.rs index ac6db87057..959ab9412e 100644 --- a/crates/tracedecay-domain/src/research/id.rs +++ b/crates/tracedecay-domain/src/research/id.rs @@ -182,10 +182,6 @@ validated_string_newtype!( FactEvidenceId, FactEventId, RetrievalAnchorId, - CanonicalSourceOccurrenceSetIdV1, - RetrieverContributionIdV1, - EvidenceSpanProjectionReceiptIdV1, - EvidenceAssemblyPublicationReceiptIdV1, PrivacyDomainId, ShardId, ActorId, diff --git a/crates/tracedecay-domain/src/research/mod.rs b/crates/tracedecay-domain/src/research/mod.rs index dc2f5e14cc..34a44252c8 100644 --- a/crates/tracedecay-domain/src/research/mod.rs +++ b/crates/tracedecay-domain/src/research/mod.rs @@ -1,8 +1,7 @@ //! Immutable research-provenance and retrieval-anchor contracts. //! -//! This module is a compatibility facade. Ownership-aligned implementation -//! modules remain directly addressable while all existing -//! `tracedecay_domain::research::Type` imports continue to resolve. +//! Each contract family lives in its own submodule; this module re-exports +//! them so `tracedecay_domain::research::Type` is the one import path. pub mod anchor; pub mod branch_stack; diff --git a/crates/tracedecay-domain/src/research/resolution.rs b/crates/tracedecay-domain/src/research/resolution.rs index 5249bc8910..38de9e3289 100644 --- a/crates/tracedecay-domain/src/research/resolution.rs +++ b/crates/tracedecay-domain/src/research/resolution.rs @@ -1,4 +1,6 @@ use std::cmp::Ordering; +use std::collections::HashMap; +use std::sync::{LazyLock, RwLock}; use serde::{Deserialize, Deserializer, Serialize}; @@ -10,6 +12,7 @@ use super::id::{ }; use super::retrieval::{PayloadAccessState, PrivacyDomainBoundLocatorDigest}; use super::watermark::VectorWatermark; +use crate::observation::PayloadReferenceV1; /// Deterministic relationship between an observed store state and the state /// frozen into a retrieval anchor. @@ -84,9 +87,68 @@ impl ResolutionAuthorizationV1 { self.capability_id.validate()?; self.canonical_request_digest.validate() } + + /// The authorization an authority namespace grants one canonical request. + /// Every field but the request digest is a function of the namespace. + pub fn for_authority( + authority: &str, + canonical_request_digest: PrivacyDomainBoundLocatorDigest, + ) -> Result { + Ok(Self { + resolved_scope_id: ScopeResolutionId::new(format!("scope.{authority}"))?, + privacy_domain_id: PrivacyDomainId::new(format!("privacy.{authority}"))?, + access_policy_digest: authority_access_policy_digest(authority)?, + capability_id: CapabilityId::new(format!("capability.{authority}"))?, + canonical_request_digest, + }) + } + + /// The namespace this authorization is the [`Self::for_authority`] + /// derivation of, if it is one. + pub fn derived_authority(&self) -> Option<&str> { + let authority = self.resolved_scope_id.as_str().strip_prefix("scope.")?; + Self::for_authority(authority, self.canonical_request_digest.clone()) + .is_ok_and(|derived| &derived == self) + .then_some(authority) + } +} + +/// Upper bound on memoized access-policy digests. Authority namespaces are +/// compile-time constants in production; the bound only stops a caller with +/// unbounded namespaces from growing the memo. +const MAX_MEMOIZED_ACCESS_POLICY_DIGESTS: usize = 64; + +/// Access-policy digests keyed by authority namespace. The digest binds only +/// the authorization domain and the namespace, so every resolution in one +/// namespace shares it; deriving it per anchor would put a canonical-JSON +/// encode and a SHA-256 on every anchor read. +static ACCESS_POLICY_DIGESTS: LazyLock>> = + LazyLock::new(|| RwLock::new(HashMap::new())); + +/// The access-policy digest an authority namespace binds its resolutions to. +pub fn authority_access_policy_digest(authority: &str) -> Result { + if let Ok(memo) = ACCESS_POLICY_DIGESTS.read() + && let Some(digest) = memo.get(authority) + { + return AccessPolicyDigest::new(digest.clone()); + } + let digest = PayloadReferenceV1::for_payload(&serde_json::json!({ + "domain": "tracedecay.observation-anchor.authorization.v1", + "authority": authority, + })) + .map_err(|error| DomainError::CanonicalSerialization(error.to_string()))? + .digest() + .as_str() + .to_owned(); + if let Ok(mut memo) = ACCESS_POLICY_DIGESTS.write() + && memo.len() < MAX_MEMOIZED_ACCESS_POLICY_DIGESTS + { + memo.insert(authority.to_owned(), digest.clone()); + } + AccessPolicyDigest::new(digest) } -/// Outcome of resolving a V2 anchor. This describes identity resolution and +/// Outcome of resolving an anchor. This describes identity resolution and /// freshness, while [`PayloadAccessState`] independently describes whether the /// retained payload may be accessed. #[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)] diff --git a/crates/tracedecay-domain/src/session/context.rs b/crates/tracedecay-domain/src/session/context.rs index 11a56a9bf0..b056644ff2 100644 --- a/crates/tracedecay-domain/src/session/context.rs +++ b/crates/tracedecay-domain/src/session/context.rs @@ -22,7 +22,7 @@ pub enum HydrationStateV1 { RetentionExpired, Unauthorized, Locked, - UnverifiableLegacy, + Unverifiable, } impl HydrationStateV1 { @@ -35,7 +35,7 @@ impl HydrationStateV1 { Self::RetentionExpired => "retention_expired", Self::Unauthorized => "unauthorized", Self::Locked => "locked", - Self::UnverifiableLegacy => "unverifiable_legacy", + Self::Unverifiable => "unverifiable", } } } diff --git a/crates/tracedecay-domain/tests/domain_suite/canonical_identity_wire_stability.rs b/crates/tracedecay-domain/tests/domain_suite/canonical_identity_wire_stability.rs index 563eae6e76..32c0fc9a1a 100644 --- a/crates/tracedecay-domain/tests/domain_suite/canonical_identity_wire_stability.rs +++ b/crates/tracedecay-domain/tests/domain_suite/canonical_identity_wire_stability.rs @@ -7,7 +7,7 @@ use tracedecay_domain::code_intelligence::{CodeGenerationId, ContentDigest}; use tracedecay_domain::configuration::UserProfileId; -use tracedecay_domain::feedback::{FeedbackCycleId, GitHubReviewIdV1, ProximityWarningIdV1}; +use tracedecay_domain::feedback::{FeedbackCycleId, GitHubReviewIdV1, ProximityContributionIdV1}; use tracedecay_domain::observation::CanonicalObservationIdV1; use tracedecay_domain::research::{DomainError, EntityId, canonical_sha256}; use tracedecay_domain::retrieval::{PrincipalId, RetrievalContractError}; @@ -38,7 +38,7 @@ fn identity_families_serialize_transparently() { "\"cycle-1\"", ), ( - serde_json::to_string(&ProximityWarningIdV1::new("warn-1").unwrap()).unwrap(), + serde_json::to_string(&ProximityContributionIdV1::new("warn-1").unwrap()).unwrap(), "\"warn-1\"", ), ( @@ -61,7 +61,7 @@ fn identity_families_digest_is_stable() { PrincipalId::new("principal-1").unwrap(), UserProfileId::new("profile-1").unwrap(), FeedbackCycleId::new("cycle-1").unwrap(), - ProximityWarningIdV1::new("warn-1").unwrap(), + ProximityContributionIdV1::new("warn-1").unwrap(), GitHubReviewIdV1::new("review-1").unwrap(), )) .unwrap(); @@ -149,8 +149,8 @@ fn identity_families_reject_the_same_values() { "{bad:?}" ); assert_eq!( - ProximityWarningIdV1::new(bad).unwrap_err(), - domain("proximity warning id"), + ProximityContributionIdV1::new(bad).unwrap_err(), + domain("proximity contribution id"), "{bad:?}" ); assert_eq!( diff --git a/crates/tracedecay-domain/tests/domain_suite/git_topology_anchor_contract.rs b/crates/tracedecay-domain/tests/domain_suite/git_topology_anchor_contract.rs index 85e8afa664..8fafa12349 100644 --- a/crates/tracedecay-domain/tests/domain_suite/git_topology_anchor_contract.rs +++ b/crates/tracedecay-domain/tests/domain_suite/git_topology_anchor_contract.rs @@ -1,8 +1,8 @@ use std::collections::BTreeMap; use tracedecay_domain::{ - AccessPolicyDigest, AnchorDurabilityClass, AnchorLineageRefV2, AnchorProvenanceRelationV2, - AnchorSourceGenerationV2, CapabilityId, CheckSnapshotAnchorRefV1, CiFailureBranchEvidenceV1, + AccessPolicyDigest, AnchorDurabilityClass, AnchorLineageRef, AnchorProvenanceRelation, + AnchorSourceGeneration, CapabilityId, CheckSnapshotAnchorRefV1, CiFailureBranchEvidenceV1, CiFailureCoverageV1, CiFailureGenerationEvidenceV1, CiFailureKindV1, CiFailureLocalizationResultV1, CiFailureLocalizationStateV1, CiFailureParserIdentityV1, CiFailureRunIdentityV1, CommitId, CoverageReportV1, EvidenceAvailabilityV1, EvidenceClass, @@ -21,10 +21,10 @@ use tracedecay_domain::{ RefSnapshotKindV1, RepositoryCaptureAnchorRefV1, RepositoryDirtyStateV1, RepositoryEvidenceV1, RepositoryId, RepositoryIndexSnapshotV1, RepositoryIndexStateV1, RepositoryProvenanceV1, RepositoryRemoteIdentityV1, RepositoryStateSnapshotV1, RepositoryWorkingTreeSnapshotV1, - RepositoryWorkingTreeStateV1, ResolutionAuthorizationV1, RetentionClass, - RetrievalAnchorRecordV2, RetrievalAnchorRecordV2Parts, RetrievalAnchorTargetV2, - ScopeResolutionId, ShardId, UtcMicros, VectorWatermark, WorktreeCaptureAnchorRefV1, WorktreeId, - canonical_sha256, derive_git_topology_anchor_id, + RepositoryWorkingTreeStateV1, ResolutionAuthorizationV1, RetentionClass, RetrievalAnchorRecord, + RetrievalAnchorRecordParts, RetrievalAnchorTarget, ScopeResolutionId, ShardId, UtcMicros, + VectorWatermark, WorktreeCaptureAnchorRefV1, WorktreeId, canonical_sha256, + derive_git_topology_anchor_id, }; use tracedecay_domain::test_fixtures::id; @@ -113,7 +113,7 @@ fn authorization() -> ResolutionAuthorizationV1 { } } -fn record(target: GitTopologyAnchorTargetV1) -> RetrievalAnchorRecordV2 { +fn record(target: GitTopologyAnchorTargetV1) -> RetrievalAnchorRecord { let owner = ObservationScopeV1::Project { project_id: id("project.fixture"), }; @@ -123,17 +123,17 @@ fn record(target: GitTopologyAnchorTargetV1) -> RetrievalAnchorRecordV2 { .ordered_sources() .iter() .map(|source| { - AnchorLineageRefV2::new( - AnchorProvenanceRelationV2::Observed, + AnchorLineageRef::new( + AnchorProvenanceRelation::Observed, source.anchor_id.clone(), owner.clone(), ) .expect("ordered source lineage is canonical") }) .collect(); - RetrievalAnchorRecordV2::new(RetrievalAnchorRecordV2Parts { - source_generation: AnchorSourceGenerationV2::GitTopology(target.generation()), - target: RetrievalAnchorTargetV2::GitTopology(Box::new(target)), + RetrievalAnchorRecord::new(RetrievalAnchorRecordParts { + source_generation: AnchorSourceGeneration::GitTopology(target.generation()), + target: RetrievalAnchorTarget::GitTopology(Box::new(target)), owner, aliases: vec![], occurred_at: None, @@ -181,7 +181,7 @@ fn worktree_snapshot_anchor_rekeys_on_exact_generation_change() { let mut tampered = serde_json::to_value(&first_record).unwrap(); tampered["source_generation"]["generation"]["binding"]["snapshot_id"] = serde_json::to_value(second_snapshot_id).unwrap(); - assert!(serde_json::from_value::(tampered).is_err()); + assert!(serde_json::from_value::(tampered).is_err()); } #[test] diff --git a/crates/tracedecay-domain/tests/domain_suite/integration_catalog_contract.rs b/crates/tracedecay-domain/tests/domain_suite/integration_catalog_contract.rs index 369eabaf34..40e57dc413 100644 --- a/crates/tracedecay-domain/tests/domain_suite/integration_catalog_contract.rs +++ b/crates/tracedecay-domain/tests/domain_suite/integration_catalog_contract.rs @@ -193,30 +193,6 @@ fn stable_direct_host_integration_ids_match_provider_ids() { #[test] fn stock_host_kinds_project_only_fixture_backed_observation_integrations() { - assert_eq!( - HostKindV1::ALL.map(|host| serde_json::to_value(host).unwrap()), - [ - "claude_code", - "cursor_desktop", - "cursor_cloud", - "codex", - "hermes", - "kiro", - "cline_family", - "cline", - "roo_code", - "kilo", - "kimi_code", - "open_code", - "gemini", - "copilot", - "devin", - "zed", - "antigravity", - "vibe", - ] - .map(Value::from) - ); assert_eq!( HostKindV1::ClaudeCode.fixture_backed_observation_integration_id(), Some(HostIntegrationIdV1::Claude) diff --git a/crates/tracedecay-domain/tests/domain_suite/observation_contract.rs b/crates/tracedecay-domain/tests/domain_suite/observation_contract.rs index ce9941a5b1..c51f1760de 100644 --- a/crates/tracedecay-domain/tests/domain_suite/observation_contract.rs +++ b/crates/tracedecay-domain/tests/domain_suite/observation_contract.rs @@ -8,8 +8,8 @@ use tracedecay_domain::{ CanonicalClaudeSanitizationReceiptMaterialV1, CanonicalMessageRoleV1, CanonicalObservationEnvelopeV1, CanonicalObservationEvidenceV1, CanonicalObservationFactV1, CanonicalObservationIdV1, CanonicalObservationRelationsV1, CanonicalReasoningVisibilityV1, - CanonicalWorkflowSemanticKindV1, ClineTranscriptStream, ComponentVersion, - DurableClaudeObservationV1, MAX_CANONICAL_OBSERVATION_FACTS_V1, MAX_OBSERVATION_RECORD_BYTES, + CanonicalWorkflowSemanticKindV1, ClineTranscriptStream, ComponentVersion, DurableObservationV1, + MAX_CANONICAL_OBSERVATION_FACTS_V1, MAX_OBSERVATION_RECORD_BYTES, MAX_OBSERVATION_STRUCTURE_DEPTH, MAX_OBSERVATION_STRUCTURE_VALUES, ObservationCollisionOutcomeV1, ObservationContractError, ObservationId, ObservationIdentityMaterialV1, ObservationOrderingDomainV1, ObservationPositionalOccurrenceV1, @@ -61,8 +61,8 @@ fn accepted_receipt(payload: &Value) -> SanitizationReceiptV1 { .unwrap() } -fn durable(material: ObservationIdentityMaterialV1, payload: Value) -> DurableClaudeObservationV1 { - DurableClaudeObservationV1::new( +fn durable(material: ObservationIdentityMaterialV1, payload: Value) -> DurableObservationV1 { + DurableObservationV1::new( material, accepted_receipt(&payload), RetentionClass::new("transcript.fixture").unwrap(), @@ -588,10 +588,6 @@ fn receipt_derivation_is_canonical_and_generation_bound() { "privacy.claude.v1.2ef774a1d81493c05616a42ac8cf08856f230c7aa4f4e9d8224512d05ded88a8" ); assert_eq!(receipt.sanitizer_version().as_str(), "sanitizer.fixture.v1"); - assert_eq!(SanitizerDispositionV1::Accepted.as_str(), "accepted"); - assert_eq!(SanitizerDispositionV1::Redacted.as_str(), "redacted"); - assert_eq!(SanitizerDispositionV1::Rejected.as_str(), "rejected"); - assert_eq!(SanitizerDispositionV1::Quarantined.as_str(), "quarantined"); let changed_generation = ObservationIdentityMaterialV1::new( source("session.fixture"), @@ -639,12 +635,12 @@ fn idempotency_wire_field_is_a_canonical_identity_alias() { legacy_wire["idempotency_key"] = Value::String( "sha256:13b3a18339fe0dbf5a1ccc894e24cf1626ca88babef32869bf7dc85f6a626abb".to_owned(), ); - let decoded: DurableClaudeObservationV1 = serde_json::from_value(legacy_wire).unwrap(); + let decoded: DurableObservationV1 = serde_json::from_value(legacy_wire).unwrap(); assert_eq!(decoded.idempotency_key(), decoded.observation_id()); let mut invalid_wire = wire; invalid_wire["idempotency_key"] = Value::String(format!("sha256:{}", "0".repeat(64))); - assert!(serde_json::from_value::(invalid_wire).is_err()); + assert!(serde_json::from_value::(invalid_wire).is_err()); } #[test] @@ -677,7 +673,14 @@ fn scope_participates_in_identity_and_invalid_positions_are_rejected() { fn source_cursors_enforce_their_comparison_domain() { let generation = ObservationSourceGenerationV1::new(2).unwrap(); let byte_cursor = |session: &str, scope, generation, offset| { - ObservationSourceCursorV1::new(source(session), scope, generation, offset).unwrap() + ObservationSourceCursorV1::for_ordering( + source(session), + scope, + generation, + ObservationOrderingDomainV1::FileBytes, + offset, + ) + .unwrap() }; let first = byte_cursor( "session.fixture", @@ -741,10 +744,11 @@ fn source_cursors_enforce_their_comparison_domain() { #[test] fn source_cursor_resume_checkpoints_round_trip_without_breaking_legacy_json() { - let legacy = ObservationSourceCursorV1::new( + let legacy = ObservationSourceCursorV1::for_ordering( source("session.fixture"), ObservationScopeV1::Profile, ObservationSourceGenerationV1::new(2).unwrap(), + ObservationOrderingDomainV1::FileBytes, 20, ) .unwrap(); @@ -805,7 +809,7 @@ fn receipts_and_durable_observations_enforce_sanitization_binding() { SanitizationReceiptV1::new(receipt_ref(), disposition, SensitivityV1::Sensitive, None) .unwrap(); assert!( - DurableClaudeObservationV1::new( + DurableObservationV1::new( profile_material(), receipt, RetentionClass::new("transcript.fixture").unwrap(), @@ -820,7 +824,7 @@ fn receipts_and_durable_observations_enforce_sanitization_binding() { json!({"message": "longer value"}), ] { assert!( - DurableClaudeObservationV1::new( + DurableObservationV1::new( profile_material(), accepted_receipt(&payload), RetentionClass::new("transcript.fixture").unwrap(), @@ -845,7 +849,7 @@ fn durable_round_trip_preserves_unknown_provider_evidence_and_canonical_bytes() let observation = durable(profile_material(), payload.clone()); let canonical = observation.canonical_payload_bytes().unwrap(); let encoded = serde_json::to_vec(&observation).unwrap(); - let decoded: DurableClaudeObservationV1 = serde_json::from_slice(&encoded).unwrap(); + let decoded: DurableObservationV1 = serde_json::from_slice(&encoded).unwrap(); assert_eq!(decoded.identity(), observation.identity()); assert_eq!(decoded.receipt(), observation.receipt()); @@ -939,7 +943,7 @@ fn durable_observations_written_before_native_identity_still_decode() { wire["observation_id"] = json!(legacy_observation_id); wire["idempotency_key"] = json!(legacy_observation_id); - let decoded: DurableClaudeObservationV1 = + let decoded: DurableObservationV1 = serde_json::from_value(wire.clone()).expect("a pre-change row must still decode"); assert_eq!(decoded.identity(), observation.identity()); assert_eq!(decoded.payload(), &payload); @@ -950,7 +954,7 @@ fn durable_observations_written_before_native_identity_still_decode() { let mut forged = wire.clone(); forged[field] = arbitrary.clone(); assert!( - serde_json::from_value::(forged).is_err(), + serde_json::from_value::(forged).is_err(), "an id matching no derivation must still be rejected in {field}" ); } @@ -982,7 +986,7 @@ fn durable_observations_decode_under_every_historical_derivation() { let mut row = wire.clone(); row["observation_id"] = json!(id); row["idempotency_key"] = json!(id); - let decoded: DurableClaudeObservationV1 = serde_json::from_value(row) + let decoded: DurableObservationV1 = serde_json::from_value(row) .unwrap_or_else(|error| panic!("a row derived as {id} must decode: {error}")); assert_eq!(decoded.identity(), observation.identity()); } @@ -1046,7 +1050,7 @@ fn decoded_observations_report_the_identity_they_are_stored_under() { let mut row = wire.clone(); row["observation_id"] = json!(stored_id); row["idempotency_key"] = json!(stored_id); - let decoded: DurableClaudeObservationV1 = + let decoded: DurableObservationV1 = serde_json::from_value(row).expect("a row under any accepted derivation must decode"); assert_eq!( @@ -1084,7 +1088,7 @@ fn cline_transition_observation( stream: ClineTranscriptStream, native_source: bool, range: ObservationSourceRangeV1, -) -> DurableClaudeObservationV1 { +) -> DurableObservationV1 { let source = if native_source { stream .source_identity( @@ -1161,10 +1165,10 @@ fn cline_transition_observation( } fn cline_payload_change( - observation: &DurableClaudeObservationV1, + observation: &DurableObservationV1, pointer: &str, replacement: Value, -) -> DurableClaudeObservationV1 { +) -> DurableObservationV1 { let mut payload = observation.payload().clone(); *payload.pointer_mut(pointer).unwrap() = replacement; durable(observation.identity().clone(), payload) @@ -1399,7 +1403,7 @@ fn cline_native_transition_preserves_sanitization_authority() { Some(new.payload_reference().clone()), ) .unwrap(); - let changed = DurableClaudeObservationV1::new( + let changed = DurableObservationV1::new( new.identity().clone(), receipt, new.retention_class().clone(), diff --git a/crates/tracedecay-domain/tests/session_contract.rs b/crates/tracedecay-domain/tests/session_contract.rs index f34b65c816..3980c2ae07 100644 --- a/crates/tracedecay-domain/tests/session_contract.rs +++ b/crates/tracedecay-domain/tests/session_contract.rs @@ -18,19 +18,13 @@ use tracedecay_domain::{ TemporalValidityV1, UtcMicros, }; -fn assert_json_round_trip(value: T) +/// Pins the literal wire form and proves that literal decodes to the value. +fn assert_wire(value: &T, wire: Value) where T: Serialize + DeserializeOwned + PartialEq + Debug, { - let encoded = serde_json::to_value(&value).unwrap(); - let decoded = serde_json::from_value::(encoded).unwrap(); - assert_eq!(decoded, value); -} - -macro_rules! assert_json_round_trip { - ($value:expr) => { - assert_json_round_trip($value); - }; + assert_eq!(serde_json::to_value(value).unwrap(), wire); + assert_eq!(&serde_json::from_value::(wire).unwrap(), value); } fn observation_id() -> CanonicalObservationIdV1 { @@ -69,16 +63,19 @@ fn evidence() -> SessionEvidenceMetadataV1 { serde_json::from_value(evidence_wire("provider_declared")).unwrap() } -fn summary_publication() -> SummaryPublicationMetadataV1 { - serde_json::from_value(json!({ +fn summary_publication_wire() -> Value { + json!({ "model_route": "summary.model.fixture", "configuration_digest": format!("sha256:{}", "3".repeat(64)), "sanitization_receipt": { "receipt_id": "receipt.fixture", "sanitizer_version": "sanitizer.fixture" } - })) - .unwrap() + }) +} + +fn summary_publication() -> SummaryPublicationMetadataV1 { + serde_json::from_value(summary_publication_wire()).unwrap() } fn occurrence_record_wire() -> Value { @@ -160,7 +157,7 @@ fn temporal_modes_round_trip_and_unknown_valid_time_is_not_representative_as_of( fn exact_byte_ranges_are_canonical_half_open_domain_values() { let range = ByteRangeV1::new(3, 11).expect("ordered non-empty byte range"); assert_eq!((range.start(), range.end()), (3, 11)); - assert_json_round_trip!(range); + assert_wire(&range, json!({"start": 3, "end": 11})); assert_eq!( ByteRangeV1::new(3, 3), Err(SessionContractError::InvalidByteRange) @@ -194,25 +191,41 @@ fn exact_byte_range_deserialization_rejects_invalid_domain_values() { } #[test] -fn temporal_values_round_trip_every_variant_and_reject_unknown_variants() { - for mode in [ - TemporalModeV1::Current, - TemporalModeV1::AsOf { - cutoff: UtcMicros(50), - }, - TemporalModeV1::Evolution, - TemporalModeV1::Forensic, +fn temporal_values_have_literal_wire_forms_and_reject_unknown_variants() { + for (mode, kind, wire) in [ + ( + TemporalModeV1::Current, + "current", + json!({"kind": "current"}), + ), + ( + TemporalModeV1::AsOf { + cutoff: UtcMicros(50), + }, + "as_of", + json!({"kind": "as_of", "cutoff": 50}), + ), + ( + TemporalModeV1::Evolution, + "evolution", + json!({"kind": "evolution"}), + ), + ( + TemporalModeV1::Forensic, + "forensic", + json!({"kind": "forensic"}), + ), ] { - assert_json_round_trip!(mode); + assert_wire(&mode, wire); + assert_eq!(mode.as_str(), kind); } - for validity in [ - TemporalValidityV1::Known { + assert_wire( + &TemporalValidityV1::Known { valid_at: UtcMicros(40), }, - TemporalValidityV1::Unknown, - ] { - assert_json_round_trip!(validity); - } + json!({"kind": "known", "valid_at": 40}), + ); + assert_wire(&TemporalValidityV1::Unknown, json!({"kind": "unknown"})); assert!(serde_json::from_value::(json!({"kind": "future"})).is_err()); assert!(serde_json::from_value::(json!({"kind": "future"})).is_err()); } @@ -222,22 +235,43 @@ fn copy_proofs_and_copy_records_round_trip_and_reject_invalid_links() { let source = occurrence(0); let target = occurrence(1); let proofs = [ - CopyProofV1::ProviderLinkage { - source_occurrence_id: source.clone(), - provider_record_id: ObservationId::new("provider.message.1").unwrap(), - }, - CopyProofV1::ParentMessageLinkage { - source_occurrence_id: source.clone(), - parent_message_id: MessageId::new("message.parent.1").unwrap(), - }, - CopyProofV1::ExplicitAnchorAssertion { - source_occurrence_id: source.clone(), - assertion_anchor_id: anchor("anchor.copy.proof"), - }, + ( + CopyProofV1::ProviderLinkage { + source_occurrence_id: source.clone(), + provider_record_id: ObservationId::new("provider.message.1").unwrap(), + }, + json!({ + "kind": "provider_linkage", + "source_occurrence_id": source, + "provider_record_id": "provider.message.1" + }), + ), + ( + CopyProofV1::ParentMessageLinkage { + source_occurrence_id: source.clone(), + parent_message_id: MessageId::new("message.parent.1").unwrap(), + }, + json!({ + "kind": "parent_message_linkage", + "source_occurrence_id": source, + "parent_message_id": "message.parent.1" + }), + ), + ( + CopyProofV1::ExplicitAnchorAssertion { + source_occurrence_id: source.clone(), + assertion_anchor_id: anchor("anchor.copy.proof"), + }, + json!({ + "kind": "explicit_anchor_assertion", + "source_occurrence_id": source, + "assertion_anchor_id": "anchor.copy.proof" + }), + ), ]; - for proof in proofs { + for (proof, proof_wire) in proofs { assert_eq!(proof.source_occurrence_id(), &source); - assert_json_round_trip!(proof.clone()); + assert_wire(&proof, proof_wire.clone()); let copy = LogicalCopyRecordV1 { occurrence_id: target.clone(), copied_from_occurrence_id: source.clone(), @@ -246,7 +280,16 @@ fn copy_proofs_and_copy_records_round_trip_and_reject_invalid_links() { valid_time: TemporalValidityV1::Unknown, }; copy.validate().unwrap(); - assert_json_round_trip!(copy); + assert_wire( + ©, + json!({ + "occurrence_id": target, + "copied_from_occurrence_id": source, + "proof": proof_wire, + "knowledge_at": 50, + "valid_time": {"kind": "unknown"} + }), + ); } assert!( serde_json::from_value::(json!({ @@ -268,7 +311,6 @@ fn copy_proofs_and_copy_records_round_trip_and_reject_invalid_links() { valid_time: TemporalValidityV1::Unknown, }; copy.validate().unwrap(); - assert_json_round_trip!(copy.clone()); let self_copy = LogicalCopyRecordV1 { occurrence_id: target.clone(), @@ -351,15 +393,24 @@ fn summaries_canonicalize_sources_and_reject_self_predecessors() { serde_json::to_value(&canonical).unwrap(), serde_json::to_value(&reordered).unwrap() ); - assert_json_round_trip!(canonical.clone()); - assert_json_round_trip!(SummarySourceHorizonV1 { - knowledge_through: UtcMicros(50), - valid_through: None, - }); - assert_json_round_trip!(SummarySourceHorizonV1 { - knowledge_through: UtcMicros(50), - valid_through: Some(UtcMicros(50)), + let canonical_wire = json!({ + "summary_id": "summary.fixture", + "session_id": "session.fixture", + "summary_anchor_id": "anchor.summary", + "source_anchors": ["anchor.source.a", "anchor.source.b"], + "source_horizon": {"knowledge_through": 50, "valid_through": 40}, + "created_at": 60, + "predecessor_summary_id": null, + "publication": null }); + assert_wire(&canonical, canonical_wire.clone()); + assert_wire( + &SummarySourceHorizonV1 { + knowledge_through: UtcMicros(50), + valid_through: None, + }, + json!({"knowledge_through": 50, "valid_through": null}), + ); let empty = SessionSummaryRecordV1::new( SessionSummaryIdV1::new("summary.empty").unwrap(), @@ -414,7 +465,19 @@ fn summaries_canonicalize_sources_and_reject_self_predecessors() { UtcMicros(70), ) .expect("valid time may extend beyond knowledge time"); - assert_json_round_trip!(future_effective_horizon); + assert_wire( + &future_effective_horizon, + json!({ + "summary_id": "summary.future-effective-horizon", + "session_id": "session.fixture", + "summary_anchor_id": "anchor.summary.future-effective-horizon", + "source_anchors": ["anchor.source.future-effective-horizon"], + "source_horizon": {"knowledge_through": 50, "valid_through": 60}, + "created_at": 70, + "predecessor_summary_id": null, + "publication": null + }), + ); assert!( serde_json::from_value::(json!({ "knowledge_through": 50 @@ -432,12 +495,17 @@ fn summaries_canonicalize_sources_and_reject_self_predecessors() { .clone() .with_predecessor(SessionSummaryIdV1::new("summary.predecessor").unwrap()) .unwrap(); - assert_json_round_trip!(predecessor); - assert_json_round_trip!( - canonical + let mut predecessor_wire = canonical_wire.clone(); + predecessor_wire["predecessor_summary_id"] = json!("summary.predecessor"); + assert_wire(&predecessor, predecessor_wire); + let mut published_wire = canonical_wire; + published_wire["publication"] = summary_publication_wire(); + assert_wire( + &canonical .clone() .with_publication(summary_publication()) - .unwrap() + .unwrap(), + published_wire, ); let mut self_predecessor = serde_json::to_value(canonical).unwrap(); @@ -447,17 +515,32 @@ fn summaries_canonicalize_sources_and_reject_self_predecessors() { #[test] fn typed_ids_and_signed_cursors_round_trip_and_reject_invalid_values() { - assert_json_round_trip!(SessionSummaryIdV1::new("summary.fixture").unwrap()); - assert_json_round_trip!(TemporalAssertionIdV1::new("assertion.fixture").unwrap()); - assert_json_round_trip!(SessionRefreshOperationIdV1::new("refresh.fixture").unwrap()); - assert_json_round_trip!(SessionProjectionGenerationV1::new(1).unwrap()); - assert_json_round_trip!(SessionCursorKeyIdV1::new("cursor.key.fixture").unwrap()); + assert_wire( + &SessionSummaryIdV1::new("summary.fixture").unwrap(), + json!("summary.fixture"), + ); + assert_wire( + &TemporalAssertionIdV1::new("assertion.fixture").unwrap(), + json!("assertion.fixture"), + ); + assert_wire( + &SessionRefreshOperationIdV1::new("refresh.fixture").unwrap(), + json!("refresh.fixture"), + ); + assert_wire(&SessionProjectionGenerationV1::new(1).unwrap(), json!(1)); + assert_wire( + &SessionCursorKeyIdV1::new("cursor.key.fixture").unwrap(), + json!("cursor.key.fixture"), + ); let signed_cursor = SignedCursorKeyRefV1 { key_id: SessionCursorKeyIdV1::new("cursor.key.fixture").unwrap(), version: SessionCursorVersionV1::new(1).unwrap(), }; - assert_json_round_trip!(signed_cursor); + assert_wire( + &signed_cursor, + json!({"key_id": "cursor.key.fixture", "version": 1}), + ); assert!(serde_json::from_value::(json!(0)).is_err()); assert!(serde_json::from_value::(json!(0)).is_err()); assert_eq!( @@ -510,39 +593,59 @@ fn typed_ids_and_signed_cursors_round_trip_and_reject_invalid_values() { } } -/// `as_str` is what callers log, key, and route on, so it must be the same -/// string the wire carries. Sweeping `ALL` keeps a new variant covered without -/// a test edit. -macro_rules! assert_as_str_is_the_wire_value_for_all_variants { - ($type:ty) => { - for variant in <$type>::ALL { - assert_json_round_trip!(variant); - assert_eq!(serde_json::to_value(variant).unwrap(), variant.as_str()); - } - }; -} - #[test] -fn enum_as_str_matches_serde_for_every_variant() { - assert_as_str_is_the_wire_value_for_all_variants!(RetrievalGrainV1); - assert_as_str_is_the_wire_value_for_all_variants!(SessionAuthorityClassV1); - assert_as_str_is_the_wire_value_for_all_variants!(TemporalAssertionKindV1); - - // These two carry data, so they serialize as tagged objects and `as_str` - // names the tag rather than the whole value. - for mode in [ - TemporalModeV1::Current, - TemporalModeV1::AsOf { - cutoff: UtcMicros(50), - }, - TemporalModeV1::Evolution, - TemporalModeV1::Forensic, +fn session_enums_have_literal_wire_spellings_and_reject_unknown_values() { + for (grain, wire) in [ + (RetrievalGrainV1::Occurrence, "occurrence"), + (RetrievalGrainV1::LogicalMessage, "logical_message"), + (RetrievalGrainV1::Turn, "turn"), + (RetrievalGrainV1::Session, "session"), + (RetrievalGrainV1::Thread, "thread"), + (RetrievalGrainV1::Agent, "agent"), + (RetrievalGrainV1::Summary, "summary"), + ] { + assert_wire(&grain, json!(wire)); + assert_eq!(grain.as_str(), wire); + } + for (authority, wire) in [ + (SessionAuthorityClassV1::ProviderNative, "provider_native"), + ( + SessionAuthorityClassV1::CanonicalObservation, + "canonical_observation", + ), + ( + SessionAuthorityClassV1::ExplicitAnchorAssertion, + "explicit_anchor_assertion", + ), + ( + SessionAuthorityClassV1::DerivedProjection, + "derived_projection", + ), + ( + SessionAuthorityClassV1::ImmutableSummary, + "immutable_summary", + ), ] { - assert_eq!(serde_json::to_value(mode).unwrap()["kind"], mode.as_str()); + assert_wire(&authority, json!(wire)); + assert_eq!(authority.as_str(), wire); } - for grouping in [GroupingProvenanceV1::ProviderNative, derived_grouping()] { - assert_json_round_trip!(grouping); + for (kind, wire) in [ + (TemporalAssertionKindV1::Corrects, "corrects"), + (TemporalAssertionKindV1::Supersedes, "supersedes"), + (TemporalAssertionKindV1::Contradicts, "contradicts"), + (TemporalAssertionKindV1::Supports, "supports"), + ] { + assert_wire(&kind, json!(wire)); + assert_eq!(kind.as_str(), wire); } + assert_wire( + &GroupingProvenanceV1::ProviderNative, + json!({"kind": "provider_native"}), + ); + assert_wire( + &derived_grouping(), + json!({"kind": "derived_role_boundary", "projector_version": "projector.fixture"}), + ); assert!(serde_json::from_value::(json!("paragraph")).is_err()); assert!(serde_json::from_value::(json!("untrusted")).is_err()); @@ -563,22 +666,23 @@ fn evidence_and_assertion_records_round_trip_and_reject_invalid_anchors() { let metadata: SessionEvidenceMetadataV1 = serde_json::from_value(evidence_wire(evidence_class)).unwrap(); metadata.validate().unwrap(); - assert_json_round_trip!(metadata); + assert_eq!( + serde_json::to_value(&metadata).unwrap(), + evidence_wire(evidence_class) + ); } let metadata = evidence(); - metadata.validate().unwrap(); - assert_json_round_trip!(metadata.clone()); let mut invalid_metadata = serde_json::to_value(&metadata).unwrap(); invalid_metadata["source_anchor_id"] = json!(" "); assert!(serde_json::from_value::(invalid_metadata).is_err()); - for kind in [ - TemporalAssertionKindV1::Corrects, - TemporalAssertionKindV1::Supersedes, - TemporalAssertionKindV1::Contradicts, - TemporalAssertionKindV1::Supports, + for (kind, kind_wire) in [ + (TemporalAssertionKindV1::Corrects, "corrects"), + (TemporalAssertionKindV1::Supersedes, "supersedes"), + (TemporalAssertionKindV1::Contradicts, "contradicts"), + (TemporalAssertionKindV1::Supports, "supports"), ] { let assertion = TemporalAssertionRecordV1 { assertion_id: TemporalAssertionIdV1::new("assertion.fixture").unwrap(), @@ -592,7 +696,18 @@ fn evidence_and_assertion_records_round_trip_and_reject_invalid_anchors() { evidence: metadata.clone(), }; assertion.validate().unwrap(); - assert_json_round_trip!(assertion.clone()); + assert_wire( + &assertion, + json!({ + "assertion_id": "assertion.fixture", + "kind": kind_wire, + "subject_anchor_id": "anchor.assertion.subject", + "object_anchor_id": "anchor.assertion.object", + "knowledge_at": 50, + "valid_time": {"kind": "known", "valid_at": 40}, + "evidence": evidence_wire("provider_declared") + }), + ); let self_assertion = TemporalAssertionRecordV1 { object_anchor_id: assertion.subject_anchor_id.clone(), @@ -614,7 +729,6 @@ fn occurrence_records_round_trip_with_independent_grouping_and_reject_orphans() let record: MessageOccurrenceRecordV1 = serde_json::from_value(wire.clone()).unwrap(); record.validate().unwrap(); assert_eq!(serde_json::to_value(&record).unwrap(), wire); - assert_json_round_trip!(record.clone()); let mut invalid_occurrence_record = record.clone(); invalid_occurrence_record.occurrence_id = occurrence(1); @@ -664,43 +778,64 @@ fn occurrence_records_round_trip_with_independent_grouping_and_reject_orphans() } #[test] -fn hydration_and_omission_values_round_trip_every_variant_and_reject_unknown_values() { - for state in [ - HydrationStateV1::Available, - HydrationStateV1::RetainedButUnavailable, - HydrationStateV1::Redacted, - HydrationStateV1::Deleted, - HydrationStateV1::RetentionExpired, - HydrationStateV1::Unauthorized, - HydrationStateV1::Locked, - HydrationStateV1::UnverifiableLegacy, +fn hydration_and_omission_values_have_literal_wire_spellings_and_reject_unknown_values() { + for (state, wire) in [ + (HydrationStateV1::Available, "available"), + ( + HydrationStateV1::RetainedButUnavailable, + "retained_but_unavailable", + ), + (HydrationStateV1::Redacted, "redacted"), + (HydrationStateV1::Deleted, "deleted"), + (HydrationStateV1::RetentionExpired, "retention_expired"), + (HydrationStateV1::Unauthorized, "unauthorized"), + (HydrationStateV1::Locked, "locked"), + (HydrationStateV1::Unverifiable, "unverifiable"), ] { - assert_json_round_trip!(state); - assert_eq!(serde_json::to_value(state).unwrap(), state.as_str()); + assert_wire(&state, json!(wire)); + assert_eq!(state.as_str(), wire); } - for reason in [ - ContextOmissionReasonV1::ByteBudget, - ContextOmissionReasonV1::TokenBudget, - ContextOmissionReasonV1::Unauthorized, - ContextOmissionReasonV1::Redacted, - ContextOmissionReasonV1::Deleted, - ContextOmissionReasonV1::RetentionExpired, - ContextOmissionReasonV1::Locked, - ContextOmissionReasonV1::Unavailable, - ContextOmissionReasonV1::SummaryHorizonMismatch, - ContextOmissionReasonV1::DuplicateRepresentative, - ContextOmissionReasonV1::RootContinuationUnavailable, + for (reason, wire) in [ + (ContextOmissionReasonV1::ByteBudget, "byte_budget"), + (ContextOmissionReasonV1::TokenBudget, "token_budget"), + (ContextOmissionReasonV1::Unauthorized, "unauthorized"), + (ContextOmissionReasonV1::Redacted, "redacted"), + (ContextOmissionReasonV1::Deleted, "deleted"), + ( + ContextOmissionReasonV1::RetentionExpired, + "retention_expired", + ), + (ContextOmissionReasonV1::Locked, "locked"), + (ContextOmissionReasonV1::Unavailable, "unavailable"), + ( + ContextOmissionReasonV1::SummaryHorizonMismatch, + "summary_horizon_mismatch", + ), + ( + ContextOmissionReasonV1::DuplicateRepresentative, + "duplicate_representative", + ), + ( + ContextOmissionReasonV1::RootContinuationUnavailable, + "root_continuation_unavailable", + ), ] { - assert_json_round_trip!(CompactContextOmissionV1 { - anchor_id: Some(anchor("anchor.omission")), - reason, - }); - assert_eq!(serde_json::to_value(reason).unwrap(), reason.as_str()); + assert_wire( + &CompactContextOmissionV1 { + anchor_id: Some(anchor("anchor.omission")), + reason, + }, + json!({"anchor_id": "anchor.omission", "reason": wire}), + ); + assert_eq!(reason.as_str(), wire); } - assert_json_round_trip!(CompactContextOmissionV1 { - anchor_id: None, - reason: ContextOmissionReasonV1::ByteBudget, - }); + assert_wire( + &CompactContextOmissionV1 { + anchor_id: None, + reason: ContextOmissionReasonV1::ByteBudget, + }, + json!({"anchor_id": null, "reason": "byte_budget"}), + ); assert!(serde_json::from_value::(json!("incomplete")).is_err()); assert!(serde_json::from_value::(json!("stale")).is_err()); @@ -727,8 +862,20 @@ fn compact_context_recomputes_bytes_and_validates_omission_anchors() { hydration: HydrationStateV1::RetainedButUnavailable, encoded_bytes: 5, }; - assert_json_round_trip!(first.clone()); - assert_json_round_trip!(second.clone()); + let first_wire = json!({ + "anchor_id": "anchor.context.first", + "grain": "occurrence", + "hydration": "available", + "encoded_bytes": 3 + }); + let second_wire = json!({ + "anchor_id": "anchor.context.second", + "grain": "summary", + "hydration": "retained_but_unavailable", + "encoded_bytes": 5 + }); + assert_wire(&first, first_wire.clone()); + assert_wire(&second, second_wire.clone()); let bundle = CompactContextBundleV1 { records: vec![first.clone(), second], @@ -759,7 +906,29 @@ fn compact_context_recomputes_bytes_and_validates_omission_anchors() { encoded_bytes: 8, }; bundle.validate().unwrap(); - assert_json_round_trip!(bundle.clone()); + assert_wire( + &bundle, + json!({ + "records": [first_wire, second_wire], + "omissions": [{"anchor_id": "anchor.context.omitted", "reason": "token_budget"}], + "continuation_anchors": ["anchor.context.continuation"], + "coverage": {"visible": 1, "hidden": 2, "unknown": 3, "redacted": 4}, + "conflicts": [{ + "anchor_id": "anchor.context.first", + "supporting_anchor_ids": ["anchor.context.support"] + }], + "lineage": [{ + "kind": "corrects", + "subject_anchor_id": "anchor.context.first", + "object_anchor_id": "anchor.context.predecessor", + "knowledge_at": 42, + "authority": "canonical_observation", + "authorized": true, + "supporting_anchor_ids": ["anchor.context.support"] + }], + "encoded_bytes": 8 + }), + ); let mut incorrect_total = bundle.clone(); incorrect_total.encoded_bytes = 9; @@ -1040,7 +1209,10 @@ fn coverage_and_anchor_entity_kinds_have_stable_wire_values() { }; assert_eq!(coverage.total(), Some(10)); assert!(coverage.has_withheld_or_unknown()); - assert_json_round_trip!(coverage); + assert_wire( + &coverage, + json!({"visible": 3, "hidden": 2, "unknown": 1, "redacted": 4}), + ); let kinds = [ (EntityKind::Thread, "thread"), diff --git a/crates/tracedecay-global-db/Cargo.toml b/crates/tracedecay-global-db/Cargo.toml index 0935da236e..0375c09999 100644 --- a/crates/tracedecay-global-db/Cargo.toml +++ b/crates/tracedecay-global-db/Cargo.toml @@ -31,7 +31,7 @@ hex = "0.4" hmac = { version = "0.13.0", features = ["zeroize"] } hotpath.workspace = true rusqlite = { version = "0.40.1", default-features = false, features = ["backup"] } -schemars = "1.2.1" +schemars.workspace = true serde = { version = "1", features = ["derive"] } serde_json = "1" sha2 = "0.11" diff --git a/crates/tracedecay-global-db/benches/lcm_expansion.rs b/crates/tracedecay-global-db/benches/lcm_expansion.rs index d921deb283..702f6bc214 100644 --- a/crates/tracedecay-global-db/benches/lcm_expansion.rs +++ b/crates/tracedecay-global-db/benches/lcm_expansion.rs @@ -20,13 +20,14 @@ use tracedecay_lcm::{ use tracedecay_session_temporal_store::RegisteredGlobalDbSessionTemporalExecution; use tracedecay_store::{ AnchoredObservationWrite, ObservationProjectionStore, ObservationStore, ObservationWrite, - build_observation_resolution_authorization_v1, build_observation_retrieval_anchor_v2, -}; -use tracedecay_temporal_query::ports::{ - BindingDigest, ExecutionControl, KernelVersions, TemporalExecutionSnapshot, - TemporalSnapshotRequest, TemporalWatermarks, + build_observation_resolution_authorization_v1, build_observation_retrieval_anchor, }; +use tracedecay_temporal_query::execution::{BindingDigest, ExecutionControl}; +use tracedecay_temporal_query::ports::TemporalSnapshotRequest; use tracedecay_temporal_query::resolution::ValidatedAuthorization; +use tracedecay_temporal_query::snapshot::{ + KernelVersions, TemporalExecutionSnapshot, TemporalWatermarks, +}; const PROVIDER: &str = "lcm-benchmark"; const SESSION_ID: &str = "session.lcm-expansion"; @@ -132,7 +133,7 @@ async fn persist_observation( let authorization = build_observation_resolution_authorization_v1(&observation, AUTHORITY) .expect("benchmark observation authorization is valid"); let access_digest = authorization.access_policy_digest.as_str().to_owned(); - let anchor = build_observation_retrieval_anchor_v2( + let anchor = build_observation_retrieval_anchor( &observation, projection_generation.clone(), UtcMicros(1), diff --git a/crates/tracedecay-global-db/benches/session_activity_reads.rs b/crates/tracedecay-global-db/benches/session_activity_reads.rs index 3b5cbbdf11..17a0c5c698 100644 --- a/crates/tracedecay-global-db/benches/session_activity_reads.rs +++ b/crates/tracedecay-global-db/benches/session_activity_reads.rs @@ -44,9 +44,10 @@ async fn seed_fixture(profile: &tempfile::TempDir) -> RegisteredGlobalDbTestRunt UNION ALL SELECT value + 1 FROM rows WHERE value < {} ) - INSERT INTO session_messages( - provider, message_id, session_id, role, timestamp, ordinal, text, - kind, model, tool_names, source_path, source_offset, metadata_json + INSERT INTO lcm_raw_messages( + provider, message_id, session_id, role, timestamp, ordinal, content, + kind, model, tool_names, source_path, source_offset, metadata_json, + content_hash, storage_kind ) SELECT '{PROVIDER}', @@ -61,7 +62,9 @@ async fn seed_fixture(profile: &tempfile::TempDir) -> RegisteredGlobalDbTestRunt 'tool', NULL, NULL, - NULL + NULL, + 'hash', + 'inline' FROM rows;", TOTAL_ROWS - 1 )) diff --git a/crates/tracedecay-global-db/src/configuration/contracts/mod.rs b/crates/tracedecay-global-db/src/configuration/contracts/mod.rs index 5665f13597..042be22c3a 100644 --- a/crates/tracedecay-global-db/src/configuration/contracts/mod.rs +++ b/crates/tracedecay-global-db/src/configuration/contracts/mod.rs @@ -9,9 +9,9 @@ pub mod ports; pub mod types; pub use ports::{ - ConfigurationClock, ConfigurationControlStore, ConfigurationCurrentStateV1, - ConfigurationMutationAuthorizationPort, ConfigurationOperationFuture, - CurrentConfigurationMutationAuthorizationV1, ScopeResolutionPort, ScopeRevalidationEvidenceV1, + ConfigurationControlStore, ConfigurationCurrentStateV1, ConfigurationMutationAuthorizationPort, + ConfigurationOperationFuture, CurrentConfigurationMutationAuthorizationV1, ScopeResolutionPort, + ScopeRevalidationEvidenceV1, }; pub use types::{ ActivationDriftV1, AuthorizedActor, CONFIGURATION_AUDIT_PAGE_LIMIT, diff --git a/crates/tracedecay-global-db/src/configuration/contracts/ports.rs b/crates/tracedecay-global-db/src/configuration/contracts/ports.rs index b0d75375ab..350ed382d2 100644 --- a/crates/tracedecay-global-db/src/configuration/contracts/ports.rs +++ b/crates/tracedecay-global-db/src/configuration/contracts/ports.rs @@ -56,10 +56,6 @@ pub trait ScopeResolutionPort: Sync { ) -> ConfigurationOperationFuture<'a, ScopeRevalidationEvidenceV1>; } -pub trait ConfigurationClock: Sync { - fn now(&self) -> UtcMicros; -} - #[derive(Clone, Debug, PartialEq, Eq)] pub struct CurrentConfigurationMutationAuthorizationV1 { pub grant_revision: u64, diff --git a/crates/tracedecay-global-db/src/configuration/registry.rs b/crates/tracedecay-global-db/src/configuration/registry.rs index 9b8926c86d..a921baac94 100644 --- a/crates/tracedecay-global-db/src/configuration/registry.rs +++ b/crates/tracedecay-global-db/src/configuration/registry.rs @@ -12,21 +12,21 @@ use tracedecay_domain::configuration::{ INDEX_EXCLUDE_SETTING_KEY, INDEX_EXTRACT_DOCSTRINGS_SETTING_KEY, INDEX_GIT_IGNORE_SETTING_KEY, INDEX_INCLUDE_SETTING_KEY, INDEX_MAX_FILE_SIZE_SETTING_KEY, INDEX_NATIVE_GRAPH_ACTIVATION_SETTING_KEY, INDEX_TRACK_CALL_SITES_SETTING_KEY, + LCM_SUMMARIZER_EXECUTABLES_SETTING_KEY, LcmSummarizerExecutablesV1, PROJECT_WORK_EXPERTISE_CONSENT_SETTING_KEY, RestartRequirementV1, SOURCE_BINDINGS_SETTING_KEY, SYNC_AUTO_INIT_SETTING_KEY, SYNC_AUTO_TRACK_PR_BRANCHES_SETTING_KEY, SYNC_AUTO_TRACK_PR_POLL_SECS_SETTING_KEY, SYNC_AUTO_WATCH_SETTING_KEY, SYNC_BACKSTOP_INTERVAL_MINS_SETTING_KEY, SYNC_BRANCH_GC_DAYS_SETTING_KEY, SYNC_FULL_SYNC_ESCALATION_FILES_SETTING_KEY, SYNC_MAX_CONCURRENT_SYNCS_SETTING_KEY, - SYNC_ORPHAN_DB_GC_DAYS_SETTING_KEY, SYNC_READ_COOLDOWN_SECS_SETTING_KEY, - SYNC_READ_REFRESH_SETTING_KEY, SYNC_SESSION_START_STALE_THRESHOLD_SECS_SETTING_KEY, - SYNC_SESSION_START_SYNC_SETTING_KEY, SYNC_WATCH_DEBOUNCE_MS_SETTING_KEY, - SYNC_WATCH_LINKED_WORKTREES_SETTING_KEY, SYNC_WATCH_MAX_DELAY_MS_SETTING_KEY, - SYNC_WATCH_MAX_PROJECTS_SETTING_KEY, SettingDefinitionV1, SettingKey, SettingScopeV1, - SettingSensitivityV1, TELEMETRY_TIMINGS_SETTING_KEY, USER_CODE_INDEX_WORKERS_SETTING_KEY, - USER_EXTRACTION_TIMEOUT_SECS_SETTING_KEY, USER_UPLOAD_ENABLED_SETTING_KEY, - USER_WATCHER_DEBOUNCE_MS_SETTING_KEY, USER_WORK_EXPERTISE_CONSENT_SETTING_KEY, - WORK_EXECUTABLE_BINDINGS_SETTING_KEY, WORK_TOPOLOGY_POLICY_SETTING_KEY, WorkExpertiseConsentV1, - safe_work_topology_policy_v1, + SYNC_READ_COOLDOWN_SECS_SETTING_KEY, SYNC_READ_REFRESH_SETTING_KEY, + SYNC_SESSION_START_STALE_THRESHOLD_SECS_SETTING_KEY, SYNC_SESSION_START_SYNC_SETTING_KEY, + SYNC_WATCH_DEBOUNCE_MS_SETTING_KEY, SYNC_WATCH_LINKED_WORKTREES_SETTING_KEY, + SYNC_WATCH_MAX_DELAY_MS_SETTING_KEY, SYNC_WATCH_MAX_PROJECTS_SETTING_KEY, SettingDefinitionV1, + SettingKey, SettingScopeV1, SettingSensitivityV1, TELEMETRY_TIMINGS_SETTING_KEY, + USER_CODE_INDEX_WORKERS_SETTING_KEY, USER_EXTRACTION_TIMEOUT_SECS_SETTING_KEY, + USER_UPLOAD_ENABLED_SETTING_KEY, USER_WATCHER_DEBOUNCE_MS_SETTING_KEY, + USER_WORK_EXPERTISE_CONSENT_SETTING_KEY, WORK_EXECUTABLE_BINDINGS_SETTING_KEY, + WORK_TOPOLOGY_POLICY_SETTING_KEY, WorkExpertiseConsentV1, safe_work_topology_policy_v1, }; use tracedecay_domain::feedback::PROXIMITY_RISK_THRESHOLD_SETTING_KEY_V1; @@ -167,6 +167,21 @@ impl ConfigurationRegistry { restart_requirement: RestartRequirementV1::None, deprecation: DeprecationStateV1::Active, })?; + // On-demand LCM summarization launches a host CLI only through this + // explicit binding; the unconfigured default keeps compaction pending + // rather than resolving a binary from the daemon's environment. + registry.register(SettingDefinitionV1 { + key: setting_key(LCM_SUMMARIZER_EXECUTABLES_SETTING_KEY)?, + schema_revision: CONFIGURATION_REGISTRY_SCHEMA_REVISION, + value_kind: ConfigurationValueKindV1::LcmSummarizerExecutables, + default_value: ConfigurationValueV1::LcmSummarizerExecutables( + LcmSummarizerExecutablesV1::unconfigured(), + ), + sensitivity: SettingSensitivityV1::Sensitive, + scope: SettingScopeV1::Project, + restart_requirement: RestartRequirementV1::None, + deprecation: DeprecationStateV1::Active, + })?; registry.register(SettingDefinitionV1 { key: setting_key(PROXIMITY_RISK_THRESHOLD_SETTING_KEY_V1)?, schema_revision: CONFIGURATION_REGISTRY_SCHEMA_REVISION, @@ -416,7 +431,6 @@ struct SyncDefaults { full_sync_escalation_files: usize, max_concurrent_syncs: usize, branch_gc_days: u64, - orphan_db_gc_days: u64, auto_init: bool, auto_track_pr_branches: bool, auto_track_pr_poll_secs: u64, @@ -438,7 +452,6 @@ impl Default for SyncDefaults { full_sync_escalation_files: 500, max_concurrent_syncs: 2, branch_gc_days: 14, - orphan_db_gc_days: 7, auto_init: true, auto_track_pr_branches: false, auto_track_pr_poll_secs: 300, @@ -606,12 +619,6 @@ fn register_project_settings( SettingSensitivityV1::Public, RestartRequirementV1::DaemonRestart, ), - ( - SYNC_ORPHAN_DB_GC_DAYS_SETTING_KEY, - ConfigurationValueV1::Unsigned(sync.orphan_db_gc_days), - SettingSensitivityV1::Public, - RestartRequirementV1::DaemonRestart, - ), ( SYNC_AUTO_INIT_SETTING_KEY, ConfigurationValueV1::Boolean(sync.auto_init), diff --git a/crates/tracedecay-global-db/src/configuration/store.rs b/crates/tracedecay-global-db/src/configuration/store.rs index c4132ea73b..08d17e0c97 100644 --- a/crates/tracedecay-global-db/src/configuration/store.rs +++ b/crates/tracedecay-global-db/src/configuration/store.rs @@ -19,11 +19,12 @@ use tracedecay_domain::configuration::{ ConfigurationAuditEventKindV1, ConfigurationCandidateV1, ConfigurationIdempotencyKey, ConfigurationLayerIdV1, ConfigurationReceiptId, ConfigurationRevisionId, ConfigurationSnapshotV1, ConfigurationValueV1, INDEX_NATIVE_GRAPH_ACTIVATION_SETTING_KEY, - ProtectedChange, ProtectedChangePlan, ProtectedChangeSnapshotError, - RETIRED_CORE_SETTING_KEYS_V1, RedactedConfigurationChangeV1, RollbackModeV1, RuleEffect, - SOURCE_BINDINGS_SETTING_KEY, SYNC_WATCH_LINKED_WORKTREES_SETTING_KEY, ScopeControlOperationV1, - ScopeSourceBinding, SettingKey, SourceKindV1, USER_CODE_INDEX_WORKERS_SETTING_KEY, - UserProfileId, WORK_TOPOLOGY_POLICY_SETTING_KEY, + LCM_SUMMARIZER_EXECUTABLES_SETTING_KEY, ProtectedChange, ProtectedChangePlan, + ProtectedChangeSnapshotError, RETIRED_CORE_SETTING_KEYS_V1, RedactedConfigurationChangeV1, + RollbackModeV1, RuleEffect, SOURCE_BINDINGS_SETTING_KEY, + SYNC_WATCH_LINKED_WORKTREES_SETTING_KEY, ScopeControlOperationV1, ScopeSourceBinding, + SettingKey, SourceKindV1, USER_CODE_INDEX_WORKERS_SETTING_KEY, UserProfileId, + WORK_TOPOLOGY_POLICY_SETTING_KEY, }; use tracedecay_domain::{AccessPolicyDigest, ActorId, ManifestDigest, UtcMicros, canonical_sha256}; #[cfg(test)] @@ -360,6 +361,7 @@ impl<'db> GlobalDbConfigurationControlStore<'db> { let registry = ConfigurationRegistry::core().map_err(ConfigurationError::validation)?; let additive_keys = [ INDEX_NATIVE_GRAPH_ACTIVATION_SETTING_KEY, + LCM_SUMMARIZER_EXECUTABLES_SETTING_KEY, SYNC_WATCH_LINKED_WORKTREES_SETTING_KEY, ] .into_iter() diff --git a/crates/tracedecay-global-db/src/configuration/store/codec.rs b/crates/tracedecay-global-db/src/configuration/store/codec.rs index 3057ad0033..7fe9933132 100644 --- a/crates/tracedecay-global-db/src/configuration/store/codec.rs +++ b/crates/tracedecay-global-db/src/configuration/store/codec.rs @@ -71,7 +71,7 @@ impl From for ConfigurationProtectedOpe #[derive(Debug, Serialize, Deserialize)] #[serde(deny_unknown_fields)] -pub(super) struct StoredConfigurationPlanPayloadV2 { +pub(super) struct StoredConfigurationPlanPayload { pub(super) schema_version: u16, pub(super) plan: ProtectedChangePlan, pub(super) operation: StoredConfigurationProtectedOperationV1, @@ -493,7 +493,7 @@ pub(super) fn decode_plan_row( "configuration plan operation payload is missing", )); }; - let payload = serde_json::from_slice::(&sealed_payload) + let payload = serde_json::from_slice::(&sealed_payload) .map_err(|error| { invalid_store_data(format!("decode configuration plan payload: {error}")) })?; diff --git a/crates/tracedecay-global-db/src/configuration/store/write.rs b/crates/tracedecay-global-db/src/configuration/store/write.rs index c31a71b797..56f5483b79 100644 --- a/crates/tracedecay-global-db/src/configuration/store/write.rs +++ b/crates/tracedecay-global-db/src/configuration/store/write.rs @@ -2,7 +2,7 @@ use std::collections::BTreeSet; use super::codec::{ CONFIGURATION_PLAN_PAYLOAD_SCHEMA_VERSION, CONFIGURATION_SNAPSHOT_ENTRY_PAYLOAD_SCHEMA_VERSION, - StoredConfigurationPlanPayloadV2, StoredConfigurationSnapshotEntryV1, + StoredConfigurationPlanPayload, StoredConfigurationSnapshotEntryV1, }; use super::{ CandidateDispositionV1, ConfigurationCandidateV1, ConfigurationLayerIdV1, @@ -88,7 +88,7 @@ fn encode_plan_payload( plan: &ConfigurationProtectedPlanRecordV1, ) -> ConfigurationStoreResult> { plan.validate().map_err(ConfigurationStoreError::from)?; - serde_json::to_vec(&StoredConfigurationPlanPayloadV2 { + serde_json::to_vec(&StoredConfigurationPlanPayload { schema_version: CONFIGURATION_PLAN_PAYLOAD_SCHEMA_VERSION, plan: plan.plan.clone(), operation: (&plan.operation).into(), diff --git a/crates/tracedecay-global-db/src/git_correlation_adapter.rs b/crates/tracedecay-global-db/src/git_correlation_adapter.rs index 0d1965deb9..ca5e8016dc 100644 --- a/crates/tracedecay-global-db/src/git_correlation_adapter.rs +++ b/crates/tracedecay-global-db/src/git_correlation_adapter.rs @@ -67,18 +67,12 @@ pub struct GitEvidenceConvergenceStats { /// Conservative signal: a full page means another retained-history page /// may exist and callers must not describe this pass as fully drained. pub backfill_page_saturated: bool, - /// The verified head predated the indexed projector and this pass - /// re-published its unchanged content under the current projector so - /// bounded reads can serve it. - pub reprojected_legacy_head: bool, } impl GitEvidenceConvergenceStats { /// Whether this pass durably changed Git evidence or its session frontier. pub fn committed_progress(&self) -> bool { - self.replayed_publications > 0 - || self.backfill.committed_progress() - || self.reprojected_legacy_head + self.replayed_publications > 0 || self.backfill.committed_progress() } } @@ -284,7 +278,6 @@ where pending_publications: None, backfill: BackfillStats::default(), backfill_page_saturated: false, - reprojected_legacy_head: false, }, later_failure: error, }); @@ -298,7 +291,6 @@ where pending_publications, backfill: BackfillStats::default(), backfill_page_saturated: false, - reprojected_legacy_head: false, }, Some(later_failure), ); @@ -313,58 +305,19 @@ where pending_publications, backfill: BackfillStats::default(), backfill_page_saturated: false, - reprojected_legacy_head: false, }, later_failure: error, }); } Err(error) => return Err(error), }; - let mut progress = GitEvidenceConvergenceStats { + let progress = GitEvidenceConvergenceStats { replayed_publications, pending_publications, backfill_page_saturated: backfill_outcome.stats.sessions_scanned == backfill_session_limit, backfill: backfill_outcome.stats, - reprojected_legacy_head: false, }; - if let Some(later_failure) = backfill_outcome.later_failure { - return settle_git_evidence_convergence(progress, Some(later_failure)); - } - match reproject_legacy_git_evidence_head(session_store).await { - Ok(reprojected) => { - progress.reprojected_legacy_head = reprojected; - settle_git_evidence_convergence(progress, None) - } - Err(error) => settle_git_evidence_convergence(progress, Some(error)), - } -} - -/// Re-publishes a verified head that predates the indexed projector. Every -/// ordinary publication re-projects the head as a side effect; this covers a -/// project whose evidence never changes again, so its bounded reads do not -/// stay unavailable indefinitely. -async fn reproject_legacy_git_evidence_head( - session_store: &S, -) -> Result { - let identity = - git_evidence_projection_identity(GraphNamespace::new(GIT_EVIDENCE_GRAPH_NAMESPACE)?)?; - match open_git_evidence_graph_view( - session_store.graph_runtime()?, - &identity, - Arc::new(NeverCancelled), - )? { - GitEvidenceGraphHead::Legacy { .. } => { - session_store - .publish_graph_evidence_owned( - "projector-upgrade".to_owned(), - Vec::new(), - Vec::new(), - ) - .await?; - Ok(true) - } - GitEvidenceGraphHead::Unpublished | GitEvidenceGraphHead::Indexed(_) => Ok(false), - } + settle_git_evidence_convergence(progress, backfill_outcome.later_failure) } /// Adapter over an already-open project-sessions database. @@ -538,9 +491,7 @@ where } /// Opens the bounded, generation-bound query view without decoding any - /// span or commit payload. `Ok(None)` is the never-published empty start; - /// a head published by the pre-index projector is a typed unavailable - /// state until the next publication (or convergence pass) re-projects it. + /// span or commit payload. `Ok(None)` is the never-published empty start. #[hotpath::measure(label = "global_db.git_correlation.graph_view")] pub fn git_evidence_graph_view( &self, @@ -554,11 +505,6 @@ where )? { GitEvidenceGraphHead::Indexed(view) => Ok(Some(view)), GitEvidenceGraphHead::Unpublished => Ok(None), - GitEvidenceGraphHead::Legacy { generation } => { - Err(GitCorrelationError::Unavailable(format!( - "verified Git evidence generation `{generation}` predates the indexed projector; the next publication re-projects it" - ))) - } } } @@ -961,8 +907,8 @@ mod tests { use tokio::sync::Notify; use tracedecay_domain::ProjectId; use tracedecay_graph_db::{ - GraphDbError, GraphGenerationManifest, GraphIdempotencyKey, GraphNamespace, - GraphProjectionIdentity, NeverCancelled, VerifiedGraphSnapshot, + GraphDbError, GraphGenerationManifest, GraphIdempotencyKey, GraphProjectionIdentity, + NeverCancelled, VerifiedGraphSnapshot, }; use tracedecay_runtime_core::RuntimeOperationTaskOwnerV1; use tracedecay_runtime_core::db::{ @@ -972,10 +918,8 @@ mod tests { use tracedecay_runtime_core::shard_runtime::VerifiedGraphRuntimePortV1; use tracedecay_sessions::runtime::SessionRecord; use tracedecay_sessions::runtime::git_correlation::{ - CommitRelationFilter, GitCorrelationError, GitEvidenceProjectionV1, - GitEvidenceProjectorRevision, GitRefFilter, GitReflogSource, GitScopeFilter, - SessionGitSpan, SessionsForQuery, SpanObservation, SpanSource, SystemGit, - git_evidence_projection_identity, legacy_git_evidence_manifest_for_test, + CommitRelationFilter, GitCorrelationError, GitRefFilter, GitScopeFilter, SessionsForQuery, + SpanObservation, SpanSource, SystemGit, }; use tracedecay_store::{FactReadControl, StoreRuntimeBindingV1, VerifiedStoreLocatorV1}; @@ -1218,7 +1162,6 @@ mod tests { pending_publications: Some(0), backfill: Default::default(), backfill_page_saturated: false, - reprojected_legacy_head: false, }; let failure = GitCorrelationError::Unavailable("git log failed".to_owned()); @@ -1271,37 +1214,6 @@ mod tests { assert_eq!(projection.projection().spans()[0].last_ts, 12); } - /// A repository whose history is readable but empty, so the convergence - /// pass's attribution sweep can run against fixture worktrees. - struct EmptyHistoryGit; - - impl GitReflogSource for EmptyHistoryGit { - fn reflog(&self, _worktree: &std::path::Path) -> Option { - Some(String::new()) - } - - fn current_branch(&self, _worktree: &std::path::Path) -> Option { - Some("main".to_owned()) - } - - fn commit_reference_exists( - &self, - _worktree: &std::path::Path, - _reference: &str, - ) -> Result { - Ok(true) - } - - fn commit_log( - &self, - _worktree: &std::path::Path, - _branch: &str, - _since: i64, - ) -> Option { - Some(String::new()) - } - } - fn released_snapshot_gate() -> mpsc::Receiver<()> { let (release, blocked) = mpsc::channel(); release.send(()).unwrap(); @@ -1333,11 +1245,6 @@ mod tests { .git_evidence_projection() .unwrap() .expect("published evidence"); - assert_eq!( - full.projector_revision(), - GitEvidenceProjectorRevision::Current - ); - let health = fixture.store.correlation_index_health().await.unwrap(); assert_eq!(health, full.health(None)); assert!(health.projection_available); @@ -1383,121 +1290,6 @@ mod tests { ); } - #[tokio::test(flavor = "current_thread")] - async fn legacy_head_is_unavailable_until_convergence_reprojects_it() { - let fixture = - GitEvidenceRuntimeFixture::open("legacy-head", released_snapshot_gate(), false).await; - let identity = - git_evidence_projection_identity(GraphNamespace::new("project").unwrap()).unwrap(); - // The attribution sweep only scans worktrees that exist on disk. - let worktree = fixture._root.path().to_string_lossy().into_owned(); - let projection = GitEvidenceProjectionV1::new( - "legacy-watermark", - vec![SessionGitSpan { - span_id: "legacy-span".to_owned(), - provider: "codex".to_owned(), - session_id: "session.legacy".to_owned(), - thread_id: None, - branch: Some("main".to_owned()), - worktree, - first_ts: 10, - last_ts: 12, - event_count: 2, - source: SpanSource::Ingest, - }], - Vec::new(), - ) - .unwrap(); - let legacy = legacy_git_evidence_manifest_for_test(identity, &projection).unwrap(); - let legacy_generation = legacy.generation.clone(); - fixture - .runtime - .publish_verified_manifest( - &legacy, - GraphIdempotencyKey::new("legacy-head").unwrap(), - Arc::new(AtomicBool::new(false)), - ) - .unwrap(); - - for error in [ - fixture - .store - .sessions_for_with_relation(&branch_query("main"), CommitRelationFilter::Produced) - .await - .unwrap_err(), - fixture.store.correlation_index_health().await.unwrap_err(), - fixture - .store - .sessions_for_with_relation_and_presence( - &branch_query("main"), - CommitRelationFilter::Produced, - ) - .await - .unwrap_err(), - ] { - assert!( - matches!(&error, GitCorrelationError::Unavailable(detail) - if detail.contains("predates the indexed projector")), - "{error}" - ); - } - // The rows themselves stay fully recoverable. - let recovered = fixture - .store - .git_evidence_projection() - .unwrap() - .expect("legacy head recovers"); - assert_eq!( - recovered.projector_revision(), - GitEvidenceProjectorRevision::LegacyV1 - ); - assert_eq!(recovered.projection(), &projection); - - let convergence = fixture - .store - .converge_session_git_evidence(&EmptyHistoryGit, 1, 1) - .await - .unwrap(); - assert!(convergence.later_failure().is_none(), "{convergence:?}"); - assert!(convergence.stats().reprojected_legacy_head); - assert!(convergence.committed_progress()); - - let health = fixture.store.correlation_index_health().await.unwrap(); - assert_eq!((health.span_count, health.commit_count), (1, 0)); - assert_ne!( - health.generation.as_deref(), - Some(legacy_generation.as_str()) - ); - let hits = fixture - .store - .sessions_for_with_relation(&branch_query("main"), CommitRelationFilter::Produced) - .await - .unwrap(); - assert_eq!( - hits.iter() - .map(|hit| hit.session_id.as_str()) - .collect::>(), - vec!["session.legacy"] - ); - assert_eq!( - fixture - .store - .git_evidence_projection() - .unwrap() - .unwrap() - .projector_revision(), - GitEvidenceProjectorRevision::Current - ); - - // An indexed head is not re-projected again. - let settled = fixture - .store - .converge_session_git_evidence(&EmptyHistoryGit, 1, 1) - .await - .unwrap(); - assert!(!settled.stats().reprojected_legacy_head); - } - #[tokio::test(flavor = "multi_thread", worker_threads = 1)] async fn cancelled_one_worker_span_caller_is_joined_before_owner_shutdown() { let (release, blocked) = mpsc::channel(); diff --git a/crates/tracedecay-global-db/src/git_topology_anchor.rs b/crates/tracedecay-global-db/src/git_topology_anchor.rs index 1cf5ef7b8b..fd9b7355c5 100644 --- a/crates/tracedecay-global-db/src/git_topology_anchor.rs +++ b/crates/tracedecay-global-db/src/git_topology_anchor.rs @@ -3,22 +3,22 @@ use std::collections::BTreeMap; use tracedecay_contracts::retrieval::{ - GitTopologyAnchorAuthorityErrorV2, GitTopologyAnchorAuthorityV2, GitTopologyAnchorFutureV2, - GitTopologyAnchorPublicationOutcomeV2, GitTopologyAnchorPublicationV2, - GitTopologyAnchorResolutionOutcomeV2, GitTopologyAnchorResolutionV2, + GitTopologyAnchorAuthority, GitTopologyAnchorAuthorityError, GitTopologyAnchorFuture, + GitTopologyAnchorPublication, GitTopologyAnchorPublicationOutcome, GitTopologyAnchorResolution, + GitTopologyAnchorResolutionOutcome, }; -use tracedecay_domain::{ObservationScopeV1, RetrievalAnchorRecordV2, RetrievalAnchorTargetV2}; +use tracedecay_domain::{ObservationScopeV1, RetrievalAnchorRecord, RetrievalAnchorTarget}; use tracedecay_runtime_core::db::engine::{params, params_from_iter}; use tracedecay_store::StoreShardScopeV1; use crate::{RegisteredGlobalDb, RegisteredGlobalDbLeaseV1}; #[derive(Clone)] -pub struct RegisteredGitTopologyAnchorAuthorityV2 { +pub struct RegisteredGitTopologyAnchorAuthority { database: RegisteredGlobalDbLeaseV1, } -impl RegisteredGitTopologyAnchorAuthorityV2 { +impl RegisteredGitTopologyAnchorAuthority { pub fn new(database: RegisteredGlobalDbLeaseV1) -> Self { Self { database } } @@ -26,10 +26,10 @@ impl RegisteredGitTopologyAnchorAuthorityV2 { #[hotpath::skip] async fn publish_records( &self, - publication: GitTopologyAnchorPublicationV2, - ) -> Result { + publication: GitTopologyAnchorPublication, + ) -> Result { if !binding_matches_owner(&self.database, publication.owner()) { - return Err(GitTopologyAnchorAuthorityErrorV2::Unavailable); + return Err(GitTopologyAnchorAuthorityError::Unavailable); } let transaction = self .database @@ -51,16 +51,16 @@ impl RegisteredGitTopologyAnchorAuthorityV2 { transaction .rollback() .await - .map_err(|_| GitTopologyAnchorAuthorityErrorV2::Unavailable)?; - return Err(GitTopologyAnchorAuthorityErrorV2::Conflict); + .map_err(|_| GitTopologyAnchorAuthorityError::Unavailable)?; + return Err(GitTopologyAnchorAuthorityError::Conflict); } None => {} } let anchor_json = serde_json::to_string(&candidate) - .map_err(|_| GitTopologyAnchorAuthorityErrorV2::Conflict)?; + .map_err(|_| GitTopologyAnchorAuthorityError::Conflict)?; let owner_json = candidate .owner_column_json() - .map_err(|_| GitTopologyAnchorAuthorityErrorV2::Conflict)?; + .map_err(|_| GitTopologyAnchorAuthorityError::Conflict)?; transaction .execute( "INSERT INTO retrieval_anchors ( @@ -79,19 +79,19 @@ impl RegisteredGitTopologyAnchorAuthorityV2 { } transaction.commit().await.map_err(map_engine_error)?; Ok(if published { - GitTopologyAnchorPublicationOutcomeV2::Published + GitTopologyAnchorPublicationOutcome::Published } else { - GitTopologyAnchorPublicationOutcomeV2::Replayed + GitTopologyAnchorPublicationOutcome::Replayed }) } #[hotpath::skip] async fn resolve_record( &self, - resolution: GitTopologyAnchorResolutionV2, - ) -> Result { + resolution: GitTopologyAnchorResolution, + ) -> Result { if !binding_matches_owner(&self.database, &resolution.owner) { - return Err(GitTopologyAnchorAuthorityErrorV2::Unavailable); + return Err(GitTopologyAnchorAuthorityError::Unavailable); } let snapshot = self .database @@ -99,36 +99,36 @@ impl RegisteredGitTopologyAnchorAuthorityV2 { .await .map_err(map_database_error)?; let Some(record) = read_record(&snapshot, resolution.anchor_id.as_str()).await? else { - return Ok(GitTopologyAnchorResolutionOutcomeV2::Unavailable); + return Ok(GitTopologyAnchorResolutionOutcome::Unavailable); }; if record.owner() != &resolution.owner { - return Ok(GitTopologyAnchorResolutionOutcomeV2::Unavailable); + return Ok(GitTopologyAnchorResolutionOutcome::Unavailable); } if !matches!( record.target(), - RetrievalAnchorTargetV2::GitTopology(_) - | RetrievalAnchorTargetV2::ExactRepositoryCommit { .. } + RetrievalAnchorTarget::GitTopology(_) + | RetrievalAnchorTarget::ExactRepositoryCommit { .. } ) { - return Ok(GitTopologyAnchorResolutionOutcomeV2::Unavailable); + return Ok(GitTopologyAnchorResolutionOutcome::Unavailable); } - Ok(GitTopologyAnchorResolutionOutcomeV2::Resolved(Box::new( + Ok(GitTopologyAnchorResolutionOutcome::Resolved(Box::new( record, ))) } } -impl GitTopologyAnchorAuthorityV2 for RegisteredGitTopologyAnchorAuthorityV2 { +impl GitTopologyAnchorAuthority for RegisteredGitTopologyAnchorAuthority { fn publish<'a>( &'a self, - publication: GitTopologyAnchorPublicationV2, - ) -> GitTopologyAnchorFutureV2<'a, GitTopologyAnchorPublicationOutcomeV2> { + publication: GitTopologyAnchorPublication, + ) -> GitTopologyAnchorFuture<'a, GitTopologyAnchorPublicationOutcome> { Box::pin(async move { self.publish_records(publication).await }) } fn resolve<'a>( &'a self, - resolution: GitTopologyAnchorResolutionV2, - ) -> GitTopologyAnchorFutureV2<'a, GitTopologyAnchorResolutionOutcomeV2> { + resolution: GitTopologyAnchorResolution, + ) -> GitTopologyAnchorFuture<'a, GitTopologyAnchorResolutionOutcome> { Box::pin(async move { self.resolve_record(resolution).await }) } } @@ -136,7 +136,7 @@ impl GitTopologyAnchorAuthorityV2 for RegisteredGitTopologyAnchorAuthorityV2 { async fn read_record( connection: &impl tracedecay_runtime_core::db::engine::QueryExecutor, anchor_id: &str, -) -> Result, GitTopologyAnchorAuthorityErrorV2> { +) -> Result, GitTopologyAnchorAuthorityError> { let mut rows = connection .query( "SELECT anchor_json, owner_json, projection_generation @@ -152,7 +152,7 @@ async fn read_record( let owner_json = row.get::(1).map_err(map_engine_error)?; let projection_generation = row.get::(2).map_err(map_engine_error)?; if rows.next().await.map_err(map_engine_error)?.is_some() { - return Err(GitTopologyAnchorAuthorityErrorV2::ResetRequired); + return Err(GitTopologyAnchorAuthorityError::ResetRequired); } decode_record(&anchor_json, &owner_json, &projection_generation).map(Some) } @@ -164,7 +164,7 @@ async fn read_record( async fn read_records( connection: &impl tracedecay_runtime_core::db::engine::QueryExecutor, anchor_ids: &[String], -) -> Result, GitTopologyAnchorAuthorityErrorV2> { +) -> Result, GitTopologyAnchorAuthorityError> { if anchor_ids.is_empty() { return Ok(BTreeMap::new()); } @@ -194,7 +194,7 @@ async fn read_records( let projection_generation = row.get::(3).map_err(map_engine_error)?; let record = decode_record(&anchor_json, &owner_json, &projection_generation)?; if record.anchor_id().as_str() != anchor_id || records.insert(anchor_id, record).is_some() { - return Err(GitTopologyAnchorAuthorityErrorV2::ResetRequired); + return Err(GitTopologyAnchorAuthorityError::ResetRequired); } } Ok(records) @@ -204,16 +204,16 @@ fn decode_record( anchor_json: &str, owner_json: &str, projection_generation: &str, -) -> Result { - let record = serde_json::from_str::(anchor_json) - .map_err(|_| GitTopologyAnchorAuthorityErrorV2::ResetRequired)?; +) -> Result { + let record = serde_json::from_str::(anchor_json) + .map_err(|_| GitTopologyAnchorAuthorityError::ResetRequired)?; record .validate() - .map_err(|_| GitTopologyAnchorAuthorityErrorV2::ResetRequired)?; + .map_err(|_| GitTopologyAnchorAuthorityError::ResetRequired)?; if !record.owner_column_matches(owner_json) || record.projection_generation().as_str() != projection_generation { - return Err(GitTopologyAnchorAuthorityErrorV2::ResetRequired); + return Err(GitTopologyAnchorAuthorityError::ResetRequired); } Ok(record) } @@ -233,22 +233,22 @@ fn binding_matches_owner(database: &RegisteredGlobalDb, owner: &ObservationScope fn map_database_error( error: tracedecay_domain::errors::TraceDecayError, -) -> GitTopologyAnchorAuthorityErrorV2 { +) -> GitTopologyAnchorAuthorityError { if error.reset_required_context().is_some() { - GitTopologyAnchorAuthorityErrorV2::ResetRequired + GitTopologyAnchorAuthorityError::ResetRequired } else { - GitTopologyAnchorAuthorityErrorV2::Unavailable + GitTopologyAnchorAuthorityError::Unavailable } } fn map_engine_error( error: tracedecay_runtime_core::db::engine::Error, -) -> GitTopologyAnchorAuthorityErrorV2 { +) -> GitTopologyAnchorAuthorityError { let detail = error.to_string(); if detail.contains("no such table") || detail.contains("no such column") { - GitTopologyAnchorAuthorityErrorV2::ResetRequired + GitTopologyAnchorAuthorityError::ResetRequired } else { - GitTopologyAnchorAuthorityErrorV2::Unavailable + GitTopologyAnchorAuthorityError::Unavailable } } diff --git a/crates/tracedecay-global-db/src/lib.rs b/crates/tracedecay-global-db/src/lib.rs index d8e822b3c9..ff9dddf612 100644 --- a/crates/tracedecay-global-db/src/lib.rs +++ b/crates/tracedecay-global-db/src/lib.rs @@ -47,7 +47,7 @@ pub use discovery_queue::HostDiscoveryQueueEntry; pub use git_correlation_adapter::{ GitEvidenceConvergenceOutcome, GitEvidenceConvergenceStats, GlobalDbGitCorrelationStore, }; -pub use git_topology_anchor::RegisteredGitTopologyAnchorAuthorityV2; +pub use git_topology_anchor::RegisteredGitTopologyAnchorAuthority; pub use observability_rollup::{ ObservabilityRollupCompactionCandidateV1, ObservabilityRollupCompactionReceiptV1, ObservabilityRollupCompactionV1, ObservabilityRollupDirtyDayClaimV1, @@ -73,7 +73,6 @@ mod registered_analytics; mod registered_dashboard; mod registered_lcm; mod registered_lcm_privacy; -mod registered_legacy_relations; mod registered_session_sync; mod registered_sessions; pub mod registry_maintenance; @@ -168,9 +167,6 @@ use support::{ global_db_operation_error, global_db_operation_message, like_pattern, normalize_git_remote_url, push_optional_analytics_filter, repo_identity_aliases, row_to_analytics_event, }; -/// Compatibility re-export: workflow search filters now live beside the -/// workflow-index contracts in [`tracedecay_sessions::runtime::workflow_index`]. -pub use tracedecay_sessions::runtime::workflow_index::WorkflowScopeFilter; #[cfg(all(test, not(windows)))] #[allow(clippy::unwrap_used, clippy::expect_used)] mod checkpoint_tests; diff --git a/crates/tracedecay-global-db/src/observation/codec.rs b/crates/tracedecay-global-db/src/observation/codec.rs index 8ac4a74d98..2eb1904a41 100644 --- a/crates/tracedecay-global-db/src/observation/codec.rs +++ b/crates/tracedecay-global-db/src/observation/codec.rs @@ -1,5 +1,5 @@ use tracedecay_domain::{ - EvidenceAvailabilityV1, GenerationBoundRepositoryProvenanceV1, RetrievalAnchorRecordV2, + EvidenceAvailabilityV1, GenerationBoundRepositoryProvenanceV1, RetrievalAnchorRecord, }; use tracedecay_store::{ ObservationStoreError, ObservationStoreResult, RepositoryProvenanceAttachmentV1, @@ -46,7 +46,7 @@ pub(super) fn decode_repository_provenance_attachment( RepositoryProvenanceAttachmentV1::new( availability, anchor_json - .map(|anchor| decode::(anchor, operation)) + .map(|anchor| decode::(anchor, operation)) .transpose()?, ) } diff --git a/crates/tracedecay-global-db/src/observation/mod.rs b/crates/tracedecay-global-db/src/observation/mod.rs index 9dfef35a6e..2be27fd6af 100644 --- a/crates/tracedecay-global-db/src/observation/mod.rs +++ b/crates/tracedecay-global-db/src/observation/mod.rs @@ -1,21 +1,18 @@ mod codec; mod persist; mod refusal_census; -mod reset; pub mod retention; mod schema; pub use refusal_census::{ ObservationRefusalCensusV1, ObservationRefusalCountV1, ingest_refusal_read_from_censuses, }; -pub use reset::{ObservationAuthorityResetV1, reset_refused_observation_authority}; pub(super) use schema::ensure_observation_schema; -pub use schema::{OBSERVATION_AUTHORITY, OBSERVATION_NATIVE_SOURCE_SCHEME_MIGRATION}; use tracedecay_domain::{ - AnchorSourceGenerationV2, CanonicalObservationIdV1, ObservationScopeV1, - ObservationSourceGenerationV1, RetrievalAnchorId, RetrievalAnchorRecordV2, - RetrievalAnchorTargetV2, VectorWatermark, + AnchorSourceGeneration, CanonicalObservationIdV1, ObservationScopeV1, + ObservationSourceGenerationV1, RetrievalAnchorId, RetrievalAnchorRecord, RetrievalAnchorTarget, + VectorWatermark, }; use tracedecay_store::{ ObservationStoreError, ObservationStoreResult, ObservedEvidenceAnchorResolution, @@ -72,7 +69,7 @@ async fn resolve_owner_bound_anchor_record( conn: &impl QueryExecutor, owner: &ObservationScopeV1, anchor_id: &RetrievalAnchorId, -) -> ObservationStoreResult> { +) -> ObservationStoreResult> { let Some(observation_id) = read_observation_id_for_retrieval_anchor(conn, anchor_id).await? else { return Ok(None); @@ -116,19 +113,19 @@ async fn resolve_owner_bound_anchor_record( } fn validate_exact_observation_provenance( - target: &RetrievalAnchorTargetV2, - source_generation: &AnchorSourceGenerationV2, + target: &RetrievalAnchorTarget, + source_generation: &AnchorSourceGeneration, source_observations: &[CanonicalObservationIdV1], observation_id: &CanonicalObservationIdV1, observation_generation: ObservationSourceGenerationV1, ) -> ObservationStoreResult<()> { - let RetrievalAnchorTargetV2::ExactObservation(target_observation_id) = target else { + let RetrievalAnchorTarget::ExactObservation(target_observation_id) = target else { return Ok(()); }; if target_observation_id != observation_id { return Err(ObservationStoreError::RetrievalAnchorObservationMismatch); } - if source_generation != &AnchorSourceGenerationV2::Observation(observation_generation) { + if source_generation != &AnchorSourceGeneration::Observation(observation_generation) { return Err(ObservationStoreError::RetrievalAnchorSourceGenerationMismatch); } if source_observations != std::slice::from_ref(observation_id) { @@ -182,7 +179,7 @@ impl super::RegisteredGlobalDb { &self, owner: &ObservationScopeV1, anchor_id: &RetrievalAnchorId, - ) -> ObservationStoreResult> { + ) -> ObservationStoreResult> { let snapshot = self .read_snapshot() .await @@ -232,8 +229,8 @@ mod tests { let canonical_id = observation_id("a"); let other_id = observation_id("b"); let generation = ObservationSourceGenerationV1::new(7).unwrap(); - let target = RetrievalAnchorTargetV2::ExactObservation(canonical_id.clone()); - let source_generation = AnchorSourceGenerationV2::Observation(generation); + let target = RetrievalAnchorTarget::ExactObservation(canonical_id.clone()); + let source_generation = AnchorSourceGeneration::Observation(generation); assert!( validate_exact_observation_provenance( @@ -248,7 +245,7 @@ mod tests { assert!(matches!( validate_exact_observation_provenance( &target, - &AnchorSourceGenerationV2::Observation( + &AnchorSourceGeneration::Observation( ObservationSourceGenerationV1::new(8).unwrap(), ), std::slice::from_ref(&canonical_id), @@ -269,7 +266,7 @@ mod tests { )); assert!(matches!( validate_exact_observation_provenance( - &RetrievalAnchorTargetV2::ExactObservation(observation_id("c")), + &RetrievalAnchorTarget::ExactObservation(observation_id("c")), &source_generation, std::slice::from_ref(&canonical_id), &canonical_id, diff --git a/crates/tracedecay-global-db/src/observation/reset.rs b/crates/tracedecay-global-db/src/observation/reset.rs deleted file mode 100644 index 78195cea87..0000000000 --- a/crates/tracedecay-global-db/src/observation/reset.rs +++ /dev/null @@ -1,768 +0,0 @@ -//! Scoped operator recovery for a refused observation authority. -//! -//! Admission refuses a sessions store whose `observations` or -//! `source_cursor_advances` table carries a pre-release branch-local shape, -//! or whose retained rows were committed under a superseded native-source -//! scheme (see [`OBSERVATION_NATIVE_SOURCE_SCHEME_MIGRATION`]), both typed -//! `ResetRequired` naming [`OBSERVATION_AUTHORITY`]. Because the refusal -//! fires before any runtime can mount the store, recovery runs offline over a -//! plain connection while the operator holds the profile's exclusive -//! maintenance lease. -//! -//! The reset is scoped to exactly the refused authority: it drops the -//! observation-authority tables plus their pure projection derivations, -//! recreates every one of them empty at the canonical shape (through the same -//! DDL, index, and trigger authorities the schema installer uses, attach -//! only validates existing stores, it never reinstalls), clears the -//! `session_messages` projector output (classified `Recoverable`; the cleared -//! evidence re-derives by re-ingesting provider transcripts), and preserves -//! everything else in the store, transcripts, LCM content, configuration, -//! registry, workflow, and the session-temporal state that is not a -//! projection of observations. The session-temporal projection *is* one, so it -//! resets with the stream it projects (see -//! [`OBSERVATION_DERIVED_TEMPORAL_DELETES`]) rather than being orphaned or -//! left advertising coverage of rows that no longer exist. So are the -//! retrieval anchors the reset observations bound and the native-record -//! aliases that resolve to them (see [`OBSERVATION_ANCHOR_BINDING_COLUMNS`]): -//! an anchor is verified field-for-field when its observation is admitted -//! again, so a retained one whose source generation moved (the transcript -//! file was replaced) fails every re-admission as a storage collision, and a -//! retained alias whose record was revised refuses it deterministically, -//! either way the rebuild the reset promises never happens. -//! -//! The derived usage (`observation_provider_usage`), the admission cursors -//! (`source_cursors`, `source_cursor_advances`), and the native-source -//! scheduling cursors in `parse_offsets` (see -//! [`NATIVE_SOURCE_SCHEDULING_CURSOR_DELETE`]) always reset together: leaving -//! any of them behind is what would let the rebuilt authority double-count or -//! skip the native events it re-reads. -//! -//! The host-observation external-source journal is the same class. Those -//! receipts and current-state rows attest observation commits the reset just -//! destroyed. Re-admission of an unchanged transcript reuses the same -//! logical-effect idempotency key (derived from the stable observation id) -//! with a new request digest, new anchors, cursors, and sanitization -//! receipts, so the surviving journal reports a conflict and the admission -//! worker retries the whole batch on its fixed cadence. That is not a -//! supersession of the prior command: the attested state was deliberately -//! removed. The receipts therefore reset with the stream they describe (see -//! [`OBSERVATION_DERIVED_EXTERNAL_SOURCE_DELETES`]), together with the -//! writer-ledger rows that name those same `external-source.` keys. Other -//! ledger identities stay; the exclusive maintenance transaction is the -//! lease that authorizes exactly this scoped retirement. -//! -//! Two invariants bound the deletion. Rows the reset preserves must never be -//! left pointing at rows it removes: [`PRESERVED_DEPENDENT_TABLES`] refuses -//! atomically for the one such dependency that has no safe scoped treatment, -//! and `PRAGMA foreign_key_check` proves the rest before the transaction -//! commits (the reset suspends per-statement enforcement for its own -//! intermediate drop states, and `PRAGMA integrity_check` does not cover -//! foreign keys). And every step, trigger removal, deletion, temporal -//! invalidation, trigger restoration, scheme enrollment, runs inside that one -//! transaction, so a failure anywhere leaves the store exactly as refused. - -use std::collections::BTreeSet; -use std::time::{SystemTime, UNIX_EPOCH}; - -use tracedecay_domain::errors::TraceDecayError; - -use super::schema::{ - OBSERVATION_AUTHORITY, OBSERVATION_AUTHORITY_SCHEMA_SQL, OBSERVATION_CANONICAL_COLUMNS, - OBSERVATION_NATIVE_SOURCE_SCHEME_MIGRATION, OBSERVATION_SCHEMA_MIGRATION, - SOURCE_CURSOR_ADVANCES_CANONICAL_COLUMNS, -}; -use crate::observation_projection::{ - OBSERVATION_PROJECTION_BINDING_TRIGGERS_SQL, OBSERVATION_PROJECTION_PERFORMANCE_INDEX_SQL, - OBSERVATION_PROJECTION_SCHEMA_SQL, -}; -use crate::schema_contract::{ - invariant_trigger_names_for_tables, invariant_trigger_sql_for_tables, -}; -use tracedecay_runtime_core::db::retrieval_anchor_schema::{ - RETRIEVAL_ANCHOR_DELETE_GUARD_TRIGGERS, RETRIEVAL_ANCHOR_IMMUTABILITY_TRIGGERS_SQL, -}; - -const OPERATION: &str = "reset refused observation authority"; - -/// Tables owned by `ensure_observation_schema`; the next admission recreates -/// every one of them empty at the canonical shape. -const OBSERVATION_AUTHORITY_TABLES: &[&str] = &[ - "observations", - "sanitization_receipts", - "source_cursors", - "source_cursor_advances", - "observation_admission_refusals", - "projection_queue", - "remote_writer_fences", - "remote_observation_events", - "observation_retrieval_anchors", - "observation_repository_provenance", - "observation_repository_captures", -]; - -/// Pure derivations of the observation stream owned by -/// `ensure_observation_projection_schema`. They are outputs of the projection -/// and rebuild machinery over `observations`, so they reset with it; the next -/// admission recreates them empty. -const OBSERVATION_PROJECTION_TABLES: &[&str] = &[ - "observation_projection_provenance", - "observation_projection_checkpoints", - "observation_projection_aliases", - "observation_projection_dispositions", - "observation_workflow_facts", - "observation_provider_usage", - "observation_projection_rebuilds", - "observation_projection_rebuild_provider_usage", - "observation_projection_rebuild_aliases", - "observation_projection_rebuild_sessions", - "observation_projection_rebuild_messages", - "observation_projection_rebuild_provenance", - "observation_projection_rebuild_dispositions", - "observation_projection_rebuild_workflow_facts", -]; - -/// Derived-temporal tables whose invariant triggers would abort the deletes -/// below. Their triggers are dropped and reinstalled from the same canonical -/// authority inside the reset transaction, the way the observation tables are -/// dropped and recreated: immutability guards ordinary writers, not the -/// authority rebuild itself. Runtime immutability is unchanged, the store -/// leaves this function carrying the same trigger set it arrived with. -/// -/// `session_occurrences` is deliberately absent: its triggers keep the -/// external-content `session_occurrences_fts` index in step with the deletes, -/// so they must stay installed while the rows go. -const IMMUTABLE_DERIVED_TEMPORAL_TABLES: &[&str] = &[ - "session_refresh_batch_bindings", - "session_refresh_bindings", - "session_refresh_progress", - "session_refresh_receipts", - "session_refresh_operations", - "session_temporal_projection_receipts", - "session_temporal_generations", - "session_temporal_observation_effects", -]; - -/// The whole session-temporal projection of the observation stream, in -/// dependency order (foreign-key enforcement is suspended, so the cascades -/// these rows normally ride do not fire; each table is named explicitly and -/// `PRAGMA foreign_key_check` proves nothing was missed). -/// -/// Every generation is projector output over `observations`: its occurrences -/// and their anchors, turn/thread/agent grouping, assertions and their -/// supersession, current entities, span and burst evidence, summary -/// availability, and the projection receipts that certify exactly those -/// counts and digests. Deleting occurrences while preserving the generation -/// would leave those receipts certifying coverage that no longer exists and -/// `validate_final_projection_receipt` recomputing a disagreement, and a -/// retained active frontier of 100 would exclude the re-ingested effects -/// 1..=100 from `pending_session_temporal_refresh_page_result`, suppressing -/// the rebuild it is supposed to trigger. So the generation is invalidated -/// outright, together with the relation receipts and refresh bindings that -/// authorize serving or resuming against it; the operator-visible refresh -/// history in `session_refresh_operations` goes with them because a `running` -/// row would keep the rebuilt stream out of discovery forever. -/// -/// Everything session-temporal that is not projector output stays: summary -/// nodes and their FTS index, external payload manifests (see -/// [`PRESERVED_DEPENDENT_TABLES`]), retained cursor keys, and the retrieval anchors -/// the rebuilt projection re-attaches to. The active cursor key rotates so a -/// rebuilt generation cannot alias a pre-reset frozen snapshot. -const OBSERVATION_DERIVED_TEMPORAL_DELETES: &[&str] = &[ - "DELETE FROM session_refresh_batch_bindings", - "DELETE FROM session_refresh_bindings", - "DELETE FROM session_refresh_progress", - "DELETE FROM session_refresh_receipts", - "DELETE FROM session_refresh_operations", - "DELETE FROM session_derived_evidence_members", - "DELETE FROM session_derived_evidence", - "DELETE FROM session_current_entities", - "DELETE FROM session_assertion_supersession", - "DELETE FROM session_assertions", - "DELETE FROM session_turn_members", - "DELETE FROM session_summary_availability", - "DELETE FROM session_occurrences", - "DELETE FROM session_turns", - "DELETE FROM session_threads", - "DELETE FROM session_agents", - "DELETE FROM session_temporal_projection_receipts", - "DELETE FROM session_relation_effect_journal", - "DELETE FROM session_relation_receipts", - "DELETE FROM session_temporal_generations", - "DELETE FROM session_temporal_observation_effects", -]; - -/// Every `(table, column)` through which a reset observation table binds a -/// `retrieval_anchors` row: the exact-observation anchor, the repository -/// capture anchor, and the projector's message anchors. The anchors those -/// columns name, and the aliases resolving native records to them, are -/// re-derived and re-verified by the next admission of the same records; they -/// go with the stream. Anchors owned by preserved rows, summary anchors, -/// git-topology anchors, are never named here and stay. -const OBSERVATION_ANCHOR_BINDING_COLUMNS: &[(&str, &str)] = &[ - ("observation_retrieval_anchors", "anchor_id"), - ("observation_repository_provenance", "retrieval_anchor_id"), - ("observation_projection_provenance", "retrieval_anchor_id"), - ("observation_workflow_facts", "retrieval_anchor_id"), - ( - "observation_projection_rebuild_provenance", - "retrieval_anchor_id", - ), - ( - "observation_projection_rebuild_workflow_facts", - "retrieval_anchor_id", - ), -]; - -/// The `parse_offsets` rows that schedule native-source admission, by key -/// namespace: per-provider coverage verdicts (`host-coverage://`), host -/// discovery frontiers (`host-frontier://`), the discovery queue -/// (`host-discovery-queue://`), and the internal history frontiers, corpus -/// epochs, and provider-rotation cursors (`tracedecay-internal:`). A retained -/// Codex epoch that says the corpus was swept, or a coverage row that says -/// `complete`, makes the rebuilt authority skip exactly the transcripts the -/// reset promised to re-read, so they reset with the observation cursors. -/// The only other tenant of the table, the hook-analytics import cursor, feeds -/// `analytics_events` and is not a derivation of observations; it stays. -const NATIVE_SOURCE_SCHEDULING_CURSOR_DELETE: &str = - "DELETE FROM parse_offsets WHERE file_path NOT LIKE 'hook_analytics:%'"; - -/// Host-observation projector tables that attest the observation stream. -/// -/// Current-state and receipt rows are not reconstructible from the -/// transcripts the way LCM content is: they name the anchors, frontiers, and -/// sanitization receipts the reset recreates empty. Leaving them makes the -/// next admission of the same observation id a conflicting reuse of the -/// prior command rather than a rebuild. Definition and binding revisions -/// stay, they are the source contract, not observation content, and the -/// next commit `INSERT OR IGNORE`s them. -const OBSERVATION_DERIVED_EXTERNAL_SOURCE_DELETES: &[&str] = &[ - "DELETE FROM external_source_acquisition_queue_v1", - "DELETE FROM external_source_pending_projections_v1", - "DELETE FROM external_source_projection_effects_v2", - "DELETE FROM external_source_projection_lineage_v1", - "DELETE FROM external_source_projected_objects_v2", - "DELETE FROM external_source_projection_publications_v2", - "DELETE FROM external_source_mutations_v1", - "DELETE FROM external_source_lineage_v1", - "DELETE FROM external_source_objects_v2", - "DELETE FROM external_source_commit_receipts_v2", - "DELETE FROM external_source_frontiers_v1", - "DELETE FROM external_source_authority_receipts_v1", - "DELETE FROM external_source_states_v1", -]; - -/// Writer-ledger identities minted for external-source commits -/// (`external-source.{logical-effect-suffix}`). A remount that keeps the -/// same incarnation and epoch would otherwise replay those rows as a -/// conflict even after the receipt tables are empty. Other ledger keys, -/// including newer or foreign markers, are not named here. -const EXTERNAL_SOURCE_RUNTIME_IDEMPOTENCY_DELETE: &str = - "DELETE FROM td_runtime_writer_idempotency_v2 WHERE idempotency_key LIKE 'external-source.%'"; - -/// Preserved rows that would be orphaned by the reset, with the authority -/// they would be orphaned from. -/// -/// `session_external_payload_manifests` is LCM publication metadata, durable, -/// immutable by trigger, and outside every deletion above, whose `receipt_id` -/// names a `sanitization_receipts` row the reset drops with the observation -/// authority. There is no scoped treatment that keeps both coherent: the -/// manifests are not reconstructible from the transcripts, and deleting -/// external-payload metadata to make the reset succeed would destroy exactly -/// the evidence a scoped reset promises to preserve. Such a store refuses -/// atomically and needs an LCM-side remediation first. -const PRESERVED_DEPENDENT_TABLES: &[(&str, &str)] = &[( - "session_external_payload_manifests", - "sanitization_receipts", -)]; - -/// Outcome of one completed scoped reset. -#[derive(Debug)] -pub struct ObservationAuthorityResetV1 { - /// Tables dropped and recreated empty at the canonical shape. - pub reset_tables: Vec, - /// `session_messages` projector-output rows cleared (`Recoverable`; the - /// external-content FTS index is synchronized by its delete trigger). - pub cleared_session_message_rows: u64, - /// Session-temporal projection rows cleared because they derive from the - /// reset observation stream (see `OBSERVATION_DERIVED_TEMPORAL_DELETES`). - pub cleared_derived_temporal_rows: u64, - /// Retrieval anchors (and the native-record aliases resolving to them) - /// cleared because the reset observation stream bound them (see - /// `OBSERVATION_ANCHOR_BINDING_COLUMNS`). - pub cleared_retrieval_anchor_rows: u64, - /// Native-source scheduling cursors cleared so the next open re-reads - /// every provider transcript (see - /// `NATIVE_SOURCE_SCHEDULING_CURSOR_DELETE`). - pub cleared_native_source_cursor_rows: u64, - /// Host-observation external-source receipts, current-state rows, and - /// matching writer-ledger identities cleared so re-admission is a rebuild - /// rather than a conflicting reuse of the destroyed stream (see - /// `OBSERVATION_DERIVED_EXTERNAL_SOURCE_DELETES`). - pub cleared_external_source_rows: u64, -} - -fn reset_storage(error: impl std::fmt::Display) -> TraceDecayError { - TraceDecayError::Database { - operation: OPERATION.to_string(), - message: error.to_string(), - } -} - -/// Offline reset cannot reuse the async ensure-active adapter: it must rotate -/// even an existing healthy key, in this exact maintenance transaction. The -/// canonical INSERT triggers retire the previous key without deleting history. -fn rotate_session_cursor_key(conn: &rusqlite::Connection) -> Result<(), TraceDecayError> { - if !table_exists(conn, "session_query_cursor_keys")? { - return Ok(()); - } - let (count, active, version, latest_time): (i64, i64, i64, i64) = conn - .query_row( - "SELECT COUNT(*), COALESCE(SUM(retired_at IS NULL), 0), - COALESCE(MAX(key_version), 0), - COALESCE(MAX(MAX(created_at, COALESCE(retired_at, created_at))), 0) - FROM session_query_cursor_keys", - [], - |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), - ) - .map_err(reset_storage)?; - if count == 0 { - return Ok(()); - } - if active != 1 || version < 1 { - return Err(reset_storage( - "session cursor key rotation state is invalid", - )); - } - let next_version = version - .checked_add(1) - .and_then(|value| u16::try_from(value).ok()) - .ok_or_else(|| reset_storage("session cursor key version exhausted"))?; - let minimum_created_at = latest_time - .checked_add(1) - .ok_or_else(|| reset_storage("session cursor key timestamp exhausted"))?; - let observed_at = i64::try_from( - SystemTime::now() - .duration_since(UNIX_EPOCH) - .map_err(reset_storage)? - .as_micros(), - ) - .map_err(reset_storage)?; - let created_at = observed_at.max(minimum_created_at); - let mut random = [0_u8; 48]; - getrandom::getrandom(&mut random).map_err(reset_storage)?; - let key_id = format!("cursor-key-{next_version}-{}", hex::encode(&random[..16])); - conn.execute( - "INSERT INTO session_query_cursor_keys - (key_id, key_version, key_material, created_at, retired_at) - VALUES (?1, ?2, ?3, ?4, NULL)", - rusqlite::params![key_id, i64::from(next_version), &random[16..], created_at], - ) - .map_err(reset_storage)?; - Ok(()) -} - -fn table_exists(conn: &rusqlite::Connection, table: &str) -> Result { - conn.query_row( - "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?1)", - [table], - |row| row.get::<_, bool>(0), - ) - .map_err(reset_storage) -} - -fn migration_recorded( - conn: &rusqlite::Connection, - migration: &str, -) -> Result { - if !table_exists(conn, "global_schema_migrations")? { - return Ok(false); - } - conn.query_row( - "SELECT EXISTS(SELECT 1 FROM global_schema_migrations WHERE migration = ?1)", - [migration], - |row| row.get::<_, bool>(0), - ) - .map_err(reset_storage) -} - -fn table_columns( - conn: &rusqlite::Connection, - table: &str, -) -> Result, TraceDecayError> { - let mut statement = conn - .prepare("SELECT name FROM pragma_table_xinfo(?1)") - .map_err(reset_storage)?; - let columns = statement - .query_map([table], |row| row.get::<_, String>(0)) - .map_err(reset_storage)? - .collect::, _>>() - .map_err(reset_storage)?; - Ok(columns) -} - -fn row_count(conn: &rusqlite::Connection, table: &str) -> Result { - let count = conn - .query_row(&format!("SELECT COUNT(*) FROM \"{table}\""), [], |row| { - row.get::<_, i64>(0) - }) - .map_err(reset_storage)?; - u64::try_from(count).map_err(|_| TraceDecayError::Database { - operation: OPERATION.to_string(), - message: format!("{table} row count was negative"), - }) -} - -fn canonical(columns: &[&str]) -> BTreeSet { - columns.iter().map(|column| (*column).to_string()).collect() -} - -/// Whether the store currently carries a shape the observation authority -/// refuses at admission. Mirrors the refusal predicates in `super::schema` -/// through the shared canonical column sets. -fn observation_authority_refused(conn: &rusqlite::Connection) -> Result { - if table_exists(conn, "observations")? { - if !migration_recorded(conn, OBSERVATION_SCHEMA_MIGRATION)? - || table_columns(conn, "observations")? != canonical(OBSERVATION_CANONICAL_COLUMNS) - { - return Ok(true); - } - // Content identity, not shape: rows written under a superseded - // native-source scheme refuse admission because re-offering them - // would double-count. Mirrors the schema-side predicate. - let populated = row_count(conn, "observations")? > 0 - || (table_exists(conn, "source_cursors")? && row_count(conn, "source_cursors")? > 0); - if populated && !migration_recorded(conn, OBSERVATION_NATIVE_SOURCE_SCHEME_MIGRATION)? { - return Ok(true); - } - } - if table_exists(conn, "source_cursor_advances")? - && table_columns(conn, "source_cursor_advances")? - != canonical(SOURCE_CURSOR_ADVANCES_CANONICAL_COLUMNS) - { - return Ok(true); - } - Ok(false) -} - -/// Whether any preserved row is left pointing at a row that no longer exists. -/// -/// The reset suspends per-statement foreign-key enforcement for its own -/// intermediate drop states, so this is what proves the committed store is -/// referentially coherent. `PRAGMA integrity_check` does not look at foreign -/// keys, and `foreign_key_check` reports violations whether or not -/// enforcement is on. -fn require_referential_integrity(conn: &rusqlite::Connection) -> Result<(), TraceDecayError> { - let mut statement = conn - .prepare("PRAGMA foreign_key_check") - .map_err(reset_storage)?; - let mut rows = statement.query([]).map_err(reset_storage)?; - let Some(row) = rows.next().map_err(reset_storage)? else { - return Ok(()); - }; - let table = row.get::<_, String>(0).map_err(reset_storage)?; - let parent = row.get::<_, String>(2).map_err(reset_storage)?; - Err(TraceDecayError::Config { - message: format!( - "a scoped {OBSERVATION_AUTHORITY} reset would leave {table} row(s) referencing \ - missing {parent} row(s); this store needs a remediation for that dependency \ - first, so nothing was reset" - ), - }) -} - -/// Resets exactly the refused observation authority in one transaction. -/// -/// Fails closed, mutating nothing, when the authority is not actually in a -/// refused shape (protecting healthy data from an accidental reset), when a -/// preserved dependency has no safe scoped treatment, or when the resulting -/// store would not be referentially coherent. -pub fn reset_refused_observation_authority( - conn: &mut rusqlite::Connection, -) -> Result { - // The reset drops the refused tables together with every table that - // references them, so per-statement foreign-key enforcement would only - // reject the intermediate drop states of an exclusive maintenance - // connection; `require_referential_integrity` proves the committed result - // instead. The setting belongs to the connection, not to this operation, - // so it is restored before the caller reuses it, on both paths. - let enforced_foreign_keys = conn - .query_row("PRAGMA foreign_keys", [], |row| row.get::<_, bool>(0)) - .map_err(reset_storage)?; - conn.pragma_update(None, "foreign_keys", false) - .map_err(reset_storage)?; - let outcome = reset_within_maintenance_transaction(conn); - let restored = conn - .pragma_update(None, "foreign_keys", enforced_foreign_keys) - .map_err(reset_storage); - let report = outcome?; - restored?; - Ok(report) -} - -fn reset_within_maintenance_transaction( - conn: &mut rusqlite::Connection, -) -> Result { - let transaction = conn - .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate) - .map_err(reset_storage)?; - if !observation_authority_refused(&transaction)? { - return Err(TraceDecayError::Config { - message: format!( - "the {OBSERVATION_AUTHORITY} authority in this store is not in a refused state; \ - nothing was reset" - ), - }); - } - for (table, parent) in PRESERVED_DEPENDENT_TABLES { - if !table_exists(&transaction, table)? { - continue; - } - let rows = row_count(&transaction, table)?; - if rows > 0 { - return Err(TraceDecayError::Config { - message: format!( - "a scoped {OBSERVATION_AUTHORITY} reset would orphan {rows} preserved row(s) \ - in {table} from the {parent} the reset recreates empty; this store needs a \ - remediation for that dependency first, so nothing was reset" - ), - }); - } - } - rotate_session_cursor_key(&transaction)?; - // The session-temporal projection derives from the observation stream, so - // it resets with it rather than being orphaned or refused over. - let mut cleared_derived_temporal_rows = 0u64; - for name in invariant_trigger_names_for_tables(IMMUTABLE_DERIVED_TEMPORAL_TABLES) { - transaction - .execute_batch(&format!("DROP TRIGGER IF EXISTS \"{name}\"")) - .map_err(reset_storage)?; - } - for statement in OBSERVATION_DERIVED_TEMPORAL_DELETES { - let table = statement - .strip_prefix("DELETE FROM ") - .and_then(|rest| rest.split_whitespace().next()) - .expect("each derived-temporal statement names its table"); - if !table_exists(&transaction, table)? { - continue; - } - let deleted = transaction.execute(statement, []).map_err(reset_storage)?; - cleared_derived_temporal_rows = - cleared_derived_temporal_rows.saturating_add(u64::try_from(deleted).map_err(|_| { - TraceDecayError::Database { - operation: OPERATION.to_string(), - message: format!("{table} delete count overflowed"), - } - })?); - } - for sql in invariant_trigger_sql_for_tables(IMMUTABLE_DERIVED_TEMPORAL_TABLES) { - transaction.execute_batch(sql).map_err(reset_storage)?; - } - let cleared_retrieval_anchor_rows = clear_observation_bound_anchors(&transaction)?; - let cleared_native_source_cursor_rows = if table_exists(&transaction, "parse_offsets")? { - u64::try_from( - transaction - .execute(NATIVE_SOURCE_SCHEDULING_CURSOR_DELETE, []) - .map_err(reset_storage)?, - ) - .map_err(|_| TraceDecayError::Database { - operation: OPERATION.to_string(), - message: "parse_offsets delete count overflowed".to_string(), - })? - } else { - 0 - }; - let cleared_external_source_rows = clear_observation_derived_external_source(&transaction)?; - - // Clear the recoverable projector output before dropping the projection - // tables: the audit-invalidation trigger on `session_messages` reads - // `observation_projection_provenance` and must still resolve while the - // deletes run. - let cleared_session_message_rows = if table_exists(&transaction, "session_messages")? { - u64::try_from( - transaction - .execute("DELETE FROM session_messages", []) - .map_err(reset_storage)?, - ) - .map_err(|_| TraceDecayError::Database { - operation: OPERATION.to_string(), - message: "session_messages delete count overflowed".to_string(), - })? - } else { - 0 - }; - let mut reset_tables = Vec::new(); - for table in OBSERVATION_AUTHORITY_TABLES - .iter() - .chain(OBSERVATION_PROJECTION_TABLES) - { - if table_exists(&transaction, table)? { - transaction - .execute_batch(&format!("DROP TABLE \"{table}\"")) - .map_err(reset_storage)?; - } - reset_tables.push((*table).to_string()); - } - // Recreate the authority empty at the canonical shape through the same - // DDL, index, and invariant-trigger authorities the schema installer - // uses. Attach only validates an existing store, it never reinstalls, - // so the reset itself must leave the store at the final contract. - transaction - .execute_batch(OBSERVATION_AUTHORITY_SCHEMA_SQL) - .map_err(reset_storage)?; - transaction - .execute_batch(OBSERVATION_PROJECTION_SCHEMA_SQL) - .map_err(reset_storage)?; - transaction - .execute_batch(OBSERVATION_PROJECTION_BINDING_TRIGGERS_SQL) - .map_err(reset_storage)?; - for sql in OBSERVATION_PROJECTION_PERFORMANCE_INDEX_SQL { - transaction.execute_batch(sql).map_err(reset_storage)?; - } - let reset_table_names = reset_tables.iter().map(String::as_str).collect::>(); - for sql in invariant_trigger_sql_for_tables(&reset_table_names) { - transaction.execute_batch(sql).map_err(reset_storage)?; - } - for migration in [ - OBSERVATION_SCHEMA_MIGRATION, - OBSERVATION_NATIVE_SOURCE_SCHEME_MIGRATION, - ] { - transaction - .execute( - "INSERT OR IGNORE INTO global_schema_migrations(migration) VALUES (?1)", - [migration], - ) - .map_err(reset_storage)?; - } - // The observation-authority audit checkpoint attests to rows that no - // longer exist; clear it so convergence re-audits the recreated authority - // from the start. - if table_exists(&transaction, "authority_audit_checkpoints")? { - transaction - .execute( - "DELETE FROM authority_audit_checkpoints WHERE audit_name = 'observation-authority'", - [], - ) - .map_err(reset_storage)?; - } - require_referential_integrity(&transaction)?; - transaction.commit().map_err(reset_storage)?; - Ok(ObservationAuthorityResetV1 { - reset_tables, - cleared_session_message_rows, - cleared_derived_temporal_rows, - cleared_retrieval_anchor_rows, - cleared_native_source_cursor_rows, - cleared_external_source_rows, - }) -} - -/// Removes the host-observation journal that attested the reset stream. -/// Runs inside the maintenance transaction with foreign keys suspended, the -/// same way the temporal projection and scheduling cursors are retired. -fn clear_observation_derived_external_source( - transaction: &rusqlite::Transaction<'_>, -) -> Result { - let mut cleared = 0u64; - for statement in OBSERVATION_DERIVED_EXTERNAL_SOURCE_DELETES { - let table = statement - .strip_prefix("DELETE FROM ") - .and_then(|rest| rest.split_whitespace().next()) - .expect("each external-source statement names its table"); - if !table_exists(transaction, table)? { - continue; - } - let deleted = transaction.execute(statement, []).map_err(reset_storage)?; - cleared = cleared.saturating_add(u64::try_from(deleted).map_err(|_| { - TraceDecayError::Database { - operation: OPERATION.to_string(), - message: format!("{table} delete count overflowed"), - } - })?); - } - if table_exists(transaction, "td_runtime_writer_idempotency_v2")? { - let deleted = transaction - .execute(EXTERNAL_SOURCE_RUNTIME_IDEMPOTENCY_DELETE, []) - .map_err(reset_storage)?; - cleared = cleared.saturating_add(u64::try_from(deleted).map_err(|_| { - TraceDecayError::Database { - operation: OPERATION.to_string(), - message: "td_runtime_writer_idempotency_v2 delete count overflowed".to_string(), - } - })?); - } - Ok(cleared) -} - -/// Removes the retrieval anchors the reset observation stream bound, and the -/// aliases resolving native records to them, while their binding tables still -/// exist to name them. Runs inside the maintenance transaction: the two -/// delete guards come off, the rows go, and every anchor guard is reinstalled -/// from the schema authority before the transaction can commit. -fn clear_observation_bound_anchors( - transaction: &rusqlite::Transaction<'_>, -) -> Result { - if !table_exists(transaction, "retrieval_anchors")? { - return Ok(0); - } - let mut bound = BTreeSet::new(); - for (table, column) in OBSERVATION_ANCHOR_BINDING_COLUMNS { - if !table_exists(transaction, table)? { - continue; - } - if !table_columns(transaction, table)?.contains(*column) { - return Err(TraceDecayError::Config { - message: format!( - "a scoped {OBSERVATION_AUTHORITY} reset expects {table}.{column} to name \ - the retrieval anchors it binds; this store carries a shape the reset \ - does not know how to invalidate, so nothing was reset" - ), - }); - } - let mut statement = transaction - .prepare(&format!( - "SELECT DISTINCT \"{column}\" FROM \"{table}\" WHERE \"{column}\" IS NOT NULL" - )) - .map_err(reset_storage)?; - let anchors = statement - .query_map([], |row| row.get::<_, String>(0)) - .map_err(reset_storage)? - .collect::, _>>() - .map_err(reset_storage)?; - bound.extend(anchors); - } - if bound.is_empty() { - return Ok(0); - } - for trigger in RETRIEVAL_ANCHOR_DELETE_GUARD_TRIGGERS { - transaction - .execute_batch(&format!("DROP TRIGGER IF EXISTS \"{trigger}\"")) - .map_err(reset_storage)?; - } - let mut cleared = 0u64; - let has_aliases = table_exists(transaction, "retrieval_anchor_aliases")?; - for anchor_id in &bound { - if has_aliases { - let aliases = transaction - .execute( - "DELETE FROM retrieval_anchor_aliases WHERE anchor_id = ?1", - [anchor_id], - ) - .map_err(reset_storage)?; - cleared = cleared.saturating_add(u64::try_from(aliases).unwrap_or(u64::MAX)); - } - let anchors = transaction - .execute( - "DELETE FROM retrieval_anchors WHERE anchor_id = ?1", - [anchor_id], - ) - .map_err(reset_storage)?; - cleared = cleared.saturating_add(u64::try_from(anchors).unwrap_or(u64::MAX)); - } - transaction - .execute_batch(RETRIEVAL_ANCHOR_IMMUTABILITY_TRIGGERS_SQL) - .map_err(reset_storage)?; - Ok(cleared) -} - -#[cfg(test)] -mod tests; diff --git a/crates/tracedecay-global-db/src/observation/reset/tests.rs b/crates/tracedecay-global-db/src/observation/reset/tests.rs deleted file mode 100644 index 73c2d417ce..0000000000 --- a/crates/tracedecay-global-db/src/observation/reset/tests.rs +++ /dev/null @@ -1,1253 +0,0 @@ -use rusqlite::OptionalExtension; -use tempfile::TempDir; - -use crate::schema_contract::invariants::test_fixture::authority_fixture; -use crate::tests::harness::open_registered_test_database_fixture; -use tracedecay_domain::errors::TraceDecayError; -use tracedecay_runtime_core::db::TestDatabaseRuntimeScope; - -use super::reset_refused_observation_authority; - -async fn install_registered_store(path: &std::path::Path) { - let admitted = - open_registered_test_database_fixture(path, TestDatabaseRuntimeScope::ProfileSessions) - .await - .expect("install the registered sessions schema"); - drop(admitted); -} - -/// Replaces the canonical `observations` table with the pre-release -/// `idempotency_key` shape that admission refuses. -fn install_legacy_observation_shape(conn: &rusqlite::Connection) { - conn.pragma_update(None, "foreign_keys", false) - .expect("disable foreign keys for fixture seeding"); - conn.execute_batch( - "DROP TABLE observations; - CREATE TABLE observations ( - sequence INTEGER PRIMARY KEY AUTOINCREMENT, - observation_id TEXT NOT NULL UNIQUE, - idempotency_key TEXT NOT NULL UNIQUE, - payload_digest TEXT NOT NULL, - receipt_id TEXT NOT NULL, - observation_json TEXT NOT NULL, - committed_cursor_json TEXT NOT NULL, - FOREIGN KEY(receipt_id) REFERENCES sanitization_receipts(receipt_id) - ); - INSERT INTO observations - (observation_id, idempotency_key, payload_digest, receipt_id, - observation_json, committed_cursor_json) - VALUES ('observation.legacy', 'idempotency.legacy', 'digest.legacy', - 'receipt.legacy', '{}', '{}');", - ) - .expect("install the refused pre-release observation shape"); -} - -fn seed_preserved_transcript_rows(conn: &rusqlite::Connection) { - conn.execute_batch( - "INSERT INTO sessions(provider, session_id, project_key, project_path) - VALUES ('claude', 'session.fixture', 'project.fixture', '/project/fixture'); - INSERT INTO session_messages(provider, message_id, session_id, role, ordinal, text) - VALUES ('claude', 'message.fixture', 'session.fixture', 'user', 0, 'projected output');", - ) - .expect("seed transcript metadata and one projector-output message"); -} - -/// Seeds the session-temporal state a store that has ever projected carries: -/// an active generation frozen at a high projection frontier, the batch -/// receipt certifying that generation's occurrence/current/FTS coverage, one -/// occurrence, an applied relation receipt, and a running refresh operation. -/// Every guard trigger is satisfied on the way in, the generation walks -/// `building -> ready -> active` and the receipt lands while it is building, -/// so the fixture cannot be weaker than production. -fn seed_active_temporal_generation(conn: &rusqlite::Connection) { - conn.execute_batch( - "INSERT INTO retrieval_anchors - (anchor_id, anchor_json, owner_json, projection_generation) - VALUES ('anchor.fixture', '{}', '{}', 'generation.fixture'); - INSERT INTO session_temporal_generations - (session_id, generation, state, frozen_watermarks_json, created_at) - VALUES ('session.fixture', 1, 'building', - '{\"projection_frontier\":100}', 1); - INSERT INTO session_temporal_projection_receipts - (session_id, generation, batch_ordinal, batch_digest, - frozen_watermarks_json, source_through, projection_through, - occurrence_count, occurrence_digest, dimension_count, dimension_digest, - copy_count, copy_digest, assertion_count, assertion_digest, - supersession_count, supersession_digest, current_count, current_digest, - fts_count, fts_digest, committed_at) - VALUES ('session.fixture', 1, 0, 'digest.batch', - '{\"projection_frontier\":100}', 100, 100, - 1, 'digest.occurrence', 0, 'digest.dimension', 0, 'digest.copy', - 0, 'digest.assertion', 0, 'digest.supersession', - 1, 'digest.current', 1, 'digest.fts', 2); - UPDATE session_temporal_generations - SET state = 'ready', ready_at = 3 WHERE session_id = 'session.fixture'; - UPDATE session_temporal_generations - SET state = 'active', activated_at = 4 WHERE session_id = 'session.fixture'; - INSERT INTO session_occurrences - (session_id, generation, occurrence_id, source_observation_id, - source_provider, projection_output_ordinal, retrieval_anchor_id, - role, knowledge_at, valid_time_json, evidence_json, - sanitized_content_digest, sanitized_content_bytes, snippet_text, index_text) - VALUES ('session.fixture', 1, 'occurrence.fixture', 'observation.legacy', - 'claude', 0, 'anchor.fixture', 'user', 5, - '{\"kind\":\"unknown\"}', '{}', - '0000000000000000000000000000000000000000000000000000000000000000', - 0, 'snippet', 'index'); - INSERT INTO session_current_entities - (session_id, generation, entity_kind, entity_id, current_occurrence_id, - coverage_json) - VALUES ('session.fixture', 1, 'occurrence_anchor', 'anchor.fixture', - 'occurrence.fixture', '{}'); - INSERT INTO session_relation_receipts - (session_id, generation, scope_kind, scope_id, expected_graph_watermark, - state, graph_watermark, created_at, applied_at) - VALUES ('session.fixture', 1, 'profile_sessions', 'scope.fixture', - 'watermark.fixture', 'applied', 'watermark.fixture', 6, 7); - INSERT INTO session_refresh_operations - (session_id, operation_id, request_digest, target_frontier_json, - state, created_at, updated_at) - VALUES ('session.fixture', 'operation.fixture', 'digest.request', - '{\"observed_through\":100,\"committed_through\":100}', - 'running', 8, 8);", - ) - .expect("seed a populated active temporal generation"); -} - -/// Mirrors the frontier predicate of -/// `pending_session_temporal_refresh_page_result`: an output-producing effect -/// past the active generation's frozen `projection_frontier`, for a session -/// with no running refresh operation. A retained generation at frontier 100 -/// is exactly what would exclude re-ingested effects 1..=100 from the rebuild. -fn refresh_discovery_frontier(conn: &rusqlite::Connection, session_id: &str) -> Option { - conn.query_row( - "SELECT COALESCE(active.projection_frontier, 0) - FROM session_temporal_observation_effects AS effect - LEFT JOIN ( - SELECT session_id, - CAST(json_extract(frozen_watermarks_json, '$.projection_frontier') - AS INTEGER) AS projection_frontier - FROM session_temporal_generations - WHERE state = 'active' - ) AS active ON active.session_id = effect.session_id - WHERE NOT EXISTS ( - SELECT 1 FROM session_refresh_operations AS running - WHERE running.session_id = effect.session_id AND running.state = 'running' - ) - AND effect.output_count > 0 - AND effect.observation_sequence > COALESCE(active.projection_frontier, 0) - AND effect.session_id = ?1 - GROUP BY effect.session_id", - [session_id], - |row| row.get::<_, i64>(0), - ) - .optional() - .unwrap() -} - -fn foreign_key_violations(conn: &rusqlite::Connection) -> Vec { - let mut statement = conn.prepare("PRAGMA foreign_key_check").unwrap(); - let violations = statement - .query_map([], |row| row.get::<_, String>(0)) - .unwrap() - .collect::, _>>(); - violations.unwrap() -} - -fn trigger_exists(conn: &rusqlite::Connection, trigger: &str) -> bool { - conn.query_row( - "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type = 'trigger' AND name = ?1)", - [trigger], - |row| row.get::<_, bool>(0), - ) - .unwrap() -} - -fn scheme_migration_recorded(conn: &rusqlite::Connection) -> bool { - conn.query_row( - "SELECT EXISTS(SELECT 1 FROM global_schema_migrations WHERE migration = ?1)", - [super::OBSERVATION_NATIVE_SOURCE_SCHEME_MIGRATION], - |row| row.get::<_, bool>(0), - ) - .unwrap() -} - -fn table_exists(conn: &rusqlite::Connection, table: &str) -> bool { - conn.query_row( - "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?1)", - [table], - |row| row.get::<_, bool>(0), - ) - .unwrap() -} - -fn count(conn: &rusqlite::Connection, table: &str) -> i64 { - conn.query_row(&format!("SELECT COUNT(*) FROM \"{table}\""), [], |row| { - row.get::<_, i64>(0) - }) - .unwrap() -} - -/// Seeds one canonical-shape observation and cursor whose source names -/// `provider`, then removes the native-source scheme marker so the store looks -/// exactly like one written before the Cline/Roo/Kilo `ui_messages` source -/// existed. `source_key` is the observation's source key (`None` omits it). -fn seed_unmarked_native_source_rows( - conn: &rusqlite::Connection, - provider: &str, - source_key: Option<&str>, -) { - let source = match source_key { - Some(key) => format!( - r#"{{"provider":"{provider}","session_id":"session.fixture","source_key":"{key}"}}"# - ), - None => format!(r#"{{"provider":"{provider}","session_id":"session.fixture"}}"#), - }; - let observation = - format!(r#"{{"identity":{{"source":{source},"scope":{{"kind":"profile"}}}}}}"#); - conn.pragma_update(None, "foreign_keys", false) - .expect("disable foreign keys for fixture seeding"); - conn.execute_batch( - "INSERT INTO sanitization_receipts - (receipt_id, sanitizer_version, payload_digest, receipt_json) - VALUES ('receipt.fixture', 'v1', 'digest.fixture', '{}');", - ) - .expect("seed a receipt"); - conn.execute( - "INSERT INTO observations - (observation_id, payload_digest, receipt_id, observation_json, - committed_cursor_json) - VALUES ('observation.fixture', 'digest.fixture', 'receipt.fixture', ?1, '{}')", - [&observation], - ) - .expect("seed an observation"); - conn.execute( - "INSERT INTO source_cursors(source_json, scope_json, cursor_json) - VALUES (?1, '{\"kind\":\"profile\"}', '{}')", - [&source], - ) - .expect("seed a cursor"); - conn.execute( - "DELETE FROM global_schema_migrations WHERE migration = ?1", - [super::OBSERVATION_NATIVE_SOURCE_SCHEME_MIGRATION], - ) - .expect("make the fixture an old-scheme store"); -} - -async fn reopen_registered_store(path: &std::path::Path) -> tracedecay_domain::errors::Result<()> { - open_registered_test_database_fixture(path, TestDatabaseRuntimeScope::ProfileSessions) - .await - .map(drop) -} - -/// The native-source scheme change only ever applied to Cline, Roo Code and -/// Kilo tasks. A populated store whose observations and cursors name none of -/// those hosts cannot double-count anything under the new scheme, so attach -/// enrolls it instead of demanding a reset that would discard derived history -/// for no reason. Rows are untouched. The rows are real committed Codex -/// observations (the same fixture the authority audit uses), because attach -/// audits every retained row after the shape check admits the store. -#[tokio::test] -async fn populated_store_without_cline_like_sources_enrolls_on_attach() { - let directory = TempDir::new().unwrap(); - let database_path = directory.path().join("sessions.db"); - install_registered_store(&database_path).await; - { - let raw = rusqlite::Connection::open(&database_path).unwrap(); - for index in 0..=super::super::schema::SOURCE_CURSOR_CENSUS_PAGE_ROWS { - let (observation, cursor) = - authority_fixture(u64::try_from(index).unwrap(), &format!("enroll-{index}")); - let receipt = observation.receipt(); - let payload_digest = observation.payload_reference().digest().as_str().to_owned(); - raw.execute( - "INSERT INTO sanitization_receipts - (receipt_id, sanitizer_version, payload_digest, receipt_json) - VALUES (?1, ?2, ?3, ?4)", - rusqlite::params![ - receipt.receipt().receipt_id().as_str(), - receipt.receipt().sanitizer_version().as_str(), - payload_digest.as_str(), - serde_json::to_string(receipt).unwrap() - ], - ) - .expect("seed a committed receipt"); - raw.execute( - "INSERT INTO observations - (observation_id, payload_digest, receipt_id, observation_json, - committed_cursor_json) - VALUES (?1, ?2, ?3, ?4, ?5)", - rusqlite::params![ - observation.observation_id().as_str(), - payload_digest.as_str(), - receipt.receipt().receipt_id().as_str(), - serde_json::to_string(&observation).unwrap(), - serde_json::to_string(&cursor).unwrap() - ], - ) - .expect("seed a committed Codex observation"); - raw.execute( - "INSERT INTO source_cursors(source_json, scope_json, cursor_json) - VALUES (?1, ?2, ?3)", - rusqlite::params![ - serde_json::to_string(cursor.source()).unwrap(), - serde_json::to_string(cursor.scope()).unwrap(), - serde_json::to_string(&cursor).unwrap() - ], - ) - .expect("seed the committed cursor"); - } - raw.execute( - "DELETE FROM global_schema_migrations WHERE migration = ?1", - [super::OBSERVATION_NATIVE_SOURCE_SCHEME_MIGRATION], - ) - .expect("make the fixture an old-scheme store"); - assert!(!scheme_migration_recorded(&raw)); - } - - reopen_registered_store(&database_path) - .await - .expect("a Codex-only old-scheme store must attach"); - - let raw = rusqlite::Connection::open(&database_path).unwrap(); - assert!( - scheme_migration_recorded(&raw), - "attach must enroll the scheme for a store the change never applied to" - ); - let expected_rows = super::super::schema::SOURCE_CURSOR_CENSUS_PAGE_ROWS + 1; - assert_eq!(count(&raw, "observations"), expected_rows); - assert_eq!(count(&raw, "source_cursors"), expected_rows); - assert!( - super::reset_refused_observation_authority( - &mut rusqlite::Connection::open(&database_path).unwrap() - ) - .is_err(), - "an enrolled store is healthy and the scoped reset must refuse it" - ); -} - -/// The observations scan is independently authoritative: a cursor can be -/// absent after a committed observation, and a Cline-like row beyond the -/// first bounded page must still refuse enrollment. -#[tokio::test] -async fn paged_census_finds_cline_observation_without_source_cursor() { - let directory = TempDir::new().unwrap(); - let database_path = directory.path().join("sessions.db"); - install_registered_store(&database_path).await; - { - let raw = rusqlite::Connection::open(&database_path).unwrap(); - raw.pragma_update(None, "foreign_keys", false) - .expect("disable foreign keys for fixture seeding"); - raw.execute_batch( - "INSERT INTO sanitization_receipts - (receipt_id, sanitizer_version, payload_digest, receipt_json) - VALUES ('receipt.census', 'v1', 'digest.census', '{}');", - ) - .expect("seed census receipt"); - let rows = super::super::schema::OBSERVATION_SOURCE_CENSUS_PAGE_ROWS + 1; - for index in 0..rows { - let provider = if index + 1 == rows { "cline" } else { "codex" }; - let observation = format!( - r#"{{"identity":{{"source":{{"provider":"{provider}","session_id":"session.{index}"}},"scope":{{"kind":"profile"}}}}}}"# - ); - raw.execute( - "INSERT INTO observations - (observation_id, payload_digest, receipt_id, observation_json, - committed_cursor_json) - VALUES (?1, 'digest.census', 'receipt.census', ?2, '{}')", - rusqlite::params![format!("observation.census-{index}"), observation], - ) - .expect("seed census observation"); - } - raw.execute( - "DELETE FROM global_schema_migrations WHERE migration = ?1", - [super::OBSERVATION_NATIVE_SOURCE_SCHEME_MIGRATION], - ) - .expect("make the fixture an old-scheme store"); - assert_eq!(count(&raw, "source_cursors"), 0); - } - - let error = reopen_registered_store(&database_path) - .await - .expect_err("a paged Cline observation census must refuse admission"); - let (authority, reason) = error - .reset_required_context() - .unwrap_or_else(|| panic!("expected typed ResetRequired, got: {error}")); - assert_eq!(authority, super::OBSERVATION_AUTHORITY); - assert!(reason.contains("ui_messages.json")); - - let raw = rusqlite::Connection::open(&database_path).unwrap(); - assert!(!scheme_migration_recorded(&raw)); - assert_eq!(count(&raw, "source_cursors"), 0); -} - -/// A store that did admit a Cline-like task under the combined `` source -/// carries no record of which scheme wrote those rows, so it must still refuse -/// with the typed `ResetRequired` state naming the observation authority, -/// whether the host shows up as an observation provider or only as a cursor. -#[tokio::test] -async fn populated_store_with_cline_like_sources_still_refuses_without_the_marker() { - for (provider, source_key) in [ - ("cline", None), - ("roo-code", None), - ("kilo", Some("task.fixture:ui_messages")), - ] { - let directory = TempDir::new().unwrap(); - let database_path = directory.path().join("sessions.db"); - install_registered_store(&database_path).await; - { - let raw = rusqlite::Connection::open(&database_path).unwrap(); - seed_unmarked_native_source_rows(&raw, provider, source_key); - } - - let error = reopen_registered_store(&database_path) - .await - .expect_err("an old-scheme Cline-like store must refuse admission"); - let (authority, reason) = error.reset_required_context().unwrap_or_else(|| { - panic!("expected the typed ResetRequired state for {provider}, got: {error}") - }); - assert_eq!(authority, super::OBSERVATION_AUTHORITY); - assert!( - reason.contains("ui_messages.json"), - "the refusal must name the scheme change for {provider}: {reason}" - ); - let raw = rusqlite::Connection::open(&database_path).unwrap(); - assert!( - !scheme_migration_recorded(&raw), - "a refused {provider} store must not be enrolled behind the operator's back" - ); - } -} - -#[tokio::test] -async fn refused_observation_shape_resets_scoped_and_readmits() { - let directory = TempDir::new().unwrap(); - let database_path = directory.path().join("sessions.db"); - install_registered_store(&database_path).await; - { - let raw = rusqlite::Connection::open(&database_path).unwrap(); - seed_preserved_transcript_rows(&raw); - install_legacy_observation_shape(&raw); - } - - let refusal = match open_registered_test_database_fixture( - &database_path, - TestDatabaseRuntimeScope::ProfileSessions, - ) - .await - { - Ok(_) => panic!("the pre-release observation shape must refuse admission"), - Err(error) => error, - }; - let (authority, reason) = refusal - .reset_required_context() - .unwrap_or_else(|| panic!("expected the typed ResetRequired state, got: {refusal}")); - assert_eq!(authority, "observations"); - assert!( - reason.contains("no sanctioned migration") || reason.contains("branch-local"), - "the refusal must say why no migration exists: {reason}" - ); - - let report = { - let mut raw = rusqlite::Connection::open(&database_path).unwrap(); - reset_refused_observation_authority(&mut raw) - .expect("scoped reset of the refused authority") - }; - assert!( - report - .reset_tables - .iter() - .any(|table| table == "observations"), - "the refused table must be part of the reset: {report:?}" - ); - assert_eq!(report.cleared_session_message_rows, 1); - - let readmitted = open_registered_test_database_fixture( - &database_path, - TestDatabaseRuntimeScope::ProfileSessions, - ) - .await - .expect("the reset store must readmit at the canonical schema"); - drop(readmitted); - - let raw = rusqlite::Connection::open(&database_path).unwrap(); - assert_eq!( - count(&raw, "observations"), - 0, - "the refused authority must be recreated empty" - ); - let has_idempotency_column = raw - .query_row( - "SELECT EXISTS( - SELECT 1 FROM pragma_table_xinfo('observations') - WHERE name = 'idempotency_key' - )", - [], - |row| row.get::<_, bool>(0), - ) - .unwrap(); - assert!( - !has_idempotency_column, - "the recreated table must carry the canonical shape" - ); - assert_eq!( - count(&raw, "sessions"), - 1, - "transcript metadata outside the refused authority must be preserved" - ); - assert_eq!( - count(&raw, "session_messages"), - 0, - "recoverable projector output must be cleared with its provenance" - ); - assert_eq!( - count(&raw, "remote_deletion_tombstones"), - 0, - "unrelated authorities must survive the scoped reset with their schema intact" - ); -} - -/// A retained admission-refusal terminal names an observation row by id and -/// digest. After a scoped reset recreates the observation authority empty, -/// a leftover terminal would falsely suppress the re-ingested record whose -/// rewritten payload happens to match the stale refusal signature, so the -/// scoped reset must clear the refusal authority with the rest. -#[tokio::test] -async fn scoped_reset_clears_retained_admission_refusals() { - let directory = TempDir::new().unwrap(); - let database_path = directory.path().join("sessions.db"); - install_registered_store(&database_path).await; - { - let raw = rusqlite::Connection::open(&database_path).unwrap(); - raw.pragma_update(None, "foreign_keys", false) - .expect("disable foreign keys for fixture seeding"); - raw.execute_batch( - "INSERT INTO observation_admission_refusals - (observation_id, refused_payload_digest, retained_payload_digest, refused_at) - VALUES ('observation.legacy', 'digest.refused', 'digest.retained', 1);", - ) - .expect("seed one retained admission refusal"); - install_legacy_observation_shape(&raw); - } - - let report = { - let mut raw = rusqlite::Connection::open(&database_path).unwrap(); - reset_refused_observation_authority(&mut raw) - .expect("scoped reset of the refused authority") - }; - assert!( - report - .reset_tables - .iter() - .any(|table| table == "observation_admission_refusals"), - "the refusal authority must be part of the scoped reset: {report:?}" - ); - - let raw = rusqlite::Connection::open(&database_path).unwrap(); - assert!( - table_exists(&raw, "observation_admission_refusals"), - "the refusal authority must be recreated at the canonical shape" - ); - assert_eq!( - count(&raw, "observation_admission_refusals"), - 0, - "a scoped reset must leave no stale refusal terminal that could \ - falsely suppress re-ingested records" - ); -} - -#[tokio::test] -async fn healthy_observation_authority_refuses_the_scoped_reset() { - let directory = TempDir::new().unwrap(); - let database_path = directory.path().join("sessions.db"); - install_registered_store(&database_path).await; - - let mut raw = rusqlite::Connection::open(&database_path).unwrap(); - raw.execute( - "INSERT INTO session_query_cursor_keys - (key_id, key_version, key_material, created_at, retired_at) - VALUES ('cursor-key-healthy', 1, ?1, 1, NULL)", - rusqlite::params![vec![9_u8; 32]], - ) - .unwrap(); - let error = reset_refused_observation_authority(&mut raw) - .expect_err("a healthy authority must never be reset"); - assert!( - matches!( - &error, - TraceDecayError::Config { message } if message.contains("not in a refused state") - ), - "unexpected error resetting a healthy authority: {error}" - ); - assert_eq!(count(&raw, "session_query_cursor_keys"), 1); - let unchanged_key: (String, Vec, Option) = raw - .query_row( - "SELECT key_id, key_material, retired_at FROM session_query_cursor_keys", - [], - |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), - ) - .unwrap(); - assert_eq!( - unchanged_key, - ("cursor-key-healthy".to_owned(), vec![9_u8; 32], None) - ); - assert!( - table_exists(&raw, "observations"), - "a refused reset must mutate nothing" - ); -} - -/// The session-temporal projection is projector output over the observation -/// stream, so it resets with the stream it projects. Refusing over it instead -/// made the reset unreachable on any store that had ever ingested; preserving -/// the generation while deleting its occurrences would be worse, the frozen -/// frontier of the retained active generation would exclude every re-ingested -/// effect from rebuild discovery, and its immutable batch receipt would go on -/// certifying occurrence, current-entity and FTS counts for rows that no -/// longer exist. So the generation, its receipts, its relation receipt and -/// its refresh operation are invalidated outright, and replay is rediscovered -/// from zero. -#[tokio::test] -async fn populated_temporal_generation_is_invalidated_and_replay_rediscovered() { - let directory = TempDir::new().unwrap(); - let database_path = directory.path().join("sessions.db"); - install_registered_store(&database_path).await; - { - let raw = rusqlite::Connection::open(&database_path).unwrap(); - seed_preserved_transcript_rows(&raw); - install_legacy_observation_shape(&raw); - seed_active_temporal_generation(&raw); - raw.execute_batch( - "INSERT INTO session_temporal_observation_effects - (observation_id, observation_sequence, session_id, receipt_id, - effect_digest, output_count, recorded_at) - VALUES ('observation.legacy', 1, 'session.fixture', 'receipt.legacy', - 'digest.effect', 1, 1);", - ) - .expect("seed one observation-derived temporal effect"); - assert_eq!( - refresh_discovery_frontier(&raw, "session.fixture"), - None, - "the seeded store must start with the rebuild suppressed, or this \ - test proves nothing" - ); - } - - let mut raw = rusqlite::Connection::open(&database_path).unwrap(); - let report = reset_refused_observation_authority(&mut raw) - .expect("a populated session-temporal projection must not block the scoped reset"); - assert_eq!( - report.cleared_derived_temporal_rows, 7, - "every seeded projection row must be accounted for: {report:?}" - ); - for table in [ - "session_temporal_generations", - "session_temporal_projection_receipts", - "session_relation_receipts", - "session_refresh_operations", - "session_occurrences", - "session_current_entities", - "session_temporal_observation_effects", - ] { - assert_eq!( - count(&raw, table), - 0, - "{table} must not survive the reset advertising coverage of deleted rows" - ); - } - assert!( - foreign_key_violations(&raw).is_empty(), - "the committed reset must be referentially coherent" - ); - assert_eq!( - count(&raw, "sessions"), - 1, - "state outside the observation projection must be preserved" - ); - - // Re-ingest one native event under the rebuilt authority: the frontier - // that used to be frozen at 100 must no longer exclude sequence 1. - raw.execute_batch( - "INSERT INTO sanitization_receipts - (receipt_id, sanitizer_version, payload_digest, receipt_json) - VALUES ('receipt.rebuilt', 'v1', 'digest.payload', '{}'); - INSERT INTO observations - (observation_id, payload_digest, receipt_id, observation_json, - committed_cursor_json) - VALUES ('observation.rebuilt', 'digest.payload', 'receipt.rebuilt', '{}', '{}'); - INSERT INTO session_temporal_observation_effects - (observation_id, observation_sequence, session_id, receipt_id, - effect_digest, output_count, recorded_at) - VALUES ('observation.rebuilt', 1, 'session.fixture', 'receipt.rebuilt', - 'digest.effect', 1, 9);", - ) - .expect("re-ingest one native event under the rebuilt authority"); - assert_eq!( - refresh_discovery_frontier(&raw, "session.fixture"), - Some(0), - "the rebuilt stream must be rediscovered from zero, not excluded by the \ - frontier of the generation the reset invalidated" - ); -} - -/// Seeds the anchor state one admitted observation leaves behind: its -/// exact-observation anchor bound through `observation_retrieval_anchors` and -/// `observation_projection_provenance`, the native-record alias resolving to -/// it, and a repository-capture anchor bound through -/// `observation_repository_provenance`. -fn seed_observation_bound_anchors(conn: &rusqlite::Connection) { - conn.pragma_update(None, "foreign_keys", false) - .expect("disable foreign keys for fixture seeding"); - conn.execute_batch( - "INSERT INTO retrieval_anchors - (anchor_id, anchor_json, owner_json, projection_generation) - VALUES ('anchor.observation', '{}', '{\"kind\":\"profile\"}', 'generation.fixture'), - ('anchor.capture', '{}', '{\"kind\":\"profile\"}', 'generation.fixture'); - INSERT INTO retrieval_anchor_aliases - (owner_json, alias_kind, locator_digest, anchor_id) - VALUES ('{\"kind\":\"profile\"}', '\"provider_record\"', '\"sha256:record\"', - 'anchor.observation'); - INSERT INTO observation_retrieval_anchors (observation_id, anchor_id) - VALUES ('observation.legacy', 'anchor.observation'); - INSERT INTO observation_repository_provenance - (observation_id, availability_json, capture_json, retrieval_anchor_id, owner_json) - VALUES ('observation.legacy', '{}', '{}', 'anchor.capture', '{\"kind\":\"profile\"}'); - INSERT INTO observation_projection_provenance - (projector_version, observation_id, output_ordinal, retrieval_anchor_id, - receipt_id, output_provider, output_message_id, output_digest, message_created) - VALUES ('projector.v1', 'observation.legacy', 0, 'anchor.observation', - 'receipt.legacy', 'claude', 'message.fixture', 'digest.output', 1);", - ) - .expect("seed the anchors one admitted observation binds"); -} - -/// The anchors an admitted observation binds, and the native-record aliases -/// resolving to them, are re-derived by the next admission of the same -/// records, and verified field-for-field against whatever row already holds -/// the anchor id. A retained anchor whose transcript file was since replaced -/// (a new source generation) fails that verification as a storage collision -/// on every retry; a retained alias whose record was revised refuses it -/// deterministically. Either leaves the rebuild the reset promises undone, so -/// they go with the observation stream, while anchors preserved rows own -/// (here a summary anchor) stay, the immutability guards return, and the -/// committed store is referentially coherent. -#[tokio::test] -async fn observation_bound_anchors_and_aliases_reset_with_the_stream() { - let directory = TempDir::new().unwrap(); - let database_path = directory.path().join("sessions.db"); - install_registered_store(&database_path).await; - { - let raw = rusqlite::Connection::open(&database_path).unwrap(); - seed_preserved_transcript_rows(&raw); - install_legacy_observation_shape(&raw); - seed_active_temporal_generation(&raw); - seed_observation_bound_anchors(&raw); - raw.execute_batch( - "INSERT INTO session_summary_nodes - (summary_id, session_id, summary_anchor_id, summary_text, - index_text, source_horizon_json, created_at) - VALUES ('summary.fixture', 'session.fixture', 'anchor.fixture', - 'summary', 'index', '{}', 1);", - ) - .expect("seed a preserved summary naming its own anchor"); - assert_eq!(count(&raw, "retrieval_anchors"), 3); - assert_eq!(count(&raw, "retrieval_anchor_aliases"), 1); - } - - let mut raw = rusqlite::Connection::open(&database_path).unwrap(); - let report = - reset_refused_observation_authority(&mut raw).expect("scoped reset of an anchored store"); - assert_eq!( - report.cleared_retrieval_anchor_rows, 3, - "two observation-bound anchors and one alias must be accounted for: {report:?}" - ); - assert_eq!( - raw.query_row( - "SELECT group_concat(anchor_id, ',') FROM retrieval_anchors", - [], - |row| row.get::<_, String>(0), - ) - .unwrap(), - "anchor.fixture", - "only the anchor a preserved summary names may survive" - ); - assert_eq!( - count(&raw, "retrieval_anchor_aliases"), - 0, - "no alias may keep resolving a native record to an anchor that is gone" - ); - for trigger in [ - "retrieval_anchors_immutable_delete", - "retrieval_anchor_aliases_immutable_delete", - "retrieval_anchors_immutable_update", - ] { - assert!( - trigger_exists(&raw, trigger), - "{trigger} must be reinstalled before the reset commits" - ); - } - assert!( - raw.execute("DELETE FROM retrieval_anchors", []).is_err(), - "runtime immutability must be back in force" - ); - assert!( - foreign_key_violations(&raw).is_empty(), - "the committed reset must be referentially coherent" - ); - assert_eq!(count(&raw, "session_summary_nodes"), 1); - - // Re-admitting the same native record under a moved source generation - // must now be able to write its anchor and alias afresh. - raw.execute_batch( - "INSERT INTO retrieval_anchors - (anchor_id, anchor_json, owner_json, projection_generation) - VALUES ('anchor.observation', '{\"generation\":2}', '{\"kind\":\"profile\"}', - 'generation.fixture'); - INSERT INTO retrieval_anchor_aliases - (owner_json, alias_kind, locator_digest, anchor_id) - VALUES ('{\"kind\":\"profile\"}', '\"provider_record\"', '\"sha256:record\"', - 'anchor.observation');", - ) - .expect("the rebuilt authority must own the anchor identity again"); -} - -/// The rebuilt authority re-reads every native transcript only if nothing -/// tells it the corpus was already swept. The Codex history frontier and -/// corpus epoch, per-provider coverage verdicts, host discovery frontiers, and -/// queued discovery paths all live in `parse_offsets`; a surviving `complete` -/// verdict would leave the reset store empty forever while every read reports -/// a healthy, current, empty projection. The hook-analytics import cursor -/// shares the table but feeds `analytics_events`, so it stays. -#[tokio::test] -async fn native_source_scheduling_cursors_reset_with_the_stream() { - let directory = TempDir::new().unwrap(); - let database_path = directory.path().join("sessions.db"); - install_registered_store(&database_path).await; - { - let raw = rusqlite::Connection::open(&database_path).unwrap(); - install_legacy_observation_shape(&raw); - raw.execute_batch( - "INSERT INTO parse_offsets (file_path, byte_offset, mtime, file_id) VALUES - ('host-coverage://codex/v1', 0, 2, 1), - ('tracedecay-internal:codex-history-frontier:v2', 0, 0, 3), - ('tracedecay-internal:codex-history-epoch:v2', -5565623358034642743, - -2743919526740859943, 1), - ('tracedecay-internal:project-ingest-provider-frontier:v1', 11, 0, 1), - ('host-frontier://kimi/discovery/v1', 0, 4, 0), - ('host-discovery-queue://codex/v1/L2hvbWUvcm9sbG91dA', 1, 0, 1), - ('hook_analytics:/project/.tracedecay/hook_analytics.jsonl', 4096, 7, 1);", - ) - .expect("seed every parse-offset tenant"); - } - - let mut raw = rusqlite::Connection::open(&database_path).unwrap(); - let report = reset_refused_observation_authority(&mut raw) - .expect("scoped reset of a store with scheduling cursors"); - assert_eq!( - report.cleared_native_source_cursor_rows, 6, - "every native-source scheduling cursor must be accounted for: {report:?}" - ); - assert_eq!( - raw.query_row( - "SELECT group_concat(file_path, ',') FROM parse_offsets", - [], - |row| row.get::<_, String>(0), - ) - .unwrap(), - "hook_analytics:/project/.tracedecay/hook_analytics.jsonl", - "only the analytics import cursor may survive the reset" - ); -} - -/// A disposition (for example a redaction) recorded against an observation -/// anchor is preserved evidence the reset cannot rebind, so a store carrying -/// one refuses atomically instead of orphaning it. -#[tokio::test] -async fn anchor_dispositions_on_observation_anchors_refuse_atomically() { - let directory = TempDir::new().unwrap(); - let database_path = directory.path().join("sessions.db"); - install_registered_store(&database_path).await; - { - let raw = rusqlite::Connection::open(&database_path).unwrap(); - seed_preserved_transcript_rows(&raw); - install_legacy_observation_shape(&raw); - seed_observation_bound_anchors(&raw); - raw.execute_batch( - "INSERT INTO retrieval_anchor_dispositions - (disposition_id, anchor_id, owner_json, state, superseded_by, - reason_class, effective_at, record_json) - VALUES ('disposition.redacted', 'anchor.observation', '{\"kind\":\"profile\"}', - 'redacted', NULL, 'redaction', 1, '{}');", - ) - .expect("seed a redaction against the observation anchor"); - } - - let mut raw = rusqlite::Connection::open(&database_path).unwrap(); - let error = reset_refused_observation_authority(&mut raw) - .expect_err("an anchor disposition the reset cannot rebind must refuse"); - assert!( - matches!( - &error, - TraceDecayError::Config { message } - if message.contains("retrieval_anchor_dispositions") - && message.contains("nothing was reset") - ), - "unexpected error for a preserved anchor disposition: {error}" - ); - assert_eq!(count(&raw, "retrieval_anchors"), 2); - assert_eq!(count(&raw, "retrieval_anchor_aliases"), 1); - assert_eq!(count(&raw, "retrieval_anchor_dispositions"), 1); - assert!( - trigger_exists(&raw, "retrieval_anchors_immutable_delete"), - "a rolled-back reset must leave the anchor guards in place" - ); -} - -/// A scoped reset must never orphan preserved evidence. External payload -/// manifests are durable LCM publication metadata whose receipt lives in the -/// `sanitization_receipts` table the reset recreates empty, and they are not -/// reconstructible from the transcripts, so a store holding one refuses -/// atomically instead of having its external-payload metadata deleted to make -/// the reset succeed. Everything else preserved keeps its evidence, and -/// `PRAGMA foreign_key_check` proves it. -#[tokio::test] -async fn preserved_dependencies_refuse_atomically_and_stay_coherent() { - let directory = TempDir::new().unwrap(); - let database_path = directory.path().join("with-manifest.db"); - install_registered_store(&database_path).await; - { - let raw = rusqlite::Connection::open(&database_path).unwrap(); - seed_preserved_transcript_rows(&raw); - install_legacy_observation_shape(&raw); - seed_active_temporal_generation(&raw); - raw.execute_batch( - "INSERT INTO lcm_external_payloads - (payload_ref, provider, session_id, message_id, kind, content_hash, - byte_count, char_count) - VALUES ('payload.fixture', 'claude', 'session.fixture', - 'message.fixture', 'text', 'digest.payload', 1, 1); - INSERT INTO session_external_payload_manifests - (payload_ref, session_id, payload_digest, manifest_json, receipt_id, - created_at) - VALUES ('payload.fixture', 'session.fixture', 'digest.payload', '{}', - 'receipt.legacy', 1);", - ) - .expect("seed one external payload manifest"); - } - - let mut raw = rusqlite::Connection::open(&database_path).unwrap(); - let error = reset_refused_observation_authority(&mut raw) - .expect_err("a preserved dependency with no safe treatment must refuse"); - assert!( - matches!( - &error, - TraceDecayError::Config { message } - if message.contains("session_external_payload_manifests") - && message.contains("nothing was reset") - ), - "unexpected error for a preserved dependency: {error}" - ); - assert_eq!( - count(&raw, "session_temporal_generations"), - 1, - "a refused reset must mutate nothing" - ); - assert_eq!( - count(&raw, "session_external_payload_manifests"), - 1, - "external-payload metadata must never be deleted to make a reset succeed" - ); - - // A store without that dependency resets, and everything preserved keeps - // the evidence it needs. - let preserved_path = directory.path().join("preserved.db"); - install_registered_store(&preserved_path).await; - { - let raw = rusqlite::Connection::open(&preserved_path).unwrap(); - seed_preserved_transcript_rows(&raw); - install_legacy_observation_shape(&raw); - seed_active_temporal_generation(&raw); - raw.execute_batch( - "INSERT INTO session_summary_nodes - (summary_id, session_id, summary_anchor_id, summary_text, - index_text, source_horizon_json, created_at) - VALUES ('summary.fixture', 'session.fixture', 'anchor.fixture', - 'summary', 'index', '{}', 1);", - ) - .expect("seed preserved LCM content"); - } - let mut preserved = rusqlite::Connection::open(&preserved_path).unwrap(); - reset_refused_observation_authority(&mut preserved) - .expect("scoped reset of a store with no unresolvable dependency"); - assert_eq!( - count(&preserved, "session_summary_nodes"), - 1, - "preserved LCM content must survive the scoped reset" - ); - assert_eq!( - count(&preserved, "retrieval_anchors"), - 1, - "the anchors preserved summaries name must survive with them" - ); - assert_eq!( - count(&preserved, "session_summary_availability"), - 0, - "per-generation availability verdicts go with the generation they judged" - ); - assert!( - foreign_key_violations(&preserved).is_empty(), - "every preserved row must still resolve its evidence" - ); -} - -/// The reset removes immutability triggers, deletes the projection, restores -/// the triggers and enrolls the new scheme inside one transaction. A failure -/// after the trigger removal and after the deletion, here the referential -/// integrity check, the last step before commit, must therefore leave the -/// store exactly as refused: no missing trigger, no partial deletion, and no -/// premature scheme enrollment that would let the old-scheme rows readmit. -#[tokio::test] -async fn a_failure_after_deletion_leaves_the_refused_store_unchanged() { - let directory = TempDir::new().unwrap(); - let database_path = directory.path().join("sessions.db"); - install_registered_store(&database_path).await; - { - let raw = rusqlite::Connection::open(&database_path).unwrap(); - seed_preserved_transcript_rows(&raw); - install_legacy_observation_shape(&raw); - seed_active_temporal_generation(&raw); - raw.execute( - "INSERT INTO session_query_cursor_keys - (key_id, key_version, key_material, created_at, retired_at) - VALUES ('cursor-key-reset-rollback', 1, ?1, 1, NULL)", - rusqlite::params![vec![7_u8; 32]], - ) - .unwrap(); - // Old-scheme rows: the enrollment marker this reset would add is - // absent, so its premature appearance would be visible. - raw.execute( - "DELETE FROM global_schema_migrations WHERE migration = ?1", - [super::OBSERVATION_NATIVE_SOURCE_SCHEME_MIGRATION], - ) - .expect("make the fixture an old-scheme store"); - // A preserved summary naming an anchor that does not exist: the reset - // cannot repair it, so its integrity check fails after everything else - // in the transaction has already run. - raw.execute_batch( - "INSERT INTO session_summary_nodes - (summary_id, session_id, summary_anchor_id, summary_text, - index_text, source_horizon_json, created_at) - VALUES ('summary.orphan', 'session.fixture', 'anchor.missing', - 'summary', 'index', '{}', 1);", - ) - .expect("seed a preserved row the reset cannot make coherent"); - } - - let mut raw = rusqlite::Connection::open(&database_path).unwrap(); - let error = reset_refused_observation_authority(&mut raw) - .expect_err("an incoherent result must never commit"); - assert!( - matches!( - &error, - TraceDecayError::Config { message } - if message.contains("retrieval_anchors") && message.contains("nothing was reset") - ), - "unexpected error for an unresolvable dependency: {error}" - ); - - // Reopen: the store is the refused one it was, whole. - let mut reopened = rusqlite::Connection::open(&database_path).unwrap(); - for trigger in [ - "session_temporal_observation_effects_immutable_delete_v1", - "session_temporal_generations_delete_guard_v1", - "session_temporal_projection_receipts_immutable_delete_v1", - "session_refresh_bindings_immutable_delete_v1", - ] { - assert!( - trigger_exists(&reopened, trigger), - "{trigger} must be back before the transaction that dropped it ends" - ); - } - assert_eq!(count(&reopened, "session_query_cursor_keys"), 1); - let retained_key: (String, Vec, Option) = reopened - .query_row( - "SELECT key_id, key_material, retired_at FROM session_query_cursor_keys", - [], - |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), - ) - .unwrap(); - assert_eq!( - retained_key, - ("cursor-key-reset-rollback".to_owned(), vec![7_u8; 32], None) - ); - assert_eq!(count(&reopened, "session_temporal_generations"), 1); - assert_eq!(count(&reopened, "session_occurrences"), 1); - assert_eq!(count(&reopened, "session_refresh_operations"), 1); - assert!( - !scheme_migration_recorded(&reopened), - "a rolled-back reset must not enroll the new native-source scheme" - ); - assert!( - reset_refused_observation_authority(&mut reopened).is_err(), - "the store must still be refused after a rolled-back reset" - ); -} - -/// The reset's promise is that the rebuilt authority re-reads what it lost. -/// Row counts alone do not prove it: an ingestion cursor, an advance ledger -/// entry, or a projector checkpoint that outlived the reset would each silently -/// skip exactly the native events the rebuild depends on re-offering. Every -/// pre-reset cursor must therefore be refused after the reset, the tables come -/// back at the canonical shape and empty, the identical advance re-presents as -/// new rather than deduping against a survivor, and the temporal refresh query -/// discovers the re-ingested stream from zero instead of the frozen frontier. -#[tokio::test] -async fn pre_reset_cursors_are_refused_and_the_rebuilt_stream_is_rediscovered() { - const ADVANCE: (&str, &str, &str) = ( - r#"{"provider":"claude"}"#, - r#"{"project":"project.fixture"}"#, - r#"{"through":100}"#, - ); - - let directory = TempDir::new().unwrap(); - let database_path = directory.path().join("sessions.db"); - install_registered_store(&database_path).await; - { - let raw = rusqlite::Connection::open(&database_path).unwrap(); - seed_preserved_transcript_rows(&raw); - install_legacy_observation_shape(&raw); - seed_active_temporal_generation(&raw); - raw.execute( - "INSERT INTO source_cursors(source_json, scope_json, cursor_json) - VALUES (?1, ?2, ?3)", - rusqlite::params![ADVANCE.0, ADVANCE.1, ADVANCE.2], - ) - .expect("seed the pre-reset ingestion cursor"); - raw.execute( - "INSERT INTO source_cursor_advances - (source_json, scope_json, coverage_json, reason, receipt_id) - VALUES (?1, ?2, ?3, 'admitted', 'receipt.legacy')", - rusqlite::params![ADVANCE.0, ADVANCE.1, ADVANCE.2], - ) - .expect("seed the pre-reset advance ledger entry"); - raw.execute_batch( - "INSERT INTO observation_projection_checkpoints(projector_version, last_sequence) - VALUES ('projector.v1', 100);", - ) - .expect("seed the pre-reset projector checkpoint"); - assert_eq!( - refresh_discovery_frontier(&raw, "session.fixture"), - None, - "the seeded store must start with the rebuild suppressed, or this \ - test proves nothing" - ); - } - - let mut raw = rusqlite::Connection::open(&database_path).unwrap(); - let report = - reset_refused_observation_authority(&mut raw).expect("scoped reset of a cursored store"); - for table in [ - "source_cursors", - "source_cursor_advances", - "observation_projection_checkpoints", - ] { - assert!( - report.reset_tables.iter().any(|reset| reset == table), - "{table} decides what gets re-offered and must be part of the reset: {report:?}" - ); - assert!( - table_exists(&raw, table), - "{table} must be recreated at the canonical shape" - ); - assert_eq!( - count(&raw, table), - 0, - "no pre-reset cursor may survive to skip the events the rebuild re-reads" - ); - } - - // Re-ingest the native event the pre-reset advance already claimed to - // cover, re-presenting that exact advance identity. A surviving row would - // collide on the advance primary key; a refused one lets the rebuilt - // authority record its own coverage. - raw.execute_batch( - "INSERT INTO sanitization_receipts - (receipt_id, sanitizer_version, payload_digest, receipt_json) - VALUES ('receipt.rebuilt', 'v1', 'digest.payload', '{}'); - INSERT INTO observations - (observation_id, payload_digest, receipt_id, observation_json, - committed_cursor_json) - VALUES ('observation.rebuilt', 'digest.payload', 'receipt.rebuilt', '{}', '{}'); - INSERT INTO session_temporal_observation_effects - (observation_id, observation_sequence, session_id, receipt_id, - effect_digest, output_count, recorded_at) - VALUES ('observation.rebuilt', 1, 'session.fixture', 'receipt.rebuilt', - 'digest.effect', 1, 9);", - ) - .expect("re-ingest one native event under the rebuilt authority"); - raw.execute( - "INSERT INTO source_cursor_advances - (source_json, scope_json, coverage_json, reason, receipt_id) - VALUES (?1, ?2, ?3, 'admitted', 'receipt.rebuilt')", - rusqlite::params![ADVANCE.0, ADVANCE.1, ADVANCE.2], - ) - .expect("the rebuilt authority must be able to record the same coverage again"); - - assert_eq!( - refresh_discovery_frontier(&raw, "session.fixture"), - Some(0), - "the rebuilt stream must be rediscovered from zero, not excluded by the \ - frontier of the generation the reset invalidated" - ); - assert!( - foreign_key_violations(&raw).is_empty(), - "the re-ingested stream must be referentially coherent" - ); -} - -/// The host-observation journal attests the stream the reset destroys. -/// Leaving those receipts makes the next admission of the same observation -/// id a conflicting reuse. The writer ledger is installed only when the -/// runtime mounts a store; this offline fixture covers the receipt tables -/// the registered schema always carries. -#[tokio::test] -async fn host_observation_journal_resets_with_the_stream() { - let directory = TempDir::new().unwrap(); - let database_path = directory.path().join("sessions.db"); - install_registered_store(&database_path).await; - { - let raw = rusqlite::Connection::open(&database_path).unwrap(); - install_legacy_observation_shape(&raw); - raw.execute_batch( - "INSERT INTO external_source_states_v1 ( - binding_id, source_id, owner_kind, owner_id, definition_revision, - definition_digest, binding_revision, binding_digest, - source_frontier_digest, source_frontier_json, - latest_source_receipt_digest - ) VALUES ( - 'binding.host', 'source.host-observation.codex', 'project', - 'project.fixture', 1, 'digest.definition', 1, 'digest.binding', - 'digest.frontier', '{}', 'digest.receipt' - ); - INSERT INTO external_source_commit_receipts_v2 ( - binding_id, idempotency_key, request_digest, definition_revision, - binding_revision, predecessor_frontier_digest, - successor_frontier_digest, receipt_digest, receipt_json - ) VALUES ( - 'binding.host', 'sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', - 'sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', - 1, 1, 'digest.pred', 'digest.succ', 'digest.receipt', '{}' - );", - ) - .expect("seed a host-observation journal"); - } - - let mut raw = rusqlite::Connection::open(&database_path).unwrap(); - let report = reset_refused_observation_authority(&mut raw) - .expect("scoped reset of a store with a host-observation journal"); - assert_eq!( - report.cleared_external_source_rows, 2, - "state and receipt must be accounted for: {report:?}" - ); - assert_eq!(count(&raw, "external_source_states_v1"), 0); - assert_eq!(count(&raw, "external_source_commit_receipts_v2"), 0); -} diff --git a/crates/tracedecay-global-db/src/observation/retention.rs b/crates/tracedecay-global-db/src/observation/retention.rs index e462ace133..8850848ed5 100644 --- a/crates/tracedecay-global-db/src/observation/retention.rs +++ b/crates/tracedecay-global-db/src/observation/retention.rs @@ -137,13 +137,6 @@ const CREATE_PROVENANCE_UPDATE_TRIGGER: &str = "CREATE TRIGGER IF NOT EXISTS \ observation_repository_provenance BEGIN SELECT RAISE(ABORT, \ 'observation repository provenance is immutable'); END"; -const DROP_CURSOR_ADVANCE_DELETE_TRIGGER: &str = - "DROP TRIGGER IF EXISTS source_cursor_advances_immutable_delete_v1"; -const CREATE_CURSOR_ADVANCE_DELETE_TRIGGER: &str = "CREATE TRIGGER \ - source_cursor_advances_immutable_delete_v1 BEFORE DELETE ON \ - source_cursor_advances BEGIN SELECT RAISE(ABORT, \ - 'source cursor advances are immutable'); END"; - mod restore; pub use restore::replay_current_release_state_for_restore; @@ -995,7 +988,6 @@ async fn run_cursor_advance_pass( transaction, "apply mode requires an open cursor advance retention transaction", )?; - execute_required(&txn, DROP_CURSOR_ADVANCE_DELETE_TRIGGER).await?; for chunk in targets.chunks(RETENTION_DML_CHUNK) { let placeholders = vec!["?"; chunk.len()].join(","); let sql = format!("DELETE FROM source_cursor_advances WHERE rowid IN ({placeholders})"); @@ -1024,7 +1016,6 @@ async fn run_cursor_advance_pass( )), } } - execute_required(&txn, CREATE_CURSOR_ADVANCE_DELETE_TRIGGER).await?; commit_transaction(txn).await?; Ok(report) } diff --git a/crates/tracedecay-global-db/src/observation/retention/tests.rs b/crates/tracedecay-global-db/src/observation/retention/tests.rs index 3c4a1be1b8..af3dc5371f 100644 --- a/crates/tracedecay-global-db/src/observation/retention/tests.rs +++ b/crates/tracedecay-global-db/src/observation/retention/tests.rs @@ -12,6 +12,8 @@ use tracedecay_domain::{ }; use tracedecay_store::observation::ObservationCoverageV1; +use crate::schema_contract::invariants::SOURCE_CURSOR_ADVANCE_DELETE_GUARD_SQL; + const DAY: i64 = 24 * 60 * 60; const NOW: i64 = 1_900_000_000; const OWNER: &str = "{\"owner\":\"o1\"}"; @@ -221,23 +223,22 @@ async fn seed_cursor_advance_history(conn: &RetentionTestStore) -> Result<(), St let scope = ObservationScopeV1::Profile; let source_json = serde_json::to_string(&source).unwrap(); let scope_json = serde_json::to_string(&scope).unwrap(); - conn.execute_batch( + conn.execute_batch(&format!( "CREATE TRIGGER IF NOT EXISTS source_cursor_advances_immutable_update_v1 BEFORE UPDATE ON source_cursor_advances BEGIN SELECT RAISE(ABORT, 'source cursor advances are immutable'); END; - CREATE TRIGGER IF NOT EXISTS source_cursor_advances_immutable_delete_v1 - BEFORE DELETE ON source_cursor_advances BEGIN - SELECT RAISE(ABORT, 'source cursor advances are immutable'); - END;", - ) + DROP TRIGGER IF EXISTS source_cursor_advances_immutable_delete_v1; + {SOURCE_CURSOR_ADVANCE_DELETE_GUARD_SQL};" + )) .await .map_err(|error| format!("install cursor immutability: {error}"))?; let current_generation = 1_u64; - let current_cursor = ObservationSourceCursorV1::new( + let current_cursor = ObservationSourceCursorV1::for_ordering( source.clone(), scope.clone(), ObservationSourceGenerationV1::new(current_generation).unwrap(), + ObservationOrderingDomainV1::FileBytes, 30, ) .unwrap(); diff --git a/crates/tracedecay-global-db/src/observation/schema.rs b/crates/tracedecay-global-db/src/observation/schema.rs index 278fd1e9d5..8b9508c656 100644 --- a/crates/tracedecay-global-db/src/observation/schema.rs +++ b/crates/tracedecay-global-db/src/observation/schema.rs @@ -1,39 +1,11 @@ -use std::collections::BTreeSet; - -use tracedecay_domain::integration::NativeHostIdentityV1; use tracedecay_runtime_core::db::engine::{Executor, QueryExecutor, params}; use super::super::global_db_operation_error; -/// Typed reset authority for the observation store. No observation shape has -/// ever shipped in a published release (`observations` is absent from both the -/// v0.0.66 package and `origin/master`), so any schema drift here is a -/// branch-local development artifact and refuses admission with -/// [`ResetRequired`](tracedecay_domain::errors::TraceDecayError::ResetRequired) -/// instead of migrating. -pub const OBSERVATION_AUTHORITY: &str = "observations"; - /// Marker proving `observations` was created with the canonical AUTOINCREMENT /// DDL below; the authority schema contract's AUTOINCREMENT invariant consumes /// it. It is recorded at creation, never by rewriting an existing table. -pub(super) const OBSERVATION_SCHEMA_MIGRATION: &str = "observations-v2-canonical-autoincrement"; - -/// Identity of the native-source scheme the committed observations were -/// written under. It is *content* identity, not table shape: since -/// `ff5c895ae` a Cline/Roo/Kilo task's `ui_messages.json` is its own native -/// source (`:ui_messages`, its own generation, in-file ordinals) rather -/// than sharing the API history's combined `` source. A store holding -/// rows written under the old scheme would re-admit every one of those native -/// UI events a second time under the new source key and silently double-count -/// their usage facts, so admission refuses it with -/// [`ResetRequired`](tracedecay_domain::errors::TraceDecayError::ResetRequired) -/// instead. The marker is recorded for any authority that holds no rows yet, -/// and for one whose retained rows and cursors name no Cline/Roo/Kilo source at -/// all, the scheme change touched only those hosts, so such a store cannot -/// double-count anything (see `cline_like_sources_present`). Only stores -/// carrying old-scheme rows from those hosts refuse. -pub const OBSERVATION_NATIVE_SOURCE_SCHEME_MIGRATION: &str = - "observations-native-source-scheme-v2-cline-ui-messages"; +const OBSERVATION_SCHEMA_MIGRATION: &str = "observations-v2-canonical-autoincrement"; /// Marker proving every `observation_repository_provenance` row references /// its repository capture through `observation_repository_captures` instead of @@ -42,7 +14,7 @@ pub const OBSERVATION_NATIVE_SOURCE_SCHEME_MIGRATION: &str = /// `availability_json.value`, repeated a handful of distinct captures hundreds /// of thousands of times. Rows written before the marker are split in place on /// the next schema admission. -pub(super) const OBSERVATION_REPOSITORY_CAPTURE_DEDUPE_MIGRATION: &str = +const OBSERVATION_REPOSITORY_CAPTURE_DEDUPE_MIGRATION: &str = "observation-repository-captures-v1-shared"; /// Moves embedded repository captures into `observation_repository_captures` @@ -66,42 +38,7 @@ const REPOSITORY_CAPTURE_DEDUPE_SQL: &str = " SELECT RAISE(ABORT, 'observation repository provenance is immutable'); END;"; -/// Source-key suffix of the native `ui_messages.json` source a Cline-like task -/// gained under [`OBSERVATION_NATIVE_SOURCE_SCHEME_MIGRATION`] (the sessions -/// crate's `ui_messages_source_key`). -const CLINE_LIKE_UI_MESSAGES_SOURCE_SUFFIX: &str = ":ui_messages"; - -/// Rows examined by one native-source census query. The schema transaction's -/// long lease renews after each bounded query completes, so the census must -/// expose progress between pages instead of running one full-table JSON scan. -/// Observation payloads may approach the 1 MiB authority limit; keep their -/// page smaller than the cursor-only page. -pub(super) const OBSERVATION_SOURCE_CENSUS_PAGE_ROWS: i64 = 48; -pub(super) const SOURCE_CURSOR_CENSUS_PAGE_ROWS: i64 = 128; - -/// Canonical `observations` column set. Shared by the admission refusal below -/// and the scoped operator reset in [`super::reset`] so the two can never -/// disagree about what counts as a refused shape. -pub(super) const OBSERVATION_CANONICAL_COLUMNS: &[&str] = &[ - "sequence", - "observation_id", - "payload_digest", - "receipt_id", - "observation_json", - "committed_cursor_json", -]; - -/// Canonical provider-neutral `source_cursor_advances` column set, shared with -/// [`super::reset`] like [`OBSERVATION_CANONICAL_COLUMNS`]. -pub(super) const SOURCE_CURSOR_ADVANCES_CANONICAL_COLUMNS: &[&str] = &[ - "source_json", - "scope_json", - "coverage_json", - "reason", - "receipt_id", -]; - -pub(super) const OBSERVATION_SCHEMA_OPERATION: &str = "ensure observation authority schema"; +const OBSERVATION_SCHEMA_OPERATION: &str = "ensure observation authority schema"; async fn observation_table_exists( conn: &impl QueryExecutor, @@ -119,144 +56,7 @@ async fn observation_table_exists( .map_err(|error| global_db_operation_error(OBSERVATION_SCHEMA_OPERATION, error)) } -/// Whether the authority already carries native-source identity written under -/// whatever scheme was current when it was committed: retained observations, -/// or the source cursors that decide what gets re-offered. Only these stores -/// can double-count when the scheme changes; an empty authority just enrolls. -/// Both tables are created by [`OBSERVATION_AUTHORITY_SCHEMA_SQL`], which runs -/// before every caller of this helper. -async fn observation_authority_populated( - conn: &impl QueryExecutor, -) -> tracedecay_domain::errors::Result { - let mut rows = conn - .query( - "SELECT 1 WHERE EXISTS(SELECT 1 FROM observations) - OR EXISTS(SELECT 1 FROM source_cursors)", - (), - ) - .await - .map_err(|error| global_db_operation_error(OBSERVATION_SCHEMA_OPERATION, error))?; - rows.next() - .await - .map(|row| row.is_some()) - .map_err(|error| global_db_operation_error(OBSERVATION_SCHEMA_OPERATION, error)) -} - -/// Whether any retained observation or admission cursor names a Cline, Roo -/// Code, or Kilo source, the only hosts whose admission scheme changed under -/// [`OBSERVATION_NATIVE_SOURCE_SCHEME_MIGRATION`]. A populated authority -/// without such rows was written by a scheme that never applied to it, so -/// enrolling it is exact rather than a migration of ambiguous data; one with -/// such rows carries no record of which scheme wrote them and must reset. -#[hotpath::measure(future = true, label = "global_db.observation.native_source_census")] -async fn cline_like_sources_present( - conn: &impl QueryExecutor, -) -> tracedecay_domain::errors::Result { - let providers = [ - NativeHostIdentityV1::Cline.hook_key(), - NativeHostIdentityV1::RooCode.hook_key(), - NativeHostIdentityV1::Kilo.hook_key(), - ]; - let ui_messages_pattern = format!("%{CLINE_LIKE_UI_MESSAGES_SOURCE_SUFFIX}"); - - let mut cursor_rowid = i64::MIN; - loop { - let mut rows = conn - .query( - "SELECT rowid, - COALESCE( - json_extract(source_json, '$.provider') IN (?1, ?2, ?3) - OR json_extract(source_json, '$.source_key') LIKE ?4, - 0 - ) - FROM source_cursors - WHERE rowid > ?5 ORDER BY rowid LIMIT ?6", - params![ - providers[0], - providers[1], - providers[2], - &ui_messages_pattern, - cursor_rowid, - SOURCE_CURSOR_CENSUS_PAGE_ROWS - ], - ) - .await - .map_err(|error| global_db_operation_error(OBSERVATION_SCHEMA_OPERATION, error))?; - let mut page_rows = 0_i64; - while let Some(row) = rows - .next() - .await - .map_err(|error| global_db_operation_error(OBSERVATION_SCHEMA_OPERATION, error))? - { - page_rows += 1; - cursor_rowid = row - .get::(0) - .map_err(|error| global_db_operation_error(OBSERVATION_SCHEMA_OPERATION, error))?; - if row - .get::(1) - .map_err(|error| global_db_operation_error(OBSERVATION_SCHEMA_OPERATION, error))? - != 0 - { - return Ok(true); - } - } - drop(rows); - if page_rows < SOURCE_CURSOR_CENSUS_PAGE_ROWS { - break; - } - } - - let mut observation_sequence = 0_i64; - loop { - let mut rows = conn - .query( - "SELECT sequence, - COALESCE( - json_extract(observation_json, '$.identity.source.provider') - IN (?1, ?2, ?3) - OR json_extract(observation_json, '$.identity.source.source_key') - LIKE ?4, - 0 - ) - FROM observations - WHERE sequence > ?5 ORDER BY sequence LIMIT ?6", - params![ - providers[0], - providers[1], - providers[2], - &ui_messages_pattern, - observation_sequence, - OBSERVATION_SOURCE_CENSUS_PAGE_ROWS - ], - ) - .await - .map_err(|error| global_db_operation_error(OBSERVATION_SCHEMA_OPERATION, error))?; - let mut page_rows = 0_i64; - while let Some(row) = rows - .next() - .await - .map_err(|error| global_db_operation_error(OBSERVATION_SCHEMA_OPERATION, error))? - { - page_rows += 1; - observation_sequence = row - .get::(0) - .map_err(|error| global_db_operation_error(OBSERVATION_SCHEMA_OPERATION, error))?; - if row - .get::(1) - .map_err(|error| global_db_operation_error(OBSERVATION_SCHEMA_OPERATION, error))? - != 0 - { - return Ok(true); - } - } - drop(rows); - if page_rows < OBSERVATION_SOURCE_CENSUS_PAGE_ROWS { - return Ok(false); - } - } -} - -pub(super) async fn migration_recorded( +async fn migration_recorded( conn: &impl QueryExecutor, migration: &str, ) -> tracedecay_domain::errors::Result { @@ -273,91 +73,8 @@ pub(super) async fn migration_recorded( .map_err(|error| global_db_operation_error(OBSERVATION_SCHEMA_OPERATION, error)) } -async fn table_columns( - conn: &impl QueryExecutor, - table: &str, -) -> tracedecay_domain::errors::Result> { - let mut rows = conn - .query("SELECT name FROM pragma_table_xinfo(?1)", params![table]) - .await - .map_err(|error| global_db_operation_error(OBSERVATION_SCHEMA_OPERATION, error))?; - let mut columns = BTreeSet::new(); - while let Some(row) = rows - .next() - .await - .map_err(|error| global_db_operation_error(OBSERVATION_SCHEMA_OPERATION, error))? - { - columns.insert( - row.get::(0) - .map_err(|error| global_db_operation_error(OBSERVATION_SCHEMA_OPERATION, error))?, - ); - } - Ok(columns) -} - -fn canonical_column_set(columns: &[&str]) -> BTreeSet { - columns.iter().map(|column| (*column).to_string()).collect() -} - -/// Refuses a store whose `observations` or `source_cursor_advances` table -/// carries anything but the canonical shape (plus, for `observations`, its -/// creation marker). The alternative shapes, the `idempotency_key` column -/// era, unmarked non-AUTOINCREMENT tables, and the byte-offset -/// `source_cursor_advances` predecessor, were branch-local and never shipped -/// in a published release, so there is no sanctioned migration: the store -/// surfaces a typed `ResetRequired` naming this authority instead of -/// rewriting data in place. Runs at schema installation for fresh stores and -/// at the attach boundary for existing ones. -async fn require_admitted_observation_shape( - conn: &impl QueryExecutor, -) -> tracedecay_domain::errors::Result<()> { - if observation_table_exists(conn).await? { - let columns = table_columns(conn, "observations").await?; - let recorded = migration_recorded(conn, OBSERVATION_SCHEMA_MIGRATION).await?; - if columns != canonical_column_set(OBSERVATION_CANONICAL_COLUMNS) || !recorded { - return Err(tracedecay_domain::errors::TraceDecayError::reset_required( - OBSERVATION_AUTHORITY, - "observations carries a pre-release branch-local shape that no \ - published binary ever wrote; there is no sanctioned migration, \ - reset the observation authority to recreate it at the canonical \ - schema", - )); - } - } - let advances = table_columns(conn, "source_cursor_advances").await?; - if !advances.is_empty() - && advances != canonical_column_set(SOURCE_CURSOR_ADVANCES_CANONICAL_COLUMNS) - { - return Err(tracedecay_domain::errors::TraceDecayError::reset_required( - OBSERVATION_AUTHORITY, - "source_cursor_advances carries a pre-release branch-local shape \ - that no published binary ever wrote; there is no sanctioned \ - migration, reset the observation authority to recreate it at the \ - canonical schema", - )); - } - if observation_authority_populated(conn).await? - && !migration_recorded(conn, OBSERVATION_NATIVE_SOURCE_SCHEME_MIGRATION).await? - { - return Err(tracedecay_domain::errors::TraceDecayError::reset_required( - OBSERVATION_AUTHORITY, - "these observations were committed before a Cline/Roo/Kilo task's \ - ui_messages.json became its own native source; re-offering that \ - file under the :ui_messages source would admit every one of \ - its native UI events a second time and double-count their usage \ - facts. There is no sanctioned migration, reset the observation \ - authority so the derived usage and the admission cursors rebuild \ - together from the preserved transcripts", - )); - } - Ok(()) -} - -/// Canonical observation-authority DDL. Shared with the scoped operator reset -/// in [`super::reset`], which recreates these tables after dropping a refused -/// authority, so the installer and the reset can never produce different -/// shapes. -pub(super) const OBSERVATION_AUTHORITY_SCHEMA_SQL: &str = +/// Canonical observation-authority DDL. +const OBSERVATION_AUTHORITY_SCHEMA_SQL: &str = "CREATE TABLE IF NOT EXISTS global_schema_migrations ( migration TEXT PRIMARY KEY ); @@ -496,24 +213,6 @@ pub async fn ensure_observation_schema( .await .map_err(|error| global_db_operation_error(OBSERVATION_SCHEMA_OPERATION, error))?; } - // The retained marker already certifies the native-source scheme. Reopening - // an enrolled authority must not scan historical JSON again while holding - // schema admission's writer transaction. - // Enroll the native-source scheme wherever it cannot double-count: an - // authority with no rows, or one whose rows and cursors never came from a - // Cline-like host. Only a populated authority that does carry such rows is - // left unmarked, and `require_admitted_observation_shape` refuses it. - if !migration_recorded(conn, OBSERVATION_NATIVE_SOURCE_SCHEME_MIGRATION).await? - && (!observation_authority_populated(conn).await? - || !cline_like_sources_present(conn).await?) - { - conn.execute( - "INSERT OR IGNORE INTO global_schema_migrations(migration) VALUES (?1)", - params![OBSERVATION_NATIVE_SOURCE_SCHEME_MIGRATION], - ) - .await - .map_err(|error| global_db_operation_error(OBSERVATION_SCHEMA_OPERATION, error))?; - } if !migration_recorded(conn, OBSERVATION_REPOSITORY_CAPTURE_DEDUPE_MIGRATION).await? { conn.execute_batch(REPOSITORY_CAPTURE_DEDUPE_SQL) .await @@ -525,135 +224,13 @@ pub async fn ensure_observation_schema( .await .map_err(|error| global_db_operation_error(OBSERVATION_SCHEMA_OPERATION, error))?; } - require_admitted_observation_shape(conn).await?; Ok(()) } #[cfg(test)] mod tests { - use std::sync::atomic::{AtomicUsize, Ordering}; - - use tracedecay_runtime_core::db::engine::{IntoParams, Rows}; - use super::*; - struct CountQueries<'a, T> { - inner: &'a T, - count: AtomicUsize, - } - - impl QueryExecutor for CountQueries<'_, T> { - async fn query

( - &self, - sql: &str, - params: P, - ) -> tracedecay_runtime_core::db::engine::Result - where - P: IntoParams, - { - self.count.fetch_add(1, Ordering::Relaxed); - self.inner.query(sql, params).await - } - } - - impl Executor for CountQueries<'_, T> { - async fn execute

( - &self, - sql: &str, - params: P, - ) -> tracedecay_runtime_core::db::engine::Result - where - P: IntoParams, - { - self.inner.execute(sql, params).await - } - - async fn execute_batch( - &self, - sql: &str, - ) -> tracedecay_runtime_core::db::engine::Result<()> { - self.inner.execute_batch(sql).await - } - } - - #[tokio::test] - async fn enrolled_schema_admission_cost_does_not_grow_with_retained_observations() { - let directory = tempfile::TempDir::new().unwrap(); - let fixture = crate::tests::harness::open_registered_test_fixture( - &directory.path().join("sessions.db"), - tracedecay_runtime_core::db::TestDatabaseRuntimeScope::ProfileSessions, - ) - .await - .unwrap(); - let transaction = fixture.database().begin_write_transaction().await.unwrap(); - let measured = CountQueries { - inner: &transaction, - count: AtomicUsize::new(0), - }; - ensure_observation_schema(&measured).await.unwrap(); - let empty_queries = measured.count.swap(0, Ordering::Relaxed); - - for index in 0..=OBSERVATION_SOURCE_CENSUS_PAGE_ROWS * 2 { - let (observation, cursor) = - crate::schema_contract::invariants::test_fixture::authority_fixture( - index as u64, - &format!("enrolled-{index}"), - ); - let receipt = observation.receipt(); - transaction - .execute( - "INSERT INTO sanitization_receipts - (receipt_id, sanitizer_version, payload_digest, receipt_json) - VALUES (?1, ?2, ?3, ?4)", - params![ - receipt.receipt().receipt_id().as_str(), - receipt.receipt().sanitizer_version().as_str(), - observation.payload_reference().digest().as_str(), - serde_json::to_string(receipt).unwrap() - ], - ) - .await - .unwrap(); - transaction.execute( - "INSERT INTO observations - (observation_id, payload_digest, receipt_id, observation_json, committed_cursor_json) - VALUES (?1, ?2, ?3, ?4, ?5)", - params![observation.observation_id().as_str(), observation.payload_reference().digest().as_str(), - receipt.receipt().receipt_id().as_str(), serde_json::to_string(&observation).unwrap(), - serde_json::to_string(&cursor).unwrap()], - ).await.unwrap(); - } - ensure_observation_schema(&measured).await.unwrap(); - let populated_queries = measured.count.swap(0, Ordering::Relaxed); - - transaction - .execute( - "DELETE FROM global_schema_migrations WHERE migration = ?1", - params![OBSERVATION_NATIVE_SOURCE_SCHEME_MIGRATION], - ) - .await - .unwrap(); - ensure_observation_schema(&measured).await.unwrap(); - let unenrolled_queries = measured.count.swap(0, Ordering::Relaxed); - println!( - "schema queries: empty={empty_queries}, populated enrolled={populated_queries}, unenrolled={unenrolled_queries}" - ); - assert!( - unenrolled_queries > populated_queries, - "unmarked content must still be inspected" - ); - assert!( - populated_queries <= empty_queries + 1, - "enrolled admission must not page through historical content: empty={empty_queries}, populated={populated_queries}" - ); - assert!( - migration_recorded(&transaction, OBSERVATION_NATIVE_SOURCE_SCHEME_MIGRATION) - .await - .unwrap() - ); - transaction.rollback().await.unwrap(); - } - /// A provenance row written before the shared-capture migration embeds the /// same capture twice; re-admission must split it into a shared row plus a /// slim reference and still hydrate the original documents for readers. diff --git a/crates/tracedecay-global-db/src/observation_adapter.rs b/crates/tracedecay-global-db/src/observation_adapter.rs index a95f123409..b966990a66 100644 --- a/crates/tracedecay-global-db/src/observation_adapter.rs +++ b/crates/tracedecay-global-db/src/observation_adapter.rs @@ -10,7 +10,7 @@ use tracedecay_domain::{ GenerationBoundRepositoryProvenanceV1, ManifestDigest, ObservationCollisionOutcomeV1, ObservationIdentityMaterialV1, ObservationScopeV1, ObservationSourceCursorV1, ObservationSourceIdentityV1, PayloadDigestV1, PayloadReferenceV1, ProjectionGenerationId, - RetrievalAnchorId, RetrievalAnchorRecordV2, SanitizationReceiptV1, canonical_json_bytes, + RetrievalAnchorId, RetrievalAnchorRecord, SanitizationReceiptV1, canonical_json_bytes, canonical_json_bytes_and_sha256, canonical_sha256, classify_observation_collision, cline_native_source_successor_id, cline_task_native_observation_id, is_canonical_payload_revision_replay, prove_cline_native_source_transition, sha256_hex_suffix, @@ -24,27 +24,27 @@ use tracedecay_store::{ BACKGROUND_BATCH_MAX_BYTES, BACKGROUND_BATCH_MAX_OPERATIONS, CommandDigestV1, ConsistencyModeV1, CursorAdvanceLedgerDisagreementV1, CursorAdvanceLedgerIdentityV1, DurabilityClassV1, FOREGROUND_BATCH_MAX_BYTES, IdempotencyIdentityV1, - ObservationBatchFallbackCause, ObservationBatchPersistOutcome, ObservationCommitReceipt, - ObservationPersistOutcome, ObservationProjectionStatus, ObservationProjectionStore, - ObservationReadOperationV1, ObservationReadResultV1, ObservationReplayRequest, - ObservationStore, ObservationStoreError, ObservationStoreResult, OperationPriorityV1, - ProjectReadOperationV1, ProjectReadResultV1, ProjectionCheckpoint, ProjectionPersistOutcome, - ProjectionPredecessorConvergence, ProjectionRebuildOutcome, ProjectionStoreResult, - RepositoryOperationEnvelopeV1, RepositoryProvenanceAttachmentV1, RepositoryReadOperationV1, - RepositoryReadResultV1, RepositoryWritePayloadV1, RuntimeBatchCompatibilityV1, - RuntimeCancellationIdV1, RuntimeCancellationIdentityV1, RuntimeDeadlineIdV1, RuntimeDeadlineV1, - RuntimeInterruptionV1, RuntimeReadCoverageV1, RuntimeReadOperationV1, RuntimeReadRequestV1, - RuntimeReadResultV1, RuntimeRequestControlV1, RuntimeRequestProbeV1, RuntimeSubmitOutcomeV1, - RuntimeSubmitRequestV1, RuntimeTransactionIdV1, RuntimeTransactionScopeV1, - StorageRuntimeErrorV1, StoreClientIdV1, StoreIdempotencyKeyV1, StoreOperationIdV1, - StoreOperationMetadataV1, StoredObservation, StoredObservationRowV1, + ObservationBatchPersistOutcome, ObservationCommitReceipt, ObservationPersistOutcome, + ObservationProjectionStatus, ObservationProjectionStore, ObservationReadOperationV1, + ObservationReadResultV1, ObservationReplayRequest, ObservationStore, ObservationStoreError, + ObservationStoreResult, OperationPriorityV1, ProjectReadOperationV1, ProjectReadResultV1, + ProjectionCheckpoint, ProjectionPersistOutcome, ProjectionPredecessorConvergence, + ProjectionRebuildOutcome, ProjectionStoreResult, RepositoryOperationEnvelopeV1, + RepositoryProvenanceAttachmentV1, RepositoryReadOperationV1, RepositoryReadResultV1, + RepositoryWritePayloadV1, RuntimeBatchCompatibilityV1, RuntimeCancellationIdV1, + RuntimeCancellationIdentityV1, RuntimeDeadlineIdV1, RuntimeDeadlineV1, RuntimeInterruptionV1, + RuntimeReadCoverageV1, RuntimeReadOperationV1, RuntimeReadRequestV1, RuntimeReadResultV1, + RuntimeRequestControlV1, RuntimeRequestProbeV1, RuntimeSubmitOutcomeV1, RuntimeSubmitRequestV1, + RuntimeTransactionIdV1, RuntimeTransactionScopeV1, StorageRuntimeErrorV1, StoreClientIdV1, + StoreIdempotencyKeyV1, StoreOperationIdV1, StoreOperationMetadataV1, StoredObservation, + StoredObservationRowV1, }; use tracedecay_runtime_core::db::{Database, DatabaseEngineReadSnapshot, DatabaseRuntimeClientV1}; use tracedecay_runtime_core::shard_runtime::registry::StoreRuntimeRegistryFailure; use tracedecay_rusqlite_runtime::repository::observation_cursor_authority::{ - COMMIT_SOURCE_CURSOR_SQL, READ_CURSOR_ADVANCE_SQL, READ_SOURCE_CURSOR_SQL, - RECORD_CURSOR_ADVANCE_SQL, cursor_advance_ledger_row_matches, + COMMIT_SOURCE_CURSOR_SQL, PRUNE_SUPERSEDED_CURSOR_ADVANCES_SQL, READ_CURSOR_ADVANCE_SQL, + READ_SOURCE_CURSOR_SQL, RECORD_CURSOR_ADVANCE_SQL, cursor_advance_ledger_row_matches, }; use tracedecay_rusqlite_runtime::repository::{ REPOSITORY_PROVENANCE_CAPTURE_JOIN, REPOSITORY_PROVENANCE_HYDRATED_COLUMNS, @@ -298,6 +298,16 @@ impl GlobalDbObservationStore { ) .await .map_err(|error| runtime_storage_error(OPERATION, error))?; + transaction + .execute( + PRUNE_SUPERSEDED_CURSOR_ADVANCES_SQL, + tracedecay_runtime_core::db::engine::params![ + source_json.as_str(), + scope_json.as_str() + ], + ) + .await + .map_err(|error| runtime_storage_error(OPERATION, error))?; transaction .commit() .await @@ -417,6 +427,16 @@ impl GlobalDbObservationStore { ) .await .map_err(|error| runtime_storage_error(OPERATION, error))?; + transaction + .execute( + PRUNE_SUPERSEDED_CURSOR_ADVANCES_SQL, + tracedecay_runtime_core::db::engine::params![ + source_json.as_str(), + scope_json.as_str() + ], + ) + .await + .map_err(|error| runtime_storage_error(OPERATION, error))?; transaction .commit() .await @@ -507,9 +527,9 @@ impl GlobalDbObservationStore { && !canonical_payload_revision { if pending.is_some() { - return Err(ObservationStoreError::BatchRequiresScalarFallback { - cause: ObservationBatchFallbackCause::IntraBatchIdentityCollision, - }); + return Ok(PreparedObservationPersist::AwaitsDurablePredecessor( + Box::new(write), + )); } let retained_digest = pending .as_ref() @@ -547,8 +567,10 @@ impl GlobalDbObservationStore { observation.payload_reference().digest().clone(), )); } - if let Some(fallback) = durable_frontier_owned_by_batch(&known_cursor) { - return Err(fallback); + if known_cursor.is_some() { + return Ok(PreparedObservationPersist::AwaitsDurablePredecessor( + Box::new(write), + )); } if let RefusalCoverageOutcome::NotAtFrontier { actual } = self .record_refusal_with_coverage(&write, retained_digest, cursor.as_ref()) @@ -597,8 +619,10 @@ impl GlobalDbObservationStore { existing, )); } - if let Some(fallback) = durable_frontier_owned_by_batch(&known_cursor) { - return Err(fallback); + if known_cursor.is_some() { + return Ok(PreparedObservationPersist::AwaitsDurablePredecessor( + Box::new(write), + )); } let mut advance = ObservationCursorAdvance::for_ordering_with_sanitization_receipt( identity.source().clone(), @@ -657,9 +681,9 @@ impl GlobalDbObservationStore { })) { if pending.is_some() { - return Err(ObservationStoreError::BatchRequiresScalarFallback { - cause: ObservationBatchFallbackCause::IntraBatchSanitizationReceiptCollision, - }); + return Ok(PreparedObservationPersist::AwaitsDurablePredecessor( + Box::new(write), + )); } return Err(ObservationStoreError::SanitizationReceiptCollision); } @@ -667,9 +691,9 @@ impl GlobalDbObservationStore { .pending_receipt(observation.receipt()) .is_some_and(|retained| retained != observation.receipt()) { - return Err(ObservationStoreError::BatchRequiresScalarFallback { - cause: ObservationBatchFallbackCause::IntraBatchSanitizationReceiptCollision, - }); + return Ok(PreparedObservationPersist::AwaitsDurablePredecessor( + Box::new(write), + )); } for alias in write.retrieval_anchor().aliases() { if let Some(existing) = @@ -677,10 +701,9 @@ impl GlobalDbObservationStore { && existing.anchor_id != *write.retrieval_anchor_id() { if existing.pending { - return Err(ObservationStoreError::BatchRequiresScalarFallback { - cause: - ObservationBatchFallbackCause::IntraBatchRetrievalAnchorAliasCollision, - }); + return Ok(PreparedObservationPersist::AwaitsDurablePredecessor( + Box::new(write), + )); } if !preflight.accepts_pending_cline_alias(&write, &existing.anchor_id)? { return Err(ObservationStoreError::RetrievalAnchorAliasCollision { @@ -735,6 +758,95 @@ impl GlobalDbObservationStore { } Ok(PreparedObservationPersist::Submit(Box::new(write))) } + + /// Persists the longest prefix of `writes` that settles in one runtime + /// batch, returning its outcomes in input order and the writes that must + /// be prepared again once that prefix is durable. + #[hotpath::skip] + async fn persist_observation_segment( + &self, + writes: Vec, + ) -> ObservationStoreResult<( + Vec, + Vec, + )> { + crate::hotpath_observe::record_transaction_rows(1); + let preflight = load_observation_preflight(&self.database, &writes).await?; + let mut batch_state = ObservationBatchState::from_preflight(&preflight); + let mut published_cursors = HashMap::< + (ObservationSourceIdentityV1, ObservationScopeV1), + ObservationSourceCursorV1, + >::new(); + let mut outcomes: Vec> = + Vec::with_capacity(writes.len()); + let mut submits = Vec::new(); + let mut deferred_exact_duplicates = Vec::new(); + let mut awaiting = Vec::new(); + let mut writes = writes.into_iter(); + while let Some(write) = writes.next() { + let key = ( + write.observation().source().clone(), + write.observation().scope().clone(), + ); + let known_cursor = published_cursors.get(&key).cloned().map(Some); + let next_cursor = write.next_cursor().clone(); + match self + .prepare_observation_persist(write, &preflight, &mut batch_state, known_cursor) + .await? + { + PreparedObservationPersist::Ready(outcome) => outcomes.push(Some(*outcome)), + PreparedObservationPersist::Submit(write) => { + submits.push((outcomes.len(), *write)); + outcomes.push(None); + } + PreparedObservationPersist::DeferredExactDuplicate(write) => { + deferred_exact_duplicates.push((outcomes.len(), *write)); + outcomes.push(None); + } + PreparedObservationPersist::AwaitsDurablePredecessor(write) => { + if outcomes.is_empty() { + return Err(runtime_storage_error( + "persist_observations", + "the first write of a batch segment has no batch predecessor", + )); + } + awaiting.push(*write); + awaiting.extend(writes); + break; + } + } + published_cursors.insert(key, next_cursor); + } + if !submits.is_empty() { + let submitted = submit_observation_writes( + &self.database, + &self.runtime, + submits, + deferred_exact_duplicates, + ) + .await?; + for (slot, outcome) in submitted { + outcomes[slot] = Some(outcome); + } + } else if !deferred_exact_duplicates.is_empty() { + return Err(runtime_storage_error( + "persist_observations", + "deferred duplicate has no preceding batch submission", + )); + } + let outcomes = outcomes + .into_iter() + .map(|outcome| { + outcome.ok_or_else(|| { + runtime_storage_error( + "persist_observations", + "batch slot was not settled by writer authority", + ) + }) + }) + .collect::>>()?; + Ok((outcomes, awaiting)) + } } /// Immutable read authority for one bounded admission window. @@ -898,7 +1010,7 @@ impl ObservationBatchState { fn retrieval_anchor_by_alias( &self, scope: &ObservationScopeV1, - alias: &tracedecay_domain::NativeAliasV2, + alias: &tracedecay_domain::NativeAlias, ) -> ObservationStoreResult> { let key = retrieval_alias_key(scope, alias)?; Ok(self.retrieval_aliases.get(&key).cloned()) @@ -1057,7 +1169,7 @@ async fn read_cline_supersessions_from_snapshot( .validate() .map_err(|error| runtime_storage_error(operation, error))?; if let Some(owner) = anchors.get(record.anchor_id()) - && record.owner().v2() == Some(&tracedecay_domain::FactOwnerV1::from(owner.clone())) + && *record.owner() == tracedecay_domain::FactOwnerV1::from(owner.clone()) && record.state() == AnchorDispositionStateV1::Superseded && record.reason_class() == AnchorDispositionReasonClassV1::Correction && let Some(successor) = record.superseded_by() @@ -1205,7 +1317,7 @@ async fn read_source_cursors_from_snapshot( fn retrieval_alias_key( scope: &ObservationScopeV1, - alias: &tracedecay_domain::NativeAliasV2, + alias: &tracedecay_domain::NativeAlias, ) -> ObservationStoreResult<(String, String, String)> { Ok(( serde_json::to_string(scope) @@ -1373,7 +1485,7 @@ async fn read_stored_observations_from_snapshot( "observation committed cursor binding mismatch", )); } - let retrieval_anchor: RetrievalAnchorRecordV2 = decode_json( + let retrieval_anchor: RetrievalAnchorRecord = decode_json( row.get::>(4) .map_err(|error| runtime_storage_error(operation, error))? .ok_or_else(|| { @@ -1422,7 +1534,7 @@ async fn read_stored_observations_from_snapshot( .map_err(|error| runtime_storage_error(operation, error))?; let expected_repository_owner = repository_anchor .as_ref() - .map(RetrievalAnchorRecordV2::owner_column_json) + .map(RetrievalAnchorRecord::owner_column_json) .transpose() .map_err(|error| runtime_storage_error(operation, error))?; if repository_owner != expected_repository_owner { @@ -1466,6 +1578,11 @@ enum PreparedObservationPersist { Ready(Box), Submit(Box), DeferredExactDuplicate(Box), + /// The write collides with, or must compare-and-set a source cursor + /// published by, an earlier member of this batch that is not durable yet. + /// The batch commits its prefix first and re-prepares this write against + /// the durable result, so it settles exactly as it would alone. + AwaitsDurablePredecessor(Box), } impl PreparedObservationPersist { @@ -1513,72 +1630,14 @@ impl ObservationStore for GlobalDbObservationStore { writes = writes.len() ); async move { - crate::hotpath_observe::record_transaction_rows(1); - let preflight = load_observation_preflight(&self.database, &writes).await?; - let mut batch_state = ObservationBatchState::from_preflight(&preflight); - let mut published_cursors = HashMap::< - (ObservationSourceIdentityV1, ObservationScopeV1), - ObservationSourceCursorV1, - >::new(); - let mut prepared = Vec::with_capacity(writes.len()); - for write in writes { - let key = ( - write.observation().source().clone(), - write.observation().scope().clone(), - ); - let known_cursor = published_cursors.get(&key).cloned().map(Some); - let next_cursor = write.next_cursor().clone(); - let item = self - .prepare_observation_persist(write, &preflight, &mut batch_state, known_cursor) - .await?; - published_cursors.insert(key, next_cursor); - prepared.push(item); + let mut settled = Vec::with_capacity(writes.len()); + let mut remaining = writes; + while !remaining.is_empty() { + let (outcomes, awaiting) = self.persist_observation_segment(remaining).await?; + settled.extend(outcomes); + remaining = awaiting; } - let mut outcomes: Vec> = - Vec::with_capacity(prepared.len()); - let mut submits = Vec::new(); - let mut deferred_exact_duplicates = Vec::new(); - for item in prepared { - match item { - PreparedObservationPersist::Ready(outcome) => outcomes.push(Some(*outcome)), - PreparedObservationPersist::Submit(write) => { - submits.push((outcomes.len(), *write)); - outcomes.push(None); - } - PreparedObservationPersist::DeferredExactDuplicate(write) => { - deferred_exact_duplicates.push((outcomes.len(), *write)); - outcomes.push(None); - } - } - } - if !submits.is_empty() { - let submitted = submit_observation_writes( - &self.database, - &self.runtime, - submits, - deferred_exact_duplicates, - ) - .await?; - for (slot, outcome) in submitted { - outcomes[slot] = Some(outcome); - } - } else if !deferred_exact_duplicates.is_empty() { - return Err(runtime_storage_error( - "persist_observations", - "deferred duplicate has no preceding batch submission", - )); - } - outcomes - .into_iter() - .map(|outcome| { - outcome.ok_or_else(|| { - runtime_storage_error( - "persist_observations", - "batch slot was not settled by writer authority", - ) - }) - }) - .collect() + Ok(settled) } .instrument(span) .await @@ -1909,23 +1968,6 @@ enum RefusalCoverageOutcome { }, } -/// Whether an earlier member of this batch already published the source cursor -/// this write would compare-and-set against. -/// -/// The collision paths below read the *durable* frontier, but a published -/// batch cursor only becomes durable when the batch submits. Replaying the -/// batch as scalar writes lets each earlier write land first, instead of -/// refusing the whole window as a cursor conflict and wedging the frontier. -fn durable_frontier_owned_by_batch( - known_cursor: &Option>, -) -> Option { - known_cursor - .is_some() - .then_some(ObservationStoreError::BatchRequiresScalarFallback { - cause: ObservationBatchFallbackCause::IntraBatchDurableFrontier, - }) -} - fn refused_scan_frontier( write: &AnchoredObservationWrite, actual_cursor: Option<&ObservationSourceCursorV1>, diff --git a/crates/tracedecay-global-db/src/observation_batch_tests.rs b/crates/tracedecay-global-db/src/observation_batch_tests.rs index dd19274bf0..c57687201b 100644 --- a/crates/tracedecay-global-db/src/observation_batch_tests.rs +++ b/crates/tracedecay-global-db/src/observation_batch_tests.rs @@ -13,15 +13,13 @@ use tracedecay_domain::{ ObservationCollisionOutcomeV1, ObservationId, ObservationIdentityMaterialV1, ObservationOrderingDomainV1, ObservationScopeV1, ObservationSourceCursorV1, ObservationSourceGenerationV1, ObservationSourceIdentityV1, ObservationSourceRangeV1, - PayloadReferenceV1, ProjectionGenerationId, ProviderId, RetentionClass, - RetrievalAnchorRecordV2, RetrievalAnchorRecordV2Parts, SanitizationReceiptId, - SanitizationReceiptRefV1, SanitizationReceiptV1, SanitizerDispositionV1, SensitivityV1, - SessionId, UtcMicros, + PayloadReferenceV1, ProjectionGenerationId, ProviderId, RetentionClass, RetrievalAnchorRecord, + RetrievalAnchorRecordParts, SanitizationReceiptId, SanitizationReceiptRefV1, + SanitizationReceiptV1, SanitizerDispositionV1, SensitivityV1, SessionId, UtcMicros, }; use tracedecay_store::{ - AnchoredObservationWrite, FOREGROUND_BATCH_MAX_OPERATIONS, ObservationBatchFallbackCause, - ObservationBatchPersistOutcome, ObservationPersistOutcome, ObservationStore, - ObservationStoreError, ObservationWrite, + AnchoredObservationWrite, FOREGROUND_BATCH_MAX_OPERATIONS, ObservationBatchPersistOutcome, + ObservationPersistOutcome, ObservationStore, ObservationStoreError, ObservationWrite, }; use tracing::field::{Field, Visit}; use tracing::span::{Attributes, Id, Record}; @@ -108,10 +106,12 @@ async fn persist_with_work_census( ) } +/// Committed writer operations, read from the shard checkpoint: every +/// operation advances its commit sequence by one and opens its own +/// `RuntimeTransactionScopeV1`, whether or not the ledger keeps a replay row. #[derive(Clone, Copy, Debug, PartialEq, Eq)] struct WriterTxnCensus { operations: i64, - scopes: i64, } async fn writer_txn_census(runtime: &HostAdmissionTestRuntimeV1) -> WriterTxnCensus { @@ -121,8 +121,7 @@ async fn writer_txn_census(runtime: &HostAdmissionTestRuntimeV1) -> WriterTxnCen let snapshot = database.read_snapshot().await.expect("read snapshot"); let mut rows = snapshot .query( - "SELECT COUNT(*), COUNT(DISTINCT transaction_scope_json) - FROM td_runtime_writer_idempotency_v2", + "SELECT COALESCE(SUM(commit_sequence), 0) FROM td_runtime_writer_checkpoint_v1", (), ) .await @@ -134,7 +133,6 @@ async fn writer_txn_census(runtime: &HostAdmissionTestRuntimeV1) -> WriterTxnCen .expect("writer ledger census row"); WriterTxnCensus { operations: row.get::(0).expect("operation count"), - scopes: row.get::(1).expect("distinct transaction scopes"), } } @@ -152,10 +150,7 @@ async fn initialize_writer_authority( )); assert_eq!( writer_txn_census(runtime).await, - WriterTxnCensus { - operations: 1, - scopes: 1, - } + WriterTxnCensus { operations: 1 } ); } @@ -241,7 +236,7 @@ fn anchored_write( "observation-batch-test", ) .unwrap(); - let anchor = tracedecay_store::build_observation_retrieval_anchor_v2( + let anchor = tracedecay_store::build_observation_retrieval_anchor( write.observation(), projection_generation.clone(), UtcMicros(1), @@ -273,10 +268,10 @@ fn sequential_writes_with_text( fn with_retrieval_alias( write: &AnchoredObservationWrite, - alias: tracedecay_domain::NativeAliasV2, + alias: tracedecay_domain::NativeAlias, ) -> AnchoredObservationWrite { let retained = write.retrieval_anchor(); - let anchor = RetrievalAnchorRecordV2::new(RetrievalAnchorRecordV2Parts { + let anchor = RetrievalAnchorRecord::new(RetrievalAnchorRecordParts { target: retained.target().clone(), owner: retained.owner().clone(), aliases: vec![alias], @@ -369,11 +364,6 @@ async fn n_persist_observation_calls_open_n_writer_transactions() { } let after = writer_txn_census(&runtime).await; assert_eq!(after.operations - before.operations, BATCH_SIZE as i64); - assert_eq!( - after.scopes - before.scopes, - BATCH_SIZE as i64, - "one persist_observation still opens one RuntimeTransactionScopeV1" - ); } #[tokio::test] @@ -420,11 +410,6 @@ async fn persist_observations_opens_one_writer_transaction_for_the_batch() { 1, "the bounded batch must be one admitted writer operation" ); - assert_eq!( - after.scopes - before.scopes, - 1, - "the bounded batch must share one RuntimeTransactionScopeV1" - ); } #[tokio::test] @@ -459,11 +444,13 @@ async fn intra_batch_exact_duplicate_is_hydrated_as_a_duplicate() { assert_eq!(outcomes[0].stored(), outcomes[1].stored()); let after = writer_txn_census(&runtime).await; assert_eq!(after.operations - before.operations, 1); - assert_eq!(after.scopes - before.scopes, 1); } -#[tokio::test] -async fn intra_batch_identity_rewrite_is_typed_and_commits_no_prefix() { +async fn open_profile_store() -> ( + TempDir, + HostAdmissionTestRuntimeV1, + crate::GlobalDbObservationStore, +) { let tmp = TempDir::new().unwrap(); let runtime = HostAdmissionTestRuntimeV1::profile(tmp.path()) .await @@ -472,51 +459,128 @@ async fn intra_batch_identity_rewrite_is_typed_and_commits_no_prefix() { .observation_store(HostAdmissionScope::Profile) .unwrap(); initialize_writer_authority(&runtime, &store).await; + (tmp, runtime, store) +} + +/// Durable effect of admitting writes in order: each settled outcome kind up +/// to the first error, that error, which observations are retained, and the +/// resulting source cursor of the first write's source. +#[derive(Debug, PartialEq, Eq)] +struct AdmissionEffect { + outcomes: Vec>, + error: Option, + retained: Vec, + cursor: Option, +} + +async fn admission_effect( + store: &crate::GlobalDbObservationStore, + writes: &[AnchoredObservationWrite], + outcomes: Vec>, + error: Option, +) -> AdmissionEffect { + let mut retained = Vec::with_capacity(writes.len()); + for write in writes { + retained.push( + store + .get_observation(write.observation().observation_id()) + .await + .unwrap() + .is_some(), + ); + } + let source = writes[0].observation(); + let cursor = store + .get_source_cursor(source.source(), source.scope()) + .await + .unwrap() + .map(|cursor| format!("{cursor:?}")); + AdmissionEffect { + outcomes, + error, + retained, + cursor, + } +} + +/// Admits `writes` as one batch and, on a separate fresh store, as sequential +/// single writes, and requires both to leave the same durable effect. Returns +/// the batch result for case-specific assertions. +async fn assert_batch_settles_like_sequential_writes( + writes: Vec, +) -> ( + Result, ObservationStoreError>, + AdmissionEffect, +) { + let (_batch_tmp, _batch_runtime, batch_store) = open_profile_store().await; + let batch = batch_store.persist_observations(writes.clone()).await; + let batch_effect = match &batch { + Ok(outcomes) => { + admission_effect( + &batch_store, + &writes, + outcomes + .iter() + .map(|outcome| std::mem::discriminant(outcome.outcome())) + .collect(), + None, + ) + .await + } + Err(error) => { + admission_effect( + &batch_store, + &writes, + Vec::new(), + Some(format!("{error:?}")), + ) + .await + } + }; + + let (_scalar_tmp, _scalar_runtime, scalar_store) = open_profile_store().await; + let mut outcomes = Vec::new(); + let mut error = None; + for write in writes.iter().cloned() { + match scalar_store.persist_observation(write).await { + Ok(outcome) => outcomes.push(std::mem::discriminant(&outcome)), + Err(scalar_error) => { + error = Some(format!("{scalar_error:?}")); + outcomes.clear(); + break; + } + } + } + let scalar_effect = admission_effect(&scalar_store, &writes, outcomes, error).await; + assert_eq!( + batch_effect, scalar_effect, + "a batch must settle exactly as the same writes admitted one at a time" + ); + (batch, batch_effect) +} + +#[tokio::test] +async fn intra_batch_identity_rewrite_settles_like_sequential_writes() { let session_id = SessionId::new("session.observation-batch.intra-rewrite").unwrap(); let first = sequential_writes(&session_id, 1) .pop() .expect("intra-batch retained observation"); let rewritten = colliding_rewrite(&session_id, Some(first.next_cursor().clone())); - let before = writer_txn_census(&runtime).await; - let error = store - .persist_observations(vec![first.clone(), rewritten]) - .await - .unwrap_err(); + let (batch, effect) = assert_batch_settles_like_sequential_writes(vec![first, rewritten]).await; assert!(matches!( - error, - ObservationStoreError::BatchRequiresScalarFallback { - cause: ObservationBatchFallbackCause::IntraBatchIdentityCollision, - } + batch, + Err(ObservationStoreError::ObservationCollision { .. }) )); - assert_eq!(writer_txn_census(&runtime).await, before); assert!( - store - .get_observation(first.observation().observation_id()) - .await - .unwrap() - .is_none() - ); - assert!( - store - .get_source_cursor(first.observation().source(), first.observation().scope()) - .await - .unwrap() - .is_none() + effect.retained[0], + "the committed predecessor stays durable" ); } #[tokio::test] -async fn intra_batch_receipt_collision_requests_typed_scalar_fallback() { - let tmp = TempDir::new().unwrap(); - let runtime = HostAdmissionTestRuntimeV1::profile(tmp.path()) - .await - .unwrap(); - let store = runtime - .observation_store(HostAdmissionScope::Profile) - .unwrap(); - initialize_writer_authority(&runtime, &store).await; +async fn intra_batch_receipt_collision_settles_like_sequential_writes() { let session_id = SessionId::new("session.observation-batch.intra-receipt").unwrap(); let mut writes = sequential_writes(&session_id, 2); let first = writes.remove(0); @@ -538,39 +602,16 @@ async fn intra_batch_receipt_collision_requests_typed_scalar_fallback() { ) .unwrap(); let conflicting = anchored_write(conflicting_observation, second.expected_cursor().cloned()); - let before = writer_txn_census(&runtime).await; - let error = store - .persist_observations(vec![first.clone(), conflicting]) - .await - .unwrap_err(); + let (batch, effect) = + assert_batch_settles_like_sequential_writes(vec![first, conflicting]).await; - assert!(matches!( - error, - ObservationStoreError::BatchRequiresScalarFallback { - cause: ObservationBatchFallbackCause::IntraBatchSanitizationReceiptCollision, - } - )); - assert_eq!(writer_txn_census(&runtime).await, before); - assert!( - store - .get_observation(first.observation().observation_id()) - .await - .unwrap() - .is_none() - ); + assert!(batch.is_err(), "a reused receipt id must not commit"); + assert_eq!(effect.retained, vec![true, false]); } #[tokio::test] -async fn intra_batch_alias_collision_is_typed_and_commits_no_prefix() { - let tmp = TempDir::new().unwrap(); - let runtime = HostAdmissionTestRuntimeV1::profile(tmp.path()) - .await - .unwrap(); - let store = runtime - .observation_store(HostAdmissionScope::Profile) - .unwrap(); - initialize_writer_authority(&runtime, &store).await; +async fn intra_batch_alias_collision_settles_like_sequential_writes() { let session_id = SessionId::new("session.observation-batch.intra-alias").unwrap(); let mut writes = sequential_writes(&session_id, 2); let first = writes.remove(0); @@ -581,27 +622,14 @@ async fn intra_batch_alias_collision_is_typed_and_commits_no_prefix() { .cloned() .expect("observation retrieval alias"); let second = with_retrieval_alias(&writes.remove(0), alias); - let before = writer_txn_census(&runtime).await; - let error = store - .persist_observations(vec![first.clone(), second]) - .await - .unwrap_err(); + let (batch, effect) = assert_batch_settles_like_sequential_writes(vec![first, second]).await; assert!(matches!( - error, - ObservationStoreError::BatchRequiresScalarFallback { - cause: ObservationBatchFallbackCause::IntraBatchRetrievalAnchorAliasCollision, - } + batch, + Err(ObservationStoreError::RetrievalAnchorAliasCollision { .. }) )); - assert_eq!(writer_txn_census(&runtime).await, before); - assert!( - store - .get_observation(first.observation().observation_id()) - .await - .unwrap() - .is_none() - ); + assert_eq!(effect.retained, vec![true, false]); } #[tokio::test(flavor = "current_thread")] @@ -649,7 +677,6 @@ async fn persist_observations_dispatches_one_runtime_command_independent_of_batc } let after = writer_txn_census(&runtime).await; assert_eq!(after.operations - before.operations, 1); - assert_eq!(after.scopes - before.scopes, 1); } } @@ -686,7 +713,6 @@ async fn persist_observations_partitions_large_windows_by_exact_admission_bytes( after.operations - before.operations, runtime_commands as i64 ); - assert_eq!(after.scopes - before.scopes, runtime_commands as i64); } #[tokio::test] diff --git a/crates/tracedecay-global-db/src/observation_collision_tests.rs b/crates/tracedecay-global-db/src/observation_collision_tests.rs index bb54331b92..5f041d7b27 100644 --- a/crates/tracedecay-global-db/src/observation_collision_tests.rs +++ b/crates/tracedecay-global-db/src/observation_collision_tests.rs @@ -64,6 +64,7 @@ use tracing::field::{Field, Visit}; use tracing::span::{Attributes, Id, Record}; use tracing::{Dispatch, Event, Metadata, Subscriber}; +use crate::schema_contract::invariants::SOURCE_CURSOR_ADVANCE_DELETE_GUARD_SQL; use crate::tests::harness::{HostAdmissionScope, HostAdmissionTestRuntimeV1}; use tracedecay_runtime_core::db::engine::params; use tracedecay_rusqlite_runtime::repository::observation_cursor_authority::COMMIT_SOURCE_CURSOR_SQL; @@ -339,7 +340,7 @@ fn anchored_write_with_cursor( "collision-test", ) .unwrap(); - let anchor = tracedecay_store::build_observation_retrieval_anchor_v2( + let anchor = tracedecay_store::build_observation_retrieval_anchor( write.observation(), projection_generation.clone(), UtcMicros(1), @@ -667,7 +668,7 @@ type ProjectedSessionRow = ( Option, ); -/// Projected `session_messages` row captured verbatim from a clean drain. +/// Projected message row captured verbatim from a clean drain. type ProjectedMessageRow = ( String, String, @@ -682,6 +683,7 @@ type ProjectedMessageRow = ( Option, Option, Option, + String, ); async fn provenance_rows(runtime: &HostAdmissionTestRuntimeV1) -> Vec { @@ -1327,9 +1329,10 @@ async fn drain_provenance_collision_with_existing_output_converges_to_durable_sk .unwrap(); transaction .execute( - "INSERT INTO session_messages - (provider, message_id, session_id, role, ordinal, text) - VALUES (?1, ?2, ?3, 'assistant', 0, 'stale era output')", + "INSERT INTO lcm_raw_messages + (provider, message_id, session_id, role, ordinal, content, content_hash, + storage_kind) + VALUES (?1, ?2, ?3, 'assistant', 0, 'stale era output', 'h', 'inline')", params![COLLISION_PROVIDER, "stale-era-output", session_id.as_str()], ) .await @@ -1398,7 +1401,7 @@ async fn drain_provenance_collision_with_existing_output_converges_to_durable_sk 0 ); assert_eq!(table_count(&runtime, "observation_workflow_facts").await, 0); - assert_eq!(table_count(&runtime, "session_messages").await, 1); + assert_eq!(table_count(&runtime, "lcm_raw_messages").await, 1); assert_eq!(table_count(&runtime, "sessions").await, 1); // The retained observation row itself stays immutable. let stored = store @@ -1442,7 +1445,7 @@ async fn drain_provenance_collision_with_existing_output_converges_to_durable_sk } /// A provenance binding that names a different output but has no backing -/// `session_messages` row is corrupt authority, not an existing-output +/// message row is corrupt authority, not an existing-output /// collision. It must stay a hard `ProvenanceCollision`: the checkpoint and /// queue remain in place and the ghost provenance row is not deleted. #[tokio::test] @@ -1499,7 +1502,7 @@ async fn drain_keeps_ghost_provenance_binding_a_hard_error() { transaction.commit().await.unwrap(); let stale_rows = provenance_rows(&runtime).await; assert_eq!(stale_rows.len(), 1); - assert_eq!(table_count(&runtime, "session_messages").await, 0); + assert_eq!(table_count(&runtime, "lcm_raw_messages").await, 0); let error = store .project_observation(observation.observation_id()) @@ -2605,14 +2608,14 @@ fn replace_vibe_eof(transcript: &Path, body: &str) { } async fn run_vibe_trigger( - source: &tracedecay_sessions::runtime::vibe::VibeSource, + source: &tracedecay_sessions::runtime::hosts::vibe::VibeSource, workspace: &Path, admission: &ProductionJsonlAdmission, ) -> Result< - tracedecay_sessions::runtime::vibe::VibeCaptureOutcome, + tracedecay_sessions::runtime::hosts::vibe::VibeCaptureOutcome, tracedecay_sessions::runtime::source::TranscriptIngestError, > { - tracedecay_sessions::runtime::vibe::capture_vibe_observations( + tracedecay_sessions::runtime::hosts::vibe::capture_vibe_observations( admission, source, workspace, @@ -2637,7 +2640,7 @@ async fn vibe_jsonl_eof_refusal_survives_retention_generation_and_restart_withou std::fs::create_dir_all(&workspace).unwrap(); let vibe_home = tmp.path().join("vibe-home"); let transcript = write_vibe_fixture(&vibe_home, &workspace, "original eof record"); - let source = tracedecay_sessions::runtime::vibe::VibeSource::with_vibe_home(&vibe_home) + let source = tracedecay_sessions::runtime::hosts::vibe::VibeSource::with_vibe_home(&vibe_home) .for_user_scope(Vec::new()); let runtime = HostAdmissionTestRuntimeV1::profile_with_session_capture_resources( tmp.path().join("profile"), @@ -2941,11 +2944,7 @@ async fn failed_coverage_advance_leaves_no_visible_refusal_marker() { .await .unwrap(); transaction - .execute_batch( - "CREATE TRIGGER source_cursor_advances_immutable_delete_v1 BEFORE DELETE ON \ - source_cursor_advances BEGIN SELECT RAISE(ABORT, \ - 'source cursor advances are immutable'); END", - ) + .execute_batch(SOURCE_CURSOR_ADVANCE_DELETE_GUARD_SQL) .await .unwrap(); transaction.commit().await.unwrap(); @@ -3316,10 +3315,11 @@ async fn already_positioned_cursor_replay_with_new_command_bytes_is_a_duplicate( .unwrap(); let generation = ObservationSourceGenerationV1::new(7).unwrap(); let cursor_at = |offset: u64, resume_fingerprint: u64| { - ObservationSourceCursorV1::new( + ObservationSourceCursorV1::for_ordering( source.clone(), ObservationScopeV1::Profile, generation, + ObservationOrderingDomainV1::FileBytes, offset, ) .unwrap() @@ -3541,9 +3541,9 @@ async fn drain_keeps_corrupt_provenance_with_matching_output_a_hard_error() { drop(rows); let mut rows = scratch_snapshot .query( - "SELECT provider, message_id, session_id, role, timestamp, ordinal, text, kind, - model, tool_names, source_path, source_offset, metadata_json - FROM session_messages", + "SELECT provider, message_id, session_id, role, timestamp, ordinal, content, kind, + model, tool_names, source_path, source_offset, metadata_json, content_hash + FROM lcm_raw_messages", (), ) .await @@ -3563,6 +3563,7 @@ async fn drain_keeps_corrupt_provenance_with_matching_output_a_hard_error() { message_row.get(10).unwrap(), message_row.get(11).unwrap(), message_row.get(12).unwrap(), + message_row.get(13).unwrap(), ); drop(rows); @@ -3625,10 +3626,11 @@ async fn drain_keeps_corrupt_provenance_with_matching_output_a_hard_error() { .unwrap(); transaction .execute( - "INSERT INTO session_messages - (provider, message_id, session_id, role, timestamp, ordinal, text, kind, model, - tool_names, source_path, source_offset, metadata_json) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13)", + "INSERT INTO lcm_raw_messages + (provider, message_id, session_id, role, timestamp, ordinal, content, kind, model, + tool_names, source_path, source_offset, metadata_json, content_hash, + storage_kind) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, 'inline')", params![ projected_message.0.as_str(), projected_message.1.as_str(), @@ -3643,6 +3645,7 @@ async fn drain_keeps_corrupt_provenance_with_matching_output_a_hard_error() { projected_message.10.as_deref(), projected_message.11, projected_message.12.as_deref(), + projected_message.13.as_str(), ], ) .await diff --git a/crates/tracedecay-global-db/src/observation_projection.rs b/crates/tracedecay-global-db/src/observation_projection.rs index 7d84eb109a..63bf7ab3ad 100644 --- a/crates/tracedecay-global-db/src/observation_projection.rs +++ b/crates/tracedecay-global-db/src/observation_projection.rs @@ -15,10 +15,7 @@ pub use rebuild::{ converge_projection_predecessor, project_observation, project_queued_observations, rebuild_projection, }; -pub(crate) use schema::{ - OBSERVATION_PROJECTION_BINDING_TRIGGERS_SQL, OBSERVATION_PROJECTION_PERFORMANCE_INDEX_SQL, - OBSERVATION_PROJECTION_SCHEMA_SQL, -}; +pub(crate) use schema::OBSERVATION_PROJECTION_PERFORMANCE_INDEX_SQL; pub(super) use schema::{ ensure_observation_projection_performance_indexes, ensure_observation_projection_schema, }; @@ -28,5 +25,6 @@ pub(crate) use state::rearm_queued_projection_retries; pub(super) use state::verify_projection_rows; pub(super) use state::{ ProjectionOutputAuthority, ProjectionRowsBatch, load_verified_session, read_output_authorities, - read_projection_rows_batch, resolve_output_projection, verify_projection_rows_from_records, + read_projection_rows_batch, resolve_output_projection, stored_output_digest, + stored_row_matches, verify_projection_rows_from_records, }; diff --git a/crates/tracedecay-global-db/src/observation_projection/apply.rs b/crates/tracedecay-global-db/src/observation_projection/apply.rs index e09d65ca8e..c92dbfa072 100644 --- a/crates/tracedecay-global-db/src/observation_projection/apply.rs +++ b/crates/tracedecay-global-db/src/observation_projection/apply.rs @@ -19,7 +19,8 @@ use tracedecay_sessions::runtime::store_access::find_preceding_codex_goal_respon use super::state::{ canonicalize_session_project_paths, read_message, read_output_state, read_session, - reconcile_session_rows_detailed, storage, storage_message, verify_output_state, + reconcile_session_rows_detailed, storage, storage_message, stored_output_digest, + verify_output_state, }; use super::transition::{ MessageTransition, MessageTransitionState, WorkflowFactTarget, WorkflowFactTransition, @@ -36,7 +37,7 @@ fn decode_canonical_envelope( pub(in super::super) fn derive_projection( observation: &DurableObservationV1, ) -> ProjectionStoreResult { - tracedecay_session_temporal_store::derive_projection(observation) + tracedecay_store::derive_canonical_projection(observation) } #[derive(Debug, Clone, PartialEq, Eq)] @@ -553,16 +554,16 @@ pub(super) async fn apply_session( } } -/// Aligns a provenance-owned raw twin onto the projection's session before +/// Aligns a provenance-owned message row onto the projection's session before /// the content upsert. /// /// The ingest upsert refuses a row whose `session_id` differs, so a drifted -/// twin blocks the rewrite that uniquely owned current provenance authorizes. +/// row blocks the rewrite that uniquely owned current provenance authorizes. /// `(provider, message_id)` is that ownership key; `session_id` is a field of -/// the twin, not a second owner. Callers reach this only after that ownership +/// the row, not a second owner. Callers reach this only after that ownership /// is already proven (an existing projected message, or released-rendering /// convergence). A first insert of an unowned identity must not adopt a -/// foreign twin and does not call this. +/// foreign row and does not call this. async fn adopt_owned_projection_raw_session( conn: &impl Executor, message: &SessionMessageRecord, @@ -693,44 +694,47 @@ async fn reconcile_projected_codex_goal_response( /// Replaces the stored output row with the one this binary derives. /// /// The projected message row is derived state, so every field but its identity -/// is rewritten from the record. Shared with released-rendering convergence, -/// which reaches the same row through a different admission path and must not -/// write it a second way. +/// is rewritten from the record. A projector that owns the output also owns the +/// row's session, so the row is first moved onto the projection's session. +/// A Hermes body belongs to the Hermes LCM turn authority: an existing Hermes +/// row keeps its body and takes only the projection's session columns. Shared +/// with released-rendering convergence, which reaches the same row through a +/// different admission path and must not write it a second way. pub(super) async fn supersede_projected_message( conn: &impl Executor, message: &SessionMessageRecord, -) -> ProjectionStoreResult { - conn.execute( - "UPDATE session_messages - SET session_id = ?3, role = ?4, timestamp = ?5, ordinal = ?6, - text = ?7, kind = ?8, model = ?9, tool_names = ?10, - source_path = ?11, source_offset = ?12, metadata_json = ?13 - WHERE provider = ?1 AND message_id = ?2", - params![ - message.provider.as_str(), - message.message_id.as_str(), - message.session_id.as_str(), - message.role.as_str(), - message.timestamp, - message.ordinal, - message.text.as_str(), - message.kind.as_deref(), - message.model.as_deref(), - message.tool_names.as_deref(), - message.source_path.as_deref(), - message.source_offset, - message.metadata_json.as_deref(), - ], - ) - .await - .map_err(|error| storage("supersede projected message", error)) +) -> ProjectionStoreResult<()> { + adopt_owned_projection_raw_session(conn, message).await?; + if message.provider == "hermes" { + let updated = conn + .execute( + "UPDATE lcm_raw_messages + SET role = ?3, timestamp = ?4, ordinal = ?5, kind = ?6, model = ?7, + tool_names = ?8, source_path = ?9, source_offset = ?10 + WHERE provider = ?1 AND message_id = ?2", + params![ + message.provider.as_str(), + message.message_id.as_str(), + message.role.as_str(), + message.timestamp, + message.ordinal, + message.kind.as_deref(), + message.model.as_deref(), + message.tool_names.as_deref(), + message.source_path.as_deref(), + message.source_offset, + ], + ) + .await + .map_err(|error| storage("supersede projected Hermes message", error))?; + if updated == 1 { + return Ok(()); + } + } + upsert_projected_raw_message(conn, message).await } -/// Removes one projected output row together with its LCM raw twin. -/// -/// The pair is the unit: a raw row without its message is unreachable and a -/// message without its raw twin is unhydratable, so every retirement path drops -/// both here rather than spelling the two deletes itself. +/// Removes one projected output row; every retirement path drops it here. async fn delete_projected_output( conn: &impl Executor, provider: &str, @@ -741,12 +745,6 @@ async fn delete_projected_output( params![provider, message_id], ) .await - .map_err(|error| storage("remove retired projection raw message", error))?; - conn.execute( - "DELETE FROM session_messages WHERE provider = ?1 AND message_id = ?2", - params![provider, message_id], - ) - .await .map(|_| ()) .map_err(|error| storage("remove retired projection message", error)) } @@ -771,8 +769,8 @@ pub(in super::super) enum ConvergedRendering { /// as a shipped rendering: provenance still carries the digest of the output /// this store holds, or it carries this binary's digest while the mutable row /// is still that shipped rendering. A row that matches neither is refused -/// before this write. The message row and its LCM raw twin are pure -/// derivations of the durable observation, so rewriting them loses nothing; +/// before this write. The message row is a pure derivation of the durable +/// observation, so rewriting it loses nothing; /// the digest is re-stamped last so an interrupted transaction leaves the /// released pairing intact. /// @@ -789,46 +787,15 @@ pub(in super::super) async fn converge_released_output_rendering( // the canonical insert authority; `apply_session` also preserves richer // compatible session metadata when the row already exists. apply_session(conn, projection.session()).await?; - let message = projection.message(); - if supersede_projected_message(conn, message).await? == 0 { - // The creator's provenance survived an interrupted write that never - // landed the message row. Update cannot restore a missing row. - conn.execute( - "INSERT INTO session_messages - (provider, message_id, session_id, role, timestamp, ordinal, text, kind, model, - tool_names, source_path, source_offset, metadata_json) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13)", - params![ - message.provider.as_str(), - message.message_id.as_str(), - message.session_id.as_str(), - message.role.as_str(), - message.timestamp, - message.ordinal, - message.text.as_str(), - message.kind.as_deref(), - message.model.as_deref(), - message.tool_names.as_deref(), - message.source_path.as_deref(), - message.source_offset, - message.metadata_json.as_deref(), - ], - ) - .await - .map_err(|error| storage("insert missing projected message", error))?; - } - if message.provider != "hermes" { - adopt_owned_projection_raw_session(conn, message).await?; - match upsert_projected_raw_message(conn, message).await { - Ok(()) => {} - Err(ProjectionStoreError::SanitizationRefused { - quarantined: true, .. - }) => { - retire_quarantined_projection(conn, projection).await?; - return Ok(ConvergedRendering::Quarantined); - } - Err(error) => return Err(error), + match supersede_projected_message(conn, projection.message()).await { + Ok(()) => {} + Err(ProjectionStoreError::SanitizationRefused { + quarantined: true, .. + }) => { + retire_quarantined_projection(conn, projection).await?; + return Ok(ConvergedRendering::Quarantined); } + Err(error) => return Err(error), } let provenance = projection.provenance(); conn.execute( @@ -839,7 +806,7 @@ pub(in super::super) async fn converge_released_output_rendering( provenance.projector_version(), provenance.observation_id().as_str(), projection.output_ordinal(), - projection.output_digest()?.as_str(), + stored_output_digest(projection)?.as_str(), ], ) .await @@ -856,7 +823,7 @@ pub(in super::super) async fn converge_released_output_rendering( /// `sanitization_refused` against the observation /// (`persist_projection_rejection_on_database`). So the released store reaches /// byte-identical state by removing the outputs this observation created with -/// their LCM raw twins, dropping its provenance and workflow rows, and writing +/// their message rows, dropping its provenance and workflow rows, and writing /// that disposition in their place. An output row a *different* observation /// created keeps its own provenance and is not this retirement's to remove, /// a fresh capture would not have created it either. @@ -946,7 +913,7 @@ async fn apply_rows( state.projector_owned, ) }); - let (transition, preserve_protected_payload) = message_transition( + let transition = message_transition( conn, sequence, projection, @@ -955,57 +922,10 @@ async fn apply_rows( ) .await?; match transition { - MessageTransition::Insert => { - conn.execute( - "INSERT INTO session_messages - (provider, message_id, session_id, role, timestamp, ordinal, text, kind, model, - tool_names, source_path, source_offset, metadata_json) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13)", - params![ - message.provider.as_str(), - message.message_id.as_str(), - message.session_id.as_str(), - message.role.as_str(), - message.timestamp, - message.ordinal, - message.text.as_str(), - message.kind.as_deref(), - message.model.as_deref(), - message.tool_names.as_deref(), - message.source_path.as_deref(), - message.source_offset, - message.metadata_json.as_deref(), - ], - ) - .await - .map_err(|error| storage("insert projected message", error))?; - } - MessageTransition::Supersede => { - supersede_projected_message(conn, message).await?; - } + MessageTransition::Insert => upsert_projected_raw_message(conn, message).await?, + MessageTransition::Supersede => supersede_projected_message(conn, message).await?, MessageTransition::Retain => {} } - let projected_message = match transition { - MessageTransition::Insert | MessageTransition::Supersede => message, - MessageTransition::Retain => { - existing - .as_ref() - .ok_or_else(|| ProjectionStoreError::OutputCollision { - provider: message.provider.clone(), - message_id: message.message_id.clone(), - })? - } - }; - if projected_message.provider != "hermes" && !preserve_protected_payload { - // Message-row presence is not projector ownership. An equal - // pre-existing row with no output state is retained without this - // projector ever having claimed the output, so its twin keeps the - // upsert's session guard and a disagreement stays a typed refusal. - if state.is_some_and(|state| state.projector_owned) { - adopt_owned_projection_raw_session(conn, projected_message).await?; - } - upsert_projected_raw_message(conn, projected_message).await?; - } Ok(transition == MessageTransition::Insert) } @@ -1236,7 +1156,7 @@ pub(super) async fn verify_provenance( provenance.receipt_id().to_string(), message.provider.clone(), message.message_id.clone(), - projection.output_digest()?.as_str().to_string(), + stored_output_digest(projection)?.as_str().to_string(), ); if actual == expected { Ok(()) @@ -1256,7 +1176,7 @@ async fn read_provenance_output_binding( .query( "SELECT provenance.output_provider, provenance.output_message_id, EXISTS( - SELECT 1 FROM session_messages AS message + SELECT 1 FROM lcm_raw_messages AS message WHERE message.provider = provenance.output_provider AND message.message_id = provenance.output_message_id ) @@ -1317,7 +1237,7 @@ async fn apply_provenance( provenance.receipt_id(), message.provider.as_str(), message.message_id.as_str(), - projection.output_digest()?.as_str(), + stored_output_digest(projection)?.as_str(), i64::from(message_created), ], ) diff --git a/crates/tracedecay-global-db/src/observation_projection/rebuild.rs b/crates/tracedecay-global-db/src/observation_projection/rebuild.rs index f82f230726..cd399ae6e8 100644 --- a/crates/tracedecay-global-db/src/observation_projection/rebuild.rs +++ b/crates/tracedecay-global-db/src/observation_projection/rebuild.rs @@ -1,9 +1,7 @@ use std::sync::atomic::{AtomicBool, Ordering}; use tracedecay_domain::{CanonicalObservationIdV1, DurableObservationV1}; -use tracedecay_lcm::retrieval_content::{ - derived_text_for_index, derived_text_for_snippet, projected_content_hash, -}; +use tracedecay_lcm::retrieval_content::projected_content_hash; use tracedecay_runtime_core::db::{ Database, engine::{Executor, QueryExecutor, Row, params}, @@ -23,10 +21,10 @@ use super::apply::{ }; use super::state::{ canonicalize_session_project_paths, consume_projection_queue_item, decode_observation_row, - decode_sequence, ensure_projection_output_state_cache, projection_retry_state, queued_sequence, - read_checkpoint, read_message, read_observation, read_session, - reaggregate_output_state_for_output, reconcile_session_rows_detailed, - schedule_projection_retry, storage, storage_message, write_checkpoint, + decode_sequence, ensure_projection_output_state_cache, projected_stored_message, + projection_retry_state, queued_sequence, read_checkpoint, read_message, read_observation, + read_session, reaggregate_output_state_for_output, reconcile_session_rows_detailed, + schedule_projection_retry, storage, storage_message, stored_output_digest, write_checkpoint, }; use super::transition::{ MessageTransition, MessageTransitionState, WorkflowFactTarget, WorkflowFactTransition, @@ -59,18 +57,19 @@ const SESSION_JSON_FIELDS: &[&str] = &[ "parent_tool_use_id", ]; -const MESSAGE_JSON_FIELDS: &[&str] = &[ +/// Staged message fields stored as same-named `lcm_raw_messages` columns. The +/// staged `text` and `metadata_json` are the row's body and protected +/// metadata, which a Hermes row keeps from the Hermes LCM turn authority. +const MESSAGE_SESSION_JSON_FIELDS: &[&str] = &[ "session_id", "role", "timestamp", "ordinal", - "text", "kind", "model", "tool_names", "source_path", "source_offset", - "metadata_json", ]; fn json_extract_expr(column: &str, field: &str) -> String { @@ -1342,18 +1341,14 @@ async fn write_staged_message( ) -> ProjectionStoreResult<()> { let json = encode_json(message, "encode staged projection message")?; let content_hash = projected_content_hash(&message.text); - let snippet = derived_text_for_snippet(&message.text); - let index = derived_text_for_index(&message.text); conn.execute( "INSERT INTO observation_projection_rebuild_messages ( projector_version, generation, output_provider, output_message_id, - message_json, content_hash, snippet_text, index_text - ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8) + message_json, content_hash + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6) ON CONFLICT(projector_version, generation, output_provider, output_message_id) DO UPDATE SET message_json = excluded.message_json, - content_hash = excluded.content_hash, - snippet_text = excluded.snippet_text, - index_text = excluded.index_text", + content_hash = excluded.content_hash", params![ SESSION_MESSAGE_PROJECTOR_VERSION, generation, @@ -1361,8 +1356,6 @@ async fn write_staged_message( message.message_id.as_str(), json.as_str(), content_hash.as_str(), - snippet.as_str(), - index.as_str(), ], ) .await @@ -1556,7 +1549,7 @@ async fn stage_rebuild_provenance( provenance.receipt_id(), message.provider.as_str(), message.message_id.as_str(), - projection.output_digest()?.as_str(), + stored_output_digest(projection)?.as_str(), i64::from(message_created), ], ) @@ -1606,7 +1599,7 @@ async fn stage_rebuild_provenance( provenance.receipt_id().to_owned(), message.provider.clone(), message.message_id.clone(), - projection.output_digest()?.as_str().to_owned(), + stored_output_digest(projection)?.as_str().to_owned(), ); if actual == expected { Ok(()) @@ -1637,7 +1630,7 @@ async fn stage_rebuild_message( state.projector_owned, ) }); - let (transition, _) = message_transition( + let transition = message_transition( conn, sequence, projection, @@ -1647,7 +1640,7 @@ async fn stage_rebuild_message( .await?; match transition { MessageTransition::Insert | MessageTransition::Supersede => { - write_staged_message(conn, generation, message).await?; + write_staged_message(conn, generation, &projected_stored_message(message)?).await?; } MessageTransition::Retain => {} } @@ -1865,7 +1858,7 @@ async fn clear_active_projection( .map_err(|error| storage("materialize cleared projection outputs", error))?; conn.execute( "DELETE FROM lcm_raw_messages - WHERE provider <> 'hermes' AND EXISTS ( + WHERE EXISTS ( SELECT 1 FROM temp.observation_projection_rebuild_cleared_outputs AS cleared WHERE cleared.output_provider = lcm_raw_messages.provider AND cleared.output_message_id = lcm_raw_messages.message_id @@ -1873,17 +1866,7 @@ async fn clear_active_projection( (), ) .await - .map_err(|error| storage("clear projected LCM raw rows for rebuild", error))?; - conn.execute( - "DELETE FROM session_messages WHERE EXISTS ( - SELECT 1 FROM temp.observation_projection_rebuild_cleared_outputs AS cleared - WHERE cleared.output_provider = session_messages.provider - AND cleared.output_message_id = session_messages.message_id - )", - (), - ) - .await - .map_err(|error| storage("clear projection message rows for rebuild", error))?; + .map_err(|error| storage("clear projected message rows for rebuild", error))?; conn.execute( "DELETE FROM observation_projection_provenance WHERE projector_version = ?1 AND NOT EXISTS ( @@ -2145,7 +2128,7 @@ async fn prepare_rebuild_output_activation( ) SELECT staged.output_provider, staged.output_message_id, EXISTS ( - SELECT 1 FROM session_messages AS active + SELECT 1 FROM lcm_raw_messages AS active WHERE active.provider = staged.output_provider AND active.message_id = staged.output_message_id ), @@ -2175,8 +2158,20 @@ async fn prepare_rebuild_output_activation( .await .map_err(|error| storage("materialize preexisting projection outputs", error))?; - let message_conflicts = - json_extract_neq_predicates("active", STAGED_MESSAGE_JSON_COLUMN, MESSAGE_JSON_FIELDS); + let message_conflicts = format!( + "{} + OR (active.provider <> 'hermes' AND ( + COALESCE(active.content, active.placeholder_text, '') IS NOT {} + OR active.metadata_json IS NOT {} + ))", + json_extract_neq_predicates( + "active", + STAGED_MESSAGE_JSON_COLUMN, + MESSAGE_SESSION_JSON_FIELDS + ), + json_extract_expr(STAGED_MESSAGE_JSON_COLUMN, "text"), + json_extract_expr(STAGED_MESSAGE_JSON_COLUMN, "metadata_json"), + ); let mut conflicts = conn .query( &format!( @@ -2185,7 +2180,7 @@ async fn prepare_rebuild_output_activation( JOIN temp.observation_projection_rebuild_preexisting_outputs AS ownership ON ownership.output_provider = staged.output_provider AND ownership.output_message_id = staged.output_message_id - LEFT JOIN session_messages AS active + LEFT JOIN lcm_raw_messages AS active ON active.provider = staged.output_provider AND active.message_id = staged.output_message_id WHERE staged.projector_version = ?1 AND staged.generation = ?2 @@ -2226,88 +2221,65 @@ async fn activate_rebuild_messages( conn: &impl Executor, generation: &str, ) -> ProjectionStoreResult<()> { - let message_extracts = json_extract_select_list(MESSAGE_JSON_COLUMN, MESSAGE_JSON_FIELDS); - conn.execute( - &format!( - "INSERT INTO session_messages ( - provider, message_id, session_id, role, timestamp, ordinal, text, kind, - model, tool_names, source_path, source_offset, metadata_json - ) - SELECT output_provider, output_message_id, - {message_extracts} - FROM observation_projection_rebuild_messages - WHERE projector_version = ?1 AND generation = ?2 - ON CONFLICT(provider, message_id) DO UPDATE SET - session_id = excluded.session_id, - role = excluded.role, - timestamp = excluded.timestamp, - ordinal = excluded.ordinal, - text = excluded.text, - kind = excluded.kind, - model = excluded.model, - tool_names = excluded.tool_names, - source_path = excluded.source_path, - source_offset = excluded.source_offset, - metadata_json = excluded.metadata_json - WHERE session_messages.session_id IS NOT excluded.session_id - OR session_messages.role IS NOT excluded.role - OR session_messages.timestamp IS NOT excluded.timestamp - OR session_messages.ordinal IS NOT excluded.ordinal - OR session_messages.text IS NOT excluded.text - OR session_messages.kind IS NOT excluded.kind - OR session_messages.model IS NOT excluded.model - OR session_messages.tool_names IS NOT excluded.tool_names - OR session_messages.source_path IS NOT excluded.source_path - OR session_messages.source_offset IS NOT excluded.source_offset - OR session_messages.metadata_json IS NOT excluded.metadata_json" - ), - params![SESSION_MESSAGE_PROJECTOR_VERSION, generation], - ) - .await - .map_err(|error| storage("activate rebuilt projection messages", error))?; - let lcm_session_id = json_extract_expr(MESSAGE_JSON_COLUMN, "session_id"); - let lcm_role = json_extract_expr(MESSAGE_JSON_COLUMN, "role"); - let lcm_ordinal = json_extract_expr(MESSAGE_JSON_COLUMN, "ordinal"); - let lcm_timestamp = json_extract_expr(MESSAGE_JSON_COLUMN, "timestamp"); - let lcm_text = json_extract_expr(MESSAGE_JSON_COLUMN, "text"); - let lcm_metadata = json_extract_expr(MESSAGE_JSON_COLUMN, "metadata_json"); + let session_columns = MESSAGE_SESSION_JSON_FIELDS.join(", "); + let session_extracts = + json_extract_select_list(MESSAGE_JSON_COLUMN, MESSAGE_SESSION_JSON_FIELDS); + let session_updates = MESSAGE_SESSION_JSON_FIELDS + .iter() + .map(|field| format!("{field} = excluded.{field}")) + .collect::>() + .join(",\n "); + let session_changed = MESSAGE_SESSION_JSON_FIELDS + .iter() + .map(|field| format!("lcm_raw_messages.{field} IS NOT excluded.{field}")) + .collect::>() + .join("\n OR "); + let body_update = |column: &str| { + format!( + "{column} = CASE WHEN lcm_raw_messages.provider = 'hermes' + THEN lcm_raw_messages.{column} ELSE excluded.{column} END" + ) + }; + let body_updates = [ + "content", + "content_hash", + "storage_kind", + "payload_ref", + "placeholder_text", + "metadata_json", + ] + .map(body_update) + .join(",\n "); + let text = json_extract_expr(MESSAGE_JSON_COLUMN, "text"); + let metadata = json_extract_expr(MESSAGE_JSON_COLUMN, "metadata_json"); + // A Hermes body belongs to the Hermes LCM turn authority, so an existing + // Hermes row takes only the rebuilt session columns. conn.execute( &format!( "INSERT INTO lcm_raw_messages ( - provider, message_id, session_id, role, ordinal, timestamp, content, - content_hash, storage_kind, payload_ref, snippet_text, index_text, - legacy_source, legacy_truncated, metadata_json + provider, message_id, {session_columns}, content, content_hash, storage_kind, + payload_ref, placeholder_text, metadata_json ) SELECT output_provider, output_message_id, - {lcm_session_id}, - {lcm_role}, - {lcm_ordinal}, - {lcm_timestamp}, - {lcm_text}, content_hash, 'inline', NULL, - snippet_text, index_text, 0, 0, - {lcm_metadata} + {session_extracts}, + {text}, content_hash, 'inline', NULL, NULL, {metadata} FROM observation_projection_rebuild_messages - WHERE projector_version = ?1 AND generation = ?2 AND output_provider <> 'hermes' + WHERE projector_version = ?1 AND generation = ?2 ON CONFLICT(provider, message_id) DO UPDATE SET - session_id = excluded.session_id, - role = excluded.role, - ordinal = excluded.ordinal, - timestamp = excluded.timestamp, - content = excluded.content, - content_hash = excluded.content_hash, - storage_kind = excluded.storage_kind, - payload_ref = excluded.payload_ref, - snippet_text = excluded.snippet_text, - index_text = excluded.index_text, - legacy_source = 0, - legacy_truncated = 0, - metadata_json = excluded.metadata_json" + {session_updates}, + {body_updates} + WHERE {session_changed} + OR (lcm_raw_messages.provider <> 'hermes' AND ( + COALESCE(lcm_raw_messages.content, lcm_raw_messages.placeholder_text, '') + IS NOT excluded.content + OR lcm_raw_messages.metadata_json IS NOT excluded.metadata_json + ))" ), params![SESSION_MESSAGE_PROJECTOR_VERSION, generation], ) .await .map(|_| ()) - .map_err(|error| storage("activate rebuilt projected LCM raw messages", error)) + .map_err(|error| storage("activate rebuilt projection messages", error)) } async fn activate_rebuild_provenance( diff --git a/crates/tracedecay-global-db/src/observation_projection/schema.rs b/crates/tracedecay-global-db/src/observation_projection/schema.rs index 31a2795267..32e14afbb4 100644 --- a/crates/tracedecay-global-db/src/observation_projection/schema.rs +++ b/crates/tracedecay-global-db/src/observation_projection/schema.rs @@ -3,11 +3,8 @@ use tracedecay_runtime_core::{ ports::registered_schema::RegisteredSchemaInstallationV1, }; -/// Final V4 observation-projection DDL. Shared with the scoped observation -/// reset in `crate::observation::reset`, which recreates these tables after -/// dropping a refused authority, so the installer and the reset can never -/// produce different shapes. -pub(crate) const OBSERVATION_PROJECTION_SCHEMA_SQL: &str = +/// Final V4 observation-projection DDL. +const OBSERVATION_PROJECTION_SCHEMA_SQL: &str = "CREATE TABLE IF NOT EXISTS observation_projection_provenance ( projector_version TEXT NOT NULL, observation_id TEXT NOT NULL, @@ -220,8 +217,6 @@ pub(crate) const OBSERVATION_PROJECTION_SCHEMA_SQL: &str = output_message_id TEXT NOT NULL, message_json TEXT NOT NULL CHECK(json_valid(message_json)), content_hash TEXT NOT NULL, - snippet_text TEXT NOT NULL, - index_text TEXT NOT NULL, PRIMARY KEY(projector_version, generation, output_provider, output_message_id), FOREIGN KEY(projector_version, generation) REFERENCES observation_projection_rebuilds(projector_version, generation) @@ -294,13 +289,12 @@ pub(crate) const OBSERVATION_PROJECTION_SCHEMA_SQL: &str = FOREIGN KEY(receipt_id) REFERENCES sanitization_receipts(receipt_id) );"; -/// Anchor-binding triggers, shared with the scoped observation reset like -/// [`OBSERVATION_PROJECTION_SCHEMA_SQL`]. -pub(crate) const OBSERVATION_PROJECTION_BINDING_TRIGGERS_SQL: &str = +/// Anchor-binding triggers of [`OBSERVATION_PROJECTION_SCHEMA_SQL`]. +const OBSERVATION_PROJECTION_BINDING_TRIGGERS_SQL: &str = include_str!("projection_binding_triggers.sql"); /// Historical-data indexes the schema contract requires on the projection -/// authority, shared with the scoped observation reset. +/// authority. pub(crate) const OBSERVATION_PROJECTION_PERFORMANCE_INDEX_SQL: &[&str] = &[ "CREATE INDEX IF NOT EXISTS idx_observation_projection_provenance_output ON observation_projection_provenance @@ -413,8 +407,6 @@ const CURRENT_REBUILD_MESSAGE_COLUMNS: &[&str] = &[ "output_message_id", "message_json", "content_hash", - "snippet_text", - "index_text", ]; const CURRENT_REBUILD_PROVIDER_USAGE_COLUMNS: &[&str] = &[ "projector_version", diff --git a/crates/tracedecay-global-db/src/observation_projection/source_transition.rs b/crates/tracedecay-global-db/src/observation_projection/source_transition.rs index 4c6a3177ea..62530fd221 100644 --- a/crates/tracedecay-global-db/src/observation_projection/source_transition.rs +++ b/crates/tracedecay-global-db/src/observation_projection/source_transition.rs @@ -2,7 +2,7 @@ use std::collections::BTreeSet; use tracedecay_domain::{ CanonicalObservationIdV1, ClineTranscriptStream, DurableObservationV1, FactOwnerV1, - ObservationIdentityMaterialV1, RetrievalAnchorRecordV2, RetrievalAnchorTargetV2, + ObservationIdentityMaterialV1, RetrievalAnchorRecord, RetrievalAnchorTarget, cline_task_native_observation_id, prove_cline_native_source_transition, }; use tracedecay_runtime_core::db::engine::{Executor, QueryExecutor, params}; @@ -386,7 +386,7 @@ pub(super) async fn settle_native_source_transition( async fn observation_anchor( conn: &impl QueryExecutor, observation: &DurableObservationV1, -) -> ProjectionStoreResult { +) -> ProjectionStoreResult { let mut rows = conn .query( "SELECT anchor.anchor_json FROM observation_retrieval_anchors AS binding @@ -404,11 +404,11 @@ async fn observation_anchor( let json: String = row .get(0) .map_err(|error| storage("read native source anchor", error))?; - let anchor: RetrievalAnchorRecordV2 = serde_json::from_str(&json) + let anchor: RetrievalAnchorRecord = serde_json::from_str(&json) .map_err(|error| storage("decode native source anchor", error))?; anchor.validate()?; if anchor.target() - != &RetrievalAnchorTargetV2::ExactObservation(observation.observation_id().clone()) + != &RetrievalAnchorTarget::ExactObservation(observation.observation_id().clone()) { return Err(ProjectionStoreError::ProvenanceCollision); } @@ -419,7 +419,7 @@ async fn read_transition_anchors( conn: &impl QueryExecutor, predecessor: &DurableObservationV1, successor: &DurableObservationV1, -) -> ProjectionStoreResult<(RetrievalAnchorRecordV2, RetrievalAnchorRecordV2)> { +) -> ProjectionStoreResult<(RetrievalAnchorRecord, RetrievalAnchorRecord)> { let old = observation_anchor(conn, predecessor).await?; let new = observation_anchor(conn, successor).await?; let old_auth = old.authorization(); diff --git a/crates/tracedecay-global-db/src/observation_projection/state.rs b/crates/tracedecay-global-db/src/observation_projection/state.rs index bc2b846466..2591ad2d63 100644 --- a/crates/tracedecay-global-db/src/observation_projection/state.rs +++ b/crates/tracedecay-global-db/src/observation_projection/state.rs @@ -1,15 +1,16 @@ use std::borrow::Cow; use std::collections::{BTreeSet, HashMap}; -use tracedecay_domain::{CanonicalObservationIdV1, DurableObservationV1}; +use tracedecay_domain::{CanonicalObservationIdV1, DurableObservationV1, PayloadDigestV1}; use tracedecay_store::{ - ObservationProjection, ProjectionCheckpoint, ProjectionStoreError, ProjectionStoreResult, - SESSION_MESSAGE_PROJECTOR_VERSION, SESSION_MESSAGE_PROJECTOR_VERSION_V4, - SessionMessageProjection, SessionMessageRecord, SessionRecord, + EDITED_FILES_KEY, ObservationProjection, ProjectionCheckpoint, ProjectionStoreError, + ProjectionStoreResult, SESSION_MESSAGE_PROJECTOR_VERSION, SESSION_MESSAGE_PROJECTOR_VERSION_V4, + SessionMessageProjection, SessionMessageRecord, SessionRecord, message_output_digest, }; -use tracedecay_lcm::LcmStorageKind; -use tracedecay_lcm::retrieval_content::{derived_text_for_index, projected_content_hash}; +use tracedecay_lcm::raw::stored_message_record_select_columns; +use tracedecay_lcm::retrieval_content::projected_content_hash; +use tracedecay_lcm::{LcmError, LcmStorageKind}; use tracedecay_runtime_core::db::engine::{Executor, QueryExecutor, Row, params}; use tracedecay_sessions::runtime::shared::durable_project_path_key; use tracedecay_sessions::runtime::store_access::{ @@ -290,13 +291,13 @@ pub(super) async fn read_message( provider: &str, message_id: &str, ) -> ProjectionStoreResult> { + let sql = format!( + "SELECT {} + FROM lcm_raw_messages AS message WHERE provider = ?1 AND message_id = ?2", + stored_message_record_select_columns("message") + ); let mut rows = conn - .query( - "SELECT provider, message_id, session_id, role, timestamp, ordinal, text, kind, - model, tool_names, source_path, source_offset, metadata_json - FROM session_messages WHERE provider = ?1 AND message_id = ?2", - params![provider, message_id], - ) + .query(&sql, params![provider, message_id]) .await .map_err(|error| storage("read projected message", error))?; let Some(row) = rows @@ -762,7 +763,8 @@ pub(in super::super) async fn verify_projection_rows_from_records( let message = projection.message(); let compatible = match actual_message { Some(actual) => { - actual == message || protected_message_rows_compatible(conn, actual, message).await? + stored_row_matches(actual, message)? + || protected_message_rows_compatible(conn, actual, message).await? } None => false, }; @@ -820,10 +822,10 @@ pub(in super::super) struct ProjectionOutputAuthority { pub(in super::super) canonical: DurableObservationV1, } -/// The LCM raw twin stored beside one projected message. Not part of the -/// output digest; current provenance still authorizes it because the twin is -/// derived from the same observation. -pub(in super::super) struct ProjectionRawTwin { +/// The storage columns of one projected message row. Not part of the output +/// digest; current provenance still authorizes them because they are derived +/// from the same observation. +pub(in super::super) struct ProjectionStorageColumns { pub(in super::super) session_id: String, pub(in super::super) storage_kind: String, pub(in super::super) content: String, @@ -835,7 +837,7 @@ pub(in super::super) struct ProjectionRawTwin { pub(in super::super) struct ProjectionRowsBatch { sessions: HashMap<(String, String), SessionRecord>, messages: HashMap<(String, String), SessionMessageRecord>, - raw_twins: HashMap<(String, String), ProjectionRawTwin>, + storage_columns: HashMap<(String, String), ProjectionStorageColumns>, } impl ProjectionRowsBatch { @@ -857,12 +859,12 @@ impl ProjectionRowsBatch { .get(&(provider.to_owned(), message_id.to_owned())) } - pub(in super::super) fn raw_twin( + pub(in super::super) fn storage_columns( &self, provider: &str, message_id: &str, - ) -> Option<&ProjectionRawTwin> { - self.raw_twins + ) -> Option<&ProjectionStorageColumns> { + self.storage_columns .get(&(provider.to_owned(), message_id.to_owned())) } } @@ -872,7 +874,7 @@ pub(in super::super) async fn read_projection_rows_batch( outputs: &BTreeSet<(String, String)>, ) -> ProjectionStoreResult { let mut messages = HashMap::with_capacity(outputs.len()); - let mut raw_twins = HashMap::with_capacity(outputs.len()); + let mut storage_columns = HashMap::with_capacity(outputs.len()); let requested_keys = outputs.iter().collect::>(); for chunk in requested_keys.chunks(OUTPUT_AUTHORITY_BATCH_KEYS) { let requested = serde_json::to_string( @@ -884,18 +886,17 @@ pub(in super::super) async fn read_projection_rows_batch( .collect::>(), ) .map_err(|error| storage("encode projected message request", error))?; + let sql = format!( + "SELECT {}, message.storage_kind, COALESCE(message.content, ''), + message.content_hash, message.snippet_text, message.index_text + FROM json_each(?1) AS requested + CROSS JOIN lcm_raw_messages AS message + WHERE message.provider = json_extract(requested.value, '$.provider') + AND message.message_id = json_extract(requested.value, '$.message_id')", + stored_message_record_select_columns("message") + ); let mut rows = conn - .query( - "SELECT message.provider, message.message_id, message.session_id, - message.role, message.timestamp, message.ordinal, message.text, - message.kind, message.model, message.tool_names, message.source_path, - message.source_offset, message.metadata_json - FROM json_each(?1) AS requested - CROSS JOIN session_messages AS message - WHERE message.provider = json_extract(requested.value, '$.provider') - AND message.message_id = json_extract(requested.value, '$.message_id')", - params![requested.as_str()], - ) + .query(&sql, params![requested.as_str()]) .await .map_err(|error| storage("read projected messages", error))?; while let Some(row) = rows @@ -905,59 +906,21 @@ pub(in super::super) async fn read_projection_rows_batch( { let message = message_record_from_row(&row, 0) .map_err(|error| storage("decode projected messages", error.source))?; - messages.insert( - (message.provider.clone(), message.message_id.clone()), - message, - ); - } - drop(rows); - let mut rows = conn - .query( - "SELECT raw.provider, raw.message_id, raw.session_id, raw.storage_kind, - COALESCE(raw.content, ''), raw.content_hash, raw.snippet_text, - raw.index_text - FROM json_each(?1) AS requested - CROSS JOIN lcm_raw_messages AS raw - WHERE raw.provider = json_extract(requested.value, '$.provider') - AND raw.message_id = json_extract(requested.value, '$.message_id')", - params![requested.as_str()], - ) - .await - .map_err(|error| storage("read projected raw twins", error))?; - while let Some(row) = rows - .next() - .await - .map_err(|error| storage("read projected raw twins", error))? - { - let provider = row - .get::(0) - .map_err(|error| storage("decode projected raw twins", error))?; - let message_id = row - .get::(1) - .map_err(|error| storage("decode projected raw twins", error))?; - raw_twins.insert( - (provider, message_id), - ProjectionRawTwin { - session_id: row - .get(2) - .map_err(|error| storage("decode projected raw twins", error))?, - storage_kind: row - .get(3) - .map_err(|error| storage("decode projected raw twins", error))?, - content: row - .get(4) - .map_err(|error| storage("decode projected raw twins", error))?, - content_hash: row - .get(5) - .map_err(|error| storage("decode projected raw twins", error))?, - snippet_text: row - .get(6) - .map_err(|error| storage("decode projected raw twins", error))?, - index_text: row - .get(7) - .map_err(|error| storage("decode projected raw twins", error))?, - }, - ); + let decode = |index: i32| { + row.get::(index) + .map_err(|error| storage("decode projected message storage", error)) + }; + let columns = ProjectionStorageColumns { + session_id: message.session_id.clone(), + storage_kind: decode(13)?, + content: decode(14)?, + content_hash: decode(15)?, + snippet_text: decode(16)?, + index_text: decode(17)?, + }; + let key = (message.provider.clone(), message.message_id.clone()); + storage_columns.insert(key.clone(), columns); + messages.insert(key, message); } } @@ -1009,7 +972,7 @@ pub(in super::super) async fn read_projection_rows_batch( Ok(ProjectionRowsBatch { sessions, messages, - raw_twins, + storage_columns, }) } @@ -1309,6 +1272,18 @@ fn reconcile_metadata( *actual_value = merged; } } + // Each record contributes its own file-edit entries; the session + // row keeps the union in first-seen order (re-applying a record + // adds nothing). + Some(serde_json::Value::Array(actual_files)) if key == EDITED_FILES_KEY => { + if let serde_json::Value::Array(expected_files) = expected_value { + for file in expected_files { + if !actual_files.contains(&file) { + actual_files.push(file); + } + } + } + } // Host ingest keeps the first annotation (`merge_session_metadata`). // A later observation's source, cwd, or hook label is not a different // session. Session identity stays on provider and session id. @@ -1349,127 +1324,98 @@ fn reconcile_usage( } } +/// The message row the projector stores for `message`: its sanitized body +/// and protected metadata beside the projection's session columns. +pub(in super::super) fn projected_stored_message( + message: &SessionMessageRecord, +) -> ProjectionStoreResult { + tracedecay_lcm::raw::projection_stored_message(message).map_err(|error| match error { + LcmError::SanitizationRefused { + reason, + quarantined, + } => ProjectionStoreError::SanitizationRefused { + reason, + quarantined, + }, + error => storage("derive projected message row", error), + }) +} + +/// Provenance digest of one projected output: the canonical output digest over +/// the row the projector stores, so a stored row is always digestible into the +/// provenance that pairs with it. +pub(in super::super) fn stored_output_digest( + projection: &SessionMessageProjection, +) -> ProjectionStoreResult { + message_output_digest( + projection.session(), + &projected_stored_message(projection.message())?, + projection.output_ordinal(), + ) +} + +/// Whether `actual` is exactly the row the projector stores for `message`. +/// +/// A Hermes body is written by the Hermes LCM turn authority rather than the +/// projector, so a Hermes row is compared on its session columns alone. A +/// body the sanitizer refuses has no projected row to match. +pub(in super::super) fn stored_row_matches( + actual: &SessionMessageRecord, + message: &SessionMessageRecord, +) -> ProjectionStoreResult { + if message.provider == "hermes" { + return Ok(same_session_columns(actual, message)); + } + match projected_stored_message(message) { + Ok(expected) => Ok(*actual == expected), + Err(ProjectionStoreError::SanitizationRefused { .. }) => Ok(false), + Err(error) => Err(error), + } +} + +fn same_session_columns(actual: &SessionMessageRecord, expected: &SessionMessageRecord) -> bool { + actual.provider == expected.provider + && actual.message_id == expected.message_id + && actual.session_id == expected.session_id + && actual.role == expected.role + && actual.timestamp == expected.timestamp + && actual.ordinal == expected.ordinal + && actual.kind == expected.kind + && actual.model == expected.model + && actual.tool_names == expected.tool_names + && actual.source_path == expected.source_path + && actual.source_offset == expected.source_offset +} + +/// Whether a row the transcript ingest wrote (externalized or with its own +/// protected metadata) is a protected rendering of `expected`. pub(super) async fn protected_message_rows_compatible( conn: &impl QueryExecutor, actual: &SessionMessageRecord, expected: &SessionMessageRecord, ) -> ProjectionStoreResult { - if actual == expected { - return Ok(false); - } - if actual.provider != expected.provider - || actual.message_id != expected.message_id - || actual.session_id != expected.session_id - || actual.role != expected.role - || actual.timestamp != expected.timestamp - || actual.ordinal != expected.ordinal - || actual.kind != expected.kind - || actual.model != expected.model - || actual.tool_names != expected.tool_names - || actual.source_path != expected.source_path - || actual.source_offset != expected.source_offset - { + if stored_row_matches(actual, expected)? || !same_session_columns(actual, expected) { return Ok(false); } - let Some(metadata) = actual - .metadata_json - .as_deref() - .and_then(|encoded| serde_json::from_str::(encoded).ok()) - else { - return Ok(false); - }; - let payload_ref = metadata - .get("payload_ref") - .and_then(serde_json::Value::as_str); - let expected_hash = projected_content_hash(&expected.text); - let external = metadata - .get("external_payload") - .and_then(serde_json::Value::as_bool) - == Some(true) - && metadata.get("sha256").and_then(serde_json::Value::as_str) - == Some(expected_hash.as_str()) - && payload_ref.is_some_and(|payload_ref| actual.text.contains(payload_ref)); - if !external { - // A twin that fails its own receipt is not a protected rendering of - // this projection. Callers treat that as an ordinary output mismatch - // and, when current provenance uniquely owns the output, rewrite it. - // A database fault is still a fault. - let raw = match tracedecay_lcm::schema::load_raw_message( - conn, - &actual.provider, - &actual.message_id, - ) - .await + // A row that fails its own receipt is not a protected rendering of this + // projection. Callers treat that as an ordinary output mismatch and, when + // current provenance uniquely owns the output, rewrite it. A database + // fault is still a fault. + let raw = + match tracedecay_lcm::schema::load_raw_message(conn, &actual.provider, &actual.message_id) + .await { - Ok(raw) => raw, - Err(tracedecay_lcm::LcmError::PayloadIntegrityMismatch) => return Ok(false), + Ok(Some(raw)) => raw, + Ok(None) | Err(LcmError::PayloadIntegrityMismatch) => return Ok(false), Err(error) => return Err(storage("read protected projection output", error)), }; - let Some(raw) = raw else { - return Ok(false); - }; - let Ok(protected) = tracedecay_privacy::sanitize_lcm_payload_text(&expected.text) else { - return Ok(false); - }; - return Ok(raw.storage_kind == LcmStorageKind::Inline - && raw.provider == actual.provider - && raw.message_id == actual.message_id - && raw.session_id == actual.session_id - && raw.role == actual.role - && raw.timestamp == actual.timestamp - && raw.ordinal == actual.ordinal - && raw.content == protected.sanitized_text() - && actual.text == derived_text_for_index(&raw.content) - && actual.metadata_json == raw.metadata_json); - } - let Some(payload_ref) = payload_ref else { - return Ok(false); - }; - let mut rows = conn - .query( - "SELECT CAST(COALESCE(content, '') AS TEXT), - CAST(COALESCE(content_hash, '') AS TEXT), - CAST(COALESCE(storage_kind, '') AS TEXT), - CAST(COALESCE(payload_ref, '') AS TEXT) - FROM lcm_raw_messages - WHERE provider = ?1 AND message_id = ?2 - LIMIT 2", - params![actual.provider.as_str(), actual.message_id.as_str()], - ) - .await - .map_err(|error| storage("read protected projection output", error))?; - let Some(row) = rows - .next() - .await - .map_err(|error| storage("read protected projection output", error))? - else { - return Ok(false); - }; - let compatible = row - .get::(0) - .map_err(|error| storage("read protected projection output", error))? - .is_empty() - && row - .get::(1) - .map_err(|error| storage("read protected projection output", error))? - == expected_hash - && row - .get::(2) - .map_err(|error| storage("read protected projection output", error))? - == "external" - && row - .get::(3) - .map_err(|error| storage("read protected projection output", error))? - == payload_ref; - if rows - .next() - .await - .map_err(|error| storage("read protected projection output", error))? - .is_some() - { - return Ok(false); - } - Ok(compatible) + Ok(match raw.storage_kind { + LcmStorageKind::Inline => tracedecay_privacy::sanitize_lcm_payload_text(&expected.text) + .is_ok_and(|protected| raw.content == protected.sanitized_text()), + LcmStorageKind::External => { + raw.content_hash == projected_content_hash(&expected.text) && raw.payload_ref.is_some() + } + }) } #[cfg(test)] @@ -1514,6 +1460,47 @@ mod reconcile_tests { } } + /// Each record's file-edit rollup joins the session row's array: the union + /// keeps every distinct edit, a re-applied record adds nothing, and the + /// other first-annotation-wins keys are untouched. + #[test] + fn edited_files_rollups_union_across_records() { + let with_metadata = |metadata: serde_json::Value| SessionRecord { + metadata_json: Some(metadata.to_string()), + ..record("/work/project") + }; + let first_edit = serde_json::json!({"path": "/work/a.rs", "edited_at_micros": 1_000}); + let second_edit = serde_json::json!({"path": "/work/b.rs", "edited_at_micros": 2_000}); + let later_a = serde_json::json!({"path": "/work/a.rs", "edited_at_micros": 3_000}); + let actual = with_metadata(serde_json::json!({ + "source": "claude_transcript", + "edited_files": [first_edit.clone()] + })); + let expected = with_metadata(serde_json::json!({ + "source": "other", + "edited_files": [second_edit.clone(), first_edit.clone(), later_a.clone()] + })); + + let merged = reconcile_session_rows_detailed(&actual, &expected).unwrap(); + let metadata: serde_json::Value = + serde_json::from_str(merged.metadata_json.as_deref().unwrap()).unwrap(); + assert_eq!( + metadata["edited_files"], + serde_json::json!([first_edit, second_edit, later_a]), + "distinct edits of one path are separate events, duplicates collapse" + ); + assert_eq!(metadata["source"], "claude_transcript"); + + let again = reconcile_session_rows_detailed(&merged, &expected).unwrap(); + assert_eq!(again.metadata_json, merged.metadata_json); + + let no_edits = with_metadata(serde_json::json!({"source": "other"})); + let merged = reconcile_session_rows_detailed(&no_edits, &actual).unwrap(); + let metadata: serde_json::Value = + serde_json::from_str(merged.metadata_json.as_deref().unwrap()).unwrap(); + assert_eq!(metadata["edited_files"].as_array().unwrap().len(), 1); + } + #[cfg(unix)] #[test] fn symlinked_family_roots_reconcile_after_ingest_normalization() { diff --git a/crates/tracedecay-global-db/src/observation_projection/transition.rs b/crates/tracedecay-global-db/src/observation_projection/transition.rs index 6f55948966..6e1189a3ab 100644 --- a/crates/tracedecay-global-db/src/observation_projection/transition.rs +++ b/crates/tracedecay-global-db/src/observation_projection/transition.rs @@ -8,7 +8,7 @@ use tracedecay_runtime_core::db::engine::{Executor, QueryExecutor, params}; use super::state::{ has_other_projector_output_owner, protected_message_rows_compatible, same_projection_lineage, - storage, + storage, stored_row_matches, }; const LIVE_WORKFLOW_FACT_INSERT: &str = "WITH ignored_generation(generation) AS (VALUES (?2)) @@ -216,36 +216,40 @@ pub(super) async fn message_transition( projection: &SessionMessageProjection, existing: Option<&SessionMessageRecord>, state: Option, -) -> ProjectionStoreResult<(MessageTransition, bool)> { - let protected_compatibility = match existing { - Some(actual) => { - protected_message_rows_compatible(conn, actual, projection.message()).await? - } - None => false, - }; - let classified_existing = if protected_compatibility { - Some(projection.message()) - } else { - existing +) -> ProjectionStoreResult { + let message = projection.message(); + // A Hermes body is written by the Hermes LCM turn authority. A row it + // wrote before any projector claimed the output takes the projection's + // session columns without the projector claiming its body. + if message.provider == "hermes" && existing.is_some() && state.is_none() { + return Ok(MessageTransition::Supersede); + } + let existing_matches = match existing { + Some(actual) => Some( + stored_row_matches(actual, message)? + || protected_message_rows_compatible(conn, actual, message).await?, + ), + None => None, }; - let transition = - classify_message_transition(sequence, projection.message(), classified_existing, state)?; + let transition = classify_message_transition(sequence, message, existing_matches, state)?; if transition == MessageTransition::Supersede && has_other_projector_output_owner(conn, projection).await? { - return Err(output_collision(projection.message())); + return Err(output_collision(message)); } - Ok((transition, protected_compatibility)) + Ok(transition) } +/// `existing_matches` is `None` when no row is stored, otherwise whether the +/// stored row is the projector's rendering of `message`. fn classify_message_transition( sequence: u64, message: &SessionMessageRecord, - existing: Option<&SessionMessageRecord>, + existing_matches: Option, state: Option, ) -> ProjectionStoreResult { - match (existing, state) { - (Some(actual), Some(state)) => { + match (existing_matches, state) { + (Some(matches), Some(state)) => { if !state.same_lineage { return Err(output_collision(message)); } @@ -253,20 +257,20 @@ fn classify_message_transition( return Ok(MessageTransition::Retain); } if state.same_generation { - return if actual == message { + return if matches { Ok(MessageTransition::Retain) } else { Err(output_collision(message)) }; } - if !state.projector_owned || actual == message { + if !state.projector_owned || matches { Ok(MessageTransition::Retain) } else { Ok(MessageTransition::Supersede) } } - (Some(actual), None) if actual == message => Ok(MessageTransition::Retain), - (Some(_), None) | (None, Some(_)) => Err(output_collision(message)), + (Some(true), None) => Ok(MessageTransition::Retain), + (Some(false), None) | (None, Some(_)) => Err(output_collision(message)), (None, None) => Ok(MessageTransition::Insert), } } @@ -322,12 +326,7 @@ mod tests { MessageTransition::Insert ); assert!(matches!( - classify_message_transition( - 1, - &expected, - Some(&expected), - Some(state(1, true, true, true)) - ), + classify_message_transition(1, &expected, Some(true), Some(state(1, true, true, true))), Ok(MessageTransition::Retain) )); assert!(matches!( @@ -338,13 +337,12 @@ mod tests { #[test] fn same_generation_requires_compatible_rows() { - let actual = message("actual"); let expected = message("expected"); assert!(matches!( classify_message_transition( 2, &expected, - Some(&actual), + Some(false), Some(state(1, true, true, true)) ), Err(ProjectionStoreError::OutputCollision { .. }) @@ -353,7 +351,7 @@ mod tests { classify_message_transition( 2, &expected, - Some(&expected), + Some(true), Some(state(1, false, true, true)) ), Err(ProjectionStoreError::OutputCollision { .. }) @@ -362,13 +360,12 @@ mod tests { #[test] fn rollover_only_supersedes_exclusively_owned_rows() { - let actual = message("actual"); let expected = message("expected"); assert_eq!( classify_message_transition( 2, &expected, - Some(&actual), + Some(false), Some(state(1, true, false, true)) ) .unwrap(), @@ -378,7 +375,7 @@ mod tests { classify_message_transition( 2, &expected, - Some(&actual), + Some(false), Some(state(1, true, false, false)) ) .unwrap(), @@ -388,7 +385,7 @@ mod tests { classify_message_transition( 0, &expected, - Some(&actual), + Some(false), Some(state(1, true, false, true)) ) .unwrap(), diff --git a/crates/tracedecay-global-db/src/registered_lcm.rs b/crates/tracedecay-global-db/src/registered_lcm.rs index 982d5484cf..bdb7a4c04d 100644 --- a/crates/tracedecay-global-db/src/registered_lcm.rs +++ b/crates/tracedecay-global-db/src/registered_lcm.rs @@ -13,9 +13,11 @@ use tracedecay_lcm::{ types::{LcmImmutableSummaryPublication, LcmSummaryPublicationReceipt}, }; use tracedecay_sessions::runtime::{SessionMessageRecord, SessionStoreAccess}; -use tracedecay_temporal_query::ports::{ExecutionControl, TemporalPortError}; +use tracedecay_temporal_query::execution::ExecutionControl; +use tracedecay_temporal_query::ports::TemporalPortError; use super::RegisteredGlobalDb; +use tracedecay_session_temporal_store::SessionTemporalAccess; use tracedecay_session_temporal_store::operations as session_temporal_operations; use tracedecay_session_temporal_store::seed_session_relation_projection; use tracedecay_session_temporal_store::store::execution_control_graph_cancellation; @@ -94,10 +96,9 @@ impl RegisteredGlobalDb { #[hotpath::measure(future = true, label = "global_db.registered.lcm.grep")] pub async fn lcm_grep(&self, request: LcmGrepRequest) -> Result { - let git_scope_session_ids = - tracedecay_session_temporal_store::SessionTemporalAccess::new(self) - .git_scope_session_ids(&request.git_filter) - .map_err(|error| LcmError::Db(error.to_string()))?; + let git_scope_session_ids = SessionTemporalAccess::new(self) + .git_scope_session_ids(&request.git_filter) + .map_err(|error| LcmError::Db(error.to_string()))?; SessionStoreAccess::new(self) .lcm_grep(request, git_scope_session_ids.as_deref()) .await @@ -210,16 +211,17 @@ impl RegisteredGlobalDb { before_commit()?; transaction.commit().await?; check_execution(control)?; - self.apply_active_session_relation_projection( - &session_id, - execution_control_graph_cancellation(control), - ) - .await - .map_err(|error| { - LcmError::Db(format!( - "apply native LCM summary relation projection: {error}" - )) - })?; + SessionTemporalAccess::new(self) + .apply_active_session_relation_projection( + &session_id, + execution_control_graph_cancellation(control), + ) + .await + .map_err(|error| { + LcmError::Db(format!( + "apply native LCM summary relation projection: {error}" + )) + })?; check_execution(control)?; Ok(receipt) } @@ -297,14 +299,15 @@ impl RegisteredGlobalDb { payload_rollback.disarm(); if !response.summary_nodes.is_empty() { check_execution(control)?; - self.apply_active_session_relation_projection( - &session_id, - execution_control_graph_cancellation(control), - ) - .await - .map_err(|error| { - LcmError::Db(format!("apply native LCM relation projection: {error}")) - })?; + SessionTemporalAccess::new(self) + .apply_active_session_relation_projection( + &session_id, + execution_control_graph_cancellation(control), + ) + .await + .map_err(|error| { + LcmError::Db(format!("apply native LCM relation projection: {error}")) + })?; response.relation_projection_status = LcmRelationProjectionStatus::Applied; check_execution(control)?; } diff --git a/crates/tracedecay-global-db/src/registered_lcm_privacy.rs b/crates/tracedecay-global-db/src/registered_lcm_privacy.rs index cc1bb3c6e0..2a48ec819f 100644 --- a/crates/tracedecay-global-db/src/registered_lcm_privacy.rs +++ b/crates/tracedecay-global-db/src/registered_lcm_privacy.rs @@ -89,7 +89,7 @@ pub struct LcmPrivacyRescanReceiptV1 { pub unavailable_payload_rows: u64, } -/// One at-rest raw row joined with its optional `session_messages` twin. +/// One at-rest message row. struct RescanRow { store_id: i64, provider: String, @@ -102,11 +102,11 @@ struct RescanRow { storage_kind: LcmStorageKind, payload_ref: Option, metadata_json: Option, - projection_kind: Option, - projection_model: Option, - projection_tool_names: Option, - projection_source_path: Option, - projection_source_offset: Option, + kind: Option, + model: Option, + tool_names: Option, + source_path: Option, + source_offset: Option, } /// The rescan input recovered from one row's at-rest bytes. @@ -241,17 +241,12 @@ impl RegisteredGlobalDb { let snapshot = self.lcm_read_snapshot().await?; let mut rows = snapshot .query( - "SELECT raw.store_id, raw.provider, raw.message_id, raw.session_id, - raw.role, raw.ordinal, raw.timestamp, raw.content, - raw.storage_kind, raw.payload_ref, raw.metadata_json, - message.kind, message.model, message.tool_names, - message.source_path, message.source_offset - FROM lcm_raw_messages AS raw - LEFT JOIN session_messages AS message - ON message.provider = raw.provider - AND message.message_id = raw.message_id - WHERE raw.store_id > ?1 - ORDER BY raw.store_id + "SELECT store_id, provider, message_id, session_id, role, ordinal, timestamp, + content, storage_kind, payload_ref, metadata_json, kind, model, + tool_names, source_path, source_offset + FROM lcm_raw_messages + WHERE store_id > ?1 + ORDER BY store_id LIMIT ?2", params![after_store_id, RESCAN_PAGE_LIMIT], ) @@ -274,11 +269,11 @@ impl RegisteredGlobalDb { storage_kind, payload_ref: row.get(9)?, metadata_json: row.get(10)?, - projection_kind: row.get(11)?, - projection_model: row.get(12)?, - projection_tool_names: row.get(13)?, - projection_source_path: row.get(14)?, - projection_source_offset: row.get(15)?, + kind: row.get(11)?, + model: row.get(12)?, + tool_names: row.get(13)?, + source_path: row.get(14)?, + source_offset: row.get(15)?, }); } Ok(page) @@ -342,7 +337,7 @@ impl RegisteredGlobalDb { } /// Re-ingests one dirty row through the canonical staging and commit - /// path, resynchronizes its projection twin, and tombstones a replaced + /// path and tombstones a replaced /// external payload so the superseded bytes leave the disk. #[hotpath::skip] async fn remediate_row( @@ -361,11 +356,11 @@ impl RegisteredGlobalDb { timestamp: row.timestamp, ordinal: row.ordinal, text, - kind: row.projection_kind.clone(), - model: row.projection_model.clone(), - tool_names: row.projection_tool_names.clone(), - source_path: row.projection_source_path.clone(), - source_offset: row.projection_source_offset, + kind: row.kind.clone(), + model: row.model.clone(), + tool_names: row.tool_names.clone(), + source_path: row.source_path.clone(), + source_offset: row.source_offset, metadata_json: provider_metadata_json, }; let mut payload_rollback = @@ -379,19 +374,7 @@ impl RegisteredGlobalDb { .begin_write_transaction() .await .map_err(|error| LcmError::Db(error.to_string()))?; - let upsert = raw::commit_staged_raw_message(&transaction, &record, staged).await?; - transaction - .execute( - "UPDATE session_messages SET text = ?3, metadata_json = ?4 - WHERE provider = ?1 AND message_id = ?2", - params![ - record.provider.as_str(), - record.message_id.as_str(), - upsert.projection_text.as_str(), - upsert.projection_metadata_json.as_deref(), - ], - ) - .await?; + raw::commit_staged_raw_message(&transaction, &record, staged).await?; // An external row remediates only when its body changed, and payload // refs are content-addressed, so the re-ingest can never reuse the // replaced ref: the superseded payload is always safe to delete. diff --git a/crates/tracedecay-global-db/src/registered_legacy_relations.rs b/crates/tracedecay-global-db/src/registered_legacy_relations.rs deleted file mode 100644 index f962504697..0000000000 --- a/crates/tracedecay-global-db/src/registered_legacy_relations.rs +++ /dev/null @@ -1,162 +0,0 @@ -use tracedecay_domain::errors::TraceDecayError; -use tracedecay_runtime_core::db::engine::QueryExecutor; -use tracedecay_store::{StoreRuntimeBindingV1, StoreShardScopeV1}; - -const LEGACY_SESSION_RELATION_TABLES: [&str; 5] = [ - "session_summary_sources", - "session_summary_successors", - "session_logical_copy_edges", - "session_thread_hierarchy_edges", - "session_agent_hierarchy_edges", -]; - -pub(crate) async fn reject_legacy_session_relation_shape( - connection: &impl QueryExecutor, - binding: &StoreRuntimeBindingV1, -) -> tracedecay_domain::errors::Result<()> { - if !matches!( - &binding.shard_id.scope, - StoreShardScopeV1::ProjectSessions { .. } | StoreShardScopeV1::ProfileSessions - ) { - return Ok(()); - } - for table in LEGACY_SESSION_RELATION_TABLES { - let mut rows = connection - .query( - "SELECT 1 - FROM sqlite_master - WHERE type = 'table' AND name = ?1 - LIMIT 1", - [table], - ) - .await - .map_err(inspection_error)?; - if rows.next().await.map_err(inspection_error)?.is_some() { - return Err(TraceDecayError::reset_required( - "registered session relation store", - format!( - "registered session store contains retired relational authority \ - '{table}'; reset this session shard before daemon admission" - ), - )); - } - } - Ok(()) -} - -fn inspection_error(error: impl std::fmt::Display) -> TraceDecayError { - TraceDecayError::Database { - operation: "inspect legacy session relation shape".to_owned(), - message: error.to_string(), - } -} - -#[cfg(test)] -mod tests { - use std::fs; - use std::path::Path; - use std::sync::Arc; - - use tempfile::TempDir; - use tracedecay_runtime_core::RuntimeOperationTaskOwnerV1; - use tracedecay_runtime_core::db::{ - Database, DatabaseAuthority, TestDatabaseRuntimeMode, TestDatabaseRuntimeScope, - enter_daemon_database_scope, - }; - - use super::TraceDecayError; - use crate::RegisteredGlobalDbOwnerV1; - - fn schema_snapshot(path: &Path) -> Vec<(String, String, String)> { - let connection = rusqlite::Connection::open(path).expect("open schema snapshot"); - let mut statement = connection - .prepare( - "SELECT type, name, COALESCE(sql, '') - FROM sqlite_master - ORDER BY type, name", - ) - .expect("prepare schema snapshot"); - statement - .query_map([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?))) - .expect("query schema snapshot") - .collect::, _>>() - .expect("read schema snapshot") - } - - #[tokio::test] - async fn installation_requires_typed_reset_without_mutating_legacy_profile_shape() { - for daemon_attach in [false, true] { - crate::register_registered_schema_installer(); - let directory = TempDir::new().expect("temporary profile"); - let database_path = directory.path().join("sessions.db"); - { - let connection = - rusqlite::Connection::open(&database_path).expect("create legacy profile"); - connection - .execute_batch( - "CREATE TABLE session_summary_sources ( - summary_id TEXT NOT NULL, - source_ordinal INTEGER NOT NULL - ); - INSERT INTO session_summary_sources VALUES ('legacy-summary', 0);", - ) - .expect("legacy relation shape"); - } - let _scope = - enter_daemon_database_scope(directory.path(), 1, "legacy-relation-shape-test") - .expect("database scope"); - let authority = DatabaseAuthority::acquire_test( - &database_path, - "legacy relation shape test runtime", - ) - .expect("database authority"); - let fixture = Database::publish_registered_test_runtime_with_retirement_control( - &database_path, - &authority, - TestDatabaseRuntimeMode::Existing, - TestDatabaseRuntimeScope::ProfileSessions, - ) - .await - .expect("existing registered runtime"); - let (database_owner, runtime, _retirement) = fixture.into_parts(); - drop(runtime); - // The runtime open itself configures the journal mode, so capture - // the untouched shape after publication and before registered - // attach admission rejects the retired relation authority. - let before_schema = schema_snapshot(&database_path); - let before_bytes = fs::read(&database_path).expect("legacy database bytes"); - let before_len = fs::metadata(&database_path) - .expect("legacy database metadata") - .len(); - let error = if daemon_attach { - match RegisteredGlobalDbOwnerV1::admit_and_attach_for_daemon( - database_owner, - Arc::new(RuntimeOperationTaskOwnerV1::new()), - ) - .await - { - Ok(_) => panic!("daemon attach must reject legacy relation shape"), - Err(error) => error, - } - } else { - match RegisteredGlobalDbOwnerV1::admit_and_attach(database_owner).await { - Ok(_) => panic!("attach must reject legacy relation shape"), - Err(error) => error, - } - }; - - assert!(matches!(error, TraceDecayError::ResetRequired { .. })); - assert_eq!(schema_snapshot(&database_path), before_schema); - assert_eq!( - fs::metadata(&database_path) - .expect("post-refusal database metadata") - .len(), - before_len - ); - assert_eq!( - fs::read(&database_path).expect("post-refusal database bytes"), - before_bytes - ); - } - } -} diff --git a/crates/tracedecay-global-db/src/registered_provider_usage.rs b/crates/tracedecay-global-db/src/registered_provider_usage.rs index 711edebb91..277d54d721 100644 --- a/crates/tracedecay-global-db/src/registered_provider_usage.rs +++ b/crates/tracedecay-global-db/src/registered_provider_usage.rs @@ -400,8 +400,14 @@ mod tests { payload, ) .unwrap(); - let cursor = - ObservationSourceCursorV1::new(source, scope, generation, range.end()).unwrap(); + let cursor = ObservationSourceCursorV1::for_ordering( + source, + scope, + generation, + ObservationOrderingDomainV1::FileBytes, + range.end(), + ) + .unwrap(); (observation, cursor) } diff --git a/crates/tracedecay-global-db/src/registered_sessions.rs b/crates/tracedecay-global-db/src/registered_sessions.rs index 235fe63300..61b2e1fdbf 100644 --- a/crates/tracedecay-global-db/src/registered_sessions.rs +++ b/crates/tracedecay-global-db/src/registered_sessions.rs @@ -208,7 +208,7 @@ mod tests { ); } - async fn insert_interleaved_session_messages(database: &RegisteredGlobalDb, rows: i64) { + async fn insert_interleaved_messages(database: &RegisteredGlobalDb, rows: i64) { let final_value = rows - 1; let transaction = database.begin_write_transaction().await.unwrap(); transaction @@ -218,9 +218,10 @@ mod tests { UNION ALL SELECT value + 1 FROM rows WHERE value < {final_value} ) - INSERT INTO session_messages( - provider, message_id, session_id, role, timestamp, ordinal, text, - kind, model, tool_names, source_path, source_offset, metadata_json + INSERT INTO lcm_raw_messages( + provider, message_id, session_id, role, timestamp, ordinal, content, + kind, model, tool_names, source_path, source_offset, metadata_json, + content_hash, storage_kind ) SELECT 'claude', @@ -229,7 +230,8 @@ mod tests { 'assistant', 1700000000 + ({final_value} - value / 8), CASE WHEN value % 2 = 0 THEN value / 4 ELSE value / 2 END, - 'payload', NULL, NULL, printf('tool-%04d', {final_value} - value), NULL, NULL, NULL + 'payload', NULL, NULL, printf('tool-%04d', {final_value} - value), NULL, NULL, NULL, + 'hash', 'inline' FROM rows;" )) .await @@ -382,7 +384,7 @@ mod tests { .await ); - insert_interleaved_session_messages(database, 2_048).await; + insert_interleaved_messages(database, 2_048).await; let activities = database .session_messages_after("claude", "target", 1_700_000_000, 512) @@ -413,7 +415,7 @@ mod tests { ], ) .await; - assert_scoped_index_plan(&activity_plan, "idx_session_messages_session_activity_v2"); + assert_scoped_index_plan(&activity_plan, "idx_lcm_raw_session_activity"); // The index orders the scan; `metadata_json` is fetched from the table // for the bounded page rather than duplicated into the index. assert!( diff --git a/crates/tracedecay-global-db/src/registry_maintenance.rs b/crates/tracedecay-global-db/src/registry_maintenance.rs index a0665d2685..ad17b521ed 100644 --- a/crates/tracedecay-global-db/src/registry_maintenance.rs +++ b/crates/tracedecay-global-db/src/registry_maintenance.rs @@ -10,9 +10,8 @@ use crate::{ }; use tracedecay_runtime_core::branch_meta; use tracedecay_runtime_core::storage::{ - STORE_MANIFEST_FILENAME, STORE_MANIFEST_SCHEMA_VERSION, StorageMode, StoreKind, - read_legacy_enrollment_marker, read_repository_identity_marker, read_store_manifest, - validate_project_id, + STORE_MANIFEST_FILENAME, STORE_MANIFEST_SCHEMA_VERSION, StoreKind, + read_repository_identity_marker, read_store_manifest, validate_project_id, }; mod lifecycle; diff --git a/crates/tracedecay-global-db/src/registry_maintenance/orphan.rs b/crates/tracedecay-global-db/src/registry_maintenance/orphan.rs index 82fef14cab..46763067cb 100644 --- a/crates/tracedecay-global-db/src/registry_maintenance/orphan.rs +++ b/crates/tracedecay-global-db/src/registry_maintenance/orphan.rs @@ -228,48 +228,19 @@ fn classify_project_root( ); } }; - // Legacy read-only evidence: markers written before the working-tree - // cutover still vouch for orphan re-adoption; nothing rewrites them. - let enrollment = match read_legacy_enrollment_marker(&canonical_root) { - Ok(marker) => marker.map(|marker| marker.project_id), - Err(error) => { - return ( - RegistryOrphanRelinkStatus::Blocked, - Some(format!("could not validate enrollment marker: {error}")), - canonical_root, - ); - } - }; - let identity = match (repository_identity.as_deref(), enrollment.as_deref()) { - (Some(repository), Some(enrolled)) if repository != enrolled => ( - RegistryOrphanRelinkStatus::Blocked, - Some(format!( - "repository identity project '{repository}' disagrees with enrollment project '{enrolled}'" - )), - ), - (Some(repository), Some(_)) if repository == project_id => { - (RegistryOrphanRelinkStatus::Eligible, None) - } - (Some(repository), Some(_)) => ( + let identity = match repository_identity.as_deref() { + Some(owner) if owner == project_id => (RegistryOrphanRelinkStatus::Eligible, None), + Some(owner) => ( RegistryOrphanRelinkStatus::Retired, Some(format!( - "repository identity and enrollment name retired project '{repository}' instead of manifest project '{project_id}'" + "repository identity names retired project '{owner}' instead of manifest project '{project_id}'" )), ), - (Some(owner), None) | (None, Some(owner)) if owner == project_id => { - (RegistryOrphanRelinkStatus::Eligible, None) - } - (Some(owner), None) | (None, Some(owner)) => ( - RegistryOrphanRelinkStatus::Retired, - Some(format!( - "project marker names retired project '{owner}' instead of manifest project '{project_id}'" - )), - ), - (None, None) if reject_ephemeral_root => ( + None if reject_ephemeral_root => ( RegistryOrphanRelinkStatus::Blocked, - Some("project has no repository identity or enrollment marker".to_string()), + Some("project has no repository identity marker".to_string()), ), - (None, None) => (RegistryOrphanRelinkStatus::Eligible, None), + None => (RegistryOrphanRelinkStatus::Eligible, None), }; (identity.0, identity.1, canonical_root) } @@ -294,13 +265,6 @@ fn validate_manifest_shape( manifest.store_kind )); } - if manifest.storage_mode != StorageMode::ProfileSharded { - issues.push(format!( - "store manifest '{}' is {:?}, not profile_sharded", - manifest_path.display(), - manifest.storage_mode - )); - } if strip_profile_root(profile_root, &manifest.data_root).is_none() { issues.push(format!( "store data root '{}' is outside profile root '{}'", diff --git a/crates/tracedecay-global-db/src/schema_contract/definitions.rs b/crates/tracedecay-global-db/src/schema_contract/definitions.rs index 8133cb6d0d..6f1d652cc8 100644 --- a/crates/tracedecay-global-db/src/schema_contract/definitions.rs +++ b/crates/tracedecay-global-db/src/schema_contract/definitions.rs @@ -5,6 +5,9 @@ pub(super) struct Column { pub(super) not_null: bool, pub(super) default_value: Option<&'static str>, pub(super) primary_key_ordinal: i64, + /// `pragma_table_xinfo.hidden`: 0 for a stored column, 2 for a virtual + /// generated column, which occupies no row bytes. + pub(super) hidden: i64, } const fn column( @@ -20,6 +23,18 @@ const fn column( not_null, default_value, primary_key_ordinal, + hidden: 0, + } +} + +const fn virtual_column(name: &'static str, declared_type: &'static str, not_null: bool) -> Column { + Column { + name, + declared_type, + not_null, + default_value: None, + primary_key_ordinal: 0, + hidden: 2, } } @@ -74,49 +89,6 @@ macro_rules! table { }; } -pub(super) const SESSION_TEMPORAL_PROJECTION_RECEIPTS_V3: Table = table!( - "session_temporal_projection_receipts", - [ - column("session_id", "TEXT", true, None, 1), - column("generation", "INTEGER", true, None, 2), - column("batch_ordinal", "INTEGER", true, None, 3), - column("batch_digest", "TEXT", true, None, 0), - column("frozen_watermarks_json", "TEXT", true, None, 0), - column("source_through", "INTEGER", true, None, 0), - column("projection_through", "INTEGER", true, None, 0), - column("occurrence_count", "INTEGER", true, None, 0), - column("occurrence_digest", "TEXT", true, None, 0), - column("dimension_count", "INTEGER", true, None, 0), - column("dimension_digest", "TEXT", true, None, 0), - column("copy_count", "INTEGER", true, None, 0), - column("copy_digest", "TEXT", true, None, 0), - column("assertion_count", "INTEGER", true, None, 0), - column("assertion_digest", "TEXT", true, None, 0), - column("supersession_count", "INTEGER", true, None, 0), - column("supersession_digest", "TEXT", true, None, 0), - column("current_count", "INTEGER", true, None, 0), - column("current_digest", "TEXT", true, None, 0), - column("fts_count", "INTEGER", true, None, 0), - column("fts_digest", "TEXT", true, None, 0), - column("committed_at", "INTEGER", true, None, 0), - ], - [ - foreign_key( - "session_id", - "session_temporal_generations", - "session_id", - "CASCADE" - ), - foreign_key_sequence( - "generation", - "session_temporal_generations", - "generation", - "CASCADE", - 1 - ), - ] -); - const SESSION_TEMPORAL_PROJECTION_RECEIPTS_V4: Table = table!( "session_temporal_projection_receipts", [ @@ -848,8 +820,6 @@ pub(super) const TABLES: &[Table] = &[ column("output_message_id", "TEXT", true, None, 4), column("message_json", "TEXT", true, None, 0), column("content_hash", "TEXT", true, None, 0), - column("snippet_text", "TEXT", true, None, 0), - column("index_text", "TEXT", true, None, 0), ], [ foreign_key( @@ -1029,9 +999,18 @@ pub(super) const TABLES: &[Table] = &[ [ column("summary_id", "TEXT", false, None, 1), column("session_id", "TEXT", true, None, 0), + column("provider", "TEXT", true, None, 0), + column("conversation_id", "TEXT", true, None, 0), + column("depth", "INTEGER", true, None, 0), column("summary_anchor_id", "TEXT", true, None, 0), column("summary_text", "TEXT", true, None, 0), - column("index_text", "TEXT", true, None, 0), + column("summary_hash", "TEXT", true, None, 0), + column("summary_token_count", "INTEGER", true, None, 0), + column("source_token_count", "INTEGER", true, None, 0), + column("source_time_start", "INTEGER", false, None, 0), + column("source_time_end", "INTEGER", false, None, 0), + column("expand_hint", "TEXT", false, None, 0), + column("metadata_json", "TEXT", false, None, 0), column("source_horizon_json", "TEXT", true, None, 0), column("publication_json", "TEXT", false, None, 0), column("created_at", "INTEGER", true, None, 0), @@ -1043,6 +1022,21 @@ pub(super) const TABLES: &[Table] = &[ "NO ACTION" )] ), + table!( + "session_summary_sources", + [ + column("summary_id", "TEXT", true, None, 1), + column("ordinal", "INTEGER", true, None, 2), + column("source_kind", "TEXT", true, None, 0), + column("source_id", "TEXT", true, None, 0), + ], + [foreign_key( + "summary_id", + "session_summary_nodes", + "summary_id", + "NO ACTION" + )] + ), SESSION_RELATION_RECEIPTS, table!( "session_relation_effect_journal", @@ -1401,7 +1395,7 @@ pub(super) const TABLES: &[Table] = &[ column("evidence_json", "TEXT", true, None, 0), column("sanitized_content_digest", "TEXT", true, None, 0), column("sanitized_content_bytes", "INTEGER", true, None, 0), - column("snippet_text", "TEXT", true, None, 0), + virtual_column("snippet_text", "TEXT", true), column("index_text", "TEXT", true, None, 0), ], [ @@ -2061,6 +2055,40 @@ pub(super) const INDEXES: &[Index] = &[ origin: "c", columns: &["created_at", "session_id", "summary_id"], }, + Index { + table: "session_summary_nodes", + name: Some("idx_session_summary_nodes_session_depth_time"), + unique: false, + origin: "c", + columns: &[ + "provider", + "session_id", + "depth", + "source_time_start", + "source_time_end", + "created_at", + ], + }, + Index { + table: "session_summary_nodes", + name: Some("idx_session_summary_nodes_depth_tokens"), + unique: false, + origin: "c", + columns: &[ + "provider", + "session_id", + "depth", + "summary_token_count", + "source_token_count", + ], + }, + Index { + table: "session_summary_sources", + name: Some("idx_session_summary_sources_source"), + unique: false, + origin: "c", + columns: &["source_kind", "source_id", "summary_id"], + }, Index { table: "session_external_payload_manifests", name: Some("idx_session_external_payload_manifests_session"), @@ -2409,71 +2437,3 @@ pub(super) const INDEXES: &[Index] = &[ columns: &["session_id", "generation", "availability"], }, ]; - -#[cfg(test)] -mod tests { - use super::{INDEXES, TABLES}; - - const REBUILD_TABLES: &[&str] = &[ - "observation_projection_rebuild_provider_usage", - "observation_projection_rebuilds", - "observation_projection_rebuild_aliases", - "observation_projection_rebuild_sessions", - "observation_projection_rebuild_messages", - "observation_projection_rebuild_provenance", - "observation_projection_rebuild_dispositions", - "observation_projection_rebuild_workflow_facts", - ]; - - #[test] - fn rebuild_schema_contract_registration_is_complete() { - let tables = TABLES - .iter() - .map(|table| table.name) - .filter(|name| name.starts_with("observation_projection_rebuild")) - .collect::>(); - assert_eq!(tables, REBUILD_TABLES); - - let indexes = INDEXES - .iter() - .filter(|index| index.table.starts_with("observation_projection_rebuild")) - .map(|index| (index.table, index.name, index.unique, index.columns)) - .collect::>(); - assert_eq!( - indexes, - vec![ - ( - "observation_projection_rebuilds", - None, - true, - &["projector_version", "generation"] as &[_], - ), - ( - "observation_projection_rebuild_provenance", - Some("idx_projection_rebuild_provenance_output"), - false, - &[ - "projector_version", - "generation", - "output_provider", - "output_message_id", - ], - ), - ( - "observation_projection_rebuild_workflow_facts", - Some("idx_projection_rebuild_workflow_goal"), - false, - &[ - "projector_version", - "generation", - "provider", - "session_id", - "semantic_kind", - "provider_reference", - "observation_sequence", - ], - ), - ] - ); - } -} diff --git a/crates/tracedecay-global-db/src/schema_contract/invariants.rs b/crates/tracedecay-global-db/src/schema_contract/invariants.rs index b3b720ba61..c67d69b463 100644 --- a/crates/tracedecay-global-db/src/schema_contract/invariants.rs +++ b/crates/tracedecay-global-db/src/schema_contract/invariants.rs @@ -15,6 +15,9 @@ mod rows; pub(crate) mod test_fixture; mod triggers; +#[cfg(test)] +pub(crate) use triggers::SOURCE_CURSOR_ADVANCE_DELETE_GUARD_SQL; + use audit::{ AuditCheckpoint, AuditProgress, audit_checkpoint_is_plausible, ensure_audit_checkpoint_schema, read_audit_checkpoint, validate_projection_authority_chunk, @@ -32,11 +35,8 @@ use rows::{ validate_receipt_authority_rows, validate_source_cursor_authority_chunk, validate_source_cursor_authority_rows, }; -pub use triggers::released_v3_invariant_triggers_intact; use triggers::{FOREIGN_KEY_AUDIT_QUERY, replace_trigger, trigger_contracts_intact}; pub(super) use triggers::{INVARIANTS, Trigger}; -pub(crate) use triggers::{invariant_trigger_names_for_tables, invariant_trigger_sql_for_tables}; - const OPERATION: &str = "ensure global database authority invariants"; const INCOMPLETE_EXHAUSTIVE_PASS: i64 = -1; const FOREIGN_KEY_AUDIT_PROGRESS: &str = "authority-invariants"; diff --git a/crates/tracedecay-global-db/src/schema_contract/invariants/audit.rs b/crates/tracedecay-global-db/src/schema_contract/invariants/audit.rs index aa60b7c721..2e96d670c0 100644 --- a/crates/tracedecay-global-db/src/schema_contract/invariants/audit.rs +++ b/crates/tracedecay-global-db/src/schema_contract/invariants/audit.rs @@ -829,17 +829,31 @@ async fn validate_message_projection_row( .is_some(); match verify_owner_output_rows(conn, resolved, &owner_projection).await { Ok(()) => { - // Message equality is not the whole output. The raw twin is - // derived from the same observation and is not covered by the - // digest, so a matching message can still sit on a stale twin. - // Protected rows are not this arm: their stored message differs - // from the projection, and that compatibility already checked - // the twin. - if resolved + // Message equality is not the whole output. The row's hash and + // storage kind are not compared by the message record, so a + // matching message can still sit on stale storage columns. + // Protected rows are not this arm: they differ from the + // projected row, and that compatibility already checked them. + let stored_matches = match resolved .projection_rows .message(&owner_message.provider, &owner_message.message_id) - .is_some_and(|stored| stored == owner_message) - && owned_raw_twin_needs_rewrite(&owner_projection, &resolved.projection_rows)? + { + Some(stored) => { + crate::observation_projection::stored_row_matches(stored, owner_message) + .map_err(|error| { + authority_violation(format!( + "projection output rows disagree with deterministic output: \ + {error}" + )) + })? + } + None => false, + }; + if stored_matches + && owned_storage_columns_need_rewrite( + &owner_projection, + &resolved.projection_rows, + )? { resolved.released.record(&owner_projection); } @@ -869,7 +883,15 @@ async fn validate_message_projection_row( .projection_rows .message(&provider, &message_id) .is_some_and(|stored| { - stored_message_is_shipped_release_rendering(&authority.canonical, stored) + // A sanitizer fault is not a match; the row then + // falls through to the named hard failure below. + stored_message_is_shipped_release_rendering( + &authority.canonical, + |released| { + crate::observation_projection::stored_row_matches(stored, released) + .is_ok_and(|matches| matches) + }, + ) }) => { // Provenance already carries this binary's digest, and the @@ -948,13 +970,14 @@ async fn verify_owner_output_rows( .await } -/// Whether the LCM raw twin of a message that already matches this projection -/// is not the twin a fresh projection write would store. +/// Whether the storage columns of a message row that already matches this +/// projection are not the ones a fresh projection write would store. /// -/// Hermes projections have no raw twin. A sanitizer quarantine is itself the +/// A Hermes body belongs to the Hermes LCM turn authority, not the projector. +/// A sanitizer quarantine is itself the /// current rendering, so the caller records the projection for the same /// converge path a fresh capture uses. A sanitizer fault stays a typed refusal. -fn owned_raw_twin_needs_rewrite( +fn owned_storage_columns_need_rewrite( projection: &SessionMessageProjection, rows: &ProjectionRowsBatch, ) -> tracedecay_domain::errors::Result { @@ -967,11 +990,11 @@ fn owned_raw_twin_needs_rewrite( Err(error) if error.is_quarantine_verdict() => return Ok(true), Err(error) => { return Err(authority_violation(format!( - "projection raw twin sanitizer failed: {error}" + "projection message sanitizer failed: {error}" ))); } }; - let Some(raw) = rows.raw_twin(&message.provider, &message.message_id) else { + let Some(raw) = rows.storage_columns(&message.provider, &message.message_id) else { return Ok(true); }; // The derived columns are pure functions of the same sanitized body, so a @@ -1925,7 +1948,7 @@ mod tests { AnchoredObservationWrite, ObservationPersistOutcome, ObservationProjection, ObservationProjectionStore, ObservationStore, ObservationWrite, SESSION_MESSAGE_PROJECTOR_VERSION, build_observation_resolution_authorization_v1, - build_observation_retrieval_anchor_v2, + build_observation_retrieval_anchor, }; use super::{ @@ -2271,7 +2294,7 @@ mod tests { let generation = ProjectionGenerationId::new("projection.audit-test").unwrap(); let authorization = build_observation_resolution_authorization_v1(&observation, "audit-test").unwrap(); - let anchor = build_observation_retrieval_anchor_v2( + let anchor = build_observation_retrieval_anchor( &observation, generation.clone(), UtcMicros(1), @@ -2637,7 +2660,7 @@ mod tests { "audit-batch", ) .unwrap(); - let anchor = tracedecay_store::build_observation_retrieval_anchor_v2( + let anchor = tracedecay_store::build_observation_retrieval_anchor( write.observation(), generation.clone(), UtcMicros(1), @@ -2743,7 +2766,7 @@ mod tests { let authorization = build_observation_resolution_authorization_v1(write.observation(), "cursor-fixture") .unwrap(); - let anchor = build_observation_retrieval_anchor_v2( + let anchor = build_observation_retrieval_anchor( write.observation(), projection_generation.clone(), UtcMicros(1), @@ -2802,8 +2825,7 @@ mod tests { .execute( "UPDATE lcm_raw_messages SET content = ?3, content_hash = ?4, storage_kind = 'inline', - payload_ref = NULL, snippet_text = ?3, index_text = ?3, - metadata_json = ?5 + payload_ref = NULL, placeholder_text = NULL, metadata_json = ?5 WHERE provider = ?1 AND message_id = ?2", params![ "cursor", @@ -2826,10 +2848,7 @@ mod tests { .await .expect("read receipt-bound stored message") .expect("receipt-bound stored message"); - assert_eq!( - stored_before.text, - tracedecay_lcm::retrieval_content::derived_text_for_index(protected.sanitized_text()) - ); + assert_eq!(stored_before.text, protected.sanitized_text()); assert_ne!(stored_before, *projection.message()); super::super::ensure_authority_invariants(database.runtime_database(), true, false) @@ -2865,7 +2884,9 @@ mod tests { .expect("preserved deterministic provenance"); assert_eq!( row.get::(0).unwrap(), - projection.output_digest().unwrap().as_str() + crate::observation_projection::stored_output_digest(&projection) + .unwrap() + .as_str() ); assert_eq!(row.get::(1).unwrap(), 1); assert!(rows.next().await.unwrap().is_none()); @@ -2884,7 +2905,7 @@ mod tests { let transaction = database.begin_write_transaction().await.unwrap(); let deleted = transaction .execute( - "DELETE FROM session_messages WHERE provider = ?1 AND message_id = ?2", + "DELETE FROM lcm_raw_messages WHERE provider = ?1 AND message_id = ?2", params!["cursor", CURSOR_COLLISION_MESSAGE_ID], ) .await @@ -2962,7 +2983,7 @@ mod tests { "projected sessions must reuse the page authority instead of reading per output" ); assert_eq!( - counting.issued("FROM session_messages WHERE provider = ?1 AND message_id = ?2"), + counting.issued("FROM lcm_raw_messages WHERE provider = ?1 AND message_id = ?2"), 0, "projected messages must reuse the page authority instead of reading per output" ); diff --git a/crates/tracedecay-global-db/src/schema_contract/invariants/released_rendering.rs b/crates/tracedecay-global-db/src/schema_contract/invariants/released_rendering.rs index 7b6e6d096c..394877efbf 100644 --- a/crates/tracedecay-global-db/src/schema_contract/invariants/released_rendering.rs +++ b/crates/tracedecay-global-db/src/schema_contract/invariants/released_rendering.rs @@ -14,6 +14,11 @@ //! | -------------------------------- | --------------------------- | ------------ | ---------------- | //! | v0.1.0-beta.25 .. v0.1.0-beta.37 | `claude-session-message-v5` | unchanged | unchanged | //! +//! Since LCM schema 12 the digested message is the stored message row (its +//! sanitized body and protected metadata), so a stored row is digestible into +//! the provenance that pairs with it; stores from those tags are reset rather +//! than converged. +//! //! What did change, after the newest tag, is one *rendering*: //! `provider_message_semantics` gives a Codex user message carrying an //! `` block a typed rendering, role @@ -52,12 +57,15 @@ use std::sync::Mutex; use tracedecay_runtime_core::db::engine::Executor; use tracedecay_store::{ - SESSION_MESSAGE_PROJECTOR_VERSION, SessionMessageProjection, message_output_digest, + ProjectionStoreError, SESSION_MESSAGE_PROJECTOR_VERSION, SessionMessageProjection, + message_output_digest, }; use super::audit::ProjectionProvenanceRow; use super::rows::authority_violation; -use crate::observation_projection::{ConvergedRendering, ProjectionRowsBatch}; +use crate::observation_projection::{ + ConvergedRendering, ProjectionRowsBatch, stored_output_digest, +}; /// What this binary must do with one stored provenance row. #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -226,11 +234,18 @@ pub(super) fn classify_provenance_rendering( if let Some(field) = provenance_identity_disagreement(actual, projection) { return Err(disagreement(field, actual, projection)); } - let derived_digest = projection.output_digest().map_err(|_| { - authority_violation("projection output digest is not canonically derivable") - })?; - if actual.output_digest == derived_digest.as_str() { - return Ok(StoredProvenanceRendering::Current); + // A rendering the sanitizer now withholds has no current row, so only a + // released stored row can still pair with its provenance. + match stored_output_digest(projection) { + Ok(derived_digest) if actual.output_digest == derived_digest.as_str() => { + return Ok(StoredProvenanceRendering::Current); + } + Ok(_) | Err(ProjectionStoreError::SanitizationRefused { .. }) => {} + Err(_) => { + return Err(authority_violation( + "projection output digest is not canonically derivable", + )); + } } let message = projection.message(); let stored_digest = stored @@ -401,7 +416,7 @@ mod tests { "codex-goal-context", ) .unwrap(); - let anchor = tracedecay_store::build_observation_retrieval_anchor_v2( + let anchor = tracedecay_store::build_observation_retrieval_anchor( write.observation(), generation.clone(), UtcMicros(1), @@ -423,8 +438,8 @@ mod tests { .map(|_| ()) } - /// Every persisted byte of one projected output: the message row, its LCM - /// raw twin's indexed text, and the provenance digest that pairs them. + /// Every persisted byte of one projected output: the message row, its + /// indexed text, and the provenance digest that pairs them. #[derive(Debug, Eq, PartialEq)] struct StoredOutput { session_id: String, @@ -445,12 +460,11 @@ mod tests { async fn stored_output(conn: &impl QueryExecutor, message_id: &str) -> StoredOutput { let mut rows = conn .query( - "SELECT m.session_id, m.role, m.timestamp, m.ordinal, m.text, m.kind, m.model, + "SELECT m.session_id, m.role, m.timestamp, m.ordinal, + COALESCE(m.content, m.placeholder_text, ''), m.kind, m.model, m.tool_names, m.source_path, m.source_offset, m.metadata_json, - raw.index_text, p.output_digest - FROM session_messages AS m - JOIN lcm_raw_messages AS raw - ON raw.provider = m.provider AND raw.message_id = m.message_id + m.index_text, p.output_digest + FROM lcm_raw_messages AS m JOIN observation_projection_provenance AS p ON p.output_provider = m.provider AND p.output_message_id = m.message_id WHERE m.provider = 'codex' AND m.message_id = ?1", @@ -480,6 +494,25 @@ mod tests { } } + /// Provenance digest a release's stored output pairs with: the output + /// digest over the row the projector stores for the released message. + fn stored_release_digest(session: &SessionRecord, message: &SessionMessageRecord) -> String { + let stored = tracedecay_lcm::raw::projection_stored_message(message) + .expect("the released rendering must be storable by this binary's sanitizer"); + message_output_digest(session, &stored, 0) + .expect("digest the released output") + .as_str() + .to_owned() + } + + fn released_digest() -> String { + let fixture = released(); + stored_release_digest( + &serde_json::from_value(fixture["released_session"].clone()).unwrap(), + &serde_json::from_value(fixture["released_message"].clone()).unwrap(), + ) + } + /// Rewrites the drained output to the bytes a release persisted, and arms /// an exhaustive audit. This is the shipped store the report describes: the /// released rendering, paired with the digest that release computed for it. @@ -493,33 +526,9 @@ mod tests { message.session_id, session.session_id, "the fixture's released output must belong to its released session" ); - conn.execute( - "UPDATE session_messages - SET session_id = ?3, role = ?4, timestamp = ?5, ordinal = ?6, - text = ?7, kind = ?8, model = ?9, tool_names = ?10, - source_path = ?11, source_offset = ?12, metadata_json = ?13 - WHERE provider = ?1 AND message_id = ?2", - tracedecay_runtime_core::params![ - message.provider.as_str(), - message.message_id.as_str(), - message.session_id.as_str(), - message.role.as_str(), - message.timestamp, - message.ordinal, - message.text.as_str(), - message.kind.as_deref(), - message.model.as_deref(), - message.tool_names.as_deref(), - message.source_path.as_deref(), - message.source_offset, - message.metadata_json.as_deref(), - ], - ) - .await - .expect("restore the released message row"); tracedecay_lcm::raw::upsert_projection_raw_message(conn, &message) .await - .expect("restore the released LCM raw twin"); + .expect("restore the released message row"); conn.execute( "UPDATE observation_projection_provenance SET output_digest = ?2 WHERE projector_version = ?1 AND observation_id = ?3", @@ -556,8 +565,8 @@ mod tests { let snapshot = database.read_snapshot().await.unwrap(); let current = stored_output(&snapshot, RECORD_ID).await; drop(snapshot); - let fixture = released(); - let released_digest = fixture["released_output_digest"].as_str().unwrap(); + let released_digest = released_digest(); + let released_digest = released_digest.as_str(); assert_eq!( current.role, "system", "this binary renders a Codex goal-context record as typed goal context" @@ -629,11 +638,7 @@ mod tests { .begin_write_transaction("seed current provenance over a stale output") .await .unwrap(); - downgrade_to_released( - &transaction, - released()["released_output_digest"].as_str().unwrap(), - ) - .await; + downgrade_to_released(&transaction, &released_digest()).await; transaction .execute( "UPDATE observation_projection_provenance SET output_digest = ?2 @@ -665,61 +670,11 @@ mod tests { assert_eq!(stored_output(&snapshot, RECORD_ID).await, current); } - /// The digest covers the message, not its LCM raw twin. A twin can be - /// rewritten under a still-current message and provenance; reopen has to - /// restore the twin a fresh projection write stores. - #[tokio::test] - async fn current_provenance_repairs_a_stale_raw_twin() { - let directory = TempDir::new().unwrap(); - let runtime = HostAdmissionTestRuntimeV1::profile(directory.path()) - .await - .unwrap(); - seed(&runtime, &observation()).await.unwrap(); - let database = runtime - .registered_database(HostAdmissionScope::Profile) - .expect("registered profile database"); - let snapshot = database.read_snapshot().await.unwrap(); - let current = stored_output(&snapshot, RECORD_ID).await; - drop(snapshot); - - let transaction = database - .runtime_database() - .begin_write_transaction("stale the raw twin under current provenance") - .await - .unwrap(); - let updated = transaction - .execute( - "UPDATE lcm_raw_messages - SET content = 'stale raw body', content_hash = 'stale', - snippet_text = 'stale raw body', index_text = 'stale raw body' - WHERE provider = 'codex' AND message_id = ?1", - tracedecay_runtime_core::params![RECORD_ID], - ) - .await - .expect("stale the raw twin"); - assert_eq!(updated, 1); - transaction.commit().await.unwrap(); - - let snapshot = database.read_snapshot().await.unwrap(); - let stale = stored_output(&snapshot, RECORD_ID).await; - drop(snapshot); - assert_eq!(stale.digest, current.digest); - assert_eq!(stale.text, current.text); - assert_eq!(stale.raw_index_text, "stale raw body"); - - super::super::ensure_authority_invariants(database.runtime_database(), false, false) - .await - .expect("current provenance must repair its stale raw twin"); - - let snapshot = database.read_snapshot().await.unwrap(); - assert_eq!(stored_output(&snapshot, RECORD_ID).await, current); - } - - /// A twin whose content survived but whose derived columns did not still + /// A row whose content survived but whose content hash did not still /// fails hydration with `PayloadIntegrityMismatch`. Content equality alone - /// is not the twin a fresh projection write stores. + /// is not the row a fresh projection write stores. #[tokio::test] - async fn current_provenance_repairs_a_raw_twin_with_a_stale_hash() { + async fn current_provenance_repairs_a_row_with_a_stale_hash() { let directory = TempDir::new().unwrap(); let runtime = HostAdmissionTestRuntimeV1::profile(directory.path()) .await @@ -734,13 +689,13 @@ mod tests { let transaction = database .runtime_database() - .begin_write_transaction("stale the raw twin derivations") + .begin_write_transaction("stale the stored content hash") .await .unwrap(); let updated = transaction .execute( "UPDATE lcm_raw_messages - SET content_hash = 'stale', index_text = 'stale index' + SET content_hash = 'stale' WHERE provider = 'codex' AND message_id = ?1", tracedecay_runtime_core::params![RECORD_ID], ) @@ -751,7 +706,7 @@ mod tests { super::super::ensure_authority_invariants(database.runtime_database(), false, false) .await - .expect("current provenance must repair a twin with stale derivations"); + .expect("current provenance must repair a row with a stale content hash"); let snapshot = database.read_snapshot().await.unwrap(); assert_eq!(stored_output(&snapshot, RECORD_ID).await, current); @@ -762,18 +717,18 @@ mod tests { tracedecay_runtime_core::params![RECORD_ID], ) .await - .expect("read the repaired twin"); + .expect("read the repaired row"); let row = rows .next() .await - .expect("read the repaired twin") - .expect("raw twin row"); + .expect("read the repaired row") + .expect("message row"); assert_eq!( row.get::(0).unwrap(), tracedecay_lcm::retrieval_content::projected_content_hash( &row.get::(1).unwrap() ), - "the repaired twin must carry the hash its content hydrates against" + "the repaired row must carry the hash its content hydrates against" ); } @@ -798,7 +753,7 @@ mod tests { .unwrap(); let updated = transaction .execute( - "UPDATE session_messages SET text = 'tampered projection body' + "UPDATE lcm_raw_messages SET content = 'tampered projection body' WHERE provider = 'codex' AND message_id = ?1", tracedecay_runtime_core::params![RECORD_ID], ) @@ -867,7 +822,7 @@ mod tests { .expect("create stale session identity"); transaction .execute( - "UPDATE session_messages SET session_id = ?2 + "UPDATE lcm_raw_messages SET session_id = ?2 WHERE provider = 'codex' AND session_id = ?1", tracedecay_runtime_core::params![SESSION, "stale-session-identity"], ) @@ -916,7 +871,6 @@ mod tests { #[derive(Debug, Eq, PartialEq)] struct ProjectionOutcome { message_rows: i64, - raw_rows: i64, provenance_rows: i64, disposition: Option<(String, String)>, } @@ -929,8 +883,6 @@ mod tests { let mut rows = conn .query( "SELECT - (SELECT COUNT(*) FROM session_messages - WHERE provider = 'codex' AND message_id = ?2), (SELECT COUNT(*) FROM lcm_raw_messages WHERE provider = 'codex' AND message_id = ?2), (SELECT COUNT(*) FROM observation_projection_provenance @@ -952,12 +904,11 @@ mod tests { .await .expect("read the projection authority row") .expect("the aggregate row is always present"); - let receipt_id: Option = row.get(3).unwrap(); - let reason: Option = row.get(4).unwrap(); + let receipt_id: Option = row.get(2).unwrap(); + let reason: Option = row.get(3).unwrap(); ProjectionOutcome { message_rows: row.get(0).unwrap(), - raw_rows: row.get(1).unwrap(), - provenance_rows: row.get(2).unwrap(), + provenance_rows: row.get(1).unwrap(), disposition: receipt_id.zip(reason), } } @@ -1008,36 +959,10 @@ mod tests { ) .await .expect("install the released session row"); - conn.execute( - "INSERT INTO session_messages - (provider, message_id, session_id, role, timestamp, ordinal, text, kind, model, - tool_names, source_path, source_offset, metadata_json) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13)", - tracedecay_runtime_core::params![ - message.provider.as_str(), - message.message_id.as_str(), - message.session_id.as_str(), - message.role.as_str(), - message.timestamp, - message.ordinal, - message.text.as_str(), - message.kind.as_deref(), - message.model.as_deref(), - message.tool_names.as_deref(), - message.source_path.as_deref(), - message.source_offset, - message.metadata_json.as_deref(), - ], - ) - .await - .expect("install the released message row"); tracedecay_lcm::raw::upsert_projection_raw_message(conn, &message) .await .expect("the released rendering must still be servable by this binary's sanitizer"); - let digest = message_output_digest(&session, &message, 0) - .expect("digest the released output") - .as_str() - .to_owned(); + let digest = stored_release_digest(&session, &message); let anchor = derive_exact_observation_anchor_id(observation.scope(), observation.observation_id()) .unwrap(); @@ -1130,7 +1055,6 @@ mod tests { fresh_capture, ProjectionOutcome { message_rows: 0, - raw_rows: 0, provenance_rows: 0, disposition: Some(( observation diff --git a/crates/tracedecay-global-db/src/schema_contract/invariants/rows.rs b/crates/tracedecay-global-db/src/schema_contract/invariants/rows.rs index 5521db3eaf..91f48572af 100644 --- a/crates/tracedecay-global-db/src/schema_contract/invariants/rows.rs +++ b/crates/tracedecay-global-db/src/schema_contract/invariants/rows.rs @@ -808,11 +808,12 @@ mod tests { created_at, ready_at, activated_at, completed_at ) VALUES ('availability-session', 1, 'building', '{}', 1, NULL, NULL, NULL); INSERT INTO session_summary_nodes ( - summary_id, session_id, summary_anchor_id, summary_text, index_text, + summary_id, session_id, provider, conversation_id, depth, summary_anchor_id, + summary_text, summary_hash, summary_token_count, source_token_count, source_horizon_json, publication_json, created_at ) VALUES ( - 'summary-owned-elsewhere', 'summary-session', 'summary-anchor', - 'summary', 'summary', '{}', NULL, 1 + 'summary-owned-elsewhere', 'summary-session', 'test', 'summary-session', 0, + 'summary-anchor', 'summary', 'hash', 1, 1, '{}', NULL, 1 ); INSERT INTO session_summary_availability ( session_id, generation, summary_id, availability, diff --git a/crates/tracedecay-global-db/src/schema_contract/invariants/test_fixture.rs b/crates/tracedecay-global-db/src/schema_contract/invariants/test_fixture.rs index c1c18cbbc5..7e19569219 100644 --- a/crates/tracedecay-global-db/src/schema_contract/invariants/test_fixture.rs +++ b/crates/tracedecay-global-db/src/schema_contract/invariants/test_fixture.rs @@ -87,9 +87,14 @@ pub(crate) fn authority_fixture( payload, ) .expect("durable observation"); - let cursor = - ObservationSourceCursorV1::new(source, ObservationScopeV1::Profile, generation, end) - .expect("committed source cursor"); + let cursor = ObservationSourceCursorV1::for_ordering( + source, + ObservationScopeV1::Profile, + generation, + ObservationOrderingDomainV1::FileBytes, + end, + ) + .expect("committed source cursor"); (observation, cursor) } @@ -136,10 +141,11 @@ pub(super) async fn seed_observation( /// The same cursor shifted along its ordering domain. pub(super) fn shift(cursor: &ObservationSourceCursorV1, delta: i64) -> ObservationSourceCursorV1 { let position = u64::try_from(i64::try_from(cursor.position()).unwrap() + delta).unwrap(); - ObservationSourceCursorV1::new( + ObservationSourceCursorV1::for_ordering( cursor.source().clone(), cursor.scope().clone(), cursor.generation(), + ObservationOrderingDomainV1::FileBytes, position, ) .expect("shifted source cursor") diff --git a/crates/tracedecay-global-db/src/schema_contract/invariants/triggers.rs b/crates/tracedecay-global-db/src/schema_contract/invariants/triggers.rs index 0924d3a151..e8c75061f5 100644 --- a/crates/tracedecay-global-db/src/schema_contract/invariants/triggers.rs +++ b/crates/tracedecay-global-db/src/schema_contract/invariants/triggers.rs @@ -1,6 +1,6 @@ use tracedecay_runtime_core::db::engine::{Executor, QueryExecutor, params}; -use crate::{global_db_operation_error, global_db_operation_message}; +use crate::global_db_operation_error; use super::{OPERATION, normalize_trigger_sql}; @@ -54,6 +54,25 @@ const RECEIPT_IMMUTABILITY: &[Trigger] = &[ }, ]; +/// Only advances the durable cursor strictly supersedes may be deleted: the +/// row supporting the current frontier and rows beyond it stay immutable. +pub(crate) const SOURCE_CURSOR_ADVANCE_DELETE_GUARD_SQL: &str = + "CREATE TRIGGER source_cursor_advances_immutable_delete_v1 + BEFORE DELETE ON source_cursor_advances + WHEN NOT EXISTS ( + SELECT 1 FROM source_cursors AS cursor + WHERE cursor.source_json = OLD.source_json + AND cursor.scope_json = OLD.scope_json + AND (json_extract(cursor.cursor_json, '$.generation') + IS NOT json_extract(OLD.coverage_json, '$.generation') + OR (COALESCE(json_extract(cursor.cursor_json, '$.ordering_domain'), 'file_bytes') + = json_extract(OLD.coverage_json, '$.ordering_domain') + AND json_extract(cursor.cursor_json, '$.byte_offset') + > json_extract(OLD.coverage_json, '$.range.end'))) + ) BEGIN + SELECT RAISE(ABORT, 'source cursor advances are immutable'); + END"; + const SOURCE_CURSOR_ADVANCE_IMMUTABILITY: &[Trigger] = &[ Trigger { name: "source_cursor_advances_immutable_update_v1", @@ -66,10 +85,7 @@ const SOURCE_CURSOR_ADVANCE_IMMUTABILITY: &[Trigger] = &[ Trigger { name: "source_cursor_advances_immutable_delete_v1", table: "source_cursor_advances", - create_sql: "CREATE TRIGGER source_cursor_advances_immutable_delete_v1 - BEFORE DELETE ON source_cursor_advances BEGIN - SELECT RAISE(ABORT, 'source cursor advances are immutable'); - END", + create_sql: SOURCE_CURSOR_ADVANCE_DELETE_GUARD_SQL, }, ]; @@ -199,38 +215,10 @@ const PROJECTION_AUDIT_INVALIDATION: &[Trigger] = &[ WHERE audit_name = 'observation-authority'; END", }, - Trigger { - name: "projection_output_audit_invalidate_update_v1", - table: "session_messages", - create_sql: "CREATE TRIGGER projection_output_audit_invalidate_update_v1 - AFTER UPDATE ON session_messages - WHEN EXISTS ( - SELECT 1 FROM observation_projection_provenance - WHERE output_provider = OLD.provider - AND output_message_id = OLD.message_id - ) BEGIN - DELETE FROM authority_audit_checkpoints - WHERE audit_name = 'observation-authority'; - END", - }, - Trigger { - name: "projection_output_audit_invalidate_delete_v1", - table: "session_messages", - create_sql: "CREATE TRIGGER projection_output_audit_invalidate_delete_v1 - AFTER DELETE ON session_messages - WHEN EXISTS ( - SELECT 1 FROM observation_projection_provenance - WHERE output_provider = OLD.provider - AND output_message_id = OLD.message_id - ) BEGIN - DELETE FROM authority_audit_checkpoints - WHERE audit_name = 'observation-authority'; - END", - }, - // The message-row triggers above do not see the LCM raw twin. A twin can - // drift (content, session identity) while the message row and the current - // provenance digest stay put, and the trusted checkpoint would then skip - // it forever. Invalidate on the same ownership predicate. + // Message rows live in `lcm_raw_messages`. A row can drift (content, + // session identity) while the current provenance digest stays put, and + // the trusted checkpoint would then skip it forever. Invalidate on the + // ownership predicate. Trigger { name: "projection_raw_audit_invalidate_update_v1", table: "lcm_raw_messages", @@ -1202,8 +1190,8 @@ const SESSION_TEMPORAL_FTS: &[Trigger] = &[ table: "session_occurrences", create_sql: "CREATE TRIGGER session_occurrences_fts_insert_v1 AFTER INSERT ON session_occurrences BEGIN - INSERT INTO session_occurrences_fts(rowid, index_text, snippet_text) - VALUES (NEW.rowid, NEW.index_text, NEW.snippet_text); + INSERT INTO session_occurrences_fts(rowid, index_text) + VALUES (NEW.rowid, NEW.index_text); END", }, Trigger { @@ -1211,23 +1199,19 @@ const SESSION_TEMPORAL_FTS: &[Trigger] = &[ table: "session_occurrences", create_sql: "CREATE TRIGGER session_occurrences_fts_delete_v1 AFTER DELETE ON session_occurrences BEGIN - INSERT INTO session_occurrences_fts( - session_occurrences_fts, rowid, index_text, snippet_text - ) - VALUES ('delete', OLD.rowid, OLD.index_text, OLD.snippet_text); + INSERT INTO session_occurrences_fts(session_occurrences_fts, rowid, index_text) + VALUES ('delete', OLD.rowid, OLD.index_text); END", }, Trigger { name: "session_occurrences_fts_update_v1", table: "session_occurrences", create_sql: "CREATE TRIGGER session_occurrences_fts_update_v1 - AFTER UPDATE OF index_text, snippet_text ON session_occurrences BEGIN - INSERT INTO session_occurrences_fts( - session_occurrences_fts, rowid, index_text, snippet_text - ) - VALUES ('delete', OLD.rowid, OLD.index_text, OLD.snippet_text); - INSERT INTO session_occurrences_fts(rowid, index_text, snippet_text) - VALUES (NEW.rowid, NEW.index_text, NEW.snippet_text); + AFTER UPDATE OF index_text ON session_occurrences BEGIN + INSERT INTO session_occurrences_fts(session_occurrences_fts, rowid, index_text) + VALUES ('delete', OLD.rowid, OLD.index_text); + INSERT INTO session_occurrences_fts(rowid, index_text) + VALUES (NEW.rowid, NEW.index_text); END", }, Trigger { @@ -1235,8 +1219,8 @@ const SESSION_TEMPORAL_FTS: &[Trigger] = &[ table: "session_summary_nodes", create_sql: "CREATE TRIGGER session_summary_nodes_fts_insert_v1 AFTER INSERT ON session_summary_nodes BEGIN - INSERT INTO session_summary_nodes_fts(rowid, summary_text, index_text) - VALUES (NEW.rowid, NEW.summary_text, NEW.index_text); + INSERT INTO session_summary_nodes_fts(rowid, summary_text) + VALUES (NEW.rowid, NEW.summary_text); END", }, Trigger { @@ -1245,22 +1229,22 @@ const SESSION_TEMPORAL_FTS: &[Trigger] = &[ create_sql: "CREATE TRIGGER session_summary_nodes_fts_delete_v1 AFTER DELETE ON session_summary_nodes BEGIN INSERT INTO session_summary_nodes_fts( - session_summary_nodes_fts, rowid, summary_text, index_text + session_summary_nodes_fts, rowid, summary_text ) - VALUES ('delete', OLD.rowid, OLD.summary_text, OLD.index_text); + VALUES ('delete', OLD.rowid, OLD.summary_text); END", }, Trigger { name: "session_summary_nodes_fts_update_v1", table: "session_summary_nodes", create_sql: "CREATE TRIGGER session_summary_nodes_fts_update_v1 - AFTER UPDATE OF summary_text, index_text ON session_summary_nodes BEGIN + AFTER UPDATE OF summary_text ON session_summary_nodes BEGIN INSERT INTO session_summary_nodes_fts( - session_summary_nodes_fts, rowid, summary_text, index_text + session_summary_nodes_fts, rowid, summary_text ) - VALUES ('delete', OLD.rowid, OLD.summary_text, OLD.index_text); - INSERT INTO session_summary_nodes_fts(rowid, summary_text, index_text) - VALUES (NEW.rowid, NEW.summary_text, NEW.index_text); + VALUES ('delete', OLD.rowid, OLD.summary_text); + INSERT INTO session_summary_nodes_fts(rowid, summary_text) + VALUES (NEW.rowid, NEW.summary_text); END", }, ]; @@ -1605,36 +1589,6 @@ pub(in crate::schema_contract) const INVARIANTS: &[Invariant] = &[ }, ]; -/// CREATE-trigger statements for every authority invariant trigger installed -/// on one of the given tables. The scoped observation reset restores the -/// triggers that dropped with their tables through this single authority, so -/// it can never install a trigger shape the contract validator would refuse. -pub(crate) fn invariant_trigger_sql_for_tables(tables: &[&str]) -> Vec<&'static str> { - INVARIANTS - .iter() - .flat_map(|invariant| invariant.triggers) - .filter(|trigger| { - tables - .iter() - .any(|table| table.eq_ignore_ascii_case(trigger.table)) - }) - .map(|trigger| trigger.create_sql) - .collect() -} - -pub(crate) fn invariant_trigger_names_for_tables(tables: &[&str]) -> Vec<&'static str> { - INVARIANTS - .iter() - .flat_map(|invariant| invariant.triggers) - .filter(|trigger| { - tables - .iter() - .any(|table| table.eq_ignore_ascii_case(trigger.table)) - }) - .map(|trigger| trigger.name) - .collect() -} - pub(super) async fn replace_trigger( conn: &impl Executor, trigger: &Trigger, @@ -1664,119 +1618,6 @@ pub(super) async fn trigger_contracts_intact( Ok(true) } -/// One authority trigger body a session-temporal v3 store carries in the shape -/// that shipped rather than the current one. -/// -/// Every release that persisted schema marker 3, v0.1.0-beta.25 through -/// v0.1.0-beta.37, the newest tag, published one identical 81-trigger -/// authority inventory (the exact SQL lives in -/// `tests/fixtures/session-temporal-released-v3-triggers.sql`). No trigger name -/// changed, so a v3 store differs from the current contract in exactly these -/// bodies and matches it everywhere else. Each released body is reconstructed -/// from the current contract so every trigger keeps one definition; a fragment -/// that is not present exactly once is a typed error, never a store admitted or -/// refused against a body nobody published. -struct ReleasedV3TriggerDrift { - trigger: &'static str, - current: &'static str, - released: &'static str, -} - -/// Triggers added after the v3 inventory. A released store is admitted on the -/// published bodies, then schema convergence installs these and the missing -/// contract forces the exhaustive repair pass. Requiring them at admission -/// would reset every beta.25–beta.37 profile. -const POST_RELEASED_V3_TRIGGERS: &[&str] = &[ - "projection_raw_audit_invalidate_update_v1", - "projection_raw_audit_invalidate_delete_v1", -]; - -const RELEASED_V3_TRIGGER_DRIFT: &[ReleasedV3TriggerDrift] = &[ - ReleasedV3TriggerDrift { - trigger: "session_refresh_progress_insert_guard_v1", - current: "AND NEW.committed_records = receipt.committed_item_count", - released: "AND NEW.committed_records = - receipt.occurrence_count - + receipt.copy_count - + receipt.assertion_count", - }, - ReleasedV3TriggerDrift { - trigger: "projection_output_audit_invalidate_update_v1", - current: "WHERE output_provider = OLD.provider", - released: "WHERE projector_version = 'claude-session-message-v4' - AND output_provider = OLD.provider", - }, - ReleasedV3TriggerDrift { - trigger: "projection_output_audit_invalidate_delete_v1", - current: "WHERE output_provider = OLD.provider", - released: "WHERE projector_version = 'claude-session-message-v4' - AND output_provider = OLD.provider", - }, -]; - -/// The released-v3 body of every drifted authority trigger, keyed by name. -fn released_v3_trigger_contracts() -> tracedecay_domain::errors::Result> -{ - RELEASED_V3_TRIGGER_DRIFT - .iter() - .map(|drift| { - let Some(trigger) = INVARIANTS - .iter() - .flat_map(|invariant| invariant.triggers) - .find(|trigger| trigger.name == drift.trigger) - else { - return Err(global_db_operation_message( - OPERATION, - format!( - "released v3 trigger '{}' is not a defined authority invariant trigger", - drift.trigger - ), - )); - }; - if trigger.create_sql.matches(drift.current).count() != 1 { - return Err(global_db_operation_message( - OPERATION, - format!( - "released v3 '{}' trigger contract is unavailable", - drift.trigger - ), - )); - } - Ok(( - drift.trigger, - trigger - .create_sql - .replacen(drift.current, drift.released, 1), - )) - }) - .collect() -} - -#[hotpath::measure( - future = true, - label = "global_db.schema_contract.triggers.released_v3_intact" -)] -pub async fn released_v3_invariant_triggers_intact( - conn: &impl QueryExecutor, -) -> tracedecay_domain::errors::Result { - let released = released_v3_trigger_contracts()?; - for invariant in INVARIANTS { - for trigger in invariant.triggers { - if POST_RELEASED_V3_TRIGGERS.contains(&trigger.name) { - continue; - } - let expected = released - .iter() - .find(|(name, _)| *name == trigger.name) - .map_or(trigger.create_sql, |(_, sql)| sql.as_str()); - if !trigger_matches_sql(conn, trigger, expected).await? { - return Ok(false); - } - } - } - Ok(true) -} - async fn trigger_matches( conn: &impl QueryExecutor, trigger: &Trigger, diff --git a/crates/tracedecay-global-db/src/schema_contract/mod.rs b/crates/tracedecay-global-db/src/schema_contract/mod.rs index 3cf13000f2..2e2c6e2168 100644 --- a/crates/tracedecay-global-db/src/schema_contract/mod.rs +++ b/crates/tracedecay-global-db/src/schema_contract/mod.rs @@ -20,21 +20,17 @@ fn normalize_trigger_sql(sql: &str) -> String { pub(crate) use definitions::SESSION_RELATION_RECEIPT_RECOVERY_COLUMNS; pub(crate) use invariants::{ - authority_invariant_triggers_intact, released_v3_invariant_triggers_intact, - validate_authority_rows_exhaustive, + authority_invariant_triggers_intact, validate_authority_rows_exhaustive, }; pub use invariants::{ ensure_authority_audit_checkpoint_schema, ensure_authority_invariant_schema, require_foreign_key_audit, }; -pub(crate) use invariants::{ - ensure_authority_invariants, ensure_fresh_authority_invariants, - invariant_trigger_names_for_tables, invariant_trigger_sql_for_tables, -}; +pub(crate) use invariants::{ensure_authority_invariants, ensure_fresh_authority_invariants}; pub use validation::validate_registry_schema_contract; pub(crate) use validation::{ - validate_authority_schema_contract, validate_released_v3_temporal_projection_receipt_contract, - validate_remote_deletion_schema_contract, validate_session_graph_publication_schema_contract, + validate_authority_schema_contract, validate_remote_deletion_schema_contract, + validate_session_graph_publication_schema_contract, validate_session_relation_receipts_without_recovery_contract, validate_session_temporal_schema_contract, }; diff --git a/crates/tracedecay-global-db/src/schema_contract/validation.rs b/crates/tracedecay-global-db/src/schema_contract/validation.rs index a018ce1dad..48d0281184 100644 --- a/crates/tracedecay-global-db/src/schema_contract/validation.rs +++ b/crates/tracedecay-global-db/src/schema_contract/validation.rs @@ -7,8 +7,7 @@ use super::super::{global_db_operation_error, global_db_operation_message}; use super::definitions::{ Column, INDEX_DESCENDING_COLUMNS, INDEX_EXPRESSION_COLUMN, INDEXES, Index, REGISTRY_TABLE_NAMES, SESSION_RELATION_RECEIPTS_RECOVERY_DUE_INDEX, - SESSION_RELATION_RECEIPTS_WITHOUT_RECOVERY, SESSION_TEMPORAL_PROJECTION_RECEIPTS_V3, TABLES, - Table, + SESSION_RELATION_RECEIPTS_WITHOUT_RECOVERY, TABLES, Table, }; use super::pragma::{ ActualColumn, ActualForeignKey, ActualIndex, ActualTableMetadata, read_table_metadata, @@ -180,7 +179,7 @@ fn validate_table( } fn column_metadata_matches(actual: &ActualColumn, expected: &Column) -> bool { - actual.hidden == 0 + actual.hidden == expected.hidden && actual .declared_type .eq_ignore_ascii_case(expected.declared_type) @@ -493,12 +492,6 @@ pub async fn validate_session_temporal_schema_contract( validate_named_tables_and_indexes(conn, table_names).await } -pub async fn validate_released_v3_temporal_projection_receipt_contract( - conn: &impl QueryExecutor, -) -> tracedecay_domain::errors::Result<()> { - validate_contracts(conn, &[&SESSION_TEMPORAL_PROJECTION_RECEIPTS_V3]).await -} - /// Validates the exact v4 `session_relation_receipts` shape persisted before /// receipt recovery: the final columns minus the recovery columns, and the /// final index inventory minus the recovery-due index. diff --git a/crates/tracedecay-global-db/src/schema_stages.rs b/crates/tracedecay-global-db/src/schema_stages.rs index 56aeff0f89..65df360760 100644 --- a/crates/tracedecay-global-db/src/schema_stages.rs +++ b/crates/tracedecay-global-db/src/schema_stages.rs @@ -21,11 +21,7 @@ use tracedecay_runtime_core::{ }, }; use tracedecay_rusqlite_runtime::repository::AUTHORIZED_SCOPE_SET_SCHEMA_V1; -use tracedecay_rusqlite_runtime::runtime_ledger::{ - COPY_RETIRED_IDEMPOTENCY_LEDGER_PAGE_SQL, DELETE_CONVERGED_IDEMPOTENCY_LEDGER_PAGE_SQL, - DROP_RETIRED_IDEMPOTENCY_LEDGER_SQL, RETIRED_IDEMPOTENCY_LEDGER_PRESENT_SQL, - RUNTIME_LEDGER_SCHEMA, -}; +use tracedecay_rusqlite_runtime::runtime_ledger::RUNTIME_LEDGER_SCHEMA; use tracedecay_rusqlite_runtime::work::{ RETIRE_WORK_EVENT_JOURNAL_V1, WORK_PRODUCT_SCHEMA_V1 as WORK_PRODUCT_GRAPH_JOURNAL_SCHEMA_V1, WORK_SCHEMA_V1, @@ -228,88 +224,13 @@ const TRANSCRIPT_SCHEMA: &str = " CREATE INDEX IF NOT EXISTS idx_sessions_active_project_path ON sessions(project_path, provider, session_id) WHERE ended_at IS NULL; - CREATE TABLE IF NOT EXISTS session_messages ( - provider TEXT NOT NULL, - message_id TEXT NOT NULL, - session_id TEXT NOT NULL, - role TEXT NOT NULL, - timestamp INTEGER, - ordinal INTEGER NOT NULL, - text TEXT NOT NULL, - kind TEXT, - model TEXT, - tool_names TEXT, - source_path TEXT, - source_offset INTEGER, - metadata_json TEXT, - PRIMARY KEY(provider, message_id), - FOREIGN KEY(provider, session_id) - REFERENCES sessions(provider, session_id) ON DELETE CASCADE - ); - CREATE INDEX IF NOT EXISTS idx_session_messages_session - ON session_messages(provider, session_id, ordinal); - CREATE INDEX IF NOT EXISTS idx_session_messages_timestamp - ON session_messages(timestamp); - CREATE INDEX IF NOT EXISTS idx_session_messages_source - ON session_messages(source_path); CREATE TABLE IF NOT EXISTS session_backfill_meta ( key TEXT PRIMARY KEY, value TEXT NOT NULL, updated_at INTEGER NOT NULL DEFAULT (unixepoch()) ); - CREATE VIRTUAL TABLE IF NOT EXISTS session_messages_fts USING fts5( - text, role, kind, model, tool_names, - content='session_messages', content_rowid='rowid' - ); - CREATE TRIGGER IF NOT EXISTS session_messages_fts_insert - AFTER INSERT ON session_messages BEGIN - INSERT INTO session_messages_fts(rowid, text, role, kind, model, tool_names) - VALUES (NEW.rowid, NEW.text, NEW.role, NEW.kind, NEW.model, NEW.tool_names); - END; - CREATE TRIGGER IF NOT EXISTS session_messages_fts_delete - AFTER DELETE ON session_messages BEGIN - INSERT INTO session_messages_fts( - session_messages_fts, rowid, text, role, kind, model, tool_names - ) - VALUES ( - 'delete', OLD.rowid, OLD.text, OLD.role, OLD.kind, OLD.model, OLD.tool_names - ); - END; - CREATE TRIGGER IF NOT EXISTS session_messages_fts_update - AFTER UPDATE ON session_messages BEGIN - INSERT INTO session_messages_fts( - session_messages_fts, rowid, text, role, kind, model, tool_names - ) - VALUES ( - 'delete', OLD.rowid, OLD.text, OLD.role, OLD.kind, OLD.model, OLD.tool_names - ); - INSERT INTO session_messages_fts(rowid, text, role, kind, model, tool_names) - VALUES (NEW.rowid, NEW.text, NEW.role, NEW.kind, NEW.model, NEW.tool_names); - END; "; -/// The activity read fetches a bounded LIMIT of rows per session, so a -/// covering index buys little there, and copying `metadata_json` (kilobytes -/// per message) into it doubled the table's footprint: on one store the index -/// alone was 0.77 GB against 0.66 GB of table. The replacement covers the -/// ordering columns and lets the scan fetch the blob from the table. -/// -/// Both statements are store-sized, on a store with 184k messages the build -/// measured 33 s and dropping the blob-covering predecessor 1 m 54 s, so -/// neither belongs in the leased schema transaction, where each one outran -/// the per-statement execution limit and failed every open. They run as -/// separate independently durable batches on the long-lease migration writer: -/// the replacement is durable before the predecessor is dropped, an -/// interrupted migration never redoes a completed statement, and the activity -/// read uses whichever of the two is present, so no reader waits for this. -const SESSION_ACTIVITY_INDEX_MIGRATION_SQL: &[&str] = &[ - "CREATE INDEX IF NOT EXISTS idx_session_messages_session_activity_v2 - ON session_messages( - provider, session_id, timestamp, ordinal, message_id, kind, tool_names - );", - "DROP INDEX IF EXISTS idx_session_messages_session_activity;", -]; - const DELIVERY_SETTLEMENT_SCHEMA: &str = " CREATE TABLE IF NOT EXISTS delivery_fanout_events ( project_id TEXT NOT NULL, @@ -428,18 +349,7 @@ pub async fn ensure_registered_schema( "new registered schema installation was not classified fresh", )); } - ensure_fresh_authority_invariants(installation).await?; - // A fresh store's session table is empty, so the statements that cost a - // migration on a populated store cost nothing here: run them and the - // store is installed already converged, with the activity index present - // for its first read. Nothing can be carrying the retired external-source - // tables a fresh install never creates. - for sql in SESSION_ACTIVITY_INDEX_MIGRATION_SQL { - installation.execute_batch(sql).await.map_err(|error| { - global_db_operation_error("install the session activity index", error) - })?; - } - Ok(()) + ensure_fresh_authority_invariants(installation).await } #[derive(Clone, Copy)] @@ -484,17 +394,13 @@ struct RegisteredSchemaAdmissionClassification { #[hotpath::measure(future = true, label = "global_db.schema.query.classify")] async fn classify_registered_schema_admission( connection: &impl QueryExecutor, - binding: &tracedecay_store::StoreRuntimeBindingV1, ) -> tracedecay_domain::errors::Result { - Box::pin(classify_registered_schema_authorities(connection, binding)).await + Box::pin(classify_registered_schema_authorities(connection)).await } async fn classify_registered_schema_authorities( connection: &impl QueryExecutor, - binding: &tracedecay_store::StoreRuntimeBindingV1, ) -> tracedecay_domain::errors::Result { - crate::registered_legacy_relations::reject_legacy_session_relation_shape(connection, binding) - .await?; // The LCM authority classifies profile content first: a legacy or // version-skewed session store must surface its own ProfileResetRequired // state instead of being masked by the coarser workflow/configuration @@ -558,6 +464,32 @@ async fn classify_registered_schema_authorities( }) } +/// Authority named by the typed reset an existing store receives when the +/// composed authority schema (observation, projection, and anchor tables with +/// their invariant triggers) drifted from the contract. No such shape migrates +/// in place: the store stays untouched for the operator's reset decision. A +/// fresh install failing the same check is a programming error and keeps its +/// storage classification. +const AUTHORITY_SCHEMA_AUTHORITY: &str = "authority schema"; + +async fn validate_admitted_authority_schema( + conn: &impl QueryExecutor, + is_fresh: bool, +) -> tracedecay_domain::errors::Result<()> { + validate_authority_schema_contract(conn) + .await + .map_err(|error| { + if is_fresh { + error + } else { + tracedecay_domain::errors::TraceDecayError::reset_required( + AUTHORITY_SCHEMA_AUTHORITY, + error.to_string(), + ) + } + }) +} + /// Installs the minimum schema and write guards required before a registered /// runtime may be published. Historical convergence remains separately /// resumable so daemon admission never waits for whole-store scans. @@ -570,7 +502,7 @@ pub async fn ensure_registered_schema_for_admission( configuration_fresh, temporal_admission, workflow_admission, - } = classify_registered_schema_admission(installation, installation.binding()).await?; + } = classify_registered_schema_admission(installation).await?; let is_fresh = configuration_fresh.is_some(); let force_exhaustive = !authority_invariant_triggers_intact(installation).await?; let transaction = installation @@ -606,7 +538,7 @@ pub async fn ensure_registered_schema_for_admission( global_db_operation_error("initialize LCM status performance indexes", error) })?; } - validate_authority_schema_contract(installation).await?; + validate_admitted_authority_schema(installation, is_fresh).await?; Ok(RegisteredSchemaConvergence { force_exhaustive, is_fresh, @@ -950,128 +882,9 @@ async fn converge_registered_schema_on( database: &Database, convergence: RegisteredSchemaConvergence, ) -> tracedecay_domain::errors::Result<()> { - converge_store_sized_migrations(database).await?; ensure_authority_invariants(database, convergence.force_exhaustive, convergence.is_fresh).await } -/// Copies the released WITHOUT ROWID ledger in bounded transactions after -/// admission. Each committed page releases the canonical writer, and runtime -/// submissions continue to consult V1 until the final exact-schema retirement. -#[hotpath::measure( - future = true, - label = "global_db.schema.persist.converge_runtime_ledger" -)] -pub async fn converge_runtime_writer_ledger( - database: &Database, -) -> tracedecay_domain::errors::Result<()> { - let installation = database - .begin_write_transaction("install current runtime writer ledger") - .await?; - installation - .execute_batch(RUNTIME_LEDGER_SCHEMA) - .await - .map_err(|error| { - global_db_operation_error("install current runtime writer ledger", error) - })?; - installation.commit().await?; - - loop { - let transaction = database - .begin_write_transaction("converge runtime writer ledger page") - .await?; - let mut presence = transaction - .query(RETIRED_IDEMPOTENCY_LEDGER_PRESENT_SQL, ()) - .await - .map_err(|error| { - global_db_operation_error("inspect retired runtime writer ledger", error) - })?; - let present = presence - .next() - .await - .map_err(|error| { - global_db_operation_error("read retired runtime writer ledger state", error) - })? - .is_some(); - drop(presence); - if !present { - transaction.commit().await?; - return Ok(()); - } - - let copied = transaction - .execute(COPY_RETIRED_IDEMPOTENCY_LEDGER_PAGE_SQL, ()) - .await - .map_err(|error| { - global_db_operation_error("copy retired runtime writer ledger page", error) - })?; - let retired = transaction - .execute(DELETE_CONVERGED_IDEMPOTENCY_LEDGER_PAGE_SQL, ()) - .await - .map_err(|error| { - global_db_operation_error("retire converged runtime writer ledger page", error) - })?; - let mut remaining = transaction - .query("SELECT 1 FROM td_runtime_writer_idempotency_v1 LIMIT 1", ()) - .await - .map_err(|error| { - global_db_operation_error("inspect runtime writer ledger convergence", error) - })?; - let has_remaining = remaining - .next() - .await - .map_err(|error| { - global_db_operation_error("read runtime writer ledger convergence", error) - })? - .is_some(); - drop(remaining); - transaction.commit().await?; - - if !has_remaining { - // Current binaries never append V1. Once the last bounded page is - // committed, no producer can repopulate the retired table before - // this separately authorized exact-schema statement removes it. - database - .execute_authority_revalidated_batch( - "retire converged runtime writer ledger", - DROP_RETIRED_IDEMPOTENCY_LEDGER_SQL, - ) - .await?; - return Ok(()); - } - if copied == 0 && retired == 0 { - return Err(global_db_operation_message( - "converge runtime writer ledger", - "retired and current idempotency authorities disagree", - )); - } - } -} - -/// Runs the migrations whose cost scales with the store rather than with the -/// schema. -/// -/// Installation is cheap idempotent `CREATE ... IF NOT EXISTS` and belongs in -/// the leased admission transaction. Rebuilding an index or rewriting rows -/// does not: each of these measured tens of seconds to minutes on a real -/// store, so inside that transaction they tripped the per-statement execution -/// limit and made every open of a large store fail. They run here instead, -/// after the fail-closed admission checks, on the long-lease migration -/// writer, releasing the writer between units, so admission, retrieval, and -/// ordinary writes never wait for them. -#[hotpath::measure(future = true, label = "global_db.schema.persist.converge_migrations")] -async fn converge_store_sized_migrations( - database: &Database, -) -> tracedecay_domain::errors::Result<()> { - converge_migration_batches( - database, - "migrate the session activity index", - SESSION_ACTIVITY_INDEX_MIGRATION_SQL, - ) - .await?; - tracedecay_runtime_core::db::migrate_retired_mutation_copy_tables(database).await?; - converge_runtime_writer_ledger(database).await -} - /// Synchronously converges an attached existing store's historical schema. /// /// Short-lived attaches have no background maintenance task, so they build the @@ -1128,8 +941,7 @@ pub async fn ensure_attached_registered_schema( configuration_fresh, temporal_admission, workflow_admission, - } = classify_registered_schema_admission(&read_connection, database.registered_binding()) - .await?; + } = classify_registered_schema_admission(&read_connection).await?; let force_exhaustive = !authority_invariant_triggers_intact(&read_connection).await?; let transaction = database .begin_bulk_write_transaction("install attached registered global database schema") @@ -1156,7 +968,7 @@ pub async fn ensure_attached_registered_schema( })?; transaction.commit().await?; } - validate_authority_schema_contract(&read_connection).await?; + validate_admitted_authority_schema(&read_connection, configuration_fresh.is_some()).await?; Ok(RegisteredSchemaConvergence { force_exhaustive, is_fresh: configuration_fresh.is_some(), diff --git a/crates/tracedecay-global-db/src/session_temporal_handle.rs b/crates/tracedecay-global-db/src/session_temporal_handle.rs index 80d7e097be..67eb2c3ee0 100644 --- a/crates/tracedecay-global-db/src/session_temporal_handle.rs +++ b/crates/tracedecay-global-db/src/session_temporal_handle.rs @@ -15,8 +15,7 @@ use tracedecay_session_temporal_store::relations::{ SessionRelationGraphStore, SessionRelationScope, }; use tracedecay_session_temporal_store::{ - SessionTemporalAccess, SessionTemporalExec, SessionTemporalQuery, SessionTemporalRegisteredDb, - SessionTemporalWriteTxn, + SessionTemporalExec, SessionTemporalQuery, SessionTemporalRegisteredDb, SessionTemporalWriteTxn, }; use crate::{RegisteredGlobalDb, RegisteredGlobalDbWriteTransaction}; @@ -93,204 +92,3 @@ impl SessionTemporalRegisteredDb for RegisteredGlobalDb { RegisteredGlobalDb::project_graph_runtime(self) } } - -/// Composition wrappers so existing `RegisteredGlobalDb` call sites keep -/// working. This is not a module re-export of the temporal crate. -impl RegisteredGlobalDb { - pub fn git_scope_session_ids( - &self, - filter: &tracedecay_sessions::runtime::git_correlation::GitScopeFilter, - ) -> Result< - Option>, - tracedecay_sessions::runtime::git_correlation::GitCorrelationError, - > { - SessionTemporalAccess::new(self).git_scope_session_ids(filter) - } - - pub fn git_scope_session_ids_bounded( - &self, - filter: &tracedecay_sessions::runtime::git_correlation::GitScopeFilter, - maximum: usize, - ) -> Result< - Option>, - tracedecay_sessions::runtime::git_correlation::GitCorrelationError, - > { - SessionTemporalAccess::new(self).git_scope_session_ids_bounded(filter, maximum) - } - - #[hotpath::measure(future = true, label = "global_db.session_temporal.doctor_health")] - pub async fn session_temporal_doctor_health( - &self, - ) -> tracedecay_session_temporal_store::SessionTemporalHealthReport { - SessionTemporalAccess::new(self) - .session_temporal_doctor_health() - .await - } - - #[hotpath::measure(future = true, label = "global_db.session_temporal.ensure_cursor_key")] - pub async fn ensure_active_session_cursor_key_result( - &self, - ) -> tracedecay_store::SessionStoreResult { - SessionTemporalAccess::new(self) - .ensure_active_session_cursor_key_result() - .await - } - - #[hotpath::measure( - future = true, - label = "global_db.session_temporal.load_cursor_key_provider" - )] - pub async fn load_session_cursor_key_provider_result( - &self, - ) -> Result< - tracedecay_session_temporal_store::SessionTemporalCursorKeyProvider, - tracedecay_session_temporal_store::SessionTemporalCursorKeyProviderError, - > { - SessionTemporalAccess::new(self) - .load_session_cursor_key_provider_result() - .await - } - - #[hotpath::measure( - future = true, - label = "global_db.session_temporal.load_preprovisioned_cursor_key" - )] - pub async fn load_preprovisioned_session_cursor_key_provider_result( - &self, - ) -> Result< - tracedecay_session_temporal_store::SessionTemporalCursorKeyProvider, - tracedecay_session_temporal_store::SessionTemporalCursorKeyProviderError, - > { - SessionTemporalAccess::new(self) - .load_preprovisioned_session_cursor_key_provider_result() - .await - } - - #[hotpath::measure( - future = true, - label = "global_db.session_temporal.pending_refresh_page" - )] - pub async fn pending_session_temporal_refresh_page_result( - &self, - limit: usize, - active_scan_slots: usize, - active_after: Option<&tracedecay_domain::SessionId>, - ) -> tracedecay_store::SessionStoreResult< - tracedecay_session_temporal_store::SessionTemporalRefreshDiscoveryPage, - > { - SessionTemporalAccess::new(self) - .pending_session_temporal_refresh_page_result(limit, active_scan_slots, active_after) - .await - } - - #[hotpath::measure( - future = true, - label = "global_db.session_temporal.materialize_refresh_batch" - )] - pub async fn materialize_session_temporal_refresh_batch_result( - &self, - recovery: &tracedecay_session_temporal_store::SessionRefreshRecoveryV1, - ) -> tracedecay_store::SessionStoreResult< - Option<( - tracedecay_store::SessionRefreshProgressV1, - tracedecay_store::SessionTemporalProjectionBatchV1, - )>, - > { - SessionTemporalAccess::new(self) - .materialize_session_temporal_refresh_batch_result(recovery) - .await - } - - #[hotpath::measure(future = true, label = "global_db.session_temporal.freeze_snapshot")] - pub async fn freeze_session_temporal_snapshot_result( - &self, - request: tracedecay_store::SessionTemporalSnapshotRequestV1, - ) -> tracedecay_store::SessionStoreResult { - SessionTemporalAccess::new(self) - .freeze_session_temporal_snapshot_result(request) - .await - } - - #[hotpath::skip] - pub async fn active_session_summary_relations( - &self, - session_id: &tracedecay_domain::SessionId, - summary_ids: &[String], - max_relations: usize, - cancellation: std::sync::Arc, - ) -> tracedecay_store::SessionStoreResult<( - tracedecay_domain::SessionProjectionGenerationV1, - Vec, - )> { - SessionTemporalAccess::new(self) - .active_session_summary_relations(session_id, summary_ids, max_relations, cancellation) - .await - } - - #[hotpath::measure( - future = true, - label = "global_db.session_temporal.apply_relation_projection" - )] - pub async fn apply_active_session_relation_projection( - &self, - session_id: &tracedecay_domain::SessionId, - cancellation: std::sync::Arc, - ) -> tracedecay_store::SessionStoreResult { - SessionTemporalAccess::new(self) - .apply_active_session_relation_projection(session_id, cancellation) - .await - } - - #[hotpath::measure( - future = true, - label = "global_db.session_temporal.recover_relation_projections" - )] - pub async fn recover_pending_session_relation_projections( - &self, - limit: usize, - cancellation: std::sync::Arc, - ) -> tracedecay_store::SessionStoreResult { - SessionTemporalAccess::new(self) - .recover_pending_session_relation_projections(limit, cancellation) - .await - } - - #[hotpath::measure( - future = true, - label = "global_db.session_temporal.recover_relation_projection_page" - )] - pub async fn recover_pending_session_relation_projection_page( - &self, - limit: usize, - cancellation: std::sync::Arc, - ) -> tracedecay_store::SessionStoreResult< - tracedecay_session_temporal_store::SessionRelationRecoveryPage, - > { - SessionTemporalAccess::new(self) - .recover_pending_session_relation_projection_page(limit, cancellation) - .await - } - - #[hotpath::measure(future = true, label = "global_db.session_temporal.refresh_recovery")] - pub async fn session_refresh_recovery_result( - &self, - session_id: &tracedecay_domain::SessionId, - ) -> tracedecay_store::SessionStoreResult< - Option, - > { - SessionTemporalAccess::new(self) - .session_refresh_recovery_result(session_id) - .await - } - - #[hotpath::measure(future = true, label = "global_db.session_temporal.complete_refresh")] - pub async fn complete_session_refresh_result( - &self, - request: tracedecay_store::SessionRefreshCompletionRequestV1, - execution_control: tracedecay_temporal_query::ports::ExecutionControl, - ) -> tracedecay_store::SessionStoreResult { - SessionTemporalAccess::new(self) - .complete_session_refresh_result(request, execution_control) - .await - } -} diff --git a/crates/tracedecay-global-db/src/session_temporal_schema.rs b/crates/tracedecay-global-db/src/session_temporal_schema.rs index e33df3667c..965dbc0505 100644 --- a/crates/tracedecay-global-db/src/session_temporal_schema.rs +++ b/crates/tracedecay-global-db/src/session_temporal_schema.rs @@ -14,17 +14,16 @@ use admission::{validate_temporal_fts_contracts, validate_temporal_fts_match}; const OPERATION: &str = "initialize session temporal schema"; const MIGRATION_NAME: &str = "session-temporal"; const SESSION_TEMPORAL_AUTHORITY: &str = "session temporal"; -const RELEASED_SESSION_TEMPORAL_SCHEMA_VERSION: i64 = 3; pub(crate) use tracedecay_session_temporal_store::SESSION_TEMPORAL_SCHEMA_VERSION; const TEMPORAL_FTS_CONTRACTS: &[(&str, &str)] = &[ ( "session_occurrences_fts", - "createvirtualtablesession_occurrences_ftsusingfts5(index_text,snippet_text,content='session_occurrences',content_rowid='rowid')", + "createvirtualtablesession_occurrences_ftsusingfts5(index_text,content='session_occurrences',content_rowid='rowid')", ), ( "session_summary_nodes_fts", - "createvirtualtablesession_summary_nodes_ftsusingfts5(summary_text,index_text,content='session_summary_nodes',content_rowid='rowid')", + "createvirtualtablesession_summary_nodes_ftsusingfts5(summary_text,content='session_summary_nodes',content_rowid='rowid')", ), ]; @@ -34,12 +33,26 @@ const TEMPORAL_SCHEMA_DDL: &str = r" version INTEGER NOT NULL CHECK(version > 0), applied_at INTEGER NOT NULL ); + -- The one summary authority. LCM reads (grep, describe, expand, replay, + -- status, DAG) and temporal retrieval share these rows; a summary is + -- visible to LCM reads only while its availability in the session's + -- active generation is 'available'. Rows are immutable; retirement is an + -- availability state, never a delete. CREATE TABLE IF NOT EXISTS session_summary_nodes ( summary_id TEXT PRIMARY KEY, session_id TEXT NOT NULL, + provider TEXT NOT NULL, + conversation_id TEXT NOT NULL, + depth INTEGER NOT NULL, summary_anchor_id TEXT NOT NULL, summary_text TEXT NOT NULL, - index_text TEXT NOT NULL, + summary_hash TEXT NOT NULL, + summary_token_count INTEGER NOT NULL, + source_token_count INTEGER NOT NULL, + source_time_start INTEGER, + source_time_end INTEGER, + expand_hint TEXT, + metadata_json TEXT, source_horizon_json TEXT NOT NULL CHECK(json_valid(source_horizon_json)), publication_json TEXT CHECK(publication_json IS NULL OR json_valid(publication_json)), created_at INTEGER NOT NULL, @@ -49,6 +62,31 @@ const TEMPORAL_SCHEMA_DDL: &str = r" ON session_summary_nodes(session_id, created_at); CREATE INDEX IF NOT EXISTS idx_session_summary_nodes_root_created_order ON session_summary_nodes(created_at, session_id, summary_id); + CREATE INDEX IF NOT EXISTS idx_session_summary_nodes_session_depth_time + ON session_summary_nodes( + provider, session_id, depth, source_time_start, source_time_end, created_at + ); + -- Covers the lcm_status depth rollup (COUNT + SUM per depth) so status + -- never reads summary_text records. + CREATE INDEX IF NOT EXISTS idx_session_summary_nodes_depth_tokens + ON session_summary_nodes( + provider, session_id, depth, summary_token_count, source_token_count + ); + + -- Ordered lineage of one summary: `source_id` is the raw `store_id` + -- rendered as text for 'raw_message' sources and the child `summary_id` + -- for 'summary_node' sources. Retention treats a raw row as + -- projection-durable once a row here covers it. + CREATE TABLE IF NOT EXISTS session_summary_sources ( + summary_id TEXT NOT NULL, + ordinal INTEGER NOT NULL CHECK(ordinal >= 0), + source_kind TEXT NOT NULL CHECK(source_kind IN ('raw_message', 'summary_node')), + source_id TEXT NOT NULL, + PRIMARY KEY(summary_id, ordinal), + FOREIGN KEY(summary_id) REFERENCES session_summary_nodes(summary_id) + ); + CREATE INDEX IF NOT EXISTS idx_session_summary_sources_source + ON session_summary_sources(source_kind, source_id, summary_id); CREATE TABLE IF NOT EXISTS session_relation_receipts ( session_id TEXT NOT NULL, @@ -358,7 +396,7 @@ const TEMPORAL_SCHEMA_DDL: &str = r" AND sanitized_content_digest NOT GLOB '*[^0-9a-f]*' ), sanitized_content_bytes INTEGER NOT NULL CHECK(sanitized_content_bytes >= 0), - snippet_text TEXT NOT NULL, + snippet_text TEXT NOT NULL GENERATED ALWAYS AS (index_text) VIRTUAL, index_text TEXT NOT NULL, PRIMARY KEY(session_id, generation, occurrence_id), FOREIGN KEY(session_id, generation) @@ -570,13 +608,11 @@ const TEMPORAL_SCHEMA_DDL: &str = r" CREATE VIRTUAL TABLE IF NOT EXISTS session_occurrences_fts USING fts5( index_text, - snippet_text, content='session_occurrences', content_rowid='rowid' ); CREATE VIRTUAL TABLE IF NOT EXISTS session_summary_nodes_fts USING fts5( summary_text, - index_text, content='session_summary_nodes', content_rowid='rowid' ); diff --git a/crates/tracedecay-global-db/src/session_temporal_schema/admission.rs b/crates/tracedecay-global-db/src/session_temporal_schema/admission.rs index e5aa349e04..6db874c499 100644 --- a/crates/tracedecay-global-db/src/session_temporal_schema/admission.rs +++ b/crates/tracedecay-global-db/src/session_temporal_schema/admission.rs @@ -5,9 +5,7 @@ use tracedecay_runtime_core::db::engine::{QueryExecutor, params}; use crate::configuration::FreshConfigurationStoreEvidence; use crate::schema_contract::{ - SESSION_RELATION_RECEIPT_RECOVERY_COLUMNS, invariant_trigger_names_for_tables, - released_v3_invariant_triggers_intact, starts_with_ignore_ascii_case, - validate_released_v3_temporal_projection_receipt_contract, + SESSION_RELATION_RECEIPT_RECOVERY_COLUMNS, starts_with_ignore_ascii_case, validate_session_graph_publication_schema_contract, validate_session_relation_receipts_without_recovery_contract, validate_session_temporal_schema_contract, @@ -15,9 +13,8 @@ use crate::schema_contract::{ use crate::{global_db_operation_error, global_db_operation_message}; use super::{ - MIGRATION_NAME, OPERATION, RELEASED_SESSION_TEMPORAL_SCHEMA_VERSION, - SESSION_TEMPORAL_AUTHORITY, SESSION_TEMPORAL_SCHEMA_VERSION, TEMPORAL_FTS_CONTRACTS, - TEMPORAL_TABLE_COLUMNS, + MIGRATION_NAME, OPERATION, SESSION_TEMPORAL_AUTHORITY, SESSION_TEMPORAL_SCHEMA_VERSION, + TEMPORAL_FTS_CONTRACTS, TEMPORAL_TABLE_COLUMNS, }; const TEMPORAL_FTS_SHADOW_TABLES: &[&str] = &[ @@ -41,118 +38,10 @@ const SESSION_RELATION_RECEIPTS_TABLE: &str = "session_relation_receipts"; const SESSION_RELATION_RECEIPTS_WITHOUT_RECOVERY_DIGEST: &str = "867dc83c80264f4b13aeab7f1ac51572a88ee5d614739a701ebddbb8dcb84a80"; -// Exact normalized CREATE TABLE authority published by every v3 release -// (v0.1.0-beta.25 through v0.1.0-beta.37 ship one byte-identical temporal DDL). -// The structural PRAGMA contract cannot observe CHECK expressions, so -// released-v3 admission also pins every durable temporal table definition by -// digest. -const RELEASED_V3_TEMPORAL_TABLE_DIGESTS: &[(&str, &str)] = &[ - ( - "session_agents", - "94d2e78d1ea2030560a21360e7ee6dc12a03d0e82ea14e2ddee08baec83bf367", - ), - ( - "session_assertion_supersession", - "8bc0df1352864a9777c9a2dbaf33abd34f2c8fc74751a23f2b7c74348ec8f93e", - ), - ( - "session_assertions", - "e4c4b5dba6ea971f33079249a724cbf76c2da7a9316c083adb42aaba29e4d963", - ), - ( - "session_current_entities", - "ae412c1be5634a6667a3223f3e9687d72fd111e46c7b179d3814531727ebad89", - ), - ( - "session_derived_evidence", - "d0b67a582d4c25bef3337dfc7f5150c2449bc6754cf63e4f56fe052f6342ad36", - ), - ( - "session_derived_evidence_members", - "985db57f82cb2693097e4b779916316b05b0432e17e7fb30adedcb89d731402b", - ), - ( - "session_external_payload_manifests", - "35941f805b9f94bd7acac50226e6bf181f3fd0f6578f3ba2779f300703cdee50", - ), - ( - "session_occurrences", - "e1eda19c4d136c5d64480a562867dd2de6bd509a389df75f3269df3b3026c565", - ), - ( - "session_query_cursor_keys", - "b50e9d3ea86a4c0fb675a3e2a7b814e00a38d6ab52d62b2423f9271cd3fab1b7", - ), - ( - "session_refresh_batch_bindings", - "0ca52e44d29058dda06caeca4287653ef2238b0d2b4f45669ab19462443552ed", - ), - ( - "session_refresh_bindings", - "542a3e5138627d7ffe8a97b50bb0f8f43d4b5e8806b2d2c62a597f5787122558", - ), - ( - "session_refresh_operations", - "abceb5507d1470be9e552ae36a8f508e349200b8d3f34b905aafeadbd715f27f", - ), - ( - "session_refresh_progress", - "b5a85a981f23b186c0106e160b22dba2db9a8e233232c51725fcb7eec101a6a1", - ), - ( - "session_refresh_receipts", - "8a01bd241fb669833630b2ba7976da8296f7602768caeffddbb15e6a270f3039", - ), - ( - "session_relation_effect_journal", - "73e372d47f338bae3e25d461c76f14e8b9b7a1606184993450fbe6a9965c7e12", - ), - ( - SESSION_RELATION_RECEIPTS_TABLE, - SESSION_RELATION_RECEIPTS_WITHOUT_RECOVERY_DIGEST, - ), - ( - "session_summary_availability", - "e1da9569afbb4bb2829404ec1deefccc35d21ce3c6a65c3ca28800cca0d8c79e", - ), - ( - "session_summary_nodes", - "9cd68c6f2f0e4224db1a0cc107f562157c62942b2349320c573be85128a040e6", - ), - ( - "session_temporal_generations", - "731f643fa5d08f7902ea3ae06fbc672cd52923acaaabffdb9985b2c806143c34", - ), - ( - "session_temporal_observation_effects", - "614ca8fb3b21b2e0d8c08dc009c7e78ae1c55e87d52804a673bc84a8e52f375e", - ), - ( - "session_temporal_projection_receipts", - "63f8c00ff8b62d060c57fe09046acd223b24a56ebf4cfa8a6f99eb2d938d23f1", - ), - ( - "session_temporal_schema_migrations", - "8e9267a82387fa36ed23b4b339bb23baf0fd75d125b53e114851bfee2b515619", - ), - ( - "session_threads", - "f94bfebac5f60f42dab193fe09348865fb416685d48a952093ac566d4c89ea98", - ), - ( - "session_turn_members", - "9b4bc136a68ab8af12eb26bdfb832d7a09f68f0b2494cabeb3fd3b52e118f023", - ), - ( - "session_turns", - "d67b106b4b594c91f5443e8a5f9cbee643349a495766adfb312b758c6b7bb1a5", - ), -]; - /// Read-only admission result for the final session-temporal schema. /// -/// Non-final shapes, including the published v3 marker and the unreleased -/// pre-recovery v4 receipt table, are not variants: admission returns +/// Non-final shapes, including every earlier version and the unreleased +/// pre-recovery receipt table, are not variants: admission returns /// `ResetRequired` before any conversion. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum SessionTemporalSchemaAdmission { @@ -184,14 +73,6 @@ pub(crate) async fn require_admissible_session_temporal_schema( validate_current_session_temporal_schema(conn).await?; Ok(SessionTemporalSchemaAdmission::Current) } - Some(RELEASED_SESSION_TEMPORAL_SCHEMA_VERSION) => { - validate_released_v3_session_temporal_schema(conn).await?; - Err(session_temporal_reset_required( - "persisted session temporal schema is the published v3 shape; the final shape \ - is required and there is no sanctioned conversion, reset the session temporal \ - authority", - )) - } Some(version) => Err(session_temporal_reset_required(format!( "persisted schema version {version} does not match final version {SESSION_TEMPORAL_SCHEMA_VERSION}" ))), @@ -297,61 +178,6 @@ async fn validate_temporal_namespace_and_fts( .map_err(|error| session_temporal_reset_required(error.to_string())) } -pub(super) async fn validate_released_v3_session_temporal_schema( - conn: &impl QueryExecutor, -) -> tracedecay_domain::errors::Result<()> { - let tables = TEMPORAL_TABLE_COLUMNS - .iter() - .map(|(table, _)| *table) - .filter(|table| { - !table.ends_with("_fts") - && *table != "session_temporal_projection_receipts" - && *table != "session_relation_receipts" - }) - .collect::>(); - validate_session_temporal_schema_contract(conn, &tables) - .await - .map_err(|error| session_temporal_reset_required(error.to_string()))?; - validate_released_v3_temporal_projection_receipt_contract(conn) - .await - .map_err(|error| session_temporal_reset_required(error.to_string()))?; - validate_released_v3_temporal_table_definitions(conn).await?; - if !released_v3_invariant_triggers_intact(conn) - .await - .map_err(|error| session_temporal_reset_required(error.to_string()))? - { - return Err(session_temporal_reset_required( - "released v3 authority trigger contracts are absent or incompatible", - )); - } - validate_released_v3_temporal_trigger_inventory(conn).await?; - validate_temporal_namespace_and_fts(conn).await -} - -async fn validate_released_v3_temporal_table_definitions( - conn: &impl QueryExecutor, -) -> tracedecay_domain::errors::Result<()> { - let expected_tables = TEMPORAL_TABLE_COLUMNS - .iter() - .map(|(table, _)| *table) - .filter(|table| !table.ends_with("_fts")) - .collect::>(); - let contract_tables = RELEASED_V3_TEMPORAL_TABLE_DIGESTS - .iter() - .map(|(table, _)| *table) - .collect::>(); - if contract_tables != expected_tables { - return Err(global_db_operation_message( - OPERATION, - "released v3 CREATE TABLE authority is incomplete", - )); - } - for (table, expected_digest) in RELEASED_V3_TEMPORAL_TABLE_DIGESTS { - validate_temporal_table_definition_digest(conn, table, expected_digest).await?; - } - Ok(()) -} - /// Pins a persisted CREATE TABLE definition by normalized digest so CHECK /// expressions, which the PRAGMA contract cannot observe, are admitted exactly. async fn validate_temporal_table_definition_digest( @@ -387,53 +213,6 @@ async fn validate_temporal_table_definition_digest( Ok(()) } -async fn validate_released_v3_temporal_trigger_inventory( - conn: &impl QueryExecutor, -) -> tracedecay_domain::errors::Result<()> { - let temporal_tables = TEMPORAL_TABLE_COLUMNS - .iter() - .map(|(table, _)| *table) - .collect::>(); - let expected = invariant_trigger_names_for_tables(&temporal_tables) - .into_iter() - .collect::>(); - let temporal_tables = temporal_tables.into_iter().collect::>(); - let mut actual = BTreeSet::new(); - let mut rows = conn - .query( - "SELECT name, tbl_name FROM sqlite_master WHERE type = 'trigger' ORDER BY name", - (), - ) - .await - .map_err(|error| global_db_operation_error(OPERATION, error))?; - while let Some(row) = rows - .next() - .await - .map_err(|error| global_db_operation_error(OPERATION, error))? - { - let name = row - .get::(0) - .map_err(|error| global_db_operation_error(OPERATION, error))?; - let table = row - .get::(1) - .map_err(|error| global_db_operation_error(OPERATION, error))?; - if temporal_tables.contains(table.as_str()) { - actual.insert(name); - } - } - if actual.len() != expected.len() - || actual - .iter() - .map(String::as_str) - .ne(expected.iter().copied()) - { - return Err(session_temporal_reset_required( - "released v3 temporal trigger inventory is not exact", - )); - } - Ok(()) -} - async fn validate_temporal_namespace_tables( conn: &impl QueryExecutor, ) -> tracedecay_domain::errors::Result<()> { diff --git a/crates/tracedecay-global-db/src/store_registration.rs b/crates/tracedecay-global-db/src/store_registration.rs index ec66007f8d..51a5c0516b 100644 --- a/crates/tracedecay-global-db/src/store_registration.rs +++ b/crates/tracedecay-global-db/src/store_registration.rs @@ -94,10 +94,6 @@ pub async fn register_project_store( ) -> Result<()> { static REGISTRY_WRITE_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); - if store_layout.storage_mode != storage::StorageMode::ProfileSharded { - return Ok(()); - } - let project_id = store_layout.identity.project_id.as_deref().ok_or_else(|| { registry_registration_error("profile-sharded store has no project identity") })?; diff --git a/crates/tracedecay-global-db/src/tests/harness.rs b/crates/tracedecay-global-db/src/tests/harness.rs index 6d1b7a6ccd..c20d40d47c 100644 --- a/crates/tracedecay-global-db/src/tests/harness.rs +++ b/crates/tracedecay-global-db/src/tests/harness.rs @@ -6,6 +6,7 @@ use std::sync::atomic::{AtomicU64, Ordering}; use tempfile::TempDir; use crate::{RegisteredGlobalDb, RegisteredGlobalDbLeaseV1, RegisteredGlobalDbOwnerV1}; +use tracedecay_domain::canonical_text::sha256_hex; use tracedecay_runtime_core::db::DaemonDatabaseScope; #[cfg(test)] use tracedecay_runtime_core::db::engine::{Executor, IntoParams, QueryExecutor, Rows}; @@ -583,7 +584,7 @@ impl HostAdmissionTestRuntimeV1 { HOST_ADMISSION_TEST_BACKGROUND_CPU .get_or_init(|| Arc::new(ProcessBackgroundCpuV1::new(NonZeroUsize::MIN))), ); - tracedecay_sessions::runtime::codex::CodexDiscoveryHub::default() + tracedecay_sessions::runtime::hosts::codex::CodexDiscoveryHub::default() .configure_preparation_resources(memory, background_cpu) .map_err( |error| tracedecay_domain::errors::TraceDecayError::Database { @@ -809,7 +810,7 @@ impl HostAdmissionTestRuntimeV1 { provider: &str, session_id: &str, transcript_path: &std::path::Path, - ) -> tracedecay_domain::errors::Result<(i64, i64, i64, i64, i64, i64, i64)> { + ) -> tracedecay_domain::errors::Result<(i64, i64, i64, i64, i64, i64)> { let snapshot = self .session_database_for_test(scope)? .read_snapshot() @@ -819,8 +820,6 @@ impl HostAdmissionTestRuntimeV1 { "SELECT (SELECT COUNT(*) FROM sessions WHERE provider = ?1 AND session_id = ?2), - (SELECT COUNT(*) FROM session_messages - WHERE provider = ?1 AND session_id = ?2), (SELECT COUNT(*) FROM lcm_raw_messages WHERE provider = ?1 AND session_id = ?2), (SELECT COUNT(*) FROM lcm_raw_messages_fts @@ -828,14 +827,16 @@ impl HostAdmissionTestRuntimeV1 { ON raw.store_id = lcm_raw_messages_fts.rowid WHERE raw.provider = ?1 AND raw.session_id = ?2), (SELECT COUNT(*) FROM lcm_raw_messages_fts), - (SELECT COUNT(*) FROM lcm_summary_nodes + (SELECT COUNT(*) FROM session_summary_nodes WHERE provider = ?1 AND session_id = ?2), (SELECT COUNT(*) FROM parse_offsets WHERE file_path = ?3)", tracedecay_runtime_core::db::engine::params![ provider, session_id, - transcript_path.to_string_lossy().as_ref() + tracedecay_sessions::runtime::shared::path_identity_key( + transcript_path.to_string_lossy().as_ref() + ) ], ) .await?; @@ -852,7 +853,6 @@ impl HostAdmissionTestRuntimeV1 { row.get(3)?, row.get(4)?, row.get(5)?, - row.get(6)?, )) } @@ -868,7 +868,7 @@ impl HostAdmissionTestRuntimeV1 { .await?; let deleted = transaction .execute( - "DELETE FROM session_messages WHERE provider = ?1 AND message_id = ?2", + "DELETE FROM lcm_raw_messages WHERE provider = ?1 AND message_id = ?2", tracedecay_runtime_core::db::engine::params![provider, message_id], ) .await?; @@ -999,10 +999,10 @@ impl HostAdmissionTestRuntimeV1 { } let external_content = "canonical external payload"; - let external_hash = tracedecay_lcm::util::sha256_hex(external_content.as_bytes()); - let raw_hash = tracedecay_lcm::util::sha256_hex(b"canonical raw message"); - let child_summary_hash = tracedecay_lcm::util::sha256_hex(b"canonical child summary"); - let parent_summary_hash = tracedecay_lcm::util::sha256_hex(b"canonical parent summary"); + let external_hash = sha256_hex(external_content.as_bytes()); + let raw_hash = sha256_hex(b"canonical raw message"); + let child_summary_hash = sha256_hex(b"canonical child summary"); + let parent_summary_hash = sha256_hex(b"canonical parent summary"); let payload_dir = database .db_path() .parent() @@ -1033,48 +1033,64 @@ impl HostAdmissionTestRuntimeV1 { ); INSERT INTO lcm_raw_messages( provider, message_id, session_id, store_id, role, ordinal, timestamp, - content, content_hash, storage_kind, payload_ref, snippet_text, - index_text, legacy_source, legacy_truncated, metadata_json + content, content_hash, storage_kind, payload_ref, metadata_json ) VALUES ( 'codex', 'message-a', 'session-a', 11, 'assistant', 0, 11, 'canonical raw message', '{raw_hash}', 'inline', NULL, - 'canonical raw message', 'canonical raw message', 0, 0, '{raw_message_metadata}' ); INSERT INTO lcm_raw_messages( provider, message_id, session_id, store_id, role, ordinal, timestamp, - content, content_hash, storage_kind, payload_ref, snippet_text, - index_text, legacy_source, legacy_truncated, metadata_json + content, content_hash, storage_kind, payload_ref, placeholder_text, + metadata_json ) VALUES ( 'codex', 'message-b', 'session-a', 12, 'tool', 1, 12, NULL, '{external_hash}', 'external', 'payload-a', - 'canonical external payload', 'canonical external payload', 0, 0, + 'canonical external payload', '{external_message_metadata}' ); - INSERT INTO lcm_summary_nodes( - node_id, provider, conversation_id, session_id, depth, summary_text, - summary_hash, summary_token_count, source_token_count, - source_time_start, source_time_end, expand_hint, metadata_json, created_at + INSERT INTO retrieval_anchors ( + anchor_id, anchor_json, owner_json, projection_generation + ) VALUES ('summary-child-anchor', '{{}}', '{{}}', 'test'), + ('summary-parent-anchor', '{{}}', '{{}}', 'test'); + INSERT INTO session_summary_nodes( + summary_id, session_id, provider, conversation_id, depth, + summary_anchor_id, summary_text, summary_hash, summary_token_count, + source_token_count, source_time_start, source_time_end, expand_hint, + metadata_json, source_horizon_json, created_at ) VALUES ( - 'summary-child', 'codex', 'session-a', 'session-a', 0, - 'canonical child summary', '{child_summary_hash}', 3, 3, - 11, 11, NULL, NULL, 13 + 'summary-child', 'session-a', 'codex', 'session-a', 0, + 'summary-child-anchor', 'canonical child summary', + '{child_summary_hash}', 3, 3, 11, 11, NULL, NULL, '{{}}', 13 ); - INSERT INTO lcm_summary_nodes( - node_id, provider, conversation_id, session_id, depth, summary_text, - summary_hash, summary_token_count, source_token_count, - source_time_start, source_time_end, expand_hint, metadata_json, created_at + INSERT INTO session_summary_nodes( + summary_id, session_id, provider, conversation_id, depth, + summary_anchor_id, summary_text, summary_hash, summary_token_count, + source_token_count, source_time_start, source_time_end, expand_hint, + metadata_json, source_horizon_json, created_at ) VALUES ( - 'summary-parent', 'codex', 'session-a', 'session-a', 1, - 'canonical parent summary', '{parent_summary_hash}', 3, 6, - 11, 12, NULL, NULL, 14 + 'summary-parent', 'session-a', 'codex', 'session-a', 1, + 'summary-parent-anchor', 'canonical parent summary', + '{parent_summary_hash}', 3, 6, 11, 12, NULL, NULL, '{{}}', 14 ); - INSERT INTO lcm_summary_sources(node_id, source_kind, source_id, ordinal) + INSERT INTO session_summary_sources(summary_id, source_kind, source_id, ordinal) VALUES ('summary-child', 'raw_message', '11', 0); - INSERT INTO lcm_summary_sources(node_id, source_kind, source_id, ordinal) + INSERT INTO session_summary_sources(summary_id, source_kind, source_id, ordinal) VALUES ('summary-parent', 'summary_node', 'summary-child', 0); - INSERT INTO lcm_summary_sources(node_id, source_kind, source_id, ordinal) - VALUES ('summary-parent', 'raw_message', '12', 1);", + INSERT INTO session_summary_sources(summary_id, source_kind, source_id, ordinal) + VALUES ('summary-parent', 'raw_message', '12', 1); + INSERT INTO session_temporal_generations( + session_id, generation, state, frozen_watermarks_json, created_at + ) VALUES ('session-a', 1, 'building', '{{}}', 10); + UPDATE session_temporal_generations SET state = 'ready', ready_at = 10 + WHERE session_id = 'session-a' AND generation = 1; + UPDATE session_temporal_generations SET state = 'active', activated_at = 10 + WHERE session_id = 'session-a' AND generation = 1; + INSERT INTO session_summary_availability( + session_id, generation, summary_id, availability, + source_horizon_json, checked_at + ) VALUES ('session-a', 1, 'summary-child', 'available', '{{}}', 13), + ('session-a', 1, 'summary-parent', 'available', '{{}}', 14);", byte_count = external_content.len(), char_count = external_content.chars().count(), )) diff --git a/crates/tracedecay-global-db/src/tests/lcm_privacy_rescan.rs b/crates/tracedecay-global-db/src/tests/lcm_privacy_rescan.rs index 82472d1943..a277c68868 100644 --- a/crates/tracedecay-global-db/src/tests/lcm_privacy_rescan.rs +++ b/crates/tracedecay-global-db/src/tests/lcm_privacy_rescan.rs @@ -58,7 +58,7 @@ async fn seed_session(harness: &RegisteredGlobalDbHarness, session_id: &str) { .expect("seed session"); } -/// Persists one inline raw row plus its projection twin exactly as an older +/// Persists one inline message row exactly as an older /// ingest could have: receipt-bound bytes the current rules never evaluated. async fn seed_legacy_inline_row( harness: &RegisteredGlobalDbHarness, @@ -72,22 +72,13 @@ async fn seed_legacy_inline_row( }) .to_string(); let writer = harness.registered.writer_connection().expect("writer"); - writer - .execute( - "INSERT INTO session_messages(provider, message_id, session_id, role, ordinal, text) - VALUES ('cursor', ?1, ?2, 'user', 1, ?3)", - params![message_id, session_id, content], - ) - .await - .expect("seed legacy projection twin"); writer .execute( "INSERT INTO lcm_raw_messages( provider, message_id, session_id, role, ordinal, timestamp, - content, content_hash, storage_kind, payload_ref, snippet_text, - index_text, legacy_source, legacy_truncated, metadata_json + content, content_hash, storage_kind, payload_ref, metadata_json ) - VALUES ('cursor', ?1, ?2, 'user', 1, 10, ?3, ?4, 'inline', NULL, ?3, ?3, 0, 0, ?5)", + VALUES ('cursor', ?1, ?2, 'user', 1, 10, ?3, ?4, 'inline', NULL, ?5)", params![ message_id, session_id, @@ -154,10 +145,10 @@ async fn seed_legacy_external_row( .execute( "INSERT INTO lcm_raw_messages( provider, message_id, session_id, role, ordinal, timestamp, - content, content_hash, storage_kind, payload_ref, snippet_text, - index_text, legacy_source, legacy_truncated, metadata_json + content, content_hash, storage_kind, payload_ref, placeholder_text, + metadata_json ) - VALUES ('cursor', ?1, ?2, 'user', 2, 20, NULL, ?3, 'external', ?4, ?5, ?5, 0, 0, ?6)", + VALUES ('cursor', ?1, ?2, 'user', 2, 20, NULL, ?3, 'external', ?4, ?5, ?6)", params![ message_id, session_id, @@ -183,22 +174,13 @@ async fn seed_unreceipted_row( content: &str, ) { let writer = harness.registered.writer_connection().expect("writer"); - writer - .execute( - "INSERT INTO session_messages(provider, message_id, session_id, role, ordinal, text) - VALUES ('cursor', ?1, ?2, 'assistant', 3, ?3)", - params![message_id, session_id, content], - ) - .await - .expect("seed unreceipted projection twin"); writer .execute( "INSERT INTO lcm_raw_messages( provider, message_id, session_id, role, ordinal, timestamp, - content, content_hash, storage_kind, payload_ref, snippet_text, - index_text, legacy_source, legacy_truncated, metadata_json + content, content_hash, storage_kind, payload_ref, metadata_json ) - VALUES ('cursor', ?1, ?2, 'assistant', 3, 30, ?3, ?4, 'inline', NULL, ?3, ?3, 0, 0, NULL)", + VALUES ('cursor', ?1, ?2, 'assistant', 3, 30, ?3, ?4, 'inline', NULL, NULL)", params![ message_id, session_id, @@ -220,9 +202,7 @@ async fn count_rows_holding(harness: &RegisteredGlobalDbHarness, needle: &str) - WHERE COALESCE(content, '') LIKE ?1 OR snippet_text LIKE ?1 OR index_text LIKE ?1 - OR COALESCE(metadata_json, '') LIKE ?1) - + (SELECT COUNT(*) FROM session_messages - WHERE text LIKE ?1 OR COALESCE(metadata_json, '') LIKE ?1)", + OR COALESCE(metadata_json, '') LIKE ?1)", params![pattern], ) .await @@ -327,8 +307,8 @@ async fn at_rest_rescan_remediates_legacy_rows_and_settles_watermark() { assert_eq!(receipt.remediated_rows, 2); assert_eq!(receipt.unavailable_payload_rows, 0); - // The detector hit is gone from every at-rest surface: raw rows, the - // projection twin, and the payload directory (the replaced payload file + // The detector hit is gone from every at-rest surface: message rows and + // the payload directory (the replaced payload file // is deleted, not merely superseded). assert_eq!(count_rows_holding(&harness, &secret()).await, 0); assert!(!payload_dir_holds(&storage_root, &secret())); diff --git a/crates/tracedecay-global-db/src/tests/lcm_schema/lcm_schema_contract.rs b/crates/tracedecay-global-db/src/tests/lcm_schema/lcm_schema_contract.rs index 32ee3c9b44..e0c053fca0 100644 --- a/crates/tracedecay-global-db/src/tests/lcm_schema/lcm_schema_contract.rs +++ b/crates/tracedecay-global-db/src/tests/lcm_schema/lcm_schema_contract.rs @@ -93,25 +93,17 @@ async fn short_lived_attach_convergence_rebuilds_queryable_lcm_status_indexes() VALUES ('cursor', 'status-index-session', 'project.status-index', '/status-index'); INSERT INTO lcm_raw_messages ( provider, message_id, session_id, role, ordinal, content, - content_hash, storage_kind, snippet_text, index_text, - legacy_truncated, metadata_json + content_hash, storage_kind, metadata_json ) VALUES ( 'cursor', 'legacy-message', 'status-index-session', 'assistant', 1, - 'legacy body', 'legacy-hash', 'inline', 'legacy', 'legacy', 1, NULL + 'legacy body', 'legacy-hash', 'inline', NULL ), ( 'cursor', 'lossy-message', 'status-index-session', 'assistant', 2, - 'lossy body', 'lossy-hash', 'inline', 'lossy', 'lossy', 0, + 'lossy body', 'lossy-hash', 'inline', '{"ingest_protection":{"lossy":true}}' ); - INSERT INTO lcm_summary_nodes ( - node_id, provider, conversation_id, session_id, depth, - summary_text, summary_hash, summary_token_count, source_token_count - ) VALUES ( - 'summary-node', 'cursor', 'conversation', 'status-index-session', 2, - 'summary body', 'summary-hash', 3, 5 - ); INSERT INTO lcm_external_payloads ( payload_ref, provider, session_id, message_id, kind, content_hash, byte_count, char_count @@ -119,9 +111,7 @@ async fn short_lived_attach_convergence_rebuilds_queryable_lcm_status_indexes() 'payload-ref', 'cursor', 'status-index-session', 'payload-message', 'text', 'payload-hash', 11, 7 ); - DROP INDEX idx_lcm_raw_legacy_truncated; DROP INDEX idx_lcm_raw_lossy_ingest; - DROP INDEX idx_lcm_summary_nodes_depth_tokens; DROP INDEX idx_lcm_external_payloads_owner_bytes; CREATE INDEX idx_lcm_external_payloads_owner ON lcm_external_payloads(provider, session_id);"#, @@ -134,18 +124,11 @@ async fn short_lived_attach_convergence_rebuilds_queryable_lcm_status_indexes() .await .expect("pre-index store reopen"); let raw_indexes = table_index_names(&reopened, "lcm_raw_messages").await; - for index in ["idx_lcm_raw_legacy_truncated", "idx_lcm_raw_lossy_ingest"] { - assert!( - raw_indexes.iter().any(|name| name == index), - "short-lived convergence did not build {index}; raw message indexes: {raw_indexes:?}" - ); - } - let summary_indexes = table_index_names(&reopened, "lcm_summary_nodes").await; assert!( - summary_indexes + raw_indexes .iter() - .any(|name| name == "idx_lcm_summary_nodes_depth_tokens"), - "short-lived convergence did not build the summary depth/token index: {summary_indexes:?}" + .any(|name| name == "idx_lcm_raw_lossy_ingest"), + "short-lived convergence did not build idx_lcm_raw_lossy_ingest; raw message indexes: {raw_indexes:?}" ); let payload_indexes = table_index_names(&reopened, "lcm_external_payloads").await; assert!( @@ -161,12 +144,6 @@ async fn short_lived_attach_convergence_rebuilds_queryable_lcm_status_indexes() "short-lived convergence left the superseded payload owner index in place: {payload_indexes:?}" ); for (index, query) in [ - ( - "idx_lcm_raw_legacy_truncated", - "SELECT COUNT(*) - FROM lcm_raw_messages INDEXED BY idx_lcm_raw_legacy_truncated - WHERE provider = ?1 AND session_id = ?2 AND legacy_truncated != 0", - ), ( "idx_lcm_raw_lossy_ingest", "SELECT COUNT(*) @@ -176,12 +153,6 @@ async fn short_lived_attach_convergence_rebuilds_queryable_lcm_status_indexes() AND json_valid(metadata_json) AND json_type(metadata_json, '$.ingest_protection.lossy') = 'true'", ), - ( - "idx_lcm_summary_nodes_depth_tokens", - "SELECT COUNT(*) - FROM lcm_summary_nodes INDEXED BY idx_lcm_summary_nodes_depth_tokens - WHERE provider = ?1 AND session_id = ?2", - ), ( "idx_lcm_external_payloads_owner_bytes", "SELECT COUNT(*) diff --git a/crates/tracedecay-global-db/src/tests/lcm_schema/mod.rs b/crates/tracedecay-global-db/src/tests/lcm_schema/mod.rs index 1bb5ec5567..8e719b4351 100644 --- a/crates/tracedecay-global-db/src/tests/lcm_schema/mod.rs +++ b/crates/tracedecay-global-db/src/tests/lcm_schema/mod.rs @@ -244,146 +244,6 @@ async fn normalized_trigger_sql(db_path: &Path, trigger: &str) -> String { .collect::() } -const TEMPORAL_SCHEMA_OBJECTS: &[(&str, &str)] = &[ - ("index", "idx_session_assertion_supersession_successor"), - ("index", "idx_session_assertions_generation_order"), - ("index", "idx_session_assertions_kind_order"), - ("index", "idx_session_assertions_object_order"), - ("index", "idx_session_assertions_subject"), - ("index", "idx_session_current_entities_assertion"), - ("index", "idx_session_current_entities_occurrence"), - ("index", "idx_session_external_payload_manifests_session"), - ("index", "idx_session_occurrences_agent"), - ("index", "idx_session_occurrences_anchor_order"), - ("index", "idx_session_occurrences_generation_order"), - ("index", "idx_session_occurrences_message"), - ("index", "idx_session_occurrences_root_generation_order"), - ("index", "idx_session_occurrences_session_time"), - ("index", "idx_session_occurrences_thread"), - ("index", "idx_session_occurrences_turn"), - ("index", "idx_session_query_cursor_keys_active"), - ("index", "idx_session_refresh_operations_join"), - ("index", "idx_session_refresh_operations_one_running"), - ("index", "idx_session_refresh_operations_state"), - ("index", "idx_session_refresh_receipts_session"), - ("index", "idx_session_relation_receipts_pending"), - ("index", "idx_session_relation_receipts_recovery_due"), - ("index", "idx_session_summary_availability_generation"), - ("index", "idx_session_summary_nodes_root_created_order"), - ("index", "idx_session_summary_nodes_session_created"), - ("index", "idx_session_temporal_generations_one_active"), - ("index", "idx_session_temporal_generations_session_state"), - ("index", "idx_session_temporal_observation_effects_session"), - ("index", "idx_session_turn_members_occurrence"), - ("table", "session_agents"), - ("table", "session_assertion_supersession"), - ("table", "session_assertions"), - ("table", "session_current_entities"), - ("table", "session_external_payload_manifests"), - ("table", "session_occurrences"), - ("table", "session_occurrences_fts"), - ("table", "session_query_cursor_keys"), - ("table", "session_refresh_batch_bindings"), - ("table", "session_refresh_bindings"), - ("table", "session_refresh_operations"), - ("table", "session_refresh_progress"), - ("table", "session_refresh_receipts"), - ("table", "session_relation_effect_journal"), - ("table", "session_relation_receipts"), - ("table", "session_summary_availability"), - ("table", "session_summary_nodes"), - ("table", "session_summary_nodes_fts"), - ("table", "session_temporal_generations"), - ("table", "session_temporal_observation_effects"), - ("table", "session_temporal_projection_receipts"), - ("table", "session_temporal_schema_migrations"), - ("table", "session_threads"), - ("table", "session_turn_members"), - ("table", "session_turns"), - ( - "trigger", - "session_external_payload_manifests_immutable_delete_v1", - ), - ( - "trigger", - "session_external_payload_manifests_immutable_update_v1", - ), - ("trigger", "session_occurrences_fts_delete_v1"), - ("trigger", "session_occurrences_fts_insert_v1"), - ("trigger", "session_occurrences_fts_update_v1"), - ("trigger", "session_query_cursor_keys_immutable_delete_v1"), - ("trigger", "session_query_cursor_keys_insert_guard_v1"), - ("trigger", "session_query_cursor_keys_retire_update_v1"), - ("trigger", "session_query_cursor_keys_rotate_insert_v1"), - ( - "trigger", - "session_refresh_batch_bindings_immutable_delete_v1", - ), - ( - "trigger", - "session_refresh_batch_bindings_immutable_update_v1", - ), - ("trigger", "session_refresh_batch_bindings_insert_guard_v1"), - ("trigger", "session_refresh_bindings_immutable_delete_v1"), - ("trigger", "session_refresh_bindings_immutable_update_v1"), - ("trigger", "session_refresh_bindings_insert_guard_v1"), - ("trigger", "session_refresh_operations_delete_guard_v1"), - ("trigger", "session_refresh_operations_insert_guard_v1"), - ("trigger", "session_refresh_operations_state_guard_v1"), - ("trigger", "session_refresh_progress_insert_guard_v1"), - ("trigger", "session_refresh_progress_immutable_delete_v1"), - ("trigger", "session_refresh_progress_immutable_update_v1"), - ("trigger", "session_refresh_receipts_insert_guard_v1"), - ("trigger", "session_refresh_receipts_immutable_delete_v1"), - ("trigger", "session_refresh_receipts_immutable_update_v1"), - ("trigger", "session_summary_nodes_fts_delete_v1"), - ("trigger", "session_summary_nodes_fts_insert_v1"), - ("trigger", "session_summary_nodes_fts_update_v1"), - ("trigger", "session_summary_nodes_immutable_delete_v1"), - ("trigger", "session_summary_nodes_immutable_update_v1"), - ("trigger", "session_summary_availability_owner_insert_v1"), - ("trigger", "session_summary_availability_owner_update_v1"), - ( - "trigger", - "session_external_payload_manifests_owner_guard_v1", - ), - ("trigger", "session_temporal_generations_delete_guard_v1"), - ("trigger", "session_temporal_generations_insert_guard_v1"), - ( - "trigger", - "session_temporal_generations_single_active_insert_v1", - ), - ( - "trigger", - "session_temporal_generations_single_active_update_v1", - ), - ("trigger", "session_temporal_generations_state_guard_v1"), - ( - "trigger", - "session_temporal_observation_effects_immutable_delete_v1", - ), - ( - "trigger", - "session_temporal_observation_effects_immutable_update_v1", - ), - ( - "trigger", - "session_temporal_observation_effects_insert_guard_v1", - ), - ( - "trigger", - "session_temporal_projection_receipts_immutable_delete_v1", - ), - ( - "trigger", - "session_temporal_projection_receipts_immutable_update_v1", - ), - ( - "trigger", - "session_temporal_projection_receipts_insert_guard_v1", - ), -]; - async fn temporal_schema_object_catalog(db_path: &Path) -> Vec<(String, String)> { let db = TestConnection::open(db_path); let conn = (*db).clone(); diff --git a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog.rs b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog.rs index 5a05263614..995ec5ed76 100644 --- a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog.rs +++ b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog.rs @@ -1,7 +1,7 @@ use super::*; #[tokio::test] -async fn temporal_schema_complete_object_catalog() { +async fn fresh_temporal_schema_stores_and_searches_a_summary_node() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); @@ -9,17 +9,54 @@ async fn temporal_schema_complete_object_catalog() { .await .expect("temporal schema initialization should not error"); drop(db); - - let mut expected = TEMPORAL_SCHEMA_OBJECTS - .iter() - .map(|(object_type, object_name)| ((*object_type).to_string(), (*object_name).to_string())) - .collect::>(); - expected.sort(); - assert_eq!(temporal_schema_object_catalog(&db_path).await, expected); assert!( table_exists(&db_path, "lcm_raw_messages").await, "fresh initialization must compose the final temporal and LCM schemas" ); + + let raw_db = TestConnection::open(&db_path); + let conn = (*raw_db).clone(); + conn.execute_batch( + "INSERT INTO retrieval_anchors ( + anchor_id, anchor_json, owner_json, projection_generation + ) VALUES ('fresh-anchor', '{}', '{}', 'test'); + INSERT INTO session_summary_nodes ( + summary_id, session_id, provider, conversation_id, depth, summary_anchor_id, + summary_text, summary_hash, summary_token_count, source_token_count, + source_horizon_json, created_at + ) VALUES ( + 'fresh-summary', 'fresh-session', 'test', 'fresh-session', 0, 'fresh-anchor', + 'fresh summary text quokka migration notes', 'hash', 1, 1, '{}', 100 + );", + ) + .await + .unwrap(); + let mut rows = conn + .query( + "SELECT n.summary_id, n.session_id, n.summary_text, n.created_at + FROM session_summary_nodes_fts + JOIN session_summary_nodes AS n ON n.rowid = session_summary_nodes_fts.rowid + WHERE session_summary_nodes_fts MATCH 'quokka'", + (), + ) + .await + .unwrap(); + let row = rows.next().await.unwrap().expect("FTS finds the summary"); + assert_eq!( + ( + row.get::(0).unwrap(), + row.get::(1).unwrap(), + row.get::(2).unwrap(), + row.get::(3).unwrap(), + ), + ( + "fresh-summary".to_string(), + "fresh-session".to_string(), + "fresh summary text quokka migration notes".to_string(), + 100, + ) + ); + assert!(rows.next().await.unwrap().is_none()); } #[tokio::test] @@ -382,8 +419,7 @@ async fn temporal_schema_root_retrieval_indexes_cover_catalog_and_large_query_sh session_id, generation, occurrence_id, source_observation_id, source_provider, projection_output_ordinal, retrieval_anchor_id, role, knowledge_at, valid_time_json, evidence_json, - sanitized_content_digest, sanitized_content_bytes, - snippet_text, index_text + sanitized_content_digest, sanitized_content_bytes, index_text ) SELECT printf('root-session-%02d', value % 8), @@ -399,7 +435,6 @@ async fn temporal_schema_root_retrieval_indexes_cover_catalog_and_large_query_sh '{{}}', '0000000000000000000000000000000000000000000000000000000000000000', 15, - 'root occurrence', 'root occurrence' FROM sequence;" )) @@ -415,15 +450,21 @@ async fn temporal_schema_root_retrieval_indexes_cover_catalog_and_large_query_sh SELECT value + 1 FROM sequence WHERE value < {end} ) INSERT INTO session_summary_nodes ( - summary_id, session_id, summary_anchor_id, summary_text, index_text, + summary_id, session_id, provider, conversation_id, depth, summary_anchor_id, + summary_text, summary_hash, summary_token_count, source_token_count, source_horizon_json, created_at ) SELECT printf('root-summary-%06d', value), printf('root-session-%02d', value % 8), + 'test', + printf('root-session-%02d', value % 8), + 0, 'root-anchor', 'root summary', - 'root summary', + 'hash', + 1, + 1, '{{}}', value / 8 FROM sequence;" @@ -693,11 +734,12 @@ async fn temporal_schema_rejects_missing_fts_without_rebuilding_it() { anchor_id, anchor_json, owner_json, projection_generation ) VALUES ('fts-anchor', '{}', '{}', 'test'); INSERT INTO session_summary_nodes ( - summary_id, session_id, summary_anchor_id, summary_text, index_text, + summary_id, session_id, provider, conversation_id, depth, summary_anchor_id, + summary_text, summary_hash, summary_token_count, source_token_count, source_horizon_json, created_at ) VALUES ( - 'fts-summary', 'fts-session', 'fts-anchor', - 'existing summary', 'migration-search summary', '{}', 100 + 'fts-summary', 'fts-session', 'test', 'fts-session', 0, 'fts-anchor', + 'existing summary', 'hash', 1, 1, '{}', 100 );", ) .await diff --git a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog/admission.rs b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog/admission.rs index 639e75c9c6..ae7c8f7f3f 100644 --- a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog/admission.rs +++ b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog/admission.rs @@ -48,203 +48,6 @@ async fn schema_object_sql(db_path: &Path, object_type: &str, name: &str) -> Str .unwrap() } -async fn suspend_schema_triggers(db_path: &Path) -> Vec { - let raw_db = TestConnection::open(db_path); - let conn = (*raw_db).clone(); - let mut rows = conn - .query( - "SELECT name, sql FROM sqlite_master - WHERE type = 'trigger' AND sql IS NOT NULL ORDER BY name", - (), - ) - .await - .unwrap(); - let mut triggers = Vec::new(); - let mut names = Vec::new(); - while let Some(row) = rows.next().await.unwrap() { - names.push(row.get::(0).unwrap()); - triggers.push(row.get::(1).unwrap()); - } - drop(rows); - for name in names { - conn.execute(&format!("DROP TRIGGER \"{name}\""), ()) - .await - .unwrap(); - } - triggers -} - -async fn restore_schema_triggers(db_path: &Path, triggers: &[String]) { - let raw_db = TestConnection::open(db_path); - let conn = (*raw_db).clone(); - for sql in triggers { - conn.execute_batch(sql).await.unwrap(); - } -} - -async fn table_trigger_sql(conn: &Connection, table: &str) -> Vec { - let mut rows = conn - .query( - "SELECT sql FROM sqlite_master - WHERE type = 'trigger' AND tbl_name = ?1 AND sql IS NOT NULL ORDER BY name", - params![table], - ) - .await - .unwrap(); - let mut triggers = Vec::new(); - while let Some(row) = rows.next().await.unwrap() { - triggers.push(row.get::(0).unwrap()); - } - triggers -} - -/// Authority-invariant trigger bodies exactly as every v3 release published -/// them, extracted verbatim from the tagged source. The file header carries the -/// tag-to-inventory table. Deriving these from the current contract instead -/// would make the fixture agree with whatever -/// `released_v3_invariant_triggers_intact` reconstructs, including a -/// reconstruction that no release ever wrote to disk. -const RELEASED_V3_AUTHORITY_TRIGGERS_SQL: &str = - include_str!("../../../../tests/fixtures/session-temporal-released-v3-triggers.sql"); - -/// Rebuilds the two tables the v4 migration widened, and the three authority -/// triggers whose bodies changed after v3, from their exact published -/// definitions. Recreating the tables costs two statement parses; the -/// equivalent `DROP COLUMN` sequence reparses the whole 700-object schema -/// several times per column, which under a full parallel test run exceeded the -/// exact-SQL statement budget. -async fn convert_final_temporal_schema_to_released_v3(db_path: &Path) { - let raw_db = TestConnection::open(db_path); - let conn = (*raw_db).clone(); - // Dropping the table drops its triggers; their text is identical in v3. - let projection_receipt_triggers = - table_trigger_sql(&conn, "session_temporal_projection_receipts").await; - assert_eq!(projection_receipt_triggers.len(), 3); - - conn.execute_batch( - "DROP TABLE session_temporal_projection_receipts; - DROP TABLE session_relation_receipts;", - ) - .await - .unwrap(); - conn.execute_batch(SESSION_TEMPORAL_PROJECTION_RECEIPTS_V3_DDL) - .await - .unwrap(); - conn.execute_batch(SESSION_RELATION_RECEIPTS_WITHOUT_RECOVERY_DDL) - .await - .unwrap(); - for trigger in &projection_receipt_triggers { - conn.execute_batch(trigger).await.unwrap(); - } - conn.execute_batch(RELEASED_V3_AUTHORITY_TRIGGERS_SQL) - .await - .unwrap(); - conn.execute( - "UPDATE session_temporal_schema_migrations - SET version = 3, applied_at = 100 - WHERE name = 'session-temporal'", - (), - ) - .await - .unwrap(); -} - -/// The authority triggers whose published v3 body differs from the current -/// contract. Two guard `session_messages`, one guards `session_refresh_progress`. -const RELEASED_V3_DRIFTED_TRIGGERS: [&str; 3] = [ - "projection_output_audit_invalidate_delete_v1", - "projection_output_audit_invalidate_update_v1", - "session_refresh_progress_insert_guard_v1", -]; - -/// Seeds the retained user sessions and messages a reset would destroy. -/// `session_messages` carries two of the three drifted authority triggers, so -/// these rows also prove the migration replaces a trigger without touching the -/// table under it. -async fn seed_retained_sessions_and_messages(db_path: &Path) { - let raw_db = TestConnection::open(db_path); - let conn = (*raw_db).clone(); - conn.execute_batch( - "INSERT INTO sessions - (provider, session_id, project_key, project_path, title, started_at, - ended_at, transcript_path, metadata_json) - VALUES - ('claude', 'mac-session-one', '/Users/mac/project', '/Users/mac/project', - 'first mac session', 1000, 1100, '/Users/mac/.claude/one.jsonl', - '{\"host\":\"mac\"}'), - ('cursor', 'mac-session-two', '/Users/mac/project', '/Users/mac/project', - NULL, 2000, NULL, NULL, NULL); - INSERT INTO session_messages - (provider, message_id, session_id, role, timestamp, ordinal, text, - kind, model, tool_names, source_path, source_offset, metadata_json) - VALUES - ('claude', 'mac-message-one', 'mac-session-one', 'user', 1001, 0, - 'keep my sessions', 'text', NULL, NULL, - '/Users/mac/.claude/one.jsonl', 0, NULL), - ('claude', 'mac-message-two', 'mac-session-one', 'assistant', 1002, 1, - 'byte-exact after migration', 'text', 'claude-opus', 'Read,Edit', - '/Users/mac/.claude/one.jsonl', 512, '{\"turn\":1}'), - ('cursor', 'mac-message-three', 'mac-session-two', 'user', 2001, 0, - 'second provider', NULL, NULL, NULL, NULL, NULL, NULL);", - ) - .await - .unwrap(); -} - -/// Every retained session and message column, as a stable comparable snapshot. -async fn retained_sessions_and_messages(db_path: &Path) -> Vec { - let raw_db = TestConnection::open(db_path); - let conn = (*raw_db).clone(); - let mut snapshot = Vec::new(); - for sql in [ - "SELECT json_array(provider, session_id, project_key, project_path, title, - started_at, ended_at, transcript_path, metadata_json, - parent_session_id, is_subagent, agent_id, parent_tool_use_id) - FROM sessions ORDER BY provider, session_id", - "SELECT json_array(provider, message_id, session_id, role, timestamp, ordinal, - text, kind, model, tool_names, source_path, source_offset, - metadata_json) - FROM session_messages ORDER BY provider, message_id", - ] { - let mut rows = conn.query(sql, ()).await.unwrap(); - while let Some(row) = rows.next().await.unwrap() { - snapshot.push(row.get::(0).unwrap()); - } - } - snapshot -} - -/// `session_temporal_projection_receipts` exactly as published in beta.37. -const SESSION_TEMPORAL_PROJECTION_RECEIPTS_V3_DDL: &str = " - CREATE TABLE session_temporal_projection_receipts ( - session_id TEXT NOT NULL, - generation INTEGER NOT NULL, - batch_ordinal INTEGER NOT NULL CHECK(batch_ordinal >= 0), - batch_digest TEXT NOT NULL, - frozen_watermarks_json TEXT NOT NULL CHECK(json_valid(frozen_watermarks_json)), - source_through INTEGER NOT NULL CHECK(source_through >= 0), - projection_through INTEGER NOT NULL CHECK(projection_through >= 0), - occurrence_count INTEGER NOT NULL CHECK(occurrence_count >= 0), - occurrence_digest TEXT NOT NULL, - dimension_count INTEGER NOT NULL CHECK(dimension_count >= 0), - dimension_digest TEXT NOT NULL, - copy_count INTEGER NOT NULL CHECK(copy_count >= 0), - copy_digest TEXT NOT NULL, - assertion_count INTEGER NOT NULL CHECK(assertion_count >= 0), - assertion_digest TEXT NOT NULL, - supersession_count INTEGER NOT NULL CHECK(supersession_count >= 0), - supersession_digest TEXT NOT NULL, - current_count INTEGER NOT NULL CHECK(current_count >= 0), - current_digest TEXT NOT NULL, - fts_count INTEGER NOT NULL CHECK(fts_count >= 0), - fts_digest TEXT NOT NULL, - committed_at INTEGER NOT NULL, - PRIMARY KEY(session_id, generation, batch_ordinal), - UNIQUE(session_id, generation, batch_digest), - FOREIGN KEY(session_id, generation) - REFERENCES session_temporal_generations(session_id, generation) ON DELETE CASCADE - );"; - /// `session_relation_receipts` exactly as persisted by v4 stores before /// receipt recovery (byte-identical to the beta.37 published definition). const SESSION_RELATION_RECEIPTS_WITHOUT_RECOVERY_DDL: &str = @@ -335,398 +138,6 @@ fn expected_retained_relation_receipts() -> Vec<(String, i64, String, String)> { ] } -async fn insert_released_v3_projection_receipts(db_path: &Path, second_counts: (i64, i64, i64)) { - let raw_db = TestConnection::open(db_path); - let conn = (*raw_db).clone(); - conn.execute_batch( - "INSERT INTO session_temporal_generations ( - session_id, generation, state, frozen_watermarks_json, created_at - ) VALUES ('released-v3', 1, 'building', '{}', 100); - INSERT INTO session_temporal_projection_receipts ( - session_id, generation, batch_ordinal, batch_digest, frozen_watermarks_json, - source_through, projection_through, - occurrence_count, occurrence_digest, dimension_count, dimension_digest, - copy_count, copy_digest, assertion_count, assertion_digest, - supersession_count, supersession_digest, current_count, current_digest, - fts_count, fts_digest, committed_at - ) VALUES ( - 'released-v3', 1, 0, - 'sha256:1000000000000000000000000000000000000000000000000000000000000000', - '{}', 5, 5, - 2, 'sha256:1100000000000000000000000000000000000000000000000000000000000000', - 0, 'sha256:1200000000000000000000000000000000000000000000000000000000000000', - 1, 'sha256:1300000000000000000000000000000000000000000000000000000000000000', - 1, 'sha256:1400000000000000000000000000000000000000000000000000000000000000', - 0, 'sha256:1500000000000000000000000000000000000000000000000000000000000000', - 0, 'sha256:1600000000000000000000000000000000000000000000000000000000000000', - 0, 'sha256:1700000000000000000000000000000000000000000000000000000000000000', - 101 - );", - ) - .await - .unwrap(); - conn.execute( - "INSERT INTO session_temporal_projection_receipts ( - session_id, generation, batch_ordinal, batch_digest, frozen_watermarks_json, - source_through, projection_through, - occurrence_count, occurrence_digest, dimension_count, dimension_digest, - copy_count, copy_digest, assertion_count, assertion_digest, - supersession_count, supersession_digest, current_count, current_digest, - fts_count, fts_digest, committed_at - ) VALUES ( - 'released-v3', 1, 1, - 'sha256:2000000000000000000000000000000000000000000000000000000000000000', - '{}', 10, 10, - ?1, 'sha256:2100000000000000000000000000000000000000000000000000000000000000', - 0, 'sha256:2200000000000000000000000000000000000000000000000000000000000000', - ?2, 'sha256:2300000000000000000000000000000000000000000000000000000000000000', - ?3, 'sha256:2400000000000000000000000000000000000000000000000000000000000000', - 0, 'sha256:2500000000000000000000000000000000000000000000000000000000000000', - 0, 'sha256:2600000000000000000000000000000000000000000000000000000000000000', - 0, 'sha256:2700000000000000000000000000000000000000000000000000000000000000', - 102 - )", - params![second_counts.0, second_counts.1, second_counts.2], - ) - .await - .unwrap(); -} - -async fn insert_seeded_active_released_v3_refresh_receipts( - db_path: &Path, - second_candidate_counts: (i64, i64, i64), -) { - let second_candidate_total = - second_candidate_counts.0 + second_candidate_counts.1 + second_candidate_counts.2; - let triggers = suspend_schema_triggers(db_path).await; - let raw_db = TestConnection::open(db_path); - let conn = (*raw_db).clone(); - conn.execute_batch( - "INSERT INTO session_temporal_generations ( - session_id, generation, state, frozen_watermarks_json, created_at, - ready_at, activated_at - ) VALUES ( - 'released-v3', 1, 'active', - '{\"active_generation\":1,\"source_frontier\":5,\"projection_frontier\":5,\"summary_frontier\":0,\"cursor_key\":null}', - 90, 91, 92 - ); - INSERT INTO session_temporal_generations ( - session_id, generation, state, frozen_watermarks_json, created_at - ) VALUES ( - 'released-v3', 2, 'building', - '{\"active_generation\":1,\"source_frontier\":5,\"projection_frontier\":5,\"summary_frontier\":0,\"cursor_key\":null}', - 100 - ); - INSERT INTO session_temporal_projection_receipts ( - session_id, generation, batch_ordinal, batch_digest, frozen_watermarks_json, - source_through, projection_through, - occurrence_count, occurrence_digest, dimension_count, dimension_digest, - copy_count, copy_digest, assertion_count, assertion_digest, - supersession_count, supersession_digest, current_count, current_digest, - fts_count, fts_digest, committed_at - ) VALUES ( - 'released-v3', 1, 0, - 'sha256:0100000000000000000000000000000000000000000000000000000000000000', - '{\"active_generation\":1,\"source_frontier\":5,\"projection_frontier\":5,\"summary_frontier\":0,\"cursor_key\":null}', - 5, 5, - 3, 'sha256:0200000000000000000000000000000000000000000000000000000000000000', - 0, 'sha256:0300000000000000000000000000000000000000000000000000000000000000', - 1, 'sha256:0400000000000000000000000000000000000000000000000000000000000000', - 1, 'sha256:0500000000000000000000000000000000000000000000000000000000000000', - 0, 'sha256:0600000000000000000000000000000000000000000000000000000000000000', - 0, 'sha256:0700000000000000000000000000000000000000000000000000000000000000', - 0, 'sha256:0800000000000000000000000000000000000000000000000000000000000000', - 99 - ); - INSERT INTO session_refresh_operations ( - session_id, operation_id, request_digest, target_frontier_json, - state, created_at, updated_at, terminal_at - ) VALUES ( - 'released-v3', 'refresh-v3-baseline', - 'sha256:a000000000000000000000000000000000000000000000000000000000000000', - '{\"observed_through\":5,\"committed_through\":0}', - 'complete', 90, 99, 99 - ); - INSERT INTO session_refresh_bindings ( - session_id, operation_id, scope_kind, source_frontier, target_frontier, - projector_version, config_digest, generation, frozen_watermarks_json, - binding_digest, created_at - ) VALUES ( - 'released-v3', 'refresh-v3-baseline', 'session_store', 0, 5, - 'session-temporal-projector.v1', - 'sha256:a100000000000000000000000000000000000000000000000000000000000000', - 1, - '{\"active_generation\":1,\"source_frontier\":5,\"projection_frontier\":5,\"summary_frontier\":0,\"cursor_key\":null}', - 'sha256:a000000000000000000000000000000000000000000000000000000000000000', - 90 - ); - INSERT INTO session_refresh_progress ( - session_id, operation_id, progress_ordinal, frontier_json, coverage_json, - committed_batches, committed_records, recorded_at - ) VALUES ( - 'released-v3', 'refresh-v3-baseline', 0, - '{\"observed_through\":5,\"committed_through\":5}', - '{\"visible\":5,\"hidden\":0,\"unknown\":0,\"redacted\":0}', - 1, 5, 95 - ); - INSERT INTO session_refresh_batch_bindings ( - session_id, operation_id, progress_ordinal, generation, batch_ordinal - ) VALUES ('released-v3', 'refresh-v3-baseline', 0, 1, 0); - INSERT INTO session_refresh_receipts ( - session_id, operation_id, terminal_state, frontier_json, coverage_json, - failure_code, terminal_at - ) VALUES ( - 'released-v3', 'refresh-v3-baseline', 'complete', - '{\"observed_through\":5,\"committed_through\":5}', - '{\"visible\":5,\"hidden\":0,\"unknown\":0,\"redacted\":0}', - NULL, 99 - ); - INSERT INTO session_temporal_projection_receipts ( - session_id, generation, batch_ordinal, batch_digest, frozen_watermarks_json, - source_through, projection_through, - occurrence_count, occurrence_digest, dimension_count, dimension_digest, - copy_count, copy_digest, assertion_count, assertion_digest, - supersession_count, supersession_digest, current_count, current_digest, - fts_count, fts_digest, committed_at - ) VALUES ( - 'released-v3', 2, 0, - 'sha256:1000000000000000000000000000000000000000000000000000000000000000', - '{\"active_generation\":1,\"source_frontier\":5,\"projection_frontier\":5,\"summary_frontier\":0,\"cursor_key\":null}', - 7, 7, - 6, 'sha256:1100000000000000000000000000000000000000000000000000000000000000', - 0, 'sha256:1200000000000000000000000000000000000000000000000000000000000000', - 1, 'sha256:1300000000000000000000000000000000000000000000000000000000000000', - 2, 'sha256:1400000000000000000000000000000000000000000000000000000000000000', - 0, 'sha256:1500000000000000000000000000000000000000000000000000000000000000', - 0, 'sha256:1600000000000000000000000000000000000000000000000000000000000000', - 0, 'sha256:1700000000000000000000000000000000000000000000000000000000000000', - 101 - ); - INSERT INTO session_refresh_operations ( - session_id, operation_id, request_digest, target_frontier_json, - state, created_at, updated_at - ) VALUES ( - 'released-v3', 'refresh-v3', - 'sha256:9000000000000000000000000000000000000000000000000000000000000000', - '{\"observed_through\":10,\"committed_through\":5}', - 'running', 100, 100 - ); - INSERT INTO session_refresh_bindings ( - session_id, operation_id, scope_kind, source_frontier, target_frontier, - projector_version, config_digest, generation, frozen_watermarks_json, - binding_digest, created_at - ) VALUES ( - 'released-v3', 'refresh-v3', 'session_store', 5, 10, - 'session-temporal-projector.v1', - 'sha256:9100000000000000000000000000000000000000000000000000000000000000', - 2, - '{\"active_generation\":1,\"source_frontier\":5,\"projection_frontier\":5,\"summary_frontier\":0,\"cursor_key\":null}', - 'sha256:9000000000000000000000000000000000000000000000000000000000000000', - 100 - ); - INSERT INTO session_refresh_progress ( - session_id, operation_id, progress_ordinal, frontier_json, coverage_json, - committed_batches, committed_records, recorded_at - ) VALUES ( - 'released-v3', 'refresh-v3', 0, - '{\"observed_through\":10,\"committed_through\":7}', - '{\"visible\":9,\"hidden\":0,\"unknown\":0,\"redacted\":0}', - 1, 9, 101 - ); - INSERT INTO session_refresh_batch_bindings ( - session_id, operation_id, progress_ordinal, generation, batch_ordinal - ) VALUES ('released-v3', 'refresh-v3', 0, 2, 0);", - ) - .await - .unwrap(); - conn.execute( - "INSERT INTO session_temporal_projection_receipts ( - session_id, generation, batch_ordinal, batch_digest, frozen_watermarks_json, - source_through, projection_through, - occurrence_count, occurrence_digest, dimension_count, dimension_digest, - copy_count, copy_digest, assertion_count, assertion_digest, - supersession_count, supersession_digest, current_count, current_digest, - fts_count, fts_digest, committed_at - ) VALUES ( - 'released-v3', 2, 1, - 'sha256:2000000000000000000000000000000000000000000000000000000000000000', - '{\"active_generation\":1,\"source_frontier\":5,\"projection_frontier\":5,\"summary_frontier\":0,\"cursor_key\":null}', - 10, 10, - ?1, 'sha256:2100000000000000000000000000000000000000000000000000000000000000', - 0, 'sha256:2200000000000000000000000000000000000000000000000000000000000000', - ?2, 'sha256:2300000000000000000000000000000000000000000000000000000000000000', - ?3, 'sha256:2400000000000000000000000000000000000000000000000000000000000000', - 0, 'sha256:2500000000000000000000000000000000000000000000000000000000000000', - 0, 'sha256:2600000000000000000000000000000000000000000000000000000000000000', - 0, 'sha256:2700000000000000000000000000000000000000000000000000000000000000', - 102 - )", - params![ - second_candidate_counts.0, - second_candidate_counts.1, - second_candidate_counts.2 - ], - ) - .await - .unwrap(); - conn.execute( - "INSERT INTO session_refresh_progress ( - session_id, operation_id, progress_ordinal, frontier_json, coverage_json, - committed_batches, committed_records, recorded_at - ) VALUES ( - 'released-v3', 'refresh-v3', 1, - '{\"observed_through\":10,\"committed_through\":10}', - json_object('visible', ?1, 'hidden', 0, 'unknown', 0, 'redacted', 0), - 2, ?1, 102 - )", - params![second_candidate_total], - ) - .await - .unwrap(); - conn.execute_batch( - "INSERT INTO session_refresh_batch_bindings ( - session_id, operation_id, progress_ordinal, generation, batch_ordinal - ) VALUES ('released-v3', 'refresh-v3', 1, 2, 1);", - ) - .await - .unwrap(); - drop(conn); - drop(raw_db); - restore_schema_triggers(db_path, &triggers).await; -} - -#[tokio::test] -async fn released_v3_temporal_receipts_are_refused_without_conversion() { - let tmp = TempDir::new().unwrap(); - let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) - .await - .expect("fresh initialization should install the final temporal schema"); - drop(db); - convert_final_temporal_schema_to_released_v3(&db_path).await; - insert_seeded_active_released_v3_refresh_receipts(&db_path, (8, 2, 2)).await; - let before_sql = - schema_object_sql(&db_path, "table", "session_temporal_projection_receipts").await; - - let error = match open_global_db(&db_path).await { - Ok(_) => panic!("the published v3 temporal shape must not be converted"), - Err(error) => error, - }; - let (authority, reason) = error - .reset_required_context() - .expect("published v3 must return typed reset-required"); - assert_eq!(authority, "session temporal"); - assert!( - reason.contains("no sanctioned conversion"), - "unexpected reason: {reason}" - ); - assert!( - reason.contains("published v3"), - "unexpected reason: {reason}" - ); - assert_eq!(temporal_schema_version(&db_path).await, 3); - assert_eq!( - schema_object_sql(&db_path, "table", "session_temporal_projection_receipts").await, - before_sql, - "typed refusal must not add v4 batch-count columns" - ); - assert!( - !persisted_column_names(&db_path, "session_temporal_projection_receipts") - .await - .iter() - .any(|column| column == "batch_item_count") - ); -} - -/// A profile written by a released v3 binary carries the authority triggers -/// that release published, not the bodies the tip contracts. That shape is not -/// the final contract, so admission returns `ResetRequired` before rewriting -/// triggers or session rows. The operator resets explicitly; bytes stay for -/// inspection. -#[tokio::test] -async fn published_v3_authority_triggers_are_refused_without_rewriting_sessions() { - let tmp = TempDir::new().unwrap(); - let fresh_path = tmp.path().join(".tracedecay").join("fresh.db"); - let db = open_global_db(&fresh_path) - .await - .expect("fresh initialization should install the final temporal schema"); - drop(db); - let fresh_catalog = temporal_schema_object_catalog(&fresh_path).await; - let mut current_drifted = Vec::new(); - for trigger in RELEASED_V3_DRIFTED_TRIGGERS { - current_drifted.push(normalized_trigger_sql(&fresh_path, trigger).await); - } - - let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path).await.unwrap(); - drop(db); - seed_retained_sessions_and_messages(&db_path).await; - let retained = retained_sessions_and_messages(&db_path).await; - assert_eq!( - retained.len(), - 5, - "the fixture must seed sessions to retain" - ); - convert_final_temporal_schema_to_released_v3(&db_path).await; - insert_seeded_active_released_v3_refresh_receipts(&db_path, (8, 2, 2)).await; - - let mut published_drifted = Vec::new(); - for trigger in RELEASED_V3_DRIFTED_TRIGGERS { - published_drifted.push(normalized_trigger_sql(&db_path, trigger).await); - } - assert!( - published_drifted - .iter() - .zip(¤t_drifted) - .all(|(published, current)| published != current), - "the fixture must present the published bodies, not the current contract" - ); - - let before_catalog = temporal_schema_object_catalog(&db_path).await; - let error = match open_global_db(&db_path).await { - Ok(_) => panic!("a store carrying the published v3 triggers must reset, not migrate"), - Err(error) => error, - }; - let (authority, reason) = error - .reset_required_context() - .expect("published v3 triggers must return typed reset-required"); - assert_eq!(authority, "session temporal"); - assert!( - reason.contains("no sanctioned conversion"), - "unexpected reason: {reason}" - ); - assert_eq!(temporal_schema_version(&db_path).await, 3); - for (trigger, published) in RELEASED_V3_DRIFTED_TRIGGERS.iter().zip(&published_drifted) { - assert_eq!( - &normalized_trigger_sql(&db_path, trigger).await, - published, - "refusal must leave '{trigger}' at the published body" - ); - } - assert!( - published_drifted - .iter() - .zip(¤t_drifted) - .all(|(published, current)| published != current), - "refusal must not rewrite published trigger bodies onto the current contract" - ); - assert_eq!( - temporal_schema_object_catalog(&db_path).await, - before_catalog, - "typed refusal must not rewrite the published temporal catalog" - ); - assert_eq!( - retained_sessions_and_messages(&db_path).await, - retained, - "every retained session and message row must survive byte-exact" - ); - assert_ne!( - temporal_schema_object_catalog(&db_path).await, - fresh_catalog, - "refusing v3 must not install the fresh temporal catalog" - ); -} - #[tokio::test] async fn v4_receipts_without_recovery_columns_are_refused_without_conversion() { let tmp = TempDir::new().unwrap(); @@ -745,7 +156,10 @@ async fn v4_receipts_without_recovery_columns_are_refused_without_conversion() { persisted_column_names(&db_path, "session_relation_receipts").await, SESSION_RELATION_RECEIPT_COLUMNS_WITHOUT_RECOVERY ); - assert_eq!(temporal_schema_version(&db_path).await, 4); + assert_eq!( + temporal_schema_version(&db_path).await, + tracedecay_session_temporal_store::SESSION_TEMPORAL_SCHEMA_VERSION + ); let before_catalog = temporal_schema_object_catalog(&db_path).await; let error = match open_global_db(&db_path).await { @@ -764,7 +178,10 @@ async fn v4_receipts_without_recovery_columns_are_refused_without_conversion() { reason.contains("pre-recovery"), "unexpected reason: {reason}" ); - assert_eq!(temporal_schema_version(&db_path).await, 4); + assert_eq!( + temporal_schema_version(&db_path).await, + tracedecay_session_temporal_store::SESSION_TEMPORAL_SCHEMA_VERSION + ); assert_eq!( persisted_column_names(&db_path, "session_relation_receipts").await, SESSION_RELATION_RECEIPT_COLUMNS_WITHOUT_RECOVERY @@ -830,7 +247,10 @@ async fn v4_receipts_with_partial_recovery_columns_are_refused_without_mutation( reason.contains("session_relation_receipts"), "unexpected reason: {reason}" ); - assert_eq!(temporal_schema_version(&db_path).await, 4); + assert_eq!( + temporal_schema_version(&db_path).await, + tracedecay_session_temporal_store::SESSION_TEMPORAL_SCHEMA_VERSION + ); assert_eq!( schema_object_sql(&db_path, "table", "session_relation_receipts").await, before_sql, @@ -902,7 +322,10 @@ async fn v4_receipts_with_retyped_column_are_refused_without_mutation() { reason.contains("session_relation_receipts"), "unexpected reason: {reason}" ); - assert_eq!(temporal_schema_version(&db_path).await, 4); + assert_eq!( + temporal_schema_version(&db_path).await, + tracedecay_session_temporal_store::SESSION_TEMPORAL_SCHEMA_VERSION + ); assert_eq!( schema_object_sql(&db_path, "table", "session_relation_receipts").await, before_sql, @@ -918,344 +341,6 @@ async fn v4_receipts_with_retyped_column_are_refused_without_mutation() { ); } -#[tokio::test] -async fn released_v3_changed_check_constraint_is_refused_without_mutation() { - let tmp = TempDir::new().unwrap(); - let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path).await.unwrap(); - drop(db); - convert_final_temporal_schema_to_released_v3(&db_path).await; - let released_sql = - schema_object_sql(&db_path, "table", "session_temporal_projection_receipts").await; - let drifted_sql = - released_sql.replacen("CHECK(batch_ordinal >= 0)", "CHECK(batch_ordinal >= -1)", 1); - assert_ne!(drifted_sql, released_sql); - let triggers = suspend_schema_triggers(&db_path).await; - let raw_db = TestConnection::open(&db_path); - let conn = (*raw_db).clone(); - conn.execute_batch("DROP TABLE session_temporal_projection_receipts;") - .await - .unwrap(); - conn.execute_batch(&drifted_sql).await.unwrap(); - drop(conn); - drop(raw_db); - restore_schema_triggers(&db_path, &triggers).await; - let persisted_drifted_sql = - schema_object_sql(&db_path, "table", "session_temporal_projection_receipts").await; - - let error = match open_global_db(&db_path).await { - Ok(_) => panic!("changed released-v3 CHECK constraint must be refused"), - Err(error) => error, - }; - assert_eq!( - error - .reset_required_context() - .map(|(authority, _)| authority), - Some("session temporal") - ); - assert_eq!(temporal_schema_version(&db_path).await, 3); - assert_eq!( - schema_object_sql(&db_path, "table", "session_temporal_projection_receipts").await, - persisted_drifted_sql - ); -} - -#[tokio::test] -async fn released_v3_extra_temporal_trigger_is_refused_without_mutation() { - let tmp = TempDir::new().unwrap(); - let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path).await.unwrap(); - drop(db); - convert_final_temporal_schema_to_released_v3(&db_path).await; - let raw_db = TestConnection::open(&db_path); - let conn = (*raw_db).clone(); - conn.execute_batch( - "CREATE TRIGGER branch_local_temporal_trigger - BEFORE INSERT ON session_temporal_projection_receipts BEGIN SELECT 1; END;", - ) - .await - .unwrap(); - drop(conn); - drop(raw_db); - - let error = match open_global_db(&db_path).await { - Ok(_) => panic!("extra released-v3 temporal trigger must be refused"), - Err(error) => error, - }; - assert_eq!( - error - .reset_required_context() - .map(|(authority, _)| authority), - Some("session temporal") - ); - assert_eq!(temporal_schema_version(&db_path).await, 3); - assert!(schema_object_exists(&db_path, "trigger", "branch_local_temporal_trigger").await); -} - -#[tokio::test] -async fn unbound_released_v3_receipts_are_refused_without_fabricated_batch_counts() { - let tmp = TempDir::new().unwrap(); - let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path).await.unwrap(); - drop(db); - convert_final_temporal_schema_to_released_v3(&db_path).await; - insert_released_v3_projection_receipts(&db_path, (5, 3, 2)).await; - - let error = match open_global_db(&db_path).await { - Ok(_) => panic!("unbound released-v3 receipts must be refused"), - Err(error) => error, - }; - assert_eq!( - error - .reset_required_context() - .map(|(authority, _)| authority), - Some("session temporal") - ); - assert_eq!(temporal_schema_version(&db_path).await, 3); - assert!( - !persisted_column_names(&db_path, "session_temporal_projection_receipts") - .await - .iter() - .any(|column| column == "batch_item_count") - ); -} - -#[tokio::test] -async fn valid_watermarks_unbound_released_v3_receipts_refuse_duplicate_batch_semantics() { - let tmp = TempDir::new().unwrap(); - let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path).await.unwrap(); - drop(db); - convert_final_temporal_schema_to_released_v3(&db_path).await; - insert_released_v3_projection_receipts(&db_path, (5, 3, 2)).await; - let triggers = suspend_schema_triggers(&db_path).await; - let raw_db = TestConnection::open(&db_path); - let conn = (*raw_db).clone(); - conn.execute( - "UPDATE session_temporal_generations - SET frozen_watermarks_json = - '{\"active_generation\":1,\"source_frontier\":10,\"projection_frontier\":10,\"summary_frontier\":0,\"cursor_key\":null}' - WHERE session_id = 'released-v3' AND generation = 1", - (), - ) - .await - .unwrap(); - conn.execute( - "UPDATE session_temporal_projection_receipts - SET frozen_watermarks_json = - '{\"active_generation\":1,\"source_frontier\":10,\"projection_frontier\":10,\"summary_frontier\":0,\"cursor_key\":null}' - WHERE session_id = 'released-v3' AND generation = 1", - (), - ) - .await - .unwrap(); - drop(conn); - drop(raw_db); - restore_schema_triggers(&db_path, &triggers).await; - - let error = match open_global_db(&db_path).await { - Ok(_) => panic!("unbound v3 duplicate-batch semantics must not be fabricated"), - Err(error) => error, - }; - let (authority, reason) = error.reset_required_context().unwrap(); - assert_eq!(authority, "session temporal"); - assert!( - reason.contains("no sanctioned conversion"), - "unexpected reason: {reason}" - ); - assert_eq!(temporal_schema_version(&db_path).await, 3); - assert!( - !persisted_column_names(&db_path, "session_temporal_projection_receipts") - .await - .iter() - .any(|column| column == "batch_item_count") - ); -} - -#[tokio::test] -async fn ambiguous_released_v3_refresh_progress_rolls_back_without_batch_counts() { - let tmp = TempDir::new().unwrap(); - let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path).await.unwrap(); - drop(db); - convert_final_temporal_schema_to_released_v3(&db_path).await; - insert_seeded_active_released_v3_refresh_receipts(&db_path, (8, 2, 2)).await; - let triggers = suspend_schema_triggers(&db_path).await; - let raw_db = TestConnection::open(&db_path); - let conn = (*raw_db).clone(); - conn.execute( - "UPDATE session_refresh_progress - SET committed_records = 11, - coverage_json = '{\"visible\":11,\"hidden\":0,\"unknown\":0,\"redacted\":0}' - WHERE session_id = 'released-v3' AND operation_id = 'refresh-v3' - AND progress_ordinal = 1", - (), - ) - .await - .unwrap(); - drop(conn); - drop(raw_db); - restore_schema_triggers(&db_path, &triggers).await; - - let error = match open_global_db(&db_path).await { - Ok(_) => panic!("ambiguous released-v3 refresh progress must be refused"), - Err(error) => error, - }; - let (authority, reason) = error.reset_required_context().unwrap(); - assert_eq!(authority, "session temporal"); - assert!( - reason.contains("no sanctioned conversion"), - "unexpected reason: {reason}" - ); - assert_eq!(temporal_schema_version(&db_path).await, 3); - assert!( - !persisted_column_names(&db_path, "session_temporal_projection_receipts") - .await - .iter() - .any(|column| column == "batch_item_count") - ); -} - -#[tokio::test] -async fn non_monotonic_released_v3_receipts_roll_back_the_v4_migration() { - let tmp = TempDir::new().unwrap(); - let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) - .await - .expect("fresh initialization should install the final temporal schema"); - drop(db); - convert_final_temporal_schema_to_released_v3(&db_path).await; - insert_seeded_active_released_v3_refresh_receipts(&db_path, (1, 1, 1)).await; - - let error = match open_global_db(&db_path).await { - Ok(_) => panic!("non-monotonic released-v3 receipts must not migrate"), - Err(error) => error, - }; - let (authority, reason) = error - .reset_required_context() - .expect("invalid released-v3 progress must return typed reset-required"); - assert_eq!(authority, "session temporal"); - assert!( - reason.contains("no sanctioned conversion"), - "unexpected reason: {reason}" - ); - assert_eq!(temporal_schema_version(&db_path).await, 3); - assert_eq!( - persisted_column_names(&db_path, "session_temporal_projection_receipts").await, - [ - "session_id", - "generation", - "batch_ordinal", - "batch_digest", - "frozen_watermarks_json", - "source_through", - "projection_through", - "occurrence_count", - "occurrence_digest", - "dimension_count", - "dimension_digest", - "copy_count", - "copy_digest", - "assertion_count", - "assertion_digest", - "supersession_count", - "supersession_digest", - "current_count", - "current_digest", - "fts_count", - "fts_digest", - "committed_at", - ] - ); -} - -#[tokio::test] -async fn drifted_released_v3_temporal_shape_is_refused_without_mutation() { - let tmp = TempDir::new().unwrap(); - let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) - .await - .expect("fresh initialization should install the final temporal schema"); - drop(db); - convert_final_temporal_schema_to_released_v3(&db_path).await; - let raw_db = TestConnection::open(&db_path); - let conn = (*raw_db).clone(); - conn.execute_batch( - "ALTER TABLE session_temporal_projection_receipts - ADD COLUMN branch_local_count INTEGER;", - ) - .await - .unwrap(); - drop(conn); - drop(raw_db); - - let error = match open_global_db(&db_path).await { - Ok(_) => panic!("drifted v3 receipt storage must not migrate"), - Err(error) => error, - }; - assert_eq!( - error - .reset_required_context() - .map(|(authority, _)| authority), - Some("session temporal") - ); - assert_eq!(temporal_schema_version(&db_path).await, 3); - assert_eq!( - persisted_column_names(&db_path, "session_temporal_projection_receipts") - .await - .last() - .map(String::as_str), - Some("branch_local_count") - ); -} - -#[tokio::test] -async fn drifted_released_v3_temporal_trigger_is_refused_without_repair() { - let tmp = TempDir::new().unwrap(); - let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) - .await - .expect("fresh initialization should install the final temporal schema"); - drop(db); - convert_final_temporal_schema_to_released_v3(&db_path).await; - let raw_db = TestConnection::open(&db_path); - let conn = (*raw_db).clone(); - conn.execute_batch( - "DROP TRIGGER session_refresh_progress_insert_guard_v1; - CREATE TRIGGER session_refresh_progress_insert_guard_v1 - BEFORE INSERT ON session_refresh_progress BEGIN SELECT 1; END;", - ) - .await - .unwrap(); - drop(conn); - drop(raw_db); - let drifted_guard = - normalized_trigger_sql(&db_path, "session_refresh_progress_insert_guard_v1").await; - - let error = match open_global_db(&db_path).await { - Ok(_) => panic!("drifted v3 temporal triggers must not be repaired and migrated"), - Err(error) => error, - }; - assert_eq!( - error - .reset_required_context() - .map(|(authority, _)| authority), - Some("session temporal") - ); - assert_eq!(temporal_schema_version(&db_path).await, 3); - assert_eq!( - normalized_trigger_sql(&db_path, "session_refresh_progress_insert_guard_v1").await, - drifted_guard, - "refused v3 trigger drift must remain untouched" - ); - assert!( - !persisted_column_names(&db_path, "session_temporal_projection_receipts") - .await - .iter() - .any(|column| column == "batch_item_count") - ); -} - #[tokio::test] async fn temporal_schema_accepts_only_fresh_or_exact_final_stores() { let tmp = TempDir::new().unwrap(); @@ -1379,7 +464,11 @@ async fn temporal_schema_rejects_extra_marker_rows_without_mutating_them() { assert_eq!( marker_rows, [ - ("session-temporal".to_string(), 4, 90), + ( + "session-temporal".to_string(), + tracedecay_session_temporal_store::SESSION_TEMPORAL_SCHEMA_VERSION, + 90, + ), ("unexpected-temporal-marker".to_string(), 4, 91), ], "typed refusal must preserve every temporal marker row" @@ -1525,7 +614,7 @@ async fn temporal_schema_rejects_transition_storage_without_mutating_it() { } #[tokio::test] -async fn temporal_schema_rejects_retired_summary_sources_without_mutation() { +async fn temporal_schema_rejects_foreign_summary_namespace_tables_without_mutation() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); let db = open_global_db(&db_path) @@ -1536,8 +625,8 @@ async fn temporal_schema_rejects_retired_summary_sources_without_mutation() { let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); conn.execute_batch( - "CREATE TABLE session_summary_sources (retired_row INTEGER NOT NULL); - INSERT INTO session_summary_sources(retired_row) VALUES (91);", + "CREATE TABLE session_summary_source_bindings (retired_row INTEGER NOT NULL); + INSERT INTO session_summary_source_bindings(retired_row) VALUES (91);", ) .await .unwrap(); @@ -1547,29 +636,25 @@ async fn temporal_schema_rejects_retired_summary_sources_without_mutation() { let before_version = temporal_schema_version(&db_path).await; let error = match open_global_db(&db_path).await { - Ok(_) => panic!("retired summary-source storage must require reset"), + Ok(_) => panic!("foreign summary-namespace storage must require reset"), Err(error) => error, }; let (authority, reason) = error .reset_required_context() - .expect("retired summary-source storage must return typed reset-required"); - // `session_summary_sources` is the retired pre-Grafeo relational - // authority, so the session-relation authority claims it ahead of the - // temporal namespace scan, the same order production admission has - // always used on reopen. - assert_eq!(authority, "registered session relation store"); + .expect("foreign summary-namespace storage must return typed reset-required"); + assert_eq!(authority, "session temporal"); assert!( - reason.contains("session_summary_sources"), + reason.contains("session_summary_source_bindings"), "unexpected reason: {reason}" ); assert!( - table_exists(&db_path, "session_summary_sources").await, - "typed refusal must not delete retired summary-source storage" + table_exists(&db_path, "session_summary_source_bindings").await, + "typed refusal must not delete foreign summary-namespace storage" ); assert_eq!( - row_count(&db_path, "session_summary_sources").await, + row_count(&db_path, "session_summary_source_bindings").await, 1, - "typed refusal must not rewrite retired summary-source rows" + "typed refusal must not rewrite foreign summary-namespace rows" ); assert_eq!(temporal_schema_version(&db_path).await, before_version); assert_eq!( diff --git a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_constraints.rs b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_constraints.rs index af2eebad96..47b643c6a7 100644 --- a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_constraints.rs +++ b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_constraints.rs @@ -46,25 +46,24 @@ async fn temporal_schema_rejects_cross_session_and_generation_rows() { session_id, generation, occurrence_id, source_observation_id, source_provider, projection_output_ordinal, retrieval_anchor_id, role, knowledge_at, valid_time_json, evidence_json, - sanitized_content_digest, sanitized_content_bytes, - snippet_text, index_text + sanitized_content_digest, sanitized_content_bytes, index_text ) VALUES ('session-one', 1, 'occurrence-one', 'observation-one', 'test', 0, 'anchor-one', 'assistant', 100, json_object('kind', 'unknown'), '{}', '0000000000000000000000000000000000000000000000000000000000000000', - 3, 'one', 'one'), + 3, 'one'), ('session-one', 2, 'occurrence-two', 'observation-one', 'test', 0, 'anchor-one', 'assistant', 100, json_object('kind', 'unknown'), '{}', '0000000000000000000000000000000000000000000000000000000000000000', - 3, 'two', 'two'), + 3, 'two'), ('session-two', 1, 'occurrence-three', 'observation-one', 'test', 0, 'anchor-one', 'assistant', 100, json_object('kind', 'unknown'), '{}', '0000000000000000000000000000000000000000000000000000000000000000', - 5, 'three', 'three');", + 5, 'three');", ) .await .unwrap(); @@ -170,15 +169,14 @@ async fn temporal_schema_rejects_invalid_current_assertion_and_valid_time_rows() session_id, generation, occurrence_id, source_observation_id, source_provider, projection_output_ordinal, retrieval_anchor_id, role, knowledge_at, valid_time_json, evidence_json, - sanitized_content_digest, sanitized_content_bytes, - snippet_text, index_text + sanitized_content_digest, sanitized_content_bytes, index_text ) VALUES ( 'session-one', 1, 'occurrence-one', 'observation-one', 'test', 0, 'anchor-subject', 'assistant', 100, json_object('kind', 'known', 'valid_at', 100), '{}', '0000000000000000000000000000000000000000000000000000000000000000', - 3, 'one', 'one' + 3, 'one' ); INSERT INTO session_assertions ( session_id, generation, assertion_id, assertion_kind, @@ -256,15 +254,14 @@ async fn temporal_schema_rejects_invalid_current_assertion_and_valid_time_rows() session_id, generation, occurrence_id, source_observation_id, source_provider, projection_output_ordinal, retrieval_anchor_id, role, knowledge_at, valid_time_json, evidence_json, - sanitized_content_digest, sanitized_content_bytes, - snippet_text, index_text + sanitized_content_digest, sanitized_content_bytes, index_text ) VALUES ( 'session-one', 1, 'occurrence-invalid-time', 'observation-one', 'test', 1, 'anchor-subject', 'assistant', 101, json_object('kind', 'unknown', 'valid_at', 101), '{}', '0000000000000000000000000000000000000000000000000000000000000000', - 3, 'bad', 'bad' + 3, 'bad' )", "unknown occurrence valid time must not include valid_at", ), @@ -1291,11 +1288,12 @@ async fn temporal_schema_keeps_append_only_authority_immutable() { anchor_id, anchor_json, owner_json, projection_generation ) VALUES ('append-anchor', '{}', '{}', 'test'); INSERT INTO session_summary_nodes ( - summary_id, session_id, summary_anchor_id, summary_text, index_text, + summary_id, session_id, provider, conversation_id, depth, summary_anchor_id, + summary_text, summary_hash, summary_token_count, source_token_count, source_horizon_json, created_at ) VALUES ( - 'append-summary', 'append-session', 'append-anchor', - 'summary', 'summary', '{}', 100 + 'append-summary', 'append-session', 'test', 'append-session', 0, 'append-anchor', + 'summary', 'hash', 1, 1, '{}', 100 ); INSERT INTO session_temporal_generations ( session_id, generation, state, frozen_watermarks_json, created_at diff --git a/crates/tracedecay-global-db/src/tests/session_sync.rs b/crates/tracedecay-global-db/src/tests/session_sync.rs index b70ffd2394..977996f093 100644 --- a/crates/tracedecay-global-db/src/tests/session_sync.rs +++ b/crates/tracedecay-global-db/src/tests/session_sync.rs @@ -16,9 +16,12 @@ async fn registered_session_message_batch_executes_json_rowset_with_exact_provid } writer .execute( - "INSERT INTO session_messages(provider, message_id, session_id, role, ordinal, text) - VALUES ('cursor', 'comp:b2', 'session.fixture', 'user', 1, 'cursor'), - ('codex', 'comp:b1', 'session.fixture', 'user', 1, 'codex')", + "INSERT INTO lcm_raw_messages( + provider, message_id, session_id, role, ordinal, content, content_hash, + storage_kind + ) + VALUES ('cursor', 'comp:b2', 'session.fixture', 'user', 1, 'cursor', 'h', 'inline'), + ('codex', 'comp:b1', 'session.fixture', 'user', 1, 'codex', 'h', 'inline')", (), ) .await @@ -50,10 +53,11 @@ async fn session_sync_journal_survives_remount_and_compare_and_swap() { let scope = tracedecay_domain::ObservationScopeV1::Project { project_id: tracedecay_domain::ProjectId::new("project.fixture").unwrap(), }; - let cursor = tracedecay_domain::ObservationSourceCursorV1::new( + let cursor = tracedecay_domain::ObservationSourceCursorV1::for_ordering( source.clone(), scope.clone(), tracedecay_domain::ObservationSourceGenerationV1::new(1).unwrap(), + tracedecay_domain::ObservationOrderingDomainV1::FileBytes, 72, ) .unwrap(); diff --git a/crates/tracedecay-global-db/tests/hotpath_coverage.rs b/crates/tracedecay-global-db/tests/hotpath_coverage.rs index 3efe7f9a7d..e8d2adb1f3 100644 --- a/crates/tracedecay-global-db/tests/hotpath_coverage.rs +++ b/crates/tracedecay-global-db/tests/hotpath_coverage.rs @@ -51,15 +51,16 @@ fn exercise_measured_hot_paths() { .expect("begin coverage transaction"); transaction .execute_batch(&format!( - "INSERT INTO session_messages( - provider, message_id, session_id, role, timestamp, ordinal, text, - kind, model, tool_names, source_path, source_offset, metadata_json + "INSERT INTO lcm_raw_messages( + provider, message_id, session_id, role, timestamp, ordinal, content, + kind, model, tool_names, source_path, source_offset, metadata_json, + content_hash, storage_kind ) VALUES ('{PROVIDER}', 'message-000000', '{SESSION}', 'assistant', 1, 1, - 'payload', 'activity', NULL, 'tool', NULL, NULL, NULL), + 'payload', 'activity', NULL, 'tool', NULL, NULL, NULL, 'hash', 'inline'), ('{PROVIDER}', 'message-000001', '{SESSION}', 'assistant', 2, 2, - 'payload', 'activity', NULL, 'tool', NULL, NULL, NULL);" + 'payload', 'activity', NULL, 'tool', NULL, NULL, NULL, 'hash', 'inline');" )) .await .expect("seed coverage session activity"); diff --git a/crates/tracedecay-global-db/tests/schema_convergence_hotpath.rs b/crates/tracedecay-global-db/tests/schema_convergence_hotpath.rs index 4dec6df889..2abc6918fb 100644 --- a/crates/tracedecay-global-db/tests/schema_convergence_hotpath.rs +++ b/crates/tracedecay-global-db/tests/schema_convergence_hotpath.rs @@ -79,9 +79,14 @@ fn authority_fixture( payload, ) .expect("durable observation"); - let cursor = - ObservationSourceCursorV1::new(source, ObservationScopeV1::Profile, generation, end) - .expect("committed source cursor"); + let cursor = ObservationSourceCursorV1::for_ordering( + source, + ObservationScopeV1::Profile, + generation, + ObservationOrderingDomainV1::FileBytes, + end, + ) + .expect("committed source cursor"); (observation, cursor) } diff --git a/crates/tracedecay-graph-db/src/bundle.rs b/crates/tracedecay-graph-db/src/bundle.rs index dd6bdf313d..908112ca6b 100644 --- a/crates/tracedecay-graph-db/src/bundle.rs +++ b/crates/tracedecay-graph-db/src/bundle.rs @@ -16,6 +16,12 @@ //! absent without error, so bundles written before a new artifact existed //! stay serveable. //! +//! Artifact bytes carry no generation of their own, so they are stored once +//! under their content digest in an artifact root shared by every worktree +//! scope of a project, while each generation's manifest stays in its own +//! generations root. Retiring a bundle removes only its manifest; the +//! project's segment sweep collects artifacts no manifest names. +//! //! A missing or mismatched artifact is a TYPED state //! ([`SealedReadBundleArtifactStateV1::Absent`] / //! [`SealedReadBundleArtifactStateV1::Stale`]), never a silent fallback: @@ -108,6 +114,7 @@ pub enum SealedReadBundleArtifactStateV1 { /// artifact was recorded in `staged`. pub struct SealedReadBundleWriterV1 { root: PathBuf, + artifact_root: PathBuf, sealed: SealedGraphStateDigest, staged: Vec<(SealedReadBundleArtifactV1, PathBuf)>, pending: Option, @@ -115,15 +122,22 @@ pub struct SealedReadBundleWriterV1 { } impl SealedReadBundleWriterV1 { - pub fn create(root: &Path, sealed: &SealedGraphStateDigest) -> Result { - if !root.is_dir() { + /// `root` holds this generation's manifest; `artifact_root` holds the + /// content-addressed artifacts every scope of the project shares. + pub fn create( + root: &Path, + artifact_root: &Path, + sealed: &SealedGraphStateDigest, + ) -> Result { + if !root.is_dir() || !artifact_root.is_dir() { return Err(GraphDbError::unavailable( "sealed read bundle root is not a directory", )); } - sweep_aborted_sealed_read_bundle_temporaries(root, sealed)?; + sweep_aborted_sealed_read_bundle_temporaries(root, artifact_root, sealed)?; Ok(Self { root: root.to_path_buf(), + artifact_root: artifact_root.to_path_buf(), sealed: sealed.clone(), staged: Vec::new(), pending: None, @@ -133,7 +147,7 @@ impl SealedReadBundleWriterV1 { fn temporary_path(&self, name: &str) -> Result { Ok(bundle_tmp_path( - &self.root, + &self.artifact_root, &sealed_hex(&self.sealed)?, name, )) @@ -232,9 +246,13 @@ impl SealedReadBundleWriterV1 { Ok(()) } - /// Renames every staged artifact into place, then writes the manifest - /// bound to `identity` and syncs the directory. The manifest write is the - /// commit point: without a manifest the artifacts are invisible. + /// Places every staged artifact under its content address, then writes + /// the manifest bound to `identity` and syncs both roots. The manifest + /// write is the commit point: without a manifest the artifacts are + /// invisible. An artifact another scope already placed is verified and + /// reused. The caller must keep the project's segment sweep out from the + /// first placement until this returns, since a placed artifact is + /// unreferenced until the manifest lands. pub fn commit( mut self, identity: &GraphGenerationManifestIdentity, @@ -245,15 +263,15 @@ impl SealedReadBundleWriterV1 { let identity_digest = generation_identity_frames_digest(identity, check)?; let mut artifacts = Vec::with_capacity(self.staged.len()); for (artifact, temporary) in std::mem::take(&mut self.staged) { - let target = self.root.join(artifact_file_name(&hex, &artifact.name)); - remove_stale_regular_file(&target)?; - std::fs::rename(&temporary, &target).map_err(|error| { - GraphDbError::unavailable(format!( - "failed to place sealed read bundle artifact: {error}" - )) - })?; + let placed = + place_content_addressed_artifact(&temporary, &self.artifact_root, &artifact, check); + if placed.is_err() { + let _ = std::fs::remove_file(&temporary); + } + placed?; artifacts.push(artifact); } + sync_bundle_directory(&self.artifact_root)?; artifacts.sort_by(|left, right| left.name.cmp(&right.name)); let manifest = SealedReadBundleManifestV1 { format: SEALED_READ_BUNDLE_FORMAT_V1.to_owned(), @@ -286,14 +304,14 @@ impl SealedReadBundleWriterV1 { )) }) }; + // A placed artifact may already back another scope's manifest, so a + // failed commit leaves it to the sweep rather than unlinking it. if let Err(error) = write_manifest() { self.abort_pending(); - self.remove_committed_artifacts(&hex, &manifest); return Err(error); } if let Err(error) = std::fs::rename(&temporary, self.root.join(manifest_file_name(&hex))) { self.abort_pending(); - self.remove_committed_artifacts(&hex, &manifest); return Err(GraphDbError::unavailable(format!( "failed to place sealed read bundle manifest: {error}" ))); @@ -303,12 +321,79 @@ impl SealedReadBundleWriterV1 { self.committed = true; Ok(manifest) } +} + +/// Moves `temporary` to the artifact's content address, or verifies the +/// artifact already there. A same-name file with other bytes fails closed. +fn place_content_addressed_artifact( + temporary: &Path, + artifact_root: &Path, + artifact: &SealedReadBundleArtifactV1, + check: &dyn Fn() -> Result<(), GraphDbError>, +) -> Result<(), GraphDbError> { + let target = + artifact_root.join(artifact_content_file_name(&artifact.digest).ok_or_else(|| { + GraphDbError::invalid("sealed read bundle artifact digest is not sha256") + })?); + match target.symlink_metadata() { + Ok(metadata) if metadata.file_type().is_file() => { + let (digest, bytes) = digest_file(&target, check)?; + if bytes != artifact.bytes || digest != artifact.digest { + return Err(GraphDbError::unavailable( + "existing sealed read bundle artifact does not match its content address", + )); + } + std::fs::remove_file(temporary).map_err(|error| { + GraphDbError::unavailable(format!( + "failed to withdraw a duplicate sealed read bundle artifact: {error}" + )) + }) + } + Ok(_) => Err(GraphDbError::unavailable( + "sealed read bundle artifact path is not a regular file", + )), + Err(error) if error.kind() == io::ErrorKind::NotFound => { + std::fs::rename(temporary, &target).map_err(|error| { + GraphDbError::unavailable(format!( + "failed to place sealed read bundle artifact: {error}" + )) + }) + } + Err(error) => Err(GraphDbError::unavailable(format!( + "failed to inspect sealed read bundle artifact: {error}" + ))), + } +} - fn remove_committed_artifacts(&self, hex: &str, manifest: &SealedReadBundleManifestV1) { - for artifact in &manifest.artifacts { - let _ = std::fs::remove_file(self.root.join(artifact_file_name(hex, &artifact.name))); +fn digest_file( + path: &Path, + check: &dyn Fn() -> Result<(), GraphDbError>, +) -> Result<(String, u64), GraphDbError> { + let mut file = std::fs::File::open(path).map_err(|error| { + GraphDbError::unavailable(format!( + "failed to open sealed read bundle artifact: {error}" + )) + })?; + let mut digest = Sha256::new(); + let mut bytes = 0_u64; + let mut chunk = vec![0_u8; IO_CHUNK_BYTES]; + loop { + check()?; + let read = file.read(&mut chunk).map_err(|error| { + GraphDbError::unavailable(format!( + "failed to read sealed read bundle artifact: {error}" + )) + })?; + if read == 0 { + break; } + digest.update(&chunk[..read]); + bytes = bytes.saturating_add(read as u64); } + Ok(( + format!("sha256:{}", encode_lowercase_hex(&digest.finalize())), + bytes, + )) } impl Drop for SealedReadBundleWriterV1 { @@ -328,6 +413,7 @@ impl Drop for SealedReadBundleWriterV1 { /// any byte is trusted. pub fn load_sealed_read_bundle_artifact( root: &Path, + artifact_root: &Path, sealed: &SealedGraphStateDigest, identity: &GraphGenerationManifestIdentity, name: &str, @@ -394,7 +480,12 @@ pub fn load_sealed_read_bundle_artifact( detail: "sealed read bundle artifact exceeds its byte bound".to_owned(), }); } - let path = root.join(artifact_file_name(&hex, &artifact.name)); + let Some(file_name) = artifact_content_file_name(&artifact.digest) else { + return Ok(SealedReadBundleArtifactStateV1::Stale { + detail: format!("sealed read bundle artifact `{name}` digest is not sha256"), + }); + }; + let path = artifact_root.join(file_name); let file = match std::fs::File::open(&path) { Ok(file) => file, Err(error) if error.kind() == io::ErrorKind::NotFound => { @@ -461,9 +552,11 @@ pub fn load_sealed_read_bundle_artifact( }) } -/// Removes the generation's bundle manifest and every bundle file filed under -/// its sealed digest. Idempotent: an absent bundle retires successfully. -/// Called from the same retirement pass that collects the generation itself. +/// Removes the generation's bundle manifest and every file filed under its +/// sealed digest in `root`. Shared artifacts stay for the project's sweep, +/// which keeps them while any other manifest names them. Idempotent: an +/// absent bundle retires successfully. Called from the same retirement pass +/// that collects the generation itself. pub fn retire_sealed_read_bundle( root: &Path, sealed: &SealedGraphStateDigest, @@ -509,6 +602,18 @@ pub fn retire_sealed_read_bundle( /// a committed bundle. Activation retries call this (via [`SealedReadBundleWriterV1::create`]) /// so a prior OOM or cancelled catalog write cannot stack `.tmp` files. pub fn sweep_aborted_sealed_read_bundle_temporaries( + root: &Path, + artifact_root: &Path, + sealed: &SealedGraphStateDigest, +) -> Result<(), GraphDbError> { + sweep_aborted_temporaries_in(root, sealed)?; + if artifact_root != root { + sweep_aborted_temporaries_in(artifact_root, sealed)?; + } + Ok(()) +} + +fn sweep_aborted_temporaries_in( root: &Path, sealed: &SealedGraphStateDigest, ) -> Result<(), GraphDbError> { @@ -664,12 +769,15 @@ fn windows_process_is_dead(pid: u32) -> bool { use std::ffi::c_void; const PROCESS_QUERY_LIMITED_INFORMATION: u32 = 0x1000; + const STILL_ACTIVE: u32 = 259; const ERROR_INVALID_PARAMETER: i32 = 87; #[link(name = "kernel32")] unsafe extern "system" { #[link_name = "OpenProcess"] fn open_process(access: u32, inherit_handle: i32, process_id: u32) -> *mut c_void; + #[link_name = "GetExitCodeProcess"] + fn get_exit_code_process(process: *mut c_void, exit_code: *mut u32) -> i32; #[link_name = "CloseHandle"] fn close_handle(handle: *mut c_void) -> i32; } @@ -678,8 +786,14 @@ fn windows_process_is_dead(pid: u32) -> bool { if process.is_null() { return std::io::Error::last_os_error().raw_os_error() == Some(ERROR_INVALID_PARAMETER); } + // An exited process stays openable for as long as anyone (its parent, + // typically) holds a handle to it, so a successful open is not liveness. + let mut exit_code = 0_u32; + // SAFETY: `process` is a live owned handle, `exit_code` is writable for + // the duration of the call, and the handle is closed exactly once. + let read = unsafe { get_exit_code_process(process, &mut exit_code) }; let _ = unsafe { close_handle(process) }; - false + read != 0 && exit_code != STILL_ACTIVE } fn bundle_tmp_path(root: &Path, hex: &str, name: &str) -> PathBuf { @@ -700,8 +814,78 @@ fn manifest_file_name(hex: &str) -> String { format!("read-bundle-{hex}.json") } -fn artifact_file_name(hex: &str, name: &str) -> String { - format!("read-bundle-{hex}.{name}.bin") +fn artifact_content_file_name(digest: &str) -> Option { + let hex = sha256_hex_suffix(digest)?; + is_sha256_hex(hex).then(|| format!("{ARTIFACT_FILE_PREFIX}{hex}{ARTIFACT_FILE_SUFFIX}")) +} + +fn is_sha256_hex(hex: &str) -> bool { + hex.len() == 64 + && hex + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) +} + +const ARTIFACT_FILE_PREFIX: &str = "read-bundle-artifact-"; +const ARTIFACT_FILE_SUFFIX: &str = ".bin"; + +/// The `sha256:` digest a shared artifact file name addresses, or `None` for +/// any other name. +#[must_use] +pub fn sealed_read_bundle_artifact_file_digest(file_name: &str) -> Option { + let hex = file_name + .strip_prefix(ARTIFACT_FILE_PREFIX)? + .strip_suffix(ARTIFACT_FILE_SUFFIX)?; + is_sha256_hex(hex).then(|| format!("sha256:{hex}")) +} + +/// The artifact digests a bundle manifest at `path` names, or `None` when +/// `path` is not a bundle manifest. A manifest that cannot be parsed names +/// nothing: loading treats it as stale and never reads an artifact through it. +pub fn sealed_read_bundle_manifest_artifact_digests( + path: &Path, +) -> Result>, GraphDbError> { + let is_manifest = path + .file_name() + .and_then(|name| name.to_str()) + .and_then(|name| name.strip_prefix("read-bundle-")) + .and_then(|name| name.strip_suffix(".json")) + .is_some_and(is_sha256_hex); + if !is_manifest { + return Ok(None); + } + let metadata = match path.symlink_metadata() { + Ok(metadata) => metadata, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(Some(Vec::new())), + Err(error) => { + return Err(GraphDbError::unavailable(format!( + "failed to stat sealed read bundle manifest: {error}" + ))); + } + }; + if !metadata.is_file() || metadata.len() > MAX_SEALED_READ_BUNDLE_MANIFEST_BYTES_V1 { + return Ok(Some(Vec::new())); + } + let bytes = match std::fs::read(path) { + Ok(bytes) => bytes, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(Some(Vec::new())), + Err(error) => { + return Err(GraphDbError::unavailable(format!( + "failed to read sealed read bundle manifest: {error}" + ))); + } + }; + Ok(Some( + serde_json::from_slice::(&bytes) + .map(|manifest| { + manifest + .artifacts + .into_iter() + .map(|artifact| artifact.digest) + .collect() + }) + .unwrap_or_default(), + )) } fn validate_artifact_name(name: &str) -> Result<(), GraphDbError> { @@ -718,25 +902,6 @@ fn validate_artifact_name(name: &str) -> Result<(), GraphDbError> { Ok(()) } -fn remove_stale_regular_file(path: &Path) -> Result<(), GraphDbError> { - match path.symlink_metadata() { - Ok(metadata) if metadata.file_type().is_file() => { - std::fs::remove_file(path).map_err(|error| { - GraphDbError::unavailable(format!( - "failed to replace stale sealed read bundle file: {error}" - )) - }) - } - Ok(_) => Err(GraphDbError::unavailable( - "sealed read bundle path is not a regular file", - )), - Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), - Err(error) => Err(GraphDbError::unavailable(format!( - "failed to inspect sealed read bundle path: {error}" - ))), - } -} - fn remove_bundle_file(path: &Path) -> Result { match path.symlink_metadata() { Ok(metadata) if metadata.file_type().is_file() => { @@ -791,11 +956,40 @@ mod tests { } fn sealed() -> SealedGraphStateDigest { - SealedGraphStateDigest::try_from(format!("sha256:{}", "ab".repeat(32))).unwrap() + sealed_of("ab") + } + + fn sealed_of(byte: &str) -> SealedGraphStateDigest { + SealedGraphStateDigest::try_from(format!("sha256:{}", byte.repeat(32))).unwrap() + } + + /// The shared artifact root beside `root`, created on first use. + fn artifacts(root: &Path) -> PathBuf { + let artifacts = root.join("artifacts"); + std::fs::create_dir_all(&artifacts).unwrap(); + artifacts + } + + fn entries(root: &Path) -> Vec { + let mut names: Vec = std::fs::read_dir(root) + .unwrap() + .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) + .collect(); + names.sort(); + names } fn write_bundle(root: &Path, generation: &str, payload: &[u8]) -> SealedReadBundleManifestV1 { - let mut writer = SealedReadBundleWriterV1::create(root, &sealed()).unwrap(); + write_bundle_for(root, &sealed(), generation, payload) + } + + fn write_bundle_for( + root: &Path, + sealed: &SealedGraphStateDigest, + generation: &str, + payload: &[u8], + ) -> SealedReadBundleManifestV1 { + let mut writer = SealedReadBundleWriterV1::create(root, &artifacts(root), sealed).unwrap(); writer .stage_artifact("interactive-catalog", &mut |out| { out.write_all(payload) @@ -815,6 +1009,7 @@ mod tests { let state = load_sealed_read_bundle_artifact( temp.path(), + &artifacts(temp.path()), &sealed(), &identity("generation-a"), "interactive-catalog", @@ -835,6 +1030,7 @@ mod tests { let temp = TempDir::new().unwrap(); let state = load_sealed_read_bundle_artifact( temp.path(), + &artifacts(temp.path()), &sealed(), &identity("generation-a"), "interactive-catalog", @@ -849,6 +1045,7 @@ mod tests { write_bundle(temp.path(), "generation-a", b"catalog-bytes"); let state = load_sealed_read_bundle_artifact( temp.path(), + &artifacts(temp.path()), &sealed(), &identity("generation-a"), "identity-index", @@ -869,6 +1066,7 @@ mod tests { write_bundle(temp.path(), "generation-a", b"catalog-bytes"); let state = load_sealed_read_bundle_artifact( temp.path(), + &artifacts(temp.path()), &sealed(), &identity("generation-b"), "interactive-catalog", @@ -886,14 +1084,13 @@ mod tests { #[test] fn tampered_artifact_bytes_are_typed_stale() { let temp = TempDir::new().unwrap(); - write_bundle(temp.path(), "generation-a", b"catalog-bytes"); - let artifact = temp.path().join(format!( - "read-bundle-{}.interactive-catalog.bin", - "ab".repeat(32) - )); + let manifest = write_bundle(temp.path(), "generation-a", b"catalog-bytes"); + let artifact = artifacts(temp.path()) + .join(artifact_content_file_name(&manifest.artifacts[0].digest).unwrap()); std::fs::write(&artifact, b"tampered-byte").unwrap(); let state = load_sealed_read_bundle_artifact( temp.path(), + &artifacts(temp.path()), &sealed(), &identity("generation-a"), "interactive-catalog", @@ -909,9 +1106,9 @@ mod tests { } #[test] - fn retirement_removes_manifest_artifacts_and_stage_leftovers() { + fn retirement_removes_manifest_and_stage_leftovers_but_keeps_shared_artifacts() { let temp = TempDir::new().unwrap(); - write_bundle(temp.path(), "generation-a", b"catalog-bytes"); + let manifest = write_bundle(temp.path(), "generation-a", b"catalog-bytes"); let hex = "ab".repeat(32); std::fs::write( temp.path().join(format!(".read-bundle-{hex}.orphan.1.tmp")), @@ -922,11 +1119,18 @@ mod tests { retire_sealed_read_bundle(temp.path(), &sealed()).unwrap(); - let remaining: Vec = std::fs::read_dir(temp.path()) - .unwrap() - .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) - .collect(); - assert_eq!(remaining, vec!["generation-unrelated.json".to_owned()]); + assert_eq!( + entries(temp.path()), + vec![ + "artifacts".to_owned(), + "generation-unrelated.json".to_owned() + ] + ); + assert_eq!( + entries(&artifacts(temp.path())), + vec![artifact_content_file_name(&manifest.artifacts[0].digest).unwrap()], + "another scope's manifest may still name the artifact; only the sweep collects it" + ); // Idempotent on an absent bundle. retire_sealed_read_bundle(temp.path(), &sealed()).unwrap(); @@ -936,7 +1140,9 @@ mod tests { fn dropped_writer_removes_staged_temporaries() { let temp = TempDir::new().unwrap(); { - let mut writer = SealedReadBundleWriterV1::create(temp.path(), &sealed()).unwrap(); + let mut writer = + SealedReadBundleWriterV1::create(temp.path(), &artifacts(temp.path()), &sealed()) + .unwrap(); writer .stage_artifact("interactive-catalog", &mut |out| { out.write_all(b"asdf") @@ -944,13 +1150,16 @@ mod tests { }) .unwrap(); } - assert_eq!(std::fs::read_dir(temp.path()).unwrap().count(), 0); + assert_eq!(entries(temp.path()), vec!["artifacts".to_owned()]); + assert!(entries(&artifacts(temp.path())).is_empty()); } #[test] fn aborted_write_removes_in_progress_temporary() { let temp = TempDir::new().unwrap(); - let mut writer = SealedReadBundleWriterV1::create(temp.path(), &sealed()).unwrap(); + let mut writer = + SealedReadBundleWriterV1::create(temp.path(), &artifacts(temp.path()), &sealed()) + .unwrap(); let error = writer .stage_artifact("interactive-catalog", &mut |out| { out.write_all(&[0u8; 4096]) @@ -960,14 +1169,17 @@ mod tests { .expect_err("aborted staging must fail"); assert!(error.to_string().contains("catalog write aborted")); drop(writer); - assert_eq!(std::fs::read_dir(temp.path()).unwrap().count(), 0); + assert_eq!(entries(temp.path()), vec!["artifacts".to_owned()]); + assert!(entries(&artifacts(temp.path())).is_empty()); } #[test] fn panic_during_write_removes_in_progress_temporary() { let temp = TempDir::new().unwrap(); let panicked = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - let mut writer = SealedReadBundleWriterV1::create(temp.path(), &sealed()).unwrap(); + let mut writer = + SealedReadBundleWriterV1::create(temp.path(), &artifacts(temp.path()), &sealed()) + .unwrap(); writer .stage_artifact("interactive-catalog", &mut |out| { out.write_all(&[0u8; 4096]) @@ -977,7 +1189,8 @@ mod tests { .expect("panic is the abort path"); })); assert!(panicked.is_err()); - assert_eq!(std::fs::read_dir(temp.path()).unwrap().count(), 0); + assert_eq!(entries(temp.path()), vec!["artifacts".to_owned()]); + assert!(entries(&artifacts(temp.path())).is_empty()); } #[test] @@ -1021,21 +1234,173 @@ mod tests { ) .unwrap(); - drop(SealedReadBundleWriterV1::create(temp.path(), &sealed()).unwrap()); + drop( + SealedReadBundleWriterV1::create(temp.path(), &artifacts(temp.path()), &sealed()) + .unwrap(), + ); - let mut remaining: Vec = std::fs::read_dir(temp.path()) - .unwrap() - .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) - .collect(); - remaining.sort(); assert_eq!( - remaining, + entries(temp.path()), vec![ live_name, format!(".read-bundle-{foreign}.interactive-catalog.1.tmp"), - format!("read-bundle-{hex}.interactive-catalog.bin"), + "artifacts".to_owned(), format!("read-bundle-{hex}.json"), ] ); } + + fn load_catalog(root: &Path, sealed: &SealedGraphStateDigest, generation: &str) -> Vec { + match load_sealed_read_bundle_artifact( + root, + &artifacts(root), + sealed, + &identity(generation), + "interactive-catalog", + &|| Ok(()), + ) + .unwrap() + { + SealedReadBundleArtifactStateV1::Loaded { bytes, .. } => bytes, + other => panic!("expected loaded artifact, got {other:?}"), + } + } + + /// Two worktree scopes filing bundles under their own generations in one + /// shared artifact root: identical bytes are stored once, and each scope + /// still loads exactly the bytes its own manifest names. + #[test] + fn identical_artifacts_of_two_generations_share_one_file_and_divergent_ones_do_not() { + let temp = TempDir::new().unwrap(); + let first = temp.path().join("first"); + let second = temp.path().join("second"); + let third = temp.path().join("third"); + for root in [&first, &second, &third] { + std::fs::create_dir_all(root).unwrap(); + } + let shared = temp.path().join("shared"); + std::fs::create_dir_all(&shared).unwrap(); + let write = + |root: &Path, sealed: &SealedGraphStateDigest, generation: &str, bytes: &[u8]| { + let mut writer = SealedReadBundleWriterV1::create(root, &shared, sealed).unwrap(); + writer + .stage_artifact("interactive-catalog", &mut |out| { + out.write_all(bytes) + .map_err(|error| GraphDbError::unavailable(error.to_string())) + }) + .unwrap(); + writer.commit(&identity(generation), &|| Ok(())).unwrap() + }; + let a = write(&first, &sealed_of("aa"), "generation-a", b"same-catalog"); + let b = write(&second, &sealed_of("bb"), "generation-b", b"same-catalog"); + let c = write( + &third, + &sealed_of("cc"), + "generation-c", + b"divergent-catalog", + ); + assert_eq!(a.artifacts, b.artifacts); + assert_ne!(a.artifacts, c.artifacts); + assert_eq!( + entries(&shared).len(), + 2, + "one file per distinct artifact: {:?}", + entries(&shared) + ); + + let load = |root: &Path, sealed: &SealedGraphStateDigest, generation: &str| { + match load_sealed_read_bundle_artifact( + root, + &shared, + sealed, + &identity(generation), + "interactive-catalog", + &|| Ok(()), + ) + .unwrap() + { + SealedReadBundleArtifactStateV1::Loaded { bytes, .. } => bytes, + other => panic!("expected loaded artifact, got {other:?}"), + } + }; + assert_eq!( + load(&first, &sealed_of("aa"), "generation-a"), + b"same-catalog" + ); + assert_eq!( + load(&third, &sealed_of("cc"), "generation-c"), + b"divergent-catalog" + ); + + retire_sealed_read_bundle(&second, &sealed_of("bb")).unwrap(); + assert_eq!( + load(&first, &sealed_of("aa"), "generation-a"), + b"same-catalog", + "retiring one scope's bundle keeps the artifact its sibling names" + ); + } + + #[test] + fn a_same_name_artifact_with_other_bytes_fails_closed() { + let temp = TempDir::new().unwrap(); + let manifest = write_bundle(temp.path(), "generation-a", b"catalog-bytes"); + let target = artifacts(temp.path()) + .join(artifact_content_file_name(&manifest.artifacts[0].digest).unwrap()); + std::fs::write(&target, b"other-bytes!!").unwrap(); + let mut writer = SealedReadBundleWriterV1::create( + temp.path(), + &artifacts(temp.path()), + &sealed_of("cd"), + ) + .unwrap(); + writer + .stage_artifact("interactive-catalog", &mut |out| { + out.write_all(b"catalog-bytes") + .map_err(|error| GraphDbError::unavailable(error.to_string())) + }) + .unwrap(); + let error = writer + .commit(&identity("generation-b"), &|| Ok(())) + .expect_err("a mismatched content address must not be reused"); + assert!( + error + .to_string() + .contains("does not match its content address"), + "{error}" + ); + assert_eq!(std::fs::read(&target).unwrap(), b"other-bytes!!"); + assert_eq!( + entries(&artifacts(temp.path())).len(), + 1, + "the refused stage must not linger" + ); + } + + #[test] + fn manifest_artifact_digests_name_exactly_the_shared_files() { + let temp = TempDir::new().unwrap(); + let manifest = write_bundle(temp.path(), "generation-a", b"catalog-bytes"); + let manifest_path = temp.path().join(manifest_file_name(&"ab".repeat(32))); + let digests = sealed_read_bundle_manifest_artifact_digests(&manifest_path) + .unwrap() + .expect("a bundle manifest"); + assert_eq!(digests, vec![manifest.artifacts[0].digest.clone()]); + let files = entries(&artifacts(temp.path())); + assert_eq!( + files + .iter() + .map(|name| sealed_read_bundle_artifact_file_digest(name)) + .collect::>(), + vec![Some(digests[0].clone())] + ); + assert_eq!( + sealed_read_bundle_manifest_artifact_digests(&temp.path().join("generation-x.json")) + .unwrap(), + None + ); + assert_eq!( + load_catalog(temp.path(), &sealed(), "generation-a"), + b"catalog-bytes" + ); + } } diff --git a/crates/tracedecay-graph-db/src/corrupt_store.rs b/crates/tracedecay-graph-db/src/corrupt_store.rs new file mode 100644 index 0000000000..e23cb0cab6 --- /dev/null +++ b/crates/tracedecay-graph-db/src/corrupt_store.rs @@ -0,0 +1,420 @@ +//! Deletion of a deterministically corrupt registry-owned graph container. +//! +//! The registry-owned `.grafeo` container is a derived index in every +//! namespace it serves: verified code and memory projections replay from the +//! relational publication journal and canonical sealed-generation seals, and +//! session relation projections re-materialize from the relational session +//! store. Permanent container corruption (a torn WAL write, a CRC fault in a +//! serialized block) therefore never destroys canonical data, but left alone +//! it permanently disables the mount: every open of the same bytes fails with +//! the identical typed [`GraphDbError::Corrupt`] and the store never heals. +//! +//! This module turns that deterministic verdict into a bounded recovery: +//! +//! 1. The corruption decision is serialized across incarnations by an +//! exclusive advisory lock on a sibling lock file. A holder elsewhere +//! means another authority is mid-decision, so this attempt reports a +//! retryable unavailable state and touches nothing. +//! 2. Under the lock, the deciding authority re-runs the identical failing +//! open itself. Only a second corruption verdict with the byte-identical +//! fault message, same GRAFEO code, same block, same CRC pair, proves +//! the fault deterministic. A successful reopen is served; a drifting +//! fault stays a terminal typed `Corrupt` for the operator, because a +//! fault that changes between attempts is hardware-shaped and a rebuild +//! onto the same medium would only re-corrupt. +//! 3. The container family is deleted: WAL sidecar, verified marker, spill +//! directory, then the container itself, and the `store_corrupt_deleted` +//! event records the fault fingerprint. No copy is kept. +//! +//! The caller then reopens the now-vacant path as a fresh store and the +//! ordinary publication and reconcile paths re-project every generation from +//! their canonical replay authorities. + +use std::fs::OpenOptions; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use tracedecay_domain::canonical_text::sha256_hex; +use tracedecay_private_fs::FileLease; +use tracedecay_private_fs::framed_log::{DirectorySyncPolicy, sync_directory}; + +use crate::{GraphDb, GraphDbError}; + +const CORRUPTION_DECISION_LOCK_SUFFIX: &str = ".corruption-lock"; + +/// Outcome of the corruption recovery protocol for one mount attempt. +#[derive(Debug)] +pub(crate) enum CorruptStoreRecovery> { + /// The verification reopen succeeded: the first verdict did not + /// reproduce, so the live database is served and nothing was touched. + Reopened(T), + /// The fault reproduced byte-identically and the container family was + /// deleted. The live path is vacant and the caller reopens it fresh. + Deleted, +} + +/// Runs the deterministic-corruption recovery protocol for `container` after +/// a mount-time open failed with the typed corruption verdict `first_fault`. +/// +/// `verification_open` must re-run the identical open the first verdict came +/// from; the protocol never trusts the first failure alone. +pub(crate) fn recover_deterministically_corrupt_container( + container: &Path, + first_fault: &str, + verification_open: &dyn Fn() -> Result, GraphDbError>, +) -> Result { + recover_deterministically_corrupt_container_with(container, first_fault, verification_open) +} + +pub(crate) fn recover_deterministically_corrupt_container_with( + container: &Path, + first_fault: &str, + verification_open: &dyn Fn() -> Result, +) -> Result, GraphDbError> { + let _decision_lock = acquire_corruption_decision_lock(container)?; + + // Re-verify under the decision lock: deletion only adopts a store whose + // corruption this exact authority reproduced. + let second_fault = match verification_open() { + Ok(database) => return Ok(CorruptStoreRecovery::Reopened(database)), + Err(GraphDbError::Corrupt { message }) => message, + Err(other) => return Err(other), + }; + if second_fault != first_fault { + return Err(GraphDbError::Corrupt { + message: format!( + "graph container corruption is not deterministic; refusing deletion: \ + first fault `{first_fault}`, second fault `{second_fault}`" + ), + }); + } + + delete_container_family(container, first_fault)?; + Ok(CorruptStoreRecovery::Deleted) +} + +/// Holds the exclusive cross-incarnation corruption-decision lock while the +/// verdict is re-proven and the family deleted. The lock file persists after +/// release: unlinking a held advisory lock would let a racer lock a fresh +/// inode while this holder still believes it owns the decision. +struct CorruptionDecisionLock { + _file: FileLease, +} + +fn corruption_decision_lock_path(container: &Path) -> Result { + let file_name = container_file_name(container)?; + Ok(container.with_file_name(format!("{file_name}{CORRUPTION_DECISION_LOCK_SUFFIX}"))) +} + +fn acquire_corruption_decision_lock( + container: &Path, +) -> Result { + let path = corruption_decision_lock_path(container)?; + let file = OpenOptions::new() + .create(true) + .read(true) + .write(true) + .truncate(false) + .open(&path) + .map_err(|error| { + GraphDbError::unavailable(format!( + "graph store corruption decision lock is unavailable at {}: {error}", + path.display() + )) + })?; + match file.try_lock().map_err(std::io::Error::from) { + Ok(()) => Ok(CorruptionDecisionLock { + _file: FileLease::held(file, "graph_db.corruption_decision"), + }), + // Windows LockFileEx reports ERROR_LOCK_VIOLATION (33) instead of + // WouldBlock. AccessDenied and sharing violations stay generic + // unavailable, not "another authority holds". + Err(error) if tracedecay_private_fs::is_lock_contended(&error) => { + Err(GraphDbError::unavailable(format!( + "another authority holds the graph store corruption decision for {}; \ + leaving the store untouched", + container.display() + ))) + } + Err(error) => Err(GraphDbError::unavailable(format!( + "graph store corruption decision lock failed for {}: {error}", + container.display() + ))), + } +} + +/// Deletes the container family. The container goes last: it is the fault +/// authority, so an interruption mid-delete leaves the corrupt container in +/// place for the next deciding authority rather than a vacant path beside +/// stranded sidecars. +fn delete_container_family(container: &Path, fault: &str) -> Result<(), GraphDbError> { + match container.symlink_metadata() { + Ok(metadata) if metadata.is_file() => {} + Ok(_) => { + return Err(GraphDbError::unavailable(format!( + "graph container at {} is no longer a regular file; refusing deletion", + container.display() + ))); + } + Err(error) => { + return Err(GraphDbError::unavailable(format!( + "graph container at {} disappeared during the corruption decision: {error}", + container.display() + ))); + } + } + + for sidecar in [ + wal_sidecar_path(container), + container.with_extension("verified"), + container.with_extension("spill"), + ] { + remove_family_member(&sidecar)?; + } + remove_family_member(container)?; + if let Some(parent) = container.parent() { + sync_directory(parent, DirectorySyncPolicy::Strict).map_err(|error| { + GraphDbError::DurabilityUncertain { + message: format!( + "corrupt graph container {} was deleted but its directory sync failed: \ + {error}", + container.display() + ), + } + })?; + } + + tracing::warn!( + event = "store_corrupt_deleted", + container = %container.display(), + fault_fingerprint = %format!("sha256:{}", sha256_hex(fault.as_bytes())), + fault = %fault, + "deterministically corrupt graph container deleted; \ + a fresh store rebuilds from the canonical replay authorities" + ); + Ok(()) +} + +/// Removes one family member without following a symlink. Every failure is +/// retryable: the container is removed last, so it still carries the verdict. +fn remove_family_member(member: &Path) -> Result<(), GraphDbError> { + let removed = match member.symlink_metadata() { + Ok(metadata) if metadata.is_dir() => std::fs::remove_dir_all(member), + Ok(_) => std::fs::remove_file(member), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(error) => Err(error), + }; + removed.map_err(|error| { + GraphDbError::unavailable(format!( + "corrupt graph store member {} could not be deleted: {error}", + member.display() + )) + }) +} + +/// `graph.grafeo` -> `graph.grafeo.wal`, matching Grafeo's sidecar layout. +fn wal_sidecar_path(container: &Path) -> PathBuf { + let mut sidecar = container.as_os_str().to_owned(); + sidecar.push(".wal"); + PathBuf::from(sidecar) +} + +fn container_file_name(container: &Path) -> Result<&str, GraphDbError> { + container + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| { + GraphDbError::invalid(format!( + "graph container path {} has no UTF-8 file name", + container.display() + )) + }) +} + +#[cfg(test)] +mod tests { + use std::sync::atomic::{AtomicUsize, Ordering}; + + use super::*; + + fn corrupt(message: &str) -> GraphDbError { + GraphDbError::Corrupt { + message: message.to_owned(), + } + } + + fn seeded_family(root: &Path) -> PathBuf { + let container = root.join("graph.grafeo"); + std::fs::write(&container, b"torn container bytes").unwrap(); + std::fs::create_dir(wal_sidecar_path(&container)).unwrap(); + std::fs::write( + wal_sidecar_path(&container).join("wal_00000001.log"), + b"wal", + ) + .unwrap(); + std::fs::write(container.with_extension("verified"), b"marker").unwrap(); + container + } + + fn directory_names(root: &Path) -> Vec { + let mut names = std::fs::read_dir(root) + .unwrap() + .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) + .collect::>(); + names.sort(); + names + } + + #[test] + fn identical_second_verdict_deletes_the_whole_family_without_a_copy() { + let temp = tempfile::tempdir().unwrap(); + let container = seeded_family(temp.path()); + let neighbour = temp.path().join("sessions.db"); + std::fs::write(&neighbour, b"live sibling").unwrap(); + let fault = "GRAFEO-X002: Serialization error: block 18 CRC mismatch: \ + expected 7d877cc5, got 5a475db3"; + + let outcome = + recover_deterministically_corrupt_container(&container, fault, &|| Err(corrupt(fault))) + .unwrap(); + + assert!(matches!(outcome, CorruptStoreRecovery::Deleted)); + assert_eq!( + directory_names(temp.path()), + vec![ + "graph.grafeo.corruption-lock".to_owned(), + "sessions.db".to_owned() + ], + "only the decision lock and unrelated siblings may remain" + ); + assert_eq!(std::fs::read(&neighbour).unwrap(), b"live sibling"); + } + + #[test] + fn successful_verification_reopen_touches_nothing() { + let temp = tempfile::tempdir().unwrap(); + let container = seeded_family(temp.path()); + + let outcome = + recover_deterministically_corrupt_container_with(&container, "fault", &|| Ok(())) + .unwrap(); + + assert!(matches!(outcome, CorruptStoreRecovery::Reopened(()))); + assert_eq!(std::fs::read(&container).unwrap(), b"torn container bytes"); + assert!(wal_sidecar_path(&container).is_dir()); + assert!(container.with_extension("verified").is_file()); + } + + #[test] + fn drifting_fault_refuses_deletion_and_stays_typed_corrupt() { + let temp = tempfile::tempdir().unwrap(); + let container = seeded_family(temp.path()); + + let error = recover_deterministically_corrupt_container( + &container, + "block 18 CRC mismatch", + &|| Err(corrupt("block 7 CRC mismatch")), + ) + .unwrap_err(); + + assert!( + matches!(&error, GraphDbError::Corrupt { message } + if message.contains("not deterministic") + && message.contains("block 18 CRC mismatch") + && message.contains("block 7 CRC mismatch")), + "a drifting fault is terminal and names both verdicts, got {error:?}" + ); + assert!(container.exists(), "a drifting fault must not delete bytes"); + assert!(wal_sidecar_path(&container).is_dir()); + } + + #[test] + fn non_corrupt_verification_failure_propagates_untouched() { + let temp = tempfile::tempdir().unwrap(); + let container = seeded_family(temp.path()); + + let error = recover_deterministically_corrupt_container(&container, "fault", &|| { + Err(GraphDbError::Cancelled) + }) + .unwrap_err(); + + assert_eq!(error, GraphDbError::Cancelled); + assert!(container.exists()); + } + + #[test] + fn held_decision_lock_reports_retryable_unavailable_without_verifying() { + let temp = tempfile::tempdir().unwrap(); + let container = seeded_family(temp.path()); + let foreign_holder = OpenOptions::new() + .create(true) + .read(true) + .write(true) + .truncate(false) + .open(corruption_decision_lock_path(&container).unwrap()) + .unwrap(); + foreign_holder + .try_lock() + .map_err(std::io::Error::from) + .unwrap(); + + let verification_attempts = AtomicUsize::new(0); + let error = recover_deterministically_corrupt_container(&container, "fault", &|| { + verification_attempts.fetch_add(1, Ordering::SeqCst); + Err(corrupt("fault")) + }) + .unwrap_err(); + + assert!( + matches!(&error, GraphDbError::Unavailable { message } + if message.contains("another authority holds")), + "a held decision lock is a retryable typed state, got {error:?}" + ); + assert_eq!( + verification_attempts.load(Ordering::SeqCst), + 0, + "a non-holder must not re-open the store it does not own" + ); + assert!(container.exists(), "a non-holder must not delete bytes"); + foreign_holder.unlock().unwrap(); + } + + #[test] + fn vanished_container_under_the_lock_is_a_retryable_abort() { + let temp = tempfile::tempdir().unwrap(); + let container = temp.path().join("graph.grafeo"); + + let error = recover_deterministically_corrupt_container(&container, "fault", &|| { + Err(corrupt("fault")) + }) + .unwrap_err(); + + assert!( + matches!(&error, GraphDbError::Unavailable { message } + if message.contains("disappeared during the corruption decision")), + "an already-recovered path must abort cleanly, got {error:?}" + ); + } + + #[cfg(unix)] + #[test] + fn symlinked_sidecar_is_unlinked_without_touching_its_target() { + let temp = tempfile::tempdir().unwrap(); + let outside = tempfile::tempdir().unwrap(); + let container = temp.path().join("graph.grafeo"); + std::fs::write(&container, b"torn").unwrap(); + std::fs::create_dir(outside.path().join("wal")).unwrap(); + std::fs::write(outside.path().join("wal/segment"), b"foreign").unwrap(); + std::os::unix::fs::symlink(outside.path().join("wal"), wal_sidecar_path(&container)) + .unwrap(); + + recover_deterministically_corrupt_container(&container, "fault", &|| Err(corrupt("fault"))) + .unwrap(); + + assert!(wal_sidecar_path(&container).symlink_metadata().is_err()); + assert_eq!( + std::fs::read(outside.path().join("wal/segment")).unwrap(), + b"foreign" + ); + } +} diff --git a/crates/tracedecay-graph-db/src/generation_runtime.rs b/crates/tracedecay-graph-db/src/generation_runtime.rs index db9acd4070..dbcfecac37 100644 --- a/crates/tracedecay-graph-db/src/generation_runtime.rs +++ b/crates/tracedecay-graph-db/src/generation_runtime.rs @@ -17,7 +17,6 @@ use crate::lease::{ }; use crate::limits::{ MAX_NATIVE_GENERATION_STAGE_LIVE_BYTES, MAX_NATIVE_GENERATION_STAGE_MUTATIONS, - MAX_VERIFIED_GENERATION_BATCH_LIVE_BYTES, MAX_VERIFIED_GENERATION_BATCH_MUTATIONS, }; use crate::projection::graph_properties_live_bytes; use crate::recovery::{ @@ -187,7 +186,6 @@ struct GenerationStagePlan<'a> { expected: &'a GraphRecoveredGenerationDigestV1, context: &'a GenerationStageContext, pages: &'a [GenerationStagePage], - adopt_legacy_partial: bool, restaging_sealed_only: bool, } @@ -630,13 +628,6 @@ impl GraphDb { let pages = generation_stage_pages(&manifest)?; context.replay_every_page = self .generation_stage_release_interrupted(&identity, expected, &context, &pages, check)?; - let adopt_legacy_partial = self.has_exact_legacy_stage_prefix( - &manifest, - &identity, - expected, - &context, - pages.first(), - )?; #[cfg(feature = "hotpath")] { let generation_bytes = pages.iter().map(GenerationStagePage::live_bytes).sum(); @@ -651,7 +642,6 @@ impl GraphDb { expected, context: &context, pages: &pages, - adopt_legacy_partial, restaging_sealed_only, }; match Arc::try_unwrap(manifest) { @@ -783,7 +773,6 @@ impl GraphDb { expected, context, pages, - adopt_legacy_partial, restaging_sealed_only, } = plan; let mut prepared_next = None; @@ -791,13 +780,7 @@ impl GraphDb { check()?; let first_page_blocked = !restaging_sealed_only && index == 0 - && self.generation_stage_first_page_blocked_without_legacy( - identity, - expected, - context, - page, - adopt_legacy_partial, - )?; + && self.generation_stage_first_page_blocked(identity, expected, context, page)?; let current = if first_page_blocked { None } else { @@ -838,7 +821,6 @@ impl GraphDb { context, index.checked_sub(1).and_then(|prior| pages.get(prior)), page, - adopt_legacy_partial && index == 0, current, check, )?; @@ -877,7 +859,6 @@ impl GraphDb { expected, context, pages, - adopt_legacy_partial, restaging_sealed_only, } = plan; let mut rows = OwnedGenerationStageRows::new(entities, relations); @@ -885,13 +866,7 @@ impl GraphDb { return rows.finish(); }; let first_page_blocked = !restaging_sealed_only - && self.generation_stage_first_page_blocked_without_legacy( - identity, - expected, - context, - first_page, - adopt_legacy_partial, - )?; + && self.generation_stage_first_page_blocked(identity, expected, context, first_page)?; let first_rows = rows.take_page(first_page)?; let mut current = if first_page_blocked || self @@ -954,7 +929,6 @@ impl GraphDb { context, index.checked_sub(1).and_then(|prior| pages.get(prior)), page, - adopt_legacy_partial && index == 0, current.take(), check, )?; @@ -986,24 +960,24 @@ impl GraphDb { construct_generation_stage_page(manifest, identity, context, page, check).map(Some) } - fn generation_stage_first_page_blocked_without_legacy( + fn generation_stage_first_page_blocked( &self, identity: &GraphGenerationManifestIdentity, expected: &GraphRecoveredGenerationDigestV1, context: &GenerationStageContext, page: &GenerationStagePage, - adopt_legacy_partial: bool, ) -> Result { let guard = self.read_guard()?; let database = guard.as_ref().ok_or(GraphDbError::Closed)?; - let Some(existing) = latest_projection( + if latest_projection( database, &context.physical_namespace, &identity.projection.projection, )? - else { + .is_none() + { return Ok(false); - }; + } // A durable receipt that binds this exact identity, recovered digest // and page says the leftover projection commit is this generation's // own: the rows behind it were released after its sealed artifact @@ -1012,14 +986,9 @@ impl GraphDb { // same conclusion inside the batch (`reuse_receipt`), so the // pre-flight decision has to agree with it or the page it refuses to // prepare is a page the batch then demands. - if Self::generation_stage_page_receipt_binds(database, identity, expected, context, page)? { - return Ok(false); - } - let exact_incomplete_legacy = adopt_legacy_partial - && existing.commit.source_generation == identity.source_generation - && existing.commit.watermark == identity.watermark - && existing.commit.generation_dependency_digest.is_none(); - Ok(!exact_incomplete_legacy) + Ok(!Self::generation_stage_page_receipt_binds( + database, identity, expected, context, page, + )?) } /// Whether a durable receipt for `page` already binds this exact manifest @@ -1083,33 +1052,6 @@ impl GraphDb { }) } - #[cfg(test)] - #[allow(clippy::too_many_arguments)] - #[hotpath::measure(label = "graph_db.generation.page_apply", impl_type = "GraphDb")] - fn apply_generation_stage_page_with_context( - &self, - manifest: &GraphGenerationManifest, - identity: &GraphGenerationManifestIdentity, - expected: &GraphRecoveredGenerationDigestV1, - context: &GenerationStageContext, - predecessor: Option<&GenerationStagePage>, - page: &GenerationStagePage, - adopt_legacy_partial: bool, - check: &dyn Fn() -> Result<(), GraphDbError>, - ) -> Result { - self.apply_prepared_generation_stage_page( - Some(manifest), - identity, - expected, - context, - predecessor, - page, - adopt_legacy_partial, - None, - check, - ) - } - #[allow(clippy::too_many_arguments)] #[hotpath::measure( label = "graph_db.generation.page_apply_prepared", @@ -1123,7 +1065,6 @@ impl GraphDb { context: &GenerationStageContext, predecessor: Option<&GenerationStagePage>, page: &GenerationStagePage, - adopt_legacy_partial: bool, prepared: Option, check: &dyn Fn() -> Result<(), GraphDbError>, ) -> Result { @@ -1185,26 +1126,19 @@ impl GraphDb { "graph generation stage predecessor rows are absent", )); } - } else if let Some(existing) = latest_projection( - database, - &context.physical_namespace, - &identity.projection.projection, - )? && !reuse_receipt + } else if !reuse_receipt + && latest_projection( + database, + &context.physical_namespace, + &identity.projection.projection, + )? + .is_some() { - // A finalized generation always carries its dependency - // digest. Only an exact unfinished legacy stage may let - // the wider first page replace its old prefix. - let exact_incomplete_legacy = adopt_legacy_partial - && existing.commit.source_generation == identity.source_generation - && existing.commit.watermark == identity.watermark - && existing.commit.generation_dependency_digest.is_none(); - if !exact_incomplete_legacy { - return Err(self.sealed_write_refusal(&context.locator).unwrap_or( - GraphDbError::conflict( - "generation_runtime.apply_generation_stage_page_with_context", - ), - )); - } + return Err(self.sealed_write_refusal(&context.locator).unwrap_or( + GraphDbError::conflict( + "generation_runtime.apply_generation_stage_page_with_context", + ), + )); } let (batch, endpoint_namespaces, digest) = match prepared { Some(prepared) => ( @@ -1246,49 +1180,6 @@ impl GraphDb { ) } - #[allow(clippy::too_many_arguments)] - fn has_exact_legacy_stage_prefix( - &self, - manifest: &GraphGenerationManifest, - identity: &GraphGenerationManifestIdentity, - expected: &GraphRecoveredGenerationDigestV1, - context: &GenerationStageContext, - native_first: Option<&GenerationStagePage>, - ) -> Result { - let legacy_first = first_generation_stage_page_with_limits( - manifest, - MAX_VERIFIED_GENERATION_BATCH_MUTATIONS, - MAX_VERIFIED_GENERATION_BATCH_LIVE_BYTES, - )?; - let Some(legacy_first) = legacy_first.as_ref() else { - return Ok(false); - }; - if native_first == Some(legacy_first) { - return Ok(false); - } - // The legacy receipt binds the exact manifest identity, recovered - // digest, page range, and live-byte count. Its presence is the durable - // proof that replacing the obsolete prefix does not adopt foreign rows. - let (legacy_key, legacy_input) = - generation_stage_page_receipt(identity, expected, legacy_first)?; - let guard = self.read_guard()?; - let database = guard.as_ref().ok_or(GraphDbError::Closed)?; - let Some(existing) = - crate::state::publication(database, &context.physical_namespace, &legacy_key)? - else { - return Ok(false); - }; - if existing.input_digest != legacy_input - || existing.commit.source_generation != identity.source_generation - || existing.commit.watermark != identity.watermark - { - return Err(GraphDbError::conflict( - "generation_runtime.has_exact_legacy_stage_prefix", - )); - } - Ok(true) - } - /// Binds the dependency metadata in one empty batch, after every page /// receipt is durable. Reads only the identity, so the staged rows are /// already released by the time this runs. @@ -2528,40 +2419,6 @@ fn generation_stage_pages_with_limits( Ok(pages) } -fn first_generation_stage_page_with_limits( - manifest: &GraphGenerationManifest, - maximum_mutations: usize, - maximum_live_bytes: usize, -) -> Result, GraphDbError> { - let mut pages = Vec::with_capacity(2); - if manifest.entities.is_empty() { - append_generation_stage_pages_with_limits( - &mut pages, - GenerationStagePageKind::Relations, - manifest - .relations - .len() - .min(maximum_mutations.saturating_add(1)), - |index| generation_relation_live_bytes(&manifest.relations[index]), - maximum_mutations, - maximum_live_bytes, - )?; - } else { - append_generation_stage_pages_with_limits( - &mut pages, - GenerationStagePageKind::Entities, - manifest - .entities - .len() - .min(maximum_mutations.saturating_add(1)), - |index| generation_entity_live_bytes(&manifest.entities[index]), - maximum_mutations, - maximum_live_bytes, - )?; - } - Ok(pages.into_iter().next()) -} - fn generation_entity_live_bytes(entity: &crate::GraphEntity) -> Result { entity .labels @@ -3884,7 +3741,6 @@ mod tests { expected: &sealed, context: &context, pages: &pages, - adopt_legacy_partial: false, restaging_sealed_only: false, }, &|| Ok(()), @@ -4024,115 +3880,6 @@ mod tests { second_owner.close().unwrap(); } - #[test] - fn wider_native_stage_adopts_an_exact_legacy_partial_receipt() { - let mut manifest = large_manifest("legacy-page-resume"); - manifest.entities.extend((5_000..9_000).map(|index| { - GraphEntity::new( - GraphEntityId::new(format!("entity:{index:05}")).unwrap(), - BTreeSet::new(), - BTreeMap::new(), - ) - .unwrap() - })); - let sealed = sealed_digest(&manifest); - let temp = TempDir::new().unwrap(); - let (owner, database) = persistent_database(&temp); - let legacy_pages = super::generation_stage_pages_with_limits( - &manifest, - MAX_VERIFIED_GENERATION_BATCH_MUTATIONS, - crate::MAX_VERIFIED_GENERATION_BATCH_LIVE_BYTES, - ) - .unwrap(); - assert_eq!(legacy_pages.len(), 3); - assert_eq!(legacy_pages[0].range, 0..4_096); - assert_eq!(legacy_pages[1].range, 4_096..8_192); - assert_eq!( - super::first_generation_stage_page_with_limits( - &manifest, - MAX_VERIFIED_GENERATION_BATCH_MUTATIONS, - crate::MAX_VERIFIED_GENERATION_BATCH_LIVE_BYTES, - ) - .unwrap(), - Some(legacy_pages[0].clone()), - "the bounded compatibility probe must reproduce the legacy first receipt" - ); - let context = super::GenerationStageContext { - locator: GenerationLocator::new( - manifest.projection.clone(), - manifest.generation.clone(), - ), - physical_namespace: manifest.identity().physical_namespace().unwrap(), - dependency_namespaces: database - .require_exact_dependencies(&manifest.identity()) - .unwrap(), - dependency_digest: manifest.dependency_closure_digest(&|| Ok(())).unwrap(), - replay_every_page: false, - }; - for (index, legacy_page) in legacy_pages.iter().take(2).enumerate() { - database - .apply_generation_stage_page_with_context( - &manifest, - &manifest.identity(), - &sealed, - &context, - index - .checked_sub(1) - .and_then(|prior| legacy_pages.get(prior)), - legacy_page, - false, - &|| Ok(()), - ) - .unwrap(); - } - - let mut divergent = manifest.clone(); - divergent.source_generation = SourceGeneration::new("source-divergent").unwrap(); - let divergent_sealed = sealed_digest(&divergent); - reset_batch_canonicalizations(); - assert!( - matches!( - database.apply_generation_unverified_with_digest_observed( - arc_manifest(&divergent), - &divergent_sealed, - &|| Ok(()) - ), - Err(GraphDbError::Conflict { .. }) - ), - "a legacy prefix may be replaced only by its exact source authority" - ); - assert_eq!( - batch_canonicalizations(), - 0, - "a divergent legacy migration must fail before writing" - ); - - reset_batch_canonicalizations(); - let outcome = database - .apply_generation_unverified_with_digest_observed( - arc_manifest(&manifest), - &sealed, - &|| Ok(()), - ) - .expect("an exact legacy partial stage must migrate to the wider page layout"); - assert!(matches!(outcome, GenerationStageOutcome::Applied(_))); - assert_eq!( - batch_canonicalizations(), - 2, - "migration must write one wide data page and one final metadata bind" - ); - let (_, recovered) = database - .reopen_and_verify_existing_generation( - &manifest.identity(), - &sealed, - manifest.row_counts(), - &|| Ok(()), - ) - .unwrap(); - assert_eq!(recovered, sealed); - owner.close().unwrap(); - } - #[test] fn near_complete_cancelled_stage_retires_in_bounded_idempotent_pages() { let manifest = large_manifest("bounded-retirement"); diff --git a/crates/tracedecay-graph-db/src/lib.rs b/crates/tracedecay-graph-db/src/lib.rs index 5d5e36f9e6..c3145aea01 100644 --- a/crates/tracedecay-graph-db/src/lib.rs +++ b/crates/tracedecay-graph-db/src/lib.rs @@ -1,6 +1,7 @@ mod adjacency_id_index; mod backup; mod bundle; +mod corrupt_store; mod epoch_cache; mod error; mod generation; @@ -22,7 +23,6 @@ mod runtime; mod schema; mod sealed_store; mod state; -mod store_quarantine; mod traversal; mod verified_marker; @@ -31,6 +31,7 @@ pub use bundle::{ MAX_SEALED_READ_BUNDLE_ARTIFACT_BYTES_V1, SEALED_READ_BUNDLE_FORMAT_V1, SealedReadBundleArtifactStateV1, SealedReadBundleArtifactV1, SealedReadBundleManifestV1, SealedReadBundleWriterV1, load_sealed_read_bundle_artifact, retire_sealed_read_bundle, + sealed_read_bundle_artifact_file_digest, sealed_read_bundle_manifest_artifact_digests, sweep_aborted_sealed_read_bundle_temporaries, }; pub use error::{ @@ -88,9 +89,7 @@ pub(crate) use publication::{ }; pub use recovery::VerifiedGraphCommit; pub use registry::{ - CODE_GRAPH_SHARD_NAMESPACE_PREFIX, LEGACY_PER_GENERATION_CODE_GRAPH_NAMESPACE_PREFIX, - code_graph_shard_namespace, is_code_graph_shard_namespace, - is_legacy_per_generation_code_graph_namespace, + CODE_GRAPH_SHARD_NAMESPACE_PREFIX, code_graph_shard_namespace, is_code_graph_shard_namespace, }; pub use registry::{ GraphDbOwnerRegistrationV1, GraphDbRegistration, GraphDbRegistry, GraphDbRegistryCapacity, diff --git a/crates/tracedecay-graph-db/src/registry.rs b/crates/tracedecay-graph-db/src/registry.rs index 2c5e3be71f..5c4acef20e 100644 --- a/crates/tracedecay-graph-db/src/registry.rs +++ b/crates/tracedecay-graph-db/src/registry.rs @@ -40,9 +40,7 @@ mod publication_support; #[path = "registry/support.rs"] mod support; pub use code_graph_namespace::{ - CODE_GRAPH_SHARD_NAMESPACE_PREFIX, LEGACY_PER_GENERATION_CODE_GRAPH_NAMESPACE_PREFIX, - code_graph_shard_namespace, is_code_graph_shard_namespace, - is_legacy_per_generation_code_graph_namespace, + CODE_GRAPH_SHARD_NAMESPACE_PREFIX, code_graph_shard_namespace, is_code_graph_shard_namespace, }; pub use publication::{GraphPublicationPreparationV1, ProvenGraphPublicationV1}; diff --git a/crates/tracedecay-graph-db/src/registry/code_graph_namespace.rs b/crates/tracedecay-graph-db/src/registry/code_graph_namespace.rs index 6038105b3c..97b3bd5b3d 100644 --- a/crates/tracedecay-graph-db/src/registry/code_graph_namespace.rs +++ b/crates/tracedecay-graph-db/src/registry/code_graph_namespace.rs @@ -1,25 +1,10 @@ -//! The canonical code-graph namespace and the legacy layout it replaced. +//! The canonical code-graph namespace. //! //! A code graph is projected into the project's graph container under a //! namespace derived from the code shard alone. Every generation of one shard //! therefore publishes into the *same* projection, so publishing generation //! N+1 supersedes N through the ordinary verified-head compare-and-swap and N //! becomes historical replay the ordinary retirement path reclaims. -//! -//! The retired layout hashed the code generation into the namespace as well, -//! which gave every generation a projection of its own. A generation was then -//! the permanent verified head of that projection: `retire_replay` always -//! answered `CurrentVerifiedHead`, nothing ever superseded anything, and the -//! shared staging container accumulated the rows of every generation ever -//! published (issue #836). -//! -//! The two layouts occupy disjoint namespace prefixes on purpose. The prefix -//! is the migration discriminator: a persisted namespace read back from a -//! store is classified by inspection as canonical or legacy, with no reliance -//! on digest-space accidents and no naming-based compatibility. Nothing -//! derives, reads, or writes a legacy namespace; the classifier exists so the -//! retirement sweep can *recognize* legacy-layout rows it reclaims and report -//! the migration instead of leaving it silent. use tracedecay_store::StoreShardIdV1; @@ -28,11 +13,6 @@ use crate::{GraphDbError, GraphNamespace}; /// Prefix of the canonical, generation-agnostic code-graph namespace. pub const CODE_GRAPH_SHARD_NAMESPACE_PREFIX: &str = "code-shard:"; -/// Prefix of the retired per-generation code-graph namespace. -/// -/// Only persisted state still carries it. It is never produced again. -pub const LEGACY_PER_GENERATION_CODE_GRAPH_NAMESPACE_PREFIX: &str = "code-scope:"; - const CODE_GRAPH_SHARD_NAMESPACE_DOMAIN: &str = "tracedecay.code-graph.shard.v2"; /// The one canonical namespace of a code shard's graph projection. @@ -67,22 +47,6 @@ pub(crate) fn is_code_graph_shard_namespace_str(namespace: &str) -> bool { namespace.starts_with(CODE_GRAPH_SHARD_NAMESPACE_PREFIX) } -/// Whether `namespace` was persisted under the retired per-generation layout. -/// -/// True only for rows written before the cutover. The retirement sweep uses it -/// to report that a reclaimed projection was legacy-layout residue rather than -/// an ordinary superseded generation. -#[must_use] -pub fn is_legacy_per_generation_code_graph_namespace(namespace: &GraphNamespace) -> bool { - is_legacy_per_generation_code_graph_namespace_str(namespace.as_str()) -} - -/// [`is_legacy_per_generation_code_graph_namespace`] over a namespace already -/// read back as a string from a persisted relational projection identity. -pub(crate) fn is_legacy_per_generation_code_graph_namespace_str(namespace: &str) -> bool { - namespace.starts_with(LEGACY_PER_GENERATION_CODE_GRAPH_NAMESPACE_PREFIX) -} - #[cfg(test)] mod tests { use super::*; @@ -104,17 +68,10 @@ mod tests { } #[test] - fn canonical_and_legacy_layouts_are_disjoint_by_prefix() { + fn canonical_namespace_carries_the_shard_prefix() { let canonical = code_graph_shard_namespace(&code_shard("worktree.primary")).unwrap(); assert!(is_code_graph_shard_namespace(&canonical)); - assert!(!is_legacy_per_generation_code_graph_namespace(&canonical)); - - let legacy = GraphNamespace::new(format!( - "{LEGACY_PER_GENERATION_CODE_GRAPH_NAMESPACE_PREFIX}{}", - "a".repeat(64) - )) - .unwrap(); - assert!(is_legacy_per_generation_code_graph_namespace(&legacy)); - assert!(!is_code_graph_shard_namespace(&legacy)); + let foreign = GraphNamespace::new(format!("code-scope:{}", "a".repeat(64))).unwrap(); + assert!(!is_code_graph_shard_namespace(&foreign)); } } diff --git a/crates/tracedecay-graph-db/src/registry/publication.rs b/crates/tracedecay-graph-db/src/registry/publication.rs index e644ea2e57..1e664199e0 100644 --- a/crates/tracedecay-graph-db/src/registry/publication.rs +++ b/crates/tracedecay-graph-db/src/registry/publication.rs @@ -15,7 +15,6 @@ use tracedecay_store::runtime::{ MAX_GRAPH_PUBLICATION_PROJECTION_PAGE_RECORDS_V1, MAX_GRAPH_REPLAY_PAGE_RECORDS_V1, }; -use super::code_graph_namespace::is_legacy_per_generation_code_graph_namespace_str; use super::path::canonical_graph_database_file; use super::publication_support::{ RegisteredGraphDbOperationV1, check_all, clear_retiring_fence, collect_closure, @@ -39,66 +38,6 @@ use crate::{ SupersededReplayRetirement, VerifiedGraphCommit, }; -/// Exact persisted identity emitted by the shipped per-generation code-graph -/// layout. This predicate gates destructive cleanup, so the broader reporting -/// classifier is deliberately insufficient here. -fn is_shipped_legacy_code_graph_projection(projection: &GraphProjectionIdentityV1) -> bool { - let Some(digest) = projection - .namespace - .as_str() - .strip_prefix(crate::LEGACY_PER_GENERATION_CODE_GRAPH_NAMESPACE_PREFIX) - else { - return false; - }; - projection.projection.as_str() == "code-graph" - && digest.len() == 64 - && digest - .bytes() - .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) -} - -#[cfg(test)] -mod legacy_cleanup_identity_tests { - use super::is_shipped_legacy_code_graph_projection; - use tracedecay_domain::{BrainId, ProjectId, UserProfileId}; - use tracedecay_store::{ - GraphNamespaceV1, GraphProjectionIdV1, GraphProjectionIdentityV1, StoreShardIdV1, - }; - - fn projection(namespace: String, projection: &str) -> GraphProjectionIdentityV1 { - GraphProjectionIdentityV1 { - shard_id: StoreShardIdV1::project( - BrainId::new("brain.legacy-cleanup").unwrap(), - UserProfileId::new("profile.legacy-cleanup").unwrap(), - ProjectId::new("project.legacy-cleanup").unwrap(), - ), - namespace: GraphNamespaceV1::new(namespace).unwrap(), - projection: GraphProjectionIdV1::new(projection).unwrap(), - } - } - - #[test] - fn destructive_legacy_cleanup_requires_the_exact_shipped_projection_identity() { - let prefix = crate::LEGACY_PER_GENERATION_CODE_GRAPH_NAMESPACE_PREFIX; - assert!(is_shipped_legacy_code_graph_projection(&projection( - format!("{prefix}{}", "1a".repeat(32)), - "code-graph", - ))); - assert!(!is_shipped_legacy_code_graph_projection(&projection( - format!("{prefix}{}", "a".repeat(63)), - "code-graph", - ))); - assert!(!is_shipped_legacy_code_graph_projection(&projection( - format!("{prefix}{}", "A".repeat(64)), - "code-graph", - ))); - assert!(!is_shipped_legacy_code_graph_projection(&projection( - format!("{prefix}{}", "b".repeat(64)), - "other-projection", - ))); - } -} - /// A publication whose durable generation proof completed but whose /// relational verified-head CAS has not yet run. /// @@ -236,93 +175,23 @@ impl GraphDbRegistry { let relational_head = authority .verified_head(projection, context) .map_err(GraphDbError::from)?; - let (key, direct_dependencies, canonical_replay_source, relational_recovered_digest) = - if let Some(head) = &relational_head { - let replay = authority - .replay(&head.key, context) - .map_err(GraphDbError::from)?; - let replay = require_active_replay_evidence( - replay, - "verified graph head has no durable active replay", - )?; - require_head_replay(head, &replay)?; - ( - replay.publication.key, - replay.publication.direct_dependency_generations, - replay.publication.canonical_replay_source, - head.recovered_digest.clone(), - ) - } else { - if !is_shipped_legacy_code_graph_projection(projection) - || authority - .pending_replay(projection, context) - .map_err(GraphDbError::from)? - .is_some() - { - return Ok(SealedStagingRelease::Retained( - SealedStagingRetentionReason::NoVerifiedLease, - )); - } - // The shipped pre-cutover layout put one code generation in one - // projection. Its head and active replay are retired atomically; - // the retained cleanup tombstone is the durable proof that the - // publication completed and that only derived native bytes remain. - // An active replay without a head is always pending in the - // production authority and can never authorize deletion. - let mut selected = None; - let mut after = None; - loop { - let request = GraphPublicationRetiredCleanupPageRequestV1::new( - projection.clone(), - after.clone(), - MAX_GRAPH_REPLAY_PAGE_RECORDS_V1, - ) - .map_err(|error| GraphDbError::invalid(error.to_string()))?; - let page = authority - .retired_cleanup_page(&request, context) - .map_err(GraphDbError::from)?; - for tombstone in page.records { - if tombstone.key.projection != *projection { - return Err(GraphDbError::Corrupt { - message: "legacy graph cleanup page escaped its projection" - .to_owned(), - }); - } - if selected.replace(tombstone).is_some() { - return Ok(SealedStagingRelease::Retained( - SealedStagingRetentionReason::NoVerifiedLease, - )); - } - } - let Some(continuation) = page.continuation else { - break; - }; - validate_replay_cursor( - projection, - after.as_ref(), - &continuation, - "legacy graph cleanup staging release", - )?; - after = Some(continuation); - } - let Some(tombstone) = selected else { - return Ok(SealedStagingRelease::Retained( - SealedStagingRetentionReason::NoVerifiedLease, - )); - }; - let source = - tombstone - .canonical_replay_source - .ok_or_else(|| GraphDbError::Corrupt { - message: "legacy graph cleanup lost its replay source".to_owned(), - })?; - ( - tombstone.key, - tombstone.direct_dependency_generations, - source, - tombstone.expected_recovered_digest, - ) - }; + let Some(head) = &relational_head else { + return Ok(SealedStagingRelease::Retained( + SealedStagingRetentionReason::NoVerifiedLease, + )); + }; + let replay = authority + .replay(&head.key, context) + .map_err(GraphDbError::from)?; + let replay = require_active_replay_evidence( + replay, + "verified graph head has no durable active replay", + )?; + require_head_replay(head, &replay)?; + let key = replay.publication.key; + let direct_dependencies = replay.publication.direct_dependency_generations; + let canonical_replay_source = replay.publication.canonical_replay_source; + let relational_recovered_digest = head.recovered_digest.clone(); if !direct_dependencies.is_empty() { return Ok(SealedStagingRelease::Retained( SealedStagingRetentionReason::DependencyBearing, @@ -338,11 +207,9 @@ impl GraphDbRegistry { )); } let locator = locator_from_key(&key)?; - // Relational publication evidence is the authority for which sealed - // artifact may stand in for the staging rows: normally the verified - // head, or the unique cleanup tombstone for a shipped legacy - // per-generation projection after its head and replay were retired. - // Release authorizes from that evidence plus the on-disk receipt or a + // Relational publication evidence (the verified head) is the + // authority for which sealed artifact may stand in for the staging + // rows. Release authorizes from that evidence plus the on-disk receipt or a // seated reader already in this process. It does not recover or prove // the sealed generation; that work belongs to activation. if database.installed_verified_generation(&locator)?.is_none() @@ -644,34 +511,15 @@ impl GraphDbRegistry { match retirement_outcome { GraphReplayRetirementOutcomeV1::Retired(_) | GraphReplayRetirementOutcomeV1::ExactReplay(_) => { - let legacy_layout = is_legacy_per_generation_code_graph_namespace_str( - replay.publication.key.projection.namespace.as_str(), - ); if selected_head.is_some() { tracing::info!( event = "graph_replay_head_retired", generation = generation.as_str(), graph_generation = %locator.generation, replay_sequence = replay.sequence.get(), - legacy_layout, "verified per-generation graph replay head retired" ); } - if legacy_layout { - // Migration evidence for issue #836: this projection was - // written under the retired per-generation namespace, so - // reclaiming it is the explicit drain of pre-cutover - // persisted state, not ordinary supersession. - tracing::info!( - event = "graph_legacy_code_graph_projection_retired", - generation = generation.as_str(), - graph_generation = %locator.generation, - namespace = replay.publication.key.projection.namespace.as_str(), - head_retired = selected_head.is_some(), - "reclaimed a code-graph projection persisted under the retired \ - per-generation namespace layout" - ); - } // Retirement is the linearization point. A failure after it // may leak derived bytes, but cannot destroy the source of an // active relational replay. A hibernated engine must not be diff --git a/crates/tracedecay-graph-db/src/registry/support.rs b/crates/tracedecay-graph-db/src/registry/support.rs index 1ba228a650..130f0f1d42 100644 --- a/crates/tracedecay-graph-db/src/registry/support.rs +++ b/crates/tracedecay-graph-db/src/registry/support.rs @@ -126,7 +126,7 @@ pub(super) fn open_registered_graph_lazy( } /// Opens the registry-owned database, running the deterministic-corruption -/// quarantine protocol when a preexisting container reports the typed +/// deletion protocol when a preexisting container reports the typed /// corruption verdict, then reopening the vacated path as a fresh store that /// the canonical replay authorities re-project into. fn open_registered_database( @@ -146,26 +146,23 @@ fn open_registered_database( Err(GraphDbError::Corrupt { message }) if persistent_store_state == PersistentGraphStoreState::Existing => { - let recovery = crate::store_quarantine::recover_deterministically_corrupt_container( + let recovery = crate::corrupt_store::recover_deterministically_corrupt_container( path, &message, &|| open(PersistentGraphStoreState::Existing), )?; match recovery { - crate::store_quarantine::CorruptStoreRecovery::Reopened(database) => { + crate::corrupt_store::CorruptStoreRecovery::Reopened(database) => { Ok((database, PersistentGraphStoreState::Existing)) } - crate::store_quarantine::CorruptStoreRecovery::Quarantined { - quarantine_directory, - } => { + crate::corrupt_store::CorruptStoreRecovery::Deleted => { let fresh_state = inspect_graph_database_file(path)?; let database = open(fresh_state)?; tracing::info!( - event = "store_rebuilt_after_quarantine", + event = "store_rebuilt_after_corruption", container = %path.display(), - quarantine = %quarantine_directory.display(), - "fresh graph store opened after corruption quarantine; canonical \ - replay authorities re-project its generations" + "fresh graph store opened after deleting a corrupt container; \ + canonical replay authorities re-project its generations" ); Ok((database, fresh_state)) } diff --git a/crates/tracedecay-graph-db/src/runtime.rs b/crates/tracedecay-graph-db/src/runtime.rs index c30858627f..47535730e7 100644 --- a/crates/tracedecay-graph-db/src/runtime.rs +++ b/crates/tracedecay-graph-db/src/runtime.rs @@ -1502,25 +1502,22 @@ impl GraphDb { let path = validated.config.path.as_deref().ok_or_else(|| { GraphDbError::unavailable("persistent graph database has no container path") })?; - match crate::store_quarantine::recover_deterministically_corrupt_container_with( + match crate::corrupt_store::recover_deterministically_corrupt_container_with( path, &message, &|| open_validated_graph(&validated, GraphEngineOpenSite::LazyFirstUse), )? { - crate::store_quarantine::CorruptStoreRecovery::Reopened(opened) => opened, - crate::store_quarantine::CorruptStoreRecovery::Quarantined { - quarantine_directory, - } => { + crate::corrupt_store::CorruptStoreRecovery::Reopened(opened) => opened, + crate::corrupt_store::CorruptStoreRecovery::Deleted => { let mut fresh = validated.clone(); fresh.preexisting_store = false; let opened = open_validated_graph(&fresh, GraphEngineOpenSite::LazyFirstUse)?; tracing::info!( - event = "store_rebuilt_after_quarantine", + event = "store_rebuilt_after_corruption", container = %path.display(), - quarantine = %quarantine_directory.display(), - "fresh graph store opened after corruption quarantine; canonical \ - replay authorities re-project its generations" + "fresh graph store opened after deleting a corrupt container; \ + canonical replay authorities re-project its generations" ); opened } diff --git a/crates/tracedecay-graph-db/src/store_quarantine.rs b/crates/tracedecay-graph-db/src/store_quarantine.rs deleted file mode 100644 index eff07461e3..0000000000 --- a/crates/tracedecay-graph-db/src/store_quarantine.rs +++ /dev/null @@ -1,567 +0,0 @@ -//! Quarantine of a deterministically corrupt registry-owned graph container. -//! -//! The registry-owned `.grafeo` container is a derived index in every -//! namespace it serves: verified code and memory projections replay from the -//! relational publication journal and canonical sealed-generation seals, and -//! session relation projections re-materialize from the relational session -//! store. Permanent container corruption (a torn WAL write, a CRC fault in a -//! serialized block) therefore never destroys canonical data, but before -//! this module it permanently disabled the mount: every open of the same -//! bytes failed with the identical typed [`GraphDbError::Corrupt`], every -//! activation retried through the same fault, and the store never healed. -//! -//! This module turns that deterministic verdict into a bounded recovery: -//! -//! 1. The corruption decision is serialized across incarnations by an -//! exclusive advisory lock on a sibling lock file. A holder elsewhere -//! means another authority is mid-decision, so this attempt reports a -//! retryable unavailable state and touches nothing. -//! 2. Under the lock, the deciding authority re-runs the identical failing -//! open itself. Only a second corruption verdict with the byte-identical -//! fault message, same GRAFEO code, same block, same CRC pair, proves -//! the fault deterministic. A successful reopen is served; a drifting -//! fault stays a terminal typed `Corrupt` for the operator, because a -//! fault that changes between attempts is hardware-shaped and a rebuild -//! onto the same medium would only re-corrupt. -//! 3. Quarantine moves the container family, WAL sidecar, verified marker, -//! spill directory, then the container itself, into one timestamped -//! sibling directory named with the canonical `.corrupt-` incident-debris -//! segment, writes a durable `store-quarantined.json` receipt carrying -//! the fault fingerprint, and emits the `store_quarantined` event. -//! Nothing is ever deleted: the operator keeps the forensic bytes and -//! retention may age the quarantine later. -//! -//! The caller then reopens the now-vacant path as a fresh store and the -//! ordinary publication and reconcile paths re-project every generation from -//! their canonical replay authorities. - -use std::fs::{File, OpenOptions}; -use std::path::{Path, PathBuf}; -use std::sync::Arc; -use std::time::{SystemTime, UNIX_EPOCH}; - -use serde::{Deserialize, Serialize}; -use tracedecay_domain::canonical_text::sha256_hex; -use tracedecay_private_fs::framed_log::{DirectorySyncPolicy, atomic_write, sync_directory}; - -use crate::{GraphDb, GraphDbError}; - -const STORE_QUARANTINE_RECEIPT_VERSION: &str = "tracedecay.graph-store-quarantine.v1"; -const STORE_QUARANTINE_RECEIPT_FILE: &str = "store-quarantined.json"; -const STORE_QUARANTINE_LOCK_SUFFIX: &str = ".quarantine-lock"; - -/// Durable journal record written into the quarantine directory. -/// -/// This is the `store_quarantined` event's durable form: it binds the exact -/// fault message and its fingerprint to the members that were moved, so a -/// doctor or an operator reading the quarantine later sees why the family -/// was adopted and what the deciding authority verified. -#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] -#[serde(deny_unknown_fields)] -struct GraphStoreQuarantineReceiptV1 { - version: String, - /// The live container path the family was quarantined from. - container: PathBuf, - quarantined_at_micros: i64, - /// The exact typed corruption message both open attempts reported. - fault: String, - /// `sha256:` fingerprint of the fault message. - fault_fingerprint: String, - /// Consecutive identical corruption verdicts the deciding authority - /// observed before adopting the store. - verification_attempts: u32, - /// File names moved into the quarantine directory, in move order. - members: Vec, -} - -/// Outcome of the corruption recovery protocol for one mount attempt. -#[derive(Debug)] -pub(crate) enum CorruptStoreRecovery> { - /// The verification reopen succeeded: the first verdict did not - /// reproduce, so the live database is served and nothing was moved. - Reopened(T), - /// The fault reproduced byte-identically and the container family was - /// moved into the returned quarantine directory. The live path is vacant - /// and the caller reopens it as a fresh store. - Quarantined { quarantine_directory: PathBuf }, -} - -/// Runs the deterministic-corruption recovery protocol for `container` after -/// a mount-time open failed with the typed corruption verdict `first_fault`. -/// -/// `verification_open` must re-run the identical open the first verdict came -/// from; the protocol never trusts the first failure alone. -pub(crate) fn recover_deterministically_corrupt_container( - container: &Path, - first_fault: &str, - verification_open: &dyn Fn() -> Result, GraphDbError>, -) -> Result { - recover_deterministically_corrupt_container_with(container, first_fault, verification_open) -} - -pub(crate) fn recover_deterministically_corrupt_container_with( - container: &Path, - first_fault: &str, - verification_open: &dyn Fn() -> Result, -) -> Result, GraphDbError> { - let _decision_lock = acquire_quarantine_decision_lock(container)?; - - // Re-verify under the decision lock: quarantine only adopts a store - // whose corruption this exact authority reproduced. - let second_fault = match verification_open() { - Ok(database) => return Ok(CorruptStoreRecovery::Reopened(database)), - Err(GraphDbError::Corrupt { message }) => message, - Err(other) => return Err(other), - }; - if second_fault != first_fault { - return Err(GraphDbError::Corrupt { - message: format!( - "graph container corruption is not deterministic; refusing quarantine: \ - first fault `{first_fault}`, second fault `{second_fault}`" - ), - }); - } - - let quarantine_directory = quarantine_container_family(container, first_fault)?; - Ok(CorruptStoreRecovery::Quarantined { - quarantine_directory, - }) -} - -/// Holds the exclusive cross-incarnation corruption-decision lock while the -/// verdict is re-proven and the family moved. The lock file persists after -/// release: unlinking a held advisory lock would let a racer lock a fresh -/// inode while this holder still believes it owns the decision. -struct QuarantineDecisionLock { - file: File, -} - -impl Drop for QuarantineDecisionLock { - fn drop(&mut self) { - let _ = self.file.unlock(); - } -} - -fn quarantine_decision_lock_path(container: &Path) -> Result { - let file_name = container_file_name(container)?; - Ok(container.with_file_name(format!("{file_name}{STORE_QUARANTINE_LOCK_SUFFIX}"))) -} - -fn acquire_quarantine_decision_lock( - container: &Path, -) -> Result { - let path = quarantine_decision_lock_path(container)?; - let file = OpenOptions::new() - .create(true) - .read(true) - .write(true) - .truncate(false) - .open(&path) - .map_err(|error| { - GraphDbError::unavailable(format!( - "graph store quarantine decision lock is unavailable at {}: {error}", - path.display() - )) - })?; - match file.try_lock().map_err(std::io::Error::from) { - Ok(()) => Ok(QuarantineDecisionLock { file }), - // Windows LockFileEx reports ERROR_LOCK_VIOLATION (33) instead of - // WouldBlock. AccessDenied and sharing violations stay generic - // unavailable, not "another authority holds". - Err(error) if tracedecay_private_fs::is_lock_contended(&error) => { - Err(GraphDbError::unavailable(format!( - "another authority holds the graph store corruption quarantine decision for {}; \ - leaving the store untouched", - container.display() - ))) - } - Err(error) => Err(GraphDbError::unavailable(format!( - "graph store quarantine decision lock failed for {}: {error}", - container.display() - ))), - } -} - -/// Moves the container family into a fresh timestamped quarantine directory -/// and journals the receipt. The container moves last: it is the fault -/// authority, so an interruption mid-move leaves the corrupt container in -/// place for the next deciding authority rather than a vacant path beside -/// stranded sidecars. -fn quarantine_container_family(container: &Path, fault: &str) -> Result { - let container_name = container_file_name(container)?.to_owned(); - match container.symlink_metadata() { - Ok(metadata) if metadata.is_file() => {} - Ok(_) => { - return Err(GraphDbError::unavailable(format!( - "graph container at {} is no longer a regular file; refusing quarantine", - container.display() - ))); - } - Err(error) => { - return Err(GraphDbError::unavailable(format!( - "graph container at {} disappeared during the quarantine decision: {error}", - container.display() - ))); - } - } - - let quarantined_at_micros = current_wall_micros()?; - let quarantine_directory = - container.with_file_name(format!("{container_name}.corrupt-{quarantined_at_micros}")); - std::fs::create_dir(&quarantine_directory).map_err(|error| { - GraphDbError::unavailable(format!( - "graph store quarantine directory {} could not be created: {error}", - quarantine_directory.display() - )) - })?; - - let mut members = Vec::new(); - let mut sidecars = vec![wal_sidecar_path(container)]; - sidecars.push(container.with_extension("verified")); - sidecars.push(container.with_extension("spill")); - for sidecar in sidecars { - move_family_member(&sidecar, &quarantine_directory, &mut members)?; - } - move_family_member(container, &quarantine_directory, &mut members)?; - - let fault_fingerprint = fault_fingerprint(fault); - let receipt = GraphStoreQuarantineReceiptV1 { - version: STORE_QUARANTINE_RECEIPT_VERSION.to_owned(), - container: container.to_path_buf(), - quarantined_at_micros, - fault: fault.to_owned(), - fault_fingerprint: fault_fingerprint.clone(), - verification_attempts: 2, - members, - }; - let payload = serde_json::to_vec_pretty(&receipt).map_err(|error| { - quarantine_durability_failure(&quarantine_directory, "receipt encoding failed", &error) - })?; - atomic_write( - &quarantine_directory.join(STORE_QUARANTINE_RECEIPT_FILE), - "graph store quarantine receipt", - &payload, - DirectorySyncPolicy::Strict, - ) - .map_err(|error| { - quarantine_durability_failure(&quarantine_directory, "receipt write failed", &error) - })?; - if let Some(parent) = container.parent() { - sync_directory(parent, DirectorySyncPolicy::Strict).map_err(|error| { - quarantine_durability_failure( - &quarantine_directory, - "store directory sync failed", - &error, - ) - })?; - } - - tracing::warn!( - event = "store_quarantined", - container = %container.display(), - quarantine = %quarantine_directory.display(), - fault_fingerprint = %fault_fingerprint, - fault = %fault, - "deterministically corrupt graph container quarantined for forensics; \ - a fresh store rebuilds from the canonical replay authorities" - ); - Ok(quarantine_directory) -} - -fn move_family_member( - source: &Path, - quarantine_directory: &Path, - members: &mut Vec, -) -> Result<(), GraphDbError> { - match source.symlink_metadata() { - Ok(_) => {} - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), - Err(error) => { - return Err(member_move_failure( - source, - quarantine_directory, - members, - &format!("inspection failed: {error}"), - )); - } - } - let Some(file_name) = source.file_name() else { - return Err(member_move_failure( - source, - quarantine_directory, - members, - "member has no file name", - )); - }; - std::fs::rename(source, quarantine_directory.join(file_name)).map_err(|error| { - member_move_failure( - source, - quarantine_directory, - members, - &format!("rename failed: {error}"), - ) - })?; - members.push(file_name.to_string_lossy().into_owned()); - Ok(()) -} - -/// A move failure before anything moved is a clean retryable abort; after the -/// first member moved the family is split across two directories and the -/// durable outcome can no longer be described as either state. -fn member_move_failure( - source: &Path, - quarantine_directory: &Path, - members: &[String], - detail: &str, -) -> GraphDbError { - let message = format!( - "graph store quarantine could not move {} into {}: {detail}", - source.display(), - quarantine_directory.display() - ); - if members.is_empty() { - GraphDbError::unavailable(message) - } else { - GraphDbError::DurabilityUncertain { - message: format!( - "{message}; members already quarantined: {}", - members.join(", ") - ), - } - } -} - -fn quarantine_durability_failure( - quarantine_directory: &Path, - context: &str, - error: &dyn std::fmt::Display, -) -> GraphDbError { - GraphDbError::DurabilityUncertain { - message: format!( - "graph store family moved into {} but the quarantine journal is incomplete: \ - {context}: {error}", - quarantine_directory.display() - ), - } -} - -fn fault_fingerprint(fault: &str) -> String { - format!("sha256:{}", sha256_hex(fault.as_bytes())) -} - -/// `graph.grafeo` -> `graph.grafeo.wal`, matching Grafeo's sidecar layout. -fn wal_sidecar_path(container: &Path) -> PathBuf { - let mut sidecar = container.as_os_str().to_owned(); - sidecar.push(".wal"); - PathBuf::from(sidecar) -} - -fn container_file_name(container: &Path) -> Result<&str, GraphDbError> { - container - .file_name() - .and_then(|name| name.to_str()) - .ok_or_else(|| { - GraphDbError::invalid(format!( - "graph container path {} has no UTF-8 file name", - container.display() - )) - }) -} - -fn current_wall_micros() -> Result { - let elapsed = SystemTime::now() - .duration_since(UNIX_EPOCH) - .map_err(|error| { - GraphDbError::unavailable(format!( - "graph store quarantine cannot read the system clock: {error}" - )) - })?; - i64::try_from(elapsed.as_micros()).map_err(|_| { - GraphDbError::unavailable("graph store quarantine timestamp exceeds the journal range") - }) -} - -#[cfg(test)] -mod tests { - use std::sync::atomic::{AtomicUsize, Ordering}; - - use super::*; - - fn corrupt(message: &str) -> GraphDbError { - GraphDbError::Corrupt { - message: message.to_owned(), - } - } - - fn seeded_family(root: &Path) -> PathBuf { - let container = root.join("graph.grafeo"); - std::fs::write(&container, b"torn container bytes").unwrap(); - std::fs::create_dir(wal_sidecar_path(&container)).unwrap(); - std::fs::write( - wal_sidecar_path(&container).join("wal_00000001.log"), - b"wal", - ) - .unwrap(); - std::fs::write(container.with_extension("verified"), b"marker").unwrap(); - container - } - - #[test] - fn identical_second_verdict_quarantines_the_whole_family_with_a_receipt() { - let temp = tempfile::tempdir().unwrap(); - let container = seeded_family(temp.path()); - let fault = "GRAFEO-X002: Serialization error: block 18 CRC mismatch: \ - expected 7d877cc5, got 5a475db3"; - - let outcome = - recover_deterministically_corrupt_container(&container, fault, &|| Err(corrupt(fault))) - .unwrap(); - let CorruptStoreRecovery::Quarantined { - quarantine_directory, - } = outcome - else { - panic!("identical deterministic verdicts must quarantine"); - }; - - assert!( - !container.exists(), - "the live container path must be vacant" - ); - assert!(!wal_sidecar_path(&container).exists()); - assert!(!container.with_extension("verified").exists()); - assert_eq!( - std::fs::read(quarantine_directory.join("graph.grafeo")).unwrap(), - b"torn container bytes", - "forensic bytes must move, never be deleted or rewritten" - ); - assert_eq!( - std::fs::read( - quarantine_directory - .join("graph.grafeo.wal") - .join("wal_00000001.log") - ) - .unwrap(), - b"wal" - ); - assert!(quarantine_directory.join("graph.verified").is_file()); - - let receipt: GraphStoreQuarantineReceiptV1 = serde_json::from_slice( - &std::fs::read(quarantine_directory.join(STORE_QUARANTINE_RECEIPT_FILE)).unwrap(), - ) - .unwrap(); - assert_eq!(receipt.version, STORE_QUARANTINE_RECEIPT_VERSION); - assert_eq!(receipt.container, container); - assert_eq!(receipt.fault, fault); - assert_eq!(receipt.fault_fingerprint, fault_fingerprint(fault)); - assert_eq!(receipt.verification_attempts, 2); - assert_eq!( - receipt.members, - vec![ - "graph.grafeo.wal".to_owned(), - "graph.verified".to_owned(), - "graph.grafeo".to_owned(), - ] - ); - let directory_name = quarantine_directory - .file_name() - .unwrap() - .to_str() - .unwrap() - .to_owned(); - assert!( - directory_name.starts_with("graph.grafeo.corrupt-"), - "quarantine directory must carry the canonical corrupt-incident segment: \ - {directory_name}" - ); - } - - #[test] - fn drifting_fault_refuses_quarantine_and_stays_typed_corrupt() { - let temp = tempfile::tempdir().unwrap(); - let container = seeded_family(temp.path()); - - let error = recover_deterministically_corrupt_container( - &container, - "block 18 CRC mismatch", - &|| Err(corrupt("block 7 CRC mismatch")), - ) - .unwrap_err(); - - assert!( - matches!(&error, GraphDbError::Corrupt { message } - if message.contains("not deterministic") - && message.contains("block 18 CRC mismatch") - && message.contains("block 7 CRC mismatch")), - "a drifting fault is terminal and names both verdicts, got {error:?}" - ); - assert!(container.exists(), "a drifting fault must not move bytes"); - } - - #[test] - fn non_corrupt_verification_failure_propagates_untouched() { - let temp = tempfile::tempdir().unwrap(); - let container = seeded_family(temp.path()); - - let error = recover_deterministically_corrupt_container(&container, "fault", &|| { - Err(GraphDbError::Cancelled) - }) - .unwrap_err(); - - assert_eq!(error, GraphDbError::Cancelled); - assert!(container.exists()); - } - - #[test] - fn held_decision_lock_reports_retryable_unavailable_without_verifying() { - let temp = tempfile::tempdir().unwrap(); - let container = seeded_family(temp.path()); - let foreign_holder = OpenOptions::new() - .create(true) - .read(true) - .write(true) - .truncate(false) - .open(quarantine_decision_lock_path(&container).unwrap()) - .unwrap(); - foreign_holder - .try_lock() - .map_err(std::io::Error::from) - .unwrap(); - - let verification_attempts = AtomicUsize::new(0); - let error = recover_deterministically_corrupt_container(&container, "fault", &|| { - verification_attempts.fetch_add(1, Ordering::SeqCst); - Err(corrupt("fault")) - }) - .unwrap_err(); - - assert!( - matches!(&error, GraphDbError::Unavailable { message } - if message.contains("another authority holds")), - "a held decision lock is a retryable typed state, got {error:?}" - ); - assert_eq!( - verification_attempts.load(Ordering::SeqCst), - 0, - "a non-holder must not re-open the store it does not own" - ); - assert!(container.exists(), "a non-holder must not move bytes"); - foreign_holder.unlock().unwrap(); - } - - #[test] - fn vanished_container_under_the_lock_is_a_retryable_abort() { - let temp = tempfile::tempdir().unwrap(); - let container = temp.path().join("graph.grafeo"); - - let error = recover_deterministically_corrupt_container(&container, "fault", &|| { - Err(corrupt("fault")) - }) - .unwrap_err(); - - assert!( - matches!(&error, GraphDbError::Unavailable { message } - if message.contains("disappeared during the quarantine decision")), - "an already-recovered path must abort cleanly, got {error:?}" - ); - } -} diff --git a/crates/tracedecay-graph-db/tests/graph_db_suite/durability_crash_contract.rs b/crates/tracedecay-graph-db/tests/graph_db_suite/durability_crash_contract.rs index f9be4bc8ab..92d8d4f85a 100644 --- a/crates/tracedecay-graph-db/tests/graph_db_suite/durability_crash_contract.rs +++ b/crates/tracedecay-graph-db/tests/graph_db_suite/durability_crash_contract.rs @@ -533,39 +533,29 @@ fn write_non_final_shape(path: &std::path::Path) { raw.close().unwrap(); } -/// Locates the timestamped quarantine directory the corrupt-mount recovery -/// created beside the container, or `None` before any quarantine ran. -fn quarantine_directory(root: &std::path::Path) -> Option { - let mut directories: Vec<_> = std::fs::read_dir(root) +/// Corrupt-mount recovery deletes the container family outright; no +/// `.corrupt-` copy may ever appear beside the store. +fn assert_no_corrupt_copy(root: &std::path::Path) { + let copies: Vec<_> = std::fs::read_dir(root) .unwrap() - .map(|entry| entry.unwrap()) - .filter(|entry| { - entry.file_type().unwrap().is_dir() - && entry - .file_name() - .to_str() - .is_some_and(|name| name.starts_with("graph.grafeo.corrupt-")) - }) - .map(|entry| entry.path()) + .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) + .filter(|name| name.contains(".corrupt-")) .collect(); - directories.sort(); - directories.pop() -} - -fn quarantine_receipt(quarantine: &std::path::Path) -> serde_json::Value { - serde_json::from_slice(&std::fs::read(quarantine.join("store-quarantined.json")).unwrap()) - .unwrap() + assert!( + copies.is_empty(), + "corrupt store copies were kept: {copies:?}" + ); } /// GitHub issue #763: a deterministic corruption verdict on the durable /// container was retried identically forever, every mount refaulted, every /// activation refused, and only manual surgery (move the store and WAL aside, /// restart) recovered the project. This pins the automatic form of exactly -/// that recovery: the second identical verdict quarantines the container for -/// forensics, the mount reopens fresh, and the relational replay journal -/// re-projects the verified generation without ever advancing the head. +/// that recovery: the second identical verdict deletes the container, the +/// mount reopens fresh, and the relational replay journal re-projects the +/// verified generation without ever advancing the head. #[test] -fn torn_durable_store_is_quarantined_and_rebuilt_from_the_replay_journal() { +fn torn_durable_store_is_deleted_and_rebuilt_from_the_replay_journal() { let temp = TempDir::new().unwrap(); let registered = RegisteredGraph::new_mounted(temp.path()).unwrap(); let (control, probe) = control_and_probe(); @@ -607,7 +597,7 @@ fn torn_durable_store_is_quarantined_and_rebuilt_from_the_replay_journal() { // A foreign or corrupt WAL sidecar left beside a checkpointed store is not // durable evidence: the reopen serves the last verified generation from the - // single file, the relational head is untouched, and nothing quarantines. + // single file, the relational head is untouched, and nothing is deleted. std::fs::create_dir_all(&sidecar).unwrap(); std::fs::write(sidecar.join("000001.wal"), vec![0xAB_u8; 4096]).unwrap(); registered.mount().unwrap(); @@ -627,7 +617,8 @@ fn torn_durable_store_is_quarantined_and_rebuilt_from_the_replay_journal() { assert_eq!(marker_of(&after_foreign_sidecar, &identity), "g1"); drop(after_foreign_sidecar); assert_eq!(authority.head(&projection_key), Some(&verified_head)); - assert!(quarantine_directory(temp.path()).is_none()); + assert_no_corrupt_copy(temp.path()); + assert!(sidecar.join("000001.wal").is_file()); assert!(registered.close().unwrap()); // A torn write in the single durable file is the real crash surface. @@ -639,33 +630,15 @@ fn torn_durable_store_is_quarantined_and_rebuilt_from_the_replay_journal() { let torn = bytes[..bytes.len() / 3].to_vec(); std::fs::write(&path, &torn).unwrap(); - // The mount re-proves the fault itself and quarantines instead of - // faulting: retry #2 with the identical verdict is the terminal state + // The mount re-proves the fault itself and deletes the container instead + // of faulting: retry #2 with the identical verdict is the terminal state // for these bytes, never retry #25. registered.mount().unwrap(); - let quarantine = quarantine_directory(temp.path()) - .expect("a deterministically corrupt container must be quarantined"); - assert_eq!( - std::fs::read(quarantine.join("graph.grafeo")).unwrap(), - torn, - "the forensic bytes must move into quarantine unmodified" - ); - let receipt = quarantine_receipt(&quarantine); - assert_eq!( - receipt["version"].as_str().unwrap(), - "tracedecay.graph-store-quarantine.v1" - ); - assert_eq!(receipt["verification_attempts"].as_u64().unwrap(), 2); - assert!( - receipt["fault"].as_str().unwrap().contains("corrupt") - || !receipt["fault"].as_str().unwrap().is_empty(), - "the receipt journals the exact fault" - ); - assert!( - receipt["fault_fingerprint"] - .as_str() - .unwrap() - .starts_with("sha256:") + assert_no_corrupt_copy(temp.path()); + assert_ne!( + std::fs::read(&path).ok().as_deref(), + Some(torn.as_slice()), + "the torn container must be gone, not served or kept" ); // The fresh store refuses the recovered head with a typed mismatch until @@ -732,11 +705,11 @@ fn torn_durable_store_is_quarantined_and_rebuilt_from_the_replay_journal() { /// the durable WAL phase and exits without closing the store. The abandoned /// image is a current-version container plus a live WAL sidecar. After the /// parent copies that image, this test corrupts a serialized block so the -/// CRC fault is deterministic; quarantine must adopt the container *and* -/// the WAL sidecar so the forensic pair stays together, and the fresh store -/// must rebuild from the replay journal. +/// CRC fault is deterministic; recovery must delete the container *and* its +/// WAL sidecar so no stale journal replays into the fresh store, and the +/// fresh store must rebuild from the replay journal. #[test] -fn crc_faulted_store_is_quarantined_with_its_wal_sidecar_and_rebuilt() { +fn crc_faulted_store_is_deleted_with_its_wal_sidecar_and_rebuilt() { if let Some(root) = crash_child_root() { let registered = write_published_g1_leaving_wal(&root, "crash", "crc"); mark_durable_phase(&root); @@ -795,38 +768,21 @@ fn crc_faulted_store_is_quarantined_with_its_wal_sidecar_and_rebuilt() { let crashed = RegisteredGraph::new(crash.path()).unwrap(); crashed.mount().unwrap(); - let quarantine = - quarantine_directory(crash.path()).expect("a CRC-faulted container must be quarantined"); - assert_eq!( - std::fs::read(quarantine.join("graph.grafeo")).unwrap(), - corrupted, - "the corrupted container bytes are forensic evidence and move unmodified" + assert_no_corrupt_copy(crash.path()); + assert_ne!( + std::fs::read(&crashed_container).ok().as_deref(), + Some(corrupted.as_slice()), + "the corrupted container must be deleted" ); - let quarantined_sidecar = quarantine.join("graph.grafeo.wal"); + // The fresh WalSync store legitimately creates its own new sidecar at the + // live path, and its format-header segment is byte-identical to the old + // one; the old journal as a whole must be gone. assert!( - quarantined_sidecar.is_dir(), - "the WAL sidecar must move with its container" + wal_segments.iter().any(|(segment, old_bytes)| { + std::fs::read(crashed_sidecar.join(segment)).ok().as_ref() != Some(old_bytes) + }), + "the old WAL journal must be deleted with its container" ); - // The quarantined segments are the exact pre-mount forensic bytes. The - // fresh WalSync store legitimately creates its own new sidecar at the - // live path; the old journal is provably not beside it because every - // old segment now lives in quarantine with unmodified content. - for (segment, expected_bytes) in &wal_segments { - assert_eq!( - &std::fs::read(quarantined_sidecar.join(segment)).unwrap(), - expected_bytes, - "WAL segment {segment} must be retained in quarantine unmodified" - ); - } - let receipt = quarantine_receipt(&quarantine); - let members: Vec<&str> = receipt["members"] - .as_array() - .unwrap() - .iter() - .map(|member| member.as_str().unwrap()) - .collect(); - assert!(members.contains(&"graph.grafeo")); - assert!(members.contains(&"graph.grafeo.wal")); // The journal rebuild serves the exact verified generation again. let rebuilt = crashed @@ -849,13 +805,13 @@ fn crc_faulted_store_is_quarantined_with_its_wal_sidecar_and_rebuilt() { assert_eq!(authority.head(&projection_key), Some(&verified_head)); } -/// The compare-and-swap discipline for the quarantine decision itself: an +/// The compare-and-swap discipline for the corruption decision itself: an /// authority that does not hold the decision lock must neither re-verify nor /// sweep the store, another incarnation may be mid-recovery. The refusal is /// a retryable typed unavailable, not a retained terminal fault, so the next /// mount attempt (after the holder releases) completes the recovery. #[test] -fn held_quarantine_decision_defers_the_mount_and_the_next_attempt_recovers() { +fn held_corruption_decision_defers_the_mount_and_the_next_attempt_recovers() { let temp = TempDir::new().unwrap(); let registered = RegisteredGraph::new_mounted(temp.path()).unwrap(); let (control, probe) = control_and_probe(); @@ -891,8 +847,8 @@ fn held_quarantine_decision_defers_the_mount_and_the_next_attempt_recovers() { let torn = bytes[..bytes.len() / 3].to_vec(); std::fs::write(&path, &torn).unwrap(); - // A foreign incarnation holds the quarantine decision. - let lock_path = temp.path().join("graph.grafeo.quarantine-lock"); + // A foreign incarnation holds the corruption decision. + let lock_path = temp.path().join("graph.grafeo.corruption-lock"); let foreign_holder = std::fs::OpenOptions::new() .create(true) .read(true) @@ -914,19 +870,18 @@ fn held_quarantine_decision_defers_the_mount_and_the_next_attempt_recovers() { assert_eq!( std::fs::read(&path).unwrap(), torn, - "a non-holder must not move or modify the store" + "a non-holder must not delete or modify the store" ); - assert!(quarantine_directory(temp.path()).is_none()); // Once the holder releases, the same mount request completes the - // quarantine and rebuild instead of remaining faulted. + // deletion and rebuild instead of remaining faulted. foreign_holder.unlock().unwrap(); registered.mount().unwrap(); - let quarantine = quarantine_directory(temp.path()) - .expect("the released decision lock lets the next mount quarantine"); - assert_eq!( - std::fs::read(quarantine.join("graph.grafeo")).unwrap(), - torn + assert_no_corrupt_copy(temp.path()); + assert_ne!( + std::fs::read(&path).ok().as_deref(), + Some(torn.as_slice()), + "the released decision lock lets the next mount delete the torn store" ); } diff --git a/crates/tracedecay-graph-db/tests/graph_db_suite/registry_contract.rs b/crates/tracedecay-graph-db/tests/graph_db_suite/registry_contract.rs index 583b5687de..19cb22def3 100644 --- a/crates/tracedecay-graph-db/tests/graph_db_suite/registry_contract.rs +++ b/crates/tracedecay-graph-db/tests/graph_db_suite/registry_contract.rs @@ -1348,12 +1348,11 @@ fn reset_required_fault_is_retained_and_cannot_reopen() { /// A preexisting zero-byte container is the corruption class a crash or a /// full disk leaves behind. It is still never a silent reopen: the mount -/// re-proves the deterministic verdict, quarantines the empty file with its -/// receipt for forensics, and serves a fresh store that the canonical replay -/// authorities re-project, instead of the pre-#763 terminal fault retried -/// on every activation forever. +/// re-proves the deterministic verdict, deletes the empty file, and serves a +/// fresh store that the canonical replay authorities re-project, instead of +/// the pre-#763 terminal fault retried on every activation forever. #[test] -fn preexisting_empty_graph_file_is_quarantined_and_remounted_fresh() { +fn preexisting_empty_graph_file_is_deleted_and_remounted_fresh() { let temp = TempDir::new().unwrap(); std::fs::File::create(graph_path(temp.path())).unwrap(); @@ -1363,28 +1362,14 @@ fn preexisting_empty_graph_file_is_quarantined_and_remounted_fresh() { let database = mount_and_resolve(®istry, request).unwrap(); drop(database); - let quarantine = std::fs::read_dir(temp.path()) + let copies: Vec<_> = std::fs::read_dir(temp.path()) .unwrap() - .map(|entry| entry.unwrap()) - .find(|entry| { - entry.file_type().unwrap().is_dir() - && entry - .file_name() - .to_str() - .is_some_and(|name| name.starts_with("graph.grafeo.corrupt-")) - }) - .expect("the empty container must be quarantined, not silently replaced") - .path(); - assert_eq!( - std::fs::metadata(quarantine.join("graph.grafeo")) - .unwrap() - .len(), - 0, - "the forensic zero-byte container is retained" - ); - assert!(quarantine.join("store-quarantined.json").is_file()); + .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) + .filter(|name| name.contains(".corrupt-")) + .collect(); + assert!(copies.is_empty(), "no corrupt copy may be kept: {copies:?}"); assert!( - graph_path(temp.path()).is_file(), + std::fs::metadata(graph_path(temp.path())).unwrap().len() > 0, "a fresh store now serves at the canonical path" ); } diff --git a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/code_graph_layout.rs b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/code_graph_layout.rs index 03026d26fb..f672560b27 100644 --- a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/code_graph_layout.rs +++ b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/code_graph_layout.rs @@ -2,102 +2,19 @@ //! //! The canonical code-graph namespace is derived from the code shard alone, so //! every generation of one scope publishes into a single projection. These -//! tests pin the two consequences that make the layout worth having: -//! -//! * publishing generation N+1 supersedes N as an ordinary verified-head -//! replacement, and N is then reclaimed through the ordinary -//! `retire_replay` path, never through the head-retirement escape hatch; -//! * a store persisted under the retired per-generation layout still opens, -//! and its immortal per-generation head is drained through the existing -//! superseded-head retirement path without disturbing the canonical -//! projection the code index republished into. +//! tests pin the consequence that makes the layout worth having: publishing +//! generation N+1 supersedes N as an ordinary verified-head replacement, and +//! N is then reclaimed through the ordinary `retire_replay` path, never +//! through the head-retirement escape hatch. -use rusqlite::Savepoint; use tracedecay_graph_db::{ - LEGACY_PER_GENERATION_CODE_GRAPH_NAMESPACE_PREFIX, SupersededReplayRetirement, - VerifiedGraphCommit, code_graph_shard_namespace, is_code_graph_shard_namespace, - is_legacy_per_generation_code_graph_namespace, -}; -use tracedecay_rusqlite_runtime::{ - ExistingWriterLocator, PersistentWriter, StorageOperationExecutor, - exact_sql::ExactSqlHandle, - reader::{ExistingReaderLocator, ReaderPool, ReaderQueryExecutor}, - repository::{GRAPH_PUBLICATION_SCHEMA_V1, GraphPublicationExactSqlStorage}, -}; -use tracedecay_store::{ - AdmissionConfigV1, CodeShardScopeV1, RepositoryWritePayloadV1, RuntimeReadOutcomeV1, - RuntimeReadRequestV1, StorageRuntimeErrorV1, StoreShardScopeV1, VerifiedStoreLocatorV1, - canonical_store_locator_digest, + SupersededReplayRetirement, VerifiedGraphCommit, code_graph_shard_namespace, + is_code_graph_shard_namespace, }; +use tracedecay_store::{CodeShardScopeV1, StoreShardScopeV1}; use super::*; -struct NoPublicationWrites; - -impl StorageOperationExecutor for NoPublicationWrites { - fn execute( - &mut self, - _savepoint: &Savepoint<'_>, - _payload: &RepositoryWritePayloadV1, - ) -> rusqlite::Result<()> { - Ok(()) - } -} - -#[derive(Clone)] -struct NoPublicationReads; - -impl ReaderQueryExecutor for NoPublicationReads { - fn execute_read( - &mut self, - _snapshot: &rusqlite::Transaction<'_>, - _request: &RuntimeReadRequestV1, - ) -> Result { - unreachable!("exact SQL graph publication queries bypass product reads") - } -} - -struct ExactPublicationAuthority { - _writer: PersistentWriter, - _readers: ReaderPool, - storage: GraphPublicationExactSqlStorage, -} - -impl ExactPublicationAuthority { - fn new(root: &std::path::Path, binding: &tracedecay_store::StoreRuntimeBindingV1) -> Self { - let path = root.join("graph-publication-authority.sqlite3"); - drop(rusqlite::Connection::open(&path).unwrap()); - let path = path.canonicalize().unwrap(); - let locator = VerifiedStoreLocatorV1::new( - binding.shard_id.clone(), - binding.incarnation, - canonical_store_locator_digest(&path).unwrap(), - ); - let writer = PersistentWriter::start( - ExistingWriterLocator::new(binding.clone(), locator.clone(), path.clone()).unwrap(), - AdmissionConfigV1::default(), - NoPublicationWrites, - ) - .unwrap(); - let readers = ReaderPool::start( - ExistingReaderLocator::new(binding.clone(), locator, path).unwrap(), - AdmissionConfigV1::default().readers, - NoPublicationReads, - ) - .unwrap(); - let handle = ExactSqlHandle::attach(&writer, &readers).unwrap(); - handle - .execute_batch(GRAPH_PUBLICATION_SCHEMA_V1.to_owned()) - .unwrap(); - let storage = GraphPublicationExactSqlStorage::from_authorized_handle(handle).unwrap(); - Self { - _writer: writer, - _readers: readers, - storage, - } - } -} - fn code_shard(worktree: &str) -> StoreShardIdV1 { StoreShardIdV1::new( tracedecay_domain::BrainId::new("brain.code-graph-layout").unwrap(), @@ -121,19 +38,6 @@ fn canonical_projection(worktree: &str) -> GraphProjectionIdentity { ) } -/// A projection exactly as a pre-cutover store persisted it: the code -/// generation hashed into the namespace, so the generation owns the projection. -fn legacy_per_generation_projection(digest_byte: char) -> GraphProjectionIdentity { - GraphProjectionIdentity::new( - GraphNamespace::new(format!( - "{LEGACY_PER_GENERATION_CODE_GRAPH_NAMESPACE_PREFIX}{}", - digest_byte.to_string().repeat(64) - )) - .unwrap(), - GraphProjectionId::new("code-graph").unwrap(), - ) -} - fn sealed_source( generation: &CodeGenerationId, digest: &SealedGraphStateDigest, @@ -188,21 +92,13 @@ fn fresh_context<'a>( GraphPublicationOperationContextV1::new(control, probe).unwrap() } -/// The canonical namespace is generation-agnostic and never collides with the -/// retired per-generation layout it replaced. +/// The canonical namespace is generation-agnostic and distinct per shard. #[test] -fn canonical_code_graph_namespace_is_per_shard_and_disjoint_from_the_legacy_layout() { +fn canonical_code_graph_namespace_is_per_shard() { let primary = canonical_projection("worktree.primary"); let linked = canonical_projection("worktree.linked"); - assert_eq!(primary, canonical_projection("worktree.primary")); assert_ne!(primary, linked); assert!(is_code_graph_shard_namespace(&primary.namespace)); - assert!(!is_legacy_per_generation_code_graph_namespace( - &primary.namespace - )); - assert!(is_legacy_per_generation_code_graph_namespace( - &legacy_per_generation_projection('a').namespace - )); } /// Publishing a second generation of one code shard supersedes the first head, @@ -337,292 +233,6 @@ fn second_generation_supersedes_the_head_and_the_first_retires_without_head_reti ); } -/// A store persisted under the retired per-generation layout still opens, and -/// its immortal per-generation head is drained through the existing -/// superseded-head retirement path without touching the canonical projection -/// the code index republished into after the cutover. -#[test] -fn a_store_persisted_under_the_legacy_layout_opens_and_drains_its_per_generation_head() { - let temp = TempDir::new().unwrap(); - let registered = RegisteredGraph::new_mounted(temp.path()).unwrap(); - let mut authority = RelationalAuthority::default(); - let legacy_identity = legacy_per_generation_projection('c'); - let canonical_identity = canonical_projection("worktree.primary"); - let sealed_digest = - SealedGraphStateDigest::try_from(format!("sha256:{}", "6".repeat(64))).unwrap(); - let legacy_generation = CodeGenerationId::new("code-generation.pre-cutover").unwrap(); - let current_generation = CodeGenerationId::new("code-generation.post-cutover").unwrap(); - - // Pre-cutover state: the generation owns a projection of its own and is - // its permanent verified head. - let legacy = manifest( - legacy_identity.clone(), - "legacy-g1", - "legacy", - vec![], - vec![], - ); - let legacy_record = stage_manifest( - &mut authority, - ®istered.binding, - &legacy, - "publish:legacy-g1", - None, - '3', - ); - let (control, probe) = control_and_probe(); - let legacy_commit = registered - .registry - .publish_verified( - registration(registered.binding.clone(), temp.path()), - &mut authority, - &fresh_context(&control, &probe), - &legacy_record.publication.key, - None, - ) - .unwrap(); - drop(legacy_commit); - bind_sealed_source( - &mut authority, - ®istered.binding, - &legacy, - &legacy_record, - "publish:legacy-g1", - None, - '3', - &legacy_generation, - &sealed_digest, - ); - - // Post-cutover: the canonical per-shard projection has no head, so the - // code index republishes the live generation into it. The legacy - // projection is untouched by that publication. - let current = manifest( - canonical_identity.clone(), - "canonical-g1", - "canonical", - vec![], - vec![], - ); - let current_record = stage_manifest( - &mut authority, - ®istered.binding, - ¤t, - "publish:canonical-g1", - None, - '4', - ); - let (control, probe) = control_and_probe(); - let current_commit = registered - .registry - .publish_verified( - registration(registered.binding.clone(), temp.path()), - &mut authority, - &fresh_context(&control, &probe), - ¤t_record.publication.key, - None, - ) - .unwrap(); - let current_head = current_commit.head.clone(); - drop(current_commit); - bind_sealed_source( - &mut authority, - ®istered.binding, - ¤t, - ¤t_record, - "publish:canonical-g1", - None, - '4', - ¤t_generation, - &sealed_digest, - ); - - // Remount: the legacy-layout rows must survive a close and reopen. - assert!(registered.close().unwrap()); - drop(registered); - let registered = RegisteredGraph::new_mounted(temp.path()).unwrap(); - - // Draining the pre-cutover generation goes through the superseded-head - // retirement path, because a legacy projection's only replay is its head. - let (control, probe) = control_and_probe(); - assert_eq!( - registered.registry.retire_one_code_generation_replay( - registration(registered.binding.clone(), temp.path()), - &mut authority, - &fresh_context(&control, &probe), - &legacy_generation, - &sealed_digest, - ), - Ok(GraphReplayCollectionOutcome::Retired(Box::new( - tracedecay_graph_db::GraphGenerationReplaySource::SealedCodeGeneration(sealed_source( - &legacy_generation, - &sealed_digest, - )) - ))) - ); - assert_eq!( - authority.head_retirement_calls, 1, - "a legacy per-generation head is reclaimed by the head-retirement path", - ); - assert!( - !authority - .heads - .contains_key(&legacy_record.publication.key.projection), - "the legacy per-generation head is gone", - ); - assert_eq!( - authority - .heads - .get(¤t_record.publication.key.projection), - Some(¤t_head), - "draining legacy-layout residue leaves the canonical head standing", - ); - - // Nothing legacy-layout is left for the sweep to find. - let (control, probe) = control_and_probe(); - assert_eq!( - registered.registry.retire_one_code_generation_replay( - registration(registered.binding.clone(), temp.path()), - &mut authority, - &fresh_context(&control, &probe), - &legacy_generation, - &sealed_digest, - ), - Ok(GraphReplayCollectionOutcome::Absent) - ); -} - -/// A retired legacy replay remains sufficient authority to release its -/// duplicate staging rows after head retirement. The sealed artifact must -/// still reproduce the tombstone's exact digest; an active, dependency- -/// bearing, ambiguous, or non-legacy replay stays fail-closed. -/// -/// Fails if release checks only `verified_head` and returns -/// `NoVerifiedLease` before inspecting the durable cleanup tombstone. -#[test] -fn retired_legacy_replay_without_a_head_releases_its_verified_sealed_staging_rows() { - let temp = TempDir::new().unwrap(); - let registered = RegisteredGraph::new_mounted(temp.path()).unwrap(); - let mut authority = ExactPublicationAuthority::new(temp.path(), ®istered.binding); - let identity = legacy_per_generation_projection('d'); - let code_generation = CodeGenerationId::new("code-generation.legacy-no-head").unwrap(); - let sealed_digest = - SealedGraphStateDigest::try_from(format!("sha256:{}", "8".repeat(64))).unwrap(); - let generation = manifest( - identity, - "legacy-no-head-g1", - "legacy-no-head", - vec![], - vec![], - ); - let publication = generation - .relational_sealed_replay( - registered.binding.shard_id.clone(), - GraphIdempotencyKey::new("publish:legacy-no-head-g1").unwrap(), - digest('8'), - None, - sealed_source(&code_generation, &sealed_digest), - &|| Ok(()), - ) - .unwrap(); - let (control, probe) = control_and_probe(); - let replay = match authority - .storage - .append_replay(&publication, &fresh_context(&control, &probe)) - .unwrap() - { - GraphReplayAppendOutcomeV1::Appended(replay) => replay, - outcome => panic!("fresh exact-SQL authority must append: {outcome:?}"), - }; - let (control, probe) = control_and_probe(); - assert_eq!( - registered.registry.release_sealed_generation_staging_rows( - registration(registered.binding.clone(), temp.path()), - &mut authority.storage, - &fresh_context(&control, &probe), - &replay.publication.key.projection, - ), - Ok(SealedStagingRelease::Retained( - SealedStagingRetentionReason::NoVerifiedLease, - )), - "production no-head replay semantics classify the sole active replay as pending" - ); - // The legacy shape under test: the generation's rows already sit in the - // staging database when its sealed artifact is built. - registered - .registry - .resolve(registration(registered.binding.clone(), temp.path())) - .unwrap() - .stage_generation_rows_unpublished(Arc::new(generation.clone())) - .unwrap(); - let (control, probe) = control_and_probe(); - let commit = registered - .registry - .publish_verified( - registration(registered.binding.clone(), temp.path()), - &mut authority.storage, - &fresh_context(&control, &probe), - &replay.publication.key, - Some(Arc::new(generation.clone())), - ) - .unwrap(); - let retirement = GraphPublicationReplayRetirementV1::new( - replay.publication.key.clone(), - replay.publication.input_digest.clone(), - replay - .publication - .dependency_generation_closure_digest - .clone(), - replay.publication.direct_dependency_generations.clone(), - replay.publication.expected_prior_head.clone(), - replay.publication.expected_recovered_digest.clone(), - replay.publication.canonical_replay_source_digest.clone(), - ) - .unwrap(); - let (control, probe) = control_and_probe(); - assert!(matches!( - authority - .storage - .retire_verified_head_replay( - &retirement, - &commit.head, - &fresh_context(&control, &probe), - ) - .unwrap(), - GraphReplayRetirementOutcomeV1::Retired(_) - )); - drop(commit); - - assert!(registered.close().unwrap()); - drop(registered); - - let registered = RegisteredGraph::new_mounted(temp.path()).unwrap(); - let (control, probe) = control_and_probe(); - assert_eq!( - registered.registry.release_sealed_generation_staging_rows( - registration(registered.binding.clone(), temp.path()), - &mut authority.storage, - &fresh_context(&control, &probe), - &replay.publication.key.projection, - ), - Ok(SealedStagingRelease::Released { - entities: 1, - relations: 0, - }) - ); - let database = registered - .registry - .resolve(registration(registered.binding.clone(), temp.path())) - .unwrap(); - assert_eq!( - database - .staging_generation_row_counts(&generation.identity()) - .unwrap(), - (0, 0), - "the replay-verified sealed artifact makes the staging copy redundant" - ); -} - /// Number of sealed generation artifacts currently on disk under the store. fn sealed_generation_count(root: &std::path::Path) -> usize { std::fs::read_dir(support::graph_path(root).with_extension("sealed")) diff --git a/crates/tracedecay-graph-query/Cargo.toml b/crates/tracedecay-graph-query/Cargo.toml index ac88f43c10..aa23ac6795 100644 --- a/crates/tracedecay-graph-query/Cargo.toml +++ b/crates/tracedecay-graph-query/Cargo.toml @@ -15,11 +15,6 @@ serde_json = "1" thiserror = "2" tokio = { version = "1", features = ["rt", "process"] } tracedecay-contracts = { path = "../tracedecay-contracts", version = "0.1.0" } -# Grammar-free entry only: `markdown_structure` compiles with no language -# feature enabled, so section structure reaches retrieval without linking a -# tree-sitter bundle into this crate; production callers already unify a -# grammar tier through the composition root. -tracedecay-code-extraction = { path = "../tracedecay-code-extraction", version = "0.1.0", default-features = false } tracedecay-code-index = { path = "../tracedecay-code-index", version = "0.1.0", default-features = false } tracedecay-domain = { path = "../tracedecay-domain", version = "0.1.0" } tracedecay-graph-db = { path = "../tracedecay-graph-db", version = "0.1.0" } diff --git a/crates/tracedecay-graph-query/src/context/markdown_sections.rs b/crates/tracedecay-graph-query/src/context/markdown_sections.rs deleted file mode 100644 index bb289d9e68..0000000000 --- a/crates/tracedecay-graph-query/src/context/markdown_sections.rs +++ /dev/null @@ -1,531 +0,0 @@ -//! Section preview, retrieval handle, and structure for markdown symbols. -//! -//! A markdown heading is one symbol in the code graph, and the graph stays -//! heading-level on purpose. But a heading alone is not a useful retrieval -//! result: a reader scanning `tracedecay_outline docs/plans/.../NEXT.md` wants -//! to know *what is under* "Remaining work" without pulling the whole file, and -//! then wants exactly one section's complete body. -//! -//! So each markdown section symbol carries a `section` lane with: -//! -//! - the title and the section's 1-based line span, so -//! `tracedecay_read mode=lines` is always the zero-magic alternative; -//! - a truncated body preview bounded by [`SECTION_PREVIEW_CHARS`]; -//! - a retrieval handle for the *full* body, minted through the same -//! response-handle cache that reversible MCP truncation uses -//! ([`tracedecay_session_memory::response_handles::store_response_handle`]), so the reader -//! dereferences it with the existing `tracedecay_retrieve` tool and no -//! parallel mechanism exists; -//! - the section's load-bearing structure, task-list checkboxes with their -//! checked state, nested bullets, ordered steps, tables, block quotes and -//! fenced code, parsed by -//! [`tracedecay_code_extraction::markdown_structure`], so "which checklist -//! items under 'Remaining work' are unchecked" is answerable from the -//! retrieval payload instead of by re-reading the file. -//! -//! A handle is minted only when the preview actually truncates: when the whole -//! body already fits in the preview, the reader is holding the full body and a -//! handle would be a pointless durable write. [`MAX_SECTION_HANDLES`] bounds how -//! many one response may mint, so a 300-heading document cannot turn one outline -//! call into 300 fsyncs; sections past the cap keep preview and line span. - -use std::path::Path; - -use serde_json::{Value, json}; -use tracedecay_code_extraction::markdown_structure::{ - MarkdownSectionStructure, parse_section_structure, -}; -use tracedecay_domain::collapse_whitespace; - -use tracedecay_session_memory::response_handles::store_response_handle; - -/// Characters of section body carried inline before the preview truncates. -pub const SECTION_PREVIEW_CHARS: usize = 320; - -/// The existing tool that dereferences a minted handle. Never a new tool. -pub const SECTION_RETRIEVE_TOOL: &str = "tracedecay_retrieve"; - -/// Handles minted per response. Past this, sections keep preview + line span. -pub const MAX_SECTION_HANDLES: usize = 64; - -/// The graph kind markdown headings are published as. -const MARKDOWN_SECTION_KIND: &str = "module"; - -/// `true` when `path` is a file the markdown extractor claims. -pub fn is_markdown_file(path: &str) -> bool { - let extension = path.rsplit_once('.').map(|(_, extension)| extension); - matches!( - extension.map(str::to_ascii_lowercase).as_deref(), - Some("md" | "markdown") - ) -} - -/// Budgeted minting across one response's sections. -pub struct SectionEnrichment<'a> { - project_root: Option<&'a Path>, - now: i64, - handles_minted: usize, -} - -impl<'a> SectionEnrichment<'a> { - pub fn new(project_root: Option<&'a Path>, now: i64) -> Self { - Self { - project_root, - now, - handles_minted: 0, - } - } - - /// Enriches every markdown section symbol in a `{"symbols": [...]}` payload - /// in place, using `source` as the file's current text. - /// - /// This is an *enrichment*: a symbol the source cannot explain (no span, a - /// span past the end of the file, a non-section kind) is left untouched - /// rather than failing the surface that carries it. - #[hotpath::measure(label = "usecases.context.markdown.enrich")] - pub fn enrich_symbol_array(&mut self, symbols: &mut [Value], source: &str) { - for symbol in symbols { - if let Some(section) = self.section_for_symbol(symbol, source) { - symbol["section"] = section; - } - } - } - - fn section_for_symbol(&mut self, symbol: &Value, source: &str) -> Option { - let kind = symbol.get("kind").and_then(Value::as_str)?; - if !kind.eq_ignore_ascii_case(MARKDOWN_SECTION_KIND) { - return None; - } - let title = symbol.get("name").and_then(Value::as_str)?; - let start_line = symbol.get("line").and_then(Value::as_u64)? as u32; - let end_line = symbol - .get("end_line") - .and_then(Value::as_u64) - .unwrap_or(u64::from(start_line)) as u32; - Some(self.section_value(source, title, start_line, end_line)) - } - - /// Builds one section's `section` lane. `start_line` is the 1-based heading - /// line and `end_line` the 1-based inclusive last line of the section. - pub fn section_value( - &mut self, - source: &str, - title: &str, - start_line: u32, - end_line: u32, - ) -> Value { - let body_start = start_line.saturating_add(1); - let body = section_body(source, body_start, end_line); - let body_chars = body.chars().count(); - let (preview, preview_truncated) = preview_of(body); - - let mut value = json!({ - "title": title, - "heading_line": start_line, - "body_start_line": body_start, - "body_end_line": end_line.max(start_line), - "body_chars": body_chars, - "preview": preview, - "preview_truncated": preview_truncated, - }); - // The span is published whether or not a handle exists, so the reader - // always has the zero-magic route into the full body. - if end_line >= body_start { - value["read_lines"] = json!(format!("{body_start}-{end_line}")); - } - - if preview_truncated { - self.attach_handle(&mut value, body); - } else { - value["body_handle"] = Value::Null; - } - - let structure = parse_section_structure(body, body_start); - if !structure.is_empty() { - value["structure"] = structure_value(&structure); - } - value - } - - /// Mints the full-body handle through the shared response-handle cache. - fn attach_handle(&mut self, value: &mut Value, body: &str) { - let Some(root) = self.project_root else { - hotpath::gauge!("usecases.context.markdown.handle_unavailable").inc(1.0); - value["body_handle"] = Value::Null; - value["body_handle_unavailable"] = json!({ - "reason_code": "handle_storage_unavailable", - "message": "This section preview was produced without a project-local cache path, so no retrieval handle could be created.", - "retryable": true, - "retry_instruction": "Re-run from a project-scoped tracedecay session, or read the section directly with tracedecay_read mode=lines.", - }); - return; - }; - if self.handles_minted >= MAX_SECTION_HANDLES { - hotpath::gauge!("usecases.context.markdown.handle_unavailable").inc(1.0); - value["body_handle"] = Value::Null; - value["body_handle_unavailable"] = json!({ - "reason_code": "handle_budget_exhausted", - "message": format!( - "This response already minted {MAX_SECTION_HANDLES} section handles; read this section with tracedecay_read mode=lines instead." - ), - "retryable": true, - "retry_instruction": "Narrow the request (for example with the outline `kinds` filter) so fewer sections compete for the handle budget.", - }); - return; - } - // Each mint is one durable write with an fsync; the span makes that - // cost visible against the rest of the markdown enrichment pass. - let stored = hotpath::measure_block!( - "usecases.context.markdown.mint_handle", - store_response_handle(root, body, self.now) - ); - match stored { - Ok(record) => { - hotpath::gauge!("usecases.context.markdown.handles_minted").inc(1.0); - self.handles_minted += 1; - value["body_handle"] = json!(record.handle); - value["body_handle_expires_at"] = json!(record.expires_at); - value["retrieve_with"] = json!(SECTION_RETRIEVE_TOOL); - } - // The handle cache records the typed error in its own telemetry. - // Public output must not disclose project-local filesystem paths. - Err(_) => { - hotpath::gauge!("usecases.context.markdown.handle_unavailable").inc(1.0); - value["body_handle"] = Value::Null; - value["body_handle_unavailable"] = json!({ - "reason_code": "handle_store_failed", - "message": "The full section body could not be cached locally, so no retrieval handle is available.", - "retryable": true, - "retry_instruction": "Read the section with tracedecay_read mode=lines, or fix the local project cache path and re-run.", - }); - } - } - } -} - -/// Unchecked checklist items named inline before the summary elides the rest. -const SUMMARY_UNCHECKED_LIMIT: usize = 8; - -/// Renders one section lane as human-facing summary lines, for a surface that -/// lists symbols as bullets. -/// -/// The lines are returned rather than written so the transport layer keeps -/// ownership of its own markdown builder: an adapter emits each line under the -/// symbol's bullet. The preview is collapsed to a single line here because the -/// surrounding surface is a list; the verbatim preview stays in the JSON. -pub fn section_summary_lines(section: &Value) -> Vec { - let mut lines = Vec::new(); - let preview = collapse_whitespace(field_str(section, "preview")); - if !preview.is_empty() { - let chars = section - .get("body_chars") - .and_then(Value::as_u64) - .unwrap_or_default(); - lines.push(format!("preview ({chars} chars): {preview}")); - } - let read_lines = field_str(section, "read_lines"); - match section.get("body_handle").and_then(Value::as_str) { - Some(handle) => lines.push(format!( - "full body: `{SECTION_RETRIEVE_TOOL}` handle `{handle}` (or `tracedecay_read mode=lines lines={read_lines}`)" - )), - None if section - .get("preview_truncated") - .and_then(Value::as_bool) - .unwrap_or(false) => - { - let reason = section - .get("body_handle_unavailable") - .and_then(|value| value.get("reason_code")) - .and_then(Value::as_str) - .unwrap_or("handle_unavailable"); - lines.push(format!( - "full body: no handle ({reason}); read `mode=lines lines={read_lines}`" - )); - } - None => {} - } - if let Some(structure) = section.get("structure") { - push_structure_lines(&mut lines, structure); - } - lines -} - -fn push_structure_lines(lines: &mut Vec, structure: &Value) { - if let Some(checklist) = structure.get("checklist") { - let total = checklist - .get("total") - .and_then(Value::as_u64) - .unwrap_or_default(); - let checked = checklist - .get("checked") - .and_then(Value::as_u64) - .unwrap_or_default(); - let mut line = format!("checklist: {checked}/{total} checked"); - let unchecked = checklist - .get("items") - .and_then(Value::as_array) - .map(|items| { - items - .iter() - .filter(|item| { - !item - .get("checked") - .and_then(Value::as_bool) - .unwrap_or(false) - }) - .take(SUMMARY_UNCHECKED_LIMIT) - .map(|item| { - format!( - "L{} {}", - item.get("line").and_then(Value::as_u64).unwrap_or_default(), - collapse_whitespace(field_str(item, "text")) - ) - }) - .collect::>() - }) - .unwrap_or_default(); - if !unchecked.is_empty() { - line.push_str(" · unchecked: "); - line.push_str(&unchecked.join("; ")); - } - lines.push(line); - } - let counts = [ - ("bullets", "bullets"), - ("ordered", "ordered items"), - ("tables", "tables"), - ("block_quotes", "block quotes"), - ("code_blocks", "code blocks"), - ] - .into_iter() - .filter_map(|(key, label)| { - let count = structure.get(key).and_then(Value::as_array)?.len(); - Some(format!("{count} {label}")) - }) - .collect::>(); - if !counts.is_empty() { - lines.push(format!("structure: {}", counts.join(" · "))); - } -} - -fn field_str<'a>(value: &'a Value, key: &str) -> &'a str { - value.get(key).and_then(Value::as_str).unwrap_or_default() -} - -/// The section body: 1-based inclusive lines `start ..= end`, empty when the -/// heading carries no body or the span points past the end of the file. -pub fn section_body(source: &str, start: u32, end: u32) -> &str { - if end < start || start == 0 { - return ""; - } - let mut offset = 0usize; - let mut body_start = None; - for (index, line) in source.split_inclusive('\n').enumerate() { - let line_number = index as u32 + 1; - if line_number == start { - body_start = Some(offset); - } - if line_number == end { - let from = match body_start { - Some(from) => from, - None => return "", - }; - return &source[from..offset + line.len()]; - } - offset += line.len(); - } - // A span that runs past the last line still describes real content: return - // the tail rather than dropping the body. - match body_start { - Some(from) => &source[from..], - None => "", - } -} - -/// `(preview, truncated)` for a section body. -fn preview_of(body: &str) -> (String, bool) { - let trimmed = body.trim(); - let mut preview = String::new(); - let mut chars = trimmed.chars(); - for _ in 0..SECTION_PREVIEW_CHARS { - match chars.next() { - Some(ch) => preview.push(ch), - None => return (preview, false), - } - } - if chars.next().is_none() { - return (preview, false); - } - // The body outran the preview budget. Truncation is reported against the - // *whole* body, not the trimmed prefix, so `body_handle` is the only way to - // see the rest. - preview.push('…'); - (preview, true) -} - -/// Publishes the parsed structure as JSON, with the counts a reader needs to -/// decide whether to pull the full body. -fn structure_value(structure: &MarkdownSectionStructure) -> Value { - let mut value = json!({}); - if !structure.checklist.is_empty() { - let checked = structure - .checklist - .iter() - .filter(|item| item.checked) - .count(); - value["checklist"] = json!({ - "total": structure.checklist.len(), - "checked": checked, - "unchecked": structure.checklist.len() - checked, - "items": structure.checklist, - }); - } - if !structure.bullets.is_empty() { - value["bullets"] = json!(structure.bullets); - } - if !structure.ordered.is_empty() { - value["ordered"] = json!(structure.ordered); - } - if !structure.tables.is_empty() { - value["tables"] = json!(structure.tables); - } - if !structure.block_quotes.is_empty() { - value["block_quotes"] = json!(structure.block_quotes); - } - if !structure.code_blocks.is_empty() { - value["code_blocks"] = json!(structure.code_blocks); - } - value -} - -#[cfg(test)] -mod tests { - use super::*; - - const DOC: &str = "\ -# Plan - -Intro line. - -## Remaining work - -- [x] wire the extractor -- [ ] mint the handle - - [ ] nested follow-up -- prose bullet - -| lane | owner | -| ---- | ----- | -| index | zack | - -```rust -fn probe() {} -``` - -## Done - -Nothing left. -"; - - fn enrichment() -> SectionEnrichment<'static> { - SectionEnrichment::new(None, 0) - } - - #[test] - fn section_body_is_the_lines_after_the_heading() { - // "## Remaining work" is 1-based line 5, section ends at line 19. - let body = section_body(DOC, 6, 19); - assert!(body.starts_with("\n- [x] wire the extractor")); - assert!(body.contains("fn probe()")); - assert!(!body.contains("## Done")); - } - - #[test] - fn section_publishes_span_preview_and_structure() { - let value = enrichment().section_value(DOC, "Remaining work", 5, 19); - - assert_eq!(value["title"], "Remaining work"); - assert_eq!(value["heading_line"], 5); - assert_eq!(value["body_start_line"], 6); - assert_eq!(value["read_lines"], "6-19"); - - let checklist = &value["structure"]["checklist"]; - assert_eq!(checklist["total"], 3); - assert_eq!(checklist["checked"], 1); - assert_eq!(checklist["unchecked"], 2); - // Checklist lines are absolute and 1-based, so they address the same - // rows `tracedecay_read mode=lines` does. - assert_eq!(checklist["items"][0]["line"], 7); - assert_eq!(checklist["items"][0]["checked"], true); - assert_eq!(checklist["items"][1]["text"], "mint the handle"); - assert_eq!(checklist["items"][2]["depth"], 1); - - assert_eq!(value["structure"]["bullets"][0]["text"], "prose bullet"); - assert_eq!(value["structure"]["tables"][0]["rows"], 1); - assert_eq!( - value["structure"]["code_blocks"][0]["language"].as_str(), - Some("rust") - ); - } - - #[test] - fn short_sections_carry_the_whole_body_and_no_handle() { - // "## Done" is 1-based line 20; its body is lines 21-22. - let value = enrichment().section_value(DOC, "Done", 20, 22); - - assert_eq!(value["preview"], "Nothing left."); - assert_eq!(value["preview_truncated"], false); - assert_eq!(value["read_lines"], "21-22"); - // A body that already fits the preview needs no durable handle: the - // reader is holding the whole section. - assert_eq!(value["body_handle"], Value::Null); - assert!(value.get("body_handle_unavailable").is_none()); - } - - #[test] - fn oversized_sections_report_why_no_handle_exists_without_a_project_root() { - let long = format!("# Big\n\n{}\n", "word ".repeat(400)); - let value = enrichment().section_value(&long, "Big", 1, 3); - - assert_eq!(value["preview_truncated"], true); - assert_eq!(value["body_handle"], Value::Null); - assert_eq!( - value["body_handle_unavailable"]["reason_code"], - "handle_storage_unavailable" - ); - } - - #[test] - fn only_markdown_section_symbols_are_enriched() { - let mut symbols = vec![ - json!({"kind": "module", "name": "Remaining work", "line": 5, "end_line": 19}), - json!({"kind": "function", "name": "probe", "line": 5, "end_line": 19}), - ]; - enrichment().enrich_symbol_array(&mut symbols, DOC); - - assert_eq!(symbols[0]["section"]["title"], "Remaining work"); - assert!(symbols[1].get("section").is_none()); - } - - #[test] - fn summary_lines_name_the_open_checklist_items_and_the_read_route() { - let value = enrichment().section_value(DOC, "Remaining work", 5, 19); - let lines = section_summary_lines(&value); - let joined = lines.join("\n"); - - assert!(joined.contains("checklist: 1/3 checked"), "{joined}"); - assert!(joined.contains("L8 mint the handle"), "{joined}"); - assert!(joined.contains("L9 nested follow-up"), "{joined}"); - assert!(joined.contains("1 code blocks"), "{joined}"); - // Every line must stay a single line: these are bullet continuations. - assert!(lines.iter().all(|line| !line.contains('\n')), "{joined}"); - } - - #[test] - fn symbols_without_a_published_span_are_left_untouched() { - let mut symbols = vec![json!({"kind": "module", "name": "Orphan", "line": Value::Null})]; - enrichment().enrich_symbol_array(&mut symbols, DOC); - - assert!(symbols[0].get("section").is_none()); - } -} diff --git a/crates/tracedecay-graph-query/src/context/mod.rs b/crates/tracedecay-graph-query/src/context/mod.rs index 840df6b273..84e8757b5e 100644 --- a/crates/tracedecay-graph-query/src/context/mod.rs +++ b/crates/tracedecay-graph-query/src/context/mod.rs @@ -1,8 +1,7 @@ -//! Code-index-backed source-read helpers (`source_read`, `read_modes`, -//! `markdown_sections`) consumed by [`crate::VerifiedGraphQuery`] and the -//! usecases source primitives. The request-context value types and read -//! cache these compose with live in `tracedecay_session_memory::context`. +//! Code-index-backed source-read helpers (`source_read`, `read_modes`) +//! consumed by [`crate::VerifiedGraphQuery`] and the usecases source +//! primitives. The request-context value types and read cache these compose +//! with live in `tracedecay_session_memory::context`. -pub mod markdown_sections; pub mod read_modes; pub mod source_read; diff --git a/crates/tracedecay-graph-query/src/lib.rs b/crates/tracedecay-graph-query/src/lib.rs index d5e94e8fd7..934ed2dfaa 100644 --- a/crates/tracedecay-graph-query/src/lib.rs +++ b/crates/tracedecay-graph-query/src/lib.rs @@ -1,6 +1,6 @@ //! Generation-pinned verified code-graph queries over daemon-resolved //! projections, plus the code-index-backed source readers -//! (`context::{read_modes, source_read, markdown_sections}`) that hydrate +//! (`context::{read_modes, source_read}`) that hydrate //! source evidence for those queries. //! //! This crate sits below the transport adapters (`tracedecay-mcp`, the root diff --git a/crates/tracedecay-graph-query/src/source_authority.rs b/crates/tracedecay-graph-query/src/source_authority.rs index 11dfa82f76..8f490cb17b 100644 --- a/crates/tracedecay-graph-query/src/source_authority.rs +++ b/crates/tracedecay-graph-query/src/source_authority.rs @@ -2,16 +2,14 @@ //! //! The [`SourceReadContext`] wired at composition enters a verified graph //! query exactly once, at admitted open, where it is frozen into -//! [`AdmittedSourceAuthority`]: root, database authority, read-only posture, -//! and project identity are copied once and used exclusively thereafter, so -//! no later API accepts a substitute. +//! [`AdmittedSourceAuthority`]: root and project identity are copied once and +//! used exclusively thereafter, so no later API accepts a substitute. use std::path::{Path, PathBuf}; use tracedecay_contracts::RequestContext; use tracedecay_domain::ProjectId; use tracedecay_domain::errors::{Result, TraceDecayError}; -use tracedecay_runtime_core::db::Database; use crate::SourceReadContext; @@ -22,8 +20,6 @@ use crate::SourceReadContext; /// is the admission-validated capture inside [`super::open_verified_graph_query`]. pub(crate) struct AdmittedSourceAuthority { project_root: PathBuf, - db: Database, - read_only: bool, project_id: ProjectId, } @@ -37,8 +33,6 @@ impl AdmittedSourceAuthority { } Ok(Self { project_root: source.project_root, - db: source.db, - read_only: source.read_only, project_id: context.scope().project_id.clone(), }) } @@ -47,14 +41,6 @@ impl AdmittedSourceAuthority { &self.project_root } - pub(crate) fn db(&self) -> &Database { - &self.db - } - - pub(crate) fn read_only(&self) -> bool { - self.read_only - } - pub(crate) fn project_id(&self) -> &str { self.project_id.as_str() } diff --git a/crates/tracedecay-graph-query/src/verified_query.rs b/crates/tracedecay-graph-query/src/verified_query.rs index c6f2f3acbc..d8923b6261 100644 --- a/crates/tracedecay-graph-query/src/verified_query.rs +++ b/crates/tracedecay-graph-query/src/verified_query.rs @@ -6,7 +6,6 @@ use std::path::{Path, PathBuf}; use std::pin::Pin; use std::sync::Arc; -use serde_json::Value; use tracedecay_code_index::chunks::CodeIndexImportEvidenceV1; use tracedecay_code_index::graph_projection::{ CodeGraphImpactBatchV1, CodeGraphInteractiveReader, CodeGraphSemanticEdgeV1, @@ -22,6 +21,7 @@ use tracedecay_domain::{ SymbolOccurrenceId, }; use tracedecay_graph_db::GraphCancellation; +use tracedecay_runtime_core::path_safety::plain_host_path; use super::queries::GraphQueryManager; use super::source_authority::{ @@ -33,8 +33,7 @@ use super::{ map_projection_error, }; use crate::SourceReadContext; -use crate::context::read_modes; -use crate::context::source_read::{self, SourceReadOutput, SourceReadRequest}; +use crate::context::source_read; use tracedecay_session_memory::context::{RequestInterruption, run_deadline_signal_interruptible}; /// Inputs required to admit and open one verified graph query. @@ -232,15 +231,6 @@ impl VerifiedGraphQuery { Ok(self.bound_source()?.project_id()) } - pub fn read_indexed_source_file(&self, file: &str) -> Result { - let (absolute, _) = self.resolve_indexed_source_file(file)?; - tracedecay_runtime_core::sync::read_source_file(&absolute).map_err(|error| { - TraceDecayError::Config { - message: format!("cannot read indexed source file '{file}': {error}"), - } - }) - } - #[hotpath::measure(label = "usecases.graph.verified.dead_code", future = true)] pub async fn find_dead_code( &self, @@ -291,23 +281,6 @@ impl VerifiedGraphQuery { .await } - #[hotpath::skip] - pub async fn read_source(&self, request: SourceReadRequest<'_>) -> Result { - let source = self.bound_source()?; - if request.project_id != source.project_id() { - return Err(graph_source_scope_mismatch()); - } - self.await_bound(source_read::read_source( - source.project_root(), - source.db(), - source.read_only(), - &self.reader, - Arc::clone(&self.cancellation), - request, - )) - .await - } - pub fn resolve_indexed_source_file(&self, file: &str) -> Result<(PathBuf, String)> { let source = self.bound_source()?; let (absolute, display) = source_read::resolve_indexed_source_file( @@ -316,22 +289,14 @@ impl VerifiedGraphQuery { Arc::clone(&self.cancellation), file, )?; - if !absolute.starts_with(source.project_root()) { + // The resolved file is always `canonicalize`d, `\\?\`-prefixed on + // Windows, while the admitted root may carry the plain identity. + if !plain_host_path(&absolute).starts_with(plain_host_path(source.project_root())) { return Err(graph_source_scope_mismatch()); } Ok((absolute, display)) } - pub fn render_map(&self, file_path: &str, kinds: Option<&[String]>) -> Result { - self.refuse_if_bound_closed()?; - read_modes::render_map( - &self.reader, - Arc::clone(&self.cancellation), - file_path, - kinds, - ) - } - pub fn generation(&self) -> &CodeGenerationId { self.reader.generation() } diff --git a/crates/tracedecay-graph-query/src/verified_query_source_tests.rs b/crates/tracedecay-graph-query/src/verified_query_source_tests.rs index 4682ab8d17..2f2803b780 100644 --- a/crates/tracedecay-graph-query/src/verified_query_source_tests.rs +++ b/crates/tracedecay-graph-query/src/verified_query_source_tests.rs @@ -20,8 +20,6 @@ use super::verified_query_test_support::{ }; use super::{VerifiedGraphQuery, VerifiedGraphQueryRequest, open_verified_graph_query}; use crate::SourceReadContext; -use crate::context::read_modes::ReadMode; -use crate::context::source_read::SourceReadRequest; async fn test_database(path: &Path) -> Database { crate::register_test_schema_installer(); @@ -96,17 +94,6 @@ fn assert_denied(error: tracedecay_domain::errors::TraceDecayError) { assert!(!retryable); } -fn full_read_request(project_id: &str) -> SourceReadRequest<'_> { - SourceReadRequest { - file: "src/lib.rs", - mode: ReadMode::Full, - line_range: None, - raw_lines: None, - include_symbols: false, - project_id, - } -} - #[test] fn unbound_query_refuses_source_reads() { let query = fixture_query("project.verified-query-source.a"); @@ -144,29 +131,6 @@ async fn resolve_rejects_absolute_path_under_another_project_root() { ); } -#[tokio::test] -async fn read_source_rejects_request_project_id_outside_bound_source() { - let home = tempfile::tempdir().expect("temp"); - let project_a = home.path().join("project-a"); - std::fs::create_dir_all(&project_a).expect("project a"); - let db = test_database(&project_a.join("bound.db")).await; - let query = - fixture_query("project.verified-query-source.a").with_source(SourceReadContext::new( - project_a, - db, - true, - "project.verified-query-source.a".to_owned(), - )); - let error = match query - .read_source(full_read_request("project.verified-query-source.b")) - .await - { - Ok(_) => panic!("foreign request project id must be denied"), - Err(error) => error, - }; - assert_denied(error); -} - #[tokio::test] async fn open_denies_cross_project_source_at_bind() { let home = tempfile::tempdir().expect("temp"); diff --git a/crates/tracedecay-hooks/src/admission_ledger.rs b/crates/tracedecay-hooks/src/admission_ledger.rs index 6c21310ae9..5558108c79 100644 --- a/crates/tracedecay-hooks/src/admission_ledger.rs +++ b/crates/tracedecay-hooks/src/admission_ledger.rs @@ -29,13 +29,15 @@ use std::path::{Path, PathBuf}; use serde::{Deserialize, Serialize}; use thiserror::Error; use tracedecay_domain::{UtcMicros, canonical_json_bytes, framed_log::checksum as frame_checksum}; +use tracedecay_private_fs::FileLease; use tracedecay_private_fs::framed_log::{ DirectorySyncPolicy, append_durable, atomic_write as shared_atomic_write, read_bounded as shared_read_bounded, sync_directory as shared_sync_directory, truncate_file as shared_truncate_file, validate_regular_or_missing as shared_validate_regular, }; -use crate::{HookEventEnvelopeV2, HookHostV1, MAX_SPOOL_AGE_MICROS, MAX_SPOOL_RECORDS_PER_HOST}; +use crate::{HookEventEnvelopeV2, MAX_SPOOL_AGE_MICROS, MAX_SPOOL_RECORDS_PER_HOST}; +use tracedecay_domain::NativeHostIdentityV1; const LEDGER_MAGIC: &[u8; 4] = b"TDL1"; const LEDGER_FORMAT_VERSION: u16 = 1; @@ -155,26 +157,20 @@ pub fn hook_admission_digest( #[derive(Debug)] pub struct HookAdmissionLedgerV1 { root: PathBuf, - _writer_lock: fs::File, - host: HookHostV1, + _writer_lock: FileLease, + host: NativeHostIdentityV1, limits: HookAdmissionLedgerLimitsV1, entries: BTreeMap<[u8; IDENTITY_BYTES], LedgerEntry>, completed_work: BTreeSet<[u8; IDENTITY_BYTES]>, next_order: u64, } -impl Drop for HookAdmissionLedgerV1 { - fn drop(&mut self) { - let _ = self._writer_lock.unlock(); - } -} - impl HookAdmissionLedgerV1 { /// Open (and bounded-recover) the ledger for one host. #[hotpath::measure(label = "hooks.admission.open")] pub fn open( root: impl Into, - host: HookHostV1, + host: NativeHostIdentityV1, limits: HookAdmissionLedgerLimitsV1, now: UtcMicros, ) -> Result<(Self, HookAdmissionLedgerOpenReportV1), HookAdmissionLedgerError> { @@ -256,7 +252,7 @@ impl HookAdmissionLedgerV1 { Ok((ledger, report)) } - pub fn host(&self) -> HookHostV1 { + pub fn host(&self) -> NativeHostIdentityV1 { self.host } @@ -460,7 +456,7 @@ fn lock_path(root: &Path) -> PathBuf { root.join(LOCK_FILE) } -fn acquire_writer_lock(root: &Path) -> Result { +fn acquire_writer_lock(root: &Path) -> Result { let path = lock_path(root); shared_validate_regular(&path).map_err(|_| HookAdmissionLedgerError::UnsafePath)?; let file = fs::OpenOptions::new() @@ -471,7 +467,7 @@ fn acquire_writer_lock(root: &Path) -> Result Ok(file), + Ok(()) => Ok(FileLease::held(file, "hooks.admission.writer")), Err(std::fs::TryLockError::WouldBlock) => { hotpath::gauge!("hooks.admission.lock.contended").inc(1); Err(HookAdmissionLedgerError::Busy) @@ -637,7 +633,7 @@ mod tests { HookEventEnvelopeV2 { schema_version: HOOK_EVENT_SCHEMA_VERSION, event_id: [event_id; 16], - producer: HookHostV1::ClaudeCode, + producer: NativeHostIdentityV1::ClaudeCode, protected_session_id: [7; 32], project_id: [1; 16], repository_id: [2; 16], @@ -655,7 +651,7 @@ mod tests { fn open(root: &Path, now: UtcMicros) -> HookAdmissionLedgerV1 { HookAdmissionLedgerV1::open( root, - HookHostV1::ClaudeCode, + NativeHostIdentityV1::ClaudeCode, HookAdmissionLedgerLimitsV1::stock(), now, ) @@ -746,7 +742,7 @@ mod tests { "contended" => assert!(matches!( HookAdmissionLedgerV1::open( &root, - HookHostV1::ClaudeCode, + NativeHostIdentityV1::ClaudeCode, HookAdmissionLedgerLimitsV1::stock(), UtcMicros(2), ), @@ -755,7 +751,7 @@ mod tests { "released" => { HookAdmissionLedgerV1::open( &root, - HookHostV1::ClaudeCode, + NativeHostIdentityV1::ClaudeCode, HookAdmissionLedgerLimitsV1::stock(), UtcMicros(3), ) @@ -885,10 +881,14 @@ mod tests { max_records: 8, max_age_micros: MAX_SPOOL_AGE_MICROS, }; - let mut ledger = - HookAdmissionLedgerV1::open(root.path(), HookHostV1::ClaudeCode, limits, UtcMicros(1)) - .unwrap() - .0; + let mut ledger = HookAdmissionLedgerV1::open( + root.path(), + NativeHostIdentityV1::ClaudeCode, + limits, + UtcMicros(1), + ) + .unwrap() + .0; for index in 1..=9u8 { assert_eq!( ledger @@ -901,10 +901,14 @@ mod tests { assert!(ledger.live_records() <= 8); // The newest identity is still deduplicated after eviction + reopen. drop(ledger); - let mut reopened = - HookAdmissionLedgerV1::open(root.path(), HookHostV1::ClaudeCode, limits, UtcMicros(20)) - .unwrap() - .0; + let mut reopened = HookAdmissionLedgerV1::open( + root.path(), + NativeHostIdentityV1::ClaudeCode, + limits, + UtcMicros(20), + ) + .unwrap() + .0; assert_eq!( reopened.admit(&envelope(9, 5), UtcMicros(21)).unwrap(), HookAdmissionDecisionV1::ExactDuplicate @@ -925,7 +929,7 @@ mod tests { let (mut ledger, report) = HookAdmissionLedgerV1::open( root.path(), - HookHostV1::ClaudeCode, + NativeHostIdentityV1::ClaudeCode, HookAdmissionLedgerLimitsV1::stock(), UtcMicros(3), ) diff --git a/crates/tracedecay-hooks/src/capture.rs b/crates/tracedecay-hooks/src/capture.rs index b4bd39eaf6..6347719d5b 100644 --- a/crates/tracedecay-hooks/src/capture.rs +++ b/crates/tracedecay-hooks/src/capture.rs @@ -11,11 +11,12 @@ use tracedecay_domain::UtcMicros; use crate::{ HookConfigurationFileReaderV1, HookConfigurationReadOutcomeV1, HookConfigurationSubscriberV1, - HookEventEnvelopeV2, HookHostV1, HookScopeBindingV1, HookSpoolConfigV1, HookSpoolError, - HookSpoolV1, NativeEnvelopeMaterialV1, NativeHookDecodeError, OpenCodePluginSurfaceV1, + HookEventEnvelopeV2, HookScopeBindingV1, HookSpoolConfigV1, HookSpoolError, HookSpoolV1, + NativeEnvelopeMaterialV1, NativeHookDecodeError, OpenCodePluginSurfaceV1, decode_native_hook_event, decode_opencode_plugin_event, hook_configuration_path, hook_v2_spool_root, }; +use tracedecay_domain::NativeHostIdentityV1; /// The real host surface that supplied native hook bytes. /// @@ -23,15 +24,15 @@ use crate::{ /// though it produces the same host-neutral envelope as its event-bus route. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum NativeHookCaptureSourceV1 { - Host(HookHostV1), + Host(NativeHostIdentityV1), OpenCodeToolExecuteAfter, } impl NativeHookCaptureSourceV1 { - pub const fn host(self) -> HookHostV1 { + pub const fn host(self) -> NativeHostIdentityV1 { match self { Self::Host(host) => host, - Self::OpenCodeToolExecuteAfter => HookHostV1::OpenCode, + Self::OpenCodeToolExecuteAfter => NativeHostIdentityV1::OpenCode, } } } diff --git a/crates/tracedecay-hooks/src/config.rs b/crates/tracedecay-hooks/src/config.rs index f459687816..64586a12f0 100644 --- a/crates/tracedecay-hooks/src/config.rs +++ b/crates/tracedecay-hooks/src/config.rs @@ -12,7 +12,8 @@ use thiserror::Error; use tracedecay_domain::{UtcMicros, encode_lowercase_hex}; use tracedecay_private_fs::framed_log::{DirectorySyncPolicy, atomic_write, read_bounded}; -use crate::{HookHostV1, HookScopeBindingV1}; +use crate::HookScopeBindingV1; +use tracedecay_domain::NativeHostIdentityV1; pub const HOOK_CONFIGURATION_SCHEMA_VERSION: u16 = 1; pub const MAX_HOOK_CONFIGURATION_BYTES: usize = 64 * 1024; @@ -21,7 +22,7 @@ const DIRECTORY_SYNC_POLICY: DirectorySyncPolicy = DirectorySyncPolicy::Tolerate pub fn hook_configuration_path( data_root: &Path, worktree_id: [u8; 16], - host: HookHostV1, + host: NativeHostIdentityV1, ) -> PathBuf { data_root .join("hook-configurations") @@ -97,7 +98,7 @@ pub trait HookConfigurationPublicationStoreV1 { pub trait HookConfigurationReadStoreV1 { fn load( &self, - host: HookHostV1, + host: NativeHostIdentityV1, ) -> Result, HookConfigurationPublicationError>; } @@ -146,7 +147,11 @@ where /// bounded read, so its cost and outcome mix are what separate "hook is /// unbound/stale" from a spool refusal when hooks fall silent. #[hotpath::measure(label = "hooks.config.load")] - pub fn load_current(&self, host: HookHostV1, now: UtcMicros) -> HookConfigurationReadOutcomeV1 { + pub fn load_current( + &self, + host: NativeHostIdentityV1, + now: UtcMicros, + ) -> HookConfigurationReadOutcomeV1 { let outcome = self.load_current_inner(host, now); #[cfg(feature = "hotpath")] { @@ -164,7 +169,7 @@ where fn load_current_inner( &self, - host: HookHostV1, + host: NativeHostIdentityV1, now: UtcMicros, ) -> HookConfigurationReadOutcomeV1 { let snapshot = match self.store.load(host) { @@ -215,7 +220,7 @@ impl HookConfigurationPublicationStoreV1 for HookConfigurationFileWriterV1 { .parent() .ok_or(HookConfigurationPublicationError::Unavailable)?; std::fs::create_dir_all(parent) - .and_then(|_| tracedecay_private_fs::make_private_directory(parent).map(drop)) + .and_then(|_| tracedecay_private_fs::make_private_directory(parent)) .map_err(|_| HookConfigurationPublicationError::Unavailable)?; let current = match read_snapshot(&self.path) { Ok(current) => current, @@ -261,7 +266,7 @@ impl HookConfigurationFileReaderV1 { impl HookConfigurationReadStoreV1 for HookConfigurationFileReaderV1 { fn load( &self, - _host: HookHostV1, + _host: NativeHostIdentityV1, ) -> Result, HookConfigurationPublicationError> { read_snapshot(&self.path) } @@ -330,7 +335,7 @@ mod tests { impl HookConfigurationReadStoreV1 for Store { fn load( &self, - _host: HookHostV1, + _host: NativeHostIdentityV1, ) -> Result, HookConfigurationPublicationError> { Ok(self.0.lock().unwrap().clone()) @@ -367,7 +372,7 @@ mod tests { published_at: UtcMicros(1), expires_at: UtcMicros(expires_at), binding: HookScopeBindingV1 { - host: HookHostV1::ClaudeCode, + host: NativeHostIdentityV1::ClaudeCode, project_id: [1; 16], repository_id: [2; 16], worktree_id: [3; 16], @@ -404,7 +409,7 @@ mod tests { ); let restarted_subscriber = HookConfigurationSubscriberV1::new(store); assert_eq!( - restarted_subscriber.load_current(HookHostV1::ClaudeCode, UtcMicros(2)), + restarted_subscriber.load_current(NativeHostIdentityV1::ClaudeCode, UtcMicros(2)), HookConfigurationReadOutcomeV1::Bound(published) ); } @@ -428,13 +433,13 @@ mod tests { let subscriber = HookConfigurationSubscriberV1::new(store.clone()); *store.0.lock().unwrap() = Some(snapshot(1, 2)); assert_eq!( - subscriber.load_current(HookHostV1::ClaudeCode, UtcMicros(2)), + subscriber.load_current(NativeHostIdentityV1::ClaudeCode, UtcMicros(2)), HookConfigurationReadOutcomeV1::Stale ); *store.0.lock().unwrap() = Some(snapshot(1, 100)); assert_eq!( - subscriber.load_current(HookHostV1::Codex, UtcMicros(2)), + subscriber.load_current(NativeHostIdentityV1::Codex, UtcMicros(2)), HookConfigurationReadOutcomeV1::Corrupted ); } @@ -456,7 +461,7 @@ mod tests { assert_eq!(value["revision"], 2); assert_eq!( HookConfigurationSubscriberV1::new(reader.clone()) - .load_current(HookHostV1::ClaudeCode, UtcMicros(2)), + .load_current(NativeHostIdentityV1::ClaudeCode, UtcMicros(2)), HookConfigurationReadOutcomeV1::Bound(published) ); assert_eq!( @@ -482,7 +487,7 @@ mod tests { fs::write(&path, vec![b'x'; MAX_HOOK_CONFIGURATION_BYTES + 1]).unwrap(); assert_eq!( HookConfigurationSubscriberV1::new(reader) - .load_current(HookHostV1::ClaudeCode, UtcMicros(2)), + .load_current(NativeHostIdentityV1::ClaudeCode, UtcMicros(2)), HookConfigurationReadOutcomeV1::Corrupted ); } @@ -496,7 +501,7 @@ mod tests { #[test] fn a_later_worktree_publication_does_not_replace_an_earlier_worktree_binding() { let data_root = TestDir::new(); - let host = HookHostV1::ClaudeCode; + let host = NativeHostIdentityV1::ClaudeCode; let earlier = snapshot(10, 100); let mut later = snapshot(11, 100); later.binding.worktree_id = [7; 16]; diff --git a/crates/tracedecay-hooks/src/core_events.rs b/crates/tracedecay-hooks/src/core_events.rs index 87e27ce7e3..f5cb95b818 100644 --- a/crates/tracedecay-hooks/src/core_events.rs +++ b/crates/tracedecay-hooks/src/core_events.rs @@ -8,8 +8,7 @@ use std::path::PathBuf; use serde::{Deserialize, Serialize}; -/// A domain-catalogued host whose lifecycle hooks notify the daemon. -pub use tracedecay_domain::HostIntegrationIdV1 as HookAgent; +use tracedecay_domain::HostIntegrationIdV1; pub const HOOK_EVENT_METHOD: &str = "tracedecay/hookEvent"; @@ -67,7 +66,7 @@ pub struct DaemonHookEvent { impl DaemonHookEvent { fn new( - agent: HookAgent, + agent: HostIntegrationIdV1, event: &'static str, rel_paths: Vec, command: Option, @@ -92,7 +91,7 @@ impl DaemonHookEvent { pub fn cursor_after_shell_execution(cwd: PathBuf) -> Self { Self::new( - HookAgent::Cursor, + HostIntegrationIdV1::Cursor, "afterShellExecution", Vec::new(), None, @@ -102,18 +101,22 @@ impl DaemonHookEvent { /// A provider session started: let the daemon own branch tracking and /// index refresh for the session's actual working directory. - pub fn session_start(agent: HookAgent, cwd: PathBuf) -> Self { + pub fn session_start(agent: HostIntegrationIdV1, cwd: PathBuf) -> Self { Self::new(agent, "sessionStart", Vec::new(), None, Some(cwd)) } /// A file-edit tool finished: request targeted sync of the edited paths. - pub fn post_tool_use_edit(agent: HookAgent, rel_paths: Vec, cwd: PathBuf) -> Self { + pub fn post_tool_use_edit( + agent: HostIntegrationIdV1, + rel_paths: Vec, + cwd: PathBuf, + ) -> Self { Self::new(agent, "postToolUseEdit", rel_paths, None, Some(cwd)) } /// A shell command finished. Command text is deliberately discarded: /// native daemon state, not shell parsing, owns Git reconciliation. - pub fn post_tool_use_shell(agent: HookAgent, cwd: PathBuf) -> Self { + pub fn post_tool_use_shell(agent: HostIntegrationIdV1, cwd: PathBuf) -> Self { Self::new(agent, "postToolUseShell", Vec::new(), None, Some(cwd)) } } diff --git a/crates/tracedecay-hooks/src/delivery_spool.rs b/crates/tracedecay-hooks/src/delivery_spool.rs index f37da94b2e..dd210973c6 100644 --- a/crates/tracedecay-hooks/src/delivery_spool.rs +++ b/crates/tracedecay-hooks/src/delivery_spool.rs @@ -4,7 +4,7 @@ //! writer has flushed successfully. The daemon settles files through the //! project delivery authority and removes them only after that durable CAS. -use std::fs::{self, File, OpenOptions}; +use std::fs::{self, OpenOptions}; use std::path::{Path, PathBuf}; use std::time::{Duration, Instant}; @@ -16,6 +16,7 @@ use tracedecay_domain::{ DeliverySettlementOutcomeV1, DeliverySettlementV1, DeliverySurfaceFamilyV1, canonical_json_bytes, canonical_sha256, sha256_hex_suffix, }; +use tracedecay_private_fs::FileLease; use tracedecay_private_fs::framed_log::{ DirectorySyncPolicy, atomic_write, is_owned_temporary_name, read_bounded, remove_abandoned_temporaries, sync_directory, validate_regular_or_missing, @@ -118,15 +119,7 @@ pub enum HookDeliverySpoolError { #[derive(Debug)] pub struct HookDeliveryReceiptSpoolV1 { root: PathBuf, - _lock: File, -} - -impl Drop for HookDeliveryReceiptSpoolV1 { - fn drop(&mut self) { - if let Err(error) = self._lock.unlock() { - tracing::warn!(error = %error, "hook delivery receipt spool lock could not be released"); - } - } + _lock: FileLease, } impl HookDeliveryReceiptSpoolV1 { @@ -187,7 +180,10 @@ impl HookDeliveryReceiptSpoolV1 { )?; } } - let spool = Self { root, _lock: lock }; + let spool = Self { + root, + _lock: FileLease::held(lock, "hooks.delivery.writer"), + }; // The lock is held now, so every staging temporary still in the root // was abandoned by a killed publisher rather than owned by a live one. remove_abandoned_temporaries(&spool.root, DIRECTORY_POLICY) @@ -340,7 +336,10 @@ impl HookDeliveryReceiptSpoolV1 { } } -pub fn hook_delivery_receipt_spool_root(data_root: &Path, host: crate::HookHostV1) -> PathBuf { +pub fn hook_delivery_receipt_spool_root( + data_root: &Path, + host: tracedecay_domain::NativeHostIdentityV1, +) -> PathBuf { data_root.join("hook-delivery-spool").join(host.hook_key()) } diff --git a/crates/tracedecay-hooks/src/hook_v2_replay.rs b/crates/tracedecay-hooks/src/hook_v2_replay.rs index 02a0bbb304..ba46f5980f 100644 --- a/crates/tracedecay-hooks/src/hook_v2_replay.rs +++ b/crates/tracedecay-hooks/src/hook_v2_replay.rs @@ -19,9 +19,10 @@ use tracedecay_domain::UtcMicros; use crate::{ HookConfigurationFileReaderV1, HookConfigurationReadOutcomeV1, HookConfigurationSubscriberV1, - HookEventEnvelopeV2, HookHostV1, HookScopeBindingV1, HookSpoolAckDispositionV1, HookSpoolAckV1, + HookEventEnvelopeV2, HookScopeBindingV1, HookSpoolAckDispositionV1, HookSpoolAckV1, HookSpoolRecordV1, HookSpoolV1, hook_configuration_path, validate_replay_batch, }; +use tracedecay_domain::NativeHostIdentityV1; /// Fair sessions leased per host per pass. The spool caps this at four. const REPLAY_SESSIONS_PER_PASS: usize = 4; @@ -70,14 +71,14 @@ pub enum HookReplayAdmissionOutcomeV1 { Unavailable, } -pub fn hook_v2_spool_root(data_root: &Path, host: HookHostV1) -> PathBuf { +pub fn hook_v2_spool_root(data_root: &Path, host: NativeHostIdentityV1) -> PathBuf { data_root.join("hook-v2-spool").join(host.hook_key()) } pub fn published_hook_scope_binding( data_root: &Path, worktree_id: [u8; 16], - host: HookHostV1, + host: NativeHostIdentityV1, now: UtcMicros, ) -> Option { let subscriber = HookConfigurationSubscriberV1::new(HookConfigurationFileReaderV1::new( @@ -317,7 +318,7 @@ where } fn log_tombstone( - host: HookHostV1, + host: NativeHostIdentityV1, record: &HookSpoolRecordV1, reason: HookReplayTombstoneReasonV1, ) { @@ -376,7 +377,7 @@ mod tests { HookSpoolConfigV1, stock_event_support, }; - const HOST: HookHostV1 = HookHostV1::ClaudeCode; + const HOST: NativeHostIdentityV1 = NativeHostIdentityV1::ClaudeCode; const PROJECT_ID: [u8; 16] = [1; 16]; const WORKTREE_ID: [u8; 16] = [3; 16]; @@ -385,11 +386,11 @@ mod tests { let data_root = Path::new("/tmp/tracedecay-hook-v2"); assert_eq!( - hook_v2_spool_root(data_root, HookHostV1::CursorDesktop), + hook_v2_spool_root(data_root, NativeHostIdentityV1::CursorDesktop), data_root.join("hook-v2-spool").join("cursor-desktop") ); assert_eq!( - hook_v2_spool_root(data_root, HookHostV1::CursorCloud), + hook_v2_spool_root(data_root, NativeHostIdentityV1::CursorCloud), data_root.join("hook-v2-spool").join("cursor-cloud") ); } @@ -541,17 +542,17 @@ mod tests { let root = TestRoot::new("lifecycle-suggestion"); let now = UtcMicros(1_000); let mut binding = binding(7); - binding.host = HookHostV1::OpenCode; + binding.host = NativeHostIdentityV1::OpenCode; binding.capabilities = binding .capabilities .iter() .map(|capability| HookCapabilityV1 { family: capability.family, - support: stock_event_support(HookHostV1::OpenCode, capability.family), + support: stock_event_support(NativeHostIdentityV1::OpenCode, capability.family), }) .collect(); let mut tool_after = envelope(9, &binding); - tool_after.producer = HookHostV1::OpenCode; + tool_after.producer = NativeHostIdentityV1::OpenCode; tool_after.protected_session_id = protected_session_id("session.native.replay"); tool_after.event = HookEventV2::ToolLifecycle { tool_id: [8; 16], @@ -565,10 +566,10 @@ mod tests { ) .unwrap(); publish_binding(root.path(), &binding, now); - let spool_root = hook_v2_spool_root(root.path(), HookHostV1::OpenCode); + let spool_root = hook_v2_spool_root(root.path(), NativeHostIdentityV1::OpenCode); let (mut spool, _) = HookSpoolV1::open( &spool_root, - HookSpoolConfigV1::stock(HookHostV1::OpenCode), + HookSpoolConfigV1::stock(NativeHostIdentityV1::OpenCode), now, ) .unwrap(); @@ -582,7 +583,7 @@ mod tests { let report = drain_host_spool_once( HookSpoolV1::open( &spool_root, - HookSpoolConfigV1::stock(HookHostV1::OpenCode), + HookSpoolConfigV1::stock(NativeHostIdentityV1::OpenCode), now, ) .unwrap() @@ -628,7 +629,7 @@ mod tests { ); let (spool, report) = HookSpoolV1::open( spool_root, - HookSpoolConfigV1::stock(HookHostV1::OpenCode), + HookSpoolConfigV1::stock(NativeHostIdentityV1::OpenCode), now, ) .unwrap(); @@ -816,8 +817,12 @@ mod tests { async fn kimi_and_opencode_replay_preserve_native_session_and_provider_order() { let seen = Arc::new(StdMutex::new(Vec::new())); for (host, session, sequence) in [ - (HookHostV1::KimiCode, "session.kimi.replay", 41), - (HookHostV1::OpenCode, "session.opencode.replay", 42), + (NativeHostIdentityV1::KimiCode, "session.kimi.replay", 41), + ( + NativeHostIdentityV1::OpenCode, + "session.opencode.replay", + 42, + ), ] { let mut host_binding = binding(7); host_binding.host = host; @@ -867,10 +872,10 @@ mod tests { } let seen = seen.lock().unwrap(); - assert_eq!(seen[0].0, HookHostV1::KimiCode); + assert_eq!(seen[0].0, NativeHostIdentityV1::KimiCode); assert_eq!(seen[0].1, HookOrderingV1::ProviderSequence(41)); assert_eq!(seen[0].2.as_str(), "session.kimi.replay"); - assert_eq!(seen[1].0, HookHostV1::OpenCode); + assert_eq!(seen[1].0, NativeHostIdentityV1::OpenCode); assert_eq!(seen[1].1, HookOrderingV1::ProviderSequence(42)); assert_eq!(seen[1].2.as_str(), "session.opencode.replay"); } diff --git a/crates/tracedecay-hooks/src/lib.rs b/crates/tracedecay-hooks/src/lib.rs index d709c19b16..c96f120d2a 100644 --- a/crates/tracedecay-hooks/src/lib.rs +++ b/crates/tracedecay-hooks/src/lib.rs @@ -32,7 +32,7 @@ pub use config::{ HookConfigurationSnapshotV1, HookConfigurationSubscriberV1, hook_configuration_path, }; pub use core_events::{ - DaemonHookEvent, HOOK_EVENT_METHOD, HookAgent, HookEventNotifyOutcomeV1, HookRouteMetadata, + DaemonHookEvent, HOOK_EVENT_METHOD, HookEventNotifyOutcomeV1, HookRouteMetadata, HookTerminalReceipt, }; pub use delivery_spool::{ @@ -80,11 +80,6 @@ pub const MAX_REPLAY_BATCH_RECORDS: u16 = 64; pub const MAX_REPLAY_BATCH_BYTES: u32 = 256 * 1024; pub const MAX_SUGGESTION_BYTES: usize = 4 * 1024; -/// Canonical native host identity used by hook decoding, configuration, and -/// persisted spool state. The alias preserves the Hook V2 API name while -/// preventing a second host vocabulary from drifting from the domain catalog. -pub type HookHostV1 = NativeHostIdentityV1; - /// Event families that a host hook itself may emit. #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] @@ -110,40 +105,46 @@ pub enum HookEventSupportV1 { /// Checked-in native host matrix. `Unavailable` is truthful absence and never /// permission to infer an event from command text or another host surface. -pub const fn stock_event_support(host: HookHostV1, family: HookEventFamily) -> HookEventSupportV1 { +pub const fn stock_event_support( + host: NativeHostIdentityV1, + family: HookEventFamily, +) -> HookEventSupportV1 { use HookEventFamily::{ PromptBoundary, SavedEdit, SessionBoundary, TestLifecycle, ToolLifecycle, }; use HookEventSupportV1::{Native, ReceiptDerived, Unavailable}; match (host, family) { - (HookHostV1::ClaudeCode, SessionBoundary | ToolLifecycle) => Native, - (HookHostV1::ClaudeCode, SavedEdit | TestLifecycle) => ReceiptDerived, - (HookHostV1::ClaudeCode, PromptBoundary) => Unavailable, - (HookHostV1::Codex, SessionBoundary | ToolLifecycle) => Native, - (HookHostV1::Codex, SavedEdit | TestLifecycle) => ReceiptDerived, - (HookHostV1::Codex, PromptBoundary) => Unavailable, - (HookHostV1::CursorDesktop, SessionBoundary | SavedEdit) => Native, - (HookHostV1::CursorDesktop, TestLifecycle) => ReceiptDerived, - (HookHostV1::CursorDesktop, PromptBoundary | ToolLifecycle) => Unavailable, + (NativeHostIdentityV1::ClaudeCode, SessionBoundary | ToolLifecycle) => Native, + (NativeHostIdentityV1::ClaudeCode, SavedEdit | TestLifecycle) => ReceiptDerived, + (NativeHostIdentityV1::ClaudeCode, PromptBoundary) => Unavailable, + (NativeHostIdentityV1::Codex, SessionBoundary | ToolLifecycle) => Native, + (NativeHostIdentityV1::Codex, SavedEdit | TestLifecycle) => ReceiptDerived, + (NativeHostIdentityV1::Codex, PromptBoundary) => Unavailable, + (NativeHostIdentityV1::CursorDesktop, SessionBoundary | SavedEdit) => Native, + (NativeHostIdentityV1::CursorDesktop, TestLifecycle) => ReceiptDerived, + (NativeHostIdentityV1::CursorDesktop, PromptBoundary | ToolLifecycle) => Unavailable, ( - HookHostV1::CursorCloud, + NativeHostIdentityV1::CursorCloud, SessionBoundary | PromptBoundary | ToolLifecycle | SavedEdit | TestLifecycle, ) => Unavailable, - (HookHostV1::Hermes, SessionBoundary | ToolLifecycle) => Native, - (HookHostV1::Hermes, SavedEdit | TestLifecycle) => ReceiptDerived, - (HookHostV1::Hermes, PromptBoundary) => Unavailable, - (HookHostV1::Kiro, PromptBoundary) => Native, - (HookHostV1::Kiro, SessionBoundary | ToolLifecycle | SavedEdit | TestLifecycle) => { - Unavailable - } - (HookHostV1::KimiCode, ToolLifecycle | SavedEdit) => Native, - (HookHostV1::KimiCode, SessionBoundary) => Native, - (HookHostV1::KimiCode, PromptBoundary | TestLifecycle) => Unavailable, - (HookHostV1::OpenCode, SessionBoundary | ToolLifecycle | SavedEdit) => Native, - (HookHostV1::OpenCode, PromptBoundary | TestLifecycle) => Unavailable, + (NativeHostIdentityV1::Hermes, SessionBoundary | ToolLifecycle) => Native, + (NativeHostIdentityV1::Hermes, SavedEdit | TestLifecycle) => ReceiptDerived, + (NativeHostIdentityV1::Hermes, PromptBoundary) => Unavailable, + (NativeHostIdentityV1::Kiro, PromptBoundary) => Native, + ( + NativeHostIdentityV1::Kiro, + SessionBoundary | ToolLifecycle | SavedEdit | TestLifecycle, + ) => Unavailable, + (NativeHostIdentityV1::KimiCode, ToolLifecycle | SavedEdit) => Native, + (NativeHostIdentityV1::KimiCode, SessionBoundary) => Native, + (NativeHostIdentityV1::KimiCode, PromptBoundary | TestLifecycle) => Unavailable, + (NativeHostIdentityV1::OpenCode, SessionBoundary | ToolLifecycle | SavedEdit) => Native, + (NativeHostIdentityV1::OpenCode, PromptBoundary | TestLifecycle) => Unavailable, ( - HookHostV1::Cline | HookHostV1::RooCode | HookHostV1::Kilo, + NativeHostIdentityV1::Cline + | NativeHostIdentityV1::RooCode + | NativeHostIdentityV1::Kilo, SessionBoundary | PromptBoundary | ToolLifecycle | SavedEdit | TestLifecycle, ) => Unavailable, } @@ -231,7 +232,7 @@ impl HookEventV2 { pub struct HookEventEnvelopeV2 { pub schema_version: u16, pub event_id: [u8; 16], - pub producer: HookHostV1, + pub producer: NativeHostIdentityV1, pub protected_session_id: [u8; 32], pub project_id: [u8; 16], pub repository_id: [u8; 16], @@ -306,7 +307,7 @@ pub struct HookCapabilityV1 { #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct HookScopeBindingV1 { - pub host: HookHostV1, + pub host: NativeHostIdentityV1, pub project_id: [u8; 16], pub repository_id: [u8; 16], pub worktree_id: [u8; 16], @@ -434,7 +435,7 @@ mod tests { fn binding() -> HookScopeBindingV1 { HookScopeBindingV1 { - host: HookHostV1::CursorDesktop, + host: NativeHostIdentityV1::CursorDesktop, project_id: [1; 16], repository_id: [2; 16], worktree_id: [3; 16], @@ -451,7 +452,7 @@ mod tests { HookEventEnvelopeV2 { schema_version: HOOK_EVENT_SCHEMA_VERSION, event_id: [8; 16], - producer: HookHostV1::CursorDesktop, + producer: NativeHostIdentityV1::CursorDesktop, protected_session_id: [9; 32], project_id: [1; 16], repository_id: [2; 16], @@ -500,49 +501,61 @@ mod tests { #[test] fn host_matrix_matches_checked_in_native_capture_authority() { assert_eq!( - stock_event_support(HookHostV1::Kiro, HookEventFamily::ToolLifecycle), + stock_event_support(NativeHostIdentityV1::Kiro, HookEventFamily::ToolLifecycle), HookEventSupportV1::Unavailable ); assert_eq!( - stock_event_support(HookHostV1::Kiro, HookEventFamily::SavedEdit), + stock_event_support(NativeHostIdentityV1::Kiro, HookEventFamily::SavedEdit), HookEventSupportV1::Unavailable ); assert_eq!( - stock_event_support(HookHostV1::Kiro, HookEventFamily::PromptBoundary), + stock_event_support(NativeHostIdentityV1::Kiro, HookEventFamily::PromptBoundary), HookEventSupportV1::Native, "the checked-in Kiro userPromptSubmit capture proves this native family" ); assert_eq!( - stock_event_support(HookHostV1::KimiCode, HookEventFamily::SessionBoundary), + stock_event_support( + NativeHostIdentityV1::KimiCode, + HookEventFamily::SessionBoundary + ), HookEventSupportV1::Native, "the checked-in Kimi Stop capture proves this native family" ); assert_eq!( - stock_event_support(HookHostV1::CursorCloud, HookEventFamily::SessionBoundary), + stock_event_support( + NativeHostIdentityV1::CursorCloud, + HookEventFamily::SessionBoundary + ), HookEventSupportV1::Unavailable, "Cursor Desktop captures cannot prove a Cursor Cloud callback" ); assert_eq!( - stock_event_support(HookHostV1::Hermes, HookEventFamily::TestLifecycle), + stock_event_support(NativeHostIdentityV1::Hermes, HookEventFamily::TestLifecycle), HookEventSupportV1::ReceiptDerived ); assert_eq!( - stock_event_support(HookHostV1::ClaudeCode, HookEventFamily::ToolLifecycle), + stock_event_support( + NativeHostIdentityV1::ClaudeCode, + HookEventFamily::ToolLifecycle + ), HookEventSupportV1::Native, "the checked-in Claude PostToolUse capture proves this native family" ); assert_eq!( - stock_event_support(HookHostV1::Hermes, HookEventFamily::ToolLifecycle), + stock_event_support(NativeHostIdentityV1::Hermes, HookEventFamily::ToolLifecycle), HookEventSupportV1::Native, "the checked-in Hermes post_tool_call capture proves this native family" ); assert_eq!( - stock_event_support(HookHostV1::Codex, HookEventFamily::ToolLifecycle), + stock_event_support(NativeHostIdentityV1::Codex, HookEventFamily::ToolLifecycle), HookEventSupportV1::Native, "the checked-in Codex PostToolUse capture proves this native family" ); assert_eq!( - stock_event_support(HookHostV1::CursorDesktop, HookEventFamily::SavedEdit), + stock_event_support( + NativeHostIdentityV1::CursorDesktop, + HookEventFamily::SavedEdit + ), HookEventSupportV1::Native, "the checked-in Cursor afterFileEdit capture proves this native family" ); diff --git a/crates/tracedecay-hooks/src/native.rs b/crates/tracedecay-hooks/src/native.rs index 948f761e86..08fb54d69e 100644 --- a/crates/tracedecay-hooks/src/native.rs +++ b/crates/tracedecay-hooks/src/native.rs @@ -61,7 +61,8 @@ impl NativeContextScoutLifecycleV1 { pub fn matches_envelope(&self, envelope: &HookEventEnvelopeV2) -> bool { matches!( envelope.producer, - crate::HookHostV1::KimiCode | crate::HookHostV1::OpenCode + tracedecay_domain::NativeHostIdentityV1::KimiCode + | tracedecay_domain::NativeHostIdentityV1::OpenCode ) && <[u8; 32]>::from(Sha256::digest(self.session_id.as_str().as_bytes())) == envelope.protected_session_id && self.event_id == envelope.event_id diff --git a/crates/tracedecay-hooks/src/spool/checkpoint.rs b/crates/tracedecay-hooks/src/spool/checkpoint.rs index f3ef4adddb..fd31364e68 100644 --- a/crates/tracedecay-hooks/src/spool/checkpoint.rs +++ b/crates/tracedecay-hooks/src/spool/checkpoint.rs @@ -13,9 +13,8 @@ use tracedecay_domain::{ }; use tracedecay_private_fs::framed_log::atomic_write_accelerator; -use crate::{ - HookHostV1, MAX_HOOK_PAYLOAD_BYTES, MAX_SPOOL_BYTES_PER_HOST, MAX_SPOOL_RECORDS_PER_HOST, -}; +use crate::{MAX_HOOK_PAYLOAD_BYTES, MAX_SPOOL_BYTES_PER_HOST, MAX_SPOOL_RECORDS_PER_HOST}; +use tracedecay_domain::NativeHostIdentityV1; use super::{ CHECKPOINT_FILE, CHECKPOINT_FORMAT_VERSION, HookSpoolConfigV1, HookSpoolError, TRANSITION_FILE, @@ -138,7 +137,7 @@ mod revision_time { #[serde(deny_unknown_fields)] struct HookSpoolCheckpointHeaderV1 { version: u16, - host: HookHostV1, + host: NativeHostIdentityV1, records_revision: Option, record_count: u32, } diff --git a/crates/tracedecay-hooks/src/spool/frame.rs b/crates/tracedecay-hooks/src/spool/frame.rs index c7669c5cd5..f897dbf41d 100644 --- a/crates/tracedecay-hooks/src/spool/frame.rs +++ b/crates/tracedecay-hooks/src/spool/frame.rs @@ -6,11 +6,12 @@ use tracedecay_domain::{UtcMicros, framed_log::checksum as frame_checksum}; use tracedecay_private_fs::framed_log::{append_durable, truncate_file as shared_truncate_file}; use crate::{ - HOOK_EVENT_SCHEMA_VERSION, HookEventEnvelopeV2, HookHostV1, MAX_HOOK_PAYLOAD_BYTES, + HOOK_EVENT_SCHEMA_VERSION, HookEventEnvelopeV2, MAX_HOOK_PAYLOAD_BYTES, NativeContextScoutLifecycleV1, }; use serde::{Deserialize, Serialize}; use serde_json::Value; +use tracedecay_domain::NativeHostIdentityV1; use super::types::{HookSpoolRecordV1, PendingRecordV1, ScanResult}; use super::{ @@ -237,7 +238,7 @@ pub(super) fn encode_frame( pub(super) fn decode_complete_frame( frame: &[u8], file_offset: u64, - host: HookHostV1, + host: NativeHostIdentityV1, ) -> Result { let minimum = FRAME_LENGTH_BYTES + FRAME_HEADER_BYTES + FRAME_CHECKSUM_BYTES; if frame.len() < minimum { diff --git a/crates/tracedecay-hooks/src/spool/lease.rs b/crates/tracedecay-hooks/src/spool/lease.rs index f88939aeec..9d0668b488 100644 --- a/crates/tracedecay-hooks/src/spool/lease.rs +++ b/crates/tracedecay-hooks/src/spool/lease.rs @@ -1,10 +1,11 @@ -use std::fs::{File, OpenOptions}; +use std::fs::OpenOptions; use std::path::Path; use std::time::{Duration, Instant}; use crate::lock_admission::{LockAdmissionError, lock_until}; use tracedecay_domain::UtcMicros; +use tracedecay_private_fs::FileLease; use super::types::HookSpoolWriterLeaseV1; use super::{HookSpoolError, HookSpoolV1, lease_path, next_token, validate_regular_or_missing}; @@ -43,7 +44,7 @@ pub(super) fn acquire_lease( root: &Path, lease_duration_micros: i64, now: UtcMicros, -) -> Result<(HookSpoolWriterLeaseV1, File), HookSpoolError> { +) -> Result<(HookSpoolWriterLeaseV1, FileLease), HookSpoolError> { acquire_lease_bounded(root, lease_duration_micros, now, None) } @@ -55,7 +56,7 @@ pub(super) fn acquire_lease_bounded( lease_duration_micros: i64, now: UtcMicros, wait_budget: Option, -) -> Result<(HookSpoolWriterLeaseV1, File), HookSpoolError> { +) -> Result<(HookSpoolWriterLeaseV1, FileLease), HookSpoolError> { let expires_at = UtcMicros( now.0 .checked_add(lease_duration_micros) @@ -93,7 +94,7 @@ pub(super) fn acquire_lease_bounded( // persisting advisory ownership would add a durability barrier without // strengthening exclusion, while records, metadata and replay cursors keep // their independent fsync-before-return contracts. - Ok((candidate, file)) + Ok((candidate, FileLease::held(file, "hooks.spool.writer"))) } pub(super) fn map_try_lock_error(error: std::fs::TryLockError) -> HookSpoolError { diff --git a/crates/tracedecay-hooks/src/spool/meta.rs b/crates/tracedecay-hooks/src/spool/meta.rs index d30bc0fba4..cff3a8b1be 100644 --- a/crates/tracedecay-hooks/src/spool/meta.rs +++ b/crates/tracedecay-hooks/src/spool/meta.rs @@ -5,8 +5,8 @@ use tracedecay_domain::framed_log::checksum as frame_checksum; use tracedecay_domain::framed_log::partial_tail_matches_prefix; use tracedecay_private_fs::framed_log::atomic_write as shared_atomic_write; -use crate::HookHostV1; use serde_json::Value; +use tracedecay_domain::NativeHostIdentityV1; use super::frame::decode_complete_frame; use super::types::{ @@ -89,7 +89,7 @@ pub(super) fn partial_tail_matches_intent( pub(super) fn reconcile_append_intent( meta: &mut HookSpoolMetaV1, records: &[PendingRecordV1], - host: HookHostV1, + host: NativeHostIdentityV1, ) -> Result<(), HookSpoolError> { let Some(intent) = meta.append_intent.clone() else { return Ok(()); @@ -118,7 +118,7 @@ pub(super) fn reconcile_append_intent( pub(super) fn validate_meta( meta: &HookSpoolMetaV1, limits: HookSpoolLimitsV1, - host: HookHostV1, + host: NativeHostIdentityV1, ) -> Result<(), HookSpoolError> { if meta.next_sequence == 0 || meta.next_sequence <= meta.committed_through @@ -142,7 +142,7 @@ pub(super) fn validate_meta( Ok(()) } -pub(super) fn valid_append_intent(intent: &AppendIntentV1, host: HookHostV1) -> bool { +pub(super) fn valid_append_intent(intent: &AppendIntentV1, host: NativeHostIdentityV1) -> bool { let minimum = FRAME_LENGTH_BYTES + FRAME_HEADER_BYTES + FRAME_CHECKSUM_BYTES; if intent.sequence == 0 || intent.frame.len() < minimum diff --git a/crates/tracedecay-hooks/src/spool/mod.rs b/crates/tracedecay-hooks/src/spool/mod.rs index 637604cfcf..09058dd215 100644 --- a/crates/tracedecay-hooks/src/spool/mod.rs +++ b/crates/tracedecay-hooks/src/spool/mod.rs @@ -17,6 +17,7 @@ use tracedecay_domain::{ UtcMicros, framed_log::{self, checksum as frame_checksum}, }; +use tracedecay_private_fs::FileLease; use tracedecay_private_fs::framed_log::{ DirectorySyncPolicy, atomic_write as shared_atomic_write, read_bounded as shared_read_bounded, sync_directory as shared_sync_directory, @@ -92,7 +93,7 @@ pub struct HookSpoolV1 { root: PathBuf, config: HookSpoolConfigV1, lease: HookSpoolWriterLeaseV1, - lease_file: File, + _lease_file: FileLease, meta: HookSpoolMetaV1, checkpoint: Option, observed_records_revision: Option, @@ -187,7 +188,7 @@ impl HookSpoolV1 { hotpath::measure_block!("hooks.spool.fsync.directory", { shared_sync_directory(&root, DIRECTORY_POLICY).map_err(|_| HookSpoolError::Io) })?; - lease_file.unlock().map_err(|_| HookSpoolError::Io)?; + lease_file.release().map_err(|_| HookSpoolError::Io)?; Ok(()) } @@ -237,7 +238,7 @@ impl HookSpoolV1 { root: PathBuf, config: HookSpoolConfigV1, lease: HookSpoolWriterLeaseV1, - lease_file: File, + lease_file: FileLease, _now: UtcMicros, ) -> Result<(Self, HookSpoolOpenReportV1), HookSpoolError> { let stored_meta = read_meta(&root)?; @@ -393,7 +394,7 @@ impl HookSpoolV1 { root, config, lease, - lease_file, + _lease_file: lease_file, meta, checkpoint, observed_records_revision, @@ -1012,12 +1013,6 @@ impl HookSpoolV1 { } } -impl Drop for HookSpoolV1 { - fn drop(&mut self) { - let _ = self.lease_file.unlock(); - } -} - fn records_path(root: &Path) -> PathBuf { root.join(RECORDS_FILE) } @@ -1047,13 +1042,9 @@ fn ensure_root(root: &Path) -> Result<(), HookSpoolError> { Ok(metadata) if metadata.file_type().is_symlink() || !metadata.is_dir() => { return Err(HookSpoolError::UnsafePath); } - Ok(_) => { - // An existing root must end private to the current owner. Foreign - // ownership stays UnsafePath; an owned but permissive directory - // (template copies under a group umask, legacy layouts) is healed - // through the same authority Hook configuration publication uses. - return ensure_existing_private_root(root); - } + // An existing root must already be private to the current owner; a + // permissive or foreign-owned one is refused, never re-permissioned. + Ok(_) => return ensure_existing_private_root(root), Err(error) if error.kind() == io::ErrorKind::NotFound => {} Err(_) => return Err(HookSpoolError::Io), } @@ -1063,7 +1054,7 @@ fn ensure_root(root: &Path) -> Result<(), HookSpoolError> { match tracedecay_private_fs::create_private_directory(root) { Ok(()) => {} // A concurrent opener may win the creation race; the directory is - // acceptable only if it is (or can be healed to) private. + // acceptable only if it is private. Err(error) if error.kind() == io::ErrorKind::AlreadyExists => { ensure_existing_private_root(root)?; } @@ -1077,17 +1068,12 @@ fn ensure_root(root: &Path) -> Result<(), HookSpoolError> { fn ensure_existing_private_root(root: &Path) -> Result<(), HookSpoolError> { match tracedecay_private_fs::validate_private_directory(root) { Ok(()) => Ok(()), - Err(error) if error.kind() == io::ErrorKind::PermissionDenied => { - tracedecay_private_fs::make_private_directory(root) - .map(|_| ()) - .map_err(|heal_error| match heal_error.kind() { - io::ErrorKind::PermissionDenied | io::ErrorKind::InvalidInput => { - HookSpoolError::UnsafePath - } - _ => HookSpoolError::Io, - }) - } - Err(error) if error.kind() == io::ErrorKind::InvalidInput => { + Err(error) + if matches!( + error.kind(), + io::ErrorKind::PermissionDenied | io::ErrorKind::InvalidInput + ) => + { Err(HookSpoolError::UnsafePath) } Err(_) => Err(HookSpoolError::Io), diff --git a/crates/tracedecay-hooks/src/spool/tests.rs b/crates/tracedecay-hooks/src/spool/tests.rs index 497acadf69..31edaab16a 100644 --- a/crates/tracedecay-hooks/src/spool/tests.rs +++ b/crates/tracedecay-hooks/src/spool/tests.rs @@ -4,9 +4,8 @@ use std::process::Command; use sha2::{Digest, Sha256}; use super::*; -use crate::{ - HookCapabilityV1, HookEventFamily, HookEventSupportV1, HookEventV2, HookHostV1, HookOrderingV1, -}; +use crate::{HookCapabilityV1, HookEventFamily, HookEventSupportV1, HookEventV2, HookOrderingV1}; +use tracedecay_domain::NativeHostIdentityV1; struct TestDir(PathBuf); @@ -33,7 +32,7 @@ impl Drop for TestDir { fn config() -> HookSpoolConfigV1 { HookSpoolConfigV1 { - host: HookHostV1::CursorDesktop, + host: NativeHostIdentityV1::CursorDesktop, limits: HookSpoolLimitsV1 { max_host_records: 8, max_host_bytes: 32 * 1024, @@ -46,7 +45,7 @@ fn config() -> HookSpoolConfigV1 { fn binding() -> HookScopeBindingV1 { HookScopeBindingV1 { - host: HookHostV1::CursorDesktop, + host: NativeHostIdentityV1::CursorDesktop, project_id: [1; 16], repository_id: [2; 16], worktree_id: [3; 16], @@ -69,7 +68,7 @@ fn envelope(event: u8, session: u8) -> HookEventEnvelopeV2 { HookEventEnvelopeV2 { schema_version: crate::HOOK_EVENT_SCHEMA_VERSION, event_id: [event; 16], - producer: HookHostV1::CursorDesktop, + producer: NativeHostIdentityV1::CursorDesktop, protected_session_id: [session; 32], project_id: [1; 16], repository_id: [2; 16], @@ -122,7 +121,7 @@ fn lifecycle_envelope(event: u32, session: u8) -> HookEventEnvelopeV2 { } fn binding_for_envelopes( - host: HookHostV1, + host: NativeHostIdentityV1, envelopes: &[HookEventEnvelopeV2], ) -> HookScopeBindingV1 { let mut binding = binding(); @@ -183,21 +182,26 @@ fn checksum_is_real_sha256() { ); } -/// An owned but group-writable spool root is healed to owner-private on open. -/// Foreign-owned permissive directories still fail closed inside -/// `make_private_directory`; per-file modes cannot protect members while the -/// directory itself stays group-writable. +/// A group-writable existing spool root fails closed on open and keeps its +/// mode: per-file modes cannot protect members while the directory itself +/// stays group-writable, and the spool never re-permissions a root it did not +/// create. #[cfg(unix)] #[test] -fn open_heals_an_owned_group_writable_existing_root() { +fn open_refuses_a_group_writable_existing_root_without_rewriting_it() { use std::os::unix::fs::PermissionsExt; let root = TestDir::new("permissive-root"); fs::set_permissions(&root.0, fs::Permissions::from_mode(0o770)).unwrap(); - let (spool, _) = HookSpoolV1::open(&root.0, config(), UtcMicros(10)).expect("heal and open"); - drop(spool); + assert!(matches!( + HookSpoolV1::open(&root.0, config(), UtcMicros(10)), + Err(HookSpoolError::UnsafePath) + )); let mode = fs::metadata(&root.0).unwrap().permissions().mode() & 0o777; - assert_eq!(mode, 0o700, "owned permissive roots must be tightened"); + assert_eq!( + mode, 0o770, + "a refused root keeps the mode it was found with" + ); } #[test] @@ -484,7 +488,7 @@ fn checkpoint_rewrites_are_amortized_across_dispatches() { const DISPATCHES: u32 = 24; let root = TestDir::new("checkpoint-amortized"); - let config = HookSpoolConfigV1::stock(HookHostV1::CursorDesktop); + let config = HookSpoolConfigV1::stock(NativeHostIdentityV1::CursorDesktop); let (mut spool, _) = HookSpoolV1::open(&root.0, config, UtcMicros(10)).unwrap(); for event in 1..=INITIAL_RECORDS { spool @@ -539,19 +543,19 @@ fn checkpoint_stores_a_fixed_width_index_without_envelopes() { let compact_root = TestDir::new("checkpoint-fixed-compact"); let large_root = TestDir::new("checkpoint-fixed-large"); - let compact_config = HookSpoolConfigV1::stock(HookHostV1::Hermes); - let large_config = HookSpoolConfigV1::stock(HookHostV1::Hermes); + let compact_config = HookSpoolConfigV1::stock(NativeHostIdentityV1::Hermes); + let large_config = HookSpoolConfigV1::stock(NativeHostIdentityV1::Hermes); let compact = (1..=RECORDS) .map(|event| { let mut envelope = numbered_envelope(event, 9); - envelope.producer = HookHostV1::Hermes; + envelope.producer = NativeHostIdentityV1::Hermes; envelope }) .collect::>(); let large = (1..=RECORDS) .map(|event| { let mut envelope = lifecycle_envelope(event, 9); - envelope.producer = HookHostV1::Hermes; + envelope.producer = NativeHostIdentityV1::Hermes; envelope }) .collect::>(); @@ -606,7 +610,7 @@ fn open_after_one_dispatch_decodes_only_the_appended_suffix() { const DISPATCHES: u32 = 5; let root = TestDir::new("checkpoint-suffix-materialization"); - let config = HookSpoolConfigV1::stock(HookHostV1::CursorDesktop); + let config = HookSpoolConfigV1::stock(NativeHostIdentityV1::CursorDesktop); let initial = (1..=INITIAL_RECORDS) .map(|event| numbered_envelope(event, (event % 251) as u8 + 1)) .collect::>(); @@ -645,7 +649,7 @@ fn open_after_one_dispatch_decodes_only_the_appended_suffix() { #[test] fn hydration_detects_a_corrupted_checkpointed_frame() { let root = TestDir::new("checkpoint-hydration-corruption"); - let config = HookSpoolConfigV1::stock(HookHostV1::CursorDesktop); + let config = HookSpoolConfigV1::stock(NativeHostIdentityV1::CursorDesktop); let first = numbered_envelope(1, 9); publish_checkpoint(&root.0, config, &[first.clone(), numbered_envelope(2, 9)]); let (mut spool, report) = HookSpoolV1::open(&root.0, config, UtcMicros(12)).unwrap(); @@ -684,7 +688,7 @@ fn hydration_detects_a_corrupted_checkpointed_frame() { #[test] fn hydration_rejects_a_wrong_checkpoint_index_without_poisoning_records() { let root = TestDir::new("checkpoint-hydration-index-mismatch"); - let config = HookSpoolConfigV1::stock(HookHostV1::CursorDesktop); + let config = HookSpoolConfigV1::stock(NativeHostIdentityV1::CursorDesktop); let first = numbered_envelope(1, 9); let envelopes = [first.clone(), numbered_envelope(2, 10)]; publish_checkpoint(&root.0, config, &envelopes); @@ -724,7 +728,7 @@ fn hydration_rejects_a_wrong_checkpoint_index_without_poisoning_records() { #[test] fn checkpoint_body_version_one_is_rejected_and_rewritten() { let root = TestDir::new("checkpoint-old-body"); - let config = HookSpoolConfigV1::stock(HookHostV1::CursorDesktop); + let config = HookSpoolConfigV1::stock(NativeHostIdentityV1::CursorDesktop); let envelopes = [numbered_envelope(1, 9), numbered_envelope(2, 9)]; publish_checkpoint(&root.0, config, &envelopes); @@ -750,7 +754,7 @@ fn checkpoint_body_version_one_is_rejected_and_rewritten() { #[test] fn compaction_copies_surviving_checkpointed_frames_byte_exactly() { let root = TestDir::new("checkpoint-compaction-bytes"); - let config = HookSpoolConfigV1::stock(HookHostV1::CursorDesktop); + let config = HookSpoolConfigV1::stock(NativeHostIdentityV1::CursorDesktop); let envelopes = [ numbered_envelope(1, 9), numbered_envelope(2, 10), @@ -801,7 +805,7 @@ fn compaction_copies_surviving_checkpointed_frames_byte_exactly() { #[test] fn replay_hydrates_only_checkpointed_records_in_the_batch() { let root = TestDir::new("checkpoint-replay-hydration"); - let config = HookSpoolConfigV1::stock(HookHostV1::CursorDesktop); + let config = HookSpoolConfigV1::stock(NativeHostIdentityV1::CursorDesktop); let envelopes = (1..=6) .map(|event| numbered_envelope(event, if event <= 3 { 9 } else { 10 })) .collect::>(); @@ -826,9 +830,9 @@ fn replay_hydrates_only_checkpointed_records_in_the_batch() { #[test] fn checkpointed_replay_hydrates_native_lifecycle() { let root = TestDir::new("checkpoint-native-lifecycle"); - let config = HookSpoolConfigV1::stock(HookHostV1::OpenCode); + let config = HookSpoolConfigV1::stock(NativeHostIdentityV1::OpenCode); let mut envelope = numbered_envelope(1, 9); - envelope.producer = HookHostV1::OpenCode; + envelope.producer = NativeHostIdentityV1::OpenCode; envelope.protected_session_id = Sha256::digest(b"session.native.checkpoint").into(); envelope.event = HookEventV2::ToolLifecycle { tool_id: [8; 16], @@ -866,7 +870,7 @@ fn checkpointed_replay_hydrates_native_lifecycle() { #[test] fn transition_extended_anchor_detects_corrupted_prefix() { let root = TestDir::new("checkpoint-extended-corrupt-prefix"); - let config = HookSpoolConfigV1::stock(HookHostV1::CursorDesktop); + let config = HookSpoolConfigV1::stock(NativeHostIdentityV1::CursorDesktop); let (mut spool, _) = HookSpoolV1::open(&root.0, config, UtcMicros(10)).unwrap(); for event in 1..=CHECKPOINT_REWRITE_FRAME_THRESHOLD { spool diff --git a/crates/tracedecay-hooks/src/spool/types.rs b/crates/tracedecay-hooks/src/spool/types.rs index 99600b75bb..2db9c48174 100644 --- a/crates/tracedecay-hooks/src/spool/types.rs +++ b/crates/tracedecay-hooks/src/spool/types.rs @@ -3,10 +3,10 @@ use thiserror::Error; use tracedecay_domain::UtcMicros; use crate::{ - HookContractError, HookEventEnvelopeV2, HookHostV1, MAX_SPOOL_BYTES_PER_HOST, - MAX_SPOOL_BYTES_PER_SESSION, MAX_SPOOL_RECORDS_PER_HOST, MAX_SPOOL_RECORDS_PER_SESSION, - NativeContextScoutLifecycleV1, + HookContractError, HookEventEnvelopeV2, MAX_SPOOL_BYTES_PER_HOST, MAX_SPOOL_BYTES_PER_SESSION, + MAX_SPOOL_RECORDS_PER_HOST, MAX_SPOOL_RECORDS_PER_SESSION, NativeContextScoutLifecycleV1, }; +use tracedecay_domain::NativeHostIdentityV1; use super::{FRAME_CHECKSUM_BYTES, FRAME_HEADER_BYTES, FRAME_LENGTH_BYTES}; @@ -54,13 +54,13 @@ impl HookSpoolLimitsV1 { #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct HookSpoolConfigV1 { - pub host: HookHostV1, + pub host: NativeHostIdentityV1, pub limits: HookSpoolLimitsV1, pub writer_lease_micros: i64, } impl HookSpoolConfigV1 { - pub const fn stock(host: HookHostV1) -> Self { + pub const fn stock(host: NativeHostIdentityV1) -> Self { Self { host, limits: HookSpoolLimitsV1::stock(), diff --git a/crates/tracedecay-host-admission/src/host_admission_batch_test.rs b/crates/tracedecay-host-admission/src/host_admission_batch_test.rs index 66f9b821c8..2487a5699e 100644 --- a/crates/tracedecay-host-admission/src/host_admission_batch_test.rs +++ b/crates/tracedecay-host-admission/src/host_admission_batch_test.rs @@ -16,12 +16,12 @@ use tracedecay_domain::{ SensitivityV1, SessionId, UtcMicros, }; use tracedecay_global_db::tests::harness::HostAdmissionTestRuntimeV1; -use tracedecay_privacy::{ClaudeRecordParseErrorV1, parse_normalized_observation_record_v1}; +use tracedecay_privacy::{ObservationRecordParseErrorV1, parse_normalized_observation_record_v1}; use tracedecay_runtime_core::background_cpu::ProcessBackgroundCpuV1; use tracedecay_sessions::admission::{HostAdmission, HostAdmissionScope}; use tracedecay_store::{ AnchoredObservationWrite, ObservationPersistOutcome, ObservationWrite, - build_observation_resolution_authorization_v1, build_observation_retrieval_anchor_v2, + build_observation_resolution_authorization_v1, build_observation_retrieval_anchor, }; use super::*; @@ -116,7 +116,7 @@ fn sequential_capture_requests( }], CanonicalObservationEvidenceV1::new(ordering_domain, range), ) - .map_err(|_| ClaudeRecordParseErrorV1::NormalizationFailed) + .map_err(|_| ObservationRecordParseErrorV1::NormalizationFailed) }, ) .unwrap(); @@ -314,7 +314,7 @@ async fn canonical_message_projection_succeeds_while_git_graph_is_unavailable() ) .with_native_timestamp(1_785_000_000), ) - .map_err(|_| ClaudeRecordParseErrorV1::NormalizationFailed) + .map_err(|_| ObservationRecordParseErrorV1::NormalizationFailed) } }, ) @@ -487,7 +487,7 @@ fn anchored_write( let authorization = build_observation_resolution_authorization_v1(write.observation(), "host-admission-batch") .unwrap(); - let anchor = build_observation_retrieval_anchor_v2( + let anchor = build_observation_retrieval_anchor( write.observation(), projection_generation.clone(), UtcMicros(1), diff --git a/crates/tracedecay-host-admission/src/lib.rs b/crates/tracedecay-host-admission/src/lib.rs index 0e90fb01ac..a72022b3d8 100644 --- a/crates/tracedecay-host-admission/src/lib.rs +++ b/crates/tracedecay-host-admission/src/lib.rs @@ -1109,7 +1109,7 @@ fn classify_git_evidence_error( fn accepted_for_external_source_replay( outcome: CaptureObservationOutcome, - receipt: tracedecay_store::SourceCommitReceiptV1, + receipt: tracedecay_store::SourceCommitReceiptSummaryV1, ) -> Result { let CaptureObservationOutcome::Persisted { outcome, @@ -1124,7 +1124,7 @@ fn accepted_for_external_source_replay( }; let durable_observation_id = outcome.receipt().observation().observation_id().clone(); let retry_handle = ExternalSourceProjectionRetryHandleV1::new( - receipt.source_frontier().binding().clone(), + receipt.binding().clone(), receipt.receipt_digest().clone(), ); Ok(CaptureObservationOutcome::AcceptedForReplay { @@ -1140,9 +1140,6 @@ fn accepted_for_external_source_replay( fn classify_store_error(error: &ObservationStoreError) -> HostAdmissionOutcome { let reason_code = match error { - ObservationStoreError::BatchRequiresScalarFallback { cause } => { - return HostAdmissionOutcome::batch_requires_scalar_fallback(*cause); - } ObservationStoreError::ObservationCollision { .. } => { return HostAdmissionOutcome::deterministic_content_refusal( "observation_identity_collision", diff --git a/crates/tracedecay-host-admission/src/projection_drain.rs b/crates/tracedecay-host-admission/src/projection_drain.rs index c59f2000de..82c8945de7 100644 --- a/crates/tracedecay-host-admission/src/projection_drain.rs +++ b/crates/tracedecay-host-admission/src/projection_drain.rs @@ -342,7 +342,6 @@ mod tests { ..Default::default() }, backfill_page_saturated: false, - reprojected_legacy_head: false, }, ); @@ -357,7 +356,6 @@ mod tests { ..Default::default() }, backfill_page_saturated: false, - reprojected_legacy_head: false, }, ); assert!(git_evidence_convergence_deferred(&transient)); diff --git a/crates/tracedecay-host-admission/src/spool/tests.rs b/crates/tracedecay-host-admission/src/spool/tests.rs index 1adfbc3268..b598f08f6d 100644 --- a/crates/tracedecay-host-admission/src/spool/tests.rs +++ b/crates/tracedecay-host-admission/src/spool/tests.rs @@ -31,7 +31,6 @@ fn write_frames(path: &Path, sequences: &[u64]) { #[test] fn frame_encoding_is_deterministic_and_checksummed() { let frame = encode_frame(7, b"cursor", b"{\"event\":1}").unwrap(); - assert_eq!(frame, encode_frame(7, b"cursor", b"{\"event\":1}").unwrap()); assert_eq!(&frame[0..4], FRAME_MAGIC); let checksum_at = frame.len() - CHECKSUM_BYTES; assert_eq!( @@ -40,14 +39,6 @@ fn frame_encoding_is_deterministic_and_checksummed() { ); } -#[test] -fn production_defaults_reserve_capacity_across_sources() { - let bounds = SpoolBounds::default(); - assert!(bounds.max_records_per_source < bounds.max_records); - assert!(bounds.max_spool_bytes_per_source < bounds.max_spool_bytes); - assert!(bounds.max_record_bytes <= bounds.max_spool_bytes_per_source); -} - #[test] fn append_reopen_ack_and_reopen_are_exact() { let (temp, mut spool) = open_temp(); diff --git a/crates/tracedecay-host-admission/tests/host_capture_background_cpu_admission.rs b/crates/tracedecay-host-admission/tests/host_capture_background_cpu_admission.rs index 9c872141f9..e18a04a6ce 100644 --- a/crates/tracedecay-host-admission/tests/host_capture_background_cpu_admission.rs +++ b/crates/tracedecay-host-admission/tests/host_capture_background_cpu_admission.rs @@ -30,7 +30,7 @@ use tracedecay_domain::{ }; use tracedecay_global_db::tests::harness::HostAdmissionTestRuntimeV1; use tracedecay_host_admission::{HostAdmissionAuthorities, HostAdmissionFacade}; -use tracedecay_privacy::{ClaudeRecordParseErrorV1, parse_normalized_observation_record_v1}; +use tracedecay_privacy::{ObservationRecordParseErrorV1, parse_normalized_observation_record_v1}; use tracedecay_runtime_core::background_cpu::ProcessBackgroundCpuV1; use tracedecay_sessions::admission::{HostAdmissionScope, HostAdmissionStatus}; use tracedecay_sessions::observation::{ @@ -78,7 +78,7 @@ fn capture_requests(session_id: &SessionId, count: usize) -> Vec, - pub rollback_boundary: HostBundleRollbackBoundaryV1, - #[serde(default)] - pub rollback_history: Vec<[u8; 16]>, -} - -/// Durable, content-free inventory for an operator-requested host-component -/// backup. Artifact bytes live in the lifecycle directory; the receipt binds -/// their exact digests and the manifest needed to restore them. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct HostBundleBackupReceiptV1 { - pub schema_version: u16, - pub operation_id: [u8; 16], - pub host: HostKindV1, - pub component: HostComponentV1, - pub manifest: HostBundleManifestV1, - pub source_receipt_digest: [u8; 32], - pub artifacts: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct HostBundleBackupArtifactV1 { - pub relative_path: String, - pub artifact_digest: [u8; 32], - pub ownership_marker: String, - pub snapshot_name: String, -} - -/// Durable proof that a named backup was restored through the rollback-safe -/// lifecycle writer. The embedded install receipt remains the ownership -/// authority for the restored component. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct HostBundleRestoreReceiptV1 { - pub schema_version: u16, - pub operation_id: [u8; 16], - pub backup_operation_id: [u8; 16], - pub restored_receipt: HostBundleInstallReceiptV1, -} - -/// Durable aggregate commit marker for a complete host component set. The root -/// adapter owns the aggregate transaction; this contract binds its receipts. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct HostComponentSetReceiptV1 { - pub schema_version: u16, - pub operation_id: [u8; 16], - pub host: HostKindV1, - pub operation: HostBundleLifecycleOpV1, - pub component_manifests: Vec, - pub component_receipts: Vec, - #[serde(default)] - pub confirmed_plan_digest: Option<[u8; 32]>, - #[serde(default)] - pub base_registration_revision: Option<[u8; 32]>, - #[serde(default)] - pub current_registration_revision: Option<[u8; 32]>, - #[serde(default)] - pub artifact_state_revision: Option<[u8; 32]>, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum HostBundleRollbackBoundaryV1 { - Pending, - Passed, -} - -/// Serialized single-component recovery state. Root adapters own opening, -/// writing, and recovering this journal; this crate owns its stable schema. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum HostBundleJournalStateV1 { - Prepared, - Committed, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct HostBundleJournalEntryV1 { - pub relative_path: String, - pub backup_name: Option, - pub backup_created: bool, - pub wrote_new: bool, - pub installed_digest: Option<[u8; 32]>, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct HostBundleJournalV1 { - pub schema_version: u16, - pub operation_id: [u8; 16], - pub host: HostKindV1, - pub component: HostComponentV1, - pub operation: HostBundleLifecycleOpV1, - pub manifest_digest: [u8; 32], - pub state: HostBundleJournalStateV1, - pub previous_receipt: Option, - pub entries: Vec, -} - -/// Serialized aggregate recovery state. Its filesystem lifecycle remains a -/// root adapter responsibility, so this is intentionally only data. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum HostComponentSetJournalStateV1 { - Prepared, - Staged, - Applied, - Verified, - Committed, - RolledBack, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct HostComponentSetJournalComponentV1 { - pub manifest: HostBundleManifestV1, - pub previous_receipt: Option, - pub entries: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct HostComponentSetJournalV1 { - pub schema_version: u16, - pub operation_id: [u8; 16], - pub host: HostKindV1, - pub operation: HostBundleLifecycleOpV1, - /// Exact operator authority admitted before any lifecycle mutation. - /// - /// Recovery must replay this value rather than manufacturing confirmation. - #[serde(default)] - pub explicit_confirmation: bool, - /// Exact Hermes profile binding admitted with the original request. - #[serde(default)] - pub hermes_profile_bindings: u8, - /// Canonical configuration/runtime preview authority, when the operation - /// was applied through confirmed preview. - #[serde(default)] - pub confirmed_plan_digest: Option<[u8; 32]>, - #[serde(default)] - pub base_registration_revision: Option<[u8; 32]>, - #[serde(default)] - pub current_registration_revision: Option<[u8; 32]>, - #[serde(default)] - pub artifact_state_revision: Option<[u8; 32]>, - pub state: HostComponentSetJournalStateV1, - pub registration_staged: bool, - pub registration_applied: bool, - pub components: Vec, -} - -impl HostComponentSetJournalV1 { - /// Whether the recorded phase and the two registration flags describe a - /// combination a writer can actually produce. - /// - /// The writer raises each flag immediately *before* invoking the - /// registration hook it names and advances `state` only *after* that hook - /// returns, so every phase implies the flags of the phases behind it: - /// - /// - `Prepared` precedes `registration.apply`, so `registration_applied` - /// can never be set there. - /// - `Staged` and later are reached only after `registration.stage` was - /// invoked, which requires `registration_staged`. - /// - `Applied` and later are reached only after `registration.apply` was - /// invoked, which requires `registration_applied`. - /// - `registration_applied` is never raised without `registration_staged`. - /// - /// `RolledBack` is deliberately unconstrained: rollback preserves whichever - /// flags the failed attempt had reached, so every combination is authentic - /// there. Recovery must therefore not read the flags as proof that a - /// rolled-back journal needs no compensation - see - /// [`Self::registration_compensation_required`]. - #[must_use] - pub fn registration_flags_match_state(&self) -> bool { - let staged_required = matches!( - self.state, - HostComponentSetJournalStateV1::Staged - | HostComponentSetJournalStateV1::Applied - | HostComponentSetJournalStateV1::Verified - | HostComponentSetJournalStateV1::Committed - ); - let applied_required = matches!( - self.state, - HostComponentSetJournalStateV1::Applied - | HostComponentSetJournalStateV1::Verified - | HostComponentSetJournalStateV1::Committed - ); - if self.registration_applied && !self.registration_staged { - return false; - } - if staged_required && !self.registration_staged { - return false; - } - if applied_required && !self.registration_applied { - return false; - } - !(self.state == HostComponentSetJournalStateV1::Prepared && self.registration_applied) - } - - /// Whether recovery must attempt host-native registration compensation. - /// - /// Only a `Prepared` journal proves registration was never entered: its - /// flags are raised before the hooks they name, so `Prepared` with both - /// flags clear is the single state where skipping compensation is sound. - /// Every other phase - including `RolledBack`, whose flags describe the - /// interrupted attempt rather than the work still outstanding - must - /// re-attempt rollback. That re-attempt is already load-bearing today, - /// because a crash between `rollback_component_set` and journal cleanup - /// replays the same compensation; the registration adapter contract is - /// idempotent and no-ops when it finds no staged backup. - #[must_use] - pub fn registration_compensation_required(&self) -> bool { - self.registration_staged - || self.registration_applied - || self.state != HostComponentSetJournalStateV1::Prepared - } -} diff --git a/crates/tracedecay-host-integration/src/lib.rs b/crates/tracedecay-host-integration/src/lib.rs index 36723a3c49..4c9b4726ff 100644 --- a/crates/tracedecay-host-integration/src/lib.rs +++ b/crates/tracedecay-host-integration/src/lib.rs @@ -2,7 +2,7 @@ //! //! The application binary composes its checked-in plugin assets with //! `include_bytes!` / `include_str!`, then passes the resulting evidence here. -//! This crate owns immutable manifest, receipt, journal, and capability-evidence +//! This crate owns immutable manifest, receipt, and capability-evidence //! contracts; root adapters retain CLI dispatch and filesystem mutation. use thiserror::Error; @@ -13,8 +13,8 @@ pub use tracedecay_domain::{ }; mod evidence; -mod journal; mod manifest; +mod receipt; #[cfg(test)] pub(crate) use evidence::HOST_REGISTRATIONS; @@ -28,19 +28,16 @@ pub use evidence::{ native_host_edit_stop_conformance_evidence_from_embedded_assets, stock_host_native_fixture_evidence_from_embedded_assets, stock_host_registration_evidence, }; -pub use journal::{ - HOST_BUNDLE_RECEIPT_SCHEMA_VERSION, HostBundleBackupArtifactV1, HostBundleBackupReceiptV1, - HostBundleInstallReceiptV1, HostBundleJournalEntryV1, HostBundleJournalStateV1, - HostBundleJournalV1, HostBundleReceiptArtifactV1, HostBundleRestoreReceiptV1, - HostBundleRollbackBoundaryV1, HostComponentSetJournalComponentV1, - HostComponentSetJournalStateV1, HostComponentSetJournalV1, HostComponentSetReceiptV1, -}; pub use manifest::{ HOST_BUNDLE_SCHEMA_VERSION, HostBundleArtifactContentV1, HostBundleArtifactV1, HostBundleLifecycleOpV1, HostBundleManifestV1, HostBundleVerificationAdapterV1, MAX_ARTIFACT_CONTENT_BYTES, MAX_HOST_COMPONENTS, MAX_IDENTIFIER_BYTES, MAX_MANIFEST_ARTIFACTS, MAX_RELATIVE_PATH_BYTES, validate_identifier, validate_relative_install_path, }; +pub use receipt::{ + HOST_BUNDLE_RECEIPT_SCHEMA_VERSION, HostBundleInstallReceiptV1, HostBundleReceiptArtifactV1, + HostComponentSetReceiptV1, +}; /// Builds a [`HostBundleError::StorageFailure`] tagged with the `file:line` of /// the site that observed the failure. @@ -60,25 +57,6 @@ macro_rules! host_bundle_storage_failure { }; } -/// Builds a [`HostBundleError::RecoveryRequired`] tagged with the `file:line` of -/// the site that refused to mutate. -/// -/// Dozens of journal, receipt, and rollback probes all fail closed with -/// `RecoveryRequired`. Without a per-site tag, an operator staring at "requires -/// recovery before mutation" cannot tell an genuinely interrupted operation from -/// a probe that misread clean state. Always construct the variant through this -/// macro. -#[macro_export] -macro_rules! host_bundle_recovery_required { - () => { - $crate::HostBundleError::RecoveryRequired(::core::concat!( - ::core::file!(), - ":", - ::core::line!() - )) - }; -} - /// Builds a [`HostBundleError::StalePreview`] tagged with the `file:line` of the /// site that observed the drift. /// @@ -139,28 +117,25 @@ pub enum HostBundleError { InvalidHermesProfileBinding, #[error("bundle artifact content is missing, oversized, duplicated, or digest-mismatched")] ArtifactContentMismatch, - #[error("host bundle receipt or operation journal is invalid")] + #[error("host bundle receipt is invalid")] ReceiptCorrupted, + /// A receipt was written with an older receipt schema. Nothing migrates + /// it: the operator reinstalls, which discards the stale receipts. + #[error( + "host bundle receipts were written by an older TraceDecay; run `tracedecay install --yes --adopt` to replace them" + )] + ReinstallRequired, /// An atomic filesystem step failed. The payload names the source site that /// observed the failure so the ~100 construction sites stay distinguishable /// in user-facing output and bug reports; build it with /// [`host_bundle_storage_failure!`] rather than by hand. #[error("host bundle atomic filesystem operation failed at {0}")] StorageFailure(&'static str), - /// A mutation refused because an earlier operation looks interrupted. The - /// payload names the probe that refused, so a false positive on clean state - /// is distinguishable from a genuine interrupted operation; build it with - /// [`host_bundle_recovery_required!`] rather than by hand. - #[error("host bundle interrupted operation requires recovery before mutation (at {0})")] - RecoveryRequired(&'static str), + /// Another process holds this host's lifecycle writer lock. #[error( - "a backed-up host configuration directory vanished and could not be recreated safely; restore the directory or its parent and retry recovery" + "another TraceDecay process is changing this host's integration; retry when it finishes" )] - RecoveryDirectoryUnavailable, - #[error( - "host recovery backup format is unsupported; use the TraceDecay version that created it or restore the host configuration from backup" - )] - UnsupportedRecoveryFormat, + HostWriterBusy, /// Apply observed drift from the confirmed preview. The payload names the /// matching layer that rejected, so genuine host drift is distinguishable /// from a lifecycle bug; build it with [`host_bundle_stale_preview!`] rather @@ -239,29 +214,6 @@ mod tests { assert_eq!(gemini.edit.native_fixture_digest, None); } - /// Kimi MCP is the installer-owned session/user `mcp.json` route. The - /// packaged plugin manifest omits `mcpServers` on purpose, so citing that - /// file would treat omission as supported-MCP evidence. - #[test] - fn kimi_mcp_evidence_is_installer_user_config_route() { - let evidence = stock_host_registration_evidence(HostKindV1::KimiCode); - let mcp = evidence - .iter() - .find(|row| row.route == HostRegistrationRouteV1::Mcp) - .expect("Kimi MCP registration row"); - let hook = evidence - .iter() - .find(|row| row.route == HostRegistrationRouteV1::Hook) - .expect("Kimi hook registration row"); - assert_eq!(mcp.state, HostCapabilityStateV1::Supported); - assert_eq!(mcp.evidence_ref, "src/agents/kimi.rs"); - assert_eq!(hook.evidence_ref, "plugin/.kimi-plugin/plugin.json"); - assert_ne!( - mcp.evidence_ref, hook.evidence_ref, - "Kimi MCP must not reuse the plugin-manifest hook authority" - ); - } - /// Every `HostKindV1` variant owns table rows: a CLI route first plus at /// least one daemon route, each route once, each with a non-empty evidence /// reference. A host-specific route (Claude/OpenCode LSP, Cursor native @@ -350,88 +302,4 @@ mod tests { Some("native_fixture_missing") ); } - - fn component_set_journal( - state: HostComponentSetJournalStateV1, - registration_staged: bool, - registration_applied: bool, - ) -> HostComponentSetJournalV1 { - HostComponentSetJournalV1 { - schema_version: 1, - operation_id: [7; 16], - host: HostKindV1::OpenCode, - operation: HostBundleLifecycleOpV1::Update, - explicit_confirmation: true, - hermes_profile_bindings: 0, - confirmed_plan_digest: None, - base_registration_revision: None, - current_registration_revision: None, - artifact_state_revision: None, - state, - registration_staged, - registration_applied, - components: Vec::new(), - } - } - - /// The flags are raised before the hook they name and the phase advances - /// after it returns, so each phase implies the flags behind it. `RolledBack` - /// is the one state that keeps whatever the failed attempt reached. - #[test] - fn component_set_journal_phases_imply_their_registration_flags() { - use HostComponentSetJournalStateV1 as State; - - for (state, staged, applied, representable) in [ - (State::Prepared, false, false, true), - (State::Prepared, true, false, true), - (State::Prepared, false, true, false), - (State::Prepared, true, true, false), - (State::Staged, true, false, true), - (State::Staged, true, true, true), - (State::Staged, false, false, false), - (State::Applied, true, true, true), - (State::Applied, true, false, false), - (State::Verified, true, true, true), - (State::Verified, false, true, false), - (State::Committed, true, true, true), - (State::Committed, false, false, false), - (State::RolledBack, false, false, true), - (State::RolledBack, true, false, true), - (State::RolledBack, true, true, true), - (State::RolledBack, false, true, false), - ] { - assert_eq!( - component_set_journal(state, staged, applied).registration_flags_match_state(), - representable, - "{state:?} staged={staged} applied={applied}" - ); - } - } - - /// Only a `Prepared` journal with both flags clear proves registration was - /// never entered. Every other journal - a rolled-back one above all - still - /// owes an idempotent compensation attempt. - #[test] - fn only_an_untouched_prepared_journal_skips_registration_compensation() { - use HostComponentSetJournalStateV1 as State; - - assert!( - !component_set_journal(State::Prepared, false, false) - .registration_compensation_required() - ); - for (state, staged, applied) in [ - (State::Prepared, true, false), - (State::Staged, true, false), - (State::Applied, true, true), - (State::Verified, true, true), - (State::Committed, true, true), - (State::RolledBack, false, false), - (State::RolledBack, true, true), - ] { - assert!( - component_set_journal(state, staged, applied).registration_compensation_required(), - "{state:?} staged={staged} applied={applied}" - ); - } - } } diff --git a/crates/tracedecay-host-integration/src/manifest.rs b/crates/tracedecay-host-integration/src/manifest.rs index 2a4831dbba..9c7d245077 100644 --- a/crates/tracedecay-host-integration/src/manifest.rs +++ b/crates/tracedecay-host-integration/src/manifest.rs @@ -168,7 +168,7 @@ pub fn validate_relative_install_path(path: &Path) -> Result<(), HostBundleError } /// Bytes obtained from the verified embedded host bundle. They are checked /// against the cataloged artifact digest before any host path is touched and -/// are never copied into receipts or journals. +/// are never copied into receipts. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct HostBundleArtifactContentV1 { pub relative_path: String, diff --git a/crates/tracedecay-host-integration/src/receipt.rs b/crates/tracedecay-host-integration/src/receipt.rs new file mode 100644 index 0000000000..7ca005cb8b --- /dev/null +++ b/crates/tracedecay-host-integration/src/receipt.rs @@ -0,0 +1,51 @@ +use serde::{Deserialize, Serialize}; +use tracedecay_domain::{HostComponentV1, HostKindV1}; + +use crate::manifest::{HostBundleLifecycleOpV1, HostBundleManifestV1}; + +/// Version 2 dropped the rollback boundary and rollback history: receipts are +/// ownership records only, and no operation keeps prior bytes after it ends. +pub const HOST_BUNDLE_RECEIPT_SCHEMA_VERSION: u16 = 2; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct HostBundleReceiptArtifactV1 { + pub relative_path: String, + pub artifact_digest: [u8; 32], + pub ownership_marker: String, +} + +/// Durable local receipt. It is a host-install ownership record, not a +/// product/configuration store and contains no artifact content or credentials. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct HostBundleInstallReceiptV1 { + pub schema_version: u16, + pub operation_id: [u8; 16], + pub host: HostKindV1, + pub component: HostComponentV1, + pub operation: HostBundleLifecycleOpV1, + pub manifest_digest: [u8; 32], + pub artifacts: Vec, +} + +/// Durable aggregate commit marker for a complete host component set. The root +/// adapter owns the aggregate transaction; this contract binds its receipts. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct HostComponentSetReceiptV1 { + pub schema_version: u16, + pub operation_id: [u8; 16], + pub host: HostKindV1, + pub operation: HostBundleLifecycleOpV1, + pub component_manifests: Vec, + pub component_receipts: Vec, + #[serde(default)] + pub confirmed_plan_digest: Option<[u8; 32]>, + #[serde(default)] + pub base_registration_revision: Option<[u8; 32]>, + #[serde(default)] + pub current_registration_revision: Option<[u8; 32]>, + #[serde(default)] + pub artifact_state_revision: Option<[u8; 32]>, +} diff --git a/crates/tracedecay-lcm/src/compression.rs b/crates/tracedecay-lcm/src/compression.rs index 3d0f531970..7538b91749 100644 --- a/crates/tracedecay-lcm/src/compression.rs +++ b/crates/tracedecay-lcm/src/compression.rs @@ -1658,7 +1658,6 @@ fn compression_response_with_attempt_state( replay_token_estimate, replay_over_budget: replay_exceeds_budget(replay_token_estimate, max_assembly_tokens), compression_attempts, - fallback_used: false, context_recovery_hint, retry_status: retry_status.map(str::to_string), relation_projection_status, @@ -1876,26 +1875,28 @@ async fn load_condensation_candidates( let mut rows = conn .query( "WITH source_order AS ( - SELECT lcm_summary_sources.node_id, MIN(CAST(source_id AS INTEGER)) AS first_source_id - FROM lcm_summary_sources + SELECT session_summary_sources.summary_id, + MIN(CAST(source_id AS INTEGER)) AS first_source_id + FROM session_summary_sources WHERE source_kind = 'raw_message' - GROUP BY lcm_summary_sources.node_id + GROUP BY session_summary_sources.summary_id ), unparented AS ( - SELECT n.node_id, n.provider, n.conversation_id, n.session_id, n.depth, n.summary_text, - n.summary_hash, n.summary_token_count, n.source_token_count, n.source_time_start, - n.source_time_end, n.expand_hint, n.metadata_json, n.created_at, + SELECT n.summary_id, n.provider, n.conversation_id, n.session_id, n.depth, + n.summary_text, n.summary_hash, n.summary_token_count, + n.source_token_count, n.source_time_start, n.source_time_end, + n.expand_hint, n.metadata_json, n.created_at, source_order.first_source_id - FROM lcm_summary_nodes n + FROM session_summary_nodes n JOIN session_temporal_generations generation ON generation.session_id = n.session_id AND generation.state = 'active' JOIN session_summary_availability availability ON availability.session_id = generation.session_id AND availability.generation = generation.generation - AND availability.summary_id = n.node_id + AND availability.summary_id = n.summary_id AND availability.availability = 'available' - LEFT JOIN source_order ON source_order.node_id = n.node_id + LEFT JOIN source_order ON source_order.summary_id = n.summary_id WHERE n.provider = ?1 AND n.session_id = ?2 -- Fail closed only while a raw revision's invalidation -- closure is partially applied: the walk enqueues @@ -1913,14 +1914,14 @@ async fn load_condensation_candidates( ) AND NOT EXISTS ( SELECT 1 - FROM lcm_summary_sources s + FROM session_summary_sources s JOIN session_summary_availability parent_availability ON parent_availability.session_id = generation.session_id AND parent_availability.generation = generation.generation - AND parent_availability.summary_id = s.node_id + AND parent_availability.summary_id = s.summary_id AND parent_availability.availability = 'available' WHERE s.source_kind = 'summary_node' - AND s.source_id = n.node_id + AND s.source_id = n.summary_id ) ), eligible_depth AS ( @@ -1932,24 +1933,18 @@ async fn load_condensation_candidates( ORDER BY depth LIMIT 1 ) - SELECT node_id, provider, conversation_id, session_id, depth, summary_text, + SELECT summary_id, provider, conversation_id, session_id, depth, summary_text, summary_hash, summary_token_count, source_token_count, source_time_start, source_time_end, expand_hint, metadata_json, created_at FROM unparented WHERE depth = (SELECT depth FROM eligible_depth) ORDER BY source_time_start IS NULL, source_time_start, first_source_id IS NULL, first_source_id, - created_at, node_id + created_at, summary_id LIMIT ?3", - params![ - provider, - session_id, - fan_in as i64, - incremental_max_depth - ], + params![provider, session_id, fan_in as i64, incremental_max_depth], ) - .await - ?; + .await?; let mut nodes = Vec::new(); while let Some(row) = rows.next().await? { nodes.push(LcmSummaryNode { @@ -2513,7 +2508,7 @@ async fn load_raw_messages_for_session( .query( "SELECT provider, message_id, session_id, store_id, role, ordinal, timestamp, content, content_hash, storage_kind, payload_ref, - snippet_text, legacy_source, legacy_truncated, metadata_json + snippet_text, metadata_json FROM lcm_raw_messages WHERE provider = ?1 AND session_id = ?2 ORDER BY store_id", @@ -2557,7 +2552,7 @@ async fn load_raw_messages_for_session_page( .query( "SELECT provider, message_id, session_id, store_id, role, ordinal, timestamp, content, content_hash, storage_kind, payload_ref, - snippet_text, legacy_source, legacy_truncated, metadata_json, + snippet_text, metadata_json, length(CAST(COALESCE(content, '') AS BLOB)) + length(CAST(snippet_text AS BLOB)) + length(CAST(index_text AS BLOB)) @@ -2573,7 +2568,7 @@ async fn load_raw_messages_for_session_page( let mut bytes_scanned = 0_u64; let mut byte_limited = false; while let Some(row) = rows.next().await? { - let row_bytes = u64::try_from(row.get::(15)?).map_err(|error| { + let row_bytes = u64::try_from(row.get::(13)?).map_err(|error| { LcmError::Db(format!("invalid retained compression byte count: {error}")) })?; if bytes_scanned.saturating_add(row_bytes) > limit.byte_limit { @@ -2769,21 +2764,10 @@ mod authority_tests { .unwrap(); schema::ensure_lcm_schema(&conn).await.unwrap(); conn.execute_batch( - "CREATE TABLE session_temporal_generations ( - session_id TEXT NOT NULL, - generation INTEGER NOT NULL, - state TEXT NOT NULL - ); - CREATE TABLE session_summary_availability ( - session_id TEXT NOT NULL, - generation INTEGER NOT NULL, - summary_id TEXT NOT NULL, - availability TEXT NOT NULL - ); - INSERT INTO session_temporal_generations(session_id, generation, state) + "INSERT INTO session_temporal_generations(session_id, generation, state) VALUES ('active-condensation', 2, 'active'); - INSERT INTO lcm_summary_nodes( - node_id, provider, conversation_id, session_id, depth, + INSERT INTO session_summary_nodes( + summary_id, provider, conversation_id, session_id, depth, summary_text, summary_hash, summary_token_count, source_token_count, created_at ) VALUES @@ -2791,7 +2775,7 @@ mod authority_tests { 'current summary', 'current-hash', 2, 4, 1), ('stale-parent', 'cursor', 'active-condensation', 'active-condensation', 1, 'stale summary', 'stale-hash', 2, 4, 2); - INSERT INTO lcm_summary_sources(node_id, source_kind, source_id, ordinal) VALUES + INSERT INTO session_summary_sources(summary_id, source_kind, source_id, ordinal) VALUES ('current', 'raw_message', '1', 0), ('stale-parent', 'summary_node', 'current', 0); INSERT INTO session_summary_availability( diff --git a/crates/tracedecay-lcm/src/compression_decision.rs b/crates/tracedecay-lcm/src/compression_decision.rs index f7993a58d0..34b2ddc148 100644 --- a/crates/tracedecay-lcm/src/compression_decision.rs +++ b/crates/tracedecay-lcm/src/compression_decision.rs @@ -360,8 +360,6 @@ mod tests { content_hash: "hash".to_string(), storage_kind: super::super::LcmStorageKind::Inline, payload_ref: None, - legacy_source: false, - legacy_truncated: false, metadata_json: None, }]; diff --git a/crates/tracedecay-lcm/src/contracts.rs b/crates/tracedecay-lcm/src/contracts.rs index ea5f2f8a41..1fd12b7d98 100644 --- a/crates/tracedecay-lcm/src/contracts.rs +++ b/crates/tracedecay-lcm/src/contracts.rs @@ -22,8 +22,6 @@ pub struct LcmRawMessage { pub content_hash: String, pub storage_kind: LcmStorageKind, pub payload_ref: Option, - pub legacy_source: bool, - pub legacy_truncated: bool, pub metadata_json: Option, } @@ -39,8 +37,6 @@ pub struct LcmRawMessageMetadata { pub content_hash: String, pub storage_kind: LcmStorageKind, pub payload_ref: Option, - pub legacy_source: bool, - pub legacy_truncated: bool, pub metadata_json: Option, } @@ -57,8 +53,6 @@ impl LcmRawMessage { content_hash: self.content_hash, storage_kind: self.storage_kind, payload_ref: self.payload_ref, - legacy_source: self.legacy_source, - legacy_truncated: self.legacy_truncated, metadata_json: self.metadata_json, } } @@ -95,8 +89,6 @@ impl LcmRawMessageMetadata { content_hash: self.content_hash, storage_kind: self.storage_kind, payload_ref: self.payload_ref, - legacy_source: self.legacy_source, - legacy_truncated: self.legacy_truncated, metadata_json: self.metadata_json, } } @@ -288,11 +280,6 @@ pub struct LcmExpandResponse { /// Mirrors hermes-lcm `from_current_session`; raw-message targets only. #[serde(default, skip_serializing_if = "Option::is_none")] pub from_current_session: Option, - /// Legacy compatibility note mirrored from hermes-lcm payloads. Modern - /// cross-session expansion flows should rely on `payload_ref` + - /// `raw_message.session_id` and remain note-free. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub externalized_note: Option, /// Source-list coverage metadata (summary-node targets only). #[serde(default, skip_serializing_if = "Option::is_none")] pub source_pagination: Option, diff --git a/crates/tracedecay-lcm/src/dag.rs b/crates/tracedecay-lcm/src/dag.rs index 2dd46a1f31..4c22b3bed2 100644 --- a/crates/tracedecay-lcm/src/dag.rs +++ b/crates/tracedecay-lcm/src/dag.rs @@ -15,7 +15,7 @@ use tracedecay_runtime_core::db::engine::{QueryExecutor, Value, params}; use super::types::{LcmImmutableSummaryPublication, LcmSummaryPublicationReceipt}; use super::{ LcmError, LcmExpandedSummarySource, LcmRawMessage, LcmRawMessageMetadata, LcmSourceRef, - LcmSummaryExpansion, LcmSummaryNode, LcmSummaryNodeDraft, raw, util, + LcmSummaryExpansion, LcmSummaryNode, LcmSummaryNodeDraft, raw, schema, util, }; #[derive(Clone)] @@ -176,7 +176,7 @@ async fn expand_summary_nodes_with_content( return Ok(Vec::new()); } let requested = hotpath::future!( - load_summary_nodes_by_ids(conn, node_ids, include_content), + load_summary_nodes_by_ids(conn, node_ids, include_content, SummaryRead::Visible), label = "sessions.lcm.expand.summary.fetch" ) .await?; @@ -209,7 +209,7 @@ async fn expand_summary_nodes_with_content( ) .await?; let child_sources = hotpath::future!( - load_summary_nodes_by_ids(conn, &child_node_ids, include_content), + load_summary_nodes_by_ids(conn, &child_node_ids, include_content, SummaryRead::Lineage), label = "sessions.lcm.expand.summary.fetch" ) .await?; @@ -334,7 +334,7 @@ pub struct LcmUncondensedSummaryNode { /// CTE prefix shared by [`load_uncondensed_summary_nodes`] and its work /// measurement: the available unparented roots of one session and, per root, -/// the descendants reachable through `lcm_summary_sources`. +/// the descendants reachable through `session_summary_sources`. /// /// `lineage` holds one `(root_id, source_kind, source_id)` row per *distinct* /// descendant. The recursive `UNION` (not `UNION ALL`) makes SQLite's queue a @@ -345,18 +345,18 @@ pub struct LcmUncondensedSummaryNode { /// corrupted cyclic lineage revisits nothing and still yields the complete /// reachable minimum instead of a depth-truncated one. const UNCONDENSED_LINEAGE_CTE: &str = "WITH RECURSIVE unparented AS ( - SELECT n.node_id, n.provider, n.conversation_id, n.session_id, n.depth, + SELECT n.summary_id, n.provider, n.conversation_id, n.session_id, n.depth, n.summary_text, n.summary_hash, n.summary_token_count, n.source_token_count, n.source_time_start, n.source_time_end, n.expand_hint, n.metadata_json, n.created_at - FROM lcm_summary_nodes n + FROM session_summary_nodes n JOIN session_temporal_generations generation ON generation.session_id = n.session_id AND generation.state = 'active' JOIN session_summary_availability availability ON availability.session_id = generation.session_id AND availability.generation = generation.generation - AND availability.summary_id = n.node_id + AND availability.summary_id = n.summary_id AND availability.availability = 'available' WHERE n.provider = ?1 AND n.session_id = ?2 -- Fail closed only while a raw revision's invalidation @@ -375,25 +375,25 @@ const UNCONDENSED_LINEAGE_CTE: &str = "WITH RECURSIVE unparented AS ( ) AND NOT EXISTS ( SELECT 1 - FROM lcm_summary_sources s + FROM session_summary_sources s JOIN session_summary_availability parent_availability ON parent_availability.session_id = generation.session_id AND parent_availability.generation = generation.generation - AND parent_availability.summary_id = s.node_id + AND parent_availability.summary_id = s.summary_id AND parent_availability.availability = 'available' WHERE s.source_kind = 'summary_node' - AND s.source_id = n.node_id + AND s.source_id = n.summary_id ) ), lineage(root_id, source_kind, source_id) AS ( - SELECT s.node_id, s.source_kind, s.source_id - FROM lcm_summary_sources s - JOIN unparented u ON u.node_id = s.node_id + SELECT s.summary_id, s.source_kind, s.source_id + FROM session_summary_sources s + JOIN unparented u ON u.summary_id = s.summary_id UNION SELECT l.root_id, s.source_kind, s.source_id FROM lineage l - JOIN lcm_summary_sources s - ON l.source_kind = 'summary_node' AND s.node_id = l.source_id + JOIN session_summary_sources s + ON l.source_kind = 'summary_node' AND s.summary_id = l.source_id ), first_raw AS ( SELECT root_id, MIN(CAST(source_id AS INTEGER)) AS first_source_store_id @@ -407,18 +407,18 @@ const UNCONDENSED_LINEAGE_CTE: &str = "WITH RECURSIVE unparented AS ( fn uncondensed_summary_nodes_sql() -> String { format!( "{UNCONDENSED_LINEAGE_CTE} - SELECT u.node_id, u.provider, u.conversation_id, u.session_id, u.depth, + SELECT u.summary_id, u.provider, u.conversation_id, u.session_id, u.depth, u.summary_text, u.summary_hash, u.summary_token_count, u.source_token_count, u.source_time_start, u.source_time_end, u.expand_hint, u.metadata_json, u.created_at, first_raw.first_source_store_id FROM unparented u - LEFT JOIN first_raw ON first_raw.root_id = u.node_id + LEFT JOIN first_raw ON first_raw.root_id = u.summary_id ORDER BY first_raw.first_source_store_id IS NULL, first_raw.first_source_store_id, u.depth DESC, u.source_time_start IS NULL, u.source_time_start, - u.created_at, u.node_id" + u.created_at, u.summary_id" ) } @@ -556,10 +556,21 @@ async fn load_raw_messages_by_store_ids( } } +/// Which summary rows a by-id load may return. +#[derive(Clone, Copy)] +enum SummaryRead { + /// Entry points: only summaries visible under [`schema::SUMMARY_VISIBLE_SQL`]. + Visible, + /// Children of an already-visible summary: its immutable lineage, read as + /// published so the parent's expansion stays complete. + Lineage, +} + async fn load_summary_nodes_by_ids( conn: &(impl QueryExecutor + ?Sized), node_ids: &[String], include_content: bool, + read: SummaryRead, ) -> Result, LcmError> { let unique_node_ids = node_ids .iter() @@ -571,7 +582,7 @@ async fn load_summary_nodes_by_ids( return Ok(BTreeMap::new()); } let summary_text = if include_content { - "summary_text" + "n.summary_text" } else { "'' AS summary_text" }; @@ -581,12 +592,17 @@ async fn load_summary_nodes_by_ids( continue; } let placeholders = util::sql_in_placeholders(chunk.len()); + let visibility = match read { + SummaryRead::Visible => schema::SUMMARY_VISIBLE_SQL, + SummaryRead::Lineage => "1", + }; let node_sql = format!( - "SELECT node_id, provider, conversation_id, session_id, depth, {summary_text}, - summary_hash, summary_token_count, source_token_count, source_time_start, - source_time_end, expand_hint, metadata_json, created_at - FROM lcm_summary_nodes - WHERE node_id IN ({placeholders})" + "SELECT n.summary_id, n.provider, n.conversation_id, n.session_id, n.depth, + {summary_text}, n.summary_hash, n.summary_token_count, + n.source_token_count, n.source_time_start, n.source_time_end, + n.expand_hint, n.metadata_json, n.created_at + FROM session_summary_nodes n + WHERE n.summary_id IN ({placeholders}) AND {visibility}" ); let values = chunk .iter() @@ -618,10 +634,10 @@ async fn load_summary_nodes_by_ids( nodes.insert(node_id, node); } let source_sql = format!( - "SELECT node_id, source_kind, source_id - FROM lcm_summary_sources - WHERE node_id IN ({placeholders}) - ORDER BY node_id, ordinal" + "SELECT summary_id, source_kind, source_id + FROM session_summary_sources + WHERE summary_id IN ({placeholders}) + ORDER BY summary_id, ordinal" ); let mut source_rows = conn.query(&source_sql, values).await?; while let Some(row) = source_rows.next().await? { @@ -751,9 +767,6 @@ mod lineage_tests { ) .await .expect("session schema"); - conn.execute_batch(test_support::SESSION_GENERATION_SCHEMA) - .await - .expect("session generation schema"); schema::ensure_lcm_schema(&conn).await.expect("lcm schema"); conn.execute( "INSERT INTO sessions(provider, session_id, project_key, project_path) @@ -769,8 +782,8 @@ mod lineage_tests { async fn seed(conn: &TestConnection, session_id: &str, nodes: &[FixtureNode]) { for fixture in nodes { conn.execute( - "INSERT INTO lcm_summary_nodes( - node_id, provider, conversation_id, session_id, depth, summary_text, + "INSERT INTO session_summary_nodes( + summary_id, provider, conversation_id, session_id, depth, summary_text, summary_hash, summary_token_count, source_token_count, created_at ) VALUES (?1, ?2, ?3, ?3, ?4, ?1, ?5, 1, 1, ?6)", params![ @@ -806,7 +819,7 @@ mod lineage_tests { LcmSourceRef::SummaryNode { node_id } => ("summary_node", node_id.clone()), }; conn.execute( - "INSERT INTO lcm_summary_sources(node_id, source_kind, source_id, ordinal) + "INSERT INTO session_summary_sources(summary_id, source_kind, source_id, ordinal) VALUES (?1, ?2, ?3, ?4)", params![ fixture.node_id.as_str(), diff --git a/crates/tracedecay-lcm/src/gc.rs b/crates/tracedecay-lcm/src/gc.rs index 42be3fa5f5..4630e0e85a 100644 --- a/crates/tracedecay-lcm/src/gc.rs +++ b/crates/tracedecay-lcm/src/gc.rs @@ -1067,20 +1067,19 @@ async fn tombstone_dangling_refs_in_transaction( continue; } let store_id = row.store_id; - let (content, snippet_text, index_text, metadata_json, changed) = + let (content, placeholder_text, metadata_json, changed) = tombstone_row_for_refs(row, dangling); if changed == 0 { continue; } conn.execute( "UPDATE lcm_raw_messages - SET content = ?2, snippet_text = ?3, index_text = ?4, metadata_json = ?5 + SET content = ?2, placeholder_text = ?3, metadata_json = ?4 WHERE store_id = ?1", params![ store_id, content.as_deref(), - snippet_text, - index_text, + placeholder_text.as_deref(), metadata_json.as_deref() ], ) @@ -1093,23 +1092,28 @@ async fn tombstone_dangling_refs_in_transaction( fn tombstone_row_for_refs( row: PlaceholderTextRow, payload_refs: &BTreeSet, -) -> (Option, String, String, Option, usize) { +) -> (Option, Option, Option, usize) { let mut changed = 0usize; let content = row.content.map(|text| { let (tombstoned, field_changes) = tombstone_text_for_refs(&text, payload_refs); changed += field_changes; tombstoned }); - let (snippet_text, snippet_changes) = tombstone_text_for_refs(&row.snippet_text, payload_refs); + // The snippet and index columns derive from the tombstoned body; they are + // counted because they are retrieval text a reader sees change. + let (_, snippet_changes) = tombstone_text_for_refs(&row.snippet_text, payload_refs); changed += snippet_changes; - let (index_text, index_changes) = tombstone_text_for_refs(&row.index_text, payload_refs); + let (_, index_changes) = tombstone_text_for_refs(&row.index_text, payload_refs); changed += index_changes; + let placeholder_text = row + .placeholder_text + .map(|text| tombstone_text_for_refs(&text, payload_refs).0); let metadata_json = row.metadata_json.map(|text| { let (tombstoned, field_changes) = tombstone_text_for_refs(&text, payload_refs); changed += field_changes; tombstoned }); - (content, snippet_text, index_text, metadata_json, changed) + (content, placeholder_text, metadata_json, changed) } fn tombstone_text_for_refs(text: &str, payload_refs: &BTreeSet) -> (String, usize) { diff --git a/crates/tracedecay-lcm/src/gc/placeholder_scan.rs b/crates/tracedecay-lcm/src/gc/placeholder_scan.rs index ce85ca5fa2..b519f5774f 100644 --- a/crates/tracedecay-lcm/src/gc/placeholder_scan.rs +++ b/crates/tracedecay-lcm/src/gc/placeholder_scan.rs @@ -22,6 +22,7 @@ pub(crate) enum PlaceholderScanScope<'a> { pub(crate) struct PlaceholderTextRow { pub store_id: i64, pub content: Option, + pub placeholder_text: Option, pub snippet_text: String, pub index_text: String, pub metadata_json: Option, @@ -201,7 +202,8 @@ async fn drive_placeholder_text_scan( loop { let sql = format!( "WITH page AS ( - SELECT store_id, content, snippet_text, index_text, metadata_json + SELECT store_id, content, snippet_text, index_text, metadata_json, + placeholder_text FROM lcm_raw_messages WHERE {scope_sql} AND store_id > ? @@ -211,15 +213,18 @@ async fn drive_placeholder_text_scan( ), bounded AS ( SELECT store_id, content, snippet_text, index_text, metadata_json, + placeholder_text, ROW_NUMBER() OVER (ORDER BY store_id) AS page_row, SUM(length(CAST(COALESCE(content, '') AS BLOB)) + length(CAST(COALESCE(snippet_text, '') AS BLOB)) + length(CAST(COALESCE(index_text, '') AS BLOB)) - + length(CAST(COALESCE(metadata_json, '') AS BLOB))) + + length(CAST(COALESCE(metadata_json, '') AS BLOB)) + + length(CAST(COALESCE(placeholder_text, '') AS BLOB))) OVER (ORDER BY store_id) AS cumulative_bytes FROM page ) - SELECT store_id, content, snippet_text, index_text, metadata_json + SELECT store_id, content, snippet_text, index_text, metadata_json, + placeholder_text FROM bounded WHERE cumulative_bytes <= ? OR page_row = 1 ORDER BY store_id" @@ -242,10 +247,11 @@ async fn drive_placeholder_text_scan( page_rows += 1; let visited = PlaceholderTextRow { store_id, - content: row.get(1).unwrap_or(None), + content: row.get(1)?, + placeholder_text: row.get(5)?, snippet_text: row.get(2)?, index_text: row.get(3)?, - metadata_json: row.get(4).unwrap_or(None), + metadata_json: row.get(4)?, }; if matches!(visit(visited), PlaceholderScanFlow::Stop) { return Ok(()); diff --git a/crates/tracedecay-lcm/src/gc/tests.rs b/crates/tracedecay-lcm/src/gc/tests.rs index f6d58fc322..af9e21ebdb 100644 --- a/crates/tracedecay-lcm/src/gc/tests.rs +++ b/crates/tracedecay-lcm/src/gc/tests.rs @@ -2,6 +2,7 @@ use std::fs; use crate::schema; use crate::util::{self, file_mtime_seconds}; +use tracedecay_domain::canonical_text::sha256_hex; use tracedecay_runtime_core::db::engine::{Connection, TestConnection, TransactionBehavior}; use super::pending_delete::{PENDING_PAYLOAD_DELETE_ERROR_PREFIX, pending_payload_delete_key}; @@ -68,19 +69,6 @@ async fn ensure_gc_test_schema(conn: &Connection) -> Result<(), String> { title TEXT, started_at INTEGER, PRIMARY KEY(provider, session_id) - ); - CREATE TABLE IF NOT EXISTS session_messages ( - provider TEXT NOT NULL, - message_id TEXT NOT NULL, - session_id TEXT NOT NULL, - role TEXT NOT NULL, - timestamp INTEGER, - ordinal INTEGER NOT NULL, - text TEXT NOT NULL, - metadata_json TEXT, - PRIMARY KEY(provider, message_id), - FOREIGN KEY(provider, session_id) - REFERENCES sessions(provider, session_id) ON DELETE CASCADE );", ) .await @@ -114,8 +102,7 @@ struct RawMessage<'a> { storage_kind: &'a str, payload_ref: Option<&'a str>, content: Option<&'a str>, - snippet_text: &'a str, - index_text: &'a str, + placeholder_text: Option<&'a str>, metadata_json: Option<&'a str>, } @@ -123,9 +110,9 @@ async fn insert_raw_message(conn: &Connection, message: RawMessage<'_>) -> Resul conn.execute( "INSERT INTO lcm_raw_messages ( provider, message_id, session_id, role, ordinal, timestamp, - content, content_hash, storage_kind, payload_ref, snippet_text, - index_text, legacy_source, legacy_truncated, metadata_json - ) VALUES (?1, ?2, ?3, 'assistant', 1, 2, ?4, ?5, ?6, ?7, ?8, ?9, 0, 0, ?10)", + content, content_hash, storage_kind, payload_ref, placeholder_text, + metadata_json + ) VALUES (?1, ?2, ?3, 'assistant', 1, 2, ?4, ?5, ?6, ?7, ?8, ?9)", params![ PROVIDER, message.message_id, @@ -134,8 +121,7 @@ async fn insert_raw_message(conn: &Connection, message: RawMessage<'_>) -> Resul format!("{}-hash", message.message_id), message.storage_kind, message.payload_ref, - message.snippet_text, - message.index_text, + message.placeholder_text, message.metadata_json ], ) @@ -176,8 +162,7 @@ async fn seed_payload( storage_kind: "external", payload_ref: Some(&payload_ref.payload_ref), content: None, - snippet_text: &placeholder, - index_text: &placeholder, + placeholder_text: Some(&placeholder), metadata_json: Some(&placeholder), }, ) @@ -270,8 +255,7 @@ async fn referenced_payload_refs_ignores_tombstoned_placeholders() -> Result<(), storage_kind: "inline", payload_ref: None, content: Some(&live), - snippet_text: &live, - index_text: &live, + placeholder_text: None, metadata_json: None, }, ) @@ -284,8 +268,7 @@ async fn referenced_payload_refs_ignores_tombstoned_placeholders() -> Result<(), storage_kind: "inline", payload_ref: None, content: Some(&tombstoned), - snippet_text: &tombstoned, - index_text: &tombstoned, + placeholder_text: None, metadata_json: None, }, ) @@ -471,7 +454,7 @@ async fn committed_payload_delete_drain_failure_returns_pending_then_retries() - let removed = drain_pending_payload_delete(&store.conn, &store.storage_root, &payload_ref) .await .map_err(|err| err.to_string())?; - assert!(removed.is_some()); + assert_eq!(removed, Some("body to retry".len() as u64)); assert!(!payload_path(&store, &payload_ref).exists()); Ok(()) } @@ -1210,7 +1193,7 @@ fn committed_delete_quarantine_preserves_rename_replacement() -> Result<(), Stri let path = dir.join(PRIMARY_REF); let original = b"original payload"; fs::write(&path, original).map_err(|err| err.to_string())?; - let expected_hash = util::sha256_hex(original); + let expected_hash = sha256_hex(original); let removal = payload::remove_committed_payload_file_with( temp.path(), @@ -1244,7 +1227,7 @@ fn committed_delete_quarantine_restores_in_place_rewrite() -> Result<(), String> let path = dir.join(PRIMARY_REF); let original = b"original payload"; fs::write(&path, original).map_err(|err| err.to_string())?; - let expected_hash = util::sha256_hex(original); + let expected_hash = sha256_hex(original); let removal = payload::remove_committed_payload_file_with( temp.path(), @@ -1272,7 +1255,7 @@ fn committed_delete_quarantine_restores_in_place_rewrite() -> Result<(), String> #[test] fn committed_delete_requires_exact_hash_byte_and_char_sizes() -> Result<(), String> { let original = "héllo 雪"; - let expected_hash = util::sha256_hex(original.as_bytes()); + let expected_hash = sha256_hex(original.as_bytes()); let expected_bytes = original.len() as u64; let expected_chars = original.chars().count() as u64; for (hash, bytes, chars) in [ @@ -1333,7 +1316,7 @@ fn committed_delete_retry_succeeds_after_same_id_content_restore() -> Result<(), fs::create_dir(&dir).map_err(|err| err.to_string())?; let path = dir.join(PRIMARY_REF); let original = b"original"; - let expected_hash = util::sha256_hex(original); + let expected_hash = sha256_hex(original); fs::write(&path, original).map_err(|err| err.to_string())?; let first = payload::remove_committed_payload_file_with( @@ -1795,8 +1778,7 @@ async fn residual_sweep_rows_visited(decoys: usize) -> Result { storage_kind: "inline", payload_ref: None, content: Some(&live), - snippet_text: &live, - index_text: &live, + placeholder_text: None, metadata_json: Some(&live), }, ) @@ -1812,8 +1794,7 @@ async fn residual_sweep_rows_visited(decoys: usize) -> Result { storage_kind: "inline", payload_ref: None, content: Some(&prose), - snippet_text: &prose, - index_text: &prose, + placeholder_text: None, metadata_json: Some(&prose), }, ) @@ -1913,8 +1894,7 @@ async fn narrowed_prefilter_rewrites_the_same_rows() -> Result<(), String> { storage_kind: "inline", payload_ref: None, content: Some(&live), - snippet_text: &live, - index_text: &live, + placeholder_text: None, metadata_json: Some(&live), }, ) @@ -1927,8 +1907,7 @@ async fn narrowed_prefilter_rewrites_the_same_rows() -> Result<(), String> { storage_kind: "inline", payload_ref: None, content: Some(&already_gcd), - snippet_text: &already_gcd, - index_text: &already_gcd, + placeholder_text: None, metadata_json: Some(&already_gcd), }, ) @@ -1941,8 +1920,7 @@ async fn narrowed_prefilter_rewrites_the_same_rows() -> Result<(), String> { storage_kind: "inline", payload_ref: None, content: Some(&prose), - snippet_text: &prose, - index_text: &prose, + placeholder_text: None, metadata_json: Some(&prose), }, ) diff --git a/crates/tracedecay-lcm/src/payload.rs b/crates/tracedecay-lcm/src/payload.rs index c0243a119e..a7c4ffe947 100644 --- a/crates/tracedecay-lcm/src/payload.rs +++ b/crates/tracedecay-lcm/src/payload.rs @@ -1,10 +1,11 @@ use std::path::{Path, PathBuf}; pub use crate::contracts::validate_payload_ref; +use tracedecay_domain::canonical_text::sha256_hex; use tracedecay_runtime_core::db::engine::{Executor, QueryExecutor, params}; use tracedecay_runtime_core::tracedecay::current_timestamp; -use super::{LcmError, LcmPayloadExpansion, LcmPayloadRef, gc, util}; +use super::{LcmError, LcmPayloadExpansion, LcmPayloadRef, gc}; mod delete_recovery; mod filesystem_authority; @@ -159,10 +160,9 @@ fn write_external_payload_inner( content, metadata_json, } = write; - let content_hash = util::sha256_hex(content.as_bytes()); - let owner_hash = util::sha256_hex( - format!("{provider}\0{session_id}\0{message_id}\0{content_hash}").as_bytes(), - ); + let content_hash = sha256_hex(content.as_bytes()); + let owner_hash = + sha256_hex(format!("{provider}\0{session_id}\0{message_id}\0{content_hash}").as_bytes()); let payload_ref = format!("payload_{owner_hash}.payload"); validate_payload_ref(&payload_ref)?; diff --git a/crates/tracedecay-lcm/src/payload/delete_recovery.rs b/crates/tracedecay-lcm/src/payload/delete_recovery.rs index cce5c9b4dc..c07bdea24c 100644 --- a/crates/tracedecay-lcm/src/payload/delete_recovery.rs +++ b/crates/tracedecay-lcm/src/payload/delete_recovery.rs @@ -6,7 +6,8 @@ use super::filesystem_authority::{ ensure_contained, existing_payload_dir_opt, inspect_payload_file_for_delete, read_payload_file_for_verify, remove_verified_payload_file, verify_payload_file_authority, }; -use super::{LcmError, gc, load_payload_metadata, util, validate_payload_ref}; +use super::{LcmError, gc, load_payload_metadata, validate_payload_ref}; +use tracedecay_domain::canonical_text::sha256_hex; #[cfg(test)] use tracedecay_runtime_core::db::engine::{Connection, TransactionBehavior}; use tracedecay_runtime_core::db::engine::{Executor, Value as SqlValue, params}; @@ -345,7 +346,7 @@ where }; let quarantine_name = format!( ".tracedecay-pending-delete-{}", - util::sha256_hex(payload_ref.as_bytes()) + sha256_hex(payload_ref.as_bytes()) ); let quarantine = dir.join(&quarantine_name); ensure_contained(&dir, &quarantine)?; @@ -438,7 +439,7 @@ pub fn payload_file_fingerprint( }; let text = std::str::from_utf8(&content).map_err(|_| LcmError::PayloadIntegrityMismatch)?; Ok(( - util::sha256_hex(&content), + sha256_hex(&content), content.len() as u64, text.chars().count() as u64, )) @@ -458,7 +459,8 @@ async fn tombstone_residual_placeholders( let like_patterns = gc::live_prefix_ref_like_patterns(payload_ref); let like_sql = gc::placeholder_text_like_sql(like_patterns.len()); let sql = format!( - "SELECT store_id, storage_kind, payload_ref, content, snippet_text, index_text, metadata_json + "SELECT store_id, storage_kind, payload_ref, content, snippet_text, index_text, metadata_json, + placeholder_text FROM lcm_raw_messages WHERE payload_ref = ? OR {like_sql}" ); @@ -482,14 +484,15 @@ async fn tombstone_residual_placeholders( } tombstoned }); - let new_snippet = gc::tombstone_placeholder_in_text(&snippet_text, payload_ref); - if new_snippet != snippet_text { + if gc::tombstone_placeholder_in_text(&snippet_text, payload_ref) != snippet_text { changed += 1; } - let new_index = gc::tombstone_placeholder_in_text(&index_text, payload_ref); - if new_index != index_text { + if gc::tombstone_placeholder_in_text(&index_text, payload_ref) != index_text { changed += 1; } + let placeholder_text: Option = row.get(7)?; + let new_placeholder = + placeholder_text.map(|text| gc::tombstone_placeholder_in_text(&text, payload_ref)); let new_metadata = metadata_json.map(|text| { let tombstoned = gc::tombstone_placeholder_in_text(&text, payload_ref); if tombstoned != text { @@ -509,8 +512,7 @@ async fn tombstone_residual_placeholders( store_id, clear_raw_ref, new_content, - new_snippet, - new_index, + new_placeholder, new_metadata, changed, )); @@ -518,19 +520,16 @@ async fn tombstone_residual_placeholders( } let mut changed_total = 0usize; - for (store_id, clear_raw_ref, content, snippet_text, index_text, metadata_json, changed) in - updates - { + for (store_id, clear_raw_ref, content, placeholder_text, metadata_json, changed) in updates { if clear_raw_ref { conn.execute( "UPDATE lcm_raw_messages - SET storage_kind = 'inline', payload_ref = NULL, content = ?2, snippet_text = ?3, index_text = ?4, metadata_json = ?5 + SET storage_kind = 'inline', payload_ref = NULL, content = ?2, placeholder_text = ?3, metadata_json = ?4 WHERE store_id = ?1", params![ store_id, content.as_deref(), - snippet_text, - index_text, + placeholder_text.as_deref(), metadata_json.as_deref() ], ) @@ -538,13 +537,12 @@ async fn tombstone_residual_placeholders( } else { conn.execute( "UPDATE lcm_raw_messages - SET content = ?2, snippet_text = ?3, index_text = ?4, metadata_json = ?5 + SET content = ?2, placeholder_text = ?3, metadata_json = ?4 WHERE store_id = ?1", params![ store_id, content.as_deref(), - snippet_text, - index_text, + placeholder_text.as_deref(), metadata_json.as_deref() ], ) diff --git a/crates/tracedecay-lcm/src/payload/filesystem_authority.rs b/crates/tracedecay-lcm/src/payload/filesystem_authority.rs index 487d617319..2dc8a7c41a 100644 --- a/crates/tracedecay-lcm/src/payload/filesystem_authority.rs +++ b/crates/tracedecay-lcm/src/payload/filesystem_authority.rs @@ -1372,11 +1372,13 @@ mod windows_tests { #[cfg(test)] mod authority_tests { + use tracedecay_domain::canonical_text::sha256_hex; + use super::*; fn expectation(content: &str) -> (String, u64, u64) { ( - super::super::util::sha256_hex(content.as_bytes()), + sha256_hex(content.as_bytes()), content.len() as u64, content.chars().count() as u64, ) diff --git a/crates/tracedecay-lcm/src/payload/filesystem_authority/payload_stream.rs b/crates/tracedecay-lcm/src/payload/filesystem_authority/payload_stream.rs index ef049b3b7f..ed135de9d5 100644 --- a/crates/tracedecay-lcm/src/payload/filesystem_authority/payload_stream.rs +++ b/crates/tracedecay-lcm/src/payload/filesystem_authority/payload_stream.rs @@ -218,7 +218,7 @@ fn io_error(error: std::io::Error) -> LcmError { #[cfg(test)] mod tests { use super::*; - use crate::util::sha256_hex; + use tracedecay_domain::canonical_text::sha256_hex; const WINDOW: usize = 4 * 1024; @@ -427,13 +427,34 @@ mod tests { let mut rewritten = content.clone(); rewritten[WINDOW + 1] ^= 0x20; - fs::write(&path, &rewritten).unwrap(); + if rewrite_while_held(&path, &rewritten) { + let error = collect(stream, &mut window).unwrap_err(); + assert_eq!( + error, + PayloadStreamError::Payload(LcmError::PayloadIntegrityMismatch) + ); + } else { + assert_eq!(collect(stream, &mut window).unwrap().0, content); + } + } - let error = collect(stream, &mut window).unwrap_err(); - assert_eq!( - error, - PayloadStreamError::Payload(LcmError::PayloadIntegrityMismatch) - ); + /// Rewrites the payload in place while a proven stream holds it, reporting + /// whether the write landed. The stream's Windows handle shares only read + /// and delete access, so there the writer is refused with a sharing + /// violation and the proven bytes cannot change under the stream. + fn rewrite_while_held(path: &Path, bytes: &[u8]) -> bool { + const ERROR_SHARING_VIOLATION: i32 = 32; + match fs::write(path, bytes) { + Ok(()) if !cfg!(windows) => true, + Ok(()) => panic!("a held payload must exclude writers"), + Err(error) => { + assert!( + cfg!(windows) && error.raw_os_error() == Some(ERROR_SHARING_VIOLATION), + "unexpected rewrite failure: {error}" + ); + false + } + } } #[test] @@ -445,32 +466,37 @@ mod tests { let mut window = vec![0_u8; WINDOW]; let stream = open(&path, &content, &mut window).unwrap().unwrap(); - fs::write(&path, &content[..WINDOW]).unwrap(); + let truncated = rewrite_while_held(&path, &content[..WINDOW]); let mut emitted = 0; - let error = stream - .emit(&mut window, &mut ok, &mut |chunk| { - emitted += chunk.len(); - Ok(()) - }) - .unwrap_err(); - assert_eq!( - error, - PayloadStreamError::Payload(LcmError::PayloadIntegrityMismatch) - ); - assert_eq!( - emitted, 0, - "a size change is refused before the first window" - ); + let result = stream.emit(&mut window, &mut ok, &mut |chunk| { + emitted += chunk.len(); + Ok(()) + }); + if truncated { + assert_eq!( + result.unwrap_err(), + PayloadStreamError::Payload(LcmError::PayloadIntegrityMismatch) + ); + assert_eq!( + emitted, 0, + "a size change is refused before the first window" + ); + } else { + assert_eq!(result.unwrap(), content.len() as u64); + } fs::write(&path, &content).unwrap(); let stream = open(&path, &content, &mut window).unwrap().unwrap(); let mut grown = content.clone(); grown.extend_from_slice(b"!"); - fs::write(&path, &grown).unwrap(); - assert_eq!( - collect(stream, &mut window).unwrap_err(), - PayloadStreamError::Payload(LcmError::PayloadIntegrityMismatch) - ); + if rewrite_while_held(&path, &grown) { + assert_eq!( + collect(stream, &mut window).unwrap_err(), + PayloadStreamError::Payload(LcmError::PayloadIntegrityMismatch) + ); + } else { + assert_eq!(collect(stream, &mut window).unwrap().0, content); + } } #[test] diff --git a/crates/tracedecay-lcm/src/payload/filesystem_authority/verified_read.rs b/crates/tracedecay-lcm/src/payload/filesystem_authority/verified_read.rs index 36c46b6f3a..6732d112f9 100644 --- a/crates/tracedecay-lcm/src/payload/filesystem_authority/verified_read.rs +++ b/crates/tracedecay-lcm/src/payload/filesystem_authority/verified_read.rs @@ -260,6 +260,7 @@ impl Utf8State { mod tests { use super::*; use std::fs; + use tracedecay_domain::canonical_text::sha256_hex; #[test] fn utf8_scalar_split_at_verification_window_is_accepted_once() { @@ -315,7 +316,7 @@ mod tests { let path = temp.path().join("payload.payload"); let content = vec![b'x'; 512 * 1024]; fs::write(&path, &content).unwrap(); - let hash = super::super::super::util::sha256_hex(&content); + let hash = sha256_hex(&content); let mut checkpoints = 0; let error = read_verified_payload_file_with_checkpoint( &path, diff --git a/crates/tracedecay-lcm/src/payload/rollback_tests.rs b/crates/tracedecay-lcm/src/payload/rollback_tests.rs index 72de88e4b4..481d4431d2 100644 --- a/crates/tracedecay-lcm/src/payload/rollback_tests.rs +++ b/crates/tracedecay-lcm/src/payload/rollback_tests.rs @@ -72,17 +72,6 @@ async fn direct_store_failure_rolls_back_metadata_and_payload_file() { project_key TEXT NOT NULL, project_path TEXT NOT NULL, PRIMARY KEY(provider, session_id) - ); - CREATE TABLE session_messages ( - provider TEXT NOT NULL, - message_id TEXT NOT NULL, - session_id TEXT NOT NULL, - role TEXT NOT NULL, - timestamp INTEGER, - ordinal INTEGER NOT NULL, - text TEXT NOT NULL, - metadata_json TEXT, - PRIMARY KEY(provider, message_id) );", ) .await diff --git a/crates/tracedecay-lcm/src/payload/tombstone_probe_tests.rs b/crates/tracedecay-lcm/src/payload/tombstone_probe_tests.rs index b7a1c1771d..9d1136c7af 100644 --- a/crates/tracedecay-lcm/src/payload/tombstone_probe_tests.rs +++ b/crates/tracedecay-lcm/src/payload/tombstone_probe_tests.rs @@ -105,19 +105,6 @@ async fn probe_store() -> ProbeStore { title TEXT, started_at INTEGER, PRIMARY KEY(provider, session_id) - ); - CREATE TABLE IF NOT EXISTS session_messages ( - provider TEXT NOT NULL, - message_id TEXT NOT NULL, - session_id TEXT NOT NULL, - role TEXT NOT NULL, - timestamp INTEGER, - ordinal INTEGER NOT NULL, - text TEXT NOT NULL, - metadata_json TEXT, - PRIMARY KEY(provider, message_id), - FOREIGN KEY(provider, session_id) - REFERENCES sessions(provider, session_id) ON DELETE CASCADE );", ) .await @@ -149,11 +136,9 @@ async fn seed_raw_messages(conn: &Connection, texts: &[String]) { batch.push_str(&format!( "INSERT INTO lcm_raw_messages ( provider, message_id, session_id, role, ordinal, timestamp, - content, content_hash, storage_kind, payload_ref, snippet_text, - index_text, legacy_source, legacy_truncated, metadata_json + content, content_hash, storage_kind, payload_ref, metadata_json ) VALUES ('{PROVIDER}', '{message_id}', '{SESSION}', 'assistant', 1, 2, - {literal}, '{message_id}-hash', 'inline', NULL, {literal}, - {literal}, 0, 0, NULL);\n" + {literal}, '{message_id}-hash', 'inline', NULL, NULL);\n" )); } conn.execute_batch(&batch) diff --git a/crates/tracedecay-lcm/src/query.rs b/crates/tracedecay-lcm/src/query.rs index 59815f9c6f..08848af14c 100644 --- a/crates/tracedecay-lcm/src/query.rs +++ b/crates/tracedecay-lcm/src/query.rs @@ -683,7 +683,17 @@ async fn count_summary_nodes( provider: &str, session_id: Option<&str>, ) -> Result { - util::count_by_provider_session(conn, "lcm_summary_nodes", provider, session_id).await + util::fetch_i64( + conn, + &format!( + "SELECT COUNT(*) FROM session_summary_nodes n + WHERE n.provider = ?1 AND (?2 IS NULL OR n.session_id = ?2) AND {}", + schema::SUMMARY_VISIBLE_SQL + ), + params![provider, util::opt_text(session_id)], + "summary count query returned no rows", + ) + .await } async fn count_external_payloads( @@ -1093,9 +1103,6 @@ mod tests { ) .await .expect("session schema"); - conn.execute_batch(test_support::SESSION_GENERATION_SCHEMA) - .await - .expect("session generation schema"); schema::ensure_lcm_schema(&conn).await.expect("LCM schema"); conn.execute( "INSERT INTO sessions(provider, session_id, project_key, project_path) @@ -1193,7 +1200,7 @@ mod tests { ) INSERT INTO lcm_raw_messages ( provider, message_id, session_id, role, ordinal, timestamp, - content, content_hash, storage_kind, snippet_text, index_text + content, content_hash, storage_kind ) SELECT 'cursor', printf('background-%09d', value), @@ -1203,9 +1210,7 @@ mod tests { value, 'retained background history', printf('hash-%09d', value), - 'inline', - 'retained background history', - 'retained background history' + 'inline' FROM fixture", start = seeded + 1, end = seeded + batch, @@ -1219,14 +1224,12 @@ mod tests { conn.execute( "INSERT INTO lcm_raw_messages ( provider, message_id, session_id, role, ordinal, timestamp, - content, content_hash, storage_kind, snippet_text, index_text + content, content_hash, storage_kind ) VALUES ('cursor', 'direct-user-match', 'session-direct-user', 'user', ?1, ?1, - 'unique:needle direct user', 'direct-user-hash', 'inline', - 'unique:needle direct user', 'unique:needle direct user'), + 'unique:needle direct user', 'direct-user-hash', 'inline'), ('cursor', 'single-session-match', 'session-single', 'assistant', ?2, ?2, - 'unique:needle single session', 'single-session-hash', 'inline', - 'unique:needle single session', 'unique:needle single session')", + 'unique:needle single session', 'single-session-hash', 'inline')", params![rows + 1, rows + 2], ) .await @@ -1375,24 +1378,19 @@ mod tests { "{}alert:marker lossless tail", "filler ".repeat(crate::MAX_DERIVED_TEXT_CHARS) ); - let index_text = crate::derived_text_for_index(&content); assert!( - !index_text.contains("alert:marker"), + !crate::derived_text_for_index(&content).contains("alert:marker"), "fixture must place the exact term beyond the FTS-derived text cap" ); conn.execute( "INSERT INTO lcm_raw_messages ( provider, message_id, session_id, role, ordinal, timestamp, - content, content_hash, storage_kind, snippet_text, index_text + content, content_hash, storage_kind ) VALUES ( 'cursor', 'tail-match', 'session-a', 'assistant', 1, 1, - ?1, 'hash', 'inline', ?2, ?3 + ?1, 'hash', 'inline' )", - params![ - content, - crate::retrieval_content::derived_text_for_snippet(&index_text), - index_text - ], + params![content], ) .await .expect("lossless raw fixture"); @@ -1437,10 +1435,10 @@ mod tests { conn.execute( "INSERT INTO lcm_raw_messages ( provider, message_id, session_id, role, ordinal, timestamp, - content, content_hash, storage_kind, snippet_text, index_text + content, content_hash, storage_kind ) VALUES ( 'cursor', ?1, 'session-a', 'assistant', ?2, ?2, - '雪 candidate', 'hash', 'inline', '雪 candidate', '雪 candidate' + '雪 candidate', 'hash', 'inline' )", params![format!("message-{ordinal}"), ordinal], ) @@ -1735,8 +1733,8 @@ mod tests { let summary_text = format!("summary {ordinal}"); let summary_hash = crate::retrieval_content::projected_content_hash(&summary_text); conn.execute( - "INSERT INTO lcm_summary_nodes ( - node_id, provider, conversation_id, session_id, depth, summary_text, + "INSERT INTO session_summary_nodes ( + summary_id, provider, conversation_id, session_id, depth, summary_text, summary_hash, summary_token_count, source_token_count ) VALUES (?1, 'cursor', 'conversation-a', 'session-a', 0, ?2, ?3, 1, 1)", params![ @@ -1747,6 +1745,7 @@ mod tests { ) .await .expect("summary node"); + test_support::mark_summary_available(&conn, "session-a", &node_id).await; node_ids.push(node_id); } @@ -1785,8 +1784,8 @@ mod tests { ) { let summary_hash = crate::retrieval_content::projected_content_hash(summary_text); conn.execute( - "INSERT INTO lcm_summary_nodes ( - node_id, provider, conversation_id, session_id, depth, summary_text, + "INSERT INTO session_summary_nodes ( + summary_id, provider, conversation_id, session_id, depth, summary_text, summary_hash, summary_token_count, source_token_count, created_at ) VALUES (?1, 'cursor', 'conversation-a', 'session-a', ?2, ?3, ?4, 1, 1, ?5)", params![ @@ -1806,7 +1805,7 @@ mod tests { LcmSourceRef::SummaryNode { node_id } => ("summary_node", node_id.clone()), }; conn.execute( - "INSERT INTO lcm_summary_sources (node_id, source_kind, source_id, ordinal) + "INSERT INTO session_summary_sources (summary_id, source_kind, source_id, ordinal) VALUES (?1, ?2, ?3, ?4)", params![node_id, source_kind, source_id.as_str(), ordinal as i64], ) @@ -1938,8 +1937,8 @@ mod tests { .expect("foreign session"); let foreign_text = "foreign child summary"; conn.execute( - "INSERT INTO lcm_summary_nodes ( - node_id, provider, conversation_id, session_id, depth, summary_text, + "INSERT INTO session_summary_nodes ( + summary_id, provider, conversation_id, session_id, depth, summary_text, summary_hash, summary_token_count, source_token_count, created_at ) VALUES ('child-foreign', 'cursor', 'conversation-b', 'session-foreign', 0, ?1, ?2, 1, 1, 10)", diff --git a/crates/tracedecay-lcm/src/query/describe.rs b/crates/tracedecay-lcm/src/query/describe.rs index c410aafcc5..34cb5f3919 100644 --- a/crates/tracedecay-lcm/src/query/describe.rs +++ b/crates/tracedecay-lcm/src/query/describe.rs @@ -46,14 +46,17 @@ pub(super) async fn summary_overviews( ) -> Result, LcmError> { let mut rows = conn .query( - "SELECT n.node_id, n.conversation_id, n.depth, n.summary_text, n.created_at, - COUNT(s.source_id) - FROM lcm_summary_nodes n - LEFT JOIN lcm_summary_sources s ON s.node_id = n.node_id - WHERE n.provider = ?1 AND n.session_id = ?2 - GROUP BY n.node_id, n.conversation_id, n.depth, n.summary_text, n.created_at - ORDER BY n.depth, n.created_at, n.node_id - LIMIT 20", + &format!( + "SELECT n.summary_id, n.conversation_id, n.depth, n.summary_text, n.created_at, + COUNT(s.source_id) + FROM session_summary_nodes n + LEFT JOIN session_summary_sources s ON s.summary_id = n.summary_id + WHERE n.provider = ?1 AND n.session_id = ?2 AND {} + GROUP BY n.summary_id, n.conversation_id, n.depth, n.summary_text, n.created_at + ORDER BY n.depth, n.created_at, n.summary_id + LIMIT 20", + schema::SUMMARY_VISIBLE_SQL + ), params![provider, session_id], ) .await?; @@ -82,11 +85,14 @@ pub(super) async fn describe_summary_node( ) -> Result { let mut rows = conn .query( - "SELECT node_id, conversation_id, depth, summary_token_count, - source_token_count, source_time_start, source_time_end, - expand_hint, metadata_json, created_at - FROM lcm_summary_nodes - WHERE provider = ?1 AND session_id = ?2 AND node_id = ?3", + &format!( + "SELECT n.summary_id, n.conversation_id, n.depth, n.summary_token_count, + n.source_token_count, n.source_time_start, n.source_time_end, + n.expand_hint, n.metadata_json, n.created_at + FROM session_summary_nodes n + WHERE n.provider = ?1 AND n.session_id = ?2 AND n.summary_id = ?3 AND {}", + schema::SUMMARY_VISIBLE_SQL + ), params![provider, session_id, node_id], ) .await?; @@ -122,8 +128,8 @@ async fn describe_summary_sources( let mut rows = conn .query( "SELECT source_kind, source_id - FROM lcm_summary_sources - WHERE node_id = ?1 + FROM session_summary_sources + WHERE summary_id = ?1 ORDER BY ordinal", params![node_id], ) @@ -246,10 +252,12 @@ async fn load_describe_summary_nodes( if chunk.is_empty() { continue; } + // Children are the lineage of the described (visible) summary and are + // read as published. let sql = format!( - "SELECT node_id, summary_token_count, source_token_count, expand_hint - FROM lcm_summary_nodes - WHERE provider = ? AND session_id = ? AND node_id IN ({})", + "SELECT summary_id, summary_token_count, source_token_count, expand_hint + FROM session_summary_nodes + WHERE provider = ? AND session_id = ? AND summary_id IN ({})", sql_in_placeholders(chunk.len()) ); let mut values = vec![ diff --git a/crates/tracedecay-lcm/src/query/expand.rs b/crates/tracedecay-lcm/src/query/expand.rs index e226c01300..81f5eececf 100644 --- a/crates/tracedecay-lcm/src/query/expand.rs +++ b/crates/tracedecay-lcm/src/query/expand.rs @@ -31,7 +31,6 @@ pub async fn expand( summary_sources: Vec::new(), payload_ref: None, from_current_session: Some(true), - externalized_note: None, source_pagination: None, }) } @@ -64,7 +63,6 @@ pub async fn expand( summary_sources: Vec::new(), payload_ref, from_current_session: Some(from_current_session), - externalized_note: None, source_pagination: None, }) } @@ -92,7 +90,6 @@ pub async fn expand( summary_sources, payload_ref: None, from_current_session: None, - externalized_note: None, source_pagination: Some(source_pagination), }) } @@ -131,7 +128,6 @@ pub async fn expand( summary_sources: Vec::new(), payload_ref: Some(expansion.payload_ref), from_current_session: None, - externalized_note: None, source_pagination: None, }) } diff --git a/crates/tracedecay-lcm/src/query/grep.rs b/crates/tracedecay-lcm/src/query/grep.rs index 1f88161f9b..07193d8d95 100644 --- a/crates/tracedecay-lcm/src/query/grep.rs +++ b/crates/tracedecay-lcm/src/query/grep.rs @@ -222,7 +222,16 @@ pub(super) async fn raw_grep_hits( ) .await; } - let mut values = vec![Value::Text(query_plan.fts_query.clone())]; + let content_query = if query_plan.fts_query.is_empty() { + String::new() + } else { + format!( + "{}({})", + crate::schema::RAW_FTS_CONTENT_COLUMN_FILTER, + query_plan.fts_query + ) + }; + let mut values = vec![Value::Text(content_query)]; let mut filters = Vec::new(); push_grep_provider_filter(request, "r.provider", &mut filters, &mut values); push_raw_grep_filters( @@ -306,12 +315,12 @@ pub(super) async fn summary_grep_hits( }; let order_by = grep_order_by(request.sort, SUMMARY_GREP_RECENCY_EXPR, None); let sql = format!( - "SELECT n.provider, n.session_id, n.node_id, n.summary_text - FROM lcm_summary_nodes_fts - JOIN lcm_summary_nodes n ON n.rowid = lcm_summary_nodes_fts.rowid - WHERE lcm_summary_nodes_fts MATCH ? + "SELECT n.provider, n.session_id, n.summary_id, n.summary_text + FROM session_summary_nodes_fts + JOIN session_summary_nodes n ON n.rowid = session_summary_nodes_fts.rowid + WHERE session_summary_nodes_fts MATCH ? {filter_sql} - ORDER BY {order_by}, n.node_id + ORDER BY {order_by}, n.summary_id LIMIT ?" ); let mut rows = conn.query(&sql, values).await?; @@ -591,10 +600,10 @@ async fn summary_like_grep_hits( values.push(Value::Integer(fetch_limit as i64)); let order_by = grep_order_by(request.sort, SUMMARY_GREP_RECENCY_EXPR, None); let sql = format!( - "SELECT n.provider, n.session_id, n.node_id, n.summary_text, 0.0 AS rank - FROM lcm_summary_nodes n + "SELECT n.provider, n.session_id, n.summary_id, n.summary_text, 0.0 AS rank + FROM session_summary_nodes n WHERE {} - ORDER BY {order_by}, n.node_id + ORDER BY {order_by}, n.summary_id LIMIT ?", filters.join(" AND "), ); @@ -678,22 +687,15 @@ fn push_summary_grep_filters( values: &mut Vec, ) { // Immutable lineage remains retained after supersession; only active, - // available summaries are eligible for current retrieval. - filters.push( - "EXISTS ( - SELECT 1 FROM session_temporal_generations generation - JOIN session_summary_availability availability - ON availability.session_id = generation.session_id - AND availability.generation = generation.generation - WHERE generation.session_id = n.session_id AND generation.state = 'active' - AND availability.summary_id = n.node_id - AND availability.availability = 'available' - ) AND NOT EXISTS ( + // available summaries are eligible for current retrieval, and none while + // the session has an unconverged raw revision. + filters.push(format!( + "{} AND NOT EXISTS ( SELECT 1 FROM lcm_summary_convergence_dirty_raw dirty WHERE dirty.provider = n.provider AND dirty.session_id = n.session_id - )" - .to_string(), - ); + )", + crate::schema::SUMMARY_VISIBLE_SQL + )); if let Some(session_id) = session_id { filters.push("n.session_id = ?".to_string()); values.push(Value::Text(session_id.to_string())); @@ -707,11 +709,11 @@ fn push_summary_grep_filters( filters.push( "EXISTS ( SELECT 1 - FROM lcm_summary_sources ss + FROM session_summary_sources ss JOIN lcm_raw_messages sr ON ss.source_kind = 'raw_message' AND sr.store_id = CAST(ss.source_id AS INTEGER) - WHERE ss.node_id = n.node_id + WHERE ss.summary_id = n.summary_id AND (json_extract(sr.metadata_json, '$.source') = ? OR sr.metadata_json LIKE ?) )" .to_string(), diff --git a/crates/tracedecay-lcm/src/query/payload_health.rs b/crates/tracedecay-lcm/src/query/payload_health.rs index 33c2b05a12..813612cb81 100644 --- a/crates/tracedecay-lcm/src/query/payload_health.rs +++ b/crates/tracedecay-lcm/src/query/payload_health.rs @@ -2,6 +2,7 @@ use std::collections::HashMap; use std::fs; use std::path::Path; +use tracedecay_domain::canonical_text::sha256_hex; use tracedecay_runtime_core::db::engine::{Value, params_from_iter}; use super::scope::LcmScopeSql; @@ -669,7 +670,7 @@ async fn payload_has_integrity_mismatch( return Ok(true); } let bytes = fs::read(&path).map_err(|err| LcmError::Io(err.to_string()))?; - Ok(util::sha256_hex(&bytes) != metadata.content_hash) + Ok(sha256_hex(&bytes) != metadata.content_hash) } fn payload_root_contained(storage_root: &Path) -> bool { diff --git a/crates/tracedecay-lcm/src/query/session.rs b/crates/tracedecay-lcm/src/query/session.rs index d0429bee11..d8c8fdd309 100644 --- a/crates/tracedecay-lcm/src/query/session.rs +++ b/crates/tracedecay-lcm/src/query/session.rs @@ -77,7 +77,7 @@ fn load_session_query(request: &LcmLoadSessionRequest, fetch_limit: usize) -> (S let sql = format!( "SELECT provider, message_id, session_id, store_id, role, ordinal, timestamp, content, content_hash, storage_kind, payload_ref, - snippet_text, legacy_source, legacy_truncated, metadata_json + snippet_text, metadata_json FROM lcm_raw_messages {scope} AND store_id > ? @@ -257,11 +257,14 @@ async fn replay_slice_summary_nodes( } let mut rows = conn .query( - "SELECT node_id, depth, created_at, summary_text, summary_hash - FROM lcm_summary_nodes - WHERE provider = ?1 AND session_id = ?2 - ORDER BY depth DESC, created_at DESC, node_id - LIMIT ?3", + &format!( + "SELECT n.summary_id, n.depth, n.created_at, n.summary_text, n.summary_hash + FROM session_summary_nodes n + WHERE n.provider = ?1 AND n.session_id = ?2 AND {} + ORDER BY n.depth DESC, n.created_at DESC, n.summary_id + LIMIT ?3", + schema::SUMMARY_VISIBLE_SQL + ), params![ request.provider.as_str(), request.session_id.as_str(), @@ -310,8 +313,6 @@ fn load_message_from_raw( content_hash, storage_kind, payload_ref, - legacy_source, - legacy_truncated, metadata_json, } = raw; let (content, content_range) = slice_content_owned(content, slice); @@ -328,8 +329,6 @@ fn load_message_from_raw( content_hash, storage_kind, payload_ref, - legacy_source, - legacy_truncated, metadata_json, } } diff --git a/crates/tracedecay-lcm/src/query/status.rs b/crates/tracedecay-lcm/src/query/status.rs index 957f309cad..f9c83295d7 100644 --- a/crates/tracedecay-lcm/src/query/status.rs +++ b/crates/tracedecay-lcm/src/query/status.rs @@ -39,7 +39,6 @@ struct StatusCounts { maintenance_debt_count: i64, lifecycle_state_count: i64, frontier_count: i64, - legacy_truncated_count: i64, lossy_ingest_records: i64, summary_pending_count: i64, summary_retryable_count: i64, @@ -197,11 +196,12 @@ fn status_counts_query(provider: &str, session_id: Option<&str>) -> (String, Vec let lifecycle_where = lifecycle.where_clause(); let lifecycle_and = lifecycle.and_clause(); let debt_where = debt.where_clause(); + let visible = schema::SUMMARY_VISIBLE_SQL; let sql = format!( "SELECT CASE WHEN EXISTS (SELECT 1 FROM lcm_raw_messages {content_where}) - OR EXISTS (SELECT 1 FROM lcm_summary_nodes {content_where}) + OR EXISTS (SELECT 1 FROM session_summary_nodes n WHERE {visible}{content_and}) OR EXISTS (SELECT 1 FROM lcm_external_payloads {content_where}) OR EXISTS (SELECT 1 FROM lcm_lifecycle_state {lifecycle_where}) THEN 1 ELSE 0 END, @@ -209,8 +209,8 @@ fn status_counts_query(provider: &str, session_id: Option<&str>) -> (String, Vec FROM lcm_raw_messages {content_where}), (SELECT COUNT(*) - FROM lcm_summary_nodes - {content_where}), + FROM session_summary_nodes n + WHERE {visible}{content_and}), (SELECT COUNT(*) FROM lcm_maintenance_debt d JOIN lcm_lifecycle_state s @@ -223,9 +223,6 @@ fn status_counts_query(provider: &str, session_id: Option<&str>) -> (String, Vec (SELECT COUNT(*) FROM lcm_lifecycle_state WHERE current_frontier_store_id IS NOT NULL{lifecycle_and}), - (SELECT COUNT(*) - FROM lcm_raw_messages - WHERE legacy_truncated != 0{content_and}), (SELECT COUNT(*) FROM lcm_raw_messages WHERE metadata_json IS NOT NULL @@ -249,11 +246,11 @@ fn status_counts_query(provider: &str, session_id: Option<&str>) -> (String, Vec WHERE state = 'permanent'{content_and})" ); // Bound in the placeholders' textual order: the four EXISTS probes, the - // raw/summary counts, the debt join, both lifecycle counts, two redaction - // counts, and five disjoint summary-convergence states. - let scopes_in_sql_order: [&LcmScopeSql; 16] = [ + // raw/summary counts, the debt join, both lifecycle counts, the lossy + // ingest count, and five disjoint summary-convergence states. + let scopes_in_sql_order: [&LcmScopeSql; 15] = [ &content, &content, &content, &lifecycle, &content, &content, &debt, &lifecycle, - &lifecycle, &content, &content, &content, &content, &content, &content, &content, + &lifecycle, &content, &content, &content, &content, &content, &content, ]; let mut values = Vec::new(); for scope in scopes_in_sql_order { @@ -281,13 +278,12 @@ async fn status_counts( maintenance_debt_count: row.get(3)?, lifecycle_state_count: row.get(4)?, frontier_count: row.get(5)?, - legacy_truncated_count: row.get(6)?, - lossy_ingest_records: row.get(7)?, - summary_pending_count: row.get(8)?, - summary_retryable_count: row.get(9)?, - summary_current_count: row.get(10)?, - summary_unavailable_count: row.get(11)?, - summary_permanent_count: row.get(12)?, + lossy_ingest_records: row.get(6)?, + summary_pending_count: row.get(7)?, + summary_retryable_count: row.get(8)?, + summary_current_count: row.get(9)?, + summary_unavailable_count: row.get(10)?, + summary_permanent_count: row.get(11)?, }) } @@ -299,7 +295,7 @@ fn status_from_parts( payload_health: PayloadHealthDetail, lifecycle_metadata: LcmLifecycleMetadata, ) -> LcmStatus { - let lossy_records = counts.legacy_truncated_count + counts.lossy_ingest_records; + let lossy_records = counts.lossy_ingest_records; LcmStatus { schema_version, raw_message_count: counts.raw_message_count, @@ -336,7 +332,6 @@ fn status_from_parts( redaction: LcmRedactionStatus { enabled: lossy_records > 0, lossy_records, - legacy_truncated_count: counts.legacy_truncated_count, }, } } @@ -391,7 +386,6 @@ fn merge_lcm_status(target: &mut LcmStatus, source: LcmStatus) { target.summary_convergence.permanent_session_count += source.summary_convergence.permanent_session_count; target.redaction.lossy_records += source.redaction.lossy_records; - target.redaction.legacy_truncated_count += source.redaction.legacy_truncated_count; } #[cfg(test)] @@ -561,7 +555,6 @@ pub(super) fn empty_status(schema_version: i64, gc_config: &LcmGcConfig) -> LcmS redaction: LcmRedactionStatus { enabled: false, lossy_records: 0, - legacy_truncated_count: 0, }, } } @@ -757,17 +750,19 @@ async fn store_message_count( Ok(row.get(0)?) } -/// DAG depth rollup, covered by `idx_lcm_summary_nodes_depth_tokens` so the -/// aggregate never reads `summary_text` records. +/// DAG depth rollup over visible summaries, scoped through +/// `idx_session_summary_nodes_depth_tokens` so the aggregate never reads +/// `summary_text` records. fn dag_status_query(provider: &str, session_id: Option<&str>) -> (String, Vec) { - let scope = LcmScopeSql::new("provider", "session_id", provider, session_id); + let scope = LcmScopeSql::new("n.provider", "n.session_id", provider, session_id); let sql = format!( - "SELECT depth, COUNT(*), SUM(summary_token_count), SUM(source_token_count) - FROM lcm_summary_nodes - {scope} - GROUP BY depth - ORDER BY depth", - scope = scope.where_clause() + "SELECT n.depth, COUNT(*), SUM(n.summary_token_count), SUM(n.source_token_count) + FROM session_summary_nodes n + WHERE {visible}{scope} + GROUP BY n.depth + ORDER BY n.depth", + visible = schema::SUMMARY_VISIBLE_SQL, + scope = scope.and_clause(), ); (sql, scope.into_values()) } @@ -908,22 +903,6 @@ mod tests { transcript_path TEXT, metadata_json TEXT, PRIMARY KEY(provider, session_id) - ); - CREATE TABLE session_messages ( - provider TEXT NOT NULL, - message_id TEXT NOT NULL, - session_id TEXT NOT NULL, - role TEXT NOT NULL, - timestamp INTEGER, - ordinal INTEGER NOT NULL, - text TEXT NOT NULL, - kind TEXT, - model TEXT, - tool_names TEXT, - source_path TEXT, - source_offset INTEGER, - metadata_json TEXT, - PRIMARY KEY(provider, message_id) );", ) .await @@ -951,17 +930,15 @@ mod tests { conn.execute( "INSERT INTO lcm_raw_messages ( provider, message_id, session_id, role, ordinal, timestamp, - content, content_hash, storage_kind, payload_ref, snippet_text, - index_text, legacy_source, legacy_truncated, metadata_json + content, content_hash, storage_kind, payload_ref, metadata_json ) - VALUES (?1, ?2, ?3, 'assistant', 1, 1, ?4, ?5, 'inline', NULL, ?4, ?4, 0, ?6, ?7)", + VALUES (?1, ?2, ?3, 'assistant', 1, 1, ?4, ?5, 'inline', NULL, ?6)", params![ provider.clone(), message_id, session_id.clone(), format!("provider {index} message"), format!("hash-{index:02}"), - i64::from(index.is_multiple_of(2)), if index.is_multiple_of(3) { Some(r#"{"ingest_protection":{"lossy":true}}"#.to_string()) } else { @@ -972,13 +949,13 @@ mod tests { .await .expect("insert raw message"); conn.execute( - "INSERT INTO lcm_summary_nodes ( - node_id, provider, conversation_id, session_id, depth, + "INSERT INTO session_summary_nodes ( + summary_id, provider, conversation_id, session_id, depth, summary_text, summary_hash, summary_token_count, source_token_count ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)", params![ - node_id, + node_id.clone(), provider.clone(), conversation_id.clone(), session_id.clone(), @@ -991,6 +968,8 @@ mod tests { ) .await .expect("insert summary node"); + crate::test_support::seed_active_generation(conn, &session_id).await; + crate::test_support::mark_summary_available(conn, &session_id, &node_id).await; conn.execute( "INSERT INTO lcm_lifecycle_state ( provider, conversation_id, current_session_id, @@ -1157,13 +1136,11 @@ mod tests { ) INSERT INTO lcm_raw_messages ( provider, message_id, session_id, role, ordinal, timestamp, - content, content_hash, storage_kind, payload_ref, snippet_text, - index_text, legacy_source, legacy_truncated, metadata_json + content, content_hash, storage_kind, payload_ref, metadata_json ) SELECT 'cursor', printf('message-%05d', value), 'session-paged-status', 'assistant', value, value, 'one token', - printf('hash-%05d', value), 'inline', NULL, 'one token', - 'one token', 0, 0, NULL + printf('hash-%05d', value), 'inline', NULL, NULL FROM fixture", (), ) @@ -1196,11 +1173,10 @@ mod tests { conn.execute( "INSERT INTO lcm_raw_messages ( provider, message_id, session_id, role, ordinal, timestamp, - content, content_hash, storage_kind, payload_ref, snippet_text, - index_text, legacy_source, legacy_truncated, metadata_json + content, content_hash, storage_kind, payload_ref, metadata_json ) VALUES ( 'cursor', ?1, 'session-byte-budget', 'assistant', ?2, ?2, - ?3, ?4, 'inline', NULL, ?3, ?3, 0, 0, NULL + ?3, ?4, 'inline', NULL, NULL )", params![ format!("byte-budget-message-{ordinal}"), @@ -1245,13 +1221,11 @@ mod tests { ) INSERT INTO lcm_raw_messages ( provider, message_id, session_id, role, ordinal, timestamp, - content, content_hash, storage_kind, payload_ref, snippet_text, - index_text, legacy_source, legacy_truncated, metadata_json + content, content_hash, storage_kind, payload_ref, metadata_json ) SELECT 'cursor', printf('message-%05d', value), ?1, 'assistant', value, value, 'one token', - printf('hash-%05d', value), 'inline', NULL, 'one token', - 'one token', 0, 0, NULL + printf('hash-%05d', value), 'inline', NULL, NULL FROM fixture" ), params![session_id], @@ -1379,14 +1353,12 @@ mod tests { ) INSERT INTO lcm_raw_messages ( provider, message_id, session_id, role, ordinal, timestamp, - content, content_hash, storage_kind, payload_ref, snippet_text, - index_text, legacy_source, legacy_truncated, metadata_json + content, content_hash, storage_kind, payload_ref, metadata_json ) SELECT 'cursor', printf('message-%05d', value), 'session-paged-payloads', 'assistant', value, value, 'one token', printf('hash-%05d', value), 'external', - printf('payload-%05d', value), 'one token', - 'one token', 0, 0, NULL + printf('payload-%05d', value), NULL FROM fixture" ), (), @@ -1451,7 +1423,7 @@ mod tests { for line in plan { for table in [ "lcm_raw_messages", - "lcm_summary_nodes", + "session_summary_nodes", "lcm_external_payloads", ] { if line.contains(table) && line.contains("SCAN") && !line.contains("INDEX") { @@ -1510,13 +1482,12 @@ mod tests { for (provider, session_id) in scopes { let (sql, values) = status_counts_query(provider, session_id); let counts_plan = status_plan_lines(&conn, &sql, values).await; - for partial_index in ["idx_lcm_raw_legacy_truncated", "idx_lcm_raw_lossy_ingest"] { - assert!( - counts_plan.iter().any(|line| line.contains(partial_index)), - "status counts no longer substitute {partial_index} for {provider:?}/{session_id:?}; plan:\n{}", - counts_plan.join("\n") - ); - } + let partial_index = "idx_lcm_raw_lossy_ingest"; + assert!( + counts_plan.iter().any(|line| line.contains(partial_index)), + "status counts no longer substitute {partial_index} for {provider:?}/{session_id:?}; plan:\n{}", + counts_plan.join("\n") + ); } } @@ -1544,8 +1515,7 @@ mod tests { ) INSERT INTO lcm_raw_messages ( provider, message_id, session_id, role, ordinal, timestamp, - content, content_hash, storage_kind, payload_ref, snippet_text, - index_text, legacy_source, legacy_truncated, metadata_json + content, content_hash, storage_kind, payload_ref, metadata_json ) SELECT CASE WHEN (1 + (value % 200)) % 4 = 0 THEN 'claude' ELSE 'cursor' END, printf('message-%09d', value), @@ -1554,9 +1524,6 @@ mod tests { value, value, hex(randomblob(1536)), printf('hash-%09d', value), 'inline', NULL, - 'snippet', 'index text', - 0, - CASE WHEN value % 9000 = 0 THEN 1 ELSE 0 END, CASE WHEN value % 7 = 0 THEN NULL WHEN value % 5000 = 0 THEN @@ -1579,8 +1546,8 @@ mod tests { "WITH RECURSIVE fixture(value) AS ( SELECT 1 UNION ALL SELECT value + 1 FROM fixture WHERE value < {summaries} ) - INSERT INTO lcm_summary_nodes ( - node_id, provider, conversation_id, session_id, depth, + INSERT INTO session_summary_nodes ( + summary_id, provider, conversation_id, session_id, depth, summary_text, summary_hash, summary_token_count, source_token_count ) SELECT printf('node-%09d', value), @@ -1598,6 +1565,16 @@ mod tests { ) .await .expect("seed summary nodes"); + conn.execute_batch( + "INSERT INTO session_temporal_generations(session_id, generation, state) + SELECT DISTINCT session_id, 1, 'active' FROM session_summary_nodes; + INSERT INTO session_summary_availability( + session_id, generation, summary_id, availability + ) + SELECT session_id, 1, summary_id, 'available' FROM session_summary_nodes;", + ) + .await + .expect("seed summary availability"); conn.execute( &format!( "WITH RECURSIVE fixture(value) AS ( @@ -1727,13 +1704,9 @@ mod tests { (SELECT COUNT(*) FROM lcm_raw_messages WHERE (?1 = 'all' OR provider = ?1) AND (?2 IS NULL OR session_id = ?2)), - (SELECT COUNT(*) FROM lcm_summary_nodes + (SELECT COUNT(*) FROM session_summary_nodes WHERE (?1 = 'all' OR provider = ?1) AND (?2 IS NULL OR session_id = ?2)), - (SELECT COUNT(*) FROM lcm_raw_messages - WHERE (?1 = 'all' OR provider = ?1) - AND (?2 IS NULL OR session_id = ?2) - AND legacy_truncated != 0), (SELECT COUNT(*) FROM lcm_raw_messages WHERE (?1 = 'all' OR provider = ?1) AND (?2 IS NULL OR session_id = ?2) @@ -1786,9 +1759,7 @@ mod tests { } conn.execute_batch( - "DROP INDEX IF EXISTS idx_lcm_raw_legacy_truncated; - DROP INDEX IF EXISTS idx_lcm_raw_lossy_ingest; - DROP INDEX IF EXISTS idx_lcm_summary_nodes_depth_tokens; + "DROP INDEX IF EXISTS idx_lcm_raw_lossy_ingest; DROP INDEX IF EXISTS idx_lcm_external_payloads_owner_bytes; CREATE INDEX idx_lcm_external_payloads_owner ON lcm_external_payloads(provider, session_id);", diff --git a/crates/tracedecay-lcm/src/raw.rs b/crates/tracedecay-lcm/src/raw.rs index c228436dde..15b12e7ea0 100644 --- a/crates/tracedecay-lcm/src/raw.rs +++ b/crates/tracedecay-lcm/src/raw.rs @@ -24,11 +24,65 @@ use super::{ pub const RAW_MESSAGE_SELECT_COLUMNS: &str = "provider, message_id, session_id, store_id, role, ordinal, timestamp, content, content_hash, storage_kind, payload_ref, - snippet_text, legacy_source, legacy_truncated, metadata_json"; + snippet_text, metadata_json"; +fn record_select_columns(alias: &str, text: &str, metadata: &str) -> String { + format!( + "{alias}.provider, {alias}.message_id, {alias}.session_id, {alias}.role, + {alias}.timestamp, {alias}.ordinal, {text}, {alias}.kind, {alias}.model, + {alias}.tool_names, {alias}.source_path, {alias}.source_offset, {metadata}" + ) +} + +/// Provider metadata as read surfaces return it: the ingest-protection +/// receipts are storage bookkeeping for the raw authority, not message +/// metadata. +fn served_metadata(alias: &str) -> String { + format!( + "CASE WHEN json_valid({alias}.metadata_json) + THEN NULLIF(json_remove({alias}.metadata_json, '$.ingest_protection'), '{{}}') + ELSE {alias}.metadata_json END" + ) +} + +/// Reads a message row as a [`SessionMessageRecord`] in its field order for +/// search surfaces: the text is the bounded retrieval text (`index_text`), +/// the stored body capped at [`crate::MAX_DERIVED_TEXT_CHARS`], or the +/// placeholder of a body stored outside the row. Lossless bodies load through +/// [`load_raw_message_by_identity`]. +pub fn message_record_select_columns(alias: &str) -> String { + record_select_columns( + alias, + &format!("{alias}.index_text"), + &served_metadata(alias), + ) +} + +/// Reads one message row as a [`SessionMessageRecord`] carrying its whole +/// stored body, or the placeholder of a body stored outside the row. +pub fn message_body_record_select_columns(alias: &str) -> String { + record_select_columns( + alias, + &format!("COALESCE({alias}.content, {alias}.placeholder_text, '')"), + &served_metadata(alias), + ) +} + +/// Reads a message row as the [`SessionMessageRecord`] its writer stored: the +/// whole stored body (or the placeholder of a body stored outside the row) +/// and the protected metadata with its receipts. Writers and verifiers compare +/// and re-ingest through this form. +pub fn stored_message_record_select_columns(alias: &str) -> String { + record_select_columns( + alias, + &format!("COALESCE({alias}.content, {alias}.placeholder_text, '')"), + &format!("{alias}.metadata_json"), + ) +} + pub const RAW_MESSAGE_METADATA_SELECT_COLUMNS: &str = "provider, message_id, session_id, store_id, role, ordinal, timestamp, NULL AS content, content_hash, storage_kind, payload_ref, - '' AS snippet_text, legacy_source, legacy_truncated, metadata_json"; + '' AS snippet_text, metadata_json"; /// Two variables per identity stay below SQLite's default 999-variable ceiling. const RAW_MESSAGE_IDENTITY_BATCH_SIZE: usize = 400; @@ -48,9 +102,7 @@ pub fn raw_message_metadata_from_row(row: &Row) -> Result(12)? != 0, - legacy_truncated: row.get::(13)? != 0, - metadata_json: row.get(14)?, + metadata_json: row.get(12)?, }) } @@ -395,8 +447,6 @@ async fn upsert_inline_raw_message( text: &str, metadata_json: Option<&str>, ) -> Result<(), LcmError> { - let snippet = derived_text_for_snippet(text); - let index = derived_text_for_index(text); let content_hash = projected_content_hash(text); upsert_owned_raw_message( conn, @@ -406,8 +456,7 @@ async fn upsert_inline_raw_message( content_hash: content_hash.as_str(), storage_kind: LcmStorageKind::Inline, payload_ref: None, - snippet: snippet.as_str(), - index_text: index.as_str(), + placeholder: None, metadata_json, }, ) @@ -419,8 +468,9 @@ struct OwnedRawMessageWrite<'a> { content_hash: &'a str, storage_kind: LcmStorageKind, payload_ref: Option<&'a str>, - snippet: &'a str, - index_text: &'a str, + /// Retrieval text of a body stored outside the row; the snippet and + /// index columns derive from it in place of `content`. + placeholder: Option<&'a str>, metadata_json: Option<&'a str>, } @@ -433,10 +483,10 @@ async fn upsert_owned_raw_message( .execute( "INSERT INTO lcm_raw_messages ( provider, message_id, session_id, role, ordinal, timestamp, - content, content_hash, storage_kind, payload_ref, snippet_text, - index_text, legacy_source, legacy_truncated, metadata_json + content, content_hash, storage_kind, payload_ref, placeholder_text, + metadata_json, kind, model, tool_names, source_path, source_offset ) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, 0, 0, ?13) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17) ON CONFLICT(provider, message_id) DO UPDATE SET session_id = excluded.session_id, role = excluded.role, @@ -446,11 +496,13 @@ async fn upsert_owned_raw_message( content_hash = excluded.content_hash, storage_kind = excluded.storage_kind, payload_ref = excluded.payload_ref, - snippet_text = excluded.snippet_text, - index_text = excluded.index_text, - legacy_source = 0, - legacy_truncated = 0, - metadata_json = excluded.metadata_json + placeholder_text = excluded.placeholder_text, + metadata_json = excluded.metadata_json, + kind = excluded.kind, + model = excluded.model, + tool_names = excluded.tool_names, + source_path = excluded.source_path, + source_offset = excluded.source_offset WHERE lcm_raw_messages.session_id = excluded.session_id", params![ message.provider.as_str(), @@ -463,9 +515,13 @@ async fn upsert_owned_raw_message( write.content_hash, write.storage_kind.as_str(), write.payload_ref, - write.snippet, - write.index_text, + write.placeholder, write.metadata_json, + message.kind.as_deref(), + message.model.as_deref(), + message.tool_names.as_deref(), + message.source_path.as_deref(), + message.source_offset, ], ) .await?; @@ -728,6 +784,18 @@ pub async fn upsert_projection_raw_message( conn: &(impl Executor + ?Sized), message: &SessionMessageRecord, ) -> Result<(), LcmError> { + let stored = projection_stored_message(message)?; + upsert_inline_raw_message(conn, message, &stored.text, stored.metadata_json.as_deref()).await +} + +/// The message row exactly as [`upsert_projection_raw_message`] stores it and +/// [`stored_message_record_select_columns`] reads it back: the body is the sanitized +/// text and the metadata carries its ingest-protection receipts. Verifiers +/// compare a stored row against this rather than against the unsanitized +/// projection record. +pub fn projection_stored_message( + message: &SessionMessageRecord, +) -> Result { // `sanitize_lcm_payload_text` is a pure function of the text: a failure // here is a deterministic content refusal, not a storage fault, and must // never be retried as one. @@ -748,14 +816,12 @@ pub async fn upsert_projection_raw_message( quarantine_kind: None, pending_payload_refs: Vec::new(), }; - prepared.metadata_json = protected_metadata_json(message.metadata_json.as_deref(), &prepared)?; - upsert_inline_raw_message( - conn, - message, - &prepared.text, - prepared.metadata_json.as_deref(), - ) - .await + let metadata_json = protected_metadata_json(message.metadata_json.as_deref(), &prepared)?; + Ok(SessionMessageRecord { + text: std::mem::take(&mut prepared.text), + metadata_json, + ..message.clone() + }) } pub fn stage_raw_message_with_payload_tracked( @@ -834,8 +900,7 @@ pub async fn commit_staged_raw_message( content_hash: whole_message.payload_ref.content_hash.as_str(), storage_kind: LcmStorageKind::External, payload_ref: Some(whole_message.payload_ref.payload_ref.as_str()), - snippet: whole_message.placeholder.as_str(), - index_text: whole_message.placeholder.as_str(), + placeholder: Some(whole_message.placeholder.as_str()), metadata_json: Some(whole_message.metadata_json.as_str()), }, ) diff --git a/crates/tracedecay-lcm/src/raw/ingest_protection_defaults_tests.rs b/crates/tracedecay-lcm/src/raw/ingest_protection_defaults_tests.rs index 956a1ba782..c1852d2fc4 100644 --- a/crates/tracedecay-lcm/src/raw/ingest_protection_defaults_tests.rs +++ b/crates/tracedecay-lcm/src/raw/ingest_protection_defaults_tests.rs @@ -13,11 +13,15 @@ const RAW_MESSAGE_TEST_SCHEMA: &str = "CREATE TABLE lcm_raw_messages ( content_hash TEXT NOT NULL, storage_kind TEXT NOT NULL, payload_ref TEXT, - snippet_text TEXT NOT NULL, - index_text TEXT NOT NULL, - legacy_source INTEGER NOT NULL, - legacy_truncated INTEGER NOT NULL, - metadata_json TEXT + placeholder_text TEXT, + snippet_text TEXT NOT NULL DEFAULT '', + index_text TEXT NOT NULL DEFAULT '', + metadata_json TEXT, + kind TEXT, + model TEXT, + tool_names TEXT, + source_path TEXT, + source_offset INTEGER );"; #[tokio::test] @@ -31,11 +35,11 @@ async fn exact_identity_reader_rejects_tampered_inline_content() { "INSERT INTO lcm_raw_messages ( provider, message_id, session_id, role, ordinal, timestamp, content, content_hash, storage_kind, payload_ref, - snippet_text, index_text, legacy_source, legacy_truncated + snippet_text, index_text ) VALUES ( 'cursor', 'message-1', 'session-1', 'assistant', 1, 1, 'canary-secret', 'not-the-content-hash', 'inline', NULL, - 'canary-secret', 'canary-secret', 0, 0 + 'canary-secret', 'canary-secret' )", (), ) @@ -58,11 +62,11 @@ async fn exact_identity_reader_rejects_missing_inline_content() { "INSERT INTO lcm_raw_messages ( provider, message_id, session_id, role, ordinal, timestamp, content, content_hash, storage_kind, payload_ref, - snippet_text, index_text, legacy_source, legacy_truncated + snippet_text, index_text ) VALUES ( 'cursor', 'message-1', 'session-1', 'assistant', 1, 1, NULL, 'not-an-empty-content-hash', 'inline', NULL, - '', '', 0, 0 + '', '' )", (), ) @@ -133,11 +137,13 @@ async fn predecessor_range_skips_policy_anchor_roles() { content_hash TEXT NOT NULL, storage_kind TEXT NOT NULL, payload_ref TEXT, - snippet_text TEXT NOT NULL, - index_text TEXT NOT NULL, - legacy_source INTEGER NOT NULL, - legacy_truncated INTEGER NOT NULL, + placeholder_text TEXT, metadata_json TEXT, + kind TEXT, + model TEXT, + tool_names TEXT, + source_path TEXT, + source_offset INTEGER, UNIQUE(provider, message_id) ); CREATE TABLE lcm_raw_predecessor_ranges ( diff --git a/crates/tracedecay-lcm/src/replay_transactions.rs b/crates/tracedecay-lcm/src/replay_transactions.rs index 7cacc7e5af..93e447312a 100644 --- a/crates/tracedecay-lcm/src/replay_transactions.rs +++ b/crates/tracedecay-lcm/src/replay_transactions.rs @@ -300,8 +300,7 @@ fn active_replay_message_from_metadata(message: &LcmRawMessage) -> Option let mut replay = metadata .get(ACTIVE_REPLAY_MESSAGE_KEY) .and_then(Value::as_object) - .cloned() - .or_else(|| legacy_active_replay_message_from_metadata(&metadata))?; + .cloned()?; if !replay.contains_key("content") { replay.insert( "content".to_string(), @@ -335,19 +334,6 @@ pub fn strip_disposable_assistant_replay_sidecars( } } -fn legacy_active_replay_message_from_metadata(metadata: &Value) -> Option> { - let mut replay = metadata.as_object()?.clone(); - replay.remove(ACTIVE_REPLAY_METADATA_KEY); - replay.remove(ACTIVE_REPLAY_MESSAGE_KEY); - replay.remove("ingest_protection"); - replay.remove("external_payload"); - replay.remove("payload_ref"); - replay.remove("byte_count"); - replay.remove("char_count"); - replay.remove("sha256"); - Some(replay) -} - #[cfg(test)] mod tests { use super::*; @@ -366,8 +352,6 @@ mod tests { content_hash: "hash".to_string(), storage_kind: LcmStorageKind::Inline, payload_ref: None, - legacy_source: false, - legacy_truncated: false, metadata_json: Some( json!({ ACTIVE_REPLAY_METADATA_KEY: true, diff --git a/crates/tracedecay-lcm/src/retention.rs b/crates/tracedecay-lcm/src/retention.rs index 88e1c1331c..413840489c 100644 --- a/crates/tracedecay-lcm/src/retention.rs +++ b/crates/tracedecay-lcm/src/retention.rs @@ -1,54 +1,37 @@ //! Projection-durability-aware session retention (plan 38 §3 and §4). //! -//! The session store keeps the *same* conversation content in several places -//! at rest, forever: -//! -//! * `lcm_raw_messages`, the lossless raw ingest of every message. -//! * `session_messages`, the projected/queryable twin of each raw message, -//! keyed by the same `(provider, message_id)`. -//! * FTS shadow tables over each (`lcm_raw_messages_fts`, -//! `session_messages_fts`), maintained by triggers. -//! -//! One observed `sessions.db` reached 15 GB carrying both full copies plus their -//! FTS shadows. Plan 38 §4 ("one content copy") makes carrying both raw and -//! projected content indefinitely a defect: the projection must reference the -//! raw content or be superseded once durable. Plan 38 §3 ("session retention -//! policy") makes raw rows retained only until their LCM projection/summary -//! lineage is durable, then payload-offloaded or dropped under a configurable -//! window, with the projected twin obeying the same window. +//! `lcm_raw_messages` is the one stored copy of each message body; session +//! message readers and the FTS index read that same row. Plan 38 §3 +//! ("session retention policy") keeps raw rows only until their LCM +//! summary lineage is durable, then offloads or drops them under a +//! configurable window. //! //! # Projection durability is the safety invariant //! //! A raw row is *projection-durable* when a summary node's lineage covers it, //! i.e. its `store_id` appears as a `raw_message` source in -//! `lcm_summary_sources` (see -//! `tracedecay_session_temporal_store::operations::summary_projection`, which persists +//! `session_summary_sources` (see +//! `tracedecay_session_temporal_store::operations::publication`, which persists //! `LcmSourceRef::RawMessage { store_id }` as `('raw_message', store_id)`). //! Only projection-durable rows are ever acted on. Rows with no summary lineage //! are live, un-projected evidence and are **never** touched, this is the //! plan's non-goal ("no lossy deletion of live, referenced evidence") expressed //! directly in SQL. //! -//! # One content copy (§4), supersede-after-durability via content addressing +//! # Passes //! -//! This module supersedes the redundant copies rather than duplicating a fourth -//! storage scheme, because the store already owns a content-addressed external -//! payload lifecycle (`lcm_external_payloads` keyed by a content-hash-derived -//! `payload_ref`, with a full reaping GC in [`super::gc`]). The three passes, -//! in reclaim order: +//! Both passes reuse the store's content-addressed external payload lifecycle +//! (`lcm_external_payloads` keyed by a content-hash-derived `payload_ref`, +//! with a full reaping GC in [`super::gc`]). In reclaim order: //! //! 1. **Drop** (terminal, longest window): projection-durable raw rows past -//! `drop_after_days` are deleted along with their projected twin. Any now -//! unreferenced external payload is reaped by the existing payload GC. -//! 2. **Offload** (recoverable, medium window): projection-durable *inline* raw -//! rows past `offload_after_days` have their bulky `content` externalized to -//! the content-addressed store (deduplicated by hash) and replaced with a -//! recoverable placeholder, reclaiming the inline column and its FTS shadow. -//! 3. **Projected dedupe** (shortest window): a projected `session_messages` -//! row is eligible only when its raw twin is still present and that raw row -//! has durable summary lineage. It is then pure duplication of the retained -//! raw copy, so it is dropped, the raw row is the single content copy and -//! the projected form is reconstructable from it. +//! `drop_after_days` are deleted. Any now unreferenced external payload is +//! reaped by the existing payload GC. +//! 2. **Offload** (recoverable, shorter window): projection-durable *inline* +//! raw rows past `offload_after_days` have their bulky `content` +//! externalized to the content-addressed store (deduplicated by hash) and +//! replaced with a recoverable placeholder, reclaiming the inline column and +//! its FTS shadow. //! //! Every pass is bounded (`max_batch_size`) and incremental so the daemon can //! schedule it off the hot path without competing with foreground writes, and a @@ -78,7 +61,7 @@ const SECONDS_PER_DAY: i64 = 24 * 60 * 60; /// the raw row's `store_id` is covered by a durable summary node's lineage. /// `source_id` for a `raw_message` source is the `store_id` rendered as text. const PROJECTION_DURABLE: &str = "EXISTS ( - SELECT 1 FROM lcm_summary_sources s + SELECT 1 FROM session_summary_sources s WHERE s.source_kind = 'raw_message' AND s.source_id = CAST(r.store_id AS TEXT) )"; @@ -87,9 +70,9 @@ const PROJECTION_DURABLE: &str = "EXISTS ( const OFFLOAD_KIND: &str = "retention_offload"; /// Per-table/per-store retention windows for the session store. Defaults keep -/// a six-month recovery horizon for projection-durable raw evidence, offload -/// its bulky inline payload after 30 days, and remove the redundant projected -/// copy after 30 days. Rows without durable summary lineage remain untouched. +/// a six-month recovery horizon for projection-durable raw evidence and +/// offload its bulky inline payload after 30 days. Rows without durable +/// summary lineage remain untouched. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub struct LcmRetentionConfig { /// Master switch. When `false`, [`run_session_retention`] is a @@ -101,15 +84,10 @@ pub struct LcmRetentionConfig { /// offload pass. #[serde(default = "default_offload_after_days")] pub offload_after_days: Option, - /// Window after which a projection-durable raw row (and its projected twin) - /// is dropped. `None` disables the drop pass. + /// Window after which a projection-durable raw row is dropped. `None` + /// disables the drop pass. #[serde(default = "default_drop_after_days")] pub drop_after_days: Option, - /// Window after which a projected `session_messages` row whose raw twin is - /// still present is dropped as a redundant second content copy. `None` - /// disables the dedupe pass. - #[serde(default = "default_dedupe_projected_after_days")] - pub dedupe_projected_after_days: Option, /// Upper bound on rows touched per pass, keeping each run incremental and /// off the hot path. #[serde(default = "default_max_batch_size")] @@ -134,18 +112,12 @@ fn default_drop_after_days() -> Option { Some(180) } -#[allow(clippy::unnecessary_wraps)] -fn default_dedupe_projected_after_days() -> Option { - Some(30) -} - impl Default for LcmRetentionConfig { fn default() -> Self { Self { enabled: default_retention_enabled(), offload_after_days: default_offload_after_days(), drop_after_days: default_drop_after_days(), - dedupe_projected_after_days: default_dedupe_projected_after_days(), max_batch_size: default_max_batch_size(), } } @@ -159,9 +131,7 @@ impl LcmRetentionConfig { /// Whether any pass has a window configured. When false, an enabled pass /// still reports zero work rather than scanning. fn any_window(&self) -> bool { - self.offload_after_days.is_some() - || self.drop_after_days.is_some() - || self.dedupe_projected_after_days.is_some() + self.offload_after_days.is_some() || self.drop_after_days.is_some() } } @@ -213,13 +183,9 @@ pub struct LcmRetentionReport { pub ended_at: i64, pub dropped: LcmRetentionPhaseReport, pub offloaded: LcmRetentionPhaseReport, - pub projected_deduped: LcmRetentionPhaseReport, /// `lcm_raw_messages` row count before/after the run. pub raw_rows_before: u64, pub raw_rows_after: u64, - /// `session_messages` row count before/after the run. - pub projected_rows_before: u64, - pub projected_rows_after: u64, /// Database `PRAGMA freelist_count` before/after (freed pages are the /// measurable, VACUUM-free signal that space was reclaimed). pub freelist_before: u64, @@ -236,7 +202,6 @@ impl LcmRetentionReport { self.dropped .bytes_reclaimed .saturating_add(self.offloaded.bytes_reclaimed) - .saturating_add(self.projected_deduped.bytes_reclaimed) } } @@ -247,8 +212,7 @@ fn cutoff_secs(window_days: u32, now_secs: i64) -> i64 { /// Observe retention-eligible session bytes without mutating the store. /// /// The raw-row query unions the configured drop and offload predicates so a -/// row eligible for both policies is counted once. The projected-row query -/// mirrors the dedupe safety predicate. A configured window emits a zero-byte +/// row eligible for both policies is counted once. A configured window emits a zero-byte /// record when clean, allowing Doctor to distinguish complete clean coverage /// from an unwired source. pub async fn read_session_retention_backlog( @@ -310,37 +274,6 @@ pub async fn read_session_retention_backlog( }); } - if let Some(days) = config.dedupe_projected_after_days { - let watermark = cutoff_secs(days, now); - let sql = format!( - "SELECT MIN(sm.timestamp), - COALESCE(SUM(LENGTH(COALESCE(sm.text, ''))), 0) - FROM session_messages sm - WHERE sm.timestamp IS NOT NULL - AND sm.timestamp < ?1 - AND EXISTS ( - SELECT 1 FROM lcm_raw_messages r - WHERE r.provider = sm.provider - AND r.message_id = sm.message_id - AND {PROJECTION_DURABLE} - )" - ); - let mut rows = conn.query(&sql, params![watermark]).await?; - let row = rows.next().await?.ok_or_else(|| { - LcmError::Db("retention backlog projection aggregate returned no row".to_string()) - })?; - let oldest = row.get::>(0)?.unwrap_or(watermark); - let bytes = row.get::(1)?.max(0) as u64; - records.push(RetentionBacklogRecordV1 { - store, - table: TableNameV1::new("session_messages") - .map_err(|error| LcmError::Db(error.to_string()))?, - past_window_bytes: StorageByteSizeV1(bytes), - oldest_past_window_at: UtcMicros(oldest.saturating_mul(1_000_000)), - window_watermark_at: UtcMicros(watermark.saturating_mul(1_000_000)), - }); - } - Ok(records) } @@ -453,8 +386,6 @@ async fn run_session_retention_inner( ) -> Result { let read = store.read_connection(); let raw_rows_before = scoped_row_count(&read, "lcm_raw_messages", provider, session_id).await; - let projected_rows_before = - scoped_row_count(&read, "session_messages", provider, session_id).await; let freelist_before = pragma_u64(&read, "freelist_count").await; let page_count_before = pragma_u64(&read, "page_count").await; @@ -466,11 +397,8 @@ async fn run_session_retention_inner( ended_at: now, dropped: LcmRetentionPhaseReport::disabled(), offloaded: LcmRetentionPhaseReport::disabled(), - projected_deduped: LcmRetentionPhaseReport::disabled(), raw_rows_before, raw_rows_after: raw_rows_before, - projected_rows_before, - projected_rows_after: projected_rows_before, freelist_before, freelist_after: freelist_before, page_count_before, @@ -481,7 +409,6 @@ async fn run_session_retention_inner( if !config.enabled || !config.any_window() { report.dropped.window_days = config.drop_after_days; report.offloaded.window_days = config.offload_after_days; - report.projected_deduped.window_days = config.dedupe_projected_after_days; return Ok(report); } @@ -510,26 +437,10 @@ async fn run_session_retention_inner( authorize, ) .await?; - report.projected_deduped = run_dedupe_pass( - store, - provider, - session_id, - config, - mode, - now, - &mut report.errors, - authorize, - ) - .await?; - if mode.is_apply() { // Consume the staged GC/reporting meta cards: record the last run so a // scheduler and Doctor can report retention backlog without a rescan. - let acted = report - .dropped - .acted - .saturating_add(report.offloaded.acted) - .saturating_add(report.projected_deduped.acted); + let acted = report.dropped.acted.saturating_add(report.offloaded.acted); let transaction = store .begin_memory_write_transaction("begin session retention metadata", authorize) .await?; @@ -540,8 +451,6 @@ async fn run_session_retention_inner( report.ended_at = now; let read = store.read_connection(); report.raw_rows_after = scoped_row_count(&read, "lcm_raw_messages", provider, session_id).await; - report.projected_rows_after = - scoped_row_count(&read, "session_messages", provider, session_id).await; report.freelist_after = pragma_u64(&read, "freelist_count").await; report.page_count_after = pragma_u64(&read, "page_count").await; crate::metrics::record_lcm_retention(report.bytes_reclaimed()); @@ -751,8 +660,6 @@ impl<'a> RetentionStore<'a> { struct DropRow { store_id: i64, - provider: String, - message_id: String, timestamp: i64, content_len: u64, } @@ -777,8 +684,7 @@ async fn run_drop_pass( }; let cutoff = cutoff_secs(window, now); let sql = format!( - "SELECT r.store_id, r.provider, r.message_id, r.timestamp, - LENGTH(COALESCE(r.content, '')) AS content_len + "SELECT r.store_id, r.timestamp, LENGTH(COALESCE(r.content, '')) AS content_len FROM lcm_raw_messages r WHERE (?1 = 'all' OR r.provider = ?1) AND (?2 IS NULL OR r.session_id = ?2) @@ -816,10 +722,8 @@ async fn run_drop_pass( while let Some(row) = rows.next().await? { targets.push(DropRow { store_id: row.get(0)?, - provider: row.get(1)?, - message_id: row.get(2)?, - timestamp: row.get(3)?, - content_len: row.get::(4)?.max(0) as u64, + timestamp: row.get(1)?, + content_len: row.get::(2)?.max(0) as u64, }); } report.eligible = targets.len() as u64; @@ -836,19 +740,8 @@ async fn run_drop_pass( if let Some(authorize) = authorize { authorize("drop session retention row")?; } - // Drop the projected twin first (its FTS delete trigger fires), then - // the raw row (its FTS delete trigger fires). Any external payload the + // The FTS delete trigger fires with the row. Any external payload the // raw row referenced becomes unreferenced and is reaped by payload GC. - if let Err(err) = txn - .execute( - "DELETE FROM session_messages WHERE provider = ?1 AND message_id = ?2", - params![target.provider.as_str(), target.message_id.as_str()], - ) - .await - { - errors.push(format!("drop projected twin {}: {err}", target.message_id)); - continue; - } match txn .execute( "DELETE FROM lcm_raw_messages WHERE store_id = ?1", @@ -1007,8 +900,7 @@ async fn offload_one( content_hash = ?2, storage_kind = 'external', payload_ref = ?3, - snippet_text = ?4, - index_text = ?4 + placeholder_text = ?4 WHERE r.store_id = ?1 AND r.provider = ?5 AND r.session_id = ?6 @@ -1047,123 +939,5 @@ async fn offload_one( Ok(byte_len) } -#[allow(clippy::too_many_arguments)] -async fn run_dedupe_pass( - store: RetentionStore<'_>, - provider: &str, - session_id: Option<&str>, - config: &LcmRetentionConfig, - mode: RetentionMode, - now: i64, - errors: &mut Vec, - authorize: Option<&RetentionAuthorization<'_>>, -) -> Result { - let mut report = LcmRetentionPhaseReport { - window_days: config.dedupe_projected_after_days, - ..LcmRetentionPhaseReport::default() - }; - let Some(window) = config.dedupe_projected_after_days else { - return Ok(report); - }; - let cutoff = cutoff_secs(window, now); - // Only dedupe a projected row whose raw twin is still present and covered - // by durable summary lineage. A projected row without both proofs remains - // immediately queryable and is never touched here. - let sql = format!( - "SELECT sm.provider, sm.message_id, sm.timestamp, - LENGTH(COALESCE(sm.text, '')) AS text_len - FROM session_messages sm - WHERE (?1 = 'all' OR sm.provider = ?1) - AND (?2 IS NULL OR sm.session_id = ?2) - AND sm.timestamp IS NOT NULL AND sm.timestamp < ?3 - AND EXISTS ( - SELECT 1 FROM lcm_raw_messages r - WHERE r.provider = sm.provider - AND r.message_id = sm.message_id - AND {PROJECTION_DURABLE} - ) - ORDER BY sm.timestamp ASC, sm.message_id ASC - LIMIT ?4" - ); - let transaction = if mode.is_apply() { - Some( - store - .begin_memory_write_transaction("begin session retention dedupe pass", authorize) - .await?, - ) - } else { - None - }; - let read = store.read_connection(); - let query_executor = match transaction.as_ref() { - Some(transaction) => RetentionQueryExecutor::Transaction(transaction), - None => RetentionQueryExecutor::Read(&read), - }; - let mut rows = query_executor - .query( - &sql, - params![ - provider, - util::opt_text(session_id), - cutoff, - config.batch_limit() - ], - ) - .await?; - let mut targets: Vec<(String, String, i64, u64)> = Vec::new(); - while let Some(row) = rows.next().await? { - targets.push(( - row.get(0)?, - row.get(1)?, - row.get(2)?, - row.get::(3)?.max(0) as u64, - )); - } - report.eligible = targets.len() as u64; - report.oldest_eligible_at = targets.iter().map(|(_, _, timestamp, _)| *timestamp).min(); - if !mode.is_apply() { - report.bytes_reclaimed = targets.iter().map(|(_, _, _, len)| *len).sum(); - return Ok(report); - } - - let txn = transaction.ok_or_else(|| { - LcmError::Db("apply mode did not start a session retention dedupe transaction".to_owned()) - })?; - let delete_sql = format!( - "DELETE FROM session_messages - WHERE provider = ?1 - AND message_id = ?2 - AND EXISTS ( - SELECT 1 FROM lcm_raw_messages r - WHERE r.provider = session_messages.provider - AND r.message_id = session_messages.message_id - AND {PROJECTION_DURABLE} - )" - ); - for (provider_val, message_id, _, text_len) in &targets { - if let Some(authorize) = authorize { - authorize("dedupe session retention projected row")?; - } - match txn - .execute( - &delete_sql, - params![provider_val.as_str(), message_id.as_str()], - ) - .await - { - Ok(1) => { - report.acted += 1; - report.bytes_reclaimed = report.bytes_reclaimed.saturating_add(*text_len); - } - Ok(changed) => errors.push(format!( - "dedupe projected {message_id} changed {changed} rows" - )), - Err(err) => errors.push(format!("dedupe projected {message_id}: {err}")), - } - } - commit_authorized(txn, authorize, "commit session retention dedupe pass").await?; - Ok(report) -} - #[cfg(test)] mod tests; diff --git a/crates/tracedecay-lcm/src/retention/tests.rs b/crates/tracedecay-lcm/src/retention/tests.rs index dbd4bf2b18..6c5876a4e2 100644 --- a/crates/tracedecay-lcm/src/retention/tests.rs +++ b/crates/tracedecay-lcm/src/retention/tests.rs @@ -2,6 +2,7 @@ use std::path::PathBuf; use crate::{LcmSourceRef, dag, schema}; use tracedecay_domain::HydrationStateV1; +use tracedecay_domain::canonical_text::sha256_hex; use tracedecay_privacy::sanitize_lcm_payload_text; use tracedecay_runtime_core::db::engine::{ Connection, Executor, IntoParams, QueryExecutor, TestConnection, params, @@ -38,39 +39,7 @@ async fn test_store() -> Result { title TEXT, started_at INTEGER, PRIMARY KEY(provider, session_id) - ); - CREATE TABLE session_messages ( - provider TEXT NOT NULL, - message_id TEXT NOT NULL, - session_id TEXT NOT NULL, - role TEXT NOT NULL, - timestamp INTEGER, - ordinal INTEGER NOT NULL, - text TEXT NOT NULL, - kind TEXT, - model TEXT, - tool_names TEXT, - source_path TEXT, - source_offset INTEGER, - metadata_json TEXT, - PRIMARY KEY(provider, message_id), - FOREIGN KEY(provider, session_id) - REFERENCES sessions(provider, session_id) ON DELETE CASCADE - ); - CREATE VIRTUAL TABLE session_messages_fts USING fts5( - text, role, kind, model, tool_names, - content='session_messages', content_rowid='rowid' - ); - CREATE TRIGGER session_messages_fts_insert - AFTER INSERT ON session_messages BEGIN - INSERT INTO session_messages_fts(rowid, text, role, kind, model, tool_names) - VALUES (NEW.rowid, NEW.text, NEW.role, NEW.kind, NEW.model, NEW.tool_names); - END; - CREATE TRIGGER session_messages_fts_delete - AFTER DELETE ON session_messages BEGIN - INSERT INTO session_messages_fts(session_messages_fts, rowid, text, role, kind, model, tool_names) - VALUES ('delete', OLD.rowid, OLD.text, OLD.role, OLD.kind, OLD.model, OLD.tool_names); - END;", + );", ) .await .map_err(|err| format!("seed sessions schema: {err}"))?; @@ -84,6 +53,7 @@ async fn test_store() -> Result { ) .await .map_err(|err| format!("insert session: {err}"))?; + crate::test_support::seed_active_generation(&runtime, SESSION).await; Ok(TestStore { conn, _runtime: runtime, @@ -92,8 +62,7 @@ async fn test_store() -> Result { }) } -/// Inserts an inline raw message (and its projected `session_messages` twin) -/// with the given age. Returns the assigned `store_id`. The row carries a real +/// Inserts an inline message row with the given age. Returns the assigned `store_id`. The row carries a real /// ingest sanitization receipt so verified reads (summary expansion) accept it. async fn insert_message( conn: &(impl Executor + ?Sized), @@ -106,7 +75,7 @@ async fn insert_message( let sanitization = sanitize_lcm_payload_text(content).map_err(|err| format!("sanitize: {err}"))?; let content = sanitization.sanitized_text(); - let hash = crate::util::sha256_hex(content.as_bytes()); + let hash = sha256_hex(content.as_bytes()); let metadata = serde_json::json!({ "ingest_protection": { "sanitization_receipt": sanitization.receipt() } }) @@ -114,10 +83,9 @@ async fn insert_message( conn.execute( "INSERT INTO lcm_raw_messages ( provider, message_id, session_id, role, ordinal, timestamp, - content, content_hash, storage_kind, payload_ref, snippet_text, - index_text, metadata_json + content, content_hash, storage_kind, payload_ref, metadata_json ) - VALUES (?1, ?2, ?3, 'assistant', ?4, ?5, ?6, ?7, 'inline', NULL, ?6, ?6, ?8)", + VALUES (?1, ?2, ?3, 'assistant', ?4, ?5, ?6, ?7, 'inline', NULL, ?8)", params![ PROVIDER, message_id.as_str(), @@ -131,20 +99,6 @@ async fn insert_message( ) .await .map_err(|err| format!("insert raw: {err}"))?; - conn.execute( - "INSERT INTO session_messages(provider, message_id, session_id, role, timestamp, ordinal, text) - VALUES (?1, ?2, ?3, 'assistant', ?4, ?5, ?6)", - params![ - PROVIDER, - message_id.as_str(), - SESSION, - timestamp, - ordinal, - content - ], - ) - .await - .map_err(|err| format!("insert projected: {err}"))?; let store_id = fetch_i64( conn, "SELECT store_id FROM lcm_raw_messages WHERE provider = ?1 AND message_id = ?2", @@ -163,8 +117,8 @@ async fn make_projection_durable( let node_id = format!("node-{store_id}"); let summary_hash = crate::retrieval_content::projected_content_hash(SUMMARY_TEXT); conn.execute( - "INSERT INTO lcm_summary_nodes( - node_id, provider, conversation_id, session_id, depth, summary_text, + "INSERT INTO session_summary_nodes( + summary_id, provider, conversation_id, session_id, depth, summary_text, summary_hash, summary_token_count, source_token_count ) VALUES (?1, ?2, 'conv', ?3, 0, ?4, ?5, 1, 1)", @@ -179,12 +133,23 @@ async fn make_projection_durable( .await .map_err(|err| format!("insert summary node: {err}"))?; conn.execute( - "INSERT INTO lcm_summary_sources(node_id, source_kind, source_id, ordinal) + "INSERT INTO session_summary_sources(summary_id, source_kind, source_id, ordinal) VALUES (?1, 'raw_message', ?2, 0)", params![node_id.as_str(), store_id.to_string()], ) .await .map_err(|err| format!("insert summary source: {err}"))?; + conn.execute( + "INSERT INTO session_summary_availability(session_id, generation, summary_id, availability) + VALUES (?1, ?2, ?3, 'available')", + params![ + SESSION, + crate::test_support::FIXTURE_GENERATION, + node_id.as_str() + ], + ) + .await + .map_err(|err| format!("insert summary availability: {err}"))?; Ok(node_id) } @@ -210,7 +175,6 @@ fn drop_config(days: u32) -> LcmRetentionConfig { LcmRetentionConfig { enabled: true, drop_after_days: Some(days), - dedupe_projected_after_days: None, ..LcmRetentionConfig::default() } } @@ -286,7 +250,6 @@ async fn authority_loss_before_commit_rolls_back_retention_mutations() -> Result "authority is revoked only after the drop mutations, at precommit" ); assert_eq!(count(&store.conn, "lcm_raw_messages").await?, 1); - assert_eq!(count(&store.conn, "session_messages").await?, 1); Ok(()) } @@ -317,7 +280,7 @@ async fn expansion_survives_sources_dropped_by_retention() -> Result<(), String> ); assert_eq!(count(conn, "lcm_raw_messages").await?, 0); assert_eq!( - count(conn, "lcm_summary_sources").await?, + count(conn, "session_summary_sources").await?, 1, "lineage still points at the dropped store_id" ); @@ -485,99 +448,41 @@ async fn backlog_read_emits_clean_zero_record_for_configured_window() -> Result< Ok(()) } -// (c)-analogue for one-content-copy: the projected twin obeys the window while -// the raw copy is retained, proving raw and projected do not both persist. +// One content copy: a durable row's body is stored once, and dropping the row +// removes its FTS entries with it. #[tokio::test] -async fn dedupe_drops_projected_duplicate_and_keeps_raw() -> Result<(), String> { +async fn drop_removes_the_single_content_copy_and_its_index() -> Result<(), String> { let store = test_store().await?; let conn = &store.conn; - let store_id = insert_message(conn, 1, 90, "duplicated content").await?; + let store_id = insert_message(conn, 1, 90, "stored once").await?; make_projection_durable(conn, store_id).await?; - - let config = LcmRetentionConfig { - enabled: true, - dedupe_projected_after_days: Some(30), - ..LcmRetentionConfig::default() + let live = insert_message(conn, 2, 90, "stored once too").await?; + let indexed = |conn: &Connection| { + let conn = conn.clone(); + async move { + fetch_i64( + &conn, + "SELECT COUNT(*) FROM lcm_raw_messages_fts WHERE lcm_raw_messages_fts MATCH 'stored'", + (), + ) + .await + } }; - let fts_before = count(conn, "session_messages_fts").await?; - let report = run_apply(conn, &store.storage_root, &config).await?; - - assert_eq!(report.projected_deduped.acted, 1); - assert_eq!( - count(conn, "session_messages").await?, - 0, - "projected twin dropped" - ); - assert_eq!( - count(conn, "lcm_raw_messages").await?, - 1, - "raw copy retained" - ); - // The projected FTS shadow obeys the same window (trigger cleaned it). - let fts_after = count(conn, "session_messages_fts").await?; - assert!(fts_after < fts_before, "projected FTS shadow shrank"); - Ok(()) -} + assert_eq!(indexed(conn).await?, 2); -#[tokio::test] -async fn dedupe_retains_projected_copy_until_summary_lineage_is_durable() -> Result<(), String> { - let store = test_store().await?; - let conn = &store.conn; - insert_message(conn, 1, 90, "not durable yet").await?; - - let config = LcmRetentionConfig::default(); - let backlog = read_session_retention_backlog( - conn, - tracedecay_contracts::storage::StoreKeyV1::new("sessions.db") - .map_err(|error| error.to_string())?, - &config, - NOW, - ) - .await - .map_err(|error| error.to_string())?; - let projected = backlog - .iter() - .find(|record| record.table.as_str() == "session_messages") - .ok_or_else(|| "missing projected retention backlog".to_string())?; - assert_eq!(projected.past_window_bytes.get(), 0); + let report = run_apply(conn, &store.storage_root, &drop_config(30)).await?; - let report = run_apply(conn, &store.storage_root, &config).await?; - assert_eq!(report.projected_deduped.eligible, 0); - assert_eq!(report.projected_deduped.acted, 0); + assert_eq!(report.dropped.acted, 1); + assert_eq!(count(conn, "lcm_raw_messages").await?, 1); assert_eq!( - count(conn, "session_messages").await?, - 1, - "non-durable projection remains the only immediately queryable copy" + fetch_i64(conn, "SELECT store_id FROM lcm_raw_messages", ()).await?, + live, + "the row without durable lineage is retained" ); - Ok(()) -} - -// A projected row with NO raw twin is the sole copy and must never be deduped. -#[tokio::test] -async fn dedupe_never_touches_sole_projected_copy() -> Result<(), String> { - let store = test_store().await?; - let conn = &store.conn; - // Insert a projected-only row (no raw twin), aged past the window. - conn.execute( - "INSERT INTO session_messages(provider, message_id, session_id, role, timestamp, ordinal, text) - VALUES (?1, 'lonely', ?2, 'assistant', ?3, 1, 'sole copy')", - params![PROVIDER, SESSION, NOW - 90 * DAY], - ) - .await - .map_err(|e| e.to_string())?; - - let config = LcmRetentionConfig { - enabled: true, - dedupe_projected_after_days: Some(30), - ..LcmRetentionConfig::default() - }; - let report = run_apply(conn, &store.storage_root, &config).await?; - - assert_eq!(report.projected_deduped.acted, 0); assert_eq!( - count(conn, "session_messages").await?, + indexed(conn).await?, 1, - "sole copy retained" + "the dropped row left the FTS index" ); Ok(()) } @@ -643,13 +548,12 @@ async fn offload_cas_preserves_revived_row_and_rolls_back_payload() -> Result<() content: original, }; let revived = "revived content"; - let revived_hash = crate::util::sha256_hex(revived.as_bytes()); + let revived_hash = sha256_hex(revived.as_bytes()); store .conn .execute( "UPDATE lcm_raw_messages - SET timestamp = ?2, content = ?3, content_hash = ?4, - snippet_text = ?3, index_text = ?3 + SET timestamp = ?2, content = ?3, content_hash = ?4 WHERE store_id = ?1", params![store_id, NOW, revived, revived_hash.as_str()], ) diff --git a/crates/tracedecay-lcm/src/retrieval_content.rs b/crates/tracedecay-lcm/src/retrieval_content.rs index 885d0d013b..83003982b5 100644 --- a/crates/tracedecay-lcm/src/retrieval_content.rs +++ b/crates/tracedecay-lcm/src/retrieval_content.rs @@ -228,9 +228,8 @@ mod tests { #[test] fn derived_index_text_caps_without_mutating_source_content() { - // Deterministic replacement for the deleted LCM ingest source-scan - // guard: derived index text is capped through the application contract - // while the authoritative raw payload remains lossless. + // Derived index text is capped while the authoritative raw payload + // remains lossless. let content = format!("{}{}", "a".repeat(300_000), "::lossless-tail"); let derived = derived_text_for_index(&content); assert!( @@ -249,17 +248,6 @@ mod tests { content.chars().count(), 300_000 + "::lossless-tail".chars().count() ); - assert_eq!( - crate::MAX_DERIVED_TEXT_CHARS, - MAX_DERIVED_TEXT_CHARS, - "LCM must re-export the application derived-text cap, not redefine it" - ); - assert_eq!(crate::DERIVED_TRUNCATION_MARKER, DERIVED_TRUNCATION_MARKER); - assert_eq!( - crate::derived_text_for_index(&content), - derived, - "LCM derived_text_for_index must be the application helper" - ); } #[test] diff --git a/crates/tracedecay-lcm/src/schema.rs b/crates/tracedecay-lcm/src/schema.rs index 79643b8585..be68883cd7 100644 --- a/crates/tracedecay-lcm/src/schema.rs +++ b/crates/tracedecay-lcm/src/schema.rs @@ -9,15 +9,48 @@ use super::{LcmError, LcmRawMessage, raw}; #[cfg(test)] use super::util; -pub const LCM_SCHEMA_VERSION: i64 = 8; +/// Message and raw rows carry no copy of their observation envelope; readers +/// join it from the `observations` row. Raw rows store their body once. +/// `snippet_text` and `index_text` are virtual columns computing +/// [`crate::retrieval_content::derived_text_for_snippet`] and +/// [`crate::retrieval_content::derived_text_for_index`] from `content`, or +/// from `placeholder_text` when the body lives outside the row. Each cursor +/// commit deletes the cursor advances the durable cursor strictly supersedes. +/// `lcm_raw_messages` also holds the session message projection, so each +/// message body is stored once beside the session-only columns (`kind`, +/// `model`, `tool_names`, `source_path`, `source_offset`), and one FTS index +/// serves both LCM grep and session message search. There are no LCM summary +/// tables. Every summary read joins the canonical `session_summary_nodes` / +/// `session_summary_sources` authority (session temporal schema) through +/// [`SUMMARY_VISIBLE_SQL`]. Stores at an older version require a profile +/// reset. +pub const LCM_SCHEMA_VERSION: i64 = 13; + +/// Visibility rule for every LCM summary read, over a `session_summary_nodes` +/// row aliased `n`: a summary surfaces iff its availability in the session's +/// active generation is `available`. Retirement writes `unavailable` (and +/// supersession or a raw revision writes `stale`), so the immutable row stays +/// for audit while grep, describe, expand, replay, status, and the DAG stop +/// returning it. There is no other deletion signal. +pub const SUMMARY_VISIBLE_SQL: &str = "EXISTS ( + SELECT 1 + FROM session_temporal_generations visible_generation + JOIN session_summary_availability visible_availability + ON visible_availability.session_id = visible_generation.session_id + AND visible_availability.generation = visible_generation.generation + WHERE visible_generation.session_id = n.session_id + AND visible_generation.state = 'active' + AND visible_availability.summary_id = n.summary_id + AND visible_availability.availability = 'available' +)"; const MIGRATION_NAME: &str = "lcm"; /// Indexes that keep expensive LCM reads off the message-body table pages. /// /// `lcm_status` aggregates whole-store counts on every probe. Without these -/// indexes four of its components scan the full `lcm_raw_messages` / -/// `lcm_summary_nodes` / `lcm_external_payloads` records, multi-gigabyte +/// indexes its components scan the full `lcm_raw_messages` / +/// `lcm_external_payloads` records, multi-gigabyte /// body reads on a long-lived profile store for a one-row answer (issue #767 /// measured 10.65 s daemon-side). Each entry is one independently committed /// idempotent batch. Fresh stores install the final index shape with the @@ -32,9 +65,6 @@ const MIGRATION_NAME: &str = "lcm"; /// live in [`super::query`]; the raw direct-user candidate predicate lives in /// [`super::query::grep`]. pub const LCM_STATUS_PERFORMANCE_INDEX_SQL: &[&str] = &[ - "CREATE INDEX IF NOT EXISTS idx_lcm_raw_legacy_truncated - ON lcm_raw_messages(provider, session_id) - WHERE legacy_truncated != 0;", "CREATE INDEX IF NOT EXISTS idx_lcm_raw_lossy_ingest ON lcm_raw_messages(provider, session_id) WHERE metadata_json IS NOT NULL @@ -48,10 +78,6 @@ pub const LCM_STATUS_PERFORMANCE_INDEX_SQL: &[&str] = &[ "CREATE INDEX IF NOT EXISTS idx_lcm_raw_direct_user_candidate ON lcm_raw_messages(provider, store_id) WHERE role = 'user';", - "CREATE INDEX IF NOT EXISTS idx_lcm_summary_nodes_depth_tokens - ON lcm_summary_nodes( - provider, session_id, depth, summary_token_count, source_token_count - );", // The byte-count variant covers the status COUNT+SUM without touching // payload metadata rows and fully supersedes the plain owner index // (same leading columns), so the replacement and the drop commit as one @@ -61,37 +87,50 @@ pub const LCM_STATUS_PERFORMANCE_INDEX_SQL: &[&str] = &[ DROP INDEX IF EXISTS idx_lcm_external_payloads_owner;", ]; -/// Raw-message FTS structure (schema v3): index only `index_text`, matching -/// hermes-lcm `build_message_fts_spec` (store.py:173-204), which indexes -/// nothing but the message content column. Earlier schemas also indexed -/// `role` and `metadata_json`, so an unqualified MATCH over-matched rows via -/// role names or metadata text. Role and source filtering happen as plain -/// SQL predicates on `lcm_raw_messages`, never through the FTS index. +/// The single message FTS index. Session message search ranks over every +/// column (`bm25` weights 10/2/1/1/1), while LCM grep keeps hermes-lcm +/// `build_message_fts_spec` (store.py:173-204) semantics by qualifying its +/// MATCH with [`RAW_FTS_CONTENT_COLUMN_FILTER`]: an unqualified LCM MATCH would +/// over-match rows through role, kind, model, or tool names. const RAW_FTS_DDL: &str = "CREATE VIRTUAL TABLE IF NOT EXISTS lcm_raw_messages_fts USING fts5( - index_text, + index_text, role, kind, model, tool_names, content='lcm_raw_messages', content_rowid='store_id' ); CREATE TRIGGER IF NOT EXISTS lcm_raw_messages_fts_insert AFTER INSERT ON lcm_raw_messages BEGIN - INSERT INTO lcm_raw_messages_fts(rowid, index_text) - VALUES (NEW.store_id, NEW.index_text); + INSERT INTO lcm_raw_messages_fts(rowid, index_text, role, kind, model, tool_names) + VALUES (NEW.store_id, NEW.index_text, NEW.role, NEW.kind, NEW.model, NEW.tool_names); END; CREATE TRIGGER IF NOT EXISTS lcm_raw_messages_fts_delete AFTER DELETE ON lcm_raw_messages BEGIN - INSERT INTO lcm_raw_messages_fts(lcm_raw_messages_fts, rowid, index_text) - VALUES ('delete', OLD.store_id, OLD.index_text); + INSERT INTO lcm_raw_messages_fts( + lcm_raw_messages_fts, rowid, index_text, role, kind, model, tool_names + ) + VALUES ( + 'delete', OLD.store_id, OLD.index_text, OLD.role, OLD.kind, OLD.model, + OLD.tool_names + ); END; CREATE TRIGGER IF NOT EXISTS lcm_raw_messages_fts_update AFTER UPDATE ON lcm_raw_messages BEGIN - INSERT INTO lcm_raw_messages_fts(lcm_raw_messages_fts, rowid, index_text) - VALUES ('delete', OLD.store_id, OLD.index_text); - INSERT INTO lcm_raw_messages_fts(rowid, index_text) - VALUES (NEW.store_id, NEW.index_text); + INSERT INTO lcm_raw_messages_fts( + lcm_raw_messages_fts, rowid, index_text, role, kind, model, tool_names + ) + VALUES ( + 'delete', OLD.store_id, OLD.index_text, OLD.role, OLD.kind, OLD.model, + OLD.tool_names + ); + INSERT INTO lcm_raw_messages_fts(rowid, index_text, role, kind, model, tool_names) + VALUES (NEW.store_id, NEW.index_text, NEW.role, NEW.kind, NEW.model, NEW.tool_names); END;"; +/// FTS5 column filter that restricts a MATCH to the message body, e.g. +/// `format!("{RAW_FTS_CONTENT_COLUMN_FILTER}({query})")`. +pub const RAW_FTS_CONTENT_COLUMN_FILTER: &str = "index_text : "; + /// Returns whether the raw-message FTS table and all three synchronization -/// triggers use the v3 content-only contracts. +/// triggers use the current five-column contracts. pub async fn raw_fts_structure_is_current(conn: &(impl QueryExecutor + ?Sized)) -> Option { let mut rows = conn .query( @@ -119,41 +158,28 @@ pub async fn raw_fts_structure_is_current(conn: &(impl QueryExecutor + ?Sized)) "lcm_raw_messages_fts" => { table_current = object_type == "table" && sql.contains( - "usingfts5(index_text,content='lcm_raw_messages',content_rowid='store_id')", + "usingfts5(index_text,role,kind,model,tool_names,\ + content='lcm_raw_messages',content_rowid='store_id')", ); } "lcm_raw_messages_fts_insert" => { insert_current = object_type == "trigger" && table_name == "lcm_raw_messages" && sql.contains("afterinsertonlcm_raw_messagesbegin") - && sql.contains( - "insertintolcm_raw_messages_fts(rowid,index_text)\ - values(new.store_id,new.index_text)", - ); + && sql.contains(RAW_FTS_INSERT_NEW); } "lcm_raw_messages_fts_delete" => { delete_current = object_type == "trigger" && table_name == "lcm_raw_messages" && sql.contains("afterdeleteonlcm_raw_messagesbegin") - && sql.contains( - "insertintolcm_raw_messages_fts\ - (lcm_raw_messages_fts,rowid,index_text)\ - values('delete',old.store_id,old.index_text)", - ); + && sql.contains(RAW_FTS_DELETE_OLD); } "lcm_raw_messages_fts_update" => { update_current = object_type == "trigger" && table_name == "lcm_raw_messages" && sql.contains("afterupdateonlcm_raw_messagesbegin") - && sql.contains( - "insertintolcm_raw_messages_fts\ - (lcm_raw_messages_fts,rowid,index_text)\ - values('delete',old.store_id,old.index_text)", - ) - && sql.contains( - "insertintolcm_raw_messages_fts(rowid,index_text)\ - values(new.store_id,new.index_text)", - ); + && sql.contains(RAW_FTS_DELETE_OLD) + && sql.contains(RAW_FTS_INSERT_NEW); } _ => {} } @@ -161,6 +187,13 @@ pub async fn raw_fts_structure_is_current(conn: &(impl QueryExecutor + ?Sized)) Some(table_current && insert_current && delete_current && update_current) } +const RAW_FTS_INSERT_NEW: &str = "insertintolcm_raw_messages_fts\ + (rowid,index_text,role,kind,model,tool_names)\ + values(new.store_id,new.index_text,new.role,new.kind,new.model,new.tool_names)"; +const RAW_FTS_DELETE_OLD: &str = "insertintolcm_raw_messages_fts\ + (lcm_raw_messages_fts,rowid,index_text,role,kind,model,tool_names)\ + values('delete',old.store_id,old.index_text,old.role,old.kind,old.model,old.tool_names)"; + fn compact_sql(sql: &str) -> String { sql.chars() .filter(|character| !character.is_ascii_whitespace()) @@ -194,9 +227,13 @@ pub async fn rebuild_raw_fts(conn: &(impl Executor + ?Sized)) -> Option<()> { } /// Test-only convenience wrapper: production schema creation runs through -/// [`ensure_lcm_schema_in_transaction`] inside the callers' own transactions. +/// [`ensure_lcm_schema_in_transaction`] inside the callers' own transactions, +/// after the session temporal schema this crate's summary reads join against; +/// unit fixtures install that fixture shape here. #[cfg(test)] pub async fn ensure_lcm_schema(conn: &Connection) -> Result<(), LcmError> { + conn.execute_batch(crate::test_support::SESSION_GENERATION_SCHEMA) + .await?; let transaction = conn .transaction_with_behavior(TransactionBehavior::Immediate) .await?; @@ -279,11 +316,29 @@ pub async fn ensure_lcm_schema_in_transaction( content_hash TEXT NOT NULL, storage_kind TEXT NOT NULL CHECK(storage_kind IN ('inline', 'external')), payload_ref TEXT, - snippet_text TEXT NOT NULL, - index_text TEXT NOT NULL, - legacy_source INTEGER NOT NULL DEFAULT 0, - legacy_truncated INTEGER NOT NULL DEFAULT 0, + placeholder_text TEXT, + snippet_text TEXT NOT NULL GENERATED ALWAYS AS ( + CASE + WHEN content IS NULL THEN COALESCE(placeholder_text, '') + WHEN length(content) <= 4096 THEN content + ELSE substr(content, 1, 4054) + || char(10) || '[derived snippet truncated by tracedecay]' + END + ) VIRTUAL, + index_text TEXT NOT NULL GENERATED ALWAYS AS ( + CASE + WHEN content IS NULL THEN COALESCE(placeholder_text, '') + WHEN length(content) <= 65536 THEN content + ELSE substr(content, 1, 65494) + || char(10) || '[derived snippet truncated by tracedecay]' + END + ) VIRTUAL, metadata_json TEXT, + kind TEXT, + model TEXT, + tool_names TEXT, + source_path TEXT, + source_offset INTEGER, UNIQUE(provider, message_id), FOREIGN KEY(provider, session_id) REFERENCES sessions(provider, session_id) ON DELETE CASCADE @@ -292,6 +347,14 @@ pub async fn ensure_lcm_schema_in_transaction( ON lcm_raw_messages(provider, session_id, store_id); CREATE INDEX IF NOT EXISTS idx_lcm_raw_session_id ON lcm_raw_messages(session_id); + CREATE INDEX IF NOT EXISTS idx_lcm_raw_session_ordinal + ON lcm_raw_messages(provider, session_id, ordinal); + CREATE INDEX IF NOT EXISTS idx_lcm_raw_session_activity + ON lcm_raw_messages( + provider, session_id, timestamp, ordinal, message_id, kind, tool_names + ); + CREATE INDEX IF NOT EXISTS idx_lcm_raw_timestamp + ON lcm_raw_messages(timestamp); CREATE TABLE IF NOT EXISTS lcm_external_payloads ( payload_ref TEXT PRIMARY KEY, provider TEXT NOT NULL, @@ -317,72 +380,6 @@ pub async fn ensure_lcm_schema_in_transaction( key TEXT PRIMARY KEY, value TEXT NOT NULL ); - CREATE TABLE IF NOT EXISTS lcm_summary_nodes ( - node_id TEXT PRIMARY KEY, - provider TEXT NOT NULL, - conversation_id TEXT NOT NULL, - session_id TEXT NOT NULL, - depth INTEGER NOT NULL, - summary_text TEXT NOT NULL, - summary_hash TEXT NOT NULL, - summary_token_count INTEGER NOT NULL, - source_token_count INTEGER NOT NULL, - source_time_start INTEGER, - source_time_end INTEGER, - expand_hint TEXT, - metadata_json TEXT, - created_at INTEGER NOT NULL DEFAULT (unixepoch()), - FOREIGN KEY(provider, session_id) - REFERENCES sessions(provider, session_id) ON DELETE CASCADE - ); - CREATE TABLE IF NOT EXISTS lcm_summary_sources ( - node_id TEXT NOT NULL, - source_kind TEXT NOT NULL CHECK(source_kind IN ('raw_message', 'summary_node')), - source_id TEXT NOT NULL, - ordinal INTEGER NOT NULL, - PRIMARY KEY(node_id, ordinal), - FOREIGN KEY(node_id) REFERENCES lcm_summary_nodes(node_id) ON DELETE CASCADE - ); - CREATE INDEX IF NOT EXISTS idx_lcm_summary_nodes_session_depth_time - ON lcm_summary_nodes( - provider, session_id, depth, source_time_start, source_time_end, created_at - ); - CREATE INDEX idx_lcm_summary_nodes_codex_pending_session_order - ON lcm_summary_nodes( - session_id, - (CASE - WHEN json_valid(metadata_json) THEN - json_extract(metadata_json, '$.source') = 'codex_context_compacted' - AND COALESCE( - json_extract(metadata_json, '$.tracedecay_summary_source'), - '' - ) <> 'codex_app_server' - ELSE 0 - END), - depth DESC, - created_at DESC, - node_id - ) - WHERE provider = 'codex'; - CREATE INDEX idx_lcm_summary_nodes_codex_pending_root_order - ON lcm_summary_nodes( - (CASE - WHEN json_valid(metadata_json) THEN - json_extract(metadata_json, '$.source') = 'codex_context_compacted' - AND COALESCE( - json_extract(metadata_json, '$.tracedecay_summary_source'), - '' - ) <> 'codex_app_server' - ELSE 0 - END), - created_at DESC, - depth DESC, - node_id, - session_id - ) - WHERE provider = 'codex'; - CREATE INDEX IF NOT EXISTS idx_lcm_summary_sources_source - ON lcm_summary_sources(source_kind, source_id); CREATE TABLE IF NOT EXISTS lcm_lifecycle_state ( provider TEXT NOT NULL, conversation_id TEXT NOT NULL, @@ -411,33 +408,7 @@ pub async fn ensure_lcm_schema_in_transaction( REFERENCES lcm_lifecycle_state(provider, conversation_id) ON DELETE CASCADE ); CREATE INDEX IF NOT EXISTS idx_lcm_maintenance_debt_kind - ON lcm_maintenance_debt(provider, debt_kind, created_at); - CREATE VIRTUAL TABLE IF NOT EXISTS lcm_summary_nodes_fts USING fts5( - summary_text, expand_hint, metadata_json, - content='lcm_summary_nodes', - content_rowid='rowid' - ); - CREATE TRIGGER IF NOT EXISTS lcm_summary_nodes_fts_insert - AFTER INSERT ON lcm_summary_nodes BEGIN - INSERT INTO lcm_summary_nodes_fts(rowid, summary_text, expand_hint, metadata_json) - VALUES (NEW.rowid, NEW.summary_text, NEW.expand_hint, NEW.metadata_json); - END; - CREATE TRIGGER IF NOT EXISTS lcm_summary_nodes_fts_delete - AFTER DELETE ON lcm_summary_nodes BEGIN - INSERT INTO lcm_summary_nodes_fts( - lcm_summary_nodes_fts, rowid, summary_text, expand_hint, metadata_json - ) - VALUES ('delete', OLD.rowid, OLD.summary_text, OLD.expand_hint, OLD.metadata_json); - END; - CREATE TRIGGER IF NOT EXISTS lcm_summary_nodes_fts_update - AFTER UPDATE ON lcm_summary_nodes BEGIN - INSERT INTO lcm_summary_nodes_fts( - lcm_summary_nodes_fts, rowid, summary_text, expand_hint, metadata_json - ) - VALUES ('delete', OLD.rowid, OLD.summary_text, OLD.expand_hint, OLD.metadata_json); - INSERT INTO lcm_summary_nodes_fts(rowid, summary_text, expand_hint, metadata_json) - VALUES (NEW.rowid, NEW.summary_text, NEW.expand_hint, NEW.metadata_json); - END;", + ON lcm_maintenance_debt(provider, debt_kind, created_at);", ) .await?; ensure_raw_identity_schema(conn).await?; @@ -633,8 +604,6 @@ mod tests { payload_ref TEXT, snippet_text TEXT NOT NULL, index_text TEXT NOT NULL, - legacy_source INTEGER NOT NULL DEFAULT 0, - legacy_truncated INTEGER NOT NULL DEFAULT 0, metadata_json TEXT, UNIQUE(provider, message_id) );", @@ -673,7 +642,11 @@ mod tests { conn.execute_batch( "CREATE TABLE lcm_raw_messages ( store_id INTEGER PRIMARY KEY, - index_text TEXT NOT NULL + index_text TEXT NOT NULL, + role TEXT NOT NULL, + kind TEXT, + model TEXT, + tool_names TEXT );", ) .await @@ -922,9 +895,8 @@ mod tests { .map_err(|error| error.to_string())?; for index in [ - "idx_lcm_raw_legacy_truncated", "idx_lcm_raw_lossy_ingest", - "idx_lcm_summary_nodes_depth_tokens", + "idx_lcm_raw_direct_user_candidate", "idx_lcm_external_payloads_owner_bytes", ] { assert!( @@ -1022,6 +994,89 @@ mod tests { Ok(()) } + #[tokio::test] + async fn raw_retrieval_columns_derive_exactly_what_the_application_derives() + -> Result<(), String> { + let temp = tempfile::tempdir().map_err(|error| error.to_string())?; + let conn = TestConnection::open(&temp.path().join("sessions.db")); + conn.execute_batch( + "CREATE TABLE sessions ( + provider TEXT NOT NULL, + session_id TEXT NOT NULL, + PRIMARY KEY(provider, session_id) + ); + INSERT INTO sessions VALUES ('cursor', 'session-1');", + ) + .await + .map_err(|error| error.to_string())?; + ensure_lcm_schema(&conn) + .await + .map_err(|error| error.to_string())?; + let cap = crate::MAX_DERIVED_TEXT_CHARS; + let snippet_cap = crate::retrieval_content::MAX_DERIVED_SNIPPET_CHARS; + let bodies = [ + String::new(), + "short body".to_owned(), + "a".repeat(snippet_cap), + "a".repeat(snippet_cap + 1), + "雪🦀é".repeat(snippet_cap), + "x".repeat(cap), + "雪".repeat(cap + 1), + ]; + for (ordinal, body) in bodies.iter().enumerate() { + conn.execute( + "INSERT INTO lcm_raw_messages ( + provider, message_id, session_id, role, ordinal, content, + content_hash, storage_kind + ) VALUES ('cursor', ?1, 'session-1', 'user', ?2, ?3, 'hash', 'inline')", + params![format!("inline-{ordinal}"), ordinal as i64, body.as_str()], + ) + .await + .map_err(|error| error.to_string())?; + } + conn.execute( + "INSERT INTO lcm_raw_messages ( + provider, message_id, session_id, role, ordinal, content, + content_hash, storage_kind, payload_ref, placeholder_text + ) VALUES ('cursor', 'external', 'session-1', 'user', 99, NULL, + 'hash', 'external', 'ref', '[payload ref=ref]')", + (), + ) + .await + .map_err(|error| error.to_string())?; + + let mut rows = conn + .query( + "SELECT message_id, content, placeholder_text, snippet_text, index_text + FROM lcm_raw_messages ORDER BY ordinal", + (), + ) + .await + .map_err(|error| error.to_string())?; + let mut checked = 0; + while let Some(row) = rows.next().await.map_err(|error| error.to_string())? { + let message_id: String = row.get(0).map_err(|error| error.to_string())?; + let content: Option = row.get(1).map_err(|error| error.to_string())?; + let placeholder: Option = row.get(2).map_err(|error| error.to_string())?; + let snippet: String = row.get(3).map_err(|error| error.to_string())?; + let index: String = row.get(4).map_err(|error| error.to_string())?; + let source = content.or(placeholder).unwrap_or_default(); + assert_eq!( + snippet, + crate::retrieval_content::derived_text_for_snippet(&source), + "{message_id} snippet" + ); + assert_eq!( + index, + crate::retrieval_content::derived_text_for_index(&source), + "{message_id} index" + ); + checked += 1; + } + assert_eq!(checked, bodies.len() + 1); + Ok(()) + } + async fn sqlite_schema_fingerprint( conn: &Connection, ) -> Result, String> { diff --git a/crates/tracedecay-lcm/src/summarizer.rs b/crates/tracedecay-lcm/src/summarizer.rs index b70f504b28..65485b0dd9 100644 --- a/crates/tracedecay-lcm/src/summarizer.rs +++ b/crates/tracedecay-lcm/src/summarizer.rs @@ -145,8 +145,6 @@ mod tests { content_hash: format!("hash-{store_id}"), storage_kind: LcmStorageKind::Inline, payload_ref: None, - legacy_source: false, - legacy_truncated: false, metadata_json: None, } } diff --git a/crates/tracedecay-lcm/src/summary_convergence.rs b/crates/tracedecay-lcm/src/summary_convergence.rs index fd524b9531..e8a6b2881b 100644 --- a/crates/tracedecay-lcm/src/summary_convergence.rs +++ b/crates/tracedecay-lcm/src/summary_convergence.rs @@ -65,8 +65,6 @@ CREATE INDEX IF NOT EXISTS idx_lcm_summary_convergence_due next_attempt_at_ms, attempt_generation, queue_id ) WHERE state IN ('pending', 'retryable'); -CREATE INDEX IF NOT EXISTS idx_lcm_summary_sources_source_node - ON lcm_summary_sources(source_kind, source_id, node_id); CREATE TRIGGER IF NOT EXISTS lcm_summary_convergence_dirty_raw_seed AFTER INSERT ON lcm_summary_convergence_dirty_raw BEGIN INSERT INTO lcm_summary_convergence_invalidation_work ( diff --git a/crates/tracedecay-lcm/src/summary_convergence_tests.rs b/crates/tracedecay-lcm/src/summary_convergence_tests.rs index 9300d7d556..9e6aa1d9d2 100644 --- a/crates/tracedecay-lcm/src/summary_convergence_tests.rs +++ b/crates/tracedecay-lcm/src/summary_convergence_tests.rs @@ -72,17 +72,6 @@ async fn backfill_page_upserts_each_session_once_and_idles_without_work() { project_path TEXT NOT NULL, PRIMARY KEY(provider, session_id) ); - CREATE TABLE session_messages ( - provider TEXT NOT NULL, - message_id TEXT NOT NULL, - session_id TEXT NOT NULL, - role TEXT NOT NULL, - timestamp INTEGER, - ordinal INTEGER NOT NULL, - text TEXT NOT NULL, - metadata_json TEXT, - PRIMARY KEY(provider, message_id) - ); INSERT INTO sessions(provider, session_id, project_key, project_path) VALUES ('cursor', 'session-a', 'project', '/p'), ('cursor', 'session-b', 'project', '/p'), @@ -102,9 +91,9 @@ async fn backfill_page_upserts_each_session_once_and_idles_without_work() { conn.execute( "INSERT INTO lcm_raw_messages ( provider, message_id, session_id, role, ordinal, content, - content_hash, storage_kind, snippet_text, index_text, metadata_json + content_hash, storage_kind, metadata_json ) VALUES ('cursor', ?1, ?2, 'assistant', ?3, 'body', - ?1, 'inline', 'body', 'body', '{}')", + ?1, 'inline', '{}')", params![format!("message-{ordinal}"), session_for(ordinal), ordinal], ) .await @@ -251,9 +240,9 @@ async fn backfill_page_upserts_each_session_once_and_idles_without_work() { conn.execute( "INSERT INTO lcm_raw_messages ( provider, message_id, session_id, role, ordinal, content, - content_hash, storage_kind, snippet_text, index_text, metadata_json + content_hash, storage_kind, metadata_json ) VALUES ('cursor', 'message-301', 'session-a', 'assistant', 301, 'body', - 'message-301', 'inline', 'body', 'body', '{}')", + 'message-301', 'inline', '{}')", (), ) .await @@ -289,10 +278,9 @@ async fn seed_preserved_role_filter_store(conn: &TestConnection) { conn.execute( "INSERT INTO lcm_raw_messages ( store_id, provider, message_id, session_id, role, ordinal, - content, content_hash, storage_kind, snippet_text, index_text, - metadata_json + content, content_hash, storage_kind, metadata_json ) VALUES (?1, 'claude', ?2, 'preserved', ?3, ?1, 'body', ?2, - 'inline', 'body', 'body', '{}')", + 'inline', '{}')", params![store_id, message_id, role], ) .await @@ -312,17 +300,6 @@ async fn create_session_host_tables(conn: &TestConnection) { project_key TEXT NOT NULL, project_path TEXT NOT NULL, PRIMARY KEY(provider, session_id) - ); - CREATE TABLE session_messages ( - provider TEXT NOT NULL, - message_id TEXT NOT NULL, - session_id TEXT NOT NULL, - role TEXT NOT NULL, - timestamp INTEGER, - ordinal INTEGER NOT NULL, - text TEXT NOT NULL, - metadata_json TEXT, - PRIMARY KEY(provider, message_id) );", ) .await @@ -513,17 +490,6 @@ async fn retained_queue_page_is_keyset_bounded_and_candidate_read_avoids_raw_cor project_path TEXT NOT NULL, PRIMARY KEY(provider, session_id) ); - CREATE TABLE session_messages ( - provider TEXT NOT NULL, - message_id TEXT NOT NULL, - session_id TEXT NOT NULL, - role TEXT NOT NULL, - timestamp INTEGER, - ordinal INTEGER NOT NULL, - text TEXT NOT NULL, - metadata_json TEXT, - PRIMARY KEY(provider, message_id) - ); INSERT INTO sessions(provider, session_id, project_key, project_path) VALUES ('cursor', 'large-corpus', 'project.large', '/large');", ) @@ -541,9 +507,9 @@ async fn retained_queue_page_is_keyset_bounded_and_candidate_read_avoids_raw_cor .execute( "INSERT INTO lcm_raw_messages ( provider, message_id, session_id, role, ordinal, content, - content_hash, storage_kind, snippet_text, index_text, metadata_json + content_hash, storage_kind, metadata_json ) VALUES ('cursor', ?1, 'large-corpus', 'assistant', ?2, 'body', - ?1, 'inline', 'body', 'body', '{}')", + ?1, 'inline', '{}')", params![format!("message-{ordinal}"), ordinal], ) .await @@ -623,17 +589,6 @@ async fn current_profiles_install_the_unreleased_queue_shape_in_place() { project_key TEXT NOT NULL, project_path TEXT NOT NULL, PRIMARY KEY(provider, session_id) - ); - CREATE TABLE session_messages ( - provider TEXT NOT NULL, - message_id TEXT NOT NULL, - session_id TEXT NOT NULL, - role TEXT NOT NULL, - timestamp INTEGER, - ordinal INTEGER NOT NULL, - text TEXT NOT NULL, - metadata_json TEXT, - PRIMARY KEY(provider, message_id) );", ) .await @@ -747,7 +702,6 @@ async fn current_profiles_install_the_unreleased_queue_shape_in_place() { for object in [ "lcm_summary_convergence_invalidation_work", "lcm_summary_convergence_dirty_raw_seed", - "idx_lcm_summary_sources_source_node", ] { let mut rows = conn .query( @@ -790,17 +744,6 @@ async fn protected_content_revision_requeues_a_current_session() { project_path TEXT NOT NULL, PRIMARY KEY(provider, session_id) ); - CREATE TABLE session_messages ( - provider TEXT NOT NULL, - message_id TEXT NOT NULL, - session_id TEXT NOT NULL, - role TEXT NOT NULL, - timestamp INTEGER, - ordinal INTEGER NOT NULL, - text TEXT NOT NULL, - metadata_json TEXT, - PRIMARY KEY(provider, message_id) - ); INSERT INTO sessions(provider, session_id, project_key, project_path) VALUES ('cursor', 'revised-session', 'project.revised', '/revised');", ) @@ -810,9 +753,9 @@ async fn protected_content_revision_requeues_a_current_session() { conn.execute( "INSERT INTO lcm_raw_messages ( provider, message_id, session_id, role, ordinal, content, - content_hash, storage_kind, snippet_text, index_text, metadata_json + content_hash, storage_kind, metadata_json ) VALUES ('cursor', 'message-1', 'revised-session', 'assistant', 1, - 'old content', 'old-hash', 'inline', 'old content', 'old content', + 'old content', 'old-hash', 'inline', '{\"ingest_protection\":{\"sanitization_receipt\":{}}}')", (), ) @@ -843,8 +786,7 @@ async fn protected_content_revision_requeues_a_current_session() { conn.execute( "UPDATE lcm_raw_messages - SET content = 'revised content', content_hash = 'revised-hash', - snippet_text = 'revised content', index_text = 'revised content' + SET content = 'revised content', content_hash = 'revised-hash' WHERE provider = 'cursor' AND message_id = 'message-1'", (), ) @@ -884,17 +826,6 @@ async fn protection_progress_cannot_overwrite_a_concurrent_raw_rewind() { project_path TEXT NOT NULL, PRIMARY KEY(provider, session_id) ); - CREATE TABLE session_messages ( - provider TEXT NOT NULL, - message_id TEXT NOT NULL, - session_id TEXT NOT NULL, - role TEXT NOT NULL, - timestamp INTEGER, - ordinal INTEGER NOT NULL, - text TEXT NOT NULL, - metadata_json TEXT, - PRIMARY KEY(provider, message_id) - ); INSERT INTO sessions(provider, session_id, project_key, project_path) VALUES ('cursor', 'protection-cas', 'project.cas', '/cas');", ) @@ -904,9 +835,9 @@ async fn protection_progress_cannot_overwrite_a_concurrent_raw_rewind() { conn.execute( "INSERT INTO lcm_raw_messages ( provider, message_id, session_id, role, ordinal, content, - content_hash, storage_kind, snippet_text, index_text, metadata_json + content_hash, storage_kind, metadata_json ) VALUES ('cursor', 'message-1', 'protection-cas', 'assistant', 1, - 'old', 'old-hash', 'inline', 'old', 'old', + 'old', 'old-hash', 'inline', '{\"ingest_protection\":{\"sanitization_receipt\":{}}}')", (), ) @@ -953,17 +884,6 @@ async fn disjoint_raw_revisions_drain_as_distinct_restart_safe_work_items() { project_path TEXT NOT NULL, PRIMARY KEY(provider, session_id) ); - CREATE TABLE session_messages ( - provider TEXT NOT NULL, - message_id TEXT NOT NULL, - session_id TEXT NOT NULL, - role TEXT NOT NULL, - timestamp INTEGER, - ordinal INTEGER NOT NULL, - text TEXT NOT NULL, - metadata_json TEXT, - PRIMARY KEY(provider, message_id) - ); INSERT INTO sessions(provider, session_id, project_key, project_path) VALUES ('cursor', 'disjoint-revisions', 'project.revised', '/revised');", ) @@ -974,9 +894,9 @@ async fn disjoint_raw_revisions_drain_as_distinct_restart_safe_work_items() { conn.execute( "INSERT INTO lcm_raw_messages ( provider, message_id, session_id, role, ordinal, content, - content_hash, storage_kind, snippet_text, index_text, metadata_json + content_hash, storage_kind, metadata_json ) VALUES ('cursor', ?1, 'disjoint-revisions', 'assistant', ?2, - ?1, ?1, 'inline', ?1, ?1, + ?1, ?1, 'inline', '{\"ingest_protection\":{\"sanitization_receipt\":{}}}')", params![format!("message-{ordinal}"), ordinal], ) diff --git a/crates/tracedecay-lcm/src/test_support.rs b/crates/tracedecay-lcm/src/test_support.rs index a7c7612503..02dd79d8dc 100644 --- a/crates/tracedecay-lcm/src/test_support.rs +++ b/crates/tracedecay-lcm/src/test_support.rs @@ -4,7 +4,7 @@ use std::path::Path; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; -use tracedecay_runtime_core::db::engine::{TestConnection, Value, params}; +use tracedecay_runtime_core::db::engine::{Executor, Value, params}; fn sqlite_value(value: &Value) -> rusqlite::types::Value { match value { @@ -54,8 +54,10 @@ pub(crate) fn sqlite_vm_steps(database_path: &Path, sql: &str, values: &[Value]) /// The session-temporal tables LCM summary reads join against. They are owned /// by `tracedecay-session-temporal-store`, which this crate cannot depend on, -/// so unit fixtures declare the columns the lineage and availability queries -/// touch and seed one active generation per session. +/// so unit fixtures declare the columns the summary, lineage, and availability +/// queries touch and seed one active generation per session. The canonical +/// publication-only columns (`summary_anchor_id`, `source_horizon_json`, +/// `publication_json`) default here because no LCM read consults them. pub(crate) const SESSION_GENERATION_SCHEMA: &str = "CREATE TABLE IF NOT EXISTS session_temporal_generations ( session_id TEXT NOT NULL, @@ -67,12 +69,52 @@ pub(crate) const SESSION_GENERATION_SCHEMA: &str = generation INTEGER NOT NULL, summary_id TEXT NOT NULL, availability TEXT NOT NULL - );"; + ); + CREATE TABLE IF NOT EXISTS session_summary_nodes ( + summary_id TEXT PRIMARY KEY, + session_id TEXT NOT NULL, + provider TEXT NOT NULL, + conversation_id TEXT NOT NULL, + depth INTEGER NOT NULL, + summary_anchor_id TEXT NOT NULL DEFAULT '', + summary_text TEXT NOT NULL, + summary_hash TEXT NOT NULL, + summary_token_count INTEGER NOT NULL, + source_token_count INTEGER NOT NULL, + source_time_start INTEGER, + source_time_end INTEGER, + expand_hint TEXT, + metadata_json TEXT, + source_horizon_json TEXT NOT NULL DEFAULT '{}', + publication_json TEXT, + created_at INTEGER NOT NULL DEFAULT (unixepoch()) + ); + CREATE INDEX IF NOT EXISTS idx_session_summary_nodes_depth_tokens + ON session_summary_nodes( + provider, session_id, depth, summary_token_count, source_token_count + ); + CREATE TABLE IF NOT EXISTS session_summary_sources ( + summary_id TEXT NOT NULL, + ordinal INTEGER NOT NULL, + source_kind TEXT NOT NULL, + source_id TEXT NOT NULL, + PRIMARY KEY(summary_id, ordinal) + ); + CREATE INDEX IF NOT EXISTS idx_session_summary_sources_source + ON session_summary_sources(source_kind, source_id, summary_id); + CREATE VIRTUAL TABLE IF NOT EXISTS session_summary_nodes_fts USING fts5( + summary_text, content='session_summary_nodes', content_rowid='rowid' + ); + CREATE TRIGGER IF NOT EXISTS session_summary_nodes_fts_insert_v1 + AFTER INSERT ON session_summary_nodes BEGIN + INSERT INTO session_summary_nodes_fts(rowid, summary_text) + VALUES (NEW.rowid, NEW.summary_text); + END;"; /// Generation every fixture session is active in. pub(crate) const FIXTURE_GENERATION: i64 = 1; -pub(crate) async fn seed_active_generation(conn: &TestConnection, session_id: &str) { +pub(crate) async fn seed_active_generation(conn: &(impl Executor + ?Sized), session_id: &str) { conn.execute( "INSERT INTO session_temporal_generations(session_id, generation, state) VALUES (?1, ?2, 'active')", @@ -84,7 +126,11 @@ pub(crate) async fn seed_active_generation(conn: &TestConnection, session_id: &s /// Publish `node_id` as available in the fixture generation; summary reads /// join on this row, so a seeded node without it is invisible by design. -pub(crate) async fn mark_summary_available(conn: &TestConnection, session_id: &str, node_id: &str) { +pub(crate) async fn mark_summary_available( + conn: &(impl Executor + ?Sized), + session_id: &str, + node_id: &str, +) { conn.execute( "INSERT INTO session_summary_availability(session_id, generation, summary_id, availability) VALUES (?1, ?2, ?3, 'available')", diff --git a/crates/tracedecay-lcm/src/types.rs b/crates/tracedecay-lcm/src/types.rs index ad6fa6c71c..dafc49dc82 100644 --- a/crates/tracedecay-lcm/src/types.rs +++ b/crates/tracedecay-lcm/src/types.rs @@ -103,8 +103,6 @@ pub struct LcmLoadSessionMessage { pub content_hash: String, pub storage_kind: LcmStorageKind, pub payload_ref: Option, - pub legacy_source: bool, - pub legacy_truncated: bool, pub metadata_json: Option, } @@ -621,7 +619,6 @@ pub struct LcmLifecycleStatus { pub struct LcmRedactionStatus { pub enabled: bool, pub lossy_records: i64, - pub legacy_truncated_count: i64, } #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] @@ -797,7 +794,6 @@ pub struct LcmCompressionResponse { pub replay_token_estimate: i64, pub replay_over_budget: bool, pub compression_attempts: usize, - pub fallback_used: bool, #[serde(default, skip_serializing_if = "Option::is_none")] pub context_recovery_hint: Option, #[serde(default, skip_serializing_if = "Option::is_none")] diff --git a/crates/tracedecay-lcm/src/util.rs b/crates/tracedecay-lcm/src/util.rs index 56b2e90644..9273f7bfeb 100644 --- a/crates/tracedecay-lcm/src/util.rs +++ b/crates/tracedecay-lcm/src/util.rs @@ -29,10 +29,6 @@ pub fn file_mtime_seconds(metadata: &std::fs::Metadata) -> i64 { .unwrap_or_default() } -pub fn sha256_hex(content: &[u8]) -> String { - tracedecay_domain::canonical_text::sha256_hex(content) -} - pub async fn fetch_i64( conn: &(impl QueryExecutor + ?Sized), sql: &str, diff --git a/crates/tracedecay-lcm/tests/lcm_suite/compression_policy.rs b/crates/tracedecay-lcm/tests/lcm_suite/compression_policy.rs index 176d6da370..3791cb98b6 100644 --- a/crates/tracedecay-lcm/tests/lcm_suite/compression_policy.rs +++ b/crates/tracedecay-lcm/tests/lcm_suite/compression_policy.rs @@ -19,8 +19,6 @@ fn raw_message(store_id: i64, role: &str, content: &str) -> LcmRawMessage { content_hash: format!("hash-{store_id}"), storage_kind: LcmStorageKind::Inline, payload_ref: None, - legacy_source: false, - legacy_truncated: false, metadata_json: None, } } diff --git a/crates/tracedecay-lsp/src/analyzer/client.rs b/crates/tracedecay-lsp/src/analyzer/client.rs index 0e08f3b4fc..b11b12053b 100644 --- a/crates/tracedecay-lsp/src/analyzer/client.rs +++ b/crates/tracedecay-lsp/src/analyzer/client.rs @@ -29,6 +29,7 @@ use tokio::sync::Mutex; use tokio::task::JoinHandle; use tokio_util::codec::{FramedRead, FramedWrite}; use tracedecay_daemon_protocol::{ConnectionLocalRequestSequence, FramePoll}; +use tracedecay_runtime_core::path_safety::plain_host_path; use super::broker::{CodeDiagnostic, DiagnosticSeverity}; use super::error::{ @@ -1335,7 +1336,9 @@ fn lsp_initialization_options(command: &str) -> Value { /// percent-encoding. Handles POSIX paths, Windows drive paths (`C:/…`), and UNC /// (`//server/share`) prefixes. Shared with the Kiro installer. pub fn file_uri_from_path_text(path: &str) -> String { - let normalized = path.replace('\\', "/"); + let normalized = plain_host_path(Path::new(path)) + .to_string_lossy() + .replace('\\', "/"); let encoded = percent_encode_file_uri_path(&normalized); if normalized.starts_with("//") { format!("file:{encoded}") @@ -1508,6 +1511,11 @@ mod tests { file_uri_from_path_text("/tmp/100% real.rs"), "file:///tmp/100%25%20real.rs" ); + assert_eq!( + file_uri_from_path_text(r"\\?\D:\repo\src\main.rs"), + "file:///D:/repo/src/main.rs", + "a canonicalized Windows path must not become a `?` URL host" + ); } #[test] diff --git a/crates/tracedecay-lsp/src/analyzer/settings.rs b/crates/tracedecay-lsp/src/analyzer/settings.rs index cc76faf835..f34d433973 100644 --- a/crates/tracedecay-lsp/src/analyzer/settings.rs +++ b/crates/tracedecay-lsp/src/analyzer/settings.rs @@ -126,16 +126,6 @@ pub async fn save_settings( "failed to serialize code diagnostics settings: {error}" )) })?; - if tokio::fs::try_exists(&path).await.unwrap_or(false) { - let backup = path.with_extension("json.bak"); - tokio::fs::copy(&path, &backup).await.map_err(|error| { - AnalyzerRuntimeError::new(format!( - "failed to back up code diagnostics settings '{}' to '{}': {error}", - path.display(), - backup.display() - )) - })?; - } let staged = path.with_extension("json.pending"); let publish_path = path.clone(); tokio::task::spawn_blocking(move || publish_settings(&staged, &publish_path, &bytes)) @@ -267,9 +257,11 @@ mod tests { save_settings(temp.path(), &replacement).await.unwrap(); assert_eq!(load_settings(temp.path()).await.unwrap(), replacement); - let backup = settings_path(temp.path()).with_extension("json.bak"); - let backup: CodeDiagnosticsSettings = - serde_json::from_slice(&tokio::fs::read(backup).await.unwrap()).unwrap(); - assert_eq!(backup, initial); + assert!( + !settings_path(temp.path()) + .with_extension("json.bak") + .exists(), + "replacing settings keeps no copy of the prior bytes" + ); } } diff --git a/crates/tracedecay-lsp/src/catalog.rs b/crates/tracedecay-lsp/src/catalog.rs index 5fe989f090..e861bfd8de 100644 --- a/crates/tracedecay-lsp/src/catalog.rs +++ b/crates/tracedecay-lsp/src/catalog.rs @@ -6,9 +6,7 @@ use tracedecay_contracts::{ ApplicationContractError, ApplicationHandlerDescriptors, application_catalog_contributions, application_handler_descriptors, }; -use tracedecay_tool_catalog::{ - BindingId, BindingStatus, BindingSurface, CatalogContributionV1, SurfaceBindingV1, -}; +use tracedecay_tool_catalog::{BindingId, BindingSurface, CatalogContributionV1, SurfaceBindingV1}; use crate::dispatch::LspClientMethod; @@ -80,9 +78,7 @@ impl LspCatalogAdmission { operation.to_owned(), )); } - if matches!(binding.status(), BindingStatus::Current) - && !binding.is_alias() - && binding.protocol_revisions().contains(1) + if binding.protocol_revisions().contains(1) && admit_binding(contribution, binding, handlers).is_err() { return Err(LspCatalogAdmissionError::BindingUnavailable( @@ -123,10 +119,7 @@ fn admit_binding( binding: &SurfaceBindingV1, handlers: &ApplicationHandlerDescriptors, ) -> Result<(), LspCatalogBindingRejection> { - if !matches!(binding.status(), BindingStatus::Current) - || binding.is_alias() - || !binding.protocol_revisions().contains(1) - { + if !binding.protocol_revisions().contains(1) { return Err(LspCatalogBindingRejection::Stale); } let capability = contribution @@ -156,8 +149,8 @@ mod tests { application_catalog_contributions, application_handler_descriptors, }; use tracedecay_tool_catalog::{ - BindingDeprecation, BindingStatus, BindingSurface, CatalogContributionInputV1, - CatalogContributionV1, SurfaceBindingInputV1, SurfaceBindingV1, + BindingSurface, CatalogContributionInputV1, CatalogContributionV1, ProtocolRevisionRange, + SurfaceBindingInputV1, SurfaceBindingV1, }; use super::{LspCatalogAdmission, LspCatalogBindingRejection}; @@ -178,9 +171,8 @@ mod tests { #[test] fn stale_context_binding_is_rejected() { - let contributions = context_binding_fixture(Some(BindingStatus::Deprecated { - deprecation: BindingDeprecation::new(2).unwrap(), - })); + let contributions = + context_binding_fixture(Some(ProtocolRevisionRange::new(2, 2).unwrap())); let handlers = application_handler_descriptors().unwrap(); let admission = LspCatalogAdmission::from_parts(&contributions, &handlers).unwrap(); @@ -190,7 +182,9 @@ mod tests { ); } - fn context_binding_fixture(status: Option) -> Vec { + fn context_binding_fixture( + protocol_revisions: Option, + ) -> Vec { let mut contributions = application_catalog_contributions().unwrap(); let context = contributions .iter_mut() @@ -203,17 +197,15 @@ mod tests { .iter() .position(|binding| binding.operation().as_str() == CONTEXT_METHOD) .unwrap(); - if let Some(status) = status { + if let Some(protocol_revisions) = protocol_revisions { let binding = &bindings[index]; bindings[index] = SurfaceBindingV1::new(SurfaceBindingInputV1 { binding_id: binding.binding_id().clone(), capability_id: binding.capability_id().clone(), surface: BindingSurface::Lsp, operation: binding.operation().clone(), - protocol_revisions: binding.protocol_revisions().clone(), + protocol_revisions, required_features: binding.required_features().to_vec(), - status, - alias_of: binding.alias_of().cloned(), }) .unwrap(); } else { diff --git a/crates/tracedecay-lsp/src/overlay/retained_parse.rs b/crates/tracedecay-lsp/src/overlay/retained_parse.rs index 6cc40a8c3c..f8683a9c9e 100644 --- a/crates/tracedecay-lsp/src/overlay/retained_parse.rs +++ b/crates/tracedecay-lsp/src/overlay/retained_parse.rs @@ -2,6 +2,7 @@ use std::sync::{Arc, OnceLock}; +use tracedecay_code_extraction::ExtractionArtifactV1; use tracedecay_code_extraction::incremental::{ ParseDocumentIdentity, ParseError, ParseInputEdit, ParseLimits, ParseReport, RetainedParseDocument, @@ -97,7 +98,7 @@ impl PartialEq for OverlayExtractionState { pub(super) struct RetainedOverlayParse { document: Option, parse_state: OverlayParseState, - prior_raw_extraction: Option>, + prior_artifact: Option, extraction_state: OverlayExtractionState, } @@ -121,7 +122,7 @@ impl RetainedOverlayParse { let mut retained = Self { document: Some(document), parse_state: OverlayParseState::Ready(Box::new(report.clone())), - prior_raw_extraction: None, + prior_artifact: None, extraction_state: OverlayExtractionState::Unavailable( OverlayParseUnavailable::StaleReport, ), @@ -195,15 +196,15 @@ impl RetainedOverlayParse { #[hotpath::measure(label = "lsp_overlay_extract", impl_type = "RetainedOverlayParse")] fn extract(&mut self, extractor: &dyn LanguageExtractor, report: &ParseReport) { let Some(document) = self.document.as_ref() else { - self.prior_raw_extraction = None; + self.prior_artifact = None; self.extraction_state = OverlayExtractionState::Unavailable(OverlayParseUnavailable::StaleReport); return; }; - match document.extract_canonical(extractor, report, self.prior_raw_extraction.as_deref()) { + match document.extract_canonical_artifact(extractor, report, self.prior_artifact.as_ref()) { Ok(extraction) => { - let result = Arc::new(extraction.result); - self.prior_raw_extraction = Some(Arc::clone(&result)); + let result = Arc::new(extraction.artifact.result.clone()); + self.prior_artifact = Some(extraction.artifact); self.extraction_state = OverlayExtractionState::Ready { result, disposition: extraction.disposition, @@ -211,7 +212,7 @@ impl RetainedOverlayParse { }; } Err(error) => { - self.prior_raw_extraction = None; + self.prior_artifact = None; self.extraction_state = OverlayExtractionState::Unavailable((&error).into()); } } @@ -221,7 +222,7 @@ impl RetainedOverlayParse { Self { document: None, parse_state: OverlayParseState::Unavailable(reason), - prior_raw_extraction: None, + prior_artifact: None, extraction_state: OverlayExtractionState::Unavailable(reason), } } diff --git a/crates/tracedecay-lsp/tests/lsp_suite/analyzer_runtime.rs b/crates/tracedecay-lsp/tests/lsp_suite/analyzer_runtime.rs index b15cd4dac1..2b1c31889b 100644 --- a/crates/tracedecay-lsp/tests/lsp_suite/analyzer_runtime.rs +++ b/crates/tracedecay-lsp/tests/lsp_suite/analyzer_runtime.rs @@ -359,13 +359,15 @@ fn broker_exposes_project_scoped_readiness_without_lsp_transport() { let readiness = authority.analyzer_readiness(); // The slot is keyed by the canonical project root, so its supervisor names - // that root; compare by path, as the gateway admits roots. + // that root; compare by path, as the gateway admits roots. A file URL names + // the root's identity, never the `\\?\` spelling Windows `canonicalize` + // returns. assert_eq!( url::Url::parse(readiness.root().uri()) .unwrap() .to_file_path() .unwrap(), - project.path().canonicalize().unwrap() + tracedecay_runtime_core::path_safety::canonical_root_identity(project.path()) ); assert_eq!(readiness.state(), AnalyzerState::AwaitingStart); assert_eq!(readiness.failure_evidence(), None); @@ -448,10 +450,11 @@ async fn settings_persist_under_dashboard_root() { lsp::settings::load_settings(temp.path()).await.unwrap(), replacement ); - let backup = lsp::settings::settings_path(temp.path()).with_extension("json.bak"); - let backup: lsp::settings::CodeDiagnosticsSettings = - serde_json::from_slice(&tokio::fs::read(backup).await.unwrap()).unwrap(); - assert_eq!(backup, settings); + assert!( + !lsp::settings::settings_path(temp.path()) + .with_extension("json.bak") + .exists() + ); } #[tokio::test] diff --git a/crates/tracedecay-maintenance/src/generation.rs b/crates/tracedecay-maintenance/src/generation.rs index 5b01404695..89795c2548 100644 --- a/crates/tracedecay-maintenance/src/generation.rs +++ b/crates/tracedecay-maintenance/src/generation.rs @@ -2,9 +2,7 @@ use crate::compaction_receipt::record_live_compaction_outcome; use crate::lease::ProjectStoreMaintenanceLeaseV1; -use crate::store_maintenance::{ - CodeGenerationRetentionOutcomeV1, run_branch_compaction, run_code_generation_retention, -}; +use crate::store_maintenance::{CodeGenerationRetentionOutcomeV1, run_code_generation_retention}; use crate::telemetry::StoreTelemetrySamplingRegistry; use crate::tick::{MaintenanceContinuation, MaintenanceTickOutcome}; use tracedecay_contracts::storage::compaction::CompactionThresholdConfig; @@ -20,7 +18,7 @@ pub async fn run_project_generation_maintenance( lease: &ProjectStoreMaintenanceLeaseV1, code_index_schedulers: &tracedecay_code_index_runtime::code_index_scheduler::CodeIndexSchedulerRegistryV1, maintenance_observations: &StoreTelemetrySamplingRegistry, - cancellation: &tracedecay_session_memory::context::CancellationToken, + cancellation: &tracedecay_runtime_core::cancellation::CancellationToken, compaction: Option<&CompactionThresholdConfig>, continuation: Option, ) -> MaintenanceTickOutcome { @@ -66,12 +64,6 @@ pub async fn run_project_generation_maintenance( if !project_compacted { outcome = MaintenanceTickOutcome::Retry; } - if !cancellation.is_cancelled() { - let branch_compacted = run_branch_compaction(lease, compaction); - if !branch_compacted { - outcome = MaintenanceTickOutcome::Retry; - } - } }); } finalize_generation_outcome(outcome, cancellation) @@ -81,7 +73,7 @@ pub async fn run_project_generation_maintenance( /// silently retries forever is exactly the waste being diagnosed. fn finalize_generation_outcome( outcome: MaintenanceTickOutcome, - cancellation: &tracedecay_session_memory::context::CancellationToken, + cancellation: &tracedecay_runtime_core::cancellation::CancellationToken, ) -> MaintenanceTickOutcome { if cancellation.is_cancelled() { hotpath::gauge!("daemon.maintenance.generation.cancelled_total").inc(1_u64); diff --git a/crates/tracedecay-maintenance/src/loop_run.rs b/crates/tracedecay-maintenance/src/loop_run.rs index ec443e8f01..f7ef2127bf 100644 --- a/crates/tracedecay-maintenance/src/loop_run.rs +++ b/crates/tracedecay-maintenance/src/loop_run.rs @@ -125,7 +125,7 @@ impl MaintenanceWake { /// Park on cancel / wake / cadence, then run the next admitted tick. pub async fn run_maintenance_loop( - cancellation: &tracedecay_session_memory::context::CancellationToken, + cancellation: &tracedecay_runtime_core::cancellation::CancellationToken, wake: &MaintenanceWake, interval: Duration, mut run_tick: F, diff --git a/crates/tracedecay-maintenance/src/profile_backup.rs b/crates/tracedecay-maintenance/src/profile_backup.rs index 1d99c72884..15c354c143 100644 --- a/crates/tracedecay-maintenance/src/profile_backup.rs +++ b/crates/tracedecay-maintenance/src/profile_backup.rs @@ -593,7 +593,6 @@ fn rebound_store_manifest( if manifest.schema_version != tracedecay_runtime_core::storage::STORE_MANIFEST_SCHEMA_VERSION || manifest.project_id.as_deref() != Some(project_id) || manifest.store_kind != tracedecay_runtime_core::storage::StoreKind::CodeProject - || manifest.storage_mode != tracedecay_runtime_core::storage::StorageMode::ProfileSharded { return Err(ProfileBackupError::corrupt(format!( "restored store manifest '{}' does not match its enrollment", @@ -1099,14 +1098,9 @@ fn restrict_private_directory(path: &Path) -> Result<(), ProfileBackupError> { /// private" step, not a repair of foreign material. #[cfg(windows)] fn restrict_private_directory(path: &Path) -> Result<(), ProfileBackupError> { - tracedecay_private_fs::make_private_directory(path) - .map(drop) - .map_err(|error| { - ProfileBackupError::unavailable(format!( - "restrict directory '{}': {error}", - path.display() - )) - }) + tracedecay_private_fs::make_private_directory(path).map_err(|error| { + ProfileBackupError::unavailable(format!("restrict directory '{}': {error}", path.display())) + }) } #[cfg(not(any(unix, windows)))] diff --git a/crates/tracedecay-maintenance/src/profile_backup/identity.rs b/crates/tracedecay-maintenance/src/profile_backup/identity.rs index 30a631f1b3..e1ffc66bc9 100644 --- a/crates/tracedecay-maintenance/src/profile_backup/identity.rs +++ b/crates/tracedecay-maintenance/src/profile_backup/identity.rs @@ -79,10 +79,7 @@ pub(super) fn collect_project_identities( manifest_path.display() )) })?; - if manifest.project_id.as_deref() != Some(project_id) - || manifest.storage_mode - != tracedecay_runtime_core::storage::StorageMode::ProfileSharded - { + if manifest.project_id.as_deref() != Some(project_id) { return Err(ProfileBackupError::corrupt(format!( "project store manifest '{}' does not match its final enrollment identity", manifest_path.display() diff --git a/crates/tracedecay-maintenance/src/retention.rs b/crates/tracedecay-maintenance/src/retention.rs index 11162fca67..0d387744c9 100644 --- a/crates/tracedecay-maintenance/src/retention.rs +++ b/crates/tracedecay-maintenance/src/retention.rs @@ -1,12 +1,12 @@ //! Conservative, opt-in retention for the largest append-only telemetry //! tables. //! -//! Three tables grow without bound and had no scheduled pruning: +//! Two tables grow without bound and had no scheduled pruning: //! //! * `analytics_events`, hook/tool/skill telemetry. Derived, reconstructable //! signal, so it carries a **safe default retention of 180 days**. -//! * `session_messages` and `lcm_raw_messages`, legacy session copies retained -//! for a six-month recovery horizon. Current session stores additionally use +//! * `lcm_raw_messages`, the stored session messages, retained for a +//! six-month recovery horizon. Current session stores additionally use //! projection-durability-aware retention in `tracedecay_lcm::retention`. //! //! Every window is expressed in whole days. Rows are pruned only when their @@ -22,24 +22,19 @@ use std::fmt; use serde::Serialize; -pub use tracedecay_automation::config::{ - DEFAULT_ANALYTICS_EVENTS_RETENTION_DAYS, DEFAULT_LEGACY_SESSION_RETENTION_DAYS, RetentionConfig, -}; +pub use tracedecay_automation::config::RetentionConfig; use tracedecay_domain::errors::{Result, TraceDecayError}; use tracedecay_global_db::RegisteredGlobalDb; use tracedecay_runtime_core::db::engine::{Executor, params}; -/// Free-page compaction for tracked branch databases, off the hot path -/// (plan 38, §6). -pub mod branch_compaction; /// Bounded retention for unmounted profile-sharded stores. pub mod cold_store; /// Read-only diagnostics over retention-owned state. pub mod diagnostics; /// Exact-liveness mark-and-sweep for immutable derived code generations. -/// Store-owned quarantine and collection for corruption/recovery artifacts -/// found beside live databases (plan 38, §5). +/// Detection and deletion of corruption/recovery artifacts found beside live +/// databases (plan 38, §5). pub mod incident_debris; /// Bounded compaction for stores retained by live runtime authorities. pub mod live_compaction; @@ -67,8 +62,6 @@ const TIMESTAMP_COLUMN: &str = "timestamp"; pub enum RetentionTable { /// `analytics_events` (global DB), pruned by `timestamp`. AnalyticsEvents, - /// `session_messages` (global DB), pruned by `timestamp`. - SessionMessages, /// `lcm_raw_messages` (per-store LCM DB), pruned by `timestamp`. LcmRawMessages, } @@ -76,23 +69,17 @@ pub enum RetentionTable { fn retention_window_days(config: &RetentionConfig, table: RetentionTable) -> Option { match table { RetentionTable::AnalyticsEvents => config.analytics_events_days, - RetentionTable::SessionMessages => config.session_messages_days, RetentionTable::LcmRawMessages => config.lcm_raw_messages_days, } } impl RetentionTable { - /// The three tables that live in the global database. - pub const GLOBAL_TABLES: [RetentionTable; 3] = [ - Self::AnalyticsEvents, - Self::SessionMessages, - Self::LcmRawMessages, - ]; + /// The tables that live in the global database. + pub const GLOBAL_TABLES: [RetentionTable; 2] = [Self::AnalyticsEvents, Self::LcmRawMessages]; pub fn table_name(self) -> &'static str { match self { Self::AnalyticsEvents => "analytics_events", - Self::SessionMessages => "session_messages", Self::LcmRawMessages => "lcm_raw_messages", } } @@ -162,25 +149,14 @@ async fn delete_slice( .map_err(|error| retention_error(name, "delete", &error)) } -/// Legacy session windows still obey the current projection-durability -/// authority. Age alone never makes lossless content eligible. +/// Session windows still obey the current projection-durability authority. +/// Age alone never makes lossless content eligible. fn retention_eligibility(table: RetentionTable) -> &'static str { match table { RetentionTable::AnalyticsEvents => "1 = 1", - RetentionTable::SessionMessages => { - "EXISTS ( - SELECT 1 - FROM lcm_raw_messages AS raw - JOIN lcm_summary_sources AS source - ON source.source_kind = 'raw_message' - AND source.source_id = CAST(raw.store_id AS TEXT) - WHERE raw.provider = session_messages.provider - AND raw.message_id = session_messages.message_id - )" - } RetentionTable::LcmRawMessages => { "EXISTS ( - SELECT 1 FROM lcm_summary_sources AS source + SELECT 1 FROM session_summary_sources AS source WHERE source.source_kind = 'raw_message' AND source.source_id = CAST(lcm_raw_messages.store_id AS TEXT) )" @@ -227,9 +203,7 @@ impl std::error::Error for RetentionPassInterruption { /// Applies global-database retention for [`RetentionTable::GLOBAL_TABLES`] in /// bounded, separately committed slices. /// -/// Tables run in declaration order so `session_messages` is evaluated while -/// its `lcm_raw_messages` lineage still exists. The cutoff is captured once -/// per table; each slice re-checks eligibility in its own transaction, so +/// The cutoff is captured once per table; each slice re-checks eligibility in its own transaction, so /// rows that gain or lose lineage between slices are judged by the current /// authority. Disabled windows never acquire the writer. #[hotpath::measure(label = "maintenance.retention.prune_global", future = true)] @@ -339,7 +313,6 @@ mod tests { fn config_days(days: Option) -> RetentionConfig { RetentionConfig { analytics_events_days: days, - session_messages_days: None, lcm_raw_messages_days: None, } } @@ -473,7 +446,6 @@ mod tests { applied: true, rows: eligible - abort_after, }, - RetentionTableReport::skipped(RetentionTable::SessionMessages), RetentionTableReport::skipped(RetentionTable::LcmRawMessages), ], "the resumed pass drains exactly the remainder without double counting" @@ -518,51 +490,37 @@ mod tests { } #[tokio::test] - async fn legacy_windows_require_durable_summary_lineage() { + async fn session_windows_require_durable_summary_lineage() { let directory = tempfile::tempdir().unwrap(); let conn = test_conn(&directory); let now = 1_000_000_000; conn.execute_batch( - "CREATE TABLE session_messages ( - provider TEXT NOT NULL, - message_id TEXT NOT NULL, - timestamp INTEGER - ); - CREATE TABLE lcm_raw_messages ( + "CREATE TABLE lcm_raw_messages ( store_id INTEGER PRIMARY KEY, provider TEXT NOT NULL, message_id TEXT NOT NULL, timestamp INTEGER ); - CREATE TABLE lcm_summary_sources ( + CREATE TABLE session_summary_sources ( source_kind TEXT NOT NULL, source_id TEXT NOT NULL ); - INSERT INTO session_messages VALUES - ('claude', 'durable', 1), - ('claude', 'live', 1); INSERT INTO lcm_raw_messages VALUES (1, 'claude', 'durable', 1), (2, 'claude', 'live', 1); - INSERT INTO lcm_summary_sources VALUES ('raw_message', '1');", + INSERT INTO session_summary_sources VALUES ('raw_message', '1');", ) .await .unwrap(); let config = RetentionConfig::default(); - for table in [ - RetentionTable::SessionMessages, - RetentionTable::LcmRawMessages, - ] { - let window = retention_window_days(&config, table).unwrap(); - delete_slice(&*conn, table, cutoff_secs(window, now)) - .await - .unwrap(); - } + let table = RetentionTable::LcmRawMessages; + let window = retention_window_days(&config, table).unwrap(); + delete_slice(&*conn, table, cutoff_secs(window, now)) + .await + .unwrap(); - assert_eq!(count_message(&conn, "session_messages", "durable").await, 0); assert_eq!(count_message(&conn, "lcm_raw_messages", "durable").await, 0); - assert_eq!(count_message(&conn, "session_messages", "live").await, 1); assert_eq!(count_message(&conn, "lcm_raw_messages", "live").await, 1); } } diff --git a/crates/tracedecay-maintenance/src/retention/branch_compaction.rs b/crates/tracedecay-maintenance/src/retention/branch_compaction.rs deleted file mode 100644 index cb70b13e72..0000000000 --- a/crates/tracedecay-maintenance/src/retention/branch_compaction.rs +++ /dev/null @@ -1,526 +0,0 @@ -//! Free-page compaction for tracked branch databases (plan 38 §6). -//! -//! `compact_project_store` and -//! `compact_registered_store` already compact the -//! live graph store and `global.db` off the hot path, through their retained -//! writer runtimes. -//! Every *other* tracked branch gets its own `SQLite` family under -//! `branches/`, cloned wholesale from an ancestor at `branch add` time and -//! then never revisited by any compaction pass. This is the exact bloat class -//! the owner's storage audit measured directly: 2.4 GB of free-page bloat -//! with individual branch databases sitting at 87-91% free pages. -//! -//! Those files are not mounted by any live daemon runtime between syncs, so -//! this module compacts them directly: open a short-lived, best-effort -//! `rusqlite` connection, sample the free-page ratio, and run -//! `PRAGMA incremental_vacuum(N)` when the same -//! [`tracedecay_contracts::storage::compaction::CompactionTriggerPolicyV1`] -//! threshold used for the live stores is met. This never competes with a live -//! writer: a busy/locked file is skipped, not an error, and the next -//! maintenance tick retries it. -//! -//! Branch databases inherit `PRAGMA auto_vacuum = INCREMENTAL` from the -//! ancestor they were cloned from (every fresh store is created with it, see -//! `tracedecay-runtime-core/src/db/migrations.rs::configure_fresh_auto_vacuum`), -//! so `incremental_vacuum` -//! reclaims pages here exactly as it does on the live graph store. A branch -//! database predating that migration carries `auto_vacuum = NONE`, which makes -//! `incremental_vacuum` a *silent no-op* -- reclaiming its free pages would -//! need a full `VACUUM` rewrite, out of scope for a bounded, hot-path-safe -//! pass. That case is precisely the one the owner's audit measured, so this -//! pass refuses to report it as work done: the mode is checked up front and a -//! database that cannot be incrementally vacuumed is skipped with -//! `BranchCompactionSkipReason::IncrementalVacuumUnavailable`, which the -//! daemon logs. Silently "compacting" zero pages would have reported success -//! over exactly the bloat this exists to remove. -//! -//! Nothing here is destructive: `incremental_vacuum` only returns already-free -//! pages to the filesystem and never touches live rows, so this pass has no -//! dry-run mode to gate -- there is no state it can destroy. Branch databases -//! are project graph stores (durable `memory_*` tables and all); compaction is -//! safe on them precisely because it preserves every row. - -use std::path::{Path, PathBuf}; - -use rusqlite::{Connection, OpenFlags}; -use tracedecay_contracts::storage::compaction::{ - CompactionThresholdConfig, CompactionTriggerPolicyV1, -}; -use tracedecay_contracts::storage::identity::{FreePageRatioV1, StorageByteSizeV1, StoreKeyV1}; -use tracedecay_contracts::storage::telemetry::StoreSizeSampleV1; -use tracedecay_domain::UtcMicros; -use tracedecay_runtime_core::sqlite_read_snapshot::{BOUNDED_PROBE_BUSY_TIMEOUT, pragma_u64}; - -/// `PRAGMA auto_vacuum` mode in which `incremental_vacuum` actually reclaims -/// pages. `0` is `NONE` and `1` is `FULL`; only `2` (`INCREMENTAL`) responds. -const AUTO_VACUUM_INCREMENTAL: u64 = 2; - -/// One tracked branch database file, other than the currently-active/mounted -/// one, eligible for direct-file compaction. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct BranchDbCandidate { - pub branch: String, - pub db_path: PathBuf, -} - -/// One branch database this pass actually vacuumed. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct BranchCompactionOutcome { - pub branch: String, - pub db_path: PathBuf, - pub freed_pages: u64, -} - -/// Why a candidate was left untouched this tick. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum BranchCompactionSkipReason { - /// The file could not be opened or locked briefly enough to sample, - /// almost always a concurrent writer (branch add/sync/gc). Expected and - /// transient; the next tick retries. - Busy, - /// The file opened but its page/freelist pragmas could not be read. - SampleFailed, - /// The threshold was met but `PRAGMA incremental_vacuum` failed. - VacuumFailed, - /// The threshold was met but the database is not in - /// `auto_vacuum = INCREMENTAL` mode, so `incremental_vacuum` would - /// reclaim nothing. Reclaiming this file needs a full `VACUUM` rewrite, - /// which this bounded pass deliberately does not do -- see module docs. - IncrementalVacuumUnavailable, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct BranchCompactionSkip { - pub branch: String, - pub db_path: PathBuf, - pub reason: BranchCompactionSkipReason, -} - -#[derive(Debug, Clone, Default)] -pub struct BranchCompactionReport { - pub compacted: Vec, - pub skipped: Vec, - /// The configured thresholds could not be turned into a valid trigger - /// policy, so no candidate was even sampled. Surfaced rather than - /// swallowed: a mistyped `free_page_ratio_threshold` (say `90` for a - /// percentage) would otherwise disable branch compaction permanently and - /// silently, and look identical to "nothing needed compacting". - pub policy_invalid: bool, -} - -/// Selects every tracked branch database file other than `active_db_path` -/// (the currently-mounted store, already compacted through the live daemon -/// runtime, see module docs). -/// -/// The active store is excluded by *resolved* path, not by string equality: -/// `active_db_path` comes from a mounted handle while candidates are rebuilt -/// from `branch-meta.json`, and the two can name the same file through -/// different symlinks or non-normalized components. A missed exclusion would -/// not corrupt anything (`SQLite` locking is sound across processes) but it -/// would put this pass in contention with the daemon's own writer actor over -/// the one file it was written to stay away from. -#[hotpath::measure(label = "maintenance.branch_compaction.select")] -pub fn select_branch_db_candidates( - tracedecay_dir: &Path, - meta: &tracedecay_runtime_core::branch_meta::BranchMeta, - active_db_path: &Path, -) -> Vec { - let active_resolved = active_db_path.canonicalize().ok(); - let is_active = |db_path: &Path| { - if db_path == active_db_path { - return true; - } - match (db_path.canonicalize().ok(), active_resolved.as_ref()) { - (Some(resolved), Some(active)) => &resolved == active, - _ => false, - } - }; - let mut candidates: Vec<_> = meta - .branches - .iter() - .filter_map(|(name, entry)| { - let db_path = tracedecay_dir.join(&entry.db_file); - if is_active(&db_path) { - return None; - } - Some(BranchDbCandidate { - branch: name.clone(), - db_path, - }) - }) - .collect(); - candidates.sort_by(|left, right| left.branch.cmp(&right.branch)); - candidates -} - -/// Runs bounded incremental-vacuum compaction over every candidate whose -/// free-page ratio crosses `config`'s threshold. Each file is handled -/// independently: one busy or failing file never blocks the rest. -#[hotpath::measure(label = "maintenance.branch_compaction.compact")] -pub fn compact_branch_databases( - candidates: &[BranchDbCandidate], - config: &CompactionThresholdConfig, -) -> BranchCompactionReport { - let mut report = BranchCompactionReport::default(); - let Some(policy) = resolve_policy(config) else { - report.policy_invalid = true; - return report; - }; - for candidate in candidates { - if !candidate.db_path.is_file() { - continue; - } - match compact_one(candidate, &policy, config.max_pages_per_tick) { - Ok(Some(outcome)) => report.compacted.push(outcome), - Ok(None) => {} - Err(reason) => report.skipped.push(BranchCompactionSkip { - branch: candidate.branch.clone(), - db_path: candidate.db_path.clone(), - reason, - }), - } - } - report -} - -/// Turns the configured thresholds into a validated trigger policy once for -/// the whole pass. `None` means the configuration itself is unusable, the -/// caller reports that rather than silently treating every database as -/// ineligible. -/// -/// Both rejections matter and neither is hypothetical: a ratio outside -/// `[0.0, 1.0]` fails [`FreePageRatioV1::new`], and a ratio of exactly `0.0` -/// fails [`CompactionTriggerPolicyV1::validate`] (it would schedule every -/// store on every pass). Validating here rather than per-file is what turns -/// the second case from "silently compacts nothing, forever" into a reported -/// configuration error. -fn resolve_policy(config: &CompactionThresholdConfig) -> Option { - let policy = CompactionTriggerPolicyV1 { - free_page_ratio_threshold: FreePageRatioV1::new(config.free_page_ratio_threshold).ok()?, - minimum_reclaimable_bytes: StorageByteSizeV1(config.minimum_reclaimable_bytes), - }; - policy.validate().ok()?; - Some(policy) -} - -/// This one is read-*write* on purpose because `incremental_vacuum` has to -/// write, so it cannot use the shared read-only probe, but it holds the same -/// bounded busy timeout: a locked branch database is skipped, never waited on. -fn open_bounded(path: &Path) -> Result { - let connection = Connection::open_with_flags(path, OpenFlags::SQLITE_OPEN_READ_WRITE) - .map_err(|_| BranchCompactionSkipReason::Busy)?; - connection - .busy_timeout(BOUNDED_PROBE_BUSY_TIMEOUT) - .map_err(|_| BranchCompactionSkipReason::Busy)?; - Ok(connection) -} - -fn compact_one( - candidate: &BranchDbCandidate, - policy: &CompactionTriggerPolicyV1, - max_pages_per_tick: u32, -) -> Result, BranchCompactionSkipReason> { - let connection = open_bounded(&candidate.db_path)?; - let page_size = - pragma_u64(&connection, "page_size").ok_or(BranchCompactionSkipReason::SampleFailed)?; - let page_count = - pragma_u64(&connection, "page_count").ok_or(BranchCompactionSkipReason::SampleFailed)?; - let freelist = pragma_u64(&connection, "freelist_count") - .ok_or(BranchCompactionSkipReason::SampleFailed)?; - if page_size == 0 || page_count == 0 { - return Ok(None); - } - if !is_compaction_scheduled(&candidate.branch, page_size, page_count, freelist, policy) { - return Ok(None); - } - // Only `auto_vacuum = INCREMENTAL` responds to `incremental_vacuum`. - // Checking before instead of inferring from "freed nothing" keeps a - // legacy branch database from being reported as successfully compacted - // when its free pages are in fact unreachable to this pass. - if pragma_u64(&connection, "auto_vacuum").ok_or(BranchCompactionSkipReason::SampleFailed)? - != AUTO_VACUUM_INCREMENTAL - { - return Err(BranchCompactionSkipReason::IncrementalVacuumUnavailable); - } - let pages = max_pages_per_tick.max(1); - connection - .execute_batch(&format!("PRAGMA incremental_vacuum({pages})")) - .map_err(|_| BranchCompactionSkipReason::VacuumFailed)?; - let freelist_after = pragma_u64(&connection, "freelist_count").unwrap_or(freelist); - Ok(Some(BranchCompactionOutcome { - branch: candidate.branch.clone(), - db_path: candidate.db_path.clone(), - freed_pages: freelist.saturating_sub(freelist_after), - })) -} - -fn is_compaction_scheduled( - branch: &str, - page_size: u64, - page_count: u64, - freelist: u64, - policy: &CompactionTriggerPolicyV1, -) -> bool { - // The logical store key convention for a branch-scoped store, so telemetry - // built from this sample names the branch rather than a shared constant. - let Ok(store_key) = StoreKeyV1::new(format!("branches/{branch}")) else { - return false; - }; - let Ok(page_size_bytes) = u32::try_from(page_size) else { - return false; - }; - let sample = StoreSizeSampleV1 { - store: store_key, - page_size_bytes, - page_count, - freelist_pages: freelist, - observed_at: UtcMicros(0), - }; - policy - .decide(&sample) - .is_ok_and(|decision| decision.is_scheduled()) -} - -#[cfg(test)] -#[allow(clippy::unwrap_used, clippy::expect_used)] -mod tests { - use super::*; - use std::collections::HashMap; - - fn config(threshold: f64) -> CompactionThresholdConfig { - CompactionThresholdConfig { - free_page_ratio_threshold: threshold, - minimum_reclaimable_bytes: 0, - max_pages_per_tick: 1024, - } - } - - fn bloated_db(path: &Path) { - bloated_db_with_auto_vacuum(path, "INCREMENTAL"); - } - - fn bloated_db_with_auto_vacuum(path: &Path, mode: &str) { - let connection = Connection::open(path).unwrap(); - connection - .execute_batch(&format!("PRAGMA auto_vacuum = {mode};")) - .unwrap(); - connection - .execute_batch("CREATE TABLE fixture (id INTEGER PRIMARY KEY, payload BLOB);") - .unwrap(); - let payload = vec![7u8; 64 * 1024]; - for id in 0..32i64 { - connection - .execute( - "INSERT INTO fixture (id, payload) VALUES (?1, ?2)", - rusqlite::params![id, payload], - ) - .unwrap(); - } - connection.execute_batch("DELETE FROM fixture;").unwrap(); - } - - #[test] - fn select_branch_db_candidates_excludes_active_and_includes_others() { - let dir = tempfile::tempdir().unwrap(); - let mut branches = HashMap::new(); - branches.insert( - "main".to_string(), - tracedecay_runtime_core::branch_meta::BranchEntry { - db_file: tracedecay_runtime_core::config::DB_FILENAME.to_string(), - parent: None, - created_at: "0".to_string(), - last_synced_at: "0".to_string(), - gc_protected: false, - graph_source: None, - }, - ); - branches.insert( - "feature".to_string(), - tracedecay_runtime_core::branch_meta::BranchEntry { - db_file: "branches/feature.db".to_string(), - parent: Some("main".to_string()), - created_at: "0".to_string(), - last_synced_at: "0".to_string(), - gc_protected: false, - graph_source: None, - }, - ); - let meta = tracedecay_runtime_core::branch_meta::BranchMeta { - default_branch: "main".to_string(), - branches, - }; - let active = dir - .path() - .join(tracedecay_runtime_core::config::DB_FILENAME); - let candidates = select_branch_db_candidates(dir.path(), &meta, &active); - assert_eq!(candidates.len(), 1); - assert_eq!(candidates[0].branch, "feature"); - assert_eq!( - candidates[0].db_path, - dir.path().join("branches/feature.db") - ); - } - - #[test] - fn compacts_a_bloated_branch_database_over_threshold() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("feature.db"); - bloated_db(&path); - let candidates = vec![BranchDbCandidate { - branch: "feature".to_string(), - db_path: path.clone(), - }]; - - let before = Connection::open(&path).unwrap(); - let freelist_before = pragma_u64(&before, "freelist_count").unwrap(); - assert!(freelist_before > 0, "fixture must create reclaimable pages"); - drop(before); - - let report = compact_branch_databases(&candidates, &config(0.5)); - assert!(report.skipped.is_empty(), "skipped: {:?}", report.skipped); - assert_eq!(report.compacted.len(), 1); - assert!(report.compacted[0].freed_pages > 0); - - let after = Connection::open(&path).unwrap(); - let freelist_after = pragma_u64(&after, "freelist_count").unwrap(); - assert!(freelist_after < freelist_before); - } - - /// A *valid* threshold the database does not meet leaves it alone. The - /// earlier version of this test used a threshold of `1.5`, which - /// `FreePageRatioV1::new` rejects outright, it passed because the policy - /// failed to build, not because the database was under threshold, and so - /// asserted nothing about the policy at all. - #[test] - fn leaves_a_database_under_threshold_untouched() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("feature.db"); - bloated_db(&path); - let candidates = vec![BranchDbCandidate { - branch: "feature".to_string(), - db_path: path.clone(), - }]; - - let before = Connection::open(&path).unwrap(); - let freelist_before = pragma_u64(&before, "freelist_count").unwrap(); - drop(before); - - // Ratio met, but the reclaimable-bytes floor is unreachable, so the - // real policy returns NotEligible. - let under = CompactionThresholdConfig { - free_page_ratio_threshold: 0.5, - minimum_reclaimable_bytes: u64::MAX, - max_pages_per_tick: 1024, - }; - let report = compact_branch_databases(&candidates, &under); - assert!(!report.policy_invalid); - assert!(report.compacted.is_empty()); - assert!(report.skipped.is_empty()); - - let after = Connection::open(&path).unwrap(); - assert_eq!( - pragma_u64(&after, "freelist_count").unwrap(), - freelist_before, - "an ineligible database must not be vacuumed" - ); - } - - /// A threshold of exactly `0.0` is in range for `FreePageRatioV1` but - /// rejected by `CompactionTriggerPolicyV1::validate` (it would schedule - /// every store on every pass). The inherited version of this module built - /// the policy per file and swallowed that rejection into "not eligible", - /// so a zero threshold compacted nothing and looked like success, the - /// same bug that left this module's own headline test failing unnoticed. - #[test] - fn a_zero_threshold_is_reported_invalid_rather_than_compacting_nothing() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("feature.db"); - bloated_db(&path); - let candidates = vec![BranchDbCandidate { - branch: "feature".to_string(), - db_path: path.clone(), - }]; - - let report = compact_branch_databases(&candidates, &config(0.0)); - - assert!(report.policy_invalid); - assert!(report.compacted.is_empty()); - assert!(report.skipped.is_empty()); - } - - /// The legacy-bloat case the module exists for: `auto_vacuum = NONE` - /// makes `incremental_vacuum` a no-op, and reporting that as a successful - /// compaction would claim to have reclaimed the exact free pages it - /// cannot reach. - #[test] - fn a_database_without_incremental_auto_vacuum_is_skipped_not_reported_compacted() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("legacy.db"); - bloated_db_with_auto_vacuum(&path, "NONE"); - let candidates = vec![BranchDbCandidate { - branch: "legacy".to_string(), - db_path: path.clone(), - }]; - - let report = compact_branch_databases(&candidates, &config(0.5)); - - assert!( - report.compacted.is_empty(), - "compacted: {:?}", - report.compacted - ); - assert_eq!(report.skipped.len(), 1); - assert_eq!( - report.skipped[0].reason, - BranchCompactionSkipReason::IncrementalVacuumUnavailable - ); - } - - /// The active store is the daemon writer-actor's; excluding it only by - /// string equality misses a path that reaches the same file through a - /// symlinked profile directory. - #[cfg(unix)] - #[test] - fn the_active_database_is_excluded_through_a_symlinked_profile_dir() { - let dir = tempfile::tempdir().unwrap(); - let real = dir.path().join("real"); - std::fs::create_dir_all(&real).unwrap(); - std::fs::write(real.join(tracedecay_runtime_core::config::DB_FILENAME), b"").unwrap(); - - let linked = dir.path().join("linked"); - #[cfg(unix)] - std::os::unix::fs::symlink(&real, &linked).unwrap(); - #[cfg(not(unix))] - return; - - let mut branches = HashMap::new(); - branches.insert( - "main".to_string(), - tracedecay_runtime_core::branch_meta::BranchEntry { - db_file: tracedecay_runtime_core::config::DB_FILENAME.to_string(), - parent: None, - created_at: "0".to_string(), - last_synced_at: "0".to_string(), - gc_protected: false, - graph_source: None, - }, - ); - let meta = tracedecay_runtime_core::branch_meta::BranchMeta { - default_branch: "main".to_string(), - branches, - }; - - // The mounted handle names the file through `real/`; branch-meta - // rebuilds it through the symlinked `linked/`. - let candidates = select_branch_db_candidates( - &linked, - &meta, - &real.join(tracedecay_runtime_core::config::DB_FILENAME), - ); - - assert!( - candidates.is_empty(), - "the active store must be excluded through a symlinked path: {candidates:?}" - ); - } -} diff --git a/crates/tracedecay-maintenance/src/retention/cold_store.rs b/crates/tracedecay-maintenance/src/retention/cold_store.rs index 53d600ed09..83d56701b6 100644 --- a/crates/tracedecay-maintenance/src/retention/cold_store.rs +++ b/crates/tracedecay-maintenance/src/retention/cold_store.rs @@ -58,7 +58,6 @@ pub async fn run_cold_store_page( profile_root: &Path, profile_database: &RegisteredGlobalDb, orphan_store_gc_days: Option, - incident_debris_retention_days: Option, cancellation: &CancellationToken, ) -> tracedecay_domain::errors::Result { let checkpoint_path = checkpoint_path(profile_root); @@ -72,16 +71,6 @@ pub async fn run_cold_store_page( COLD_STORE_PAGE_LIMIT, ) .await?; - let retention_now = - if orphan_store_gc_days.is_some() || incident_debris_retention_days.is_some() { - Some(now_secs_i64().map_err(|message| { - tracedecay_domain::errors::TraceDecayError::Config { - message: message.to_owned(), - } - })?) - } else { - None - }; let mut report = ColdStorePageReportV1::default(); for entry in &page.entries { let outcome = classify_cold_store_state( @@ -106,12 +95,12 @@ pub async fn run_cold_store_page( } } if let Some(days) = orphan_store_gc_days { - let findings = orphan_stores::classify_stores( - &page.entries, - retention_now.ok_or_else(|| tracedecay_domain::errors::TraceDecayError::Config { - message: "maintenance retention clock unavailable".to_owned(), - })?, - ); + let now = now_secs_i64().map_err(|message| { + tracedecay_domain::errors::TraceDecayError::Config { + message: message.to_owned(), + } + })?; + let findings = orphan_stores::classify_stores(&page.entries, now); let plan = orphan_stores::plan_collection(findings, retention_window_secs(days)); let (outcome, _) = orphan_stores::execute_registered_collection(profile_database, &plan, profile_root) @@ -126,22 +115,13 @@ pub async fn run_cold_store_page( report.outcome = ColdStorePageOutcomeV1::Unreadable; } } - if let Some(days) = incident_debris_retention_days { - let sweep = incident_debris::sweep_incident_debris( - &page.entries, - profile_root, - retention_window_secs(days), - retention_now.ok_or_else(|| tracedecay_domain::errors::TraceDecayError::Config { - message: "maintenance retention clock unavailable".to_owned(), - })?, - ); - report.reclaimed_bytes = report.reclaimed_bytes.saturating_add(sweep.reclaimed_bytes); - report.unavailable_stores = report - .unavailable_stores - .saturating_add(sweep.errors.len() as u64); - if !sweep.errors.is_empty() { - report.outcome = ColdStorePageOutcomeV1::Unreadable; - } + let sweep = incident_debris::sweep_incident_debris(&page.entries, profile_root); + report.reclaimed_bytes = report.reclaimed_bytes.saturating_add(sweep.reclaimed_bytes); + report.unavailable_stores = report + .unavailable_stores + .saturating_add(sweep.errors.len() as u64); + if !sweep.errors.is_empty() { + report.outcome = ColdStorePageOutcomeV1::Unreadable; } let project_ids = page .entries diff --git a/crates/tracedecay-maintenance/src/retention/incident_debris.rs b/crates/tracedecay-maintenance/src/retention/incident_debris.rs index b3c2fd861f..d44119b97b 100644 --- a/crates/tracedecay-maintenance/src/retention/incident_debris.rs +++ b/crates/tracedecay-maintenance/src/retention/incident_debris.rs @@ -1,54 +1,26 @@ -//! Durable incident-debris quarantine and retention collection (Plan 38 §5). +//! Incident-debris detection and deletion (Plan 38 §5). -use std::io::{self, Read, Write}; +use std::io; use std::path::{Path, PathBuf}; -use std::time::UNIX_EPOCH; #[cfg(not(windows))] use cap_fs_ext::OpenOptionsMaybeDirExt; -use cap_fs_ext::{ - DirExt, FollowSymlinks, OpenOptionsFollowExt, OpenOptionsSyncExt, ambient_authority, -}; -use cap_std::fs::{Dir, DirBuilder, OpenOptions}; -#[cfg(unix)] -use cap_std::fs::{DirBuilderExt, OpenOptionsExt}; -use serde::{Deserialize, Serialize}; -use sha2::{Digest, Sha256}; +use cap_fs_ext::ambient_authority; +use cap_std::fs::Dir; +#[cfg(not(windows))] +use cap_std::fs::OpenOptions; use tracedecay_contracts::storage::{ IncidentDebrisArtifactV1, IncidentDebrisKindV1, IncidentDebrisScanV1, RelativeArtifactPathV1, StorageByteSizeV1, StoreKeyV1, }; use tracedecay_domain::UtcMicros; -use tracedecay_domain::canonical_text::encode_lowercase_hex; use super::orphan_stores::StoreCensusEntry; -pub const INCIDENT_DEBRIS_QUARANTINE_DIR: &str = ".incident-debris"; -const METADATA_SCHEMA_V1: &str = "tracedecay.incident-debris.v1"; -const HASH_BUFFER_BYTES: usize = 64 * 1024; - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct IncidentDebrisMetadataV1 { - pub schema: String, - pub record_id: String, - pub store_id: String, - pub original_name: String, - pub kind: IncidentDebrisKindV1, - pub content_sha256: String, - pub size_bytes: u64, - pub quarantined_at_secs: i64, - pub collection_eligible_at_secs: i64, -} - #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum IncidentDebrisFailureKind { OutsideProfile, InspectFailed, - MetadataInvalid, - MetadataWriteFailed, - MoveFailed, - IntegrityMismatch, RemoveFailed, } @@ -60,9 +32,7 @@ pub struct IncidentDebrisFailure { #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct IncidentDebrisSweepReport { - pub quarantined: usize, pub collected: usize, - pub retained: usize, pub reclaimed_bytes: u64, pub errors: Vec, } @@ -97,51 +67,18 @@ impl StoreDebrisCapability { root, }) } - - fn quarantine_dir(&self, create: bool) -> io::Result> { - match self.root.open_dir_nofollow(INCIDENT_DEBRIS_QUARANTINE_DIR) { - Ok(directory) => Ok(Some(directory)), - Err(error) if error.kind() == io::ErrorKind::NotFound && !create => Ok(None), - Err(error) if error.kind() == io::ErrorKind::NotFound => { - #[allow(unused_mut)] - let mut builder = DirBuilder::new(); - #[cfg(unix)] - builder.mode(0o700); - match self - .root - .create_dir_with(INCIDENT_DEBRIS_QUARANTINE_DIR, &builder) - { - Ok(()) => {} - Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {} - Err(error) => return Err(error), - } - self.root - .open_dir_nofollow(INCIDENT_DEBRIS_QUARANTINE_DIR) - .map(Some) - } - Err(error) => Err(error), - } - } } +/// Deletes every classified loose debris file beside each census store. +/// Only regular files reached through the store capability without following +/// symlinks are removed; directories are never debris. #[must_use] #[hotpath::measure(label = "maintenance.incident_debris.sweep")] pub fn sweep_incident_debris( census: &[StoreCensusEntry], profile_root: &Path, - retention_secs: i64, - now: i64, ) -> IncidentDebrisSweepReport { let mut report = IncidentDebrisSweepReport::default(); - if retention_secs <= 0 { - report - .errors - .extend(census.iter().map(|entry| IncidentDebrisFailure { - store_id: entry.store_id.clone(), - kind: IncidentDebrisFailureKind::MetadataInvalid, - })); - return observed_sweep_report(report); - } let profile = match canonical_profile_root(profile_root) { Ok(profile) => profile, Err(kind) => { @@ -152,31 +89,21 @@ pub fn sweep_incident_debris( } }; for entry in census { - let capability = match StoreDebrisCapability::open(entry, &profile) { - Ok(capability) => capability, - Err(kind) => { - report.errors.push(failure(entry, kind)); - continue; - } - }; - quarantine_loose(&capability, retention_secs, now, &mut report); - collect_due(&capability, now, &mut report); - report.retained = report - .retained - .saturating_add(retained_count(&capability, &mut report.errors)); + match StoreDebrisCapability::open(entry, &profile) { + Ok(capability) => delete_loose_debris(&capability, &mut report), + Err(kind) => report.errors.push(failure(entry, kind)), + } } observed_sweep_report(report) } -/// Items-moved/removed census for the one outer sweep wall span, including -/// the fail-closed early exits that touch nothing but report every store. +/// Items-removed census for the one outer sweep wall span, including the +/// fail-closed early exits that touch nothing but report every store. fn observed_sweep_report(report: IncidentDebrisSweepReport) -> IncidentDebrisSweepReport { - hotpath::gauge!("maintenance.incident_debris.quarantined_total").inc(report.quarantined); hotpath::gauge!("maintenance.incident_debris.collected_total").inc(report.collected); hotpath::gauge!("maintenance.incident_debris.failed_total").inc(report.errors.len()); hotpath::gauge!("maintenance.incident_debris.reclaimed_bytes_total") .inc(report.reclaimed_bytes); - hotpath::gauge!("maintenance.incident_debris.retained").set(report.retained); report } @@ -188,7 +115,7 @@ pub fn scan_incident_debris( ) -> Result { let capability = StoreDebrisCapability::open(entry, &canonical_profile_root(profile_root)?)?; let store = StoreKeyV1::new(entry.store_id.clone()) - .map_err(|_| IncidentDebrisFailureKind::MetadataInvalid)?; + .map_err(|_| IncidentDebrisFailureKind::InspectFailed)?; let observed_at = UtcMicros(now.saturating_mul(1_000_000)); let mut artifacts = Vec::new(); let mut listing_complete = true; @@ -210,9 +137,6 @@ pub fn scan_incident_debris( listing_complete = false; continue; }; - if name == INCIDENT_DEBRIS_QUARANTINE_DIR { - continue; - } let file_type = match listed.file_type() { Ok(file_type) => file_type, Err(_) => { @@ -220,38 +144,24 @@ pub fn scan_incident_debris( continue; } }; + // Store subdirectories are never debris; anything that is neither a + // directory nor a regular file makes the listing partial. + if file_type.is_dir() { + continue; + } + if !file_type.is_file() { + listing_complete = false; + continue; + } let Some(kind) = IncidentDebrisKindV1::classify(name) else { - // Ordinary live files and store subdirectories (branches/, - // payloads/, ...) are not debris; anything else unclassifiable - // makes the listing partial. - if !file_type.is_dir() && !file_type.is_file() { - listing_complete = false; - } continue; }; - let size_bytes = if file_type.is_file() { - match listed.metadata() { - Ok(metadata) => metadata.len(), - Err(_) => { - listing_complete = false; - continue; - } - } - } else if file_type.is_dir() { - // A store-quarantine directory (for example the graph mount's - // `tracedecay.grafeo.corrupt-/` family with its WAL and - // receipt) is one debris artifact whose recursive payload is the - // retained forensic evidence. - match capability.root.open_dir_nofollow(name) { - Ok(directory) => directory_debris_bytes(&directory, &mut listing_complete), - Err(_) => { - listing_complete = false; - continue; - } + let size_bytes = match listed.metadata() { + Ok(metadata) => metadata.len(), + Err(_) => { + listing_complete = false; + continue; } - } else { - listing_complete = false; - continue; }; if let Some(artifact) = application_artifact(&store, name, kind, size_bytes, observed_at) { artifacts.push(artifact); @@ -260,40 +170,6 @@ pub fn scan_incident_debris( } } - if let Some(quarantine) = capability - .quarantine_dir(false) - .map_err(|_| IncidentDebrisFailureKind::InspectFailed)? - { - for metadata_name in metadata_names(&quarantine, &mut listing_complete) { - let metadata = match read_metadata(&quarantine, &metadata_name) { - Ok(metadata) if validate_metadata(&metadata, &capability.store_id) => metadata, - _ => { - listing_complete = false; - continue; - } - }; - let artifact_name = artifact_name(&metadata.record_id); - match quarantine.symlink_metadata(&artifact_name) { - Ok(file) if file.is_file() => {} - _ => { - listing_complete = false; - continue; - } - } - if let Some(artifact) = application_artifact( - &store, - &metadata.original_name, - metadata.kind, - metadata.size_bytes, - UtcMicros(metadata.quarantined_at_secs.saturating_mul(1_000_000)), - ) { - artifacts.push(artifact); - } else { - listing_complete = false; - } - } - } - Ok(IncidentDebrisScanV1 { store, artifacts, @@ -301,46 +177,6 @@ pub fn scan_incident_debris( }) } -/// Recursive byte total of one quarantined debris directory. Symlinks are -/// never followed; anything unreadable makes the surrounding scan partial so -/// an incomplete walk can never assert a clean or fully-sized result. -fn directory_debris_bytes(directory: &Dir, listing_complete: &mut bool) -> u64 { - let entries = match directory.read_dir(".") { - Ok(entries) => entries, - Err(_) => { - *listing_complete = false; - return 0; - } - }; - let mut total = 0_u64; - for listed in entries { - let Ok(listed) = listed else { - *listing_complete = false; - continue; - }; - let Ok(file_type) = listed.file_type() else { - *listing_complete = false; - continue; - }; - if file_type.is_file() { - match listed.metadata() { - Ok(metadata) => total = total.saturating_add(metadata.len()), - Err(_) => *listing_complete = false, - } - } else if file_type.is_dir() { - match directory.open_dir_nofollow(listed.file_name()) { - Ok(child) => { - total = total.saturating_add(directory_debris_bytes(&child, listing_complete)); - } - Err(_) => *listing_complete = false, - } - } else { - *listing_complete = false; - } - } - total -} - fn failure(entry: &StoreCensusEntry, kind: IncidentDebrisFailureKind) -> IncidentDebrisFailure { IncidentDebrisFailure { store_id: entry.store_id.clone(), @@ -359,12 +195,7 @@ fn push_failure( }); } -fn quarantine_loose( - capability: &StoreDebrisCapability, - retention_secs: i64, - now: i64, - report: &mut IncidentDebrisSweepReport, -) { +fn delete_loose_debris(capability: &StoreDebrisCapability, report: &mut IncidentDebrisSweepReport) { let entries = match capability.root.read_dir(".") { Ok(entries) => entries, Err(_) => { @@ -376,7 +207,7 @@ fn quarantine_loose( return; } }; - let mut names = Vec::new(); + let mut debris = Vec::new(); for listed in entries { let Ok(listed) = listed else { push_failure( @@ -390,149 +221,24 @@ fn quarantine_loose( let Some(name) = name.to_str() else { continue; }; - if listed.file_type().is_ok_and(|kind| kind.is_file()) - && IncidentDebrisKindV1::classify(name).is_some() - { - names.push(name.to_string()); - } - } - names.sort(); - - for name in names { - let Some(kind) = IncidentDebrisKindV1::classify(&name) else { - continue; - }; - let record = match quarantine_record(capability, &name, kind, retention_secs, now) { - Ok(record) => record, - Err(kind) => { - push_failure(report, &capability.store_id, kind); - continue; - } - }; - let quarantine = match capability.quarantine_dir(true) { - Ok(Some(quarantine)) => quarantine, - _ => { - push_failure( - report, - &capability.store_id, - IncidentDebrisFailureKind::MetadataWriteFailed, - ); - continue; - } - }; - let metadata_name = metadata_name(&record.record_id); - if write_metadata(&quarantine, &metadata_name, &record).is_err() { - push_failure( - report, - &capability.store_id, - IncidentDebrisFailureKind::MetadataWriteFailed, - ); - continue; - } - let artifact_name = artifact_name(&record.record_id); - if quarantine.symlink_metadata(&artifact_name).is_ok() { - push_failure( - report, - &capability.store_id, - IncidentDebrisFailureKind::MoveFailed, - ); - continue; - } - if capability - .root - .rename(&name, &quarantine, &artifact_name) - .is_err() - { - let _ = quarantine.remove_file(&metadata_name); - let _ = sync_dir(&quarantine); - push_failure( - report, - &capability.store_id, - IncidentDebrisFailureKind::MoveFailed, - ); - continue; - } - if verify_artifact(&quarantine, &artifact_name, &record).is_err() { - let _ = quarantine.rename(&artifact_name, &capability.root, &name); - let _ = quarantine.remove_file(&metadata_name); - let _ = sync_dir(&quarantine); - let _ = sync_dir(&capability.root); - push_failure( - report, - &capability.store_id, - IncidentDebrisFailureKind::IntegrityMismatch, - ); + if IncidentDebrisKindV1::classify(name).is_none() { continue; } - if sync_dir(&quarantine).is_err() || sync_dir(&capability.root).is_err() { - push_failure( - report, - &capability.store_id, - IncidentDebrisFailureKind::MoveFailed, - ); - continue; - } - report.quarantined = report.quarantined.saturating_add(1); - } -} - -fn collect_due( - capability: &StoreDebrisCapability, - now: i64, - report: &mut IncidentDebrisSweepReport, -) { - let quarantine = match capability.quarantine_dir(false) { - Ok(Some(quarantine)) => quarantine, - Ok(None) => return, - Err(_) => { - push_failure( + // `DirEntry` metadata describes the entry itself, so a symlink named + // like debris is never treated as a regular file. + match listed.metadata() { + Ok(metadata) if metadata.is_file() => debris.push((name.to_string(), metadata.len())), + Ok(_) => {} + Err(_) => push_failure( report, &capability.store_id, IncidentDebrisFailureKind::InspectFailed, - ); - return; + ), } - }; - let mut complete = true; - for metadata_name in metadata_names(&quarantine, &mut complete) { - let record = match read_metadata(&quarantine, &metadata_name) { - Ok(record) if validate_metadata(&record, &capability.store_id) => record, - _ => { - push_failure( - report, - &capability.store_id, - IncidentDebrisFailureKind::MetadataInvalid, - ); - continue; - } - }; - if now < record.collection_eligible_at_secs { - continue; - } - let artifact_name = artifact_name(&record.record_id); - match verify_artifact(&quarantine, &artifact_name, &record) { - Ok(()) => {} - Err(error) if error.kind() == io::ErrorKind::NotFound => { - if quarantine.remove_file(&metadata_name).is_err() || sync_dir(&quarantine).is_err() - { - push_failure( - report, - &capability.store_id, - IncidentDebrisFailureKind::RemoveFailed, - ); - } - continue; - } - Err(_) => { - push_failure( - report, - &capability.store_id, - IncidentDebrisFailureKind::IntegrityMismatch, - ); - continue; - } - } - if quarantine.remove_file(&artifact_name).is_err() || sync_dir(&quarantine).is_err() { + } + let mut removed = false; + for (name, size_bytes) in debris { + if capability.root.remove_file(&name).is_err() { push_failure( report, &capability.store_id, @@ -540,249 +246,19 @@ fn collect_due( ); continue; } + removed = true; report.collected = report.collected.saturating_add(1); - report.reclaimed_bytes = report.reclaimed_bytes.saturating_add(record.size_bytes); - if quarantine.remove_file(&metadata_name).is_err() || sync_dir(&quarantine).is_err() { - push_failure( - report, - &capability.store_id, - IncidentDebrisFailureKind::RemoveFailed, - ); - } + report.reclaimed_bytes = report.reclaimed_bytes.saturating_add(size_bytes); } - if !complete { + if removed && sync_dir(&capability.root).is_err() { push_failure( report, &capability.store_id, - IncidentDebrisFailureKind::InspectFailed, + IncidentDebrisFailureKind::RemoveFailed, ); } } -fn retained_count( - capability: &StoreDebrisCapability, - errors: &mut Vec, -) -> usize { - let quarantine = match capability.quarantine_dir(false) { - Ok(Some(quarantine)) => quarantine, - Ok(None) => return 0, - Err(_) => { - errors.push(IncidentDebrisFailure { - store_id: capability.store_id.clone(), - kind: IncidentDebrisFailureKind::InspectFailed, - }); - return 0; - } - }; - let mut complete = true; - let count = metadata_names(&quarantine, &mut complete) - .into_iter() - .filter_map(|name| read_metadata(&quarantine, &name).ok()) - .filter(|metadata| validate_metadata(metadata, &capability.store_id)) - .count(); - if !complete { - errors.push(IncidentDebrisFailure { - store_id: capability.store_id.clone(), - kind: IncidentDebrisFailureKind::InspectFailed, - }); - } - count -} - -fn quarantine_record( - capability: &StoreDebrisCapability, - name: &str, - kind: IncidentDebrisKindV1, - retention_secs: i64, - now: i64, -) -> Result { - if !is_component(name) { - return Err(IncidentDebrisFailureKind::InspectFailed); - } - let mut options = OpenOptions::new(); - options.read(true).follow(FollowSymlinks::No); - let mut file = capability - .root - .open_with(name, &options) - .map_err(|_| IncidentDebrisFailureKind::InspectFailed)?; - let metadata = file - .metadata() - .map_err(|_| IncidentDebrisFailureKind::InspectFailed)?; - if !metadata.is_file() { - return Err(IncidentDebrisFailureKind::InspectFailed); - } - let content_sha256 = - sha256_reader(&mut file).map_err(|_| IncidentDebrisFailureKind::InspectFailed)?; - let modified_nanos = metadata - .modified() - .ok() - .and_then(|modified| modified.into_std().duration_since(UNIX_EPOCH).ok()) - .map_or(0u128, |elapsed| elapsed.as_nanos()); - let mut identity = Sha256::new(); - identity.update(b"tracedecay.incident-debris.record.v1\0"); - identity.update(capability.store_id.as_bytes()); - identity.update(b"\0"); - identity.update(name.as_bytes()); - identity.update(b"\0"); - identity.update(metadata.len().to_le_bytes()); - identity.update(modified_nanos.to_le_bytes()); - identity.update(content_sha256.as_bytes()); - let record_id = encode_lowercase_hex(&identity.finalize()); - Ok(IncidentDebrisMetadataV1 { - schema: METADATA_SCHEMA_V1.to_string(), - record_id, - store_id: capability.store_id.clone(), - original_name: name.to_string(), - kind, - content_sha256, - size_bytes: metadata.len(), - quarantined_at_secs: now, - collection_eligible_at_secs: now.saturating_add(retention_secs), - }) -} - -fn write_metadata( - quarantine: &Dir, - name: &str, - metadata: &IncidentDebrisMetadataV1, -) -> io::Result<()> { - match quarantine.symlink_metadata(name) { - Ok(existing) if existing.is_file() => { - return if read_metadata(quarantine, name)? == *metadata { - Ok(()) - } else { - Err(io::Error::new( - io::ErrorKind::AlreadyExists, - "incident debris metadata identity collision", - )) - }; - } - Ok(_) => { - return Err(io::Error::new( - io::ErrorKind::AlreadyExists, - "incident debris metadata is not a regular file", - )); - } - Err(error) if error.kind() == io::ErrorKind::NotFound => {} - Err(error) => return Err(error), - } - let bytes = serde_json::to_vec_pretty(metadata) - .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?; - let temporary = format!(".{}.tmp", metadata.record_id); - let _ = remove_regular_file_if_exists(quarantine, &temporary); - let mut options = OpenOptions::new(); - options - .write(true) - .create_new(true) - .follow(FollowSymlinks::No) - .sync(true); - #[cfg(unix)] - options.mode(0o600); - { - let mut file = quarantine.open_with(&temporary, &options)?; - file.write_all(&bytes)?; - file.sync_all()?; - } - quarantine.rename(&temporary, quarantine, name)?; - sync_dir(quarantine) -} - -fn read_metadata(quarantine: &Dir, name: &str) -> io::Result { - if !is_component(name) { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "incident debris metadata name is not a component", - )); - } - let mut options = OpenOptions::new(); - options.read(true).follow(FollowSymlinks::No); - let mut file = quarantine.open_with(name, &options)?; - let mut bytes = Vec::new(); - file.read_to_end(&mut bytes)?; - serde_json::from_slice(&bytes) - .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error)) -} - -fn metadata_names(quarantine: &Dir, complete: &mut bool) -> Vec { - let entries = match quarantine.read_dir(".") { - Ok(entries) => entries, - Err(_) => { - *complete = false; - return Vec::new(); - } - }; - let mut names = Vec::new(); - for listed in entries { - let Ok(listed) = listed else { - *complete = false; - continue; - }; - let name = listed.file_name(); - let Some(name) = name.to_str() else { - *complete = false; - continue; - }; - if name.ends_with(".json") { - if listed.file_type().is_ok_and(|kind| kind.is_file()) { - names.push(name.to_string()); - } else { - *complete = false; - } - } else if !name.ends_with(".artifact") && !name.ends_with(".tmp") { - *complete = false; - } - } - names.sort(); - names -} - -fn validate_metadata(metadata: &IncidentDebrisMetadataV1, expected_store_id: &str) -> bool { - metadata.schema == METADATA_SCHEMA_V1 - && metadata.store_id == expected_store_id - && is_sha256(&metadata.record_id) - && is_sha256(&metadata.content_sha256) - && is_component(&metadata.original_name) - && IncidentDebrisKindV1::classify(&metadata.original_name) == Some(metadata.kind) - && metadata.collection_eligible_at_secs > metadata.quarantined_at_secs -} - -fn verify_artifact( - quarantine: &Dir, - name: &str, - metadata: &IncidentDebrisMetadataV1, -) -> io::Result<()> { - let mut options = OpenOptions::new(); - options.read(true).follow(FollowSymlinks::No); - let mut file = quarantine.open_with(name, &options)?; - let observed = file.metadata()?; - if !observed.is_file() || observed.len() != metadata.size_bytes { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "incident debris artifact size mismatch", - )); - } - if sha256_reader(&mut file)? != metadata.content_sha256 { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "incident debris artifact digest mismatch", - )); - } - Ok(()) -} - -fn sha256_reader(mut reader: impl Read) -> io::Result { - let mut hasher = Sha256::new(); - let mut buffer = vec![0u8; HASH_BUFFER_BYTES].into_boxed_slice(); - loop { - let read = reader.read(&mut buffer)?; - if read == 0 { - break; - } - hasher.update(&buffer[..read]); - } - Ok(encode_lowercase_hex(&hasher.finalize())) -} - fn application_artifact( store: &StoreKeyV1, name: &str, @@ -802,38 +278,6 @@ fn application_artifact( .filter(|artifact| artifact.kind == kind) } -fn metadata_name(record_id: &str) -> String { - format!("{record_id}.json") -} - -fn artifact_name(record_id: &str) -> String { - format!("{record_id}.artifact") -} - -fn is_sha256(value: &str) -> bool { - value.len() == 64 && value.bytes().all(|byte| byte.is_ascii_hexdigit()) -} - -fn is_component(value: &str) -> bool { - !value.is_empty() - && value != "." - && value != ".." - && !value.contains('/') - && !value.contains('\\') -} - -fn remove_regular_file_if_exists(directory: &Dir, name: &str) -> io::Result<()> { - match directory.symlink_metadata(name) { - Ok(metadata) if metadata.is_file() => directory.remove_file(name), - Ok(_) => Err(io::Error::new( - io::ErrorKind::InvalidInput, - "incident debris path is not a regular file", - )), - Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), - Err(error) => Err(error), - } -} - fn sync_dir(directory: &Dir) -> io::Result<()> { #[cfg(windows)] { @@ -861,7 +305,6 @@ mod tests { }; const NOW: i64 = 1_800_000_000; - const DAY: i64 = 24 * 60 * 60; fn entry(store_root: &Path) -> StoreCensusEntry { StoreCensusEntry { @@ -887,137 +330,93 @@ mod tests { } } - fn quarantine_files(store_root: &Path) -> Vec { - let quarantine = store_root.join(INCIDENT_DEBRIS_QUARANTINE_DIR); - let mut files = std::fs::read_dir(quarantine) - .unwrap() - .map(|item| item.unwrap().path()) - .collect::>(); - files.sort(); - files - } - #[test] - fn sweep_quarantines_loose_debris_with_metadata_and_preserves_live_files() { + fn sweep_deletes_loose_debris_immediately_and_preserves_live_files() { let profile = tempfile::tempdir().unwrap(); let store_root = profile.path().join("stores/store.debris"); - std::fs::create_dir_all(&store_root).unwrap(); + std::fs::create_dir_all(store_root.join("payloads")).unwrap(); let debris = store_root.join("sessions.db.corrupt-incident"); + let scratch = store_root.join("recovery-scratch-incident"); let live = store_root.join("sessions.db"); + let live_wal = store_root.join("sessions.db-wal"); std::fs::write(&debris, b"debris payload").unwrap(); + std::fs::write(&scratch, b"scratch").unwrap(); std::fs::write(&live, b"live database").unwrap(); + std::fs::write(&live_wal, b"live wal").unwrap(); - let report = sweep_incident_debris(&[entry(&store_root)], profile.path(), 7 * DAY, NOW); - - assert_eq!(report.quarantined, 1); - assert_eq!(report.collected, 0); - assert_eq!(report.retained, 1); - assert!(report.errors.is_empty(), "{:?}", report.errors); - assert!(!debris.exists(), "loose debris must move into quarantine"); - assert!(live.exists(), "live store files must remain untouched"); - let files = quarantine_files(&store_root); - assert_eq!(files.len(), 2, "artifact plus durable metadata"); - let metadata_path = files + let before = scan_incident_debris(&entry(&store_root), profile.path(), NOW).unwrap(); + let mut kinds = before + .artifacts .iter() - .find(|path| { - path.extension() - .is_some_and(|extension| extension == "json") - }) - .unwrap(); - let metadata: IncidentDebrisMetadataV1 = - serde_json::from_slice(&std::fs::read(metadata_path).unwrap()).unwrap(); - assert_eq!(metadata.store_id, "store.debris"); - assert_eq!(metadata.original_name, "sessions.db.corrupt-incident"); - assert_eq!(metadata.kind, IncidentDebrisKindV1::Corrupt); - assert_eq!(metadata.size_bytes, b"debris payload".len() as u64); - assert_eq!(metadata.quarantined_at_secs, NOW); - assert_eq!(metadata.collection_eligible_at_secs, NOW + 7 * DAY); + .map(|artifact| artifact.kind) + .collect::>(); + kinds.sort(); + assert_eq!( + kinds, + [ + IncidentDebrisKindV1::Corrupt, + IncidentDebrisKindV1::RecoveryScratch + ] + ); - let scan = scan_incident_debris(&entry(&store_root), profile.path(), NOW).unwrap(); - assert!(scan.listing_complete); - assert_eq!(scan.artifact_count(), 1); + let report = sweep_incident_debris(&[entry(&store_root)], profile.path()); + + assert_eq!(report.collected, 2); assert_eq!( - scan.artifacts[0].path.as_str(), - "sessions.db.corrupt-incident" + report.reclaimed_bytes, + (b"debris payload".len() + b"scratch".len()) as u64 ); + assert!(report.errors.is_empty(), "{:?}", report.errors); + assert!(!debris.exists() && !scratch.exists()); + assert_eq!(std::fs::read(&live).unwrap(), b"live database"); + assert_eq!(std::fs::read(&live_wal).unwrap(), b"live wal"); + assert!(store_root.join("payloads").is_dir()); + let entries = std::fs::read_dir(&store_root) + .unwrap() + .map(|entry| entry.unwrap().file_name()) + .collect::>(); + assert_eq!(entries.len(), 3, "no copy of the debris is left behind"); + + let after = scan_incident_debris(&entry(&store_root), profile.path(), NOW).unwrap(); + assert!(after.listing_complete); + assert!(after.is_empty()); } - /// The graph mount's corruption quarantine (issue #763) moves the - /// container family into a `.corrupt-` directory beside the - /// fresh store. Doctor must surface that directory as one typed Corrupt - /// debris artifact sized by its retained forensic payload, and the sweep - /// must leave it in place, it is already quarantined evidence, never - /// loose debris to relocate or collect. #[test] - fn scan_surfaces_a_graph_store_quarantine_directory_and_sweep_retains_it() { + fn directories_are_never_debris_even_with_a_debris_name() { let profile = tempfile::tempdir().unwrap(); let store_root = profile.path().join("stores/store.debris"); - let quarantine = store_root.join("tracedecay.grafeo.corrupt-1721692800000000"); - std::fs::create_dir_all(quarantine.join("tracedecay.grafeo.wal")).unwrap(); - std::fs::write(store_root.join("sessions.db"), b"live database").unwrap(); - std::fs::write(quarantine.join("tracedecay.grafeo"), b"corrupt container").unwrap(); - std::fs::write( - quarantine - .join("tracedecay.grafeo.wal") - .join("wal_00000001.log"), - b"torn wal segment", - ) - .unwrap(); - std::fs::write(quarantine.join("store-quarantined.json"), b"{}").unwrap(); - let expected_bytes = - (b"corrupt container".len() + b"torn wal segment".len() + b"{}".len()) as u64; + let named = store_root.join("graph.db.recovered-dir"); + std::fs::create_dir_all(&named).unwrap(); + std::fs::write(named.join("payload"), b"store data").unwrap(); let scan = scan_incident_debris(&entry(&store_root), profile.path(), NOW).unwrap(); - assert!(scan.listing_complete); - assert_eq!(scan.artifact_count(), 1); - assert_eq!( - scan.artifacts[0].path.as_str(), - "tracedecay.grafeo.corrupt-1721692800000000" - ); - assert_eq!(scan.artifacts[0].kind, IncidentDebrisKindV1::Corrupt); - assert_eq!(scan.artifacts[0].size_bytes.get(), expected_bytes); + assert!(scan.is_empty()); - let report = sweep_incident_debris(&[entry(&store_root)], profile.path(), 7 * DAY, NOW); - assert_eq!( - report.quarantined, 0, - "the directory is already quarantined" - ); + let report = sweep_incident_debris(&[entry(&store_root)], profile.path()); assert_eq!(report.collected, 0); assert!(report.errors.is_empty(), "{:?}", report.errors); - assert_eq!( - std::fs::read(quarantine.join("tracedecay.grafeo")).unwrap(), - b"corrupt container", - "forensic evidence must survive the sweep untouched" - ); + assert_eq!(std::fs::read(named.join("payload")).unwrap(), b"store data"); } + #[cfg(unix)] #[test] - fn sweep_collects_only_after_the_quarantine_window() { + fn sweep_never_follows_a_debris_named_symlink() { let profile = tempfile::tempdir().unwrap(); + let outside = tempfile::tempdir().unwrap(); let store_root = profile.path().join("stores/store.debris"); std::fs::create_dir_all(&store_root).unwrap(); - std::fs::write( - store_root.join("graph.db.recovered-incident"), - b"recoverable debris", - ) - .unwrap(); - let census = [entry(&store_root)]; + let target = outside.path().join("precious.db"); + std::fs::write(&target, b"not debris").unwrap(); + let link = store_root.join("graph.db.recovered-link"); + std::os::unix::fs::symlink(&target, &link).unwrap(); - let first = sweep_incident_debris(&census, profile.path(), 7 * DAY, NOW); - assert_eq!(first.quarantined, 1); - assert_eq!(first.retained, 1); - let early = sweep_incident_debris(&census, profile.path(), 7 * DAY, NOW + 7 * DAY - 1); - assert_eq!(early.collected, 0); - assert_eq!(early.retained, 1); - assert_eq!(quarantine_files(&store_root).len(), 2); + let report = sweep_incident_debris(&[entry(&store_root)], profile.path()); - let due = sweep_incident_debris(&census, profile.path(), 7 * DAY, NOW + 7 * DAY); - assert_eq!(due.collected, 1); - assert_eq!(due.reclaimed_bytes, b"recoverable debris".len() as u64); - assert_eq!(due.retained, 0); - assert!(due.errors.is_empty(), "{:?}", due.errors); - assert!(quarantine_files(&store_root).is_empty()); + assert_eq!(report.collected, 0); + assert!(link.symlink_metadata().is_ok()); + assert_eq!(std::fs::read(&target).unwrap(), b"not debris"); } #[test] @@ -1027,9 +426,9 @@ mod tests { let debris = outside.path().join("sessions.db.corrupt-outside"); std::fs::write(&debris, b"outside").unwrap(); - let report = sweep_incident_debris(&[entry(outside.path())], profile.path(), 7 * DAY, NOW); + let report = sweep_incident_debris(&[entry(outside.path())], profile.path()); - assert_eq!(report.quarantined, 0); + assert_eq!(report.collected, 0); assert_eq!( report.errors, vec![IncidentDebrisFailure { @@ -1039,40 +438,4 @@ mod tests { ); assert!(debris.exists()); } - - #[test] - fn collection_refuses_tampered_quarantine_content() { - let profile = tempfile::tempdir().unwrap(); - let store_root = profile.path().join("stores/store.debris"); - std::fs::create_dir_all(&store_root).unwrap(); - std::fs::write( - store_root.join("recovery-scratch-incident"), - b"original debris", - ) - .unwrap(); - let census = [entry(&store_root)]; - let first = sweep_incident_debris(&census, profile.path(), DAY, NOW); - assert!(first.errors.is_empty()); - let artifact = quarantine_files(&store_root) - .into_iter() - .find(|path| { - path.extension() - .is_some_and(|extension| extension == "artifact") - }) - .unwrap(); - std::fs::write(&artifact, b"tampered").unwrap(); - - let due = sweep_incident_debris(&census, profile.path(), DAY, NOW + DAY); - - assert_eq!(due.collected, 0); - assert_eq!(due.retained, 1); - assert_eq!( - due.errors, - vec![IncidentDebrisFailure { - store_id: "store.debris".to_string(), - kind: IncidentDebrisFailureKind::IntegrityMismatch, - }] - ); - assert!(artifact.exists(), "tampered evidence must fail closed"); - } } diff --git a/crates/tracedecay-maintenance/src/retention/orphan_stores.rs b/crates/tracedecay-maintenance/src/retention/orphan_stores.rs index 63518a5e08..eed152075f 100644 --- a/crates/tracedecay-maintenance/src/retention/orphan_stores.rs +++ b/crates/tracedecay-maintenance/src/retention/orphan_stores.rs @@ -23,22 +23,20 @@ use std::path::{Path, PathBuf}; use tracedecay_global_db::registry_maintenance::{RootLivenessV1, probe_root}; +mod collection; mod fence; mod pages; -mod quarantine; mod unregistered_page; pub use fence::{ StoreContentEntry, StoreContentEntryKind, StoreContentFence, StoreContentInventory, StoreDirectoryFence, StoreFileIdentity, StoreRootIdentity, }; -#[cfg(test)] -pub(crate) use quarantine::read_pending_quarantine_receipts; pub use unregistered_page::UnregisteredSweepCompletionV1; +pub(super) use unregistered_page::read_project_directory_page; pub use unregistered_page::{ DEFAULT_UNREGISTERED_STORE_PAGE_LIMIT, UnregisteredStoreSweepReport, UnregisteredStoreSweepRequestV1, sweep_unregistered_store_page, }; -pub(super) use unregistered_page::{ProjectDirectoryWorkV1, read_project_directory_page}; /// One profile-sharded store observed on disk, paired with the registry /// identity that points at it. This is the pure input to classification so the @@ -80,7 +78,7 @@ pub struct StoreCensusEntry { /// prior store's eligibility merely by copying its payload mtimes. pub expected_data_root_fence: StoreDirectoryFence, /// Complete no-follow child content/identity fence. Collection rechecks it - /// only after atomically moving the store into a same-parent quarantine. + /// on the opened store directory immediately before deleting it. pub expected_content_fence: StoreContentFence, pub expected_manifest_bytes: Option>, /// Registered graph-scope database paths, relative to `data_root`. Scopes @@ -270,76 +268,31 @@ pub struct CollectedStore { pub size_bytes: u64, } -/// The exact filesystem mutation that failed during orphan-store retirement. +/// The exact filesystem mutation that failed while deleting an orphan store. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum CollectionMutationOperation { - ReserveQuarantineName, - PublishQuarantineJournal, - PublishQuarantineRenameMarker, - RenameLiveLeafToQuarantine, - RestoreLiveLeafFromQuarantine, - ProbeRecoveryJournal, - ValidateRestoredStoreIdentity, - ClearRecoveryJournal, - MarkRetirementCommitted, RecursiveRemove, ParentSync, } -/// Whether a mutation failure is a known external-owner deferral. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum CollectionMutationFailureClassification { - RetryableDeferred, - NonRetryable, -} - /// Structured evidence for a failed orphan-store filesystem mutation. #[derive(Debug, Clone, PartialEq, Eq)] pub struct CollectionMutationFailure { pub operation: CollectionMutationOperation, pub raw_os_error: Option, pub target_path: PathBuf, - pub expected_root_identity: Option, - pub classification: CollectionMutationFailureClassification, } impl CollectionMutationFailure { - pub fn retryable(&self) -> bool { - self.classification == CollectionMutationFailureClassification::RetryableDeferred - } - pub(crate) fn from_io_error( operation: CollectionMutationOperation, target_path: PathBuf, - expected_root_identity: Option, error: &std::io::Error, - ) -> Self { - let raw_os_error = error.raw_os_error(); - let classification = if cfg!(windows) && matches!(raw_os_error, Some(5 | 32 | 33)) { - CollectionMutationFailureClassification::RetryableDeferred - } else { - CollectionMutationFailureClassification::NonRetryable - }; - Self { - operation, - raw_os_error, - target_path, - expected_root_identity, - classification, - } - } - - pub(crate) fn without_native_error( - operation: CollectionMutationOperation, - target_path: PathBuf, - expected_root_identity: Option, ) -> Self { Self { operation, - raw_os_error: None, + raw_os_error: error.raw_os_error(), target_path, - expected_root_identity, - classification: CollectionMutationFailureClassification::NonRetryable, } } } @@ -370,36 +323,11 @@ pub struct CollectionFailure { pub kind: CollectionFailureKind, } -/// A truthful recovery receipt for a store moved to the retention quarantine. -/// A failed post-move proof never becomes an invisible failure: either the -/// original name was restored, or the moved bytes remain at the named sibling -/// for a later reconciliation pass. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum CollectionRecoveryAction { - Restored, - RetainedForRecovery, - /// Registry retirement committed, but the irreversible delete has not yet - /// been durably confirmed. A journal-backed retry owns this state. - DeleteUnconfirmed, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct CollectionRecoveryReceipt { - pub store_id: String, - pub original_path: PathBuf, - pub quarantine_path: PathBuf, - /// The path that currently owns the bytes (or, after a remove/sync - /// ambiguity, the exact path whose deletion remains unconfirmed). - pub actual_path: PathBuf, - pub action: CollectionRecoveryAction, -} - #[derive(Debug, Clone, PartialEq, Eq, Default)] pub struct CollectionOutcome { pub collected: Vec, pub reclaimed_bytes: u64, pub errors: Vec, - pub recovery_receipts: Vec, /// A bounded pass may have completed only a prefix of its plan. This is /// never reported as a successful empty collection. pub completion: CollectionCompletionV1, @@ -413,6 +341,13 @@ pub enum CollectionCompletionV1 { DeadlineExceeded, } +pub use collection::execute_registered_collection; +pub(crate) use collection::{CollectionControl, execute_unregistered_collection_controlled}; +#[cfg(test)] +pub(crate) use collection::{ + execute_registered_collection_controlled, execute_unregistered_collection, + unbounded_collection_control, +}; pub use pages::{ OrphanSweepReport, StoreCensusPageV1, UnregisteredCollectionPlan, UnregisteredStoreFinding, build_store_census, build_store_census_page, plan_unregistered_collection, @@ -424,13 +359,6 @@ pub(crate) use pages::{ dir_size_bytes, dir_size_bytes_controlled, manifest_names_abandoned_root, newest_mtime_secs_controlled, }; -pub use quarantine::execute_registered_collection; -pub(crate) use quarantine::{CollectionControl, execute_unregistered_collection_controlled}; -#[cfg(test)] -pub(crate) use quarantine::{ - execute_registered_collection_controlled, execute_unregistered_collection, - unbounded_collection_control, -}; #[cfg(test)] #[allow(clippy::unwrap_used, clippy::expect_used)] diff --git a/crates/tracedecay-maintenance/src/retention/orphan_stores/collection.rs b/crates/tracedecay-maintenance/src/retention/orphan_stores/collection.rs new file mode 100644 index 0000000000..bc172a4520 --- /dev/null +++ b/crates/tracedecay-maintenance/src/retention/orphan_stores/collection.rs @@ -0,0 +1,1111 @@ +//! Registered and unregistered orphan-store collection under the census fences. + +use std::future::Future; +use std::path::{Path, PathBuf}; +use std::time::Instant; + +use cap_std::fs::Dir; +use tracedecay_global_db::RegisteredGlobalDb; +use tracedecay_private_fs::capability_dir::{remove_open_dir_all_nofollow, sync_directory}; +use tracedecay_runtime_core::cancellation::{CancellationToken, MonotonicDeadline}; + +use super::fence::{ + StoreContentFence, StoreDirectoryFence, capture_store_content_fence_in_dir_controlled, + capture_store_directory_fence, data_root_fence_matches, open_store_directory_nofollow, + profile_relative_store_path, +}; +use super::pages::newest_mtime_secs_controlled; +use super::{ + CollectedStore, CollectionCompletionV1, CollectionFailure, CollectionFailureKind, + CollectionMutationFailure, CollectionMutationOperation, CollectionOutcome, CollectionPlan, + OrphanStoreFinding, UnregisteredCollectionPlan, UnregisteredStoreFinding, +}; + +/// Cooperative budget carried through every expensive retention read and +/// apply boundary. The database writer is acquired only after content hashing +/// and durable-memory inspection have completed under this control. +#[derive(Clone, Copy)] +pub(crate) struct CollectionControl<'a> { + cancellation: &'a CancellationToken, + deadline: MonotonicDeadline, +} + +impl<'a> CollectionControl<'a> { + pub(crate) const fn new( + cancellation: &'a CancellationToken, + deadline: MonotonicDeadline, + ) -> Self { + Self { + cancellation, + deadline, + } + } + + pub(crate) fn completion(self) -> Option { + if self.cancellation.is_cancelled() { + Some(CollectionCompletionV1::Cancelled) + } else if self.deadline.is_elapsed_at(Instant::now()) { + Some(CollectionCompletionV1::DeadlineExceeded) + } else { + None + } + } + + /// Adapt the retention admission to the canonical `SQLite` read-snapshot + /// control. The snapshot layer may copy/materialize a foreign database in + /// `spawn_blocking`, so it must observe the same live cancellation and + /// deadline rather than an unbounded root-shim control. + pub(crate) fn snapshot_read_control( + self, + ) -> tracedecay_runtime_core::sqlite_read_snapshot::SnapshotReadControl { + let cancellation = (*self.cancellation).clone(); + tracedecay_runtime_core::sqlite_read_snapshot::SnapshotReadControl::new( + self.deadline.instant(), + move || cancellation.is_cancelled(), + ) + } + + /// Race an awaitable inspection or `SQLite` command against the admission's + /// cancellation/deadline. Losing the race never authorizes the following + /// destructive phase; a later pass inspects the store afresh. + pub(crate) async fn race( + self, + future: impl Future, + ) -> Result { + if let Some(completion) = self.completion() { + return Err(completion); + } + tokio::select! { + biased; + () = self.cancellation.cancelled() => Err(CollectionCompletionV1::Cancelled), + () = tokio::time::sleep_until(tokio::time::Instant::from_std(self.deadline.instant())) => { + Err(CollectionCompletionV1::DeadlineExceeded) + } + result = future => { + self.completion().map_or(Ok(result), Err) + } + } + } +} + +pub(crate) fn unbounded_collection_control() -> CollectionControl<'static> { + static CANCELLATION: std::sync::OnceLock = std::sync::OnceLock::new(); + CollectionControl::new( + CANCELLATION.get_or_init(CancellationToken::new), + MonotonicDeadline::at(Instant::now() + std::time::Duration::from_hours(24)), + ) +} + +pub(crate) fn store_finding_is_profile_contained( + finding: &OrphanStoreFinding, + profile_root: &Path, +) -> bool { + profile_relative_store_path(profile_root, &finding.data_root) + .is_ok_and(|relative| relative == Path::new(&finding.expected_store_relpath)) + && matches!( + capture_store_directory_fence(profile_root, &finding.data_root), + Ok(StoreDirectoryFence::Missing | StoreDirectoryFence::Present { .. }) + ) +} + +fn registered_payload_fence_matches( + finding: &OrphanStoreFinding, + profile_root: &Path, + control: CollectionControl<'_>, +) -> Result { + if !data_root_fence_matches( + &finding.expected_data_root_fence, + profile_root, + &finding.data_root, + )? { + return Ok(false); + } + match &finding.expected_data_root_fence { + StoreDirectoryFence::Missing => Ok(true), + StoreDirectoryFence::Present { .. } => { + Ok(newest_mtime_secs_controlled(&finding.data_root, control)? + == finding.expected_payload_mtime_secs) + } + StoreDirectoryFence::Unverifiable => Err(CollectionFailureKind::InspectFailed), + } +} + +fn unregistered_payload_fence_matches( + finding: &UnregisteredStoreFinding, + profile_root: &Path, + control: CollectionControl<'_>, +) -> Result { + if !data_root_fence_matches( + &finding.expected_data_root_fence, + profile_root, + &finding.data_root, + )? { + return Ok(false); + } + match &finding.expected_data_root_fence { + StoreDirectoryFence::Missing => Ok(true), + StoreDirectoryFence::Present { .. } => { + Ok(newest_mtime_secs_controlled(&finding.data_root, control)? + == finding.expected_payload_mtime_secs) + } + StoreDirectoryFence::Unverifiable => Err(CollectionFailureKind::InspectFailed), + } +} + +/// The censused store leaf, opened no-follow and proven byte-for-byte equal to +/// its census content fence. Removal goes through this handle, so a path +/// swapped in after the proof is never the directory that gets deleted. +pub(super) struct VerifiedStore { + parent: Dir, + root: Dir, + data_root: PathBuf, +} + +impl VerifiedStore { + /// Runs to completion once started: callers invoke it only after the + /// registry authority is retired, and a cancelled removal would leave a + /// half-deleted store with no owner. + pub(super) fn remove(self) -> Result<(), CollectionMutationFailure> { + let Self { + parent, + root, + data_root, + } = self; + remove_open_dir_all_nofollow(root, &mut || Ok(())).map_err(|error| { + CollectionMutationFailure::from_io_error( + CollectionMutationOperation::RecursiveRemove, + data_root.clone(), + &error, + ) + })?; + sync_directory(&parent).map_err(|error| { + CollectionMutationFailure::from_io_error( + CollectionMutationOperation::ParentSync, + data_root + .parent() + .map_or_else(PathBuf::new, Path::to_path_buf), + &error, + ) + }) + } +} + +/// Opens `data_root` and proves its exact content still equals `expected`. +/// `Ok(None)` means the census already observed the store absent. +pub(super) fn open_verified_store( + profile_root: &Path, + data_root: &Path, + expected: &StoreContentFence, + control: CollectionControl<'_>, +) -> Result, CollectionFailureKind> { + match expected { + StoreContentFence::Missing => return Ok(None), + StoreContentFence::Unverifiable => return Err(CollectionFailureKind::InspectFailed), + StoreContentFence::Present(_) => {} + } + let capability = open_store_directory_nofollow(profile_root, data_root)?; + match capture_store_content_fence_in_dir_controlled(&capability.root, Some(control)) { + Ok(actual) if matches!(expected, StoreContentFence::Present(inventory) if *inventory == actual) => { + Ok(Some(VerifiedStore { + parent: capability.parent, + root: capability.root, + data_root: data_root.to_path_buf(), + })) + } + Err(error) if error.kind() == std::io::ErrorKind::Interrupted => { + Err(CollectionFailureKind::Cancelled) + } + Ok(_) | Err(_) => Err(CollectionFailureKind::PayloadChanged), + } +} + +/// Terminal state of one finding within a bounded collection pass. +enum FindingStep { + Collected, + /// The registry authority was retired but the bytes were not fully + /// removed; the failure names the exact filesystem operation. + RemoveFailed(CollectionMutationFailure), + Refused(CollectionFailureKind), + Interrupted(CollectionCompletionV1), +} + +fn interrupted(control: CollectionControl<'_>) -> FindingStep { + FindingStep::Interrupted( + control + .completion() + .unwrap_or(CollectionCompletionV1::Cancelled), + ) +} + +fn payload_fence_step( + matches: Result, + control: CollectionControl<'_>, +) -> Option { + match matches { + Ok(true) => None, + Ok(false) => Some(FindingStep::Refused(CollectionFailureKind::PayloadChanged)), + Err(CollectionFailureKind::Cancelled) => Some(interrupted(control)), + Err(kind) => Some(FindingStep::Refused(kind)), + } +} + +fn durable_memory_step( + check: DurableMemoryCheck, + control: CollectionControl<'_>, +) -> Option { + match check { + DurableMemoryCheck::Empty => None, + DurableMemoryCheck::Present | DurableMemoryCheck::Unverifiable => Some( + FindingStep::Refused(CollectionFailureKind::DurableDataProtected), + ), + DurableMemoryCheck::Interrupted => Some(interrupted(control)), + } +} + +fn verified_store_step( + opened: Result, CollectionFailureKind>, + control: CollectionControl<'_>, +) -> Result, FindingStep> { + match opened { + Ok(store) => Ok(store), + Err(CollectionFailureKind::Cancelled) => Err(interrupted(control)), + Err(kind) => Err(FindingStep::Refused(kind)), + } +} + +fn remove_step(store: Option) -> FindingStep { + match store.map(VerifiedStore::remove) { + None | Some(Ok(())) => FindingStep::Collected, + Some(Err(failure)) => FindingStep::RemoveFailed(failure), + } +} + +/// Records one step; returns `false` when the pass must stop. +fn record_step(outcome: &mut CollectionOutcome, step: FindingStep, store: CollectedStore) -> bool { + let kind = match step { + FindingStep::Collected => { + outcome.reclaimed_bytes = outcome.reclaimed_bytes.saturating_add(store.size_bytes); + outcome.collected.push(store); + return true; + } + FindingStep::Interrupted(completion) => { + outcome.completion = completion; + return false; + } + FindingStep::RemoveFailed(failure) => CollectionFailureKind::RemoveFailed(failure), + FindingStep::Refused(kind) => kind, + }; + outcome.errors.push(CollectionFailure { + store_id: store.store_id, + kind, + }); + true +} + +/// Expensive inspection (payload, manifest, durable memory, exact content) +/// runs without a writer; a short final transaction then retires the exact +/// registry row before the verified store directory is deleted. +pub async fn execute_registered_collection( + db: &RegisteredGlobalDb, + plan: &CollectionPlan, + profile_root: &Path, +) -> tracedecay_domain::errors::Result<(CollectionOutcome, usize)> { + execute_registered_collection_controlled(db, plan, profile_root, unbounded_collection_control()) + .await +} + +#[hotpath::measure(label = "maintenance.orphan_stores.collect_registered", future = true)] +pub(crate) async fn execute_registered_collection_controlled( + db: &RegisteredGlobalDb, + plan: &CollectionPlan, + profile_root: &Path, + control: CollectionControl<'_>, +) -> tracedecay_domain::errors::Result<(CollectionOutcome, usize)> { + let mut outcome = CollectionOutcome::default(); + let mut retired = 0usize; + for finding in &plan.collect { + if let Some(completion) = control.completion() { + outcome.completion = completion; + break; + } + let step = collect_registered_finding(db, finding, profile_root, control).await?; + if matches!(step, FindingStep::Collected | FindingStep::RemoveFailed(_)) { + retired = retired.saturating_add(1); + } + let store = CollectedStore { + project_id: finding.project_id.clone(), + store_id: finding.store_id.clone(), + data_root: finding.data_root.clone(), + size_bytes: finding.size_bytes, + }; + if !record_step(&mut outcome, step, store) { + break; + } + } + Ok((outcome, retired)) +} + +async fn collect_registered_finding( + db: &RegisteredGlobalDb, + finding: &OrphanStoreFinding, + profile_root: &Path, + control: CollectionControl<'_>, +) -> tracedecay_domain::errors::Result { + if !store_finding_is_profile_contained(finding, profile_root) { + return Ok(FindingStep::Refused(CollectionFailureKind::OutsideProfile)); + } + if let Some(step) = payload_fence_step( + registered_payload_fence_matches(finding, profile_root, control), + control, + ) { + return Ok(step); + } + let expected_row = Some(( + finding.expected_store_relpath.clone(), + finding.expected_created_at, + finding.expected_last_write_at, + )); + let current_stores = match control + .race(db.try_list_store_instances_for_project(&finding.project_id)) + .await + { + Ok(stores) => stores?, + Err(completion) => return Ok(FindingStep::Interrupted(completion)), + }; + let current = current_stores + .into_iter() + .find(|store| store.store_id == finding.store_id) + .map(|store| (store.store_relpath, store.created_at, store.last_write_at)); + if current != expected_row { + return Ok(FindingStep::Refused(CollectionFailureKind::RegistryChanged)); + } + + let manifest_path = finding + .data_root + .join(tracedecay_runtime_core::storage::STORE_MANIFEST_FILENAME); + let current_manifest = match read_regular_file(&manifest_path) { + RegularFileSnapshot::Bytes(bytes) => Some(bytes), + RegularFileSnapshot::Missing => None, + RegularFileSnapshot::Unverifiable => { + return Ok(FindingStep::Refused(CollectionFailureKind::InspectFailed)); + } + }; + if current_manifest != finding.expected_manifest_bytes { + return Ok(FindingStep::Refused(CollectionFailureKind::ManifestChanged)); + } + + let check = check_store_durable_memory( + &finding.data_root, + finding.expected_manifest_bytes.as_deref(), + &finding.graph_scope_relpaths, + &durable_check_scratch_root(profile_root), + control, + ) + .await; + if let Some(step) = durable_memory_step(check, control) { + return Ok(step); + } + // The durable inventory can take a private snapshot and therefore leaves + // a window for a concurrent replacement. + if let Some(step) = payload_fence_step( + registered_payload_fence_matches(finding, profile_root, control), + control, + ) { + return Ok(step); + } + let store = match verified_store_step( + open_verified_store( + profile_root, + &finding.data_root, + &finding.expected_content_fence, + control, + ), + control, + ) { + Ok(store) => store, + Err(step) => return Ok(step), + }; + + let transaction = match control.race(db.begin_write_transaction()).await { + Ok(transaction) => transaction?, + Err(completion) => return Ok(FindingStep::Interrupted(completion)), + }; + let mut rows = match control + .race(transaction.query( + "SELECT store_relpath, created_at, last_write_at + FROM store_instances + WHERE project_id = ?1 AND store_id = ?2", + tracedecay_runtime_core::db::engine::params![ + finding.project_id.as_str(), + finding.store_id.as_str() + ], + )) + .await + { + Ok(rows) => rows.map_err(|error| orphan_db_error("confirm orphan registry", error))?, + Err(completion) => return Ok(FindingStep::Interrupted(completion)), + }; + let next = match control.race(rows.next()).await { + Ok(next) => next.map_err(|error| orphan_db_error("read orphan registry", error))?, + Err(completion) => return Ok(FindingStep::Interrupted(completion)), + }; + let current = match next { + Some(row) => Some(( + row.get::(0) + .map_err(|error| orphan_db_error("decode orphan store relpath", error))?, + row.get::(1) + .map_err(|error| orphan_db_error("decode orphan store generation", error))?, + row.get::>(2) + .map_err(|error| orphan_db_error("decode orphan last write", error))?, + )), + None => None, + }; + drop(rows); + if current != expected_row { + transaction + .rollback() + .await + .map_err(|error| orphan_db_error("rollback changed orphan", error))?; + return Ok(FindingStep::Refused(CollectionFailureKind::RegistryChanged)); + } + let deleted = match control + .race(transaction.execute( + "DELETE FROM store_instances + WHERE project_id = ?1 AND store_id = ?2 + AND store_relpath = ?3 AND created_at = ?4 + AND last_write_at IS ?5", + tracedecay_runtime_core::db::engine::params![ + finding.project_id.as_str(), + finding.store_id.as_str(), + finding.expected_store_relpath.as_str(), + finding.expected_created_at, + finding.expected_last_write_at + ], + )) + .await + { + Ok(deleted) => { + deleted.map_err(|error| orphan_db_error("retire collected orphan store", error))? + } + Err(completion) => return Ok(FindingStep::Interrupted(completion)), + }; + if deleted != 1 { + transaction + .rollback() + .await + .map_err(|error| orphan_db_error("rollback raced orphan retirement", error))?; + return Ok(FindingStep::Refused(CollectionFailureKind::RegistryChanged)); + } + match control + .race(transaction.execute( + "DELETE FROM code_projects + WHERE project_id = ?1 + AND NOT EXISTS ( + SELECT 1 FROM store_instances WHERE project_id = ?1 + )", + tracedecay_runtime_core::db::engine::params![finding.project_id.as_str()], + )) + .await + { + Ok(result) => { + result.map_err(|error| orphan_db_error("retire empty collected project", error))?; + } + Err(completion) => return Ok(FindingStep::Interrupted(completion)), + } + // Not raced: abandoning an in-flight commit would leave the retirement + // ambiguous while the verified bytes stay on disk. + transaction + .commit() + .await + .map_err(|error| orphan_db_error("commit collected orphan retirement", error))?; + Ok(remove_step(store)) +} + +fn orphan_db_error( + operation: &'static str, + error: impl std::fmt::Display, +) -> tracedecay_domain::errors::TraceDecayError { + tracedecay_domain::errors::TraceDecayError::Database { + operation: operation.to_string(), + message: error.to_string(), + } +} + +/// Result of checking a store's graph database for durable memory rows. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum DurableMemoryCheck { + /// Cooperative cancellation/deadline interrupted recursive discovery or a + /// bounded database probe before any mutation. + Interrupted, + /// No durable memory table has any row (including: none of the tables + /// exist, or the database file itself does not exist). Safe to collect. + Empty, + /// At least one durable memory table has at least one row. + Present, + /// The check could not prove the store is free of durable memory rows + /// (I/O error, corrupt/locked database, the source changed mid-check). + /// Fails closed: treated exactly like `Present` by every caller. + Unverifiable, +} + +/// Every database under a store that can carry durable rows, or a typed +/// statement that the inventory itself could not be trusted. +/// +/// The databases registered as project authorities for durable memory. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) enum DurableDatabaseInventoryV1 { + /// The bounded scan stopped before it could establish a complete durable + /// database inventory. This is not an unverifiable green light: callers + /// preserve the exact cancellation/deadline state for the coordinator. + Interrupted, + /// The complete set of database paths, relative to the store's data root. + Resolved(Vec), + /// The set could not be enumerated, a missing or malformed manifest, or a + /// directory that could not be listed. Never a green light for deletion. + Unverifiable, +} + +/// A regular-file read that preserves the difference between an absent +/// optional artifact and an unsafe/unreadable one. In particular, `read()` +/// follows symlinks; retention must never turn a symlinked manifest into a +/// trusted manifest snapshot. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) enum RegularFileSnapshot { + Missing, + Bytes(Vec), + Unverifiable, +} + +pub(super) fn read_regular_file(path: &Path) -> RegularFileSnapshot { + let metadata = match std::fs::symlink_metadata(path) { + Ok(metadata) => metadata, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return RegularFileSnapshot::Missing; + } + Err(_) => return RegularFileSnapshot::Unverifiable, + }; + if metadata.file_type().is_symlink() || !metadata.file_type().is_file() { + return RegularFileSnapshot::Unverifiable; + } + let Ok(bytes) = std::fs::read(path) else { + return RegularFileSnapshot::Unverifiable; + }; + match std::fs::symlink_metadata(path) { + Ok(metadata) if !metadata.file_type().is_symlink() && metadata.file_type().is_file() => { + RegularFileSnapshot::Bytes(bytes) + } + _ => RegularFileSnapshot::Unverifiable, + } +} + +/// Store manifests and registry-provided graph scopes are path authorities, +/// not arbitrary filesystem paths. Only normalized, non-empty relative paths +/// made entirely from normal components are accepted; `..`, `.`, roots, +/// prefixes, and empty paths all fail closed before joining. +fn safe_store_relative_path(path: &Path) -> bool { + let mut saw_normal = false; + let mut normalized = PathBuf::new(); + for component in path.components() { + if let std::path::Component::Normal(component) = component { + saw_normal = true; + normalized.push(component); + } else { + return false; + } + } + saw_normal && normalized == path +} + +/// Reject symlinked directory components as well as a symlinked final file. +/// A lexical relative-path check alone is insufficient when an intermediate +/// directory redirects outside the store. +fn safe_store_path(data_root: &Path, relative: &Path) -> bool { + if !safe_store_relative_path(relative) { + return false; + } + let mut current = data_root.to_path_buf(); + for component in relative.components() { + let std::path::Component::Normal(component) = component else { + return false; + }; + current.push(component); + match std::fs::symlink_metadata(¤t) { + Ok(metadata) if metadata.file_type().is_symlink() => return false, + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return true, + Err(_) => return false, + } + } + true +} + +/// Enumerates every durable database a store's manifest and registered graph +/// scopes name, relative to its data root. +/// +/// Fails closed. The manifest is the store's own record of where its graph +/// lives; if it is absent or will not parse, guessing the default filename +/// would check the wrong file (or no file) and report "empty" for a store whose +/// real graph sits elsewhere. +pub(super) fn durable_database_inventory( + manifest_bytes: Option<&[u8]>, + graph_scope_relpaths: &[PathBuf], + control: CollectionControl<'_>, +) -> DurableDatabaseInventoryV1 { + if control.completion().is_some() { + return DurableDatabaseInventoryV1::Interrupted; + } + let Some(bytes) = manifest_bytes else { + return DurableDatabaseInventoryV1::Unverifiable; + }; + let manifest = + match serde_json::from_slice::(bytes) { + Ok(manifest) => manifest, + Err(_) if control.completion().is_some() => { + return DurableDatabaseInventoryV1::Interrupted; + } + Err(_) => return DurableDatabaseInventoryV1::Unverifiable, + }; + if control.completion().is_some() { + return DurableDatabaseInventoryV1::Interrupted; + } + + if !safe_store_relative_path(&manifest.graph_db_relpath) { + return DurableDatabaseInventoryV1::Unverifiable; + } + + let mut inventory = vec![manifest.graph_db_relpath]; + for relpath in graph_scope_relpaths { + if control.completion().is_some() { + return DurableDatabaseInventoryV1::Interrupted; + } + if !safe_store_relative_path(relpath) { + return DurableDatabaseInventoryV1::Unverifiable; + } + if !inventory.contains(relpath) { + inventory.push(relpath.clone()); + } + } + + DurableDatabaseInventoryV1::Resolved(inventory) +} + +/// Runs [`check_durable_memory_rows`] over every database in the store's +/// inventory. Any single `Present` or `Unverifiable` protects the whole store. +pub(super) async fn check_store_durable_memory( + data_root: &Path, + manifest_bytes: Option<&[u8]>, + graph_scope_relpaths: &[PathBuf], + scratch_root: &Path, + control: CollectionControl<'_>, +) -> DurableMemoryCheck { + if control.completion().is_some() { + return DurableMemoryCheck::Interrupted; + } + let inventory = match durable_database_inventory(manifest_bytes, graph_scope_relpaths, control) + { + DurableDatabaseInventoryV1::Interrupted => return DurableMemoryCheck::Interrupted, + DurableDatabaseInventoryV1::Resolved(inventory) => inventory, + DurableDatabaseInventoryV1::Unverifiable => return DurableMemoryCheck::Unverifiable, + }; + for relpath in inventory { + if control.completion().is_some() { + return DurableMemoryCheck::Interrupted; + } + match check_durable_memory_rows(data_root, &relpath, scratch_root, control).await { + DurableMemoryCheck::Empty => {} + protected => return protected, + } + } + DurableMemoryCheck::Empty +} + +/// The read-snapshot scratch directory for durable-memory checks. +/// +/// It lives under the *profile* root, never inside the store being examined. +/// Two reasons, both load-bearing: the store is a deletion candidate, and +/// writing into it bumps the newest mtime that +/// [`walk_store_stats`] uses as the revival fence, a store that failed one +/// check would have its age reset by the check itself and could never mature +/// past the retention window again. +pub(super) fn durable_check_scratch_root(profile_root: &Path) -> PathBuf { + profile_root.join("scratch").join("sqlite-read") +} + +/// Checks whether `data_root`'s graph database carries rows in any canonical +/// `memory_*` table. This intentionally discovers tables from the schema +/// instead of maintaining a fixed list: both legacy memory and Memory V2 add +/// durable tables, and a newly added table must be protected automatically. +/// Side-effect-free with respect to the store: opens the database through +/// [`tracedecay_runtime_core::sqlite_read_snapshot`], so the live store is never mutated or +/// locked against a concurrent writer. +async fn check_durable_memory_rows( + data_root: &Path, + graph_db_relpath: &Path, + scratch_root: &Path, + control: CollectionControl<'_>, +) -> DurableMemoryCheck { + if control.completion().is_some() { + return DurableMemoryCheck::Interrupted; + } + if !safe_store_path(data_root, graph_db_relpath) { + return DurableMemoryCheck::Unverifiable; + } + let graph_db_path = data_root.join(graph_db_relpath); + match std::fs::symlink_metadata(&graph_db_path) { + Ok(metadata) if metadata.file_type().is_symlink() || !metadata.file_type().is_file() => { + return DurableMemoryCheck::Unverifiable; + } + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + // No database file at all: there is no schema that could carry + // durable rows. + return DurableMemoryCheck::Empty; + } + Err(_) => return DurableMemoryCheck::Unverifiable, + } + // The snapshot layer creates only the final scratch component, so its + // parent must exist first. Without this the snapshot fails NotFound, the + // check fails closed as `Unverifiable`, and because `Unverifiable` is + // treated exactly like `Present`, *every* collection is refused. That is + // safe, but it silently disables orphan reclamation entirely. + if control.completion().is_some() || std::fs::create_dir_all(scratch_root).is_err() { + return if control.completion().is_some() { + DurableMemoryCheck::Interrupted + } else { + DurableMemoryCheck::Unverifiable + }; + } + let snapshot = match control + .race( + tracedecay_runtime_core::sqlite_read_snapshot::open_foreign_in( + &graph_db_path, + scratch_root, + control.snapshot_read_control(), + ), + ) + .await + { + Err(_) => return DurableMemoryCheck::Interrupted, + Ok(Ok(snapshot)) => snapshot, + Ok(Err(_)) => return DurableMemoryCheck::Unverifiable, + }; + if control.completion().is_some() { + return DurableMemoryCheck::Interrupted; + } + let connection = snapshot.connection(); + let mut rows = match control + .race(connection.query( + "SELECT name + FROM pragma_table_list + WHERE schema = 'main' + AND type = 'table' + AND name LIKE ?1 ESCAPE '\\' + ORDER BY name", + tracedecay_runtime_core::db::engine::params!["memory\\_%"], + )) + .await + { + Err(_) => return DurableMemoryCheck::Interrupted, + Ok(Ok(rows)) => rows, + Ok(Err(_)) => return DurableMemoryCheck::Unverifiable, + }; + let mut present_tables = Vec::new(); + loop { + let next = match control.race(rows.next()).await { + Err(_) => return DurableMemoryCheck::Interrupted, + Ok(Ok(next)) => next, + Ok(Err(_)) => return DurableMemoryCheck::Unverifiable, + }; + match next { + Some(row) => match row.get::(0) { + Ok(name) => present_tables.push(name), + Err(_) => return DurableMemoryCheck::Unverifiable, + }, + None => break, + } + } + drop(rows); + for table in present_tables { + // `pragma_table_list.type = 'table'` intentionally excludes FTS + // virtual/shadow tables, whose internal config rows are derived and + // exist even when there is no durable memory. Identifiers cannot be + // SQL parameters, so only interpolate TraceDecay's canonical shape; + // an unexpected name fails closed rather than becoming SQL text. + if !is_memory_table_identifier(&table) { + return DurableMemoryCheck::Unverifiable; + } + let probe_sql = format!("SELECT 1 FROM \"{table}\" LIMIT 1"); + let mut probe_rows = match control.race(connection.query(&probe_sql, ())).await { + Err(_) => return DurableMemoryCheck::Interrupted, + Ok(Ok(rows)) => rows, + Ok(Err(_)) => return DurableMemoryCheck::Unverifiable, + }; + match control.race(probe_rows.next()).await { + Err(_) => return DurableMemoryCheck::Interrupted, + Ok(Ok(Some(_))) => return DurableMemoryCheck::Present, + Ok(Ok(None)) => {} + Ok(Err(_)) => return DurableMemoryCheck::Unverifiable, + } + } + if control.completion().is_some() { + return DurableMemoryCheck::Interrupted; + } + if snapshot.validate_source().is_err() { + // The file changed under us mid-check: cannot trust an empty result. + return DurableMemoryCheck::Unverifiable; + } + DurableMemoryCheck::Empty +} + +fn is_memory_table_identifier(table: &str) -> bool { + table.strip_prefix("memory_").is_some_and(|suffix| { + !suffix.is_empty() + && suffix + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'_') + }) +} + +#[cfg(test)] +pub(crate) async fn execute_unregistered_collection( + db: &RegisteredGlobalDb, + plan: &UnregisteredCollectionPlan, + profile_root: &Path, +) -> tracedecay_domain::errors::Result { + execute_unregistered_collection_controlled( + db, + plan, + profile_root, + unbounded_collection_control(), + ) + .await +} + +#[hotpath::measure( + label = "maintenance.orphan_stores.collect_unregistered", + future = true +)] +pub(crate) async fn execute_unregistered_collection_controlled( + db: &RegisteredGlobalDb, + plan: &UnregisteredCollectionPlan, + profile_root: &Path, + control: CollectionControl<'_>, +) -> tracedecay_domain::errors::Result { + let mut outcome = CollectionOutcome::default(); + for finding in &plan.collect { + if let Some(completion) = control.completion() { + outcome.completion = completion; + break; + } + let step = collect_unregistered_finding(db, finding, profile_root, control).await?; + let store = CollectedStore { + project_id: finding.project_dir_name.clone(), + store_id: finding.project_dir_name.clone(), + data_root: finding.data_root.clone(), + size_bytes: finding.size_bytes, + }; + if !record_step(&mut outcome, step, store) { + break; + } + } + Ok(outcome) +} + +async fn collect_unregistered_finding( + db: &RegisteredGlobalDb, + finding: &UnregisteredStoreFinding, + profile_root: &Path, + control: CollectionControl<'_>, +) -> tracedecay_domain::errors::Result { + // Containment + shape: only ever delete an exact, safely-named + // `/projects/` leaf. + let expected = profile_root + .join("projects") + .join(&finding.project_dir_name); + if expected != finding.data_root + || tracedecay_runtime_core::storage::validate_project_id(&finding.project_dir_name).is_err() + { + return Ok(FindingStep::Refused(CollectionFailureKind::OutsideProfile)); + } + if let Some(step) = payload_fence_step( + unregistered_payload_fence_matches(finding, profile_root, control), + control, + ) { + return Ok(step); + } + match control + .race(db.code_project_exists(&finding.project_dir_name)) + .await + { + Ok(exists) => { + if exists? { + return Ok(FindingStep::Refused(CollectionFailureKind::RegistryChanged)); + } + } + Err(completion) => return Ok(FindingStep::Interrupted(completion)), + } + + let scratch_root = durable_check_scratch_root(profile_root); + // An unreadable manifest must not be swallowed into "no manifest": the + // inventory then fails closed instead of checking a guessed database. A + // manifestless directory is different: only an exact empty-tree inventory + // proves that it carries no durable authority. Arbitrary payload files + // remain unverifiable, while any discovered `.db` family is inspected + // directly and remains fail-closed on error. + let manifest_path = finding + .data_root + .join(tracedecay_runtime_core::storage::STORE_MANIFEST_FILENAME); + let check = match read_regular_file(&manifest_path) { + RegularFileSnapshot::Bytes(manifest_bytes) => { + // An unregistered store has no registry graph scopes by + // definition; the manifest remains the canonical graph path. + check_store_durable_memory( + &finding.data_root, + Some(&manifest_bytes), + &[], + &scratch_root, + control, + ) + .await + } + RegularFileSnapshot::Missing => { + check_manifestless_store_durable_memory(&finding.data_root, &scratch_root, control) + .await + } + RegularFileSnapshot::Unverifiable => DurableMemoryCheck::Unverifiable, + }; + if let Some(step) = durable_memory_step(check, control) { + return Ok(step); + } + // The durable-data inspection is not a deletion lock: an in-profile + // replacement or symlink swap must not inherit its decision. + if let Some(step) = payload_fence_step( + unregistered_payload_fence_matches(finding, profile_root, control), + control, + ) { + return Ok(step); + } + let store = match verified_store_step( + open_verified_store( + profile_root, + &finding.data_root, + &finding.expected_content_fence, + control, + ), + control, + ) { + Ok(store) => store, + Err(step) => return Ok(step), + }; + + // The writer transaction orders this final absence proof after any + // in-flight registration commit. + let transaction = match control.race(db.begin_write_transaction()).await { + Ok(transaction) => transaction?, + Err(completion) => return Ok(FindingStep::Interrupted(completion)), + }; + let mut rows = match control + .race(transaction.query( + "SELECT 1 FROM code_projects WHERE project_id = ?1", + tracedecay_runtime_core::db::engine::params![finding.project_dir_name.as_str()], + )) + .await + { + Ok(rows) => rows.map_err(|error| orphan_db_error("confirm unregistered store", error))?, + Err(completion) => return Ok(FindingStep::Interrupted(completion)), + }; + let now_registered = match control.race(rows.next()).await { + Ok(next) => next + .map_err(|error| orphan_db_error("read unregistered store", error))? + .is_some(), + Err(completion) => return Ok(FindingStep::Interrupted(completion)), + }; + drop(rows); + transaction + .rollback() + .await + .map_err(|error| orphan_db_error("release unregistered store fence", error))?; + if now_registered { + return Ok(FindingStep::Refused(CollectionFailureKind::RegistryChanged)); + } + Ok(remove_step(store)) +} + +/// Inspects a manifestless unregistered directory without inventing a graph +/// path. An exactly empty directory is provably free of durable rows. Any +/// arbitrary payload, symlink, or unreadable entry remains unverifiable; +/// when a SQLite-looking file is present, every such file is treated as a +/// possible durable authority and inspected fail-closed. +async fn check_manifestless_store_durable_memory( + data_root: &Path, + scratch_root: &Path, + control: CollectionControl<'_>, +) -> DurableMemoryCheck { + let mut databases = Vec::new(); + if control.completion().is_some() { + return DurableMemoryCheck::Interrupted; + } + if collect_sqlite_candidates(data_root, data_root, &mut databases, control).is_err() { + return if control.completion().is_some() { + DurableMemoryCheck::Interrupted + } else { + DurableMemoryCheck::Unverifiable + }; + } + if databases.is_empty() { + return DurableMemoryCheck::Empty; + } + for relpath in databases { + if control.completion().is_some() { + return DurableMemoryCheck::Interrupted; + } + match check_durable_memory_rows(data_root, &relpath, scratch_root, control).await { + DurableMemoryCheck::Empty => {} + protected => return protected, + } + } + DurableMemoryCheck::Empty +} + +/// Finds only regular `.db` files below a store and never follows symlinks. +/// The manifestless path deliberately does not guess a single filename, so a +/// custom legacy graph cannot be mistaken for payload-only debris. Any other +/// file shape is an unverifiable durable-data candidate, not disposable dust. +fn collect_sqlite_candidates( + root: &Path, + current: &Path, + output: &mut Vec, + control: CollectionControl<'_>, +) -> std::io::Result<()> { + let entries = std::fs::read_dir(current)?; + for entry in entries { + if control.completion().is_some() { + return Err(std::io::Error::new( + std::io::ErrorKind::Interrupted, + "retention durable-data inventory interrupted", + )); + } + let entry = entry?; + let file_type = entry.file_type()?; + if file_type.is_symlink() { + return Err(std::io::Error::other( + "manifestless store contains a symlink", + )); + } + let path = entry.path(); + if file_type.is_dir() { + collect_sqlite_candidates(root, &path, output, control)?; + } else if file_type.is_file() + && path.extension().and_then(|extension| extension.to_str()) == Some("db") + && let Ok(relative) = path.strip_prefix(root) + { + output.push(relative.to_path_buf()); + } else { + return Err(std::io::Error::other( + "manifestless store contains an unrecognized payload", + )); + } + } + output.sort(); + Ok(()) +} diff --git a/crates/tracedecay-maintenance/src/retention/orphan_stores/fence.rs b/crates/tracedecay-maintenance/src/retention/orphan_stores/fence.rs index 13354b3858..0956bfb2ae 100644 --- a/crates/tracedecay-maintenance/src/retention/orphan_stores/fence.rs +++ b/crates/tracedecay-maintenance/src/retention/orphan_stores/fence.rs @@ -52,7 +52,7 @@ pub struct StoreContentInventory { pub entries: Vec, } -/// Exact content identity carried from census to post-quarantine verification. +/// Exact content identity carried from census to the delete boundary. #[derive(Debug, Clone, PartialEq, Eq)] pub enum StoreContentFence { Missing, @@ -219,8 +219,8 @@ fn capture_store_content_fence_impl( .map_err(|error| failure_from_io(error, control)) } -/// Captures an inventory from an already-open directory; quarantine uses this -/// after its same-parent rename so the proof applies to the moved bytes. +/// Captures an inventory from an already-open directory so the proof applies +/// to the exact handle collection later deletes. pub(super) fn capture_store_content_fence_in_dir_controlled( root: &Dir, control: Option>, diff --git a/crates/tracedecay-maintenance/src/retention/orphan_stores/pages.rs b/crates/tracedecay-maintenance/src/retention/orphan_stores/pages.rs index 328395e94b..d99a08f921 100644 --- a/crates/tracedecay-maintenance/src/retention/orphan_stores/pages.rs +++ b/crates/tracedecay-maintenance/src/retention/orphan_stores/pages.rs @@ -5,11 +5,11 @@ use std::path::{Path, PathBuf}; use tracedecay_global_db::RegisteredGlobalDb; use tracedecay_runtime_core::cancellation::{CancellationToken, MonotonicDeadline}; +use super::collection::{RegularFileSnapshot, read_regular_file}; use super::fence::{ StoreContentFence, StoreDirectoryFence, capture_store_content_fence, capture_store_content_fence_controlled, capture_store_directory_fence, }; -use super::quarantine::{RegularFileSnapshot, read_regular_file}; use super::unregistered_page::{ DEFAULT_UNREGISTERED_STORE_PAGE_LIMIT, UnregisteredStoreSweepReport, UnregisteredStoreSweepRequestV1, UnregisteredSweepCompletionV1, sweep_unregistered_store_page, @@ -20,8 +20,8 @@ use super::{ }; #[cfg(test)] use super::{ - CollectionFailure, classify_stores, execute_registered_collection, plan_collection, - quarantine::store_finding_is_profile_contained, + CollectionFailure, classify_stores, collection::store_finding_is_profile_contained, + execute_registered_collection, plan_collection, }; pub(super) struct StoreWalkStats { @@ -623,11 +623,10 @@ pub(crate) fn manifest_names_abandoned_root(data_root: &Path, profile_root: &Pat ) } -/// Deletes unregistered directories through the same two-phase boundary: -/// content/durable inspection and quarantine first, then a short final -/// still-unregistered confirmation before the irreversible phase. +/// Deletes unregistered directories after content/durable inspection and a +/// short final still-unregistered confirmation. /// -/// Compatibility convenience for one bounded read/apply page. The daemon uses +/// Convenience for one bounded read/apply page. The daemon uses /// [`sweep_unregistered_store_page`] directly so it can persist the returned /// cursor across maintenance cadences; Doctor deliberately receives one /// bounded preview rather than a hidden full-profile traversal. diff --git a/crates/tracedecay-maintenance/src/retention/orphan_stores/quarantine.rs b/crates/tracedecay-maintenance/src/retention/orphan_stores/quarantine.rs deleted file mode 100644 index eb8ec05c0c..0000000000 --- a/crates/tracedecay-maintenance/src/retention/orphan_stores/quarantine.rs +++ /dev/null @@ -1,3948 +0,0 @@ -//! Durable two-phase quarantine for destructive orphan-store retention. - -use std::collections::HashSet; -use std::ffi::{OsStr, OsString}; -use std::io::{Read, Write}; -use std::path::{Path, PathBuf}; -use std::sync::atomic::{AtomicU64, Ordering}; - -use cap_fs_ext::{DirExt, FollowSymlinks, OpenOptionsFollowExt}; -use cap_std::fs::{Dir, OpenOptions}; -use serde::{Deserialize, Serialize}; -use std::future::Future; -use std::time::Instant; -use tracedecay_private_fs::capability_dir::{ - remove_open_dir_all_nofollow, rename_noreplace, sync_directory, -}; - -use tracedecay_global_db::{RegisteredGlobalDb, RegisteredGlobalDbWriteTransaction}; -use tracedecay_runtime_core::cancellation::{CancellationToken, MonotonicDeadline}; - -use super::fence::{ - StoreContentFence, StoreDirectoryFence, capture_store_content_fence_in_dir_controlled, - capture_store_directory_fence, data_root_fence_matches, open_store_directory_nofollow, - open_store_parent_nofollow, profile_relative_store_path, store_root_identity, -}; -use super::pages::newest_mtime_secs_controlled; -use super::{ - CollectedStore, CollectionCompletionV1, CollectionFailure, CollectionFailureKind, - CollectionMutationFailure, CollectionMutationOperation, CollectionOutcome, CollectionPlan, - CollectionRecoveryAction, CollectionRecoveryReceipt, OrphanStoreFinding, StoreRootIdentity, - UnregisteredCollectionPlan, UnregisteredStoreFinding, -}; - -static QUARANTINE_SEQUENCE: AtomicU64 = AtomicU64::new(1); -const QUARANTINE_ATTEMPTS: usize = 32; -const MAX_RECOVERY_JOURNAL_BYTES: u64 = 64 * 1024; -const MAX_REGISTERED_QUARANTINE_INTENTS: usize = 16_384; -const JOURNAL_SUFFIX: &str = ".receipt-v1.json"; -const RENAMED_SUFFIX: &str = ".renamed"; -const RETIRED_SUFFIX: &str = ".retired"; - -/// The database decision that must be durable before the quarantined bytes may -/// be removed. `Unregistered` has no row to delete, but it still records the -/// final absence confirmation before its irreversible phase. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -pub(crate) enum QuarantineKindV1 { - Registered, - Unregistered, -} -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub(super) struct QuarantineRegistryFenceV1 { - pub(super) store_relpath: String, - pub(super) created_at: i64, - pub(super) last_write_at: Option, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -struct QuarantineJournalV1 { - version: u8, - kind: QuarantineKindV1, - project_id: String, - store_id: String, - original_name: String, - expected_root_identity: StoreRootIdentity, - registry_fence: Option, -} - -/// One validated registered retirement intent discovered independently of the -/// current registry census. The registry fence is the only authority the -/// caller may use to classify the interrupted database commit. -#[derive(Debug, Clone, PartialEq, Eq)] -pub(super) struct RegisteredQuarantineIntentV1 { - pub(super) project_id: String, - pub(super) store_id: String, - pub(super) quarantine_name: String, - pub(super) quarantine_path: PathBuf, - pub(super) original_path: PathBuf, - pub(super) registry_fence: QuarantineRegistryFenceV1, - pub(super) expected_root_identity: StoreRootIdentity, -} - -pub(super) enum RegisteredQuarantineInventoryV1 { - Complete(Vec), - Interrupted, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(super) enum RegisteredQuarantineDecisionV1 { - Restore, - Remove, - Retain, -} - -/// Test projection of a readable on-disk recovery record. -#[derive(Debug, Clone, PartialEq, Eq)] -#[cfg(test)] -pub(crate) struct PendingQuarantineReceiptV1 { - pub(crate) quarantine_path: PathBuf, - /// The live filesystem location observed when the receipt was read. A - /// rename can succeed before its parent-directory sync fails, leaving the - /// bytes at `original_path` while the journal remains pending. - pub(crate) actual_path: PathBuf, - pub(crate) retirement_committed: bool, -} - -/// The result of moving one exact store leaf out of its live name and proving -/// that the moved bytes still equal the census fence. -pub(super) enum QuarantineStoreOutcome { - Missing, - Verified(QuarantinedStore), - Interrupted { - quarantine_path: PathBuf, - failure: Option, - }, - Restored { - restored_path: PathBuf, - failure: Option, - }, - Retained { - quarantine_path: PathBuf, - failure: CollectionMutationFailure, - }, -} - -/// A durable quarantine found on a later maintenance admission. -#[derive(Debug, Clone, PartialEq, Eq)] -pub(super) enum QuarantineRecoveryOutcome { - Removed { - quarantine_path: PathBuf, - journal_failure: Option, - }, - Restored { - restored_path: PathBuf, - failure: Option, - }, - Retained { - quarantine_path: PathBuf, - actual_path: PathBuf, - failure: Option, - }, -} - -pub(super) enum QuarantineFinalizeOutcome { - Removed { - journal_failure: Option, - }, - Interrupted { - quarantine_path: PathBuf, - }, - DeleteUnconfirmed { - quarantine_path: PathBuf, - failure: CollectionMutationFailure, - }, -} - -/// A verified moved directory plus its immutable, sibling journal. The -/// journal is written and synced before this value is returned; after that, -/// no crash can make the quarantine invisible to the production reader. -pub(super) struct QuarantinedStore { - parent: Dir, - root: Dir, - quarantine_path: PathBuf, - journal_name: String, - expected_root_identity: Option, -} - -impl QuarantinedStore { - pub(super) fn quarantine_path(&self) -> &Path { - &self.quarantine_path - } - - /// Publish the database-commit phase before removal. This marker is - /// additive/no-replace, so a crash cannot turn a committed retirement back - /// into an apparently prepared one by tearing an overwrite. - pub(super) fn mark_retirement_committed(&self) -> Result<(), CollectionMutationFailure> { - let marker_name = retired_marker_name(&self.journal_name); - write_empty_marker( - &self.parent, - self.quarantine_path - .parent() - .map_or_else(PathBuf::new, Path::to_path_buf) - .as_path(), - &marker_name, - CollectionMutationOperation::MarkRetirementCommitted, - self.expected_root_identity.clone(), - ) - } - - /// The irreversible phase runs only after the caller's registry commit. - /// If recursive removal or its parent sync fails, the journal is retained - /// and reports `DeleteUnconfirmed`; a later reconciliation retries from - /// the exact same capability boundary rather than claiming reclaimed data. - pub(super) fn finalize(self, control: CollectionControl<'_>) -> QuarantineFinalizeOutcome { - let Self { - parent, - root, - quarantine_path, - journal_name, - expected_root_identity, - .. - } = self; - if control.completion().is_some() { - return QuarantineFinalizeOutcome::Interrupted { quarantine_path }; - } - // The descent is capability-relative and no-follow; the interrupt - // check runs before every child operation so a cancelled admission - // leaves the journal and remaining bytes for the mounted reconciler. - let interrupted = &mut || { - if control.completion().is_some() { - Err(interrupted_remove_error()) - } else { - Ok(()) - } - }; - match remove_open_dir_all_nofollow(root, interrupted) { - Ok(()) => {} - Err(error) if error.kind() == std::io::ErrorKind::Interrupted => { - return QuarantineFinalizeOutcome::Interrupted { quarantine_path }; - } - Err(error) => { - let failure = CollectionMutationFailure::from_io_error( - CollectionMutationOperation::RecursiveRemove, - quarantine_path.clone(), - expected_root_identity, - &error, - ); - return QuarantineFinalizeOutcome::DeleteUnconfirmed { - quarantine_path, - failure, - }; - } - } - // Once the final child disappears, synchronizing the parent is part - // of the same irreversible operation. It must complete even if the - // admission is cancelled concurrently; otherwise a completed delete - // could be reported without its durability boundary. - if let Err(error) = sync_directory(&parent) { - let failure = CollectionMutationFailure::from_io_error( - CollectionMutationOperation::ParentSync, - quarantine_path - .parent() - .map_or_else(PathBuf::new, Path::to_path_buf), - expected_root_identity, - &error, - ); - return QuarantineFinalizeOutcome::DeleteUnconfirmed { - quarantine_path, - failure, - }; - } - let journal_failure = clear_committed_journal( - &parent, - quarantine_path - .parent() - .map_or_else(PathBuf::new, Path::to_path_buf) - .as_path(), - &journal_name, - expected_root_identity, - ) - .err(); - QuarantineFinalizeOutcome::Removed { journal_failure } - } -} - -fn interrupted_remove_error() -> std::io::Error { - std::io::Error::new( - std::io::ErrorKind::Interrupted, - "retention quarantine finalization interrupted", - ) -} - -/// Atomically moves `data_root` to a unique sibling, persists a prepared -/// journal, then proves its exact content inventory. The caller performs the -/// short registry transaction only after this potentially expensive hashing. -#[hotpath::measure(label = "maintenance.orphan_stores.quarantine")] -pub(super) fn quarantine_store_for_verified_collection_controlled( - profile_root: &Path, - data_root: &Path, - expected: &StoreContentFence, - kind: QuarantineKindV1, - project_id: &str, - store_id: &str, - registry_fence: Option, - control: CollectionControl<'_>, -) -> Result { - if control.completion().is_some() { - return Ok(QuarantineStoreOutcome::Interrupted { - quarantine_path: data_root.to_path_buf(), - failure: None, - }); - } - if *expected == StoreContentFence::Unverifiable { - return Err(CollectionFailureKind::InspectFailed); - } - if *expected == StoreContentFence::Missing { - return Ok(QuarantineStoreOutcome::Missing); - } - let expected_root_identity = match expected { - StoreContentFence::Present(inventory) => Some(inventory.root.clone()), - StoreContentFence::Missing | StoreContentFence::Unverifiable => None, - }; - let capability = open_store_directory_nofollow(profile_root, data_root)?; - // The leaf handle only proved the store is present and readable. The - // content proof re-opens the leaf under its quarantine name, so release - // the probe before the rename: cap-std opens directories without - // `FILE_SHARE_DELETE`, and Windows refuses to rename a directory while - // such a handle is live. - drop(capability.root); - let original_name = capability - .leaf_name - .to_str() - .ok_or(CollectionFailureKind::InspectFailed)? - .to_owned(); - let quarantine_name = reserve_quarantine_name( - &capability.parent, - data_root, - &capability.leaf_name, - expected_root_identity.clone(), - ) - .map_err(CollectionFailureKind::RemoveFailed)?; - let quarantine_path = data_root - .parent() - .ok_or(CollectionFailureKind::OutsideProfile)? - .join(&quarantine_name); - let journal_name = journal_name(&quarantine_name); - let journal = QuarantineJournalV1 { - version: 1, - kind, - project_id: project_id.to_owned(), - store_id: store_id.to_owned(), - original_name, - expected_root_identity: match expected { - StoreContentFence::Present(inventory) => inventory.root.clone(), - StoreContentFence::Missing | StoreContentFence::Unverifiable => { - return Err(CollectionFailureKind::InspectFailed); - } - }, - registry_fence, - }; - // The journal is the intent record for the following destructive rename. - // Publishing it first eliminates the old crash window where a synced - // quarantine existed with no discoverable recovery authority. - write_journal( - &capability.parent, - quarantine_path - .parent() - .ok_or(CollectionFailureKind::OutsideProfile)?, - &journal_name, - &journal, - expected_root_identity.clone(), - ) - .map_err(CollectionFailureKind::RemoveFailed)?; - - if let Err(error) = rename_noreplace( - &capability.parent, - &capability.leaf_name, - &capability.parent, - OsStr::new(&quarantine_name), - ) { - let failure = CollectionMutationFailure::from_io_error( - CollectionMutationOperation::RenameLiveLeafToQuarantine, - data_root.to_path_buf(), - expected_root_identity.clone(), - &error, - ); - let _ = clear_journal( - &capability.parent, - quarantine_path - .parent() - .ok_or(CollectionFailureKind::OutsideProfile)?, - &journal_name, - expected_root_identity, - ); - // The live-leaf rename is the primary failure. Best-effort journal - // cleanup is secondary and must never replace its operation or code. - return Err(CollectionFailureKind::RemoveFailed(failure)); - } - if let Err(error) = sync_directory(&capability.parent) { - let parent_path = quarantine_path - .parent() - .map_or_else(PathBuf::new, Path::to_path_buf); - let failure = CollectionMutationFailure::from_io_error( - CollectionMutationOperation::ParentSync, - parent_path, - expected_root_identity.clone(), - &error, - ); - return Ok(recover_original_name( - capability.parent, - capability.leaf_name, - quarantine_name, - quarantine_path, - Some(journal_name), - expected_root_identity, - Some(failure), - )); - } - let renamed_marker = renamed_marker_name(&journal_name); - if let Err(failure) = write_empty_marker( - &capability.parent, - quarantine_path - .parent() - .ok_or(CollectionFailureKind::OutsideProfile)?, - &renamed_marker, - CollectionMutationOperation::PublishQuarantineRenameMarker, - expected_root_identity.clone(), - ) { - return Ok(QuarantineStoreOutcome::Interrupted { - quarantine_path, - failure: Some(failure), - }); - } - let moved_root = match capability.parent.open_dir_nofollow(&quarantine_name) { - Ok(root) => root, - Err(_) => { - return Ok(recover_original_name( - capability.parent, - capability.leaf_name, - quarantine_name, - quarantine_path, - Some(journal_name), - expected_root_identity, - None, - )); - } - }; - let verified = capture_store_content_fence_in_dir_controlled(&moved_root, Some(control)) - .map(StoreContentFence::Present); - match verified { - Ok(actual) if actual == *expected => { - Ok(QuarantineStoreOutcome::Verified(QuarantinedStore { - parent: capability.parent, - root: moved_root, - quarantine_path, - journal_name, - expected_root_identity, - })) - } - Err(error) if error.kind() == std::io::ErrorKind::Interrupted => { - drop(moved_root); - Ok(QuarantineStoreOutcome::Interrupted { - quarantine_path, - failure: None, - }) - } - Ok(_) | Err(_) => { - drop(moved_root); - Ok(recover_original_name( - capability.parent, - capability.leaf_name, - quarantine_name, - quarantine_path, - Some(journal_name), - expected_root_identity, - None, - )) - } - } -} - -#[cfg(test)] -pub(super) fn quarantine_store_for_verified_collection( - profile_root: &Path, - data_root: &Path, - expected: &StoreContentFence, -) -> Result { - let cancellation = CancellationToken::new(); - quarantine_store_for_verified_collection_controlled( - profile_root, - data_root, - expected, - QuarantineKindV1::Unregistered, - "test-project", - "test-store", - None, - CollectionControl::new( - &cancellation, - MonotonicDeadline::at(std::time::Instant::now() + std::time::Duration::from_hours(24)), - ), - ) -} - -/// Renames the quarantined leaf back to its original name. Every handle on -/// the leaf must already be closed (see the probe release before the -/// forward rename); the callers drop `moved_root` before arriving here. -fn recover_original_name( - parent: Dir, - original_name: OsString, - quarantine_name: String, - quarantine_path: PathBuf, - journal_name: Option, - expected_root_identity: Option, - primary_failure: Option, -) -> QuarantineStoreOutcome { - match rename_noreplace( - &parent, - OsStr::new(&quarantine_name), - &parent, - &original_name, - ) { - Ok(()) => { - // A directory sync failure occurs after the atomic rename. Preserve - // any journal and return the true, restored path for that state. - let parent_path = quarantine_path - .parent() - .map_or_else(PathBuf::new, Path::to_path_buf); - let secondary_failure = match sync_directory(&parent) { - Ok(()) => journal_name.and_then(|name| { - clear_journal(&parent, &parent_path, &name, expected_root_identity.clone()) - .err() - }), - Err(error) => Some(CollectionMutationFailure::from_io_error( - CollectionMutationOperation::ParentSync, - parent_path, - expected_root_identity, - &error, - )), - }; - // One flat failure preserves the initiating error; restore sync - // and cleanup errors fill the slot only when no primary exists. - let failure = primary_failure.or(secondary_failure); - let restored_path = quarantine_path - .parent() - .map_or_else(PathBuf::new, |parent| parent.join(&original_name)); - QuarantineStoreOutcome::Restored { - restored_path, - failure, - } - } - Err(error) => { - let restore_failure = CollectionMutationFailure::from_io_error( - CollectionMutationOperation::RestoreLiveLeafFromQuarantine, - quarantine_path - .parent() - .map_or_else(PathBuf::new, |parent| parent.join(&original_name)), - expected_root_identity, - &error, - ); - QuarantineStoreOutcome::Retained { - quarantine_path, - failure: primary_failure.unwrap_or(restore_failure), - } - } - } -} - -fn reserve_quarantine_name( - parent: &Dir, - data_root: &Path, - original: &OsStr, - expected_root_identity: Option, -) -> Result { - let Some(original) = original.to_str() else { - return Err(CollectionMutationFailure::without_native_error( - CollectionMutationOperation::ReserveQuarantineName, - data_root.to_path_buf(), - expected_root_identity, - )); - }; - reserve_quarantine_name_with_sequence( - parent, - data_root, - original, - expected_root_identity, - || QUARANTINE_SEQUENCE.fetch_add(1, Ordering::Relaxed), - ) -} - -pub(super) fn reserve_quarantine_name_with_sequence( - parent: &Dir, - data_root: &Path, - original: &str, - expected_root_identity: Option, - mut next_sequence: impl FnMut() -> u64, -) -> Result { - for _ in 0..QUARANTINE_ATTEMPTS { - let sequence = next_sequence(); - let candidate = format!( - ".tracedecay-orphan-quarantine-{original}-{}-{sequence}", - std::process::id() - ); - match quarantine_candidate_namespace_available(parent, &candidate) { - Ok(true) => return Ok(candidate), - Ok(false) => {} - Err(error) => { - return Err(CollectionMutationFailure::from_io_error( - CollectionMutationOperation::ReserveQuarantineName, - data_root - .parent() - .map_or_else(PathBuf::new, |parent| parent.join(candidate)), - expected_root_identity, - &error, - )); - } - } - } - Err(CollectionMutationFailure::without_native_error( - CollectionMutationOperation::ReserveQuarantineName, - data_root.to_path_buf(), - expected_root_identity, - )) -} - -pub(super) fn quarantine_candidate_namespace_available( - parent: &Dir, - candidate: &str, -) -> std::io::Result { - // A journal or marker carries authority over the candidate name even when - // its directory is gone. Reusing any part of that namespace could let a - // new quarantine inherit stale rename or retirement authority. - let journal = journal_name(candidate); - let renamed_marker = renamed_marker_name(&journal); - let retired_marker = retired_marker_name(&journal); - for name in [ - candidate, - journal.as_str(), - renamed_marker.as_str(), - retired_marker.as_str(), - ] { - match parent.symlink_metadata(name) { - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} - Ok(_) => return Ok(false), - Err(error) => return Err(error), - } - } - Ok(true) -} - -fn journal_name(quarantine_name: &str) -> String { - format!("{quarantine_name}{JOURNAL_SUFFIX}") -} - -fn retired_marker_name(journal_name: &str) -> String { - format!("{journal_name}{RETIRED_SUFFIX}") -} - -fn renamed_marker_name(journal_name: &str) -> String { - format!("{journal_name}{RENAMED_SUFFIX}") -} - -fn write_journal( - parent: &Dir, - parent_path: &Path, - name: &str, - journal: &QuarantineJournalV1, - expected_root_identity: Option, -) -> Result<(), CollectionMutationFailure> { - let target_path = parent_path.join(name); - let publish_failure = |error: &std::io::Error| { - CollectionMutationFailure::from_io_error( - CollectionMutationOperation::PublishQuarantineJournal, - target_path.clone(), - expected_root_identity.clone(), - error, - ) - }; - let bytes = serde_json::to_vec(journal).map_err(|error| { - publish_failure(&std::io::Error::other(format!( - "serialize retention journal: {error}" - ))) - })?; - let temporary = format!( - ".{name}.tmp-{}-{}", - std::process::id(), - QUARANTINE_SEQUENCE.fetch_add(1, Ordering::Relaxed) - ); - let mut options = OpenOptions::new(); - options.write(true).create_new(true); - let mut file = parent - .open_with(&temporary, &options) - .map_err(|error| publish_failure(&error))?; - if let Err(error) = file.write_all(&bytes).and_then(|()| file.sync_all()) { - // Preserve the publish error; temporary cleanup is best-effort only. - let _ = parent.remove_file(&temporary); - return Err(publish_failure(&error)); - } - drop(file); - if let Err(error) = rename_noreplace(parent, OsStr::new(&temporary), parent, OsStr::new(name)) { - // Preserve the publish error; temporary cleanup is best-effort only. - let _ = parent.remove_file(&temporary); - return Err(publish_failure(&error)); - } - sync_directory(parent).map_err(|error| { - CollectionMutationFailure::from_io_error( - CollectionMutationOperation::ParentSync, - parent_path.to_path_buf(), - expected_root_identity, - &error, - ) - }) -} - -fn write_empty_marker( - parent: &Dir, - parent_path: &Path, - name: &str, - operation: CollectionMutationOperation, - expected_root_identity: Option, -) -> Result<(), CollectionMutationFailure> { - let target_path = parent_path.join(name); - let marker_failure = |error: &std::io::Error| { - CollectionMutationFailure::from_io_error( - operation, - target_path.clone(), - expected_root_identity.clone(), - error, - ) - }; - let mut options = OpenOptions::new(); - options.write(true).create_new(true); - match parent.open_with(name, &options) { - Ok(file) => { - file.sync_all().map_err(|error| marker_failure(&error))?; - sync_directory(parent).map_err(|error| { - CollectionMutationFailure::from_io_error( - CollectionMutationOperation::ParentSync, - parent_path.to_path_buf(), - expected_root_identity, - &error, - ) - }) - } - Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => Ok(()), - Err(error) => Err(marker_failure(&error)), - } -} - -fn clear_journal( - parent: &Dir, - parent_path: &Path, - journal_name: &str, - expected_root_identity: Option, -) -> Result<(), CollectionMutationFailure> { - clear_journal_in_order( - parent, - parent_path, - journal_name, - expected_root_identity, - JournalCleanupState::Recoverable, - ) -} - -fn clear_committed_journal( - parent: &Dir, - parent_path: &Path, - journal_name: &str, - expected_root_identity: Option, -) -> Result<(), CollectionMutationFailure> { - clear_journal_in_order( - parent, - parent_path, - journal_name, - expected_root_identity, - JournalCleanupState::DeletionConfirmed, - ) -} - -#[derive(Clone, Copy)] -enum JournalCleanupState { - Recoverable, - DeletionConfirmed, -} - -fn journal_cleanup_names(journal_name: &str, state: JournalCleanupState) -> [String; 3] { - let renamed = renamed_marker_name(journal_name); - let retired = retired_marker_name(journal_name); - match state { - // Restore and pre-delete cleanup must keep the journal as the final - // recovery authority if either marker cleanup is interrupted. - JournalCleanupState::Recoverable => [renamed, retired, journal_name.to_owned()], - // Once exact deletion is confirmed, the retired marker must remain - // authoritative until the journal is removed. It becomes ignorable - // orphan debris as soon as journal-driven inventory cannot see it. - JournalCleanupState::DeletionConfirmed => [renamed, journal_name.to_owned(), retired], - } -} - -fn clear_journal_in_order( - parent: &Dir, - parent_path: &Path, - journal_name: &str, - expected_root_identity: Option, - state: JournalCleanupState, -) -> Result<(), CollectionMutationFailure> { - for name in journal_cleanup_names(journal_name, state) { - match parent.remove_file(&name) { - Ok(()) => {} - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} - Err(error) => { - return Err(CollectionMutationFailure::from_io_error( - CollectionMutationOperation::ClearRecoveryJournal, - parent_path.join(&name), - expected_root_identity, - &error, - )); - } - } - } - sync_directory(parent).map_err(|error| { - CollectionMutationFailure::from_io_error( - CollectionMutationOperation::ParentSync, - parent_path.to_path_buf(), - expected_root_identity, - &error, - ) - }) -} - -/// Legacy pre-journal quarantines are restored. Unregistered journal recovery -/// uses its durable retirement marker; registered journal recovery remains -/// pending until the caller supplies a decision from the exact global row. -/// Neither path proceeds until the opened quarantine matches the journal's -/// root identity. -pub(super) fn recover_existing_store_quarantine( - profile_root: &Path, - data_root: &Path, - control: CollectionControl<'_>, -) -> Result, CollectionFailureKind> { - let capability = open_store_parent_nofollow(profile_root, data_root)?; - let original = capability - .leaf_name - .to_str() - .ok_or(CollectionFailureKind::InspectFailed)?; - let parent_path = data_root - .parent() - .ok_or(CollectionFailureKind::OutsideProfile)?; - let mut outcomes = Vec::new(); - let mut recovered_names = HashSet::new(); - for entry in capability - .parent - .read_dir(".") - .map_err(|_| CollectionFailureKind::InspectFailed)? - { - if control.completion().is_some() { - break; - } - let entry = entry.map_err(|_| CollectionFailureKind::InspectFailed)?; - let file_name = entry.file_name(); - let Some(name) = file_name.to_str() else { - continue; - }; - let quarantine_name = name.strip_suffix(JOURNAL_SUFFIX).unwrap_or(name); - if quarantine_original_name(quarantine_name) != Some(original) - || !recovered_names.insert(quarantine_name.to_owned()) - { - continue; - } - if let Some(outcome) = recover_named_store_quarantine( - profile_root, - data_root, - OsStr::new(quarantine_name), - parent_path, - control, - )? { - outcomes.push(outcome); - } - } - Ok(outcomes) -} - -/// Returns the original project id encoded in an orphan-store quarantine name. -pub(super) fn quarantined_project_id(name: &str) -> Option { - let project_id = quarantine_original_name(name)?; - tracedecay_runtime_core::storage::validate_project_id(project_id).ok()?; - Some(project_id.to_owned()) -} - -pub(super) fn quarantine_recovery_entry(name: &str) -> Option<(String, String)> { - let quarantine_name = name.strip_suffix(JOURNAL_SUFFIX).unwrap_or(name); - quarantined_project_id(quarantine_name) - .map(|project_id| (project_id, quarantine_name.to_owned())) -} - -/// Inventories registered journal intents directly under `stores/`. Every -/// journal is opened no-follow, size-bounded, and fully validated before its -/// fields are exposed. Unregistered journals belong to the existing projects -/// pager and are deliberately not returned here. -pub(super) fn read_registered_quarantine_intents_controlled( - profile_root: &Path, - control: CollectionControl<'_>, -) -> Result { - let stores_path = profile_root.join("stores"); - let stores = match open_store_directory_nofollow(profile_root, &stores_path) { - Ok(capability) => capability.root, - Err(CollectionFailureKind::PayloadChanged) => { - return Ok(RegisteredQuarantineInventoryV1::Complete(Vec::new())); - } - Err(kind) => return Err(kind), - }; - let listing = stores - .open_dir(Path::new(".")) - .map_err(|_| CollectionFailureKind::InspectFailed)?; - let entries = listing - .entries() - .map_err(|_| CollectionFailureKind::InspectFailed)?; - let mut intents = Vec::new(); - for entry in entries { - if control.completion().is_some() { - return Ok(RegisteredQuarantineInventoryV1::Interrupted); - } - let entry = entry.map_err(|_| CollectionFailureKind::InspectFailed)?; - let Ok(name) = entry.file_name().into_string() else { - continue; - }; - let Some(quarantine_name) = name.strip_suffix(JOURNAL_SUFFIX) else { - continue; - }; - let Some(original_name) = quarantine_original_name(quarantine_name) else { - return Err(CollectionFailureKind::InspectFailed); - }; - let journal = read_recovery_journal( - &stores, - &stores_path, - &name, - quarantine_name, - OsStr::new(original_name), - ) - .map_err(CollectionFailureKind::RemoveFailed)? - .ok_or(CollectionFailureKind::InspectFailed)?; - if journal.kind == QuarantineKindV1::Unregistered { - continue; - } - let Some(registry_fence) = journal.registry_fence else { - return Err(CollectionFailureKind::RemoveFailed( - CollectionMutationFailure::without_native_error( - CollectionMutationOperation::ProbeRecoveryJournal, - stores_path.join(name), - Some(journal.expected_root_identity), - ), - )); - }; - let original_path = stores_path.join(&journal.original_name); - let expected_relpath = profile_relative_store_path(profile_root, &original_path)?; - if tracedecay_runtime_core::storage::validate_project_id(&journal.project_id).is_err() - || Path::new(®istry_fence.store_relpath) != expected_relpath - { - return Err(CollectionFailureKind::RemoveFailed( - CollectionMutationFailure::without_native_error( - CollectionMutationOperation::ProbeRecoveryJournal, - stores_path.join(name), - Some(journal.expected_root_identity), - ), - )); - } - if intents.len() == MAX_REGISTERED_QUARANTINE_INTENTS { - return Err(CollectionFailureKind::RemoveFailed( - CollectionMutationFailure::without_native_error( - CollectionMutationOperation::ProbeRecoveryJournal, - stores_path, - None, - ), - )); - } - intents.push(RegisteredQuarantineIntentV1 { - project_id: journal.project_id, - store_id: journal.store_id, - quarantine_name: quarantine_name.to_owned(), - quarantine_path: stores_path.join(quarantine_name), - original_path, - registry_fence, - expected_root_identity: journal.expected_root_identity, - }); - } - if control.completion().is_some() { - Ok(RegisteredQuarantineInventoryV1::Interrupted) - } else { - Ok(RegisteredQuarantineInventoryV1::Complete(intents)) - } -} - -fn quarantine_original_name(name: &str) -> Option<&str> { - let rest = name.strip_prefix(".tracedecay-orphan-quarantine-")?; - let (rest, sequence) = rest.rsplit_once('-')?; - sequence.parse::().ok()?; - let (original, process_id) = rest.rsplit_once('-')?; - process_id.parse::().ok()?; - (!original.is_empty()).then_some(original) -} - -pub(super) fn recover_named_store_quarantine( - profile_root: &Path, - data_root: &Path, - quarantine_name: &OsStr, - parent_path: &Path, - control: CollectionControl<'_>, -) -> Result, CollectionFailureKind> { - recover_named_store_quarantine_inner( - profile_root, - data_root, - quarantine_name, - parent_path, - None, - control, - || {}, - ) -} - -pub(super) fn recover_registered_quarantine_intent_controlled( - profile_root: &Path, - intent: &RegisteredQuarantineIntentV1, - decision: RegisteredQuarantineDecisionV1, - control: CollectionControl<'_>, -) -> Result, CollectionFailureKind> { - let parent_path = intent - .original_path - .parent() - .ok_or(CollectionFailureKind::OutsideProfile)?; - recover_named_store_quarantine_inner( - profile_root, - &intent.original_path, - OsStr::new(&intent.quarantine_name), - parent_path, - Some((intent, decision)), - control, - || {}, - ) -} - -#[cfg(test)] -pub(super) fn recover_named_store_quarantine_controlled( - profile_root: &Path, - data_root: &Path, - quarantine_name: &OsStr, - parent_path: &Path, - after_rename: impl FnOnce(), -) -> Result, CollectionFailureKind> { - recover_named_store_quarantine_inner( - profile_root, - data_root, - quarantine_name, - parent_path, - None, - super::unbounded_collection_control(), - after_rename, - ) -} - -fn recover_named_store_quarantine_inner( - profile_root: &Path, - data_root: &Path, - quarantine_name: &OsStr, - parent_path: &Path, - registered: Option<( - &RegisteredQuarantineIntentV1, - RegisteredQuarantineDecisionV1, - )>, - control: CollectionControl<'_>, - after_rename: impl FnOnce(), -) -> Result, CollectionFailureKind> { - let capability = open_store_parent_nofollow(profile_root, data_root)?; - let quarantine_path = parent_path.join(quarantine_name); - let quarantine_name_str = quarantine_name - .to_str() - .ok_or(CollectionFailureKind::InspectFailed)?; - let journal_name = journal_name(quarantine_name_str); - let journal = match read_recovery_journal( - &capability.parent, - parent_path, - &journal_name, - quarantine_name_str, - &capability.leaf_name, - ) { - Ok(journal) => journal, - Err(failure) => { - let actual_path = receipt_actual_path(data_root, &quarantine_path); - return Ok(Some(QuarantineRecoveryOutcome::Retained { - actual_path, - quarantine_path, - failure: Some(failure), - })); - } - }; - let quarantine_root = match capability.parent.open_dir_nofollow(quarantine_name) { - Ok(root) => root, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => { - let Some(journal) = journal else { - return Ok(None); - }; - // The commit authority is independent of either filesystem name. - // Classify it before requiring a live-name identity: a completed - // recursive delete legitimately leaves both names absent. - let decision = match journal.kind { - QuarantineKindV1::Registered => match registered { - Some((intent, decision)) - if registered_intent_matches_journal(intent, &journal) => - { - decision - } - Some((intent, _)) => { - return Ok(Some(QuarantineRecoveryOutcome::Retained { - failure: Some(CollectionMutationFailure::without_native_error( - CollectionMutationOperation::ProbeRecoveryJournal, - parent_path.join(&journal_name), - Some(intent.expected_root_identity.clone()), - )), - actual_path: quarantine_path.clone(), - quarantine_path, - })); - } - None => RegisteredQuarantineDecisionV1::Retain, - }, - QuarantineKindV1::Unregistered => { - match probe_regular_recovery_marker( - &capability.parent, - parent_path, - &retired_marker_name(&journal_name), - &journal.expected_root_identity, - ) { - Ok(true) => RegisteredQuarantineDecisionV1::Remove, - Ok(false) => RegisteredQuarantineDecisionV1::Restore, - Err(failure) => { - return Ok(Some(QuarantineRecoveryOutcome::Retained { - failure: Some(failure), - actual_path: quarantine_path.clone(), - quarantine_path, - })); - } - } - } - }; - let original_identity = match child_store_identity( - &capability.parent, - &capability.leaf_name, - data_root, - &journal.expected_root_identity, - ) { - Ok(identity) => identity, - Err(failure) => { - return Ok(Some(QuarantineRecoveryOutcome::Retained { - failure: Some(failure), - actual_path: data_root.to_path_buf(), - quarantine_path, - })); - } - }; - let Some(original_identity) = original_identity else { - if decision == RegisteredQuarantineDecisionV1::Remove { - let journal_failure = clear_committed_journal( - &capability.parent, - parent_path, - &journal_name, - Some(journal.expected_root_identity), - ) - .err(); - return Ok(Some(QuarantineRecoveryOutcome::Removed { - quarantine_path, - journal_failure, - })); - } - return Ok(Some(QuarantineRecoveryOutcome::Retained { - failure: Some(CollectionMutationFailure::without_native_error( - CollectionMutationOperation::ValidateRestoredStoreIdentity, - data_root.to_path_buf(), - Some(journal.expected_root_identity), - )), - actual_path: quarantine_path.clone(), - quarantine_path, - })); - }; - if original_identity != journal.expected_root_identity { - return Ok(Some(QuarantineRecoveryOutcome::Retained { - failure: Some(CollectionMutationFailure::without_native_error( - CollectionMutationOperation::ValidateRestoredStoreIdentity, - data_root.to_path_buf(), - Some(journal.expected_root_identity), - )), - actual_path: data_root.to_path_buf(), - quarantine_path, - })); - } - if decision == RegisteredQuarantineDecisionV1::Restore { - let failure = clear_journal( - &capability.parent, - parent_path, - &journal_name, - Some(journal.expected_root_identity), - ) - .err(); - return Ok(Some(QuarantineRecoveryOutcome::Restored { - restored_path: data_root.to_path_buf(), - failure, - })); - } - return Ok(Some(QuarantineRecoveryOutcome::Retained { - failure: Some(CollectionMutationFailure::without_native_error( - CollectionMutationOperation::ValidateRestoredStoreIdentity, - data_root.to_path_buf(), - Some(journal.expected_root_identity), - )), - actual_path: data_root.to_path_buf(), - quarantine_path, - })); - } - Err(error) => { - let expected_root_identity = journal - .as_ref() - .map(|journal| journal.expected_root_identity.clone()); - return Ok(Some(QuarantineRecoveryOutcome::Retained { - failure: Some(CollectionMutationFailure::from_io_error( - CollectionMutationOperation::ValidateRestoredStoreIdentity, - quarantine_path.clone(), - expected_root_identity, - &error, - )), - actual_path: quarantine_path.clone(), - quarantine_path, - })); - } - }; - let expected_root_identity = match store_root_identity(&quarantine_root) { - Ok(identity) => identity, - Err(error) => { - drop(quarantine_root); - let expected_root_identity = journal - .as_ref() - .map(|journal| journal.expected_root_identity.clone()); - return Ok(Some(QuarantineRecoveryOutcome::Retained { - failure: Some(CollectionMutationFailure::from_io_error( - CollectionMutationOperation::ValidateRestoredStoreIdentity, - quarantine_path.clone(), - expected_root_identity, - &error, - )), - actual_path: quarantine_path.clone(), - quarantine_path, - })); - } - }; - let Some(journal_record) = journal else { - drop(quarantine_root); - return Ok(Some(restore_quarantine_name( - &capability.parent, - &capability.leaf_name, - quarantine_name, - data_root, - parent_path, - &quarantine_path, - &expected_root_identity, - None, - after_rename, - ))); - }; - if let Some((intent, _)) = registered - && !registered_intent_matches_journal(intent, &journal_record) - { - drop(quarantine_root); - return Ok(Some(QuarantineRecoveryOutcome::Retained { - failure: Some(CollectionMutationFailure::without_native_error( - CollectionMutationOperation::ProbeRecoveryJournal, - parent_path.join(&journal_name), - Some(intent.expected_root_identity.clone()), - )), - actual_path: quarantine_path.clone(), - quarantine_path, - })); - } - if expected_root_identity != journal_record.expected_root_identity { - drop(quarantine_root); - return Ok(Some(QuarantineRecoveryOutcome::Retained { - failure: Some(CollectionMutationFailure::without_native_error( - CollectionMutationOperation::ValidateRestoredStoreIdentity, - quarantine_path.clone(), - Some(journal_record.expected_root_identity), - )), - actual_path: quarantine_path.clone(), - quarantine_path, - })); - } - if journal_record.kind == QuarantineKindV1::Registered { - let Some((_, decision)) = registered else { - drop(quarantine_root); - return Ok(Some(QuarantineRecoveryOutcome::Retained { - actual_path: quarantine_path.clone(), - quarantine_path, - failure: None, - })); - }; - match decision { - RegisteredQuarantineDecisionV1::Restore => { - drop(quarantine_root); - return Ok(Some(restore_quarantine_name( - &capability.parent, - &capability.leaf_name, - quarantine_name, - data_root, - parent_path, - &quarantine_path, - &expected_root_identity, - Some(&journal_name), - after_rename, - ))); - } - RegisteredQuarantineDecisionV1::Retain => { - drop(quarantine_root); - return Ok(Some(QuarantineRecoveryOutcome::Retained { - actual_path: quarantine_path.clone(), - quarantine_path, - failure: None, - })); - } - RegisteredQuarantineDecisionV1::Remove => { - let quarantine = QuarantinedStore { - parent: capability.parent, - root: quarantine_root, - quarantine_path: quarantine_path.clone(), - journal_name, - expected_root_identity: Some(expected_root_identity), - }; - return Ok(Some(match quarantine.finalize(control) { - QuarantineFinalizeOutcome::Removed { journal_failure } => { - QuarantineRecoveryOutcome::Removed { - quarantine_path, - journal_failure, - } - } - QuarantineFinalizeOutcome::Interrupted { quarantine_path } => { - QuarantineRecoveryOutcome::Retained { - actual_path: quarantine_path.clone(), - quarantine_path, - failure: None, - } - } - QuarantineFinalizeOutcome::DeleteUnconfirmed { - quarantine_path, - failure, - } => QuarantineRecoveryOutcome::Retained { - actual_path: quarantine_path.clone(), - quarantine_path, - failure: Some(failure), - }, - })); - } - } - } - let retired_name = retired_marker_name(&journal_name); - let retirement_committed = match probe_regular_recovery_marker( - &capability.parent, - parent_path, - &retired_name, - &journal_record.expected_root_identity, - ) { - Ok(retirement_committed) => retirement_committed, - Err(failure) => { - drop(quarantine_root); - return Ok(Some(QuarantineRecoveryOutcome::Retained { - actual_path: quarantine_path.clone(), - quarantine_path, - failure: Some(failure), - })); - } - }; - if !retirement_committed { - drop(quarantine_root); - return Ok(Some(restore_quarantine_name( - &capability.parent, - &capability.leaf_name, - quarantine_name, - data_root, - parent_path, - &quarantine_path, - &expected_root_identity, - Some(&journal_name), - after_rename, - ))); - } - - let quarantine = QuarantinedStore { - parent: capability.parent, - root: quarantine_root, - quarantine_path: quarantine_path.clone(), - journal_name, - expected_root_identity: Some(expected_root_identity), - }; - Ok(Some(match quarantine.finalize(control) { - QuarantineFinalizeOutcome::Removed { journal_failure } => { - QuarantineRecoveryOutcome::Removed { - quarantine_path, - journal_failure, - } - } - QuarantineFinalizeOutcome::Interrupted { quarantine_path } => { - QuarantineRecoveryOutcome::Retained { - actual_path: quarantine_path.clone(), - quarantine_path, - failure: None, - } - } - QuarantineFinalizeOutcome::DeleteUnconfirmed { - quarantine_path, - failure, - } => QuarantineRecoveryOutcome::Retained { - actual_path: quarantine_path.clone(), - quarantine_path, - failure: Some(failure), - }, - })) -} - -#[allow(clippy::too_many_arguments)] -fn restore_quarantine_name( - parent: &Dir, - live_name: &OsStr, - quarantine_name: &OsStr, - data_root: &Path, - parent_path: &Path, - quarantine_path: &Path, - expected_root_identity: &StoreRootIdentity, - journal_name: Option<&str>, - after_rename: impl FnOnce(), -) -> QuarantineRecoveryOutcome { - match rename_noreplace(parent, quarantine_name, parent, live_name) { - Ok(()) => { - after_rename(); - let restored_root = match parent.open_dir_nofollow(live_name) { - Ok(root) => root, - Err(error) => { - let failure = CollectionMutationFailure::from_io_error( - CollectionMutationOperation::ValidateRestoredStoreIdentity, - data_root.to_path_buf(), - Some(expected_root_identity.clone()), - &error, - ); - return retain_failed_legacy_restore( - parent, - live_name, - quarantine_name, - data_root, - quarantine_path, - expected_root_identity, - failure, - ); - } - }; - let restored_identity = match store_root_identity(&restored_root) { - Ok(identity) => identity, - Err(error) => { - drop(restored_root); - let failure = CollectionMutationFailure::from_io_error( - CollectionMutationOperation::ValidateRestoredStoreIdentity, - data_root.to_path_buf(), - Some(expected_root_identity.clone()), - &error, - ); - return retain_failed_legacy_restore( - parent, - live_name, - quarantine_name, - data_root, - quarantine_path, - expected_root_identity, - failure, - ); - } - }; - if restored_identity != *expected_root_identity { - drop(restored_root); - let failure = CollectionMutationFailure::without_native_error( - CollectionMutationOperation::ValidateRestoredStoreIdentity, - data_root.to_path_buf(), - Some(expected_root_identity.clone()), - ); - return retain_failed_legacy_restore( - parent, - live_name, - quarantine_name, - data_root, - quarantine_path, - expected_root_identity, - failure, - ); - } - drop(restored_root); - let failure = match sync_directory(parent) { - Ok(()) => journal_name.and_then(|journal_name| { - clear_journal( - parent, - parent_path, - journal_name, - Some(expected_root_identity.clone()), - ) - .err() - }), - Err(error) => Some(CollectionMutationFailure::from_io_error( - CollectionMutationOperation::ParentSync, - parent_path.to_path_buf(), - Some(expected_root_identity.clone()), - &error, - )), - }; - QuarantineRecoveryOutcome::Restored { - restored_path: data_root.to_path_buf(), - failure, - } - } - Err(error) => QuarantineRecoveryOutcome::Retained { - failure: Some(CollectionMutationFailure::from_io_error( - CollectionMutationOperation::RestoreLiveLeafFromQuarantine, - data_root.to_path_buf(), - Some(expected_root_identity.clone()), - &error, - )), - actual_path: quarantine_path.to_path_buf(), - quarantine_path: quarantine_path.to_path_buf(), - }, - } -} - -fn retain_failed_legacy_restore( - parent: &Dir, - live_name: &OsStr, - quarantine_name: &OsStr, - data_root: &Path, - quarantine_path: &Path, - expected_root_identity: &StoreRootIdentity, - primary_failure: CollectionMutationFailure, -) -> QuarantineRecoveryOutcome { - match rename_noreplace(parent, live_name, parent, quarantine_name) { - Ok(()) => { - let failure = match sync_directory(parent) { - Err(error) if primary_failure.raw_os_error.is_none() => { - CollectionMutationFailure::from_io_error( - CollectionMutationOperation::ParentSync, - quarantine_path - .parent() - .map_or_else(PathBuf::new, Path::to_path_buf), - Some(expected_root_identity.clone()), - &error, - ) - } - Ok(()) | Err(_) => primary_failure, - }; - QuarantineRecoveryOutcome::Retained { - actual_path: quarantine_path.to_path_buf(), - quarantine_path: quarantine_path.to_path_buf(), - failure: Some(failure), - } - } - Err(error) => { - let reverse_failure = CollectionMutationFailure::from_io_error( - CollectionMutationOperation::RestoreLiveLeafFromQuarantine, - quarantine_path.to_path_buf(), - Some(expected_root_identity.clone()), - &error, - ); - let failure = if primary_failure.raw_os_error.is_some() { - primary_failure - } else { - reverse_failure - }; - if child_has_store_identity(parent, quarantine_name, expected_root_identity) { - QuarantineRecoveryOutcome::Retained { - actual_path: quarantine_path.to_path_buf(), - quarantine_path: quarantine_path.to_path_buf(), - failure: Some(failure), - } - } else if child_has_store_identity(parent, live_name, expected_root_identity) - || parent.open_dir_nofollow(live_name).is_ok() - { - QuarantineRecoveryOutcome::Restored { - restored_path: data_root.to_path_buf(), - failure: Some(failure), - } - } else { - QuarantineRecoveryOutcome::Retained { - actual_path: quarantine_path.to_path_buf(), - quarantine_path: quarantine_path.to_path_buf(), - failure: Some(failure), - } - } - } - } -} - -fn child_has_store_identity(parent: &Dir, name: &OsStr, expected: &StoreRootIdentity) -> bool { - let Ok(root) = parent.open_dir_nofollow(name) else { - return false; - }; - store_root_identity(&root).is_ok_and(|identity| identity == *expected) -} - -fn child_store_identity( - parent: &Dir, - name: &OsStr, - path: &Path, - expected: &StoreRootIdentity, -) -> Result, CollectionMutationFailure> { - let root = match parent.open_dir_nofollow(name) { - Ok(root) => root, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), - Err(error) => { - return Err(CollectionMutationFailure::from_io_error( - CollectionMutationOperation::ValidateRestoredStoreIdentity, - path.to_path_buf(), - Some(expected.clone()), - &error, - )); - } - }; - store_root_identity(&root).map(Some).map_err(|error| { - CollectionMutationFailure::from_io_error( - CollectionMutationOperation::ValidateRestoredStoreIdentity, - path.to_path_buf(), - Some(expected.clone()), - &error, - ) - }) -} - -fn registered_intent_matches_journal( - intent: &RegisteredQuarantineIntentV1, - journal: &QuarantineJournalV1, -) -> bool { - journal.kind == QuarantineKindV1::Registered - && journal.project_id == intent.project_id - && journal.store_id == intent.store_id - && journal.original_name - == intent - .original_path - .file_name() - .and_then(OsStr::to_str) - .unwrap_or_default() - && journal.registry_fence.as_ref() == Some(&intent.registry_fence) - && journal.expected_root_identity == intent.expected_root_identity -} - -fn read_recovery_journal( - parent: &Dir, - parent_path: &Path, - journal_name: &str, - quarantine_name: &str, - expected_original_name: &OsStr, -) -> Result, CollectionMutationFailure> { - let journal_path = parent_path.join(journal_name); - let mut options = OpenOptions::new(); - options.read(true).follow(FollowSymlinks::No); - let mut file = match parent.open_with(journal_name, &options) { - Ok(file) => file, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), - Err(error) => { - return Err(CollectionMutationFailure::from_io_error( - CollectionMutationOperation::ProbeRecoveryJournal, - journal_path, - None, - &error, - )); - } - }; - let metadata = file.metadata().map_err(|error| { - CollectionMutationFailure::from_io_error( - CollectionMutationOperation::ProbeRecoveryJournal, - journal_path.clone(), - None, - &error, - ) - })?; - if !metadata.is_file() || metadata.len() > MAX_RECOVERY_JOURNAL_BYTES { - return Err(CollectionMutationFailure::without_native_error( - CollectionMutationOperation::ProbeRecoveryJournal, - journal_path, - None, - )); - } - let mut bytes = Vec::with_capacity(metadata.len() as usize); - (&mut file) - .take(MAX_RECOVERY_JOURNAL_BYTES.saturating_add(1)) - .read_to_end(&mut bytes) - .map_err(|error| { - CollectionMutationFailure::from_io_error( - CollectionMutationOperation::ProbeRecoveryJournal, - journal_path.clone(), - None, - &error, - ) - })?; - if bytes.len() as u64 != metadata.len() || bytes.len() as u64 > MAX_RECOVERY_JOURNAL_BYTES { - return Err(CollectionMutationFailure::without_native_error( - CollectionMutationOperation::ProbeRecoveryJournal, - journal_path, - None, - )); - } - let journal = serde_json::from_slice::(&bytes).map_err(|_| { - CollectionMutationFailure::without_native_error( - CollectionMutationOperation::ProbeRecoveryJournal, - journal_path.clone(), - None, - ) - })?; - if journal.version != 1 - || quarantine_original_name(quarantine_name) != Some(journal.original_name.as_str()) - || expected_original_name != OsStr::new(&journal.original_name) - { - return Err(CollectionMutationFailure::without_native_error( - CollectionMutationOperation::ProbeRecoveryJournal, - journal_path, - Some(journal.expected_root_identity), - )); - } - Ok(Some(journal)) -} - -fn probe_regular_recovery_marker( - parent: &Dir, - parent_path: &Path, - marker_name: &str, - expected_root_identity: &StoreRootIdentity, -) -> Result { - let marker_path = parent_path.join(marker_name); - let mut options = OpenOptions::new(); - options.read(true).follow(FollowSymlinks::No); - let marker = match parent.open_with(marker_name, &options) { - Ok(marker) => marker, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(false), - Err(error) => { - return Err(CollectionMutationFailure::from_io_error( - CollectionMutationOperation::ProbeRecoveryJournal, - marker_path, - Some(expected_root_identity.clone()), - &error, - )); - } - }; - let metadata = marker.metadata().map_err(|error| { - CollectionMutationFailure::from_io_error( - CollectionMutationOperation::ProbeRecoveryJournal, - marker_path.clone(), - Some(expected_root_identity.clone()), - &error, - ) - })?; - if !metadata.is_file() || metadata.len() != 0 { - return Err(CollectionMutationFailure::without_native_error( - CollectionMutationOperation::ProbeRecoveryJournal, - marker_path, - Some(expected_root_identity.clone()), - )); - } - Ok(true) -} - -#[cfg(all(test, windows))] -pub(super) fn classify_recovery_journal_probe( - probe: std::io::Result, - journal_path: PathBuf, - expected_root_identity: &StoreRootIdentity, -) -> Result { - match probe { - Ok(_) => Ok(true), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), - Err(error) => Err(CollectionFailureKind::RemoveFailed( - CollectionMutationFailure::from_io_error( - CollectionMutationOperation::ProbeRecoveryJournal, - journal_path, - Some(expected_root_identity.clone()), - &error, - ), - )), - } -} - -/// Test helper for asserting that a crash boundary left a readable durable -/// journal. Production recovery is mounted at each store's next admission. -#[cfg(test)] -pub(crate) fn read_pending_quarantine_receipts( - profile_root: &Path, -) -> Result, CollectionFailureKind> { - read_pending_quarantine_receipts_controlled(profile_root, super::unbounded_collection_control()) -} - -#[cfg(test)] -pub(super) fn read_pending_quarantine_receipts_controlled( - profile_root: &Path, - control: CollectionControl<'_>, -) -> Result, CollectionFailureKind> { - let mut receipts = Vec::new(); - for parent in [profile_root.join("stores"), profile_root.join("projects")] { - if control.completion().is_some() { - return Err(CollectionFailureKind::Cancelled); - } - let parent_capability = match open_store_directory_nofollow(profile_root, &parent) { - Ok(capability) => capability, - Err(CollectionFailureKind::PayloadChanged) => continue, - Err(kind) => return Err(kind), - }; - let entries = match std::fs::read_dir(&parent) { - Ok(entries) => entries, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, - Err(_) => return Err(CollectionFailureKind::InspectFailed), - }; - if control.completion().is_some() { - return Err(CollectionFailureKind::Cancelled); - } - let mut entries = entries; - loop { - // `ReadDir` advances lazily. Check before calling `next` so an - // interrupted admission does not fetch another receipt entry. - if control.completion().is_some() { - return Err(CollectionFailureKind::Cancelled); - } - let Some(entry) = entries.next() else { - break; - }; - if control.completion().is_some() { - return Err(CollectionFailureKind::Cancelled); - } - let entry = entry.map_err(|_| CollectionFailureKind::InspectFailed)?; - let name = entry.file_name(); - let Some(name) = name.to_str() else { - continue; - }; - let Some(quarantine_name) = name.strip_suffix(JOURNAL_SUFFIX) else { - continue; - }; - if control.completion().is_some() { - return Err(CollectionFailureKind::Cancelled); - } - let Some(original_name) = quarantine_original_name(quarantine_name) else { - return Err(CollectionFailureKind::InspectFailed); - }; - let journal = read_recovery_journal( - &parent_capability.root, - &parent, - name, - quarantine_name, - OsStr::new(original_name), - ) - .map_err(CollectionFailureKind::RemoveFailed)? - .ok_or_else(|| { - CollectionFailureKind::RemoveFailed( - CollectionMutationFailure::without_native_error( - CollectionMutationOperation::ProbeRecoveryJournal, - entry.path(), - None, - ), - ) - })?; - if control.completion().is_some() { - return Err(CollectionFailureKind::Cancelled); - } - let original_path = parent.join(&journal.original_name); - let quarantine_path = parent.join(quarantine_name); - if control.completion().is_some() { - return Err(CollectionFailureKind::Cancelled); - } - let actual_path = receipt_actual_path(&original_path, &quarantine_path); - if control.completion().is_some() { - return Err(CollectionFailureKind::Cancelled); - } - receipts.push(PendingQuarantineReceiptV1 { - actual_path, - quarantine_path, - retirement_committed: probe_regular_recovery_marker( - &parent_capability.root, - &parent, - &retired_marker_name(name), - &journal.expected_root_identity, - ) - .map_err(CollectionFailureKind::RemoveFailed)?, - }); - } - } - Ok(receipts) -} - -/// Prefer the quarantined path while it is still a regular directory. Once a -/// restore rename has succeeded, even if its parent sync or journal cleanup -/// failed, expose the original path as the bytes' actual observed location. -fn receipt_actual_path(original_path: &Path, quarantine_path: &Path) -> PathBuf { - let quarantine_is_directory = std::fs::symlink_metadata(quarantine_path) - .is_ok_and(|metadata| metadata.is_dir() && !metadata.file_type().is_symlink()); - if quarantine_is_directory { - quarantine_path.to_path_buf() - } else { - let original_is_directory = std::fs::symlink_metadata(original_path) - .is_ok_and(|metadata| metadata.is_dir() && !metadata.file_type().is_symlink()); - if original_is_directory { - original_path.to_path_buf() - } else { - quarantine_path.to_path_buf() - } - } -} - -// High-level registered/unregistered collection orchestration. - -/// Cooperative budget carried through every expensive retention read and -/// apply boundary. The database writer is acquired only after content hashing -/// and durable-memory inspection have completed under this control. -#[derive(Clone, Copy)] -pub(crate) struct CollectionControl<'a> { - cancellation: &'a CancellationToken, - deadline: MonotonicDeadline, -} - -impl<'a> CollectionControl<'a> { - pub(crate) const fn new( - cancellation: &'a CancellationToken, - deadline: MonotonicDeadline, - ) -> Self { - Self { - cancellation, - deadline, - } - } - - pub(crate) fn completion(self) -> Option { - if self.cancellation.is_cancelled() { - Some(CollectionCompletionV1::Cancelled) - } else if self.deadline.is_elapsed_at(Instant::now()) { - Some(CollectionCompletionV1::DeadlineExceeded) - } else { - None - } - } - - /// Adapt the retention admission to the canonical `SQLite` read-snapshot - /// control. The snapshot layer may copy/materialize a foreign database in - /// `spawn_blocking`, so it must observe the same live cancellation and - /// deadline rather than an unbounded root-shim control. - pub(crate) fn snapshot_read_control( - self, - ) -> tracedecay_runtime_core::sqlite_read_snapshot::SnapshotReadControl { - let cancellation = (*self.cancellation).clone(); - tracedecay_runtime_core::sqlite_read_snapshot::SnapshotReadControl::new( - self.deadline.instant(), - move || cancellation.is_cancelled(), - ) - } - - /// Race an awaitable inspection or `SQLite` command against the admission's - /// cancellation/deadline. Losing the race never authorizes the following - /// destructive phase: callers retain their quarantine journal and let a - /// later reconciliation inspect the durable state afresh. - pub(crate) async fn race( - self, - future: impl Future, - ) -> Result { - if let Some(completion) = self.completion() { - return Err(completion); - } - tokio::select! { - biased; - () = self.cancellation.cancelled() => Err(CollectionCompletionV1::Cancelled), - () = tokio::time::sleep_until(tokio::time::Instant::from_std(self.deadline.instant())) => { - Err(CollectionCompletionV1::DeadlineExceeded) - } - result = future => { - self.completion().map_or(Ok(result), Err) - } - } - } -} - -pub(crate) fn unbounded_collection_control() -> CollectionControl<'static> { - static CANCELLATION: std::sync::OnceLock = std::sync::OnceLock::new(); - CollectionControl::new( - CANCELLATION.get_or_init(CancellationToken::new), - MonotonicDeadline::at(Instant::now() + std::time::Duration::from_hours(24)), - ) -} - -pub(crate) fn store_finding_is_profile_contained( - finding: &OrphanStoreFinding, - profile_root: &Path, -) -> bool { - profile_relative_store_path(profile_root, &finding.data_root) - .is_ok_and(|relative| relative == Path::new(&finding.expected_store_relpath)) - && matches!( - capture_store_directory_fence(profile_root, &finding.data_root), - Ok(StoreDirectoryFence::Missing | StoreDirectoryFence::Present { .. }) - ) -} - -fn registered_payload_fence_matches( - finding: &OrphanStoreFinding, - profile_root: &Path, - control: CollectionControl<'_>, -) -> Result { - if !data_root_fence_matches( - &finding.expected_data_root_fence, - profile_root, - &finding.data_root, - )? { - return Ok(false); - } - match &finding.expected_data_root_fence { - StoreDirectoryFence::Missing => Ok(true), - StoreDirectoryFence::Present { .. } => { - Ok(newest_mtime_secs_controlled(&finding.data_root, control)? - == finding.expected_payload_mtime_secs) - } - StoreDirectoryFence::Unverifiable => Err(CollectionFailureKind::InspectFailed), - } -} - -fn unregistered_payload_fence_matches( - finding: &UnregisteredStoreFinding, - profile_root: &Path, - control: CollectionControl<'_>, -) -> Result { - if !data_root_fence_matches( - &finding.expected_data_root_fence, - profile_root, - &finding.data_root, - )? { - return Ok(false); - } - match &finding.expected_data_root_fence { - StoreDirectoryFence::Missing => Ok(true), - StoreDirectoryFence::Present { .. } => { - Ok(newest_mtime_secs_controlled(&finding.data_root, control)? - == finding.expected_payload_mtime_secs) - } - StoreDirectoryFence::Unverifiable => Err(CollectionFailureKind::InspectFailed), - } -} - -/// A prepared mutation is private to its same-parent quarantine but remains -/// fully recoverable. The caller must commit registry retirement before it -/// calls [`finalize_verified_quarantine`]. -enum QuarantinePreparation { - Missing, - Verified(QuarantinedStore), - Interrupted, - Failed, -} - -fn prepare_verified_quarantine( - profile_root: &Path, - data_root: &Path, - expected_content_fence: &StoreContentFence, - kind: QuarantineKindV1, - project_id: &str, - store_id: &str, - registry_fence: Option, - control: CollectionControl<'_>, - outcome: &mut CollectionOutcome, -) -> QuarantinePreparation { - match quarantine_store_for_verified_collection_controlled( - profile_root, - data_root, - expected_content_fence, - kind, - project_id, - store_id, - registry_fence, - control, - ) { - Ok(QuarantineStoreOutcome::Missing) => QuarantinePreparation::Missing, - Ok(QuarantineStoreOutcome::Verified(quarantine)) => { - QuarantinePreparation::Verified(quarantine) - } - Ok(QuarantineStoreOutcome::Interrupted { - quarantine_path, - failure, - }) => { - outcome.recovery_receipts.push(CollectionRecoveryReceipt { - store_id: store_id.to_owned(), - original_path: data_root.to_path_buf(), - actual_path: quarantine_path.clone(), - quarantine_path, - action: CollectionRecoveryAction::RetainedForRecovery, - }); - if let Some(failure) = failure { - outcome.errors.push(CollectionFailure { - store_id: store_id.to_owned(), - kind: CollectionFailureKind::RemoveFailed(failure), - }); - } - if let Some(completion) = control.completion() { - outcome.completion = completion; - } - QuarantinePreparation::Interrupted - } - Ok(QuarantineStoreOutcome::Restored { - restored_path, - failure, - }) => { - outcome.recovery_receipts.push(CollectionRecoveryReceipt { - store_id: store_id.to_owned(), - original_path: data_root.to_path_buf(), - quarantine_path: data_root.to_path_buf(), - actual_path: restored_path, - action: CollectionRecoveryAction::Restored, - }); - outcome.errors.push(CollectionFailure { - store_id: store_id.to_owned(), - kind: CollectionFailureKind::PayloadChanged, - }); - if let Some(failure) = failure { - outcome.errors.push(CollectionFailure { - store_id: store_id.to_owned(), - kind: CollectionFailureKind::RemoveFailed(failure), - }); - } - QuarantinePreparation::Failed - } - Ok(QuarantineStoreOutcome::Retained { - quarantine_path, - failure, - }) => { - outcome.recovery_receipts.push(CollectionRecoveryReceipt { - store_id: store_id.to_owned(), - original_path: data_root.to_path_buf(), - actual_path: quarantine_path.clone(), - quarantine_path, - action: CollectionRecoveryAction::RetainedForRecovery, - }); - outcome.errors.push(CollectionFailure { - store_id: store_id.to_owned(), - kind: CollectionFailureKind::PayloadChanged, - }); - outcome.errors.push(CollectionFailure { - store_id: store_id.to_owned(), - kind: CollectionFailureKind::RemoveFailed(failure), - }); - QuarantinePreparation::Failed - } - Err(kind) => { - outcome.errors.push(CollectionFailure { - store_id: store_id.to_owned(), - kind, - }); - QuarantinePreparation::Failed - } - } -} - -fn retain_interrupted_quarantine( - quarantine: Option<&QuarantinedStore>, - data_root: &Path, - store_id: &str, - completion: CollectionCompletionV1, - outcome: &mut CollectionOutcome, -) { - outcome.completion = completion; - if let Some(quarantine) = quarantine { - outcome.recovery_receipts.push(CollectionRecoveryReceipt { - store_id: store_id.to_owned(), - original_path: data_root.to_path_buf(), - quarantine_path: quarantine.quarantine_path().to_path_buf(), - actual_path: quarantine.quarantine_path().to_path_buf(), - action: CollectionRecoveryAction::RetainedForRecovery, - }); - } -} - -fn finalize_verified_quarantine( - quarantine: QuarantinedStore, - data_root: &Path, - store_id: &str, - control: CollectionControl<'_>, - outcome: &mut CollectionOutcome, -) -> bool { - if let Some(completion) = control.completion() { - outcome.completion = completion; - outcome.recovery_receipts.push(CollectionRecoveryReceipt { - store_id: store_id.to_owned(), - original_path: data_root.to_path_buf(), - quarantine_path: quarantine.quarantine_path().to_path_buf(), - actual_path: quarantine.quarantine_path().to_path_buf(), - action: CollectionRecoveryAction::RetainedForRecovery, - }); - return false; - } - if let Err(failure) = quarantine.mark_retirement_committed() { - outcome.recovery_receipts.push(CollectionRecoveryReceipt { - store_id: store_id.to_owned(), - original_path: data_root.to_path_buf(), - quarantine_path: quarantine.quarantine_path().to_path_buf(), - actual_path: quarantine.quarantine_path().to_path_buf(), - action: CollectionRecoveryAction::RetainedForRecovery, - }); - outcome.errors.push(CollectionFailure { - store_id: store_id.to_owned(), - kind: CollectionFailureKind::RemoveFailed(failure), - }); - return false; - } - match quarantine.finalize(control) { - QuarantineFinalizeOutcome::Removed { journal_failure } => { - if let Some(failure) = journal_failure { - outcome.errors.push(CollectionFailure { - store_id: store_id.to_owned(), - kind: CollectionFailureKind::RemoveFailed(failure), - }); - } - true - } - QuarantineFinalizeOutcome::Interrupted { quarantine_path } => { - if let Some(completion) = control.completion() { - outcome.completion = completion; - } - outcome.recovery_receipts.push(CollectionRecoveryReceipt { - store_id: store_id.to_owned(), - original_path: data_root.to_path_buf(), - actual_path: quarantine_path.clone(), - quarantine_path, - action: CollectionRecoveryAction::RetainedForRecovery, - }); - false - } - QuarantineFinalizeOutcome::DeleteUnconfirmed { - quarantine_path, - failure, - } => { - outcome.recovery_receipts.push(CollectionRecoveryReceipt { - store_id: store_id.to_owned(), - original_path: data_root.to_path_buf(), - actual_path: quarantine_path.clone(), - quarantine_path, - action: CollectionRecoveryAction::DeleteUnconfirmed, - }); - outcome.errors.push(CollectionFailure { - store_id: store_id.to_owned(), - kind: CollectionFailureKind::RemoveFailed(failure), - }); - false - } - } -} - -/// Reconcile a durable interrupted quarantine before applying a fresh plan for -/// this exact live-name. Unregistered journals use their durable retirement -/// marker; registered journals remain pending unless the global-registry -/// inventory pass supplied an exact database decision. A restored or retained -/// quarantine forces a later census/confirmation pass, and recovery never -/// fabricates the old plan's byte count. -pub(super) fn reconcile_existing_quarantine( - profile_root: &Path, - data_root: &Path, - store_id: &str, - outcome: &mut CollectionOutcome, - control: CollectionControl<'_>, -) -> bool { - let can_continue = match recover_existing_store_quarantine(profile_root, data_root, control) { - Ok(recoveries) if recoveries.is_empty() => true, - Ok(recoveries) => { - let mut retained_or_restored = false; - for recovery in recoveries { - let recovery_receipt = match recovery { - QuarantineRecoveryOutcome::Removed { - journal_failure, .. - } => { - if let Some(failure) = journal_failure { - outcome.errors.push(CollectionFailure { - store_id: store_id.to_owned(), - kind: CollectionFailureKind::RemoveFailed(failure), - }); - } - None - } - QuarantineRecoveryOutcome::Restored { - restored_path, - failure, - } => { - retained_or_restored = true; - let action = if failure.is_some() { - CollectionRecoveryAction::RetainedForRecovery - } else { - CollectionRecoveryAction::Restored - }; - Some((data_root.to_path_buf(), restored_path, action, failure)) - } - QuarantineRecoveryOutcome::Retained { - quarantine_path, - actual_path, - failure, - } => { - retained_or_restored = true; - Some(( - quarantine_path.clone(), - actual_path, - CollectionRecoveryAction::RetainedForRecovery, - failure, - )) - } - }; - let Some((quarantine_path, actual_path, action, failure)) = recovery_receipt else { - continue; - }; - outcome.recovery_receipts.push(CollectionRecoveryReceipt { - store_id: store_id.to_owned(), - original_path: data_root.to_path_buf(), - quarantine_path, - actual_path, - action, - }); - if let Some(failure) = failure { - outcome.errors.push(CollectionFailure { - store_id: store_id.to_owned(), - kind: CollectionFailureKind::RemoveFailed(failure), - }); - } - } - if retained_or_restored { - outcome.errors.push(CollectionFailure { - store_id: store_id.to_owned(), - kind: CollectionFailureKind::PayloadChanged, - }); - } - false - } - Err(kind) => { - outcome.errors.push(CollectionFailure { - store_id: store_id.to_owned(), - kind, - }); - false - } - }; - if let Some(completion) = control.completion() { - outcome.completion = completion; - false - } else { - can_continue - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(super) enum RegisteredQuarantineRegistryStateV1 { - Exact, - Absent, - Changed, -} - -async fn registered_quarantine_registry_state( - transaction: &RegisteredGlobalDbWriteTransaction<'_>, - intent: &RegisteredQuarantineIntentV1, - control: CollectionControl<'_>, -) -> tracedecay_domain::errors::Result< - Result, -> { - let mut rows = match control - .race(transaction.query( - "SELECT project_id, store_relpath, created_at, last_write_at - FROM store_instances - WHERE store_id = ?1", - tracedecay_runtime_core::db::engine::params![intent.store_id.as_str()], - )) - .await - { - Ok(Ok(rows)) => rows, - Ok(Err(error)) => { - return Err(orphan_db_error( - "classify registered quarantine registry row", - error, - )); - } - Err(completion) => return Ok(Err(completion)), - }; - let first = match control.race(rows.next()).await { - Ok(Ok(row)) => row, - Ok(Err(error)) => { - return Err(orphan_db_error( - "read registered quarantine registry row", - error, - )); - } - Err(completion) => return Ok(Err(completion)), - }; - let Some(row) = first else { - return Ok(Ok(RegisteredQuarantineRegistryStateV1::Absent)); - }; - let current = ( - row.get::(0) - .map_err(|error| orphan_db_error("decode registered quarantine project id", error))?, - row.get::(1).map_err(|error| { - orphan_db_error("decode registered quarantine store relpath", error) - })?, - row.get::(2) - .map_err(|error| orphan_db_error("decode registered quarantine created time", error))?, - row.get::>(3) - .map_err(|error| orphan_db_error("decode registered quarantine last write", error))?, - ); - let ambiguous = match control.race(rows.next()).await { - Ok(Ok(row)) => row.is_some(), - Ok(Err(error)) => { - return Err(orphan_db_error( - "confirm registered quarantine registry uniqueness", - error, - )); - } - Err(completion) => return Ok(Err(completion)), - }; - if ambiguous { - return Ok(Ok(RegisteredQuarantineRegistryStateV1::Changed)); - } - let expected = ( - intent.project_id.clone(), - intent.registry_fence.store_relpath.clone(), - intent.registry_fence.created_at, - intent.registry_fence.last_write_at, - ); - Ok(Ok(if current == expected { - RegisteredQuarantineRegistryStateV1::Exact - } else { - RegisteredQuarantineRegistryStateV1::Changed - })) -} - -async fn rollback_registered_quarantine_recovery( - transaction: RegisteredGlobalDbWriteTransaction<'_>, - operation: &'static str, -) -> tracedecay_domain::errors::Result<()> { - transaction - .rollback() - .await - .map_err(|error| orphan_db_error(operation, error)) -} - -fn record_registered_quarantine_recovery( - intent: &RegisteredQuarantineIntentV1, - recovery: Option, - registry_changed: bool, - outcome: &mut CollectionOutcome, -) { - if registry_changed { - outcome.errors.push(CollectionFailure { - store_id: intent.store_id.clone(), - kind: CollectionFailureKind::RegistryChanged, - }); - } - let Some(recovery) = recovery else { - return; - }; - match recovery { - QuarantineRecoveryOutcome::Removed { - journal_failure, .. - } => { - if let Some(failure) = journal_failure { - outcome.errors.push(CollectionFailure { - store_id: intent.store_id.clone(), - kind: CollectionFailureKind::RemoveFailed(failure), - }); - } - } - QuarantineRecoveryOutcome::Restored { - restored_path, - failure, - } => { - let action = if failure.is_some() { - CollectionRecoveryAction::RetainedForRecovery - } else { - CollectionRecoveryAction::Restored - }; - outcome.recovery_receipts.push(CollectionRecoveryReceipt { - store_id: intent.store_id.clone(), - original_path: intent.original_path.clone(), - quarantine_path: intent.quarantine_path.clone(), - actual_path: restored_path, - action, - }); - if let Some(failure) = failure { - outcome.errors.push(CollectionFailure { - store_id: intent.store_id.clone(), - kind: CollectionFailureKind::RemoveFailed(failure), - }); - } - } - QuarantineRecoveryOutcome::Retained { - quarantine_path, - actual_path, - failure, - } => { - outcome.recovery_receipts.push(CollectionRecoveryReceipt { - store_id: intent.store_id.clone(), - original_path: intent.original_path.clone(), - quarantine_path, - actual_path, - action: CollectionRecoveryAction::RetainedForRecovery, - }); - if let Some(failure) = failure { - outcome.errors.push(CollectionFailure { - store_id: intent.store_id.clone(), - kind: CollectionFailureKind::RemoveFailed(failure), - }); - } - } - } -} - -async fn reconcile_registered_quarantine_inventory( - db: &RegisteredGlobalDb, - profile_root: &Path, - control: CollectionControl<'_>, - outcome: &mut CollectionOutcome, -) -> tracedecay_domain::errors::Result<()> { - reconcile_registered_quarantine_inventory_inner( - db, - profile_root, - control, - outcome, - #[cfg(test)] - None, - ) - .await -} - -#[cfg(test)] -pub(super) async fn reconcile_registered_quarantine_inventory_with_classified_hook( - db: &RegisteredGlobalDb, - profile_root: &Path, - control: CollectionControl<'_>, - outcome: &mut CollectionOutcome, - mut after_classification: impl FnMut( - &RegisteredQuarantineIntentV1, - RegisteredQuarantineRegistryStateV1, - ) + Send, -) -> tracedecay_domain::errors::Result<()> { - reconcile_registered_quarantine_inventory_inner( - db, - profile_root, - control, - outcome, - Some(&mut after_classification), - ) - .await -} - -#[cfg(test)] -type RegisteredQuarantineClassifiedHook<'a> = - dyn FnMut(&RegisteredQuarantineIntentV1, RegisteredQuarantineRegistryStateV1) + Send + 'a; - -async fn reconcile_registered_quarantine_inventory_inner( - db: &RegisteredGlobalDb, - profile_root: &Path, - control: CollectionControl<'_>, - outcome: &mut CollectionOutcome, - #[cfg(test)] mut after_classification: Option<&mut RegisteredQuarantineClassifiedHook<'_>>, -) -> tracedecay_domain::errors::Result<()> { - let intents = match read_registered_quarantine_intents_controlled(profile_root, control) { - Ok(RegisteredQuarantineInventoryV1::Complete(intents)) => intents, - Ok(RegisteredQuarantineInventoryV1::Interrupted) => { - outcome.completion = control - .completion() - .unwrap_or(CollectionCompletionV1::Cancelled); - return Ok(()); - } - Err(CollectionFailureKind::Cancelled) => { - outcome.completion = control - .completion() - .unwrap_or(CollectionCompletionV1::Cancelled); - return Ok(()); - } - Err(kind) => { - outcome.errors.push(CollectionFailure { - store_id: "registered-quarantine-inventory".to_owned(), - kind, - }); - return Ok(()); - } - }; - for intent in intents { - if let Some(completion) = control.completion() { - outcome.completion = completion; - break; - } - let transaction = match control.race(db.begin_write_transaction()).await { - Ok(Ok(transaction)) => transaction, - Ok(Err(error)) => return Err(error), - Err(completion) => { - outcome.completion = completion; - break; - } - }; - let registry_state = - match registered_quarantine_registry_state(&transaction, &intent, control).await { - Ok(Ok(state)) => state, - Ok(Err(completion)) => { - rollback_registered_quarantine_recovery( - transaction, - "rollback interrupted registered quarantine classification", - ) - .await?; - outcome.completion = completion; - break; - } - Err(error) => { - if let Err(rollback_error) = transaction.rollback().await { - return Err(orphan_db_error( - "rollback failed registered quarantine classification", - format!("{error}; rollback failed: {rollback_error}"), - )); - } - return Err(error); - } - }; - #[cfg(test)] - if let Some(after_classification) = after_classification.as_deref_mut() { - after_classification(&intent, registry_state); - } - let (decision, registry_changed) = match registry_state { - RegisteredQuarantineRegistryStateV1::Exact => { - (RegisteredQuarantineDecisionV1::Restore, false) - } - RegisteredQuarantineRegistryStateV1::Absent => { - (RegisteredQuarantineDecisionV1::Remove, false) - } - RegisteredQuarantineRegistryStateV1::Changed => { - (RegisteredQuarantineDecisionV1::Retain, true) - } - }; - if let Some(completion) = control.completion() { - rollback_registered_quarantine_recovery( - transaction, - "rollback interrupted registered quarantine recovery", - ) - .await?; - outcome.completion = completion; - break; - } - let recovery = match recover_registered_quarantine_intent_controlled( - profile_root, - &intent, - decision, - control, - ) { - Ok(recovery) => recovery, - Err(CollectionFailureKind::Cancelled) => { - rollback_registered_quarantine_recovery( - transaction, - "rollback interrupted registered quarantine recovery", - ) - .await?; - outcome.completion = control - .completion() - .unwrap_or(CollectionCompletionV1::Cancelled); - break; - } - Err(kind) => { - rollback_registered_quarantine_recovery( - transaction, - "rollback failed registered quarantine recovery", - ) - .await?; - outcome.errors.push(CollectionFailure { - store_id: intent.store_id.clone(), - kind, - }); - continue; - } - }; - rollback_registered_quarantine_recovery( - transaction, - "rollback completed registered quarantine recovery", - ) - .await?; - record_registered_quarantine_recovery(&intent, recovery, registry_changed, outcome); - if let Some(completion) = control.completion() { - outcome.completion = completion; - break; - } - } - Ok(()) -} - -/// Executes registered collection in two phases: expensive inspection and a -/// same-parent quarantine run without a writer; a short final transaction then -/// retires the exact registry row before irreversible quarantine deletion. -pub async fn execute_registered_collection( - db: &RegisteredGlobalDb, - plan: &CollectionPlan, - profile_root: &Path, -) -> tracedecay_domain::errors::Result<(CollectionOutcome, usize)> { - execute_registered_collection_controlled(db, plan, profile_root, unbounded_collection_control()) - .await -} - -#[hotpath::measure(label = "maintenance.orphan_stores.collect_registered", future = true)] -pub(crate) async fn execute_registered_collection_controlled( - db: &RegisteredGlobalDb, - plan: &CollectionPlan, - profile_root: &Path, - control: CollectionControl<'_>, -) -> tracedecay_domain::errors::Result<(CollectionOutcome, usize)> { - let mut outcome = CollectionOutcome::default(); - let mut retired = 0usize; - reconcile_registered_quarantine_inventory(db, profile_root, control, &mut outcome).await?; - if outcome.completion != CollectionCompletionV1::Complete { - return Ok((outcome, retired)); - } - for finding in &plan.collect { - if let Some(completion) = control.completion() { - outcome.completion = completion; - break; - } - if !reconcile_existing_quarantine( - profile_root, - &finding.data_root, - &finding.store_id, - &mut outcome, - control, - ) { - continue; - } - if !store_finding_is_profile_contained(finding, profile_root) { - outcome.errors.push(CollectionFailure { - store_id: finding.store_id.clone(), - kind: CollectionFailureKind::OutsideProfile, - }); - continue; - } - match registered_payload_fence_matches(finding, profile_root, control) { - Ok(true) => {} - Ok(false) => { - outcome.errors.push(CollectionFailure { - store_id: finding.store_id.clone(), - kind: CollectionFailureKind::PayloadChanged, - }); - continue; - } - Err(CollectionFailureKind::Cancelled) => { - outcome.completion = control - .completion() - .unwrap_or(CollectionCompletionV1::Cancelled); - break; - } - Err(kind) => { - outcome.errors.push(CollectionFailure { - store_id: finding.store_id.clone(), - kind, - }); - continue; - } - } - - let current_stores = match control - .race(db.try_list_store_instances_for_project(&finding.project_id)) - .await - { - Ok(Ok(stores)) => stores, - Ok(Err(error)) => return Err(error), - Err(completion) => { - outcome.completion = completion; - break; - } - }; - let current = current_stores - .into_iter() - .find(|store| store.store_id == finding.store_id) - .map(|store| (store.store_relpath, store.created_at, store.last_write_at)); - if current - != Some(( - finding.expected_store_relpath.clone(), - finding.expected_created_at, - finding.expected_last_write_at, - )) - { - outcome.errors.push(CollectionFailure { - store_id: finding.store_id.clone(), - kind: CollectionFailureKind::RegistryChanged, - }); - continue; - } - - let manifest_path = finding - .data_root - .join(tracedecay_runtime_core::storage::STORE_MANIFEST_FILENAME); - let current_manifest = match read_regular_file(&manifest_path) { - RegularFileSnapshot::Bytes(bytes) => Some(bytes), - RegularFileSnapshot::Missing => None, - RegularFileSnapshot::Unverifiable => { - outcome.errors.push(CollectionFailure { - store_id: finding.store_id.clone(), - kind: CollectionFailureKind::InspectFailed, - }); - continue; - } - }; - if current_manifest != finding.expected_manifest_bytes { - outcome.errors.push(CollectionFailure { - store_id: finding.store_id.clone(), - kind: CollectionFailureKind::ManifestChanged, - }); - continue; - } - match registered_payload_fence_matches(finding, profile_root, control) { - Ok(true) => {} - Ok(false) => { - outcome.errors.push(CollectionFailure { - store_id: finding.store_id.clone(), - kind: CollectionFailureKind::PayloadChanged, - }); - continue; - } - Err(CollectionFailureKind::Cancelled) => { - outcome.completion = control - .completion() - .unwrap_or(CollectionCompletionV1::Cancelled); - break; - } - Err(kind) => { - outcome.errors.push(CollectionFailure { - store_id: finding.store_id.clone(), - kind, - }); - continue; - } - } - - let scratch_root = durable_check_scratch_root(profile_root); - match check_store_durable_memory( - &finding.data_root, - finding.expected_manifest_bytes.as_deref(), - &finding.graph_scope_relpaths, - &scratch_root, - control, - ) - .await - { - DurableMemoryCheck::Empty => {} - DurableMemoryCheck::Present | DurableMemoryCheck::Unverifiable => { - outcome.errors.push(CollectionFailure { - store_id: finding.store_id.clone(), - kind: CollectionFailureKind::DurableDataProtected, - }); - continue; - } - DurableMemoryCheck::Interrupted => { - outcome.errors.push(CollectionFailure { - store_id: finding.store_id.clone(), - kind: CollectionFailureKind::Cancelled, - }); - outcome.completion = control - .completion() - .unwrap_or(CollectionCompletionV1::Cancelled); - break; - } - } - - // The durable inventory can take a private snapshot and therefore - // leaves a window for a concurrent replacement. Re-prove the exact - // directory generation immediately before destructive removal. - match registered_payload_fence_matches(finding, profile_root, control) { - Ok(true) => {} - Ok(false) => { - outcome.errors.push(CollectionFailure { - store_id: finding.store_id.clone(), - kind: CollectionFailureKind::PayloadChanged, - }); - continue; - } - Err(CollectionFailureKind::Cancelled) => { - outcome.completion = control - .completion() - .unwrap_or(CollectionCompletionV1::Cancelled); - break; - } - Err(kind) => { - outcome.errors.push(CollectionFailure { - store_id: finding.store_id.clone(), - kind, - }); - continue; - } - } - - let quarantine = match prepare_verified_quarantine( - profile_root, - &finding.data_root, - &finding.expected_content_fence, - QuarantineKindV1::Registered, - &finding.project_id, - &finding.store_id, - Some(QuarantineRegistryFenceV1 { - store_relpath: finding.expected_store_relpath.clone(), - created_at: finding.expected_created_at, - last_write_at: finding.expected_last_write_at, - }), - control, - &mut outcome, - ) { - QuarantinePreparation::Missing => None, - QuarantinePreparation::Verified(quarantine) => Some(quarantine), - QuarantinePreparation::Interrupted | QuarantinePreparation::Failed => { - continue; - } - }; - let transaction = match control.race(db.begin_write_transaction()).await { - Ok(Ok(transaction)) => transaction, - Ok(Err(error)) => return Err(error), - Err(completion) => { - retain_interrupted_quarantine( - quarantine.as_ref(), - &finding.data_root, - &finding.store_id, - completion, - &mut outcome, - ); - break; - } - }; - let mut rows = match control - .race(transaction.query( - "SELECT store_relpath, created_at, last_write_at - FROM store_instances - WHERE project_id = ?1 AND store_id = ?2", - tracedecay_runtime_core::db::engine::params![ - finding.project_id.as_str(), - finding.store_id.as_str() - ], - )) - .await - { - Ok(Ok(rows)) => rows, - Ok(Err(error)) => { - return Err(orphan_db_error( - "confirm quarantined orphan registry", - error, - )); - } - Err(completion) => { - drop(transaction); - retain_interrupted_quarantine( - quarantine.as_ref(), - &finding.data_root, - &finding.store_id, - completion, - &mut outcome, - ); - break; - } - }; - let next = match control.race(rows.next()).await { - Ok(Ok(next)) => next, - Ok(Err(error)) => { - return Err(orphan_db_error("read quarantined orphan registry", error)); - } - Err(completion) => { - drop(rows); - drop(transaction); - retain_interrupted_quarantine( - quarantine.as_ref(), - &finding.data_root, - &finding.store_id, - completion, - &mut outcome, - ); - break; - } - }; - let current = match next { - Some(row) => Some(( - row.get::(0) - .map_err(|error| orphan_db_error("decode orphan store relpath", error))?, - row.get::(1) - .map_err(|error| orphan_db_error("decode orphan store generation", error))?, - row.get::>(2) - .map_err(|error| orphan_db_error("decode orphan last write", error))?, - )), - None => None, - }; - drop(rows); - if current - != Some(( - finding.expected_store_relpath.clone(), - finding.expected_created_at, - finding.expected_last_write_at, - )) - { - match control.race(transaction.rollback()).await { - Ok(Ok(())) => {} - Ok(Err(error)) => { - return Err(orphan_db_error( - "rollback changed quarantined orphan", - error, - )); - } - Err(completion) => { - retain_interrupted_quarantine( - quarantine.as_ref(), - &finding.data_root, - &finding.store_id, - completion, - &mut outcome, - ); - break; - } - } - outcome.errors.push(CollectionFailure { - store_id: finding.store_id.clone(), - kind: CollectionFailureKind::RegistryChanged, - }); - continue; - } - let deleted = match control - .race(transaction.execute( - "DELETE FROM store_instances - WHERE project_id = ?1 AND store_id = ?2 - AND store_relpath = ?3 AND created_at = ?4 - AND last_write_at IS ?5", - tracedecay_runtime_core::db::engine::params![ - finding.project_id.as_str(), - finding.store_id.as_str(), - finding.expected_store_relpath.as_str(), - finding.expected_created_at, - finding.expected_last_write_at - ], - )) - .await - { - Ok(Ok(deleted)) => deleted, - Ok(Err(error)) => return Err(orphan_db_error("retire collected orphan store", error)), - Err(completion) => { - drop(transaction); - retain_interrupted_quarantine( - quarantine.as_ref(), - &finding.data_root, - &finding.store_id, - completion, - &mut outcome, - ); - break; - } - }; - if deleted != 1 { - match control.race(transaction.rollback()).await { - Ok(Ok(())) => {} - Ok(Err(error)) => { - return Err(orphan_db_error("rollback raced orphan retirement", error)); - } - Err(completion) => { - retain_interrupted_quarantine( - quarantine.as_ref(), - &finding.data_root, - &finding.store_id, - completion, - &mut outcome, - ); - break; - } - } - outcome.errors.push(CollectionFailure { - store_id: finding.store_id.clone(), - kind: CollectionFailureKind::RegistryChanged, - }); - continue; - } - match control - .race(transaction.execute( - "DELETE FROM code_projects - WHERE project_id = ?1 - AND NOT EXISTS ( - SELECT 1 FROM store_instances WHERE project_id = ?1 - )", - tracedecay_runtime_core::db::engine::params![finding.project_id.as_str()], - )) - .await - { - Ok(Ok(_)) => {} - Ok(Err(error)) => return Err(orphan_db_error("retire empty collected project", error)), - Err(completion) => { - drop(transaction); - retain_interrupted_quarantine( - quarantine.as_ref(), - &finding.data_root, - &finding.store_id, - completion, - &mut outcome, - ); - break; - } - } - match control.race(transaction.commit()).await { - Ok(Ok(())) => {} - Ok(Err(error)) => { - return Err(orphan_db_error("commit collected orphan retirement", error)); - } - Err(completion) => { - retain_interrupted_quarantine( - quarantine.as_ref(), - &finding.data_root, - &finding.store_id, - completion, - &mut outcome, - ); - break; - } - } - - retired = retired.saturating_add(1); - if let Some(quarantine) = quarantine - && !finalize_verified_quarantine( - quarantine, - &finding.data_root, - &finding.store_id, - control, - &mut outcome, - ) - { - continue; - } - outcome.reclaimed_bytes = outcome.reclaimed_bytes.saturating_add(finding.size_bytes); - outcome.collected.push(CollectedStore { - project_id: finding.project_id.clone(), - store_id: finding.store_id.clone(), - data_root: finding.data_root.clone(), - size_bytes: finding.size_bytes, - }); - } - Ok((outcome, retired)) -} - -fn orphan_db_error( - operation: &'static str, - error: impl std::fmt::Display, -) -> tracedecay_domain::errors::TraceDecayError { - tracedecay_domain::errors::TraceDecayError::Database { - operation: operation.to_string(), - message: error.to_string(), - } -} - -/// Result of checking a store's graph database for durable memory rows. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(super) enum DurableMemoryCheck { - /// Cooperative cancellation/deadline interrupted recursive discovery or a - /// bounded database probe before any mutation. - Interrupted, - /// No durable memory table has any row (including: none of the tables - /// exist, or the database file itself does not exist). Safe to collect. - Empty, - /// At least one durable memory table has at least one row. - Present, - /// The check could not prove the store is free of durable memory rows - /// (I/O error, corrupt/locked database, the source changed mid-check). - /// Fails closed: treated exactly like `Present` by every caller. - Unverifiable, -} - -/// Every database under a store that can carry durable rows, or a typed -/// statement that the inventory itself could not be trusted. -/// -/// The databases registered as project authorities for durable memory. -#[derive(Debug, Clone, PartialEq, Eq)] -pub(super) enum DurableDatabaseInventoryV1 { - /// The bounded scan stopped before it could establish a complete durable - /// database inventory. This is not an unverifiable green light: callers - /// preserve the exact cancellation/deadline state for the coordinator. - Interrupted, - /// The complete set of database paths, relative to the store's data root. - Resolved(Vec), - /// The set could not be enumerated, a missing or malformed manifest, or a - /// directory that could not be listed. Never a green light for deletion. - Unverifiable, -} - -/// A regular-file read that preserves the difference between an absent -/// optional artifact and an unsafe/unreadable one. In particular, `read()` -/// follows symlinks; retention must never turn a symlinked manifest into a -/// trusted manifest snapshot. -#[derive(Debug, Clone, PartialEq, Eq)] -pub(super) enum RegularFileSnapshot { - Missing, - Bytes(Vec), - Unverifiable, -} - -pub(super) fn read_regular_file(path: &Path) -> RegularFileSnapshot { - let metadata = match std::fs::symlink_metadata(path) { - Ok(metadata) => metadata, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => { - return RegularFileSnapshot::Missing; - } - Err(_) => return RegularFileSnapshot::Unverifiable, - }; - if metadata.file_type().is_symlink() || !metadata.file_type().is_file() { - return RegularFileSnapshot::Unverifiable; - } - let Ok(bytes) = std::fs::read(path) else { - return RegularFileSnapshot::Unverifiable; - }; - match std::fs::symlink_metadata(path) { - Ok(metadata) if !metadata.file_type().is_symlink() && metadata.file_type().is_file() => { - RegularFileSnapshot::Bytes(bytes) - } - _ => RegularFileSnapshot::Unverifiable, - } -} - -/// Store manifests and registry-provided graph scopes are path authorities, -/// not arbitrary filesystem paths. Only normalized, non-empty relative paths -/// made entirely from normal components are accepted; `..`, `.`, roots, -/// prefixes, and empty paths all fail closed before joining. -fn safe_store_relative_path(path: &Path) -> bool { - let mut saw_normal = false; - let mut normalized = PathBuf::new(); - for component in path.components() { - if let std::path::Component::Normal(component) = component { - saw_normal = true; - normalized.push(component); - } else { - return false; - } - } - saw_normal && normalized == path -} - -/// Reject symlinked directory components as well as a symlinked final file. -/// A lexical relative-path check alone is insufficient when an intermediate -/// directory redirects outside the store. -fn safe_store_path(data_root: &Path, relative: &Path) -> bool { - if !safe_store_relative_path(relative) { - return false; - } - let mut current = data_root.to_path_buf(); - for component in relative.components() { - let std::path::Component::Normal(component) = component else { - return false; - }; - current.push(component); - match std::fs::symlink_metadata(¤t) { - Ok(metadata) if metadata.file_type().is_symlink() => return false, - Ok(_) => {} - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return true, - Err(_) => return false, - } - } - true -} - -/// Enumerates every durable database under `data_root`. -/// -/// Fails closed. The manifest is the store's own record of where its graph -/// lives; if it is absent or will not parse, guessing the default filename -/// would check the wrong file (or no file) and report "empty" for a store whose -/// real graph sits elsewhere. -pub(super) fn durable_database_inventory( - data_root: &Path, - manifest_bytes: Option<&[u8]>, - graph_scope_relpaths: &[PathBuf], - control: CollectionControl<'_>, -) -> DurableDatabaseInventoryV1 { - if control.completion().is_some() { - return DurableDatabaseInventoryV1::Interrupted; - } - let Some(bytes) = manifest_bytes else { - return DurableDatabaseInventoryV1::Unverifiable; - }; - let manifest = - match serde_json::from_slice::(bytes) { - Ok(manifest) => manifest, - Err(_) if control.completion().is_some() => { - return DurableDatabaseInventoryV1::Interrupted; - } - Err(_) => return DurableDatabaseInventoryV1::Unverifiable, - }; - if control.completion().is_some() { - return DurableDatabaseInventoryV1::Interrupted; - } - - if !safe_store_relative_path(&manifest.graph_db_relpath) { - return DurableDatabaseInventoryV1::Unverifiable; - } - - let mut inventory = vec![manifest.graph_db_relpath]; - for relpath in graph_scope_relpaths { - if control.completion().is_some() { - return DurableDatabaseInventoryV1::Interrupted; - } - if !safe_store_relative_path(relpath) { - return DurableDatabaseInventoryV1::Unverifiable; - } - if !inventory.contains(relpath) { - inventory.push(relpath.clone()); - } - } - - // Durable facts are project-wide and outlive the branch they were written - // on, so a branch database can hold the only surviving rows. The manifest - // does not name them; an unlistable directory is therefore unverifiable, - // not empty. - let branches = data_root.join("branches"); - let branches_metadata = std::fs::symlink_metadata(&branches); - if control.completion().is_some() { - return DurableDatabaseInventoryV1::Interrupted; - } - match branches_metadata { - Ok(metadata) if metadata.file_type().is_symlink() || !metadata.file_type().is_dir() => { - return DurableDatabaseInventoryV1::Unverifiable; - } - Ok(_) => {} - Err(error) if error.kind() == std::io::ErrorKind::NotFound => { - return DurableDatabaseInventoryV1::Resolved(inventory); - } - Err(_) => return DurableDatabaseInventoryV1::Unverifiable, - } - let branch_entries = std::fs::read_dir(&branches); - if control.completion().is_some() { - return DurableDatabaseInventoryV1::Interrupted; - } - match branch_entries { - Ok(entries) => { - let mut entries = entries; - loop { - // `ReadDir` fetches lazily, so control must be checked before - // every `next` rather than only before opening `branches`. - if control.completion().is_some() { - return DurableDatabaseInventoryV1::Interrupted; - } - let Some(entry) = entries.next() else { - break; - }; - if control.completion().is_some() { - return DurableDatabaseInventoryV1::Interrupted; - } - let Ok(entry) = entry else { - return DurableDatabaseInventoryV1::Unverifiable; - }; - let path = entry.path(); - if path.extension().and_then(|extension| extension.to_str()) != Some("db") { - continue; - } - if control.completion().is_some() { - return DurableDatabaseInventoryV1::Interrupted; - } - let Ok(file_type) = entry.file_type() else { - return DurableDatabaseInventoryV1::Unverifiable; - }; - if control.completion().is_some() { - return DurableDatabaseInventoryV1::Interrupted; - } - if file_type.is_symlink() || !file_type.is_file() { - return DurableDatabaseInventoryV1::Unverifiable; - } - let Some(name) = path.file_name() else { - continue; - }; - let relpath = Path::new("branches").join(name); - if !inventory.contains(&relpath) { - inventory.push(relpath); - } - } - } - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} - Err(_) => return DurableDatabaseInventoryV1::Unverifiable, - } - - if control.completion().is_some() { - return DurableDatabaseInventoryV1::Interrupted; - } - DurableDatabaseInventoryV1::Resolved(inventory) -} - -/// Runs [`check_durable_memory_rows`] over every database in the store's -/// inventory. Any single `Present` or `Unverifiable` protects the whole store. -pub(super) async fn check_store_durable_memory( - data_root: &Path, - manifest_bytes: Option<&[u8]>, - graph_scope_relpaths: &[PathBuf], - scratch_root: &Path, - control: CollectionControl<'_>, -) -> DurableMemoryCheck { - if control.completion().is_some() { - return DurableMemoryCheck::Interrupted; - } - let inventory = match durable_database_inventory( - data_root, - manifest_bytes, - graph_scope_relpaths, - control, - ) { - DurableDatabaseInventoryV1::Interrupted => return DurableMemoryCheck::Interrupted, - DurableDatabaseInventoryV1::Resolved(inventory) => inventory, - DurableDatabaseInventoryV1::Unverifiable => return DurableMemoryCheck::Unverifiable, - }; - for relpath in inventory { - if control.completion().is_some() { - return DurableMemoryCheck::Interrupted; - } - match check_durable_memory_rows(data_root, &relpath, scratch_root, control).await { - DurableMemoryCheck::Empty => {} - protected => return protected, - } - } - DurableMemoryCheck::Empty -} - -/// The read-snapshot scratch directory for durable-memory checks. -/// -/// It lives under the *profile* root, never inside the store being examined. -/// Two reasons, both load-bearing: the store is a deletion candidate, and -/// writing into it bumps the newest mtime that -/// [`walk_store_stats`] uses as the revival fence, a store that failed one -/// check would have its age reset by the check itself and could never mature -/// past the retention window again. -pub(super) fn durable_check_scratch_root(profile_root: &Path) -> PathBuf { - profile_root.join("scratch").join("sqlite-read") -} - -/// Checks whether `data_root`'s graph database carries rows in any canonical -/// `memory_*` table. This intentionally discovers tables from the schema -/// instead of maintaining a fixed list: both legacy memory and Memory V2 add -/// durable tables, and a newly added table must be protected automatically. -/// Side-effect-free with respect to the store: opens the database through -/// [`tracedecay_runtime_core::sqlite_read_snapshot`], so the live store is never mutated or -/// locked against a concurrent writer. -async fn check_durable_memory_rows( - data_root: &Path, - graph_db_relpath: &Path, - scratch_root: &Path, - control: CollectionControl<'_>, -) -> DurableMemoryCheck { - if control.completion().is_some() { - return DurableMemoryCheck::Interrupted; - } - if !safe_store_path(data_root, graph_db_relpath) { - return DurableMemoryCheck::Unverifiable; - } - let graph_db_path = data_root.join(graph_db_relpath); - match std::fs::symlink_metadata(&graph_db_path) { - Ok(metadata) if metadata.file_type().is_symlink() || !metadata.file_type().is_file() => { - return DurableMemoryCheck::Unverifiable; - } - Ok(_) => {} - Err(error) if error.kind() == std::io::ErrorKind::NotFound => { - // No database file at all: there is no schema that could carry - // durable rows. - return DurableMemoryCheck::Empty; - } - Err(_) => return DurableMemoryCheck::Unverifiable, - } - // The snapshot layer creates only the final scratch component, so its - // parent must exist first. Without this the snapshot fails NotFound, the - // check fails closed as `Unverifiable`, and because `Unverifiable` is - // treated exactly like `Present`, *every* collection is refused. That is - // safe, but it silently disables orphan reclamation entirely. - if control.completion().is_some() || std::fs::create_dir_all(scratch_root).is_err() { - return if control.completion().is_some() { - DurableMemoryCheck::Interrupted - } else { - DurableMemoryCheck::Unverifiable - }; - } - let snapshot = match control - .race( - tracedecay_runtime_core::sqlite_read_snapshot::open_foreign_in( - &graph_db_path, - scratch_root, - control.snapshot_read_control(), - ), - ) - .await - { - Err(_) => return DurableMemoryCheck::Interrupted, - Ok(Ok(snapshot)) => snapshot, - Ok(Err(_)) => return DurableMemoryCheck::Unverifiable, - }; - if control.completion().is_some() { - return DurableMemoryCheck::Interrupted; - } - let connection = snapshot.connection(); - let mut rows = match control - .race(connection.query( - "SELECT name - FROM pragma_table_list - WHERE schema = 'main' - AND type = 'table' - AND name LIKE ?1 ESCAPE '\\' - ORDER BY name", - tracedecay_runtime_core::db::engine::params!["memory\\_%"], - )) - .await - { - Err(_) => return DurableMemoryCheck::Interrupted, - Ok(Ok(rows)) => rows, - Ok(Err(_)) => return DurableMemoryCheck::Unverifiable, - }; - let mut present_tables = Vec::new(); - loop { - let next = match control.race(rows.next()).await { - Err(_) => return DurableMemoryCheck::Interrupted, - Ok(Ok(next)) => next, - Ok(Err(_)) => return DurableMemoryCheck::Unverifiable, - }; - match next { - Some(row) => match row.get::(0) { - Ok(name) => present_tables.push(name), - Err(_) => return DurableMemoryCheck::Unverifiable, - }, - None => break, - } - } - drop(rows); - for table in present_tables { - // `pragma_table_list.type = 'table'` intentionally excludes FTS - // virtual/shadow tables, whose internal config rows are derived and - // exist even when there is no durable memory. Identifiers cannot be - // SQL parameters, so only interpolate TraceDecay's canonical shape; - // an unexpected name fails closed rather than becoming SQL text. - if !is_memory_table_identifier(&table) { - return DurableMemoryCheck::Unverifiable; - } - let probe_sql = format!("SELECT 1 FROM \"{table}\" LIMIT 1"); - let mut probe_rows = match control.race(connection.query(&probe_sql, ())).await { - Err(_) => return DurableMemoryCheck::Interrupted, - Ok(Ok(rows)) => rows, - Ok(Err(_)) => return DurableMemoryCheck::Unverifiable, - }; - match control.race(probe_rows.next()).await { - Err(_) => return DurableMemoryCheck::Interrupted, - Ok(Ok(Some(_))) => return DurableMemoryCheck::Present, - Ok(Ok(None)) => {} - Ok(Err(_)) => return DurableMemoryCheck::Unverifiable, - } - } - if control.completion().is_some() { - return DurableMemoryCheck::Interrupted; - } - if snapshot.validate_source().is_err() { - // The file changed under us mid-check: cannot trust an empty result. - return DurableMemoryCheck::Unverifiable; - } - DurableMemoryCheck::Empty -} - -fn is_memory_table_identifier(table: &str) -> bool { - table.strip_prefix("memory_").is_some_and(|suffix| { - !suffix.is_empty() - && suffix - .bytes() - .all(|byte| byte.is_ascii_alphanumeric() || byte == b'_') - }) -} - -#[cfg(test)] -pub(crate) async fn execute_unregistered_collection( - db: &RegisteredGlobalDb, - plan: &UnregisteredCollectionPlan, - profile_root: &Path, -) -> tracedecay_domain::errors::Result { - execute_unregistered_collection_controlled( - db, - plan, - profile_root, - unbounded_collection_control(), - ) - .await -} - -#[hotpath::measure( - label = "maintenance.orphan_stores.collect_unregistered", - future = true -)] -pub(crate) async fn execute_unregistered_collection_controlled( - db: &RegisteredGlobalDb, - plan: &UnregisteredCollectionPlan, - profile_root: &Path, - control: CollectionControl<'_>, -) -> tracedecay_domain::errors::Result { - let mut outcome = CollectionOutcome::default(); - for finding in &plan.collect { - if let Some(completion) = control.completion() { - outcome.completion = completion; - break; - } - if !reconcile_existing_quarantine( - profile_root, - &finding.data_root, - &finding.project_dir_name, - &mut outcome, - control, - ) { - continue; - } - // Containment + shape: only ever delete an exact, safely-named - // `/projects/` leaf. - let expected = profile_root - .join("projects") - .join(&finding.project_dir_name); - if expected != finding.data_root - || tracedecay_runtime_core::storage::validate_project_id(&finding.project_dir_name) - .is_err() - { - outcome.errors.push(CollectionFailure { - store_id: finding.project_dir_name.clone(), - kind: CollectionFailureKind::OutsideProfile, - }); - continue; - } - match unregistered_payload_fence_matches(finding, profile_root, control) { - Ok(true) => {} - Ok(false) => { - outcome.errors.push(CollectionFailure { - store_id: finding.project_dir_name.clone(), - kind: CollectionFailureKind::PayloadChanged, - }); - continue; - } - Err(CollectionFailureKind::Cancelled) => { - outcome.completion = control - .completion() - .unwrap_or(CollectionCompletionV1::Cancelled); - break; - } - Err(kind) => { - outcome.errors.push(CollectionFailure { - store_id: finding.project_dir_name.clone(), - kind, - }); - continue; - } - } - - let now_registered = match control - .race(db.code_project_exists(&finding.project_dir_name)) - .await - { - Ok(Ok(exists)) => exists, - Ok(Err(error)) => return Err(error), - Err(completion) => { - outcome.completion = completion; - break; - } - }; - if now_registered { - outcome.errors.push(CollectionFailure { - store_id: finding.project_dir_name.clone(), - kind: CollectionFailureKind::RegistryChanged, - }); - continue; - } - match unregistered_payload_fence_matches(finding, profile_root, control) { - Ok(true) => {} - Ok(false) => { - outcome.errors.push(CollectionFailure { - store_id: finding.project_dir_name.clone(), - kind: CollectionFailureKind::PayloadChanged, - }); - continue; - } - Err(CollectionFailureKind::Cancelled) => { - outcome.completion = control - .completion() - .unwrap_or(CollectionCompletionV1::Cancelled); - break; - } - Err(kind) => { - outcome.errors.push(CollectionFailure { - store_id: finding.project_dir_name.clone(), - kind, - }); - continue; - } - } - - let scratch_root = durable_check_scratch_root(profile_root); - // An unreadable manifest must not be swallowed into "no manifest": - // the inventory then fails closed instead of checking a guessed - // database. A manifestless directory is different: only an exact - // empty-tree inventory proves that it carries no durable authority. - // Arbitrary payload files remain unverifiable, while any discovered - // `.db` family is inspected directly and remains fail-closed on error. - let manifest_path = finding - .data_root - .join(tracedecay_runtime_core::storage::STORE_MANIFEST_FILENAME); - let durable_check = match read_regular_file(&manifest_path) { - RegularFileSnapshot::Bytes(manifest_bytes) => { - // An unregistered store has no registry graph scopes by - // definition; the manifest remains the canonical graph path. - check_store_durable_memory( - &finding.data_root, - Some(&manifest_bytes), - &[], - &scratch_root, - control, - ) - .await - } - RegularFileSnapshot::Missing => { - check_manifestless_store_durable_memory(&finding.data_root, &scratch_root, control) - .await - } - RegularFileSnapshot::Unverifiable => DurableMemoryCheck::Unverifiable, - }; - match durable_check { - DurableMemoryCheck::Empty => {} - DurableMemoryCheck::Present | DurableMemoryCheck::Unverifiable => { - outcome.errors.push(CollectionFailure { - store_id: finding.project_dir_name.clone(), - kind: CollectionFailureKind::DurableDataProtected, - }); - continue; - } - DurableMemoryCheck::Interrupted => { - outcome.errors.push(CollectionFailure { - store_id: finding.project_dir_name.clone(), - kind: CollectionFailureKind::Cancelled, - }); - outcome.completion = control - .completion() - .unwrap_or(CollectionCompletionV1::Cancelled); - break; - } - } - - // The durable-data inspection is intentionally fail-closed, but it is - // not a deletion lock. Re-prove the inspected root generation at the - // final destructive boundary so an in-profile replacement or symlink - // swap cannot inherit an old empty-directory decision. - match unregistered_payload_fence_matches(finding, profile_root, control) { - Ok(true) => {} - Ok(false) => { - outcome.errors.push(CollectionFailure { - store_id: finding.project_dir_name.clone(), - kind: CollectionFailureKind::PayloadChanged, - }); - continue; - } - Err(CollectionFailureKind::Cancelled) => { - outcome.completion = control - .completion() - .unwrap_or(CollectionCompletionV1::Cancelled); - break; - } - Err(kind) => { - outcome.errors.push(CollectionFailure { - store_id: finding.project_dir_name.clone(), - kind, - }); - continue; - } - } - - let quarantine = match prepare_verified_quarantine( - profile_root, - &finding.data_root, - &finding.expected_content_fence, - QuarantineKindV1::Unregistered, - &finding.project_dir_name, - &finding.project_dir_name, - None, - control, - &mut outcome, - ) { - QuarantinePreparation::Missing => None, - QuarantinePreparation::Verified(quarantine) => Some(quarantine), - QuarantinePreparation::Interrupted | QuarantinePreparation::Failed => { - continue; - } - }; - let transaction = match control.race(db.begin_write_transaction()).await { - Ok(Ok(transaction)) => transaction, - Ok(Err(error)) => return Err(error), - Err(completion) => { - retain_interrupted_quarantine( - quarantine.as_ref(), - &finding.data_root, - &finding.project_dir_name, - completion, - &mut outcome, - ); - break; - } - }; - let mut rows = match control - .race(transaction.query( - "SELECT 1 FROM code_projects WHERE project_id = ?1", - tracedecay_runtime_core::db::engine::params![finding.project_dir_name.as_str()], - )) - .await - { - Ok(Ok(rows)) => rows, - Ok(Err(error)) => { - return Err(orphan_db_error( - "confirm quarantined unregistered store", - error, - )); - } - Err(completion) => { - drop(transaction); - retain_interrupted_quarantine( - quarantine.as_ref(), - &finding.data_root, - &finding.project_dir_name, - completion, - &mut outcome, - ); - break; - } - }; - let next = match control.race(rows.next()).await { - Ok(Ok(next)) => next, - Ok(Err(error)) => { - return Err(orphan_db_error( - "read quarantined unregistered store", - error, - )); - } - Err(completion) => { - drop(rows); - drop(transaction); - retain_interrupted_quarantine( - quarantine.as_ref(), - &finding.data_root, - &finding.project_dir_name, - completion, - &mut outcome, - ); - break; - } - }; - let now_registered = next.is_some(); - drop(rows); - if now_registered { - match control.race(transaction.rollback()).await { - Ok(Ok(())) => {} - Ok(Err(error)) => { - return Err(orphan_db_error( - "rollback newly-registered quarantined store", - error, - )); - } - Err(completion) => { - retain_interrupted_quarantine( - quarantine.as_ref(), - &finding.data_root, - &finding.project_dir_name, - completion, - &mut outcome, - ); - break; - } - } - outcome.errors.push(CollectionFailure { - store_id: finding.project_dir_name.clone(), - kind: CollectionFailureKind::RegistryChanged, - }); - continue; - } - match control.race(transaction.commit()).await { - Ok(Ok(())) => {} - Ok(Err(error)) => { - return Err(orphan_db_error("commit unregistered store fence", error)); - } - Err(completion) => { - retain_interrupted_quarantine( - quarantine.as_ref(), - &finding.data_root, - &finding.project_dir_name, - completion, - &mut outcome, - ); - break; - } - } - - if let Some(quarantine) = quarantine - && !finalize_verified_quarantine( - quarantine, - &finding.data_root, - &finding.project_dir_name, - control, - &mut outcome, - ) - { - continue; - } - outcome.reclaimed_bytes = outcome.reclaimed_bytes.saturating_add(finding.size_bytes); - outcome.collected.push(CollectedStore { - project_id: finding.project_dir_name.clone(), - store_id: finding.project_dir_name.clone(), - data_root: finding.data_root.clone(), - size_bytes: finding.size_bytes, - }); - } - Ok(outcome) -} - -/// Inspects a manifestless unregistered directory without inventing a graph -/// path. An exactly empty directory is provably free of durable rows. Any -/// arbitrary payload, symlink, or unreadable entry remains unverifiable; -/// when a SQLite-looking file is present, every such file is treated as a -/// possible durable authority and inspected fail-closed. -async fn check_manifestless_store_durable_memory( - data_root: &Path, - scratch_root: &Path, - control: CollectionControl<'_>, -) -> DurableMemoryCheck { - let mut databases = Vec::new(); - if control.completion().is_some() { - return DurableMemoryCheck::Interrupted; - } - if collect_sqlite_candidates(data_root, data_root, &mut databases, control).is_err() { - return if control.completion().is_some() { - DurableMemoryCheck::Interrupted - } else { - DurableMemoryCheck::Unverifiable - }; - } - if databases.is_empty() { - return DurableMemoryCheck::Empty; - } - for relpath in databases { - if control.completion().is_some() { - return DurableMemoryCheck::Interrupted; - } - match check_durable_memory_rows(data_root, &relpath, scratch_root, control).await { - DurableMemoryCheck::Empty => {} - protected => return protected, - } - } - DurableMemoryCheck::Empty -} - -/// Finds only regular `.db` files below a store and never follows symlinks. -/// The manifestless path deliberately does not guess a single filename, so a -/// custom legacy graph cannot be mistaken for payload-only debris. Any other -/// file shape is an unverifiable durable-data candidate, not disposable dust. -fn collect_sqlite_candidates( - root: &Path, - current: &Path, - output: &mut Vec, - control: CollectionControl<'_>, -) -> std::io::Result<()> { - let entries = std::fs::read_dir(current)?; - for entry in entries { - if control.completion().is_some() { - return Err(std::io::Error::new( - std::io::ErrorKind::Interrupted, - "retention durable-data inventory interrupted", - )); - } - let entry = entry?; - let file_type = entry.file_type()?; - if file_type.is_symlink() { - return Err(std::io::Error::other( - "manifestless store contains a symlink", - )); - } - let path = entry.path(); - if file_type.is_dir() { - collect_sqlite_candidates(root, &path, output, control)?; - } else if file_type.is_file() - && path.extension().and_then(|extension| extension.to_str()) == Some("db") - && let Ok(relative) = path.strip_prefix(root) - { - output.push(relative.to_path_buf()); - } else { - return Err(std::io::Error::other( - "manifestless store contains an unrecognized payload", - )); - } - } - output.sort(); - Ok(()) -} diff --git a/crates/tracedecay-maintenance/src/retention/orphan_stores/tests.rs b/crates/tracedecay-maintenance/src/retention/orphan_stores/tests.rs index fc21ae748a..a66e940bed 100644 --- a/crates/tracedecay-maintenance/src/retention/orphan_stores/tests.rs +++ b/crates/tracedecay-maintenance/src/retention/orphan_stores/tests.rs @@ -2,24 +2,12 @@ use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::{Duration, Instant}; +use super::collection::{ + DurableDatabaseInventoryV1, DurableMemoryCheck, check_store_durable_memory, + durable_check_scratch_root, durable_database_inventory, open_verified_store, +}; use super::fence::{capture_store_content_fence, capture_store_directory_fence}; use super::pages::walk_store_stats; -#[cfg(windows)] -use super::quarantine::classify_recovery_journal_probe; -use super::quarantine::{ - DurableDatabaseInventoryV1, DurableMemoryCheck, PendingQuarantineReceiptV1, - QuarantineFinalizeOutcome, QuarantineKindV1, QuarantineRecoveryOutcome, - QuarantineRegistryFenceV1, QuarantineStoreOutcome, RegisteredQuarantineDecisionV1, - RegisteredQuarantineInventoryV1, RegisteredQuarantineRegistryStateV1, - check_store_durable_memory, durable_check_scratch_root, durable_database_inventory, - quarantine_candidate_namespace_available, quarantine_store_for_verified_collection, - quarantine_store_for_verified_collection_controlled, - read_registered_quarantine_intents_controlled, reconcile_existing_quarantine, - reconcile_registered_quarantine_inventory_with_classified_hook, - recover_existing_store_quarantine, recover_named_store_quarantine, - recover_named_store_quarantine_controlled, recover_registered_quarantine_intent_controlled, - reserve_quarantine_name_with_sequence, -}; use super::*; use tracedecay_global_db::RegisteredGlobalDb; use tracedecay_global_db::tests::harness::RegisteredGlobalDbTestRuntime; @@ -29,10 +17,6 @@ use tracedecay_runtime_core::storage::{ }; const DAY: i64 = 24 * 60 * 60; -#[cfg(unix)] -const OCCUPIED_RENAME_RAW_OS_ERROR: i32 = 17; -#[cfg(windows)] -const OCCUPIED_RENAME_RAW_OS_ERROR: i32 = 183; /// Same shape as the production convenience caller /// (`unbounded_collection_control`): a far-future monotonic deadline so @@ -185,8 +169,8 @@ async fn seed_project( transaction.commit().await.unwrap(); } +mod collection; mod pages; -mod quarantine; #[test] fn live_root_is_never_collected() { @@ -262,88 +246,6 @@ fn live_git_common_dir_keeps_a_linked_worktree_store_live() { assert!(plan_collection(findings, 0).collect.is_empty()); } -#[tokio::test] -async fn empty_plan_retains_registered_live_source_when_exact_row_is_absent() { - let tmp = tempfile::TempDir::new().unwrap(); - let profile_root = tmp.path().join("profile"); - std::fs::create_dir_all(&profile_root).unwrap(); - let (_runtime, db) = open_registered_db(&profile_root).await; - let payload = b"absent row cannot authorize deleting contradictory live bytes"; - let (data_root, quarantine_path) = prepare_registered_quarantine( - &db, - &profile_root, - "proj_registered_live_absent", - "store_registered_live_absent", - payload, - ) - .await; - let expected = capture_store_content_fence(&profile_root, &quarantine_path).unwrap(); - let StoreContentFence::Present(expected_inventory) = &expected else { - panic!("fixture must capture an exact present-store fence"); - }; - let expected_root_identity = expected_inventory.root.clone(); - std::fs::rename(&quarantine_path, &data_root).unwrap(); - let transaction = db.begin_write_transaction().await.unwrap(); - assert_eq!( - transaction - .execute( - "DELETE FROM store_instances WHERE store_id = ?1", - tracedecay_runtime_core::db::engine::params!["store_registered_live_absent"], - ) - .await - .unwrap(), - 1 - ); - transaction.commit().await.unwrap(); - - let (outcome, retired) = - execute_registered_collection(&db, &CollectionPlan::default(), &profile_root) - .await - .unwrap(); - - assert_eq!(retired, 0); - assert_eq!( - outcome, - CollectionOutcome { - errors: vec![CollectionFailure { - store_id: "store_registered_live_absent".to_owned(), - kind: CollectionFailureKind::RemoveFailed(CollectionMutationFailure { - operation: CollectionMutationOperation::ValidateRestoredStoreIdentity, - raw_os_error: None, - target_path: data_root.clone(), - expected_root_identity: Some(expected_root_identity), - classification: CollectionMutationFailureClassification::NonRetryable, - }), - }], - recovery_receipts: vec![CollectionRecoveryReceipt { - store_id: "store_registered_live_absent".to_owned(), - original_path: data_root.clone(), - quarantine_path: quarantine_path.clone(), - actual_path: data_root.clone(), - action: CollectionRecoveryAction::RetainedForRecovery, - }], - ..CollectionOutcome::default() - } - ); - assert_eq!( - std::fs::read(data_root.join("payload.bin")).unwrap(), - payload - ); - assert_eq!( - capture_store_content_fence(&profile_root, &data_root).unwrap(), - expected - ); - assert!(!quarantine_path.exists()); - assert_eq!( - read_pending_quarantine_receipts(&profile_root).unwrap(), - vec![PendingQuarantineReceiptV1 { - quarantine_path: quarantine_path.clone(), - actual_path: data_root, - retirement_committed: false, - }] - ); -} - #[test] fn portable_inventory_other_profiles_progress_while_one_writer_is_paused() { let tmp = tempfile::TempDir::new().unwrap(); @@ -397,11 +299,7 @@ fn portable_inventory_other_profiles_progress_while_one_writer_is_paused() { .expect("another profile must progress before the paused writer is released") .unwrap() .unwrap(); - assert!(matches!( - page.entries.as_slice(), - [super::unregistered_page::ProjectDirectoryWorkV1::Project(name)] - if name == "proj_independent" - )); + assert_eq!(page.entries, ["proj_independent"]); } /// Every platform uses an append-only durable inventory. A cancelled admission @@ -496,10 +394,7 @@ fn unregistered_inventory_restart_converges_without_repeating_records() { .unwrap(); let mut scanned = first.entries_scanned; let mut observed = std::collections::HashSet::new(); - for entry in first.entries { - let super::unregistered_page::ProjectDirectoryWorkV1::Project(name) = entry else { - panic!("unexpected quarantine") - }; + for name in first.entries { assert!(observed.insert(name)); } let saved = first.next_cursor.unwrap(); @@ -520,10 +415,7 @@ fn unregistered_inventory_restart_converges_without_repeating_records() { .unwrap() .unwrap(); scanned += page.entries_scanned; - for entry in page.entries { - let super::unregistered_page::ProjectDirectoryWorkV1::Project(name) = entry else { - panic!("unexpected quarantine") - }; + for name in page.entries { assert!(observed.insert(name), "restart repeated a committed record"); } cursor = page.next_cursor; @@ -858,51 +750,3 @@ fn payload_fence_finding(data_root: PathBuf, expected_store_relpath: &str) -> Or graph_scope_relpaths: Vec::new(), } } - -async fn prepare_registered_quarantine( - db: &RegisteredGlobalDb, - profile_root: &Path, - project_id: &str, - store_id: &str, - payload: &[u8], -) -> (PathBuf, PathBuf) { - let data_root = seed_store( - db, - profile_root, - project_id, - store_id, - &profile_root.join("missing-project-root"), - 1_700_000_000, - ) - .await; - std::fs::write(data_root.join("payload.bin"), payload).unwrap(); - let row = db - .try_list_store_instances_for_project(project_id) - .await - .unwrap() - .into_iter() - .find(|row| row.store_id == store_id) - .unwrap(); - let expected = capture_store_content_fence(profile_root, &data_root).unwrap(); - let quarantine = quarantine_store_for_verified_collection_controlled( - profile_root, - &data_root, - &expected, - QuarantineKindV1::Registered, - project_id, - store_id, - Some(QuarantineRegistryFenceV1 { - store_relpath: row.store_relpath, - created_at: row.created_at, - last_write_at: row.last_write_at, - }), - unbounded_collection_control(), - ) - .unwrap(); - let QuarantineStoreOutcome::Verified(quarantine) = quarantine else { - panic!("fixture must reach a verified registered quarantine"); - }; - let quarantine_path = quarantine.quarantine_path().to_path_buf(); - drop(quarantine); - (data_root, quarantine_path) -} diff --git a/crates/tracedecay-maintenance/src/retention/orphan_stores/tests/collection.rs b/crates/tracedecay-maintenance/src/retention/orphan_stores/tests/collection.rs new file mode 100644 index 0000000000..9c164f6b1e --- /dev/null +++ b/crates/tracedecay-maintenance/src/retention/orphan_stores/tests/collection.rs @@ -0,0 +1,872 @@ +use super::*; + +/// Seed a profile with one live store and one identity-drift orphan store, then +/// prove the async sweep collects only the orphan and retires its registry row. +#[cfg(unix)] +#[tokio::test] +async fn registered_collection_refuses_same_second_directory_replacement() { + let tmp = tempfile::TempDir::new().unwrap(); + let profile_root = tmp.path().join("profile"); + std::fs::create_dir_all(&profile_root).unwrap(); + let dead_root = tmp.path().join("moved-away-repo"); + let (_runtime, db) = open_registered_db(&profile_root).await; + let data_root = seed_store( + &db, + &profile_root, + "proj_replaced", + "store_replaced", + &dead_root, + 1_700_000_000 - 100 * DAY, + ) + .await; + + let census = build_store_census(&db, &profile_root).await.unwrap(); + let plan = plan_collection(classify_stores(&census, 1_700_000_000), 7 * DAY); + assert_eq!( + plan.collect.len(), + 1, + "fixture must be eligible before replacement" + ); + + let displaced = profile_root.join("displaced-store"); + std::fs::rename(&data_root, &displaced).unwrap(); + std::fs::create_dir_all(&data_root).unwrap(); + for name in [ + "graph.db", + tracedecay_runtime_core::storage::STORE_MANIFEST_FILENAME, + ] { + let source = displaced.join(name); + let target = data_root.join(name); + std::fs::copy(&source, &target).unwrap(); + let modified = + filetime::FileTime::from_system_time(source.metadata().unwrap().modified().unwrap()); + filetime::set_file_mtime(&target, modified).unwrap(); + } + let original_directory_time = + filetime::FileTime::from_system_time(displaced.metadata().unwrap().modified().unwrap()); + filetime::set_file_mtime(&data_root, original_directory_time).unwrap(); + + let (outcome, retired) = execute_registered_collection(&db, &plan, &profile_root) + .await + .unwrap(); + + assert_eq!(retired, 0); + assert!(outcome.collected.is_empty()); + assert_eq!( + outcome.errors, + vec![CollectionFailure { + store_id: "store_replaced".to_owned(), + kind: CollectionFailureKind::PayloadChanged, + }] + ); + assert!(data_root.exists(), "replacement directory must survive"); + assert_eq!( + db.try_list_store_instances_for_project("proj_replaced") + .await + .unwrap() + .len(), + 1, + "a rejected collection must leave the registry authority intact" + ); +} + +/// A profile-contained symlink is still not a store directory authority. The +/// collector must refuse it instead of deleting the link and retiring the +/// registry row while its target payload survives without an owner. +#[cfg(unix)] +#[tokio::test] +async fn registered_collection_rejects_profile_contained_data_root_symlink() { + let tmp = tempfile::TempDir::new().unwrap(); + let profile_root = tmp.path().join("profile"); + std::fs::create_dir_all(&profile_root).unwrap(); + let dead_root = tmp.path().join("moved-away-repo"); + let (_runtime, db) = open_registered_db(&profile_root).await; + let data_root = seed_store( + &db, + &profile_root, + "proj_symlinked_root", + "store_symlinked_root", + &dead_root, + 1_700_000_000 - 100 * DAY, + ) + .await; + + let census = build_store_census(&db, &profile_root).await.unwrap(); + let plan = plan_collection(classify_stores(&census, 1_700_000_000), 7 * DAY); + assert_eq!( + plan.collect.len(), + 1, + "fixture must be eligible before replacement" + ); + + let held_payload = profile_root.join("held-payload"); + std::fs::rename(&data_root, &held_payload).unwrap(); + std::os::unix::fs::symlink(&held_payload, &data_root).unwrap(); + + let (outcome, retired) = execute_registered_collection(&db, &plan, &profile_root) + .await + .unwrap(); + + assert_eq!(retired, 0); + assert!(outcome.collected.is_empty()); + assert_eq!( + outcome.errors, + vec![CollectionFailure { + store_id: "store_symlinked_root".to_owned(), + kind: CollectionFailureKind::OutsideProfile, + }] + ); + assert!(held_payload.exists(), "the payload target must survive"); + assert!( + data_root + .symlink_metadata() + .unwrap() + .file_type() + .is_symlink() + ); + assert_eq!( + db.try_list_store_instances_for_project("proj_symlinked_root") + .await + .unwrap() + .len(), + 1, + "a rejected collection must leave the registry authority intact" + ); +} + +#[tokio::test] +async fn relink_database_failure_rolls_back_manifest_and_registry() { + let tmp = tempfile::TempDir::new().unwrap(); + let profile_root = tmp.path().join("profile"); + std::fs::create_dir_all(&profile_root).unwrap(); + let dead_root = tmp.path().join("old-repository-root"); + let live_root = tmp.path().join("registered-live-root"); + std::fs::create_dir_all(&live_root).unwrap(); + let (_runtime, db) = open_registered_db(&profile_root).await; + let store_root = seed_store( + &db, + &profile_root, + "proj_old", + "store_moved", + &dead_root, + 1_700_000_000, + ) + .await; + seed_project(&db, "proj_live", &live_root, 1_700_000_000).await; + let mut manifest = tracedecay_runtime_core::storage::read_store_manifest( + &store_root.join(tracedecay_runtime_core::storage::STORE_MANIFEST_FILENAME), + ) + .unwrap(); + manifest.project_root = live_root; + std::fs::write( + store_root.join(tracedecay_runtime_core::storage::STORE_MANIFEST_FILENAME), + serde_json::to_string_pretty(&manifest).unwrap(), + ) + .unwrap(); + db.writer_connection() + .unwrap() + .execute_batch( + "CREATE TRIGGER reject_test_relink + BEFORE INSERT ON store_instances + WHEN NEW.project_id = 'proj_live' + BEGIN SELECT RAISE(ABORT, 'test relink rejection'); END;", + ) + .await + .unwrap(); + + assert!( + sweep_orphan_stores(&db, &profile_root, 7 * DAY, 1_700_000_000, true) + .await + .is_err() + ); + + let prior = db + .try_list_store_instances_for_project("proj_old") + .await + .unwrap(); + assert_eq!(prior.len(), 1); + assert_eq!(prior[0].store_id, "store_moved"); + assert!( + db.try_list_store_instances_for_project("proj_live") + .await + .unwrap() + .is_empty() + ); + let restored_manifest = tracedecay_runtime_core::storage::read_store_manifest( + &store_root.join(tracedecay_runtime_core::storage::STORE_MANIFEST_FILENAME), + ) + .unwrap(); + assert_eq!(restored_manifest.project_id.as_deref(), Some("proj_old")); +} + +#[tokio::test] +async fn durable_memory_rows_block_orphan_store_collection() { + let tmp = tempfile::TempDir::new().unwrap(); + let profile_root = tmp.path().join("profile"); + std::fs::create_dir_all(&profile_root).unwrap(); + let dead_root = tmp.path().join("moved-away-repo"); + let (_runtime, db) = open_registered_db(&profile_root).await; + let base = 1_700_000_000i64; + let data_root = seed_store( + &db, + &profile_root, + "proj_memory", + "store_memory", + &dead_root, + base - 100 * DAY, + ) + .await; + + { + let connection = rusqlite::Connection::open(data_root.join("graph.db")).unwrap(); + connection + .execute_batch( + "CREATE TABLE memory_facts (fact_id INTEGER PRIMARY KEY, content TEXT NOT NULL); + INSERT INTO memory_facts (fact_id, content) VALUES (1, 'durable fact');", + ) + .unwrap(); + } + + let report = sweep_orphan_stores(&db, &profile_root, 7 * DAY, base, true) + .await + .unwrap(); + + assert!( + report.outcome.collected.is_empty(), + "a store with durable memory rows must never be collected" + ); + assert_eq!(report.outcome.errors.len(), 1); + assert_eq!( + report.outcome.errors[0].kind, + CollectionFailureKind::DurableDataProtected + ); + assert!( + data_root.exists(), + "durable-memory-protected store must remain on disk" + ); + assert!( + db.list_code_projects(usize::MAX) + .await + .unwrap() + .into_iter() + .any(|project| project.project_id == "proj_memory"), + "registry row for a protected store must not be retired" + ); +} + +/// The guard is schema-discovered, so current and future Memory V2 tables are +/// protected without adding every table name to a second hand-maintained list. +#[tokio::test] +async fn memory_v2_rows_block_orphan_store_collection() { + let tmp = tempfile::TempDir::new().unwrap(); + let profile_root = tmp.path().join("profile"); + std::fs::create_dir_all(&profile_root).unwrap(); + let dead_root = tmp.path().join("moved-away-repo"); + let (_runtime, db) = open_registered_db(&profile_root).await; + let base = 1_700_000_000i64; + let data_root = seed_store( + &db, + &profile_root, + "proj_memory_v2", + "store_memory_v2", + &dead_root, + base - 100 * DAY, + ) + .await; + + { + let connection = rusqlite::Connection::open(data_root.join("graph.db")).unwrap(); + connection + .execute_batch( + "CREATE TABLE memory_v2_assertions ( + assertion_id TEXT PRIMARY KEY, + payload TEXT NOT NULL + ); + INSERT INTO memory_v2_assertions (assertion_id, payload) + VALUES ('assertion-1', 'durable v2 fact');", + ) + .unwrap(); + } + + let report = sweep_orphan_stores(&db, &profile_root, 7 * DAY, base, true) + .await + .unwrap(); + + assert!(report.outcome.collected.is_empty()); + assert_eq!( + report.outcome.errors[0].kind, + CollectionFailureKind::DurableDataProtected + ); + assert!(data_root.exists()); +} + +/// A durable memory table that exists but is empty must not block collection. +/// Only an actual row does. +#[tokio::test] +async fn empty_memory_table_does_not_block_collection() { + let tmp = tempfile::TempDir::new().unwrap(); + let profile_root = tmp.path().join("profile"); + std::fs::create_dir_all(&profile_root).unwrap(); + let dead_root = tmp.path().join("moved-away-repo"); + let (_runtime, db) = open_registered_db(&profile_root).await; + let base = 1_700_000_000i64; + let data_root = seed_store( + &db, + &profile_root, + "proj_empty_memory", + "store_empty_memory", + &dead_root, + base - 100 * DAY, + ) + .await; + { + let connection = rusqlite::Connection::open(data_root.join("graph.db")).unwrap(); + connection + .execute_batch( + "CREATE TABLE memory_facts ( + fact_id INTEGER PRIMARY KEY, + content TEXT NOT NULL + ); + CREATE VIRTUAL TABLE memory_facts_fts USING fts5(content);", + ) + .unwrap(); + } + + let report = sweep_orphan_stores(&db, &profile_root, 7 * DAY, base, true) + .await + .unwrap(); + + assert_eq!( + report.outcome.collected.len(), + 1, + "empty durable-memory tables must not block collection: {report:#?}" + ); + assert!(!data_root.exists()); +} + +// === Unregistered store directories ========================================= + +#[cfg(unix)] +#[tokio::test] +async fn unregistered_collection_refuses_same_second_directory_replacement() { + let tmp = tempfile::TempDir::new().unwrap(); + let profile_root = tmp.path().join("profile"); + std::fs::create_dir_all(&profile_root).unwrap(); + let (_runtime, db) = open_registered_db(&profile_root).await; + let data_root = profile_root.join("projects/proj_replaced_unregistered"); + std::fs::create_dir_all(&data_root).unwrap(); + + let now = walk_store_stats(&data_root) + .newest_mtime_secs + .saturating_add(100 * DAY); + let findings = census_unregistered_project_dirs(&db, &profile_root, now) + .await + .unwrap(); + let plan = plan_unregistered_collection(findings, 7 * DAY); + assert_eq!( + plan.collect.len(), + 1, + "fixture must be eligible before replacement" + ); + + let displaced = profile_root.join("displaced-unregistered-store"); + let original_time = + filetime::FileTime::from_system_time(data_root.metadata().unwrap().modified().unwrap()); + std::fs::rename(&data_root, &displaced).unwrap(); + std::fs::create_dir_all(&data_root).unwrap(); + filetime::set_file_mtime(&data_root, original_time).unwrap(); + + let outcome = execute_unregistered_collection(&db, &plan, &profile_root) + .await + .unwrap(); + + assert!(outcome.collected.is_empty()); + assert_eq!( + outcome.errors, + vec![CollectionFailure { + store_id: "proj_replaced_unregistered".to_owned(), + kind: CollectionFailureKind::PayloadChanged, + }] + ); + assert!(data_root.exists(), "replacement directory must survive"); +} + +/// An unregistered leaf must not become a deletion target merely because its +/// symlink resolves back inside the profile. The physical `/projects` +/// path, rather than canonicalized containment, is the destructive authority. +#[cfg(unix)] +#[tokio::test] +async fn unregistered_collection_rejects_profile_contained_data_root_symlink() { + let tmp = tempfile::TempDir::new().unwrap(); + let profile_root = tmp.path().join("profile"); + std::fs::create_dir_all(&profile_root).unwrap(); + let (_runtime, db) = open_registered_db(&profile_root).await; + let data_root = profile_root.join("projects/proj_symlinked_unregistered"); + std::fs::create_dir_all(&data_root).unwrap(); + + let now = walk_store_stats(&data_root) + .newest_mtime_secs + .saturating_add(100 * DAY); + let findings = census_unregistered_project_dirs(&db, &profile_root, now) + .await + .unwrap(); + let plan = plan_unregistered_collection(findings, 7 * DAY); + assert_eq!( + plan.collect.len(), + 1, + "fixture must be eligible before the symlink swap" + ); + + let held_payload = profile_root.join("held-unregistered-payload"); + std::fs::rename(&data_root, &held_payload).unwrap(); + std::os::unix::fs::symlink(&held_payload, &data_root).unwrap(); + + let outcome = execute_unregistered_collection(&db, &plan, &profile_root) + .await + .unwrap(); + + assert!(outcome.collected.is_empty()); + assert_eq!( + outcome.errors, + vec![CollectionFailure { + store_id: "proj_symlinked_unregistered".to_owned(), + kind: CollectionFailureKind::OutsideProfile, + }] + ); + assert!( + held_payload.is_dir(), + "the in-profile symlink target must survive" + ); +} + +/// `SQLite` pages can change in place without changing the parent directory. +/// Hashing the opened child handles makes that post-census mutation visible at +/// the delete boundary even when the writer resets the database mtime. +#[test] +fn delete_boundary_refuses_same_second_sqlite_mutation() { + let tmp = tempfile::TempDir::new().unwrap(); + let profile_root = tmp.path().join("profile"); + let data_root = profile_root.join("stores/sqlite-race"); + std::fs::create_dir_all(&data_root).unwrap(); + let database = data_root.join("graph.db"); + let connection = rusqlite::Connection::open(&database).unwrap(); + connection + .execute_batch("CREATE TABLE facts (value TEXT NOT NULL);") + .unwrap(); + drop(connection); + let expected = capture_store_content_fence(&profile_root, &data_root).unwrap(); + let original_time = + filetime::FileTime::from_system_time(database.metadata().unwrap().modified().unwrap()); + + let connection = rusqlite::Connection::open(&database).unwrap(); + connection + .execute("INSERT INTO facts (value) VALUES ('post-census')", []) + .unwrap(); + drop(connection); + filetime::set_file_mtime(&database, original_time).unwrap(); + + let result = open_verified_store( + &profile_root, + &data_root, + &expected, + unbounded_collection_control(), + ); + assert!(matches!(result, Err(CollectionFailureKind::PayloadChanged))); + let connection = rusqlite::Connection::open(&database).unwrap(); + let rows: i64 = connection + .query_row("SELECT COUNT(*) FROM facts", [], |row| row.get(0)) + .unwrap(); + assert_eq!(rows, 1, "mutated SQLite bytes must survive"); +} + +/// Even an empty replacement is not the inspected directory. Its child list +/// is identical, so the opened root's stable identity must participate in the +/// comparison before collection can remove anything. +#[test] +fn delete_boundary_refuses_empty_directory_replacement() { + let tmp = tempfile::TempDir::new().unwrap(); + let profile_root = tmp.path().join("profile"); + let data_root = profile_root.join("stores/empty-rename-race"); + std::fs::create_dir_all(&data_root).unwrap(); + let expected = capture_store_content_fence(&profile_root, &data_root).unwrap(); + + let displaced = profile_root.join("stores/displaced-empty"); + std::fs::rename(&data_root, &displaced).unwrap(); + std::fs::create_dir_all(&data_root).unwrap(); + + let result = open_verified_store( + &profile_root, + &data_root, + &expected, + unbounded_collection_control(), + ); + + assert!(matches!(result, Err(CollectionFailureKind::PayloadChanged))); + assert!(data_root.is_dir(), "fresh empty replacement must survive"); + assert!(displaced.is_dir(), "inspected empty directory must survive"); +} + +/// Cancellation is checked before any recursive SHA-256 read. A cancelled +/// maintenance admission cannot turn a deep inventory into a partial plan or +/// an implicit deletion permit. +#[tokio::test] +async fn registered_collection_payload_fence_cancellation_is_terminal() { + let tmp = tempfile::TempDir::new().unwrap(); + let profile_root = tmp.path().join("profile"); + let data_root = profile_root.join("stores/payload-fence-cancelled"); + seed_payload_fence_work(&data_root); + let (_runtime, db) = open_registered_db(&profile_root).await; + let finding = payload_fence_finding(data_root.clone(), "stores/payload-fence-cancelled"); + let plan = CollectionPlan { + collect: vec![finding], + ..CollectionPlan::default() + }; + let cancellation = CancellationToken::new(); + let started = std::sync::Arc::new(AtomicBool::new(false)); + let started_thread = std::sync::Arc::clone(&started); + let cancellation_thread = cancellation.clone(); + let signal = std::thread::spawn(move || { + while !started_thread.load(Ordering::Acquire) { + std::thread::yield_now(); + } + std::thread::sleep(Duration::from_millis(20)); + cancellation_thread.cancel(); + }); + started.store(true, Ordering::Release); + + let (outcome, retired) = execute_registered_collection_controlled( + &db, + &plan, + &profile_root, + CollectionControl::new( + &cancellation, + MonotonicDeadline::at(Instant::now() + Duration::from_secs(5)), + ), + ) + .await + .unwrap(); + signal.join().unwrap(); + + assert_eq!(retired, 0); + assert_eq!(outcome.completion, CollectionCompletionV1::Cancelled); + assert!(outcome.errors.is_empty()); + assert!(outcome.collected.is_empty()); + assert!(data_root.exists()); +} + +#[tokio::test] +async fn unregistered_collection_payload_fence_deadline_is_distinct() { + let tmp = tempfile::TempDir::new().unwrap(); + let profile_root = tmp.path().join("profile"); + let data_root = profile_root.join("projects/proj_payload_fence_deadline"); + seed_payload_fence_work(&data_root); + let (_runtime, db) = open_registered_db(&profile_root).await; + let finding = payload_fence_finding(data_root.clone(), "projects/proj_payload_fence_deadline"); + let plan = UnregisteredCollectionPlan { + collect: vec![UnregisteredStoreFinding { + project_dir_name: "proj_payload_fence_deadline".to_owned(), + data_root: finding.data_root, + age_secs: finding.age_secs, + size_bytes: finding.size_bytes, + expected_payload_mtime_secs: finding.expected_payload_mtime_secs, + expected_data_root_fence: finding.expected_data_root_fence, + expected_content_fence: finding.expected_content_fence, + abandoned_root: false, + }], + ..UnregisteredCollectionPlan::default() + }; + + let cancellation = CancellationToken::new(); + let (outcome, deadline) = { + let deadline = MonotonicDeadline::at(Instant::now() + Duration::from_millis(20)); + let outcome = execute_unregistered_collection_controlled( + &db, + &plan, + &profile_root, + CollectionControl::new(&cancellation, deadline), + ) + .await + .unwrap(); + (outcome, deadline) + }; + + assert!(deadline.is_elapsed_at(Instant::now())); + assert_eq!(outcome.completion, CollectionCompletionV1::DeadlineExceeded); + assert!(outcome.errors.is_empty()); + assert!(outcome.collected.is_empty()); + assert!(data_root.exists()); +} + +/// A durable-memory guard applies to unregistered directories exactly as it +/// does to registered orphan stores. +#[tokio::test] +async fn sweep_unregistered_stores_never_deletes_durable_memory_rows() { + let tmp = tempfile::TempDir::new().unwrap(); + let profile_root = tmp.path().join("profile"); + std::fs::create_dir_all(&profile_root).unwrap(); + let (_runtime, db) = open_registered_db(&profile_root).await; + + let base = 1_700_000_000i64; + let dir = profile_root.join("projects").join("proj_ghost_with_memory"); + std::fs::create_dir_all(&dir).unwrap(); + { + let connection = + rusqlite::Connection::open(dir.join(tracedecay_runtime_core::config::DB_FILENAME)) + .unwrap(); + connection + .execute_batch( + "CREATE TABLE memory_facts (fact_id INTEGER PRIMARY KEY, content TEXT NOT NULL); + INSERT INTO memory_facts (fact_id, content) VALUES (1, 'durable fact');", + ) + .unwrap(); + } + filetime::set_file_mtime( + dir.join(tracedecay_runtime_core::config::DB_FILENAME), + filetime::FileTime::from_unix_time(base - 100 * DAY, 0), + ) + .unwrap(); + + let report = sweep_unregistered_stores(&db, &profile_root, 7 * DAY, base, true) + .await + .unwrap(); + assert!(report.outcome.collected.is_empty()); + assert_eq!( + report.outcome.errors[0].kind, + CollectionFailureKind::DurableDataProtected + ); + assert!(dir.exists()); +} + +/// The durable-data check covers the manifest-selected project graph and every +/// registered project graph scope, and refuses to answer when the manifest +/// that names them cannot be read. +mod durable_inventory { + use super::*; + + fn manifest_bytes(graph_db_relpath: &str) -> Vec { + let project_root = PathBuf::from("/definitely/not/here/gone"); + let manifest = StoreManifest { + schema_version: STORE_MANIFEST_SCHEMA_VERSION, + project_id: Some("proj_inventory".to_string()), + store_kind: StoreKind::CodeProject, + storage_mode: StorageMode::ProfileSharded, + project_root: project_root.clone(), + data_root: project_root, + graph_db_relpath: PathBuf::from(graph_db_relpath), + sessions_db_relpath: PathBuf::from("sessions.db"), + branch_meta_relpath: PathBuf::from( + tracedecay_runtime_core::storage::BRANCH_META_FILENAME, + ), + }; + serde_json::to_vec(&manifest).unwrap() + } + + #[test] + fn registered_graph_scopes_at_custom_paths_are_covered() { + let custom = PathBuf::from("scopes/custom-scope.db"); + + let DurableDatabaseInventoryV1::Resolved(inventory) = durable_database_inventory( + Some(&manifest_bytes("custom-main.db")), + std::slice::from_ref(&custom), + unbounded_collection_control(), + ) else { + panic!("a readable manifest must resolve an inventory"); + }; + + assert_eq!( + inventory, + [PathBuf::from("custom-main.db"), custom], + "the manifest's custom main graph path must be honoured, not the default filename" + ); + } + + #[test] + fn a_missing_manifest_fails_closed() { + assert_eq!( + durable_database_inventory(None, &[], unbounded_collection_control()), + DurableDatabaseInventoryV1::Unverifiable, + "without a manifest the store's graph path is a guess, not a fact" + ); + } + + #[test] + fn manifest_graph_path_must_be_normalized_relative() { + for graph_path in [PathBuf::from(""), PathBuf::from("../graph.db")] { + let bytes = manifest_bytes(graph_path.to_string_lossy().as_ref()); + assert_eq!( + durable_database_inventory(Some(&bytes), &[], unbounded_collection_control()), + DurableDatabaseInventoryV1::Unverifiable, + "graph path {graph_path:?} must not escape the store" + ); + } + + assert_eq!( + durable_database_inventory( + Some(&manifest_bytes("/tmp/graph.db")), + &[], + unbounded_collection_control(), + ), + DurableDatabaseInventoryV1::Unverifiable, + "an absolute graph path must not replace the store root" + ); + } + + #[test] + fn registered_graph_scope_path_must_be_normalized_relative() { + assert_eq!( + durable_database_inventory( + Some(&manifest_bytes("graph.db")), + &[PathBuf::from("scopes/../../escape.db")], + unbounded_collection_control(), + ), + DurableDatabaseInventoryV1::Unverifiable + ); + assert_eq!( + durable_database_inventory( + Some(&manifest_bytes("graph.db")), + &[PathBuf::from("/tmp/escape.db")], + unbounded_collection_control(), + ), + DurableDatabaseInventoryV1::Unverifiable + ); + } + + #[test] + fn cancelled_control_interrupts_the_inventory() { + let cancellation = CancellationToken::new(); + cancellation.cancel(); + + assert_eq!( + durable_database_inventory( + Some(&manifest_bytes("graph.db")), + &[], + CollectionControl::new( + &cancellation, + MonotonicDeadline::at(Instant::now() + Duration::from_secs(1)), + ), + ), + DurableDatabaseInventoryV1::Interrupted, + "a cancelled admission must not resolve an inventory" + ); + } + + #[tokio::test] + async fn a_store_whose_manifest_is_unreadable_is_never_reported_empty() { + let profile = tempfile::tempdir().unwrap(); + let data_root = profile.path().join("stores/unreadable"); + std::fs::create_dir_all(&data_root).unwrap(); + + let check = check_store_durable_memory( + &data_root, + Some(b"{ not json"), + &[], + &durable_check_scratch_root(profile.path()), + unbounded_collection_control(), + ) + .await; + + assert_eq!( + check, + DurableMemoryCheck::Unverifiable, + "an unverifiable inventory must protect the store, never clear it for deletion" + ); + } + + #[tokio::test] + async fn a_pre_cancelled_durable_snapshot_reports_interrupted() { + let profile = tempfile::tempdir().unwrap(); + let data_root = profile.path().join("stores/cancelled"); + std::fs::create_dir_all(&data_root).unwrap(); + rusqlite::Connection::open(data_root.join("graph.db")).unwrap(); + + let cancellation = CancellationToken::new(); + cancellation.cancel(); + let check = check_store_durable_memory( + &data_root, + Some(&manifest_bytes("graph.db")), + &[], + &durable_check_scratch_root(profile.path()), + CollectionControl::new( + &cancellation, + MonotonicDeadline::at(Instant::now() + Duration::from_secs(1)), + ), + ) + .await; + + assert_eq!( + check, + DurableMemoryCheck::Interrupted, + "a pre-cancelled durable snapshot must not report an empty database" + ); + } +} + +#[cfg(unix)] +#[tokio::test] +async fn symlink_manifest_is_unverifiable_and_never_collected() { + let tmp = tempfile::TempDir::new().unwrap(); + let profile_root = tmp.path().join("profile"); + std::fs::create_dir_all(&profile_root).unwrap(); + let dead_root = tmp.path().join("moved-away-repo"); + let (_runtime, db) = open_registered_db(&profile_root).await; + let data_root = seed_store( + &db, + &profile_root, + "proj_symlink_manifest", + "store_symlink_manifest", + &dead_root, + 1_700_000_000 - 100 * DAY, + ) + .await; + let manifest_path = data_root.join(tracedecay_runtime_core::storage::STORE_MANIFEST_FILENAME); + let target = tmp.path().join("manifest-target.json"); + std::fs::copy(&manifest_path, &target).unwrap(); + std::fs::remove_file(&manifest_path).unwrap(); + std::os::unix::fs::symlink(&target, &manifest_path).unwrap(); + + let report = sweep_orphan_stores(&db, &profile_root, 7 * DAY, 1_700_000_000, true) + .await + .unwrap(); + + assert!(report.plan.collect.is_empty()); + assert_eq!(report.plan.unverifiable.len(), 1); + assert!(report.outcome.collected.is_empty()); + assert!(data_root.exists()); +} + +#[cfg(unix)] +#[tokio::test] +async fn symlink_graph_database_is_durable_data_protected() { + let tmp = tempfile::TempDir::new().unwrap(); + let profile_root = tmp.path().join("profile"); + std::fs::create_dir_all(&profile_root).unwrap(); + let dead_root = tmp.path().join("moved-away-repo"); + let (_runtime, db) = open_registered_db(&profile_root).await; + let data_root = seed_store( + &db, + &profile_root, + "proj_symlink_graph", + "store_symlink_graph", + &dead_root, + 1_700_000_000 - 100 * DAY, + ) + .await; + let graph_path = data_root.join("graph.db"); + let target = tmp.path().join("graph-target.db"); + rusqlite::Connection::open(&target).unwrap(); + std::fs::remove_file(&graph_path).unwrap(); + std::os::unix::fs::symlink(&target, &graph_path).unwrap(); + + let report = sweep_orphan_stores(&db, &profile_root, 7 * DAY, 1_700_000_000, true) + .await + .unwrap(); + + assert_eq!(report.plan.collect.len(), 1); + assert!(report.outcome.collected.is_empty()); + assert_eq!(report.outcome.errors.len(), 1); + assert_eq!( + report.outcome.errors[0].kind, + CollectionFailureKind::DurableDataProtected + ); + assert!(data_root.exists()); +} diff --git a/crates/tracedecay-maintenance/src/retention/orphan_stores/tests/pages.rs b/crates/tracedecay-maintenance/src/retention/orphan_stores/tests/pages.rs index e5e9602bd1..0ce2dd3822 100644 --- a/crates/tracedecay-maintenance/src/retention/orphan_stores/tests/pages.rs +++ b/crates/tracedecay-maintenance/src/retention/orphan_stores/tests/pages.rs @@ -885,171 +885,3 @@ async fn unregistered_store_sweep_returns_deadline_without_mutation() { assert!(report.outcome.collected.is_empty()); assert!(profile_root.join("projects/proj_deadline").is_dir()); } - -/// The mounted unregistered pager recognizes a durable quarantine even though -/// it is not a valid `project_id` leaf. It restores the bytes, emits a typed -/// receipt, and deliberately defers deletion until a later fresh census. -#[tokio::test] -async fn unregistered_store_sweep_reconciles_interrupted_quarantine() { - let tmp = tempfile::TempDir::new().unwrap(); - let profile_root = tmp.path().join("profile"); - let projects = profile_root.join("projects"); - std::fs::create_dir_all(&projects).unwrap(); - let quarantine = projects.join(".tracedecay-orphan-quarantine-proj_paged_recovery-42-7"); - std::fs::create_dir_all(&quarantine).unwrap(); - std::fs::write(quarantine.join("payload.bin"), b"recover through pager").unwrap(); - let (_runtime, db) = open_registered_db(&profile_root).await; - let cancellation = CancellationToken::new(); - - let report = sweep_unregistered_store_page( - &db, - &profile_root, - UnregisteredStoreSweepRequestV1 { - cursor: None, - limit: 1, - retention_secs: 0, - now: 1_700_000_000, - apply: true, - cancellation: &cancellation, - deadline: MonotonicDeadline::at(Instant::now() + Duration::from_secs(1)), - }, - ) - .await - .unwrap(); - - let restored = projects.join("proj_paged_recovery"); - assert_eq!(report.completion, UnregisteredSweepCompletionV1::Complete); - assert!(report.outcome.collected.is_empty()); - assert_eq!(report.outcome.recovery_receipts.len(), 1); - assert_eq!( - std::fs::read(restored.join("payload.bin")).unwrap(), - b"recover through pager" - ); - assert!(!quarantine.exists()); -} - -#[tokio::test] -async fn unregistered_store_sweep_reports_failed_legacy_restore() { - let tmp = tempfile::TempDir::new().unwrap(); - let profile_root = tmp.path().join("profile"); - let projects = profile_root.join("projects"); - let data_root = projects.join("proj_paged_retained"); - let quarantine = projects.join(".tracedecay-orphan-quarantine-proj_paged_retained-42-7"); - std::fs::create_dir_all(&data_root).unwrap(); - std::fs::write(data_root.join("payload.bin"), b"new live bytes").unwrap(); - std::fs::create_dir_all(&quarantine).unwrap(); - std::fs::write(quarantine.join("payload.bin"), b"legacy quarantine bytes").unwrap(); - let expected = capture_store_content_fence(&profile_root, &quarantine).unwrap(); - let StoreContentFence::Present(inventory) = expected else { - panic!("fixture must capture the legacy quarantine identity"); - }; - let expected_root_identity = inventory.root; - let (_runtime, db) = open_registered_db(&profile_root).await; - let cancellation = CancellationToken::new(); - - let report = sweep_unregistered_store_page( - &db, - &profile_root, - UnregisteredStoreSweepRequestV1 { - cursor: None, - limit: 2, - retention_secs: 0, - now: 1_700_000_000, - apply: true, - cancellation: &cancellation, - deadline: functional_sweep_deadline(), - }, - ) - .await - .unwrap(); - - let failure = CollectionMutationFailure { - operation: CollectionMutationOperation::RestoreLiveLeafFromQuarantine, - raw_os_error: Some(OCCUPIED_RENAME_RAW_OS_ERROR), - target_path: data_root.clone(), - expected_root_identity: Some(expected_root_identity), - classification: CollectionMutationFailureClassification::NonRetryable, - }; - assert_eq!( - report.outcome.errors, - vec![ - CollectionFailure { - store_id: "proj_paged_retained".to_owned(), - kind: CollectionFailureKind::RemoveFailed(failure), - }, - CollectionFailure { - store_id: "proj_paged_retained".to_owned(), - kind: CollectionFailureKind::PayloadChanged, - }, - ] - ); - assert_eq!( - report.outcome.recovery_receipts, - vec![CollectionRecoveryReceipt { - store_id: "proj_paged_retained".to_owned(), - original_path: data_root.clone(), - quarantine_path: quarantine.clone(), - actual_path: quarantine.clone(), - action: CollectionRecoveryAction::RetainedForRecovery, - }] - ); - assert_eq!( - std::fs::read(data_root.join("payload.bin")).unwrap(), - b"new live bytes" - ); - assert_eq!( - std::fs::read(quarantine.join("payload.bin")).unwrap(), - b"legacy quarantine bytes" - ); -} - -/// Expiry before the quarantine entry is processed is not a fabricated -/// recovery failure: `DeadlineExceeded` carries the empty accumulated list. -#[tokio::test] -async fn unregistered_store_sweep_elapsed_deadline_reports_empty_legacy_restore_failures() { - let tmp = tempfile::TempDir::new().unwrap(); - let profile_root = tmp.path().join("profile"); - let projects = profile_root.join("projects"); - let data_root = projects.join("proj_paged_retained"); - let quarantine = projects.join(".tracedecay-orphan-quarantine-proj_paged_retained-42-7"); - std::fs::create_dir_all(&data_root).unwrap(); - std::fs::write(data_root.join("payload.bin"), b"new live bytes").unwrap(); - std::fs::create_dir_all(&quarantine).unwrap(); - std::fs::write(quarantine.join("payload.bin"), b"legacy quarantine bytes").unwrap(); - let (_runtime, db) = open_registered_db(&profile_root).await; - let cancellation = CancellationToken::new(); - - let report = sweep_unregistered_store_page( - &db, - &profile_root, - UnregisteredStoreSweepRequestV1 { - cursor: None, - limit: 2, - retention_secs: 0, - now: 1_700_000_000, - apply: true, - cancellation: &cancellation, - deadline: MonotonicDeadline::at(Instant::now()), - }, - ) - .await - .unwrap(); - - assert_eq!( - report.completion, - UnregisteredSweepCompletionV1::DeadlineExceeded - ); - assert!( - report.outcome.errors.is_empty(), - "interruption before the quarantine entry must not fabricate restore failures" - ); - assert!(report.outcome.recovery_receipts.is_empty()); - assert_eq!( - std::fs::read(data_root.join("payload.bin")).unwrap(), - b"new live bytes" - ); - assert_eq!( - std::fs::read(quarantine.join("payload.bin")).unwrap(), - b"legacy quarantine bytes" - ); -} diff --git a/crates/tracedecay-maintenance/src/retention/orphan_stores/tests/quarantine.rs b/crates/tracedecay-maintenance/src/retention/orphan_stores/tests/quarantine.rs deleted file mode 100644 index d24530082c..0000000000 --- a/crates/tracedecay-maintenance/src/retention/orphan_stores/tests/quarantine.rs +++ /dev/null @@ -1,2308 +0,0 @@ -use super::*; - -/// Seed a profile with one live store and one identity-drift orphan store, then -/// prove the async sweep collects only the orphan and retires its registry row. -#[cfg(unix)] -#[tokio::test] -async fn registered_collection_refuses_same_second_directory_replacement() { - let tmp = tempfile::TempDir::new().unwrap(); - let profile_root = tmp.path().join("profile"); - std::fs::create_dir_all(&profile_root).unwrap(); - let dead_root = tmp.path().join("moved-away-repo"); - let (_runtime, db) = open_registered_db(&profile_root).await; - let data_root = seed_store( - &db, - &profile_root, - "proj_replaced", - "store_replaced", - &dead_root, - 1_700_000_000 - 100 * DAY, - ) - .await; - - let census = build_store_census(&db, &profile_root).await.unwrap(); - let plan = plan_collection(classify_stores(&census, 1_700_000_000), 7 * DAY); - assert_eq!( - plan.collect.len(), - 1, - "fixture must be eligible before replacement" - ); - - let displaced = profile_root.join("displaced-store"); - std::fs::rename(&data_root, &displaced).unwrap(); - std::fs::create_dir_all(&data_root).unwrap(); - for name in [ - "graph.db", - tracedecay_runtime_core::storage::STORE_MANIFEST_FILENAME, - ] { - let source = displaced.join(name); - let target = data_root.join(name); - std::fs::copy(&source, &target).unwrap(); - let modified = - filetime::FileTime::from_system_time(source.metadata().unwrap().modified().unwrap()); - filetime::set_file_mtime(&target, modified).unwrap(); - } - let original_directory_time = - filetime::FileTime::from_system_time(displaced.metadata().unwrap().modified().unwrap()); - filetime::set_file_mtime(&data_root, original_directory_time).unwrap(); - - let (outcome, retired) = execute_registered_collection(&db, &plan, &profile_root) - .await - .unwrap(); - - assert_eq!(retired, 0); - assert!(outcome.collected.is_empty()); - assert_eq!( - outcome.errors, - vec![CollectionFailure { - store_id: "store_replaced".to_owned(), - kind: CollectionFailureKind::PayloadChanged, - }] - ); - assert!(data_root.exists(), "replacement directory must survive"); - assert_eq!( - db.try_list_store_instances_for_project("proj_replaced") - .await - .unwrap() - .len(), - 1, - "a rejected collection must leave the registry authority intact" - ); -} - -/// A profile-contained symlink is still not a store directory authority. The -/// collector must refuse it instead of deleting the link and retiring the -/// registry row while its target payload survives without an owner. -#[cfg(unix)] -#[tokio::test] -async fn registered_collection_rejects_profile_contained_data_root_symlink() { - let tmp = tempfile::TempDir::new().unwrap(); - let profile_root = tmp.path().join("profile"); - std::fs::create_dir_all(&profile_root).unwrap(); - let dead_root = tmp.path().join("moved-away-repo"); - let (_runtime, db) = open_registered_db(&profile_root).await; - let data_root = seed_store( - &db, - &profile_root, - "proj_symlinked_root", - "store_symlinked_root", - &dead_root, - 1_700_000_000 - 100 * DAY, - ) - .await; - - let census = build_store_census(&db, &profile_root).await.unwrap(); - let plan = plan_collection(classify_stores(&census, 1_700_000_000), 7 * DAY); - assert_eq!( - plan.collect.len(), - 1, - "fixture must be eligible before replacement" - ); - - let held_payload = profile_root.join("held-payload"); - std::fs::rename(&data_root, &held_payload).unwrap(); - std::os::unix::fs::symlink(&held_payload, &data_root).unwrap(); - - let (outcome, retired) = execute_registered_collection(&db, &plan, &profile_root) - .await - .unwrap(); - - assert_eq!(retired, 0); - assert!(outcome.collected.is_empty()); - assert_eq!( - outcome.errors, - vec![CollectionFailure { - store_id: "store_symlinked_root".to_owned(), - kind: CollectionFailureKind::OutsideProfile, - }] - ); - assert!(held_payload.exists(), "the payload target must survive"); - assert!( - data_root - .symlink_metadata() - .unwrap() - .file_type() - .is_symlink() - ); - assert_eq!( - db.try_list_store_instances_for_project("proj_symlinked_root") - .await - .unwrap() - .len(), - 1, - "a rejected collection must leave the registry authority intact" - ); -} - -#[tokio::test] -async fn relink_database_failure_rolls_back_manifest_and_registry() { - let tmp = tempfile::TempDir::new().unwrap(); - let profile_root = tmp.path().join("profile"); - std::fs::create_dir_all(&profile_root).unwrap(); - let dead_root = tmp.path().join("old-repository-root"); - let live_root = tmp.path().join("registered-live-root"); - std::fs::create_dir_all(&live_root).unwrap(); - let (_runtime, db) = open_registered_db(&profile_root).await; - let store_root = seed_store( - &db, - &profile_root, - "proj_old", - "store_moved", - &dead_root, - 1_700_000_000, - ) - .await; - seed_project(&db, "proj_live", &live_root, 1_700_000_000).await; - let mut manifest = tracedecay_runtime_core::storage::read_store_manifest( - &store_root.join(tracedecay_runtime_core::storage::STORE_MANIFEST_FILENAME), - ) - .unwrap(); - manifest.project_root = live_root; - std::fs::write( - store_root.join(tracedecay_runtime_core::storage::STORE_MANIFEST_FILENAME), - serde_json::to_string_pretty(&manifest).unwrap(), - ) - .unwrap(); - db.writer_connection() - .unwrap() - .execute_batch( - "CREATE TRIGGER reject_test_relink - BEFORE INSERT ON store_instances - WHEN NEW.project_id = 'proj_live' - BEGIN SELECT RAISE(ABORT, 'test relink rejection'); END;", - ) - .await - .unwrap(); - - assert!( - sweep_orphan_stores(&db, &profile_root, 7 * DAY, 1_700_000_000, true) - .await - .is_err() - ); - - let prior = db - .try_list_store_instances_for_project("proj_old") - .await - .unwrap(); - assert_eq!(prior.len(), 1); - assert_eq!(prior[0].store_id, "store_moved"); - assert!( - db.try_list_store_instances_for_project("proj_live") - .await - .unwrap() - .is_empty() - ); - let restored_manifest = tracedecay_runtime_core::storage::read_store_manifest( - &store_root.join(tracedecay_runtime_core::storage::STORE_MANIFEST_FILENAME), - ) - .unwrap(); - assert_eq!(restored_manifest.project_id.as_deref(), Some("proj_old")); -} - -#[tokio::test] -async fn durable_memory_rows_block_orphan_store_collection() { - let tmp = tempfile::TempDir::new().unwrap(); - let profile_root = tmp.path().join("profile"); - std::fs::create_dir_all(&profile_root).unwrap(); - let dead_root = tmp.path().join("moved-away-repo"); - let (_runtime, db) = open_registered_db(&profile_root).await; - let base = 1_700_000_000i64; - let data_root = seed_store( - &db, - &profile_root, - "proj_memory", - "store_memory", - &dead_root, - base - 100 * DAY, - ) - .await; - - { - let connection = rusqlite::Connection::open(data_root.join("graph.db")).unwrap(); - connection - .execute_batch( - "CREATE TABLE memory_facts (fact_id INTEGER PRIMARY KEY, content TEXT NOT NULL); - INSERT INTO memory_facts (fact_id, content) VALUES (1, 'durable fact');", - ) - .unwrap(); - } - - let report = sweep_orphan_stores(&db, &profile_root, 7 * DAY, base, true) - .await - .unwrap(); - - assert!( - report.outcome.collected.is_empty(), - "a store with durable memory rows must never be collected" - ); - assert_eq!(report.outcome.errors.len(), 1); - assert_eq!( - report.outcome.errors[0].kind, - CollectionFailureKind::DurableDataProtected - ); - assert!( - data_root.exists(), - "durable-memory-protected store must remain on disk" - ); - assert!( - db.list_code_projects(usize::MAX) - .await - .unwrap() - .into_iter() - .any(|project| project.project_id == "proj_memory"), - "registry row for a protected store must not be retired" - ); -} - -/// The guard is schema-discovered, so current and future Memory V2 tables are -/// protected without adding every table name to a second hand-maintained list. -#[tokio::test] -async fn memory_v2_rows_block_orphan_store_collection() { - let tmp = tempfile::TempDir::new().unwrap(); - let profile_root = tmp.path().join("profile"); - std::fs::create_dir_all(&profile_root).unwrap(); - let dead_root = tmp.path().join("moved-away-repo"); - let (_runtime, db) = open_registered_db(&profile_root).await; - let base = 1_700_000_000i64; - let data_root = seed_store( - &db, - &profile_root, - "proj_memory_v2", - "store_memory_v2", - &dead_root, - base - 100 * DAY, - ) - .await; - - { - let connection = rusqlite::Connection::open(data_root.join("graph.db")).unwrap(); - connection - .execute_batch( - "CREATE TABLE memory_v2_assertions ( - assertion_id TEXT PRIMARY KEY, - payload TEXT NOT NULL - ); - INSERT INTO memory_v2_assertions (assertion_id, payload) - VALUES ('assertion-1', 'durable v2 fact');", - ) - .unwrap(); - } - - let report = sweep_orphan_stores(&db, &profile_root, 7 * DAY, base, true) - .await - .unwrap(); - - assert!(report.outcome.collected.is_empty()); - assert_eq!( - report.outcome.errors[0].kind, - CollectionFailureKind::DurableDataProtected - ); - assert!(data_root.exists()); -} - -/// A durable memory table that exists but is empty must not block collection. -/// Only an actual row does. -#[tokio::test] -async fn empty_memory_table_does_not_block_collection() { - let tmp = tempfile::TempDir::new().unwrap(); - let profile_root = tmp.path().join("profile"); - std::fs::create_dir_all(&profile_root).unwrap(); - let dead_root = tmp.path().join("moved-away-repo"); - let (_runtime, db) = open_registered_db(&profile_root).await; - let base = 1_700_000_000i64; - let data_root = seed_store( - &db, - &profile_root, - "proj_empty_memory", - "store_empty_memory", - &dead_root, - base - 100 * DAY, - ) - .await; - { - let connection = rusqlite::Connection::open(data_root.join("graph.db")).unwrap(); - connection - .execute_batch( - "CREATE TABLE memory_facts ( - fact_id INTEGER PRIMARY KEY, - content TEXT NOT NULL - ); - CREATE VIRTUAL TABLE memory_facts_fts USING fts5(content);", - ) - .unwrap(); - } - - let report = sweep_orphan_stores(&db, &profile_root, 7 * DAY, base, true) - .await - .unwrap(); - - assert_eq!( - report.outcome.collected.len(), - 1, - "empty durable-memory tables must not block collection: {report:#?}" - ); - assert!(!data_root.exists()); -} - -// === Unregistered store directories ========================================= - -#[cfg(unix)] -#[tokio::test] -async fn unregistered_collection_refuses_same_second_directory_replacement() { - let tmp = tempfile::TempDir::new().unwrap(); - let profile_root = tmp.path().join("profile"); - std::fs::create_dir_all(&profile_root).unwrap(); - let (_runtime, db) = open_registered_db(&profile_root).await; - let data_root = profile_root.join("projects/proj_replaced_unregistered"); - std::fs::create_dir_all(&data_root).unwrap(); - - let now = walk_store_stats(&data_root) - .newest_mtime_secs - .saturating_add(100 * DAY); - let findings = census_unregistered_project_dirs(&db, &profile_root, now) - .await - .unwrap(); - let plan = plan_unregistered_collection(findings, 7 * DAY); - assert_eq!( - plan.collect.len(), - 1, - "fixture must be eligible before replacement" - ); - - let displaced = profile_root.join("displaced-unregistered-store"); - let original_time = - filetime::FileTime::from_system_time(data_root.metadata().unwrap().modified().unwrap()); - std::fs::rename(&data_root, &displaced).unwrap(); - std::fs::create_dir_all(&data_root).unwrap(); - filetime::set_file_mtime(&data_root, original_time).unwrap(); - - let outcome = execute_unregistered_collection(&db, &plan, &profile_root) - .await - .unwrap(); - - assert!(outcome.collected.is_empty()); - assert_eq!( - outcome.errors, - vec![CollectionFailure { - store_id: "proj_replaced_unregistered".to_owned(), - kind: CollectionFailureKind::PayloadChanged, - }] - ); - assert!(data_root.exists(), "replacement directory must survive"); -} - -/// An unregistered leaf must not become a deletion target merely because its -/// symlink resolves back inside the profile. The physical `/projects` -/// path, rather than canonicalized containment, is the destructive authority. -#[cfg(unix)] -#[tokio::test] -async fn unregistered_collection_rejects_profile_contained_data_root_symlink() { - let tmp = tempfile::TempDir::new().unwrap(); - let profile_root = tmp.path().join("profile"); - std::fs::create_dir_all(&profile_root).unwrap(); - let (_runtime, db) = open_registered_db(&profile_root).await; - let data_root = profile_root.join("projects/proj_symlinked_unregistered"); - std::fs::create_dir_all(&data_root).unwrap(); - - let now = walk_store_stats(&data_root) - .newest_mtime_secs - .saturating_add(100 * DAY); - let findings = census_unregistered_project_dirs(&db, &profile_root, now) - .await - .unwrap(); - let plan = plan_unregistered_collection(findings, 7 * DAY); - assert_eq!( - plan.collect.len(), - 1, - "fixture must be eligible before the symlink swap" - ); - - let held_payload = profile_root.join("held-unregistered-payload"); - std::fs::rename(&data_root, &held_payload).unwrap(); - std::os::unix::fs::symlink(&held_payload, &data_root).unwrap(); - - let outcome = execute_unregistered_collection(&db, &plan, &profile_root) - .await - .unwrap(); - - assert!(outcome.collected.is_empty()); - assert_eq!( - outcome.errors, - vec![CollectionFailure { - store_id: "proj_symlinked_unregistered".to_owned(), - kind: CollectionFailureKind::OutsideProfile, - }] - ); - assert!( - held_payload.is_dir(), - "the in-profile symlink target must survive" - ); -} - -/// `SQLite` pages can change in place without changing the parent directory. -/// Hashing the opened child handles makes that post-census mutation visible at -/// the recovery boundary even when the writer resets the database mtime. -#[test] -fn quarantine_restores_same_second_sqlite_mutation() { - let tmp = tempfile::TempDir::new().unwrap(); - let profile_root = tmp.path().join("profile"); - let data_root = profile_root.join("stores/sqlite-race"); - std::fs::create_dir_all(&data_root).unwrap(); - let database = data_root.join("graph.db"); - let connection = rusqlite::Connection::open(&database).unwrap(); - connection - .execute_batch("CREATE TABLE facts (value TEXT NOT NULL);") - .unwrap(); - drop(connection); - let expected = capture_store_content_fence(&profile_root, &data_root).unwrap(); - let original_time = - filetime::FileTime::from_system_time(database.metadata().unwrap().modified().unwrap()); - - let connection = rusqlite::Connection::open(&database).unwrap(); - connection - .execute("INSERT INTO facts (value) VALUES ('post-census')", []) - .unwrap(); - drop(connection); - filetime::set_file_mtime(&database, original_time).unwrap(); - - let result = - quarantine_store_for_verified_collection(&profile_root, &data_root, &expected).unwrap(); - assert!(matches!(result, QuarantineStoreOutcome::Restored { .. })); - let connection = rusqlite::Connection::open(&database).unwrap(); - let rows: i64 = connection - .query_row("SELECT COUNT(*) FROM facts", [], |row| row.get(0)) - .unwrap(); - assert_eq!(rows, 1, "mutated SQLite bytes must survive recovery"); -} - -/// Even an empty replacement is not the inspected directory. Its child list -/// is identical, so the moved root's stable identity must participate in the -/// post-rename comparison before collection can remove anything. -#[test] -fn quarantine_restores_empty_directory_replacement_before_delete() { - let tmp = tempfile::TempDir::new().unwrap(); - let profile_root = tmp.path().join("profile"); - let data_root = profile_root.join("stores/empty-rename-race"); - std::fs::create_dir_all(&data_root).unwrap(); - let expected = capture_store_content_fence(&profile_root, &data_root).unwrap(); - - let displaced = profile_root.join("stores/displaced-empty"); - std::fs::rename(&data_root, &displaced).unwrap(); - std::fs::create_dir_all(&data_root).unwrap(); - - let result = - quarantine_store_for_verified_collection(&profile_root, &data_root, &expected).unwrap(); - - assert!(matches!(result, QuarantineStoreOutcome::Restored { .. })); - assert!(data_root.is_dir(), "fresh empty replacement must survive"); - assert!(displaced.is_dir(), "inspected empty directory must survive"); -} - -#[test] -fn stale_retired_marker_occupies_the_whole_quarantine_candidate_namespace() { - let tmp = tempfile::TempDir::new().unwrap(); - let parent_path = tmp.path().join("stores"); - let data_root = parent_path.join("stale-authority"); - let payload_path = data_root.join("payload.bin"); - let payload = b"new store payload must survive reservation"; - std::fs::create_dir_all(&data_root).unwrap(); - std::fs::write(&payload_path, payload).unwrap(); - let parent = - cap_std::fs::Dir::open_ambient_dir(&parent_path, cap_std::ambient_authority()).unwrap(); - let candidate = format!( - ".tracedecay-orphan-quarantine-stale-authority-{}-7", - std::process::id() - ); - let retired_marker_path = parent_path.join(format!("{candidate}.receipt-v1.json.retired")); - let marker_bytes = b"stale commit authority must remain exact"; - std::fs::write(&retired_marker_path, marker_bytes).unwrap(); - - assert!( - !quarantine_candidate_namespace_available(&parent, &candidate).unwrap(), - "a retired marker reserves its candidate even when the directory and journal are absent" - ); - let mut sequences = [7, 8].into_iter(); - let reserved = - reserve_quarantine_name_with_sequence(&parent, &data_root, "stale-authority", None, || { - sequences - .next() - .expect("reservation must use only two candidates") - }) - .unwrap(); - assert_eq!( - reserved, - format!( - ".tracedecay-orphan-quarantine-stale-authority-{}-8", - std::process::id() - ), - "the stale retired marker must force the next sequence" - ); - assert_eq!( - retired_marker_path, - parent_path.join(format!( - ".tracedecay-orphan-quarantine-stale-authority-{}-7.receipt-v1.json.retired", - std::process::id() - )) - ); - assert_eq!(std::fs::read(&retired_marker_path).unwrap(), marker_bytes); - assert_eq!(std::fs::read(&payload_path).unwrap(), payload); -} - -/// A Windows directory capability denies same-parent rename because cap-std -/// deliberately omits FILE_SHARE_DELETE. The failed retirement must remain a -/// typed deferral over the exact censused store, then converge once that -/// external owner releases its handle. -#[cfg(windows)] -#[test] -fn quarantine_defers_held_windows_store_and_converges_after_release() { - let tmp = tempfile::TempDir::new().unwrap(); - let profile_root = tmp.path().join("profile"); - let data_root = profile_root.join("stores/held-capability"); - let payload_path = data_root.join("payload.bin"); - let payload = b"held store bytes remain authoritative"; - std::fs::create_dir_all(&data_root).unwrap(); - std::fs::write(&payload_path, payload).unwrap(); - let expected = capture_store_content_fence(&profile_root, &data_root).unwrap(); - let StoreContentFence::Present(expected_inventory) = &expected else { - panic!("fixture must capture an exact present-store fence"); - }; - let expected_root_identity = expected_inventory.root.clone(); - let external_owner = - cap_std::fs::Dir::open_ambient_dir(&data_root, cap_std::ambient_authority()).unwrap(); - - let failure = - match quarantine_store_for_verified_collection(&profile_root, &data_root, &expected) { - Err(failure) => failure, - Ok(_) => panic!("held live leaf must not enter quarantine"), - }; - let CollectionFailureKind::RemoveFailed(failure) = failure else { - panic!("held live leaf must report a structured mutation failure"); - }; - assert_eq!( - failure.operation, - CollectionMutationOperation::RenameLiveLeafToQuarantine - ); - assert!( - matches!(failure.raw_os_error, Some(5 | 32)), - "Windows held-directory rename must preserve access-denied/sharing violation: {failure:?}" - ); - assert!(failure.retryable()); - assert_eq!( - failure.classification, - CollectionMutationFailureClassification::RetryableDeferred - ); - assert_eq!(failure.target_path, data_root); - assert_eq!(failure.expected_root_identity, Some(expected_root_identity)); - assert_eq!(std::fs::read(&payload_path).unwrap(), payload); - assert!( - data_root.is_dir(), - "failed quarantine must not collect the store" - ); - assert!( - read_pending_quarantine_receipts(&profile_root) - .unwrap() - .is_empty(), - "the failed rename's prepared journal must still be cleared" - ); - - drop(external_owner); - let result = - quarantine_store_for_verified_collection(&profile_root, &data_root, &expected).unwrap(); - let QuarantineStoreOutcome::Verified(quarantine) = result else { - panic!("retry after releasing the external owner must verify quarantine"); - }; - assert_eq!( - std::fs::read(quarantine.quarantine_path().join("payload.bin")).unwrap(), - payload - ); - quarantine.mark_retirement_committed().unwrap(); - let QuarantineFinalizeOutcome::Removed { journal_failure } = - quarantine.finalize(unbounded_collection_control()) - else { - panic!("released quarantine must complete the durable removal sequence"); - }; - assert_eq!(journal_failure, None); - assert!(!data_root.exists()); - assert!( - read_pending_quarantine_receipts(&profile_root) - .unwrap() - .is_empty() - ); -} - -/// This simulates a process death after the registry phase was durably marked -/// but before recursive removal. The journal, quarantine path, and committed -/// phase remain readable without relying on an in-memory collection outcome. -#[test] -fn committed_quarantine_crash_boundary_has_a_readable_recovery_receipt() { - let tmp = tempfile::TempDir::new().unwrap(); - let profile_root = tmp.path().join("profile"); - let data_root = profile_root.join("stores/crash-boundary"); - std::fs::create_dir_all(&data_root).unwrap(); - std::fs::write(data_root.join("payload.bin"), b"preserve until finalize").unwrap(); - let expected = capture_store_content_fence(&profile_root, &data_root).unwrap(); - - let result = - quarantine_store_for_verified_collection(&profile_root, &data_root, &expected).unwrap(); - let QuarantineStoreOutcome::Verified(quarantine) = result else { - panic!("fixture must reach verified quarantine"); - }; - quarantine.mark_retirement_committed().unwrap(); - drop(quarantine); - - let receipts = read_pending_quarantine_receipts(&profile_root).unwrap(); - assert_eq!(receipts.len(), 1); - assert!(receipts[0].retirement_committed); - assert!(receipts[0].quarantine_path.is_dir()); - assert!(!data_root.exists(), "live name remains private after crash"); -} - -#[test] -fn prepared_journal_recovery_restores_the_exact_original_store() { - let tmp = tempfile::TempDir::new().unwrap(); - let profile_root = tmp.path().join("profile"); - let data_root = profile_root.join("stores/prepared-recovery"); - let payload = b"prepared quarantine bytes are restored exactly"; - std::fs::create_dir_all(&data_root).unwrap(); - std::fs::write(data_root.join("payload.bin"), payload).unwrap(); - let expected = capture_store_content_fence(&profile_root, &data_root).unwrap(); - - let result = - quarantine_store_for_verified_collection(&profile_root, &data_root, &expected).unwrap(); - let QuarantineStoreOutcome::Verified(quarantine) = result else { - panic!("fixture must reach verified quarantine"); - }; - let quarantine_path = quarantine.quarantine_path().to_path_buf(); - drop(quarantine); - - let outcomes = recover_existing_store_quarantine( - &profile_root, - &data_root, - unbounded_collection_control(), - ) - .unwrap(); - - assert_eq!( - outcomes, - vec![QuarantineRecoveryOutcome::Restored { - restored_path: data_root.clone(), - failure: None, - }] - ); - assert_eq!( - std::fs::read(data_root.join("payload.bin")).unwrap(), - payload - ); - assert_eq!( - capture_store_content_fence(&profile_root, &data_root).unwrap(), - expected - ); - assert!(!quarantine_path.exists()); - assert!( - read_pending_quarantine_receipts(&profile_root) - .unwrap() - .is_empty() - ); -} - -#[test] -fn committed_journal_recovery_removes_the_exact_quarantine() { - let tmp = tempfile::TempDir::new().unwrap(); - let profile_root = tmp.path().join("profile"); - let data_root = profile_root.join("stores/committed-recovery"); - let payload = b"committed quarantine bytes are removed exactly"; - std::fs::create_dir_all(&data_root).unwrap(); - std::fs::write(data_root.join("payload.bin"), payload).unwrap(); - let expected = capture_store_content_fence(&profile_root, &data_root).unwrap(); - - let result = - quarantine_store_for_verified_collection(&profile_root, &data_root, &expected).unwrap(); - let QuarantineStoreOutcome::Verified(quarantine) = result else { - panic!("fixture must reach verified quarantine"); - }; - let quarantine_path = quarantine.quarantine_path().to_path_buf(); - assert_eq!( - std::fs::read(quarantine_path.join("payload.bin")).unwrap(), - payload - ); - quarantine.mark_retirement_committed().unwrap(); - drop(quarantine); - - let outcomes = recover_existing_store_quarantine( - &profile_root, - &data_root, - unbounded_collection_control(), - ) - .unwrap(); - - assert_eq!( - outcomes, - vec![QuarantineRecoveryOutcome::Removed { - quarantine_path: quarantine_path.clone(), - journal_failure: None, - }] - ); - assert!(!data_root.exists()); - assert!(!quarantine_path.exists()); - assert!( - read_pending_quarantine_receipts(&profile_root) - .unwrap() - .is_empty() - ); -} - -#[test] -fn registered_remove_clears_journal_after_exact_quarantine_is_already_absent() { - let tmp = tempfile::TempDir::new().unwrap(); - let profile_root = tmp.path().join("profile"); - let data_root = profile_root.join("stores/registered-delete-complete"); - std::fs::create_dir_all(&data_root).unwrap(); - std::fs::write( - data_root.join("payload.bin"), - b"registered deletion completed before metadata cleanup", - ) - .unwrap(); - let expected = capture_store_content_fence(&profile_root, &data_root).unwrap(); - let quarantine = quarantine_store_for_verified_collection_controlled( - &profile_root, - &data_root, - &expected, - QuarantineKindV1::Registered, - "proj_registered_delete_complete", - "registered-delete-complete", - Some(QuarantineRegistryFenceV1 { - store_relpath: "stores/registered-delete-complete".to_owned(), - created_at: 1_700_000_000, - last_write_at: Some(1_700_000_000), - }), - unbounded_collection_control(), - ) - .unwrap(); - let QuarantineStoreOutcome::Verified(quarantine) = quarantine else { - panic!("fixture must reach verified registered quarantine"); - }; - let quarantine_path = quarantine.quarantine_path().to_path_buf(); - drop(quarantine); - let intent = match read_registered_quarantine_intents_controlled( - &profile_root, - unbounded_collection_control(), - ) - .unwrap() - { - RegisteredQuarantineInventoryV1::Complete(mut intents) => { - assert_eq!(intents.len(), 1); - intents.pop().unwrap() - } - RegisteredQuarantineInventoryV1::Interrupted => panic!("fixture inventory interrupted"), - }; - std::fs::remove_dir_all(&quarantine_path).unwrap(); - - let recovery = recover_registered_quarantine_intent_controlled( - &profile_root, - &intent, - RegisteredQuarantineDecisionV1::Remove, - unbounded_collection_control(), - ) - .unwrap(); - - assert_eq!( - recovery, - Some(QuarantineRecoveryOutcome::Removed { - quarantine_path: quarantine_path.clone(), - journal_failure: None, - }) - ); - assert!(!data_root.exists()); - assert!(!quarantine_path.exists()); - assert!( - read_pending_quarantine_receipts(&profile_root) - .unwrap() - .is_empty() - ); -} - -#[test] -fn unregistered_committed_recovery_clears_journal_after_quarantine_is_already_absent() { - let tmp = tempfile::TempDir::new().unwrap(); - let profile_root = tmp.path().join("profile"); - let data_root = profile_root.join("stores/unregistered-delete-complete"); - std::fs::create_dir_all(&data_root).unwrap(); - std::fs::write( - data_root.join("payload.bin"), - b"unregistered deletion completed before metadata cleanup", - ) - .unwrap(); - let expected = capture_store_content_fence(&profile_root, &data_root).unwrap(); - let quarantine = - quarantine_store_for_verified_collection(&profile_root, &data_root, &expected).unwrap(); - let QuarantineStoreOutcome::Verified(quarantine) = quarantine else { - panic!("fixture must reach verified unregistered quarantine"); - }; - let quarantine_path = quarantine.quarantine_path().to_path_buf(); - quarantine.mark_retirement_committed().unwrap(); - drop(quarantine); - std::fs::remove_dir_all(&quarantine_path).unwrap(); - let quarantine_name = quarantine_path.file_name().unwrap().to_str().unwrap(); - let journal_path = quarantine_path.with_file_name(format!("{quarantine_name}.receipt-v1.json")); - let renamed_marker = - quarantine_path.with_file_name(format!("{quarantine_name}.receipt-v1.json.renamed")); - let retired_marker = - quarantine_path.with_file_name(format!("{quarantine_name}.receipt-v1.json.retired")); - std::fs::remove_file(&renamed_marker).unwrap(); - assert!( - journal_path.is_file() && retired_marker.is_file(), - "a crash before journal removal retains both recovery authorities" - ); - - let recovery = recover_existing_store_quarantine( - &profile_root, - &data_root, - unbounded_collection_control(), - ) - .unwrap(); - - assert_eq!( - recovery, - vec![QuarantineRecoveryOutcome::Removed { - quarantine_path: quarantine_path.clone(), - journal_failure: None, - }] - ); - assert!(!data_root.exists()); - assert!(!quarantine_path.exists()); - assert!(!journal_path.exists()); - assert!(!retired_marker.exists()); - assert!( - read_pending_quarantine_receipts(&profile_root) - .unwrap() - .is_empty() - ); -} - -#[test] -fn unregistered_uncommitted_recovery_retains_journal_when_both_names_are_absent() { - let tmp = tempfile::TempDir::new().unwrap(); - let profile_root = tmp.path().join("profile"); - let data_root = profile_root.join("stores/unregistered-lost-before-commit"); - std::fs::create_dir_all(&data_root).unwrap(); - std::fs::write(data_root.join("payload.bin"), b"uncommitted bytes").unwrap(); - let expected = capture_store_content_fence(&profile_root, &data_root).unwrap(); - let StoreContentFence::Present(expected_inventory) = &expected else { - panic!("fixture must capture an exact present-store fence"); - }; - let expected_root_identity = expected_inventory.root.clone(); - let quarantine = - quarantine_store_for_verified_collection(&profile_root, &data_root, &expected).unwrap(); - let QuarantineStoreOutcome::Verified(quarantine) = quarantine else { - panic!("fixture must reach verified unregistered quarantine"); - }; - let quarantine_path = quarantine.quarantine_path().to_path_buf(); - drop(quarantine); - std::fs::remove_dir_all(&quarantine_path).unwrap(); - - let recovery = recover_existing_store_quarantine( - &profile_root, - &data_root, - unbounded_collection_control(), - ) - .unwrap(); - - assert_eq!( - recovery, - vec![QuarantineRecoveryOutcome::Retained { - quarantine_path: quarantine_path.clone(), - actual_path: quarantine_path.clone(), - failure: Some(CollectionMutationFailure { - operation: CollectionMutationOperation::ValidateRestoredStoreIdentity, - raw_os_error: None, - target_path: data_root.clone(), - expected_root_identity: Some(expected_root_identity), - classification: CollectionMutationFailureClassification::NonRetryable, - }), - }] - ); - assert!(!data_root.exists()); - assert!(!quarantine_path.exists()); - assert_eq!( - read_pending_quarantine_receipts(&profile_root).unwrap(), - vec![PendingQuarantineReceiptV1 { - quarantine_path: quarantine_path.clone(), - actual_path: quarantine_path, - retirement_committed: false, - }] - ); -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn registered_recovery_holds_writer_exclusion_through_exact_restore() { - let tmp = tempfile::TempDir::new().unwrap(); - let profile_root = tmp.path().join("profile"); - std::fs::create_dir_all(&profile_root).unwrap(); - let (_runtime, db) = open_registered_db(&profile_root).await; - let payload = b"registry deletion waits until exact recovery is durable"; - let (data_root, quarantine_path) = prepare_registered_quarantine( - &db, - &profile_root, - "proj_registered_serialized_restore", - "store_registered_serialized_restore", - payload, - ) - .await; - let quarantine_name = quarantine_path.file_name().unwrap().to_str().unwrap(); - let journal_path = quarantine_path.with_file_name(format!("{quarantine_name}.receipt-v1.json")); - let (classified_tx, classified_rx) = tokio::sync::oneshot::channel(); - let (release_tx, release_rx) = std::sync::mpsc::channel(); - let recovery_db = db.clone(); - let recovery_profile_root = profile_root.clone(); - let recovery = tokio::spawn(async move { - let mut classified_tx = Some(classified_tx); - let mut outcome = CollectionOutcome::default(); - reconcile_registered_quarantine_inventory_with_classified_hook( - &recovery_db, - &recovery_profile_root, - unbounded_collection_control(), - &mut outcome, - move |intent, state| { - assert_eq!(intent.store_id, "store_registered_serialized_restore"); - assert_eq!(state, RegisteredQuarantineRegistryStateV1::Exact); - classified_tx.take().unwrap().send(()).unwrap(); - release_rx - .recv_timeout(Duration::from_secs(5)) - .expect("test must release classified recovery"); - }, - ) - .await - .unwrap(); - outcome - }); - - tokio::time::timeout(Duration::from_secs(5), classified_rx) - .await - .expect("recovery must reach exact classification") - .unwrap(); - assert!(!data_root.exists()); - assert!(quarantine_path.is_dir()); - assert!(journal_path.is_file()); - - let competitor_db = db.clone(); - let (attempted_tx, attempted_rx) = tokio::sync::oneshot::channel(); - let mut competitor = tokio::spawn(async move { - attempted_tx.send(()).unwrap(); - let transaction = competitor_db.begin_write_transaction().await.unwrap(); - let deleted = transaction - .execute( - "DELETE FROM store_instances WHERE store_id = ?1", - tracedecay_runtime_core::db::engine::params!["store_registered_serialized_restore"], - ) - .await - .unwrap(); - transaction.commit().await.unwrap(); - deleted - }); - attempted_rx.await.unwrap(); - assert!( - tokio::time::timeout(Duration::from_millis(250), &mut competitor) - .await - .is_err(), - "competing registry deletion committed between classification and restore" - ); - assert!(!data_root.exists()); - assert!(quarantine_path.is_dir()); - assert!(journal_path.is_file()); - - release_tx.send(()).unwrap(); - let outcome = recovery.await.unwrap(); - assert_eq!(competitor.await.unwrap(), 1); - assert!(outcome.errors.is_empty(), "{outcome:#?}"); - assert_eq!( - outcome.recovery_receipts, - vec![CollectionRecoveryReceipt { - store_id: "store_registered_serialized_restore".to_owned(), - original_path: data_root.clone(), - quarantine_path: quarantine_path.clone(), - actual_path: data_root.clone(), - action: CollectionRecoveryAction::Restored, - }] - ); - assert_eq!( - std::fs::read(data_root.join("payload.bin")).unwrap(), - payload - ); - assert!(!quarantine_path.exists()); - assert!(!journal_path.exists()); - assert!( - db.try_list_store_instances_for_project("proj_registered_serialized_restore") - .await - .unwrap() - .is_empty() - ); - assert!( - read_pending_quarantine_receipts(&profile_root) - .unwrap() - .is_empty() - ); -} - -#[tokio::test] -async fn empty_plan_clears_registered_pre_rename_journal_when_exact_row_remains() { - let tmp = tempfile::TempDir::new().unwrap(); - let profile_root = tmp.path().join("profile"); - std::fs::create_dir_all(&profile_root).unwrap(); - let (_runtime, db) = open_registered_db(&profile_root).await; - let payload = b"pre-rename registered bytes never moved"; - let (data_root, quarantine_path) = prepare_registered_quarantine( - &db, - &profile_root, - "proj_registered_pre_rename", - "store_registered_pre_rename", - payload, - ) - .await; - let expected = capture_store_content_fence(&profile_root, &quarantine_path).unwrap(); - let quarantine_name = quarantine_path.file_name().unwrap().to_str().unwrap(); - let journal_path = quarantine_path.with_file_name(format!("{quarantine_name}.receipt-v1.json")); - let renamed_marker = - quarantine_path.with_file_name(format!("{quarantine_name}.receipt-v1.json.renamed")); - let retired_marker = - quarantine_path.with_file_name(format!("{quarantine_name}.receipt-v1.json.retired")); - std::fs::rename(&quarantine_path, &data_root).unwrap(); - std::fs::remove_file(&renamed_marker).unwrap(); - assert!(journal_path.is_file()); - - let (outcome, retired) = - execute_registered_collection(&db, &CollectionPlan::default(), &profile_root) - .await - .unwrap(); - - assert_eq!(retired, 0); - assert!(outcome.errors.is_empty(), "{outcome:#?}"); - assert!(outcome.collected.is_empty()); - assert_eq!(outcome.reclaimed_bytes, 0); - assert_eq!( - outcome.recovery_receipts, - vec![CollectionRecoveryReceipt { - store_id: "store_registered_pre_rename".to_owned(), - original_path: data_root.clone(), - quarantine_path: quarantine_path.clone(), - actual_path: data_root.clone(), - action: CollectionRecoveryAction::Restored, - }] - ); - assert_eq!( - std::fs::read(data_root.join("payload.bin")).unwrap(), - payload - ); - assert_eq!( - capture_store_content_fence(&profile_root, &data_root).unwrap(), - expected - ); - assert!(!quarantine_path.exists()); - assert!(!journal_path.exists()); - assert!(!renamed_marker.exists()); - assert!(!retired_marker.exists()); - assert!( - read_pending_quarantine_receipts(&profile_root) - .unwrap() - .is_empty() - ); - assert_eq!( - db.try_list_store_instances_for_project("proj_registered_pre_rename") - .await - .unwrap() - .len(), - 1 - ); -} - -#[tokio::test] -async fn stale_registered_retirement_marker_cannot_override_exact_row() { - let tmp = tempfile::TempDir::new().unwrap(); - let profile_root = tmp.path().join("profile"); - std::fs::create_dir_all(&profile_root).unwrap(); - let (_runtime, db) = open_registered_db(&profile_root).await; - let payload = b"the exact registry row remains the commit authority"; - let (data_root, quarantine_path) = prepare_registered_quarantine( - &db, - &profile_root, - "proj_registered_stale_marker", - "store_registered_stale_marker", - payload, - ) - .await; - let quarantine_name = quarantine_path.file_name().unwrap().to_str().unwrap(); - std::fs::write( - quarantine_path.with_file_name(format!("{quarantine_name}.receipt-v1.json.retired")), - [], - ) - .unwrap(); - - assert_eq!( - recover_existing_store_quarantine( - &profile_root, - &data_root, - unbounded_collection_control() - ) - .unwrap(), - vec![QuarantineRecoveryOutcome::Retained { - quarantine_path: quarantine_path.clone(), - actual_path: quarantine_path.clone(), - failure: None, - }], - "registered named recovery has no database authority to consume the marker" - ); - assert_eq!( - std::fs::read(quarantine_path.join("payload.bin")).unwrap(), - payload - ); - - let (outcome, retired) = - execute_registered_collection(&db, &CollectionPlan::default(), &profile_root) - .await - .unwrap(); - - assert_eq!(retired, 0); - assert!(outcome.collected.is_empty()); - assert!(outcome.errors.is_empty(), "{outcome:#?}"); - assert_eq!( - std::fs::read(data_root.join("payload.bin")).unwrap(), - payload - ); - assert!(!quarantine_path.exists()); - assert_eq!( - db.try_list_store_instances_for_project("proj_registered_stale_marker") - .await - .unwrap() - .len(), - 1 - ); - assert!( - read_pending_quarantine_receipts(&profile_root) - .unwrap() - .is_empty() - ); -} - -#[tokio::test] -async fn empty_plan_removes_registered_quarantine_when_exact_row_is_absent() { - let tmp = tempfile::TempDir::new().unwrap(); - let profile_root = tmp.path().join("profile"); - std::fs::create_dir_all(&profile_root).unwrap(); - let (_runtime, db) = open_registered_db(&profile_root).await; - let payload = b"exact registered bytes retire after the database commit"; - let (data_root, quarantine_path) = prepare_registered_quarantine( - &db, - &profile_root, - "proj_registered_remove", - "store_registered_remove", - payload, - ) - .await; - let transaction = db.begin_write_transaction().await.unwrap(); - assert_eq!( - transaction - .execute( - "DELETE FROM store_instances - WHERE project_id = ?1 AND store_id = ?2 - AND store_relpath = ?3 AND created_at = ?4 - AND last_write_at IS ?5", - tracedecay_runtime_core::db::engine::params![ - "proj_registered_remove", - "store_registered_remove", - "stores/store_registered_remove", - 1_700_000_000i64, - Some(1_700_000_000i64) - ], - ) - .await - .unwrap(), - 1 - ); - transaction.commit().await.unwrap(); - - let (outcome, retired) = - execute_registered_collection(&db, &CollectionPlan::default(), &profile_root) - .await - .unwrap(); - - assert_eq!(retired, 0); - assert!(outcome.collected.is_empty()); - assert_eq!(outcome.reclaimed_bytes, 0); - assert!(outcome.errors.is_empty(), "{outcome:#?}"); - assert!(!data_root.exists()); - assert!(!quarantine_path.exists()); - assert!( - db.try_list_store_instances_for_project("proj_registered_remove") - .await - .unwrap() - .is_empty() - ); - assert!( - read_pending_quarantine_receipts(&profile_root) - .unwrap() - .is_empty() - ); -} - -#[tokio::test] -async fn empty_plan_retains_registered_quarantine_when_row_fence_changed() { - let tmp = tempfile::TempDir::new().unwrap(); - let profile_root = tmp.path().join("profile"); - std::fs::create_dir_all(&profile_root).unwrap(); - let (_runtime, db) = open_registered_db(&profile_root).await; - let payload = b"changed registry authority cannot consume these bytes"; - let (data_root, quarantine_path) = prepare_registered_quarantine( - &db, - &profile_root, - "proj_registered_changed", - "store_registered_changed", - payload, - ) - .await; - let transaction = db.begin_write_transaction().await.unwrap(); - assert_eq!( - transaction - .execute( - "UPDATE store_instances SET last_write_at = ?3 - WHERE project_id = ?1 AND store_id = ?2", - tracedecay_runtime_core::db::engine::params![ - "proj_registered_changed", - "store_registered_changed", - 1_700_000_001i64 - ], - ) - .await - .unwrap(), - 1 - ); - transaction.commit().await.unwrap(); - - let (outcome, retired) = - execute_registered_collection(&db, &CollectionPlan::default(), &profile_root) - .await - .unwrap(); - - assert_eq!(retired, 0); - assert!(outcome.collected.is_empty()); - assert_eq!(outcome.reclaimed_bytes, 0); - assert_eq!( - outcome.errors, - vec![CollectionFailure { - store_id: "store_registered_changed".to_owned(), - kind: CollectionFailureKind::RegistryChanged, - }] - ); - assert_eq!( - outcome.recovery_receipts, - vec![CollectionRecoveryReceipt { - store_id: "store_registered_changed".to_owned(), - original_path: data_root.clone(), - quarantine_path: quarantine_path.clone(), - actual_path: quarantine_path.clone(), - action: CollectionRecoveryAction::RetainedForRecovery, - }] - ); - assert!(!data_root.exists()); - assert_eq!( - std::fs::read(quarantine_path.join("payload.bin")).unwrap(), - payload - ); - let rows = db - .try_list_store_instances_for_project("proj_registered_changed") - .await - .unwrap(); - assert_eq!(rows.len(), 1); - assert_eq!(rows[0].last_write_at, Some(1_700_000_001)); - assert_eq!( - read_pending_quarantine_receipts(&profile_root) - .unwrap() - .len(), - 1 - ); -} - -#[test] -fn registered_recovery_completes_without_fabricating_collection_totals() { - let tmp = tempfile::TempDir::new().unwrap(); - let profile_root = tmp.path().join("profile"); - let data_root = profile_root.join("stores/registered-recovery-consumer"); - std::fs::create_dir_all(&data_root).unwrap(); - std::fs::write(data_root.join("payload.bin"), b"resume exactly once").unwrap(); - let expected = capture_store_content_fence(&profile_root, &data_root).unwrap(); - - let result = - quarantine_store_for_verified_collection(&profile_root, &data_root, &expected).unwrap(); - let QuarantineStoreOutcome::Verified(quarantine) = result else { - panic!("fixture must reach verified quarantine"); - }; - quarantine.mark_retirement_committed().unwrap(); - drop(quarantine); - let mut outcome = CollectionOutcome::default(); - - assert!(!reconcile_existing_quarantine( - &profile_root, - &data_root, - "registered-recovery-consumer", - &mut outcome, - unbounded_collection_control(), - )); - assert_eq!(outcome, CollectionOutcome::default()); - assert!(reconcile_existing_quarantine( - &profile_root, - &data_root, - "registered-recovery-consumer", - &mut outcome, - unbounded_collection_control(), - )); - assert_eq!(outcome.reclaimed_bytes, 0); - assert!(outcome.collected.is_empty()); -} - -#[test] -fn committed_journal_recovery_retains_an_identity_replacement() { - let tmp = tempfile::TempDir::new().unwrap(); - let profile_root = tmp.path().join("profile"); - let data_root = profile_root.join("stores/committed-identity-replacement"); - let original_payload = b"exact committed quarantine bytes"; - let replacement_payload = b"replacement must never inherit delete authority"; - std::fs::create_dir_all(&data_root).unwrap(); - std::fs::write(data_root.join("payload.bin"), original_payload).unwrap(); - let expected = capture_store_content_fence(&profile_root, &data_root).unwrap(); - let StoreContentFence::Present(expected_inventory) = &expected else { - panic!("fixture must capture an exact present-store fence"); - }; - let expected_root_identity = expected_inventory.root.clone(); - - let result = - quarantine_store_for_verified_collection(&profile_root, &data_root, &expected).unwrap(); - let QuarantineStoreOutcome::Verified(quarantine) = result else { - panic!("fixture must reach verified quarantine"); - }; - let quarantine_path = quarantine.quarantine_path().to_path_buf(); - quarantine.mark_retirement_committed().unwrap(); - drop(quarantine); - let displaced = profile_root.join("stores/exact-committed-quarantine"); - std::fs::rename(&quarantine_path, &displaced).unwrap(); - std::fs::create_dir_all(&quarantine_path).unwrap(); - std::fs::write(quarantine_path.join("payload.bin"), replacement_payload).unwrap(); - - let outcomes = recover_existing_store_quarantine( - &profile_root, - &data_root, - unbounded_collection_control(), - ) - .unwrap(); - - assert_eq!( - outcomes, - vec![QuarantineRecoveryOutcome::Retained { - quarantine_path: quarantine_path.clone(), - actual_path: quarantine_path.clone(), - failure: Some(CollectionMutationFailure { - operation: CollectionMutationOperation::ValidateRestoredStoreIdentity, - raw_os_error: None, - target_path: quarantine_path.clone(), - expected_root_identity: Some(expected_root_identity), - classification: CollectionMutationFailureClassification::NonRetryable, - }), - }] - ); - assert_eq!( - std::fs::read(displaced.join("payload.bin")).unwrap(), - original_payload - ); - assert_eq!( - std::fs::read(quarantine_path.join("payload.bin")).unwrap(), - replacement_payload - ); - assert!(!data_root.exists()); - assert_eq!( - read_pending_quarantine_receipts(&profile_root) - .unwrap() - .len(), - 1 - ); -} - -#[test] -fn legacy_journal_without_identity_fails_closed_and_preserves_exact_bytes() { - let tmp = tempfile::TempDir::new().unwrap(); - let profile_root = tmp.path().join("profile"); - let data_root = profile_root.join("stores/legacy-journal"); - let payload = b"legacy journal bytes must remain untouched"; - std::fs::create_dir_all(&data_root).unwrap(); - std::fs::write(data_root.join("payload.bin"), payload).unwrap(); - let expected = capture_store_content_fence(&profile_root, &data_root).unwrap(); - - let result = - quarantine_store_for_verified_collection(&profile_root, &data_root, &expected).unwrap(); - let QuarantineStoreOutcome::Verified(quarantine) = result else { - panic!("fixture must reach verified quarantine"); - }; - let quarantine_path = quarantine.quarantine_path().to_path_buf(); - drop(quarantine); - let quarantine_name = quarantine_path.file_name().unwrap().to_str().unwrap(); - let journal_path = quarantine_path.with_file_name(format!("{quarantine_name}.receipt-v1.json")); - let legacy_journal = br#"{"version":1,"kind":"Unregistered","project_id":"test-project","store_id":"test-store","original_name":"legacy-journal","registry_fence":null}"#; - std::fs::write(&journal_path, legacy_journal).unwrap(); - - let outcomes = recover_existing_store_quarantine( - &profile_root, - &data_root, - unbounded_collection_control(), - ) - .unwrap(); - - assert_eq!( - outcomes, - vec![QuarantineRecoveryOutcome::Retained { - quarantine_path: quarantine_path.clone(), - actual_path: quarantine_path.clone(), - failure: Some(CollectionMutationFailure { - operation: CollectionMutationOperation::ProbeRecoveryJournal, - raw_os_error: None, - target_path: journal_path.clone(), - expected_root_identity: None, - classification: CollectionMutationFailureClassification::NonRetryable, - }), - }] - ); - assert_eq!( - std::fs::read(quarantine_path.join("payload.bin")).unwrap(), - payload - ); - assert_eq!(std::fs::read(journal_path).unwrap(), legacy_journal); - assert!(!data_root.exists()); -} - -#[test] -fn unreadable_pre_rename_journal_reports_the_observed_original_path() { - let tmp = tempfile::TempDir::new().unwrap(); - let profile_root = tmp.path().join("profile"); - let data_root = profile_root.join("stores/unreadable-pre-rename-journal"); - std::fs::create_dir_all(&data_root).unwrap(); - std::fs::write(data_root.join("payload.bin"), b"original bytes").unwrap(); - let expected = capture_store_content_fence(&profile_root, &data_root).unwrap(); - - let result = - quarantine_store_for_verified_collection(&profile_root, &data_root, &expected).unwrap(); - let QuarantineStoreOutcome::Verified(quarantine) = result else { - panic!("fixture must reach verified quarantine"); - }; - let quarantine_path = quarantine.quarantine_path().to_path_buf(); - drop(quarantine); - let quarantine_name = quarantine_path.file_name().unwrap().to_str().unwrap(); - let journal_path = quarantine_path.with_file_name(format!("{quarantine_name}.receipt-v1.json")); - std::fs::rename(&quarantine_path, &data_root).unwrap(); - std::fs::remove_file(&journal_path).unwrap(); - std::fs::create_dir(&journal_path).unwrap(); - - let outcomes = recover_existing_store_quarantine( - &profile_root, - &data_root, - unbounded_collection_control(), - ) - .unwrap(); - - let [ - QuarantineRecoveryOutcome::Retained { - actual_path, - quarantine_path: retained_quarantine_path, - failure: Some(failure), - }, - ] = outcomes.as_slice() - else { - panic!("unreadable journal must retain the observed store: {outcomes:#?}"); - }; - assert_eq!(actual_path, &data_root); - assert_eq!(retained_quarantine_path, &quarantine_path); - assert_eq!( - failure.operation, - CollectionMutationOperation::ProbeRecoveryJournal - ); - assert_eq!(failure.target_path, journal_path); - assert_eq!( - std::fs::read(data_root.join("payload.bin")).unwrap(), - b"original bytes" - ); -} - -#[test] -fn unregistered_pre_rename_journal_clears_at_exact_original() { - let tmp = tempfile::TempDir::new().unwrap(); - let profile_root = tmp.path().join("profile"); - let data_root = profile_root.join("stores/missing-quarantine"); - let payload = b"restored bytes cannot imply journal completion"; - std::fs::create_dir_all(&data_root).unwrap(); - std::fs::write(data_root.join("payload.bin"), payload).unwrap(); - let expected = capture_store_content_fence(&profile_root, &data_root).unwrap(); - - let result = - quarantine_store_for_verified_collection(&profile_root, &data_root, &expected).unwrap(); - let QuarantineStoreOutcome::Verified(quarantine) = result else { - panic!("fixture must reach verified quarantine"); - }; - let quarantine_path = quarantine.quarantine_path().to_path_buf(); - drop(quarantine); - let quarantine_name = quarantine_path.file_name().unwrap().to_str().unwrap(); - let journal_path = quarantine_path.with_file_name(format!("{quarantine_name}.receipt-v1.json")); - let renamed_marker = - quarantine_path.with_file_name(format!("{quarantine_name}.receipt-v1.json.renamed")); - let retired_marker = - quarantine_path.with_file_name(format!("{quarantine_name}.receipt-v1.json.retired")); - std::fs::rename(&quarantine_path, &data_root).unwrap(); - - let outcomes = recover_existing_store_quarantine( - &profile_root, - &data_root, - unbounded_collection_control(), - ) - .unwrap(); - - assert_eq!( - outcomes, - vec![QuarantineRecoveryOutcome::Restored { - restored_path: data_root.clone(), - failure: None, - }] - ); - assert_eq!( - std::fs::read(data_root.join("payload.bin")).unwrap(), - payload - ); - assert_eq!( - capture_store_content_fence(&profile_root, &data_root).unwrap(), - expected - ); - assert!(!quarantine_path.exists()); - assert!(!journal_path.exists()); - assert!(!renamed_marker.exists()); - assert!(!retired_marker.exists()); - assert!( - read_pending_quarantine_receipts(&profile_root) - .unwrap() - .is_empty() - ); -} - -/// A restore rename can complete before the parent-directory sync or journal -/// cleanup. The mounted reader must expose the bytes' original live path, -/// rather than the now-absent quarantine name, while retaining the receipt. -#[test] -fn pending_quarantine_reader_reports_restored_path_when_sync_is_unconfirmed() { - let tmp = tempfile::TempDir::new().unwrap(); - let profile_root = tmp.path().join("profile"); - let data_root = profile_root.join("stores/restore-sync-boundary"); - std::fs::create_dir_all(&data_root).unwrap(); - std::fs::write( - data_root.join("payload.bin"), - b"restore location is authoritative", - ) - .unwrap(); - let expected = capture_store_content_fence(&profile_root, &data_root).unwrap(); - - let result = - quarantine_store_for_verified_collection(&profile_root, &data_root, &expected).unwrap(); - let QuarantineStoreOutcome::Verified(quarantine) = result else { - panic!("fixture must reach verified quarantine"); - }; - let quarantine_path = quarantine.quarantine_path().to_path_buf(); - drop(quarantine); - // This is the persisted shape after the rename succeeds but a later - // directory sync/journal cleanup cannot be confirmed. - std::fs::rename(&quarantine_path, &data_root).unwrap(); - - let receipts = read_pending_quarantine_receipts(&profile_root).unwrap(); - assert_eq!(receipts.len(), 1); - assert_eq!(receipts[0].quarantine_path, quarantine_path); - assert_eq!(receipts[0].actual_path, data_root); -} - -/// Cancellation is checked before any recursive SHA-256 read. A cancelled -/// maintenance admission cannot turn a deep inventory into a partial plan or -/// an implicit deletion permit. -#[tokio::test] -async fn registered_collection_payload_fence_cancellation_is_terminal() { - let tmp = tempfile::TempDir::new().unwrap(); - let profile_root = tmp.path().join("profile"); - let data_root = profile_root.join("stores/payload-fence-cancelled"); - seed_payload_fence_work(&data_root); - let (_runtime, db) = open_registered_db(&profile_root).await; - let finding = payload_fence_finding(data_root.clone(), "stores/payload-fence-cancelled"); - let plan = CollectionPlan { - collect: vec![finding], - ..CollectionPlan::default() - }; - let cancellation = CancellationToken::new(); - let started = std::sync::Arc::new(AtomicBool::new(false)); - let started_thread = std::sync::Arc::clone(&started); - let cancellation_thread = cancellation.clone(); - let signal = std::thread::spawn(move || { - while !started_thread.load(Ordering::Acquire) { - std::thread::yield_now(); - } - std::thread::sleep(Duration::from_millis(20)); - cancellation_thread.cancel(); - }); - started.store(true, Ordering::Release); - - let (outcome, retired) = execute_registered_collection_controlled( - &db, - &plan, - &profile_root, - CollectionControl::new( - &cancellation, - MonotonicDeadline::at(Instant::now() + Duration::from_secs(5)), - ), - ) - .await - .unwrap(); - signal.join().unwrap(); - - assert_eq!(retired, 0); - assert_eq!(outcome.completion, CollectionCompletionV1::Cancelled); - assert!(outcome.errors.is_empty()); - assert!(outcome.collected.is_empty()); - assert!(data_root.exists()); -} - -#[tokio::test] -async fn unregistered_collection_payload_fence_deadline_is_distinct() { - let tmp = tempfile::TempDir::new().unwrap(); - let profile_root = tmp.path().join("profile"); - let data_root = profile_root.join("projects/proj_payload_fence_deadline"); - seed_payload_fence_work(&data_root); - let (_runtime, db) = open_registered_db(&profile_root).await; - let finding = payload_fence_finding(data_root.clone(), "projects/proj_payload_fence_deadline"); - let plan = UnregisteredCollectionPlan { - collect: vec![UnregisteredStoreFinding { - project_dir_name: "proj_payload_fence_deadline".to_owned(), - data_root: finding.data_root, - age_secs: finding.age_secs, - size_bytes: finding.size_bytes, - expected_payload_mtime_secs: finding.expected_payload_mtime_secs, - expected_data_root_fence: finding.expected_data_root_fence, - expected_content_fence: finding.expected_content_fence, - abandoned_root: false, - }], - ..UnregisteredCollectionPlan::default() - }; - - let cancellation = CancellationToken::new(); - let (outcome, deadline) = { - let deadline = MonotonicDeadline::at(Instant::now() + Duration::from_millis(20)); - let outcome = execute_unregistered_collection_controlled( - &db, - &plan, - &profile_root, - CollectionControl::new(&cancellation, deadline), - ) - .await - .unwrap(); - (outcome, deadline) - }; - - assert!(deadline.is_elapsed_at(Instant::now())); - assert_eq!(outcome.completion, CollectionCompletionV1::DeadlineExceeded); - assert!(outcome.errors.is_empty()); - assert!(outcome.collected.is_empty()); - assert!(data_root.exists()); -} - -/// Once SQL retirement has been marked, cancellation during recursive remove -/// retains the journal-backed quarantine rather than reporting reclaimed -/// bytes. Restart reconciliation owns the remaining irreversible work. -#[test] -fn cancelled_quarantine_finalization_retains_a_readable_recovery_record() { - let tmp = tempfile::TempDir::new().unwrap(); - let profile_root = tmp.path().join("profile"); - let data_root = profile_root.join("stores/cancelled-finalize"); - std::fs::create_dir_all(&data_root).unwrap(); - std::fs::write(data_root.join("payload.bin"), b"retain while cancelled").unwrap(); - let expected = capture_store_content_fence(&profile_root, &data_root).unwrap(); - let result = - quarantine_store_for_verified_collection(&profile_root, &data_root, &expected).unwrap(); - let QuarantineStoreOutcome::Verified(quarantine) = result else { - panic!("fixture must reach verified quarantine"); - }; - quarantine.mark_retirement_committed().unwrap(); - let quarantine_path = quarantine.quarantine_path().to_path_buf(); - let cancellation = CancellationToken::new(); - cancellation.cancel(); - - assert!(matches!( - quarantine.finalize(CollectionControl::new( - &cancellation, - MonotonicDeadline::at(Instant::now() + Duration::from_secs(1),), - )), - QuarantineFinalizeOutcome::Interrupted { .. } - )); - assert!(quarantine_path.is_dir()); - assert_eq!( - read_pending_quarantine_receipts(&profile_root) - .unwrap() - .len(), - 1 - ); -} - -#[test] -fn legacy_recovery_rejects_post_rename_identity_replacement() { - let tmp = tempfile::TempDir::new().unwrap(); - let profile_root = tmp.path().join("profile"); - let projects = profile_root.join("projects"); - let data_root = projects.join("proj_identity_race"); - let quarantine_name = ".tracedecay-orphan-quarantine-proj_identity_race-42-7"; - let quarantine = projects.join(quarantine_name); - std::fs::create_dir_all(&data_root).unwrap(); - std::fs::write(data_root.join("payload.bin"), b"exact quarantined bytes").unwrap(); - let expected = capture_store_content_fence(&profile_root, &data_root).unwrap(); - let StoreContentFence::Present(inventory) = expected else { - panic!("fixture must capture the legacy quarantine identity"); - }; - let expected_root_identity = inventory.root; - std::fs::rename(&data_root, &quarantine).unwrap(); - - let outcome = recover_named_store_quarantine_controlled( - &profile_root, - &data_root, - std::ffi::OsStr::new(quarantine_name), - &projects, - || { - std::fs::rename(&data_root, &quarantine).unwrap(); - std::fs::create_dir_all(&data_root).unwrap(); - std::fs::write(data_root.join("payload.bin"), b"replacement live bytes").unwrap(); - }, - ) - .unwrap(); - - assert_eq!( - outcome, - Some(QuarantineRecoveryOutcome::Retained { - quarantine_path: quarantine.clone(), - actual_path: quarantine.clone(), - failure: Some(CollectionMutationFailure { - operation: CollectionMutationOperation::RestoreLiveLeafFromQuarantine, - raw_os_error: Some(OCCUPIED_RENAME_RAW_OS_ERROR), - target_path: quarantine.clone(), - expected_root_identity: Some(expected_root_identity), - classification: CollectionMutationFailureClassification::NonRetryable, - }), - }) - ); - assert_eq!( - std::fs::read(quarantine.join("payload.bin")).unwrap(), - b"exact quarantined bytes" - ); - assert_eq!( - std::fs::read(data_root.join("payload.bin")).unwrap(), - b"replacement live bytes" - ); -} - -#[cfg(windows)] -#[test] -fn unreadable_recovery_journal_is_a_retryable_typed_failure() { - let journal_path = PathBuf::from( - r"C:\profile\projects\.tracedecay-orphan-quarantine-proj_journal-42-7.receipt-v1.json", - ); - let expected_root_identity = StoreRootIdentity { - device: 17, - inode: 23, - }; - - assert_eq!( - classify_recovery_journal_probe( - Err(std::io::Error::from_raw_os_error(32)), - journal_path.clone(), - &expected_root_identity, - ), - Err(CollectionFailureKind::RemoveFailed( - CollectionMutationFailure { - operation: CollectionMutationOperation::ProbeRecoveryJournal, - raw_os_error: Some(32), - target_path: journal_path, - expected_root_identity: Some(expected_root_identity), - classification: CollectionMutationFailureClassification::RetryableDeferred, - } - )) - ); -} - -/// Unregistered projects are an on-disk-only class, but their retention work -/// still advances through a bounded, resumable page rather than recursing the -/// entire profile under a single writer admission. -#[test] -fn committed_unregistered_recovery_preserves_interrupted_control_and_resumes() { - for expected_completion in [ - CollectionCompletionV1::Cancelled, - CollectionCompletionV1::DeadlineExceeded, - ] { - let tmp = tempfile::TempDir::new().unwrap(); - let profile_root = tmp.path().join("profile"); - let projects = profile_root.join("projects"); - let data_root = projects.join("proj_controlled_recovery"); - std::fs::create_dir_all(data_root.join("nested")).unwrap(); - std::fs::write( - data_root.join("nested/payload.bin"), - b"retained exact bytes", - ) - .unwrap(); - let expected = capture_store_content_fence(&profile_root, &data_root).unwrap(); - let QuarantineStoreOutcome::Verified(quarantine) = - quarantine_store_for_verified_collection(&profile_root, &data_root, &expected).unwrap() - else { - panic!("fixture must reach verified quarantine"); - }; - let quarantine_path = quarantine.quarantine_path().to_path_buf(); - quarantine.mark_retirement_committed().unwrap(); - drop(quarantine); - let pending = read_pending_quarantine_receipts(&profile_root).unwrap(); - assert_eq!(pending.len(), 1); - assert!(pending[0].retirement_committed); - let cancellation = CancellationToken::new(); - let deadline = if expected_completion == CollectionCompletionV1::DeadlineExceeded { - MonotonicDeadline::at(Instant::now()) - } else { - cancellation.cancel(); - MonotonicDeadline::at(Instant::now() + Duration::from_secs(1)) - }; - let control = CollectionControl::new(&cancellation, deadline); - let outcome = recover_named_store_quarantine( - &profile_root, - &data_root, - quarantine_path.file_name().unwrap(), - &projects, - control, - ) - .unwrap(); - assert_eq!( - outcome, - Some(QuarantineRecoveryOutcome::Retained { - quarantine_path: quarantine_path.clone(), - actual_path: quarantine_path.clone(), - failure: None, - }) - ); - assert_eq!(control.completion(), Some(expected_completion)); - assert_eq!( - std::fs::read(quarantine_path.join("nested/payload.bin")).unwrap(), - b"retained exact bytes" - ); - assert_eq!( - read_pending_quarantine_receipts(&profile_root).unwrap(), - pending - ); - assert!(!data_root.exists()); - - let mut collection = CollectionOutcome::default(); - assert!(!reconcile_existing_quarantine( - &profile_root, - &data_root, - "proj_controlled_recovery", - &mut collection, - control, - )); - assert_eq!(collection.completion, expected_completion); - assert_eq!(collection.reclaimed_bytes, 0); - assert!(collection.collected.is_empty()); - assert_eq!( - read_pending_quarantine_receipts(&profile_root).unwrap(), - pending - ); - - let fresh = CancellationToken::new(); - let resumed = recover_named_store_quarantine( - &profile_root, - &data_root, - quarantine_path.file_name().unwrap(), - &projects, - CollectionControl::new( - &fresh, - MonotonicDeadline::at(Instant::now() + Duration::from_secs(1)), - ), - ) - .unwrap(); - assert_eq!( - resumed, - Some(QuarantineRecoveryOutcome::Removed { - quarantine_path: quarantine_path.clone(), - journal_failure: None, - }) - ); - assert!(!quarantine_path.exists()); - assert!( - read_pending_quarantine_receipts(&profile_root) - .unwrap() - .is_empty() - ); - } -} - -#[tokio::test] -async fn unregistered_recovery_removal_has_no_fabricated_bytes_or_receipt() { - for already_removed in [false, true] { - let tmp = tempfile::TempDir::new().unwrap(); - let profile_root = tmp.path().join("profile"); - let data_root = profile_root.join("projects/proj_committed_recovery"); - std::fs::create_dir_all(&data_root).unwrap(); - std::fs::write(data_root.join("payload.bin"), b"remove through pager").unwrap(); - let expected = capture_store_content_fence(&profile_root, &data_root).unwrap(); - let result = - quarantine_store_for_verified_collection(&profile_root, &data_root, &expected).unwrap(); - let QuarantineStoreOutcome::Verified(quarantine) = result else { - panic!("fixture must reach verified quarantine"); - }; - let quarantine_path = quarantine.quarantine_path().to_path_buf(); - quarantine.mark_retirement_committed().unwrap(); - drop(quarantine); - if already_removed { - // Model a crash after deleting the committed leaf but before clearing - // its journal. The pager must discover the journal without a directory. - std::fs::remove_dir_all(&quarantine_path).unwrap(); - } - let (_runtime, db) = open_registered_db(&profile_root).await; - let cancellation = CancellationToken::new(); - - let report = sweep_unregistered_store_page( - &db, - &profile_root, - UnregisteredStoreSweepRequestV1 { - cursor: None, - limit: 4, - retention_secs: 0, - now: 1_700_000_000, - apply: true, - cancellation: &cancellation, - deadline: MonotonicDeadline::at(Instant::now() + Duration::from_secs(1)), - }, - ) - .await - .unwrap(); - - assert_eq!(report.completion, UnregisteredSweepCompletionV1::Complete); - assert_eq!(report.outcome.reclaimed_bytes, 0); - assert!(report.outcome.collected.is_empty()); - assert!(report.outcome.recovery_receipts.is_empty()); - assert!(report.outcome.errors.is_empty()); - assert!(!quarantine_path.exists()); - assert!( - read_pending_quarantine_receipts(&profile_root) - .unwrap() - .is_empty() - ); - } -} - -/// A durable-memory guard applies to unregistered directories exactly as it -/// does to registered orphan stores. -#[tokio::test] -async fn sweep_unregistered_stores_never_deletes_durable_memory_rows() { - let tmp = tempfile::TempDir::new().unwrap(); - let profile_root = tmp.path().join("profile"); - std::fs::create_dir_all(&profile_root).unwrap(); - let (_runtime, db) = open_registered_db(&profile_root).await; - - let base = 1_700_000_000i64; - let dir = profile_root.join("projects").join("proj_ghost_with_memory"); - std::fs::create_dir_all(&dir).unwrap(); - { - let connection = - rusqlite::Connection::open(dir.join(tracedecay_runtime_core::config::DB_FILENAME)) - .unwrap(); - connection - .execute_batch( - "CREATE TABLE memory_facts (fact_id INTEGER PRIMARY KEY, content TEXT NOT NULL); - INSERT INTO memory_facts (fact_id, content) VALUES (1, 'durable fact');", - ) - .unwrap(); - } - filetime::set_file_mtime( - dir.join(tracedecay_runtime_core::config::DB_FILENAME), - filetime::FileTime::from_unix_time(base - 100 * DAY, 0), - ) - .unwrap(); - - let report = sweep_unregistered_stores(&db, &profile_root, 7 * DAY, base, true) - .await - .unwrap(); - assert!(report.outcome.collected.is_empty()); - assert_eq!( - report.outcome.errors[0].kind, - CollectionFailureKind::DurableDataProtected - ); - assert!(dir.exists()); -} - -/// The durable-data check covers the manifest-selected project graph and every -/// registered project graph scope, and refuses to answer when the manifest -/// that names them cannot be read. -mod durable_inventory { - use super::*; - - fn manifest_bytes(graph_db_relpath: &str) -> Vec { - let project_root = PathBuf::from("/definitely/not/here/gone"); - let manifest = StoreManifest { - schema_version: STORE_MANIFEST_SCHEMA_VERSION, - project_id: Some("proj_inventory".to_string()), - store_kind: StoreKind::CodeProject, - storage_mode: StorageMode::ProfileSharded, - project_root: project_root.clone(), - data_root: project_root, - graph_db_relpath: PathBuf::from(graph_db_relpath), - sessions_db_relpath: PathBuf::from("sessions.db"), - branch_meta_relpath: PathBuf::from( - tracedecay_runtime_core::storage::BRANCH_META_FILENAME, - ), - }; - serde_json::to_vec(&manifest).unwrap() - } - - #[test] - fn registered_graph_scopes_at_custom_paths_are_covered() { - let store = tempfile::tempdir().unwrap(); - let custom = PathBuf::from("scopes/custom-scope.db"); - - let DurableDatabaseInventoryV1::Resolved(inventory) = durable_database_inventory( - store.path(), - Some(&manifest_bytes("custom-main.db")), - std::slice::from_ref(&custom), - unbounded_collection_control(), - ) else { - panic!("a readable manifest must resolve an inventory"); - }; - - assert!( - inventory.contains(&PathBuf::from("custom-main.db")), - "the manifest's custom main graph path must be honoured, not the default filename" - ); - assert!(inventory.contains(&custom)); - } - - #[test] - fn branch_databases_are_part_of_the_inventory() { - let store = tempfile::tempdir().unwrap(); - std::fs::create_dir_all(store.path().join("branches")).unwrap(); - std::fs::write(store.path().join("branches/feature-x.db"), b"").unwrap(); - std::fs::write(store.path().join("branches/main.db"), b"").unwrap(); - std::fs::write(store.path().join("branches/notes.txt"), b"").unwrap(); - - let DurableDatabaseInventoryV1::Resolved(inventory) = durable_database_inventory( - store.path(), - Some(&manifest_bytes("code.db")), - &[], - unbounded_collection_control(), - ) else { - panic!("a readable manifest must resolve an inventory"); - }; - - assert!( - inventory.contains(&PathBuf::from("branches/feature-x.db")), - "a branch database can hold the only surviving durable rows" - ); - assert!(inventory.contains(&PathBuf::from("branches/main.db"))); - assert!(!inventory.contains(&PathBuf::from("branches/notes.txt"))); - } - - #[test] - fn a_missing_manifest_fails_closed() { - let store = tempfile::tempdir().unwrap(); - assert_eq!( - durable_database_inventory(store.path(), None, &[], unbounded_collection_control()), - DurableDatabaseInventoryV1::Unverifiable, - "without a manifest the store's graph path is a guess, not a fact" - ); - } - - #[test] - fn manifest_graph_path_must_be_normalized_relative() { - for graph_path in [PathBuf::from(""), PathBuf::from("../graph.db")] { - let bytes = manifest_bytes(graph_path.to_string_lossy().as_ref()); - assert_eq!( - durable_database_inventory( - Path::new("/tmp/store"), - Some(&bytes), - &[], - unbounded_collection_control(), - ), - DurableDatabaseInventoryV1::Unverifiable, - "graph path {graph_path:?} must not escape the store" - ); - } - - assert_eq!( - durable_database_inventory( - Path::new("/tmp/store"), - Some(&manifest_bytes("/tmp/graph.db")), - &[], - unbounded_collection_control(), - ), - DurableDatabaseInventoryV1::Unverifiable, - "an absolute graph path must not replace the store root" - ); - } - - #[test] - fn registered_graph_scope_path_must_be_normalized_relative() { - assert_eq!( - durable_database_inventory( - Path::new("/tmp/store"), - Some(&manifest_bytes("graph.db")), - &[PathBuf::from("scopes/../../escape.db")], - unbounded_collection_control(), - ), - DurableDatabaseInventoryV1::Unverifiable - ); - assert_eq!( - durable_database_inventory( - Path::new("/tmp/store"), - Some(&manifest_bytes("graph.db")), - &[PathBuf::from("/tmp/escape.db")], - unbounded_collection_control(), - ), - DurableDatabaseInventoryV1::Unverifiable - ); - } - - #[test] - fn cancelled_control_interrupts_branch_database_inventory() { - let store = tempfile::tempdir().unwrap(); - std::fs::create_dir_all(store.path().join("branches")).unwrap(); - std::fs::write(store.path().join("branches/only-memory.db"), b"").unwrap(); - let cancellation = CancellationToken::new(); - cancellation.cancel(); - - assert_eq!( - durable_database_inventory( - store.path(), - Some(&manifest_bytes("graph.db")), - &[], - CollectionControl::new( - &cancellation, - MonotonicDeadline::at(Instant::now() + Duration::from_secs(1)), - ), - ), - DurableDatabaseInventoryV1::Interrupted, - "a cancelled admission must not finish the lazy branch scan" - ); - } - - #[tokio::test] - async fn a_store_whose_manifest_is_unreadable_is_never_reported_empty() { - let profile = tempfile::tempdir().unwrap(); - let data_root = profile.path().join("stores/unreadable"); - std::fs::create_dir_all(&data_root).unwrap(); - - let check = check_store_durable_memory( - &data_root, - Some(b"{ not json"), - &[], - &durable_check_scratch_root(profile.path()), - unbounded_collection_control(), - ) - .await; - - assert_eq!( - check, - DurableMemoryCheck::Unverifiable, - "an unverifiable inventory must protect the store, never clear it for deletion" - ); - } - - #[tokio::test] - async fn a_pre_cancelled_durable_snapshot_reports_interrupted() { - let profile = tempfile::tempdir().unwrap(); - let data_root = profile.path().join("stores/cancelled"); - std::fs::create_dir_all(&data_root).unwrap(); - rusqlite::Connection::open(data_root.join("graph.db")).unwrap(); - - let cancellation = CancellationToken::new(); - cancellation.cancel(); - let check = check_store_durable_memory( - &data_root, - Some(&manifest_bytes("graph.db")), - &[], - &durable_check_scratch_root(profile.path()), - CollectionControl::new( - &cancellation, - MonotonicDeadline::at(Instant::now() + Duration::from_secs(1)), - ), - ) - .await; - - assert_eq!( - check, - DurableMemoryCheck::Interrupted, - "a pre-cancelled durable snapshot must not report an empty database" - ); - } -} - -#[cfg(unix)] -#[tokio::test] -async fn symlink_manifest_is_unverifiable_and_never_collected() { - let tmp = tempfile::TempDir::new().unwrap(); - let profile_root = tmp.path().join("profile"); - std::fs::create_dir_all(&profile_root).unwrap(); - let dead_root = tmp.path().join("moved-away-repo"); - let (_runtime, db) = open_registered_db(&profile_root).await; - let data_root = seed_store( - &db, - &profile_root, - "proj_symlink_manifest", - "store_symlink_manifest", - &dead_root, - 1_700_000_000 - 100 * DAY, - ) - .await; - let manifest_path = data_root.join(tracedecay_runtime_core::storage::STORE_MANIFEST_FILENAME); - let target = tmp.path().join("manifest-target.json"); - std::fs::copy(&manifest_path, &target).unwrap(); - std::fs::remove_file(&manifest_path).unwrap(); - std::os::unix::fs::symlink(&target, &manifest_path).unwrap(); - - let report = sweep_orphan_stores(&db, &profile_root, 7 * DAY, 1_700_000_000, true) - .await - .unwrap(); - - assert!(report.plan.collect.is_empty()); - assert_eq!(report.plan.unverifiable.len(), 1); - assert!(report.outcome.collected.is_empty()); - assert!(data_root.exists()); -} - -#[cfg(unix)] -#[tokio::test] -async fn symlink_graph_database_is_durable_data_protected() { - let tmp = tempfile::TempDir::new().unwrap(); - let profile_root = tmp.path().join("profile"); - std::fs::create_dir_all(&profile_root).unwrap(); - let dead_root = tmp.path().join("moved-away-repo"); - let (_runtime, db) = open_registered_db(&profile_root).await; - let data_root = seed_store( - &db, - &profile_root, - "proj_symlink_graph", - "store_symlink_graph", - &dead_root, - 1_700_000_000 - 100 * DAY, - ) - .await; - let graph_path = data_root.join("graph.db"); - let target = tmp.path().join("graph-target.db"); - rusqlite::Connection::open(&target).unwrap(); - std::fs::remove_file(&graph_path).unwrap(); - std::os::unix::fs::symlink(&target, &graph_path).unwrap(); - - let report = sweep_orphan_stores(&db, &profile_root, 7 * DAY, 1_700_000_000, true) - .await - .unwrap(); - - assert_eq!(report.plan.collect.len(), 1); - assert!(report.outcome.collected.is_empty()); - assert_eq!(report.outcome.errors.len(), 1); - assert_eq!( - report.outcome.errors[0].kind, - CollectionFailureKind::DurableDataProtected - ); - assert!(data_root.exists()); -} diff --git a/crates/tracedecay-maintenance/src/retention/orphan_stores/unregistered_page.rs b/crates/tracedecay-maintenance/src/retention/orphan_stores/unregistered_page.rs index 6779d80509..2249acee5c 100644 --- a/crates/tracedecay-maintenance/src/retention/orphan_stores/unregistered_page.rs +++ b/crates/tracedecay-maintenance/src/retention/orphan_stores/unregistered_page.rs @@ -13,13 +13,9 @@ use super::fence::{ capture_store_content_fence_controlled, capture_store_directory_fence, open_store_directory_nofollow, }; -use super::quarantine::{ - QuarantineRecoveryOutcome, quarantine_recovery_entry, recover_named_store_quarantine, -}; use super::{ - CollectionCompletionV1, CollectionControl, CollectionFailure, CollectionFailureKind, - CollectionOutcome, CollectionRecoveryAction, CollectionRecoveryReceipt, StoreContentFence, - StoreDirectoryFence, UnregisteredCollectionPlan, UnregisteredStoreFinding, + CollectionCompletionV1, CollectionControl, CollectionFailureKind, CollectionOutcome, + StoreContentFence, StoreDirectoryFence, UnregisteredCollectionPlan, UnregisteredStoreFinding, dir_size_bytes_controlled, execute_unregistered_collection_controlled, manifest_names_abandoned_root, newest_mtime_secs_controlled, plan_unregistered_collection, }; @@ -28,16 +24,9 @@ pub const DEFAULT_UNREGISTERED_STORE_PAGE_LIMIT: usize = 8; const MAX_UNREGISTERED_STORE_PAGE_LIMIT: usize = 64; const UNREGISTERED_STORE_DIRECTORY_ENTRY_MULTIPLIER: usize = 8; -pub(in crate::retention) enum ProjectDirectoryWorkV1 { - Project(String), - Quarantine { - project_id: String, - quarantine_name: String, - }, -} - pub(in crate::retention) struct ProjectDirectoryPageV1 { - pub entries: Vec, + /// Valid `project_id` leaf names under `projects/`. + pub entries: Vec, pub next_cursor: Option, /// Raw directory or portable-inventory entries consumed to produce this /// slice. Summing pages exposes nonlinear rescans without timing heuristics. @@ -109,24 +98,21 @@ pub async fn sweep_unregistered_store_page( CollectionOutcome::default(), ))); } - let mut recovery_outcome = CollectionOutcome::default(); let census = census_unregistered_project_dirs_page( db, profile_root, request.cursor.as_deref(), limit, request.now, - request.apply, request.cancellation, request.deadline, - &mut recovery_outcome, ) .await?; let Some((findings, next_cursor)) = census else { return Ok(observed_page_report(interrupted_report( UnregisteredSweepCompletionV1::interrupted(request.cancellation, request.deadline) .unwrap_or(UnregisteredSweepCompletionV1::DeadlineExceeded), - recovery_outcome, + CollectionOutcome::default(), ))); }; let plan = plan_unregistered_collection(findings, request.retention_secs); @@ -134,7 +120,7 @@ pub async fn sweep_unregistered_store_page( return Ok(observed_page_report(UnregisteredStoreSweepReport { plan, applied: false, - outcome: recovery_outcome, + outcome: CollectionOutcome::default(), next_cursor, completion: UnregisteredSweepCompletionV1::Complete, })); @@ -145,26 +131,18 @@ pub async fn sweep_unregistered_store_page( return Ok(observed_page_report(UnregisteredStoreSweepReport { plan: UnregisteredCollectionPlan::default(), applied: false, - outcome: recovery_outcome, + outcome: CollectionOutcome::default(), next_cursor: request.cursor, completion, })); } - let mut outcome = execute_unregistered_collection_controlled( + let outcome = execute_unregistered_collection_controlled( db, &plan, profile_root, CollectionControl::new(request.cancellation, request.deadline), ) .await?; - outcome.reclaimed_bytes = outcome - .reclaimed_bytes - .saturating_add(recovery_outcome.reclaimed_bytes); - outcome.collected.extend(recovery_outcome.collected); - outcome.errors.extend(recovery_outcome.errors); - outcome - .recovery_receipts - .extend(recovery_outcome.recovery_receipts); let completion = match outcome.completion { CollectionCompletionV1::Complete => UnregisteredSweepCompletionV1::Complete, CollectionCompletionV1::Cancelled => UnregisteredSweepCompletionV1::Cancelled, @@ -228,10 +206,8 @@ async fn census_unregistered_project_dirs_page( cursor: Option<&str>, limit: usize, now: i64, - recover_interrupted_quarantines: bool, cancellation: &CancellationToken, deadline: MonotonicDeadline, - recovery_outcome: &mut CollectionOutcome, ) -> tracedecay_domain::errors::Result, Option)>> { let projects_dir = profile_root.join("projects"); let interrupted = @@ -241,166 +217,57 @@ async fn census_unregistered_project_dirs_page( return Ok(None); }; let next_cursor = page.next_cursor; - let mut recovered_project_ids = HashSet::new(); let mut findings = Vec::with_capacity(page.entries.len()); - for work in page.entries { + for name in page.entries { if UnregisteredSweepCompletionV1::interrupted(cancellation, deadline).is_some() { return Ok(None); } - let ProjectDirectoryWorkV1::Quarantine { - project_id, - quarantine_name, - } = work - else { - let ProjectDirectoryWorkV1::Project(name) = work else { - continue; - }; - let control = CollectionControl::new(cancellation, deadline); - let is_registered = match control.race(db.code_project_exists(&name)).await { - Ok(Ok(exists)) => exists, - Ok(Err(error)) => return Err(error), - Err(_) => return Ok(None), - }; - if is_registered { - continue; - } - if recovered_project_ids.contains(&name) { - continue; - } - let data_root = projects_dir.join(&name); - let metadata = match std::fs::symlink_metadata(&data_root) { - Ok(metadata) => metadata, - Err(_) => continue, - }; - if metadata.file_type().is_symlink() || !metadata.is_dir() { - continue; - } - let expected_data_root_fence = capture_store_directory_fence(profile_root, &data_root) - .unwrap_or(StoreDirectoryFence::Unverifiable); - let expected_content_fence = match capture_store_content_fence_controlled( - profile_root, - &data_root, - CollectionControl::new(cancellation, deadline), - ) { + let control = CollectionControl::new(cancellation, deadline); + let is_registered = match control.race(db.code_project_exists(&name)).await { + Ok(Ok(exists)) => exists, + Ok(Err(error)) => return Err(error), + Err(_) => return Ok(None), + }; + if is_registered { + continue; + } + let data_root = projects_dir.join(&name); + let metadata = match std::fs::symlink_metadata(&data_root) { + Ok(metadata) => metadata, + Err(_) => continue, + }; + if metadata.file_type().is_symlink() || !metadata.is_dir() { + continue; + } + let expected_data_root_fence = capture_store_directory_fence(profile_root, &data_root) + .unwrap_or(StoreDirectoryFence::Unverifiable); + let expected_content_fence = + match capture_store_content_fence_controlled(profile_root, &data_root, control) { Ok(fence) => fence, Err(CollectionFailureKind::Cancelled) => return Ok(None), Err(_) => StoreContentFence::Unverifiable, }; - if UnregisteredSweepCompletionV1::interrupted(cancellation, deadline).is_some() { - return Ok(None); - } - let last_write_secs = match newest_mtime_secs_controlled(&data_root, control) { - Ok(mtime) => mtime, - Err(CollectionFailureKind::Cancelled) => return Ok(None), - Err(_) => return Ok(None), - }; - let size_bytes = match dir_size_bytes_controlled(&data_root, control) { - Ok(size) => size, - Err(CollectionFailureKind::Cancelled) => return Ok(None), - Err(_) => return Ok(None), - }; - let abandoned_root = manifest_names_abandoned_root(&data_root, profile_root); - findings.push(UnregisteredStoreFinding { - project_dir_name: name, - data_root, - age_secs: now.saturating_sub(last_write_secs).max(0), - size_bytes, - expected_payload_mtime_secs: last_write_secs, - expected_data_root_fence, - expected_content_fence, - abandoned_root, - }); - continue; - }; - if recover_interrupted_quarantines { - if recovered_project_ids.contains(&project_id) { - continue; - } - let data_root = projects_dir.join(&project_id); - let record_recovery = match recover_named_store_quarantine( - profile_root, - &data_root, - std::ffi::OsStr::new(&quarantine_name), - &projects_dir, - CollectionControl::new(cancellation, deadline), - ) { - Ok(Some(QuarantineRecoveryOutcome::Removed { - journal_failure, .. - })) => { - recovered_project_ids.insert(project_id.clone()); - if let Some(failure) = journal_failure { - recovery_outcome.errors.push(CollectionFailure { - store_id: project_id.clone(), - kind: CollectionFailureKind::RemoveFailed(failure), - }); - } - None - } - Ok(Some(QuarantineRecoveryOutcome::Restored { - restored_path, - failure, - })) => { - recovered_project_ids.insert(project_id.clone()); - Some(( - projects_dir.join(&quarantine_name), - restored_path, - if failure.is_some() { - CollectionRecoveryAction::RetainedForRecovery - } else { - CollectionRecoveryAction::Restored - }, - failure, - )) - } - Ok(Some(QuarantineRecoveryOutcome::Retained { - quarantine_path, - actual_path, - failure, - })) => { - recovered_project_ids.insert(project_id.clone()); - Some(( - quarantine_path.clone(), - actual_path, - CollectionRecoveryAction::RetainedForRecovery, - failure, - )) - } - Ok(None) => None, - Err(kind) => { - recovery_outcome.errors.push(CollectionFailure { - store_id: project_id.clone(), - kind, - }); - None - } - }; - if let Some((quarantine_path, actual_path, action, failure)) = record_recovery { - recovery_outcome - .recovery_receipts - .push(CollectionRecoveryReceipt { - store_id: project_id.clone(), - original_path: data_root.clone(), - actual_path, - quarantine_path, - action, - }); - if let Some(failure) = failure { - recovery_outcome.errors.push(CollectionFailure { - store_id: project_id.clone(), - kind: CollectionFailureKind::RemoveFailed(failure), - }); - } - recovery_outcome.errors.push(CollectionFailure { - store_id: project_id, - kind: CollectionFailureKind::PayloadChanged, - }); - } + if UnregisteredSweepCompletionV1::interrupted(cancellation, deadline).is_some() { + return Ok(None); } + let Ok(last_write_secs) = newest_mtime_secs_controlled(&data_root, control) else { + return Ok(None); + }; + let Ok(size_bytes) = dir_size_bytes_controlled(&data_root, control) else { + return Ok(None); + }; + let abandoned_root = manifest_names_abandoned_root(&data_root, profile_root); + findings.push(UnregisteredStoreFinding { + project_dir_name: name, + data_root, + age_secs: now.saturating_sub(last_write_secs).max(0), + size_bytes, + expected_payload_mtime_secs: last_write_secs, + expected_data_root_fence, + expected_content_fence, + abandoned_root, + }); } - // Directory inventories preserve discovery order. A live leaf can precede - // its quarantine; recovery invalidates that earlier census just as it - // prevents a later one, so neither may reach collection in this admission. - findings.retain(|finding| !recovered_project_ids.contains(&finding.project_dir_name)); Ok(Some((findings, next_cursor))) } @@ -578,13 +445,8 @@ pub(in crate::retention) fn read_project_directory_page( })?; scanned = scanned.saturating_add(1); entries_scanned = entries_scanned.saturating_add(1); - if let Some((project_id, quarantine_name)) = quarantine_recovery_entry(&name) { - work.push(ProjectDirectoryWorkV1::Quarantine { - project_id, - quarantine_name, - }); - } else if tracedecay_runtime_core::storage::validate_project_id(&name).is_ok() { - work.push(ProjectDirectoryWorkV1::Project(name)); + if portable_inventory_entry_is_valid(&name) { + work.push(name); } if work.len() == limit || scanned >= scan_limit { return Ok(Some(ProjectDirectoryPageV1 { @@ -665,8 +527,7 @@ fn portable_inventory_matches(path: &Path, signature: &str) -> bool { } pub(super) fn portable_inventory_entry_is_valid(name: &str) -> bool { - quarantine_recovery_entry(name).is_some() - || tracedecay_runtime_core::storage::validate_project_id(name).is_ok() + tracedecay_runtime_core::storage::validate_project_id(name).is_ok() } fn clear_portable_inventory_complete(inventory: &Path) -> std::io::Result<()> { diff --git a/crates/tracedecay-maintenance/src/retention/storage_report.rs b/crates/tracedecay-maintenance/src/retention/storage_report.rs index d28c3978f8..a7eb303204 100644 --- a/crates/tracedecay-maintenance/src/retention/storage_report.rs +++ b/crates/tracedecay-maintenance/src/retention/storage_report.rs @@ -571,17 +571,9 @@ fn list_project_directories_page( let directories = page .entries .into_iter() - .map(|entry| match entry { - super::orphan_stores::ProjectDirectoryWorkV1::Project(name) => { - let path = projects_dir.join(&name); - (name, path) - } - super::orphan_stores::ProjectDirectoryWorkV1::Quarantine { - quarantine_name, .. - } => { - let path = projects_dir.join(&quarantine_name); - (quarantine_name, path) - } + .map(|name| { + let path = projects_dir.join(&name); + (name, path) }) .collect::>(); Ok(ProjectDirectoryPage { diff --git a/crates/tracedecay-maintenance/src/store_maintenance/graph_replay.rs b/crates/tracedecay-maintenance/src/store_maintenance/graph_replay.rs index 85ec80cb63..a18b838f54 100644 --- a/crates/tracedecay-maintenance/src/store_maintenance/graph_replay.rs +++ b/crates/tracedecay-maintenance/src/store_maintenance/graph_replay.rs @@ -133,7 +133,7 @@ pub async fn reconcile_graph_replay_releases( lease: &ProjectStoreMaintenanceLeaseV1, store_root: &Path, observations: &crate::telemetry::StoreTelemetrySamplingRegistry, - cancellation: &tracedecay_session_memory::context::CancellationToken, + cancellation: &tracedecay_runtime_core::cancellation::CancellationToken, ) -> ReconcileOutcome { let Some(project_id) = lease.store_layout().identity.project_id.as_ref() else { log_code_generation_retention_degraded( diff --git a/crates/tracedecay-maintenance/src/store_maintenance/mod.rs b/crates/tracedecay-maintenance/src/store_maintenance/mod.rs index 8e5da03c55..ff88f98799 100644 --- a/crates/tracedecay-maintenance/src/store_maintenance/mod.rs +++ b/crates/tracedecay-maintenance/src/store_maintenance/mod.rs @@ -12,7 +12,6 @@ use std::path::Path; use crate::lease::ProjectStoreMaintenanceLeaseV1; use crate::telemetry::StoreTelemetrySamplingRegistry; use tracedecay_code_index_runtime::code_index_scheduler::CodeIndexSchedulerRegistryV1; -use tracedecay_contracts::storage::compaction::CompactionThresholdConfig; use tracedecay_runtime_core::logging::log_daemon_event; mod graph_replay; @@ -21,10 +20,11 @@ use graph_replay::{defer_graph_replay_pool_busy, log_code_generation_retention_d /// Outcome of one bounded code-generation retention pass. /// /// `MoreWork` reports bounded progress with a remaining backlog, another -/// collectable superseded generation, or unconsumed graph-replay release -/// evidence, so the maintenance owner keeps the short cadence until the -/// store converges instead of parking multi-GiB debris behind the full -/// maintenance interval. +/// collectable superseded generation, superseded bytes a transient holder +/// (serving seat, in-flight text replacement) is about to release, or +/// unconsumed graph-replay release evidence, so the maintenance owner keeps +/// the short cadence until the store converges instead of parking multi-GiB +/// debris behind the full maintenance interval. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum CodeGenerationRetentionOutcomeV1 { Complete, @@ -57,7 +57,7 @@ pub async fn run_code_generation_retention( lease: &ProjectStoreMaintenanceLeaseV1, schedulers: &CodeIndexSchedulerRegistryV1, observations: &StoreTelemetrySamplingRegistry, - cancellation: &tracedecay_session_memory::context::CancellationToken, + cancellation: &tracedecay_runtime_core::cancellation::CancellationToken, ) -> CodeGenerationRetentionOutcomeV1 { use tracedecay_code_index_retention::code_index_generations::{ CodeGenerationRetentionErrorV1, CodeGenerationRetentionModeV1, @@ -90,7 +90,8 @@ pub async fn run_code_generation_retention( .graph_db() .database_path() .with_extension("graph-replay"); - let mut protected_sources = serving_generation_pins(schedulers, &layout.project_root).await; + let serving_pins = serving_generation_pins(schedulers, &layout.project_root).await; + let mut protected_sources = serving_pins.clone(); // Native previews bind retained-only candidate generations between // preflight and terminal apply. Their durable commitments are liveness // roots; omitting them lets an ordinary maintenance tick collect the exact @@ -227,10 +228,15 @@ pub async fn run_code_generation_retention( false } }; + // A superseded generation still named by the serving or text slot, or a + // text replacement still building, is released by a seat or descriptor + // publication that does not wake maintenance. Without the short cadence + // its bytes wait for the next full interval (a day by default). + let awaits_transient_release = plan.awaits_transient_release(&serving_pins); if !plan.has_collectable_work() { return if replay_reconcile_failed { CodeGenerationRetentionOutcomeV1::Failed - } else if release_backlog_remains { + } else if release_backlog_remains || awaits_transient_release { CodeGenerationRetentionOutcomeV1::MoreWork } else { CodeGenerationRetentionOutcomeV1::Complete @@ -328,6 +334,7 @@ pub async fn run_code_generation_retention( if release_reconcile_failed { CodeGenerationRetentionOutcomeV1::Failed } else if release_backlog_remains + || awaits_transient_release || report.generation_segment_batch_exhausted || !report.deleted_generations.is_empty() || !report.deleted_text_artifacts.is_empty() @@ -399,82 +406,3 @@ async fn serving_generation_pins( } pins } - -/// Runs bounded incremental-vacuum compaction over every tracked branch -/// database other than the one `cg` currently has mounted (the maintenance -/// owner compacts that store through its live-runtime authority). Best-effort -/// and independent per file: a busy or failing branch database never blocks -/// the rest, but keeps the maintenance cadence retry-eligible, see -/// `src/retention/branch_compaction.rs` for the compaction policy itself. -#[hotpath::measure(label = "daemon.git.maintenance.branch_compaction")] -pub fn run_branch_compaction( - lease: &ProjectStoreMaintenanceLeaseV1, - config: &CompactionThresholdConfig, -) -> bool { - let layout = lease.store_layout(); - let Some(meta) = tracedecay_runtime_core::branch_meta::load_branch_meta(&layout.data_root) - else { - return true; - }; - let active_db_path = layout.graph_db_path.clone(); - let candidates = crate::retention::branch_compaction::select_branch_db_candidates( - &layout.data_root, - &meta, - &active_db_path, - ); - if candidates.is_empty() { - return true; - } - let report = crate::retention::branch_compaction::compact_branch_databases(&candidates, config); - if report.policy_invalid { - // Never silent: an out-of-range threshold disables the pass entirely - // and would otherwise be indistinguishable from "nothing to compact". - log_daemon_event( - "retention_degraded", - &[ - ("pass", "branch_compaction".to_string()), - ("failure", "invalid_compaction_policy".to_string()), - ( - "free_page_ratio_threshold", - config.free_page_ratio_threshold.to_string(), - ), - ], - ); - return false; - } - if report.compacted.is_empty() && report.skipped.is_empty() { - return true; - } - let freed_pages: u64 = report - .compacted - .iter() - .map(|outcome| outcome.freed_pages) - .sum(); - let unreclaimable = report - .skipped - .iter() - .filter(|skip| { - skip.reason - == crate::retention::branch_compaction::BranchCompactionSkipReason::IncrementalVacuumUnavailable - }) - .count(); - log_daemon_event( - "retention_branch_compaction", - &[ - ("project", lease.project_root().display().to_string()), - ("compacted", report.compacted.len().to_string()), - ("freed_pages", freed_pages.to_string()), - ("skipped", report.skipped.len().to_string()), - // Branch databases predating `auto_vacuum = INCREMENTAL`: their - // free pages need a full VACUUM this pass deliberately avoids. - ("unreclaimable", unreclaimable.to_string()), - ], - ); - branch_compaction_succeeded(&report) -} - -pub fn branch_compaction_succeeded( - report: &crate::retention::branch_compaction::BranchCompactionReport, -) -> bool { - !report.policy_invalid && report.skipped.is_empty() -} diff --git a/crates/tracedecay-maintenance/src/tick.rs b/crates/tracedecay-maintenance/src/tick.rs index 2adc2c83e9..bb6cafa6b8 100644 --- a/crates/tracedecay-maintenance/src/tick.rs +++ b/crates/tracedecay-maintenance/src/tick.rs @@ -173,13 +173,3 @@ pub fn cursor_after_attempted_units( .cloned() .or_else(|| prior.map(str::to_owned)) } - -/// Whether any retention or compaction window is configured. -#[must_use] -pub fn retention_maintenance_enabled( - orphan_store_gc_days: Option, - incident_debris_retention_days: Option, - compaction: bool, -) -> bool { - orphan_store_gc_days.is_some() || incident_debris_retention_days.is_some() || compaction -} diff --git a/crates/tracedecay-mcp-catalog/Cargo.toml b/crates/tracedecay-mcp-catalog/Cargo.toml index e106271582..59dd1d168c 100644 --- a/crates/tracedecay-mcp-catalog/Cargo.toml +++ b/crates/tracedecay-mcp-catalog/Cargo.toml @@ -13,7 +13,7 @@ doctest = false [dependencies] hotpath.workspace = true # The multi-root execute tool derives its input schema from the contracts DTO. -schemars = "1.2.1" +schemars.workspace = true serde = { version = "1", features = ["derive"] } serde_json = "1" thiserror = "2" diff --git a/crates/tracedecay-mcp-catalog/src/definitions.rs b/crates/tracedecay-mcp-catalog/src/definitions.rs index e5c28ca1d8..27d6532667 100644 --- a/crates/tracedecay-mcp-catalog/src/definitions.rs +++ b/crates/tracedecay-mcp-catalog/src/definitions.rs @@ -334,9 +334,6 @@ pub fn apply_context_warming_budget(defs: &mut [ToolDefinition], budget: u8) { /// Tools whose backing dependency is missing on the current host are /// filtered out so the model never sees a tool that will immediately /// fail when called. The host `ast-grep` CLI gates rewrite support. -/// `tracedecay_outline` remains advertised and reports its runtime -/// `ast-grep outline` requirement from the handler, because the Cursor -/// plugin docs/rules intentionally teach agents to start there. pub fn get_tool_definitions() -> Result, McpCatalogError> { let mut definitions = get_maximal_tool_definitions()?; retain_host_available_tool_definitions(&mut definitions); @@ -394,8 +391,6 @@ fn build_maximal_tool_definitions() -> Result, McpCatalogErr def_ast_grep_search(), def_retrieve(), def_context(request_schema("context")?), - def_callers(), - def_callees(request_schema("callees")?), def_impact(request_schema("impact")?), def_node(request_schema("node")?), def_status(), @@ -432,7 +427,6 @@ fn build_maximal_tool_definitions() -> Result, McpCatalogErr def_commit_context(), def_pr_context(), def_test_map(), - def_type_hierarchy(), def_branch_search(), def_branch_diff(), def_branch_list(), @@ -446,12 +440,9 @@ fn build_maximal_tool_definitions() -> Result, McpCatalogErr def_runtime(), def_dsm(), def_test_risk(), - def_body(), def_todos(request_schema("todos")?), - def_callers_for(), def_by_qualified_name(), def_signature(), - def_impls(), def_diagnose(), def_derives(), def_run_affected_tests(), @@ -479,12 +470,8 @@ fn build_maximal_tool_definitions() -> Result, McpCatalogErr def_lcm_describe(), def_lcm_expand(), def_lcm_expand_query(), - def_read(), - def_outline(), - def_implementations(), def_unsafe_patterns(), def_config(), - def_signature_search(), def_constructors(), def_field_sites(), def_replace_symbol(), @@ -658,19 +645,14 @@ const FORMAT_CAPABLE_NON_APPLICATION_TOOL_NAMES: &[&str] = &[ "tracedecay_grep", "tracedecay_ast_grep_search", "tracedecay_context", - "tracedecay_callers", - "tracedecay_callees", "tracedecay_impact", "tracedecay_node", "tracedecay_similar", "tracedecay_redundancy", "tracedecay_rename_preview", - "tracedecay_implementations", - "tracedecay_callers_for", "tracedecay_find_exact_symbol", "tracedecay_by_qualified_name", "tracedecay_signature", - "tracedecay_impls", "tracedecay_derives", // info "tracedecay_status", @@ -678,12 +660,8 @@ const FORMAT_CAPABLE_NON_APPLICATION_TOOL_NAMES: &[&str] = &[ "tracedecay_project_search", "tracedecay_project_context", "tracedecay_files", - "tracedecay_body", "tracedecay_todos", - "tracedecay_read", - "tracedecay_outline", "tracedecay_config", - "tracedecay_signature_search", "tracedecay_port_status", "tracedecay_port_order", // git @@ -780,7 +758,6 @@ const FORMAT_CAPABLE_NON_APPLICATION_TOOL_NAMES: &[&str] = &[ "tracedecay_dashboard", "tracedecay_retrieve", "tracedecay_analytics", - "tracedecay_type_hierarchy", ]; static FORMAT_CAPABLE_TOOL_NAMES: LazyLock> = LazyLock::new(|| { @@ -817,10 +794,8 @@ pub fn tool_defaults_to_markdown(tool_name: &str) -> bool { | "tracedecay_fact_store_supersede" | "tracedecay_fact_store_list" | "tracedecay_files" - | "tracedecay_read" | "tracedecay_skill_list" | "tracedecay_skill_view" - | "tracedecay_type_hierarchy" ) } @@ -852,6 +827,7 @@ fn add_format_property(definitions: &mut [ToolDefinition]) -> Result<(), McpCata json!({ "type": "string", "enum": ["markdown", "json"], + "default": "markdown", "description": "Output format. Default 'markdown' (compact, LLM-optimized; no tables). 'json' for machine-readable output." }), ); diff --git a/crates/tracedecay-mcp-catalog/src/definitions/analysis.rs b/crates/tracedecay-mcp-catalog/src/definitions/analysis.rs index 3b10282008..e34738a518 100644 --- a/crates/tracedecay-mcp-catalog/src/definitions/analysis.rs +++ b/crates/tracedecay-mcp-catalog/src/definitions/analysis.rs @@ -431,9 +431,5 @@ mod tests { definition.input_schema.get("required").is_none(), "every dead-code parameter stays optional" ); - assert!( - definition.description.contains("`path`"), - "the description documents the filter it accepts" - ); } } diff --git a/crates/tracedecay-mcp-catalog/src/definitions/application.rs b/crates/tracedecay-mcp-catalog/src/definitions/application.rs index 5a4b532e1f..cafd1de414 100644 --- a/crates/tracedecay-mcp-catalog/src/definitions/application.rs +++ b/crates/tracedecay-mcp-catalog/src/definitions/application.rs @@ -23,6 +23,57 @@ const PROTECTED_CHANGE_GUIDANCE: &str = "Read the affected setting first through the daemon validates the change against the canonical ProtectedChange schema before it \ returns the redacted preview."; +/// Operations whose shipped MCP definition (input schema and description) is +/// the handwritten `def_*` in [`super`], not a projection of the catalog. +const HANDWRITTEN_DEFINITION_OPERATIONS: [ApplicationSurfaceOperation; 46] = [ + ApplicationSurfaceOperation::Context, + ApplicationSurfaceOperation::Node, + ApplicationSurfaceOperation::Impact, + ApplicationSurfaceOperation::Similar, + ApplicationSurfaceOperation::Redundancy, + ApplicationSurfaceOperation::RenamePreview, + ApplicationSurfaceOperation::PortStatus, + ApplicationSurfaceOperation::PortOrder, + ApplicationSurfaceOperation::Todos, + ApplicationSurfaceOperation::StrReplace, + ApplicationSurfaceOperation::MultiStrReplace, + ApplicationSurfaceOperation::InsertAt, + ApplicationSurfaceOperation::AstGrepRewrite, + ApplicationSurfaceOperation::ReplaceSymbol, + ApplicationSurfaceOperation::InsertAtSymbol, + ApplicationSurfaceOperation::MoveSymbol, + ApplicationSurfaceOperation::RenameSymbol, + ApplicationSurfaceOperation::SourceEditReconcile, + ApplicationSurfaceOperation::SourceEditRollback, + ApplicationSurfaceOperation::FactStoreCurate, + ApplicationSurfaceOperation::FactStoreAdd, + ApplicationSurfaceOperation::FactStoreSearch, + ApplicationSurfaceOperation::FactStoreProbe, + ApplicationSurfaceOperation::FactStoreRelated, + ApplicationSurfaceOperation::FactStoreReason, + ApplicationSurfaceOperation::FactStoreContradict, + ApplicationSurfaceOperation::FactStoreGet, + ApplicationSurfaceOperation::FactStoreUpdate, + ApplicationSurfaceOperation::FactStoreRemove, + ApplicationSurfaceOperation::FactStoreSupersede, + ApplicationSurfaceOperation::FactStoreList, + ApplicationSurfaceOperation::FactFeedback, + ApplicationSurfaceOperation::MemoryStatus, + ApplicationSurfaceOperation::SessionRefreshStatus, + ApplicationSurfaceOperation::SessionRefreshCancel, + ApplicationSurfaceOperation::SessionRefreshBegin, + ApplicationSurfaceOperation::MessageSearch, + ApplicationSurfaceOperation::SessionsFor, + ApplicationSurfaceOperation::Workflows, + ApplicationSurfaceOperation::LcmStatus, + ApplicationSurfaceOperation::LcmDoctor, + ApplicationSurfaceOperation::LcmLoadSession, + ApplicationSurfaceOperation::LcmGrep, + ApplicationSurfaceOperation::LcmDescribe, + ApplicationSurfaceOperation::LcmExpand, + ApplicationSurfaceOperation::LcmExpandQuery, +]; + /// Project every canonical application handler into its MCP transport view. /// /// Operation identity comes from `ApplicationHandlerDescriptor`, exposure and @@ -39,6 +90,7 @@ pub(super) fn application_definitions() -> Result, McpCatalo ApplicationSurfaceOperation::ALL .into_iter() + .filter(|operation| !HANDWRITTEN_DEFINITION_OPERATIONS.contains(operation)) .map(|operation| { let descriptor = handlers.for_surface_operation(operation).ok_or_else(|| { invalid_application_definition( @@ -88,8 +140,14 @@ pub(super) fn application_definitions() -> Result, McpCatalo "readOnlyHint": executable.effect().is_read_only(), "title": manifest.routing().name(), })), - meta: (operation == ApplicationSurfaceOperation::StorageStatus) - .then(|| json!({ "anthropic/alwaysLoad": true })), + // Callers stays loaded: "who calls this" is the most common + // native reflex after grep and chains straight from a node ID. + meta: matches!( + operation, + ApplicationSurfaceOperation::StorageStatus + | ApplicationSurfaceOperation::CodeCallers + ) + .then(|| json!({ "anthropic/alwaysLoad": true })), }) }) .collect() @@ -186,7 +244,9 @@ pub(super) fn def_remote_status_read() -> ToolDefinition { #[cfg(test)] mod tests { - use super::{application_definitions, application_input_schema}; + use super::{ + HANDWRITTEN_DEFINITION_OPERATIONS, application_definitions, application_input_schema, + }; use crate::definitions::{ add_format_property, add_registered_project_selector_properties, get_maximal_tool_definitions, project_input_schema, @@ -216,6 +276,48 @@ mod tests { ); } + #[test] + fn code_navigation_reads_publish_only_their_short_tool_names() { + let advertised = get_maximal_tool_definitions().expect("advertised definitions"); + for operation in [ + ApplicationSurfaceOperation::CodeCallers, + ApplicationSurfaceOperation::CodeCallees, + ApplicationSurfaceOperation::CodeTypeHierarchy, + ApplicationSurfaceOperation::CodeSignatureSearch, + ApplicationSurfaceOperation::CodeImplementations, + ] { + let short = operation.mcp_tool_name(); + let canonical = format!("tracedecay_{}", operation.as_str()); + assert_ne!(short, canonical); + assert_eq!( + advertised + .iter() + .filter(|definition| definition.name == short) + .count(), + 1, + "{short}" + ); + assert!( + advertised + .iter() + .all(|definition| definition.name != canonical), + "{canonical} must not be a second advertisement" + ); + } + let callers = advertised + .iter() + .find(|definition| definition.name == "tracedecay_callers") + .expect("callers definition"); + assert_eq!( + callers.input_schema["required"], + serde_json::json!(["node_id"]) + ); + assert_eq!( + callers.meta, + Some(serde_json::json!({ "anthropic/alwaysLoad": true })) + ); + } + #[test] fn definitions_project_each_handler_and_executable_once() { let handlers = @@ -236,15 +338,32 @@ mod tests { .get(&operation_id) .and_then(|availability| availability.binding()) .expect("executable binding"); - let definition = definitions - .iter() - .find(|definition| definition.name == operation.mcp_tool_name()) - .expect("MCP definition"); assert_eq!( executable.capability_id(), descriptor.operation().capability_id() ); + if HANDWRITTEN_DEFINITION_OPERATIONS.contains(&operation) { + assert!( + definitions + .iter() + .all(|definition| definition.name != operation.mcp_tool_name()), + "{operation:?} keeps its handwritten definition" + ); + assert_eq!( + advertised + .iter() + .filter(|published| published.name == operation.mcp_tool_name()) + .count(), + 1, + "{operation:?}" + ); + continue; + } + let definition = definitions + .iter() + .find(|definition| definition.name == operation.mcp_tool_name()) + .expect("MCP definition"); let canonical = executable.request_schema().body(); let expected_schema = match operation { // These tools deliberately adapt a shipped request or bound diff --git a/crates/tracedecay-mcp-catalog/src/definitions/ast_grep.rs b/crates/tracedecay-mcp-catalog/src/definitions/ast_grep.rs index a0e9c92899..7224260fcb 100644 --- a/crates/tracedecay-mcp-catalog/src/definitions/ast_grep.rs +++ b/crates/tracedecay-mcp-catalog/src/definitions/ast_grep.rs @@ -3,24 +3,15 @@ use serde_json::{Value, json}; use tracedecay_runtime_core::ast_grep::ast_grep_command; -const MIN_AST_GREP_OUTLINE_VERSION: (u64, u64, u64) = (0, 44, 0); - /// Outcome of probing the external `ast-grep` CLI once per process. /// -/// Each flag is one independently observed probe (`--version` ran, its -/// version meets the outline floor, `outline --help` advertises the JSON -/// flags) and `ast_grep_diagnostics_json` reports every flag verbatim to -/// `tracedecay doctor`, so the struct is that wire shape rather than a state -/// machine. -#[allow(clippy::struct_excessive_bools)] +/// `ast_grep_diagnostics_json` reports every field verbatim to +/// `tracedecay doctor`. #[derive(Debug, Clone)] pub struct AstGrepDiagnostics { pub installed: bool, pub version: Option, pub rewrite_available: bool, - pub outline_available: bool, - pub outline_version_ok: bool, - pub outline_flags_ok: bool, pub message: String, } @@ -45,7 +36,7 @@ fn parse_version_component(component: &str) -> Option { (!digits.is_empty()).then(|| digits.parse().ok()).flatten() } -fn parse_ast_grep_version(text: &str) -> Option<(String, (u64, u64, u64))> { +fn parse_ast_grep_version(text: &str) -> Option { for token in text.split_whitespace() { let token = token .trim_start_matches('v') @@ -58,7 +49,7 @@ fn parse_ast_grep_version(text: &str) -> Option<(String, (u64, u64, u64))> { continue; }; let patch = parts.next().and_then(parse_version_component).unwrap_or(0); - return Some((format!("{major}.{minor}.{patch}"), (major, minor, patch))); + return Some(format!("{major}.{minor}.{patch}")); } None } @@ -71,11 +62,8 @@ fn ast_grep_diagnostics_uncached() -> AstGrepDiagnostics { installed: false, version: None, rewrite_available: false, - outline_available: false, - outline_version_ok: false, - outline_flags_ok: false, message: format!( - "ast-grep is not installed or is not on PATH: {err}. Install ast-grep >= 0.44 for tracedecay_outline and rewrite support." + "ast-grep is not installed or is not on PATH: {err}. Install ast-grep for rewrite support." ), }; } @@ -85,53 +73,20 @@ fn ast_grep_diagnostics_uncached() -> AstGrepDiagnostics { if !version_output.status.success() { return AstGrepDiagnostics { installed: true, - version: parse_ast_grep_version(&version_text).map(|(version, _)| version), + version: parse_ast_grep_version(&version_text), rewrite_available: false, - outline_available: false, - outline_version_ok: false, - outline_flags_ok: false, message: format!( - "ast-grep --version failed. Install or repair ast-grep >= 0.44. Output: {version_text}" + "ast-grep --version failed. Install or repair ast-grep. Output: {version_text}" ), }; } - let (version, version_tuple) = - parse_ast_grep_version(&version_text).unwrap_or_else(|| (version_text.clone(), (0, 0, 0))); - let outline_version_ok = version_tuple >= MIN_AST_GREP_OUTLINE_VERSION; - let help_output = ast_grep_command().args(["outline", "--help"]).output(); - let (outline_flags_ok, help_detail) = match help_output { - Ok(output) => { - let help_text = ast_grep_output_text(&output); - ( - output.status.success() - && help_text.contains("--json") - && help_text.contains("--items") - && help_text.contains("--view"), - help_text, - ) - } - Err(err) => (false, err.to_string()), - }; - let outline_available = outline_version_ok && outline_flags_ok; - let message = if outline_available { - format!("ast-grep {version} is available with outline JSON support") - } else if !outline_version_ok { - format!("ast-grep {version} is installed, but tracedecay_outline requires ast-grep >= 0.44") - } else { - format!( - "ast-grep {version} is installed, but `ast-grep outline --help` does not advertise the required --json, --items, and --view flags. Output: {help_detail}" - ) - }; - + let version = parse_ast_grep_version(&version_text).unwrap_or(version_text); AstGrepDiagnostics { installed: true, + message: format!("ast-grep {version} is available with rewrite support"), version: Some(version), rewrite_available: true, - outline_available, - outline_version_ok, - outline_flags_ok, - message, } } @@ -147,10 +102,6 @@ pub fn ast_grep_diagnostics_json() -> Value { "installed": diagnostics.installed, "version": diagnostics.version.clone(), "rewrite_available": diagnostics.rewrite_available, - "outline_available": diagnostics.outline_available, - "outline_min_version": "0.44.0", - "outline_version_ok": diagnostics.outline_version_ok, - "outline_flags_ok": diagnostics.outline_flags_ok, "message": diagnostics.message.clone(), }) } @@ -161,9 +112,3 @@ pub fn ast_grep_diagnostics_json() -> Value { pub fn ast_grep_available() -> bool { ast_grep_diagnostics().rewrite_available } - -/// Returns true when the external `ast-grep` CLI supports `outline` JSON output -/// with the flags introduced in ast-grep 0.44. -pub fn ast_grep_outline_available() -> bool { - ast_grep_diagnostics().outline_available -} diff --git a/crates/tracedecay-mcp-catalog/src/definitions/edit.rs b/crates/tracedecay-mcp-catalog/src/definitions/edit.rs index 4bbd32443e..77f0c4d96e 100644 --- a/crates/tracedecay-mcp-catalog/src/definitions/edit.rs +++ b/crates/tracedecay-mcp-catalog/src/definitions/edit.rs @@ -439,7 +439,7 @@ pub(super) fn def_ast_grep_search() -> ToolDefinition { syntax (metavariables: `$X` one node, `$$$` many). Runs IN-PROCESS over the project \ working tree using the bundled tree-sitter grammars, no external ast-grep binary, no \ gating. Each hit resolves its enclosing symbol, so the natural next call is \ - tracedecay_body. Routing: use this when the pattern is structural (e.g. `foo($$$)`, \ + tracedecay_source_body with that symbol's node_id. Routing: use this when the pattern is structural (e.g. `foo($$$)`, \ `if ($C) { $$$ }`); for a literal/regex string use tracedecay_grep; for a symbol name \ use tracedecay_search. To rewrite a structural match, pair with tracedecay_ast_grep_rewrite.", json!({ diff --git a/crates/tracedecay-mcp-catalog/src/definitions/graph.rs b/crates/tracedecay-mcp-catalog/src/definitions/graph.rs index 19a14f9255..6857a04a48 100644 --- a/crates/tracedecay-mcp-catalog/src/definitions/graph.rs +++ b/crates/tracedecay-mcp-catalog/src/definitions/graph.rs @@ -3,8 +3,8 @@ use serde_json::{Value, json}; use super::{ - context_description, def, def_always_load, def_object, def_required_object, number_property, - required_object_schema, string_property, with_project_selector_properties, + context_description, def, def_always_load, def_required_object, string_property, + with_project_selector_properties, }; use crate::ToolDefinition; @@ -132,7 +132,7 @@ pub(super) fn def_grep() -> ToolDefinition { def_always_load( "tracedecay_grep", "Grep Content", - "grep, ripgrep, rg, text search, find string. Literal/regex content search over UTF-8 text sources in the project working tree (respects .gitignore; binary and non-UTF-8 files are outside the search scope), graph-enriched: each hit resolves the enclosing symbol so the natural next call is tracedecay_body. Bounded file or line omissions and unavailable source candidates are reported as partial coverage. Routing: use this for literal/regex content search (string literals, config keys, error messages); for symbol names use tracedecay_search; for concepts use tracedecay_context. Defaults to the active project; pass project_selector.project_id only when intentionally searching another registered project.", + "grep, ripgrep, rg, text search, find string. Literal/regex content search over UTF-8 text sources in the project working tree (respects .gitignore; binary and non-UTF-8 files are outside the search scope), graph-enriched: each hit resolves the enclosing symbol so the natural next call is tracedecay_source_body with its node_id. Bounded file or line omissions and unavailable source candidates are reported as partial coverage. Routing: use this for literal/regex content search (string literals, config keys, error messages); for symbol names use tracedecay_search; for concepts use tracedecay_context. Defaults to the active project; pass project_selector.project_id only when intentionally searching another registered project.", json!({ "type": "object", "properties": with_project_selector_properties(json!({ @@ -206,36 +206,6 @@ pub(super) fn def_context(input_schema: Value) -> ToolDefinition { ) } -pub(super) fn def_callers_for() -> ToolDefinition { - def( - "tracedecay_callers_for", - "Bulk callers", - "Returns the caller set of every supplied node ID in one round-trip. \ - Useful for clustering or similarity queries that need many caller \ - sets at once. Returns a map of {node_id: [caller_id, …]}. Defaults \ - to `calls` edges; pass `kind` to filter by `uses`, `type_of`, etc.", - json!({ - "type": "object", - "properties": { - "node_ids": { - "type": "array", - "items": { "type": "string" }, - "description": "Node IDs to look up callers for." - }, - "kind": { - "type": "string", - "description": "Edge kind to filter by (default: \"calls\"). Pass an empty string to match all kinds." - }, - "max_per_item": { - "type": "number", - "description": "Cap callers per item (default: 1000)." - } - }, - "required": ["node_ids"] - }), - ) -} - pub(super) fn def_by_qualified_name() -> ToolDefinition { def_required_object( "tracedecay_by_qualified_name", @@ -250,24 +220,6 @@ pub(super) fn def_by_qualified_name() -> ToolDefinition { ) } -pub(super) fn def_impls() -> ToolDefinition { - def_object( - "tracedecay_impls", - "Trait Implementations", - "List `impl` blocks matching a trait, a type, or both. With no filter \ - returns every impl in the graph (use sparingly). Both arguments \ - accept short names (e.g. `Display`) or qualified names. Surfaces \ - information that is otherwise hard to query: trait-method dispatch \ - targets, which types satisfy a given trait, and which traits a type \ - implements.", - json!({ - "trait": string_property("Trait name to filter by (short or qualified). Omit to include all traits."), - "type": string_property("Implementing type to filter by (short or qualified). Omit to include all types."), - "limit": number_property("Maximum number of results to return (default: 100).") - }), - ) -} - pub(super) fn def_signature() -> ToolDefinition { def( "tracedecay_signature", @@ -294,40 +246,6 @@ pub(super) fn def_signature() -> ToolDefinition { // ── Deferred tools (discovered via ToolSearch on demand) ──────────────── -pub(super) fn def_callers() -> ToolDefinition { - // alwaysLoad: "who calls this / find references" is the second-most-common - // native reflex after grep. It only needs a node_id, so keeping it loaded - // lets the model chain straight from a search/context hit into a caller - // trace. This is the 7th (and final) always-loaded tool, see def_grep. - def_always_load( - "tracedecay_callers", - "Callers", - "Who calls this, find references, find usages, call sites. Find all callers of a given node (function, method, etc.) up to a specified depth.", - required_object_schema( - json!({ - "node_id": string_property("The unique node ID to find callers for"), - "max_depth": number_property("Maximum traversal depth (default: 3)") - }), - &["node_id"], - ), - ) -} - -pub(super) fn def_callees(input_schema: Value) -> ToolDefinition { - def( - "tracedecay_callees", - "Callees", - "What does this call, outgoing calls, dependencies of a function. \ - Find all callees of a given node (function, method, etc.) up to a \ - specified depth. When a callee resolves to a trait method, the \ - concrete impl methods reachable through that trait are also \ - returned, tagged with `dispatch_via_trait: true` and a `dispatch_from` \ - pointing at the trait method. Pass `resolve_dispatch: false` to \ - disable this behaviour and get only direct call edges.", - input_schema, - ) -} - pub(super) fn def_impact(input_schema: Value) -> ToolDefinition { def( "tracedecay_impact", @@ -391,28 +309,6 @@ pub(super) fn def_redundancy(input_schema: Value) -> ToolDefinition { ) } -pub(super) fn def_type_hierarchy() -> ToolDefinition { - def( - "tracedecay_type_hierarchy", - "Type Hierarchy", - "Use when asked a trait/interface/class type-hierarchy question, trigger before manually grepping `impl X for` / `extends X` chains across files. Returns the full recursive tree of implementors and extenders for a resolved type node.", - json!({ - "type": "object", - "properties": { - "node_id": { - "type": "string", - "description": "The type node ID to build the hierarchy for" - }, - "max_depth": { - "type": "number", - "description": "Maximum inheritance depth to traverse (default: 5)" - } - }, - "required": ["node_id"] - }), - ) -} - pub(super) fn def_derives() -> ToolDefinition { def( "tracedecay_derives", @@ -441,34 +337,6 @@ pub(super) fn def_derives() -> ToolDefinition { ) } -pub(super) fn def_body() -> ToolDefinition { - def( - "tracedecay_body", - "Symbol Body", - "Return the full source body of a symbol by name (function, struct, const, etc.). \ - Collapses search + node lookup + file read into a single call. \ - When the name is ambiguous, returns multiple matches ranked by relevance.", - json!({ - "type": "object", - "properties": { - "symbol": { - "type": "string", - "description": "Symbol name to look up (e.g. 'resolve_provider_api_key', 'CCH_SEED', 'GraphStats'). Qualified names are also accepted." - }, - "limit": { - "type": "number", - "description": "Maximum number of matching bodies to return when the name is ambiguous (default: 3, max: 20)" - }, - "lazy_index_ignored_dependencies": { - "type": "boolean", - "description": "Opt in to bounded indexing of ignored dependency entry files when an import hint matches (default: false)." - } - }, - "required": ["symbol"] - }), - ) -} - pub(super) fn def_field_sites() -> ToolDefinition { def( "tracedecay_field_sites", @@ -531,51 +399,6 @@ pub(super) fn def_constructors() -> ToolDefinition { ) } -pub(super) fn def_signature_search() -> ToolDefinition { - def( - "tracedecay_signature_search", - "Signature Search", - "Find functions and methods by signature shape: return type, parameter \ - substring, async, or path. Searches the cached `signature` column on \ - every Function/Method node. Substring-matched with case-sensitive \ - compare; combine multiple criteria for narrower hits. Use \ - tracedecay_search for plain name lookups; this tool is for refactor \ - questions like 'find every function returning Result<_, MyError>' or \ - 'every async fn taking &mut self'.", - json!({ - "type": "object", - "properties": { - "returns": { - "type": "string", - "description": "Substring that must appear in the return-type portion of the signature (after '->'). E.g. 'Result<', 'impl Future', 'Vec'." - }, - "params": { - "type": "array", - "items": { "type": "string" }, - "description": "Substrings that must all appear in the parameter list portion of the signature. E.g. ['&mut self'], ['i32', 'String']." - }, - "async": { - "type": "boolean", - "description": "When true, only return functions marked async. When false, exclude them. Omit to ignore async-ness." - }, - "path": { - "type": "string", - "description": "Filter to symbols defined under this directory." - }, - "limit": { - "type": "number", - "description": "Maximum matches to return (default: 50, max: 500)." - } - }, - "anyOf": [ - { "required": ["returns"] }, - { "required": ["params"] }, - { "required": ["async"] } - ] - }), - ) -} - pub(super) fn def_config() -> ToolDefinition { def( "tracedecay_config", @@ -613,105 +436,6 @@ pub(super) fn def_config() -> ToolDefinition { ) } -pub(super) fn def_implementations() -> ToolDefinition { - def( - "tracedecay_implementations", - "Trait / Method Implementations", - "Find every type implementing a given trait, or every body of a given \ - method name. The 'trait' form returns each implementing type plus the \ - methods on its impl block. The 'method' form returns every function/ \ - method named X across the project, grouped by enclosing type when \ - present. Each result includes file, signature, and the method body.", - json!({ - "type": "object", - "properties": { - "trait": { - "type": "string", - "description": "Trait name to look up implementations of (e.g. 'LanguageExtractor', 'Display'). Mutually exclusive with 'method'." - }, - "method": { - "type": "string", - "description": "Method or function name to find every implementation of (e.g. 'extensions', 'count_complexity'). Mutually exclusive with 'trait'." - }, - "limit": { - "type": "number", - "description": "Maximum number of implementations to return (default: 20, max: 200)" - } - }, - "anyOf": [ - { "required": ["trait"] }, - { "required": ["method"] } - ] - }), - ) -} - -pub(super) fn def_outline() -> ToolDefinition { - def( - "tracedecay_outline", - "File Outline", - "Flat list of every top-level symbol defined in a file (functions, structs, \ - enums, traits, classes, impls, etc.), like a table of contents. Sorted by \ - line number; no code bodies. Includes ast-grep outline JSON when the host \ - ast-grep CLI supports outline flags from ast-grep 0.44 or newer. Optional \ - 'kinds' filter narrows to specific node kinds. Use this as the cheapest way \ - to orient before zooming into a \ - large file with tracedecay_node, tracedecay_body, or tracedecay_read.", - json!({ - "type": "object", - "properties": { - "file": { - "type": "string", - "description": "Project-relative path to the file (e.g. 'src/sync.rs')." - }, - "kinds": { - "type": "array", - "items": { "type": "string" }, - "description": "Optional filter on node kinds. Common values: 'function', 'struct', 'enum', 'trait', 'impl', 'class', 'method', 'const'. Case-insensitive. Default: all kinds." - } - }, - "required": ["file"] - }), - ) -} - -pub(super) fn def_read() -> ToolDefinition { - def( - "tracedecay_read", - "Read File (mode-aware)", - "Read a file or its symbol map. Modes: 'full' (entire file), 'lines' \ - (1-based inclusive line slice via the 'lines' arg, e.g. '120-180'), \ - 'map' (flat list of every top-level symbol from the graph, no source \ - bytes touched), 'signatures' (functions and types with their cached \ - signature). Line reads include overlapping symbol signatures by default; \ - full reads can opt in with include_symbols. Cross-session cached: a re-call \ - on an unchanged file returns a tiny stub with 'unchanged: true'.", - json!({ - "type": "object", - "properties": { - "file": { - "type": "string", - "description": "Project-relative or absolute path to the file (e.g. 'src/sync.rs')." - }, - "mode": { - "type": "string", - "enum": ["full", "lines", "map", "signatures"], - "description": "Read mode. Default: 'full'." - }, - "lines": { - "type": "string", - "description": "Required when mode='lines'. Format 'A-B' or single 'A' (1-based, inclusive). E.g. '120-180' or '42'." - }, - "include_symbols": { - "type": "boolean", - "description": "Include graph symbol context for source reads. Defaults to true for mode='lines' and false for mode='full'." - } - }, - "required": ["file"] - }), - ) -} - pub(super) fn def_find_exact_symbol() -> ToolDefinition { def( "tracedecay_find_exact_symbol", @@ -788,11 +512,5 @@ mod search_schema_tests { .as_array() .is_some_and(|fields| fields.contains(&serde_json::json!("documentation"))) ); - assert!( - definition - .description - .contains("freshness: fresh | possibly_stale"), - "the description must tell agents the first line is a freshness verdict" - ); } } diff --git a/crates/tracedecay-mcp-catalog/src/definitions/lcm.rs b/crates/tracedecay-mcp-catalog/src/definitions/lcm.rs index f20b384c41..b42519a031 100644 --- a/crates/tracedecay-mcp-catalog/src/definitions/lcm.rs +++ b/crates/tracedecay-mcp-catalog/src/definitions/lcm.rs @@ -1,6 +1,6 @@ //! LCM session-store and session health-baseline tool definitions. -use serde_json::json; +use serde_json::{Value, json}; use tracedecay_contracts::retained_surfaces::{ LcmRoleV1, MessageRelationshipScopeV1, MessageTypeFilterV1, }; @@ -8,6 +8,39 @@ use tracedecay_contracts::retained_surfaces::{ use super::{def, string_property}; use crate::ToolDefinition; +/// Wire form of the domain `TemporalModeV1`: a `kind`-tagged object whose +/// `as_of` variant carries its UTC-microsecond `cutoff`. +fn temporal_mode_schema(description: &str) -> Value { + let unit = |kind: &str| { + json!({ + "type": "object", + "properties": { "kind": { "const": kind } }, + "required": ["kind"], + "additionalProperties": false + }) + }; + json!({ + "description": description, + "oneOf": [ + unit("current"), + { + "type": "object", + "properties": { + "kind": { "const": "as_of" }, + "cutoff": { + "type": "integer", + "description": "Inclusive cutoff in UTC microseconds." + } + }, + "required": ["kind", "cutoff"], + "additionalProperties": false + }, + unit("evolution"), + unit("forensic") + ] + }) +} + pub(super) fn def_lcm_status() -> ToolDefinition { def( "tracedecay_lcm_status", @@ -67,16 +100,9 @@ pub(super) fn def_lcm_load_session() -> ToolDefinition { "minLength": 1, "description": "Authenticated opaque continuation cursor returned as next_cursor." }, - "temporal_mode": { - "type": "string", - "enum": ["current", "as_of", "evolution", "forensic"], - "description": "Canonical temporal retrieval mode. Defaults to forensic for exact-session loading." - }, - "as_of_micros": { - "type": "integer", - "minimum": 0, - "description": "Required cutoff in UTC microseconds when temporal_mode=as_of." - }, + "temporal_mode": temporal_mode_schema( + "Canonical temporal retrieval mode. Defaults to {\"kind\":\"forensic\"} for exact-session loading." + ), "limit": { "type": "integer", "minimum": 1, @@ -213,16 +239,9 @@ pub(super) fn def_lcm_grep() -> ToolDefinition { "minLength": 1, "description": "Authenticated opaque continuation cursor returned as next_cursor." }, - "temporal_mode": { - "type": "string", - "enum": ["current", "as_of", "evolution", "forensic"], - "description": "Canonical temporal retrieval mode. Defaults to current." - }, - "as_of_micros": { - "type": "integer", - "minimum": 0, - "description": "Required cutoff in UTC microseconds when temporal_mode=as_of." - }, + "temporal_mode": temporal_mode_schema( + "Canonical temporal retrieval mode. Defaults to {\"kind\":\"current\"}." + ), "branch": string_property("Optional git branch filter: only LCM snippets from sessions active on this branch (via the session-git correlation index)."), "worktree": string_property("Optional git worktree root path filter: only LCM snippets from sessions active in this worktree (via the session-git correlation index)."), "commit": string_property("Optional commit sha filter (full or >=6-char hex prefix): only LCM snippets from sessions attributed to this commit (via the session-git correlation index).") diff --git a/crates/tracedecay-mcp-catalog/src/definitions/session.rs b/crates/tracedecay-mcp-catalog/src/definitions/session.rs index a9762d3b19..e5c33c09a6 100644 --- a/crates/tracedecay-mcp-catalog/src/definitions/session.rs +++ b/crates/tracedecay-mcp-catalog/src/definitions/session.rs @@ -35,7 +35,7 @@ pub(super) fn def_message_search() -> ToolDefinition { def( "tracedecay_message_search", "Message Search", - "Read session-temporal message evidence from one authorized project or profile root. This tool never ingests or refreshes provider history. Omitted catch_up is false; explicit catch_up=true requires fresh data and returns typed refresh guidance when the selected root is stale or partial. Set goals=true to list each session's latest thread goal; goals mode makes query optional. project_scope=all_registered sweeps every registered project's own session store with per-root provenance; it cannot be combined with a project selector, cursor, or catch_up.", + "Read session-temporal message evidence from one authorized project or profile root. This tool never ingests or refreshes provider history. Omitted require_fresh is false; explicit require_fresh=true requires fresh data and returns typed refresh guidance when the selected root is stale or partial. Set goals=true to list each session's latest thread goal; goals mode makes query optional.", json!({ "type": "object", "additionalProperties": false, @@ -64,10 +64,10 @@ pub(super) fn def_message_search() -> ToolDefinition { "default": true, "description": "Whether to include child subagent sessions in results (default: true)." }, - "catch_up": { + "require_fresh": { "type": "boolean", "default": false, - "description": "Deprecated compatibility flag. Omitted/false allows stored data. Explicit true is a freshness precondition only: the read executes when fresh, while stale or partial coverage returns refresh_required and a typed tracedecay_session_refresh_begin next action. This tool never performs catch-up, refresh, or ingest." + "description": "Freshness precondition. Omitted/false allows stored data. Explicit true executes the read only when fresh; stale or partial coverage returns refresh_required and a typed tracedecay_session_refresh_begin next action. This tool never refreshes or ingests." }, "cursor": { "type": "string", @@ -80,8 +80,6 @@ pub(super) fn def_message_search() -> ToolDefinition { }, "since": time_filter_schema("Optional inclusive minimum message timestamp. Accepts Unix seconds, RFC3339, YYYY-MM-DD, or relative time like 'last hour'."), "until": time_filter_schema("Optional inclusive maximum message timestamp. Accepts Unix seconds, RFC3339, YYYY-MM-DD, or relative time like 'last hour'."), - "time_from": time_filter_schema("Alias for since."), - "time_to": time_filter_schema("Alias for until."), "scope": { "type": "string", "default": "all", @@ -104,11 +102,6 @@ pub(super) fn def_message_search() -> ToolDefinition { "project_selector": project_selector_object( "Advanced optional registered project selector. Omit to use the active project." ), - "project_scope": { - "type": "string", - "description": "all_registered fans the search out over every registered project's durable session store (bounded, deterministic merge, per-root provenance). Cannot be combined with project_selector, cursor, or catch_up.", - "enum": ["all_registered"] - }, "branch": string_property("Optional git branch filter: only messages from sessions active on this branch (via the session-git correlation index)."), "worktree": string_property("Optional git worktree root path filter: only messages from sessions active in this worktree (via the session-git correlation index)."), "commit": string_property("Optional commit sha filter (full or >=6-char hex prefix): only messages from sessions attributed to this commit (via the session-git correlation index)."), @@ -117,6 +110,7 @@ pub(super) fn def_message_search() -> ToolDefinition { "format": { "type": "string", "enum": ["markdown", "json"], + "default": "markdown", "description": "Optional output format. MCP defaults to compact Markdown; use json for the full compatibility and temporal envelopes." } }, @@ -266,25 +260,10 @@ mod message_search_definition_tests { definition.annotations.as_ref().unwrap()["readOnlyHint"], true ); - assert!( - definition - .description - .contains("never ingests or refreshes") - ); assert_eq!( - definition.input_schema["properties"]["catch_up"]["default"], + definition.input_schema["properties"]["require_fresh"]["default"], false ); - assert!( - definition.input_schema["properties"]["catch_up"]["description"] - .as_str() - .unwrap() - .contains("freshness precondition") - ); - assert_eq!( - definition.input_schema["properties"]["project_scope"]["enum"], - json!(["all_registered"]) - ); assert_closed_objects(&definition.input_schema); } diff --git a/crates/tracedecay-mcp-catalog/src/definitions/tests.rs b/crates/tracedecay-mcp-catalog/src/definitions/tests.rs index 02c598a77b..2150194569 100644 --- a/crates/tracedecay-mcp-catalog/src/definitions/tests.rs +++ b/crates/tracedecay-mcp-catalog/src/definitions/tests.rs @@ -1,4 +1,6 @@ use super::*; +use tracedecay_contracts::retained_surfaces::RetainedSurfaceRequestV1; +use tracedecay_daemon_protocol::ApplicationSurfaceRequest; #[test] fn work_and_workflow_advertise_every_executable_request_schema() { @@ -52,24 +54,6 @@ fn internal_host_ingest_is_cli_resolvable_but_not_advertised() { assert!(internal_daemon_tool_definition("tracedecay_unknown").is_none()); } -#[test] -fn retired_unused_import_scan_is_absent_while_diagnostic_reads_remain() { - let definitions = get_maximal_tool_definitions().expect("tool definitions"); - eprintln!("maximal source catalog count: {}", definitions.len()); - - assert!( - definitions - .iter() - .all(|definition| definition.name != "tracedecay_unused_imports") - ); - for name in ["tracedecay_diagnose", "tracedecay_diagnostics"] { - assert!( - definitions.iter().any(|definition| definition.name == name), - "{name} must remain available for compiler and published diagnostics" - ); - } -} - #[test] fn stack_snapshot_requires_an_exact_selection_binding() { let definition = get_tool_definitions() @@ -167,36 +151,92 @@ fn handle_gated_feedback_reads_are_advertised_with_their_request_handle() { } } +/// An argument object built from the published LCM schema must decode, through +/// the shared CLI/MCP adapter, to the same `as_of` cutoff on the typed request. #[test] -fn lcm_compatibility_definitions_expose_only_opaque_continuation_cursors() { - let load = def_lcm_load_session(); - let grep = def_lcm_grep(); - - for definition in [&load, &grep] { - let properties = definition.input_schema["properties"] +fn lcm_history_reads_accept_an_as_of_cutoff() { + let cutoff = json!({ "kind": "as_of", "cutoff": 1_700_000_000_000_000_i64 }); + for (definition, operation, arguments) in [ + ( + def_lcm_load_session(), + ApplicationSurfaceOperation::LcmLoadSession, + json!({ "session_id": "session-a", "temporal_mode": cutoff }), + ), + ( + def_lcm_grep(), + ApplicationSurfaceOperation::LcmGrep, + json!({ "query": "retention", "temporal_mode": cutoff }), + ), + ] { + let name = definition.name.as_str(); + let schema = &definition.input_schema; + let properties = schema["properties"].as_object().expect("properties"); + let supplied = arguments.as_object().expect("argument object"); + assert!( + supplied.keys().all(|key| properties.contains_key(key)), + "{name} advertises every supplied argument" + ); + assert!( + schema["required"] + .as_array() + .into_iter() + .flatten() + .all(|key| key.as_str().is_some_and(|key| supplied.contains_key(key))), + "{name} requires only supplied arguments" + ); + let as_of = properties["temporal_mode"]["oneOf"] + .as_array() + .expect("temporal mode variants") + .iter() + .find(|variant| variant["properties"]["kind"]["const"] == "as_of") + .expect("advertised as_of variant"); + let mut advertised_fields = as_of["required"] + .as_array() + .expect("as_of required fields") + .iter() + .filter_map(Value::as_str) + .collect::>(); + advertised_fields.sort_unstable(); + let mut supplied_fields = cutoff .as_object() - .expect("LCM properties"); - assert_eq!(properties["cursor"]["type"], "string"); + .expect("cutoff object") + .keys() + .map(String::as_str) + .collect::>(); + supplied_fields.sort_unstable(); + assert_eq!(advertised_fields, supplied_fields, "{name}"); + + let adapted = tracedecay_daemon_protocol::adapt_application_tool_request(name, arguments) + .expect("the shared CLI/MCP adapter accepts the arguments"); + let request = tracedecay_daemon_protocol::parse_application_surface_request( + operation, + adapted.request, + ) + .expect("the typed LCM request accepts an as_of cutoff"); + let temporal_mode = match request { + ApplicationSurfaceRequest::Retained(RetainedSurfaceRequestV1::LcmLoadSession( + request, + )) => request.temporal_mode, + ApplicationSurfaceRequest::Retained(RetainedSurfaceRequestV1::LcmGrep(request)) => { + request.temporal_mode + } + other => panic!("{name} decoded to {other:?}"), + }; assert_eq!( - properties["temporal_mode"]["enum"], - json!(["current", "as_of", "evolution", "forensic"]) + serde_json::to_value(temporal_mode).expect("temporal mode serializes"), + cutoff, + "{name}" ); - assert_eq!(properties["as_of_micros"]["minimum"], 0); } + let legacy = tracedecay_daemon_protocol::parse_application_surface_request( + ApplicationSurfaceOperation::LcmLoadSession, + json!({ "session_id": "session-a", "as_of_micros": 1_700_000_000_000_000_i64 }), + ) + .expect_err("the retired microsecond cutoff argument is refused"); assert!( - load.input_schema["properties"] - .get("after_store_id") - .is_none(), - "legacy offset pagination must not remain public" - ); - assert_eq!( - grep.input_schema["properties"]["include_summaries"]["default"], - false - ); - assert_eq!( - grep.input_schema["properties"]["sort"]["default"], - "relevance" + legacy.to_string().contains("unknown field `as_of_micros`"), + "{legacy}" ); } @@ -279,38 +319,6 @@ fn per_session_budget_does_not_leak_through_the_cached_registry() { ); } -/// Always-loaded schemas enter the model prompt on every turn. The agreed cap -/// is the small core; growing it is a context-window decision, not a drive-by. -#[test] -fn always_loaded_tools_stay_inside_the_agreed_core() { - let definitions = get_maximal_tool_definitions().expect("tool definitions"); - let mut always_loaded = definitions - .iter() - .filter(|definition| { - definition - .meta - .as_ref() - .and_then(|meta| meta.get("anthropic/alwaysLoad")) - .and_then(serde_json::Value::as_bool) - == Some(true) - }) - .map(|definition| definition.name.as_str()) - .collect::>(); - always_loaded.sort_unstable(); - assert_eq!( - always_loaded, - vec![ - "tracedecay_active_project", - "tracedecay_callers", - "tracedecay_context", - "tracedecay_grep", - "tracedecay_search", - "tracedecay_status", - "tracedecay_storage_status", - ] - ); -} - #[test] fn status_and_skill_view_default_to_summaries() { let definitions = get_tool_definitions().expect("tool definitions"); @@ -338,10 +346,4 @@ fn status_and_skill_view_default_to_summaries() { view.input_schema["properties"]["include_support_files"]["default"], serde_json::json!(false) ); - let retrieve = definitions - .iter() - .find(|definition| definition.name == "tracedecay_retrieve") - .expect("retrieve"); - assert!(retrieve.description.contains("Do not walk next_offset")); - assert!(!retrieve.description.contains("byte-exactly")); } diff --git a/crates/tracedecay-mcp-catalog/src/definitions/work.rs b/crates/tracedecay-mcp-catalog/src/definitions/work.rs index 8353679043..1aa4bf8481 100644 --- a/crates/tracedecay-mcp-catalog/src/definitions/work.rs +++ b/crates/tracedecay-mcp-catalog/src/definitions/work.rs @@ -46,44 +46,3 @@ pub(super) fn work_definitions() -> DiscoveryResult> { ), ) } - -#[cfg(test)] -mod tests { - use super::work_definitions; - use tracedecay_api::WorkOperation; - - #[test] - fn discovery_uses_discriminating_work_descriptions() { - let definitions = work_definitions().expect("Work definitions"); - assert_eq!(definitions.len(), WorkOperation::ALL.len()); - assert!(definitions.iter().all(|definition| { - !definition.description.starts_with("Invoke the Work ") - && !definition.description.trim().is_empty() - })); - - for (name, description) in [ - ( - "tracedecay_work_generate_proposal", - "Generate an evidence-calibrated proposal for one task against an exact current Work graph.", - ), - ( - "tracedecay_work_resume_attempts", - "Recover open attempts after daemon restart and fence those that require an explicit retry.", - ), - ( - "tracedecay_work_mutate_graph", - "Apply an exact prepared Work graph mutation using its preserved identity and revision pins.", - ), - ( - "tracedecay_work_release_placement", - "Release or quarantine one run's placement at the expected authority version without deleting bytes.", - ), - ] { - let definition = definitions - .iter() - .find(|definition| definition.name == name) - .unwrap_or_else(|| panic!("missing {name}")); - assert_eq!(definition.description, description); - } - } -} diff --git a/crates/tracedecay-mcp-catalog/src/definitions/workflow.rs b/crates/tracedecay-mcp-catalog/src/definitions/workflow.rs index 54db627769..a8d51dfabc 100644 --- a/crates/tracedecay-mcp-catalog/src/definitions/workflow.rs +++ b/crates/tracedecay-mcp-catalog/src/definitions/workflow.rs @@ -54,27 +54,20 @@ mod tests { use super::workflow_definitions; #[test] - fn activation_help_names_catalog_and_revision_preconditions() { + fn activation_requires_a_positive_expected_revision() { let definitions = workflow_definitions().expect("workflow definitions"); let activate = definitions .iter() .find(|definition| definition.name == "tracedecay_workflow_activate_definition") .expect("workflow activation definition"); - let revision_help = activate.input_schema["properties"]["expected_revision"]["description"] - .as_str() - .unwrap_or_default(); - assert!(revision_help.contains("candidate disposition revision")); - assert!(revision_help.contains('1')); - - let register = definitions - .iter() - .find(|definition| definition.name == "tracedecay_workflow_register_definition") - .expect("workflow registration definition"); - let catalog_help = register.input_schema["$defs"]["WorkflowDefinition"]["properties"] - ["pinned_catalog_digest"]["description"] - .as_str() - .unwrap_or_default(); - assert!(catalog_help.contains("live Work executable catalog digest")); - assert!(catalog_help.to_ascii_lowercase().contains("validation")); + let schema = &activate.input_schema; + assert_eq!(schema["properties"]["expected_revision"]["type"], "integer"); + assert_eq!(schema["properties"]["expected_revision"]["minimum"], 1); + assert!( + schema["required"] + .as_array() + .expect("required fields") + .contains(&serde_json::json!("expected_revision")) + ); } } diff --git a/crates/tracedecay-mcp-catalog/src/lib.rs b/crates/tracedecay-mcp-catalog/src/lib.rs index 3f817efc2b..e06e780e8f 100644 --- a/crates/tracedecay-mcp-catalog/src/lib.rs +++ b/crates/tracedecay-mcp-catalog/src/lib.rs @@ -34,10 +34,7 @@ mod definitions; mod project_access; pub use catalog_error::McpCatalogError; -pub use definitions::ast_grep::{ - AstGrepDiagnostics, ast_grep_available, ast_grep_diagnostics, ast_grep_diagnostics_json, - ast_grep_outline_available, -}; +pub use definitions::ast_grep::{ast_grep_available, ast_grep_diagnostics_json}; pub use definitions::{ SEARCH_MAX_LEXICAL_ANCHOR_BYTES, SEARCH_MAX_LEXICAL_ANCHORS, ToolRegistryMode, apply_context_warming_budget, context_description, context_warming_description, diff --git a/crates/tracedecay-mcp-catalog/src/project_access.rs b/crates/tracedecay-mcp-catalog/src/project_access.rs index c297021da1..5d53e66605 100644 --- a/crates/tracedecay-mcp-catalog/src/project_access.rs +++ b/crates/tracedecay-mcp-catalog/src/project_access.rs @@ -24,17 +24,12 @@ const REGISTERED_PROJECT_READER_TOOL_NAMES: &[&str] = &[ "tracedecay_impact", "tracedecay_node", "tracedecay_implementations", - "tracedecay_callers_for", "tracedecay_find_exact_symbol", "tracedecay_by_qualified_name", "tracedecay_signature", - "tracedecay_impls", "tracedecay_derives", "tracedecay_files", "tracedecay_type_hierarchy", - "tracedecay_body", - "tracedecay_read", - "tracedecay_outline", "tracedecay_signature_search", "tracedecay_call_chain", "tracedecay_file_dependents", diff --git a/crates/tracedecay-mcp/src/application_output/view.rs b/crates/tracedecay-mcp/src/application_output/view.rs index 9d88dc3c01..d76f847c7d 100644 --- a/crates/tracedecay-mcp/src/application_output/view.rs +++ b/crates/tracedecay-mcp/src/application_output/view.rs @@ -1,3 +1,5 @@ +use std::fmt::Write as _; + use serde::Serialize; use serde_json::Value; use tracedecay_contracts::{ @@ -106,6 +108,15 @@ impl CanonicalHumanView { view.code("Receipt outcome", scalar(&effect.receipt.outcome)?); view.code("Receipt actor", scalar(&effect.receipt.actor)?); } + ApplicationOutcome::Result(payload) => { + view.block("Payload", payload_preview(operation, Some(payload))?); + view.code("Status", "success"); + view.code("Operation", operation); + view.code("Binding", binding_id.as_str()); + view.code("Request", envelope.request_id.as_str()); + view.push_scope(&envelope.scope)?; + view.code("Outcome", "result"); + } }, Err(envelope) => { view.code("Operation", operation); @@ -390,6 +401,8 @@ fn payload_preview(operation: &str, payload: Option<&Value>) -> serde_json::Resu fields.get("body").and_then(Value::as_str), ) { format!("{file}:{start_line}-{end_line}\n{body}") + } else if let Some(rendered) = code_graph_page_preview(operation, payload) { + rendered } else { match payload { Value::String(value) => value.clone(), @@ -399,6 +412,132 @@ fn payload_preview(operation: &str, payload: Option<&Value>) -> serde_json::Resu Ok(bounded_payload(rendered)) } +/// One line per symbol for the code-graph navigation pages, with the type +/// hierarchy drawn as its implements/extends tree. `None` falls back to the +/// JSON payload, so an unexpected shape is never hidden. +fn code_graph_page_preview(operation: &str, payload: &Value) -> Option { + if !matches!( + operation, + "code_callers" + | "code_callees" + | "code_implementations" + | "code_type_hierarchy" + | "code_signature_search" + ) { + return None; + } + let items = payload.get("items")?.as_array()?; + let mut rendered = String::new(); + if items.is_empty() { + rendered.push_str("no matches\n"); + } else if operation == "code_type_hierarchy" { + push_hierarchy_roots(items, &mut rendered)?; + } else { + for item in items { + push_symbol_entry(item, &mut rendered)?; + } + } + push_page_trailer(payload, &mut rendered)?; + Some(rendered.trim_end().to_owned()) +} + +/// Starts a subtree at every entry whose parent is itself (the root) or is not +/// on this page, so a continuation page still renders every entry. +fn push_hierarchy_roots(items: &[Value], rendered: &mut String) -> Option<()> { + let page_ids = items.iter().filter_map(item_node_id).collect::>(); + for item in items { + let parent = item.get("parent_node_id").and_then(Value::as_str); + if parent == item_node_id(item) || parent.is_none_or(|parent| !page_ids.contains(&parent)) { + push_hierarchy_subtree(items, item, 0, rendered)?; + } + } + Some(()) +} + +/// One relation, implementation, or signature match: its symbol line, the +/// traversal annotations, then its signature and body when present. +fn push_symbol_entry(item: &Value, rendered: &mut String) -> Option<()> { + let symbol = item.get("symbol").unwrap_or(item); + rendered.push_str(&symbol_line(symbol)?); + if let Some(depth) = item.get("depth").and_then(Value::as_u64) { + write!(rendered, " depth={depth}").ok()?; + } + if item.get("dispatch_via_trait").and_then(Value::as_bool) == Some(true) + && let Some(from) = item.get("dispatch_from").and_then(Value::as_str) + { + write!(rendered, " via trait {from}").ok()?; + } + rendered.push('\n'); + if let Some(signature) = symbol.get("signature").and_then(Value::as_str) { + writeln!(rendered, " {signature}").ok()?; + } + for line in item + .get("body") + .and_then(Value::as_str) + .into_iter() + .flat_map(str::lines) + { + writeln!(rendered, " | {line}").ok()?; + } + Some(()) +} + +fn push_page_trailer(payload: &Value, rendered: &mut String) -> Option<()> { + for gap in payload + .get("support_gaps") + .and_then(Value::as_array) + .into_iter() + .flatten() + { + let reason = gap.get("reason").and_then(Value::as_str)?; + writeln!(rendered, "support gap: {reason}").ok()?; + } + if let Some(cursor) = payload.get("next_cursor").and_then(Value::as_str) { + writeln!(rendered, "next_cursor: {cursor}").ok()?; + } + Some(()) +} + +fn push_hierarchy_subtree( + items: &[Value], + item: &Value, + depth: usize, + rendered: &mut String, +) -> Option<()> { + let symbol = item.get("symbol")?; + let node_id = symbol.get("node_id").and_then(Value::as_str)?; + let line = symbol_line(symbol)?; + if depth == 0 { + writeln!(rendered, "{line}").ok()?; + } else { + let relation = item.get("edge_kind").and_then(Value::as_str)?; + let pad = " ".repeat(depth - 1); + writeln!(rendered, "{pad}|- {relation} {line}").ok()?; + } + for child in items.iter().filter(|child| { + child.get("parent_node_id").and_then(Value::as_str) == Some(node_id) + && item_node_id(child) != Some(node_id) + }) { + push_hierarchy_subtree(items, child, depth + 1, rendered)?; + } + Some(()) +} + +fn item_node_id(item: &Value) -> Option<&str> { + item.pointer("/symbol/node_id").and_then(Value::as_str) +} + +fn symbol_line(symbol: &Value) -> Option { + Some(format!( + "{} ({}) {}:{} node_id={}", + symbol.get("qualified_name").and_then(Value::as_str)?, + symbol.get("kind").and_then(Value::as_str)?, + symbol.get("file").and_then(Value::as_str)?, + symbol.get("line").and_then(Value::as_u64)?, + symbol.get("node_id").and_then(Value::as_str)?, + )) +} + fn bounded_payload(rendered: String) -> String { let Some((end, _)) = rendered.char_indices().nth(MAX_HUMAN_PAYLOAD_CHARS) else { return rendered; @@ -521,4 +660,67 @@ mod tests { assert_eq!(preview, "src/lib.rs:7-9\npub fn answer() {\n 42\n}"); } + + fn symbol(node_id: &str, name: &str, kind: &str, line: u32) -> serde_json::Value { + json!({ + "node_id": node_id, + "name": name, + "qualified_name": format!("src/lib.rs::{name}"), + "kind": kind, + "file": "src/lib.rs", + "line": line, + "end_line": line, + "signature": null, + "is_async": false, + "score": null, + }) + } + + #[test] + fn type_hierarchy_preview_draws_the_implements_tree() { + let preview = payload_preview( + "code_type_hierarchy", + Some(&json!({ + "items": [ + {"symbol": symbol("t", "Shape", "trait", 1), "parent_node_id": "t", "edge_kind": "root", "depth": 0}, + {"symbol": symbol("b", "Base", "struct", 5), "parent_node_id": "t", "edge_kind": "implements", "depth": 1}, + {"symbol": symbol("d", "Derived", "class", 9), "parent_node_id": "b", "edge_kind": "extends", "depth": 2}, + ], + "support_gaps": [], + "next_cursor": null, + })), + ) + .unwrap(); + + assert_eq!( + preview, + "src/lib.rs::Shape (trait) src/lib.rs:1 node_id=t\n\ + |- implements src/lib.rs::Base (struct) src/lib.rs:5 node_id=b\n \ + |- extends src/lib.rs::Derived (class) src/lib.rs:9 node_id=d" + ); + } + + #[test] + fn implementation_preview_carries_each_body_and_the_continuation() { + let preview = payload_preview( + "code_implementations", + Some(&json!({ + "items": [{ + "symbol": symbol("m", "area", "method", 3), + "edge_kind": "implementation", + "dispatch_from": null, + "body": "fn area() {\n 1\n}", + }], + "support_gaps": [{"provider": null, "language": null, "reason": "partial"}], + "next_cursor": "cursor.next", + })), + ) + .unwrap(); + + assert_eq!( + preview, + "src/lib.rs::area (method) src/lib.rs:3 node_id=m\n | fn area() {\n | 1\n | }\n\ + support gap: partial\nnext_cursor: cursor.next" + ); + } } diff --git a/crates/tracedecay-mcp/src/broker_stream_transport.rs b/crates/tracedecay-mcp/src/broker_stream_transport.rs index dfdefefe15..7e18fa5fda 100644 --- a/crates/tracedecay-mcp/src/broker_stream_transport.rs +++ b/crates/tracedecay-mcp/src/broker_stream_transport.rs @@ -14,7 +14,7 @@ use tracedecay_framing::{ BoundedLineReader, MAX_MCP_JSONRPC_FRAME_BYTES, MCP_OVERSIZE_ID_INSPECT_BYTES, is_wire_oversized_io_error, wire_oversized_io_error_with_prefix, }; -use tracedecay_session_memory::context::CancellationToken; +use tracedecay_runtime_core::cancellation::CancellationToken; use crate::lifecycle::ProjectServerResponseLifecycle; use crate::server::{RmcpSelectedProjectResponseAuthority, RmcpWorkDeliverySettlement}; @@ -107,12 +107,6 @@ pub struct BrokerStreamTransport { active_requests: Arc< std::sync::Mutex>>, >, - /// Whether this connection ever accepted an identified request. After the - /// peer half-closes its request side, a connection that served requests - /// and has settled every one of them has nothing left to deliver, while a - /// connection that never carried a request keeps waiting for the peer's - /// full close. - accepted_any_request: Arc, replay: VecDeque, response_lifecycle: Option>, selected_project_responses: Option>, @@ -148,7 +142,6 @@ impl BrokerStreamTransport { reader: BoundedLineReader::new(tokio::io::BufReader::new(reader)), writer: Arc::new(tokio::sync::Mutex::new(Some(writer))), active_requests: Arc::new(std::sync::Mutex::new(HashMap::new())), - accepted_any_request: Arc::new(std::sync::atomic::AtomicBool::new(false)), replay: VecDeque::new(), response_lifecycle: None, selected_project_responses: None, @@ -332,18 +325,16 @@ impl BrokerStreamTransport { .as_deref() .and_then(|settlement| settlement.attempt_for_request(value)); active.insert(request_key, delivery_attempt); - self.accepted_any_request - .store(true, std::sync::atomic::Ordering::Release); } } - /// Resolves once this connection has accepted at least one request and - /// every accepted request has settled, delivered, suppressed, or answered - /// with its typed cancellation. After the peer half-closes its request - /// side, a connection in that state has nothing left it could ever - /// deliver, so waiting for the peer's full close would only strand clients - /// that hold their read half open awaiting the daemon's EOF (a cancelling - /// client does exactly that). + /// Resolves once every accepted request has settled, delivered, + /// suppressed, or answered with its typed cancellation. After the peer + /// half-closes its request side, a connection in that state, including + /// one that only carried notifications or refused frames, has nothing left + /// it could ever deliver, so waiting for the peer's full close would only + /// strand clients that hold their read half open awaiting the daemon's EOF + /// (a cancelling client does exactly that). #[hotpath::measure(label = "daemon.broker.eof_settled_wait", future = true)] async fn wait_for_accepted_requests_settled( active_requests: Arc< @@ -351,12 +342,9 @@ impl BrokerStreamTransport { HashMap>, >, >, - accepted_any_request: Arc, ) { loop { - if accepted_any_request.load(std::sync::atomic::Ordering::Acquire) - && active_requests.lock().is_ok_and(|active| active.is_empty()) - { + if active_requests.lock().is_ok_and(|active| active.is_empty()) { return; } // Settlement lands through independently spawned response and @@ -540,10 +528,8 @@ impl rmcp::transport::Transport for BrokerStreamTransport { // connection has nothing left to deliver, and a client // that reads until daemon EOF, a cancelling client does, // needs this side to close first. - let settled = Self::wait_for_accepted_requests_settled( - Arc::clone(&self.active_requests), - Arc::clone(&self.accepted_any_request), - ); + let settled = + Self::wait_for_accepted_requests_settled(Arc::clone(&self.active_requests)); let peer_full_close = self.peer_fully_closed_after_eof(); tokio::select! { () = peer_full_close => { @@ -565,19 +551,20 @@ impl rmcp::transport::Transport for BrokerStreamTransport { } }; // The envelope rule runs before cancellation matching so a frame - // with a foreign protocol version is never treated as work. + // with a foreign protocol version is never treated as work, and a + // refused frame is answered here rather than registered as an + // accepted request that EOF would wait on. let decoded = match serde_json::from_str::(&line) { - Ok(value) => match crate::jsonrpc::validate_envelope(&value) { - Ok(()) => { - self.observe_incoming_message(&value).await; - crate::jsonrpc::decode_envelope(value) - } - Err(error) => Err(error), - }, + Ok(value) => { + crate::jsonrpc::decode_envelope(&value).map(|message| (value, message)) + } Err(error) => Err(JsonRpcDecodeError::Parse(error)), }; match decoded { - Ok(message) => return Some(message), + Ok((value, message)) => { + self.observe_incoming_message(&value).await; + return Some(message); + } Err(error) => { if let Ok(line) = serde_json::to_string(&error.into_response()) { let _ = self.write_line(&format!("{line}\n")).await; diff --git a/crates/tracedecay-mcp/src/handlers/analytics.rs b/crates/tracedecay-mcp/src/handlers/analytics.rs index 8cf9cfdce8..1ee5ef8ab2 100644 --- a/crates/tracedecay-mcp/src/handlers/analytics.rs +++ b/crates/tracedecay-mcp/src/handlers/analytics.rs @@ -30,7 +30,7 @@ use tracedecay_daemon_service::retained_owner::open_project_retained_memory_targ use tracedecay_domain::errors::{Result, TraceDecayError}; use tracedecay_global_db::{AnalyticsToolCounts, RegisteredGlobalDb}; use tracedecay_project::project::TraceDecay; -use tracedecay_project::project::current_timestamp; +use tracedecay_runtime_core::tracedecay::current_timestamp; use tracedecay_session_memory::fact_store::DatabaseFactStore; use tracedecay_store_runtime::retained_memory::MemoryTargetAccessV1; @@ -56,13 +56,11 @@ const NAVIGATION_TOOLS: &[&str] = &[ "similar", "rename_preview", "implementations", - "callers_for", "by_qualified_name", "call_chain", "file_dependents", "find_exact_symbol", "signature", - "impls", "derives", "status", "active_project", @@ -71,10 +69,10 @@ const NAVIGATION_TOOLS: &[&str] = &[ "project_list", "project_search", "project_context", - "body", + "source_body", "todos", - "read", - "outline", + "source_lines", + "source_outline", "config", "signature_search", "port_status", @@ -667,21 +665,13 @@ fn tools_section(rows: &[AnalyticsToolCounts]) -> Result { "tiers": tiers, "top_tools": top_tools, "raw_distinct_event_name_count": per_tool.len(), - // Deprecated shipped key: this was always a count of raw persisted - // event names, not a public-catalog adoption numerator. - "distinct_tools_called": per_tool.len(), "called_available_defined_tool_count": called_available_defined.len(), "available_defined_tool_count": available_defined.len(), - // Deprecated shipped key: it remains the current host-available - // catalog count, which the explicit field above now names directly. - "defined_tool_count": available_defined.len(), "maximal_defined_tool_count": maximal_defined.len(), "aliased_call_names": aliased_call_names, "bound_internal_call_names": bound_internal_call_names, "unavailable_public_call_names": unavailable_public_call_names, "unknown_or_retired_call_names": unknown_or_retired_call_names, - // Deprecated shipped key preserved as an exact object alias. - "zero_call_tools": zero_call_tools.clone(), "zero_call_available_defined_tools": zero_call_tools, })) } diff --git a/crates/tracedecay-mcp/src/handlers/edit.rs b/crates/tracedecay-mcp/src/handlers/edit.rs index fd83432c97..0605e804c7 100644 --- a/crates/tracedecay-mcp/src/handlers/edit.rs +++ b/crates/tracedecay-mcp/src/handlers/edit.rs @@ -1,282 +1,100 @@ -//! File editing tool handlers: `str_replace`, `multi_str_replace`, `insert_at`, -//! `ast_grep_rewrite`. +//! Source-edit tools served through the canonical application surface. +//! +//! Every transport decodes the arguments with the shared source-edit decoder, +//! dispatches the typed invocation, and renders the result as these tools +//! always have: typed refusals stay project-route errors, and a completed +//! edit reports its touched files and failure message. -use serde_json::{Value, json}; +use std::path::Path; + +use serde_json::Value; +use tracedecay_contracts::source_edit::SourceEditSurfaceResultV1; use tracedecay_contracts::{ - CancellationSignal, Deadline, EffectId, IdempotencyKey, RenameSymbolSurfaceRequestV1, - RequestId, SourceEditInvocationV1, SourceEditKind, SourceEditReconciliationDispositionV1, - SourceEditReconciliationInvocationV1, SourceEditRequest, SourceEditRollbackInvocationV1, + ApplicationOutcome, ApplicationProblem, ApplicationProblemRecord, CancellationSignal, Deadline, + InvocationTarget, PageRequest, RequestId, SafeDiagnostic, }; use tracedecay_daemon_protocol::{ - DaemonInvocationExecutor, DaemonInvocationOutcome, DaemonInvocationRequest, - InvocationCancellationPolicy, invocation_now_micros, + ApplicationSurfaceAdapterError, DaemonInvocationExecutor, RequestedOutputFormat, + parse_source_edit_arguments, +}; +use tracedecay_daemon_service::application_surface::{ + execute_application_surface, resolve_application_surface_dispatch_with_controls, }; -use tracedecay_domain::ManifestDigest; - use tracedecay_domain::errors::{Result, TraceDecayError}; -use tracedecay_project::project::TraceDecay; +use tracedecay_tool_catalog::{ApplicationSurfaceOperation, BindingSurface}; use crate::ToolResult; use crate::handlers::{generic_tool_result, rendered_tool_result}; use crate::tools::render; -fn missing_required_param(name: &str) -> TraceDecayError { - TraceDecayError::Config { - message: format!("missing required parameter: {name}"), - } -} - -fn required_str<'a>(args: &'a Value, name: &str) -> Result<&'a str> { - args.get(name) - .and_then(Value::as_str) - .ok_or_else(|| missing_required_param(name)) -} - -fn required_array<'a>(args: &'a Value, name: &str) -> Result<&'a [Value]> { - args.get(name) - .and_then(Value::as_array) - .map(Vec::as_slice) - .ok_or_else(|| missing_required_param(name)) -} - -/// Decodes the exact public request shape at the transport boundary. Keeping -/// this conversion here prevents MCP handlers from independently accepting a -/// looser shape than the application-owned wire model. `format` selects the -/// MCP response rendering and is therefore removed before application input -/// validation; every operation field remains subject to the DTO's strict -/// shape. -pub fn deserialize_source_edit_surface(args: &Value) -> Result -where - T: serde::de::DeserializeOwned, -{ - let mut input = args.clone(); - if let Some(object) = input.as_object_mut() { - object.remove("format"); - object.remove("__mcp_request_id"); - } - serde_json::from_value(input).map_err(|error| TraceDecayError::Config { - message: format!("invalid source edit request: {error}"), - }) -} - -/// Reads the shared `dry_run` edit flag (default `false`): when set, an edit -/// primitive validates and computes the resulting content but writes nothing, -/// returning a preview diff instead. -fn dry_run_arg(args: &Value) -> bool { - args.get("dry_run") - .and_then(Value::as_bool) - .unwrap_or(false) -} - -/// Reads the shared `verify` edit flag (default `false`): when set, a real -/// (non-dry-run) successful edit re-runs file-scoped diagnostics and attaches a -/// compact verdict to the result. Off by default to keep edits fast; compound -/// refactor tools are expected to default it on. -fn verify_arg(args: &Value) -> bool { - args.get("verify").and_then(Value::as_bool).unwrap_or(false) -} - -#[hotpath::measure(label = "mcp.edit.apply.total")] -async fn source_edit_tool_result( - cg: &TraceDecay, - args: &Value, - request: SourceEditRequest, - invocation: SourceEditInvocationContext<'_>, -) -> Result { - let idempotency_key = optional_idempotency_key(args)?; - let expected_state = optional_expected_state(args)?; - if !request.dry_run() && (idempotency_key.is_none() || expected_state.is_none()) { - return Err(TraceDecayError::Config { - message: "source edit apply requires a fresh idempotency_key and the expected_state returned by a preview" - .to_owned(), - }); - } - let SourceEditInvocationContext { - executor, - request_id, - deadline, - cancellation, - } = invocation; - let (Some(executor), Some(request_id), Some(deadline), Some(cancellation)) = - (executor, request_id, deadline, cancellation) - else { - return Err(TraceDecayError::Config { - message: "daemon-owned source edit authority is unavailable".to_owned(), - }); - }; - let result = hotpath::future!( - invoke_source_edit( - executor, - DaemonInvocationRequest::source_edit( - request_id.as_str(), - SourceEditInvocationV1 { - edit: request, - idempotency_key, - expected_state, - }, - invocation_now_micros(), - deadline.clone(), - cancellation.context(), - ), - deadline, - cancellation, - ), - label = "mcp.edit.apply.execute" - ) - .await?; - let value = source_edit_surface_value(&result)?; - let touched_files = result.outcome.touched_files(); - let success = result.outcome.success(); - let tool_result = - rendered_tool_result(Some(cg.project_root()), args, &value, touched_files, || { - result - .outcome - .as_move() - .map_or_else(|| render::generic_md(&value), move_result_md) - }) - .with_semantic_error(!success); - if success { - Ok(tool_result) - } else { - Ok(tool_result.with_failure_message(result.outcome.message())) - } -} - +/// Controls and daemon authority for one source-edit tool call. #[derive(Clone)] pub struct SourceEditInvocationContext<'a> { pub executor: Option<&'a dyn DaemonInvocationExecutor>, + pub target: InvocationTarget, pub request_id: Option, pub deadline: Option, pub cancellation: Option, } -#[hotpath::measure(label = "mcp.edit.rollback.total")] -pub async fn handle_source_edit_rollback( - cg: &TraceDecay, - args: Value, - invocation: SourceEditInvocationContext<'_>, -) -> Result { - if args.get("confirm").and_then(Value::as_bool) != Some(true) { - return Err(TraceDecayError::Config { - message: "source edit rollback requires confirm=true from the caller after it checks the receipt; do not pause for a human".to_owned(), - }); - } - let effect_id = - EffectId::new(required_str(&args, "effect_id")?).map_err(source_edit_identity_error)?; - let original_idempotency_key = - IdempotencyKey::new(required_str(&args, "original_idempotency_key")?) - .map_err(source_edit_identity_error)?; - let idempotency_key = IdempotencyKey::new(required_str(&args, "idempotency_key")?) - .map_err(source_edit_identity_error)?; - if idempotency_key == original_idempotency_key { - return Err(TraceDecayError::Config { - message: "rollback idempotency key must differ from the original edit key".to_owned(), - }); - } - let original_input_digest = ManifestDigest::new(required_str(&args, "original_input_digest")?) - .map_err(source_edit_identity_error)?; - let expected_state = ManifestDigest::new(required_str(&args, "expected_state")?) - .map_err(source_edit_identity_error)?; - let SourceEditInvocationContext { - executor, - request_id, - deadline, - cancellation, - } = invocation; - let (Some(executor), Some(request_id), Some(deadline), Some(cancellation)) = - (executor, request_id, deadline, cancellation) - else { - return Err(TraceDecayError::Config { - message: "daemon-owned source edit rollback authority is unavailable".to_owned(), - }); +fn unavailable_authority(operation: ApplicationSurfaceOperation) -> TraceDecayError { + let authority = match operation { + ApplicationSurfaceOperation::SourceEditRollback => "source edit rollback authority", + ApplicationSurfaceOperation::SourceEditReconcile => "source edit reconciliation authority", + _ => "source edit authority", }; - let result = hotpath::future!( - invoke_source_edit( - executor, - DaemonInvocationRequest::source_edit_rollback( - request_id.as_str(), - SourceEditRollbackInvocationV1 { - effect_id, - original_idempotency_key, - idempotency_key, - original_input_digest, - expected_state, - }, - invocation_now_micros(), - deadline.clone(), - cancellation.context(), - ), - deadline, - cancellation, - ), - label = "mcp.edit.rollback.execute" - ) - .await?; - let value = source_edit_surface_value(&result)?; - let success = result.outcome.success(); - let tool_result = generic_tool_result(Some(cg.project_root()), &args, &value, Vec::new()) - .with_semantic_error(!success); - if success { - Ok(tool_result) - } else { - Ok(tool_result.with_failure_message(result.outcome.message())) + TraceDecayError::Config { + message: format!("daemon-owned {authority} is unavailable"), + } +} + +fn adapter_error(error: ApplicationSurfaceAdapterError) -> TraceDecayError { + match error { + ApplicationSurfaceAdapterError::DaemonUnreachable { + reason_code, + detail, + } => ApplicationProblem::unavailable(SafeDiagnostic { + code: reason_code, + message: detail, + }) + .into_trace_decay_error(), + error => TraceDecayError::Config { + message: error.to_string(), + }, } } -#[hotpath::measure(label = "mcp.edit.reconcile.total")] -pub async fn handle_source_edit_reconcile( - cg: &TraceDecay, +/// A completed source-edit invocation: the typed result, or the daemon's +/// typed refusal. +pub type SourceEditOutcome = + std::result::Result; + +/// Run one source-edit tool on `surface` and render its tool result. +#[hotpath::measure(label = "mcp.edit.total", future = true)] +pub async fn source_edit_tool( + project_root: Option<&Path>, + surface: BindingSurface, + operation: ApplicationSurfaceOperation, args: Value, invocation: SourceEditInvocationContext<'_>, ) -> Result { - if args.get("confirm").and_then(Value::as_bool) != Some(true) { - return Err(TraceDecayError::Config { - message: "source edit reconciliation requires confirm=true from the caller after it inspects the file; do not pause for a human".to_owned(), - }); - } - let kind = serde_json::from_value::(json!(required_str(&args, "kind")?)) - .map_err(|error| TraceDecayError::Config { - message: format!("invalid source edit kind: {error}"), - })?; - let effect_id = - EffectId::new(required_str(&args, "effect_id")?).map_err(source_edit_identity_error)?; - let idempotency_key = IdempotencyKey::new(required_str(&args, "idempotency_key")?) - .map_err(source_edit_identity_error)?; - let attempt_idempotency_key = - IdempotencyKey::new(required_str(&args, "attempt_idempotency_key")?) - .map_err(source_edit_identity_error)?; - if attempt_idempotency_key == idempotency_key { - return Err(TraceDecayError::Config { - message: - "reconciliation attempt idempotency key must differ from the original edit key" - .to_owned(), - }); - } - let input_digest = ManifestDigest::new(required_str(&args, "input_digest")?) - .map_err(source_edit_identity_error)?; - let disposition = match required_str(&args, "disposition")? { - "confirm_committed" => { - let committed_state = ManifestDigest::new(required_str(&args, "committed_state")?) - .map_err(source_edit_identity_error)?; - SourceEditReconciliationDispositionV1::ConfirmCommitted { committed_state } - } - "confirm_rolled_back" => { - if args.get("committed_state").is_some() { - return Err(TraceDecayError::Config { - message: "committed_state is only valid when disposition is confirm_committed" - .to_owned(), - }); - } - SourceEditReconciliationDispositionV1::ConfirmRolledBack - } - value => { - return Err(TraceDecayError::Config { - message: format!("invalid source edit reconciliation disposition: {value}"), - }); - } - }; + let outcome = run_source_edit(surface, operation, &args, invocation).await?; + render_source_edit_outcome(project_root, operation, &args, outcome) +} + +/// Decode, dispatch, and settle one source-edit tool call. `Err` is an +/// argument or transport failure the caller reports as-is. +pub async fn run_source_edit( + surface: BindingSurface, + operation: ApplicationSurfaceOperation, + args: &Value, + invocation: SourceEditInvocationContext<'_>, +) -> Result { + let request = parse_source_edit_arguments(operation, args) + .map_err(|message| TraceDecayError::Config { message })?; let SourceEditInvocationContext { executor, + target, request_id, deadline, cancellation, @@ -284,319 +102,89 @@ pub async fn handle_source_edit_reconcile( let (Some(executor), Some(request_id), Some(deadline), Some(cancellation)) = (executor, request_id, deadline, cancellation) else { - return Err(TraceDecayError::Config { - message: "daemon-owned source edit reconciliation authority is unavailable".to_owned(), - }); + return Err(unavailable_authority(operation)); }; + let mut dispatched = resolve_application_surface_dispatch_with_controls( + surface, + operation, + request_id, + request, + PageRequest::first(10).map_err(|error| TraceDecayError::Config { + message: error.to_string(), + })?, + Some(deadline), + cancellation, + RequestedOutputFormat::Json, + ) + .map_err(adapter_error)?; + dispatched.invocation.invocation.scope = target; let result = hotpath::future!( - invoke_source_edit( - executor, - DaemonInvocationRequest::source_edit_reconcile( - request_id.as_str(), - SourceEditReconciliationInvocationV1 { - kind, - effect_id, - idempotency_key, - attempt_idempotency_key, - input_digest, - disposition, - }, - invocation_now_micros(), - deadline.clone(), - cancellation.context(), - ), - deadline, - cancellation, - ), - label = "mcp.edit.reconcile.execute" + execute_application_surface(operation, dispatched, Some(executor)), + label = "mcp.edit.execute" ) - .await?; - let value = source_edit_surface_value(&result)?; - let success = result.outcome.success(); - let tool_result = generic_tool_result(Some(cg.project_root()), &args, &value, Vec::new()) - .with_semantic_error(!success); - if success { - Ok(tool_result) - } else { - Ok(tool_result.with_failure_message(result.outcome.message())) - } + .await + .map_err(adapter_error)?; + let envelope = match result.result { + Ok(envelope) => envelope, + Err(problem) => return Ok(Err(*problem.problem)), + }; + let ApplicationOutcome::Result(value) = envelope.outcome else { + return Err(unexpected_outcome()); + }; + serde_json::from_value(value) + .map(Ok) + .map_err(|_| unexpected_outcome()) } -async fn invoke_source_edit( - executor: &dyn DaemonInvocationExecutor, - request: DaemonInvocationRequest, - deadline: Deadline, - cancellation: CancellationSignal, -) -> Result { - let response = executor - .invoke_controlled( - request, - deadline, - cancellation, - InvocationCancellationPolicy::AuthoritativeEffect, - ) - .await - .map_err(|error| error.into_application_problem().into_trace_decay_error())?; - match response.outcome { - DaemonInvocationOutcome::SourceEdit { result, .. } => Ok(result), - DaemonInvocationOutcome::ApplicationProblem { problem } => { - Err(problem.into_trace_decay_error()) - } - DaemonInvocationOutcome::Problem { problem } => Err(problem.into_trace_decay_error()), - _ => Err(TraceDecayError::Config { - message: "source edit invocation returned an unexpected outcome".to_owned(), - }), - } +/// Render a settled source edit as the edit tools always have. +pub fn render_source_edit_outcome( + project_root: Option<&Path>, + operation: ApplicationSurfaceOperation, + args: &Value, + outcome: SourceEditOutcome, +) -> Result { + let result = outcome.map_err(|problem| problem.into_source().into_trace_decay_error())?; + render_source_edit_result(project_root, operation, args, &result) } -fn source_edit_identity_error(error: impl std::fmt::Display) -> TraceDecayError { +fn unexpected_outcome() -> TraceDecayError { TraceDecayError::Config { - message: format!("invalid source edit effect identity: {error}"), + message: "source edit invocation returned an unexpected outcome".to_owned(), } } -fn source_edit_surface_value( - result: &tracedecay_contracts::source_edit::SourceEditSurfaceResultV1, -) -> Result { - Ok(serde_json::to_value(result)?) -} - -fn optional_idempotency_key(args: &Value) -> Result> { - args.get("idempotency_key") - .map(|value| { - value - .as_str() - .ok_or_else(|| missing_required_param("idempotency_key")) - .and_then(|value| { - IdempotencyKey::new(value).map_err(|error| TraceDecayError::Config { - message: format!("invalid idempotency_key: {error}"), - }) - }) - }) - .transpose() -} - -fn optional_expected_state(args: &Value) -> Result> { - args.get("expected_state") - .map(|value| { - value - .as_str() - .ok_or_else(|| missing_required_param("expected_state")) - .and_then(|value| { - ManifestDigest::new(value).map_err(|error| TraceDecayError::Config { - message: format!("invalid expected_state: {error}"), - }) - }) - }) - .transpose() -} - -pub async fn handle_str_replace( - cg: &TraceDecay, - args: Value, - invocation: SourceEditInvocationContext<'_>, -) -> Result { - let path = required_str(&args, "path")?; - let old_str = required_str(&args, "old_str")?; - let new_str = required_str(&args, "new_str")?; - let dry_run = dry_run_arg(&args); - let verify = verify_arg(&args); - - source_edit_tool_result( - cg, - &args, - SourceEditRequest::StrReplace { - path: path.to_owned(), - old_str: old_str.to_owned(), - new_str: new_str.to_owned(), - dry_run, - verify, - }, - invocation, - ) - .await -} - -pub async fn handle_multi_str_replace( - cg: &TraceDecay, - args: Value, - invocation: SourceEditInvocationContext<'_>, +fn render_source_edit_result( + project_root: Option<&Path>, + operation: ApplicationSurfaceOperation, + args: &Value, + result: &SourceEditSurfaceResultV1, ) -> Result { - let path = required_str(&args, "path")?; - let replacements = required_array(&args, "replacements")?; - let dry_run = dry_run_arg(&args); - let verify = verify_arg(&args); - - let parsed_replacements: Vec<(String, String)> = replacements - .iter() - .filter_map(|pair| { - let arr = pair.as_array()?; - if arr.len() != 2 { - return None; - } - let old = arr[0].as_str()?; - let new = arr[1].as_str()?; - Some((old.to_owned(), new.to_owned())) - }) - .collect(); - - if parsed_replacements.len() != replacements.len() { - return Err(TraceDecayError::Config { - message: "each replacement must be an array of exactly 2 strings".to_string(), - }); + let value = serde_json::to_value(result)?; + let success = result.outcome.success(); + let tool_result = match operation { + ApplicationSurfaceOperation::SourceEditRollback + | ApplicationSurfaceOperation::SourceEditReconcile => { + generic_tool_result(project_root, args, &value, Vec::new()) + } + _ => rendered_tool_result( + project_root, + args, + &value, + result.outcome.touched_files(), + || { + result + .outcome + .as_move() + .map_or_else(|| render::generic_md(&value), move_result_md) + }, + ), } - - source_edit_tool_result( - cg, - &args, - SourceEditRequest::MultiStrReplace { - path: path.to_owned(), - replacements: parsed_replacements, - dry_run, - verify, - }, - invocation, - ) - .await -} - -pub async fn handle_insert_at( - cg: &TraceDecay, - args: Value, - invocation: SourceEditInvocationContext<'_>, -) -> Result { - let path = required_str(&args, "path")?; - let anchor = required_str(&args, "anchor")?; - let content = required_str(&args, "content")?; - let dry_run = dry_run_arg(&args); - let verify = verify_arg(&args); - - let before = args.get("before").and_then(Value::as_bool).unwrap_or(false); - - source_edit_tool_result( - cg, - &args, - SourceEditRequest::InsertAt { - path: path.to_owned(), - anchor: anchor.to_owned(), - content: content.to_owned(), - before, - dry_run, - verify, - }, - invocation, - ) - .await -} - -pub async fn handle_replace_symbol( - cg: &TraceDecay, - args: Value, - invocation: SourceEditInvocationContext<'_>, -) -> Result { - let symbol = required_str(&args, "symbol")?; - let new_source = required_str(&args, "new_source")?; - let dry_run = dry_run_arg(&args); - let verify = verify_arg(&args); - - source_edit_tool_result( - cg, - &args, - SourceEditRequest::ReplaceSymbol { - symbol: symbol.to_owned(), - new_source: new_source.to_owned(), - dry_run, - verify, - }, - invocation, - ) - .await -} - -pub async fn handle_insert_at_symbol( - cg: &TraceDecay, - args: Value, - invocation: SourceEditInvocationContext<'_>, -) -> Result { - let symbol = required_str(&args, "symbol")?; - let content = required_str(&args, "content")?; - let dry_run = dry_run_arg(&args); - let verify = verify_arg(&args); - let position = args - .get("position") - .and_then(|v| v.as_str()) - .unwrap_or("after"); - - source_edit_tool_result( - cg, - &args, - SourceEditRequest::InsertAtSymbol { - symbol: symbol.to_owned(), - content: content.to_owned(), - position: position.to_owned(), - dry_run, - verify, - }, - invocation, - ) - .await -} - -pub async fn handle_move_symbol( - cg: &TraceDecay, - args: Value, - invocation: SourceEditInvocationContext<'_>, -) -> Result { - let symbol = required_str(&args, "symbol")?; - let dest_file = required_str(&args, "dest_file")?; - // The impact report is the product; applying is opt-in. - let dry_run = args.get("dry_run").and_then(Value::as_bool).unwrap_or(true); - let update_references = args - .get("update_references") - .and_then(Value::as_bool) - .unwrap_or(false); - - source_edit_tool_result( - cg, - &args, - SourceEditRequest::MoveSymbol { - symbol: symbol.to_owned(), - dest_file: dest_file.to_owned(), - dry_run, - update_references, - }, - invocation, - ) - .await -} - -pub async fn handle_rename_symbol( - cg: &TraceDecay, - args: Value, - invocation: SourceEditInvocationContext<'_>, -) -> Result { - let request: RenameSymbolSurfaceRequestV1 = deserialize_source_edit_surface(&args)?; - let binding = tracedecay_contracts::RenameSymbolBindingV1 { - node_id: request.node_id, - qualified_name: request.qualified_name, - kind: request.kind, - file: request.file, - old_name: request.old_name, - accepted_preview: request.accepted_preview, - }; - - source_edit_tool_result( - cg, - &args, - SourceEditRequest::RenameSymbol { - binding, - new_name: request.new_name, - dry_run: request.dry_run, - verify: request.verify, - }, - invocation, - ) - .await + .with_semantic_error(!success); + Ok(if success { + tool_result + } else { + tool_result.with_failure_message(result.outcome.message()) + }) } /// Human-readable markdown for a move result: the outcome line, applied @@ -640,42 +228,15 @@ fn move_result_md(result: &tracedecay_contracts::source_edit::MoveResult) -> Str out } -pub async fn handle_ast_grep_rewrite( - cg: &TraceDecay, - args: Value, - invocation: SourceEditInvocationContext<'_>, -) -> Result { - let path = required_str(&args, "path")?; - let pattern = required_str(&args, "pattern")?; - let rewrite = required_str(&args, "rewrite")?; - let dry_run = dry_run_arg(&args); - let verify = verify_arg(&args); - - source_edit_tool_result( - cg, - &args, - SourceEditRequest::AstGrepRewrite { - path: path.to_owned(), - pattern: pattern.to_owned(), - rewrite: rewrite.to_owned(), - dry_run, - verify, - }, - invocation, - ) - .await -} - #[cfg(test)] mod tests { - use std::path::Path; - use std::sync::{Arc, Mutex}; + use std::sync::Mutex; use tempfile::tempdir; - use tracedecay_domain::UtcMicros; - use tracedecay_store::ProjectId; + use tracedecay_domain::{ManifestDigest, UtcMicros}; use super::*; + use serde_json::json; use tracedecay_contracts::source_edit::EditResult; use tracedecay_contracts::source_edit::{ SourceEditSurfaceOutcomeV1, SourceEditSurfaceResultV1, @@ -684,11 +245,12 @@ mod tests { ApplicationInvocation, ApplicationInvocationExecutor, ApplicationInvocationFuture, ApplicationProblem, ApplicationResponse, InvocationError, RetryDirective, SafeDiagnostic, }; + use tracedecay_contracts::{SourceEditKind, SourceEditRequest}; use tracedecay_daemon_protocol::{ - DaemonInvocationError, DaemonInvocationExecutorFuture, DaemonInvocationPayload, - DaemonInvocationProblem, DaemonInvocationResponse, + DaemonInvocationError, DaemonInvocationExecutorFuture, DaemonInvocationOutcome, + DaemonInvocationPayload, DaemonInvocationProblem, DaemonInvocationRequest, + DaemonInvocationResponse, InvocationCancellationPolicy, }; - use tracedecay_project::project::TraceDecayOpenOptions; const EXPECTED_STATE: &str = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; @@ -714,80 +276,12 @@ mod tests { ManifestDigest::new(value).unwrap() } - async fn fixture_graph( - project_root: &Path, - ) -> (TraceDecay, tracedecay_runtime_core::db::DaemonDatabaseScope) { - tracedecay_project::runtime_ports::register_runtime_ports( - tracedecay_project::runtime_ports::fixture_daemon_client_ports(), - ) - .expect("runtime port registration"); - let profile_root = project_root.join(".tracedecay-test-profile"); - let open_options = TraceDecayOpenOptions { - profile_root: Some(profile_root.clone()), - global_db_path: Some(profile_root.join("global.db")), - }; - let identity = - tracedecay_daemon_identity::profile_identity::load_or_create(&profile_root).unwrap(); - let database_scope = tracedecay_runtime_core::db::enter_daemon_database_scope( - identity.profile_root(), - 1, - "mcp-source-edit-test-runtime", - ) - .unwrap(); - let runtime_registry = Arc::new( - tracedecay_store_runtime::DaemonSessionRuntimeRegistryV1::open(identity) - .await - .unwrap(), - ); - let profile_database = runtime_registry.profile_database().await.unwrap(); - let store_layout = TraceDecay::resolve_first_touch_configuration_layout( - project_root, - &open_options, - profile_database.as_ref(), - ) - .await - .unwrap(); - let project_id = ProjectId::new( - store_layout - .identity - .project_id - .clone() - .expect("fixture layout has a project identity"), - ) - .unwrap(); - tracedecay_runtime_core::storage::pin_fixture_repository_identity( - project_root, - project_id.as_str(), - ) - .unwrap(); - let configuration_database = runtime_registry - .project_sessions( - project_id, - [ - project_root.to_path_buf(), - store_layout.project_root.clone(), - ], - ) - .await - .unwrap(); - let graph = TraceDecay::init_with_registered_configuration( - project_root, - open_options, - store_layout, - configuration_database, - profile_database, - runtime_registry, - ) - .await - .unwrap(); - (graph, database_scope) - } - fn invocation_context( executor: Option<&dyn DaemonInvocationExecutor>, ) -> SourceEditInvocationContext<'_> { SourceEditInvocationContext { executor, + target: tracedecay_contracts::InvocationTarget::CurrentProject, request_id: Some(RequestId::new("request.mcp.source-edit.fixture").unwrap()), deadline: Some(Deadline::new(UtcMicros(i64::MAX)).unwrap()), cancellation: Some( @@ -813,12 +307,23 @@ mod tests { impl ApplicationInvocationExecutor for RecordingSourceEditExecutor { fn invoke( &self, - _invocation: ApplicationInvocation, + invocation: ApplicationInvocation, ) -> ApplicationInvocationFuture< '_, std::result::Result, > { - Box::pin(async { Err(InvocationError::Unavailable) }) + Box::pin(async move { + let (context, request) = invocation.into_parts(); + let tracedecay_contracts::ApplicationRequest::Surface { binding, payload } = + request + else { + return Err(InvocationError::Unavailable); + }; + tracedecay_daemon_protocol::invoke_application_surface( + self, context, binding, payload, + ) + .await + }) } } @@ -912,12 +417,23 @@ mod tests { impl ApplicationInvocationExecutor for RefusingSourceEditExecutor { fn invoke( &self, - _invocation: ApplicationInvocation, + invocation: ApplicationInvocation, ) -> ApplicationInvocationFuture< '_, std::result::Result, > { - Box::pin(async { Err(InvocationError::Unavailable) }) + Box::pin(async move { + let (context, request) = invocation.into_parts(); + let tracedecay_contracts::ApplicationRequest::Surface { binding, payload } = + request + else { + return Err(InvocationError::Unavailable); + }; + tracedecay_daemon_protocol::invoke_application_surface( + self, context, binding, payload, + ) + .await + }) } } @@ -955,10 +471,11 @@ mod tests { outcome: DaemonInvocationOutcome, ) -> tracedecay_domain::errors::TraceDecayError { let project = tempdir().unwrap(); - let (graph, _database_scope) = fixture_graph(project.path()).await; let executor = RefusingSourceEditExecutor { outcome }; - handle_str_replace( - &graph, + source_edit_tool( + Some(project.path()), + BindingSurface::Mcp, + ApplicationSurfaceOperation::StrReplace, json!({"path":"src/lib.rs","old_str":"old","new_str":"new","dry_run":true}), invocation_context(Some(&executor)), ) @@ -1047,7 +564,7 @@ mod tests { let (reason_code, retryable, _) = error .project_route_context() .expect("protocol refusal must stay a typed project-route error"); - assert_eq!(reason_code, "daemon_invocation.not_found_or_not_authorized"); + assert_eq!(reason_code, "not_found_or_not_authorized"); assert!(!retryable); assert!(!error.to_string().contains("NotFoundOrNotAuthorized")); } @@ -1055,60 +572,75 @@ mod tests { #[tokio::test] async fn source_edit_handlers_forward_exact_variants_defaults_and_controls() { let project = tempdir().unwrap(); - let (graph, _database_scope) = fixture_graph(project.path()).await; let executor = RecordingSourceEditExecutor::new(); - handle_str_replace( - &graph, + source_edit_tool( + Some(project.path()), + BindingSurface::Mcp, + ApplicationSurfaceOperation::StrReplace, json!({"path":"src/lib.rs","old_str":"old","new_str":"new","dry_run":true}), invocation_context(Some(&executor)), ) .await .unwrap(); - handle_multi_str_replace( - &graph, + source_edit_tool( + Some(project.path()), + BindingSurface::Mcp, + ApplicationSurfaceOperation::MultiStrReplace, json!({"path":"src/lib.rs","replacements":[["old","new"]],"dry_run":true}), invocation_context(Some(&executor)), ) .await .unwrap(); - handle_insert_at( - &graph, + source_edit_tool( + Some(project.path()), + BindingSurface::Mcp, + ApplicationSurfaceOperation::InsertAt, json!({"path":"src/lib.rs","anchor":"1","content":"new","dry_run":true}), invocation_context(Some(&executor)), ) .await .unwrap(); - handle_ast_grep_rewrite( - &graph, + source_edit_tool( + Some(project.path()), + BindingSurface::Mcp, + ApplicationSurfaceOperation::AstGrepRewrite, json!({"path":"src/lib.rs","pattern":"old","rewrite":"new","dry_run":true}), invocation_context(Some(&executor)), ) .await .unwrap(); - handle_replace_symbol( - &graph, + source_edit_tool( + Some(project.path()), + BindingSurface::Mcp, + ApplicationSurfaceOperation::ReplaceSymbol, json!({"symbol":"old","new_source":"fn new() {}","dry_run":true}), invocation_context(Some(&executor)), ) .await .unwrap(); - handle_insert_at_symbol( - &graph, + source_edit_tool( + Some(project.path()), + BindingSurface::Mcp, + ApplicationSurfaceOperation::InsertAtSymbol, json!({"symbol":"old","content":"fn new() {}","dry_run":true}), invocation_context(Some(&executor)), ) .await .unwrap(); - handle_move_symbol( - &graph, + source_edit_tool( + Some(project.path()), + BindingSurface::Mcp, + ApplicationSurfaceOperation::MoveSymbol, json!({"symbol":"old","dest_file":"src/new.rs"}), invocation_context(Some(&executor)), ) .await .unwrap(); - handle_rename_symbol( - &graph, + source_edit_tool( + Some(project.path()), + BindingSurface::Mcp, + ApplicationSurfaceOperation::RenameSymbol, json!({ "node_id": "node.fixture", "qualified_name": "old", @@ -1177,7 +709,8 @@ mod tests { invocation.request_id.as_str(), "request.mcp.source-edit.fixture" ); - assert_eq!(invocation.deadline.expires_at, UtcMicros(i64::MAX)); + // The catalog's 30 s source-edit deadline bounds the caller's. + assert!(invocation.deadline.expires_at < UtcMicros(i64::MAX)); assert_eq!( invocation.cancellation.token_id.as_str(), "cancel.mcp.source-edit.fixture" @@ -1189,10 +722,11 @@ mod tests { #[tokio::test] async fn preview_accepts_no_effect_identity_and_returns_expected_state() { let project = tempdir().unwrap(); - let (graph, _database_scope) = fixture_graph(project.path()).await; let executor = RecordingSourceEditExecutor::new(); - let result = handle_str_replace( - &graph, + let result = source_edit_tool( + Some(project.path()), + BindingSurface::Mcp, + ApplicationSurfaceOperation::StrReplace, json!({ "path": "src/lib.rs", "old_str": "old", @@ -1219,9 +753,10 @@ mod tests { #[tokio::test] async fn preview_remains_unavailable_until_source_edit_owner_is_installed() { let project = tempdir().unwrap(); - let (graph, _database_scope) = fixture_graph(project.path()).await; - let error = handle_str_replace( - &graph, + let error = source_edit_tool( + Some(project.path()), + BindingSurface::Mcp, + ApplicationSurfaceOperation::StrReplace, json!({ "path": "src/lib.rs", "old_str": "old", @@ -1243,7 +778,6 @@ mod tests { #[tokio::test] async fn apply_requires_preview_idempotency_and_expected_state() { let project = tempdir().unwrap(); - let (graph, _database_scope) = fixture_graph(project.path()).await; for args in [ json!({ "path": "src/lib.rs", @@ -1258,9 +792,15 @@ mod tests { "expected_state": EXPECTED_STATE }), ] { - let error = handle_str_replace(&graph, args, invocation_context(None)) - .await - .unwrap_err(); + let error = source_edit_tool( + Some(project.path()), + BindingSurface::Mcp, + ApplicationSurfaceOperation::StrReplace, + args, + invocation_context(None), + ) + .await + .unwrap_err(); assert!(error.to_string().contains( "requires a fresh idempotency_key and the expected_state returned by a preview" )); @@ -1270,10 +810,11 @@ mod tests { #[tokio::test] async fn apply_forwards_exact_idempotency_key_and_expected_state() { let project = tempdir().unwrap(); - let (graph, _database_scope) = fixture_graph(project.path()).await; let executor = RecordingSourceEditExecutor::new(); - handle_str_replace( - &graph, + source_edit_tool( + Some(project.path()), + BindingSurface::Mcp, + ApplicationSurfaceOperation::StrReplace, json!({ "path": "src/lib.rs", "old_str": "old", diff --git a/crates/tracedecay-mcp/src/handlers/git/mod.rs b/crates/tracedecay-mcp/src/handlers/git/mod.rs index 13abda1168..812432b960 100644 --- a/crates/tracedecay-mcp/src/handlers/git/mod.rs +++ b/crates/tracedecay-mcp/src/handlers/git/mod.rs @@ -175,7 +175,9 @@ mod tests { /// failure every other git error uses, so a caller never sees a bare hang. #[test] fn an_elapsed_dispatch_deadline_is_a_typed_semantic_failure() { - let project = fixture_project(std::path::Path::new("/unread")); + // Never read, but admission requires a host-absolute root, which a + // driveless `/unread` is not on Windows. + let project = fixture_project(&std::env::temp_dir().join("unread")); let result = git_dispatch_deadline_result(&fixture_context(&project), "tracedecay_pr_context"); diff --git a/crates/tracedecay-mcp/src/handlers/git/pr_context_cursor.rs b/crates/tracedecay-mcp/src/handlers/git/pr_context_cursor.rs index e32c5aaa51..be57706581 100644 --- a/crates/tracedecay-mcp/src/handlers/git/pr_context_cursor.rs +++ b/crates/tracedecay-mcp/src/handlers/git/pr_context_cursor.rs @@ -5,14 +5,15 @@ use tracedecay_domain::{ canonical_sha256, sha256_hex_suffix, }; use tracedecay_global_db::RegisteredGlobalDb; -use tracedecay_session_temporal_store::SessionTemporalCursorKeyProvider; +use tracedecay_session_temporal_store::{SessionTemporalAccess, SessionTemporalCursorKeyProvider}; use tracedecay_temporal_query::cursor::{CursorError, StableSortKey, encode_cursor, verify_cursor}; +use tracedecay_temporal_query::execution::BindingDigest; use tracedecay_temporal_query::ports::SessionCursorAuthenticator; -use tracedecay_temporal_query::ports::{ - BindingDigest, KernelVersions, TemporalExecutionSnapshot, TemporalSnapshotRequest, - TemporalWatermarks, -}; +use tracedecay_temporal_query::ports::TemporalSnapshotRequest; use tracedecay_temporal_query::resolution::ValidatedAuthorization; +use tracedecay_temporal_query::snapshot::{ + KernelVersions, TemporalExecutionSnapshot, TemporalWatermarks, +}; const PR_CONTEXT_CURSOR_SESSION: &str = "session.daemon.pr-context"; @@ -244,8 +245,9 @@ pub(super) async fn pr_context_cursor_authority( )); }; let session_db: &RegisteredGlobalDb = session_db; + let session_temporal = SessionTemporalAccess::new(session_db); let authenticator = hotpath::future!( - session_db.load_preprovisioned_session_cursor_key_provider_result(), + session_temporal.load_preprovisioned_session_cursor_key_provider_result(), label = "mcp.git.cursor.key_provider" ) .await @@ -706,7 +708,7 @@ mod tests { .expect("registered project lease"); // The daemon provisions this store's signing key at project open; the // authority path below reads it back exactly as production does. - lease + SessionTemporalAccess::new(&*lease) .ensure_active_session_cursor_key_result() .await .expect("provision the store's cursor signing key"); diff --git a/crates/tracedecay-mcp/src/handlers/graph/context_markdown.rs b/crates/tracedecay-mcp/src/handlers/graph/context_markdown.rs index e528508a64..97f9c09575 100644 --- a/crates/tracedecay-mcp/src/handlers/graph/context_markdown.rs +++ b/crates/tracedecay-mcp/src/handlers/graph/context_markdown.rs @@ -1,40 +1,108 @@ -//! Context markdown and plan-context enrichment for verified context. +//! Plan-context computation and the markdown every surface renders from a +//! typed context result. use std::collections::HashSet; use std::fmt::Write as _; +use std::path::Path; +use super::search_freshness::freshness_lines; use super::{ - GRAPH_RELATION_READ_LIMIT, graph_symbol_corrupt, required_graph_file_path, - required_graph_metadata, single_graph_adjacency_batch, traverse_verified_neighbors, + GRAPH_RELATION_READ_LIMIT, required_graph_file_path, required_graph_metadata, + single_graph_adjacency_batch, traverse_verified_neighbors, }; +use crate::ToolResult; use crate::context_headings::{ - CONTEXT_CODE_HEADING, CONTEXT_ENTRY_POINTS_HEADING, CONTEXT_RELATED_SYMBOLS_HEADING, + CONTEXT_CODE_HEADING, CONTEXT_ENTRY_POINTS_HEADING, CONTEXT_EXTENSION_POINTS_HEADING, + CONTEXT_RELATED_SYMBOLS_HEADING, CONTEXT_SEEN_NODE_IDS_LABEL, CONTEXT_TEST_COVERAGE_HEADING, }; +use crate::handlers::support::text_tool_result; use crate::path_tree::format_compact_path_list; +use crate::tools::render::{self, Md}; use serde_json::Value; use tracedecay_code_index::graph_projection::CodeGraphSymbolSummaryV1; +use tracedecay_contracts::retrieval::{ + ContextCodeBlockV1, ContextExtensionPointV1, ContextPlanV1, ContextResultV1, + ContextSearchMatchV1, PrimitiveSymbolLocationV1, +}; use tracedecay_domain::RelationEdgeKindV1; -use tracedecay_domain::code_intelligence::NodeKind; +use tracedecay_domain::code_intelligence::{NodeKind, Visibility}; use tracedecay_domain::errors::Result; use tracedecay_graph_query::VerifiedGraphQuery; -#[hotpath::measure(label = "mcp.graph.context_markdown")] -pub(super) fn verified_context_markdown( +use super::context_support::{context_markdown_lane_preview, insert_context_memory_section}; +use super::search::append_coverage_md; +use super::search_evidence::append_verified_graph_evidence_md; + +/// Renders a context result: the full markdown for JSON callers' fallback, +/// and a lane-bounded preview for markdown callers. +pub(crate) fn render_context( + project_root: Option<&Path>, + args: &Value, + result: &ContextResultV1, + touched_files: Vec, +) -> Result { + let value = serde_json::to_value(result)?; + let mut output = freshness_lines(&result.freshness); + output.push_str(&context_markdown( + &result.task, + &result.symbols, + &result.related_symbols, + &result.code, + )); + if result.symbols.is_empty() { + append_context_search_matches(&mut output, &result.search_matches); + } + insert_context_memory_section( + &mut output, + &result.memory_matches, + result.memory_matches_error.as_deref(), + ); + if let Some(plan) = &result.plan { + append_plan_markdown(&mut output, plan); + } + if !result.symbols.is_empty() { + let seen = result + .symbols + .iter() + .map(|symbol| symbol.node_id.as_str()) + .collect::>(); + let _ = write!( + output, + "\n{} {}\n", + CONTEXT_SEEN_NODE_IDS_LABEL, + serde_json::to_string(&seen)? + ); + } + let mut degradation = Md::new(); + append_coverage_md(&mut degradation, &value); + append_verified_graph_evidence_md(&mut degradation, &value); + let degradation = degradation.render(); + if !degradation.is_empty() { + output.push('\n'); + output.push_str(°radation); + } + let text = if render::wants_json(args) { + render::finalize(project_root, args, &value, || output) + } else { + let preview = context_markdown_lane_preview(&output); + render::markdown_preview_with_handle(project_root, &output, &preview) + }; + Ok(text_tool_result(&text, touched_files)) +} + +fn context_markdown( task: &str, - symbols: &[Value], - related: &[Value], - code: &[Value], -) -> Result { + symbols: &[PrimitiveSymbolLocationV1], + related: &[PrimitiveSymbolLocationV1], + code: &[ContextCodeBlockV1], +) -> String { let mut output = format!("# Context for {task}\n\n{CONTEXT_CODE_HEADING}\n"); if code.is_empty() { for symbol in symbols { let _ = writeln!( output, "- **{}** ({}), {}:{}", - field_str(symbol, "name")?, - field_str(symbol, "kind")?, - field_str(symbol, "file")?, - field_i64(symbol, "start_line")?, + symbol.name, symbol.kind, symbol.file, symbol.start_line, ); } } else { @@ -42,9 +110,7 @@ pub(super) fn verified_context_markdown( let _ = writeln!( output, "#### {}:{}\n```\n{}\n```", - field_str(block, "file")?, - field_i64(block, "start_line")?, - field_str(block, "code")?, + block.file, block.start_line, block.code, ); } } @@ -55,60 +121,99 @@ pub(super) fn verified_context_markdown( let _ = writeln!( output, "- **{}** ({}), {}:{}", - field_str(symbol, "name")?, - field_str(symbol, "kind")?, - field_str(symbol, "file")?, - field_i64(symbol, "start_line")?, + symbol.name, symbol.kind, symbol.file, symbol.start_line, ); } output.push('\n'); output.push_str(CONTEXT_ENTRY_POINTS_HEADING); output.push('\n'); for symbol in symbols.iter().take(5) { - let _ = writeln!(output, "- `{}`", field_str(symbol, "qualified_name")?); + let _ = writeln!(output, "- `{}`", symbol.qualified_name); + } + output +} + +fn append_context_search_matches(output: &mut String, matches: &[ContextSearchMatchV1]) { + if matches.is_empty() { + return; + } + output.push_str("\n### Available Code Search Matches\n"); + for search_match in matches { + let _ = writeln!( + output, + "- **{}** ({}), `{}` · rank {} · utility {}", + search_match.name, + search_match.kind, + search_match.file, + search_match.rank, + search_match.utility_micros, + ); } - Ok(output) } +fn append_plan_markdown(output: &mut String, plan: &ContextPlanV1) { + let _ = write!(output, "\n{CONTEXT_EXTENSION_POINTS_HEADING}\n"); + if plan.extension_points.is_empty() { + output.push_str("_No public traits/interfaces found in context._\n"); + } + for point in &plan.extension_points { + let _ = writeln!( + output, + "- **{}** ({}) - {}:{} ({} implementors)", + point.name, point.kind, point.file, point.line, point.implementor_count, + ); + } + let Some(test_files) = &plan.test_files else { + return; + }; + let _ = write!(output, "\n{CONTEXT_TEST_COVERAGE_HEADING}\n"); + if test_files.is_empty() { + output.push_str("_No test files found covering these modules._\n"); + } else { + output.push_str(&format_compact_path_list( + test_files.iter().map(String::as_str), + "- ", + "", + )); + output.push('\n'); + } +} + +/// The plan section for the selected symbols: public traits and interfaces +/// with their implementor counts, and the test files reaching the selection. #[hotpath::measure(label = "mcp.graph.plan_context")] -pub(super) fn append_verified_plan_context( +pub(super) fn verified_plan_context( graph: &VerifiedGraphQuery, symbols: &[CodeGraphSymbolSummaryV1], - output: &mut String, -) -> Result<()> { - output.push_str("\n### Extension Points\n"); - let mut found_extension = false; +) -> Result { + let mut extension_points = Vec::new(); for node in symbols { let metadata = required_graph_metadata(node)?; if matches!( NodeKind::from_str(&metadata.kind), Some(NodeKind::Trait | NodeKind::Interface | NodeKind::InterfaceType) - ) && matches!(metadata.visibility.as_str(), "pub" | "public") + ) && Visibility::from_str(&metadata.visibility) == Some(Visibility::Pub) { let implementors = single_graph_adjacency_batch(graph.callers( std::slice::from_ref(&node.occurrence), &[RelationEdgeKindV1::Implements], GRAPH_RELATION_READ_LIMIT, )?)?; - let _ = writeln!( - output, - "- **{}** ({}) - {}:{} ({} implementors)", - metadata.simple_name, - metadata.kind, - required_graph_file_path(node)?, - metadata.start_line.saturating_add(1), - implementors.len(), - ); - found_extension = true; + extension_points.push(ContextExtensionPointV1 { + name: metadata.simple_name.clone(), + kind: metadata.kind.clone(), + file: required_graph_file_path(node)?.to_owned(), + line: metadata.start_line.saturating_add(1), + implementor_count: implementors.len(), + }); } } - if !found_extension { - output.push_str("_No public traits/interfaces found in context._\n"); - } if symbols.is_empty() { - return Ok(()); + return Ok(ContextPlanV1 { + extension_points, + test_files: None, + }); } - output.push_str("\n### Test Coverage\n"); let annotated_files = graph.test_annotated_logical_files(None, 500_000, 2_000_000)?; let mut test_files = HashSet::new(); for symbol in symbols { @@ -126,33 +231,64 @@ pub(super) fn append_verified_plan_context( } } } - if test_files.is_empty() { - output.push_str("_No test files found covering these modules._\n"); - } else { - let mut sorted = test_files.into_iter().collect::>(); - sorted.sort(); - output.push_str(&format_compact_path_list( - sorted.iter().map(String::as_str), - "- ", - "", - )); - output.push('\n'); - } - Ok(()) -} - -fn field_str<'a>(value: &'a Value, key: &str) -> Result<&'a str> { - value.get(key).and_then(Value::as_str).ok_or_else(|| { - graph_symbol_corrupt(format!( - "verified context value has no string field '{key}'" - )) + let mut test_files = test_files.into_iter().collect::>(); + test_files.sort(); + Ok(ContextPlanV1 { + extension_points, + test_files: Some(test_files), }) } -fn field_i64(value: &Value, key: &str) -> Result { - value.get(key).and_then(Value::as_i64).ok_or_else(|| { - graph_symbol_corrupt(format!( - "verified context value has no integer field '{key}'" - )) - }) +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn plan_markdown_renders_the_typed_sections() { + let mut output = String::new(); + append_plan_markdown( + &mut output, + &ContextPlanV1 { + extension_points: vec![ContextExtensionPointV1 { + name: "Store".to_owned(), + kind: "trait".to_owned(), + file: "src/store.rs".to_owned(), + line: 3, + implementor_count: 2, + }], + test_files: Some(vec!["tests/store.rs".to_owned()]), + }, + ); + assert!(output.starts_with( + "\n### Extension Points\n- **Store** (trait) - src/store.rs:3 (2 implementors)\n\n### Test Coverage\n" + )); + assert!(output.contains("tests/store.rs"), "{output}"); + + let mut empty = String::new(); + append_plan_markdown( + &mut empty, + &ContextPlanV1 { + extension_points: Vec::new(), + test_files: None, + }, + ); + assert_eq!( + empty, + "\n### Extension Points\n_No public traits/interfaces found in context._\n" + ); + + let mut uncovered = String::new(); + append_plan_markdown( + &mut uncovered, + &ContextPlanV1 { + extension_points: Vec::new(), + test_files: Some(Vec::new()), + }, + ); + assert!( + uncovered + .ends_with("\n### Test Coverage\n_No test files found covering these modules._\n"), + "{uncovered}" + ); + } } diff --git a/crates/tracedecay-mcp/src/handlers/graph/context_support.rs b/crates/tracedecay-mcp/src/handlers/graph/context_support.rs index 62455f0ff2..1ce689a366 100644 --- a/crates/tracedecay-mcp/src/handlers/graph/context_support.rs +++ b/crates/tracedecay-mcp/src/handlers/graph/context_support.rs @@ -3,12 +3,13 @@ use std::fmt::Write as _; use std::sync::Arc; -use serde_json::{Value, json}; +use serde_json::Value; use tracedecay_contracts::retained_surfaces::{ FactCategoryV1, FactSearchGraphCoverageV1, FactSearchGraphDegradationV1, FactSearchHitV1, }; use tracedecay_contracts::{ - CancellationSignal, Deadline, now_micros, retained_surface_execution_problem, + CancellationSignal, ContextMemoryAnalyticsV1, Deadline, now_micros, + retained_surface_execution_problem, }; use tracedecay_domain::Confidence; use tracedecay_domain::collapse_whitespace; @@ -219,23 +220,21 @@ pub(super) fn context_memory_read_control( })))) } -pub(super) fn context_memory_analytics_value( +pub(super) fn context_memory_analytics( options: &ContextMemoryOptions, memory_matches: &[FactSearchHitV1], memory_matches_error: Option<&str>, -) -> Value { - let fact_ids: Vec = memory_matches - .iter() - .map(|hit| Value::String(hit.fact.fact_id.as_str().to_owned())) - .collect(); - json!({ - "include_memory": options.include_memory, - "limit": options.limit, - "min_trust": options.min_trust, - "match_count": fact_ids.len(), - "fact_ids": fact_ids, - "error": memory_matches_error, - }) +) -> ContextMemoryAnalyticsV1 { + ContextMemoryAnalyticsV1 { + include_memory: options.include_memory, + limit: u32::try_from(options.limit).unwrap_or(u32::MAX), + min_trust_millionths: (options.min_trust * 1_000_000.0).round() as u32, + fact_ids: memory_matches + .iter() + .map(|hit| hit.fact.fact_id.as_str().to_owned()) + .collect(), + error: memory_matches_error.map(str::to_owned), + } } pub(super) struct ContextMemoryMatches { diff --git a/crates/tracedecay-mcp/src/handlers/graph/dispatch.rs b/crates/tracedecay-mcp/src/handlers/graph/dispatch.rs index af2bb2439e..836c00dfa4 100644 --- a/crates/tracedecay-mcp/src/handlers/graph/dispatch.rs +++ b/crates/tracedecay-mcp/src/handlers/graph/dispatch.rs @@ -5,10 +5,8 @@ use tracedecay_application::code_index::CodeIndexIgnoredDependencyAdmissionPortV use tracedecay_domain::errors::Result; use super::{ - handle_by_qualified_name, handle_callees, handle_callers, handle_callers_for, handle_context, - handle_derives, handle_find_exact_symbol, handle_impact, handle_implementations, handle_impls, - handle_node, handle_redundancy, handle_rename_preview, handle_search, handle_signature, - handle_similar, + handle_by_qualified_name, handle_derives, handle_find_exact_symbol, handle_search, + handle_signature, }; use crate::ToolResult; use crate::handlers::ast_grep::handle_ast_grep_search; @@ -18,7 +16,7 @@ use crate::handlers::verified_read::{VerifiedGraphOpen, verified_read_operation use crate::tool_context::McpToolContext; /// Dispatches one graph-family tool (`tracedecay_search`, -/// `tracedecay_callers`, ...) onto its handler, opening the verified graph +/// `tracedecay_impact`, ...) onto its handler, opening the verified graph /// through `open` under the operation the catalog registers for it. pub async fn dispatch_tool( ctx: &McpToolContext<'_>, @@ -66,29 +64,6 @@ pub async fn dispatch_tool( ) .await } - "tracedecay_context" => { - handle_context(ctx, open(read("context")?), args, scope_prefix).await - } - "tracedecay_callers" => handle_callers(&open(read("code_callers")?).await?, args).await, - "tracedecay_callees" => handle_callees(&open(read("callees")?).await?, args).await, - "tracedecay_impact" => handle_impact(&open(read("impact")?).await?, args).await, - "tracedecay_node" => handle_node(&open(read("node")?).await?, args).await, - "tracedecay_similar" => handle_similar(ctx, args).await, - "tracedecay_redundancy" => handle_redundancy(ctx, args).await, - "tracedecay_rename_preview" => { - handle_rename_preview(ctx, &open(read("rename_preview")?).await?, args).await - } - "tracedecay_implementations" => { - handle_implementations( - &open(read("code_implementations")?).await?, - args, - scope_prefix, - ) - .await - } - "tracedecay_callers_for" => { - handle_callers_for(&open(read("code_callers")?).await?, args).await - } "tracedecay_find_exact_symbol" => { handle_find_exact_symbol( ctx, @@ -103,9 +78,8 @@ pub async fn dispatch_tool( handle_by_qualified_name(&open(read("qualified_name")?).await?, args).await } "tracedecay_signature" => { - handle_signature(&open(read("code_signature_search")?).await?, args).await + handle_signature(&open(read("qualified_name")?).await?, args).await } - "tracedecay_impls" => handle_impls(&open(read("code_implementations")?).await?, args).await, "tracedecay_derives" => { handle_derives(&open(read("code_type_hierarchy")?).await?, args).await } diff --git a/crates/tracedecay-mcp/src/handlers/graph/mod.rs b/crates/tracedecay-mcp/src/handlers/graph/mod.rs index 04d7708b87..594caaa7bb 100644 --- a/crates/tracedecay-mcp/src/handlers/graph/mod.rs +++ b/crates/tracedecay-mcp/src/handlers/graph/mod.rs @@ -11,22 +11,21 @@ mod search_evidence; mod search_freshness; mod verified; +pub(crate) use context_markdown::render_context; pub use dispatch::dispatch_tool; pub use navigation::{ - handle_by_qualified_name, handle_callees, handle_callers, handle_callers_for, handle_derives, - handle_impact, handle_implementations, handle_impls, handle_node, handle_signature, + compute_impact, compute_node, handle_by_qualified_name, handle_derives, handle_signature, }; pub use search::{ - handle_context, handle_find_exact_symbol, handle_redundancy, handle_rename_preview, - handle_search, handle_similar, + compute_context, compute_redundancy, compute_rename_preview, compute_similar, + handle_find_exact_symbol, handle_search, }; pub use verified::{ - GRAPH_RELATION_READ_LIMIT, VerifiedNeighbor, canonical_relation_kind, cost_to_expand_verified, - graph_name_matches, graph_occurrence_id, graph_symbol_corrupt, graph_symbol_end_line, - graph_symbol_location_value, graph_symbol_paths, graph_symbols_in_scope, line_for_byte_offset, - nodes_addressed_by_args, required_graph_file_path, required_graph_metadata, - single_graph_adjacency_batch, traverse_verified_neighbors, verified_neighbor_value, - verified_trait_dispatch_targets, + GRAPH_RELATION_READ_LIMIT, VerifiedNeighbor, cost_to_expand_verified, graph_occurrence_id, + graph_symbol_corrupt, graph_symbol_end_line, graph_symbol_location_value, graph_symbol_paths, + graph_symbols_in_scope, line_for_byte_offset, nodes_addressed_by_args, + required_graph_file_path, required_graph_metadata, single_graph_adjacency_batch, + traverse_verified_neighbors, }; use tracedecay_contracts::retrieval::PrimitiveNotFoundV1; @@ -48,15 +47,34 @@ pub(super) fn require_positive_depth(max_depth: u32) -> Result<()> { } pub fn node_not_found(node_id: &str) -> Result { - let output = PrimitiveNotFoundV1 { + not_found_tool_result(&node_not_found_result(node_id)) +} + +pub(crate) fn node_not_found_result(node_id: &str) -> PrimitiveNotFoundV1 { + PrimitiveNotFoundV1 { status: "not_found".to_owned(), reason_code: "node_not_found".to_owned(), node_id: node_id.to_owned(), message: format!("Node not found: {node_id}"), - }; + } +} + +pub fn not_found_tool_result(output: &PrimitiveNotFoundV1) -> Result { Ok( - text_tool_result(&serde_json::to_string_pretty(&output)?, vec![]) + text_tool_result(&serde_json::to_string_pretty(output)?, vec![]) .with_semantic_error(true) - .with_failure_message(format!("node not found: {node_id}")), + .with_failure_message(format!("node not found: {}", output.node_id)), ) } + +pub(crate) fn graph_tool_completion( + result: tracedecay_contracts::graph_tool::GraphToolResultV1, + touched_files: Vec, +) -> tracedecay_contracts::graph_tool::GraphToolCompletionV1 { + tracedecay_contracts::graph_tool::GraphToolCompletionV1 { + result, + touched_files, + code_graph: None, + analytics: None, + } +} diff --git a/crates/tracedecay-mcp/src/handlers/graph/navigation.rs b/crates/tracedecay-mcp/src/handlers/graph/navigation.rs index 8126abc0a2..a057b0f2a0 100644 --- a/crates/tracedecay-mcp/src/handlers/graph/navigation.rs +++ b/crates/tracedecay-mcp/src/handlers/graph/navigation.rs @@ -1,187 +1,29 @@ //! Dependency-clean graph-navigation handlers over [`VerifiedGraphQuery`]. -use std::collections::{HashMap, HashSet}; - use serde_json::{Value, json}; -use tracedecay_code_index::graph_projection::CodeGraphSymbolSummaryV1; +use tracedecay_contracts::graph_tool::{GraphToolCompletionV1, GraphToolResultV1}; use tracedecay_contracts::retrieval::{ - CalleeV1, CalleesSurfaceRequestV1, ImpactNodeV1, ImpactResultV1, ImpactSurfaceRequestV1, - NodeDetailsV1, NodeExpansionCostV1, NodeSurfaceRequestV1, + ImpactNodeV1, ImpactResultV1, NodeDepthSurfaceRequestV1, NodeDetailsV1, NodeExpansionCostV1, + NodeResultV1, NodeSurfaceRequestV1, }; -use tracedecay_contracts::{CoverageCompleteness, EvidenceDomain, Omission, OmissionReason}; -use tracedecay_domain::RelationEdgeKindV1; -use tracedecay_domain::code_intelligence::{EdgeKind, NodeKind}; use tracedecay_domain::errors::{Result, TraceDecayError}; use tracedecay_graph_query::VerifiedGraphQuery; -use crate::{ - ToolResult, decode_primitive_request, generic_tool_result, require_node_id, text_tool_result, - unique_file_paths, -}; +use crate::{ToolResult, decode_primitive_request, generic_tool_result, text_tool_result}; use super::{ - GRAPH_RELATION_READ_LIMIT, canonical_relation_kind, cost_to_expand_verified, - graph_name_matches, graph_occurrence_id, graph_symbol_corrupt, graph_symbol_end_line, - graph_symbol_location_value, graph_symbol_paths, node_not_found, nodes_addressed_by_args, - require_positive_depth, required_graph_file_path, required_graph_metadata, - single_graph_adjacency_batch, traverse_verified_neighbors, user_line, verified_neighbor_value, - verified_trait_dispatch_targets, + GRAPH_RELATION_READ_LIMIT, cost_to_expand_verified, graph_occurrence_id, graph_symbol_corrupt, + graph_symbol_end_line, graph_symbol_location_value, graph_symbol_paths, graph_tool_completion, + node_not_found_result, nodes_addressed_by_args, require_positive_depth, + required_graph_file_path, required_graph_metadata, user_line, }; -#[hotpath::measure(label = "mcp.graph.callers.total")] -pub async fn handle_callers(graph: &VerifiedGraphQuery, args: Value) -> Result { - let node_id = require_node_id(&args)?; - - let max_depth = args - .get("max_depth") - .and_then(serde_json::Value::as_u64) - .map_or(3, |v| v.min(10) as u32); - require_positive_depth(max_depth)?; - - let occurrence = graph_occurrence_id(node_id)?; - let results = hotpath::measure_block!( - "mcp.graph.callers.graph", - traverse_verified_neighbors( - graph, - occurrence.clone(), - &[RelationEdgeKindV1::Calls], - true, - max_depth as usize, - )? - ); - let summaries = results - .iter() - .map(|result| result.symbol.clone()) - .collect::>(); - let touched_files = graph_symbol_paths(&summaries)?; - let items = results - .iter() - .map(verified_neighbor_value) - .collect::>>()?; - - let mut traversed = vec![occurrence]; - traversed.extend( - results - .iter() - .filter(|result| result.depth < max_depth as usize) - .map(|result| result.symbol.occurrence.clone()), - ); - let unsupported = graph.has_unresolved_callers(&traversed)?; - let omissions = if unsupported { - vec![Omission { - domain: EvidenceDomain::Graph, - count: 1, - reason: OmissionReason::Unsupported, - }] - } else { - Vec::new() - }; - let value = hotpath::measure_block!( - "mcp.graph.callers.serialize", - json!({ - "callers": items, - "coverage": { "completeness": if unsupported { CoverageCompleteness::Partial } else { CoverageCompleteness::Complete } }, - "omissions": omissions, - }) - ); - Ok(generic_tool_result( - Some(graph.project_root()?), - &args, - &value, - touched_files, - )) -} - -/// Beyond the direct `Calls` edges, this handler also surfaces *trait -/// dispatch targets*: when a callee is a method whose enclosing scope is a -/// trait, the concrete impl methods reachable through that trait are added -/// to the result list and tagged with `dispatch_via_trait: true`. The -/// original trait-method entry is preserved so callers can still see what -/// they statically called. -/// -/// Dispatch resolution skipped when `resolve_dispatch=false` is passed. -#[hotpath::measure(label = "mcp.graph.callees.total")] -pub async fn handle_callees(graph: &VerifiedGraphQuery, args: Value) -> Result { - let request: CalleesSurfaceRequestV1 = decode_primitive_request(&args, "tracedecay_callees")?; - let max_depth = request.max_depth.map_or(3, |value| value.min(10)); - require_positive_depth(max_depth)?; - let resolve_dispatch = request.resolve_dispatch.unwrap_or(true); - - let occurrence = graph_occurrence_id(&request.node_id)?; - let results = hotpath::measure_block!( - "mcp.graph.callees.graph", - traverse_verified_neighbors( - graph, - occurrence, - &[RelationEdgeKindV1::Calls], - false, - max_depth as usize, - )? - ); - let mut seen = results - .iter() - .map(|result| result.symbol.occurrence.clone()) - .collect::>(); - - let mut items = results - .iter() - .map(|result| { - let metadata = required_graph_metadata(&result.symbol)?; - Ok(CalleeV1 { - node_id: result.symbol.occurrence.as_str().to_owned(), - name: metadata.simple_name.clone(), - kind: metadata.kind.clone(), - file: required_graph_file_path(&result.symbol)?.to_owned(), - line: user_line(metadata.start_line), - edge_kind: result.edge_kind.as_str().to_owned(), - dispatch_via_trait: false, - depth: Some(u32::try_from(result.depth).map_err(|_| { - graph_symbol_corrupt("callee traversal depth exceeds u32".to_owned()) - })?), - dispatch_from: None, - }) - }) - .collect::>>()?; - - if resolve_dispatch { - hotpath::measure_block!("mcp.graph.callees.dispatch", { - for callee in &results { - for impl_method in verified_trait_dispatch_targets(graph, &callee.symbol)? { - if !seen.insert(impl_method.occurrence.clone()) { - continue; - } - let metadata = required_graph_metadata(&impl_method)?; - items.push(CalleeV1 { - node_id: impl_method.occurrence.as_str().to_owned(), - name: metadata.simple_name.clone(), - kind: metadata.kind.clone(), - file: required_graph_file_path(&impl_method)?.to_owned(), - line: user_line(metadata.start_line), - edge_kind: "calls".to_owned(), - dispatch_via_trait: true, - depth: None, - dispatch_from: Some(callee.symbol.occurrence.as_str().to_owned()), - }); - } - } - }); - } - - let touched_files = unique_file_paths(items.iter().map(|item| item.file.as_str())); - - let value = - hotpath::measure_block!("mcp.graph.callees.serialize", serde_json::to_value(items)?); - Ok(generic_tool_result( - Some(graph.project_root()?), - &args, - &value, - touched_files, - )) -} - #[hotpath::measure(label = "mcp.graph.impact.total")] -pub async fn handle_impact(graph: &VerifiedGraphQuery, args: Value) -> Result { - let request: ImpactSurfaceRequestV1 = decode_primitive_request(&args, "tracedecay_impact")?; +pub async fn compute_impact( + graph: &VerifiedGraphQuery, + args: Value, +) -> Result { + let request: NodeDepthSurfaceRequestV1 = decode_primitive_request(&args, "tracedecay_impact")?; let max_depth = request.max_depth.map_or(3, |value| value.min(10)); require_positive_depth(max_depth)?; @@ -218,26 +60,23 @@ pub async fn handle_impact(graph: &VerifiedGraphQuery, args: Value) -> Result>>()?; - let output = hotpath::measure_block!( - "mcp.graph.impact.serialize", - serde_json::to_value(ImpactResultV1 { - node_count: nodes.len(), - complete: impact.complete, - unavailable_fields: vec!["edge_count".to_owned()], - nodes, - })? - ); - - Ok(generic_tool_result( - Some(graph.project_root()?), - &args, - &output, + let result = ImpactResultV1 { + node_count: nodes.len(), + complete: impact.complete, + unavailable_fields: vec!["edge_count".to_owned()], + nodes, + }; + Ok(graph_tool_completion( + GraphToolResultV1::Impact(result), touched_files, )) } #[hotpath::measure(label = "mcp.graph.node.total")] -pub async fn handle_node(graph: &VerifiedGraphQuery, args: Value) -> Result { +pub async fn compute_node( + graph: &VerifiedGraphQuery, + args: Value, +) -> Result { let request: NodeSurfaceRequestV1 = decode_primitive_request(&args, "tracedecay_node")?; let occurrence = graph_occurrence_id(&request.node_id)?; let node = hotpath::measure_block!("mcp.graph.node.graph", graph.symbol_summary(&occurrence)?); @@ -277,169 +116,42 @@ pub async fn handle_node(graph: &VerifiedGraphQuery, args: Value) -> Result node_not_found(&request.node_id), - } -} - -/// Bulk caller lookup over many IDs. -#[hotpath::measure(label = "mcp.graph.callers_for.total")] -pub async fn handle_callers_for(graph: &VerifiedGraphQuery, args: Value) -> Result { - let node_ids: Vec = args - .get("node_ids") - .and_then(|v| v.as_array()) - .map(|arr| { - arr.iter() - .filter_map(|v| v.as_str().map(str::to_string)) - .collect() - }) - .unwrap_or_default(); - - if node_ids.is_empty() { - return Err(TraceDecayError::Config { - message: "callers_for requires non-empty node_ids".to_string(), - }); - } - - // Default to "calls" but allow any kind (or empty string for all kinds). - let kind_arg = args.get("kind").and_then(|v| v.as_str()).unwrap_or("calls"); - let kinds: Vec = if kind_arg.is_empty() { - Vec::new() - } else { - match EdgeKind::from_str(kind_arg) { - Some(k) => vec![canonical_relation_kind(k)?], - None => { - return Err(TraceDecayError::Config { - message: format!("unknown edge kind: {kind_arg}"), - }); - } - } - }; - - let max_per_item = args - .get("max_per_item") - .and_then(serde_json::Value::as_u64) - .map_or(1000usize, |v| v.min(10_000) as usize); - - let occurrences = node_ids - .iter() - .map(|node_id| graph_occurrence_id(node_id)) - .collect::>>()?; - let batches = hotpath::measure_block!( - "mcp.graph.callers_for.graph", - graph.callers(&occurrences, &kinds, 2_000_000)? - ); - if batches.len() != node_ids.len() { - return Err(graph_symbol_corrupt(format!( - "verified graph returned {} caller batches for {} symbols", - batches.len(), - node_ids.len() - ))); - } - - let mut truncated = false; - let mut by_target = HashMap::new(); - for (target, callers) in node_ids.iter().zip(batches) { - if callers.len() > max_per_item { - truncated = true; - } - by_target.insert( - target, - callers - .into_iter() - .take(max_per_item) - .map(|edge| edge.neighbor.occurrence.as_str().to_owned()) - .collect::>(), - ); + None => Ok(graph_tool_completion( + GraphToolResultV1::Node(NodeResultV1::NotFound(node_not_found_result( + &request.node_id, + ))), + Vec::new(), + )), } - - // Ensure every requested ID appears in the response, even if no callers. - let result_map: HashMap<&String, Vec> = node_ids - .iter() - .map(|id| (id, by_target.remove(id).unwrap_or_default())) - .collect(); - - let mut unsupported = false; - if kinds.is_empty() || kinds.contains(&RelationEdgeKindV1::Calls) { - // Bulk lookup preserves an empty row for unmatched IDs, but absence of - // admitted target metadata cannot establish complete caller coverage. - for occurrence in &occurrences { - if graph - .symbol_summary(occurrence)? - .and_then(|symbol| symbol.metadata) - .is_none() - { - unsupported = true; - break; - } - } - unsupported = unsupported || graph.has_unresolved_callers(&occurrences)?; - } - let mut omissions = Vec::new(); - if unsupported { - omissions.push(Omission { - domain: EvidenceDomain::Graph, - count: 1, - reason: OmissionReason::Unsupported, - }); - } - if truncated { - omissions.push(Omission { - domain: EvidenceDomain::Graph, - count: 1, - reason: OmissionReason::Budget, - }); - } - - let output = hotpath::measure_block!( - "mcp.graph.callers_for.serialize", - json!({ - "callers": result_map, - "truncated": truncated, - "max_per_item": max_per_item, - "coverage": { "completeness": if unsupported || truncated { CoverageCompleteness::Partial } else { CoverageCompleteness::Complete } }, - "omissions": omissions, - }) - ); - Ok(generic_tool_result( - Some(graph.project_root()?), - &args, - &output, - vec![], - )) } /// Cross-run node lookup by name. @@ -517,117 +229,6 @@ pub async fn handle_signature(graph: &VerifiedGraphQuery, args: Value) -> Result )) } -/// Index of `impl Trait for Type` blocks. -/// -/// Both `trait` and `type` arguments are optional. With neither, every impl -/// in the graph is returned (capped by `limit`). Surfaces trait-dispatch -/// information that is otherwise hidden behind raw `Implements` edges. -#[hotpath::measure(label = "mcp.graph.impls.total")] -pub async fn handle_impls(graph: &VerifiedGraphQuery, args: Value) -> Result { - let trait_filter = args.get("trait").and_then(|v| v.as_str()); - let type_filter = args.get("type").and_then(|v| v.as_str()); - let limit = args - .get("limit") - .and_then(serde_json::Value::as_u64) - .map_or(100, |v| v.min(1000) as usize); - - let mut after = None; - let mut results = Vec::new(); - let mut examined = 0usize; - let mut generation_complete = false; - hotpath::measure_block!("mcp.graph.impls.graph", { - while results.len() <= limit { - if examined >= 500_000 { - return Err(TraceDecayError::ProjectRoute { - reason_code: "verified-code-graph-budget-exhausted".to_owned(), - retryable: false, - detail: "impl census exceeded 500000 verified symbols".to_owned(), - }); - } - let page = graph.symbols_page(after.as_ref(), 1_024)?; - examined = examined.saturating_add(page.symbols.len()); - after = page.symbols.last().map(|symbol| symbol.occurrence.clone()); - for impl_node in page.symbols { - let metadata = required_graph_metadata(&impl_node)?; - if metadata.kind != NodeKind::Impl.as_str() - || type_filter.is_some_and(|query| !graph_name_matches(metadata, query)) - { - continue; - } - let traits = single_graph_adjacency_batch(graph.callees( - std::slice::from_ref(&impl_node.occurrence), - &[RelationEdgeKindV1::Implements], - GRAPH_RELATION_READ_LIMIT, - )?)?; - let trait_node = traits.into_iter().next().map(|edge| edge.neighbor); - if trait_filter.is_some_and(|query| { - trait_node - .as_ref() - .and_then(|node| node.metadata.as_ref()) - .is_none_or(|metadata| !graph_name_matches(metadata, query)) - }) { - continue; - } - results.push((impl_node, trait_node)); - if results.len() > limit { - break; - } - } - if !page.has_more { - generation_complete = true; - break; - } - } - }); - let truncated = !generation_complete || results.len() > limit; - results.truncate(limit); - - let result_paths = results - .iter() - .map(|(impl_node, _)| required_graph_file_path(impl_node)) - .collect::>>()?; - let touched_files = unique_file_paths(result_paths.into_iter()); - - let items = results - .iter() - .map(|(impl_node, trait_node)| { - let metadata = required_graph_metadata(impl_node)?; - let file_path = required_graph_file_path(impl_node)?; - let trait_metadata = trait_node - .as_ref() - .map(required_graph_metadata) - .transpose()?; - Ok(json!({ - "impl_id": impl_node.occurrence.as_str(), - "type": metadata.simple_name, - "qualified_name": metadata.qualified_name, - "trait": trait_metadata.map(|value| value.simple_name.as_str()), - "trait_qualified_name": trait_metadata.map(|value| value.qualified_name.as_str()), - "trait_id": trait_node.as_ref().map(|value| value.occurrence.as_str()), - "file": file_path, - "start_line": user_line(metadata.start_line), - "end_line": user_line(graph_symbol_end_line(metadata)?), - "signature": metadata.signature, - })) - }) - .collect::>>()?; - - let output = hotpath::measure_block!( - "mcp.graph.impls.serialize", - json!({ - "count": items.len(), - "truncated": truncated, - "impls": items, - }) - ); - Ok(generic_tool_result( - Some(graph.project_root()?), - &args, - &output, - touched_files, - )) -} - /// Derive annotations attached to a symbol. Accepts `node_id` or /// `qualified_name`. Macro expansion is outside the retained syntax evidence. #[hotpath::measure(label = "mcp.graph.derives.total")] @@ -676,206 +277,7 @@ pub async fn handle_derives(graph: &VerifiedGraphQuery, args: Value) -> Result, -) -> Result { - let trait_name = args.get("trait").and_then(|v| v.as_str()); - let method_name = args.get("method").and_then(|v| v.as_str()); - - if trait_name.is_none() && method_name.is_none() { - return Err(TraceDecayError::Config { - message: "missing required parameter: 'trait' or 'method'".to_string(), - }); - } - if trait_name.is_some() && method_name.is_some() { - return Err(TraceDecayError::Config { - message: "tracedecay_implementations: 'trait' and 'method' are mutually exclusive" - .to_string(), - }); - } - - let limit = args - .get("limit") - .and_then(serde_json::Value::as_u64) - .map_or(20, |v| v.clamp(1, 200) as usize); - - let mut entries: Vec = Vec::new(); - let mut touched: Vec = Vec::new(); - - hotpath::measure_block!("mcp.graph.implementations.graph", { - if let Some(name) = trait_name { - let candidates = graph.resolve_simple_name(name, None, 50)?; - let trait_nodes: Vec<_> = candidates - .into_iter() - .filter(|node| { - node.metadata.as_ref().is_some_and(|metadata| { - matches!( - NodeKind::from_str(&metadata.kind), - Some(NodeKind::Trait | NodeKind::Interface | NodeKind::InterfaceType) - ) - }) - }) - .collect(); - if trait_nodes.is_empty() { - return Ok(text_tool_result( - &format!("No trait or interface named '{name}' found."), - vec![], - )); - } - - for trait_node in trait_nodes { - let trait_metadata = required_graph_metadata(&trait_node)?; - let implementors = single_graph_adjacency_batch(graph.callers( - std::slice::from_ref(&trait_node.occurrence), - &[RelationEdgeKindV1::Implements], - GRAPH_RELATION_READ_LIMIT, - )?)?; - for implementor in implementors { - let impl_node = implementor.neighbor; - let impl_metadata = required_graph_metadata(&impl_node)?; - let impl_file = required_graph_file_path(&impl_node)?; - if scope_prefix.is_some_and(|prefix| !impl_file.starts_with(prefix)) { - continue; - } - let methods = collect_method_bodies(graph, &impl_node)?; - if !touched.iter().any(|path| path == impl_file) { - touched.push(impl_file.to_owned()); - } - entries.push(json!({ - "type": impl_metadata.simple_name, - "qualified_name": impl_metadata.qualified_name, - "kind": impl_metadata.kind, - "file": impl_file, - "line": user_line(impl_metadata.start_line), - "trait": trait_metadata.qualified_name, - "methods": methods, - })); - if entries.len() >= limit { - break; - } - } - if entries.len() >= limit { - break; - } - } - } else if let Some(name) = method_name { - let nodes = graph.resolve_simple_name(name, None, limit.saturating_mul(4))?; - let mut method_nodes = Vec::new(); - for node in nodes { - let metadata = required_graph_metadata(&node)?; - if !matches!( - NodeKind::from_str(&metadata.kind), - Some(NodeKind::Function | NodeKind::Method) - ) { - continue; - } - let file_path = required_graph_file_path(&node)?; - if scope_prefix.is_none_or(|prefix| file_path.starts_with(prefix)) { - method_nodes.push(node); - if method_nodes.len() == limit { - break; - } - } - } - if method_nodes.is_empty() { - return Ok(text_tool_result( - &format!("No function or method named '{name}' found."), - vec![], - )); - } - for n in method_nodes { - let metadata = required_graph_metadata(&n)?; - let file_path = required_graph_file_path(&n)?; - let source = graph.read_indexed_source_file(file_path)?; - let end_line = graph_symbol_end_line(metadata)?; - let body = - crate::handlers::info::extract_lines(&source, metadata.start_line, end_line); - if !touched.iter().any(|path| path == file_path) { - touched.push(file_path.to_owned()); - } - entries.push(json!({ - "name": metadata.simple_name, - "qualified_name": metadata.qualified_name, - "kind": metadata.kind, - "file": file_path, - "line": user_line(metadata.start_line), - "end_line": user_line(end_line), - "signature": metadata.signature, - "body": body, - })); - } - } - }); - - let payload = hotpath::measure_block!( - "mcp.graph.implementations.serialize", - json!({ - "match_count": entries.len(), - "implementations": entries, - }) - ); - Ok(generic_tool_result( - Some(graph.project_root()?), - &args, - &payload, - touched, - )) -} - fn bound_source_file_len(graph: &VerifiedGraphQuery, file_path: &str) -> Result { let (absolute, _) = graph.resolve_indexed_source_file(file_path)?; Ok(std::fs::metadata(absolute)?.len()) } - -fn collect_method_bodies( - graph: &VerifiedGraphQuery, - impl_node: &CodeGraphSymbolSummaryV1, -) -> Result> { - let children = single_graph_adjacency_batch(graph.callees( - std::slice::from_ref(&impl_node.occurrence), - &[RelationEdgeKindV1::Contains], - GRAPH_RELATION_READ_LIMIT, - )?)?; - let mut methods = Vec::new(); - for child in children { - let child = child.neighbor; - let metadata = required_graph_metadata(&child)?; - if !matches!( - NodeKind::from_str(&metadata.kind), - Some(NodeKind::Method | NodeKind::Function) - ) { - continue; - } - let file_path = required_graph_file_path(&child)?.to_owned(); - methods.push((file_path, metadata.start_line, child)); - } - methods.sort_by(|left, right| { - (&left.0, left.1, &left.2.occurrence).cmp(&(&right.0, right.1, &right.2.occurrence)) - }); - - let mut out: Vec = Vec::new(); - let mut cached_path: Option = None; - let mut cached_source = String::new(); - for (file_path, _, child) in methods { - let metadata = required_graph_metadata(&child)?; - if cached_path.as_deref() != Some(file_path.as_str()) { - cached_source = graph.read_indexed_source_file(&file_path)?; - cached_path = Some(file_path); - } - let end_line = graph_symbol_end_line(metadata)?; - let body = - crate::handlers::info::extract_lines(&cached_source, metadata.start_line, end_line); - out.push(json!({ - "name": metadata.simple_name, - "kind": metadata.kind, - "line": user_line(metadata.start_line), - "signature": metadata.signature, - "body": body, - })); - } - Ok(out) -} diff --git a/crates/tracedecay-mcp/src/handlers/graph/search.rs b/crates/tracedecay-mcp/src/handlers/graph/search.rs index 9768d9b8b3..fcd1c54f38 100644 --- a/crates/tracedecay-mcp/src/handlers/graph/search.rs +++ b/crates/tracedecay-mcp/src/handlers/graph/search.rs @@ -2,39 +2,40 @@ //! `search`, `context`, `similar`, `find_exact_symbol`, `rename_preview`. use std::collections::HashMap; -use std::fmt::Write as _; use std::future::Future; use std::path::Path; use serde_json::{Value, json}; use tracedecay_code_index::graph_projection::CodeGraphSymbolSummaryV1; +use tracedecay_contracts::InvocationAnalyticsV1; +use tracedecay_contracts::graph_tool::{GraphToolCompletionV1, GraphToolResultV1}; use tracedecay_contracts::retrieval::{ ContextCodeBlockV1, ContextModeV1, ContextResultV1, ContextSearchMatchV1, ContextSurfaceRequestV1, RedundancyScopeV1, RedundancySurfaceRequestV1, RenamePreviewNodeV1, - RenamePreviewPrimitiveRequestV1, RenamePreviewPrimitiveResultV1, RenamePreviewReferenceV1, - RenamePreviewTextOnlyMatchV1, SimilarCoverageV1, SimilarFamilyV1, SimilarMatchClassV1, - SimilarOccurrenceV1, SimilarResultV1, SimilarSurfaceRequestV1, SimilarTargetV1, + RenamePreviewPrimitiveOutcomeV1, RenamePreviewPrimitiveRequestV1, + RenamePreviewPrimitiveResultV1, RenamePreviewReferenceV1, RenamePreviewTextOnlyMatchV1, + SimilarCoverageV1, SimilarFamilyV1, SimilarMatchClassV1, SimilarOccurrenceV1, SimilarResultV1, + SimilarSurfaceRequestV1, SimilarTargetV1, }; use tracedecay_domain::ExactClass; use tracedecay_domain::errors::{Result, TraceDecayError}; #[cfg(test)] use tracedecay_query::retrieval::lexical::LexicalRoutingV1; +#[cfg(test)] use crate::context_headings::CONTEXT_SEEN_NODE_IDS_LABEL; use crate::handlers::dependency_hints; use crate::handlers::support::{ - CONTEXT_MEMORY_ANALYTICS_KEY, decode_primitive_request, generic_tool_result as support_generic, - rendered_tool_result as support_rendered, retrieval_cursor, - take_internal_context_memory_analytics, text_tool_result, unique_file_paths, + decode_primitive_request, generic_tool_result as support_generic, + rendered_tool_result as support_rendered, retrieval_cursor, unique_file_paths, }; use crate::tools::render::{self, Md}; use crate::{McpToolContext, ToolResult}; -use super::context_markdown::{append_verified_plan_context, verified_context_markdown}; +use super::context_markdown::verified_plan_context; use super::context_support::{ - ContextMemoryOutcome, context_markdown_lane_preview, context_memory_analytics_value, - context_memory_options, context_memory_outcome, context_memory_read_control, - insert_context_memory_section, + ContextMemoryOutcome, context_memory_analytics, context_memory_options, context_memory_outcome, + context_memory_read_control, }; use super::primitive_surface::{ search_coverage as primitive_search_coverage, symbol_location as primitive_symbol_location, @@ -48,13 +49,13 @@ use super::search_freshness::{ use super::verified::CODE_SYMBOL_EVIDENCE_PREFIX; use super::{ graph_occurrence_id, graph_symbol_end_line, graph_symbol_paths, graph_symbols_in_scope, - line_for_byte_offset, node_not_found as node_not_found_result, required_graph_file_path, + graph_tool_completion, line_for_byte_offset, node_not_found_result, required_graph_file_path, required_graph_metadata, single_graph_adjacency_batch, user_line, }; use super::{lexical_routing, search_evidence}; #[cfg(test)] -use super::context_support::context_memory_section; +use super::context_support::{context_markdown_lane_preview, context_memory_section}; async fn execute_code_index_search( executor: Option<&tracedecay_query::code_search::CodeIndexSearchExecutor>, @@ -153,32 +154,6 @@ fn generic_tool_result( support_generic(Some(ctx.project_root()), args, value, touched_files) } -fn rendered_context_tool_result( - ctx: &McpToolContext<'_>, - args: &Value, - mut value: Value, - touched_files: Vec, - full_markdown: String, - preview_markdown: Option<&str>, -) -> ToolResult { - let internal_analytics = take_internal_context_memory_analytics(&mut value); - let text = if render::wants_json(args) { - render::finalize(Some(ctx.project_root()), args, &value, || full_markdown) - } else { - render::markdown_preview_with_handle( - Some(ctx.project_root()), - &full_markdown, - preview_markdown.unwrap_or(&full_markdown), - ) - }; - let result = text_tool_result(&text, touched_files); - if let Some(internal_analytics) = internal_analytics { - result.with_internal_analytics(internal_analytics) - } else { - result - } -} - #[hotpath::measure(label = "mcp.graph.search.total")] pub async fn handle_search( ctx: &McpToolContext<'_>, @@ -421,7 +396,7 @@ where /// Warns, in the human-facing body, that a result list is short because a lane /// was missing. A degraded page is otherwise indistinguishable from a thorough /// one, which is exactly how a partial answer gets trusted as a complete one. -fn append_coverage_md(md: &mut Md, value: &Value) { +pub(super) fn append_coverage_md(md: &mut Md, value: &Value) { let Some(coverage) = value.get("coverage") else { return; }; @@ -683,7 +658,7 @@ fn context_graph_projection( file: file_path.to_owned(), start_line: user_line(metadata.start_line), end_line: user_line(graph_symbol_end_line(metadata)?), - code: crate::handlers::info::extract_lines( + code: extract_lines( source, metadata.start_line, graph_symbol_end_line(metadata)?, @@ -699,31 +674,36 @@ fn context_graph_projection( }) } -fn append_context_search_matches(output: &mut String, matches: &[ContextSearchMatchV1]) { - if matches.is_empty() { - return; +/// Extract the source spanning tree-sitter rows `start_line..=end_line` +/// (0-based, inclusive) from `source`. Node line fields are stored as the +/// raw tree-sitter row index, so the caller passes them through unchanged. +/// Returns the empty string if the range is out of bounds. +fn extract_lines(source: &str, start_line: u32, end_line: u32) -> String { + let start = start_line as usize; + let end_exclusive = (end_line as usize).saturating_add(1); + if start >= end_exclusive { + return String::new(); } - output.push_str("\n### Available Code Search Matches\n"); - for search_match in matches { - let _ = writeln!( - output, - "- **{}** ({}), `{}` · rank {} · utility {}", - search_match.name, - search_match.kind, - search_match.file, - search_match.rank, - search_match.utility_micros, - ); + let mut selected = source.lines().skip(start).take(end_exclusive - start); + let Some(first) = selected.next() else { + return String::new(); + }; + let mut body = String::with_capacity(first.len()); + body.push_str(first); + for line in selected { + body.push('\n'); + body.push_str(line); } + body } #[hotpath::measure(label = "mcp.graph.context.total")] -pub async fn handle_context( +pub async fn compute_context( ctx: &McpToolContext<'_>, graph: F, args: Value, scope_prefix: Option<&str>, -) -> Result +) -> Result where F: Future>, { @@ -842,117 +822,58 @@ where graph_coverage: memory_graph_coverage, error: memory_matches_error, } = memory_outcome; - let seeds = projection - .selected - .iter() - .map(|symbol| symbol.occurrence.clone()) - .collect::>(); - let symbol_values = projection + let symbols = projection .selected .iter() .map(primitive_symbol_location) .collect::>>()?; - let related_values = projection + let related_symbols = projection .related .iter() .map(primitive_symbol_location) .collect::>>()?; - let symbol_render_values = symbol_values - .iter() - .map(serde_json::to_value) - .collect::, _>>()?; - let related_render_values = related_values - .iter() - .map(serde_json::to_value) - .collect::, _>>()?; - let code_render_values = projection - .code_blocks - .iter() - .map(serde_json::to_value) - .collect::, _>>()?; - let mut output = freshness_lines(&freshness); - output.push_str(&verified_context_markdown( - task, - &symbol_render_values, - &related_render_values, - &code_render_values, - )?); - if symbol_values.is_empty() { - append_context_search_matches(&mut output, &search_matches); - } - insert_context_memory_section( - &mut output, - &memory_matches, - memory_matches_error.as_deref(), + let plan = match (mode, graph.as_ref()) { + (ContextModeV1::Plan, Some(graph)) => { + Some(verified_plan_context(graph, &projection.selected)?) + } + _ => None, + }; + let touched_files = unique_file_paths( + projection.touched_files.iter().map(String::as_str).chain( + search_matches + .iter() + .map(|search_match| search_match.file.as_str()), + ), ); - if mode == ContextModeV1::Plan - && let Some(graph) = graph.as_ref() - { - append_verified_plan_context(graph, &projection.selected, &mut output)?; - } - - if !seeds.is_empty() { - let _ = write!( - output, - "\n{} {}\n", - CONTEXT_SEEN_NODE_IDS_LABEL, - serde_json::to_string(&seeds)? - ); - } - + let analytics = InvocationAnalyticsV1 { + context_memory: Some(context_memory_analytics( + &memory_options, + &memory_matches, + memory_matches_error.as_deref(), + )), + }; let result = ContextResultV1 { task: request.task, mode, freshness, code_generation, - search_matches: search_matches.clone(), - symbols: symbol_values, - related_symbols: related_values, + search_matches, + symbols, + related_symbols, code: projection.code_blocks, coverage, - memory_matches: memory_matches.clone(), + memory_matches, memory_graph_coverage, - memory_matches_error: memory_matches_error.clone(), + memory_matches_error, verified_graph_evidence, + plan, }; - let mut value = - hotpath::measure_block!("mcp.graph.context.serialize", serde_json::to_value(result)?); - if let Some(object) = value.as_object_mut() { - object.insert( - CONTEXT_MEMORY_ANALYTICS_KEY.to_string(), - json!({ - "context_memory": context_memory_analytics_value( - &memory_options, - &memory_matches, - memory_matches_error.as_deref() - ), - }), - ); - } - let mut degradation = Md::new(); - append_coverage_md(&mut degradation, &value); - search_evidence::append_verified_graph_evidence_md(&mut degradation, &value); - let degradation = degradation.render(); - if !degradation.is_empty() { - output.push('\n'); - output.push_str(°radation); - } - let touched_files = unique_file_paths( - projection.touched_files.iter().map(String::as_str).chain( - search_matches - .iter() - .map(|search_match| search_match.file.as_str()), - ), - ); - let preview = (!render::wants_json(&args)).then(|| context_markdown_lane_preview(&output)); - Ok(rendered_context_tool_result( - ctx, - &args, - value, + Ok(GraphToolCompletionV1 { + result: GraphToolResultV1::Context(Box::new(result)), touched_files, - output, - preview.as_deref(), - )) + code_graph: None, + analytics: Some(analytics), + }) } /// Bare-name lookup against `idx_nodes_name`, no BM25 scoring, no fuzzy @@ -1032,7 +953,10 @@ pub async fn handle_find_exact_symbol( } #[hotpath::measure(label = "mcp.graph.similar.total")] -pub async fn handle_similar(ctx: &McpToolContext<'_>, args: Value) -> Result { +pub async fn compute_similar( + ctx: &McpToolContext<'_>, + args: Value, +) -> Result { let request: SimilarSurfaceRequestV1 = decode_primitive_request(&args, "tracedecay_similar")?; let project_id = request.project_id; let repository_id = request.repository_id; @@ -1184,9 +1108,10 @@ pub async fn handle_similar(ctx: &McpToolContext<'_>, args: Value) -> Result, args: Value) -> Result { +pub async fn compute_redundancy( + ctx: &McpToolContext<'_>, + args: Value, +) -> Result { let request: RedundancySurfaceRequestV1 = decode_primitive_request(&args, "tracedecay_redundancy")?; if request.project_id != ctx.admitted_scope().project_id @@ -1297,11 +1225,10 @@ pub async fn handle_redundancy(ctx: &McpToolContext<'_>, args: Value) -> Result< .collect::>(); touched_files.sort(); touched_files.dedup(); - let value = hotpath::measure_block!( - "mcp.graph.redundancy.serialize", - serde_json::to_value(outcome)? - ); - Ok(generic_tool_result(ctx, &args, &value, touched_files)) + Ok(graph_tool_completion( + GraphToolResultV1::Redundancy(outcome), + touched_files, + )) } fn similar_occurrence( @@ -1410,11 +1337,11 @@ struct RenameReferenceSiteInput { /// that are NOT backed by a graph edge ("text-only matches, review /// manually"). Nothing is rewritten. #[hotpath::measure(label = "mcp.graph.rename_preview.total")] -pub async fn handle_rename_preview( +pub async fn compute_rename_preview( ctx: &McpToolContext<'_>, graph: &tracedecay_graph_query::VerifiedGraphQuery, args: Value, -) -> Result { +) -> Result { let request: RenamePreviewPrimitiveRequestV1 = decode_primitive_request(&args, "tracedecay_rename_preview")?; @@ -1428,7 +1355,12 @@ pub async fn handle_rename_preview( let (mut declaration, declaration_line, symbol_name, reference_inputs) = hotpath::measure_block!("mcp.graph.rename_preview.graph", { let Some(node) = graph.symbol_summary(&occurrence)? else { - return node_not_found_result(&request.node_id); + return Ok(graph_tool_completion( + GraphToolResultV1::RenamePreview(RenamePreviewPrimitiveOutcomeV1::NotFound( + node_not_found_result(&request.node_id), + )), + Vec::new(), + )); }; let node_metadata = required_graph_metadata(&node)?; let node_file = required_graph_file_path(&node)?; @@ -1560,26 +1492,25 @@ pub async fn handle_rename_preview( })??; declaration.snippet = decl_snippet; - let output = hotpath::measure_block!( - "mcp.graph.rename_preview.serialize", - serde_json::to_value(RenamePreviewPrimitiveResultV1 { - read_only: true, - note: "Preview only. Nothing is edited. 'references' are graph reference sites \ + let result = RenamePreviewPrimitiveResultV1 { + read_only: true, + note: "Preview only. Nothing is edited. 'references' are graph reference sites \ (the declaration is reported separately in 'node'); 'text_only_matches' are \ literal name occurrences NOT backed by a graph edge (comments, strings, \ dynamic dispatch, unresolved refs) and must be reviewed by hand. Graph \ call-edge coverage improves as the resolver does." - .to_owned(), - symbol: symbol_name, - new_name: request.new_name, - node: declaration, - reference_count: references.len(), - references, - text_only_matches, - })? - ); - - Ok(generic_tool_result(ctx, &args, &output, touched_files)) + .to_owned(), + symbol: symbol_name, + new_name: request.new_name, + node: declaration, + reference_count: references.len(), + references, + text_only_matches, + }; + Ok(graph_tool_completion( + GraphToolResultV1::RenamePreview(RenamePreviewPrimitiveOutcomeV1::Preview(result)), + touched_files, + )) } #[cfg(test)] @@ -1591,56 +1522,50 @@ mod tests { fn clone_lanes_report_one_unavailable_wire_protocol() { use tracedecay_query::code_search::CodeIndexSearchUnavailableReasonV1 as Reason; - // Retired branch-local opaque tokens, never shipped on master; must - // stay absent from the shared mapper wire (migrate-then-delete, not - // one-release alias). Request-schema / catalog `alias_of` for these - // tools lives on #1433 and is separable. - const RETIRED_OPAQUE_REASON_CODES: &[&str] = &[ - "verified-code-redundancy-unavailable", - "verified-code-similarity-unavailable", - ]; - - for reason in [ - Reason::CapabilityUnavailable, - Reason::AuthorityUnavailable, - Reason::LinkedWorktreeDisabled, - Reason::Cancelled, - Reason::TimedOut, - Reason::CapacityUnavailable, - Reason::GenerationUnavailable, - Reason::GenerationUnverified, - Reason::InvalidRequest, - Reason::CorruptionResetRequired, - Reason::Internal, + for (reason, code, retryable) in [ + ( + Reason::CapabilityUnavailable, + "code_index_unavailable", + false, + ), + (Reason::AuthorityUnavailable, "authority_unavailable", false), + ( + Reason::LinkedWorktreeDisabled, + "linked_worktree_disabled", + false, + ), + (Reason::Cancelled, "cancelled", true), + (Reason::TimedOut, "timed_out", true), + ( + Reason::CapacityUnavailable, + "search_capacity_unavailable", + true, + ), + ( + Reason::GenerationUnavailable, + "generation_unavailable", + true, + ), + (Reason::GenerationUnverified, "generation_unverified", true), + (Reason::InvalidRequest, "invalid_request", false), + ( + Reason::CorruptionResetRequired, + "index_corruption_reset_required", + false, + ), + (Reason::Internal, "search_failed", false), ] { - let similarity = clone_lane_unavailable_error("similarity", reason); - let family = clone_lane_unavailable_error("family", reason); - let (similarity_code, similarity_retryable, similarity_detail) = similarity - .project_route_context() - .expect("clone lane failures are typed project-route errors"); - let (family_code, family_retryable, family_detail) = family - .project_route_context() - .expect("clone lane failures are typed project-route errors"); - assert_eq!(similarity_code, reason.as_str()); - assert_eq!(family_code, reason.as_str()); - assert_eq!(similarity_retryable, reason.is_retryable()); - assert_eq!(family_retryable, reason.is_retryable()); - for retired in RETIRED_OPAQUE_REASON_CODES { - assert_ne!( - similarity_code, *retired, - "similarity lane must not re-emit retired opaque reason" - ); - assert_ne!( - family_code, *retired, - "family lane must not re-emit retired opaque reason" - ); - assert!( - !similarity_detail.contains(retired), - "similarity detail must not mention retired opaque reason" - ); - assert!( - !family_detail.contains(retired), - "family detail must not mention retired opaque reason" + for lane in ["similarity", "family"] { + let error = clone_lane_unavailable_error(lane, reason); + assert_eq!( + error + .project_route_context() + .expect("clone lane failures are typed project-route errors"), + ( + code, + retryable, + format!("the maintained clone {lane} lane is unavailable: {code}").as_str(), + ) ); } } @@ -1730,7 +1655,7 @@ mod tests { }, }) .expect("admitted similar binding"); - let result = handle_similar( + let result = compute_similar( &ctx, json!({ "project_id": admitted.project_id, @@ -1799,7 +1724,7 @@ mod tests { }, }) .expect("admitted redundancy binding"); - let result = handle_redundancy( + let result = compute_redundancy( &ctx, json!({ "project_id": admitted.project_id, @@ -1865,7 +1790,7 @@ mod tests { }, }) .expect("admitted similar-only binding"); - let redundancy_err = handle_redundancy( + let redundancy_err = compute_redundancy( &redundancy_ctx, json!({ "project_id": admitted.project_id, @@ -1908,7 +1833,7 @@ mod tests { }, }) .expect("admitted redundancy-only binding"); - let similar_err = handle_similar( + let similar_err = compute_similar( &similar_ctx, json!({ "project_id": admitted.project_id, diff --git a/crates/tracedecay-mcp/src/handlers/graph/verified.rs b/crates/tracedecay-mcp/src/handlers/graph/verified.rs index b7540ad57a..79d1b67aa0 100644 --- a/crates/tracedecay-mcp/src/handlers/graph/verified.rs +++ b/crates/tracedecay-mcp/src/handlers/graph/verified.rs @@ -5,7 +5,6 @@ use std::collections::HashSet; use serde_json::{Value, json}; use tracedecay_code_index::graph_projection::CodeGraphSymbolSummaryV1; use tracedecay_code_index::lineage::LineageSymbolRecordV1; -use tracedecay_domain::code_intelligence::{EdgeKind, NodeKind}; use tracedecay_domain::errors::{Result, TraceDecayError}; use tracedecay_domain::{RelationEdgeKindV1, SymbolOccurrenceId}; use tracedecay_graph_query::VerifiedGraphQuery; @@ -156,33 +155,6 @@ pub fn graph_symbol_location_value(symbol: &CodeGraphSymbolSummaryV1) -> Result< })) } -pub fn graph_name_matches(metadata: &LineageSymbolRecordV1, query: &str) -> bool { - let query = query.to_ascii_lowercase(); - metadata.simple_name.to_ascii_lowercase().contains(&query) - || metadata - .qualified_name - .to_ascii_lowercase() - .contains(&query) -} - -pub fn canonical_relation_kind(kind: EdgeKind) -> Result { - match kind { - EdgeKind::Calls => Ok(RelationEdgeKindV1::Calls), - EdgeKind::Uses => Ok(RelationEdgeKindV1::Uses), - EdgeKind::TypeOf => Ok(RelationEdgeKindV1::TypeOf), - EdgeKind::Contains => Ok(RelationEdgeKindV1::Contains), - EdgeKind::Implements => Ok(RelationEdgeKindV1::Implements), - EdgeKind::Extends => Ok(RelationEdgeKindV1::Extends), - EdgeKind::Annotates => Ok(RelationEdgeKindV1::Annotates), - EdgeKind::Returns => Ok(RelationEdgeKindV1::Returns), - EdgeKind::Receives => Ok(RelationEdgeKindV1::Receives), - EdgeKind::DerivesMacro => Err(TraceDecayError::Config { - message: "derive-macro relations are not published in the verified code graph" - .to_owned(), - }), - } -} - pub fn single_graph_adjacency_batch(mut batches: Vec>) -> Result> { if batches.len() != 1 { return Err(graph_symbol_corrupt(format!( @@ -244,90 +216,6 @@ pub fn traverse_verified_neighbors( Ok(results) } -pub fn verified_neighbor_value(result: &VerifiedNeighbor) -> Result { - let metadata = required_graph_metadata(&result.symbol)?; - Ok(json!({ - "node_id": result.symbol.occurrence.as_str(), - "name": metadata.simple_name, - "kind": metadata.kind, - "file": required_graph_file_path(&result.symbol)?, - "line": metadata.start_line.saturating_add(1), - "edge_kind": result.edge_kind.as_str(), - "depth": result.depth, - })) -} - -#[hotpath::measure(label = "mcp.graph.trait_dispatch")] -pub fn verified_trait_dispatch_targets( - graph: &VerifiedGraphQuery, - method: &CodeGraphSymbolSummaryV1, -) -> Result> { - let method_metadata = required_graph_metadata(method)?; - if !matches!( - NodeKind::from_str(&method_metadata.kind), - Some(NodeKind::Method | NodeKind::Function) - ) { - return Ok(Vec::new()); - } - let parents = single_graph_adjacency_batch(graph.callers( - std::slice::from_ref(&method.occurrence), - &[RelationEdgeKindV1::Contains], - GRAPH_RELATION_READ_LIMIT, - )?)?; - let traits = parents - .into_iter() - .map(|edge| edge.neighbor) - .filter(|parent| { - parent.metadata.as_ref().is_some_and(|metadata| { - matches!( - NodeKind::from_str(&metadata.kind), - Some(NodeKind::Trait | NodeKind::Interface | NodeKind::InterfaceType) - ) - }) - }) - .collect::>(); - let trait_occurrences = traits - .iter() - .map(|node| node.occurrence.clone()) - .collect::>(); - if trait_occurrences.is_empty() { - return Ok(Vec::new()); - } - let implementors = graph - .callers( - &trait_occurrences, - &[RelationEdgeKindV1::Implements], - GRAPH_RELATION_READ_LIMIT, - )? - .into_iter() - .flatten() - .map(|edge| edge.neighbor.occurrence) - .collect::>(); - if implementors.is_empty() { - return Ok(Vec::new()); - } - let mut targets = Vec::new(); - for child in graph - .callees( - &implementors, - &[RelationEdgeKindV1::Contains], - GRAPH_RELATION_READ_LIMIT, - )? - .into_iter() - .flatten() - { - let metadata = required_graph_metadata(&child.neighbor)?; - if matches!( - NodeKind::from_str(&metadata.kind), - Some(NodeKind::Method | NodeKind::Function) - ) && metadata.simple_name == method_metadata.simple_name - { - targets.push(child.neighbor); - } - } - Ok(targets) -} - pub fn cost_to_expand_verified( metadata: &LineageSymbolRecordV1, file_size_bytes: u64, diff --git a/crates/tracedecay-mcp/src/handlers/graph_tool.rs b/crates/tracedecay-mcp/src/handlers/graph_tool.rs new file mode 100644 index 0000000000..da96e88fa5 --- /dev/null +++ b/crates/tracedecay-mcp/src/handlers/graph_tool.rs @@ -0,0 +1,308 @@ +//! Graph and port reads served by the project's graph-tool owner. +//! +//! The owner computes each operation's typed catalog result; every surface +//! renders it here, so MCP and the CLI print the same tool result. + +use std::path::Path; + +use serde_json::Value; +use tracedecay_contracts::graph_tool::{GraphToolCompletionV1, GraphToolResultV1}; +use tracedecay_contracts::retrieval::{NodeResultV1, RenamePreviewPrimitiveOutcomeV1}; +use tracedecay_domain::errors::Result; +use tracedecay_tool_catalog::ApplicationSurfaceOperation; + +use crate::handlers::graph::{ + compute_context, compute_impact, compute_node, compute_redundancy, compute_rename_preview, + compute_similar, not_found_tool_result, render_context, +}; +use crate::handlers::info::{compute_port_order, compute_port_status, compute_todos}; +use crate::handlers::support::{generic_tool_result, unknown_tool_error}; +use crate::handlers::verified_read::{VerifiedGraphOpen, verified_read_operation as read}; +use crate::tools::response_trailers::append_code_graph_freshness; +use crate::{McpToolContext, ToolResult}; + +/// Computes one graph-tool operation's typed result on the owner's side. +pub async fn compute_graph_tool( + ctx: &McpToolContext<'_>, + open: &VerifiedGraphOpen<'_>, + operation: ApplicationSurfaceOperation, + args: Value, + scope_prefix: Option<&str>, +) -> Result { + match operation { + ApplicationSurfaceOperation::Context => { + compute_context(ctx, open(read("context")?), args, scope_prefix).await + } + ApplicationSurfaceOperation::Impact => { + compute_impact(&open(read("impact")?).await?, args).await + } + ApplicationSurfaceOperation::Node => compute_node(&open(read("node")?).await?, args).await, + ApplicationSurfaceOperation::Similar => compute_similar(ctx, args).await, + ApplicationSurfaceOperation::Redundancy => compute_redundancy(ctx, args).await, + ApplicationSurfaceOperation::RenamePreview => { + compute_rename_preview(ctx, &open(read("rename_preview")?).await?, args).await + } + ApplicationSurfaceOperation::PortStatus => { + compute_port_status(&open(read("port_status")?).await?, args).await + } + ApplicationSurfaceOperation::PortOrder => { + compute_port_order(&open(read("port_order")?).await?, args).await + } + ApplicationSurfaceOperation::Todos => { + compute_todos(&open(read("todos")?).await?, args, scope_prefix).await + } + operation => Err(unknown_tool_error(operation.mcp_tool_name())), + } +} + +/// Renders a typed graph-tool result as its tool result, with the stale-graph +/// trailer every surface appends. +/// +/// Token accounting stays with the caller that owns raw-file sizes. The MCP +/// server uses its retained cache; the CLI stats the project files. Rendering +/// them here would mark the result accounted and skip that cache. +pub fn render_graph_tool( + project_root: Option<&Path>, + args: &Value, + completion: GraphToolCompletionV1, +) -> Result { + let GraphToolCompletionV1 { + result, + touched_files, + code_graph, + analytics, + } = completion; + let mut rendered = match &result { + GraphToolResultV1::Context(context) => { + render_context(project_root, args, context, touched_files)? + } + GraphToolResultV1::Node(NodeResultV1::NotFound(not_found)) + | GraphToolResultV1::RenamePreview(RenamePreviewPrimitiveOutcomeV1::NotFound(not_found)) => { + not_found_tool_result(not_found)? + } + _ => generic_tool_result(project_root, args, &result.result_value()?, touched_files), + }; + if let Some(served) = &code_graph { + append_code_graph_freshness(&mut rendered, served); + } + Ok(match analytics { + Some(analytics) => rendered.with_internal_analytics(analytics.ledger_value()), + None => rendered, + }) +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use serde_json::json; + use tracedecay_contracts::retrieval::{ + CodeGraphReadFreshnessV1, ContextExtensionPointV1, ContextModeV1, ContextPlanV1, + ContextResultV1, PrimitiveFreshnessStateV1, PrimitiveLaneCompleteV1, PrimitiveLaneStatusV1, + PrimitiveRecallV1, PrimitiveSearchCoverageV1, PrimitiveSearchFreshnessV1, + PrimitiveSymbolLocationV1, ServedCodeGraphGenerationV1, TodoMarkerV1, TodosResultV1, + }; + use tracedecay_contracts::{ContextMemoryAnalyticsV1, InvocationAnalyticsV1}; + use tracedecay_domain::UtcMicros; + + use super::*; + + fn todos_completion(code_graph: Option) -> GraphToolCompletionV1 { + GraphToolCompletionV1 { + result: GraphToolResultV1::Todos(TodosResultV1 { + match_count: 1, + by_kind: BTreeMap::from([("TODO".to_owned(), 1)]), + markers: vec![TodoMarkerV1 { + kind: "TODO".to_owned(), + file: "src/lib.rs".to_owned(), + line: 1, + text: "// TODO: probe".to_owned(), + enclosing: None, + }], + }), + touched_files: vec!["src/lib.rs".to_owned()], + code_graph, + analytics: None, + } + } + + fn texts(result: &ToolResult) -> Vec { + result.value["content"] + .as_array() + .expect("content blocks") + .iter() + .map(|block| block["text"].as_str().expect("text block").to_owned()) + .collect() + } + + #[test] + fn a_stale_seat_renders_its_trailer_before_the_accounting_footer() { + let root = tempfile::tempdir().expect("project"); + std::fs::create_dir_all(root.path().join("src")).expect("src"); + std::fs::write(root.path().join("src/lib.rs"), "a".repeat(800)).expect("source"); + let mut rendered = render_graph_tool( + Some(root.path()), + &json!({"format": "json"}), + todos_completion(Some(ServedCodeGraphGenerationV1 { + generation: "generation.render.stale".to_owned(), + freshness: CodeGraphReadFreshnessV1::LastCompleteStale { + sealed_at: UtcMicros(0), + rebuild_in_flight: false, + }, + })), + ) + .expect("rendered"); + crate::tools::response_trailers::account_tool_result(Some(root.path()), &mut rendered); + let blocks = texts(&rendered); + assert_eq!(blocks.len(), 3, "{blocks:?}"); + assert!( + blocks[1].starts_with( + "\ncode_graph_freshness: stale, serving the last complete generation \ + generation.render.stale (sealed " + ), + "{blocks:?}" + ); + assert!( + blocks[1].ends_with( + "ago) while source freshness remains unverified; results may trail the live worktree" + ), + "{blocks:?}" + ); + let after = (blocks[0].len() + blocks[1].len()) / 4; + assert_eq!( + blocks[2], + format!("\ntracedecay_metrics: before=200 after={after}") + ); + assert_eq!(rendered.touched_files, vec!["src/lib.rs".to_owned()]); + } + + fn plan_context() -> ContextResultV1 { + ContextResultV1 { + task: "extend the store".to_owned(), + mode: ContextModeV1::Plan, + freshness: PrimitiveSearchFreshnessV1 { + state: PrimitiveFreshnessStateV1::Fresh, + indexing: None, + }, + code_generation: Some("generation.context".to_owned()), + search_matches: Vec::new(), + symbols: vec![PrimitiveSymbolLocationV1 { + node_id: "symbol.store".to_owned(), + name: "Store".to_owned(), + qualified_name: "crate::Store".to_owned(), + kind: "trait".to_owned(), + file: "src/store.rs".to_owned(), + start_line: 3, + end_line: 9, + unavailable_fields: Vec::new(), + }], + related_symbols: Vec::new(), + code: Vec::new(), + coverage: PrimitiveSearchCoverageV1 { + exact: PrimitiveLaneStatusV1::Complete(PrimitiveLaneCompleteV1::Complete), + lexical: PrimitiveLaneStatusV1::Complete(PrimitiveLaneCompleteV1::Complete), + graph: PrimitiveLaneStatusV1::Complete(PrimitiveLaneCompleteV1::Complete), + recall: PrimitiveRecallV1::Full, + }, + memory_matches: Vec::new(), + memory_graph_coverage: None, + memory_matches_error: None, + verified_graph_evidence: None, + plan: Some(ContextPlanV1 { + extension_points: vec![ContextExtensionPointV1 { + name: "Store".to_owned(), + kind: "trait".to_owned(), + file: "src/store.rs".to_owned(), + line: 3, + implementor_count: 2, + }], + test_files: Some(Vec::new()), + }), + } + } + + #[test] + fn context_renders_plan_markdown_and_records_memory_analytics_beside_it() { + let analytics = InvocationAnalyticsV1 { + context_memory: Some(ContextMemoryAnalyticsV1 { + include_memory: true, + limit: 3, + min_trust_millionths: 500_000, + fact_ids: vec!["fact.one".to_owned()], + error: None, + }), + }; + let completion = |analytics| GraphToolCompletionV1 { + result: GraphToolResultV1::Context(Box::new(plan_context())), + touched_files: Vec::new(), + code_graph: None, + analytics, + }; + let markdown = render_graph_tool(None, &json!({}), completion(Some(analytics.clone()))) + .expect("markdown"); + let text = texts(&markdown).join(""); + assert!( + text.contains( + "### Extension Points\n- **Store** (trait) - src/store.rs:3 (2 implementors)\n" + ), + "{text}" + ); + assert!( + text.contains("### Test Coverage\n_No test files found covering these modules._\n"), + "{text}" + ); + assert!(text.contains("seen_node_ids: [\"symbol.store\"]"), "{text}"); + assert_eq!( + markdown.internal_analytics(), + Some(&json!({"context_memory": { + "include_memory": true, + "limit": 3, + "min_trust": 0.5, + "match_count": 1, + "fact_ids": ["fact.one"], + "error": null, + }})) + ); + + let as_json = render_graph_tool( + None, + &json!({"format": "json"}), + completion(Some(analytics)), + ) + .expect("json"); + let payload: serde_json::Value = + serde_json::from_str(&texts(&as_json)[0]).expect("json payload"); + assert_eq!( + payload["plan"]["extension_points"][0]["implementor_count"], + 2 + ); + assert!(payload.get("context_memory").is_none(), "{payload}"); + assert!(payload.get("analytics").is_none(), "{payload}"); + } + + #[test] + fn a_current_seat_renders_only_the_accounting_footer() { + let root = tempfile::tempdir().expect("project"); + std::fs::create_dir_all(root.path().join("src")).expect("src"); + std::fs::write(root.path().join("src/lib.rs"), "b".repeat(40)).expect("source"); + let mut rendered = render_graph_tool( + Some(root.path()), + &json!({"format": "json"}), + todos_completion(Some(ServedCodeGraphGenerationV1 { + generation: "generation.render.current".to_owned(), + freshness: CodeGraphReadFreshnessV1::Current, + })), + ) + .expect("rendered"); + crate::tools::response_trailers::account_tool_result(Some(root.path()), &mut rendered); + let blocks = texts(&rendered); + assert_eq!(blocks.len(), 2, "{blocks:?}"); + assert_eq!( + blocks[1], + format!( + "\ntracedecay_metrics: before=10 after={}", + blocks[0].len() / 4 + ) + ); + } +} diff --git a/crates/tracedecay-mcp/src/handlers/health/dispatch.rs b/crates/tracedecay-mcp/src/handlers/health/dispatch.rs index 3388feb8b3..85e917897b 100644 --- a/crates/tracedecay-mcp/src/handlers/health/dispatch.rs +++ b/crates/tracedecay-mcp/src/handlers/health/dispatch.rs @@ -34,7 +34,7 @@ pub async fn dispatch_tool( handle_dependency_depth(&open(read("health_read")?).await?, args, scope_prefix).await } "tracedecay_health" => { - handle_health(&open(read("health_read")?).await?, args, scope_prefix).await + handle_health(&open(read("health_delta")?).await?, args, scope_prefix).await } "tracedecay_dsm" => { handle_dsm(&open(read("health_read")?).await?, args, scope_prefix).await diff --git a/crates/tracedecay-mcp/src/handlers/health/runtime.rs b/crates/tracedecay-mcp/src/handlers/health/runtime.rs index fad2be688f..226f47ddc2 100644 --- a/crates/tracedecay-mcp/src/handlers/health/runtime.rs +++ b/crates/tracedecay-mcp/src/handlers/health/runtime.rs @@ -5,6 +5,7 @@ use std::time::Duration; use serde_json::{Value, json}; use tracedecay_domain::errors::{Result, TraceDecayError}; use tracedecay_global_db::RegisteredGlobalDb; +use tracedecay_session_temporal_store::SessionTemporalAccess; use crate::{McpDoctorReportV1, McpToolContext, ToolResult, generic_tool_result}; @@ -18,7 +19,7 @@ async fn session_temporal_health_value( match project_session_db { Some(db) => match tokio::time::timeout( SESSION_TEMPORAL_HEALTH_BUDGET, - db.session_temporal_doctor_health(), + SessionTemporalAccess::new(db).session_temporal_doctor_health(), ) .await { diff --git a/crates/tracedecay-mcp/src/handlers/hook_runtime/admission.rs b/crates/tracedecay-mcp/src/handlers/hook_runtime/admission.rs index 20b573b27a..b2ba5c2097 100644 --- a/crates/tracedecay-mcp/src/handlers/hook_runtime/admission.rs +++ b/crates/tracedecay-mcp/src/handlers/hook_runtime/admission.rs @@ -74,7 +74,7 @@ pub(super) fn hook_v2_catchup_response(action: &str) -> Value { /// No migrated database participates. pub fn hook_v2_admission_ledger_root( data_root: &Path, - host: tracedecay_hooks::HookHostV1, + host: tracedecay_domain::NativeHostIdentityV1, ) -> std::path::PathBuf { data_root.join("hook-v2-admissions").join(host.hook_key()) } @@ -95,7 +95,7 @@ fn hook_v2_admission_ledgers() -> &'static StdMutex { fn hook_v2_pending_work_root( data_root: &Path, - host: tracedecay_hooks::HookHostV1, + host: tracedecay_domain::NativeHostIdentityV1, ) -> std::path::PathBuf { data_root.join("hook-v2-pending-work").join(host.hook_key()) } @@ -179,7 +179,7 @@ fn retain_hook_v2_pending_work( #[hotpath::measure(label = "mcp.hook_runtime.pending_work")] pub fn hook_v2_pending_work_envelopes( data_root: &Path, - host: tracedecay_hooks::HookHostV1, + host: tracedecay_domain::NativeHostIdentityV1, now: UtcMicros, ) -> Vec { let Some(_gate) = hook_v2_pending_work_gate().lock().ok() else { @@ -235,7 +235,10 @@ pub fn record_hook_v2_admission( } #[cfg(test)] -fn forget_hook_v2_admission_ledger_for_test(data_root: &Path, host: tracedecay_hooks::HookHostV1) { +fn forget_hook_v2_admission_ledger_for_test( + data_root: &Path, + host: tracedecay_domain::NativeHostIdentityV1, +) { hook_v2_admission_ledgers() .lock() .unwrap() @@ -347,9 +350,9 @@ fn ready_guidance_from_retained_claim( fn cursor_stack_wakeup_allowed( first_admission: bool, - producer: tracedecay_hooks::HookHostV1, + producer: tracedecay_domain::NativeHostIdentityV1, ) -> bool { - first_admission && producer == tracedecay_hooks::HookHostV1::CursorDesktop + first_admission && producer == tracedecay_domain::NativeHostIdentityV1::CursorDesktop } /// The project-sessions authority a hook admission may bind a native @@ -604,7 +607,7 @@ async fn admit_hook_v2_envelope_with_lifecycle_inner( let lifecycle = hook_v2_context_scout_lifecycle_for_session(envelope, native_session_id).await; let claim_authority = if host_response_available { match ( - tracedecay_agent_hosts::agents::context_scout::ports::AdmittedContextScoutHookV1::new( + tracedecay_agent_hosts::agents::context_scout::address_registry::AdmittedContextScoutHookV1::new( envelope.clone(), &snapshot.binding, ), @@ -864,7 +867,7 @@ pub(super) fn hook_v2_profile_admit( fn profile_hook_v2_binding( profile_identity: &dyn tracedecay_contracts::ProfileIdentityReadPort, - host: tracedecay_hooks::HookHostV1, + host: tracedecay_domain::NativeHostIdentityV1, ) -> tracedecay_hooks::HookScopeBindingV1 { let profile_key = format!( "{}:{}", diff --git a/crates/tracedecay-mcp/src/handlers/hook_runtime/admission/tests.rs b/crates/tracedecay-mcp/src/handlers/hook_runtime/admission/tests.rs index 8bebee8510..d1ec873821 100644 --- a/crates/tracedecay-mcp/src/handlers/hook_runtime/admission/tests.rs +++ b/crates/tracedecay-mcp/src/handlers/hook_runtime/admission/tests.rs @@ -39,9 +39,12 @@ fn daemon_admission_is_idempotent_per_identity_and_conflicts_on_different_bytes( ); assert_eq!(second.order, first.order + 1); assert!( - hook_v2_admission_ledger_root(data_root.path(), tracedecay_hooks::HookHostV1::ClaudeCode) - .join("admissions.v1.bin") - .is_file() + hook_v2_admission_ledger_root( + data_root.path(), + tracedecay_domain::NativeHostIdentityV1::ClaudeCode + ) + .join("admissions.v1.bin") + .is_file() ); } @@ -60,15 +63,20 @@ fn completion_persists_before_pending_ack_failure_and_cleanup_retries() { assert_eq!( hook_v2_pending_work_envelopes( data_root.path(), - tracedecay_hooks::HookHostV1::ClaudeCode, + tracedecay_domain::NativeHostIdentityV1::ClaudeCode, now, ), std::slice::from_ref(&envelope) ); let pending_sequence = { let (mut spool, _) = tracedecay_hooks::HookSpoolV1::open( - hook_v2_pending_work_root(data_root.path(), tracedecay_hooks::HookHostV1::ClaudeCode), - tracedecay_hooks::HookSpoolConfigV1::stock(tracedecay_hooks::HookHostV1::ClaudeCode), + hook_v2_pending_work_root( + data_root.path(), + tracedecay_domain::NativeHostIdentityV1::ClaudeCode, + ), + tracedecay_hooks::HookSpoolConfigV1::stock( + tracedecay_domain::NativeHostIdentityV1::ClaudeCode, + ), now, ) .unwrap(); @@ -95,7 +103,7 @@ fn completion_persists_before_pending_ack_failure_and_cleanup_retries() { assert_eq!( hook_v2_pending_work_envelopes( data_root.path(), - tracedecay_hooks::HookHostV1::ClaudeCode, + tracedecay_domain::NativeHostIdentityV1::ClaudeCode, now, ), std::slice::from_ref(&envelope) @@ -110,7 +118,7 @@ fn completion_persists_before_pending_ack_failure_and_cleanup_retries() { assert!( hook_v2_pending_work_envelopes( data_root.path(), - tracedecay_hooks::HookHostV1::ClaudeCode, + tracedecay_domain::NativeHostIdentityV1::ClaudeCode, now, ) .is_empty() @@ -122,7 +130,7 @@ fn completion_persists_before_pending_ack_failure_and_cleanup_retries() { ); forget_hook_v2_admission_ledger_for_test( data_root.path(), - tracedecay_hooks::HookHostV1::ClaudeCode, + tracedecay_domain::NativeHostIdentityV1::ClaudeCode, ); assert!( record_hook_v2_admission(data_root.path(), &envelope, now) @@ -145,7 +153,7 @@ fn completed_restart_duplicate_cleans_pending_without_work_redrive() { { let key = ( data_root.path().to_path_buf(), - tracedecay_hooks::HookHostV1::ClaudeCode.hook_key(), + tracedecay_domain::NativeHostIdentityV1::ClaudeCode.hook_key(), ); let mut ledgers = hook_v2_admission_ledgers().lock().unwrap(); assert!( @@ -159,7 +167,7 @@ fn completed_restart_duplicate_cleans_pending_without_work_redrive() { drop(completion); forget_hook_v2_admission_ledger_for_test( data_root.path(), - tracedecay_hooks::HookHostV1::ClaudeCode, + tracedecay_domain::NativeHostIdentityV1::ClaudeCode, ); let duplicate = record_hook_v2_admission(data_root.path(), &envelope, now).unwrap(); @@ -176,7 +184,7 @@ fn completed_restart_duplicate_cleans_pending_without_work_redrive() { assert!( hook_v2_pending_work_envelopes( data_root.path(), - tracedecay_hooks::HookHostV1::ClaudeCode, + tracedecay_domain::NativeHostIdentityV1::ClaudeCode, now, ) .is_empty(), @@ -234,16 +242,16 @@ fn hook_v2_missing_configuration_remains_transiently_unavailable() { fn github_stack_wakeup_is_cursor_desktop_only_and_first_admission_only() { assert!(cursor_stack_wakeup_allowed( true, - tracedecay_hooks::HookHostV1::CursorDesktop, + tracedecay_domain::NativeHostIdentityV1::CursorDesktop, )); assert!(!cursor_stack_wakeup_allowed( false, - tracedecay_hooks::HookHostV1::CursorDesktop, + tracedecay_domain::NativeHostIdentityV1::CursorDesktop, )); for host in [ - tracedecay_hooks::HookHostV1::Codex, - tracedecay_hooks::HookHostV1::ClaudeCode, - tracedecay_hooks::HookHostV1::CursorCloud, + tracedecay_domain::NativeHostIdentityV1::Codex, + tracedecay_domain::NativeHostIdentityV1::ClaudeCode, + tracedecay_domain::NativeHostIdentityV1::CursorCloud, ] { assert!(!cursor_stack_wakeup_allowed(true, host), "{host:?}"); } @@ -263,7 +271,7 @@ fn profile_scoped_native_admission_is_idempotent_in_the_authenticated_profile() let identity = tracedecay_daemon_identity::profile_identity::load_or_create(&profile_root).unwrap(); let decoded = tracedecay_hooks::decode_native_hook_event( - tracedecay_hooks::HookHostV1::ClaudeCode, + tracedecay_domain::NativeHostIdentityV1::ClaudeCode, br#"{"hook_event_name":"SessionStart"}"#, ) .unwrap(); diff --git a/crates/tracedecay-mcp/src/handlers/hook_runtime/context_scout.rs b/crates/tracedecay-mcp/src/handlers/hook_runtime/context_scout.rs index 7f1fd3ce93..3a091a8ca9 100644 --- a/crates/tracedecay-mcp/src/handlers/hook_runtime/context_scout.rs +++ b/crates/tracedecay-mcp/src/handlers/hook_runtime/context_scout.rs @@ -1,6 +1,9 @@ use serde_json::{Value, json}; use std::collections::BTreeMap; use std::sync::{Mutex as StdMutex, OnceLock}; +use tracedecay_agent_hosts::agents::context_scout::address_registry::{ + AdmittedContextScoutHookV1, ContextScoutLifecycleAddressV1, +}; use tracedecay_agent_hosts::agents::context_scout::{ ContextScoutControlV1, ContextScoutDurableStoreOutcomeV1, ContextScoutErrorV1, }; @@ -36,7 +39,7 @@ use super::required_value; async fn hook_v2_context_scout_lifecycle( args: &Value, envelope: &tracedecay_hooks::HookEventEnvelopeV2, -) -> Option { +) -> Option { hook_v2_context_scout_lifecycle_for_session(envelope, hook_v2_native_session_id(args, envelope)) .await } @@ -44,7 +47,7 @@ async fn hook_v2_context_scout_lifecycle( pub(super) async fn hook_v2_context_scout_lifecycle_for_session( envelope: &tracedecay_hooks::HookEventEnvelopeV2, session_id: Option, -) -> Option { +) -> Option { let session_id = session_id?; tracedecay_daemon_service::context_scout_lifecycle::lookup_registered_context_scout_lifecycle( envelope.project_id, @@ -481,12 +484,7 @@ pub(super) async fn hook_v2_scout_read( let Some(lifecycle) = hook_v2_context_scout_lifecycle(args, &envelope).await else { return Ok(json!({ "action": action, "status": "unavailable" })); }; - let Some(hook) = - tracedecay_agent_hosts::agents::context_scout::ports::AdmittedContextScoutHookV1::new( - envelope, - &snapshot.binding, - ) - else { + let Some(hook) = AdmittedContextScoutHookV1::new(envelope, &snapshot.binding) else { return Ok(json!({ "action": action, "status": "unavailable" })); }; let Some((address, _)) = cg diff --git a/crates/tracedecay-mcp/src/handlers/hook_runtime/ingest.rs b/crates/tracedecay-mcp/src/handlers/hook_runtime/ingest.rs index 1d3500711f..09f9d06786 100644 --- a/crates/tracedecay-mcp/src/handlers/hook_runtime/ingest.rs +++ b/crates/tracedecay-mcp/src/handlers/hook_runtime/ingest.rs @@ -114,7 +114,7 @@ fn project_observation_id(cg: &TraceDecay) -> Result { /// silently consuming the cap and reporting the pass as complete. async fn admit_codex_project_rollouts( admission: &HostAdmissionFacade<'_>, - source: &tracedecay_sessions::runtime::codex::CodexSource, + source: &tracedecay_sessions::runtime::hosts::codex::CodexSource, project_root: &Path, project_id: ProjectId, max_new_bytes: Option, @@ -128,7 +128,7 @@ async fn admit_codex_project_rollouts( let mut paths = source.transcript_paths(project_root).into_iter().peekable(); while let Some(path) = paths.next() { let progress = - tracedecay_sessions::runtime::codex::try_admit_codex_jsonl_observations_for_project_with_admission_and_cancellation( + tracedecay_sessions::runtime::hosts::codex::try_admit_codex_jsonl_observations_for_project_with_admission_and_cancellation( &path, project_root, project_id.clone(), @@ -180,7 +180,7 @@ async fn drain_host_observation_projections( scope: &ObservationScopeV1, cancellation: &ObservationCancellation, ) -> Result { - let stats = tracedecay_sessions::runtime::claude_observation::drain_projection_queue( + let stats = tracedecay_sessions::runtime::hosts::claude_observation::drain_projection_queue( admission, scope, cancellation, @@ -334,7 +334,7 @@ async fn admit_codex_rollouts_once( ) { return Err(rejection); } - let source = tracedecay_sessions::runtime::codex::CodexSource::new() + let source = tracedecay_sessions::runtime::hosts::codex::CodexSource::new() .ok_or_else(|| config_error("Codex transcript source is unavailable"))?; let project_id = project_observation_id(cg)?; let scope = ObservationScopeV1::Project { @@ -749,7 +749,7 @@ pub async fn ingest_transcript_with_cancellation( cg.project_root() )), cg.project_root(), - tracedecay_project::project::current_timestamp() + tracedecay_runtime_core::tracedecay::current_timestamp() ), label = "mcp.hook_runtime.hint_settle" ) diff --git a/crates/tracedecay-mcp/src/handlers/hook_runtime/ingest/kernels.rs b/crates/tracedecay-mcp/src/handlers/hook_runtime/ingest/kernels.rs index 245e6f64ba..d99ef04e7b 100644 --- a/crates/tracedecay-mcp/src/handlers/hook_runtime/ingest/kernels.rs +++ b/crates/tracedecay-mcp/src/handlers/hook_runtime/ingest/kernels.rs @@ -31,8 +31,8 @@ use tracedecay_session_memory::session::lcm::{ }; use tracedecay_sessions::admission::{HostAdmissionOutcome, HostAdmissionStatus}; use tracedecay_sessions::observation::ObservationCancellation; -use tracedecay_sessions::runtime::claude_observation::ClaudeObservationIngestStats; -use tracedecay_sessions::runtime::hermes::HermesSweepOutcome; +use tracedecay_sessions::runtime::hosts::claude_observation::ClaudeObservationIngestStats; +use tracedecay_sessions::runtime::hosts::hermes::HermesSweepOutcome; use tracedecay_sessions::runtime::snapshot_observation::SnapshotCaptureOutcome; use super::super::{required_str, required_user_db}; @@ -278,13 +278,13 @@ async fn capture_claude_profile( required_user_db(&ctx.session_authorities)?; let roots = registered_project_roots(global_db).await?; let stats = - tracedecay_sessions::runtime::claude_observation::ingest_user_sessions_with_admission( + tracedecay_sessions::runtime::hosts::claude_observation::ingest_user_sessions_with_admission( profile_root, Some(session_id), roots, ctx.facade, Some(ctx.max_new_bytes.unwrap_or( - tracedecay_sessions::runtime::claude_observation::CLAUDE_HOOK_MAX_NEW_BYTES, + tracedecay_sessions::runtime::hosts::claude_observation::CLAUDE_HOOK_MAX_NEW_BYTES, )), ctx.cancellation.clone(), ) @@ -311,8 +311,9 @@ async fn capture_codex_profile( roots, ctx.facade, Some( - ctx.max_new_bytes - .unwrap_or(tracedecay_sessions::runtime::codex::CODEX_HOOK_MAX_NEW_BYTES), + ctx.max_new_bytes.unwrap_or( + tracedecay_sessions::runtime::hosts::codex::CODEX_HOOK_MAX_NEW_BYTES, + ), ), ) .await @@ -332,7 +333,7 @@ async fn capture_cursor_profile( let event_json = required_str(ctx.args, "event_json")?; let roots = registered_project_roots(global_db).await?; let stats = - tracedecay_sessions::runtime::cursor::try_ingest_cursor_user_transcript_event_capped_with_admission( + tracedecay_sessions::runtime::hosts::cursor::try_ingest_cursor_user_transcript_event_capped_with_admission( event_json, ctx.facade, ctx.max_new_bytes, @@ -349,14 +350,15 @@ async fn capture_hermes_profile( ctx.profile_root()?; let global_db = ctx.global_db()?; let roots = registered_project_roots(global_db).await?; - let outcome = tracedecay_sessions::runtime::hermes::ingest_user_sessions_capped_with_admission( - ctx.facade, - &roots, - ctx.max_new_bytes, - ctx.cancellation, - ) - .await - .ok_or_else(|| config_error("Hermes transcript source is unavailable"))?; + let outcome = + tracedecay_sessions::runtime::hosts::hermes::ingest_user_sessions_capped_with_admission( + ctx.facade, + &roots, + ctx.max_new_bytes, + ctx.cancellation, + ) + .await + .ok_or_else(|| config_error("Hermes transcript source is unavailable"))?; hermes_capture_outcome(&outcome) } @@ -365,11 +367,11 @@ async fn capture_kiro_profile( ) -> Result { let profile_root = ctx.profile_root()?; let global_db = ctx.global_db()?; - let source = tracedecay_sessions::runtime::kiro::KiroSource::new() + let source = tracedecay_sessions::runtime::hosts::kiro::KiroSource::new() .ok_or_else(|| config_error("Kiro transcript source is unavailable"))?; let roots = registered_project_roots(global_db).await?; let source = source.for_user_scope(roots); - let capture = tracedecay_sessions::runtime::kiro::capture_kiro_snapshot_observations( + let capture = tracedecay_sessions::runtime::hosts::kiro::capture_kiro_snapshot_observations( ctx.facade, &source, profile_root, @@ -396,7 +398,7 @@ async fn capture_hermes_project( ctx: TranscriptCaptureContext<'_>, ) -> Result { let project = ctx.project()?; - let outcome = tracedecay_sessions::runtime::hermes::ingest_for_project_capped_with_admission_and_cancellation( + let outcome = tracedecay_sessions::runtime::hosts::hermes::ingest_for_project_capped_with_admission_and_cancellation( project.project_root(), project_observation_id(project)?, ctx.facade, @@ -412,7 +414,7 @@ async fn capture_codex_project( ctx: TranscriptCaptureContext<'_>, ) -> Result { let cg = ctx.project()?; - let source = tracedecay_sessions::runtime::codex::CodexSource::new() + let source = tracedecay_sessions::runtime::hosts::codex::CodexSource::new() .ok_or_else(|| config_error("Codex transcript source is unavailable"))?; let project_id = project_observation_id(cg)?; let scope = ObservationScopeV1::Project { @@ -444,7 +446,7 @@ async fn capture_cursor_project( ) -> Result { let cg = ctx.project()?; let event_json = required_str(ctx.args, "event_json")?; - let stats = tracedecay_sessions::runtime::cursor::try_ingest_cursor_transcript_event_capped_with_admission( + let stats = tracedecay_sessions::runtime::hosts::cursor::try_ingest_cursor_transcript_event_capped_with_admission( event_json, project_observation_id(cg)?, ctx.facade, @@ -456,7 +458,7 @@ async fn capture_cursor_project( } fn cursor_capture_outcome( - stats: tracedecay_sessions::runtime::cursor::CursorTranscriptIngestStats, + stats: tracedecay_sessions::runtime::hosts::cursor::CursorTranscriptIngestStats, ) -> TranscriptCaptureOutcome { TranscriptCaptureOutcome { messages_upserted: stats.messages_upserted, @@ -575,13 +577,13 @@ async fn capture_kiro_project( ctx: TranscriptCaptureContext<'_>, ) -> Result { let cg = ctx.project()?; - let source = tracedecay_sessions::runtime::kiro::KiroSource::new() + let source = tracedecay_sessions::runtime::hosts::kiro::KiroSource::new() .ok_or_else(|| config_error("Kiro transcript source is unavailable"))?; let project_id = project_observation_id(cg)?; let scope = ObservationScopeV1::Project { project_id: project_id.clone(), }; - let capture = tracedecay_sessions::runtime::kiro::capture_kiro_snapshot_observations( + let capture = tracedecay_sessions::runtime::hosts::kiro::capture_kiro_snapshot_observations( ctx.facade, &source, cg.project_root(), @@ -607,7 +609,7 @@ mod tests { #[test] fn cursor_capture_preserves_deferred_projection() { let outcome = cursor_capture_outcome( - tracedecay_sessions::runtime::cursor::CursorTranscriptIngestStats { + tracedecay_sessions::runtime::hosts::cursor::CursorTranscriptIngestStats { messages_upserted: 3, source_deferred: true, ..Default::default() diff --git a/crates/tracedecay-mcp/src/handlers/hook_runtime/test_support.rs b/crates/tracedecay-mcp/src/handlers/hook_runtime/test_support.rs index 43a43c5fdf..c366ae7a26 100644 --- a/crates/tracedecay-mcp/src/handlers/hook_runtime/test_support.rs +++ b/crates/tracedecay-mcp/src/handlers/hook_runtime/test_support.rs @@ -25,7 +25,7 @@ pub(super) fn admission_test_envelope( tracedecay_hooks::HookEventEnvelopeV2 { schema_version: tracedecay_hooks::HOOK_EVENT_SCHEMA_VERSION, event_id: [event_id; 16], - producer: tracedecay_hooks::HookHostV1::ClaudeCode, + producer: tracedecay_domain::NativeHostIdentityV1::ClaudeCode, protected_session_id: [5; 32], project_id: [1; 16], repository_id: [2; 16], @@ -41,7 +41,7 @@ pub(super) fn admission_test_envelope( } pub(super) fn admission_test_binding(epoch: u64) -> tracedecay_hooks::HookScopeBindingV1 { - let host = tracedecay_hooks::HookHostV1::ClaudeCode; + let host = tracedecay_domain::NativeHostIdentityV1::ClaudeCode; tracedecay_hooks::HookScopeBindingV1 { host, project_id: [1; 16], @@ -202,7 +202,7 @@ pub(super) fn hook_v2_snapshot() -> tracedecay_hooks::HookConfigurationSnapshotV published_at: UtcMicros(1), expires_at: UtcMicros(100), binding: tracedecay_hooks::HookScopeBindingV1 { - host: tracedecay_hooks::HookHostV1::ClaudeCode, + host: tracedecay_domain::NativeHostIdentityV1::ClaudeCode, project_id: [1; 16], repository_id: [2; 16], worktree_id: [3; 16], @@ -220,7 +220,7 @@ pub(super) fn hook_v2_envelope_for_test() -> tracedecay_hooks::HookEventEnvelope tracedecay_hooks::HookEventEnvelopeV2 { schema_version: tracedecay_hooks::HOOK_EVENT_SCHEMA_VERSION, event_id: [6; 16], - producer: tracedecay_hooks::HookHostV1::ClaudeCode, + producer: tracedecay_domain::NativeHostIdentityV1::ClaudeCode, protected_session_id: [7; 32], project_id: [1; 16], repository_id: [2; 16], diff --git a/crates/tracedecay-mcp/src/handlers/info/body.rs b/crates/tracedecay-mcp/src/handlers/info/body.rs deleted file mode 100644 index d382ec63eb..0000000000 --- a/crates/tracedecay-mcp/src/handlers/info/body.rs +++ /dev/null @@ -1,251 +0,0 @@ -//! `tracedecay_body`, source bodies for symbols matched by name. - -use crate::ToolResult; -use crate::rendered_tool_result; -use crate::tools::render::{self, Md}; -use serde_json::{Value, json}; -use tracedecay_code_index::graph_projection::CodeGraphSymbolSummaryV1; -use tracedecay_domain::code_intelligence::NodeKind; -use tracedecay_domain::errors::{Result, TraceDecayError}; -use tracedecay_graph_query::VerifiedGraphQuery; -use tracedecay_graph_query::context::read_modes::estimate_tokens; - -use super::verified::{ - end_line, info_graph_error, required_file_path, required_metadata, required_symbol_parts, -}; - -/// Extract the source spanning tree-sitter rows `start_line..=end_line` -/// (0-based, inclusive) from `source`. Node line fields are stored as the -/// raw tree-sitter row index, so the caller passes them through unchanged. -/// Returns the empty string if the range is out of bounds. -pub fn extract_lines(source: &str, start_line: u32, end_line: u32) -> String { - let start = start_line as usize; - let end_exclusive = (end_line as usize).saturating_add(1); - if start >= end_exclusive { - return String::new(); - } - let mut selected = source.lines().skip(start).take(end_exclusive - start); - let Some(first) = selected.next() else { - return String::new(); - }; - let mut body = String::with_capacity(first.len()); - body.push_str(first); - for line in selected { - body.push('\n'); - body.push_str(line); - } - body -} - -#[hotpath::measure(label = "mcp.info.body.total")] -pub async fn handle_body( - graph: &VerifiedGraphQuery, - args: Value, - scope_prefix: Option<&str>, -) -> Result { - let symbol = - args.get("symbol") - .and_then(|v| v.as_str()) - .ok_or_else(|| TraceDecayError::Config { - message: "missing required parameter: symbol".to_string(), - })?; - - let limit = args - .get("limit") - .and_then(Value::as_u64) - .map_or(3, |v| v.clamp(1, 20) as usize); - if args - .get("lazy_index_ignored_dependencies") - .and_then(Value::as_bool) - .unwrap_or(false) - { - return Err(info_graph_error( - "verified-body-lazy-indexing-unavailable", - "lazy dependency indexing cannot mutate the generation pinned for this body request", - )); - } - - let chosen = hotpath::measure_block!( - "mcp.info.body.candidates", - body_candidates(graph, symbol, limit, scope_prefix)? - ); - - if chosen.is_empty() { - return Ok(ToolResult::new( - json!({ - "content": [{ "type": "text", "text": format!("No symbol named '{symbol}' found.") }] - }), - vec![], - )); - } - - let project_root = graph.project_root()?; - let (output, touched) = hotpath::measure_block!("mcp.info.body.source", { - let mut matches: Vec = Vec::new(); - let mut touched: Vec = Vec::new(); - - for result in &chosen { - let (metadata, file_path) = required_symbol_parts(result)?; - let body = source_body_for_node( - graph, - file_path, - metadata.start_line, - end_line(metadata)?, - &mut touched, - )?; - matches.push(json!({ - "id": result.occurrence.as_str(), - "name": metadata.simple_name, - "qualified_name": metadata.qualified_name, - "kind": metadata.kind, - "file": file_path, - "start_line": metadata.start_line.saturating_add(1), - "end_line": end_line(metadata)?.saturating_add(1), - "signature": metadata.signature, - "body": body, - })); - } - - ( - json!({ - "match_count": matches.len(), - "matches": matches, - }), - touched, - ) - }); - Ok(rendered_tool_result( - Some(project_root), - &args, - &output, - touched, - || render_body_md(&output), - )) -} - -/// Renders `tracedecay_body` matches like `render_read_md` rather than dumping -/// source into a table cell with newlines collapsed: each match gets a heading, -/// a location line, an optional signature, a token count, and a fenced code -/// block tagged with the file's language extension. -fn render_body_md(value: &Value) -> String { - let mut md = Md::new(); - let matches = value.get("matches").and_then(Value::as_array); - let count = matches.map_or(0, std::vec::Vec::len); - md.heading(2, &format!("Body matches ({count})")); - - let Some(matches) = matches else { - return md.render(); - }; - for m in matches { - let name = render::field_str(m, "name"); - let kind = render::field_str(m, "kind"); - let file = render::field_str(m, "file"); - let start = render::field_i64(m, "start_line"); - let end = render::field_i64(m, "end_line"); - let signature = render::field_str(m, "signature"); - let body = render::field_str(m, "body"); - - md.blank(); - md.heading(3, &format!("{name} ({kind})")); - md.field("location", &format!("{file}:{start}-{end}")); - if !signature.is_empty() { - md.field("signature", signature); - } - md.field("tokens", &estimate_tokens(body).to_string()); - md.blank(); - let lang = file.rsplit_once('.').map_or("", |(_, ext)| ext); - md.code(lang, body); - } - md.render() -} - -fn body_candidates( - graph: &VerifiedGraphQuery, - symbol: &str, - limit: usize, - scope_prefix: Option<&str>, -) -> Result> { - let mut candidates = graph.resolve_qualified_name(symbol, None, 1_000)?; - if candidates.is_empty() { - candidates = graph.resolve_simple_name(symbol, None, 1_000)?; - } - let mut scoped = Vec::new(); - for candidate in candidates { - let path = required_file_path(&candidate)?; - let metadata = required_metadata(&candidate)?; - if scope_prefix.is_none_or(|scope| tracedecay_domain::path_matches_scope(path, Some(scope))) - { - let preference = NodeKind::from_str(&metadata.kind) - .map_or(u8::MAX, |kind| body_kind_preference(&kind)); - scoped.push((preference, candidate)); - } - } - scoped.sort_by_key(|(preference, _)| *preference); - let mut candidates = scoped - .into_iter() - .map(|(_, candidate)| candidate) - .collect::>(); - candidates.truncate(limit); - Ok(candidates) -} - -fn source_body_for_node( - graph: &VerifiedGraphQuery, - file_path: &str, - start_line: u32, - end_line: u32, - touched: &mut Vec, -) -> Result { - match graph.read_indexed_source_file(file_path) { - Ok(source) => { - if !touched.iter().any(|path| path == file_path) { - touched.push(file_path.to_string()); - } - Ok(extract_lines(&source, start_line, end_line)) - } - Err(error) => Err(TraceDecayError::Config { - message: format!("cannot read indexed source body '{file_path}': {error}"), - }), - } -} - -/// Ordering key used by `handle_body` to choose between same-named symbols. -/// Lower number = higher preference (sorted ascending). Callable kinds rank -/// best because the user almost always asks for "show me the body of X" -/// expecting a function or method; type definitions are next; fields, -/// variants, use statements come last. -fn body_kind_preference(kind: &NodeKind) -> u8 { - match kind { - NodeKind::Function - | NodeKind::Method - | NodeKind::StructMethod - | NodeKind::Constructor - | NodeKind::AbstractMethod - | NodeKind::ArrowFunction - | NodeKind::Procedure => 0, - NodeKind::Struct - | NodeKind::Enum - | NodeKind::Trait - | NodeKind::Class - | NodeKind::InnerClass - | NodeKind::Interface - | NodeKind::InterfaceType - | NodeKind::Record - | NodeKind::CaseClass - | NodeKind::DataClass - | NodeKind::SealedClass - | NodeKind::TypeAlias - | NodeKind::Union - | NodeKind::Typedef => 1, - NodeKind::Impl => 2, - NodeKind::Const | NodeKind::Static | NodeKind::Macro | NodeKind::PreprocessorDef => 3, - NodeKind::Field - | NodeKind::ValField - | NodeKind::VarField - | NodeKind::Property - | NodeKind::CSharpProperty - | NodeKind::EnumVariant => 4, - NodeKind::Use | NodeKind::Include => 5, - _ => 6, - } -} diff --git a/crates/tracedecay-mcp/src/handlers/info/dispatch.rs b/crates/tracedecay-mcp/src/handlers/info/dispatch.rs index 0f18d58970..deb3d7e41a 100644 --- a/crates/tracedecay-mcp/src/handlers/info/dispatch.rs +++ b/crates/tracedecay-mcp/src/handlers/info/dispatch.rs @@ -9,16 +9,13 @@ use serde_json::Value; use tracedecay_contracts::retrieval::{CallableCodeOperationKind, callable_code_operation}; use tracedecay_domain::errors::{Result, TraceDecayError}; -use super::{ - handle_body, handle_config, handle_files, handle_outline, handle_port_order, - handle_port_status, handle_read, handle_signature_search, handle_todos, handle_type_hierarchy, -}; +use super::{handle_config, handle_files}; use crate::ToolResult; use crate::handlers::support::unknown_tool_error; -use crate::handlers::verified_read::{VerifiedGraphOpen, verified_read_operation as read}; +use crate::handlers::verified_read::VerifiedGraphOpen; -/// Dispatches one graph-backed info tool (`tracedecay_read`, -/// `tracedecay_files`, ...) onto its handler, opening the verified graph +/// Dispatches one graph-backed info tool (`tracedecay_files`, +/// `tracedecay_todos`, ...) onto its handler, opening the verified graph /// through `open` under the operation the catalog registers for it. pub async fn dispatch_tool( project_root: &Path, @@ -35,35 +32,7 @@ pub async fn dispatch_tool( })?; handle_files(&open(operation).await?, args, scope_prefix).await } - "tracedecay_port_status" => { - handle_port_status(&open(read("port_status")?).await?, args).await - } - "tracedecay_port_order" => handle_port_order(&open(read("port_order")?).await?, args).await, - "tracedecay_type_hierarchy" => { - handle_type_hierarchy(&open(read("code_type_hierarchy")?).await?, args).await - } - "tracedecay_body" => { - handle_body(&open(read("source_body")?).await?, args, scope_prefix).await - } - "tracedecay_todos" => handle_todos(&open(read("todos")?).await?, args, scope_prefix).await, - "tracedecay_read" => { - let operation = match args.get("mode").and_then(Value::as_str).unwrap_or("full") { - "map" => "source_outline", - "signatures" => "code_signature_search", - _ => "source_lines", - }; - handle_read(&open(read(operation)?).await?, args).await - } - "tracedecay_outline" => handle_outline(&open(read("source_outline")?).await?, args).await, "tracedecay_config" => handle_config(project_root, &args).await, - "tracedecay_signature_search" => { - handle_signature_search( - &open(read("code_signature_search")?).await?, - args, - scope_prefix, - ) - .await - } _ => Err(unknown_tool_error(tool_name)), } } diff --git a/crates/tracedecay-mcp/src/handlers/info/mod.rs b/crates/tracedecay-mcp/src/handlers/info/mod.rs index 10492aaada..9f646226b4 100644 --- a/crates/tracedecay-mcp/src/handlers/info/mod.rs +++ b/crates/tracedecay-mcp/src/handlers/info/mod.rs @@ -1,92 +1,28 @@ //! Portable project-info and file-inspection tool handlers. //! -//! Each tool owns a sibling module; this module holds the shared markdown -//! enrichment helpers used by more than one sibling, and the re-exports the -//! handler dispatcher calls. +//! Each tool owns a sibling module; this module holds the constants shared by +//! more than one sibling, and the re-exports the handler dispatcher calls. -mod body; mod config; mod dispatch; mod files; -mod outline; mod port_order; mod port_status; -mod read; mod registry; mod remote_status; -mod signature_search; mod status; mod todos; -mod type_hierarchy; mod verified; -pub use body::{extract_lines, handle_body}; pub use config::handle_config; pub use dispatch::dispatch_tool; pub use files::handle_files; -pub use outline::handle_outline; -pub use port_order::handle_port_order; -pub use port_status::handle_port_status; -pub use read::handle_read; +pub use port_order::compute_port_order; +pub use port_status::compute_port_status; pub use registry::{handle_project_context, handle_project_list, handle_project_search}; pub use remote_status::handle_remote_status; -pub use signature_search::handle_signature_search; pub use status::{graph_statistics_value, handle_active_project, handle_status}; -pub use todos::handle_todos; -pub use type_hierarchy::handle_type_hierarchy; - -use std::path::Path; - -use crate::tools::render::Md; -use serde_json::Value; -use tracedecay_graph_query::context::markdown_sections::{ - SectionEnrichment, is_markdown_file, section_summary_lines, -}; -use tracedecay_runtime_core::tracedecay::current_timestamp; - -/// Adds the section lane, title, truncated preview, full-body retrieval -/// handle, line span, and parsed section structure, to every markdown section -/// symbol in a `{"symbols": [...]}` container. -/// -/// This is an enrichment of a surface that already answered: a file that cannot -/// be read, or a container with no symbol array, leaves the payload exactly as -/// it was rather than failing the outline or read that carries it. -pub(super) fn enrich_markdown_sections( - project_root: &Path, - absolute_path: &Path, - display_file: &str, - container: &mut Value, -) { - if !is_markdown_file(display_file) { - return; - } - let Some(symbols) = container - .get_mut("symbols") - .and_then(Value::as_array_mut) - .filter(|symbols| !symbols.is_empty()) - else { - return; - }; - let Ok(source) = tracedecay_runtime_core::sync::read_source_file(absolute_path) else { - return; - }; - SectionEnrichment::new(Some(project_root), current_timestamp()) - .enrich_symbol_array(symbols, &source); -} - -/// Emits one symbol's markdown-section lane under its outline/read bullet. -/// -/// The summary lines themselves are composed in -/// `tracedecay-application::context::markdown_sections`; this adapter only owns -/// the markdown builder and the two-space bullet continuation indent. -pub(super) fn render_section_md(md: &mut Md, section: Option<&Value>) { - let Some(section) = section else { - return; - }; - for line in section_summary_lines(section) { - md.line(&format!(" {line}")); - } -} +pub use todos::compute_todos; /// Default node kinds for port comparisons. pub(super) const PORT_DEFAULT_KINDS: &[&str] = &[ diff --git a/crates/tracedecay-mcp/src/handlers/info/outline.rs b/crates/tracedecay-mcp/src/handlers/info/outline.rs deleted file mode 100644 index b48945eea8..0000000000 --- a/crates/tracedecay-mcp/src/handlers/info/outline.rs +++ /dev/null @@ -1,146 +0,0 @@ -//! `tracedecay_outline`, flat symbol map for a file, enriched with ast-grep structure. - -use std::path::Path; - -use crate::ToolResult; -use crate::rendered_tool_result; -use crate::tools::render::{self, Md}; -use serde_json::{Value, json}; -use tracedecay_domain::errors::{Result, TraceDecayError}; -use tracedecay_graph_query::VerifiedGraphQuery; -use tracedecay_mcp_catalog::ast_grep_diagnostics; -use tracedecay_runtime_core::ast_grep::ast_grep_command; - -use super::{enrich_markdown_sections, render_section_md}; - -/// Flat symbol map for a file with optional `kinds` filter. -#[hotpath::measure(label = "mcp.info.outline.total")] -pub async fn handle_outline(graph: &VerifiedGraphQuery, args: Value) -> Result { - let file = - args.get("file") - .and_then(|v| v.as_str()) - .ok_or_else(|| TraceDecayError::Config { - message: "missing required parameter: file".to_string(), - })?; - - let kinds: Option> = args.get("kinds").and_then(|v| v.as_array()).map(|arr| { - arr.iter() - .filter_map(|v| v.as_str().map(str::to_string)) - .collect() - }); - - let project_root = graph.project_root()?; - let (abs_path, display_file) = graph.resolve_indexed_source_file(file)?; - - let kinds_slice: Option<&[String]> = kinds.as_deref(); - let mut value = hotpath::measure_block!( - "mcp.info.outline.map", - graph.render_map(&display_file, kinds_slice)? - ); - enrich_markdown_sections(project_root, &abs_path, &display_file, &mut value); - match hotpath::measure_block!("mcp.info.outline.ast_grep", ast_grep_outline(&abs_path)) { - Ok(outline) => { - value["ast_grep_outline"] = outline; - } - Err(err) => { - value["ast_grep_outline"] = Value::Null; - value["ast_grep_outline_error"] = json!(err.to_string()); - } - } - Ok(rendered_tool_result( - Some(project_root), - &args, - &value, - vec![display_file], - || render_outline_md(&value), - )) -} - -fn ast_grep_outline(abs_path: &Path) -> Result { - ensure_ast_grep_outline_available()?; - - let output = ast_grep_command() - .args([ - "outline", - "--json=compact", - "--items", - "structure", - "--view", - "expanded", - ]) - .arg(abs_path) - .output() - .map_err(|err| TraceDecayError::Config { - message: format!("failed to run ast-grep outline: {err}"), - })?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - let stdout = String::from_utf8_lossy(&output.stdout); - let detail = if !stderr.trim().is_empty() { - stderr.trim() - } else if !stdout.trim().is_empty() { - stdout.trim() - } else { - "no output" - }; - return Err(TraceDecayError::Config { - message: format!("ast-grep outline failed: {detail}"), - }); - } - - serde_json::from_slice::(&output.stdout).map_err(|err| TraceDecayError::Config { - message: format!("failed to parse ast-grep outline JSON: {err}"), - }) -} - -fn ensure_ast_grep_outline_available() -> Result<()> { - let diagnostics = ast_grep_diagnostics(); - if diagnostics.outline_available { - Ok(()) - } else { - Err(TraceDecayError::Config { - message: format!( - "tracedecay_outline requires ast-grep outline >= 0.44: {}", - diagnostics.message - ), - }) - } -} - -fn render_outline_md(value: &Value) -> String { - let mut md = Md::new(); - let file = render::field_str(value, "file"); - let count = render::field_i64(value, "symbol_count"); - md.heading(2, &format!("Outline, {file}")); - md.field("symbols", &count.to_string()); - md.blank(); - match value.get("symbols").and_then(Value::as_array) { - Some(symbols) if !symbols.is_empty() => { - for symbol in symbols { - let name = render::field_str(symbol, "name"); - let kind = render::field_str(symbol, "kind"); - let visibility = render::field_str(symbol, "visibility"); - let line = render::field_i64(symbol, "line"); - let end = render::field_i64(symbol, "end_line"); - let span = if end > line { - format!("{line}-{end}") - } else { - line.to_string() - }; - let signature = render::field_str(symbol, "signature"); - md.bullet(&format!( - "**{name}** ({kind}) - lines {span} - {visibility}" - )); - if !signature.is_empty() { - md.line(&format!(" `{signature}`")); - } - render_section_md(&mut md, symbol.get("section")); - } - } - _ => { - md.empty_note("No symbols."); - } - } - md.render() -} diff --git a/crates/tracedecay-mcp/src/handlers/info/port_order.rs b/crates/tracedecay-mcp/src/handlers/info/port_order.rs index 7c55603bc1..4f9a2e4871 100644 --- a/crates/tracedecay-mcp/src/handlers/info/port_order.rs +++ b/crates/tracedecay-mcp/src/handlers/info/port_order.rs @@ -2,9 +2,10 @@ use std::collections::{HashMap, HashSet}; -use crate::ToolResult; -use crate::{decode_primitive_request, generic_tool_result, unique_file_paths}; +use crate::handlers::graph::graph_tool_completion; +use crate::{decode_primitive_request, unique_file_paths}; use serde_json::Value; +use tracedecay_contracts::graph_tool::{GraphToolCompletionV1, GraphToolResultV1}; use tracedecay_contracts::retrieval::{ PortCycleAnchorV1, PortCycleFileV1, PortCycleSymbolV1, PortCycleV1, PortOrderLevelV1, PortOrderResultV1, PortOrderSurfaceRequestV1, PortOrderSymbolV1, @@ -27,7 +28,10 @@ struct PortOrderSymbol<'a> { } #[hotpath::measure(label = "mcp.info.port_order.total")] -pub async fn handle_port_order(graph: &VerifiedGraphQuery, args: Value) -> Result { +pub async fn compute_port_order( + graph: &VerifiedGraphQuery, + args: Value, +) -> Result { let request: PortOrderSurfaceRequestV1 = decode_primitive_request(&args, "tracedecay_port_order")?; let kind_strs = request.kinds.as_ref().map_or_else( @@ -81,12 +85,9 @@ pub async fn handle_port_order(graph: &VerifiedGraphQuery, args: Value) -> Resul levels: Vec::new(), cycles: Vec::new(), }; - let output = serde_json::to_value(result)?; - return Ok(generic_tool_result( - Some(graph.project_root()?), - &args, - &output, - vec![], + return Ok(graph_tool_completion( + GraphToolResultV1::PortOrder(result), + Vec::new(), )); } @@ -408,12 +409,8 @@ pub async fn handle_port_order(graph: &VerifiedGraphQuery, args: Value) -> Resul levels: result_levels, cycles, }; - let output = serde_json::to_value(result)?; - - Ok(generic_tool_result( - Some(graph.project_root()?), - &args, - &output, + Ok(graph_tool_completion( + GraphToolResultV1::PortOrder(result), touched_files, )) } diff --git a/crates/tracedecay-mcp/src/handlers/info/port_status.rs b/crates/tracedecay-mcp/src/handlers/info/port_status.rs index 3ae440c9c4..27fd1a7a05 100644 --- a/crates/tracedecay-mcp/src/handlers/info/port_status.rs +++ b/crates/tracedecay-mcp/src/handlers/info/port_status.rs @@ -2,9 +2,10 @@ use std::collections::{BTreeMap, HashMap, HashSet}; -use crate::ToolResult; -use crate::{decode_primitive_request, generic_tool_result, unique_file_paths}; +use crate::handlers::graph::graph_tool_completion; +use crate::{decode_primitive_request, unique_file_paths}; use serde_json::Value; +use tracedecay_contracts::graph_tool::{GraphToolCompletionV1, GraphToolResultV1}; use tracedecay_contracts::retrieval::{ PortMatchedSymbolV1, PortStatusResultV1, PortStatusSurfaceRequestV1, PortTargetOnlySymbolV1, PortUnmatchedSymbolV1, @@ -39,7 +40,7 @@ fn kind_compat_group(kind: &str) -> u8 { } } -/// Composite match key used by `handle_port_status`. +/// Composite match key used by `compute_port_status`. /// /// Combines the lowercased name, an optional parent qualifier (for methods, /// fields, and variants), and a kind compatibility group, so siblings whose @@ -85,7 +86,10 @@ fn port_parent_qualifier(kind: &str, qualified_name: &str) -> Option { } #[hotpath::measure(label = "mcp.info.port_status.total")] -pub async fn handle_port_status(graph: &VerifiedGraphQuery, args: Value) -> Result { +pub async fn compute_port_status( + graph: &VerifiedGraphQuery, + args: Value, +) -> Result { let request: PortStatusSurfaceRequestV1 = decode_primitive_request(&args, "tracedecay_port_status")?; let kind_strs = request.kinds.as_ref().map_or_else( @@ -232,12 +236,8 @@ pub async fn handle_port_status(graph: &VerifiedGraphQuery, args: Value) -> Resu matched_symbols, target_only_symbols: target_only, }; - let output = serde_json::to_value(result)?; - - Ok(generic_tool_result( - Some(graph.project_root()?), - &args, - &output, + Ok(graph_tool_completion( + GraphToolResultV1::PortStatus(result), touched_files, )) } diff --git a/crates/tracedecay-mcp/src/handlers/info/read.rs b/crates/tracedecay-mcp/src/handlers/info/read.rs deleted file mode 100644 index 6d5bacab0e..0000000000 --- a/crates/tracedecay-mcp/src/handlers/info/read.rs +++ /dev/null @@ -1,172 +0,0 @@ -//! `tracedecay_read`, mode-aware file read with cross-session cache. - -use crate::ToolResult; -use crate::rendered_tool_result; -use crate::tools::render::{self, Md}; -use serde_json::{Value, json}; -use tracedecay_domain::errors::{Result, TraceDecayError}; -use tracedecay_graph_query::VerifiedGraphQuery; -use tracedecay_graph_query::context::read_modes::{LineRange, ReadMode}; -use tracedecay_graph_query::context::source_read::SourceReadRequest; - -use super::{enrich_markdown_sections, render_section_md}; - -#[hotpath::measure(future = true, label = "mcp.info.read.total")] -pub async fn handle_read(graph: &VerifiedGraphQuery, args: Value) -> Result { - let file = - args.get("file") - .and_then(|v| v.as_str()) - .ok_or_else(|| TraceDecayError::Config { - message: "missing required parameter: file".to_string(), - })?; - - let mode_str = args.get("mode").and_then(|v| v.as_str()).unwrap_or("full"); - let mode = ReadMode::parse(mode_str).ok_or_else(|| TraceDecayError::Config { - message: format!("unknown mode '{mode_str}'; expected one of full, lines, map, signatures"), - })?; - let include_symbols = args - .get("include_symbols") - .and_then(Value::as_bool) - .unwrap_or(mode == ReadMode::Lines); - - let line_range = if mode == ReadMode::Lines { - let raw = - args.get("lines") - .and_then(|v| v.as_str()) - .ok_or_else(|| TraceDecayError::Config { - message: "mode='lines' requires the 'lines' argument (e.g. '120-180')" - .to_string(), - })?; - Some( - LineRange::parse(raw).ok_or_else(|| TraceDecayError::Config { - message: format!("invalid 'lines' value '{raw}'; expected 'A' or 'A-B'"), - })?, - ) - } else { - None - }; - - let project_root = graph.project_root()?.to_path_buf(); - let project_id = graph.project_id()?.to_owned(); - // The source-read future carries the whole read pipeline's state; boxing - // it keeps this handler's own future small. - let output = hotpath::future!( - Box::pin(graph.read_source(SourceReadRequest { - file, - mode, - line_range, - raw_lines: args.get("lines").and_then(Value::as_str), - include_symbols, - project_id: &project_id, - })), - label = "mcp.info.read.source" - ) - .await?; - let display_file = output.file; - let mut payload = json!({ - "file": &display_file, - "mode": output.mode.as_str(), - "mtime_ns": output.mtime_ns, - "digest": output.digest, - "token_count": output.token_count, - }); - if output.unchanged { - payload["unchanged"] = Value::Bool(true); - } - if let Some(body) = output.body { - payload["body"] = Value::String(body); - } - if let Some(mut context) = output.context { - // `display_file` is the repository-relative logical path the read - // resolved to, so this is the same file the symbol context describes. - enrich_markdown_sections( - &project_root, - &project_root.join(&display_file), - &display_file, - &mut context, - ); - payload["context"] = context; - } - Ok(rendered_tool_result( - Some(&project_root), - &args, - &payload, - vec![display_file], - || render_read_md(&payload), - )) -} - -fn render_read_md(value: &Value) -> String { - let mut md = Md::new(); - let file = render::field_str(value, "file"); - let mode = render::field_str(value, "mode"); - md.heading(2, &format!("{file} ({mode})")); - if value - .get("unchanged") - .and_then(Value::as_bool) - .unwrap_or(false) - { - md.field("unchanged", "true"); - let digest = render::field_str(value, "digest"); - if !digest.is_empty() { - md.field("digest", digest); - } - } - md.field( - "tokens", - &render::field_i64(value, "token_count").to_string(), - ); - render_read_context_md(&mut md, value.get("context")); - if value.get("body").is_none() { - return md.render(); - } - md.blank(); - let lang = file.rsplit_once('.').map_or("", |(_, ext)| ext); - md.code(lang, render::field_str(value, "body")); - md.render() -} - -fn render_read_context_md(md: &mut Md, context: Option<&Value>) { - let Some(context) = context else { - return; - }; - let Some(symbols) = context.get("symbols").and_then(Value::as_array) else { - return; - }; - if symbols.is_empty() { - return; - } - - md.blank(); - md.heading(3, "Context"); - let symbol_count = context - .get("symbol_count") - .and_then(Value::as_u64) - .unwrap_or(symbols.len() as u64); - md.field("symbols", &symbol_count.to_string()); - for symbol in symbols { - let kind = render::field_str(symbol, "kind"); - let name = render::field_str(symbol, "name"); - let line = render::field_i64(symbol, "line"); - let end_line = render::field_i64(symbol, "end_line"); - let signature = render::field_str(symbol, "signature"); - let span = if end_line > line { - format!("{line}-{end_line}") - } else { - line.to_string() - }; - if signature.is_empty() { - md.bullet(&format!("{kind} {name} {span}")); - } else { - md.bullet(&format!("{kind} {name} {span}: `{signature}`")); - } - render_section_md(md, symbol.get("section")); - } - if context - .get("truncated") - .and_then(Value::as_bool) - .unwrap_or(false) - { - md.empty_note("symbol list truncated"); - } -} diff --git a/crates/tracedecay-mcp/src/handlers/info/registry.rs b/crates/tracedecay-mcp/src/handlers/info/registry.rs index 60de7f965b..c7f5368a08 100644 --- a/crates/tracedecay-mcp/src/handlers/info/registry.rs +++ b/crates/tracedecay-mcp/src/handlers/info/registry.rs @@ -344,11 +344,18 @@ mod tests { "unexpected refusal: {error}" ); + // Git identity is reserved for a host-absolute path; `/srv/other` is + // drive-relative on Windows. + let other = if cfg!(windows) { + r"C:\srv\other" + } else { + "/srv/other" + }; assert_eq!( - project_context_selector(None, &json!({"path": "/srv/other"})) + project_context_selector(None, &json!({"path": other})) .expect("explicit path selector"), ProjectRegistrySelector::Path { - path: Path::new("/srv/other").to_path_buf(), + path: Path::new(other).to_path_buf(), allow_git_identity: true, } ); diff --git a/crates/tracedecay-mcp/src/handlers/info/signature_search.rs b/crates/tracedecay-mcp/src/handlers/info/signature_search.rs deleted file mode 100644 index 09a275d327..0000000000 --- a/crates/tracedecay-mcp/src/handlers/info/signature_search.rs +++ /dev/null @@ -1,142 +0,0 @@ -//! `tracedecay_signature_search`, substring search over cached function/method signatures. - -use crate::ToolResult; -use crate::generic_tool_result; -use serde_json::{Value, json}; -use tracedecay_domain::code_intelligence::NodeKind; -use tracedecay_domain::errors::{Result, TraceDecayError}; -use tracedecay_graph_query::VerifiedGraphQuery; - -use super::verified::{all_symbols, required_symbol_parts}; - -/// Substring search across the cached `signature` column on every -/// Function/Method node. -#[hotpath::measure(future = true, label = "mcp.info.signature_search.total")] -pub async fn handle_signature_search( - graph: &VerifiedGraphQuery, - args: Value, - scope_prefix: Option<&str>, -) -> Result { - let returns = args.get("returns").and_then(|v| v.as_str()); - let params: Vec = args - .get("params") - .and_then(|v| v.as_array()) - .map(|arr| { - arr.iter() - .filter_map(|v| v.as_str().map(str::to_string)) - .collect() - }) - .unwrap_or_default(); - let want_async = args.get("async").and_then(Value::as_bool); - let path_filter = args.get("path").and_then(|v| v.as_str()).or(scope_prefix); - let limit = args - .get("limit") - .and_then(Value::as_u64) - .map_or(50, |v| v.clamp(1, 500) as usize); - - if returns.is_none() && params.is_empty() && want_async.is_none() { - return Err(TraceDecayError::Config { - message: "missing required parameter: one of 'returns', 'params', or 'async'" - .to_string(), - }); - } - let (payload, touched) = hotpath::measure_block!("mcp.info.signature_search.scan", { - let mut entries: Vec = Vec::new(); - let mut touched: Vec = Vec::new(); - for node in all_symbols(graph)? { - let (metadata, file_path) = required_symbol_parts(&node)?; - if !matches!( - NodeKind::from_str(&metadata.kind), - Some(NodeKind::Function | NodeKind::Method) - ) { - continue; - } - if let Some(prefix) = path_filter - && !tracedecay_domain::path_matches_scope(file_path, Some(prefix)) - { - continue; - } - - let Some(sig) = metadata.signature.as_deref() else { - continue; - }; - - if want_async.is_some_and(|want_async| metadata.is_async != want_async) { - continue; - } - - if let Some(ret_pat) = returns - && !returns_substring(sig).contains(ret_pat) - { - continue; - } - - if !params.is_empty() { - let param_region = params_substring(sig); - if !params.iter().all(|p| param_region.contains(p.as_str())) { - continue; - } - } - - if !touched.iter().any(|path| path == file_path) { - touched.push(file_path.to_owned()); - } - entries.push(json!({ - "id": node.occurrence.as_str(), - "name": metadata.simple_name, - "qualified_name": metadata.qualified_name, - "kind": metadata.kind, - "file": file_path, - "line": metadata.start_line.saturating_add(1), - "signature": sig, - "is_async": metadata.is_async, - "unavailable_fields": [], - })); - if entries.len() >= limit { - break; - } - } - - ( - json!({ - "match_count": entries.len(), - "matches": entries, - }), - touched, - ) - }); - Ok(generic_tool_result( - Some(graph.project_root()?), - &args, - &payload, - touched, - )) -} - -fn returns_substring(signature: &str) -> &str { - match signature.find("->") { - Some(pos) => signature[pos + 2..].trim_start(), - None => signature, - } -} - -fn params_substring(signature: &str) -> &str { - let bytes = signature.as_bytes(); - let Some(open) = signature.find('(') else { - return signature; - }; - let mut depth = 0i32; - for (i, b) in bytes.iter().enumerate().skip(open) { - match b { - b'(' => depth += 1, - b')' => { - depth -= 1; - if depth == 0 { - return &signature[open + 1..i]; - } - } - _ => {} - } - } - signature -} diff --git a/crates/tracedecay-mcp/src/handlers/info/status.rs b/crates/tracedecay-mcp/src/handlers/info/status.rs index 4a37abbcc5..0327b61da3 100644 --- a/crates/tracedecay-mcp/src/handlers/info/status.rs +++ b/crates/tracedecay-mcp/src/handlers/info/status.rs @@ -655,7 +655,6 @@ fn active_project_context( "class": store_kind_name(&layout.store_kind), "mode": storage_mode_name(&layout.storage_mode), "data_root": display_path(&layout.data_root), - "config_path": display_path(&layout.config_path), "graph_db_path": display_path(graph_db_path), "graph_db_exists": graph_db_path.exists(), "graph_db_size_bytes": graph_db_path.metadata().map_or(0, |metadata| metadata.len()), @@ -684,7 +683,6 @@ fn active_project_context( fn storage_mode_name(mode: &StorageMode) -> &'static str { match mode { - StorageMode::ProjectLocal => "project_local", StorageMode::ProfileSharded => "profile_sharded", } } diff --git a/crates/tracedecay-mcp/src/handlers/info/todos.rs b/crates/tracedecay-mcp/src/handlers/info/todos.rs index c59d0a3ab1..b6a7f8cf9b 100644 --- a/crates/tracedecay-mcp/src/handlers/info/todos.rs +++ b/crates/tracedecay-mcp/src/handlers/info/todos.rs @@ -3,9 +3,10 @@ use std::collections::{BTreeMap, HashMap}; use std::path::Path; -use crate::ToolResult; -use crate::{decode_primitive_request, generic_tool_result}; +use crate::decode_primitive_request; +use crate::handlers::graph::graph_tool_completion; use serde_json::Value; +use tracedecay_contracts::graph_tool::{GraphToolCompletionV1, GraphToolResultV1}; use tracedecay_contracts::retrieval::{TodoMarkerV1, TodosResultV1, TodosSurfaceRequestV1}; use tracedecay_domain::errors::{Result, TraceDecayError}; use tracedecay_graph_query::VerifiedGraphQuery; @@ -48,11 +49,11 @@ fn contains_marker_word(text: &str, marker: &str) -> Option { } #[hotpath::measure(label = "mcp.info.todos.total")] -pub async fn handle_todos( +pub async fn compute_todos( graph: &VerifiedGraphQuery, args: Value, scope_prefix: Option<&str>, -) -> Result { +) -> Result { let request: TodosSurfaceRequestV1 = decode_primitive_request(&args, "tracedecay_todos")?; let kinds = request .kinds @@ -97,7 +98,6 @@ pub async fn handle_todos( // Graph phase is done. The marker walk reads every candidate source file, // so it belongs on a blocking worker like the sibling analysis scans. let project_root = graph.project_root()?.to_path_buf(); - let response_project_root = project_root.clone(); let (markers, touched, by_kind) = hotpath::future!( tokio::task::spawn_blocking(move || -> Result<_> { let mut markers = Vec::::new(); @@ -170,11 +170,8 @@ pub async fn handle_todos( by_kind, markers, }; - let output = serde_json::to_value(result)?; - Ok(generic_tool_result( - Some(&response_project_root), - &args, - &output, + Ok(graph_tool_completion( + GraphToolResultV1::Todos(result), touched, )) } diff --git a/crates/tracedecay-mcp/src/handlers/info/type_hierarchy.rs b/crates/tracedecay-mcp/src/handlers/info/type_hierarchy.rs deleted file mode 100644 index b9bf8058d0..0000000000 --- a/crates/tracedecay-mcp/src/handlers/info/type_hierarchy.rs +++ /dev/null @@ -1,155 +0,0 @@ -//! `tracedecay_type_hierarchy`, verified implements/extends tree rooted at a symbol. - -use std::collections::HashSet; -use std::fmt::Write as _; - -use crate::ToolResult; -use crate::tools::render::Md; -use crate::{rendered_tool_result, require_node_id, unique_file_paths}; -use serde_json::{Value, json}; -use tracedecay_domain::errors::{Result, TraceDecayError}; -use tracedecay_domain::{RelationEdgeKindV1, SymbolOccurrenceId}; -use tracedecay_graph_query::VerifiedGraphQuery; - -use super::verified::{INFO_RELATION_LIMIT, info_graph_error, required_symbol_parts}; - -#[hotpath::measure(label = "mcp.info.type_hierarchy.total")] -pub async fn handle_type_hierarchy(graph: &VerifiedGraphQuery, args: Value) -> Result { - let node_id = require_node_id(&args)?; - let occurrence = - SymbolOccurrenceId::new(node_id.to_owned()).map_err(|error| TraceDecayError::Config { - message: format!("invalid node_id '{node_id}': {error}"), - })?; - let max_depth = args - .get("max_depth") - .and_then(Value::as_u64) - .map_or(5, |value| value.min(10) as usize); - let root = graph - .symbol_summary(&occurrence)? - .ok_or_else(|| TraceDecayError::Config { - message: format!("node not found in verified generation: {node_id}"), - })?; - let (root_metadata, root_file) = required_symbol_parts(&root)?; - - let mut tree = format!( - "{} ({}) -- {}:{}\n", - root_metadata.simple_name, - root_metadata.kind, - root_file, - root_metadata.start_line.saturating_add(1) - ); - let root_display = format!( - "{} ({}) - {}:{}", - root_metadata.simple_name, - root_metadata.kind, - root_file, - root_metadata.start_line.saturating_add(1) - ); - let mut all_files = vec![root_file.to_owned()]; - let mut seen = HashSet::from([occurrence.clone()]); - hotpath::measure_block!( - "mcp.info.type_hierarchy.walk", - build_type_tree( - graph, - &occurrence, - max_depth, - 0, - &mut tree, - &mut all_files, - &mut seen, - )? - ); - - let touched_files = unique_file_paths(all_files.iter().map(String::as_str)); - let payload = json!({ - "root": { - "id": root.occurrence.as_str(), - "name": root_metadata.simple_name, - "kind": root_metadata.kind, - "file": root_file, - "line": root_metadata.start_line.saturating_add(1), - }, - "max_depth": max_depth, - "tree": tree, - }); - Ok(rendered_tool_result( - Some(graph.project_root()?), - &args, - &payload, - touched_files, - || render_type_hierarchy_md(&root_display, max_depth, &tree), - )) -} - -fn render_type_hierarchy_md(root_display: &str, max_depth: usize, tree: &str) -> String { - let mut md = Md::new(); - md.heading(2, "Type Hierarchy"); - md.field("root", root_display); - md.field("max_depth", &max_depth.to_string()); - md.blank().code("text", tree); - md.render() -} - -fn build_type_tree( - graph: &VerifiedGraphQuery, - node_id: &SymbolOccurrenceId, - max_depth: usize, - depth: usize, - output: &mut String, - all_files: &mut Vec, - seen: &mut HashSet, -) -> Result<()> { - if depth >= max_depth { - return Ok(()); - } - let mut batches = graph.callers( - std::slice::from_ref(node_id), - &[RelationEdgeKindV1::Implements, RelationEdgeKindV1::Extends], - INFO_RELATION_LIMIT, - )?; - if batches.len() != 1 { - return Err(info_graph_error( - "verified-type-hierarchy-adjacency-invalid", - "verified graph did not return exactly one hierarchy adjacency batch", - )); - } - let pad = " ".repeat(depth); - for edge in batches.remove(0) { - if !seen.insert(edge.neighbor.occurrence.clone()) { - continue; - } - let (metadata, file) = required_symbol_parts(&edge.neighbor)?; - let relation = match edge.edge.kind { - RelationEdgeKindV1::Implements => "implements", - RelationEdgeKindV1::Extends => "extends", - _ => { - return Err(info_graph_error( - "verified-type-hierarchy-relation-invalid", - "verified graph returned a non-hierarchy relation", - )); - } - }; - writeln!( - output, - "{pad}|- {relation} {} ({}) -- {}:{}", - metadata.simple_name, - metadata.kind, - file, - metadata.start_line.saturating_add(1), - ) - .map_err(|error| TraceDecayError::Config { - message: format!("cannot render type hierarchy: {error}"), - })?; - all_files.push(file.to_owned()); - build_type_tree( - graph, - &edge.neighbor.occurrence, - max_depth, - depth + 1, - output, - all_files, - seen, - )?; - } - Ok(()) -} diff --git a/crates/tracedecay-mcp/src/handlers/mod.rs b/crates/tracedecay-mcp/src/handlers/mod.rs index 3d57d18814..16598fcdc4 100644 --- a/crates/tracedecay-mcp/src/handlers/mod.rs +++ b/crates/tracedecay-mcp/src/handlers/mod.rs @@ -20,12 +20,12 @@ pub mod dependency_hints; pub mod edit; pub mod git; pub mod graph; +pub mod graph_tool; pub mod grep; pub mod health; pub mod hook_runtime; pub mod info; mod multi_root; -mod retained_response; mod session_authorities; pub mod skills; pub mod support; @@ -37,17 +37,13 @@ pub mod workflow_family; pub use bounded_search::run_bounded_search; pub use multi_root::handle_multi_root; pub(crate) use multi_root::multi_root_operation_for_tool; -pub use retained_response::{ - retained_problem_envelope, retained_safe_diagnostic, validated_retained_response, -}; pub use session_authorities::SessionAuthorities; #[cfg(any(test, feature = "test-helpers"))] pub use session_authorities::mcp_session_authorities; pub use support::{ - CONTEXT_MEMORY_ANALYTICS_KEY, decode_primitive_request, effective_path, generic_tool_result, - json_result, rendered_tool_result, require_node_id, require_object_args, - require_positive_limit, take_internal_context_memory_analytics, text_tool_result, tool_json, - tool_json_with_md, unique_file_paths, unknown_tool_error, + decode_primitive_request, effective_path, generic_tool_result, json_result, + rendered_tool_result, require_node_id, require_object_args, require_positive_limit, + text_tool_result, tool_json, tool_json_with_md, unique_file_paths, unknown_tool_error, }; pub use verified_read::{VerifiedGraphOpen, VerifiedGraphOpenFuture, verified_read_operation}; pub use work::handle_work; diff --git a/crates/tracedecay-mcp/src/handlers/multi_root.rs b/crates/tracedecay-mcp/src/handlers/multi_root.rs index 17c64bb07a..82e6b365b5 100644 --- a/crates/tracedecay-mcp/src/handlers/multi_root.rs +++ b/crates/tracedecay-mcp/src/handlers/multi_root.rs @@ -14,10 +14,9 @@ use tracedecay_tool_catalog::{BindingId, SchemaId}; use crate::ToolResult; use crate::handlers::support::{json_result, unknown_tool_error}; +use tracedecay_contracts::now_micros; use tracedecay_contracts::request_identity::{GlobalRequestSurface, mint_global_request_id}; -use tracedecay_daemon_protocol::{ - DaemonInvocationExecutor, InvocationCancellationPolicy, invocation_now_micros, -}; +use tracedecay_daemon_protocol::{DaemonInvocationExecutor, InvocationCancellationPolicy}; use tracedecay_daemon_protocol::{ DaemonInvocationOutcome, DaemonInvocationProblem, DaemonInvocationRequest, DaemonInvocationResponse, @@ -45,7 +44,7 @@ pub async fn handle_multi_root( } })?, }; - let observed_at = invocation_now_micros(); + let observed_at = now_micros(); let deadline = match protocol_deadline { Some(deadline) => deadline, None => Deadline::new(UtcMicros( @@ -214,6 +213,9 @@ where request_id, scope, outcome, + touched_files: Vec::new(), + code_graph: None, + analytics: None, }; let payload = json!({ "binding_id": binding_id(operation)?, diff --git a/crates/tracedecay-mcp/src/handlers/retained_response.rs b/crates/tracedecay-mcp/src/handlers/retained_response.rs deleted file mode 100644 index 3097d35dce..0000000000 --- a/crates/tracedecay-mcp/src/handlers/retained_response.rs +++ /dev/null @@ -1,130 +0,0 @@ -use tracedecay_contracts::{ - ApplicationEnvelope, ApplicationProblem, ApplicationProblemEnvelope, ApplicationResult, - RequestId, ResultContractRef, RetryDirective, SafeDiagnostic, -}; - -use tracedecay_daemon_protocol::{DaemonInvocationOutcome, DaemonInvocationProblem}; -use tracedecay_domain::errors::{Result, TraceDecayError}; - -fn retained_contract_error( - context: &'static str, - error: &tracedecay_contracts::ApplicationContractError, -) -> TraceDecayError { - TraceDecayError::Config { - message: format!("{context}: {error}"), - } -} - -pub fn retained_safe_diagnostic( - code: &'static str, - message: &'static str, -) -> Result { - SafeDiagnostic::new(code, message) - .map_err(|error| retained_contract_error("invalid retained application diagnostic", &error)) -} - -pub fn retained_problem_envelope( - contract: ResultContractRef, - request_id: RequestId, - problem: ApplicationProblem, -) -> Result { - ApplicationProblemEnvelope::new(contract, request_id, problem).map_err(|error| { - retained_contract_error("invalid retained application problem envelope", &error) - }) -} - -#[hotpath::measure(label = "mcp.retained.response_validate")] -pub fn validated_retained_response( - outcome: DaemonInvocationOutcome, - operation: tracedecay_contracts::RetainedSurfaceOperation, - request_id: &RequestId, - result_contract: &ResultContractRef, -) -> Result> { - match outcome { - DaemonInvocationOutcome::RetainedApplication { scope, outcome } - if tracedecay_contracts::retained_surface_outcome_matches_terminal( - operation, request_id, &scope, &outcome, - ) => - { - Ok(Ok(ApplicationEnvelope { - contract: result_contract.clone(), - request_id: request_id.clone(), - scope, - outcome, - })) - } - DaemonInvocationOutcome::RetainedApplicationProblem { scope, problem } - if tracedecay_contracts::retained_surface_problem_matches_terminal( - operation, - request_id, - Some(&scope), - &problem, - ) => - { - Ok(Err(retained_problem_envelope( - result_contract.clone(), - request_id.clone(), - problem, - )?)) - } - DaemonInvocationOutcome::ApplicationProblem { problem } - if tracedecay_contracts::retained_surface_problem_matches_terminal( - operation, request_id, None, &problem, - ) => - { - Ok(Err(retained_problem_envelope( - result_contract.clone(), - request_id.clone(), - problem, - )?)) - } - DaemonInvocationOutcome::Problem { problem } => { - let problem = invocation_problem(problem)?; - Ok(Err(retained_problem_envelope( - result_contract.clone(), - request_id.clone(), - problem, - )?)) - } - _ => Ok(Err(retained_problem_envelope( - result_contract.clone(), - request_id.clone(), - ApplicationProblem::unavailable(retained_safe_diagnostic( - "application.surface.invalid_response", - "The daemon returned an invalid retained application response", - )?), - )?)), - } -} - -fn invocation_problem(problem: DaemonInvocationProblem) -> Result { - Ok(match problem { - DaemonInvocationProblem::InvalidRequest | DaemonInvocationProblem::UnsupportedRevision => { - ApplicationProblem::invalid_request_without_action( - "application.surface.invalid_request", - "The daemon rejected the retained application request", - ) - } - DaemonInvocationProblem::NotFoundOrNotAuthorized => { - ApplicationProblem::not_found_or_not_authorized(RetryDirective::Never) - } - DaemonInvocationProblem::ResetRequired => { - ApplicationProblem::reset_required(retained_safe_diagnostic( - "application.surface.reset_required", - "The retained application store requires an explicit reset", - )?) - } - DaemonInvocationProblem::ApplicationContractViolation => { - ApplicationProblem::unavailable(retained_safe_diagnostic( - "application.surface.contract_violation", - "The retained application result violated its canonical contract", - )?) - } - DaemonInvocationProblem::Unavailable => { - ApplicationProblem::unavailable(retained_safe_diagnostic( - "application.surface.unavailable", - "The retained application service is unavailable", - )?) - } - }) -} diff --git a/crates/tracedecay-mcp/src/handlers/skills.rs b/crates/tracedecay-mcp/src/handlers/skills.rs index 77af8ec2d4..ac5f998a54 100644 --- a/crates/tracedecay-mcp/src/handlers/skills.rs +++ b/crates/tracedecay-mcp/src/handlers/skills.rs @@ -5,7 +5,6 @@ use std::path::Path; use serde::Serialize; use serde_json::{Value, json}; -use tracedecay_automation_runtime::ports::session_store::AutomationSessionStore; use crate::ToolResult; use tracedecay_automation_runtime::automation::hermes_skill_bridge::{ @@ -123,7 +122,7 @@ pub async fn handle_skill_list( let usage_summaries = summarize_skill_usage(&profile_root, &skills).await?; let recommendations = stale_skill_recommendations( &usage_summaries, - tracedecay_project::project::current_timestamp(), + tracedecay_runtime_core::tracedecay::current_timestamp(), STALE_SKILL_AFTER_SECS, ); let improvement_recommendations = skill_improvement_recommendations(&usage_summaries); @@ -219,7 +218,7 @@ pub async fn handle_skill_view( let usage_summary = summarize_skill_usage_for(&profile_root, &skill).await?; let stale_recommendation = stale_skill_recommendations( std::slice::from_ref(&usage_summary), - tracedecay_project::project::current_timestamp(), + tracedecay_runtime_core::tracedecay::current_timestamp(), STALE_SKILL_AFTER_SECS, ) .into_iter() @@ -300,7 +299,7 @@ async fn sync_project_skill_analytics( ingest_project_analytics_events( profile_root, cg.project_root(), - analytics_db.map(|database| database as &dyn AutomationSessionStore), + analytics_db, SKILL_ANALYTICS_IMPORT_LIMIT, ) .await diff --git a/crates/tracedecay-mcp/src/handlers/support.rs b/crates/tracedecay-mcp/src/handlers/support.rs index a1b1643de4..747624fd17 100644 --- a/crates/tracedecay-mcp/src/handlers/support.rs +++ b/crates/tracedecay-mcp/src/handlers/support.rs @@ -10,11 +10,6 @@ use tracedecay_domain::errors::{Result, TraceDecayError}; use crate::ToolResult; use crate::tools::render; -/// Key under which context handlers stash analytics that must reach the server -/// but never the client. [`rendered_tool_result`] is the one place it is lifted -/// back out, so no handler has to remember to strip it. -pub const CONTEXT_MEMORY_ANALYTICS_KEY: &str = "context_memory_analytics"; - /// Decodes a paginated read's continuation from the transport arguments. /// /// The cursor is caller-supplied, so it is bounded before parsing and then @@ -39,10 +34,9 @@ pub fn retrieval_cursor(args: &Value) -> Result String>( project_root: Option<&Path>, args: &Value, @@ -50,27 +44,8 @@ pub fn rendered_tool_result String>( touched_files: Vec, md: F, ) -> ToolResult { - let internal_analytics = value.get(CONTEXT_MEMORY_ANALYTICS_KEY).cloned(); - let public_value = internal_analytics - .as_ref() - .and_then(|_| public_value_without_internal_context_memory_analytics(value)); - let value = public_value.as_ref().unwrap_or(value); let text = render::finalize(project_root, args, value, md); - let result = text_tool_result(&text, touched_files); - if let Some(internal_analytics) = internal_analytics { - result.with_internal_analytics(internal_analytics) - } else { - result - } -} - -fn public_value_without_internal_context_memory_analytics(value: &Value) -> Option { - let mut value = value.clone(); - take_internal_context_memory_analytics(&mut value).map(|_| value) -} - -pub fn take_internal_context_memory_analytics(value: &mut Value) -> Option { - value.as_object_mut()?.remove(CONTEXT_MEMORY_ANALYTICS_KEY) + text_tool_result(&text, touched_files) } /// A compact JSON payload rendered as the tool's text content. diff --git a/crates/tracedecay-mcp/src/handlers/work.rs b/crates/tracedecay-mcp/src/handlers/work.rs index 482a7b0ce6..26be0f10d4 100644 --- a/crates/tracedecay-mcp/src/handlers/work.rs +++ b/crates/tracedecay-mcp/src/handlers/work.rs @@ -10,9 +10,9 @@ use std::future::Future; use serde_json::Value; use tracedecay_api::{HttpApplicationControls, WorkHttpRequest, WorkOperation}; +use tracedecay_contracts::now_micros; use tracedecay_contracts::request_identity::{GlobalRequestSurface, mint_global_request_id}; use tracedecay_contracts::{CancellationSignal, Deadline, RequestId}; -use tracedecay_daemon_protocol::invocation_now_micros; use tracedecay_domain::UtcMicros; use tracedecay_domain::errors::{Result, TraceDecayError}; use tracedecay_tool_catalog::OperationId; @@ -131,7 +131,7 @@ fn work_controls( "The canonical Work deadline exceeds the domain clock", ) })?; - let maximum_deadline = UtcMicros(invocation_now_micros().0.saturating_add(maximum_micros)); + let maximum_deadline = UtcMicros(now_micros().0.saturating_add(maximum_micros)); let deadline = protocol_deadline .filter(|deadline| deadline.expires_at <= maximum_deadline) .map_or_else(|| Deadline::new(maximum_deadline), Ok) diff --git a/crates/tracedecay-mcp/src/handlers/workflow.rs b/crates/tracedecay-mcp/src/handlers/workflow.rs index e1899b7221..7c9dc0372a 100644 --- a/crates/tracedecay-mcp/src/handlers/workflow.rs +++ b/crates/tracedecay-mcp/src/handlers/workflow.rs @@ -28,9 +28,10 @@ use tracedecay_application::diagnostics_store::DiagnosticsStore; use tracedecay_application::operation_stream::{ OperationEmitter, OperationEventError, operation_event_authority, }; +use tracedecay_code_index::is_test_file; use tracedecay_contracts::request_identity::{GlobalRequestSurface, mint_global_request_id}; use tracedecay_domain::errors::{Result, TraceDecayError}; -use tracedecay_project::project::{TraceDecay, is_test_file}; +use tracedecay_project::project::TraceDecay; use crate::ToolResult; use crate::handlers::{generic_tool_result, rendered_tool_result, unique_file_paths}; @@ -488,7 +489,7 @@ where { let run_args = match RunAffectedArgs::parse(&args) { Ok(run_args) => run_args, - Err(result) => return Ok(result), + Err(result) => return Ok(*result), }; let project_root = cg.project_root().to_path_buf(); @@ -498,7 +499,7 @@ where // of whether the graph projection is mounted. let changed_paths = match resolve_changed_paths(&args, run_args.explicit_paths) { Ok(paths) => paths, - Err(result) => return Ok(result), + Err(result) => return Ok(*result), }; if changed_paths.is_empty() { return Ok(empty_result(&args, "no changed files detected")); @@ -848,15 +849,15 @@ fn test_run_contract_error(error: impl std::fmt::Display) -> TraceDecayError { fn resolve_changed_paths( args: &Value, explicit_paths: Option>, -) -> std::result::Result, ToolResult> { +) -> std::result::Result, Box> { match explicit_paths { Some(paths) => Ok(paths), - None => Err(error_result( + None => Err(Box::new(error_result( args, "invalid_request", "changed_paths", "`changed_paths` is required and must explicitly scope the affected-test run", - )), + ))), } } diff --git a/crates/tracedecay-mcp/src/handlers/workflow_family.rs b/crates/tracedecay-mcp/src/handlers/workflow_family.rs index dea181858e..c2fa85d967 100644 --- a/crates/tracedecay-mcp/src/handlers/workflow_family.rs +++ b/crates/tracedecay-mcp/src/handlers/workflow_family.rs @@ -10,9 +10,9 @@ use std::future::Future; use serde_json::Value; use tracedecay_api::{HttpApplicationControls, WorkflowHttpRequest, WorkflowOperation}; +use tracedecay_contracts::now_micros; use tracedecay_contracts::request_identity::{GlobalRequestSurface, mint_global_request_id}; use tracedecay_contracts::{CancellationSignal, Deadline, RequestId}; -use tracedecay_daemon_protocol::invocation_now_micros; use tracedecay_domain::UtcMicros; use tracedecay_domain::errors::{Result, TraceDecayError}; use tracedecay_tool_catalog::OperationId; @@ -129,7 +129,7 @@ fn workflow_controls( "The canonical Workflow deadline exceeds the domain clock", ) })?; - let maximum_deadline = UtcMicros(invocation_now_micros().0.saturating_add(maximum_micros)); + let maximum_deadline = UtcMicros(now_micros().0.saturating_add(maximum_micros)); let deadline = protocol_deadline .filter(|deadline| deadline.expires_at <= maximum_deadline) .map_or_else(|| Deadline::new(maximum_deadline), Ok) diff --git a/crates/tracedecay-mcp/src/hook_events.rs b/crates/tracedecay-mcp/src/hook_events.rs index 746d7cef52..3d7e80a07b 100644 --- a/crates/tracedecay-mcp/src/hook_events.rs +++ b/crates/tracedecay-mcp/src/hook_events.rs @@ -9,8 +9,7 @@ use std::time::{Duration, Instant}; use serde::{Deserialize, Serialize}; use serde_json::Value; -/// Shared with hook emitters so the receiver accepts the same agent keys. -pub use tracedecay_hooks::core_events::HookAgent; +use tracedecay_domain::HostIntegrationIdV1; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum HookEventKind { @@ -54,7 +53,7 @@ impl HookEventKind { } pub struct HookEvent { - pub agent: HookAgent, + pub agent: HostIntegrationIdV1, pub kind: HookEventKind, pub rel_paths: Vec, pub had_command: bool, @@ -145,13 +144,13 @@ pub enum HookEventPlan { AddBranchAt { root: PathBuf, branch: String, - agent: HookAgent, + agent: HostIntegrationIdV1, }, SyncCurrentBranch { branch: String, - agent: HookAgent, + agent: HostIntegrationIdV1, }, - DebouncedIncrementalSync(HookAgent), + DebouncedIncrementalSync(HostIntegrationIdV1), RecordTerminalReceipt { route: Option, receipt: tracedecay_hooks::core_events::HookTerminalReceipt, @@ -443,19 +442,21 @@ fn runtime_plan_from_durable( .map_err(|()| DurableHookEventDecodeError::Malformed)?, branch: durable_bound_required_str(&branch, DURABLE_MAX_BRANCH_BYTES) .map_err(|()| DurableHookEventDecodeError::Malformed)?, - agent: HookAgent::from_wire(&agent).ok_or(DurableHookEventDecodeError::Malformed)?, + agent: HostIntegrationIdV1::from_wire(&agent) + .ok_or(DurableHookEventDecodeError::Malformed)?, }), DurableHookEventPlan::SyncCurrentBranch { branch, agent } => { Ok(HookEventPlan::SyncCurrentBranch { branch: durable_bound_required_str(&branch, DURABLE_MAX_BRANCH_BYTES) .map_err(|()| DurableHookEventDecodeError::Malformed)?, - agent: HookAgent::from_wire(&agent) + agent: HostIntegrationIdV1::from_wire(&agent) .ok_or(DurableHookEventDecodeError::Malformed)?, }) } DurableHookEventPlan::DebouncedIncrementalSync { agent } => { Ok(HookEventPlan::DebouncedIncrementalSync( - HookAgent::from_wire(&agent).ok_or(DurableHookEventDecodeError::Malformed)?, + HostIntegrationIdV1::from_wire(&agent) + .ok_or(DurableHookEventDecodeError::Malformed)?, )) } DurableHookEventPlan::RecordTerminalReceipt { route, receipt } => { @@ -517,7 +518,7 @@ pub fn parse_hook_event(params: Option<&Value>) -> Option { protect_hook_receipt_structural_ids(receipt).ok()?; } Some(HookEvent { - agent: HookAgent::from_wire(&event.agent)?, + agent: HostIntegrationIdV1::from_wire(&event.agent)?, kind: HookEventKind::from_wire(&event.event)?, rel_paths: safe_hook_rel_paths(&event.rel_paths), // Shell text is an untyped observation. Keep only a content-free @@ -585,7 +586,7 @@ pub fn plan_hook_event( } } -pub fn sync_marker_path(data_root: &Path, agent: HookAgent) -> PathBuf { +pub fn sync_marker_path(data_root: &Path, agent: HostIntegrationIdV1) -> PathBuf { data_root.join(agent.sync_marker_file()) } @@ -910,9 +911,10 @@ mod tests { use serde_json::json; use super::{ - AddBranchAtRootAuthError, DurableHookEventDecodeError, HookAgent, HookEvent, HookEventKind, - HookEventPlan, authorize_add_branch_at_root, decode_durable_hook_event_plan, - encode_durable_hook_event_plan, parse_hook_event, plan_hook_event, + AddBranchAtRootAuthError, DurableHookEventDecodeError, HookEvent, HookEventKind, + HookEventPlan, HostIntegrationIdV1, authorize_add_branch_at_root, + decode_durable_hook_event_plan, encode_durable_hook_event_plan, parse_hook_event, + plan_hook_event, }; fn parse_or_panic(params: &serde_json::Value) -> HookEvent { @@ -1044,7 +1046,7 @@ mod tests { "planned root {root:?} should match expected root {expected_root:?}" ); assert_eq!(branch, expected_branch); - assert_eq!(agent, HookAgent::Codex); + assert_eq!(agent, HostIntegrationIdV1::Codex); } #[test] @@ -1074,7 +1076,7 @@ mod tests { PathBuf::from("/project"), ), tracedecay_hooks::core_events::DaemonHookEvent::post_tool_use_shell( - HookAgent::Codex, + HostIntegrationIdV1::Codex, PathBuf::from("/project"), ), ] { @@ -1109,10 +1111,10 @@ mod tests { #[test] fn accepts_every_constructible_hook_agent() { for agent in [ - HookAgent::Claude, - HookAgent::Codex, - HookAgent::Cursor, - HookAgent::Kiro, + HostIntegrationIdV1::Claude, + HostIntegrationIdV1::Codex, + HostIntegrationIdV1::Cursor, + HostIntegrationIdV1::Kiro, ] { let params = json!({ "agent": agent.as_wire(), @@ -1211,13 +1213,13 @@ mod tests { HookEventPlan::AddBranchAt { root: worktree_root, branch: "feature/test".to_string(), - agent: HookAgent::Codex, + agent: HostIntegrationIdV1::Codex, }, HookEventPlan::SyncCurrentBranch { branch: "main".to_string(), - agent: HookAgent::Claude, + agent: HostIntegrationIdV1::Claude, }, - HookEventPlan::DebouncedIncrementalSync(HookAgent::Cursor), + HookEventPlan::DebouncedIncrementalSync(HostIntegrationIdV1::Cursor), HookEventPlan::RecordTerminalReceipt { route: route.clone(), receipt: receipt.clone(), @@ -1361,7 +1363,7 @@ mod tests { encode_durable_hook_event_plan(&HookEventPlan::AddBranchAt { root: PathBuf::from("/tmp/worktree/../escape"), branch: "feature".to_string(), - agent: HookAgent::Codex, + agent: HostIntegrationIdV1::Codex, }) .is_err() ); @@ -1374,7 +1376,7 @@ mod tests { // The sender-side wire shape the Hermes plugin emits; production only // deserializes these events. let params = serde_json::to_value(tracedecay_hooks::core_events::DaemonHookEvent { - agent: HookAgent::Hermes.as_wire().to_string(), + agent: HostIntegrationIdV1::Hermes.as_wire().to_string(), event: "terminalReceipt".to_string(), rel_paths: Vec::new(), command: None, diff --git a/crates/tracedecay-mcp/src/hook_runtime/errors.rs b/crates/tracedecay-mcp/src/hook_runtime/errors.rs index 634572184b..e34d1c311f 100644 --- a/crates/tracedecay-mcp/src/hook_runtime/errors.rs +++ b/crates/tracedecay-mcp/src/hook_runtime/errors.rs @@ -1,6 +1,6 @@ use tracedecay_domain::errors::TraceDecayError; use tracedecay_sessions::admission::{HostAdmissionOutcome, HostAdmissionStatus}; -use tracedecay_sessions::runtime::claude_observation::ClaudeObservationIngestError; +use tracedecay_sessions::runtime::hosts::claude_observation::ClaudeObservationIngestError; /// Builds a hook-runtime error that carries the admission status its authority /// actually reported. diff --git a/crates/tracedecay-mcp/src/jsonrpc.rs b/crates/tracedecay-mcp/src/jsonrpc.rs index 47842a3368..80477d885f 100644 --- a/crates/tracedecay-mcp/src/jsonrpc.rs +++ b/crates/tracedecay-mcp/src/jsonrpc.rs @@ -109,12 +109,22 @@ pub fn validate_envelope(value: &Value) -> std::result::Result<(), JsonRpcDecode /// after [`validate_envelope`]. Transports that parse to [`Value`] first (the /// rmcp receive loop) use this so their shape rejections carry the same id /// correlation as [`JsonRpcRequest::decode`]. +/// +/// MCP forbids a `null` request id, and the typed `rmcp` model cannot carry +/// one (it would decode as a notification and never be answered), so an +/// explicit `"id": null` is refused as `InvalidRequest` with the null id. pub fn decode_envelope( - value: Value, + value: &Value, ) -> std::result::Result { - validate_envelope(&value)?; - let id = detected_request_id(&value); - serde_json::from_value(value).map_err(|error| JsonRpcDecodeError::invalid_request(id, error)) + validate_envelope(value)?; + if value.get("id").is_some_and(Value::is_null) { + return Err(JsonRpcDecodeError::invalid_request( + Value::Null, + "MCP request id must be a string or number, not null", + )); + } + T::deserialize(value) + .map_err(|error| JsonRpcDecodeError::invalid_request(detected_request_id(value), error)) } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -412,7 +422,7 @@ mod tests { fn decode_envelope_applies_the_same_rule_to_parsed_values() { let value = json!({"jsonrpc": "2.0", "id": 3, "method": "ping"}); assert!(validate_envelope(&value).is_ok()); - let request: JsonRpcRequest = decode_envelope(value).unwrap(); + let request: JsonRpcRequest = decode_envelope(&value).unwrap(); assert_eq!(request.method, "ping"); let foreign = json!({"jsonrpc": "1.0", "id": 3, "method": "ping"}); @@ -421,7 +431,7 @@ mod tests { json!(3) ); assert_eq!( - invalid_request_id(decode_envelope::(foreign).unwrap_err()), + invalid_request_id(decode_envelope::(&foreign).unwrap_err()), json!(3) ); assert_eq!( @@ -435,9 +445,20 @@ mod tests { // A valid envelope whose body is not a request still correlates by id. assert_eq!( invalid_request_id( - decode_envelope::(json!({"jsonrpc": "2.0", "id": 5})).unwrap_err() + decode_envelope::(&json!({"jsonrpc": "2.0", "id": 5})).unwrap_err() ), json!(5) ); + // MCP forbids a null id; it must be refused, not decoded as work. + let null_id = decode_envelope::( + &json!({"jsonrpc": "2.0", "id": null, "method": "ping"}), + ) + .unwrap_err() + .into_response(); + assert_eq!(null_id.id, Value::Null); + assert_eq!( + null_id.error.unwrap().code, + ErrorCode::InvalidRequest.as_i32() + ); } } diff --git a/crates/tracedecay-mcp/src/lib.rs b/crates/tracedecay-mcp/src/lib.rs index 44532d0bbe..755a23f994 100644 --- a/crates/tracedecay-mcp/src/lib.rs +++ b/crates/tracedecay-mcp/src/lib.rs @@ -77,11 +77,9 @@ pub use context_headings::{ CONTEXT_RELATED_SYMBOLS_HEADING, CONTEXT_SEEN_NODE_IDS_LABEL, CONTEXT_TEST_COVERAGE_HEADING, }; pub use handlers::{ - CONTEXT_MEMORY_ANALYTICS_KEY, decode_primitive_request, effective_path, generic_tool_result, - handle_multi_root, handle_work, handle_workflow, rendered_tool_result, require_node_id, - require_object_args, require_positive_limit, retained_problem_envelope, - retained_safe_diagnostic, take_internal_context_memory_analytics, text_tool_result, tool_json, - tool_json_with_md, unique_file_paths, validated_retained_response, + decode_primitive_request, effective_path, generic_tool_result, handle_multi_root, handle_work, + handle_workflow, rendered_tool_result, require_node_id, require_object_args, + require_positive_limit, text_tool_result, tool_json, tool_json_with_md, unique_file_paths, }; pub use hook_runtime::{ hook_admission_error, map_claude_observation_ingest_error, map_host_admission_outcome, @@ -90,7 +88,6 @@ pub use hook_runtime::{ pub use jsonrpc::{ ErrorCode, JsonRpcDecodeError, JsonRpcError, JsonRpcRequest, JsonRpcResponse, McpTransport, }; -pub use lifecycle::{McpConnectionLifecyclePort, McpLifecycleDrainFuture, McpRequestActivity}; pub use tool_call_deadline::{ TOOL_CALL_DEADLINE_META_KEY, caller_tool_call_deadline, caller_tool_call_deadline_from_meta, tool_call_deadline_meta, @@ -112,8 +109,8 @@ pub use tools::{ pub use tracedecay_mcp_catalog::{ MAX_RESPONSE_CHARS, McpCatalogError, ToolDefinition, ToolRegistryMode, apply_context_warming_budget, ast_grep_available, ast_grep_diagnostics_json, - ast_grep_outline_available, context_description, context_warming_description, - explore_call_budget, format_capable_tool_names, get_maximal_tool_definitions, + context_description, context_warming_description, explore_call_budget, + format_capable_tool_names, get_maximal_tool_definitions, get_maximal_tool_definitions_with_budget, get_tool_definitions, get_tool_definitions_with_budget, get_tool_definitions_with_warming_budget, internal_daemon_tool_definition, mcp_input_schema, project_catalog_discovery_scope, diff --git a/crates/tracedecay-mcp/src/lifecycle.rs b/crates/tracedecay-mcp/src/lifecycle.rs index 2e134e5f7d..8be544de3d 100644 --- a/crates/tracedecay-mcp/src/lifecycle.rs +++ b/crates/tracedecay-mcp/src/lifecycle.rs @@ -1,53 +1,7 @@ -//! Server-shaped lifecycle observation ports. -//! -//! The concrete daemon lifecycle lives in `tracedecay_daemon_service::shutdown`; -//! this module adapts it to the port the MCP connection loop observes drain -//! and request admission through. - -use std::future::Future; -use std::pin::Pin; -use std::sync::Arc; - -use tracedecay_daemon_service::shutdown::DaemonLifecycle; - -/// Request-activity guard retained while one MCP request is admitted. -/// -/// Dropping the guard releases the underlying lifecycle seat. The boxed -/// retainee is the root-implemented activity token. -pub struct McpRequestActivity { - _retain: Box, -} - -impl McpRequestActivity { - pub fn retain(guard: T) -> Self { - Self { - _retain: Box::new(guard), - } - } -} +//! Server-shaped lifecycle owners: background tasks, startup catch-up, and +//! project-server response revocation. -pub type McpLifecycleDrainFuture<'a> = Pin + Send + 'a>>; - -/// Observe daemon drain and admit one request seat without naming daemon types. -pub trait McpConnectionLifecyclePort: Send + Sync { - fn accepting(&self) -> bool; - fn try_enter(&self) -> Option; - fn wait_for_draining(&self) -> McpLifecycleDrainFuture<'_>; -} - -impl McpConnectionLifecyclePort for DaemonLifecycle { - fn accepting(&self) -> bool { - DaemonLifecycle::accepting(self) - } - - fn try_enter(&self) -> Option { - DaemonLifecycle::try_enter(self).map(McpRequestActivity::retain) - } - - fn wait_for_draining(&self) -> McpLifecycleDrainFuture<'_> { - Box::pin(DaemonLifecycle::wait_for_draining(self)) - } -} +use std::sync::Arc; /// Bound on the join failures retained from tasks reaped during normal /// operation. Shutdown reports these alongside anything it drains itself; a @@ -352,16 +306,16 @@ impl StartupCatchUpMachineV1 { #[derive(Clone)] pub struct ProjectServerResponseLifecycle { response_gate: Arc>, - response_revoked: tracedecay_session_memory::context::CancellationToken, - request_abort: tracedecay_session_memory::context::CancellationToken, + response_revoked: tracedecay_runtime_core::cancellation::CancellationToken, + request_abort: tracedecay_runtime_core::cancellation::CancellationToken, } impl Default for ProjectServerResponseLifecycle { fn default() -> Self { Self { response_gate: Arc::new(tokio::sync::RwLock::new(())), - response_revoked: tracedecay_session_memory::context::CancellationToken::new(), - request_abort: tracedecay_session_memory::context::CancellationToken::new(), + response_revoked: tracedecay_runtime_core::cancellation::CancellationToken::new(), + request_abort: tracedecay_runtime_core::cancellation::CancellationToken::new(), } } } @@ -392,7 +346,7 @@ impl ProjectServerResponseLifecycle { &self.response_gate } - pub fn response_revoked(&self) -> &tracedecay_session_memory::context::CancellationToken { + pub fn response_revoked(&self) -> &tracedecay_runtime_core::cancellation::CancellationToken { &self.response_revoked } } diff --git a/crates/tracedecay-mcp/src/server/connection.rs b/crates/tracedecay-mcp/src/server/connection.rs index 2215675f5f..3ae112d4d5 100644 --- a/crates/tracedecay-mcp/src/server/connection.rs +++ b/crates/tracedecay-mcp/src/server/connection.rs @@ -1,42 +1,18 @@ -//! Transport-owned JSON-RPC connection scheduling and delivery. +//! Production connection context the `rmcp` adapter dispatches through. -use std::collections::{HashMap, HashSet, VecDeque}; use std::future::Future; use std::pin::Pin; -use std::sync::Arc; -#[cfg(test)] -use std::sync::atomic::Ordering; use serde_json::Value; -use crate::lifecycle::{McpConnectionLifecyclePort, McpRequestActivity}; -use crate::transport::{McpTransport, write_wire_oversized_rejection}; -use crate::{ErrorCode, JsonRpcRequest, JsonRpcResponse, serialize_response_line}; -use tracedecay_contracts::request_identity::mcp_connection_request_key as application_surface_request_id; -use tracedecay_domain::errors::{Result, TraceDecayError}; -use tracedecay_framing::is_wire_oversized_io_error; - -use super::{McpDispatchRequest, McpMethod, classify_mcp_method, dispatch_is_independent_read}; - -const MAX_PENDING_CANCELLABLE_REQUEST_LINES: usize = 64; - -#[hotpath::measure(label = "mcp.server.connection.read", future = true)] -async fn read_connection_line( - transport: &mut impl McpTransport, -) -> std::io::Result> { - transport.read_line().await -} +use crate::JsonRpcResponse; +use tracedecay_domain::errors::Result; -#[hotpath::measure(label = "mcp.server.connection.inflight_read", future = true)] -async fn read_inflight_connection_line( - transport: &mut impl McpTransport, -) -> std::io::Result> { - transport.read_line().await -} +use super::McpDispatchRequest; /// Selected-project response authority retained through transport delivery. pub trait McpResponseLease: Send + 'static { - fn revoked(&self) -> &tracedecay_session_memory::context::CancellationToken; + fn revoked(&self) -> &tracedecay_runtime_core::cancellation::CancellationToken; } /// Per-connection routing state owned by the production request context. @@ -46,12 +22,10 @@ pub trait McpConnectionState: Send + Sync + 'static { fn memory_request_scope(&self) -> &str; #[must_use] fn fork_for_independent_read(&self) -> Self; - #[must_use] - fn fork_for_connection_owned_read(&self) -> Self; fn take_selected_response_lease(&mut self) -> Option; } -/// The one production context required by the portable connection scheduler. +/// The one production context the `rmcp` connection adapter requires. pub trait McpConnectionContext: Send + Sync + 'static { type Connection: McpConnectionState; @@ -60,7 +34,6 @@ pub trait McpConnectionContext: Send + Sync + 'static { fn build_version(&self) -> Result<&'static str>; fn max_concurrent_reads(&self) -> usize; fn tool_is_read_only(&self, tool_name: &str) -> bool; - fn tool_supports_live_cancellation(&self, tool_name: &str) -> bool; /// The token is sticky: once cancelled it stays cancelled, so a late /// sample never misses a cancel. Sticky is not the same as interruptible. /// An implementation that only samples after resolving its route leaves @@ -80,19 +53,18 @@ pub trait McpConnectionContext: Send + Sync + 'static { /// A cancel that [`Self::cancel_request`] reports as unregistered is not /// automatically "never registerable": route resolution has not yet called /// `prepare_dispatch_control`. Transports must wait on - /// [`Self::cancellation_registered`] (as the legacy connection does) so - /// the sticky sample and selected target still settle. Abandon only when - /// the cancel can never register, no registration wait channel. + /// [`Self::cancellation_registered`] so the sticky sample and selected + /// target still settle. Abandon only when the cancel can never register, + /// no registration wait channel. fn dispatch<'a>( &'a self, request: McpDispatchRequest<'a>, timings_enabled: bool, connection: &'a mut Self::Connection, - cancellation: tracedecay_session_memory::context::CancellationToken, + cancellation: tracedecay_runtime_core::cancellation::CancellationToken, ) -> Pin> + Send + 'a>>; fn cancel_request(&self, id: &Value, connection_scope: &str) -> bool; fn cancellation_registered(&self) -> &tokio::sync::Notify; - fn take_pending_notifications(&self) -> Vec; fn run_in_connection_admission<'a, T, F>( &'a self, future: F, @@ -100,56 +72,6 @@ pub trait McpConnectionContext: Send + Sync + 'static { where T: Send + 'a, F: Future + Send + 'a; - fn shutdown(self: Arc) -> Pin + Send>>; -} - -/// A portable connection scheduler over one real production context. -pub struct McpConnectionServer { - context: Arc, -} - -impl McpConnectionServer -where - C: McpConnectionContext, -{ - pub fn new(context: Arc) -> Arc { - Arc::new(Self { context }) - } - - fn new_connection_route_state(&self) -> Result { - self.context.new_connection() - } - - fn timings_enabled(&self) -> bool { - self.context.timings_enabled() - } - - fn cancel_application_surface_request(&self, id: &Value, connection_scope: &str) -> bool { - self.context.cancel_request(id, connection_scope) - } - - async fn handle_request_for_connection<'a>( - &'a self, - request: &'a JsonRpcRequest, - timings_enabled: bool, - connection: &'a mut C::Connection, - cancellation: tracedecay_session_memory::context::CancellationToken, - ) -> Option { - self.context - .dispatch( - McpDispatchRequest::from_legacy(request), - timings_enabled, - connection, - cancellation, - ) - .await - } - - async fn shutdown_if(self: &Arc, enabled: bool) { - if enabled { - Arc::clone(&self.context).shutdown().await; - } - } } /// Races asynchronous route resolution against the request's own cancellation. @@ -173,1065 +95,12 @@ where } } -/// One buffered request line plus the identity a queued cancellation can -/// target, extracted once at enqueue so each cancellation notification does -/// not re-parse every pending line. -struct QueuedRequestLine { - line: String, - request_id: Option, - independent_read: bool, - /// `Some(id)` only when the line is a `tools/call` for a - /// live-cancellable tool, the only lines a queued cancellation matches. - cancellable_request_id: Option, - queued_at: std::time::Instant, - _depth: PendingRequestGaugeGuard, -} - -struct PendingRequestGaugeGuard { - bytes: usize, - #[cfg(test)] - observer: Option>, -} - -impl PendingRequestGaugeGuard { - fn enter(bytes: usize) -> Self { - hotpath::gauge!("mcp.server.request.queue_depth").inc(1_u64); - hotpath::gauge!("mcp.server.request.queue_bytes").inc(bytes as u64); - Self { - bytes, - #[cfg(test)] - observer: None, - } - } - - #[cfg(test)] - fn enter_observed(bytes: usize, observer: Arc) -> Self { - let mut guard = Self::enter(bytes); - observer.fetch_add(1, Ordering::AcqRel); - guard.observer = Some(observer); - guard - } -} - -impl Drop for PendingRequestGaugeGuard { - fn drop(&mut self) { - hotpath::gauge!("mcp.server.request.queue_depth").dec(1_u64); - hotpath::gauge!("mcp.server.request.queue_bytes").dec(self.bytes as u64); - #[cfg(test)] - if let Some(observer) = self.observer.as_ref() { - observer.fetch_sub(1, Ordering::AcqRel); - } - } -} - -impl QueuedRequestLine { - fn new(line: String, context: &C) -> Self - where - C: McpConnectionContext, - { - let parsed = hotpath::measure_block!( - "mcp.server.connection.queued_decode", - JsonRpcRequest::decode(line.trim()) - ); - let request = parsed.as_ref().ok(); - let request_id = request.and_then(|request| request.id.clone()); - let independent_read = - request.is_some_and(|request| request_is_independent_read(request, context)); - let cancellable_request_id = - request.and_then(|request| cancellable_queued_request_id(request, context)); - let depth = PendingRequestGaugeGuard::enter(line.len()); - Self { - line, - request_id, - independent_read, - cancellable_request_id, - queued_at: std::time::Instant::now(), - _depth: depth, - } - } - - fn from_parsed(line: String, request: Option<&JsonRpcRequest>, context: &C) -> Self - where - C: McpConnectionContext, - { - let request_id = request.and_then(|request| request.id.clone()); - let independent_read = - request.is_some_and(|request| request_is_independent_read(request, context)); - let cancellable_request_id = - request.and_then(|request| cancellable_queued_request_id(request, context)); - let depth = PendingRequestGaugeGuard::enter(line.len()); - Self { - line, - request_id, - independent_read, - cancellable_request_id, - queued_at: std::time::Instant::now(), - _depth: depth, - } - } - - #[cfg(test)] - fn new_observed(line: String, observer: Arc) -> Self { - let depth = PendingRequestGaugeGuard::enter_observed(line.len(), observer); - Self { - line, - request_id: None, - independent_read: false, - cancellable_request_id: None, - queued_at: std::time::Instant::now(), - _depth: depth, - } - } - - fn into_line(self) -> String { - hotpath::gauge!("mcp.server.request.queue_wait_us") - .set(self.queued_at.elapsed().as_micros() as u64); - self.line - } -} - -fn cancellable_queued_request_id(request: &JsonRpcRequest, context: &C) -> Option -where - C: McpConnectionContext, -{ - let cancellable = request.method == "tools/call" - && request - .params - .as_ref() - .and_then(|params| params.get("name")) - .and_then(Value::as_str) - .is_some_and(|tool_name| context.tool_supports_live_cancellation(tool_name)); - if !cancellable { - return None; - } - request.id.clone() -} - -fn queued_cancellable_request_key( - pending_lines: &VecDeque, - request_id: &Value, - connection_scope: &str, -) -> Option { - let expected = application_surface_request_id(request_id, connection_scope)?; - pending_lines - .iter() - .filter_map(|queued| queued.cancellable_request_id.as_ref()) - .any(|id| application_surface_request_id(id, connection_scope).as_ref() == Some(&expected)) - .then_some(expected) -} - -fn current_cancellable_request_key( - request: &JsonRpcRequest, - request_id: &Value, - connection_scope: &str, -) -> Option { - let current = request - .id - .as_ref() - .and_then(|id| application_surface_request_id(id, connection_scope))?; - let cancelled = application_surface_request_id(request_id, connection_scope)?; - (current == cancelled).then_some(current) -} - -#[hotpath::measure(label = "mcp.server.connection.classify")] -fn request_is_independent_read(request: &JsonRpcRequest, context: &C) -> bool -where - C: McpConnectionContext, -{ - dispatch_is_independent_read( - classify_mcp_method(&request.method), - request - .params - .as_ref() - .and_then(|params| params.get("name")) - .and_then(Value::as_str), - |tool_name| context.tool_is_read_only(tool_name), - ) -} - -struct ConcurrentReadCompletion { - request_key: Option, - _request_activity: Option, - revocable_tool_call: Option<(Value, String)>, - response: Option, - selected_response_lease: Option, - connection_scope: String, - connection_closed: bool, -} - -enum ConnectionLoopEvent { - Queued(String), - Incoming(std::io::Result>), - Completed( - Box, tokio::task::JoinError>>>, - ), - Shutdown, - PeerClosed, -} - -#[hotpath::measure(label = "mcp.server.connection.read_dispatch", future = true)] -async fn dispatch_independent_read( - server: Arc>, - request: JsonRpcRequest, - timings_enabled: bool, - mut connection: C::Connection, - request_activity: Option, - cancellation: tracedecay_session_memory::context::CancellationToken, - connection_shutdown: tracedecay_session_memory::context::CancellationToken, -) -> ConcurrentReadCompletion<::ResponseLease> -where - C: McpConnectionContext, -{ - let connection_scope = connection.memory_request_scope().to_owned(); - let request_key = request - .id - .as_ref() - .and_then(|id| application_surface_request_id(id, &connection_scope)); - let revocable_tool_call = request.id.clone().and_then(|id| { - (request.method == "tools/call").then_some(())?; - let tool_name = request.params.as_ref()?.get("name")?.as_str()?.to_owned(); - Some((id, tool_name)) - }); - let (response, connection_closed) = { - let handling = Box::pin(server.handle_request_for_connection( - &request, - timings_enabled, - &mut connection, - cancellation.clone(), - )); - tokio::pin!(handling); - let mut cancellation_waiting_for_registration = false; - loop { - let waiting_for_registration = cancellation_waiting_for_registration; - let wait_for_cancellation_registration = async { - if !waiting_for_registration { - std::future::pending::<()>().await; - return; - } - loop { - let registered = server.context.cancellation_registered().notified(); - tokio::pin!(registered); - registered.as_mut().enable(); - if let Some(id) = request.id.as_ref() - && server.cancel_application_surface_request(id, &connection_scope) - { - return; - } - registered.await; - } - }; - tokio::pin!(wait_for_cancellation_registration); - tokio::select! { - biased; - () = connection_shutdown.cancelled() => { - if let Some(id) = request.id.as_ref() { - let _ = server.cancel_application_surface_request(id, &connection_scope); - } - break (None, true); - } - response = &mut handling => break (response, false), - () = &mut wait_for_cancellation_registration => { - cancellation_waiting_for_registration = false; - } - () = cancellation.cancelled(), if !cancellation_waiting_for_registration => { - cancellation_waiting_for_registration = request - .id - .as_ref() - .is_some_and(|id| { - !server.cancel_application_surface_request(id, &connection_scope) - }); - } - } - } - }; - let selected_response_lease = connection.take_selected_response_lease(); - ConcurrentReadCompletion { - request_key, - _request_activity: request_activity, - revocable_tool_call, - response, - selected_response_lease, - connection_scope, - connection_closed, - } -} - -struct ConnectionResponseWriter; - -impl ConnectionResponseWriter { - async fn write( - server: &McpConnectionServer, - transport: &mut impl McpTransport, - completion: &mut ConcurrentReadCompletion< - ::ResponseLease, - >, - ) -> std::io::Result - where - C: McpConnectionContext, - { - let response_revoked = completion - .selected_response_lease - .as_ref() - .map(McpResponseLease::revoked); - let notifications = server.context.take_pending_notifications(); - for notification in notifications { - if let Ok(serialized) = hotpath::measure_block!( - "mcp.server.notification.serialize", - serde_json::to_string(¬ification) - ) && !server - .write_response_line_or_revoke( - transport, - &format!("{serialized}\n"), - response_revoked, - ) - .await? - { - return Ok(false); - } - } - let Some(response) = completion.response.as_ref() else { - return Ok(true); - }; - let json_line = hotpath::measure_block!( - "mcp.server.response.serialize", - serialize_response_line(response) - ); - server - .write_response_line_or_revoke(transport, &format!("{json_line}\n"), response_revoked) - .await - } -} - -async fn wait_for_peer_close( - peer_close: &mut Option< - std::pin::Pin + Send + 'static>>, - >, -) { - match peer_close { - Some(peer_close) => peer_close.await, - None => std::future::pending().await, - } -} - -impl McpConnectionServer -where - C: McpConnectionContext, -{ - #[hotpath::measure(label = "mcp.server.write", future = true)] - async fn write_response_line_or_revoke( - &self, - transport: &mut impl McpTransport, - output: &str, - response_revoked: Option<&tracedecay_session_memory::context::CancellationToken>, - ) -> std::io::Result { - hotpath::gauge!("mcp.server.response.bytes").set(output.len()); - let write = async { - hotpath::future!( - transport.write_line(output), - label = "mcp.server.response.write" - ) - .await?; - hotpath::future!(transport.flush(), label = "mcp.server.response.flush").await - }; - let Some(response_revoked) = response_revoked else { - return write.await.map(|()| true); - }; - tokio::select! { - biased; - () = response_revoked.cancelled() => Ok(false), - result = write => result.map(|()| true), - } - } - - #[allow(clippy::too_many_arguments)] - #[hotpath::measure(label = "mcp.server.request_cancellable", future = true)] - async fn handle_cancellable_application_request( - &self, - request: &JsonRpcRequest, - timings_enabled: bool, - connection: &mut C::Connection, - transport: &mut impl McpTransport, - pending_lines: &mut VecDeque, - pending_cancellations: &mut HashSet, - mut shutdown_requested: std::pin::Pin<&mut impl std::future::Future>, - ) -> Result<(Option, bool)> { - let connection_scope = connection.memory_request_scope().to_owned(); - let pre_cancelled = request - .id - .as_ref() - .and_then(|id| application_surface_request_id(id, &connection_scope)) - .is_some_and(|key| pending_cancellations.remove(&key)); - let cancellation = tracedecay_session_memory::context::CancellationToken::new(); - if pre_cancelled { - cancellation.cancel(); - } - let handling = Box::pin(self.handle_request_for_connection( - request, - timings_enabled, - connection, - cancellation.clone(), - )); - tokio::pin!(handling); - // One-shot clients (the CLI and the stdio proxy) shut down their write - // half once the request is on the wire, so end-of-input means "no more - // requests", not "peer is gone". Stop watching for cancellations and - // keep serving the in-flight response. Cancel only on actual peer loss - // (read/write I/O failure) or explicit shutdown/cancel paths. - let mut peer_close_check: Option< - std::pin::Pin + Send + 'static>>, - > = None; - loop { - if let Some(peer_close_check) = peer_close_check.as_mut() { - tokio::select! { - biased; - () = &mut shutdown_requested => { - if let Some(id) = request.id.as_ref() { - let _ = self.cancel_application_surface_request(id, &connection_scope); - } - return Ok((None, true)); - } - response = &mut handling => return Ok((response, false)), - () = peer_close_check => { - if let Some(id) = request.id.as_ref() { - let _ = self.cancel_application_surface_request(id, &connection_scope); - } - return Ok((None, true)); - } - } - } - tokio::select! { - biased; - () = &mut shutdown_requested => { - if let Some(id) = request.id.as_ref() { - let _ = self.cancel_application_surface_request(id, &connection_scope); - } - return Ok((None, true)); - } - response = &mut handling => return Ok((response, false)), - incoming = read_inflight_connection_line(transport) => { - let line = match incoming { - Ok(Some(line)) => line, - Ok(None) => { - peer_close_check = Some(Box::pin( - transport.peer_fully_closed_after_eof(), - )); - continue; - } - Err(error) => { - if let Some(id) = request.id.as_ref() { - let _ = self.cancel_application_surface_request(id, &connection_scope); - } - return Err(error.into()); - } - }; - let parsed = hotpath::measure_block!( - "mcp.server.connection.inflight_decode", - JsonRpcRequest::decode(line.trim()) - ); - if let Ok(notification) = &parsed - && matches!( - classify_mcp_method(¬ification.method), - McpMethod::Cancelled - ) - { - if let Some(id) = notification - .params - .as_ref() - .and_then(|params| params.get("requestId")) - && !self.cancel_application_surface_request(id, &connection_scope) - { - if current_cancellable_request_key( - request, - id, - &connection_scope, - ) - .is_some() - { - cancellation.cancel(); - } else if pending_cancellations.len() - < MAX_PENDING_CANCELLABLE_REQUEST_LINES - && let Some(key) = queued_cancellable_request_key( - pending_lines, - id, - &connection_scope, - ) - { - pending_cancellations.insert(key); - } - } - continue; - } - if pending_lines.len() >= MAX_PENDING_CANCELLABLE_REQUEST_LINES { - if let Some(id) = request.id.as_ref() { - let _ = self.cancel_application_surface_request(id, &connection_scope); - } - return Ok((None, true)); - } - pending_lines.push_back(QueuedRequestLine::from_parsed( - line, - parsed.as_ref().ok(), - self.context.as_ref(), - )); - } - } - } - } - - /// Runs a non-live-cancellable request while still observing connection - /// teardown. A request-side EOF is only a half-close until the transport - /// reports the peer's write side closed; this keeps one-shot CLI responses - /// intact while dropping abandoned handlers and their admission permits. - #[hotpath::measure(label = "mcp.server.request_non_cancellable", future = true)] - async fn handle_non_cancellable_application_request( - &self, - request: &JsonRpcRequest, - timings_enabled: bool, - connection: &mut C::Connection, - transport: &mut impl McpTransport, - pending_lines: &mut VecDeque, - mut shutdown_requested: std::pin::Pin<&mut impl std::future::Future>, - ) -> Result<(Option, bool)> { - let connection_scope = connection.memory_request_scope().to_owned(); - let handling = Box::pin(self.handle_request_for_connection( - request, - timings_enabled, - connection, - tracedecay_session_memory::context::CancellationToken::new(), - )); - tokio::pin!(handling); - let mut peer_close_check: Option< - std::pin::Pin + Send + 'static>>, - > = None; - loop { - if let Some(peer_close_check) = peer_close_check.as_mut() { - tokio::select! { - response = &mut handling => return Ok((response, false)), - () = &mut shutdown_requested => { - if let Some(id) = request.id.as_ref() { - let _ = self.cancel_application_surface_request( - id, - &connection_scope, - ); - } - return Ok((None, true)); - } - () = peer_close_check => { - if let Some(id) = request.id.as_ref() { - let _ = self.cancel_application_surface_request( - id, - &connection_scope, - ); - } - return Ok((None, true)); - } - } - } - tokio::select! { - response = &mut handling => return Ok((response, false)), - () = &mut shutdown_requested => { - if let Some(id) = request.id.as_ref() { - let _ = self.cancel_application_surface_request( - id, - &connection_scope, - ); - } - return Ok((None, true)); - } - incoming = read_inflight_connection_line(transport) => { - let line = match incoming { - Ok(Some(line)) => line, - Ok(None) => { - peer_close_check = Some(Box::pin( - transport.peer_fully_closed_after_eof(), - )); - continue; - } - Err(error) => { - if let Some(id) = request.id.as_ref() { - let _ = self.cancel_application_surface_request( - id, - &connection_scope, - ); - } - return Err(error.into()); - } - }; - if pending_lines.len() >= MAX_PENDING_CANCELLABLE_REQUEST_LINES { - if let Some(id) = request.id.as_ref() { - let _ = self.cancel_application_surface_request( - id, - &connection_scope, - ); - } - return Ok((None, true)); - } - pending_lines.push_back(QueuedRequestLine::new(line, self.context.as_ref())); - } - } - } - } - - /// Runs the server, reading JSON-RPC requests from stdin and writing - /// responses to stdout. Runs until stdin is closed or a shutdown signal - /// (SIGINT/SIGTERM) is received, then performs graceful cleanup. - #[hotpath::skip] - pub async fn run(self: &Arc, transport: &mut impl McpTransport) -> Result<()> { - self.run_with_shutdown_policy(transport, true, true, None, None) - .await - } - - /// Runs one client connection without shutting down the server when that - /// connection closes. Production daemon connections go through - /// [`Self::run_daemon_connection_with_timings`]; direct servers and - /// transport harnesses use this same connection loop without process - /// shutdown. - #[hotpath::skip] - pub async fn run_connection(self: &Arc, transport: &mut impl McpTransport) -> Result<()> { - self.run_with_shutdown_policy(transport, false, false, None, None) - .await - } - - #[hotpath::skip] - pub async fn run_daemon_connection_with_timings( - self: &Arc, - transport: &mut impl McpTransport, - timings_enabled: bool, - lifecycle: &dyn McpConnectionLifecyclePort, - ) -> Result<()> { - self.run_with_shutdown_policy( - transport, - false, - false, - Some(timings_enabled), - Some(lifecycle), - ) - .await - } - - #[hotpath::measure(label = "mcp.server.connection", future = true)] - pub async fn run_with_shutdown_policy( - self: &Arc, - transport: &mut impl McpTransport, - shutdown_on_exit: bool, - listen_for_process_signals: bool, - timings_override: Option, - request_lifecycle: Option<&dyn McpConnectionLifecyclePort>, - ) -> Result<()> { - Box::pin(self.run_connection_loop( - transport, - shutdown_on_exit, - listen_for_process_signals, - timings_override, - request_lifecycle, - )) - .await - } - - #[hotpath::measure(label = "mcp.server.connection.loop", future = true)] - async fn run_connection_loop( - self: &Arc, - transport: &mut impl McpTransport, - shutdown_on_exit: bool, - listen_for_process_signals: bool, - timings_override: Option, - request_lifecycle: Option<&dyn McpConnectionLifecyclePort>, - ) -> Result<()> { - let mut connection_route = self.new_connection_route_state()?; - let mut pending_lines: VecDeque = VecDeque::new(); - let mut pending_cancellations = HashSet::new(); - let mut active_reads: tokio::task::JoinSet< - ConcurrentReadCompletion<::ResponseLease>, - > = tokio::task::JoinSet::new(); - let mut active_cancellations: HashMap< - String, - tracedecay_session_memory::context::CancellationToken, - > = HashMap::new(); - let connection_shutdown = tracedecay_session_memory::context::CancellationToken::new(); - let mut input_closed = false; - let mut peer_close_check: Option< - std::pin::Pin + Send + 'static>>, - > = None; - let timings_enabled = timings_override.unwrap_or_else(|| self.timings_enabled()); - - // Install the process listeners once. This same fused future is polled - // by idle reads, active read batches, and effect barriers, so shutdown - // cannot land in an iteration gap. - let external_shutdown_requested = async { - if listen_for_process_signals { - #[cfg(unix)] - { - #[allow(clippy::expect_used)] - let mut sigterm = - tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) - .expect("failed to register SIGTERM handler"); - tokio::select! { - _ = tokio::signal::ctrl_c() => {} - _ = sigterm.recv() => {} - } - } - #[cfg(not(unix))] - { - let _ = tokio::signal::ctrl_c().await; - } - } else if let Some(lifecycle) = request_lifecycle { - lifecycle.wait_for_draining().await; - } else { - std::future::pending::<()>().await; - } - }; - tokio::pin!(external_shutdown_requested); - - 'connection: loop { - if input_closed && pending_lines.is_empty() && active_reads.is_empty() { - break; - } - - let queued_ready = pending_lines.front().is_some_and(|queued| { - if active_reads.is_empty() { - return true; - } - if !queued.independent_read - || active_reads.len() >= self.context.max_concurrent_reads() - { - return false; - } - queued - .request_id - .as_ref() - .and_then(|id| { - application_surface_request_id(id, connection_route.memory_request_scope()) - }) - .is_none_or(|key| !active_cancellations.contains_key(&key)) - }); - - let line_from_queue = queued_ready; - let event = if queued_ready { - let Some(queued) = pending_lines.pop_front() else { - continue; - }; - ConnectionLoopEvent::Queued(queued.into_line()) - } else if active_reads.is_empty() { - let incoming = read_connection_line(transport); - tokio::pin!(incoming); - tokio::select! { - biased; - () = &mut external_shutdown_requested => ConnectionLoopEvent::Shutdown, - () = wait_for_peer_close(&mut peer_close_check), if input_closed => - ConnectionLoopEvent::PeerClosed, - result = &mut incoming, if !input_closed => - ConnectionLoopEvent::Incoming(result), - } - } else { - let can_read_more = - !input_closed && pending_lines.len() < MAX_PENDING_CANCELLABLE_REQUEST_LINES; - let incoming = read_connection_line(transport); - tokio::pin!(incoming); - tokio::select! { - biased; - () = &mut external_shutdown_requested => ConnectionLoopEvent::Shutdown, - () = wait_for_peer_close(&mut peer_close_check), if input_closed => - ConnectionLoopEvent::PeerClosed, - result = active_reads.join_next() => { - ConnectionLoopEvent::Completed(Box::new(result)) - }, - result = &mut incoming, if can_read_more => - ConnectionLoopEvent::Incoming(result), - } - }; - - let line = match event { - ConnectionLoopEvent::Queued(line) - | ConnectionLoopEvent::Incoming(Ok(Some(line))) => Some(line), - ConnectionLoopEvent::Incoming(Ok(None)) => { - input_closed = true; - peer_close_check = Some(Box::pin(transport.peer_fully_closed_after_eof())); - None - } - ConnectionLoopEvent::Incoming(Err(error)) => { - connection_shutdown.cancel(); - while active_reads.join_next().await.is_some() {} - if is_wire_oversized_io_error(&error) { - let _ = write_wire_oversized_rejection(transport, &error).await; - break; - } - self.shutdown_if(shutdown_on_exit).await; - return Err(error.into()); - } - ConnectionLoopEvent::Completed(completed) => { - let Some(completed) = *completed else { - continue; - }; - let mut completion = completed.map_err(|error| TraceDecayError::Config { - message: format!("MCP concurrent read task failed: {error}"), - })?; - if let Some(request_key) = completion.request_key.as_ref() { - active_cancellations.remove(request_key); - } - if completion.connection_closed { - connection_shutdown.cancel(); - while active_reads.join_next().await.is_some() {} - break; - } - match ConnectionResponseWriter::write(self, transport, &mut completion).await { - Ok(true) => {} - Ok(false) => { - connection_shutdown.cancel(); - while active_reads.join_next().await.is_some() {} - break; - } - Err(error) => { - tracing::error!(error = %error, "failed to write MCP response"); - if let Some((id, _)) = &completion.revocable_tool_call { - let _ = self.cancel_application_surface_request( - id, - &completion.connection_scope, - ); - } - connection_shutdown.cancel(); - while active_reads.join_next().await.is_some() {} - self.shutdown_if(shutdown_on_exit).await; - return Err(error.into()); - } - } - drop(completion); - if request_lifecycle.is_some_and(|lifecycle| !lifecycle.accepting()) { - connection_shutdown.cancel(); - while active_reads.join_next().await.is_some() {} - break; - } - None - } - ConnectionLoopEvent::Shutdown | ConnectionLoopEvent::PeerClosed => { - connection_shutdown.cancel(); - while active_reads.join_next().await.is_some() {} - break; - } - }; - - let Some(line) = line else { - continue; - }; - - let line = line.trim().to_string(); - if line.is_empty() { - continue; - } - - let parsed = hotpath::measure_block!( - "mcp.server.connection.decode", - JsonRpcRequest::decode(&line) - ); - if let Ok(notification) = &parsed - && matches!( - classify_mcp_method(¬ification.method), - McpMethod::Cancelled - ) - && let Some(id) = notification - .params - .as_ref() - .and_then(|params| params.get("requestId")) - { - let connection_scope = connection_route.memory_request_scope(); - if !self.cancel_application_surface_request(id, connection_scope) - && let Some(key) = application_surface_request_id(id, connection_scope) - { - if let Some(cancellation) = active_cancellations.get(&key) { - cancellation.cancel(); - } else if pending_cancellations.len() < MAX_PENDING_CANCELLABLE_REQUEST_LINES - && queued_cancellable_request_key(&pending_lines, id, connection_scope) - .is_some() - { - pending_cancellations.insert(key); - } - } - continue; - } - - if let Ok(request) = &parsed - && request_is_independent_read(request, self.context.as_ref()) - { - let request_key = request.id.as_ref().and_then(|id| { - application_surface_request_id(id, connection_route.memory_request_scope()) - }); - let duplicate_in_flight = request_key - .as_ref() - .is_some_and(|key| active_cancellations.contains_key(key)); - if !duplicate_in_flight - && active_reads.len() < self.context.max_concurrent_reads() - && (line_from_queue || pending_lines.is_empty()) - { - let request_activity = - request_lifecycle.and_then(McpConnectionLifecyclePort::try_enter); - if request_lifecycle.is_some() && request_activity.is_none() { - let mut completion = ConcurrentReadCompletion { - request_key, - _request_activity: request_activity, - revocable_tool_call: None, - response: request.id.clone().map(|id| { - JsonRpcResponse::error( - id, - ErrorCode::InternalError, - "TraceDecay daemon is draining for upgrade; retry the request" - .to_string(), - ) - }), - selected_response_lease: None, - connection_scope: connection_route.memory_request_scope().to_owned(), - connection_closed: false, - }; - ConnectionResponseWriter::write(self, transport, &mut completion).await?; - break; - } - let cancellation = tracedecay_session_memory::context::CancellationToken::new(); - if let Some(request_key) = request_key.as_ref() { - if pending_cancellations.remove(request_key) { - cancellation.cancel(); - } - active_cancellations.insert(request_key.clone(), cancellation.clone()); - } - let context = Arc::clone(&self.context); - let dispatch = dispatch_independent_read( - Arc::clone(self), - request.clone(), - timings_enabled, - connection_route.fork_for_connection_owned_read(), - request_activity, - cancellation, - connection_shutdown.clone(), - ); - active_reads - .spawn(async move { context.run_in_connection_admission(dispatch).await }); - continue; - } - } - - if !active_reads.is_empty() { - if pending_lines.len() >= MAX_PENDING_CANCELLABLE_REQUEST_LINES { - connection_shutdown.cancel(); - while active_reads.join_next().await.is_some() {} - break; - } - pending_lines.push_back(QueuedRequestLine::from_parsed( - line, - parsed.as_ref().ok(), - self.context.as_ref(), - )); - continue; - } - - let revocable_tool_call = parsed.as_ref().ok().and_then(|request| { - (request.method == "tools/call").then_some(())?; - let id = request.id.clone()?; - let tool_name = request.params.as_ref()?.get("name")?.as_str()?.to_owned(); - Some((id, tool_name)) - }); - let request_activity = - request_lifecycle.and_then(McpConnectionLifecyclePort::try_enter); - let rejecting_for_drain = request_lifecycle.is_some() && request_activity.is_none(); - let mut peer_closed = false; - - let response = if rejecting_for_drain { - parsed.as_ref().ok().and_then(|request| { - request.id.clone().map(|id| { - JsonRpcResponse::error( - id, - ErrorCode::InternalError, - "TraceDecay daemon is draining for upgrade; retry the request" - .to_string(), - ) - }) - }) - } else { - match parsed { - Ok(request) => { - let cancellable_tool_call = request.method == "tools/call" - && request - .params - .as_ref() - .and_then(|params| params.get("name")) - .and_then(Value::as_str) - .is_some_and(|tool_name| { - self.context.tool_supports_live_cancellation(tool_name) - }); - if cancellable_tool_call { - let (response, closed) = self - .handle_cancellable_application_request( - &request, - timings_enabled, - &mut connection_route, - transport, - &mut pending_lines, - &mut pending_cancellations, - external_shutdown_requested.as_mut(), - ) - .await?; - peer_closed = closed; - response - } else { - let (response, closed) = self - .handle_non_cancellable_application_request( - &request, - timings_enabled, - &mut connection_route, - transport, - &mut pending_lines, - external_shutdown_requested.as_mut(), - ) - .await?; - peer_closed = closed; - response - } - } - Err(error) => Some(error.into_response()), - } - }; - - let selected_response_lease = connection_route.take_selected_response_lease(); - if peer_closed { - drop(request_activity); - break; - } - let mut completion = ConcurrentReadCompletion { - request_key: None, - _request_activity: request_activity, - revocable_tool_call, - response, - selected_response_lease, - connection_scope: connection_route.memory_request_scope().to_owned(), - connection_closed: false, - }; - match ConnectionResponseWriter::write(self, transport, &mut completion).await { - Ok(true) => {} - Ok(false) => break 'connection, - Err(error) => { - tracing::error!(error = %error, "failed to write MCP response"); - if let Some((id, _)) = &completion.revocable_tool_call { - let _ = self - .cancel_application_surface_request(id, &completion.connection_scope); - } - self.shutdown_if(shutdown_on_exit).await; - return Err(error.into()); - } - } - drop(completion); - if rejecting_for_drain - || request_lifecycle.is_some_and(|lifecycle| !lifecycle.accepting()) - { - break; - } - } - - self.shutdown_if(shutdown_on_exit).await; - Ok(()) - } -} - #[cfg(test)] mod tests { use std::sync::Arc; use std::sync::atomic::{AtomicIsize, Ordering}; - use super::{QueuedRequestLine, await_route_with_cancellation}; + use super::await_route_with_cancellation; #[tokio::test] async fn route_resolution_is_abandoned_when_the_request_is_cancelled() { @@ -1262,26 +131,4 @@ mod tests { assert_eq!(routed, Some("selected-server")); } - - #[test] - fn queued_request_depth_is_released_on_dequeue_and_connection_drop() { - let queued = Arc::new(AtomicIsize::new(0)); - let mut pending = std::collections::VecDeque::new(); - pending.push_back(QueuedRequestLine::new_observed( - "first".to_owned(), - Arc::clone(&queued), - )); - pending.push_back(QueuedRequestLine::new_observed( - "second".to_owned(), - Arc::clone(&queued), - )); - assert_eq!(queued.load(Ordering::Acquire), 2); - - let first = pending.pop_front().expect("first queued line").into_line(); - assert_eq!(first, "first"); - assert_eq!(queued.load(Ordering::Acquire), 1); - - drop(pending); - assert_eq!(queued.load(Ordering::Acquire), 0); - } } diff --git a/crates/tracedecay-mcp/src/server/dispatch.rs b/crates/tracedecay-mcp/src/server/dispatch.rs index 29565cab97..5140c5a592 100644 --- a/crates/tracedecay-mcp/src/server/dispatch.rs +++ b/crates/tracedecay-mcp/src/server/dispatch.rs @@ -1,25 +1,17 @@ -//! The typed internal dispatch envelope shared by every MCP transport. +//! The typed internal dispatch envelope in front of the handler catalog. //! -//! Two transports reach the same handler catalog: the legacy line-oriented -//! JSON-RPC connection loop, which parses a [`JsonRpcRequest`] off the wire, -//! and the `rmcp` adapter, whose server callbacks are handed already-typed -//! request DTOs. Before this envelope existed the `rmcp` edge re-encoded each -//! typed DTO into a `serde_json::Value` and rebuilt a second, transport-neutral -//! [`JsonRpcRequest`] purely to reach dispatch, one full JSON tree built per -//! request, and for `tools/call` a second deep clone when the handler pulled -//! `arguments` back out of it. +//! The `rmcp` adapter's server callbacks are handed already-typed request +//! DTOs; custom notifications and in-process replay arrive as a raw +//! [`JsonRpcRequest`]. The envelope carries the request identity, the method, +//! its classification, and a method-specific payload that is *either* the raw +//! JSON params or the typed DTO, so no typed DTO is re-encoded into a JSON +//! tree just to reach dispatch. Internal dispatch reads the payload only +//! through the accessors below, each of which answers the same question on +//! both representations, so there is exactly one dispatch authority. //! -//! The envelope removes that bridge without forking the handler catalog. It -//! carries the request identity, the method, its classification, and a -//! method-specific payload that is *either* the raw wire params (legacy) or -//! the typed DTO (`rmcp`). Internal dispatch reads the payload only through -//! the accessors below, each of which answers the same question on both -//! representations, so there is exactly one dispatch authority and one place -//! where a new transport has to be taught anything. -//! -//! Response rendering stays at the wire edge: the legacy transport serializes -//! the handler's [`JsonRpcResponse`], and the `rmcp` adapter materializes it -//! into the typed result DTO its `ServerHandler` signature requires. +//! Response rendering stays at the wire edge: the `rmcp` adapter materializes +//! the handler's [`JsonRpcResponse`] into the typed result DTO its +//! `ServerHandler` signature requires. use crate::transport::JsonRpcRequest; use rmcp::model::{CallToolRequestParams, InitializeRequestParams, ReadResourceRequestParams}; @@ -30,9 +22,9 @@ use super::protocol::{McpMethod, classify_mcp_method}; /// The method-specific request payload, in whichever form its transport /// already owns. pub enum McpDispatchParams<'a> { - /// Raw JSON-RPC params, borrowed from the request the legacy transport - /// parsed. This is also how `rmcp` delivers hook-event and cancellation - /// notifications, which are custom (untyped) methods on that transport. + /// Raw JSON-RPC params, borrowed from a parsed request. This is how + /// `rmcp` delivers hook-event and cancellation notifications, which are + /// custom (untyped) methods on that transport. Raw(Option<&'a Value>), /// Typed `initialize` params from the `rmcp` server callback. Initialize(&'a InitializeRequestParams), @@ -64,11 +56,12 @@ pub struct McpDispatchRequest<'a> { } impl<'a> McpDispatchRequest<'a> { - /// Wraps a request exactly as the legacy JSON-RPC transport parsed it. + /// Wraps a raw JSON-RPC request whose params stay untyped JSON, such as a + /// custom notification. /// /// Nothing is copied but the (small) request identity, which dispatch /// already cloned before consuming. - pub fn from_legacy(request: &'a JsonRpcRequest) -> Self { + pub fn raw(request: &'a JsonRpcRequest) -> Self { Self::new( request.id.clone(), &request.method, @@ -202,9 +195,9 @@ impl McpDispatchParams<'_> { /// Whether a request may run concurrently with other in-flight reads on its /// connection. /// -/// One authority for both transports: the legacy loop and the `rmcp` adapter -/// classify the same method the same way, so a read that forks connection -/// state on one transport can never take the ordered write path on the other. +/// One authority for both payload forms: a raw and a typed request classify +/// the same method the same way, so a read that forks connection state in one +/// form can never take the ordered write path in the other. pub fn dispatch_is_independent_read( method: McpMethod, tool_name: Option<&str>, @@ -244,7 +237,7 @@ mod tests { use super::*; - fn legacy(method: &str, params: Value) -> JsonRpcRequest { + fn raw_request_for(method: &str, params: Value) -> JsonRpcRequest { JsonRpcRequest { jsonrpc: "2.0".to_owned(), id: Some(json!(1)), @@ -259,7 +252,7 @@ mod tests { ClientCapabilities::default(), Implementation::new("claude-code", "1.2.3"), ); - let raw_request = legacy( + let raw_request = raw_request_for( "initialize", json!({ "protocolVersion": "2024-11-05", @@ -268,7 +261,7 @@ mod tests { }), ); - let raw = McpDispatchRequest::from_legacy(&raw_request); + let raw = McpDispatchRequest::raw(&raw_request); let typed = McpDispatchRequest::typed( json!(1), "initialize", @@ -310,9 +303,9 @@ mod tests { #[test] fn typed_and_raw_resources_read_agree_on_the_target_uri() { let typed_params = ReadResourceRequestParams::new("tracedecay://schema"); - let raw_request = legacy("resources/read", json!({"uri": "tracedecay://schema"})); + let raw_request = raw_request_for("resources/read", json!({"uri": "tracedecay://schema"})); assert_eq!( - McpDispatchRequest::from_legacy(&raw_request).resource_uri(), + McpDispatchRequest::raw(&raw_request).resource_uri(), Some("tracedecay://schema"), ); assert_eq!( @@ -340,11 +333,11 @@ mod tests { #[test] fn read_concurrency_is_decided_identically_for_both_transports() { - let raw_request = legacy( + let raw_request = raw_request_for( "tools/call", json!({"name": "tracedecay_search", "arguments": {}}), ); - let raw = McpDispatchRequest::from_legacy(&raw_request); + let raw = McpDispatchRequest::raw(&raw_request); let typed = McpDispatchRequest::typed( json!(1), "tools/call", @@ -359,11 +352,11 @@ mod tests { }), ); - let raw_write = legacy( + let raw_write = raw_request_for( "tools/call", json!({"name": "tracedecay_str_replace", "arguments": {}}), ); - let raw_write = McpDispatchRequest::from_legacy(&raw_write); + let raw_write = McpDispatchRequest::raw(&raw_write); let typed_write = McpDispatchRequest::typed( json!(1), "tools/call", diff --git a/crates/tracedecay-mcp/src/server/mod.rs b/crates/tracedecay-mcp/src/server/mod.rs index cdf4c8b431..83b1ecb33c 100644 --- a/crates/tracedecay-mcp/src/server/mod.rs +++ b/crates/tracedecay-mcp/src/server/mod.rs @@ -13,8 +13,7 @@ pub use crate::lifecycle::{ McpBackgroundTaskOwner, ProjectServerResponseLifecycle, StartupCatchUpMachineV1, }; pub use connection::{ - McpConnectionContext, McpConnectionServer, McpConnectionState, McpResponseLease, - await_route_with_cancellation, + McpConnectionContext, McpConnectionState, McpResponseLease, await_route_with_cancellation, }; pub use dispatch::{ McpDispatchParams, McpDispatchRequest, ToolCallParams, dispatch_is_independent_read, @@ -33,8 +32,8 @@ pub use read_coalescing::{ }; pub use rmcp::{ RmcpConnectionAdapter, RmcpInitializeResponseDecorator, RmcpSelectedProjectResponseAuthority, - RmcpWorkDeliverySettlement, await_dispatch_with_cancellation, project_server_retired_error, - rmcp_response_result, + RmcpWorkDeliverySettlement, attach_stateless_request_context, await_dispatch_with_cancellation, + opens_rmcp_session, project_server_retired_error, rmcp_response_result, }; pub use settlement::{ ApplicationCancellationRegistration, DispatchControl, DispatchControlRequest, DispatchFailure, diff --git a/crates/tracedecay-mcp/src/server/protocol.rs b/crates/tracedecay-mcp/src/server/protocol.rs index 0a7f5985b0..afc3ad8f4a 100644 --- a/crates/tracedecay-mcp/src/server/protocol.rs +++ b/crates/tracedecay-mcp/src/server/protocol.rs @@ -11,7 +11,7 @@ pub enum McpMethod { ToolsCall, ResourcesList, ResourcesRead, - /// `ping` / `logging/setLevel`, acknowledged with an empty result. + /// `ping`, acknowledged with an empty result. TrivialAck, /// The daemon's internal hook-event notification. HookEvent, @@ -31,7 +31,7 @@ pub fn classify_mcp_method(method: &str) -> McpMethod { "resources/list" => McpMethod::ResourcesList, "resources/read" => McpMethod::ResourcesRead, "notifications/cancelled" => McpMethod::Cancelled, - "ping" | "logging/setLevel" => McpMethod::TrivialAck, + "ping" => McpMethod::TrivialAck, _ => McpMethod::Unknown, } } @@ -45,8 +45,7 @@ pub fn initialize_result(version: &str, instructions: &str) -> Value { "tools": { "listChanged": true }, - "resources": {}, - "logging": {} + "resources": {} }, "serverInfo": { "name": "tracedecay", diff --git a/crates/tracedecay-mcp/src/server/read_coalescing.rs b/crates/tracedecay-mcp/src/server/read_coalescing.rs index f1023127ee..ba630316ae 100644 --- a/crates/tracedecay-mcp/src/server/read_coalescing.rs +++ b/crates/tracedecay-mcp/src/server/read_coalescing.rs @@ -328,7 +328,7 @@ mod tests { let coalescer = IdenticalReadCoalescer::default(); let leader = match coalescer.claim( "graph-main", - "tracedecay_outline", + "tracedecay_source_outline", &json!({"file": "src/lib.rs"}), None, ) { @@ -337,7 +337,7 @@ mod tests { }; let follower = match coalescer.claim( "graph-main", - "tracedecay_outline", + "tracedecay_source_outline", &json!({"file": "src/lib.rs"}), None, ) { @@ -350,7 +350,7 @@ mod tests { assert!(matches!( coalescer.claim( "graph-main", - "tracedecay_outline", + "tracedecay_source_outline", &json!({"file": "src/lib.rs"}), None ), @@ -363,7 +363,7 @@ mod tests { let coalescer = IdenticalReadCoalescer::default(); let _leader = match coalescer.claim( "graph-main", - "tracedecay_outline", + "tracedecay_source_outline", &json!({"file": "src/lib.rs"}), None, ) { @@ -372,7 +372,7 @@ mod tests { }; let follower = match coalescer.claim( "graph-main", - "tracedecay_outline", + "tracedecay_source_outline", &json!({"file": "src/lib.rs"}), None, ) { @@ -418,7 +418,7 @@ mod tests { ); } assert!(tool_allows_identical_read_coalescing( - "tracedecay_outline", + "tracedecay_source_outline", |_| true, )); assert!(!tool_allows_identical_read_coalescing( diff --git a/crates/tracedecay-mcp/src/server/rmcp.rs b/crates/tracedecay-mcp/src/server/rmcp.rs index fe331a9403..5da70ef729 100644 --- a/crates/tracedecay-mcp/src/server/rmcp.rs +++ b/crates/tracedecay-mcp/src/server/rmcp.rs @@ -8,10 +8,11 @@ use std::sync::Arc; use rmcp::model::{ - CallToolRequestParams, CallToolResponse, CallToolResult, CustomNotification, ErrorCode, - ErrorData, Implementation, InitializeRequestParams, InitializeResult, ListResourcesResult, - ListToolsResult, MetaObject, ReadResourceRequestParams, ReadResourceResponse, - ReadResourceResult, ServerCapabilities, ServerConfig, + CallToolRequestParams, CallToolResponse, CallToolResult, ClientCapabilities, + CustomNotification, CustomRequest, CustomResult, ErrorCode, ErrorData, Implementation, + InitializeRequestParams, InitializeResult, ListResourcesResult, ListToolsResult, MetaObject, + ProtocolVersion, ReadResourceRequestParams, ReadResourceResponse, ReadResourceResult, + RequestMetaObject, ServerCapabilities, ServerConfig, }; use rmcp::service::{NotificationContext, RequestContext}; use rmcp::{RoleServer, ServerHandler}; @@ -116,7 +117,7 @@ impl RmcpSelectedProjectResponseAuthority { } } -/// Allows daemon routing to enrich the legacy `initialize` response without +/// Allows daemon routing to enrich the `initialize` response without /// coupling this MCP module to daemon route types. pub type RmcpInitializeResponseDecorator = Arc; @@ -223,8 +224,8 @@ impl RmcpWorkDeliverySettlement { /// When cancel wins and `cancel_registered_request` finds a live registration, /// the sticky/worker path owns settlement, so this keeps awaiting `handling`. /// When cancel wins and the request is not registered yet, pass -/// `cancellation_registered` so this waits for the same notify the legacy -/// connection uses instead of dropping `handling` during route resolution. +/// `cancellation_registered` so this waits for route resolution to register +/// the request instead of dropping `handling` mid-route. /// Only a cancel that can never register (no notify channel) abandons. pub async fn await_dispatch_with_cancellation( handling: F, @@ -249,7 +250,7 @@ where } let notify = cancellation_registered?; // Cancel raced route resolution: keep polling handling while waiting for - // prepare_dispatch_control to register, same as the legacy connection. + // prepare_dispatch_control to register. loop { let registered = notify.notified(); tokio::pin!(registered); @@ -351,7 +352,7 @@ where async fn dispatch( &self, context: RequestContext, - method: &'static str, + method: &str, params: McpDispatchParams<'_>, ) -> Result { let queued_at = std::time::Instant::now(); @@ -384,7 +385,7 @@ where async fn dispatch_admitted( &self, context: RequestContext, - method: &'static str, + method: &str, params: McpDispatchParams<'_>, ) -> Result { // The wire identity is the one value internal dispatch genuinely keys @@ -423,13 +424,12 @@ where connection: &mut C::Connection, ) -> Result { let pre_cancelled = request_cancellation.is_cancelled(); - let dispatch_cancellation = tracedecay_session_memory::context::CancellationToken::new(); + let dispatch_cancellation = tracedecay_runtime_core::cancellation::CancellationToken::new(); if pre_cancelled { dispatch_cancellation.cancel(); } - // The legacy MCP route already erases this shared dispatch authority - // before awaiting it. Keep the typed RMCP route at the same ownership - // boundary: the cancellation combinator otherwise stores the complete + // Erase the shared dispatch authority before awaiting it: the + // cancellation combinator otherwise stores the complete // catalog-dispatch future inline in rmcp's generated request future. let handling = self.context.dispatch( request, @@ -501,10 +501,10 @@ where let _ = self .context .dispatch( - McpDispatchRequest::from_legacy(&request), + McpDispatchRequest::raw(&request), self.timings_enabled, &mut connection, - tracedecay_session_memory::context::CancellationToken::new(), + tracedecay_runtime_core::cancellation::CancellationToken::new(), ) .await; } @@ -540,12 +540,68 @@ where GuardedHandshakeTransport { inner: transport, handshake_settled: false, + pending_ping_answer: None, }, ) .await } } +/// Whether a daemon connection's first request opens an `rmcp` session: an +/// `initialize`, or a request carrying the SEP-2575 per-request client context +/// `rmcp` serves without one. +pub fn opens_rmcp_session(request: &JsonRpcRequest) -> bool { + request.method == "initialize" + || (request.id.is_some() + && request + .params + .as_ref() + .and_then(|params| params.get("_meta")) + .and_then(Value::as_object) + .is_some_and(|meta| { + RequestMetaObject::DRAFT_REQUIRED_KEYS + .iter() + .all(|key| meta.contains_key(*key)) + })) +} + +/// Attaches this client's SEP-2575 per-request context to `params._meta` so a +/// request sent on its own daemon connection needs no `initialize` session. +/// +/// Notifications, `initialize`, and non-object params or `_meta` are left +/// unchanged; `_meta` entries the caller already set win. Returns whether the +/// request changed. +pub fn attach_stateless_request_context(request: &mut JsonRpcRequest) -> bool { + if request.id.is_none() || request.method == "initialize" { + return false; + } + let Some(params) = request + .params + .get_or_insert_with(|| Value::Object(serde_json::Map::new())) + .as_object_mut() + else { + return false; + }; + let Some(meta) = params + .entry("_meta") + .or_insert_with(|| Value::Object(serde_json::Map::new())) + .as_object_mut() + else { + return false; + }; + let mut context = RequestMetaObject::new(); + context.set_protocol_version(ProtocolVersion::LATEST); + context.set_client_capabilities(ClientCapabilities::default()); + let mut attached = false; + for (key, value) in context.0.0 { + if let serde_json::map::Entry::Vacant(entry) = meta.entry(key) { + entry.insert(value); + attached = true; + } + } + attached +} + pub fn rmcp_response_result( response: JsonRpcResponse, ) -> Result { @@ -576,14 +632,32 @@ const MALFORMED_INITIALIZE_MESSAGE: &str = "initialize params are missing or mal /// returning a matching response", a transport mystery for what is a /// definitive protocol answer, exactly like the unparseable-handshake and /// rejected-auth refusals the daemon already writes before closing. -struct GuardedHandshakeTransport { +/// +/// The guard also answers every `ping` itself. A connection whose first +/// request carried SEP-2575 `_meta` instead of `initialize` is an inline +/// lifecycle peer, and `rmcp`'s blanket `Service` impl refuses `ping` there +/// with method-not-found (or invalid params without `_meta`) before +/// `ServerHandler::ping` runs (rmcp 3.4, `handler/server.rs`). MCP requires +/// an empty result to a ping at any time; hosts use it as a liveness probe. +struct GuardedHandshakeTransport> { inner: T, /// Set once a request that ends `rmcp`'s pre-initialize loop is forwarded. /// After that the guard is inert: a later stray `initialize` is an ordinary - /// request the adapter answers with a typed error of its own. + /// request the adapter answers with a typed error of its own. Before it, + /// non-request messages are dropped: `rmcp`'s pre-initialize loop fails on + /// them, so a pipelined `notifications/initialized` after a refused + /// initialize would end the connection the corrected handshake needs. + /// JSON-RPC forbids answering them, and nothing is in flight yet. handshake_settled: bool, + /// `rmcp` polls `receive` inside `select!`, so a ping answer is written + /// from here rather than from a receive future that may be dropped + /// mid-write. + pending_ping_answer: Option>, } +type PendingPingAnswer = + std::pin::Pin> + Send>>; + impl rmcp::transport::Transport for GuardedHandshakeTransport where T: rmcp::transport::Transport + Send + 'static, @@ -604,12 +678,28 @@ where { async move { loop { + if let Some(answer) = self.pending_ping_answer.as_mut() { + let sent = answer.await; + self.pending_ping_answer = None; + sent.ok()?; + } let message = self.inner.receive().await?; + if let rmcp::model::ClientJsonRpcMessage::Request(request) = &message + && matches!(request.request, rmcp::model::ClientRequest::PingRequest(_)) + { + self.pending_ping_answer = Some(Box::pin(self.inner.send( + rmcp::model::ServerJsonRpcMessage::response( + rmcp::model::ServerResult::EmptyResult(rmcp::model::EmptyResult {}), + request.id.clone(), + ), + ))); + continue; + } if self.handshake_settled { return Some(message); } let rmcp::model::ClientJsonRpcMessage::Request(request) = &message else { - return Some(message); + continue; }; let malformed_initialize = request.request.method() == "initialize" && !matches!( @@ -617,10 +707,7 @@ where rmcp::model::ClientRequest::InitializeRequest(_) ); if !malformed_initialize { - // `rmcp` answers a pre-initialize ping in place and keeps - // waiting; any other request ends its handshake loop. - self.handshake_settled = - !matches!(request.request, rmcp::model::ClientRequest::PingRequest(_)); + self.handshake_settled = true; return Some(message); } let refusal = rmcp::model::ServerJsonRpcMessage::error( @@ -689,15 +776,40 @@ where #[hotpath::skip] async fn call_tool( &self, - request: CallToolRequestParams, + mut request: CallToolRequestParams, context: RequestContext, ) -> Result { + // `rmcp` moves the wire `params._meta` into the request context; the + // caller deadline is read from the typed params. + if request.meta.is_none() && !context.meta.is_empty() { + request.meta = Some(context.meta.clone()); + } let started = (self.timings_enabled || self.context.timings_enabled()).then(std::time::Instant::now); - let mut result = rmcp_response_result::( - self.dispatch(context, "tools/call", McpDispatchParams::ToolsCall(request)) - .await?, - )?; + let mut response = self + .dispatch(context, "tools/call", McpDispatchParams::ToolsCall(request)) + .await?; + // `CallToolResult` has no extension members, so the typed problem the + // dispatcher attaches beside `content` travels as structured content; + // otherwise a markdown-format refusal would reach clients only as prose. + let problem = response + .result + .as_mut() + .and_then(Value::as_object_mut) + .and_then(|result| result.remove("problem")); + let mut result = rmcp_response_result::(response)?; + if let Some(problem) = problem { + match result + .structured_content + .as_mut() + .and_then(Value::as_object_mut) + { + Some(structured) => { + structured.insert("problem".to_owned(), problem); + } + None => result.structured_content = Some(json!({ "problem": problem })), + } + } if let Some(started) = started { result .meta @@ -756,6 +868,36 @@ where self.dispatch_notification(notification.method, notification.params) .await; } + + /// A hook event arrives as a stateless request on its own daemon + /// connection; it is dispatched exactly like the notification form. + /// Any other custom request is an unknown method or a known one whose + /// params did not fit its typed DTO; the dispatch authority answers it + /// (method not found, invalid params) and accounts for it. + #[hotpath::skip] + async fn on_custom_request( + &self, + request: CustomRequest, + context: RequestContext, + ) -> Result { + if request.method == tracedecay_hooks::core_events::HOOK_EVENT_METHOD { + self.dispatch_notification(request.method, request.params) + .await; + return Ok(CustomResult::new(json!({}))); + } + // `rmcp` lifts `params._meta` into the request context; the object it + // leaves empty is a request that carried no params of its own (the + // stdio proxy creates `params` only to hold that context). + let params = request + .params + .as_ref() + .filter(|params| params.as_object().is_none_or(|object| !object.is_empty())); + rmcp_response_result( + self.dispatch(context, &request.method, McpDispatchParams::Raw(params)) + .await?, + ) + .map(CustomResult::new) + } } fn rmcp_error(error: JsonRpcError) -> ErrorData { @@ -807,9 +949,9 @@ mod tests { #[tokio::test] async fn rmcp_cancel_during_route_resolution_preserves_dispatch_until_registration() { // Cancel can win while route resolution still awaits, before - // prepare_dispatch_control registers. Legacy waits on - // cancellation_registered; RMCP must too so sticky sampling and the - // selected target still settle. + // prepare_dispatch_control registers. RMCP waits on + // cancellation_registered so sticky sampling and the selected target + // still settle. let registration = Arc::new(tokio::sync::Notify::new()); let registered = Arc::new(AtomicBool::new(false)); let cancel_attempts = Arc::new(AtomicUsize::new(0)); diff --git a/crates/tracedecay-mcp/src/server/settlement.rs b/crates/tracedecay-mcp/src/server/settlement.rs index e57b4a1524..ef53afd359 100644 --- a/crates/tracedecay-mcp/src/server/settlement.rs +++ b/crates/tracedecay-mcp/src/server/settlement.rs @@ -180,8 +180,6 @@ pub struct RetainedDispatchRegistry { state: RetainedDispatchStateMutex, #[cfg(any(test, feature = "test-transport"))] retained_spawn_count: AtomicUsize, - #[cfg(any(test, feature = "test-transport"))] - connection_owned_count: AtomicUsize, } impl RetainedDispatchRegistry { @@ -203,8 +201,6 @@ impl RetainedDispatchRegistry { ), #[cfg(any(test, feature = "test-transport"))] retained_spawn_count: AtomicUsize::new(0), - #[cfg(any(test, feature = "test-transport"))] - connection_owned_count: AtomicUsize::new(0), } } @@ -305,12 +301,6 @@ impl RetainedDispatchRegistry { self.retained_spawn_count.load(Ordering::Acquire) } - #[cfg(feature = "test-transport")] - #[doc(hidden)] - pub fn connection_owned_count_for_test(&self) -> usize { - self.connection_owned_count.load(Ordering::Acquire) - } - #[cfg(test)] fn active_slot_count_for_test(&self) -> usize { self.active_slots.load(Ordering::Acquire) @@ -602,87 +592,6 @@ impl DispatchControl { self.cancellation.clone() } - pub fn permits_connection_owned_execution(&self) -> bool { - !self.carries_effect && !self.canonical_effect_settlement - } - - #[hotpath::measure(label = "mcp.server.dispatch.settlement", future = true)] - pub async fn run_connection_owned( - &self, - registry: &RetainedDispatchRegistry, - future: F, - ) -> RetainedDispatchOutcome - where - F: Future> + Send, - { - let cancelled_before_admission = self.cancellation.is_cancelled(); - if cancelled_before_admission && !self.live_cancellable { - return RetainedDispatchOutcome::failed(dispatch_cancelled_error( - &self.tool_name, - DispatchSettlement::NotStarted, - self.carries_effect, - )); - } - if tokio::time::Instant::now() >= self.deadline_at { - let _ = self - .cancellation - .cancel(tracedecay_contracts::clock::now_micros()); - return RetainedDispatchOutcome::failed(dispatch_deadline_error( - &self.tool_name, - DispatchSettlement::NotStarted, - self.carries_effect, - )); - } - - let _capacity_lease = match registry.acquire_capacity() { - Ok(lease) => lease, - Err(error) => return RetainedDispatchOutcome::failed(error), - }; - #[cfg(any(test, feature = "test-transport"))] - registry - .connection_owned_count - .fetch_add(1, Ordering::AcqRel); - let settlement = Arc::new(DispatchExecutionSettlement::not_started()); - settlement.mark_settling(); - let deadline = tokio::time::sleep_until(self.deadline_at); - let cancellation = - tracedecay_daemon_protocol::wait_for_cancellation(self.cancellation.clone()); - tokio::pin!(future); - tokio::pin!(deadline); - tokio::pin!(cancellation); - - let outcome = tokio::select! { - biased; - () = &mut cancellation, if self.live_cancellable => { - Err(DispatchFailure::new(dispatch_cancelled_error( - &self.tool_name, - if cancelled_before_admission { - DispatchSettlement::NotStarted - } else { - settlement.snapshot() - }, - self.carries_effect, - ))) - } - () = &mut deadline => { - let _ = self - .cancellation - .cancel(tracedecay_contracts::clock::now_micros()); - Err(DispatchFailure::new(dispatch_deadline_error( - &self.tool_name, - settlement.snapshot(), - self.carries_effect, - ))) - } - output = &mut future => output.map_err(DispatchFailure::new), - }; - settlement.mark_joined(); - RetainedDispatchOutcome { - result: outcome, - settlement, - } - } - #[hotpath::measure(label = "mcp.server.dispatch.settlement", future = true)] pub async fn run_retained( &self, @@ -1037,7 +946,7 @@ mod tests { let worker_flag = Arc::clone(&worker_ran); let outcome = control .run_retained(®istry, async move { - worker_flag.store(true, std::sync::atomic::Ordering::Release); + worker_flag.store(true, Ordering::Release); Ok::<_, tracedecay_domain::errors::TraceDecayError>("authority settled") }) .await; @@ -1056,7 +965,7 @@ mod tests { ); registry.shutdown().await; assert!( - worker_ran.load(std::sync::atomic::Ordering::Acquire), + worker_ran.load(Ordering::Acquire), "the admitted worker must run so the invocation authority settles it" ); } @@ -1113,7 +1022,7 @@ mod tests { let worker_flag = Arc::clone(&worker_ran); let outcome = control .run_retained(®istry, async move { - worker_flag.store(true, std::sync::atomic::Ordering::Release); + worker_flag.store(true, Ordering::Release); Ok::<_, tracedecay_domain::errors::TraceDecayError>("never admitted") }) .await; @@ -1127,7 +1036,7 @@ mod tests { ); registry.shutdown().await; assert!( - !worker_ran.load(std::sync::atomic::Ordering::Acquire), + !worker_ran.load(Ordering::Acquire), "no settlement authority exists, so the worker must never be admitted" ); } @@ -1189,12 +1098,12 @@ mod tests { } #[tokio::test] - async fn inline_read_capacity_refuses_the_next_retained_effect() { + async fn retained_read_capacity_refuses_the_next_retained_effect() { let registry = Arc::new(RetainedDispatchRegistry::new_with_capacity_for_test(1)); let read_control = dispatch_control( "tracedecay_status", deadline_after(std::time::Duration::from_mins(1)), - tracedecay_contracts::CancellationSignal::active("capacity.inline-read") + tracedecay_contracts::CancellationSignal::active("capacity.retained-read") .expect("read cancellation"), ) .expect("read control"); @@ -1205,7 +1114,7 @@ mod tests { let read_registry = Arc::clone(®istry); let read = tokio::spawn(async move { read_control - .run_connection_owned(&read_registry, async move { + .run_retained(&read_registry, async move { started.notify_one(); release.notified().await; Ok::<_, tracedecay_domain::errors::TraceDecayError>("read") @@ -1238,155 +1147,7 @@ mod tests { read_release.notify_one(); assert_eq!(read.await.expect("join read").result.expect("read"), "read"); - assert_eq!(registry.active_slot_count_for_test(), 0); registry.shutdown().await; - } - - #[tokio::test] - async fn retained_effect_capacity_refuses_the_next_inline_read() { - let registry = Arc::new(RetainedDispatchRegistry::new_with_capacity_for_test(1)); - let effect_control = dispatch_control( - "tracedecay_configuration_set", - deadline_after(std::time::Duration::from_mins(1)), - tracedecay_contracts::CancellationSignal::active("capacity.retained-owner") - .expect("effect cancellation"), - ) - .expect("effect control"); - let effect_started = Arc::new(tokio::sync::Notify::new()); - let effect_release = Arc::new(tokio::sync::Notify::new()); - let started = Arc::clone(&effect_started); - let release = Arc::clone(&effect_release); - let effect_registry = Arc::clone(®istry); - let effect = tokio::spawn(async move { - effect_control - .run_retained(&effect_registry, async move { - started.notify_one(); - release.notified().await; - Ok::<_, tracedecay_domain::errors::TraceDecayError>("effect") - }) - .await - }); - effect_started.notified().await; - assert_eq!(registry.active_slot_count_for_test(), 1); - - let read_control = dispatch_control( - "tracedecay_status", - deadline_after(std::time::Duration::from_mins(1)), - tracedecay_contracts::CancellationSignal::active("capacity.inline-refused") - .expect("read cancellation"), - ) - .expect("read control"); - let refused = read_control - .run_connection_owned(®istry, async { - Ok::<_, tracedecay_domain::errors::TraceDecayError>("read") - }) - .await; - assert_eq!( - refused - .result - .expect_err("inline read must be refused") - .project_route_context() - .map(|context| context.0), - Some("tool_dispatch_saturated") - ); - - effect_release.notify_one(); - assert_eq!( - effect.await.expect("join effect").result.expect("effect"), - "effect" - ); assert_eq!(registry.active_slot_count_for_test(), 0); - registry.shutdown().await; - } - - struct FutureDropObserver(Arc); - - impl Drop for FutureDropObserver { - fn drop(&mut self) { - self.0.store(true, Ordering::Release); - } - } - - #[tokio::test] - async fn connection_owned_cancellation_drops_the_read_and_joins_settlement() { - let registry = RetainedDispatchRegistry::new(); - let cancellation = tracedecay_contracts::CancellationSignal::active("inline.cancel-drop") - .expect("cancellation"); - let control = dispatch_control( - "tracedecay_search", - deadline_after(std::time::Duration::from_mins(1)), - cancellation.clone(), - ) - .expect("control"); - let started = Arc::new(tokio::sync::Notify::new()); - let entered = Arc::clone(&started); - let dropped = Arc::new(std::sync::atomic::AtomicBool::new(false)); - let drop_observer = Arc::clone(&dropped); - let dispatch = async move { - let _drop_observer = FutureDropObserver(drop_observer); - entered.notify_one(); - std::future::pending::>().await - }; - let runner = - tokio::spawn(async move { control.run_connection_owned(®istry, dispatch).await }); - - started.notified().await; - assert!(cancellation.cancel(tracedecay_contracts::clock::now_micros())); - let cancelled = runner.await.expect("join inline dispatch"); - assert_eq!( - cancelled - .result - .as_ref() - .expect_err("cancellation must win") - .project_route_context() - .map(|context| context.0), - Some("tool_dispatch_cancelled") - ); - assert_eq!(cancelled.settlement(), DispatchSettlement::Joined); - assert!( - dropped.load(Ordering::Acquire), - "connection-owned reads have no post-cancel owner" - ); - } - - #[tokio::test(start_paused = true)] - async fn connection_owned_deadline_drops_the_read_and_joins_settlement() { - let registry = RetainedDispatchRegistry::new(); - let control = dispatch_control( - "tracedecay_status", - deadline_after(std::time::Duration::from_secs(1)), - tracedecay_contracts::CancellationSignal::active("inline.deadline-drop") - .expect("cancellation"), - ) - .expect("control"); - let started = Arc::new(tokio::sync::Notify::new()); - let entered = Arc::clone(&started); - let dropped = Arc::new(std::sync::atomic::AtomicBool::new(false)); - let drop_observer = Arc::clone(&dropped); - let dispatch = async move { - let _drop_observer = FutureDropObserver(drop_observer); - entered.notify_one(); - std::future::pending::>().await - }; - let runner = - tokio::spawn(async move { control.run_connection_owned(®istry, dispatch).await }); - - started.notified().await; - tokio::time::advance(std::time::Duration::from_secs(1)).await; - let timed_out = runner.await.expect("join inline dispatch"); - assert_eq!( - timed_out - .result - .as_ref() - .expect_err("deadline must win") - .project_route_context() - .map(|context| context.0), - Some("tool_dispatch_deadline_exceeded") - ); - assert_eq!(timed_out.settlement(), DispatchSettlement::Joined); - assert!( - dropped.load(Ordering::Acquire), - "connection-owned reads have no post-deadline owner" - ); } } diff --git a/crates/tracedecay-mcp/src/tool_analytics.rs b/crates/tracedecay-mcp/src/tool_analytics.rs index b201e91006..cde60233fb 100644 --- a/crates/tracedecay-mcp/src/tool_analytics.rs +++ b/crates/tracedecay-mcp/src/tool_analytics.rs @@ -267,7 +267,8 @@ mod tests { use serde_json::json; - use crate::hook_events::{HookAgent, HookEvent, HookEventKind}; + use crate::hook_events::{HookEvent, HookEventKind}; + use tracedecay_domain::HostIntegrationIdV1; use tracedecay_hooks::core_events::HookRouteMetadata; use super::{ @@ -278,7 +279,7 @@ mod tests { #[test] fn hook_route_analytics_event_preserves_protected_ids_and_omits_payloads() { let event = HookEvent { - agent: HookAgent::Codex, + agent: HostIntegrationIdV1::Codex, kind: HookEventKind::Shell, rel_paths: Vec::new(), had_command: true, @@ -337,7 +338,7 @@ mod tests { #[test] fn hook_route_idempotency_key_distinguishes_durable_admissions() { let event = HookEvent { - agent: HookAgent::Codex, + agent: HostIntegrationIdV1::Codex, kind: HookEventKind::Shell, rel_paths: Vec::new(), had_command: false, diff --git a/crates/tracedecay-mcp/src/tool_context.rs b/crates/tracedecay-mcp/src/tool_context.rs index 081bcf114b..8504c18554 100644 --- a/crates/tracedecay-mcp/src/tool_context.rs +++ b/crates/tracedecay-mcp/src/tool_context.rs @@ -929,18 +929,16 @@ pub(crate) mod tests { primary_alias: root.to_path_buf(), }, store_kind: tracedecay_runtime_core::storage::StoreKind::CodeProject, - storage_mode: tracedecay_runtime_core::storage::StorageMode::ProjectLocal, + storage_mode: tracedecay_runtime_core::storage::StorageMode::ProfileSharded, project_root: root.to_path_buf(), data_root: root.join(".tracedecay"), graph_db_path: root.join("graph.db"), - config_path: root.join("config.toml"), branch_meta_path: root.join("branch-meta.json"), sessions_db_path: root.join("sessions.db"), response_handle_root: root.join("handles"), lcm_payload_root: root.join("lcm"), dashboard_root: root.join("dashboard"), manifest_path: None, - dirty_path: root.join("dirty"), sync_lock_path: root.join("sync.lock"), branch_add_lock_path: root.join("branch-add.lock"), } diff --git a/crates/tracedecay-mcp/src/tools/binding.rs b/crates/tracedecay-mcp/src/tools/binding.rs index 6752333ada..81b778edcf 100644 --- a/crates/tracedecay-mcp/src/tools/binding.rs +++ b/crates/tracedecay-mcp/src/tools/binding.rs @@ -6,9 +6,9 @@ //! projected from its executable registry instead, because that registry owns //! its complete mounted operation set and lifecycle contracts. //! -//! Canonical application tools are projected from their executable registry -//! and have no handwritten rows here. `group` is `None` only for retained -//! tools whose predicate remains their dispatch authority. +//! Canonical application tools are projected from their executable registry. +//! Their rows here carry only registered-project selector access, with `group` +//! `None`; dispatch derives from the operation identity. use std::collections::{HashMap, HashSet}; use std::sync::LazyLock; @@ -52,9 +52,7 @@ pub enum McpToolDispatchGroup { Admin, Analysis, Git, - Edit, Health, - RetainedApplication, Memory, SessionWorkflow, Work, @@ -91,15 +89,13 @@ pub fn tool_branch_sensitivity(tool_name: &str) -> BranchSensitivity { | McpToolDispatchGroup::Admin | McpToolDispatchGroup::Analysis | McpToolDispatchGroup::Git - | McpToolDispatchGroup::Edit | McpToolDispatchGroup::Health | McpToolDispatchGroup::MultiRoot | McpToolDispatchGroup::ApplicationSurface | McpToolDispatchGroup::SessionWorkflow, ) => BranchSensitivity::Sensitive, Some( - McpToolDispatchGroup::RetainedApplication - | McpToolDispatchGroup::Memory + McpToolDispatchGroup::Memory | McpToolDispatchGroup::Work | McpToolDispatchGroup::Workflow, ) => BranchSensitivity::Independent, @@ -136,6 +132,10 @@ fn application_surface_branch_sensitivity( NativeIntegrationWorktreeRemove, ObservatoryRead, QualifiedName, SessionLookup, SourceBody, SourceLines, SourceOutline, StorageStatus, TestResults, }; + use ApplicationSurfaceOperation::{ + AstGrepRewrite, InsertAt, InsertAtSymbol, MoveSymbol, MultiStrReplace, RenameSymbol, + ReplaceSymbol, SourceEditReconcile, SourceEditRollback, StrReplace, + }; match operation { // Mixed ApplicationSurface group: these operations read configuration, // host-integration lifecycle, session identity, store identity, or @@ -169,7 +169,35 @@ fn application_surface_branch_sensitivity( | NativeIntegrationApprove | NativeIntegrationApply | NativeIntegrationStatus - | NativeIntegrationCancel => BranchSensitivity::Independent, + | NativeIntegrationCancel + // Retained memory, session, and workflow authority. + | ApplicationSurfaceOperation::FactStoreCurate + | ApplicationSurfaceOperation::FactStoreAdd + | ApplicationSurfaceOperation::FactStoreSearch + | ApplicationSurfaceOperation::FactStoreProbe + | ApplicationSurfaceOperation::FactStoreRelated + | ApplicationSurfaceOperation::FactStoreReason + | ApplicationSurfaceOperation::FactStoreContradict + | ApplicationSurfaceOperation::FactStoreGet + | ApplicationSurfaceOperation::FactStoreUpdate + | ApplicationSurfaceOperation::FactStoreRemove + | ApplicationSurfaceOperation::FactStoreSupersede + | ApplicationSurfaceOperation::FactStoreList + | ApplicationSurfaceOperation::FactFeedback + | ApplicationSurfaceOperation::MemoryStatus + | ApplicationSurfaceOperation::SessionRefreshStatus + | ApplicationSurfaceOperation::SessionRefreshCancel + | ApplicationSurfaceOperation::SessionRefreshBegin + | ApplicationSurfaceOperation::MessageSearch + | ApplicationSurfaceOperation::SessionsFor + | ApplicationSurfaceOperation::Workflows + | ApplicationSurfaceOperation::LcmStatus + | ApplicationSurfaceOperation::LcmDoctor + | ApplicationSurfaceOperation::LcmLoadSession + | ApplicationSurfaceOperation::LcmGrep + | ApplicationSurfaceOperation::LcmDescribe + | ApplicationSurfaceOperation::LcmExpand + | ApplicationSurfaceOperation::LcmExpandQuery => BranchSensitivity::Independent, // Mixed ApplicationSurface group: git walks, worktree inventory, stack // snapshots, code-graph reads, source-file bodies, health, diagnostics, // and post-edit feedback all depend on the current checkout or graph. @@ -216,9 +244,28 @@ fn application_surface_branch_sensitivity( | SourceBody | SourceOutline | ModuleApi + | ApplicationSurfaceOperation::Context + | ApplicationSurfaceOperation::Node + | ApplicationSurfaceOperation::Impact + | ApplicationSurfaceOperation::Similar + | ApplicationSurfaceOperation::Redundancy + | ApplicationSurfaceOperation::RenamePreview + | ApplicationSurfaceOperation::PortStatus + | ApplicationSurfaceOperation::PortOrder + | ApplicationSurfaceOperation::Todos | HealthRead | HealthDelta - | DiagnosticsRead => BranchSensitivity::Sensitive, + | DiagnosticsRead + | StrReplace + | MultiStrReplace + | InsertAt + | AstGrepRewrite + | ReplaceSymbol + | InsertAtSymbol + | MoveSymbol + | RenameSymbol + | SourceEditReconcile + | SourceEditRollback => BranchSensitivity::Sensitive, } } @@ -257,10 +304,10 @@ macro_rules! binding_groups { const BINDING_GROUPS: &[BindingGroup] = binding_groups![ [Some(McpToolDispatchGroup::Graph), RegisteredProjectAccess::ActiveProjectOnly, "tracedecay_search", "tracedecay_grep", "tracedecay_ast_grep_search", "tracedecay_retrieve", - "tracedecay_context", "tracedecay_callers", "tracedecay_callees", "tracedecay_impact", + "tracedecay_context", "tracedecay_impact", "tracedecay_node", "tracedecay_similar", "tracedecay_redundancy", "tracedecay_rename_preview", - "tracedecay_implementations", "tracedecay_callers_for", "tracedecay_find_exact_symbol", - "tracedecay_by_qualified_name", "tracedecay_signature", "tracedecay_impls", "tracedecay_derives"], + "tracedecay_find_exact_symbol", + "tracedecay_by_qualified_name", "tracedecay_signature", "tracedecay_derives"], [Some(McpToolDispatchGroup::Info), RegisteredProjectAccess::ActiveProjectOnly, "tracedecay_status", "tracedecay_remote_status", "tracedecay_active_project", "tracedecay_project_list", "tracedecay_project_search"], @@ -268,8 +315,7 @@ const BINDING_GROUPS: &[BindingGroup] = binding_groups![ "tracedecay_project_context"], [Some(McpToolDispatchGroup::Info), RegisteredProjectAccess::ActiveProjectOnly, "tracedecay_files", "tracedecay_admin_sync", "tracedecay_port_status", "tracedecay_port_order", - "tracedecay_type_hierarchy", "tracedecay_body", "tracedecay_todos", "tracedecay_read", - "tracedecay_outline", "tracedecay_config", "tracedecay_signature_search"], + "tracedecay_todos", "tracedecay_config"], [Some(McpToolDispatchGroup::Admin), RegisteredProjectAccess::ActiveProjectOnly, "tracedecay_hook_runtime", "tracedecay_admin_cli", "tracedecay_admin_project"], [Some(McpToolDispatchGroup::Analysis), RegisteredProjectAccess::ActiveProjectOnly, @@ -281,10 +327,6 @@ const BINDING_GROUPS: &[BindingGroup] = binding_groups![ "tracedecay_admin_branch_add", "tracedecay_affected", "tracedecay_diff_context", "tracedecay_changelog", "tracedecay_commit_context", "tracedecay_pr_context", "tracedecay_branch_search", "tracedecay_branch_diff", "tracedecay_branch_list"], - [Some(McpToolDispatchGroup::Edit), RegisteredProjectAccess::ActiveProjectOnly, - "tracedecay_str_replace", "tracedecay_multi_str_replace", "tracedecay_insert_at", "tracedecay_ast_grep_rewrite", - "tracedecay_replace_symbol", "tracedecay_insert_at_symbol", "tracedecay_move_symbol", "tracedecay_rename_symbol", - "tracedecay_source_edit_reconcile", "tracedecay_source_edit_rollback"], [Some(McpToolDispatchGroup::Health), RegisteredProjectAccess::ActiveProjectOnly, "tracedecay_test_map", "tracedecay_gini", "tracedecay_dependency_depth", "tracedecay_health", "tracedecay_runtime", "tracedecay_dsm", "tracedecay_test_risk"], @@ -663,10 +705,7 @@ pub fn tool_dispatches_source_edit_effect(tool_name: &str) -> bool { } fn compute_tool_dispatches_source_edit_effect(tool_name: &str) -> bool { - matches!( - binding(tool_name).and_then(|binding| binding.group), - Some(McpToolDispatchGroup::Edit) - ) && application_capability_for_tool(tool_name) + application_capability_for_tool(tool_name) .ok() .flatten() .is_some_and(|capability| capability.effect() == EffectClass::SourceEdit) @@ -783,11 +822,6 @@ fn executable_handler_is_available( || matches!(group, Some(McpToolDispatchGroup::ApplicationSurface)) && application_capability .is_some_and(|capability| capability.availability().is_callable()) - || matches!(group, Some(McpToolDispatchGroup::Edit)) - && application_capability.is_some_and(|capability| { - capability.effect() == EffectClass::SourceEdit - && capability.availability().is_callable() - }) } fn inverse_for_tool(tool_name: &str, effect: EffectClass) -> McpInverseContract { @@ -1000,6 +1034,22 @@ mod tests { .into_iter() .map(|entry| entry.name) .collect::>(); + // One tool per binding source: a root binding, a root binding merged + // with its application operation, an application-only operation, and + // the Work and Workflow families. + for tool in [ + "tracedecay_search", + "tracedecay_callers", + "tracedecay_code_exact_occurrence", + "tracedecay_work_create", + "tracedecay_workflow_start_run", + ] { + assert_eq!( + names.iter().filter(|name| *name == tool).count(), + 1, + "{tool} must be bound exactly once" + ); + } let total = names.len(); names.sort_unstable(); names.dedup(); diff --git a/crates/tracedecay-mcp/src/tools/catalog_discovery.rs b/crates/tracedecay-mcp/src/tools/catalog_discovery.rs index 354596e11f..fd4ff67345 100644 --- a/crates/tracedecay-mcp/src/tools/catalog_discovery.rs +++ b/crates/tracedecay-mcp/src/tools/catalog_discovery.rs @@ -394,7 +394,6 @@ mod tests { definition.annotations.as_ref().unwrap()["readOnlyHint"], dispatch["read_only"] ); - assert!(dispatch["deadline"]["maximum_millis"].as_u64().unwrap() > 0); dispatch["fingerprint"].as_str().unwrap() }) .collect::>(); @@ -421,9 +420,20 @@ mod tests { let dispatch = &doctor.meta.as_ref().unwrap()["tracedecay/dispatch"]; assert_eq!(dispatch["effect"], "read"); assert_eq!(dispatch["availability"]["state"], "available"); + assert_eq!(dispatch["deadline"]["maximum_millis"], 30_000); assert!(dispatch.get("receipt").is_none()); assert!(dispatch.get("reconciliation").is_none()); + let affected_tests = definitions + .iter() + .find(|definition| definition.name == "tracedecay_run_affected_tests") + .unwrap(); + assert_eq!( + affected_tests.meta.as_ref().unwrap()["tracedecay/dispatch"]["deadline"]["maximum_millis"], + 600_000, + "a long-running tool gets the ten-minute ceiling" + ); + for retired in [ "tracedecay_lcm_preflight", "tracedecay_lcm_compress", diff --git a/crates/tracedecay-mcp/src/tools/mod.rs b/crates/tracedecay-mcp/src/tools/mod.rs index 0eb390533a..1978ef6d57 100644 --- a/crates/tracedecay-mcp/src/tools/mod.rs +++ b/crates/tracedecay-mcp/src/tools/mod.rs @@ -6,6 +6,7 @@ pub mod dispatch; pub mod dispatch_ceiling; pub mod render; pub mod renderers; +pub mod response_trailers; use serde_json::Value; use std::fmt::Write as _; @@ -35,6 +36,9 @@ pub struct ToolResult { /// populate analytics `failure_reason` without re-deriving it from /// rendered response text. failure_message: Option, + /// Set once the shared renderer accounted this result, so the transport + /// persists those figures instead of appending a second footer. + token_accounting: Option, } impl ToolResult { @@ -45,6 +49,7 @@ impl ToolResult { internal_analytics: None, semantic_error: None, failure_message: None, + token_accounting: None, } } @@ -85,6 +90,18 @@ impl ToolResult { pub fn failure_message(&self) -> Option<&str> { self.failure_message.as_deref() } + + pub(crate) fn set_token_accounting( + &mut self, + accounting: response_trailers::ToolTokenAccounting, + ) { + self.token_accounting = Some(accounting); + } + + /// The figures the shared renderer accounted, if it did. + pub fn token_accounting(&self) -> Option { + self.token_accounting + } } /// Render the CLI help shown by `tracedecay tool --help`. @@ -564,8 +581,8 @@ mod tests { let definition = tracedecay_mcp_catalog::get_tool_definitions() .expect("tool definitions") .into_iter() - .find(|definition| definition.name == "tracedecay_code_implementations") - .expect("code_implementations is advertised"); + .find(|definition| definition.name == "tracedecay_implementations") + .expect("implementations is advertised"); let help = render_tool_cli_help(&definition); let example = help diff --git a/crates/tracedecay-mcp/src/tools/response_trailers.rs b/crates/tracedecay-mcp/src/tools/response_trailers.rs new file mode 100644 index 0000000000..60e869ad2a --- /dev/null +++ b/crates/tracedecay-mcp/src/tools/response_trailers.rs @@ -0,0 +1,248 @@ +//! Blocks every surface appends beside a rendered tool result: the stale +//! code-graph trailer and the token-accounting footer. +//! +//! MCP and the `tracedecay tool` CLI render through the same functions, so +//! both print the same trailer and footer for the same typed result. + +use std::path::{Component, Path}; + +use serde_json::json; +use tracedecay_contracts::retrieval::{CodeGraphReadFreshnessV1, ServedCodeGraphGenerationV1}; + +use super::ToolResult; + +pub const TOKEN_ACCOUNTING_FOOTER_PREFIX: &str = "tracedecay_metrics:"; + +/// Token estimate for one rendered result: reading its touched files raw +/// versus the response it actually delivered. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct ToolTokenAccounting { + pub raw_file_tokens: u64, + pub response_tokens: u64, +} + +impl ToolTokenAccounting { + pub fn net_saved_tokens(self) -> u64 { + self.raw_file_tokens.saturating_sub(self.response_tokens) + } +} + +/// Appends the `code_graph_freshness` trailer when `served` is a stale seat. +pub fn append_code_graph_freshness(result: &mut ToolResult, served: &ServedCodeGraphGenerationV1) { + let CodeGraphReadFreshnessV1::LastCompleteStale { + sealed_at, + rebuild_in_flight, + } = served.freshness + else { + return; + }; + let Some(content) = result + .value + .get_mut("content") + .and_then(|content| content.as_array_mut()) + else { + return; + }; + let generation = &served.generation; + let age = seated_generation_age_label(sealed_at); + let remedy = if rebuild_in_flight { + "while the code index rebuilds" + } else { + "while source freshness remains unverified" + }; + content.push(json!({"type": "text", "text": format!( + "\ncode_graph_freshness: stale, serving the last complete generation \ + {generation} (sealed {age} ago) {remedy}; results may trail the live worktree" + )})); +} + +/// Coarse human duration between a generation's seal time and now. A routine +/// rebuild window reads in seconds or minutes; a wedged route in hours or days. +fn seated_generation_age_label(sealed_at: tracedecay_domain::UtcMicros) -> String { + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(sealed_at.0, |elapsed| { + i64::try_from(elapsed.as_micros()).unwrap_or(i64::MAX) + }); + let seconds = now.saturating_sub(sealed_at.0).max(0) / 1_000_000; + if seconds < 60 { + format!("{seconds}s") + } else if seconds < 3_600 { + format!("{}m", seconds / 60) + } else if seconds < 86_400 { + format!("{}h", seconds / 3_600) + } else { + format!("{}d", seconds / 86_400) + } +} + +/// Approximate tokens (chars / 4) across the result's text blocks. +pub fn response_token_count(result: &ToolResult) -> u64 { + let chars: usize = result + .value + .get("content") + .and_then(|content| content.as_array()) + .into_iter() + .flatten() + .filter_map(|item| item.get("text").and_then(|text| text.as_str())) + .map(str::len) + .sum(); + (chars / 4) as u64 +} + +/// Raw-read counterfactual: every touched project file read in full. +/// Absolute or escaping paths are not project files and cost nothing. +pub fn raw_file_tokens(project_root: &Path, touched_files: &[String]) -> u64 { + touched_files + .iter() + .filter(|path| !path.is_empty()) + .filter_map(|path| { + let relative = Path::new(path); + if relative.is_absolute() + || relative + .components() + .any(|component| matches!(component, Component::ParentDir)) + { + return None; + } + std::fs::metadata(project_root.join(relative)) + .ok() + .filter(std::fs::Metadata::is_file) + .map(|metadata| metadata.len() / 4) + }) + .fold(0, u64::saturating_add) +} + +/// Accounts a rendered result's tokens, records the figures on the result, and +/// appends the footer when the touched files cost anything to read raw. +pub fn account_tool_result(project_root: Option<&Path>, result: &mut ToolResult) { + let accounting = ToolTokenAccounting { + raw_file_tokens: project_root + .map_or(0, |root| raw_file_tokens(root, &result.touched_files)), + response_tokens: response_token_count(result), + }; + record_token_accounting(result, accounting); +} + +/// Records `accounting` on the result and appends its footer when the +/// touched files cost anything to read raw. +pub fn record_token_accounting(result: &mut ToolResult, accounting: ToolTokenAccounting) { + let ToolTokenAccounting { + raw_file_tokens, + response_tokens, + } = accounting; + if raw_file_tokens > 0 + && let Some(content) = result + .value + .get_mut("content") + .and_then(|content| content.as_array_mut()) + { + content.push(json!({"type": "text", "text": format!( + "\n{TOKEN_ACCOUNTING_FOOTER_PREFIX} before={raw_file_tokens} after={response_tokens}" + )})); + } + result.set_token_accounting(accounting); +} + +#[cfg(test)] +mod tests { + use serde_json::json; + use tracedecay_domain::UtcMicros; + + use super::*; + + fn text_result(text: &str, touched: Vec) -> ToolResult { + ToolResult::new( + json!({"content": [{"type": "text", "text": text}]}), + touched, + ) + } + + fn block(result: &ToolResult, index: usize) -> Option<&str> { + result.value["content"][index]["text"].as_str() + } + + #[test] + fn stale_seat_appends_the_trailer_and_a_current_seat_does_not() { + let mut stale = text_result("{}", Vec::new()); + append_code_graph_freshness( + &mut stale, + &ServedCodeGraphGenerationV1 { + generation: "generation.fixture.7".to_owned(), + freshness: CodeGraphReadFreshnessV1::LastCompleteStale { + sealed_at: UtcMicros(0), + rebuild_in_flight: true, + }, + }, + ); + let trailer = block(&stale, 1).expect("stale trailer"); + assert!( + trailer.starts_with( + "\ncode_graph_freshness: stale, serving the last complete generation \ + generation.fixture.7 (sealed " + ), + "{trailer}" + ); + assert!( + trailer.ends_with( + "ago) while the code index rebuilds; results may trail the live worktree" + ), + "{trailer}" + ); + + let mut current = text_result("{}", Vec::new()); + append_code_graph_freshness( + &mut current, + &ServedCodeGraphGenerationV1 { + generation: "generation.fixture.8".to_owned(), + freshness: CodeGraphReadFreshnessV1::Current, + }, + ); + assert_eq!(current.value["content"].as_array().map(Vec::len), Some(1)); + } + + #[test] + fn accounting_footer_counts_touched_files_and_the_response() { + let root = tempfile::tempdir().expect("root"); + std::fs::write(root.path().join("lib.rs"), "x".repeat(400)).expect("source"); + let mut result = text_result(&"y".repeat(40), vec!["lib.rs".to_owned()]); + account_tool_result(Some(root.path()), &mut result); + assert_eq!( + block(&result, 1), + Some("\ntracedecay_metrics: before=100 after=10") + ); + assert_eq!( + result.token_accounting(), + Some(ToolTokenAccounting { + raw_file_tokens: 100, + response_tokens: 10, + }) + ); + } + + #[test] + fn untouched_or_escaping_files_add_no_footer() { + let root = tempfile::tempdir().expect("root"); + let project = root.path().join("project"); + std::fs::create_dir(&project).expect("project"); + let outside = root.path().join("outside.rs"); + std::fs::write(&outside, "x".repeat(400)).expect("outside source"); + let mut result = text_result( + "body", + vec![ + "../outside.rs".to_owned(), + outside.display().to_string(), + "missing.rs".to_owned(), + ], + ); + account_tool_result(Some(&project), &mut result); + assert_eq!(result.value["content"].as_array().map(Vec::len), Some(1)); + assert_eq!( + result.token_accounting(), + Some(ToolTokenAccounting { + raw_file_tokens: 0, + response_tokens: 1, + }) + ); + } +} diff --git a/crates/tracedecay-mcp/src/workflow/test_request.rs b/crates/tracedecay-mcp/src/workflow/test_request.rs index d760728e55..7ec267f360 100644 --- a/crates/tracedecay-mcp/src/workflow/test_request.rs +++ b/crates/tracedecay-mcp/src/workflow/test_request.rs @@ -12,7 +12,7 @@ pub const MAX_TESTS_HARD_CAP: usize = 500; /// into an unbounded daemon job by selecting an arbitrarily distant deadline. pub const MAX_TEST_TIMEOUT_SECS: u64 = DEFAULT_TEST_TIMEOUT_SECS; -fn error_result(args: &Value, kind: &str, operation: &str, message: &str) -> ToolResult { +fn error_result(args: &Value, kind: &str, operation: &str, message: &str) -> Box { let value = json!({ "passed": 0, "failed": 0, @@ -24,10 +24,10 @@ fn error_result(args: &Value, kind: &str, operation: &str, message: &str) -> Too } }); let text = render::finalize(None, args, &value, || render::generic_md(&value)); - ToolResult::new( + Box::new(ToolResult::new( json!({ "content": [{ "type": "text", "text": text }] }), Vec::new(), - ) + )) } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -37,7 +37,7 @@ pub enum TestProfile { } impl TestProfile { - fn parse(args: &Value) -> std::result::Result { + fn parse(args: &Value) -> std::result::Result> { match args.get("profile") { None => Ok(Self::Debug), Some(Value::String(profile)) if profile == "debug" => Ok(Self::Debug), @@ -62,7 +62,7 @@ pub struct RunAffectedArgs { impl RunAffectedArgs { #[hotpath::measure(label = "mcp.workflow.affected_tests.request_build")] - pub fn parse(args: &Value) -> std::result::Result { + pub fn parse(args: &Value) -> std::result::Result> { let explicit_paths = match args.get("changed_paths") { Some(Value::Array(paths)) => { let mut parsed = Vec::with_capacity(paths.len()); @@ -125,7 +125,7 @@ fn bounded_positive_u64( field: &str, default: u64, maximum: u64, -) -> std::result::Result { +) -> std::result::Result> { let Some(value) = args.get(field) else { return Ok(default); }; diff --git a/crates/tracedecay-mcp/tests/server_connection.rs b/crates/tracedecay-mcp/tests/server_connection.rs deleted file mode 100644 index b45866df95..0000000000 --- a/crates/tracedecay-mcp/tests/server_connection.rs +++ /dev/null @@ -1,298 +0,0 @@ -use std::future::Future; -use std::pin::Pin; -use std::sync::Arc; -use std::sync::atomic::{AtomicBool, Ordering}; - -use serde_json::{Value, json}; -use tracedecay_mcp::server::{ - McpConnectionContext, McpConnectionServer, McpConnectionState, McpDispatchRequest, - McpResponseLease, -}; -use tracedecay_mcp::{JsonRpcResponse, McpTransport}; - -struct TestTransport { - incoming: tokio::sync::mpsc::UnboundedReceiver, - outgoing: tokio::sync::mpsc::UnboundedSender, - reads: Arc, -} - -impl McpTransport for TestTransport { - async fn read_line(&mut self) -> std::io::Result> { - let line = self.incoming.recv().await; - if line.is_some() { - self.reads.fetch_add(1, Ordering::Release); - } - Ok(line) - } - - async fn write_line(&mut self, line: &str) -> std::io::Result<()> { - self.outgoing - .send(line.to_owned()) - .map_err(|error| std::io::Error::new(std::io::ErrorKind::BrokenPipe, error)) - } - - async fn flush(&mut self) -> std::io::Result<()> { - Ok(()) - } -} - -struct TestConnection { - scope: String, -} - -struct TestLease { - revoked: tracedecay_session_memory::context::CancellationToken, -} - -impl McpResponseLease for TestLease { - fn revoked(&self) -> &tracedecay_session_memory::context::CancellationToken { - &self.revoked - } -} - -impl McpConnectionState for TestConnection { - type ResponseLease = TestLease; - - fn memory_request_scope(&self) -> &str { - &self.scope - } - - fn fork_for_independent_read(&self) -> Self { - Self { - scope: self.scope.clone(), - } - } - - fn fork_for_connection_owned_read(&self) -> Self { - self.fork_for_independent_read() - } - - fn take_selected_response_lease(&mut self) -> Option { - None - } -} - -struct TestContext { - slow_release: tokio::sync::Notify, - shutdown: AtomicBool, - cancellation_registered: tokio::sync::Notify, - max_concurrent_reads: usize, -} - -impl TestContext { - fn new() -> Self { - Self::with_max_concurrent_reads(2) - } - - fn with_max_concurrent_reads(max_concurrent_reads: usize) -> Self { - Self { - slow_release: tokio::sync::Notify::new(), - shutdown: AtomicBool::new(false), - cancellation_registered: tokio::sync::Notify::new(), - max_concurrent_reads, - } - } -} - -impl McpConnectionContext for TestContext { - type Connection = TestConnection; - - fn new_connection(&self) -> tracedecay_domain::errors::Result { - Ok(TestConnection { - scope: "test-connection".to_owned(), - }) - } - - fn timings_enabled(&self) -> bool { - false - } - - fn build_version(&self) -> tracedecay_domain::errors::Result<&'static str> { - Ok("test") - } - - fn max_concurrent_reads(&self) -> usize { - self.max_concurrent_reads - } - - fn tool_is_read_only(&self, _tool_name: &str) -> bool { - true - } - - fn tool_supports_live_cancellation(&self, _tool_name: &str) -> bool { - true - } - - fn dispatch<'a>( - &'a self, - request: McpDispatchRequest<'a>, - _timings_enabled: bool, - _connection: &'a mut Self::Connection, - cancellation: tracedecay_session_memory::context::CancellationToken, - ) -> Pin> + Send + 'a>> { - Box::pin(async move { - let id = request.cloned_id()?; - if cancellation.is_cancelled() { - return Some(JsonRpcResponse::error( - id, - tracedecay_mcp::ErrorCode::RequestCancelled, - "cancelled before dispatch".to_owned(), - )); - } - if id == json!("slow") { - self.slow_release.notified().await; - } - Some(JsonRpcResponse::success(id, json!({"served": true}))) - }) - } - - fn cancel_request(&self, _id: &Value, _connection_scope: &str) -> bool { - false - } - - fn cancellation_registered(&self) -> &tokio::sync::Notify { - &self.cancellation_registered - } - - fn take_pending_notifications(&self) -> Vec { - Vec::new() - } - - fn run_in_connection_admission<'a, T, F>( - &'a self, - future: F, - ) -> Pin + Send + 'a>> - where - T: Send + 'a, - F: Future + Send + 'a, - { - Box::pin(future) - } - - fn shutdown(self: Arc) -> Pin + Send>> { - Box::pin(async move { - self.shutdown.store(true, Ordering::Release); - }) - } -} - -async fn response_id(responses: &mut tokio::sync::mpsc::UnboundedReceiver) -> Value { - response(responses).await["id"].clone() -} - -async fn response(responses: &mut tokio::sync::mpsc::UnboundedReceiver) -> Value { - let line = tokio::time::timeout(std::time::Duration::from_secs(5), responses.recv()) - .await - .expect("response timeout") - .expect("response line"); - serde_json::from_str::(line.trim()).expect("JSON-RPC response") -} - -#[tokio::test] -async fn queued_cancellation_is_delivered_before_the_request_dispatches() { - let context = Arc::new(TestContext::with_max_concurrent_reads(1)); - let server = McpConnectionServer::new(Arc::clone(&context)); - let (request_tx, request_rx) = tokio::sync::mpsc::unbounded_channel(); - let (response_tx, mut response_rx) = tokio::sync::mpsc::unbounded_channel(); - let reads = Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let mut transport = TestTransport { - incoming: request_rx, - outgoing: response_tx, - reads: Arc::clone(&reads), - }; - let serving = tokio::spawn(async move { server.run_connection(&mut transport).await }); - - for id in [json!("slow"), json!("cancelled")] { - request_tx - .send( - json!({ - "jsonrpc": "2.0", - "id": id, - "method": "tools/call", - "params": {"name": "tracedecay_search", "arguments": {}} - }) - .to_string(), - ) - .expect("queued request"); - } - request_tx - .send( - json!({ - "jsonrpc": "2.0", - "method": "notifications/cancelled", - "params": {"requestId": "cancelled"} - }) - .to_string(), - ) - .expect("queued cancellation"); - - tokio::time::timeout(std::time::Duration::from_secs(5), async { - while reads.load(Ordering::Acquire) < 3 { - tokio::task::yield_now().await; - } - }) - .await - .expect("connection did not consume queued cancellation"); - context.slow_release.notify_one(); - assert_eq!(response_id(&mut response_rx).await, json!("slow")); - let cancelled = response(&mut response_rx).await; - assert_eq!(cancelled["id"], json!("cancelled")); - assert_eq!(cancelled["error"]["code"], json!(-32800)); - - drop(request_tx); - serving - .await - .expect("connection task") - .expect("connection result"); -} - -#[tokio::test] -async fn independent_reads_settle_out_of_order_and_half_close_drains() { - let context = Arc::new(TestContext::new()); - let server = McpConnectionServer::new(Arc::clone(&context)); - let (request_tx, request_rx) = tokio::sync::mpsc::unbounded_channel(); - let (response_tx, mut response_rx) = tokio::sync::mpsc::unbounded_channel(); - let mut transport = TestTransport { - incoming: request_rx, - outgoing: response_tx, - reads: Arc::new(std::sync::atomic::AtomicUsize::new(0)), - }; - let serving = tokio::spawn(async move { server.run_connection(&mut transport).await }); - - request_tx - .send( - json!({ - "jsonrpc": "2.0", - "id": "slow", - "method": "tools/call", - "params": {"name": "tracedecay_search", "arguments": {}} - }) - .to_string(), - ) - .expect("slow request"); - request_tx - .send( - json!({ - "jsonrpc": "2.0", - "id": "fast", - "method": "tools/call", - "params": {"name": "tracedecay_status", "arguments": {}} - }) - .to_string(), - ) - .expect("fast request"); - - assert_eq!(response_id(&mut response_rx).await, json!("fast")); - context.slow_release.notify_one(); - assert_eq!(response_id(&mut response_rx).await, json!("slow")); - - drop(request_tx); - serving - .await - .expect("connection task") - .expect("connection result"); - assert!( - !context.shutdown.load(Ordering::Acquire), - "a daemon-style connection close must not stop the shared server" - ); -} diff --git a/crates/tracedecay-mcp/tests/server_protocol.rs b/crates/tracedecay-mcp/tests/server_protocol.rs index 9fe7fc58b6..484641615b 100644 --- a/crates/tracedecay-mcp/tests/server_protocol.rs +++ b/crates/tracedecay-mcp/tests/server_protocol.rs @@ -43,8 +43,8 @@ fn initialize_payload_uses_composed_product_metadata() { } #[test] -fn typed_and_legacy_envelopes_share_read_classification() { - let legacy = JsonRpcRequest { +fn typed_and_raw_envelopes_share_read_classification() { + let raw_request = JsonRpcRequest { jsonrpc: "2.0".to_owned(), id: Some(json!(1)), method: "tools/call".to_owned(), @@ -53,7 +53,7 @@ fn typed_and_legacy_envelopes_share_read_classification() { "arguments": {"query": "server owner"} })), }; - let raw = McpDispatchRequest::from_legacy(&legacy); + let raw = McpDispatchRequest::raw(&raw_request); let typed = McpDispatchRequest::typed( json!(1), "tools/call", diff --git a/crates/tracedecay-mcp/tests/server_read_coalescing.rs b/crates/tracedecay-mcp/tests/server_read_coalescing.rs index 4c10bd2971..01bd35cab3 100644 --- a/crates/tracedecay-mcp/tests/server_read_coalescing.rs +++ b/crates/tracedecay-mcp/tests/server_read_coalescing.rs @@ -7,7 +7,7 @@ async fn identical_reads_share_only_the_in_flight_result() { let coalescer = IdenticalReadCoalescer::default(); let leader = match coalescer.claim( "graph-main", - "tracedecay_outline", + "tracedecay_source_outline", &json!({"path": "src/lib.rs"}), None, ) { @@ -16,7 +16,7 @@ async fn identical_reads_share_only_the_in_flight_result() { }; let follower = match coalescer.claim( "graph-main", - "tracedecay_outline", + "tracedecay_source_outline", &json!({"path": "src/lib.rs"}), None, ) { @@ -38,7 +38,7 @@ async fn identical_reads_share_only_the_in_flight_result() { assert!(matches!( coalescer.claim( "graph-main", - "tracedecay_outline", + "tracedecay_source_outline", &json!({"path": "src/lib.rs"}), None, ), diff --git a/crates/tracedecay-mcp/tests/server_rmcp.rs b/crates/tracedecay-mcp/tests/server_rmcp.rs index 90fe4abab5..162d35c95e 100644 --- a/crates/tracedecay-mcp/tests/server_rmcp.rs +++ b/crates/tracedecay-mcp/tests/server_rmcp.rs @@ -13,11 +13,11 @@ use tracedecay_mcp::server::{ }; struct TestLease { - revoked: tracedecay_session_memory::context::CancellationToken, + revoked: tracedecay_runtime_core::cancellation::CancellationToken, } impl McpResponseLease for TestLease { - fn revoked(&self) -> &tracedecay_session_memory::context::CancellationToken { + fn revoked(&self) -> &tracedecay_runtime_core::cancellation::CancellationToken { &self.revoked } } @@ -39,10 +39,6 @@ impl McpConnectionState for TestConnection { } } - fn fork_for_connection_owned_read(&self) -> Self { - self.fork_for_independent_read() - } - fn take_selected_response_lease(&mut self) -> Option { None } @@ -78,16 +74,12 @@ impl McpConnectionContext for TestContext { true } - fn tool_supports_live_cancellation(&self, _tool_name: &str) -> bool { - false - } - fn dispatch<'a>( &'a self, request: McpDispatchRequest<'a>, _timings_enabled: bool, _connection: &'a mut Self::Connection, - _cancellation: tracedecay_session_memory::context::CancellationToken, + _cancellation: tracedecay_runtime_core::cancellation::CancellationToken, ) -> Pin> + Send + 'a>> { Box::pin(async move { let id = request.cloned_id()?; @@ -115,10 +107,6 @@ impl McpConnectionContext for TestContext { &self.cancellation_registered } - fn take_pending_notifications(&self) -> Vec { - Vec::new() - } - fn run_in_connection_admission<'a, T, F>( &'a self, future: F, @@ -129,10 +117,6 @@ impl McpConnectionContext for TestContext { { Box::pin(future) } - - fn shutdown(self: Arc) -> Pin + Send>> { - Box::pin(async {}) - } } #[tokio::test] diff --git a/crates/tracedecay-policy/Cargo.toml b/crates/tracedecay-policy/Cargo.toml index f034be4a1c..63ed898f34 100644 --- a/crates/tracedecay-policy/Cargo.toml +++ b/crates/tracedecay-policy/Cargo.toml @@ -9,7 +9,7 @@ repository = "https://github.com/ScriptedAlchemy/tracedecay" [dependencies] hotpath.workspace = true -schemars = "1.2.1" +schemars.workspace = true serde = { version = "1", features = ["derive"] } tracedecay-domain = { path = "../tracedecay-domain", version = "0.1.0" } diff --git a/crates/tracedecay-policy/src/authorization.rs b/crates/tracedecay-policy/src/authorization.rs new file mode 100644 index 0000000000..b65ec3b98b --- /dev/null +++ b/crates/tracedecay-policy/src/authorization.rs @@ -0,0 +1,87 @@ +//! Identifiers, privacy obligations, and the canonical digest shared by the +//! policy evaluators. + +use std::collections::BTreeSet; +use std::fmt; + +use serde::{Deserialize, Deserializer, Serialize}; +use tracedecay_domain::{ManifestDigest, canonical_sha256}; + +/// A bounded, canonical identifier owned by the policy input schema. +/// +/// It represents immutable references only; it is never a path, display +/// label, provider account, branch name, or native object identifier. +#[derive(Clone, Debug, Serialize, schemars::JsonSchema, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[serde(transparent)] +pub struct PolicyIdentifierV1(String); + +impl PolicyIdentifierV1 { + pub fn new(value: impl Into) -> Result { + let value = value.into(); + Self::validate(&value)?; + Ok(Self(value)) + } + + pub fn as_str(&self) -> &str { + &self.0 + } + + pub fn is_valid(&self) -> bool { + Self::validate(&self.0).is_ok() + } + + fn validate(value: &str) -> Result<(), &'static str> { + if value.is_empty() + || value.trim() != value + || value.len() > 512 + || value.chars().any(char::is_control) + { + return Err("policy identifier must be non-empty, trimmed, bounded, and printable"); + } + Ok(()) + } +} + +impl<'de> Deserialize<'de> for PolicyIdentifierV1 { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + Self::new(String::deserialize(deserializer)?).map_err(serde::de::Error::custom) + } +} + +impl fmt::Display for PolicyIdentifierV1 { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(&self.0) + } +} + +/// Non-waivable obligations accumulate across every authorization operand. +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[serde(rename_all = "snake_case")] +pub enum PrivacyConstraintV1 { + LocalOnly, + SanitizedOnly, + NoRetention, + NoModelContext, + NoTelemetry, + NoExport, +} + +pub type PrivacyConstraintSetV1 = BTreeSet; + +/// Stable digest helper used for immutable, serializable policy inputs. +pub(crate) fn policy_digest(domain: &'static str, value: &T) -> ManifestDigest { + match canonical_sha256(&(domain, value)) { + Ok(digest) => digest, + Err(_) => { + // This can only be reached if a future serializable policy type + // violates canonical JSON requirements. Preserve a deterministic + // non-authorizing digest rather than panic or consult external + // state. + ManifestDigest::new(format!("sha256:{}", "0".repeat(64))) + .expect("static policy fallback digest is canonical") + } + } +} diff --git a/crates/tracedecay-policy/src/authorization/decision.rs b/crates/tracedecay-policy/src/authorization/decision.rs deleted file mode 100644 index b2ddfdfe60..0000000000 --- a/crates/tracedecay-policy/src/authorization/decision.rs +++ /dev/null @@ -1,565 +0,0 @@ -use serde::{Deserialize, Serialize}; -use tracedecay_domain::ManifestDigest; - -use super::grant::GrantStateAtV1; -use super::input::{ - AuthorizationCoverageV1, AuthorizationSnapshotStateV1, ExternalContentStatusV1, - PolicyIdentifierV1, SourceAuthorizationInputV1, policy_digest, -}; -use super::intersection::{ - EffectiveSourceGrantV1, IntersectionFailureV1, intersect_source_authority, -}; -use super::state::{ - PublicSourceResultShapeV1, SourceAccessDecisionV1, SourceAuthorizationDispositionV1, -}; - -/// Stable evaluator implementation identity. It is recorded with every -/// decision so exact replay can refuse a substituted evaluator revision. -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct PolicyEvaluatorVersionV1 { - pub evaluator_id: PolicyIdentifierV1, - pub evaluator_revision: u64, -} - -impl PolicyEvaluatorVersionV1 { - pub fn is_valid(&self) -> bool { - self.evaluator_id.is_valid() && self.evaluator_revision > 0 - } -} - -/// Stable machine-readable decision trace entries. Renderers may turn these -/// into text, but text never changes the authority represented by a decision. -#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)] -#[serde(rename_all = "snake_case")] -pub enum PolicyReasonCodeV1 { - InputInvalid, - InputComplete, - InputPartial, - InputMissing, - InputStale, - InputAmbiguous, - SourceDefinitionBindingMismatch, - SourcePolicySourceMismatch, - SinkPolicySinkMismatch, - OwnerScopeMismatch, - RequesterSubjectMismatch, - OperationPolicyExcluded, - SinkPolicyExcluded, - SourceGrantActive, - SourceGrantRevoked, - SourceGrantStale, - SourceGrantAmbiguous, - SourceGrantNotYetIssued, - SourceGrantExpired, - RequesterGrantActive, - RequesterGrantRevoked, - RequesterGrantStale, - RequesterGrantAmbiguous, - RequesterGrantNotYetIssued, - RequesterGrantExpired, - GrantIntersectionNonExpanding, - ResourceNotGranted, - OperationNotGranted, - SinkNotGranted, - DisclosureTooBroad, - BudgetExceeded, - MandatoryLocalPrivacyBlocksEgress, - SanitizedOnlyBlocksDisclosure, - NoModelContext, - NoRetention, - NoTelemetry, - NoExport, - SinkUnavailable, - AccessAllowed, - AuthorizationCoveragePartial, - ContentLive, - ContentPartial, - ContentTemporarilyUnavailable, - ContentAuthoritativeDeleted, - SinkPolicyDrift, - AuthorizationInputDrift, -} - -/// Decision trace over immutable source authorization facts. -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct SourceAuthorizationDecisionV1 { - pub evaluator_version: PolicyEvaluatorVersionV1, - pub input_digest: ManifestDigest, - pub policy_revision: u64, - pub policy_digest: ManifestDigest, - pub configuration_digest: ManifestDigest, - pub content_status: ExternalContentStatusV1, - pub access: SourceAccessDecisionV1, - pub authorization_coverage: AuthorizationCoverageV1, - pub disposition: SourceAuthorizationDispositionV1, - pub effective_grant: Option, - pub ordered_reason_codes: Vec, - pub evidence_references: Vec, - pub decision_digest: ManifestDigest, -} - -impl SourceAuthorizationDecisionV1 { - pub fn is_authorized(&self) -> bool { - self.access == SourceAccessDecisionV1::Authorized - } -} - -/// Borrowed projection of every decision field that the decision digest -/// covers. The digest is computed over this material before the owned -/// decision exists, so a decision is constructed exactly once with its final -/// digest and no placeholder digest is ever hashed. -#[derive(Serialize)] -struct DecisionMaterial<'a> { - evaluator_version: &'a PolicyEvaluatorVersionV1, - input_digest: &'a ManifestDigest, - policy_revision: u64, - policy_digest: &'a ManifestDigest, - configuration_digest: &'a ManifestDigest, - content_status: ExternalContentStatusV1, - access: SourceAccessDecisionV1, - authorization_coverage: AuthorizationCoverageV1, - disposition: SourceAuthorizationDispositionV1, - effective_grant: &'a Option, - ordered_reason_codes: &'a [PolicyReasonCodeV1], - evidence_references: &'a [PolicyIdentifierV1], -} - -impl DecisionMaterial<'_> { - fn digest(&self) -> ManifestDigest { - policy_digest("tracedecay.policy.source-authorization-decision.v1", self) - } -} - -/// Expected JSON truth-table projection. It intentionally asserts only public -/// stable semantics, not opaque digest bytes. -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct SourceAuthorizationExpectedDecisionV1 { - pub access: SourceAccessDecisionV1, - pub authorization_coverage: AuthorizationCoverageV1, - pub disposition: SourceAuthorizationDispositionV1, - pub ordered_reason_codes: Vec, - pub has_effective_grant: bool, - pub public_shape: PublicSourceResultShapeV1, -} - -/// Checked-in JSON truth-table row for the deterministic source evaluator. -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct SourceAuthorizationTruthTableV1 { - pub name: String, - pub source_visible: bool, - pub input: SourceAuthorizationInputV1, - pub expected: SourceAuthorizationExpectedDecisionV1, -} - -/// Pure source authorization contract. -pub trait SourceAuthorizationEvaluator { - fn evaluator_version(&self) -> &PolicyEvaluatorVersionV1; - - fn evaluate(&self, input: &SourceAuthorizationInputV1) -> SourceAuthorizationDecisionV1; -} - -/// Reviewed Rust implementation of source authorization. It has no mutable -/// state and therefore evaluates identical input bytes identically. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct SourceAuthorizationEvaluatorV1 { - version: PolicyEvaluatorVersionV1, -} - -impl Default for SourceAuthorizationEvaluatorV1 { - fn default() -> Self { - Self { - version: PolicyEvaluatorVersionV1 { - evaluator_id: PolicyIdentifierV1::new("source_authorization.v1") - .expect("static evaluator identifier is valid"), - evaluator_revision: 1, - }, - } - } -} - -impl SourceAuthorizationEvaluatorV1 { - pub fn version(&self) -> &PolicyEvaluatorVersionV1 { - &self.version - } - - fn decision( - &self, - input: &SourceAuthorizationInputV1, - access: SourceAccessDecisionV1, - coverage: AuthorizationCoverageV1, - disposition: SourceAuthorizationDispositionV1, - effective_grant: Option, - ordered_reason_codes: Vec, - ) -> SourceAuthorizationDecisionV1 { - crate::hotpath_observe::authorization_outcome(access, disposition); - let input_digest = input.input_digest(); - let evidence_references: Vec = - input.evidence_references.iter().cloned().collect(); - let decision_digest = DecisionMaterial { - evaluator_version: &self.version, - input_digest: &input_digest, - policy_revision: input.policy_revision, - policy_digest: &input.policy_digest, - configuration_digest: &input.configuration_digest, - content_status: input.content_status, - access, - authorization_coverage: coverage, - disposition, - effective_grant: &effective_grant, - ordered_reason_codes: &ordered_reason_codes, - evidence_references: &evidence_references, - } - .digest(); - SourceAuthorizationDecisionV1 { - evaluator_version: self.version.clone(), - input_digest, - policy_revision: input.policy_revision, - policy_digest: input.policy_digest.clone(), - configuration_digest: input.configuration_digest.clone(), - content_status: input.content_status, - access, - authorization_coverage: coverage, - disposition, - effective_grant, - ordered_reason_codes, - evidence_references, - decision_digest, - } - } - - fn non_authorizing( - &self, - input: &SourceAuthorizationInputV1, - access: SourceAccessDecisionV1, - disposition: SourceAuthorizationDispositionV1, - reasons: Vec, - ) -> SourceAuthorizationDecisionV1 { - self.decision( - input, - access, - input.requested_coverage, - disposition, - None, - reasons, - ) - } - - fn grant_reason( - source_grant: bool, - state: GrantStateAtV1, - ) -> (PolicyReasonCodeV1, SourceAuthorizationDispositionV1) { - match (source_grant, state) { - (true, GrantStateAtV1::Active) => ( - PolicyReasonCodeV1::SourceGrantActive, - SourceAuthorizationDispositionV1::Allow, - ), - (false, GrantStateAtV1::Active) => ( - PolicyReasonCodeV1::RequesterGrantActive, - SourceAuthorizationDispositionV1::Allow, - ), - (true, GrantStateAtV1::Revoked) => ( - PolicyReasonCodeV1::SourceGrantRevoked, - SourceAuthorizationDispositionV1::Deny, - ), - (false, GrantStateAtV1::Revoked) => ( - PolicyReasonCodeV1::RequesterGrantRevoked, - SourceAuthorizationDispositionV1::Deny, - ), - (true, GrantStateAtV1::Expired) => ( - PolicyReasonCodeV1::SourceGrantExpired, - SourceAuthorizationDispositionV1::Deny, - ), - (false, GrantStateAtV1::Expired) => ( - PolicyReasonCodeV1::RequesterGrantExpired, - SourceAuthorizationDispositionV1::Deny, - ), - (true, GrantStateAtV1::NotYetIssued) => ( - PolicyReasonCodeV1::SourceGrantNotYetIssued, - SourceAuthorizationDispositionV1::Deny, - ), - (false, GrantStateAtV1::NotYetIssued) => ( - PolicyReasonCodeV1::RequesterGrantNotYetIssued, - SourceAuthorizationDispositionV1::Deny, - ), - (true, GrantStateAtV1::Stale) => ( - PolicyReasonCodeV1::SourceGrantStale, - SourceAuthorizationDispositionV1::Indeterminate, - ), - (false, GrantStateAtV1::Stale) => ( - PolicyReasonCodeV1::RequesterGrantStale, - SourceAuthorizationDispositionV1::Indeterminate, - ), - (true, GrantStateAtV1::Ambiguous) => ( - PolicyReasonCodeV1::SourceGrantAmbiguous, - SourceAuthorizationDispositionV1::Indeterminate, - ), - (false, GrantStateAtV1::Ambiguous) => ( - PolicyReasonCodeV1::RequesterGrantAmbiguous, - SourceAuthorizationDispositionV1::Indeterminate, - ), - } - } - - fn intersection_reason(failure: IntersectionFailureV1) -> PolicyReasonCodeV1 { - match failure { - IntersectionFailureV1::OwnerMismatch => PolicyReasonCodeV1::OwnerScopeMismatch, - IntersectionFailureV1::RequesterSubjectMismatch => { - PolicyReasonCodeV1::RequesterSubjectMismatch - } - IntersectionFailureV1::ResourceNotGranted => PolicyReasonCodeV1::ResourceNotGranted, - IntersectionFailureV1::OperationNotGranted => PolicyReasonCodeV1::OperationNotGranted, - IntersectionFailureV1::SinkNotGranted => PolicyReasonCodeV1::SinkNotGranted, - IntersectionFailureV1::DisclosureTooBroad => PolicyReasonCodeV1::DisclosureTooBroad, - IntersectionFailureV1::BudgetExceeded => PolicyReasonCodeV1::BudgetExceeded, - IntersectionFailureV1::MandatoryLocalPrivacyBlocksEgress => { - PolicyReasonCodeV1::MandatoryLocalPrivacyBlocksEgress - } - IntersectionFailureV1::SanitizedOnlyBlocksDisclosure => { - PolicyReasonCodeV1::SanitizedOnlyBlocksDisclosure - } - IntersectionFailureV1::NoModelContext => PolicyReasonCodeV1::NoModelContext, - IntersectionFailureV1::NoRetention => PolicyReasonCodeV1::NoRetention, - IntersectionFailureV1::NoTelemetry => PolicyReasonCodeV1::NoTelemetry, - IntersectionFailureV1::NoExport => PolicyReasonCodeV1::NoExport, - IntersectionFailureV1::SinkUnavailable => PolicyReasonCodeV1::SinkUnavailable, - } - } -} - -impl SourceAuthorizationEvaluator for SourceAuthorizationEvaluatorV1 { - fn evaluator_version(&self) -> &PolicyEvaluatorVersionV1 { - &self.version - } - - #[hotpath::measure(label = "policy.authorization.evaluate")] - fn evaluate(&self, input: &SourceAuthorizationInputV1) -> SourceAuthorizationDecisionV1 { - if !input.is_structurally_valid() || !self.version.is_valid() { - return self.non_authorizing( - input, - SourceAccessDecisionV1::Unauthorized, - SourceAuthorizationDispositionV1::Indeterminate, - vec![PolicyReasonCodeV1::InputInvalid], - ); - } - - let mut reasons = Vec::new(); - match input.snapshot_state { - AuthorizationSnapshotStateV1::Complete => { - reasons.push(PolicyReasonCodeV1::InputComplete); - } - AuthorizationSnapshotStateV1::Partial => { - return self.non_authorizing( - input, - SourceAccessDecisionV1::Unauthorized, - SourceAuthorizationDispositionV1::Indeterminate, - vec![PolicyReasonCodeV1::InputPartial], - ); - } - AuthorizationSnapshotStateV1::Missing => { - return self.non_authorizing( - input, - SourceAccessDecisionV1::Unauthorized, - SourceAuthorizationDispositionV1::Indeterminate, - vec![PolicyReasonCodeV1::InputMissing], - ); - } - AuthorizationSnapshotStateV1::Stale => { - return self.non_authorizing( - input, - SourceAccessDecisionV1::Unauthorized, - SourceAuthorizationDispositionV1::Indeterminate, - vec![PolicyReasonCodeV1::InputStale], - ); - } - AuthorizationSnapshotStateV1::Ambiguous => { - return self.non_authorizing( - input, - SourceAccessDecisionV1::Unauthorized, - SourceAuthorizationDispositionV1::Indeterminate, - vec![PolicyReasonCodeV1::InputAmbiguous], - ); - } - } - - if &input.definition.definition.source_id != input.binding.binding.source_id() { - reasons.push(PolicyReasonCodeV1::SourceDefinitionBindingMismatch); - return self.non_authorizing( - input, - SourceAccessDecisionV1::Unauthorized, - SourceAuthorizationDispositionV1::Deny, - reasons, - ); - } - if input.definition.definition.source_id != input.source_policy.source_id { - reasons.push(PolicyReasonCodeV1::SourcePolicySourceMismatch); - return self.non_authorizing( - input, - SourceAccessDecisionV1::Unauthorized, - SourceAuthorizationDispositionV1::Deny, - reasons, - ); - } - if input.sink_policy.sink != input.requested_access.sink { - reasons.push(PolicyReasonCodeV1::SinkPolicySinkMismatch); - return self.non_authorizing( - input, - SourceAccessDecisionV1::Unauthorized, - SourceAuthorizationDispositionV1::Deny, - reasons, - ); - } - let binding_owner = input.binding.binding.owner(); - let resolved_owner = &input.resolved_owner_scope.owner; - if &binding_owner != resolved_owner - || &input.source_grant.owner != resolved_owner - || &input.requester_grant.owner != resolved_owner - { - reasons.push(PolicyReasonCodeV1::OwnerScopeMismatch); - return self.non_authorizing( - input, - SourceAccessDecisionV1::Unauthorized, - SourceAuthorizationDispositionV1::Deny, - reasons, - ); - } - if !input - .source_policy - .eligible_operations - .contains(&input.requested_access.operation) - { - reasons.push(PolicyReasonCodeV1::OperationPolicyExcluded); - return self.non_authorizing( - input, - SourceAccessDecisionV1::PolicyExcluded, - SourceAuthorizationDispositionV1::NotApplicable, - reasons, - ); - } - if !input - .source_policy - .eligible_sinks - .contains(&input.requested_access.sink) - { - reasons.push(PolicyReasonCodeV1::SinkPolicyExcluded); - return self.non_authorizing( - input, - SourceAccessDecisionV1::PolicyExcluded, - SourceAuthorizationDispositionV1::NotApplicable, - reasons, - ); - } - - let (source_reason, source_disposition) = - Self::grant_reason(true, input.source_grant.state_at(input.evaluated_at)); - reasons.push(source_reason); - if source_disposition != SourceAuthorizationDispositionV1::Allow { - return self.non_authorizing( - input, - SourceAccessDecisionV1::Unauthorized, - source_disposition, - reasons, - ); - } - let (requester_reason, requester_disposition) = - Self::grant_reason(false, input.requester_grant.state_at(input.evaluated_at)); - reasons.push(requester_reason); - if requester_disposition != SourceAuthorizationDispositionV1::Allow { - return self.non_authorizing( - input, - SourceAccessDecisionV1::Unauthorized, - requester_disposition, - reasons, - ); - } - - reasons.push(PolicyReasonCodeV1::GrantIntersectionNonExpanding); - let effective_grant = match intersect_source_authority(input) { - Ok(grant) => grant, - Err(failure) => { - let reason = Self::intersection_reason(failure); - reasons.push(reason); - let disposition = if failure == IntersectionFailureV1::SinkUnavailable { - SourceAuthorizationDispositionV1::Indeterminate - } else { - SourceAuthorizationDispositionV1::Deny - }; - return self.non_authorizing( - input, - SourceAccessDecisionV1::Unauthorized, - disposition, - reasons, - ); - } - }; - reasons.push(PolicyReasonCodeV1::AccessAllowed); - - let coverage = match (input.requested_coverage, input.content_status) { - (AuthorizationCoverageV1::Partial, _) | (_, ExternalContentStatusV1::Partial) => { - reasons.push(PolicyReasonCodeV1::AuthorizationCoveragePartial); - AuthorizationCoverageV1::Partial - } - (AuthorizationCoverageV1::Complete, _) => AuthorizationCoverageV1::Complete, - }; - let (disposition, content_reason) = match input.content_status { - ExternalContentStatusV1::Live => ( - SourceAuthorizationDispositionV1::Allow, - PolicyReasonCodeV1::ContentLive, - ), - ExternalContentStatusV1::Partial => ( - SourceAuthorizationDispositionV1::Allow, - PolicyReasonCodeV1::ContentPartial, - ), - ExternalContentStatusV1::TemporarilyUnavailable => ( - SourceAuthorizationDispositionV1::Indeterminate, - PolicyReasonCodeV1::ContentTemporarilyUnavailable, - ), - ExternalContentStatusV1::AuthoritativeDeleted => ( - SourceAuthorizationDispositionV1::Allow, - PolicyReasonCodeV1::ContentAuthoritativeDeleted, - ), - }; - reasons.push(content_reason); - self.decision( - input, - SourceAccessDecisionV1::Authorized, - coverage, - disposition, - Some(effective_grant), - reasons, - ) - } -} - -/// Apply the non-disclosure boundary after authorization. Reasons, content -/// counts, cursors, source state, and timing never appear in the -/// `NotFoundOrNotAuthorized` variant. -pub fn public_source_result_shape( - decision: &SourceAuthorizationDecisionV1, - source_visible: bool, -) -> PublicSourceResultShapeV1 { - if !source_visible || decision.access == SourceAccessDecisionV1::Unauthorized { - return PublicSourceResultShapeV1::NotFoundOrNotAuthorized; - } - if decision.access == SourceAccessDecisionV1::PolicyExcluded { - return PublicSourceResultShapeV1::PolicyExcluded; - } - if decision.authorization_coverage == AuthorizationCoverageV1::Partial - || decision.content_status == ExternalContentStatusV1::Partial - { - return PublicSourceResultShapeV1::Partial; - } - match decision.content_status { - ExternalContentStatusV1::Live => PublicSourceResultShapeV1::Live, - ExternalContentStatusV1::Partial => PublicSourceResultShapeV1::Partial, - ExternalContentStatusV1::TemporarilyUnavailable => { - PublicSourceResultShapeV1::TemporarilyUnavailable - } - ExternalContentStatusV1::AuthoritativeDeleted => { - PublicSourceResultShapeV1::AuthoritativeDeleted - } - } -} diff --git a/crates/tracedecay-policy/src/authorization/grant.rs b/crates/tracedecay-policy/src/authorization/grant.rs deleted file mode 100644 index dacc0759ee..0000000000 --- a/crates/tracedecay-policy/src/authorization/grant.rs +++ /dev/null @@ -1,78 +0,0 @@ -use std::collections::BTreeSet; - -use serde::{Deserialize, Serialize}; -use tracedecay_domain::{ActorId, ManifestDigest, UtcMicros}; - -use super::input::{ - BudgetSetV1, DisclosureClassV1, GrantIdV1, PrivacyConstraintSetV1, ResourceIdV1, SinkKindV1, - SourceOwnerV1, TypedOperationV1, -}; - -/// Explicit external grant record state. Policy cannot issue, renew, revoke, -/// widen, or reinterpret a grant; it only consumes this immutable input. -#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)] -#[serde(rename_all = "snake_case")] -pub enum GrantStateV1 { - Active, - Revoked, - Stale, - Ambiguous, -} - -/// Immutable authorization input issued outside this crate. -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct CapabilityGrantV1 { - pub grant_id: GrantIdV1, - pub issuer: ActorId, - pub subject: ActorId, - pub owner: SourceOwnerV1, - pub resources: BTreeSet, - pub operations: BTreeSet, - pub sinks: BTreeSet, - pub disclosure_ceiling: DisclosureClassV1, - pub constraints: PrivacyConstraintSetV1, - pub budgets: BudgetSetV1, - pub revision: u64, - pub issued_at: UtcMicros, - pub expires_at: UtcMicros, - pub digest: ManifestDigest, - pub state: GrantStateV1, -} - -impl CapabilityGrantV1 { - pub fn is_valid(&self) -> bool { - self.grant_id.is_valid() - && self.issuer.validate().is_ok() - && self.subject.validate().is_ok() - && self.owner.is_valid() - && !self.resources.is_empty() - && self.resources.iter().all(ResourceIdV1::is_valid) - && !self.operations.is_empty() - && !self.sinks.is_empty() - && self.revision > 0 - && self.issued_at < self.expires_at - && self.digest.validate().is_ok() - } - - pub(crate) fn state_at(&self, evaluated_at: UtcMicros) -> GrantStateAtV1 { - match self.state { - GrantStateV1::Revoked => GrantStateAtV1::Revoked, - GrantStateV1::Stale => GrantStateAtV1::Stale, - GrantStateV1::Ambiguous => GrantStateAtV1::Ambiguous, - GrantStateV1::Active if evaluated_at < self.issued_at => GrantStateAtV1::NotYetIssued, - GrantStateV1::Active if evaluated_at >= self.expires_at => GrantStateAtV1::Expired, - GrantStateV1::Active => GrantStateAtV1::Active, - } - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) enum GrantStateAtV1 { - Active, - Revoked, - Stale, - Ambiguous, - NotYetIssued, - Expired, -} diff --git a/crates/tracedecay-policy/src/authorization/input.rs b/crates/tracedecay-policy/src/authorization/input.rs deleted file mode 100644 index a5be0ffc96..0000000000 --- a/crates/tracedecay-policy/src/authorization/input.rs +++ /dev/null @@ -1,520 +0,0 @@ -use std::collections::BTreeSet; -use std::fmt; - -use serde::{Deserialize, Deserializer, Serialize}; -use tracedecay_domain::configuration::{SourceKindV1, UserProfileId}; -use tracedecay_domain::{ActorId, ManifestDigest, ProjectId, UtcMicros, canonical_sha256}; - -/// A bounded, canonical identifier owned by the policy input schema. -/// -/// It represents immutable references only; it is never a path, display -/// label, provider account, branch name, or native object identifier. -#[derive(Clone, Debug, Serialize, schemars::JsonSchema, PartialEq, Eq, PartialOrd, Ord, Hash)] -#[serde(transparent)] -pub struct PolicyIdentifierV1(String); - -impl PolicyIdentifierV1 { - pub fn new(value: impl Into) -> Result { - let value = value.into(); - Self::validate(&value)?; - Ok(Self(value)) - } - - pub fn as_str(&self) -> &str { - &self.0 - } - - pub fn is_valid(&self) -> bool { - Self::validate(&self.0).is_ok() - } - - fn validate(value: &str) -> Result<(), &'static str> { - if value.is_empty() - || value.trim() != value - || value.len() > 512 - || value.chars().any(char::is_control) - { - return Err("policy identifier must be non-empty, trimmed, bounded, and printable"); - } - Ok(()) - } -} - -impl<'de> Deserialize<'de> for PolicyIdentifierV1 { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - Self::new(String::deserialize(deserializer)?).map_err(serde::de::Error::custom) - } -} - -impl fmt::Display for PolicyIdentifierV1 { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter.write_str(&self.0) - } -} - -pub type SourceIdV1 = PolicyIdentifierV1; -pub type SourceBindingIdV1 = PolicyIdentifierV1; -pub type ResourceIdV1 = PolicyIdentifierV1; -pub type GrantIdV1 = PolicyIdentifierV1; - -/// Owner identity is typed and exact. Mutable paths, collection membership, -/// labels, provider accounts, branch names, and native object IDs cannot -/// become owner authority. -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)] -#[serde(tag = "kind", content = "id", rename_all = "snake_case")] -pub enum SourceOwnerV1 { - Project(ProjectId), - Profile(UserProfileId), -} - -impl SourceOwnerV1 { - pub fn is_valid(&self) -> bool { - match self { - Self::Project(id) => id.validate().is_ok(), - Self::Profile(id) => id.validate().is_ok(), - } - } -} - -/// Operations that a source authorization decision may consider. The closed -/// enum prevents a caller from smuggling an unreviewed generic effect through -/// the policy boundary. -#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)] -#[serde(rename_all = "snake_case")] -pub enum TypedOperationV1 { - ProviderFetch, - SourcePageContinuation, - CanonicalAdmission, - ShardSelection, - StatisticsRead, - GraphExpansion, - Hydration, - QueryPageContinuation, - AnchorResolution, - SummaryPublication, - ModelContextDelivery, - HostDelivery, - UiRendering, - Export, - TelemetryWrite, - AnalyzerAdmission, - HistoricalRead, -} - -/// A concrete sink receiving source-derived content or metadata. -#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)] -#[serde(rename_all = "snake_case")] -pub enum SinkKindV1 { - ProviderFetch, - CanonicalStore, - AnalyzerRuntime, - LocalDurableStore, - ModelContext, - HostDelivery, - UiRendering, - Export, - Telemetry, - QueryResponse, -} - -impl SinkKindV1 { - pub const fn is_egress(self) -> bool { - matches!( - self, - Self::ModelContext | Self::HostDelivery | Self::Export | Self::Telemetry - ) - } -} - -/// Ordered disclosure ceiling. Earlier variants are more restrictive. -#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)] -#[serde(rename_all = "snake_case")] -pub enum DisclosureClassV1 { - Metadata, - Summary, - SanitizedContent, - RawContent, -} - -/// Source sensitivity is an input classification, never inferred by policy. -#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)] -#[serde(rename_all = "snake_case")] -pub enum SourceSensitivityV1 { - NonSensitive, - Sensitive, - Restricted, -} - -/// Non-waivable obligations accumulate across every authorization operand. -#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)] -#[serde(rename_all = "snake_case")] -pub enum PrivacyConstraintV1 { - LocalOnly, - SanitizedOnly, - NoRetention, - NoModelContext, - NoTelemetry, - NoExport, -} - -pub type PrivacyConstraintSetV1 = BTreeSet; - -/// Explicit resource limits. Intersections take the pointwise minimum. -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct BudgetSetV1 { - pub requests: u64, - pub bytes: u64, - pub tokens: u64, -} - -impl BudgetSetV1 { - pub fn pointwise_min(&self, other: &Self) -> Self { - Self { - requests: self.requests.min(other.requests), - bytes: self.bytes.min(other.bytes), - tokens: self.tokens.min(other.tokens), - } - } - - pub fn contains(&self, requested: &Self) -> bool { - requested.requests <= self.requests - && requested.bytes <= self.bytes - && requested.tokens <= self.tokens - } -} - -/// Immutable source-capture/storage identity. It intentionally contains no -/// owner, sink, disclosure, local privacy, or grant authority. -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct SourceDefinitionV1 { - pub source_id: SourceIdV1, - pub source_kind: SourceKindV1, - pub revision: u64, - pub digest: ManifestDigest, -} - -impl SourceDefinitionV1 { - pub fn is_valid(&self) -> bool { - self.source_id.is_valid() && self.revision > 0 && self.digest.validate().is_ok() - } -} - -/// Exact immutable definition snapshot supplied by the configuration -/// authority. Policy does not create, mutate, or persist this value. -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct SourceDefinitionSnapshotV1 { - pub definition: SourceDefinitionV1, -} - -impl SourceDefinitionSnapshotV1 { - pub fn is_valid(&self) -> bool { - self.definition.is_valid() - } -} - -/// Binding identity attaches one definition to exactly one typed owner. -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(tag = "kind", rename_all = "snake_case")] -pub enum SourceBindingV1 { - Project { - binding_id: SourceBindingIdV1, - source_id: SourceIdV1, - project_id: ProjectId, - revision: u64, - digest: ManifestDigest, - }, - Profile { - binding_id: SourceBindingIdV1, - source_id: SourceIdV1, - profile_id: UserProfileId, - revision: u64, - digest: ManifestDigest, - }, -} - -impl SourceBindingV1 { - pub fn source_id(&self) -> &SourceIdV1 { - match self { - Self::Project { source_id, .. } | Self::Profile { source_id, .. } => source_id, - } - } - - pub fn owner(&self) -> SourceOwnerV1 { - match self { - Self::Project { project_id, .. } => SourceOwnerV1::Project(project_id.clone()), - Self::Profile { profile_id, .. } => SourceOwnerV1::Profile(profile_id.clone()), - } - } - - pub fn revision(&self) -> u64 { - match self { - Self::Project { revision, .. } | Self::Profile { revision, .. } => *revision, - } - } - - pub fn digest(&self) -> &ManifestDigest { - match self { - Self::Project { digest, .. } | Self::Profile { digest, .. } => digest, - } - } - - pub fn is_valid(&self) -> bool { - match self { - Self::Project { - binding_id, - source_id, - project_id, - revision, - digest, - } => { - binding_id.is_valid() - && source_id.is_valid() - && project_id.validate().is_ok() - && *revision > 0 - && digest.validate().is_ok() - } - Self::Profile { - binding_id, - source_id, - profile_id, - revision, - digest, - } => { - binding_id.is_valid() - && source_id.is_valid() - && profile_id.validate().is_ok() - && *revision > 0 - && digest.validate().is_ok() - } - } - } -} - -/// Exact immutable binding snapshot supplied by the configuration authority. -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct SourceBindingSnapshotV1 { - pub binding: SourceBindingV1, -} - -impl SourceBindingSnapshotV1 { - pub fn is_valid(&self) -> bool { - self.binding.is_valid() - } -} - -/// Typed owner resolution with an explicit revision and digest. A caller must -/// not substitute paths or display labels for this authority. -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct ResolvedOwnerScopeV1 { - pub owner: SourceOwnerV1, - pub revision: u64, - pub digest: ManifestDigest, -} - -impl ResolvedOwnerScopeV1 { - pub fn is_valid(&self) -> bool { - self.owner.is_valid() && self.revision > 0 && self.digest.validate().is_ok() - } -} - -/// Plan-20 source policy metadata. This is deliberately separate from source -/// definition identity so mutable policy cannot become capture identity. -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct SourcePolicyMetadataSnapshotV1 { - pub source_id: SourceIdV1, - pub policy_revision: u64, - pub policy_digest: ManifestDigest, - pub sensitivity: SourceSensitivityV1, - pub disclosure_ceiling: DisclosureClassV1, - pub eligible_sinks: BTreeSet, - pub eligible_operations: BTreeSet, - pub mandatory_privacy: PrivacyConstraintSetV1, -} - -impl SourcePolicyMetadataSnapshotV1 { - pub fn is_valid(&self) -> bool { - self.source_id.is_valid() - && self.policy_revision > 0 - && self.policy_digest.validate().is_ok() - } -} - -/// Current sink policy supplied by the owning configuration/application -/// authority. It is a read-only policy input to this crate. -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct SinkPolicySnapshotV1 { - pub sink: SinkKindV1, - pub policy_revision: u64, - pub policy_digest: ManifestDigest, - pub disclosure_ceiling: DisclosureClassV1, - pub mandatory_privacy: PrivacyConstraintSetV1, - pub available: bool, -} - -impl SinkPolicySnapshotV1 { - pub fn is_valid(&self) -> bool { - self.policy_revision > 0 && self.policy_digest.validate().is_ok() - } -} - -/// The exact requested subset that must be contained by the effective grant. -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct RequestedSourceAccessV1 { - pub resource: ResourceIdV1, - pub operation: TypedOperationV1, - pub sink: SinkKindV1, - pub disclosure: DisclosureClassV1, - pub budget: BudgetSetV1, -} - -impl RequestedSourceAccessV1 { - pub fn is_valid(&self) -> bool { - self.resource.is_valid() - } -} - -/// Explicit completeness/freshness of required immutable inputs. -#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)] -#[serde(rename_all = "snake_case")] -pub enum AuthorizationSnapshotStateV1 { - Complete, - Partial, - Missing, - Stale, - Ambiguous, -} - -/// Content truth is independent from access truth. Policy never infers -/// authoritative deletion from access loss or incomplete input. -#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)] -#[serde(rename_all = "snake_case")] -pub enum ExternalContentStatusV1 { - Live, - Partial, - TemporarilyUnavailable, - AuthoritativeDeleted, -} - -/// Authorization coverage is separate from content status. It captures a -/// mixed visible/authorized resource set without exposing hidden counts. -#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)] -#[serde(rename_all = "snake_case")] -pub enum AuthorizationCoverageV1 { - Complete, - Partial, -} - -/// All policy inputs are immutable values. The caller supplies the clock and -/// all source/policy/sink state; policy performs no lookup or refresh. -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct SourceAuthorizationInputV1 { - pub definition: SourceDefinitionSnapshotV1, - pub binding: SourceBindingSnapshotV1, - pub source_grant: super::grant::CapabilityGrantV1, - pub requester_grant: super::grant::CapabilityGrantV1, - pub resolved_owner_scope: ResolvedOwnerScopeV1, - pub requested_access: RequestedSourceAccessV1, - pub source_policy: SourcePolicyMetadataSnapshotV1, - pub sink_policy: SinkPolicySnapshotV1, - pub content_status: ExternalContentStatusV1, - pub requested_coverage: AuthorizationCoverageV1, - pub snapshot_state: AuthorizationSnapshotStateV1, - pub requester: ActorId, - pub policy_revision: u64, - pub policy_digest: ManifestDigest, - pub configuration_digest: ManifestDigest, - /// Immutable evidence references only. The evaluator records these in its - /// trace but never dereferences, fetches, or persists them. - #[serde(default)] - pub evidence_references: BTreeSet, - pub evaluated_at: UtcMicros, -} - -impl SourceAuthorizationInputV1 { - pub fn is_structurally_valid(&self) -> bool { - self.definition.is_valid() - && self.binding.is_valid() - && self.source_grant.is_valid() - && self.requester_grant.is_valid() - && self.resolved_owner_scope.is_valid() - && self.requested_access.is_valid() - && self.source_policy.is_valid() - && self.sink_policy.is_valid() - && self.requester.validate().is_ok() - && self.policy_revision > 0 - && self.policy_digest.validate().is_ok() - && self.configuration_digest.validate().is_ok() - } - - /// Digest of all input facts, including explicit time and content state. - #[hotpath::measure(label = "policy.authorization.input_digest")] - pub fn input_digest(&self) -> ManifestDigest { - policy_digest("tracedecay.policy.source-authorization-input.v1", self) - } - - /// Digest of every authority/configuration surface that a sink proof pins. - /// The explicit clock and content truth are intentionally excluded: they - /// are re-evaluated at the sink, rather than silently reused. - pub fn authority_fingerprint(&self) -> ManifestDigest { - #[derive(Serialize)] - struct AuthoritySurface<'a> { - definition: &'a SourceDefinitionSnapshotV1, - binding: &'a SourceBindingSnapshotV1, - source_grant: &'a super::grant::CapabilityGrantV1, - requester_grant: &'a super::grant::CapabilityGrantV1, - resolved_owner_scope: &'a ResolvedOwnerScopeV1, - requested_access: &'a RequestedSourceAccessV1, - source_policy: &'a SourcePolicyMetadataSnapshotV1, - sink_policy: &'a SinkPolicySnapshotV1, - requester: &'a ActorId, - policy_revision: u64, - policy_digest: &'a ManifestDigest, - configuration_digest: &'a ManifestDigest, - } - - policy_digest( - "tracedecay.policy.source-authorization-authority-surface.v1", - &AuthoritySurface { - definition: &self.definition, - binding: &self.binding, - source_grant: &self.source_grant, - requester_grant: &self.requester_grant, - resolved_owner_scope: &self.resolved_owner_scope, - requested_access: &self.requested_access, - source_policy: &self.source_policy, - sink_policy: &self.sink_policy, - requester: &self.requester, - policy_revision: self.policy_revision, - policy_digest: &self.policy_digest, - configuration_digest: &self.configuration_digest, - }, - ) - } -} - -/// Stable digest helper used for immutable, serializable policy inputs. -pub(crate) fn policy_digest(domain: &'static str, value: &T) -> ManifestDigest { - match canonical_sha256(&(domain, value)) { - Ok(digest) => digest, - Err(_) => { - // This can only be reached if a future serializable policy type - // violates canonical JSON requirements. Preserve a deterministic - // non-authorizing digest rather than panic or consult external - // state. - ManifestDigest::new(format!("sha256:{}", "0".repeat(64))) - .expect("static policy fallback digest is canonical") - } - } -} diff --git a/crates/tracedecay-policy/src/authorization/intersection.rs b/crates/tracedecay-policy/src/authorization/intersection.rs deleted file mode 100644 index 7aa8347d1b..0000000000 --- a/crates/tracedecay-policy/src/authorization/intersection.rs +++ /dev/null @@ -1,171 +0,0 @@ -use std::collections::BTreeSet; - -use serde::{Deserialize, Serialize}; -use tracedecay_domain::ManifestDigest; - -use super::input::{ - DisclosureClassV1, PrivacyConstraintSetV1, PrivacyConstraintV1, SinkKindV1, - SourceAuthorizationInputV1, SourceOwnerV1, -}; - -/// The non-expanding effective authority used only by a successful decision. -/// -/// Every collection is narrowed to the requested subset. The policy crate -/// cannot turn this value into an effect; application must sink-recheck it. -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct EffectiveSourceGrantV1 { - pub owner: SourceOwnerV1, - pub resources: BTreeSet, - pub operations: BTreeSet, - pub sinks: BTreeSet, - pub disclosure_ceiling: DisclosureClassV1, - pub constraints: PrivacyConstraintSetV1, - pub budgets: super::input::BudgetSetV1, - pub source_grant_digest: ManifestDigest, - pub requester_grant_digest: ManifestDigest, -} - -impl EffectiveSourceGrantV1 { - pub fn permits_requested_access(&self, input: &SourceAuthorizationInputV1) -> bool { - self.owner == input.resolved_owner_scope.owner - && self.resources.contains(&input.requested_access.resource) - && self.operations.contains(&input.requested_access.operation) - && self.sinks.contains(&input.requested_access.sink) - && input.requested_access.disclosure <= self.disclosure_ceiling - && self.budgets.contains(&input.requested_access.budget) - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) enum IntersectionFailureV1 { - OwnerMismatch, - RequesterSubjectMismatch, - ResourceNotGranted, - OperationNotGranted, - SinkNotGranted, - DisclosureTooBroad, - BudgetExceeded, - MandatoryLocalPrivacyBlocksEgress, - SanitizedOnlyBlocksDisclosure, - NoModelContext, - NoRetention, - NoTelemetry, - NoExport, - SinkUnavailable, -} - -pub(crate) fn intersect_source_authority( - input: &SourceAuthorizationInputV1, -) -> Result { - let source_grant = &input.source_grant; - let requester_grant = &input.requester_grant; - let binding_owner = input.binding.binding.owner(); - let resolved_owner = &input.resolved_owner_scope.owner; - - if &binding_owner != resolved_owner - || &source_grant.owner != resolved_owner - || &requester_grant.owner != resolved_owner - { - return Err(IntersectionFailureV1::OwnerMismatch); - } - if source_grant.subject != input.requester || requester_grant.subject != input.requester { - return Err(IntersectionFailureV1::RequesterSubjectMismatch); - } - - let resource_allowed = source_grant - .resources - .contains(&input.requested_access.resource) - && requester_grant - .resources - .contains(&input.requested_access.resource); - if !resource_allowed { - return Err(IntersectionFailureV1::ResourceNotGranted); - } - let operation_allowed = source_grant - .operations - .contains(&input.requested_access.operation) - && requester_grant - .operations - .contains(&input.requested_access.operation); - if !operation_allowed { - return Err(IntersectionFailureV1::OperationNotGranted); - } - let sink_allowed = source_grant.sinks.contains(&input.requested_access.sink) - && requester_grant.sinks.contains(&input.requested_access.sink); - if !sink_allowed { - return Err(IntersectionFailureV1::SinkNotGranted); - } - if !input.sink_policy.available { - return Err(IntersectionFailureV1::SinkUnavailable); - } - - let disclosure_ceiling = source_grant - .disclosure_ceiling - .min(requester_grant.disclosure_ceiling) - .min(input.source_policy.disclosure_ceiling) - .min(input.sink_policy.disclosure_ceiling); - if input.requested_access.disclosure > disclosure_ceiling { - return Err(IntersectionFailureV1::DisclosureTooBroad); - } - - let budgets = source_grant.budgets.pointwise_min(&requester_grant.budgets); - if !budgets.contains(&input.requested_access.budget) { - return Err(IntersectionFailureV1::BudgetExceeded); - } - - let constraints = source_grant - .constraints - .iter() - .chain(requester_grant.constraints.iter()) - .chain(input.source_policy.mandatory_privacy.iter()) - .chain(input.sink_policy.mandatory_privacy.iter()) - .copied() - .collect::(); - - if constraints.contains(&PrivacyConstraintV1::LocalOnly) - && input.requested_access.sink.is_egress() - { - return Err(IntersectionFailureV1::MandatoryLocalPrivacyBlocksEgress); - } - if constraints.contains(&PrivacyConstraintV1::SanitizedOnly) - && input.requested_access.disclosure > DisclosureClassV1::SanitizedContent - { - return Err(IntersectionFailureV1::SanitizedOnlyBlocksDisclosure); - } - if constraints.contains(&PrivacyConstraintV1::NoModelContext) - && input.requested_access.sink == SinkKindV1::ModelContext - { - return Err(IntersectionFailureV1::NoModelContext); - } - if constraints.contains(&PrivacyConstraintV1::NoRetention) - && matches!( - input.requested_access.sink, - SinkKindV1::CanonicalStore | SinkKindV1::LocalDurableStore - ) - { - return Err(IntersectionFailureV1::NoRetention); - } - if constraints.contains(&PrivacyConstraintV1::NoTelemetry) - && input.requested_access.sink == SinkKindV1::Telemetry - { - return Err(IntersectionFailureV1::NoTelemetry); - } - if constraints.contains(&PrivacyConstraintV1::NoExport) - && input.requested_access.sink == SinkKindV1::Export - { - return Err(IntersectionFailureV1::NoExport); - } - - Ok(EffectiveSourceGrantV1 { - owner: input.resolved_owner_scope.owner.clone(), - resources: BTreeSet::from([input.requested_access.resource.clone()]), - operations: BTreeSet::from([input.requested_access.operation]), - sinks: BTreeSet::from([input.requested_access.sink]), - disclosure_ceiling: input.requested_access.disclosure, - constraints, - budgets: input.requested_access.budget.clone(), - source_grant_digest: source_grant.digest.clone(), - requester_grant_digest: requester_grant.digest.clone(), - }) -} diff --git a/crates/tracedecay-policy/src/authorization/mod.rs b/crates/tracedecay-policy/src/authorization/mod.rs deleted file mode 100644 index 26d2e17b5b..0000000000 --- a/crates/tracedecay-policy/src/authorization/mod.rs +++ /dev/null @@ -1,38 +0,0 @@ -//! External-source authorization kernel. -//! -//! The state transition is intentionally one-way: -//! `input -> decision -> source proof -> sink recheck -> admission proof`. -//! Constructors for proofs are private to this module's transition functions. - -mod decision; -mod grant; -mod input; -mod intersection; -mod recheck; -mod state; - -pub(crate) use input::policy_digest; - -pub use decision::{ - PolicyEvaluatorVersionV1, PolicyReasonCodeV1, SourceAuthorizationDecisionV1, - SourceAuthorizationEvaluator, SourceAuthorizationEvaluatorV1, - SourceAuthorizationExpectedDecisionV1, SourceAuthorizationTruthTableV1, - public_source_result_shape, -}; -pub use grant::{CapabilityGrantV1, GrantStateV1}; -pub use input::{ - AuthorizationCoverageV1, AuthorizationSnapshotStateV1, BudgetSetV1, DisclosureClassV1, - ExternalContentStatusV1, GrantIdV1, PolicyIdentifierV1, PrivacyConstraintSetV1, - PrivacyConstraintV1, RequestedSourceAccessV1, ResolvedOwnerScopeV1, ResourceIdV1, SinkKindV1, - SinkPolicySnapshotV1, SourceAuthorizationInputV1, SourceBindingIdV1, SourceBindingSnapshotV1, - SourceBindingV1, SourceDefinitionSnapshotV1, SourceDefinitionV1, SourceIdV1, SourceOwnerV1, - SourcePolicyMetadataSnapshotV1, SourceSensitivityV1, TypedOperationV1, -}; -pub use intersection::EffectiveSourceGrantV1; -pub use recheck::{ - SinkAdmissionProofV1, SinkRecheckDecisionV1, SinkRecheckDispositionV1, - SourceAuthorizationProofV1, issue_source_authorization_proof, recheck_sink_admission, -}; -pub use state::{ - PublicSourceResultShapeV1, SourceAccessDecisionV1, SourceAuthorizationDispositionV1, -}; diff --git a/crates/tracedecay-policy/src/authorization/recheck.rs b/crates/tracedecay-policy/src/authorization/recheck.rs deleted file mode 100644 index 9cf0629101..0000000000 --- a/crates/tracedecay-policy/src/authorization/recheck.rs +++ /dev/null @@ -1,227 +0,0 @@ -use serde::Serialize; -use tracedecay_domain::{ManifestDigest, UtcMicros}; - -use super::decision::{ - PolicyReasonCodeV1, SourceAuthorizationDecisionV1, SourceAuthorizationEvaluator, -}; -use super::input::{ - ExternalContentStatusV1, SourceAuthorizationInputV1, TypedOperationV1, policy_digest, -}; -use super::intersection::EffectiveSourceGrantV1; -use super::state::SourceAuthorizationDispositionV1; - -/// Opaque proof emitted only from an allow decision. Its fields are private so -/// callers cannot manufacture a transition around the evaluator. -#[derive(Clone, Debug, Serialize, PartialEq, Eq)] -pub struct SourceAuthorizationProofV1 { - input_digest: ManifestDigest, - authority_fingerprint: ManifestDigest, - decision_digest: ManifestDigest, - effective_grant: EffectiveSourceGrantV1, - source_grant_expires_at: UtcMicros, - requester_grant_expires_at: UtcMicros, - sink_policy_revision: u64, - sink_policy_digest: ManifestDigest, -} - -impl SourceAuthorizationProofV1 { - pub fn input_digest(&self) -> &ManifestDigest { - &self.input_digest - } - - pub fn authority_fingerprint(&self) -> &ManifestDigest { - &self.authority_fingerprint - } - - pub fn effective_grant(&self) -> &EffectiveSourceGrantV1 { - &self.effective_grant - } - - fn expires_at(&self) -> UtcMicros { - self.source_grant_expires_at - .min(self.requester_grant_expires_at) - } -} - -/// Opaque admission proof required by effect-owning application code. It -/// proves fresh recheck only; it does not execute or authorize a side effect. -#[derive(Clone, Debug, Serialize, PartialEq, Eq)] -pub struct SinkAdmissionProofV1 { - proof_digest: ManifestDigest, - authority_fingerprint: ManifestDigest, - effective_grant: EffectiveSourceGrantV1, - admitted_at: UtcMicros, - expires_at: UtcMicros, -} - -impl SinkAdmissionProofV1 { - pub fn proof_digest(&self) -> &ManifestDigest { - &self.proof_digest - } - - pub fn effective_grant(&self) -> &EffectiveSourceGrantV1 { - &self.effective_grant - } - - pub fn expires_at(&self) -> UtcMicros { - self.expires_at - } -} - -/// Sink recheck is intentionally separate from source authorization: an old -/// allow cannot be reused after grants, binding, owner, policy, configuration, -/// privacy, or sink state drift. -#[derive(Clone, Copy, Debug, Serialize, PartialEq, Eq, PartialOrd, Ord, Hash)] -#[serde(rename_all = "snake_case")] -pub enum SinkRecheckDispositionV1 { - Admit, - Deny, - Indeterminate, -} - -#[derive(Clone, Debug, Serialize, PartialEq, Eq)] -pub struct SinkRecheckDecisionV1 { - pub disposition: SinkRecheckDispositionV1, - pub ordered_reason_codes: Vec, - admission_proof: Option, -} - -impl SinkRecheckDecisionV1 { - pub fn admission_proof(&self) -> Option<&SinkAdmissionProofV1> { - self.admission_proof.as_ref() - } -} - -/// Issue a source proof from a current, fully allowing decision. Indeterminate -/// availability, partial input, denied access, and policy exclusion cannot -/// transition into a proof. -#[hotpath::measure(label = "policy.authorization.issue_proof")] -pub fn issue_source_authorization_proof( - evaluator: &impl SourceAuthorizationEvaluator, - input: &SourceAuthorizationInputV1, - decision: &SourceAuthorizationDecisionV1, -) -> Option { - let proof = evaluate_proof_issuance(evaluator, input, decision); - crate::hotpath_observe::proof_issuance(proof.is_some()); - proof -} - -fn evaluate_proof_issuance( - evaluator: &impl SourceAuthorizationEvaluator, - input: &SourceAuthorizationInputV1, - decision: &SourceAuthorizationDecisionV1, -) -> Option { - if evaluator.evaluate(input) != *decision { - return None; - } - let effective_grant = decision.effective_grant.clone()?; - if !decision.is_authorized() - || decision.disposition != SourceAuthorizationDispositionV1::Allow - || !effective_grant.permits_requested_access(input) - || decision.input_digest != input.input_digest() - || (input.content_status == ExternalContentStatusV1::AuthoritativeDeleted - && input.requested_access.operation != TypedOperationV1::HistoricalRead) - { - return None; - } - Some(SourceAuthorizationProofV1 { - input_digest: decision.input_digest.clone(), - authority_fingerprint: input.authority_fingerprint(), - decision_digest: decision.decision_digest.clone(), - effective_grant, - source_grant_expires_at: input.source_grant.expires_at, - requester_grant_expires_at: input.requester_grant.expires_at, - sink_policy_revision: input.sink_policy.policy_revision, - sink_policy_digest: input.sink_policy.policy_digest.clone(), - }) -} - -/// Re-run authorization against current immutable facts immediately before an -/// application sink. No proof survives a revision or privacy drift. -#[hotpath::measure(label = "policy.authorization.recheck")] -pub fn recheck_sink_admission( - evaluator: &impl SourceAuthorizationEvaluator, - proof: &SourceAuthorizationProofV1, - current: &SourceAuthorizationInputV1, -) -> SinkRecheckDecisionV1 { - let decision = evaluate_sink_recheck(evaluator, proof, current); - crate::hotpath_observe::recheck_outcome(decision.disposition); - decision -} - -fn evaluate_sink_recheck( - evaluator: &impl SourceAuthorizationEvaluator, - proof: &SourceAuthorizationProofV1, - current: &SourceAuthorizationInputV1, -) -> SinkRecheckDecisionV1 { - let fresh = evaluator.evaluate(current); - if !fresh.is_authorized() || fresh.disposition != SourceAuthorizationDispositionV1::Allow { - return SinkRecheckDecisionV1 { - disposition: match fresh.disposition { - SourceAuthorizationDispositionV1::Indeterminate - | SourceAuthorizationDispositionV1::Abstain => { - SinkRecheckDispositionV1::Indeterminate - } - SourceAuthorizationDispositionV1::Allow - | SourceAuthorizationDispositionV1::Deny - | SourceAuthorizationDispositionV1::NotApplicable => SinkRecheckDispositionV1::Deny, - }, - ordered_reason_codes: fresh.ordered_reason_codes, - admission_proof: None, - }; - } - if current.content_status == ExternalContentStatusV1::AuthoritativeDeleted - && current.requested_access.operation != TypedOperationV1::HistoricalRead - { - return SinkRecheckDecisionV1 { - disposition: SinkRecheckDispositionV1::Deny, - ordered_reason_codes: vec![PolicyReasonCodeV1::AuthorizationInputDrift], - admission_proof: None, - }; - } - if current.evaluated_at >= proof.expires_at() { - return SinkRecheckDecisionV1 { - disposition: SinkRecheckDispositionV1::Deny, - ordered_reason_codes: vec![PolicyReasonCodeV1::AuthorizationInputDrift], - admission_proof: None, - }; - } - if current.sink_policy.policy_revision != proof.sink_policy_revision - || current.sink_policy.policy_digest != proof.sink_policy_digest - { - return SinkRecheckDecisionV1 { - disposition: SinkRecheckDispositionV1::Deny, - ordered_reason_codes: vec![PolicyReasonCodeV1::SinkPolicyDrift], - admission_proof: None, - }; - } - let authority_fingerprint = current.authority_fingerprint(); - if authority_fingerprint != proof.authority_fingerprint { - return SinkRecheckDecisionV1 { - disposition: SinkRecheckDispositionV1::Deny, - ordered_reason_codes: vec![PolicyReasonCodeV1::AuthorizationInputDrift], - admission_proof: None, - }; - } - let proof_digest = policy_digest( - "tracedecay.policy.sink-admission-proof.v1", - &( - &proof.input_digest, - &proof.decision_digest, - &fresh.decision_digest, - &authority_fingerprint, - current.evaluated_at, - ), - ); - SinkRecheckDecisionV1 { - disposition: SinkRecheckDispositionV1::Admit, - ordered_reason_codes: fresh.ordered_reason_codes, - admission_proof: Some(SinkAdmissionProofV1 { - proof_digest, - authority_fingerprint, - effective_grant: proof.effective_grant.clone(), - admitted_at: current.evaluated_at, - expires_at: proof.expires_at(), - }), - } -} diff --git a/crates/tracedecay-policy/src/authorization/state.rs b/crates/tracedecay-policy/src/authorization/state.rs deleted file mode 100644 index 8b71c512a4..0000000000 --- a/crates/tracedecay-policy/src/authorization/state.rs +++ /dev/null @@ -1,36 +0,0 @@ -use serde::{Deserialize, Serialize}; - -/// One exhaustive policy disposition. This conveys the evaluator result; it -/// does not itself authorize an application effect. -#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)] -#[serde(rename_all = "snake_case")] -pub enum SourceAuthorizationDispositionV1 { - Allow, - Deny, - Abstain, - NotApplicable, - Indeterminate, -} - -/// Access and content remain independent axes. In particular, -/// `AuthoritativeDeleted` is never synthesized from an authorization failure. -#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)] -#[serde(rename_all = "snake_case")] -pub enum SourceAccessDecisionV1 { - Authorized, - PolicyExcluded, - Unauthorized, -} - -/// The only public shape for resource-addressed hidden, absent, wrong-owner, -/// or unauthorized results. It intentionally has no payload fields. -#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)] -#[serde(rename_all = "snake_case")] -pub enum PublicSourceResultShapeV1 { - NotFoundOrNotAuthorized, - PolicyExcluded, - Live, - Partial, - TemporarilyUnavailable, - AuthoritativeDeleted, -} diff --git a/crates/tracedecay-policy/src/hotpath_observe.rs b/crates/tracedecay-policy/src/hotpath_observe.rs index 1e4dd8a624..02e371df79 100644 --- a/crates/tracedecay-policy/src/hotpath_observe.rs +++ b/crates/tracedecay-policy/src/hotpath_observe.rs @@ -4,63 +4,8 @@ //! reason text, or content. Every call is a no-op unless this crate's //! `hotpath` feature is selected. -use crate::authorization::{ - SinkRecheckDispositionV1, SourceAccessDecisionV1, SourceAuthorizationDispositionV1, -}; use crate::routing::CapabilityRoutingDispositionV1; -/// One bounded outcome class per source authorization decision. Allowed means -/// authorized access with an allowing disposition; every deny and -/// not-applicable disposition counts as denied; stale, ambiguous, abstaining, -/// and unavailable states count as indeterminate. -#[inline] -pub(crate) fn authorization_outcome( - access: SourceAccessDecisionV1, - disposition: SourceAuthorizationDispositionV1, -) { - match (access, disposition) { - (SourceAccessDecisionV1::Authorized, SourceAuthorizationDispositionV1::Allow) => { - hotpath::gauge!("policy.authorization.outcome.allowed").inc(1.0); - } - ( - _, - SourceAuthorizationDispositionV1::Deny - | SourceAuthorizationDispositionV1::NotApplicable, - ) => { - hotpath::gauge!("policy.authorization.outcome.denied").inc(1.0); - } - _ => { - hotpath::gauge!("policy.authorization.outcome.indeterminate").inc(1.0); - } - } -} - -/// One bounded outcome class per sink admission recheck. -#[inline] -pub(crate) fn recheck_outcome(disposition: SinkRecheckDispositionV1) { - match disposition { - SinkRecheckDispositionV1::Admit => { - hotpath::gauge!("policy.authorization.recheck.admitted").inc(1.0); - } - SinkRecheckDispositionV1::Deny => { - hotpath::gauge!("policy.authorization.recheck.denied").inc(1.0); - } - SinkRecheckDispositionV1::Indeterminate => { - hotpath::gauge!("policy.authorization.recheck.indeterminate").inc(1.0); - } - } -} - -/// Proof issuance is all-or-nothing; a refusal is recorded, never silent. -#[inline] -pub(crate) fn proof_issuance(issued: bool) { - if issued { - hotpath::gauge!("policy.authorization.proof.issued").inc(1.0); - } else { - hotpath::gauge!("policy.authorization.proof.refused").inc(1.0); - } -} - /// One bounded outcome class per capability routing decision, plus the size /// of the candidate set the evaluation walked. #[inline] diff --git a/crates/tracedecay-policy/tests/fixtures/source_authorization/core.json b/crates/tracedecay-policy/tests/fixtures/source_authorization/core.json deleted file mode 100644 index 40097a871f..0000000000 --- a/crates/tracedecay-policy/tests/fixtures/source_authorization/core.json +++ /dev/null @@ -1,782 +0,0 @@ -[ - { - "name": "project_authorized_live", - "source_visible": true, - "input": { - "definition": { - "definition": { - "source_id": "source.fixture", - "source_kind": "cursor", - "revision": 1, - "digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - } - }, - "binding": { - "binding": { - "kind": "project", - "binding_id": "binding.fixture", - "source_id": "source.fixture", - "project_id": "project.fixture", - "revision": 1, - "digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" - } - }, - "source_grant": { - "grant_id": "grant.source.fixture", - "issuer": "actor.source", - "subject": "actor.requester", - "owner": { - "kind": "project", - "id": "project.fixture" - }, - "resources": ["resource.fixture"], - "operations": ["provider_fetch"], - "sinks": ["provider_fetch"], - "disclosure_ceiling": "sanitized_content", - "constraints": [], - "budgets": { - "requests": 10, - "bytes": 10000, - "tokens": 1000 - }, - "revision": 1, - "issued_at": 0, - "expires_at": 100, - "digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", - "state": "active" - }, - "requester_grant": { - "grant_id": "grant.requester.fixture", - "issuer": "actor.authority", - "subject": "actor.requester", - "owner": { - "kind": "project", - "id": "project.fixture" - }, - "resources": ["resource.fixture"], - "operations": ["provider_fetch"], - "sinks": ["provider_fetch"], - "disclosure_ceiling": "sanitized_content", - "constraints": [], - "budgets": { - "requests": 8, - "bytes": 8000, - "tokens": 800 - }, - "revision": 1, - "issued_at": 0, - "expires_at": 100, - "digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", - "state": "active" - }, - "resolved_owner_scope": { - "owner": { - "kind": "project", - "id": "project.fixture" - }, - "revision": 1, - "digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" - }, - "requested_access": { - "resource": "resource.fixture", - "operation": "provider_fetch", - "sink": "provider_fetch", - "disclosure": "sanitized_content", - "budget": { - "requests": 1, - "bytes": 1000, - "tokens": 100 - } - }, - "source_policy": { - "source_id": "source.fixture", - "policy_revision": 1, - "policy_digest": "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", - "sensitivity": "non_sensitive", - "disclosure_ceiling": "sanitized_content", - "eligible_sinks": ["provider_fetch"], - "eligible_operations": ["provider_fetch"], - "mandatory_privacy": [] - }, - "sink_policy": { - "sink": "provider_fetch", - "policy_revision": 1, - "policy_digest": "sha256:1111111111111111111111111111111111111111111111111111111111111111", - "disclosure_ceiling": "sanitized_content", - "mandatory_privacy": [], - "available": true - }, - "content_status": "live", - "requested_coverage": "complete", - "snapshot_state": "complete", - "requester": "actor.requester", - "policy_revision": 1, - "policy_digest": "sha256:2222222222222222222222222222222222222222222222222222222222222222", - "configuration_digest": "sha256:3333333333333333333333333333333333333333333333333333333333333333", - "evaluated_at": 10 - }, - "expected": { - "access": "authorized", - "authorization_coverage": "complete", - "disposition": "allow", - "ordered_reason_codes": [ - "input_complete", - "source_grant_active", - "requester_grant_active", - "grant_intersection_non_expanding", - "access_allowed", - "content_live" - ], - "has_effective_grant": true, - "public_shape": "live" - } - }, - { - "name": "project_owner_mismatch", - "source_visible": true, - "input": { - "definition": { - "definition": { - "source_id": "source.fixture", - "source_kind": "cursor", - "revision": 1, - "digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - } - }, - "binding": { - "binding": { - "kind": "project", - "binding_id": "binding.fixture", - "source_id": "source.fixture", - "project_id": "project.other", - "revision": 1, - "digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" - } - }, - "source_grant": { - "grant_id": "grant.source.fixture", - "issuer": "actor.source", - "subject": "actor.requester", - "owner": { - "kind": "project", - "id": "project.fixture" - }, - "resources": ["resource.fixture"], - "operations": ["provider_fetch"], - "sinks": ["provider_fetch"], - "disclosure_ceiling": "sanitized_content", - "constraints": [], - "budgets": { - "requests": 10, - "bytes": 10000, - "tokens": 1000 - }, - "revision": 1, - "issued_at": 0, - "expires_at": 100, - "digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", - "state": "active" - }, - "requester_grant": { - "grant_id": "grant.requester.fixture", - "issuer": "actor.authority", - "subject": "actor.requester", - "owner": { - "kind": "project", - "id": "project.fixture" - }, - "resources": ["resource.fixture"], - "operations": ["provider_fetch"], - "sinks": ["provider_fetch"], - "disclosure_ceiling": "sanitized_content", - "constraints": [], - "budgets": { - "requests": 8, - "bytes": 8000, - "tokens": 800 - }, - "revision": 1, - "issued_at": 0, - "expires_at": 100, - "digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", - "state": "active" - }, - "resolved_owner_scope": { - "owner": { - "kind": "project", - "id": "project.fixture" - }, - "revision": 1, - "digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" - }, - "requested_access": { - "resource": "resource.fixture", - "operation": "provider_fetch", - "sink": "provider_fetch", - "disclosure": "sanitized_content", - "budget": { - "requests": 1, - "bytes": 1000, - "tokens": 100 - } - }, - "source_policy": { - "source_id": "source.fixture", - "policy_revision": 1, - "policy_digest": "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", - "sensitivity": "non_sensitive", - "disclosure_ceiling": "sanitized_content", - "eligible_sinks": ["provider_fetch"], - "eligible_operations": ["provider_fetch"], - "mandatory_privacy": [] - }, - "sink_policy": { - "sink": "provider_fetch", - "policy_revision": 1, - "policy_digest": "sha256:1111111111111111111111111111111111111111111111111111111111111111", - "disclosure_ceiling": "sanitized_content", - "mandatory_privacy": [], - "available": true - }, - "content_status": "live", - "requested_coverage": "complete", - "snapshot_state": "complete", - "requester": "actor.requester", - "policy_revision": 1, - "policy_digest": "sha256:2222222222222222222222222222222222222222222222222222222222222222", - "configuration_digest": "sha256:3333333333333333333333333333333333333333333333333333333333333333", - "evaluated_at": 10 - }, - "expected": { - "access": "unauthorized", - "authorization_coverage": "complete", - "disposition": "deny", - "ordered_reason_codes": [ - "input_complete", - "owner_scope_mismatch" - ], - "has_effective_grant": false, - "public_shape": "not_found_or_not_authorized" - } - }, - { - "name": "mandatory_local_privacy_blocks_host_egress", - "source_visible": true, - "input": { - "definition": { - "definition": { - "source_id": "source.fixture", - "source_kind": "cursor", - "revision": 1, - "digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - } - }, - "binding": { - "binding": { - "kind": "project", - "binding_id": "binding.fixture", - "source_id": "source.fixture", - "project_id": "project.fixture", - "revision": 1, - "digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" - } - }, - "source_grant": { - "grant_id": "grant.source.fixture", - "issuer": "actor.source", - "subject": "actor.requester", - "owner": { - "kind": "project", - "id": "project.fixture" - }, - "resources": ["resource.fixture"], - "operations": ["host_delivery"], - "sinks": ["host_delivery"], - "disclosure_ceiling": "sanitized_content", - "constraints": [], - "budgets": { - "requests": 10, - "bytes": 10000, - "tokens": 1000 - }, - "revision": 1, - "issued_at": 0, - "expires_at": 100, - "digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", - "state": "active" - }, - "requester_grant": { - "grant_id": "grant.requester.fixture", - "issuer": "actor.authority", - "subject": "actor.requester", - "owner": { - "kind": "project", - "id": "project.fixture" - }, - "resources": ["resource.fixture"], - "operations": ["host_delivery"], - "sinks": ["host_delivery"], - "disclosure_ceiling": "sanitized_content", - "constraints": [], - "budgets": { - "requests": 8, - "bytes": 8000, - "tokens": 800 - }, - "revision": 1, - "issued_at": 0, - "expires_at": 100, - "digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", - "state": "active" - }, - "resolved_owner_scope": { - "owner": { - "kind": "project", - "id": "project.fixture" - }, - "revision": 1, - "digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" - }, - "requested_access": { - "resource": "resource.fixture", - "operation": "host_delivery", - "sink": "host_delivery", - "disclosure": "sanitized_content", - "budget": { - "requests": 1, - "bytes": 1000, - "tokens": 100 - } - }, - "source_policy": { - "source_id": "source.fixture", - "policy_revision": 1, - "policy_digest": "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", - "sensitivity": "sensitive", - "disclosure_ceiling": "sanitized_content", - "eligible_sinks": ["host_delivery"], - "eligible_operations": ["host_delivery"], - "mandatory_privacy": ["local_only"] - }, - "sink_policy": { - "sink": "host_delivery", - "policy_revision": 1, - "policy_digest": "sha256:1111111111111111111111111111111111111111111111111111111111111111", - "disclosure_ceiling": "sanitized_content", - "mandatory_privacy": [], - "available": true - }, - "content_status": "live", - "requested_coverage": "complete", - "snapshot_state": "complete", - "requester": "actor.requester", - "policy_revision": 1, - "policy_digest": "sha256:2222222222222222222222222222222222222222222222222222222222222222", - "configuration_digest": "sha256:3333333333333333333333333333333333333333333333333333333333333333", - "evaluated_at": 10 - }, - "expected": { - "access": "unauthorized", - "authorization_coverage": "complete", - "disposition": "deny", - "ordered_reason_codes": [ - "input_complete", - "source_grant_active", - "requester_grant_active", - "grant_intersection_non_expanding", - "mandatory_local_privacy_blocks_egress" - ], - "has_effective_grant": false, - "public_shape": "not_found_or_not_authorized" - } - }, - { - "name": "expired_requester_grant", - "source_visible": true, - "input": { - "definition": { - "definition": { - "source_id": "source.fixture", - "source_kind": "cursor", - "revision": 1, - "digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - } - }, - "binding": { - "binding": { - "kind": "project", - "binding_id": "binding.fixture", - "source_id": "source.fixture", - "project_id": "project.fixture", - "revision": 1, - "digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" - } - }, - "source_grant": { - "grant_id": "grant.source.fixture", - "issuer": "actor.source", - "subject": "actor.requester", - "owner": { - "kind": "project", - "id": "project.fixture" - }, - "resources": ["resource.fixture"], - "operations": ["provider_fetch"], - "sinks": ["provider_fetch"], - "disclosure_ceiling": "sanitized_content", - "constraints": [], - "budgets": { - "requests": 10, - "bytes": 10000, - "tokens": 1000 - }, - "revision": 1, - "issued_at": 0, - "expires_at": 100, - "digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", - "state": "active" - }, - "requester_grant": { - "grant_id": "grant.requester.fixture", - "issuer": "actor.authority", - "subject": "actor.requester", - "owner": { - "kind": "project", - "id": "project.fixture" - }, - "resources": ["resource.fixture"], - "operations": ["provider_fetch"], - "sinks": ["provider_fetch"], - "disclosure_ceiling": "sanitized_content", - "constraints": [], - "budgets": { - "requests": 8, - "bytes": 8000, - "tokens": 800 - }, - "revision": 1, - "issued_at": 0, - "expires_at": 10, - "digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", - "state": "active" - }, - "resolved_owner_scope": { - "owner": { - "kind": "project", - "id": "project.fixture" - }, - "revision": 1, - "digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" - }, - "requested_access": { - "resource": "resource.fixture", - "operation": "provider_fetch", - "sink": "provider_fetch", - "disclosure": "sanitized_content", - "budget": { - "requests": 1, - "bytes": 1000, - "tokens": 100 - } - }, - "source_policy": { - "source_id": "source.fixture", - "policy_revision": 1, - "policy_digest": "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", - "sensitivity": "non_sensitive", - "disclosure_ceiling": "sanitized_content", - "eligible_sinks": ["provider_fetch"], - "eligible_operations": ["provider_fetch"], - "mandatory_privacy": [] - }, - "sink_policy": { - "sink": "provider_fetch", - "policy_revision": 1, - "policy_digest": "sha256:1111111111111111111111111111111111111111111111111111111111111111", - "disclosure_ceiling": "sanitized_content", - "mandatory_privacy": [], - "available": true - }, - "content_status": "live", - "requested_coverage": "complete", - "snapshot_state": "complete", - "requester": "actor.requester", - "policy_revision": 1, - "policy_digest": "sha256:2222222222222222222222222222222222222222222222222222222222222222", - "configuration_digest": "sha256:3333333333333333333333333333333333333333333333333333333333333333", - "evaluated_at": 10 - }, - "expected": { - "access": "unauthorized", - "authorization_coverage": "complete", - "disposition": "deny", - "ordered_reason_codes": [ - "input_complete", - "source_grant_active", - "requester_grant_expired" - ], - "has_effective_grant": false, - "public_shape": "not_found_or_not_authorized" - } - }, - { - "name": "temporarily_unavailable_is_not_deletion", - "source_visible": true, - "input": { - "definition": { - "definition": { - "source_id": "source.fixture", - "source_kind": "cursor", - "revision": 1, - "digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - } - }, - "binding": { - "binding": { - "kind": "project", - "binding_id": "binding.fixture", - "source_id": "source.fixture", - "project_id": "project.fixture", - "revision": 1, - "digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" - } - }, - "source_grant": { - "grant_id": "grant.source.fixture", - "issuer": "actor.source", - "subject": "actor.requester", - "owner": { - "kind": "project", - "id": "project.fixture" - }, - "resources": ["resource.fixture"], - "operations": ["provider_fetch"], - "sinks": ["provider_fetch"], - "disclosure_ceiling": "sanitized_content", - "constraints": [], - "budgets": { - "requests": 10, - "bytes": 10000, - "tokens": 1000 - }, - "revision": 1, - "issued_at": 0, - "expires_at": 100, - "digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", - "state": "active" - }, - "requester_grant": { - "grant_id": "grant.requester.fixture", - "issuer": "actor.authority", - "subject": "actor.requester", - "owner": { - "kind": "project", - "id": "project.fixture" - }, - "resources": ["resource.fixture"], - "operations": ["provider_fetch"], - "sinks": ["provider_fetch"], - "disclosure_ceiling": "sanitized_content", - "constraints": [], - "budgets": { - "requests": 8, - "bytes": 8000, - "tokens": 800 - }, - "revision": 1, - "issued_at": 0, - "expires_at": 100, - "digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", - "state": "active" - }, - "resolved_owner_scope": { - "owner": { - "kind": "project", - "id": "project.fixture" - }, - "revision": 1, - "digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" - }, - "requested_access": { - "resource": "resource.fixture", - "operation": "provider_fetch", - "sink": "provider_fetch", - "disclosure": "sanitized_content", - "budget": { - "requests": 1, - "bytes": 1000, - "tokens": 100 - } - }, - "source_policy": { - "source_id": "source.fixture", - "policy_revision": 1, - "policy_digest": "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", - "sensitivity": "non_sensitive", - "disclosure_ceiling": "sanitized_content", - "eligible_sinks": ["provider_fetch"], - "eligible_operations": ["provider_fetch"], - "mandatory_privacy": [] - }, - "sink_policy": { - "sink": "provider_fetch", - "policy_revision": 1, - "policy_digest": "sha256:1111111111111111111111111111111111111111111111111111111111111111", - "disclosure_ceiling": "sanitized_content", - "mandatory_privacy": [], - "available": true - }, - "content_status": "temporarily_unavailable", - "requested_coverage": "complete", - "snapshot_state": "complete", - "requester": "actor.requester", - "policy_revision": 1, - "policy_digest": "sha256:2222222222222222222222222222222222222222222222222222222222222222", - "configuration_digest": "sha256:3333333333333333333333333333333333333333333333333333333333333333", - "evaluated_at": 10 - }, - "expected": { - "access": "authorized", - "authorization_coverage": "complete", - "disposition": "indeterminate", - "ordered_reason_codes": [ - "input_complete", - "source_grant_active", - "requester_grant_active", - "grant_intersection_non_expanding", - "access_allowed", - "content_temporarily_unavailable" - ], - "has_effective_grant": true, - "public_shape": "temporarily_unavailable" - } - }, - { - "name": "policy_excluded_is_not_unauthorized", - "source_visible": true, - "input": { - "definition": { - "definition": { - "source_id": "source.fixture", - "source_kind": "cursor", - "revision": 1, - "digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - } - }, - "binding": { - "binding": { - "kind": "project", - "binding_id": "binding.fixture", - "source_id": "source.fixture", - "project_id": "project.fixture", - "revision": 1, - "digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" - } - }, - "source_grant": { - "grant_id": "grant.source.fixture", - "issuer": "actor.source", - "subject": "actor.requester", - "owner": { - "kind": "project", - "id": "project.fixture" - }, - "resources": ["resource.fixture"], - "operations": ["provider_fetch"], - "sinks": ["provider_fetch"], - "disclosure_ceiling": "sanitized_content", - "constraints": [], - "budgets": { - "requests": 10, - "bytes": 10000, - "tokens": 1000 - }, - "revision": 1, - "issued_at": 0, - "expires_at": 100, - "digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", - "state": "active" - }, - "requester_grant": { - "grant_id": "grant.requester.fixture", - "issuer": "actor.authority", - "subject": "actor.requester", - "owner": { - "kind": "project", - "id": "project.fixture" - }, - "resources": ["resource.fixture"], - "operations": ["provider_fetch"], - "sinks": ["provider_fetch"], - "disclosure_ceiling": "sanitized_content", - "constraints": [], - "budgets": { - "requests": 8, - "bytes": 8000, - "tokens": 800 - }, - "revision": 1, - "issued_at": 0, - "expires_at": 100, - "digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", - "state": "active" - }, - "resolved_owner_scope": { - "owner": { - "kind": "project", - "id": "project.fixture" - }, - "revision": 1, - "digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" - }, - "requested_access": { - "resource": "resource.fixture", - "operation": "provider_fetch", - "sink": "provider_fetch", - "disclosure": "sanitized_content", - "budget": { - "requests": 1, - "bytes": 1000, - "tokens": 100 - } - }, - "source_policy": { - "source_id": "source.fixture", - "policy_revision": 1, - "policy_digest": "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", - "sensitivity": "non_sensitive", - "disclosure_ceiling": "sanitized_content", - "eligible_sinks": ["provider_fetch"], - "eligible_operations": ["host_delivery"], - "mandatory_privacy": [] - }, - "sink_policy": { - "sink": "provider_fetch", - "policy_revision": 1, - "policy_digest": "sha256:1111111111111111111111111111111111111111111111111111111111111111", - "disclosure_ceiling": "sanitized_content", - "mandatory_privacy": [], - "available": true - }, - "content_status": "live", - "requested_coverage": "complete", - "snapshot_state": "complete", - "requester": "actor.requester", - "policy_revision": 1, - "policy_digest": "sha256:2222222222222222222222222222222222222222222222222222222222222222", - "configuration_digest": "sha256:3333333333333333333333333333333333333333333333333333333333333333", - "evaluated_at": 10 - }, - "expected": { - "access": "policy_excluded", - "authorization_coverage": "complete", - "disposition": "not_applicable", - "ordered_reason_codes": [ - "input_complete", - "operation_policy_excluded" - ], - "has_effective_grant": false, - "public_shape": "policy_excluded" - } - } -] diff --git a/crates/tracedecay-policy/tests/policy_suite/main.rs b/crates/tracedecay-policy/tests/policy_suite/main.rs index e8ed9c0f5d..b0270f0bf9 100644 --- a/crates/tracedecay-policy/tests/policy_suite/main.rs +++ b/crates/tracedecay-policy/tests/policy_suite/main.rs @@ -3,11 +3,8 @@ //! Each module was previously a standalone integration-test binary under //! `tests/.rs`; every one of them linked the same dependency closure. //! Compiled as modules of one binary, each test keeps its old binary name as -//! its module prefix. The `tests/fixtures` truth tables stay where the -//! contracts crate's shared harness also reads them. +//! its module prefix. mod curation_apply; mod routing_admission; -mod sink_recheck; -mod source_authorization; mod work_planner; diff --git a/crates/tracedecay-policy/tests/policy_suite/sink_recheck.rs b/crates/tracedecay-policy/tests/policy_suite/sink_recheck.rs deleted file mode 100644 index f6e86a3949..0000000000 --- a/crates/tracedecay-policy/tests/policy_suite/sink_recheck.rs +++ /dev/null @@ -1,112 +0,0 @@ -use tracedecay_domain::ManifestDigest; -use tracedecay_policy::authorization::{ - ExternalContentStatusV1, GrantStateV1, PolicyReasonCodeV1, SinkRecheckDispositionV1, - SourceAuthorizationEvaluator, SourceAuthorizationEvaluatorV1, SourceAuthorizationTruthTableV1, - issue_source_authorization_proof, recheck_sink_admission, -}; - -const SOURCE_AUTHORIZATION_TRUTH_TABLES: &str = - include_str!("../fixtures/source_authorization/core.json"); - -fn authorized_input() -> tracedecay_policy::authorization::SourceAuthorizationInputV1 { - serde_json::from_str::>(SOURCE_AUTHORIZATION_TRUTH_TABLES) - .expect("checked-in source authorization truth tables deserialize") - .into_iter() - .find(|row| row.name == "project_authorized_live") - .expect("allow fixture exists") - .input -} - -#[test] -fn sink_recheck_issues_a_fresh_proof_only_for_unchanged_authority() { - let evaluator = SourceAuthorizationEvaluatorV1::default(); - let input = authorized_input(); - let decision = evaluator.evaluate(&input); - let proof = issue_source_authorization_proof(&evaluator, &input, &decision) - .expect("an allow decision produces an internal source proof"); - - let mut current = input; - current.evaluated_at.0 += 1; - let recheck = recheck_sink_admission(&evaluator, &proof, ¤t); - - assert_eq!(recheck.disposition, SinkRecheckDispositionV1::Admit); - let admission = recheck - .admission_proof() - .expect("unchanged authority admits with a fresh proof"); - // The admission proof digest binds the source decision digest and the - // fresh recheck decision digest; both must be byte-identical to the - // decisions the evaluator produced before decision construction hashed - // its material once. - assert_eq!( - admission.proof_digest().as_str(), - "sha256:5edfb9bf5933f773a0716a5104a0e045d13c6f11203f20a57a6e0b286a50e25a" - ); -} - -#[test] -fn sink_policy_drift_invalidates_the_old_proof() { - let evaluator = SourceAuthorizationEvaluatorV1::default(); - let input = authorized_input(); - let decision = evaluator.evaluate(&input); - let proof = issue_source_authorization_proof(&evaluator, &input, &decision) - .expect("an allow decision produces an internal source proof"); - - let mut current = input; - current.sink_policy.policy_revision += 1; - current.sink_policy.policy_digest = - ManifestDigest::new(format!("sha256:{}", "9".repeat(64))).expect("fixture digest"); - let recheck = recheck_sink_admission(&evaluator, &proof, ¤t); - - assert_eq!(recheck.disposition, SinkRecheckDispositionV1::Deny); - assert_eq!( - recheck.ordered_reason_codes, - vec![PolicyReasonCodeV1::SinkPolicyDrift] - ); - assert!(recheck.admission_proof().is_none()); -} - -#[test] -fn revoked_grant_between_evaluation_and_sink_recheck_is_denied() { - let evaluator = SourceAuthorizationEvaluatorV1::default(); - let input = authorized_input(); - let decision = evaluator.evaluate(&input); - let proof = issue_source_authorization_proof(&evaluator, &input, &decision) - .expect("an allow decision produces an internal source proof"); - - let mut current = input; - current.requester_grant.state = GrantStateV1::Revoked; - current.evaluated_at.0 += 1; - let recheck = recheck_sink_admission(&evaluator, &proof, ¤t); - - assert_eq!(recheck.disposition, SinkRecheckDispositionV1::Deny); - assert_eq!( - recheck.ordered_reason_codes, - vec![ - PolicyReasonCodeV1::InputComplete, - PolicyReasonCodeV1::SourceGrantActive, - PolicyReasonCodeV1::RequesterGrantRevoked, - ] - ); - assert!(recheck.admission_proof().is_none()); -} - -#[test] -fn authoritative_deletion_requires_historical_authority_at_sink_recheck() { - let evaluator = SourceAuthorizationEvaluatorV1::default(); - let input = authorized_input(); - let decision = evaluator.evaluate(&input); - let proof = issue_source_authorization_proof(&evaluator, &input, &decision) - .expect("an allow decision produces an internal source proof"); - - let mut current = input; - current.content_status = ExternalContentStatusV1::AuthoritativeDeleted; - current.evaluated_at.0 += 1; - let recheck = recheck_sink_admission(&evaluator, &proof, ¤t); - - assert_eq!(recheck.disposition, SinkRecheckDispositionV1::Deny); - assert_eq!( - recheck.ordered_reason_codes, - vec![PolicyReasonCodeV1::AuthorizationInputDrift] - ); - assert!(recheck.admission_proof().is_none()); -} diff --git a/crates/tracedecay-policy/tests/policy_suite/source_authorization.rs b/crates/tracedecay-policy/tests/policy_suite/source_authorization.rs deleted file mode 100644 index f1c93e3afa..0000000000 --- a/crates/tracedecay-policy/tests/policy_suite/source_authorization.rs +++ /dev/null @@ -1,303 +0,0 @@ -use serde_json::json; -use tracedecay_policy::authorization::{ - AuthorizationCoverageV1, DisclosureClassV1, ExternalContentStatusV1, PolicyIdentifierV1, - PolicyReasonCodeV1, PublicSourceResultShapeV1, SinkKindV1, SourceAccessDecisionV1, - SourceAuthorizationEvaluator, SourceAuthorizationEvaluatorV1, SourceAuthorizationTruthTableV1, - TypedOperationV1, issue_source_authorization_proof, public_source_result_shape, -}; -const SOURCE_AUTHORIZATION_TRUTH_TABLES: &str = - include_str!("../fixtures/source_authorization/core.json"); - -/// Byte-exact `(input_digest, decision_digest)` per truth-table row. These -/// are the digests the evaluator produced before decision construction was -/// reworked to hash the decision material once; replay, proof issuance, and -/// sink recheck all bind to these bytes, so a change here is a contract break, -/// not a refactor. -const PINNED_DECISION_DIGESTS: &[(&str, &str, &str)] = &[ - ( - "project_authorized_live", - "sha256:ebd80c5f30861d41091cac8136112220a51b11c65d24fa9d39cf5b15a543a6fe", - "sha256:cc0659b2bf7f5064a771c0bc98c455308a8669cb58f5807edd732b112083849d", - ), - ( - "project_owner_mismatch", - "sha256:2a31d316680e54279e4e233317c5262c7d424a23626a8cd0d768a7ca122882b2", - "sha256:0ced4c613a4cf96bce197747246c9a7b50a93d2e90cfa9a8d1f6fc8b928b7223", - ), - ( - "mandatory_local_privacy_blocks_host_egress", - "sha256:96302fa59d141d0a383843c09c76ac0ac36b4d151502795d0deb8690e34aed33", - "sha256:59d50e411a22c56bab7dc5acfd3cfbb7e6ad6116b984d14b2e45db500451a6cc", - ), - ( - "expired_requester_grant", - "sha256:a72cf235938cd39fbf950eb8a7892435ab13bb0e9e98280b0dcf9fccf3fff555", - "sha256:00bf07d143ed55b229fd9adf327d31e69ff1dc788540e5037bdc7569aa6763af", - ), - ( - "temporarily_unavailable_is_not_deletion", - "sha256:5201a04cbcd8e18a7f62ef87807b8909d3b2cf3c22f80545f5dd0bd5f035acc6", - "sha256:ddaa027062de71c36d8c18ef5044f183851609fb90c59a6730c1c7716b2fa047", - ), - ( - "policy_excluded_is_not_unauthorized", - "sha256:633d4a84397a2d1ccd202b6f6189af5aa83b65f55cf6b70c93b6ecc92b74603c", - "sha256:f460d6e5b9647aecb9d848e03584549fcf3c4223c8addbc1e29849cc0938ae90", - ), -]; - -fn truth_tables() -> Vec { - serde_json::from_str(SOURCE_AUTHORIZATION_TRUTH_TABLES) - .expect("checked-in source authorization truth tables deserialize") -} - -#[test] -fn canonical_source_authorization_truth_tables_hold() { - let evaluator = SourceAuthorizationEvaluatorV1::default(); - - for row in truth_tables() { - let decision = evaluator.evaluate(&row.input); - let (_, input_digest, decision_digest) = PINNED_DECISION_DIGESTS - .iter() - .find(|(name, _, _)| *name == row.name) - .unwrap_or_else(|| panic!("truth-table row {} has no pinned digests", row.name)); - assert_eq!( - decision.input_digest.as_str(), - *input_digest, - "input digest drifted for {}", - row.name - ); - assert_eq!( - decision.decision_digest.as_str(), - *decision_digest, - "decision digest drifted for {}", - row.name - ); - - assert_eq!( - decision.access, row.expected.access, - "unexpected access for {}", - row.name - ); - assert_eq!( - decision.authorization_coverage, row.expected.authorization_coverage, - "unexpected coverage for {}", - row.name - ); - assert_eq!( - decision.disposition, row.expected.disposition, - "unexpected disposition for {}", - row.name - ); - assert_eq!( - decision.ordered_reason_codes, row.expected.ordered_reason_codes, - "unexpected reasons for {}", - row.name - ); - assert_eq!( - decision.effective_grant.is_some(), - row.expected.has_effective_grant, - "unexpected effective-grant presence for {}", - row.name - ); - assert_eq!( - public_source_result_shape(&decision, row.source_visible), - row.expected.public_shape, - "unexpected public shape for {}", - row.name - ); - } -} - -#[test] -fn definition_binding_and_owner_snapshots_remain_separate_authorities() { - let evaluator = SourceAuthorizationEvaluatorV1::default(); - let mut input = truth_tables() - .into_iter() - .find(|row| row.name == "project_authorized_live") - .expect("allow fixture exists") - .input; - - assert_eq!( - &input.definition.definition.source_id, - input.binding.binding.source_id() - ); - assert_eq!( - input.binding.binding.owner(), - input.resolved_owner_scope.owner - ); - - input.definition.definition.source_id = - PolicyIdentifierV1::new("source.definition.other").unwrap(); - let decision = evaluator.evaluate(&input); - - assert_eq!(decision.access, SourceAccessDecisionV1::Unauthorized); - assert_eq!( - decision.ordered_reason_codes, - [ - PolicyReasonCodeV1::InputComplete, - PolicyReasonCodeV1::SourceDefinitionBindingMismatch, - ] - ); -} - -#[test] -fn partial_snapshot_coverage_never_claims_authoritative_deletion() { - let evaluator = SourceAuthorizationEvaluatorV1::default(); - let mut input = truth_tables() - .into_iter() - .find(|row| row.name == "project_authorized_live") - .expect("allow fixture exists") - .input; - input.content_status = ExternalContentStatusV1::Partial; - input.requested_coverage = AuthorizationCoverageV1::Partial; - - let decision = evaluator.evaluate(&input); - - assert_eq!(decision.access, SourceAccessDecisionV1::Authorized); - assert_eq!( - decision.authorization_coverage, - AuthorizationCoverageV1::Partial - ); - assert_eq!( - public_source_result_shape(&decision, true), - PublicSourceResultShapeV1::Partial - ); - assert!( - decision - .ordered_reason_codes - .contains(&PolicyReasonCodeV1::ContentPartial) - ); - assert!( - !decision - .ordered_reason_codes - .contains(&PolicyReasonCodeV1::ContentAuthoritativeDeleted) - ); -} - -#[test] -fn narrowing_a_grant_cannot_widen_an_authorization_decision() { - let evaluator = SourceAuthorizationEvaluatorV1::default(); - let allowed = truth_tables() - .into_iter() - .find(|row| row.name == "project_authorized_live") - .expect("allow fixture exists"); - let baseline = evaluator.evaluate(&allowed.input); - assert_eq!(baseline.access, SourceAccessDecisionV1::Authorized); - - let mut narrowed = allowed.input; - narrowed.requester_grant.disclosure_ceiling = DisclosureClassV1::Summary; - let narrowed_decision = evaluator.evaluate(&narrowed); - - assert_ne!(narrowed_decision.access, SourceAccessDecisionV1::Authorized); - assert!(narrowed_decision.effective_grant.is_none()); -} - -#[test] -fn effective_grant_is_narrowed_to_the_exact_requested_authority() { - let evaluator = SourceAuthorizationEvaluatorV1::default(); - let allowed = truth_tables() - .into_iter() - .find(|row| row.name == "project_authorized_live") - .expect("allow fixture exists"); - let decision = evaluator.evaluate(&allowed.input); - let effective = decision.effective_grant.expect("effective grant"); - - assert_eq!( - effective.disclosure_ceiling, - allowed.input.requested_access.disclosure - ); - assert_eq!(effective.budgets, allowed.input.requested_access.budget); -} - -#[test] -fn sink_policy_must_describe_the_requested_sink() { - let evaluator = SourceAuthorizationEvaluatorV1::default(); - let mut input = truth_tables() - .into_iter() - .find(|row| row.name == "project_authorized_live") - .expect("allow fixture exists") - .input; - input.sink_policy.sink = SinkKindV1::HostDelivery; - - let decision = evaluator.evaluate(&input); - - assert_eq!(decision.access, SourceAccessDecisionV1::Unauthorized); - assert_eq!( - decision.ordered_reason_codes, - vec![ - PolicyReasonCodeV1::InputComplete, - PolicyReasonCodeV1::SinkPolicySinkMismatch, - ] - ); -} - -#[test] -fn mutated_decision_cannot_issue_an_opaque_source_proof() { - let evaluator = SourceAuthorizationEvaluatorV1::default(); - let input = truth_tables() - .into_iter() - .find(|row| row.name == "project_authorized_live") - .expect("allow fixture exists") - .input; - let mut decision = evaluator.evaluate(&input); - decision - .effective_grant - .as_mut() - .expect("effective grant") - .budgets = input.requester_grant.budgets.clone(); - - assert!(issue_source_authorization_proof(&evaluator, &input, &decision).is_none()); -} - -#[test] -fn deleted_content_requires_historical_read_authority_before_sink_admission() { - let evaluator = SourceAuthorizationEvaluatorV1::default(); - let mut input = truth_tables() - .into_iter() - .find(|row| row.name == "project_authorized_live") - .expect("allow fixture exists") - .input; - input.content_status = ExternalContentStatusV1::AuthoritativeDeleted; - - let deleted = evaluator.evaluate(&input); - assert_eq!(deleted.access, SourceAccessDecisionV1::Authorized); - assert!(issue_source_authorization_proof(&evaluator, &input, &deleted).is_none()); - - input.requested_access.operation = TypedOperationV1::HistoricalRead; - input - .source_grant - .operations - .insert(TypedOperationV1::HistoricalRead); - input - .requester_grant - .operations - .insert(TypedOperationV1::HistoricalRead); - input - .source_policy - .eligible_operations - .insert(TypedOperationV1::HistoricalRead); - let historical = evaluator.evaluate(&input); - - assert!(issue_source_authorization_proof(&evaluator, &input, &historical).is_some()); -} - -#[test] -fn unauthorized_public_result_is_indistinguishable_from_not_found() { - let evaluator = SourceAuthorizationEvaluatorV1::default(); - let denied = truth_tables() - .into_iter() - .find(|row| row.name == "project_owner_mismatch") - .expect("owner-mismatch fixture exists"); - let decision = evaluator.evaluate(&denied.input); - let public_shape = public_source_result_shape(&decision, denied.source_visible); - - assert_eq!( - public_shape, - PublicSourceResultShapeV1::NotFoundOrNotAuthorized - ); - assert_eq!( - serde_json::to_value(public_shape).expect("public shape serializes"), - json!("not_found_or_not_authorized") - ); -} diff --git a/crates/tracedecay-privacy/src/detect.rs b/crates/tracedecay-privacy/src/detect.rs index 46ef75e28b..0407a621c2 100644 --- a/crates/tracedecay-privacy/src/detect.rs +++ b/crates/tracedecay-privacy/src/detect.rs @@ -40,7 +40,7 @@ pub enum PrivacyDetectorV1 { SensitiveField, HighEntropyToken, /// Reserved for public V1 compatibility; malformed input is reported by - /// `ClaudeRecordParseErrorV1` before detector findings are constructed. + /// `ObservationRecordParseErrorV1` before detector findings are constructed. MalformedRecord, RecordSizeLimit, StructureLimit, diff --git a/crates/tracedecay-privacy/src/lib.rs b/crates/tracedecay-privacy/src/lib.rs index bb2140bd7a..36101d3d48 100644 --- a/crates/tracedecay-privacy/src/lib.rs +++ b/crates/tracedecay-privacy/src/lib.rs @@ -81,9 +81,8 @@ pub use structured_text::{ sanitize_lcm_payload_text, sanitize_provider_metadata_text, }; pub use tracedecay_capture::{ - ClaudeRecordParseErrorV1, MAX_OBSERVATION_RECORD_BYTES, ObservationRecordParseErrorV1, - ParsedClaudeRecordV1, ParsedObservationRecordV1, PreparedObservationRecordV1, - normalize_prepared_observation_record_v1, parse_claude_record_v1, + MAX_OBSERVATION_RECORD_BYTES, ObservationRecordParseErrorV1, ParsedObservationRecordV1, + PreparedObservationRecordV1, normalize_prepared_observation_record_v1, parse_normalized_observation_record_v1, parse_observation_record_v1, prepare_observation_record_v1, }; diff --git a/crates/tracedecay-privacy/src/sanitize.rs b/crates/tracedecay-privacy/src/sanitize.rs index 986c18e12a..bf8949f6f1 100644 --- a/crates/tracedecay-privacy/src/sanitize.rs +++ b/crates/tracedecay-privacy/src/sanitize.rs @@ -5,7 +5,7 @@ use sha2::{Digest, Sha256}; use thiserror::Error; use tracedecay_domain::canonical_text::encode_lowercase_hex; use tracedecay_domain::{ - CanonicalClaudeSanitizationReceiptMaterialV1, ComponentVersion, DurableClaudeObservationV1, + CanonicalClaudeSanitizationReceiptMaterialV1, ComponentVersion, DurableObservationV1, ObservationContractError, ObservationId, ObservationIdentityMaterialV1, ObservationOrderingDomainV1, ObservationSourceIdentityV1, PayloadReferenceV1, RetentionClass, SanitizationReceiptV1, SanitizerDispositionV1, SensitivityV1, SessionId, @@ -17,7 +17,7 @@ use super::detect::{ redact_sensitive_values, }; use super::structural_id::{StructuralIdProtectionError, protect_sensitive_structural_id}; -use super::{ParseLimits, ParsedClaudeRecordV1, ParsedPolicyLimitViolation}; +use super::{ParseLimits, ParsedObservationRecordV1, ParsedPolicyLimitViolation}; pub(crate) const CLAUDE_SANITIZER_VERSION_V1: &str = "privacy.claude-record.v1"; pub(crate) const OBSERVATION_SANITIZER_VERSION_V1: &str = "privacy.observation-record.v1"; @@ -229,7 +229,7 @@ impl ClaudeRecordSanitizerV1 { /// Sanitizes a parser-issued token without decoding or parsing the record again. pub fn sanitize_parsed( &self, - parsed: ParsedClaudeRecordV1, + parsed: ParsedObservationRecordV1, mut identity: ObservationIdentityMaterialV1, retention_class: RetentionClass, ) -> Result { @@ -338,7 +338,7 @@ impl ClaudeRecordSanitizerV1 { Some(payload_reference), )?; let observation = - DurableClaudeObservationV1::new(identity, receipt, retention_class, detected.payload)?; + DurableObservationV1::new(identity, receipt, retention_class, detected.payload)?; let sanitized_record = SanitizedClaudeRecordV1::issue(&observation); Ok(ClaudeSanitizationOutcomeV1::Durable { observation: Box::new(observation), @@ -592,10 +592,10 @@ fn validate_canonical_structural_identity( /// Its constructor is private so a raw `serde_json::Value` cannot be relabeled /// as sanitized by provider adapters. #[derive(Clone, Debug, PartialEq, Eq)] -pub struct SanitizedClaudeRecordV1(Box); +pub struct SanitizedClaudeRecordV1(Box); impl SanitizedClaudeRecordV1 { - fn issue(observation: &DurableClaudeObservationV1) -> Self { + fn issue(observation: &DurableObservationV1) -> Self { Self(Box::new(observation.clone())) } @@ -611,7 +611,7 @@ impl SanitizedClaudeRecordV1 { #[derive(Clone, Debug)] pub enum ClaudeSanitizationOutcomeV1 { Durable { - observation: Box, + observation: Box, sanitized_record: SanitizedClaudeRecordV1, findings: Vec, }, @@ -630,7 +630,7 @@ pub type SanitizedObservationRecordV1 = SanitizedClaudeRecordV1; pub type ObservationSanitizationOutcomeV1 = ClaudeSanitizationOutcomeV1; impl ClaudeSanitizationOutcomeV1 { - pub fn durable_observation(&self) -> Option<&DurableClaudeObservationV1> { + pub fn durable_observation(&self) -> Option<&DurableObservationV1> { match self { Self::Durable { observation, .. } => Some(observation), Self::Rejected { .. } | Self::Quarantined { .. } => None, diff --git a/crates/tracedecay-privacy/src/tests.rs b/crates/tracedecay-privacy/src/tests.rs index a00a988497..7d15d4f6f3 100644 --- a/crates/tracedecay-privacy/src/tests.rs +++ b/crates/tracedecay-privacy/src/tests.rs @@ -15,11 +15,11 @@ use super::detect::{ }; use super::sanitize::OBSERVATION_SANITIZER_VERSION_V1; use super::{ - CODE_SOURCE_SANITIZER_VERSION_V1, ClaudeRecordParseErrorV1, ClaudeRecordSanitizerV1, - ClaudeSanitizationOutcomeV1, ClaudeSanitizerPolicyV1, CodeSourceShapeV1, DetectionConfidenceV1, + CODE_SOURCE_SANITIZER_VERSION_V1, ClaudeRecordSanitizerV1, ClaudeSanitizationOutcomeV1, + ClaudeSanitizerPolicyV1, CodeSourceShapeV1, DetectionConfidenceV1, LcmSensitiveRedactionPolicyV1, MEMORY_FACT_SANITIZER_VERSION_V1, MemoryFactSanitizationV1, - PrivacyDetectorV1, PrivacySanitizerError, SanitizationActionV1, SanitizationFindingV1, - SanitizedPayloadVerificationError, parse_claude_record_v1, + ObservationRecordParseErrorV1, PrivacyDetectorV1, PrivacySanitizerError, SanitizationActionV1, + SanitizationFindingV1, SanitizedPayloadVerificationError, parse_normalized_observation_record_v1, parse_observation_record_v1, redact_lcm_sensitive_payload, sanitize_code_source_bytes, sanitize_memory_fact_payload, sanitize_provider_metadata_json, verify_memory_fact_sanitization, @@ -58,8 +58,12 @@ fn sanitize_with_identity( record: &[u8], identity: ObservationIdentityMaterialV1, ) -> ClaudeSanitizationOutcomeV1 { - let parsed = parse_claude_record_v1(record, identity.position()) - .expect("parse bounded sanitizer fixture"); + let parsed = parse_observation_record_v1( + record, + identity.position(), + ObservationOrderingDomainV1::FileBytes, + ) + .expect("parse bounded sanitizer fixture"); sanitizer .sanitize_parsed(parsed, identity, retention_class()) .expect("sanitizer should produce an outcome") @@ -238,7 +242,9 @@ fn parsed_record_token_preserves_verified_source_evidence() { let range = ObservationSourceRangeV1::new(start, start + record.len() as u64) .expect("valid parsed-token range"); - let parsed = parse_claude_record_v1(&record, range).expect("parse bounded Claude record"); + let parsed = + parse_observation_record_v1(&record, range, ObservationOrderingDomainV1::FileBytes) + .expect("parse bounded Claude record"); assert_eq!(parsed.encoded_len(), record.len()); assert_eq!(*parsed.source_range(), range); @@ -267,8 +273,9 @@ fn parsed_record_rejects_mismatched_range_and_canonical_oversize() { let mismatched = ObservationSourceRangeV1::new(0, record.len() as u64 + 1) .expect("non-empty mismatched range"); assert_eq!( - parse_claude_record_v1(record, mismatched).err(), - Some(ClaudeRecordParseErrorV1::RangeLengthMismatch) + parse_observation_record_v1(record, mismatched, ObservationOrderingDomainV1::FileBytes) + .err(), + Some(ObservationRecordParseErrorV1::RangeLengthMismatch) ); let at_limit = format!( @@ -278,8 +285,12 @@ fn parsed_record_rejects_mismatched_range_and_canonical_oversize() { assert_eq!(at_limit.len(), 1_048_576); let at_limit_range = ObservationSourceRangeV1::new(0, 1_048_576).expect("non-empty at-limit range"); - let admitted = parse_claude_record_v1(at_limit.as_bytes(), at_limit_range) - .expect("a record at the one-mebibyte limit is admitted"); + let admitted = parse_observation_record_v1( + at_limit.as_bytes(), + at_limit_range, + ObservationOrderingDomainV1::FileBytes, + ) + .expect("a record at the one-mebibyte limit is admitted"); assert_eq!( admitted.value()["message"].as_str().map(str::len), Some(1_048_576 - br#"{"message":""}"#.len()) @@ -289,8 +300,13 @@ fn parsed_record_rejects_mismatched_range_and_canonical_oversize() { let oversized_range = ObservationSourceRangeV1::new(0, oversized.len() as u64) .expect("non-empty oversized range"); assert_eq!( - parse_claude_record_v1(&oversized, oversized_range).err(), - Some(ClaudeRecordParseErrorV1::TooLarge) + parse_observation_record_v1( + &oversized, + oversized_range, + ObservationOrderingDomainV1::FileBytes + ) + .err(), + Some(ObservationRecordParseErrorV1::TooLarge) ); } @@ -299,8 +315,12 @@ fn sanitize_parsed_consumes_token_without_reparsing_raw_bytes() { let mut record = serde_json::to_vec(&json!({"message": "ordinary parsed fixture"})) .expect("serialize parsed sanitizer fixture"); let identity = identity_for(&record); - let parsed = - parse_claude_record_v1(&record, identity.position()).expect("parse sanitizer fixture once"); + let parsed = parse_observation_record_v1( + &record, + identity.position(), + ObservationOrderingDomainV1::FileBytes, + ) + .expect("parse sanitizer fixture once"); record.fill(b'!'); let outcome = ClaudeRecordSanitizerV1::claude_v1() @@ -323,7 +343,12 @@ fn sanitize_parsed_rejects_identity_range_mismatch() { .expect("serialize range fixture"); let shifted_range = ObservationSourceRangeV1::new(1, record.len() as u64 + 1).expect("valid shifted range"); - let parsed = parse_claude_record_v1(&record, shifted_range).expect("parse shifted fixture"); + let parsed = parse_observation_record_v1( + &record, + shifted_range, + ObservationOrderingDomainV1::FileBytes, + ) + .expect("parse shifted fixture"); let error = ClaudeRecordSanitizerV1::claude_v1() .expect("valid Claude V1 sanitizer") @@ -385,7 +410,7 @@ fn provider_sanitizer_uses_provider_neutral_policy_and_receipt_domain() { range, ), ) - .map_err(|_| ClaudeRecordParseErrorV1::NormalizationFailed) + .map_err(|_| ObservationRecordParseErrorV1::NormalizationFailed) }, ) .expect("parse provider fixture"); @@ -454,7 +479,7 @@ fn provider_sanitizer_allows_only_legacy_claude_to_omit_native_record_identity() range, ), ) - .map_err(|_| ClaudeRecordParseErrorV1::NormalizationFailed) + .map_err(|_| ObservationRecordParseErrorV1::NormalizationFailed) }, ) .unwrap(); @@ -520,7 +545,7 @@ fn provider_sanitizer_preserves_stable_public_structural_ids() { range, ), ) - .map_err(|_| ClaudeRecordParseErrorV1::NormalizationFailed) + .map_err(|_| ObservationRecordParseErrorV1::NormalizationFailed) }, ) .unwrap(); @@ -589,7 +614,7 @@ fn provider_sanitizer_protects_credential_shaped_structural_ids_consistently() { range, ), ) - .map_err(|_| ClaudeRecordParseErrorV1::NormalizationFailed) + .map_err(|_| ObservationRecordParseErrorV1::NormalizationFailed) }, ) .unwrap(); @@ -678,7 +703,7 @@ fn provider_neutral_workflow_fact_redaction_leaks_no_raw_secret() { range, ), ) - .map_err(|_| ClaudeRecordParseErrorV1::NormalizationFailed) + .map_err(|_| ObservationRecordParseErrorV1::NormalizationFailed) }, ) .unwrap(); @@ -822,8 +847,13 @@ fn json_is_parsed_before_unknown_fields_are_scanned() { assert_eq!(malformed_record.pop(), Some(b'}')); let malformed_range = identity_for(&malformed_record).position(); assert_eq!( - parse_claude_record_v1(&malformed_record, malformed_range).err(), - Some(ClaudeRecordParseErrorV1::Malformed) + parse_observation_record_v1( + &malformed_record, + malformed_range, + ObservationOrderingDomainV1::FileBytes + ) + .err(), + Some(ObservationRecordParseErrorV1::Malformed) ); } @@ -1270,13 +1300,17 @@ fn invalid_records_stop_at_the_parser_and_policy_limited_records_have_no_payload let scalar = serde_json::to_vec(&json!("ordinary scalar")).expect("serialize scalar fixture"); for (record, expected) in [ - (Vec::new(), ClaudeRecordParseErrorV1::Empty), - (malformed, ClaudeRecordParseErrorV1::Malformed), - (scalar, ClaudeRecordParseErrorV1::NonObject), + (Vec::new(), ObservationRecordParseErrorV1::Empty), + (malformed, ObservationRecordParseErrorV1::Malformed), + (scalar, ObservationRecordParseErrorV1::NonObject), ] { let end = u64::try_from(record.len().max(1)).expect("test record length fits"); let range = ObservationSourceRangeV1::new(0, end).expect("non-empty parser range"); - assert_eq!(parse_claude_record_v1(&record, range).err(), Some(expected)); + assert_eq!( + parse_observation_record_v1(&record, range, ObservationOrderingDomainV1::FileBytes) + .err(), + Some(expected) + ); } let limited_policy = ClaudeSanitizerPolicyV1::claude_v1() @@ -1287,8 +1321,12 @@ fn invalid_records_stop_at_the_parser_and_policy_limited_records_have_no_payload let oversized = serde_json::to_vec(&json!({ "message": "x".repeat(64) })) .expect("serialize oversized fixture"); let identity = identity_for(&oversized); - let parsed = parse_claude_record_v1(&oversized, identity.position()) - .expect("canonical parser accepts policy-limited fixture"); + let parsed = parse_observation_record_v1( + &oversized, + identity.position(), + ObservationOrderingDomainV1::FileBytes, + ) + .expect("canonical parser accepts policy-limited fixture"); let outcome = limited_sanitizer .sanitize_parsed(parsed, identity, retention_class()) .expect("limited sanitizer returns a typed outcome"); @@ -1309,8 +1347,12 @@ fn structure_bound_failures_are_quarantined_without_payloads() { let depth_record = serde_json::to_vec(&json!({ "a": { "b": { "c": "value" } } })) .expect("serialize depth fixture"); let depth_identity = identity_for(&depth_record); - let depth_parsed = parse_claude_record_v1(&depth_record, depth_identity.position()) - .expect("canonical parser accepts policy-limited depth fixture"); + let depth_parsed = parse_observation_record_v1( + &depth_record, + depth_identity.position(), + ObservationOrderingDomainV1::FileBytes, + ) + .expect("canonical parser accepts policy-limited depth fixture"); let depth_outcome = ClaudeRecordSanitizerV1::new(depth_policy) .sanitize_parsed(depth_parsed, depth_identity, retention_class()) .expect("limited sanitizer returns a typed outcome"); @@ -1328,8 +1370,12 @@ fn structure_bound_failures_are_quarantined_without_payloads() { let value_record = serde_json::to_vec(&json!({ "values": [1, 2, 3] })).expect("serialize value-count fixture"); let value_identity = identity_for(&value_record); - let value_parsed = parse_claude_record_v1(&value_record, value_identity.position()) - .expect("canonical parser accepts policy-limited value fixture"); + let value_parsed = parse_observation_record_v1( + &value_record, + value_identity.position(), + ObservationOrderingDomainV1::FileBytes, + ) + .expect("canonical parser accepts policy-limited value fixture"); let value_outcome = ClaudeRecordSanitizerV1::new(value_policy) .sanitize_parsed(value_parsed, value_identity, retention_class()) .expect("limited sanitizer returns a typed outcome"); diff --git a/crates/tracedecay-private-fs/Cargo.toml b/crates/tracedecay-private-fs/Cargo.toml index 79914a3f1c..685c012c9c 100644 --- a/crates/tracedecay-private-fs/Cargo.toml +++ b/crates/tracedecay-private-fs/Cargo.toml @@ -11,6 +11,7 @@ repository = "https://github.com/ScriptedAlchemy/tracedecay" cap-fs-ext = "4.0.2" cap-std = "4.0.2" hotpath.workspace = true +tracing = "0.1" [dev-dependencies] tempfile = "3" diff --git a/crates/tracedecay-private-fs/src/file_lease.rs b/crates/tracedecay-private-fs/src/file_lease.rs new file mode 100644 index 0000000000..a52c9e0532 --- /dev/null +++ b/crates/tracedecay-private-fs/src/file_lease.rs @@ -0,0 +1,167 @@ +use std::fs::File; +use std::io; +use std::ops::{Deref, DerefMut}; + +/// A held advisory file lock that is released explicitly, never by closing. +/// +/// Closing a descriptor releases an `flock` only once every copy of its open +/// file description is closed. A child forked by any thread holds such a copy +/// until it execs, even under `O_CLOEXEC`, so a lock released only by close can +/// outlive its holder and refuse the next acquisition as busy. Unlocking +/// releases it for every copy. +#[derive(Debug)] +pub struct FileLease { + file: File, + label: &'static str, + released: bool, +} + +impl FileLease { + /// Adopts `file`, whose lock the caller has just acquired. `label` names + /// the lease in the warning logged if the implicit release on drop fails. + pub fn held(file: File, label: &'static str) -> Self { + Self { + file, + label, + released: false, + } + } + + /// Releases the lock now, returning the failure instead of logging it. + pub fn release(mut self) -> io::Result<()> { + self.released = true; + self.file.unlock() + } +} + +impl Deref for FileLease { + type Target = File; + + fn deref(&self) -> &File { + &self.file + } +} + +impl DerefMut for FileLease { + fn deref_mut(&mut self) -> &mut File { + &mut self.file + } +} + +impl Drop for FileLease { + fn drop(&mut self) { + if self.released { + return; + } + if let Err(error) = self.file.unlock() { + tracing::warn!(lease = self.label, %error, "file lease could not be released"); + } + } +} + +#[cfg(all(test, unix))] +mod tests { + use std::fs::{File, OpenOptions, TryLockError}; + use std::path::Path; + + use super::FileLease; + + /// A forked child that has not exec'd yet: it shares every open file + /// description of the parent until the returned guard lets it exit. + struct ForkedChild { + pid: libc::pid_t, + release: libc::c_int, + } + + impl ForkedChild { + fn spawn() -> Self { + let mut fds = [0; 2]; + // SAFETY: `fds` is a writable two-element array for `pipe`. + assert_eq!(unsafe { libc::pipe(fds.as_mut_ptr()) }, 0); + let [read_end, write_end] = fds; + // SAFETY: the child only calls async-signal-safe `close`, `read`, + // and `_exit`, so forking a multithreaded test process is sound. + let pid = unsafe { libc::fork() }; + assert!(pid >= 0, "fork failed"); + if pid == 0 { + let mut byte = 0_u8; + // SAFETY: both descriptors are the child's copies of the pipe + // and `byte` is writable for one byte. + unsafe { + libc::close(write_end); + libc::read(read_end, (&raw mut byte).cast(), 1); + libc::_exit(0); + } + } + // SAFETY: `read_end` is this process's open pipe descriptor. + unsafe { libc::close(read_end) }; + Self { + pid, + release: write_end, + } + } + } + + impl Drop for ForkedChild { + fn drop(&mut self) { + let mut status = 0; + // SAFETY: closing the owned write end lets the child's `read` + // return; `pid` is this process's unreaped child. + unsafe { + libc::close(self.release); + libc::waitpid(self.pid, &raw mut status, 0); + } + } + } + + fn open(path: &Path) -> File { + OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(path) + .unwrap() + } + + fn locked(path: &Path) -> File { + let file = open(path); + file.try_lock().unwrap(); + file + } + + /// One test, so no parallel test's forked child inherits these locks. + #[test] + fn lease_is_reacquirable_after_drop_or_release_while_a_forked_child_shares_it() { + let temp = tempfile::tempdir().unwrap(); + + let closed_only = temp.path().join("closed-only.lock"); + let file = locked(&closed_only); + let child = ForkedChild::spawn(); + drop(file); + assert!( + matches!(open(&closed_only).try_lock(), Err(TryLockError::WouldBlock)), + "control: closing alone must leave the child's shared description locked" + ); + drop(child); + open(&closed_only).try_lock().unwrap(); + + let dropped = temp.path().join("dropped.lock"); + let lease = FileLease::held(locked(&dropped), "test"); + let child = ForkedChild::spawn(); + drop(lease); + open(&dropped) + .try_lock() + .expect("a dropped lease must not stay held by a forked child"); + drop(child); + + let released = temp.path().join("released.lock"); + let lease = FileLease::held(locked(&released), "test"); + let child = ForkedChild::spawn(); + lease.release().unwrap(); + open(&released) + .try_lock() + .expect("a released lease must not stay held by a forked child"); + drop(child); + } +} diff --git a/crates/tracedecay-private-fs/src/lib.rs b/crates/tracedecay-private-fs/src/lib.rs index a8794b0330..168d700025 100644 --- a/crates/tracedecay-private-fs/src/lib.rs +++ b/crates/tracedecay-private-fs/src/lib.rs @@ -6,8 +6,11 @@ use std::fs::File; use std::io; pub mod capability_dir; +mod file_lease; pub mod framed_log; mod rename_noreplace; + +pub use file_lease::FileLease; #[cfg(windows)] pub mod windows_file; @@ -51,15 +54,6 @@ impl std::error::Error for PrivateFileCreationFailure { } } -/// Receipt from [`make_private_directory`]: the pre-heal state observed on the -/// exact directory handle that was re-permissioned. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct MadePrivateDirectory { - /// Unix permission bits observed before re-permissioning; `None` on - /// platforms without Unix modes. - pub previous_unix_mode: Option, -} - #[cfg(windows)] pub mod windows; @@ -175,13 +169,12 @@ mod unix { /// opened handle, when the current user owns it. /// /// The directory analogue of [`make_private_file`]: creation-time privacy - /// belongs to [`create_private_directory`], while this converges a legacy - /// directory an older binary created under a permissive umask. It never - /// follows symlinks, refuses a directory another user owns (ownership is + /// belongs to [`create_private_directory`], while this tightens a directory + /// the caller created through an ordinary path. It never follows symlinks, refuses a directory another user owns (ownership is /// the proof the caller may tighten it), and re-validates the handle after /// tightening so a concurrent swap cannot smuggle a non-private object. #[hotpath::measure(label = "private_fs.make_private_directory")] - pub fn make_private_directory(path: &Path) -> io::Result { + pub fn make_private_directory(path: &Path) -> io::Result<()> { let mut options = fs::OpenOptions::new(); options .read(true) @@ -196,12 +189,8 @@ mod unix { "filesystem handle is not owned by the current user", )); } - let previous_mode = metadata.permissions().mode() & 0o777; file.set_permissions(fs::Permissions::from_mode(0o700))?; - validate_handle(&file, true, 0o700)?; - Ok(crate::MadePrivateDirectory { - previous_unix_mode: Some(previous_mode), - }) + validate_handle(&file, true, 0o700) } pub fn validate_directory_path(path: &Path) -> io::Result<()> { @@ -369,6 +358,7 @@ mod lock_contention_tests { #[cfg(all(test, unix))] mod tests { use std::os::unix::fs::{MetadataExt, PermissionsExt, symlink}; + use std::path::Path; use tempfile::tempdir; @@ -414,16 +404,15 @@ mod tests { } #[test] - fn owned_permissive_directory_is_healed_through_its_handle() { + fn owned_permissive_directory_is_tightened_through_its_handle() { let temp = tempdir().unwrap(); - let directory = temp.path().join("legacy"); + let directory = temp.path().join("permissive"); create_private_directory(&directory).unwrap(); std::fs::set_permissions(&directory, std::fs::Permissions::from_mode(0o775)).unwrap(); assert!(open_private_directory(&directory).is_err()); - let receipt = super::make_private_directory(&directory).unwrap(); + super::make_private_directory(&directory).unwrap(); - assert_eq!(receipt.previous_unix_mode, Some(0o775)); assert_eq!( std::fs::metadata(&directory).unwrap().permissions().mode() & 0o777, 0o700 @@ -432,17 +421,31 @@ mod tests { } #[test] - fn directory_heal_rejects_symlinks_and_non_directories() { + fn directory_tightening_rejects_symlinks_and_non_directories() { + let mode = |path: &Path| std::fs::metadata(path).unwrap().permissions().mode() & 0o777; let temp = tempdir().unwrap(); let directory = temp.path().join("target"); create_private_directory(&directory).unwrap(); + std::fs::set_permissions(&directory, std::fs::Permissions::from_mode(0o755)).unwrap(); let link = temp.path().join("link"); symlink(&directory, &link).unwrap(); - assert!(super::make_private_directory(&link).is_err()); + let refused = super::make_private_directory(&link).unwrap_err(); + // `O_NOFOLLOW | O_DIRECTORY` on a symlink fails ENOTDIR on Linux and + // ELOOP (normalized to InvalidInput) on the BSDs. + assert!( + matches!( + refused.kind(), + std::io::ErrorKind::NotADirectory | std::io::ErrorKind::InvalidInput + ), + "{refused:?}" + ); + assert_eq!(mode(&directory), 0o755, "the link target stays untouched"); let file_path = temp.path().join("regular"); drop(create_private_file(&file_path).unwrap()); - assert!(super::make_private_directory(&file_path).is_err()); + let refused = super::make_private_directory(&file_path).unwrap_err(); + assert_eq!(refused.kind(), std::io::ErrorKind::NotADirectory); + assert_eq!(mode(&file_path), 0o600, "the file stays untouched"); } #[test] diff --git a/crates/tracedecay-private-fs/src/windows.rs b/crates/tracedecay-private-fs/src/windows.rs index 5306b497fc..8699893176 100644 --- a/crates/tracedecay-private-fs/src/windows.rs +++ b/crates/tracedecay-private-fs/src/windows.rs @@ -234,12 +234,12 @@ pub fn make_private_file(path: &Path) -> io::Result { /// Protect an existing directory through its exact opened handle, when the /// current user may rewrite its owner and DACL. /// -/// The directory analogue of [`make_private_file`]: it converges a legacy -/// directory an older binary created without the protected private ACL. The +/// The directory analogue of [`make_private_file`]: it protects a directory +/// the caller created through an ordinary path. The /// `WRITE_DAC | WRITE_OWNER` open is the authorization proof, a caller that /// cannot take ownership of the object is refused by the open itself. #[hotpath::measure(label = "private_fs.make_private_directory")] -pub fn make_private_directory(path: &Path) -> io::Result { +pub fn make_private_directory(path: &Path) -> io::Result<()> { let file = open_handle_with_share( path, OPEN_EXISTING, @@ -249,10 +249,7 @@ pub fn make_private_directory(path: &Path) -> io::Result` path intact. pub use tracedecay_runtime_core::config::{ DB_FILENAME, TRACEDECAY_DIR, USER_DATA_DIR_ENV, active_data_dir_name, db_filename, - discover_project_root, get_project_db_path, get_tracedecay_dir, has_project_database, - is_ambient_project_root, user_data_dir, + discover_project_root, get_tracedecay_dir, is_ambient_project_root, user_data_dir, }; /// The shared generated/vendored segment list and its membership test moved @@ -48,106 +45,17 @@ pub use tracedecay_runtime_core::config::{GENERATED_DIR_SEGMENTS, is_generated_d /// display/routing context only; [`ProjectId`] remains the authority key. pub use tracedecay_configuration::config::RuntimeConfigurationTarget; -/// A complete resolved configuration pinned to one revision before a runtime -/// component starts. No caller may re-read mutable legacy input after holding -/// this value. -/// -/// The shared runtime settings live in the embedded -/// [`tracedecay_configuration::config::PinnedRuntimeConfiguration`], which is -/// the one validated snapshot/revision binding; the composition root only -/// layers its daemon-only policy on top. Both are materialized once at -/// construction, so the fields stay private: there is no way to hold this -/// value with settings that disagree with its snapshot. -#[derive(Clone, Debug)] -pub struct DaemonRuntimeConfiguration { - runtime: tracedecay_configuration::config::PinnedRuntimeConfiguration, - config: TraceDecayConfig, -} - -impl DaemonRuntimeConfiguration { - /// Materializes the legacy runtime shape from a complete typed snapshot. - /// The conversion rejects missing or wrongly typed settings rather than - /// adding adapter-local defaults. - pub fn new( - target: RuntimeConfigurationTarget, - revision_id: ConfigurationRevisionId, - snapshot: ConfigurationSnapshotV1, - ) -> Result { - Self::from_runtime( - tracedecay_configuration::config::PinnedRuntimeConfiguration::new( - target, - revision_id, - snapshot, - )?, - ) - } - - /// Layers the daemon-only settings over an already validated runtime pin. - /// Shared settings are taken from the pin, never decoded a second time. - #[hotpath::measure(label = "daemon.config.materialize")] - pub fn from_runtime( - runtime: tracedecay_configuration::config::PinnedRuntimeConfiguration, - ) -> Result { - let config = TraceDecayConfig::from_runtime(&runtime)?; - Ok(Self { runtime, config }) - } - - pub fn into_runtime(self) -> tracedecay_configuration::config::PinnedRuntimeConfiguration { - self.runtime - } - - pub fn target(&self) -> &RuntimeConfigurationTarget { - self.runtime.target() - } - - pub fn revision_id(&self) -> &ConfigurationRevisionId { - self.runtime.revision_id() - } - - pub fn snapshot(&self) -> &ConfigurationSnapshotV1 { - self.runtime.snapshot() - } - - pub fn config(&self) -> &TraceDecayConfig { - &self.config - } - - pub fn into_config(self) -> TraceDecayConfig { - self.config - } - - /// Splits the daemon runtime shape from the runtime pin the configuration - /// control plane retains. - pub fn into_parts( - self, - ) -> ( - TraceDecayConfig, - tracedecay_configuration::config::PinnedRuntimeConfiguration, - ) { - (self.config, self.runtime) - } - - /// The same revision and settings routed under another root of the same - /// registered project. Only the non-authoritative route and the legacy - /// `root_dir` metadata change; nothing is decoded again. - fn with_project_root(mut self, project_root: &Path) -> Self { - self.runtime = self.runtime.with_project_root(project_root); - self.config.root_dir = project_root.to_string_lossy().to_string(); - self - } -} - /// Process-local, immutable-after-publication lookup cache. The daemon owns /// refreshing it when a configuration revision activates; hook paths only /// perform an in-memory lookup. #[derive(Default)] pub struct RuntimeConfigurationCache { - by_project: RwLock>, + by_project: RwLock>, project_by_root: RwLock>, } impl RuntimeConfigurationCache { - pub fn insert(&self, configuration: DaemonRuntimeConfiguration) { + pub fn insert(&self, configuration: PinnedRuntimeConfiguration) { let project_id = configuration.target().project_id.as_str().to_owned(); let project_root = configuration.target().project_root.clone(); self.by_project @@ -160,7 +68,7 @@ impl RuntimeConfigurationCache { .insert(project_root, project_id); } - pub fn for_project(&self, project_id: &ProjectId) -> Result { + pub fn for_project(&self, project_id: &ProjectId) -> Result { self.by_project .read() .unwrap_or_else(std::sync::PoisonError::into_inner) @@ -174,7 +82,7 @@ impl RuntimeConfigurationCache { }) } - pub fn for_root(&self, project_root: &Path) -> Result { + pub fn for_root(&self, project_root: &Path) -> Result { let project_id = self .project_by_root .read() @@ -209,7 +117,7 @@ impl tracedecay_dashboard_api::config::DashboardConfigurationReadPort &self, project_root: &Path, ) -> Result { - Ok(self.for_root(project_root)?.into_runtime()) + self.for_root(project_root) } fn is_in_gitignore(&self, project_root: &Path) -> bool { @@ -230,7 +138,7 @@ pub fn install_dashboard_configuration_read_port() -> Result<()> { } /// Publishes one daemon-resolved snapshot for runtime and hook consumers. -pub fn install_pinned_runtime_configuration(configuration: DaemonRuntimeConfiguration) { +pub fn install_pinned_runtime_configuration(configuration: PinnedRuntimeConfiguration) { runtime_configuration_cache().insert(configuration); } @@ -269,7 +177,7 @@ pub fn runtime_configuration_target_for_project_id( pub fn runtime_configuration_for_layout( project_root: &Path, layout: &tracedecay_runtime_core::storage::StoreLayout, -) -> Result { +) -> Result { let target = runtime_configuration_target_for_layout(project_root, layout)?; let configuration = runtime_configuration_cache() .for_project(&target.project_id)? @@ -296,7 +204,7 @@ pub async fn resolve_runtime_configuration_for_registered_database( project_root: &Path, layout: &tracedecay_runtime_core::storage::StoreLayout, database: RegisteredGlobalDbLeaseV1, -) -> Result { +) -> Result { let target = runtime_configuration_target_for_layout(project_root, layout)?; validate_registered_configuration_database(&target, database.as_ref())?; if let Ok(configuration) = runtime_configuration_cache().for_project(&target.project_id) { @@ -323,7 +231,7 @@ pub async fn resolve_runtime_configuration_for_registered_database( /// open. Daemon composition consumes this bundle instead of opening a second /// configuration database or resolving a second snapshot. pub struct OpenedRuntimeConfiguration { - pub configuration: DaemonRuntimeConfiguration, + pub configuration: PinnedRuntimeConfiguration, /// Exact daemon-owned registered session runtime used to resolve this /// snapshot. Configuration composition retains this authority directly; /// it never reacquires the physical database by path. @@ -336,14 +244,13 @@ impl OpenedRuntimeConfiguration { pub fn into_parts( self, ) -> ( - TraceDecayConfig, + RuntimeTraceDecayConfig, tracedecay_configuration::config::OpenedRuntimeConfiguration, ) { - let (config, runtime) = self.configuration.into_parts(); ( - config, + self.configuration.config().clone(), tracedecay_configuration::config::OpenedRuntimeConfiguration::new( - runtime, + self.configuration, self.registered_database, ), ) @@ -352,28 +259,22 @@ impl OpenedRuntimeConfiguration { /// Root-owned pin cache behind the lower crate's /// [`tracedecay_configuration::config::PinnedRuntimeConfigurationCachePort`]. -/// Publication layers the daemon-only settings over the published runtime pin -/// once; a cached read hands the embedded runtime pin back without decoding. struct RootPinnedRuntimeConfigurationCache; impl tracedecay_configuration::config::PinnedRuntimeConfigurationCachePort for RootPinnedRuntimeConfigurationCache { - fn publish( - &self, - configuration: tracedecay_configuration::config::PinnedRuntimeConfiguration, - ) -> Result<()> { - install_pinned_runtime_configuration(DaemonRuntimeConfiguration::from_runtime( - configuration, - )?); + fn publish(&self, configuration: PinnedRuntimeConfiguration) -> Result<()> { + install_pinned_runtime_configuration(configuration); Ok(()) } - fn cached_for_root( - &self, - project_root: &Path, - ) -> Result { - Ok(cached_runtime_configuration(project_root)?.into_runtime()) + fn cached_for_root(&self, project_root: &Path) -> Result { + cached_runtime_configuration(project_root) + } + + fn cached_for_project(&self, project_id: &ProjectId) -> Result { + runtime_configuration_cache().for_project(project_id) } } @@ -484,7 +385,7 @@ async fn initialize_canonical_project_configuration( async fn open_runtime_configuration_from_store( target: RuntimeConfigurationTarget, store: &GlobalDbConfigurationControlStore<'_>, -) -> Result { +) -> Result { if let Err(error) = store.current().await { if !store .is_uninitialized() @@ -574,7 +475,7 @@ async fn open_runtime_configuration_from_store( } } let configuration = - DaemonRuntimeConfiguration::new(target, current.revision_id, current.snapshot)?; + PinnedRuntimeConfiguration::new(target, current.revision_id, current.snapshot)?; install_pinned_runtime_configuration(configuration.clone()); Ok(configuration) } @@ -588,7 +489,7 @@ pub async fn ensure_runtime_configuration_for_registered_database( project_root: &Path, layout: &tracedecay_runtime_core::storage::StoreLayout, database: RegisteredGlobalDbLeaseV1, -) -> Result { +) -> Result { Ok( open_runtime_configuration_for_registered_database(project_root, layout, database) .await? @@ -617,7 +518,7 @@ pub async fn open_runtime_configuration_for_registered_database_read_only( async fn open_runtime_configuration_read_only_from_store( target: RuntimeConfigurationTarget, store: &GlobalDbConfigurationControlStore<'_>, -) -> Result { +) -> Result { if store .is_uninitialized() .await @@ -630,7 +531,7 @@ async fn open_runtime_configuration_read_only_from_store( } let current = store.current().await.map_err(map_configuration_error)?; let configuration = - DaemonRuntimeConfiguration::new(target, current.revision_id, current.snapshot)?; + PinnedRuntimeConfiguration::new(target, current.revision_id, current.snapshot)?; install_pinned_runtime_configuration(configuration.clone()); Ok(configuration) } @@ -693,7 +594,7 @@ fn map_configuration_error(error: ConfigurationError) -> TraceDecayError { /// Returns a cached configuration without resolving a layout, opening a /// database, performing IPC, or reading a file. This is the hook-safe lookup. -pub fn cached_runtime_configuration(project_root: &Path) -> Result { +pub fn cached_runtime_configuration(project_root: &Path) -> Result { runtime_configuration_cache().for_root(project_root) } @@ -703,7 +604,7 @@ pub fn cached_runtime_configuration(project_root: &Path) -> Result Result { +) -> Result { let target = runtime_configuration_target_for_project_id(project_root, project_id)?; Ok(runtime_configuration_cache() .for_project(&target.project_id)? @@ -712,14 +613,16 @@ pub fn cached_runtime_configuration_for_project_id( pub fn cached_sync_config(project_root: &Path) -> Result { Ok(cached_runtime_configuration(project_root)? - .into_config() - .sync) + .config() + .sync + .clone()) } pub fn cached_telemetry_config(project_root: &Path) -> Result { Ok(cached_runtime_configuration(project_root)? - .into_config() - .telemetry) + .config() + .telemetry + .clone()) } fn config_error(message: impl Into) -> TraceDecayError { @@ -728,20 +631,6 @@ fn config_error(message: impl Into) -> TraceDecayError { } } -pub async fn get_config_path_with_identity(project_root: &Path) -> PathBuf { - if let Ok(layout) = - crate::project::TraceDecay::resolve_store_layout_for_identity(project_root).await - { - return layout.config_path; - } - get_config_path(project_root) -} - -pub async fn load_config_with_identity(project_root: &Path) -> Result { - let config_path = get_config_path_with_identity(project_root).await; - load_config_from_path(project_root, &config_path) -} - #[hotpath::measure(label = "daemon.config.discover", future = true)] pub async fn discover_project_root_with_identity(start: &Path) -> Option { if let Some(root) = discover_project_root(start) { diff --git a/crates/tracedecay-project/src/config/tests.rs b/crates/tracedecay-project/src/config/tests.rs index c18e329a23..27f350fb99 100644 --- a/crates/tracedecay-project/src/config/tests.rs +++ b/crates/tracedecay-project/src/config/tests.rs @@ -1,7 +1,6 @@ use std::fs; use std::process::Command; use tempfile::TempDir; -use tracedecay_configuration::{TraceDecayConfig, get_config_path, save_config_to_path}; #[tokio::test] async fn discover_project_root_with_identity_does_not_open_registry_only_store() { @@ -83,7 +82,7 @@ async fn discover_project_root_with_identity_does_not_open_registry_only_store() } #[tokio::test] -async fn config_path_with_identity_does_not_open_registry_without_enrollment() { +async fn store_layout_for_identity_does_not_open_registry_without_enrollment() { let _profile = super::PinnedUserDataDir::new(); let profile_root = tracedecay_runtime_core::storage::default_profile_root().unwrap(); let gdb = @@ -132,26 +131,12 @@ async fn config_path_with_identity_does_not_open_registry_without_enrollment() { }, ) .unwrap(); - save_config_to_path( - &identity_layout.config_path, - &TraceDecayConfig { - root_dir: "identity-config".to_string(), - ..TraceDecayConfig::default() - }, - ) - .unwrap(); - assert_eq!( - super::get_config_path_with_identity(&project_root).await, - get_config_path(&project_root) - ); - assert_eq!( - super::load_config_with_identity(&project_root) - .await - .unwrap() - .root_dir, - project_root.to_string_lossy() - ); + if let Ok(selected) = + crate::project::TraceDecay::resolve_store_layout_for_identity(&project_root).await + { + assert_ne!(selected.data_root, identity_layout.data_root); + } } #[tokio::test] @@ -209,15 +194,22 @@ async fn discover_project_root_with_identity_preserves_sync_fast_path() { let project_dir = TempDir::new().unwrap(); let project_root = project_dir.path().canonicalize().unwrap(); - let db_dir = super::get_tracedecay_dir(&project_root); - fs::create_dir_all(&db_dir).unwrap(); - fs::write(super::get_project_db_path(&project_root), b"").unwrap(); + let store = tracedecay_runtime_core::storage::default_profile_sharded_layout( + &project_root, + &super::user_data_dir().unwrap(), + ) + .unwrap(); + fs::create_dir_all(&store.data_root).unwrap(); + fs::write(&store.graph_db_path, b"").unwrap(); - let sync = super::discover_project_root(&project_root); - assert!(sync.is_some(), "sync resolver must see a repo-local db"); + assert_eq!( + super::discover_project_root(&project_root), + Some(project_root.clone()), + "sync resolver must see the path-local store" + ); assert_eq!( super::discover_project_root_with_identity(&project_root).await, - sync, + Some(project_root), "identity wrapper fast path must equal the sync result" ); } @@ -243,13 +235,14 @@ mod runtime_configuration_cutover { use crate::config::registry::ConfigurationRegistry; use crate::config::resolver::{ConfigurationLayerV1, resolve_configuration}; use crate::config::{ - DaemonRuntimeConfiguration, RuntimeConfigurationCache, RuntimeConfigurationTarget, - cached_runtime_configuration, cached_sync_config, cached_telemetry_config, - install_pinned_runtime_configuration, runtime_configuration_for_layout, + RuntimeConfigurationCache, RuntimeConfigurationTarget, cached_runtime_configuration, + cached_sync_config, cached_telemetry_config, install_pinned_runtime_configuration, + runtime_configuration_for_layout, }; use crate::test_support::host_admission::HostAdmissionTestRuntimeV1; use tracedecay_configuration::ProjectConfigurationRuntime; - use tracedecay_configuration::TraceDecayConfig; + use tracedecay_configuration::SyncConfig; + use tracedecay_configuration::config::PinnedRuntimeConfiguration; use tracedecay_global_db::configuration::contracts::{ ConfigurationControlStore, ConfigurationMutationAuthority, DirectConfigurationMutation, }; @@ -284,7 +277,7 @@ mod runtime_configuration_cutover { ) .expect("explicit settings layer resolves") .snapshot; - let pinned = DaemonRuntimeConfiguration::new( + let pinned = PinnedRuntimeConfiguration::new( RuntimeConfigurationTarget { project_id, project_root: root.path().to_path_buf(), @@ -318,15 +311,15 @@ mod runtime_configuration_cutover { assert_eq!( cached_runtime_configuration(root.path()) .expect("cache lookup") - .config - .root_dir, - root.path().to_string_lossy().to_string(), - "root metadata comes from the non-authoritative published route" + .target() + .project_root, + root.path(), + "the root comes from the non-authoritative published route" ); } #[test] - fn runtime_cache_retargets_legacy_root_metadata_per_cached_root() { + fn runtime_cache_retargets_the_route_per_cached_root() { let project_id = project_id("project.runtime-cache-retarget"); let root = TempDir::new().expect("temporary project root"); let first_root = root.path().join("first-worktree"); @@ -342,7 +335,7 @@ mod runtime_configuration_cutover { let revision_id = revision_id("revision.runtime-cache-retarget"); let cache = RuntimeConfigurationCache::default(); cache.insert( - DaemonRuntimeConfiguration::new( + PinnedRuntimeConfiguration::new( RuntimeConfigurationTarget { project_id: project_id.clone(), project_root: first_root.clone(), @@ -353,7 +346,7 @@ mod runtime_configuration_cutover { .expect("first snapshot materializes"), ); cache.insert( - DaemonRuntimeConfiguration::new( + PinnedRuntimeConfiguration::new( RuntimeConfigurationTarget { project_id: project_id.clone(), project_root: second_root.clone(), @@ -370,7 +363,6 @@ mod runtime_configuration_cutover { assert_eq!(second.target().project_id, project_id); assert_eq!(first.target().project_root, first_root); assert_eq!(second.target().project_root, second_root); - assert_ne!(first.config.root_dir, second.config.root_dir); } #[tokio::test] @@ -526,7 +518,7 @@ mod runtime_configuration_cutover { assert!(!root_pin.config().diagnostics_prewarm); assert_eq!( root_pin.config().sync.auto_watch, - TraceDecayConfig::default().sync.auto_watch, + SyncConfig::default().auto_watch, "daemon-only settings materialize from the same snapshot" ); @@ -586,7 +578,7 @@ mod runtime_configuration_cutover { assert!(root_pin.config().diagnostics_prewarm); assert_eq!( root_pin.config().sync.auto_watch, - TraceDecayConfig::default().sync.auto_watch, + SyncConfig::default().auto_watch, "an unrelated change must not disturb daemon-only settings" ); assert_eq!( @@ -611,9 +603,9 @@ mod runtime_configuration_cutover { // Write the opposite of the typed registry default so the stale input // stays distinguishable from the canonical resolution regardless of // the default's polarity. - let stale_auto_watch = !TraceDecayConfig::default().sync.auto_watch; + let stale_auto_watch = !SyncConfig::default().auto_watch; std::fs::write( - &layout.config_path, + layout.data_root.join("config.json"), format!(r#"{{"sync":{{"auto_watch":{stale_auto_watch}}},"max_file_size":7}}"#), ) .expect("write stale config.json input"); @@ -644,13 +636,13 @@ mod runtime_configuration_cutover { "fresh stores publish the sole canonical initial revision" ); assert_eq!( - pinned.config.sync.auto_watch, - TraceDecayConfig::default().sync.auto_watch, + pinned.config().sync.auto_watch, + SyncConfig::default().auto_watch, "stale config.json input must not enter the final configuration authority" ); - assert_eq!( - pinned.config.max_file_size, - TraceDecayConfig::default().max_file_size, + assert_ne!( + pinned.config().max_file_size, + 7, "fresh initialization uses the typed registry, not config.json" ); assert!( @@ -770,7 +762,7 @@ mod runtime_configuration_cutover { .await .expect("registered default must converge before runtime materialization"); assert_ne!(converged.revision_id(), initial.revision_id()); - assert!(converged.config.native_graph_activation); + assert!(converged.config().native_graph_activation); assert_eq!( converged.snapshot().effective_values.get(&setting), Some(&ConfigurationValueV1::Boolean(true)) diff --git a/crates/tracedecay-project/src/project.rs b/crates/tracedecay-project/src/project.rs index 414ba4da15..8ebd818876 100644 --- a/crates/tracedecay-project/src/project.rs +++ b/crates/tracedecay-project/src/project.rs @@ -8,7 +8,7 @@ use std::path::PathBuf; use std::sync::{Arc, OnceLock}; -use tracedecay_configuration::TraceDecayConfig; +use tracedecay_configuration::config::RuntimeTraceDecayConfig; use tracedecay_contracts::context_scout::ContextScoutAddressV1; use tracedecay_domain::errors::Result; use tracedecay_graph_query::SourceReadContext; @@ -44,7 +44,7 @@ pub struct TraceDecay { db: Database, profile_database: tracedecay_global_db::RegisteredGlobalDbLeaseV1, pub store_runtime_registry: Arc, - config: TraceDecayConfig, + config: RuntimeTraceDecayConfig, configuration_runtime: Arc, project_root: PathBuf, store_layout: StoreLayout, @@ -154,11 +154,11 @@ impl TraceDecay { #[hotpath::skip] pub async fn mount_current_context_scout_claim_authority( &self, - registry: Arc, - hook: &tracedecay_agent_hosts::agents::context_scout::ports::AdmittedContextScoutHookV1, - pin: tracedecay_agent_hosts::agents::context_scout::ports::ContextScoutAuthorityPinV1, + registry: Arc, + hook: &tracedecay_agent_hosts::agents::context_scout::address_registry::AdmittedContextScoutHookV1, + pin: tracedecay_agent_hosts::agents::context_scout::address_registry::ContextScoutAuthorityPinV1, context: tracedecay_contracts::RequestContext, - lifecycle: tracedecay_agent_hosts::agents::context_scout::ports::ContextScoutLifecycleAddressV1, + lifecycle: tracedecay_agent_hosts::agents::context_scout::address_registry::ContextScoutLifecycleAddressV1, address: ContextScoutAddressV1, input_watermark: [u8; 32], observed_at: tracedecay_domain::UtcMicros, @@ -193,8 +193,8 @@ impl TraceDecay { #[hotpath::skip] pub async fn resolve_current_context_scout_claim_authority( &self, - hook: &tracedecay_agent_hosts::agents::context_scout::ports::AdmittedContextScoutHookV1, - lifecycle: &tracedecay_agent_hosts::agents::context_scout::ports::ContextScoutLifecycleAddressV1, + hook: &tracedecay_agent_hosts::agents::context_scout::address_registry::AdmittedContextScoutHookV1, + lifecycle: &tracedecay_agent_hosts::agents::context_scout::address_registry::ContextScoutLifecycleAddressV1, observed_at: tracedecay_domain::UtcMicros, ) -> Option<(ContextScoutAddressV1, [u8; 32])> { let owner = self.context_scout_owner()?; @@ -218,7 +218,7 @@ impl TraceDecay { /// read inside the bounded hook acknowledgement path. pub async fn resolve_mounted_context_scout_claim_authority( &self, - lifecycle: &tracedecay_agent_hosts::agents::context_scout::ports::ContextScoutLifecycleAddressV1, + lifecycle: &tracedecay_agent_hosts::agents::context_scout::address_registry::ContextScoutLifecycleAddressV1, ) -> Option<(ContextScoutAddressV1, [u8; 32])> { let owner = self.context_scout_owner()?; let pin = owner.mounted_claim_pin(lifecycle).await?; @@ -234,8 +234,8 @@ impl TraceDecay { #[hotpath::skip] pub async fn resolve_current_context_scout_session_claim_authority( &self, - hook: &tracedecay_agent_hosts::agents::context_scout::ports::AdmittedContextScoutHookV1, - lifecycle: &tracedecay_agent_hosts::agents::context_scout::ports::ContextScoutLifecycleAddressV1, + hook: &tracedecay_agent_hosts::agents::context_scout::address_registry::AdmittedContextScoutHookV1, + lifecycle: &tracedecay_agent_hosts::agents::context_scout::address_registry::ContextScoutLifecycleAddressV1, observed_at: tracedecay_domain::UtcMicros, ) -> Option<(ContextScoutAddressV1, [u8; 32])> { let owner = self.context_scout_owner()?; @@ -260,7 +260,7 @@ impl TraceDecay { #[hotpath::skip] async fn context_scout_configuration_is_current( &self, - pin: &tracedecay_agent_hosts::agents::context_scout::ports::ContextScoutAuthorityPinV1, + pin: &tracedecay_agent_hosts::agents::context_scout::address_registry::ContextScoutAuthorityPinV1, ) -> bool { self.configuration_runtime .client() @@ -293,16 +293,3 @@ impl TraceDecayOpenOptions { storage::default_profile_root() } } - -/// Returns the current UNIX timestamp in seconds. -/// -/// Defined in `tracedecay_runtime_core::tracedecay` because the memory and -/// `memory_v2` writers stamp records with it and those layers moved into the -/// kernel crate. -pub use tracedecay_runtime_core::tracedecay::current_timestamp; - -/// Returns `true` if the file path looks like a test file. -/// -/// Re-exported from the code-index crate so the segment list has one -/// definition shared by extraction and the orchestrator's read paths. -pub use tracedecay_code_index::is_test_file; diff --git a/crates/tracedecay-project/src/project/lifecycle/identity.rs b/crates/tracedecay-project/src/project/lifecycle/identity.rs index af02bc25f7..146f1069e9 100644 --- a/crates/tracedecay-project/src/project/lifecycle/identity.rs +++ b/crates/tracedecay-project/src/project/lifecycle/identity.rs @@ -16,16 +16,14 @@ impl TraceDecay { open_options: &TraceDecayOpenOptions, registry_database: &RegisteredGlobalDb, ) -> Result { - let layout = Self::resolve_store_layout_for_authority( + Self::resolve_store_layout_for_authority( project_root, open_options, Some(registry_database), false, &MovedStoreAdoption::Never, ) - .await?; - Self::reject_split_identity_cutover(project_root, open_options, &layout)?; - Ok(layout) + .await } /// Resolves the store layout for a project that has never been enrolled, @@ -33,8 +31,8 @@ impl TraceDecay { /// `init` can bootstrap it under the daemon's authority. /// /// This differs from [`Self::resolve_registered_configuration_layout`] only - /// in that a project with no enrollment marker or registry match falls - /// through to a default identity instead of failing closed. + /// in that a project with no repository identity marker or registry match + /// falls through to a default identity instead of failing closed. #[hotpath::skip] pub async fn resolve_first_touch_configuration_layout( project_root: &Path, @@ -101,28 +99,6 @@ impl TraceDecay { )?); } - // One-time legacy adoption: a project enrolled before the working-tree - // cutover may carry a retired `/.tracedecay/enrollment.json` and - // no other resolvable identity. Adopt the identity it names so the - // following open registers it durably (registry row plus `.git/` - // marker); after that, the marker or registry resolves first and the - // legacy file is never consulted again. The file itself is left - // untouched, users may delete it. - if selected.is_none() { - let enrollment_root = - tracedecay_runtime_core::worktree::repository_identity_root(project_root) - .unwrap_or_else(|| project_root.to_path_buf()); - if let Some(marker) = storage::read_legacy_enrollment_marker(&enrollment_root)? - && marker.storage_mode == storage::StorageMode::ProfileSharded - { - selected = Some(storage::profile_sharded_layout( - project_root, - &profile_root, - &marker, - )?); - } - } - if allow_default_identity && let MovedStoreAdoption::AdoptNamed(requested) = adoption && let Some(layout) = selected.as_ref() @@ -189,15 +165,11 @@ impl TraceDecay { .and_then(|profile_root| { tracedecay_runtime_core::storage::resolve_layout(project_root, &profile_root) }) - .is_ok_and(|layout| { - layout.storage_mode == tracedecay_runtime_core::storage::StorageMode::ProfileSharded - && layout.graph_db_path.exists() - }); + .is_ok_and(|layout| layout.graph_db_path.exists()); if open_options.profile_root.is_some() || open_options.global_db_path.is_some() { return option_resolved_store_exists; } option_resolved_store_exists - || crate::config::has_project_database(project_root) || tracedecay_runtime_core::storage::has_repository_identity_marker(project_root) } @@ -268,63 +240,17 @@ impl TraceDecay { project_root: &Path, open_options: &TraceDecayOpenOptions, ) -> Result { - let layout = Self::resolve_store_layout_for_authority( + Self::resolve_store_layout_for_authority( project_root, open_options, None, true, &MovedStoreAdoption::Never, ) - .await?; - Self::reject_split_identity_cutover(project_root, open_options, &layout)?; - Ok(layout) - } - - fn reject_split_identity_cutover( - project_root: &Path, - open_options: &TraceDecayOpenOptions, - selected: &StoreLayout, - ) -> Result<()> { - let profile_root = open_options.resolved_profile_root()?; - let selected_id = selected.identity.project_id.as_deref(); - let (candidates, _, candidates_match_exact_root) = - storage::matching_legacy_profile_layouts(project_root, &profile_root, selected_id)?; - // Sibling worktree manifests share a git common dir but name a - // different checkout path. They are not a second identity for this - // exact root and must not fail a registered exact-root resolution. - if !candidates_match_exact_root { - return Ok(()); - } - let Some(legacy) = candidates - .into_iter() - .find(|layout| layout.graph_db_path.is_file()) - else { - return Ok(()); - }; - if !selected.graph_db_path.is_file() { - return Ok(()); - } - let selected_id = selected_id.unwrap_or("unknown"); - let legacy_id = legacy.identity.project_id.as_deref().unwrap_or("unknown"); - let command = format!( - "tracedecay migrate consolidate --project {} --source-project-id {legacy_id} --target-project-id {selected_id}", - shell_quote(&project_root.to_string_lossy()), - ); - Err(TraceDecayError::Config { - message: format!( - "identity cutover conflict for '{}': selected [project_id={selected_id} path='{}']; legacy [project_id={legacy_id} path='{}']; choose one shard and retire the other; run the offline dry-run `{command}` before changing the marker; both shards were preserved and no files changed", - project_root.display(), - selected.data_root.display(), - legacy.data_root.display(), - ), - }) + .await } } -fn shell_quote(value: &str) -> String { - format!("'{}'", value.replace('\'', "'\"'\"'")) -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/tracedecay-project/src/project/lifecycle/mod.rs b/crates/tracedecay-project/src/project/lifecycle/mod.rs index 2e616ef06a..b871fea1b8 100644 --- a/crates/tracedecay-project/src/project/lifecycle/mod.rs +++ b/crates/tracedecay-project/src/project/lifecycle/mod.rs @@ -389,9 +389,7 @@ impl TraceDecay { .into_parts(); let (configuration_runtime, _) = ProjectConfigurationRuntime::open(opened)?; let configuration_runtime = Arc::new(configuration_runtime); - if store_layout.storage_mode == storage::StorageMode::ProfileSharded { - storage::write_store_manifest(&store_layout)?; - } + storage::write_store_manifest(&store_layout)?; // Bootstrap branch metadata if we can detect a default branch let default_branch = active_branch.as_ref().and_then(|_| { @@ -849,7 +847,7 @@ fn configuration_runtime_unavailable() -> TraceDecayError { mod tests { use super::*; use std::collections::{BTreeMap, BTreeSet}; - use tracedecay_agent_hosts::agents::context_scout::ports::{ + use tracedecay_agent_hosts::agents::context_scout::address_registry::{ AdmittedContextScoutHookV1, ContextScoutAddressBindOutcomeV1, ContextScoutAuthorityPinV1, ContextScoutConfigurationPinV1, ContextScoutLifecycleAddressV1, ProjectContextScoutAddressRegistryV1, @@ -858,6 +856,7 @@ mod tests { CancellationContext, CapabilityGrantId, CapabilityGrantSnapshot, Deadline, DisclosureClass, RequestId, ResolvedScope, }; + use tracedecay_domain::NativeHostIdentityV1; use tracedecay_domain::canonical_sha256; use tracedecay_domain::configuration::{ CONTEXT_SCOUT_SETTINGS_SETTING_KEY, CandidateDispositionV1, ConfigurationCandidateV1, @@ -868,15 +867,15 @@ mod tests { use tracedecay_domain::{ActorId, RepositoryId, UtcMicros, WorktreeId}; use tracedecay_global_db::configuration::contracts::ConfigurationCurrentStateV1; use tracedecay_hooks::{ - HookCapabilityV1, HookEventFamily, HookHostV1, HookScopeBindingV1, - NativeEnvelopeMaterialV1, decode_bound_native_hook_event, stock_event_support, + HookCapabilityV1, HookEventFamily, HookScopeBindingV1, NativeEnvelopeMaterialV1, + decode_bound_native_hook_event, stock_event_support, }; use tracedecay_tool_catalog::{CapabilityId, UseCaseId}; async fn mount_verified_reopen_claim( owner: &tracedecay_agent_hosts::agents::context_scout::owner::ProjectContextScoutOwnerV1, ) -> ( - tracedecay_agent_hosts::agents::context_scout::ports::ContextScoutLifecycleAddressV1, + ContextScoutLifecycleAddressV1, tracedecay_contracts::context_scout::ContextScoutAddressV1, ) { use tracedecay_domain::test_fixtures::id; @@ -954,7 +953,7 @@ mod tests { ) .expect("authority pin"); let binding = HookScopeBindingV1 { - host: HookHostV1::ClaudeCode, + host: NativeHostIdentityV1::ClaudeCode, project_id: [1; 16], repository_id: [2; 16], worktree_id: [3; 16], @@ -970,12 +969,12 @@ mod tests { .into_iter() .map(|family| HookCapabilityV1 { family, - support: stock_event_support(HookHostV1::ClaudeCode, family), + support: stock_event_support(NativeHostIdentityV1::ClaudeCode, family), }) .collect(), }; let envelope = decode_bound_native_hook_event( - HookHostV1::ClaudeCode, + NativeHostIdentityV1::ClaudeCode, include_bytes!( "../../../../../tests/fixtures/packaged_host_events/claude/post_tool_use_write.json" ), @@ -1142,15 +1141,11 @@ mod tests { let db_path = initialized.store_layout().graph_db_path.clone(); initialized.close(); let connection = rusqlite::Connection::open(&db_path).expect("open graph fixture"); - // Not `SCHEMA_VERSION - 1`: that stamp is - // `PAYLOAD_DIGEST_STEP_SOURCE_VERSION`, the one sanctioned step this - // binary carries forward in place, so an open of it upgrades instead - // of refusing. Age the store one step past the sanctioned source. connection .pragma_update( None, "user_version", - tracedecay_runtime_core::db::migrations::PAYLOAD_DIGEST_STEP_SOURCE_VERSION - 1, + tracedecay_runtime_core::db::migrations::SCHEMA_VERSION - 1, ) .expect("stamp incompatible graph schema"); drop(connection); diff --git a/crates/tracedecay-project/src/project/queries/meta.rs b/crates/tracedecay-project/src/project/queries/meta.rs index 39cf22f086..e224a5e47f 100644 --- a/crates/tracedecay-project/src/project/queries/meta.rs +++ b/crates/tracedecay-project/src/project/queries/meta.rs @@ -4,7 +4,7 @@ use crate::project::TraceDecay; use tracedecay_application::tracedecay::{ add_local_counter, get_local_counter, get_tokens_saved, reset_local_counter, set_tokens_saved, }; -use tracedecay_configuration::TraceDecayConfig; +use tracedecay_configuration::config::RuntimeTraceDecayConfig; use tracedecay_domain::errors::Result; impl TraceDecay { @@ -67,7 +67,7 @@ impl TraceDecay { } /// Returns a reference to the current configuration. - pub fn get_config(&self) -> &TraceDecayConfig { + pub fn get_config(&self) -> &RuntimeTraceDecayConfig { &self.config } diff --git a/crates/tracedecay-project/src/project/source_edit_runtime.rs b/crates/tracedecay-project/src/project/source_edit_runtime.rs index 94b9656a8b..9cf1b2f009 100644 --- a/crates/tracedecay-project/src/project/source_edit_runtime.rs +++ b/crates/tracedecay-project/src/project/source_edit_runtime.rs @@ -1,59 +1,5 @@ //! Direct graph mutations are crate-internal adapters; external callers must //! use the canonical source-edit transaction. -//! -//! ```compile_fail -//! async fn direct_str_replace_is_not_public(graph: &tracedecay::project::TraceDecay) { -//! let _ = graph.str_replace("src/lib.rs", "old", "new", true).await; -//! } -//! ``` -//! ```compile_fail -//! async fn direct_multi_str_replace_is_not_public(graph: &tracedecay::project::TraceDecay) { -//! let _ = graph -//! .multi_str_replace("src/lib.rs", &[("old", "new")], true) -//! .await; -//! } -//! ``` -//! ```compile_fail -//! async fn direct_insert_at_is_not_public(graph: &tracedecay::project::TraceDecay) { -//! let _ = graph -//! .insert_at("src/lib.rs", "anchor", "content", true, true) -//! .await; -//! } -//! ``` -//! ```compile_fail -//! async fn direct_replace_symbol_is_not_public(graph: &tracedecay::project::TraceDecay) { -//! let _ = graph.replace_symbol("symbol", "fn symbol() {}", true).await; -//! } -//! ``` -//! ```compile_fail -//! async fn direct_insert_at_symbol_is_not_public(graph: &tracedecay::project::TraceDecay) { -//! let _ = graph -//! .insert_at_symbol("symbol", "content", "before", true) -//! .await; -//! } -//! ``` -//! ```compile_fail -//! async fn direct_ast_grep_rewrite_is_not_public(graph: &tracedecay::project::TraceDecay) { -//! let _ = graph -//! .ast_grep_rewrite("src/lib.rs", "$A", "$A", true) -//! .await; -//! } -//! ``` -//! ```compile_fail -//! async fn direct_move_symbol_is_not_public(graph: &tracedecay::project::TraceDecay) { -//! let _ = graph -//! .move_symbol("symbol", "src/dest.rs", true, false) -//! .await; -//! } -//! ``` -//! ```compile_fail -//! async fn direct_rename_symbol_is_not_public( -//! graph: &tracedecay::project::TraceDecay, -//! binding: &tracedecay_contracts::RenameSymbolBindingV1, -//! ) { -//! let _ = graph.rename_symbol(binding, "new_name", true).await; -//! } -//! ``` use std::path::Path; diff --git a/crates/tracedecay-project/src/runtime_ports.rs b/crates/tracedecay-project/src/runtime_ports.rs index c3c96a87ba..2dc89f37f3 100644 --- a/crates/tracedecay-project/src/runtime_ports.rs +++ b/crates/tracedecay-project/src/runtime_ports.rs @@ -74,9 +74,6 @@ pub(crate) fn require_runtime_ports() -> Result { fn register_session_ports() { use tracedecay_sessions::host_ports; - host_ports::hermes_profile_pin::register( - tracedecay_agent_hosts::agents::hermes::read_config_pinned_project_root, - ); host_ports::unregistered_admission::register(unregistered_admission); } @@ -108,9 +105,6 @@ fn register_agent_host_ports() { use tracedecay_automation_runtime::ports as automation_ports; automation_ports::codex_app_server::register(run_codex_app_server_prompt); - automation_ports::session_store::register_canonical_project_key( - tracedecay_global_db::RegisteredGlobalDb::canonical_project_key, - ); } /// The hook runtime handle for the registered daemon client, or a typed @@ -191,20 +185,21 @@ fn run_codex_app_server_prompt( thread_source: &str, response_schema: Option<&Value>, ) -> std::result::Result { - let config = tracedecay_sessions::runtime::codex_app_server::CodexAppServerSummaryConfig { - codex_bin: config.codex_bin.clone(), - model: config.model.clone(), - timeout: config.timeout, - }; + let config = + tracedecay_sessions::runtime::hosts::codex_app_server::CodexAppServerSummaryConfig { + codex_bin: config.codex_bin.to_string_lossy().into_owned(), + model: config.model.clone(), + timeout: config.timeout, + }; let result = if let Some(response_schema) = response_schema { - tracedecay_sessions::runtime::codex_app_server::run_prompt_with_codex_app_server_response_schema( + tracedecay_sessions::runtime::hosts::codex_app_server::run_prompt_with_codex_app_server_response_schema( prompt, &config, thread_source, response_schema, ) } else { - tracedecay_sessions::runtime::codex_app_server::run_prompt_with_codex_app_server( + tracedecay_sessions::runtime::hosts::codex_app_server::run_prompt_with_codex_app_server( prompt, &config, thread_source, @@ -275,38 +270,6 @@ mod tests { pinned } - /// YAML double-quoted scalars treat `\t`/`\U` as escapes. A Windows - /// native path must be written so those separators survive as separators. - fn hermes_project_root_yaml(project_root: &str) -> String { - format!( - "plugins:\n tracedecay:\n project_root: '{}'\n", - project_root.replace('\'', "''") - ) - } - - #[test] - fn hermes_profile_pin_resolves_a_pinned_root_after_registration() { - let _pinned = registered(); - let temp = tempfile::tempdir().expect("tempdir"); - let config = temp.path().join("config.yaml"); - let pinned = temp.path().join("pinned-project"); - // Single-quoted so a Windows path's backslashes are not read as YAML - // escapes (`\a` -> BEL, `\t` -> TAB). - std::fs::write( - &config, - hermes_project_root_yaml(&pinned.display().to_string()), - ) - .expect("write hermes profile config"); - - // Unwired this reads `None`, which makes legacy Hermes state stores - // skip rather than attribute to the pinned root. - assert_eq!( - tracedecay_sessions::host_ports::hermes_profile_pin::resolve(&config), - Some(pinned.display().to_string()), - "registered resolver must back the hermes profile pin port" - ); - } - /// The hook runtime is one explicit handle of adapters, so this is the /// single check that every hook capability this crate composes answers /// through it (here: the registered-identity gates for an unregistered @@ -334,6 +297,17 @@ mod tests { .await .expect("the root resolves a canonical layout for any checkout"); assert_eq!(layout.project_root, checkout); - assert!(layout.identity.project_id.is_some()); + let project_id = layout + .identity + .project_id + .expect("the layout carries a project identity"); + let again = (runtime.store_layout_resolver)(&checkout) + .await + .expect("the same checkout resolves again"); + assert_eq!( + again.identity.project_id, + Some(project_id), + "one checkout resolves to one stable project identity" + ); } } diff --git a/crates/tracedecay-project/src/test_support/host_admission.rs b/crates/tracedecay-project/src/test_support/host_admission.rs index d1e0cd45e8..e188c54ea3 100644 --- a/crates/tracedecay-project/src/test_support/host_admission.rs +++ b/crates/tracedecay-project/src/test_support/host_admission.rs @@ -19,7 +19,7 @@ use tracedecay_host_admission::{ use tracedecay_sessions::admission::{ HostAdmissionOutcome, HostAdmissionScope, HostAdmissionStatus, }; -use tracedecay_sessions::runtime::codex::CodexDiscoveryHub; +use tracedecay_sessions::runtime::hosts::codex::CodexDiscoveryHub; use crate::project::{TraceDecay, TraceDecayOpenOptions}; use tracedecay_domain::errors::{Result, TraceDecayError}; @@ -669,7 +669,7 @@ impl HostAdmissionTestRuntimeV1 { provider: &str, session_id: &str, transcript_path: &Path, - ) -> Result<(i64, i64, i64, i64, i64, i64, i64)> { + ) -> Result<(i64, i64, i64, i64, i64, i64)> { let snapshot = self .session_database_for_test(scope)? .read_snapshot() @@ -679,8 +679,6 @@ impl HostAdmissionTestRuntimeV1 { "SELECT (SELECT COUNT(*) FROM sessions WHERE provider = ?1 AND session_id = ?2), - (SELECT COUNT(*) FROM session_messages - WHERE provider = ?1 AND session_id = ?2), (SELECT COUNT(*) FROM lcm_raw_messages WHERE provider = ?1 AND session_id = ?2), (SELECT COUNT(*) FROM lcm_raw_messages_fts @@ -688,14 +686,16 @@ impl HostAdmissionTestRuntimeV1 { ON raw.store_id = lcm_raw_messages_fts.rowid WHERE raw.provider = ?1 AND raw.session_id = ?2), (SELECT COUNT(*) FROM lcm_raw_messages_fts), - (SELECT COUNT(*) FROM lcm_summary_nodes + (SELECT COUNT(*) FROM session_summary_nodes WHERE provider = ?1 AND session_id = ?2), (SELECT COUNT(*) FROM parse_offsets WHERE file_path = ?3)", tracedecay_runtime_core::db::engine::params![ provider, session_id, - transcript_path.to_string_lossy().as_ref() + tracedecay_sessions::runtime::shared::path_identity_key( + transcript_path.to_string_lossy().as_ref() + ) ], ) .await?; @@ -713,7 +713,6 @@ impl HostAdmissionTestRuntimeV1 { row.get(3)?, row.get(4)?, row.get(5)?, - row.get(6)?, )) } @@ -880,7 +879,7 @@ impl HostAdmissionTestRuntimeV1 { &self, project_root: &Path, layout: &tracedecay_runtime_core::storage::StoreLayout, - ) -> Result { + ) -> Result { crate::config::ensure_runtime_configuration_for_registered_database( project_root, layout, @@ -894,7 +893,7 @@ impl HostAdmissionTestRuntimeV1 { &self, project_root: &Path, layout: &tracedecay_runtime_core::storage::StoreLayout, - ) -> Result { + ) -> Result { crate::config::resolve_runtime_configuration_for_registered_database( project_root, layout, @@ -908,7 +907,7 @@ impl HostAdmissionTestRuntimeV1 { &self, project_root: &Path, layout: &tracedecay_runtime_core::storage::StoreLayout, - ) -> Result { + ) -> Result { crate::config::open_runtime_configuration_for_registered_database_read_only( project_root, layout, diff --git a/crates/tracedecay-project/src/test_support/host_admission/integration_test_support.rs b/crates/tracedecay-project/src/test_support/host_admission/integration_test_support.rs index d285f3909f..8d0e2103d0 100644 --- a/crates/tracedecay-project/src/test_support/host_admission/integration_test_support.rs +++ b/crates/tracedecay-project/src/test_support/host_admission/integration_test_support.rs @@ -37,10 +37,13 @@ impl HostAdmissionTestRuntimeV1 { }) } - /// Runs workflow ingestion through this runtime's exact ProjectSessions mount. + /// Runs workflow ingestion through this runtime's exact ProjectSessions + /// mount, reading Claude transcripts under the isolated `home` only. The + /// operator's real home is never consulted. #[doc(hidden)] pub async fn ingest_workflows_for_test( &self, + home: &Path, project_root: &Path, ) -> Result { let project_id = self @@ -51,11 +54,6 @@ impl HostAdmissionTestRuntimeV1 { message: "project session authority is unavailable".to_owned(), })?; let database = self.project_database_for_test()?; - let Some(home) = tracedecay_sessions::runtime::home_dir() else { - return Ok( - tracedecay_sessions::runtime::workflow_ingest::WorkflowIngestStats::default(), - ); - }; let store = tracedecay_global_db::GlobalDbWorkflowStore::new(database); Ok( tracedecay_sessions::runtime::workflow_ingest::ingest_workflow_runs_with_sink( @@ -283,8 +281,10 @@ impl HostAdmissionTestRuntimeV1 { &self, scope: HostAdmissionScope, observation: &tracedecay_store::ObservationCommitReceipt, - ) -> std::result::Result, HostAdmissionOutcome> - { + ) -> std::result::Result< + Option, + HostAdmissionOutcome, + > { let database = self.registered_database(scope).ok_or_else(|| { HostAdmissionOutcome::retained_unavailable("registered_authority_unavailable") })?; diff --git a/crates/tracedecay-project/src/test_support/host_admission/lcm_fixture_test_support.rs b/crates/tracedecay-project/src/test_support/host_admission/lcm_fixture_test_support.rs index 19d311b76f..1b231c32a2 100644 --- a/crates/tracedecay-project/src/test_support/host_admission/lcm_fixture_test_support.rs +++ b/crates/tracedecay-project/src/test_support/host_admission/lcm_fixture_test_support.rs @@ -1,9 +1,12 @@ use super::*; +use tracedecay_session_temporal_store::SessionTemporalAccess; #[doc(hidden)] #[derive(Debug, Clone, PartialEq, Eq)] pub enum LcmLineageFaultForTest { - CorruptCompatibilitySummaryText { + /// Rewrites a stored summary body without touching its hash so readers + /// must fail closed on the mismatch. + CorruptSummaryText { node_id: String, text: String, }, @@ -105,7 +108,7 @@ impl HostAdmissionTestRuntimeV1 { transaction .execute( "UPDATE lcm_raw_messages - SET content = ?2, snippet_text = ?2, index_text = ?2 + SET content = ?2 WHERE store_id = ?1", tracedecay_runtime_core::db::engine::params![store_id, poison], ) @@ -302,7 +305,7 @@ impl HostAdmissionTestRuntimeV1 { ) -> Result<()> { let statement = if enabled { "CREATE TRIGGER abort_late_summary_projection - BEFORE INSERT ON lcm_summary_nodes + BEFORE INSERT ON session_summary_sources BEGIN SELECT RAISE(ABORT, 'forced late summary projection failure'); END;" @@ -340,7 +343,7 @@ impl HostAdmissionTestRuntimeV1 { let transaction = database.begin_write_transaction().await?; transaction .execute( - "DELETE FROM lcm_summary_sources WHERE node_id = ?1", + "DELETE FROM session_summary_sources WHERE summary_id = ?1", (node_id,), ) .await @@ -359,7 +362,7 @@ impl HostAdmissionTestRuntimeV1 { let transaction = database.begin_write_transaction().await?; transaction .execute( - "INSERT INTO lcm_summary_sources (node_id, source_kind, source_id, ordinal) + "INSERT INTO session_summary_sources (summary_id, source_kind, source_id, ordinal) VALUES (?1, 'summary_node', ?2, 0)", (node_id, source_node_id), ) @@ -501,7 +504,7 @@ impl HostAdmissionTestRuntimeV1 { &draft.source_refs, &summary_hash, ); - let control = tracedecay_temporal_query::ports::ExecutionControl::default(); + let control = tracedecay_temporal_query::execution::ExecutionControl::default(); database .lcm_publish_immutable_summary_guarded( tracedecay_lcm::types::LcmImmutableSummaryPublication { @@ -546,7 +549,7 @@ impl HostAdmissionTestRuntimeV1 { let database = self .session_database_for_test(scope) .map_err(|error| tracedecay_lcm::LcmError::Db(error.to_string()))?; - let control = tracedecay_temporal_query::ports::ExecutionControl::default(); + let control = tracedecay_temporal_query::execution::ExecutionControl::default(); database .lcm_publish_immutable_summary_guarded(publication, &control, || Ok(())) .await @@ -575,10 +578,10 @@ impl HostAdmissionTestRuntimeV1 { .begin_write_transaction() .await?; let result = match fault { - LcmLineageFaultForTest::CorruptCompatibilitySummaryText { node_id, text } => { + LcmLineageFaultForTest::CorruptSummaryText { node_id, text } => { transaction .execute( - "UPDATE lcm_summary_nodes SET summary_text = ?2 WHERE node_id = ?1", + "UPDATE session_summary_nodes SET summary_text = ?2 WHERE summary_id = ?1", tracedecay_runtime_core::db::engine::params![node_id, text], ) .await @@ -758,6 +761,10 @@ impl HostAdmissionTestRuntimeV1 { fault: &LcmLineageFaultForTest, ) -> Result<()> { let (statement, operation) = match fault { + LcmLineageFaultForTest::CorruptSummaryText { .. } => ( + "DROP TRIGGER IF EXISTS session_summary_nodes_immutable_update_v1", + "prepare corrupt lcm summary text fixture", + ), LcmLineageFaultForTest::ReplaceGenerationWatermarks { .. } => ( "DROP TRIGGER IF EXISTS session_temporal_generations_state_guard_v1", "prepare changed lcm watermarks fixture", @@ -986,7 +993,7 @@ impl HostAdmissionTestRuntimeV1 { message: error.to_string(), } })?; - let (_, mut session_relations) = database + let (_, mut session_relations) = SessionTemporalAccess::new(&*database) .active_session_summary_relations( &session_id, &summary_ids, @@ -1344,7 +1351,7 @@ impl HostAdmissionTestRuntimeV1 { async fn set_lcm_summary_insert_abort_trigger_for_test(&self, enabled: bool) -> Result<()> { let statement = if enabled { "CREATE TRIGGER fail_codex_summary_successor - BEFORE INSERT ON lcm_summary_nodes + BEFORE INSERT ON session_summary_nodes BEGIN SELECT RAISE(ABORT, 'forced summary successor failure'); END;" diff --git a/crates/tracedecay-project/src/test_support/host_admission/session_test_support.rs b/crates/tracedecay-project/src/test_support/host_admission/session_test_support.rs index 00e332d95f..9204bea1c2 100644 --- a/crates/tracedecay-project/src/test_support/host_admission/session_test_support.rs +++ b/crates/tracedecay-project/src/test_support/host_admission/session_test_support.rs @@ -2,6 +2,8 @@ use std::path::Path; +use tracedecay_session_temporal_store::SessionTemporalAccess; + use super::{HostAdmissionScope, HostAdmissionTestRuntimeV1}; impl HostAdmissionTestRuntimeV1 { @@ -10,7 +12,7 @@ impl HostAdmissionTestRuntimeV1 { &self, scope: HostAdmissionScope, ) -> tracedecay_domain::errors::Result { - self.session_database_for_test(scope)? + SessionTemporalAccess::new(&*self.session_database_for_test(scope)?) .ensure_active_session_cursor_key_result() .await .map_err( @@ -338,7 +340,7 @@ impl HostAdmissionTestRuntimeV1 { let writer = self.session_database_for_test(scope)?.writer_connection()?; let statement = if enabled { "CREATE TRIGGER fail_session_message_projection - BEFORE INSERT ON session_messages + BEFORE INSERT ON lcm_raw_messages BEGIN SELECT RAISE(ABORT, 'projection failure'); END;" @@ -597,7 +599,7 @@ impl HostAdmissionTestRuntimeV1 { ) -> tracedecay_domain::errors::Result<()> { let statement = if enabled { "CREATE TRIGGER fail_session_message_projection - BEFORE INSERT ON session_messages + BEFORE INSERT ON lcm_raw_messages BEGIN SELECT RAISE(ABORT, 'projection failure'); END;" diff --git a/crates/tracedecay-query/Cargo.toml b/crates/tracedecay-query/Cargo.toml index 4de93b629c..5af789e5b8 100644 --- a/crates/tracedecay-query/Cargo.toml +++ b/crates/tracedecay-query/Cargo.toml @@ -9,6 +9,9 @@ description = "TraceDecay exact/lexical/graph code retrieval query kernel and se include = ["/src/**", "/assets/**"] [dependencies] +# Raw deflate for lexical row text; already in the tree through +# `tracedecay-code-index`. +flate2 = "1" fst = { version = "=0.4.7", default-features = false, features = ["levenshtein"] } gix = { version = "=0.87.1", default-features = false, features = ["revision", "parallel", "sha1", "sha256"] } hex = "0.4" @@ -46,13 +49,6 @@ hotpath = ["hotpath/hotpath"] # Explicit opt-in (same policy as `test-transport` on the runtime crate): # exposes `*_for_test` constructors to dependent crates' test builds only. test-helpers = ["tracedecay-temporal-query/test-helpers"] -# Evaluation-only in-memory lexical projection (`retrieval::lexical:: -# CodeLexicalProjectionAdapterV1` and its builder). Production retrieval reads -# the durable lexical artifact; only the search-quality evaluator and this -# crate's own suites build projections directly from admitted chunks. -# `tracedecay` must not depend on the evaluator unconditionally: that unifies -# this feature into every root test target, including transport. -search-eval = [] # The grammar tier the shipped product indexes with, forwarded exactly as # `tracedecay-cli`'s `production` forwards `tracedecay/production`. Only the # `tracedecay-index-bench` binary needs it: a profiling run must parse the @@ -60,9 +56,6 @@ search-eval = [] production = ["tracedecay-code-index/lite", "tracedecay-code-index/full"] [dev-dependencies] -# Self dev-dependency: the search-quality suites drive the eval-only in-memory -# lexical projection, so the crate's own test build enables it. -tracedecay-query = { path = ".", features = ["search-eval"] } static_assertions = "1.1.0" tempfile = "3" # Test-only: the relocated search-quality suites build lexical/exact fixtures diff --git a/crates/tracedecay-query/assets/runtime-root/tests/fixtures/search_quality/query-lexical-graph-workload-v1.json b/crates/tracedecay-query/assets/runtime-root/tests/fixtures/search_quality/query-lexical-graph-workload-v1.json index 7870e2a69a..bb17e8b588 100644 --- a/crates/tracedecay-query/assets/runtime-root/tests/fixtures/search_quality/query-lexical-graph-workload-v1.json +++ b/crates/tracedecay-query/assets/runtime-root/tests/fixtures/search_quality/query-lexical-graph-workload-v1.json @@ -135,8 +135,8 @@ } ], "expected_query_fallback_digests": { - "train": "sha256:f3744c56a8d35a13ddd9616eb949ae31b2bf745bf14b4d0bdc97c3e0ed139660", - "validation": "sha256:561eb59d2b58d5c935b28d0217dc18252a2d2c28e47f404cc59104e03c95625d" + "train": "sha256:89b770fc6902e8683b813e6d9f6432d4fd0404f212c488c9c9146442b8e7083e", + "validation": "sha256:e3545abc35d6626d6fc20f9bbf14f3a3cf6cc71da4dedbb679e649304eac0672" }, "profile_matrix": [ { diff --git a/crates/tracedecay-query/benches/code_lexical_catchup.rs b/crates/tracedecay-query/benches/code_lexical_catchup.rs index b02d4b6ca5..d52a81269f 100644 --- a/crates/tracedecay-query/benches/code_lexical_catchup.rs +++ b/crates/tracedecay-query/benches/code_lexical_catchup.rs @@ -4,17 +4,18 @@ use std::collections::{BTreeMap, BTreeSet}; use std::fmt; -use std::io::Cursor; use std::sync::{Arc, Mutex}; use std::time::Instant; use serde::Serialize; +use sha2::{Digest, Sha256}; use tracedecay_code_index::chunks::content_digest; use tracedecay_code_index::production::{ CodeIndexAtomicPublicationPort, CodeIndexBuildRequestV1, CodeIndexCapturedFileV1, CodeIndexExecutionControlV1, CodeIndexGenerationScopeV1, CodeIndexProductionConfigV1, - CodeIndexProductionOwnerV1, CodeIndexPublicationStoreErrorV1, CodeIndexPublishedGenerationV1, - CodeIndexRepositoryParseIdentityV1, VerifiedSealedLexicalPageReadV1, + CodeIndexProductionErrorV1, CodeIndexProductionOwnerV1, CodeIndexPublicationStoreErrorV1, + CodeIndexPublishedGenerationV1, CodeIndexRepositoryParseIdentityV1, + SealedGenerationSegmentPublicationV1, VerifiedSealedLexicalPageReadV1, VerifiedSealedLexicalPageSourceV1, VerifiedSealedLexicalPageV1, VerifiedSealedLexicalSourceReceiptV1, }; @@ -32,7 +33,7 @@ use tracedecay_domain::{ SourceNamespace, UtcMicros, }; use tracedecay_query::retrieval::lexical::{ - CodeLexicalArtifactBuilderV1, CodeLexicalArtifactFinalizationStepV1, + CodeLexicalArtifactBuilderV1, CodeLexicalArtifactFinalizationStepV1, CodeLexicalCloneRouteV1, CodeLexicalProjectionMetadataV1, VerifiedCodeLexicalArtifactV1, }; @@ -166,7 +167,7 @@ struct RunReport { sqlite_ingestion_commits: u64, artifact_bytes: u64, artifact_digest: String, - source_cumulative_digest: String, + artifact_file_digest: String, } struct RunResult { @@ -182,7 +183,7 @@ struct ComparisonReport { bounded_batch: RunReport, final_receipt_equal: bool, artifact_digest_equal: bool, - source_cumulative_digest_equal: bool, + artifact_file_digest_equal: bool, } fn main() { @@ -197,8 +198,8 @@ fn main() { let final_receipt_equal = one_page.receipt == bounded_batch.receipt; let artifact_digest_equal = one_page.receipt.artifact_digest() == bounded_batch.receipt.artifact_digest(); - let source_cumulative_digest_equal = one_page.receipt.source_cumulative_digest() - == bounded_batch.receipt.source_cumulative_digest(); + let artifact_file_digest_equal = + one_page.report.artifact_file_digest == bounded_batch.report.artifact_file_digest; let report = ComparisonReport { fixture_pages: fixture.pages.len(), batch_page_limit: BATCH_PAGE_LIMIT, @@ -206,7 +207,7 @@ fn main() { bounded_batch: bounded_batch.report, final_receipt_equal, artifact_digest_equal, - source_cumulative_digest_equal, + artifact_file_digest_equal, }; println!( @@ -214,8 +215,8 @@ fn main() { serde_json::to_string_pretty(&report).expect("serialize benchmark report") ); assert!( - final_receipt_equal && artifact_digest_equal && source_cumulative_digest_equal, - "the compared ingestion paths must produce the exact same final receipt and digests" + final_receipt_equal && artifact_digest_equal && artifact_file_digest_equal, + "the compared ingestion paths must produce the exact same final receipt and bytes" ); } @@ -316,17 +317,32 @@ fn build_fixture() -> Fixture { let generation = owner .build_and_publish(request, &control) .expect("build deterministic sealed generation"); - let sealed = generation - .encode_sealed() + let mut segments = BTreeMap::new(); + let mut evidence_pack = Vec::new(); + let manifest = generation + .encode_partitioned_sealed(|publication| { + match publication { + SealedGenerationSegmentPublicationV1::File { digest, bytes } => { + segments.insert(digest.as_str().to_owned(), bytes.to_vec()); + } + SealedGenerationSegmentPublicationV1::GenerationEvidencePage { bytes, .. } => { + evidence_pack.extend_from_slice(bytes); + } + SealedGenerationSegmentPublicationV1::GenerationEvidenceCommit { + segment_digest, + .. + } => { + segments.insert( + segment_digest.as_str().to_owned(), + std::mem::take(&mut evidence_pack), + ); + } + } + Ok(()) + }) .expect("encode deterministic sealed generation"); - let sealed_len = u64::try_from(sealed.len()).expect("sealed generation length"); - let envelope: serde_json::Value = - serde_json::from_slice(&sealed).expect("decode sealed generation envelope"); - let state_digest = id::( - envelope["state_digest"] - .as_str() - .expect("sealed generation state digest"), - ); + let state_digest = ManifestDigest::from_sha256_bytes(&Sha256::digest(&manifest)) + .expect("sealed manifest digest"); let metadata = CodeLexicalProjectionMetadataV1 { generation: generation.manifest().generation_id.clone(), repository_id: Some(repository), @@ -342,14 +358,25 @@ fn build_fixture() -> Fixture { "retriever.lexical.catchup-benchmark.v1", ), exact_score_domain: id::("score.exact.catchup-benchmark.v1"), + clone_route: Some(CodeLexicalCloneRouteV1 { + project_id: generation.manifest().project_id.clone(), + worktree_id: generation.snapshot().worktree.clone(), + snapshot_digest: generation.manifest().snapshot_digest.clone(), + }), }; - let mut source = VerifiedSealedLexicalPageSourceV1::open( - Cursor::new(sealed), - sealed_len, + let mut source = VerifiedSealedLexicalPageSourceV1::open_partitioned_sealed( + &manifest, state_digest, + move |digest, _, buffer, _control| { + let bytes = segments.get(digest.as_str()).ok_or_else(|| { + CodeIndexProductionErrorV1::Contract("benchmark segment is missing".to_owned()) + })?; + buffer.clear(); + buffer.extend_from_slice(bytes); + Ok(()) + }, 1, 1024 * 1024, - &control, ) .expect("open verified lexical page source"); let mut pages = Vec::new(); @@ -425,7 +452,9 @@ fn run(fixture: &Fixture, mode: IngestionMode) -> RunResult { sqlite_ingestion_commits, artifact_bytes: receipt.file_size_bytes(), artifact_digest: receipt.artifact_digest().as_str().to_owned(), - source_cumulative_digest: receipt.source_cumulative_digest().as_str().to_owned(), + artifact_file_digest: hex::encode(Sha256::digest( + std::fs::read(&artifact_path).expect("read finalized benchmark artifact"), + )), }, receipt, } diff --git a/crates/tracedecay-query/src/bench_support.rs b/crates/tracedecay-query/src/bench_support.rs index 297d0221f4..6c054d9053 100644 --- a/crates/tracedecay-query/src/bench_support.rs +++ b/crates/tracedecay-query/src/bench_support.rs @@ -6,20 +6,21 @@ //! indexing profiles the daemon's commit width, search profiles a narrower //! query ingest. +use std::collections::BTreeMap; use std::fmt; -use std::io::Cursor; use std::num::NonZeroUsize; use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; use std::time::Duration; +use sha2::{Digest, Sha256}; use tracedecay_code_index::languages::{LanguageRegistry, StaticLanguageRegistry}; use tracedecay_code_index::production::{ CodeIndexAtomicPublicationPort, CodeIndexExecutionControlV1, CodeIndexGenerationScopeV1, - CodeIndexPublicationStoreErrorV1, CodeIndexPublishedGenerationV1, - VerifiedSealedLexicalPageBatchBoundsV1, VerifiedSealedLexicalPageBatchReadV1, - VerifiedSealedLexicalPageSourceV1, VerifiedSealedLexicalPageV1, - VerifiedSealedLexicalSourceReceiptV1, + CodeIndexProductionErrorV1, CodeIndexPublicationStoreErrorV1, CodeIndexPublishedGenerationV1, + SealedGenerationSegmentPublicationV1, VerifiedSealedLexicalPageBatchBoundsV1, + VerifiedSealedLexicalPageBatchReadV1, VerifiedSealedLexicalPageSourceV1, + VerifiedSealedLexicalPageV1, VerifiedSealedLexicalSourceReceiptV1, }; use tracedecay_code_index::projection::{ ChunkProjectionDecisionV1, CodeChunkProjectionSink, ProjectionReceiptBuilderV1, @@ -253,24 +254,67 @@ impl CodeChunkProjectionSink for ApplyingProjectionSink { } } -pub(crate) fn sealed_state_digest(sealed: &[u8]) -> Result { - let envelope: serde_json::Value = serde_json::from_slice(sealed) - .map_err(|error| format!("decode sealed generation envelope: {error}"))?; - let digest = envelope - .get("state_digest") - .and_then(serde_json::Value::as_str) - .ok_or_else(|| "sealed generation envelope has no state digest".to_owned())?; - ManifestDigest::try_from(digest.to_owned()) - .map_err(|error| format!("sealed generation state digest: {error:?}")) +/// A partitioned sealed generation held in memory: the manifest, its content +/// address (the daemon's lexical source state digest), and every published +/// segment under its digest, with evidence pages assembled into their pack. +pub(crate) struct PartitionedSealedV1 { + manifest: Vec, + pub(crate) state_digest: ManifestDigest, + segments: Arc>>, +} + +impl PartitionedSealedV1 { + /// Manifest plus every segment, the bytes a fresh publication writes. + pub(crate) fn byte_len(&self) -> u64 { + self.segments + .values() + .map(|segment| segment.len() as u64) + .sum::() + + self.manifest.len() as u64 + } +} + +pub(crate) fn seal_partitioned( + generation: &CodeIndexPublishedGenerationV1, +) -> Result { + let mut segments = BTreeMap::new(); + let mut evidence_pack = Vec::new(); + let manifest = generation + .encode_partitioned_sealed(|publication| { + match publication { + SealedGenerationSegmentPublicationV1::File { digest, bytes } => { + segments.insert(digest.as_str().to_owned(), bytes.to_vec()); + } + SealedGenerationSegmentPublicationV1::GenerationEvidencePage { bytes, .. } => { + evidence_pack.extend_from_slice(bytes); + } + SealedGenerationSegmentPublicationV1::GenerationEvidenceCommit { + segment_digest, + .. + } => { + segments.insert( + segment_digest.as_str().to_owned(), + std::mem::take(&mut evidence_pack), + ); + } + } + Ok(()) + }) + .map_err(|error| format!("encode partitioned sealed generation: {error}"))?; + let state_digest = ManifestDigest::from_sha256_bytes(&Sha256::digest(&manifest)) + .map_err(|error| format!("sealed manifest digest: {error}"))?; + Ok(PartitionedSealedV1 { + manifest, + state_digest, + segments: Arc::new(segments), + }) } /// Drain the sealed generation through the bounded batch path the daemon uses /// to ingest an artifact. Page budgets stay with the caller so the two benches /// do not silently share a commit width. pub(crate) fn drain_pages( - sealed: &[u8], - sealed_len: u64, - state_digest: &ManifestDigest, + sealed: &PartitionedSealedV1, control: &impl CodeIndexExecutionControlV1, bounds: SealedDrainBounds, ) -> Result< @@ -285,13 +329,20 @@ pub(crate) fn drain_pages( bounds.batch_retained_bytes, ) .map_err(|error| format!("sealed lexical batch bounds: {error}"))?; - let mut source = VerifiedSealedLexicalPageSourceV1::open( - Cursor::new(sealed.to_vec()), - sealed_len, - state_digest.clone(), + let segments = Arc::clone(&sealed.segments); + let mut source = VerifiedSealedLexicalPageSourceV1::open_partitioned_sealed( + &sealed.manifest, + sealed.state_digest.clone(), + move |digest, _, buffer, _control| { + let bytes = segments.get(digest.as_str()).ok_or_else(|| { + CodeIndexProductionErrorV1::Contract("bench segment is missing".to_owned()) + })?; + buffer.clear(); + buffer.extend_from_slice(bytes); + Ok(()) + }, bounds.page_chunks, bounds.page_bytes, - control, ) .map_err(|error| format!("open sealed lexical page source: {error}"))?; let mut pages = Vec::new(); diff --git a/crates/tracedecay-query/src/bin/tracedecay_index_bench.rs b/crates/tracedecay-query/src/bin/tracedecay_index_bench.rs index 69f2af8227..14a796fdc1 100644 --- a/crates/tracedecay-query/src/bin/tracedecay_index_bench.rs +++ b/crates/tracedecay-query/src/bin/tracedecay_index_bench.rs @@ -55,7 +55,7 @@ mod artifact_bench; use artifact_bench::{ ActiveControl, AdmittedFile, ApplyingProjectionSink, MemoryPublicationStore, SealedDrainBounds, default_corpus_root, drain_pages, identity, load_corpus, millis, peak_rss_bytes, percentile, - replicate, sealed_state_digest, + replicate, seal_partitioned, }; use std::collections::{BTreeMap, BTreeSet}; use std::num::NonZeroUsize; @@ -86,9 +86,9 @@ use tracedecay_query::retrieval::lexical::{ CLONE_NEAR_MATCH_MINIMUM_COVERAGE_MILLIONTHS_V1, CLONE_NEAR_MATCH_TOKEN_WORK_BUDGET_V1, CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, CloneExactArtifactMemberV1, CodeLexicalArtifactBuilderV1, CodeLexicalArtifactFinalizationStepV1, - CodeLexicalArtifactReaderV1, CodeLexicalArtifactWriterRevisionV1, - CodeLexicalProjectionMetadataV1, MAX_CLONE_EXACT_PAGE_MEMBERS_V1, - MAX_CLONE_FINGERPRINT_PAGE_BODIES_V1, VerifiedCodeLexicalArtifactV1, + CodeLexicalArtifactReaderV1, CodeLexicalCloneRouteV1, CodeLexicalProjectionMetadataV1, + MAX_CLONE_EXACT_PAGE_MEMBERS_V1, MAX_CLONE_FINGERPRINT_PAGE_BODIES_V1, + VerifiedCodeLexicalArtifactV1, }; /// Bumped whenever the workload shape changes, so a profile comparison @@ -197,14 +197,13 @@ const HOTPATH_OUTPUT_PATH_ENV: &str = "HOTPATH_OUTPUT_PATH"; const HOTPATH_OUTPUT_FORMAT_ENV: &str = "HOTPATH_OUTPUT_FORMAT"; const USAGE: &str = "\ -usage: tracedecay-index-bench [--corpus DIR] [--replicas N] [--format-revision 14|15|16] [--clone-envelope] +usage: tracedecay-index-bench [--corpus DIR] [--replicas N] [--clone-envelope] --corpus DIR committed fixture corpus to index (default: $TRACEDECAY_INDEX_BENCH_CORPUS, else benchmark_data/index-bench/corpus beside this workspace) --replicas N index the corpus N times under distinct logical path prefixes (default: $TRACEDECAY_INDEX_BENCH_REPLICAS, else 1) - --format-revision lexical artifact revision (default: 16) --clone-envelope measure one-body refresh and clone query behavior -h, --help print this message @@ -215,8 +214,6 @@ report is written."; struct Options { corpus_root: PathBuf, replicas: usize, - writer_revision: CodeLexicalArtifactWriterRevisionV1, - format_revision: u32, clone_envelope: bool, } @@ -224,8 +221,6 @@ impl Options { fn parse(arguments: impl Iterator) -> Result, String> { let mut corpus_root: Option = None; let mut replicas: Option = None; - let mut writer_revision = CodeLexicalArtifactWriterRevisionV1::default(); - let mut format_revision = 16; let mut clone_envelope = false; let mut arguments = arguments.peekable(); while let Some(argument) = arguments.next() { @@ -243,20 +238,6 @@ impl Options { .ok_or_else(|| "--replicas needs a count".to_owned())?; replicas = Some(parse_replicas(&value)?); } - "--format-revision" => { - let value = arguments - .next() - .ok_or_else(|| "--format-revision needs 14, 15, or 16".to_owned())?; - writer_revision = match value.as_str() { - "14" => CodeLexicalArtifactWriterRevisionV1::V14, - "15" => CodeLexicalArtifactWriterRevisionV1::V15, - "16" => CodeLexicalArtifactWriterRevisionV1::V16, - _ => return Err("--format-revision needs 14, 15, or 16".to_owned()), - }; - format_revision = value - .parse() - .map_err(|error| format!("invalid artifact revision: {error}"))?; - } "--clone-envelope" => clone_envelope = true, other => return Err(format!("unrecognized argument {other:?}")), } @@ -274,8 +255,6 @@ impl Options { Ok(Some(Self { corpus_root, replicas, - writer_revision, - format_revision, clone_envelope, })) } @@ -556,20 +535,15 @@ fn run(options: &Options) -> Result { .map_err(|error| format!("read generation statistics: {error}"))?; let seal_started = Instant::now(); - let sealed = generations - .generation - .encode_sealed() - .map_err(|error| format!("encode sealed generation: {error}"))?; + let sealed = seal_partitioned(&generations.generation)?; let seal_wall = seal_started.elapsed(); - let sealed_len = sealed.len() as u64; - let state_digest = sealed_state_digest(&sealed)?; + let sealed_len = sealed.byte_len(); + let state_digest = sealed.state_digest.clone(); // Pass 3 - drain the sealed generation as bounded page batches. let drain_started = Instant::now(); let (pages, source_receipt) = drain_pages( &sealed, - sealed_len, - &state_digest, &control, SealedDrainBounds { batch_pages: BATCH_MAX_PAGES, @@ -587,10 +561,9 @@ fn run(options: &Options) -> Result { let artifact_path = scratch.path().join("lexical.sqlite"); let artifact = ingest_artifact( &artifact_path, - metadata, + metadata.clone(), &pages, &source_receipt, - options.writer_revision, &control, )?; let ingest_wall = ingest_started.elapsed(); @@ -598,7 +571,7 @@ fn run(options: &Options) -> Result { measure_clone_queries( &artifact_path, &artifact.receipt, - options.format_revision, + &metadata, &pages, generations.body_refresh.changed_path.as_deref(), &control, @@ -636,7 +609,7 @@ fn run(options: &Options) -> Result { committed_pages: artifact.committed_pages, committed_chunks: artifact.committed_chunks, artifact: &artifact.receipt, - artifact_format_revision: options.format_revision, + artifact_format_revision: artifact.receipt.format_revision(), clone_queries, corpus_wall, clean_wall: generations.clean_wall, @@ -747,6 +720,11 @@ fn projection_metadata( "retriever.lexical.index-bench.v1", ), exact_score_domain: identity::("score.exact.index-bench.v1"), + clone_route: Some(CodeLexicalCloneRouteV1 { + project_id: generation.manifest().project_id.clone(), + worktree_id: generation.snapshot().worktree.clone(), + snapshot_digest: generation.manifest().snapshot_digest.clone(), + }), } } @@ -761,15 +739,10 @@ fn ingest_artifact( metadata: CodeLexicalProjectionMetadataV1, pages: &[VerifiedSealedLexicalPageV1], source_receipt: &VerifiedSealedLexicalSourceReceiptV1, - writer_revision: CodeLexicalArtifactWriterRevisionV1, control: &ActiveControl, ) -> Result { - let mut builder = CodeLexicalArtifactBuilderV1::create_with_format_revision( - artifact_path, - metadata, - writer_revision, - ) - .map_err(|error| format!("create lexical artifact: {error}"))?; + let mut builder = CodeLexicalArtifactBuilderV1::create(artifact_path, metadata) + .map_err(|error| format!("create lexical artifact: {error}"))?; let mut progress = builder .progress() .map_err(|error| format!("read artifact progress: {error}"))?; @@ -839,34 +812,23 @@ fn clone_census(pages: &[VerifiedSealedLexicalPageV1]) -> serde_json::Value { fn measure_clone_queries( artifact_path: &Path, receipt: &VerifiedCodeLexicalArtifactV1, - format_revision: u32, + metadata: &CodeLexicalProjectionMetadataV1, pages: &[VerifiedSealedLexicalPageV1], excluded_path: Option<&str>, control: &ActiveControl, ) -> Result { let elapsed_micros = |duration: Duration| u64::try_from(duration.as_micros()).unwrap_or(u64::MAX); - // Always open the reader under --clone-envelope so peak RSS compares like - // with like across format revisions. Format 14 has no clone index, but it - // still mmaps the sealed artifact; skipping the open made v16−v14 look like - // a multi-GiB clone regression when most of the delta was "mmap vs no mmap". let open_started = Instant::now(); let reader = CodeLexicalArtifactReaderV1::open_with_control( artifact_path, receipt, + metadata, CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, control, ) .map_err(|error| format!("open clone artifact reader: {error}"))?; let first_reader_open_micros = elapsed_micros(open_started.elapsed()); - if format_revision < 15 { - drop(reader); - return Ok(serde_json::json!({ - "state": "unavailable", - "reason": "artifact_has_no_clone_index", - "first_reader_open_micros": first_reader_open_micros, - })); - } let Some(source) = pages .iter() .flat_map(VerifiedSealedLexicalPageV1::clone_bodies) @@ -925,7 +887,7 @@ fn measure_clone_queries( ) })?; - let fingerprint = if reader.has_clone_fingerprints() { + let fingerprint = { let read = reader .clone_fingerprint_page( &artifact_source.occurrence, @@ -982,11 +944,6 @@ fn measure_clone_queries( "point": cancelled.accounting.cancellation_point.map(|point| format!("{point:?}")), }, }) - } else { - serde_json::json!({ - "state": "unavailable", - "reason": "artifact_has_no_clone_fingerprints", - }) }; drop(reader); @@ -994,6 +951,7 @@ fn measure_clone_queries( let restarted = CodeLexicalArtifactReaderV1::open_with_control( artifact_path, receipt, + metadata, CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, control, ) diff --git a/crates/tracedecay-query/src/bin/tracedecay_search_bench.rs b/crates/tracedecay-query/src/bin/tracedecay_search_bench.rs index c84de10e31..b597e48889 100644 --- a/crates/tracedecay-query/src/bin/tracedecay_search_bench.rs +++ b/crates/tracedecay-query/src/bin/tracedecay_search_bench.rs @@ -33,7 +33,7 @@ mod artifact_bench; use artifact_bench::{ ActiveControl, AdmittedFile, ApplyingProjectionSink, MemoryPublicationStore, SealedDrainBounds, default_corpus_root, drain_pages, identity, load_corpus, millis, peak_rss_bytes, percentile, - replicate, sealed_state_digest, + replicate, seal_partitioned, }; use std::collections::BTreeSet; use std::io::Read; @@ -65,9 +65,9 @@ use tracedecay_query::retrieval::exact::{ }; use tracedecay_query::retrieval::lexical::{ CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, CodeLexicalArtifactBuilderV1, - CodeLexicalArtifactFinalizationStepV1, CodeLexicalArtifactReaderV1, - CodeLexicalArtifactWriterRevisionV1, CodeLexicalProjectionMetadataV1, LexicalLane, - LexicalLaneRequest, LexicalLaneRetriever, MAX_FUZZY_TERM_EXPANSIONS_V1, lexical_query_parts, + CodeLexicalArtifactFinalizationStepV1, CodeLexicalArtifactReaderV1, CodeLexicalCloneRouteV1, + CodeLexicalProjectionMetadataV1, LexicalLane, LexicalLaneRequest, LexicalLaneRetriever, + MAX_FUZZY_TERM_EXPANSIONS_V1, lexical_query_parts, }; use tracedecay_query::retrieval::ports::RetrievalExecutionControl; use tracedecay_query::retrieval::{ @@ -123,10 +123,7 @@ fn main() -> ExitCode { } }; - match options.artifact.as_ref().map_or_else( - || run(&options), - |path| run_existing_artifact(&options, path), - ) { + match run(&options) { Ok(summary) => { println!("{summary}"); ExitCode::SUCCESS @@ -159,8 +156,7 @@ fn configure_hotpath() { const USAGE: &str = "\ usage: tracedecay-search-bench [--corpus DIR] [--replicas N] [--iterations N] - [--warmups N] [--fuzzy-budget N] [--artifact FILE] - [--format-revision 11|12|13|14] + [--warmups N] [--fuzzy-budget N] [--class NAME]... [--term CLASS=QUERY]... --corpus DIR fixture corpus to index and query @@ -171,8 +167,6 @@ usage: tracedecay-search-bench [--corpus DIR] [--replicas N] [--iterations N] --iterations N timed query iterations per class (default: 40) --warmups N untimed warmup iterations per class (default: 3) --fuzzy-budget N lexical typo-recovery budget (default: production 64) - --artifact FILE reopen an existing sealed lexical artifact and skip ingest - --format-revision select the writer revision for build A/B runs (default: 14) --class NAME run only the named classes (repeatable; default: all) --term CLASS=QUERY override one class's query text (repeatable) -h, --help print this message @@ -208,8 +202,6 @@ struct Options { iterations: usize, warmups: usize, fuzzy_budget: u32, - artifact: Option, - writer_revision: CodeLexicalArtifactWriterRevisionV1, classes: Vec<(String, String)>, } @@ -222,8 +214,6 @@ impl Options { let mut fuzzy_budget = MAX_FUZZY_TERM_EXPANSIONS_V1; let mut selected: Vec = Vec::new(); let mut overrides: Vec<(String, String)> = Vec::new(); - let mut artifact = None; - let mut writer_revision = CodeLexicalArtifactWriterRevisionV1::default(); let mut arguments = arguments.peekable(); while let Some(argument) = arguments.next() { match argument.as_str() { @@ -265,24 +255,6 @@ impl Options { .ok_or_else(|| "--class needs a name".to_owned())?; selected.push(value); } - "--artifact" => { - let value = arguments - .next() - .ok_or_else(|| "--artifact needs a file".to_owned())?; - artifact = Some(PathBuf::from(value)); - } - "--format-revision" => { - let value = arguments - .next() - .ok_or_else(|| "--format-revision needs 11, 12, 13, or 14".to_owned())?; - writer_revision = match value.as_str() { - "11" => CodeLexicalArtifactWriterRevisionV1::V11, - "12" => CodeLexicalArtifactWriterRevisionV1::V12, - "13" => CodeLexicalArtifactWriterRevisionV1::V13, - "14" => CodeLexicalArtifactWriterRevisionV1::V14, - _ => return Err("--format-revision needs 11, 12, 13, or 14".to_owned()), - }; - } "--term" => { let value = arguments .next() @@ -329,8 +301,6 @@ impl Options { iterations, warmups, fuzzy_budget, - artifact, - writer_revision, classes, })) } @@ -400,18 +370,13 @@ fn run(options: &Options) -> Result { let chunk_count = generation.chunks().chunks().len() as u64; let seal_started = Instant::now(); - let sealed = generation - .encode_sealed() - .map_err(|error| format!("encode sealed generation: {error}"))?; + let sealed = seal_partitioned(&generation)?; let seal_wall = seal_started.elapsed(); - let sealed_len = sealed.len() as u64; - let state_digest = sealed_state_digest(&sealed)?; + let sealed_len = sealed.byte_len(); let drain_started = Instant::now(); let (pages, source_receipt) = drain_pages( &sealed, - sealed_len, - &state_digest, &control, SealedDrainBounds { batch_pages: BATCH_MAX_PAGES, @@ -428,10 +393,9 @@ fn run(options: &Options) -> Result { let ingest_started = Instant::now(); let receipt = ingest_artifact( &artifact_path, - metadata, + metadata.clone(), &pages, &source_receipt, - options.writer_revision, &control, )?; let ingest_wall = ingest_started.elapsed(); @@ -446,6 +410,7 @@ fn run(options: &Options) -> Result { &artifact_path, &file_digest, file_size_bytes, + &metadata, CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, &control, ) @@ -514,111 +479,6 @@ fn run(options: &Options) -> Result { serde_json::to_string_pretty(&report).map_err(|error| format!("serialize summary: {error}")) } -/// Reopen a preserved sealed artifact and measure the same query classes -/// without repeating ingest. Used to verify read-path fixes against a -/// generation-scale file. -fn run_existing_artifact(options: &Options, artifact_path: &Path) -> Result { - let started = Instant::now(); - let control = ActiveControl; - - let open_started = Instant::now(); - let (file_digest, file_size_bytes) = hash_file(artifact_path)?; - let reader = CodeLexicalArtifactReaderV1::open_content_addressed( - artifact_path, - &file_digest, - file_size_bytes, - CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, - &control, - ) - .map_err(|error| format!("reopen lexical artifact: {error}"))?; - let open_wall = open_started.elapsed(); - - let authority = CentralExactAdmissionAuthorityV1::new( - ExactAdmissionRuleRevision::new(QUERY_EXACT_RULE_REVISION_V1) - .map_err(|error| format!("exact rule revision: {error}"))?, - ); - let exact_lane = ExactLane::new(authority.clone(), reader.exact_adapter(authority.clone())); - let lexical_lane = LexicalLane::new(reader.clone()); - let generation_id = reader.metadata().generation.clone(); - let prototype = request_prototype_from_artifact(&reader)?; - - let mut class_reports = Vec::with_capacity(options.classes.len()); - for (class, query) in &options.classes { - let report = run_class(RunClassArguments { - class, - query, - options, - prototype: &prototype, - authority: &authority, - exact_lane: &exact_lane, - lexical_lane: &lexical_lane, - generation: &generation_id, - })?; - class_reports.push(report); - } - - let verified = reader.verified_artifact(); - let total_wall = started.elapsed(); - let report = serde_json::json!({ - "workload_revision": WORKLOAD_REVISION, - "reused_artifact": true, - "artifact_path": artifact_path.display().to_string(), - "chunks": verified.total_chunks(), - "artifact_bytes": file_size_bytes, - "artifact_digest": file_digest.as_str(), - "artifact_logical_digest": verified.artifact_digest().as_str(), - "iterations": options.iterations, - "warmups": options.warmups, - "fuzzy_budget": options.fuzzy_budget, - "peak_rss_bytes": peak_rss_bytes(), - "build_wall_ms": { - "artifact_reopen_verified": millis(open_wall), - "total": millis(total_wall), - }, - "classes": class_reports, - }); - serde_json::to_string_pretty(&report).map_err(|error| format!("serialize summary: {error}")) -} - -fn request_prototype_from_artifact( - reader: &CodeLexicalArtifactReaderV1, -) -> Result { - let metadata = reader.metadata(); - let verified = reader.verified_artifact(); - let repository = metadata - .repository_id - .clone() - .or_else(|| verified.repository_id().cloned()) - .unwrap_or_else(|| identity("repository.search-bench")); - Ok(RequestPrototypeV1 { - principal: identity::("principal.search-bench"), - scope: RetrievalScope { - privacy_domain: identity::("privacy.search-bench"), - root: SingleRootScopeV1 { - repository, - worktree: None, - reference: None, - }, - }, - snapshot: RetrievalSnapshot { - watermarks: VectorWatermark::default(), - freshness_digest: FreshnessVectorDigest::new(verified.source_state_digest().as_str()) - .map_err(|error| format!("freshness digest: {error}"))?, - authorization_revision: identity::( - "authorization.search-bench.v1", - ), - captured_at: metadata.freshness.observed_at, - }, - profile_id: identity::("query-fallback"), - sanitizer_revision: identity::(QUERY_SANITIZER_REVISION_V1), - normalization_revision: identity::( - QUERY_NORMALIZATION_REVISION_V1, - ), - lexical_profile_revision: identity::(QUERY_LEXICAL_PROFILE_REVISION_V1), - lexical_score_domain: identity::(QUERY_LEXICAL_SCORE_DOMAIN_V1), - }) -} - /// Query-independent request fields, cloned per iteration exactly as the /// daemon clones its own per-request base. struct RequestPrototypeV1 { @@ -919,6 +779,11 @@ fn projection_metadata( "retriever.lexical.search-bench.v1", ), exact_score_domain: identity::("score.exact.search-bench.v1"), + clone_route: Some(CodeLexicalCloneRouteV1 { + project_id: generation.manifest().project_id.clone(), + worktree_id: generation.snapshot().worktree.clone(), + snapshot_digest: generation.manifest().snapshot_digest.clone(), + }), } } @@ -927,15 +792,10 @@ fn ingest_artifact( metadata: CodeLexicalProjectionMetadataV1, pages: &[VerifiedSealedLexicalPageV1], source_receipt: &VerifiedSealedLexicalSourceReceiptV1, - writer_revision: CodeLexicalArtifactWriterRevisionV1, control: &ActiveControl, ) -> Result { - let mut builder = CodeLexicalArtifactBuilderV1::create_with_format_revision( - artifact_path, - metadata, - writer_revision, - ) - .map_err(|error| format!("create lexical artifact: {error}"))?; + let mut builder = CodeLexicalArtifactBuilderV1::create(artifact_path, metadata) + .map_err(|error| format!("create lexical artifact: {error}"))?; for batch in pages.chunks(BATCH_MAX_PAGES) { builder .append_pages(batch, control) diff --git a/crates/tracedecay-query/src/retrieval/evidence_lanes.rs b/crates/tracedecay-query/src/retrieval/evidence_lanes.rs index 25f0c1b99c..67c60cd41b 100644 --- a/crates/tracedecay-query/src/retrieval/evidence_lanes.rs +++ b/crates/tracedecay-query/src/retrieval/evidence_lanes.rs @@ -24,7 +24,8 @@ use tracedecay_domain::{ SourceOccurrenceId, TemporalLaneEvidenceV1, canonical_sha256, }; use tracedecay_temporal_query::TemporalCandidateExport; -use tracedecay_temporal_query::ports::{ExecutionControl, TemporalPortError}; +use tracedecay_temporal_query::execution::ExecutionControl; +use tracedecay_temporal_query::ports::TemporalPortError; use super::ports::{RetrievalPortError, contract_error}; diff --git a/crates/tracedecay-query/src/retrieval/lexical.rs b/crates/tracedecay-query/src/retrieval/lexical.rs index 99d5738245..c79d20d88c 100644 --- a/crates/tracedecay-query/src/retrieval/lexical.rs +++ b/crates/tracedecay-query/src/retrieval/lexical.rs @@ -46,18 +46,12 @@ pub use self::projection::{ CodeLexicalArtifactBuilderV1, CodeLexicalArtifactErrorV1, CodeLexicalArtifactFinalizationPhaseV1, CodeLexicalArtifactFinalizationStepV1, CodeLexicalArtifactOccurrenceV1, CodeLexicalArtifactReaderV1, - CodeLexicalArtifactSectionDigestV1, CodeLexicalArtifactWriterRevisionV1, - CodeLexicalCloneIndexCensusV1, CodeLexicalCloneSuccessorV1, + CodeLexicalArtifactSectionDigestV1, CodeLexicalCloneIndexCensusV1, CodeLexicalCloneRouteV1, CodeLexicalImportMembershipWitnessV1, CodeLexicalProjectionMetadataV1, MAX_CLONE_EXACT_PAGE_MEMBERS_V1, MAX_CLONE_FINGERPRINT_PAGE_BODIES_V1, PreparedCodeLexicalArtifactBatchV1, PreparedCodeLexicalArtifactPageV1, VerifiedCodeLexicalArtifactV1, code_lexical_artifact_build_memory_budget_for, -}; -#[cfg(feature = "search-eval")] -pub use self::projection::{ - CodeExactProjectionAdapterV1, CodeLexicalProjectionAdapterV1, CodeLexicalProjectionBuildStepV1, - CodeLexicalProjectionBuildV1, LEXICAL_PROJECTION_BUILD_DEADLINE_MICROS_V1, - lexical_projection_build_deadline_micros, + code_lexical_artifact_content_key, }; pub use self::routes::{ LexicalAliasV1, LexicalAlternativeReasonV1, LexicalAnchorV1, LexicalRouteErrorV1, diff --git a/crates/tracedecay-query/src/retrieval/lexical/projection.rs b/crates/tracedecay-query/src/retrieval/lexical/projection.rs index 3b61929551..f3fbfe7369 100644 --- a/crates/tracedecay-query/src/retrieval/lexical/projection.rs +++ b/crates/tracedecay-query/src/retrieval/lexical/projection.rs @@ -7,9 +7,10 @@ use tracedecay_domain::{ BoundedSanitizedText, CodeGenerationId, CodeSearchChunkAnchorV1, CodeSearchChunkGrainV1, CodeSearchChunkId, CodeSearchChunkV1, CompactCandidate, ComponentRevision, EvidenceRole, ExactAdmissionProof, ExactFieldV1, ExactTechnicalTermKindV1, ExactTechnicalTermV1, - FileOccurrenceId, FixedPointScore, LanguageDescriptorRevision, LogicalEvidenceId, RepositoryId, - RetrievalAnchorId, RetrieverKind, ScoreDomainId, SourceFreshness, SourceOccurrenceId, - exact_search_canonical, split_subtokens, technical_tokens, validate_code_logical_path, + FileOccurrenceId, FixedPointScore, LanguageDescriptorRevision, LogicalEvidenceId, + ManifestDigest, ProjectId, RepositoryId, RetrievalAnchorId, RetrieverKind, ScoreDomainId, + SourceFreshness, SourceOccurrenceId, WorktreeId, exact_search_canonical, split_subtokens, + technical_tokens, validate_code_logical_path, }; use super::{ @@ -22,8 +23,6 @@ use crate::retrieval::ports::{ }; mod artifact; -#[cfg(feature = "search-eval")] -mod in_memory; pub use artifact::{ CLONE_FINGERPRINT_CANDIDATE_BODY_BUDGET_V1, CLONE_FINGERPRINT_HOT_POSTING_THRESHOLD_V1, @@ -43,18 +42,11 @@ pub use artifact::{ CodeLexicalArtifactBuilderV1, CodeLexicalArtifactErrorV1, CodeLexicalArtifactFinalizationPhaseV1, CodeLexicalArtifactFinalizationStepV1, CodeLexicalArtifactOccurrenceV1, CodeLexicalArtifactReaderV1, - CodeLexicalArtifactSectionDigestV1, CodeLexicalArtifactWriterRevisionV1, - CodeLexicalCloneIndexCensusV1, CodeLexicalCloneSuccessorV1, + CodeLexicalArtifactSectionDigestV1, CodeLexicalCloneIndexCensusV1, CodeLexicalImportMembershipWitnessV1, MAX_CLONE_EXACT_PAGE_MEMBERS_V1, MAX_CLONE_FINGERPRINT_PAGE_BODIES_V1, PreparedCodeLexicalArtifactBatchV1, PreparedCodeLexicalArtifactPageV1, VerifiedCodeLexicalArtifactV1, - code_lexical_artifact_build_memory_budget_for, -}; -#[cfg(feature = "search-eval")] -pub use in_memory::{ - CodeExactProjectionAdapterV1, CodeLexicalProjectionAdapterV1, CodeLexicalProjectionBuildStepV1, - CodeLexicalProjectionBuildV1, LEXICAL_PROJECTION_BUILD_DEADLINE_MICROS_V1, - lexical_projection_build_deadline_micros, + code_lexical_artifact_build_memory_budget_for, code_lexical_artifact_content_key, }; const BM25_K1_MILLIS: u64 = 1_200; @@ -74,11 +66,37 @@ pub struct CodeLexicalProjectionMetadataV1 { pub exact_retriever_revision: ComponentRevision, pub lexical_retriever_revision: ComponentRevision, pub exact_score_domain: ScoreDomainId, + /// The route the projection's clone occurrences belong to; `None` when + /// the opener carries no clone authority. + pub clone_route: Option, +} + +/// Project, worktree, and snapshot of the generation an artifact is opened +/// for. Clone occurrences are stored without them, so identical trees in +/// different worktrees share one artifact and each opener supplies its own. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct CodeLexicalCloneRouteV1 { + pub project_id: ProjectId, + pub worktree_id: Option, + pub snapshot_digest: ManifestDigest, } impl CodeLexicalProjectionMetadataV1 { fn validate(&self) -> Result<(), RetrievalPortError> { self.generation.validate().map_err(contract_error)?; + if let Some(route) = &self.clone_route { + if self.repository_id.is_none() { + return Err(RetrievalPortError::Contract( + "a clone route requires the projection's repository".to_owned(), + )); + } + route.project_id.validate().map_err(contract_error)?; + if let Some(worktree_id) = &route.worktree_id { + worktree_id.validate().map_err(contract_error)?; + } + route.snapshot_digest.validate().map_err(contract_error)?; + } if let Some(repository_id) = &self.repository_id { repository_id.validate().map_err(contract_error)?; } @@ -661,7 +679,7 @@ impl LexicalFieldTextV1 for ProjectedChunkV1 { } } -/// Row identity shared by the in-memory projection and the artifact reader. +/// Row identity the artifact builder and reader share. trait LexicalIndexedRow { fn chunk_id(&self) -> &CodeSearchChunkId; fn anchor(&self) -> &CodeSearchChunkAnchorV1; @@ -863,8 +881,8 @@ fn add_score(scores: &mut BTreeMap, field: LexicalFieldV1, .or_insert(score); } -/// Shared exact/fuzzy/phrase/proximity scoring for the in-memory projection -/// and the artifact reader. Callers supply term frequencies and BM25 inputs; +/// Exact/fuzzy/phrase/proximity scoring for artifact rows. Callers supply +/// term frequencies and BM25 inputs; /// the loop, fuzzy discount, phrase boost, and echo penalty stay one place. #[allow(clippy::too_many_arguments)] fn score_lexical_row( diff --git a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact.rs b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact.rs index 4cbba3bae1..ec5a7a9bd4 100644 --- a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact.rs +++ b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact.rs @@ -12,7 +12,7 @@ use tracedecay_code_index::production::{CodeIndexExecutionControlV1, CodeIndexIn mod builder; mod clone_census; -mod clone_successor; +mod clone_codec; mod fingerprints; mod format; mod postings; @@ -27,7 +27,6 @@ pub use builder::{ PreparedCodeLexicalArtifactBatchV1, }; pub use clone_census::CodeLexicalCloneIndexCensusV1; -pub use clone_successor::CodeLexicalCloneSuccessorV1; pub use fingerprints::{ CLONE_FINGERPRINT_CANDIDATE_BODY_BUDGET_V1, CLONE_FINGERPRINT_HOT_POSTING_THRESHOLD_V1, CLONE_FINGERPRINT_POSTING_ROW_BUDGET_V1, CLONE_NEAR_MATCH_BODY_COMPARISON_BUDGET_V1, @@ -41,6 +40,7 @@ pub use fingerprints::{ pub use format::{ CodeLexicalArtifactOccurrenceV1, CodeLexicalArtifactSectionDigestV1, CodeLexicalImportMembershipWitnessV1, VerifiedCodeLexicalArtifactV1, + code_lexical_artifact_content_key, }; pub use prepared::PreparedCodeLexicalArtifactPageV1; pub use reader::{ @@ -48,7 +48,6 @@ pub use reader::{ CloneExactFamilyArtifactCandidateV1, CloneExactFamilyArtifactPageV1, CodeExactLexicalArtifactReaderV1, CodeLexicalArtifactReaderV1, MAX_CLONE_EXACT_PAGE_MEMBERS_V1, }; -pub use schema::CodeLexicalArtifactWriterRevisionV1; /// Floor for the artifact build memory ledger. /// diff --git a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/builder.rs b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/builder.rs index 1edad59958..7cf7f890a8 100644 --- a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/builder.rs +++ b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/builder.rs @@ -1,14 +1,13 @@ -use std::cmp::{Ordering as CmpOrdering, Reverse}; -use std::collections::{BTreeMap, BinaryHeap, HashMap}; +use std::cmp::Reverse; +use std::collections::{BTreeMap, BinaryHeap, HashMap, HashSet}; use std::fs::File; +use std::io::{Seek, SeekFrom, Write}; use std::num::NonZeroUsize; use std::panic::{AssertUnwindSafe, catch_unwind, resume_unwind}; use std::path::{Path, PathBuf}; use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicU8, Ordering}; use std::time::Duration; -#[cfg(feature = "hotpath")] -use std::time::Instant; use rayon::prelude::*; use rusqlite::functions::FunctionFlags; @@ -24,30 +23,38 @@ use tracedecay_code_index::production::{ VerifiedSealedLexicalSourceReceiptV1, VerifiedSealedLexicalSymbolDisplayV1, }; use tracedecay_domain::{ - CodeSearchChunkAnchorV1, CodeSearchChunkV1, ExactTechnicalTermV1, FileOccurrenceId, - ManifestDigest, + CodeGenerationId, CodeSearchChunkAnchorV1, CodeSearchChunkV1, ExactTechnicalTermV1, + FileOccurrenceId, ManifestDigest, }; use tracedecay_private_fs::framed_log::{DirectorySyncPolicy, sync_parent_directory}; use tracedecay_private_fs::{create_private_file_retained, open_private_file}; +use super::clone_codec::{digest_from_key, digest_key}; use super::format::{ - BASE_SECTION_NAMES, CodeLexicalArtifactSectionDigestV1, RECEIPT_RESERVATION_BYTES, - SECTION_NAMES, SERVING_INDEX_STEP_COUNT_V11, STATISTICS_STEP_COUNT_V11, - VerifiedCodeLexicalArtifactV1, absorb_page_base_sections_receipt, artifact_digest, - contract_number, decode_padded_receipt, decode_padded_receipt_with_control, + BASE_SECTION_NAMES, CodeLexicalArtifactSectionDigestV1, PostingListDecoderV1, + PostingListEncoderV1, RECEIPT_RESERVATION_BYTES, SECTION_NAMES, SERVING_INDEX_STEP_COUNT_V11, + STATISTICS_STEP_COUNT_V11, VerifiedCodeLexicalArtifactV1, absorb_page_base_sections_receipt, + content_metadata_bytes, contract_number, decode_fingerprint_postings, decode_ngram_bitmap, + decode_padded_receipt, decode_padded_receipt_with_control, decode_page_base_sections_receipt, + encode_document_set, encode_fingerprint_postings, encode_term_lists, finish_base_section_receipt_fold, hash_bytes, initial_base_section_receipt_fold, - metadata_digest, new_verified_receipt, padded_receipt, section_names, - verify_artifact_table_layout, verify_required_artifact_indexes, + metadata_digest, new_verified_receipt, padded_receipt, receipt_artifact_digest, + stored_metadata_digest, verify_artifact_table_layout, }; use super::postings::document_ngram_scratch; +use super::prepared::document_ngram_keys; use super::prepared::{ PreparedCloneBodyV1, PreparedCodeLexicalArtifactPageV1, PreparedTermPostingV1, prepare_page as prepare_page_values, }; +use super::row_codec::{ + ConnectionRowDictionaryV1, decode_artifact_row, decode_row_block, stored_chunk_key, + stored_symbol_key, +}; use super::schema::{ - CodeLexicalArtifactWriterRevisionV1, LexicalArtifactLayoutV1, derive_row_dictionary, - exact_field_code_from_encoded, field_code, field_code_from_encoded, intern_exact_terms, - intern_terms, stable_exact_term_id, stable_term_id, stage_row_dictionary, + CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V1, derive_row_dictionary, exact_field_code_from_encoded, + field_code, field_code_from_encoded, intern_exact_terms, require_served_revision, + stable_exact_term_id, stage_row_dictionary, }; use super::{ ARTIFACT_SQLITE_CACHE_BYTES, CODE_LEXICAL_ARTIFACT_BUILD_MEMORY_BUDGET_BYTES_V1, @@ -60,31 +67,30 @@ use super::{ }; use crate::retrieval::lexical::LexicalFieldV1; -use super::super::CodeLexicalProjectionMetadataV1; +use super::super::{CodeLexicalProjectionMetadataV1, normalized_search_text}; -const PROGRESS_TAIL_QUERY: &str = "SELECT page_ordinal, import_dictionary_digest, cumulative_digest, next_cursor \ - FROM source_pages ORDER BY page_ordinal DESC LIMIT 1"; +const SQLITE_HEADER_COMMIT_COUNTER_OFFSETS: [u64; 2] = [24, 92]; +const SEALED_COMMIT_COUNTER: u32 = 1; +const PROGRESS_TAIL_QUERY: &str = "SELECT page_ordinal, next_cursor \ + FROM source_page_cursors ORDER BY page_ordinal DESC LIMIT 1"; const FINALIZATION_PROGRESS_INTERVAL_OPS: i32 = 4_096; const FINALIZATION_CONTROL_POLL_INTERVAL: Duration = Duration::from_millis(1); -// A plan entry owns the three clustered-key integers (document id, -// content-addressed term id, integer field code, in the layout's key order) -// and one posting reference. Four words match the 64-bit layout and cover -// the 32-bit pointer; general allocator metadata remains outside the ledger -// contract. -const TERM_INSERT_PLAN_BYTES_PER_REF: usize = 4 * std::mem::size_of::(); +// A plan entry owns its sort key (borrowed term text, integer field code, +// document id) and one posting reference. Five words match the 64-bit +// layout and cover the 32-bit one; general allocator metadata remains +// outside the ledger contract. +const TERM_INSERT_PLAN_BYTES_PER_REF: usize = 5 * std::mem::size_of::(); const TERM_INSERT_CONTROL_INTERVAL: usize = 4_096; -const TERM_INSERT_SORT_RUN_ROWS: usize = 4_096; -// An exact-posting plan entry owns document, field, and term identifiers plus -// the original field/term borrowed slices needed by the revision-11 writer. -// Eight words conservatively covers both the 32-bit and 64-bit layouts; -// general allocator metadata remains outside the ledger contract. +// An exact-posting plan entry owns its document, field, and term identifiers. +// Eight words conservatively over-reserves that entry on both the 32-bit and +// 64-bit layouts; general allocator metadata remains outside the ledger +// contract. const EXACT_INSERT_PLAN_BYTES_PER_REF: usize = 8 * std::mem::size_of::(); const EXACT_INSERT_CONTROL_INTERVAL: usize = TERM_INSERT_CONTROL_INTERVAL; -const EXACT_INSERT_SORT_RUN_ROWS: usize = TERM_INSERT_SORT_RUN_ROWS; /// Rows per multi-row `INSERT ... VALUES (...), (...)` statement in the /// append phase. The per-row cost of the base-table inserts is statement -/// overhead plus the per-row builder-gate trigger, not I/O: on the -/// `term_postings` shape at production pragmas (120k sorted rows in one +/// overhead plus the per-row builder-gate trigger, not I/O: on a +/// one-row-per-posting shape at production pragmas (120k sorted rows in one /// transaction) one row per statement costs 1.47 µs/row and 32 rows per /// statement 0.86 µs/row, the trigger still firing for every row. Five /// columns × 32 rows stays under SQLite's 999-parameter floor. @@ -97,204 +103,71 @@ const BUILDER_MUTATION_GATE_FUNCTION: &str = "tracedecay_lexical_builder_append_ const BUILDER_MUTATION_IDLE: u8 = 0; const BUILDER_MUTATION_APPEND: u8 = 1; -/// One planned `term_postings` row, keyed in the layout's clustered order so -/// inserts land in tree order. Revision 13 leads with `document_id`: a -/// batch's documents are contiguous and monotone, so the whole batch appends -/// at the tail instead of touching one leaf per hashed term id. +/// One planned posting, keyed `(term, field, document_id)` so the merged +/// stream yields each `(term, field)` run of the batch contiguously and in +/// document order, ready to encode as one `term_posting_runs` list. #[derive(Clone, Copy)] struct PreparedTermInsertRefV1<'a> { - key: (i64, i64, i64), + key: (&'a str, i64, i64), posting: &'a PreparedTermPostingV1, } impl<'a> PreparedTermInsertRefV1<'a> { - fn new( - layout: LexicalArtifactLayoutV1, - document_id: i64, - term_id: i64, - field: i64, - posting: &'a PreparedTermPostingV1, - ) -> Self { - let key = if layout.clusters_term_postings_by_document() { - (document_id, term_id, field) - } else { - (term_id, field, document_id) - }; - Self { key, posting } - } - - fn key(&self) -> (i64, i64, i64) { - self.key - } - - /// `(term_id, field, document_id)` in column order. - fn columns(&self, layout: LexicalArtifactLayoutV1) -> (i64, i64, i64) { - let (first, second, third) = self.key; - if layout.clusters_term_postings_by_document() { - (second, third, first) - } else { - (first, second, third) + fn new(document_id: i64, field: i64, posting: &'a PreparedTermPostingV1) -> Self { + Self { + key: (posting.term.as_str(), field, document_id), + posting, } } -} - -#[derive(Clone, Copy)] -struct PreparedTermMergeCursorV1<'a> { - entry: PreparedTermInsertRefV1<'a>, - run_index: usize, - run_offset: usize, -} - -impl PartialEq for PreparedTermMergeCursorV1<'_> { - fn eq(&self, other: &Self) -> bool { - self.entry.key() == other.entry.key() - && self.run_index == other.run_index - && self.run_offset == other.run_offset - } -} - -impl Eq for PreparedTermMergeCursorV1<'_> {} - -impl PartialOrd for PreparedTermMergeCursorV1<'_> { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} -impl Ord for PreparedTermMergeCursorV1<'_> { - fn cmp(&self, other: &Self) -> CmpOrdering { - self.entry - .key() - .cmp(&other.entry.key()) - .then_with(|| self.run_index.cmp(&other.run_index)) - .then_with(|| self.run_offset.cmp(&other.run_offset)) + fn key(&self) -> (&'a str, i64, i64) { + self.key } } +/// Every planned term posting of one batch, sorted by key. Keys are unique +/// (one posting per document, field, and term), so the order is total. struct PreparedTermInsertPlanV1<'a> { entries: Vec>, - merge_heap: BinaryHeap>>, - /// The batch's distinct terms with the ids this plan content-addressed, - /// ascending by term text. `vocabulary` is interned in exactly that - /// order, so the sealed page layout does not depend on how the plan - /// collected them, and the intern step neither rewalks every posting nor - /// recomputes a digest this pass already produced. - interned_terms: Vec<(&'a str, i64)>, -} - -// `exact_postings` shares `term_postings`'s clustered key shape (field, -// term, document_id) so the same bounded k-way merge sort pattern applies: -// insert in `PRIMARY KEY`/`WITHOUT ROWID` clustered order instead of raw -// arrival order to avoid B-tree page splits on out-of-order inserts. +} + +// Exact postings sort the same way as term postings, keyed +// `(term_id, field, document_id)` so each `exact_posting_runs` list is +// contiguous in the sorted stream. #[derive(Clone, Copy)] -struct PreparedExactInsertRefV1<'a> { +struct PreparedExactInsertRefV1 { document_id: i64, - field: &'a str, field_code: i64, - term: &'a [u8], term_id: i64, - layout: LexicalArtifactLayoutV1, -} - -#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] -enum PreparedExactInsertKeyV1<'a> { - V11 { - field: &'a str, - term: &'a [u8], - document_id: i64, - }, - V12 { - term_id: i64, - field: i64, - document_id: i64, - }, -} - -impl PreparedExactInsertRefV1<'_> { - fn key(&self) -> PreparedExactInsertKeyV1<'_> { - match self.layout { - LexicalArtifactLayoutV1::V10 | LexicalArtifactLayoutV1::V11 => { - PreparedExactInsertKeyV1::V11 { - field: self.field, - term: self.term, - document_id: self.document_id, - } - } - LexicalArtifactLayoutV1::V12 - | LexicalArtifactLayoutV1::V13 - | LexicalArtifactLayoutV1::V14 - | LexicalArtifactLayoutV1::V15 - | LexicalArtifactLayoutV1::V16 => PreparedExactInsertKeyV1::V12 { - term_id: self.term_id, - field: self.field_code, - document_id: self.document_id, - }, - } - } -} - -#[derive(Clone, Copy)] -struct PreparedExactMergeCursorV1<'a> { - entry: PreparedExactInsertRefV1<'a>, - run_index: usize, - run_offset: usize, } -impl PartialEq for PreparedExactMergeCursorV1<'_> { - fn eq(&self, other: &Self) -> bool { - self.entry.key() == other.entry.key() - && self.run_index == other.run_index - && self.run_offset == other.run_offset - } -} - -impl Eq for PreparedExactMergeCursorV1<'_> {} - -impl PartialOrd for PreparedExactMergeCursorV1<'_> { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl Ord for PreparedExactMergeCursorV1<'_> { - fn cmp(&self, other: &Self) -> CmpOrdering { - self.entry - .key() - .cmp(&other.entry.key()) - .then_with(|| self.run_index.cmp(&other.run_index)) - .then_with(|| self.run_offset.cmp(&other.run_offset)) +impl PreparedExactInsertRefV1 { + fn key(&self) -> (i64, i64, i64) { + (self.term_id, self.field_code, self.document_id) } } struct PreparedExactInsertPlanV1<'a> { - entries: Vec>, - merge_heap: BinaryHeap>>, + entries: Vec, /// The batch's distinct exact terms with the ids this plan /// content-addressed, ascending by id, the order `exact_vocabulary` was /// always interned in. interned_terms: Vec<(&'a [u8], i64)>, } -const BUILDER_GATE_TRIGGER_LAYOUT: [(&str, &str, &str); 14] = [ +const BUILDER_GATE_TRIGGER_LAYOUT: [(&str, &str, &str); 15] = [ ("builder_gate_source_pages_insert", "source_pages", "INSERT"), - ( - "builder_gate_document_integrity_insert", - "document_integrity", - "INSERT", - ), - ( - "builder_gate_import_integrity_insert", - "import_integrity", - "INSERT", - ), ( "builder_gate_import_evidence_insert", "import_evidence", "INSERT", ), - ("builder_gate_rows_insert", "rows", "INSERT"), - ("builder_gate_rows_update", "rows", "UPDATE"), - ("builder_gate_rows_delete", "rows", "DELETE"), + ("builder_gate_row_blocks_insert", "row_blocks", "INSERT"), + ("builder_gate_row_blocks_update", "row_blocks", "UPDATE"), + ("builder_gate_row_blocks_delete", "row_blocks", "DELETE"), + ("builder_gate_row_chunks_insert", "row_chunks", "INSERT"), + ("builder_gate_row_chunks_update", "row_chunks", "UPDATE"), + ("builder_gate_row_chunks_delete", "row_chunks", "DELETE"), ( "builder_gate_term_postings_insert", "term_postings", @@ -331,6 +204,81 @@ const BUILDER_GATE_TRIGGER_LAYOUT: [(&str, &str, &str); 14] = [ "INSERT", ), ]; +/// Batches append each document's chunk id here in document order; +/// finalization sorts it into `row_chunks` and drops it, gates included. +const SOURCE_PAGE_CURSORS_BUILDER_GATE_TRIGGER_LAYOUT: [(&str, &str, &str); 1] = [( + "builder_gate_source_page_cursors_insert", + "source_page_cursors", + "INSERT", +)]; +const SOURCE_PAGE_CURSORS_IMMUTABLE_TRIGGER_LAYOUT: [(&str, &str, &str, &str); 2] = [ + ( + "immutable_source_page_cursors_update", + "source_page_cursors", + "UPDATE", + "immutable lexical source page cursors", + ), + ( + "immutable_source_page_cursors_delete", + "source_page_cursors", + "DELETE", + "immutable lexical source page cursors", + ), +]; +const ROW_CHUNK_PAGES_BUILDER_GATE_TRIGGER_LAYOUT: [(&str, &str, &str); 3] = [ + ( + "builder_gate_row_chunk_pages_insert", + "row_chunk_pages", + "INSERT", + ), + ( + "builder_gate_row_chunk_pages_update", + "row_chunk_pages", + "UPDATE", + ), + ( + "builder_gate_row_chunk_pages_delete", + "row_chunk_pages", + "DELETE", + ), +]; +/// Batches append postings in page order to these staging tables under the +/// private-builder gate; finalization merges each into its sealed serving +/// table and drops it, gates included. +const TERM_POSTING_RUNS_BUILDER_GATE_TRIGGER_LAYOUT: [(&str, &str, &str); 3] = [ + ( + "builder_gate_term_posting_runs_insert", + "term_posting_runs", + "INSERT", + ), + ( + "builder_gate_term_posting_runs_update", + "term_posting_runs", + "UPDATE", + ), + ( + "builder_gate_term_posting_runs_delete", + "term_posting_runs", + "DELETE", + ), +]; +const EXACT_POSTING_RUNS_BUILDER_GATE_TRIGGER_LAYOUT: [(&str, &str, &str); 3] = [ + ( + "builder_gate_exact_posting_runs_insert", + "exact_posting_runs", + "INSERT", + ), + ( + "builder_gate_exact_posting_runs_update", + "exact_posting_runs", + "UPDATE", + ), + ( + "builder_gate_exact_posting_runs_delete", + "exact_posting_runs", + "DELETE", + ), +]; const EXACT_VOCABULARY_BUILDER_GATE_TRIGGER_LAYOUT: [(&str, &str, &str); 3] = [ ( "builder_gate_exact_vocabulary_insert", @@ -444,7 +392,7 @@ const CLONE_IMMUTABLE_TRIGGER_LAYOUT: [(&str, &str, &str, &str); 6] = [ "immutable clone exact postings", ), ]; -const CLONE_FINGERPRINT_IMMUTABLE_TRIGGER_LAYOUT: [(&str, &str, &str, &str); 4] = [ +const CLONE_FINGERPRINT_IMMUTABLE_TRIGGER_LAYOUT: [(&str, &str, &str, &str); 2] = [ ( "immutable_clone_fingerprint_postings_update", "clone_fingerprint_postings", @@ -457,20 +405,8 @@ const CLONE_FINGERPRINT_IMMUTABLE_TRIGGER_LAYOUT: [(&str, &str, &str, &str); 4] "DELETE", "immutable clone fingerprint postings", ), - ( - "immutable_clone_fingerprint_counts_update", - "clone_fingerprint_counts", - "UPDATE", - "immutable clone fingerprint counts", - ), - ( - "immutable_clone_fingerprint_counts_delete", - "clone_fingerprint_counts", - "DELETE", - "immutable clone fingerprint counts", - ), ]; -const IMMUTABLE_TRIGGER_LAYOUT: [(&str, &str, &str, &str); 10] = [ +const IMMUTABLE_TRIGGER_LAYOUT: [(&str, &str, &str, &str); 6] = [ ( "immutable_source_pages_update", "source_pages", @@ -483,30 +419,6 @@ const IMMUTABLE_TRIGGER_LAYOUT: [(&str, &str, &str, &str); 10] = [ "DELETE", "immutable lexical source pages", ), - ( - "immutable_document_integrity_update", - "document_integrity", - "UPDATE", - "immutable lexical document integrity", - ), - ( - "immutable_document_integrity_delete", - "document_integrity", - "DELETE", - "immutable lexical document integrity", - ), - ( - "immutable_import_integrity_update", - "import_integrity", - "UPDATE", - "immutable lexical import integrity", - ), - ( - "immutable_import_integrity_delete", - "import_integrity", - "DELETE", - "immutable lexical import integrity", - ), ( "immutable_import_evidence_update", "import_evidence", @@ -688,17 +600,15 @@ enum FinalizationSectionV1 { ExactPostings, NgramPostings, FieldStatistics, - TermStatistics, Vocabulary, CloneOccurrences, CloneExactPostings, CloneBodyPayloads, - CloneFingerprintCounts, CloneFingerprintPostings, } impl FinalizationSectionV1 { - const ALL: [Self; 16] = [ + const ALL: [Self; 14] = [ Self::SourcePages, Self::DocumentIntegrity, Self::ImportIntegrity, @@ -708,12 +618,10 @@ impl FinalizationSectionV1 { Self::ExactPostings, Self::NgramPostings, Self::FieldStatistics, - Self::TermStatistics, Self::Vocabulary, Self::CloneOccurrences, Self::CloneExactPostings, Self::CloneBodyPayloads, - Self::CloneFingerprintCounts, Self::CloneFingerprintPostings, ]; @@ -739,294 +647,134 @@ impl FinalizationSectionV1 { Self::CloneOccurrences => "clone_occurrences", Self::CloneExactPostings => "clone_exact_postings", Self::CloneBodyPayloads => "clone_body_payloads", - Self::CloneFingerprintCounts => "clone_fingerprint_counts", Self::CloneFingerprintPostings => "clone_fingerprint_postings", Self::FieldStatistics => "field_stats", - Self::TermStatistics => "term_stats", Self::Vocabulary => "vocabulary", } } + /// Native digest query of every section the digest phase walks. Base + /// sections carry none: their digests are adopted from page receipts. #[hotpath::skip] - const fn full_query(self, layout: LexicalArtifactLayoutV1) -> &'static str { - match (self, layout) { - (Self::SourcePages, _) => { - "SELECT page_ordinal, page_digest, cumulative_digest, chunk_count, payload_bytes, import_count, import_payload_bytes, import_dictionary_digest, ngram_digest, base_sections_receipt, next_cursor FROM source_pages ORDER BY page_ordinal" - } - ( - Self::DocumentIntegrity, - LexicalArtifactLayoutV1::V14 - | LexicalArtifactLayoutV1::V15 - | LexicalArtifactLayoutV1::V16, - ) => "SELECT document_id, digest FROM document_integrity ORDER BY document_id", - (Self::DocumentIntegrity, _) => { - "SELECT document_id, chunk_id, digest FROM document_integrity ORDER BY document_id" - } - (Self::ImportIntegrity, _) => { - "SELECT canonical, digest FROM import_integrity ORDER BY canonical" - } - (Self::ImportEvidence, _) => { - "SELECT canonical, evidence FROM import_evidence ORDER BY canonical" - } - (Self::Rows, _) => "SELECT document_id, chunk_id, row FROM rows ORDER BY document_id", - (Self::TermPostings, LexicalArtifactLayoutV1::V10) => { - "SELECT field, term, document_id, frequency FROM term_postings ORDER BY field, term, document_id" - } - ( - Self::TermPostings, - LexicalArtifactLayoutV1::V11 - | LexicalArtifactLayoutV1::V12 - | LexicalArtifactLayoutV1::V13 - | LexicalArtifactLayoutV1::V14 - | LexicalArtifactLayoutV1::V15 - | LexicalArtifactLayoutV1::V16, - ) => { - "SELECT term_id, field, document_id, frequency FROM term_postings ORDER BY term_id, field, document_id" - } - (Self::ExactPostings, _) => { - "SELECT field, term, document_id FROM exact_postings ORDER BY field, term, document_id" - } - (Self::NgramPostings, _) => { - "SELECT page_ordinal, kind, ngram, documents, cardinality FROM ngram_postings ORDER BY page_ordinal, kind, ngram" - } - (Self::CloneOccurrences, _) => { - "SELECT symbol_occurrence_id, payload_digest, path, body_start, body_end, occurrence FROM clone_occurrences ORDER BY symbol_occurrence_id" - } - (Self::CloneExactPostings, _) => { - "SELECT class, normalization_revision, digest, symbol_occurrence_id, payload_digest FROM clone_exact_postings ORDER BY class, normalization_revision, digest, symbol_occurrence_id" - } - (Self::CloneBodyPayloads, _) => { - "SELECT payload_digest, payload FROM clone_body_payloads ORDER BY payload_digest" - } - (Self::CloneFingerprintCounts, _) => { - "SELECT language, class, normalization_revision, fingerprint, posting_count FROM clone_fingerprint_counts ORDER BY language, class, normalization_revision, fingerprint" - } - (Self::CloneFingerprintPostings, _) => { - "SELECT language, class, normalization_revision, fingerprint, symbol_occurrence_id, token_position, payload_digest, body_digest FROM clone_fingerprint_postings ORDER BY language, class, normalization_revision, fingerprint, symbol_occurrence_id, token_position" - } - (Self::FieldStatistics, _) => { - "SELECT field, total_length FROM field_stats ORDER BY field" - } - (Self::TermStatistics, LexicalArtifactLayoutV1::V10) => { - "SELECT field, term, document_frequency FROM term_stats ORDER BY field, term" - } - ( - Self::TermStatistics, - LexicalArtifactLayoutV1::V11 - | LexicalArtifactLayoutV1::V12 - | LexicalArtifactLayoutV1::V13 - | LexicalArtifactLayoutV1::V14 - | LexicalArtifactLayoutV1::V15 - | LexicalArtifactLayoutV1::V16, - ) => { - "SELECT term_id, field, document_frequency FROM term_stats ORDER BY term_id, field" - } - (Self::Vocabulary, LexicalArtifactLayoutV1::V10) => { - "SELECT term FROM vocabulary ORDER BY term" + const fn full_query(self) -> Option<&'static str> { + match self { + Self::SourcePages => Some( + "SELECT page_ordinal, chunk_count, import_count, import_payload_bytes, import_dictionary_digest, ngram_digest, base_sections_receipt FROM source_pages ORDER BY page_ordinal", + ), + Self::CloneOccurrences => Some( + "SELECT ordinal, symbol_key, payload_ordinal, path, body_start, body_end, eligibility FROM clone_occurrences ORDER BY ordinal", + ), + Self::CloneExactPostings => Some( + "SELECT class, normalization_revision, digest, occurrence_ordinal FROM clone_exact_postings ORDER BY class, normalization_revision, digest, occurrence_ordinal", + ), + Self::CloneBodyPayloads => Some( + "SELECT ordinal, payload_digest, payload FROM clone_body_payloads ORDER BY ordinal", + ), + Self::CloneFingerprintPostings => Some( + "SELECT language, class, normalization_revision, fingerprint, posting_count, postings FROM clone_fingerprint_postings ORDER BY language, class, normalization_revision, fingerprint", + ), + Self::FieldStatistics => { + Some("SELECT field, total_length FROM field_stats ORDER BY field") } - ( - Self::Vocabulary, - LexicalArtifactLayoutV1::V11 - | LexicalArtifactLayoutV1::V12 - | LexicalArtifactLayoutV1::V13 - | LexicalArtifactLayoutV1::V14 - | LexicalArtifactLayoutV1::V15 - | LexicalArtifactLayoutV1::V16, - ) => "SELECT term_id, term, in_fuzzy FROM vocabulary ORDER BY term_id", + // The vocabulary is every sealed term with its fuzzy flag; its + // posting lists belong to the adopted `term_postings` section. + Self::Vocabulary => Some("SELECT term, in_fuzzy FROM term_postings ORDER BY term"), + Self::DocumentIntegrity + | Self::ImportIntegrity + | Self::ImportEvidence + | Self::Rows + | Self::TermPostings + | Self::ExactPostings + | Self::NgramPostings => None, } } /// Bounded resumes seek a native table key, never a computed cursor. #[hotpath::skip] - const fn seek_query(self, layout: LexicalArtifactLayoutV1, after: bool) -> &'static str { - if let Some(query) = self.clone_seek_query(after) { - return query; - } + const fn seek_query(self, after: bool) -> Option<&'static str> { match (self, after) { - (Self::DocumentIntegrity, false) - if matches!( - layout, - LexicalArtifactLayoutV1::V14 - | LexicalArtifactLayoutV1::V15 - | LexicalArtifactLayoutV1::V16 - ) => - { - "SELECT document_id, digest FROM document_integrity ORDER BY document_id LIMIT ?1" - } - (Self::DocumentIntegrity, true) - if matches!( - layout, - LexicalArtifactLayoutV1::V14 - | LexicalArtifactLayoutV1::V15 - | LexicalArtifactLayoutV1::V16 - ) => - { - "SELECT document_id, digest FROM document_integrity WHERE document_id > ?1 ORDER BY document_id LIMIT ?2" - } - (Self::SourcePages, false) => { - "SELECT page_ordinal, page_digest, cumulative_digest, chunk_count, payload_bytes, import_count, import_payload_bytes, import_dictionary_digest, ngram_digest, base_sections_receipt, next_cursor FROM source_pages ORDER BY page_ordinal LIMIT ?1" - } - (Self::SourcePages, true) => { - "SELECT page_ordinal, page_digest, cumulative_digest, chunk_count, payload_bytes, import_count, import_payload_bytes, import_dictionary_digest, ngram_digest, base_sections_receipt, next_cursor FROM source_pages WHERE page_ordinal > ?1 ORDER BY page_ordinal LIMIT ?2" - } - (Self::DocumentIntegrity, false) => { - "SELECT document_id, chunk_id, digest FROM document_integrity ORDER BY document_id LIMIT ?1" - } - (Self::DocumentIntegrity, true) => { - "SELECT document_id, chunk_id, digest FROM document_integrity WHERE document_id > ?1 ORDER BY document_id LIMIT ?2" - } - (Self::ImportIntegrity, false) => { - "SELECT canonical, digest FROM import_integrity ORDER BY canonical LIMIT ?1" - } - (Self::ImportIntegrity, true) => { - "SELECT canonical, digest FROM import_integrity WHERE canonical > ?1 ORDER BY canonical LIMIT ?2" - } - (Self::ImportEvidence, false) => { - "SELECT canonical, evidence FROM import_evidence ORDER BY canonical LIMIT ?1" - } - (Self::ImportEvidence, true) => { - "SELECT canonical, evidence FROM import_evidence WHERE canonical > ?1 ORDER BY canonical LIMIT ?2" - } - (Self::Rows, false) => { - "SELECT document_id, chunk_id, row FROM rows ORDER BY document_id LIMIT ?1" - } - (Self::Rows, true) => { - "SELECT document_id, chunk_id, row FROM rows WHERE document_id > ?1 ORDER BY document_id LIMIT ?2" - } - (Self::TermPostings, false) => { - "SELECT term_id, field, document_id, frequency FROM term_postings ORDER BY term_id, field, document_id LIMIT ?1" - } - (Self::TermPostings, true) => { - "SELECT term_id, field, document_id, frequency FROM term_postings WHERE (term_id, field, document_id) > (?1, ?2, ?3) ORDER BY term_id, field, document_id LIMIT ?4" - } - (Self::ExactPostings, false) => { - "SELECT field, term, document_id FROM exact_postings ORDER BY field, term, document_id LIMIT ?1" - } - (Self::ExactPostings, true) => { - "SELECT field, term, document_id FROM exact_postings WHERE (field, term, document_id) > (?1, ?2, ?3) ORDER BY field, term, document_id LIMIT ?4" - } - (Self::NgramPostings, false) => { - "SELECT page_ordinal, kind, ngram, documents, cardinality FROM ngram_postings ORDER BY page_ordinal, kind, ngram LIMIT ?1" - } - (Self::NgramPostings, true) => { - "SELECT page_ordinal, kind, ngram, documents, cardinality FROM ngram_postings WHERE (page_ordinal, kind, ngram) > (?1, ?2, ?3) ORDER BY page_ordinal, kind, ngram LIMIT ?4" - } + (Self::SourcePages, false) => Some( + "SELECT page_ordinal, chunk_count, import_count, import_payload_bytes, import_dictionary_digest, ngram_digest, base_sections_receipt FROM source_pages ORDER BY page_ordinal LIMIT ?1", + ), + (Self::SourcePages, true) => Some( + "SELECT page_ordinal, chunk_count, import_count, import_payload_bytes, import_dictionary_digest, ngram_digest, base_sections_receipt FROM source_pages WHERE page_ordinal > ?1 ORDER BY page_ordinal LIMIT ?2", + ), (Self::FieldStatistics, false) => { - "SELECT field, total_length FROM field_stats ORDER BY field LIMIT ?1" - } - (Self::FieldStatistics, true) => { - "SELECT field, total_length FROM field_stats WHERE field > ?1 ORDER BY field LIMIT ?2" - } - (Self::TermStatistics, false) => { - "SELECT term_id, field, document_frequency FROM term_stats ORDER BY term_id, field LIMIT ?1" - } - (Self::TermStatistics, true) => { - "SELECT term_id, field, document_frequency FROM term_stats WHERE (term_id, field) > (?1, ?2) ORDER BY term_id, field LIMIT ?3" + Some("SELECT field, total_length FROM field_stats ORDER BY field LIMIT ?1") } + (Self::FieldStatistics, true) => Some( + "SELECT field, total_length FROM field_stats WHERE field > ?1 ORDER BY field LIMIT ?2", + ), (Self::Vocabulary, false) => { - "SELECT term_id, term, in_fuzzy FROM vocabulary ORDER BY term_id LIMIT ?1" - } - (Self::Vocabulary, true) => { - "SELECT term_id, term, in_fuzzy FROM vocabulary WHERE term_id > ?1 ORDER BY term_id LIMIT ?2" - } - ( - Self::CloneOccurrences - | Self::CloneExactPostings - | Self::CloneBodyPayloads - | Self::CloneFingerprintCounts - | Self::CloneFingerprintPostings, - _, - ) => { - unreachable!() + Some("SELECT term, in_fuzzy FROM term_postings ORDER BY term LIMIT ?1") } - } - } - - const fn clone_seek_query(self, after: bool) -> Option<&'static str> { - match (self, after) { + (Self::Vocabulary, true) => Some( + "SELECT term, in_fuzzy FROM term_postings WHERE term > ?1 ORDER BY term LIMIT ?2", + ), (Self::CloneOccurrences, false) => Some( - "SELECT symbol_occurrence_id, payload_digest, path, body_start, body_end, occurrence FROM clone_occurrences ORDER BY symbol_occurrence_id LIMIT ?1", + "SELECT ordinal, symbol_key, payload_ordinal, path, body_start, body_end, eligibility FROM clone_occurrences ORDER BY ordinal LIMIT ?1", ), (Self::CloneOccurrences, true) => Some( - "SELECT symbol_occurrence_id, payload_digest, path, body_start, body_end, occurrence FROM clone_occurrences WHERE symbol_occurrence_id > ?1 ORDER BY symbol_occurrence_id LIMIT ?2", + "SELECT ordinal, symbol_key, payload_ordinal, path, body_start, body_end, eligibility FROM clone_occurrences WHERE ordinal > ?1 ORDER BY ordinal LIMIT ?2", ), (Self::CloneExactPostings, false) => Some( - "SELECT class, normalization_revision, digest, symbol_occurrence_id, payload_digest FROM clone_exact_postings ORDER BY class, normalization_revision, digest, symbol_occurrence_id LIMIT ?1", + "SELECT class, normalization_revision, digest, occurrence_ordinal FROM clone_exact_postings ORDER BY class, normalization_revision, digest, occurrence_ordinal LIMIT ?1", ), (Self::CloneExactPostings, true) => Some( - "SELECT class, normalization_revision, digest, symbol_occurrence_id, payload_digest FROM clone_exact_postings WHERE (class, normalization_revision, digest, symbol_occurrence_id) > (?1, ?2, ?3, ?4) ORDER BY class, normalization_revision, digest, symbol_occurrence_id LIMIT ?5", + "SELECT class, normalization_revision, digest, occurrence_ordinal FROM clone_exact_postings WHERE (class, normalization_revision, digest, occurrence_ordinal) > (?1, ?2, ?3, ?4) ORDER BY class, normalization_revision, digest, occurrence_ordinal LIMIT ?5", ), (Self::CloneBodyPayloads, false) => Some( - "SELECT payload_digest, payload FROM clone_body_payloads ORDER BY payload_digest LIMIT ?1", + "SELECT ordinal, payload_digest, payload FROM clone_body_payloads ORDER BY ordinal LIMIT ?1", ), (Self::CloneBodyPayloads, true) => Some( - "SELECT payload_digest, payload FROM clone_body_payloads WHERE payload_digest > ?1 ORDER BY payload_digest LIMIT ?2", - ), - (Self::CloneFingerprintCounts, false) => Some( - "SELECT language, class, normalization_revision, fingerprint, posting_count FROM clone_fingerprint_counts ORDER BY language, class, normalization_revision, fingerprint LIMIT ?1", - ), - (Self::CloneFingerprintCounts, true) => Some( - "SELECT language, class, normalization_revision, fingerprint, posting_count FROM clone_fingerprint_counts WHERE (language, class, normalization_revision, fingerprint) > (?1, ?2, ?3, ?4) ORDER BY language, class, normalization_revision, fingerprint LIMIT ?5", + "SELECT ordinal, payload_digest, payload FROM clone_body_payloads WHERE ordinal > ?1 ORDER BY ordinal LIMIT ?2", ), (Self::CloneFingerprintPostings, false) => Some( - "SELECT language, class, normalization_revision, fingerprint, symbol_occurrence_id, token_position, payload_digest, body_digest FROM clone_fingerprint_postings ORDER BY language, class, normalization_revision, fingerprint, symbol_occurrence_id, token_position LIMIT ?1", + "SELECT language, class, normalization_revision, fingerprint, posting_count, postings FROM clone_fingerprint_postings ORDER BY language, class, normalization_revision, fingerprint LIMIT ?1", ), (Self::CloneFingerprintPostings, true) => Some( - "SELECT language, class, normalization_revision, fingerprint, symbol_occurrence_id, token_position, payload_digest, body_digest FROM clone_fingerprint_postings WHERE (language, class, normalization_revision, fingerprint, symbol_occurrence_id, token_position) > (?1, ?2, ?3, ?4, ?5, ?6) ORDER BY language, class, normalization_revision, fingerprint, symbol_occurrence_id, token_position LIMIT ?7", + "SELECT language, class, normalization_revision, fingerprint, posting_count, postings FROM clone_fingerprint_postings WHERE (language, class, normalization_revision, fingerprint) > (?1, ?2, ?3, ?4) ORDER BY language, class, normalization_revision, fingerprint LIMIT ?5", ), - _ => None, + ( + Self::DocumentIntegrity + | Self::ImportIntegrity + | Self::ImportEvidence + | Self::Rows + | Self::TermPostings + | Self::ExactPostings + | Self::NgramPostings, + _, + ) => None, } } + + fn walked_query(self, after: bool) -> Result<&'static str, CodeLexicalArtifactErrorV1> { + self.seek_query(after).ok_or_else(base_section_walk_error) + } +} + +fn base_section_walk_error() -> CodeLexicalArtifactErrorV1 { + CodeLexicalArtifactErrorV1::Corrupt( + "lexical artifact base sections are adopted from page receipts, never walked".to_owned(), + ) } #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] #[serde(rename_all = "snake_case", tag = "kind", content = "value")] enum PersistedFinalizationKeyV1 { Integer(i64), - Blob(Vec), Text(String), - TextTextInteger { - field: String, - term: String, - document_id: i64, - }, - TextBlobInteger { - field: String, - term: Vec, - document_id: i64, - }, - IntegerIntegerInteger { - page_ordinal: i64, - kind: i64, - ngram: i64, - }, - TextText { - field: String, - term: String, - }, - IntegerPair { - first: i64, - second: i64, - }, ClonePosting { class: i64, normalization_revision: i64, - digest: String, - symbol_occurrence_id: String, - }, - FingerprintCount { - language: String, - class: i64, - normalization_revision: i64, - fingerprint: i64, + digest: Vec, + occurrence_ordinal: i64, }, - FingerprintPosting { + Fingerprint { language: String, class: i64, normalization_revision: i64, fingerprint: i64, - symbol_occurrence_id: String, - token_position: i64, }, } @@ -1037,36 +785,18 @@ impl PersistedFinalizationKeyV1 { ( Self::Integer(_), FinalizationSectionV1::SourcePages - | FinalizationSectionV1::DocumentIntegrity - | FinalizationSectionV1::Rows - ) | ( - Self::Blob(_), - FinalizationSectionV1::ImportIntegrity | FinalizationSectionV1::ImportEvidence - ) | ( - Self::IntegerIntegerInteger { .. }, - FinalizationSectionV1::TermPostings | FinalizationSectionV1::NgramPostings - ) | ( - Self::TextBlobInteger { .. }, - FinalizationSectionV1::ExactPostings - ) | ( - Self::Integer(_), - FinalizationSectionV1::FieldStatistics | FinalizationSectionV1::Vocabulary - ) | ( - Self::IntegerPair { .. }, - FinalizationSectionV1::TermStatistics - ) | ( - Self::Text(_), - FinalizationSectionV1::CloneOccurrences | FinalizationSectionV1::CloneBodyPayloads - ) | ( - Self::ClonePosting { .. }, - FinalizationSectionV1::CloneExactPostings - ) | ( - Self::FingerprintCount { .. }, - FinalizationSectionV1::CloneFingerprintCounts - ) | ( - Self::FingerprintPosting { .. }, - FinalizationSectionV1::CloneFingerprintPostings - ) + | FinalizationSectionV1::FieldStatistics + | FinalizationSectionV1::CloneOccurrences + | FinalizationSectionV1::CloneBodyPayloads + ) | (Self::Text(_), FinalizationSectionV1::Vocabulary) + | ( + Self::ClonePosting { .. }, + FinalizationSectionV1::CloneExactPostings + ) + | ( + Self::Fingerprint { .. }, + FinalizationSectionV1::CloneFingerprintPostings + ) ) } } @@ -1118,6 +848,10 @@ struct PersistedFinalizationStateV1 { completed_rows: u64, content_epoch: i64, source_state_digest: ManifestDigest, + /// The accepted source's terminal cursor. Statistics drops the staged + /// per-page cursors before the seal, and a resumed finalization restores + /// its source to this cursor to re-mint the completion receipt. + terminal_cursor: Option>, } #[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)] @@ -1248,10 +982,6 @@ impl FinalizationWakeMetricsV1 { hotpath::gauge!("query.artifact.finalization.phase.clone_body_payloads_total") .inc(1u64); } - FinalizationSectionV1::CloneFingerprintCounts => { - hotpath::gauge!("query.artifact.finalization.phase.clone_fingerprint_counts_total") - .inc(1u64); - } FinalizationSectionV1::CloneFingerprintPostings => { hotpath::gauge!( "query.artifact.finalization.phase.clone_fingerprint_postings_total" @@ -1261,9 +991,6 @@ impl FinalizationWakeMetricsV1 { FinalizationSectionV1::FieldStatistics => { hotpath::gauge!("query.artifact.finalization.phase.field_stats_total").inc(1u64); } - FinalizationSectionV1::TermStatistics => { - hotpath::gauge!("query.artifact.finalization.phase.term_stats_total").inc(1u64); - } FinalizationSectionV1::Vocabulary => { hotpath::gauge!("query.artifact.finalization.phase.vocabulary_total").inc(1u64); } @@ -1326,11 +1053,10 @@ pub struct CodeLexicalArtifactBuilderV1 { /// Keeps the exact no-follow/private file handle alive while the SQLite /// connection is in use. Every public transition rebinds the pathname to /// this identity before it trusts the connection's contents. - _private_file: File, + private_file: File, file_identity: StableArtifactFileIdentityV1, connection: Connection, mutation_gate: Arc, - layout: LexicalArtifactLayoutV1, metadata: CodeLexicalProjectionMetadataV1, metadata_digest: ManifestDigest, memory_budget_bytes: usize, @@ -1367,39 +1093,11 @@ impl CodeLexicalArtifactBuilderV1 { ) } - pub fn create_with_format_revision( - path: impl AsRef, - metadata: CodeLexicalProjectionMetadataV1, - revision: CodeLexicalArtifactWriterRevisionV1, - ) -> Result { - Self::create_with_memory_budget_and_format_revision( - path, - metadata, - CODE_LEXICAL_ARTIFACT_BUILD_MEMORY_BUDGET_BYTES_V1, - revision, - ) - } - #[hotpath::measure(label = "query.artifact.create")] pub fn create_with_memory_budget( path: impl AsRef, metadata: CodeLexicalProjectionMetadataV1, memory_budget_bytes: usize, - ) -> Result { - Self::create_with_memory_budget_and_format_revision( - path, - metadata, - memory_budget_bytes, - CodeLexicalArtifactWriterRevisionV1::default(), - ) - } - - #[hotpath::measure(label = "query.artifact.create")] - pub fn create_with_memory_budget_and_format_revision( - path: impl AsRef, - metadata: CodeLexicalProjectionMetadataV1, - memory_budget_bytes: usize, - revision: CodeLexicalArtifactWriterRevisionV1, ) -> Result { metadata .validate() @@ -1412,15 +1110,8 @@ impl CodeLexicalArtifactBuilderV1 { "lexical artifact staging path already contains state".to_owned(), )); } - let layout = revision.layout(); let metadata_digest = metadata_digest(&metadata)?; - publish_initialized_staging( - path, - layout, - &metadata, - &metadata_digest, - memory_budget_bytes, - )?; + publish_initialized_staging(path, &metadata, &metadata_digest, memory_budget_bytes)?; let (connection, private_file, file_identity) = open_private_builder_connection(path, memory_budget_bytes)?; let mutation_gate = register_builder_mutation_gate(&connection)?; @@ -1428,11 +1119,10 @@ impl CodeLexicalArtifactBuilderV1 { crate::hotpath_metrics::Residency::Cold.record("query.artifact.residency"); Ok(Self { path: path.to_path_buf(), - _private_file: private_file, + private_file, file_identity, connection, mutation_gate, - layout, metadata, metadata_digest, memory_budget_bytes, @@ -1462,10 +1152,10 @@ impl CodeLexicalArtifactBuilderV1 { open_private_builder_connection(path, memory_budget_bytes) )?; let mutation_gate = register_builder_mutation_gate(&connection)?; - let layout = read_staged_artifact_layout(&connection)?; + require_staged_revision(&connection)?; hotpath::measure_block!("query.artifact.open.schema_verify", { require_integrity(&connection, control)?; - verify_artifact_table_layout(&connection, layout)?; + verify_artifact_table_layout(&connection)?; verify_builder_mutation_gate_schema(&connection) })?; let expected_digest = hotpath::measure_block!("query.artifact.open.metadata_restore", { @@ -1474,7 +1164,6 @@ impl CodeLexicalArtifactBuilderV1 { &connection, &expected_metadata, &expected_digest, - layout, control, )?; Ok::<_, CodeLexicalArtifactErrorV1>(expected_digest) @@ -1486,13 +1175,6 @@ impl CodeLexicalArtifactBuilderV1 { load_finalization_state(&connection)?, )) )?; - if receipt.is_some() - || finalization - .as_ref() - .is_some_and(|state| state.phase == PersistedFinalizationPhaseV1::Digest) - { - verify_required_artifact_indexes(&connection, layout)?; - } // A staging file still accepting pages must carry the per-batch field // totals; one staged without them (an older builder) cannot seal // correct statistics and is not resumed. @@ -1504,12 +1186,11 @@ impl CodeLexicalArtifactBuilderV1 { "lexical artifact was staged without incremental field statistics".to_owned(), )); } - // Likewise, a revision-16 staging file still accepting pages must - // carry the fingerprint staging table; one written straight into the - // keyed tree (an older builder) is refused as incompatible so the - // scheduler discards and restages it instead of retrying an `Io`. - if layout.has_clone_fingerprints() - && receipt.is_none() + // Likewise, a staging file still accepting pages must carry the + // fingerprint staging table; one without it is refused as + // incompatible so the scheduler discards and restages it instead of + // retrying an `Io`. + if receipt.is_none() && finalization.is_none() && !table_exists(&connection, "clone_fingerprint_postings_pages")? { @@ -1522,11 +1203,10 @@ impl CodeLexicalArtifactBuilderV1 { crate::hotpath_metrics::Residency::Rebuilding.record("query.artifact.residency"); Ok(Self { path: path.to_path_buf(), - _private_file: private_file, + private_file, file_identity, connection, mutation_gate, - layout, metadata: expected_metadata, metadata_digest: expected_digest, memory_budget_bytes, @@ -1542,6 +1222,17 @@ impl CodeLexicalArtifactBuilderV1 { progress(&self.connection) } + /// The receipt of a staging file that finished finalization and awaits + /// publication. A sealed file keeps no source cursor, so a resumed + /// publisher publishes it rather than rescanning its source. + #[hotpath::skip] + pub fn sealed_receipt( + &self, + ) -> Result, CodeLexicalArtifactErrorV1> { + self.verify_path_binding()?; + read_receipt(&self.connection) + } + /// The ledger bytes charged regardless of page content: the SQLite /// page-cache authority plus the builder-retained projection metadata. #[hotpath::skip] @@ -1807,7 +1498,6 @@ impl CodeLexicalArtifactBuilderV1 { .map(|page| page_transient_peak_bytes(&self.metadata, page, usize::MAX)) .collect::, _>>()?; let metadata = &self.metadata; - let layout = self.layout; let prepared = hotpath::measure_block!("query.artifact.batch.parallel_prepare", { tracedecay_code_index::parallelism::install(|| { fresh_pages @@ -1819,7 +1509,6 @@ impl CodeLexicalArtifactBuilderV1 { tracedecay_code_index::parallelism::with_background_cpu_permit(|| { std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { prepare_page_values( - layout, metadata, page, previous_cursor, @@ -1887,22 +1576,20 @@ impl CodeLexicalArtifactBuilderV1 { self.memory_budget_bytes, pages, )?; - let mut term_insert_plan = hotpath::measure_block!( + let term_insert_plan = hotpath::measure_block!( "query.artifact.batch.term_order", prepare_term_insert_plan( self.fixed_ledger_charge_bytes, self.memory_budget_bytes, - self.layout, pages, control, ) )?; - let mut exact_insert_plan = hotpath::measure_block!( + let exact_insert_plan = hotpath::measure_block!( "query.artifact.batch.exact_order", prepare_exact_insert_plan( self.fixed_ledger_charge_bytes, self.memory_budget_bytes, - self.layout, pages, control, ) @@ -1918,31 +1605,26 @@ impl CodeLexicalArtifactBuilderV1 { Ok::<(), CodeLexicalArtifactErrorV1>(()) })?; record_batch_import_metrics(pages); - if self.layout.has_clone_index() { - hotpath::measure_block!( - "query.artifact.batch.clone_bodies", - append_prepared_clone_bodies(&transaction, pages, control) - )?; - } - if self.layout.interns_row_dictionary() { - hotpath::measure_block!( - "query.artifact.batch.rows.stage_dictionary", - stage_row_dictionary(&transaction, pages, control) - )?; - } + hotpath::measure_block!( + "query.artifact.batch.clone_bodies", + append_prepared_clone_bodies(&transaction, pages, control) + )?; + hotpath::measure_block!( + "query.artifact.batch.rows.stage_dictionary", + stage_row_dictionary(&transaction, pages, control) + )?; hotpath::measure_block!( "query.artifact.batch.rows", - append_prepared_rows(&transaction, self.layout, pages, control) + append_prepared_rows(&transaction, pages, control) )?; record_batch_row_metrics(pages); hotpath::measure_block!( "query.artifact.batch.postings", append_prepared_postings( &transaction, - self.layout, pages, - &mut term_insert_plan, - &mut exact_insert_plan, + &term_insert_plan, + &exact_insert_plan, control, ) )?; @@ -2022,11 +1704,11 @@ impl CodeLexicalArtifactBuilderV1 { &self.connection, &self.metadata, &self.metadata_digest, - self.layout, control, )?; if let Some(receipt) = read_receipt(&self.connection)? { verify_sealed_receipt_header(&receipt, &self.metadata_digest, source)?; + self.canonicalize_sealed_header()?; let step = CodeLexicalArtifactFinalizationStepV1::Ready(Box::new(receipt)); record_finalization_step(&step); return Ok(step); @@ -2035,31 +1717,27 @@ impl CodeLexicalArtifactBuilderV1 { if load_finalization_state(&self.connection)?.is_none() { let transaction = self.connection.transaction().map_err(sqlite_error)?; let mut transaction_metrics = FinalizationTransactionMetricsV1::new(); - verify_staged_source_chain(&transaction, source, control)?; - if self.layout.has_clone_fingerprints() { - // The sorted pass and the count aggregation are the heaviest - // sorter statements in finalization; run them under the same - // sorter CPU admission as the pre-digest index wakes. - super::with_builder_sorter_cpu_admission(&transaction, || { - hotpath::measure_block!( - "query.artifact.finalization.derive_clone_fingerprint_postings", - with_cancellable_sqlite_statement(&transaction, control, || { - derive_clone_fingerprint_postings(&transaction, &self.mutation_gate) - }) - )?; - hotpath::measure_block!( - "query.artifact.finalization.derive_clone_fingerprint_counts", - with_cancellable_sqlite_statement(&transaction, control, || { - derive_clone_fingerprint_counts(&transaction) - }) - ) - })??; - } - let content_epoch = authenticated_authority_epoch(&transaction, source)?; - install_base_freeze(&transaction, self.layout)?; + let terminal_cursor = verify_staged_source_chain(&transaction, source, control)?; + // The sorted pass and the count aggregation are the heaviest + // sorter statements in finalization; run them under the same + // sorter CPU admission as the pre-digest index wakes. + super::with_builder_sorter_cpu_admission(&transaction, || { + hotpath::measure_block!( + "query.artifact.finalization.derive_clone_fingerprint_postings", + with_cancellable_sqlite_statement(&transaction, control, || { + derive_clone_fingerprint_postings( + &transaction, + &self.mutation_gate, + control, + ) + }) + ) + })??; + let content_epoch = authenticated_authority_epoch(&transaction, source, control)?; + install_base_freeze(&transaction)?; store_finalization_state( &transaction, - &PersistedFinalizationStateV1::new(content_epoch, source, self.layout)?, + &PersistedFinalizationStateV1::new(content_epoch, source, terminal_cursor)?, )?; checkpoint(control)?; commit_finalization_transaction(transaction, &mut transaction_metrics)?; @@ -2079,7 +1757,7 @@ impl CodeLexicalArtifactBuilderV1 { "lexical artifact finalization marker disappeared".to_owned(), ) })?; - validate_finalization_state(&state, self.layout)?; + validate_finalization_state(&state)?; wake_metrics.digest_pass(state.phase); ensure_content_epoch(&transaction, state.content_epoch)?; if &state.source_state_digest != source.source_state_digest() { @@ -2090,7 +1768,16 @@ impl CodeLexicalArtifactBuilderV1 { } if state.phase != PersistedFinalizationPhaseV1::Digest { super::with_builder_sorter_cpu_admission(&transaction, || { - advance_pre_digest_work(&transaction, &mut state, self.layout, control) + advance_pre_digest_work( + &transaction, + &mut state, + &ServingIndexStepAuthorityV1 { + mutation_gate: &self.mutation_gate, + generation: &self.metadata.generation, + ngram_memory_bytes: self.memory_budget_bytes / 4, + }, + control, + ) })??; store_finalization_state(&transaction, &state)?; checkpoint(control)?; @@ -2104,7 +1791,7 @@ impl CodeLexicalArtifactBuilderV1 { return Ok(step); } let mut remaining_work = maximum_work; - let section_names = section_names(self.layout); + let section_names = &SECTION_NAMES; let section_count = u64::try_from(section_names.len()).map_err(contract_number)?; while remaining_work > 0 && state.section_ordinal < section_count { checkpoint(control)?; @@ -2114,14 +1801,8 @@ impl CodeLexicalArtifactBuilderV1 { let section_name = section.name(); wake_metrics.phase(section); wake_metrics.probe(); - let rows = advance_section_rows( - &transaction, - section, - self.layout, - &mut state, - remaining_work, - control, - )?; + let rows = + advance_section_rows(&transaction, section, &mut state, remaining_work, control)?; wake_metrics.add_rows(rows)?; if rows > 0 { remaining_work = remaining_work.checked_sub(rows).ok_or_else(|| { @@ -2189,48 +1870,67 @@ impl CodeLexicalArtifactBuilderV1 { .to_owned(), )); } - let sections = state.completed_sections; + // Page placement follows the order batches arrived in, so one content + // staged through different batch sizes lays out differently. Rewriting + // the file from its content before the seal makes its bytes a function + // of the content alone, which is what lets worktrees share one file. + // A crash before the seal commits repeats this rewrite on resume. + store_finalization_state(&transaction, &state)?; + commit_finalization_transaction(transaction, &mut transaction_metrics)?; + hotpath::measure_block!("query.artifact.finalization.canonical_layout", { + with_cancellable_sqlite_statement(&self.connection, control, || { + self.connection + .execute_batch("VACUUM;") + .map_err(sqlite_error) + }) + })?; + checkpoint(control)?; + let transaction = self.connection.transaction().map_err(sqlite_error)?; + let mut transaction_metrics = FinalizationTransactionMetricsV1::new(); + let sections = state.completed_sections; verify_final_sections_against_source(§ions, source)?; - let artifact_digest = artifact_digest( - &self.metadata_digest, - source.source_state_digest(), - source.format_revision(), - source.page_count(), - source.total_chunks(), - source.total_payload_bytes(), - source.total_imports(), - source.import_payload_bytes(), - source.import_dictionary_digest(), - source.cumulative_digest(), - §ions, - self.layout.revision(), - )?; + // The resume state binds the building worktree's source; the sealed + // file keeps none of it, not even the space its row occupied. + transaction + .execute_batch("DROP TABLE finalization_state;") + .map_err(sqlite_error)?; + release_free_pages(&transaction, control)?; let file_size_bytes = sqlite_file_size(&transaction)?; let receipt = new_verified_receipt( - self.metadata.clone(), self.metadata_digest.clone(), source, - artifact_digest, sections, file_size_bytes, - self.layout, - ); + )?; transaction .execute( "UPDATE artifact_state SET receipt = ?1 WHERE singleton = 1", params![padded_receipt(&receipt)?], ) .map_err(sqlite_error)?; - transaction - .execute("DELETE FROM finalization_state WHERE singleton = 1", []) - .map_err(sqlite_error)?; checkpoint(control)?; commit_finalization_transaction(transaction, &mut transaction_metrics)?; + self.canonicalize_sealed_header()?; let step = CodeLexicalArtifactFinalizationStepV1::Ready(Box::new(receipt)); record_finalization_step(&step); Ok(step) } + /// SQLite's file change counter (header offset 24) and version-valid-for + /// number (offset 92) count commits, so one content staged through a + /// different number of transactions differs only there. Once sealed no + /// connection writes the file again, and both are set to one value. + fn canonicalize_sealed_header(&self) -> Result<(), CodeLexicalArtifactErrorV1> { + self.verify_path_binding()?; + let mut file = &self.private_file; + for offset in SQLITE_HEADER_COMMIT_COUNTER_OFFSETS { + file.seek(SeekFrom::Start(offset)) + .and_then(|_| file.write_all(&SEALED_COMMIT_COUNTER.to_be_bytes())) + .map_err(private_staging_error)?; + } + file.sync_all().map_err(private_staging_error) + } + #[hotpath::measure(label = "query.artifact.finalize")] pub fn finalize( &mut self, @@ -2274,7 +1974,6 @@ impl CodeLexicalArtifactBuilderV1 { /// its immutable sealed generation. fn publish_initialized_staging( path: &Path, - layout: LexicalArtifactLayoutV1, metadata: &CodeLexicalProjectionMetadataV1, metadata_digest: &ManifestDigest, memory_budget_bytes: usize, @@ -2287,12 +1986,11 @@ fn publish_initialized_staging( Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} Err(error) => return Err(private_staging_error(error)), } - let metadata_bytes = serde_json::to_vec(metadata) - .map_err(|error| CodeLexicalArtifactErrorV1::Contract(error.to_string()))?; + let metadata_bytes = content_metadata_bytes(metadata)?; let (connection, private_file, _) = create_private_builder_connection(&initializing, memory_budget_bytes)?; let _mutation_gate = register_builder_mutation_gate(&connection)?; - create_schema(&connection, layout)?; + create_schema(&connection)?; verify_builder_mutation_gate_schema(&connection)?; #[cfg(test)] if take_failed_staging_initialization() { @@ -2304,7 +2002,7 @@ fn publish_initialized_staging( .execute( "INSERT INTO artifact_state(singleton, format_revision, metadata, metadata_digest, receipt) VALUES (1, ?1, ?2, ?3, ?4)", params![ - i64::from(layout.revision()), + i64::from(CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V1), metadata_bytes, metadata_digest.as_str(), vec![0u8; RECEIPT_RESERVATION_BYTES], @@ -2762,15 +2460,8 @@ fn prepared_term_row_count( } fn term_insert_plan_ledger_bytes(term_rows: usize) -> Result { - let entries = term_rows + term_rows .checked_mul(TERM_INSERT_PLAN_BYTES_PER_REF) - .ok_or_else(batch_ledger_overflow)?; - let runs = term_rows.div_ceil(TERM_INSERT_SORT_RUN_ROWS); - let merge_heap = runs - .checked_mul(std::mem::size_of::>()) - .ok_or_else(batch_ledger_overflow)?; - entries - .checked_add(merge_heap) .ok_or_else(batch_ledger_overflow) } @@ -2787,15 +2478,8 @@ fn prepared_exact_row_count( } fn exact_insert_plan_ledger_bytes(exact_rows: usize) -> Result { - let entries = exact_rows + exact_rows .checked_mul(EXACT_INSERT_PLAN_BYTES_PER_REF) - .ok_or_else(batch_ledger_overflow)?; - let runs = exact_rows.div_ceil(EXACT_INSERT_SORT_RUN_ROWS); - let merge_heap = runs - .checked_mul(std::mem::size_of::>()) - .ok_or_else(batch_ledger_overflow)?; - entries - .checked_add(merge_heap) .ok_or_else(batch_ledger_overflow) } @@ -2855,7 +2539,6 @@ fn admit_prepared_page_batch( fn prepare_term_insert_plan<'a>( fixed_ledger_charge_bytes: usize, memory_budget_bytes: usize, - layout: LexicalArtifactLayoutV1, pages: &'a [PreparedCodeLexicalArtifactPageV1], control: &dyn CodeIndexExecutionControlV1, ) -> Result, CodeLexicalArtifactErrorV1> { @@ -2895,106 +2578,118 @@ fn prepare_term_insert_plan<'a>( "bounded lexical term insert plan allocation failed: {error}" )) })?; - // One digest per distinct term rather than per posting: the fixture's - // batches carry ~15 postings per distinct term. - let mut term_ids: HashMap<&str, i64> = HashMap::new(); for page in pages { checkpoint(control)?; for document in &page.documents { checkpoint(control)?; for posting in &document.term_postings { - let term_id = *term_ids - .entry(posting.term.as_str()) - .or_insert_with(|| stable_term_id(&posting.term)); entries.push(PreparedTermInsertRefV1::new( - layout, document.document_id, - term_id, field_code_from_encoded(&posting.field)?, posting, )); } } } - let mut interned_terms: Vec<(&str, i64)> = term_ids.into_iter().collect(); - interned_terms.sort_unstable_by_key(|(term, _)| *term); - for run in entries.chunks_mut(TERM_INSERT_SORT_RUN_ROWS) { - checkpoint(control)?; - run.sort_unstable_by_key(PreparedTermInsertRefV1::key); - checkpoint(control)?; - } checkpoint(control)?; + sort_insert_plan(&mut entries, PreparedTermInsertRefV1::key, control)?; + checkpoint(control)?; + Ok(PreparedTermInsertPlanV1 { entries }) +} - let run_count = term_rows.div_ceil(TERM_INSERT_SORT_RUN_ROWS); - let mut merge_heap = BinaryHeap::new(); - merge_heap.try_reserve_exact(run_count).map_err(|error| { - CodeLexicalArtifactErrorV1::Io(format!( - "bounded lexical term merge heap allocation failed: {error}" - )) - })?; - for (run_index, run) in entries.chunks(TERM_INSERT_SORT_RUN_ROWS).enumerate() { +/// Sort one batch's insert plan on the indexing pool. +/// +/// Runs of [`TERM_INSERT_CONTROL_INTERVAL`] sort in waves so cancellation is +/// observed between them, and each wave takes a background CPU permit. A +/// k-way merge then materializes the total order, checkpointing on the same +/// interval. The merge holds a second copy of the plan until the first is +/// dropped. +fn sort_insert_plan( + entries: &mut Vec, + key: impl Fn(&T) -> K + Copy + Sync, + control: &dyn CodeIndexExecutionControlV1, +) -> Result<(), CodeLexicalArtifactErrorV1> +where + T: Copy + Send, + K: Ord + Copy + Send, +{ + if entries.len() <= 1 { + return Ok(()); + } + let run = TERM_INSERT_CONTROL_INTERVAL; + let workers = tracedecay_code_index::parallelism::indexing_workers().max(1); + let wave = run.saturating_mul(workers); + let mut start = 0; + while start < entries.len() { checkpoint(control)?; - let Some(entry) = run.first().copied() else { - continue; - }; - merge_heap.push(Reverse(PreparedTermMergeCursorV1 { - entry, - run_index, - run_offset: 0, - })); + let end = start.saturating_add(wave).min(entries.len()); + let slice = &mut entries[start..end]; + tracedecay_code_index::parallelism::install(|| { + slice.par_chunks_mut(run).for_each(|chunk| { + tracedecay_code_index::parallelism::with_background_cpu_permit(|| { + chunk.sort_unstable_by_key(key); + }); + }); + }) + .map_err(|error| CodeLexicalArtifactErrorV1::Io(error.to_string()))?; + start = end; } - Ok(PreparedTermInsertPlanV1 { - entries, - merge_heap, - interned_terms, - }) + if entries.len() <= run { + return Ok(()); + } + merge_sorted_runs(entries, run, key, control) } -fn next_term_insert<'a>( - plan: &mut PreparedTermInsertPlanV1<'a>, -) -> Result>, CodeLexicalArtifactErrorV1> { - let Some(Reverse(cursor)) = plan.merge_heap.pop() else { - return Ok(None); - }; - let next_offset = cursor - .run_offset - .checked_add(1) - .ok_or_else(batch_ledger_overflow)?; - if next_offset < TERM_INSERT_SORT_RUN_ROWS { - let run_start = cursor - .run_index - .checked_mul(TERM_INSERT_SORT_RUN_ROWS) - .ok_or_else(batch_ledger_overflow)?; - let next_index = run_start - .checked_add(next_offset) - .ok_or_else(batch_ledger_overflow)?; - let run_end = run_start - .checked_add(TERM_INSERT_SORT_RUN_ROWS) - .ok_or_else(batch_ledger_overflow)? - .min(plan.entries.len()); - if next_index < run_end { - let entry = plan.entries.get(next_index).copied().ok_or_else(|| { - CodeLexicalArtifactErrorV1::Contract( - "lexical term merge cursor escaped its bounded run".to_owned(), - ) - })?; - plan.merge_heap.push(Reverse(PreparedTermMergeCursorV1 { - entry, - run_index: cursor.run_index, - run_offset: next_offset, - })); +fn merge_sorted_runs( + entries: &mut Vec, + run: usize, + key: impl Fn(&T) -> K, + control: &dyn CodeIndexExecutionControlV1, +) -> Result<(), CodeLexicalArtifactErrorV1> +where + T: Copy, + K: Ord + Copy, +{ + let mut heap = BinaryHeap::new(); + let mut run_index = 0usize; + let mut start = 0usize; + while start < entries.len() { + heap.push((Reverse(key(&entries[start])), run_index, start)); + run_index += 1; + start = start.saturating_add(run); + } + let mut merged = Vec::new(); + merged.try_reserve_exact(entries.len()).map_err(|error| { + CodeLexicalArtifactErrorV1::Io(format!( + "bounded lexical insert plan merge allocation failed: {error}" + )) + })?; + let mut emitted = 0usize; + while let Some((Reverse(_), run_index, index)) = heap.pop() { + if emitted.is_multiple_of(TERM_INSERT_CONTROL_INTERVAL) { + checkpoint(control)?; + } + merged.push(entries[index]); + emitted += 1; + let next = index + 1; + let run_end = run_index + .saturating_add(1) + .saturating_mul(run) + .min(entries.len()); + if next < run_end { + heap.push((Reverse(key(&entries[next])), run_index, next)); } } - Ok(Some(cursor.entry)) + *entries = merged; + Ok(()) } -// Mirrors `prepare_term_insert_plan`/`next_term_insert` for `exact_postings`, +// Mirrors `prepare_term_insert_plan` for `exact_postings`, // whose `PRIMARY KEY(field, term, document_id)` `WITHOUT ROWID` layout has // the same clustered-index cost for out-of-order inserts as `term_postings`. fn prepare_exact_insert_plan<'a>( fixed_ledger_charge_bytes: usize, memory_budget_bytes: usize, - layout: LexicalArtifactLayoutV1, pages: &'a [PreparedCodeLexicalArtifactPageV1], control: &dyn CodeIndexExecutionControlV1, ) -> Result, CodeLexicalArtifactErrorV1> { @@ -3049,11 +2744,8 @@ fn prepare_exact_insert_plan<'a>( .or_insert_with(|| stable_exact_term_id(term)); entries.push(PreparedExactInsertRefV1 { document_id: document.document_id, - field: field.as_str(), field_code: exact_field_code_from_encoded(field)?, - term, term_id, - layout, }); } } @@ -3063,76 +2755,15 @@ fn prepare_exact_insert_plan<'a>( // collision `intern_exact_terms` rejects, and it must reject the same one // on every run rather than whichever the hash map happened to yield first. interned_terms.sort_unstable_by_key(|(term, term_id)| (*term_id, *term)); - for run in entries.chunks_mut(EXACT_INSERT_SORT_RUN_ROWS) { - checkpoint(control)?; - run.sort_unstable_by(|left, right| left.key().cmp(&right.key())); - checkpoint(control)?; - } checkpoint(control)?; - - let run_count = exact_rows.div_ceil(EXACT_INSERT_SORT_RUN_ROWS); - let mut merge_heap = BinaryHeap::new(); - merge_heap.try_reserve_exact(run_count).map_err(|error| { - CodeLexicalArtifactErrorV1::Io(format!( - "bounded lexical exact merge heap allocation failed: {error}" - )) - })?; - for (run_index, run) in entries.chunks(EXACT_INSERT_SORT_RUN_ROWS).enumerate() { - checkpoint(control)?; - let Some(entry) = run.first().copied() else { - continue; - }; - merge_heap.push(Reverse(PreparedExactMergeCursorV1 { - entry, - run_index, - run_offset: 0, - })); - } + sort_insert_plan(&mut entries, PreparedExactInsertRefV1::key, control)?; + checkpoint(control)?; Ok(PreparedExactInsertPlanV1 { entries, - merge_heap, interned_terms, }) } -fn next_exact_insert<'a>( - plan: &mut PreparedExactInsertPlanV1<'a>, -) -> Result>, CodeLexicalArtifactErrorV1> { - let Some(Reverse(cursor)) = plan.merge_heap.pop() else { - return Ok(None); - }; - let next_offset = cursor - .run_offset - .checked_add(1) - .ok_or_else(batch_ledger_overflow)?; - if next_offset < EXACT_INSERT_SORT_RUN_ROWS { - let run_start = cursor - .run_index - .checked_mul(EXACT_INSERT_SORT_RUN_ROWS) - .ok_or_else(batch_ledger_overflow)?; - let next_index = run_start - .checked_add(next_offset) - .ok_or_else(batch_ledger_overflow)?; - let run_end = run_start - .checked_add(EXACT_INSERT_SORT_RUN_ROWS) - .ok_or_else(batch_ledger_overflow)? - .min(plan.entries.len()); - if next_index < run_end { - let entry = plan.entries.get(next_index).copied().ok_or_else(|| { - CodeLexicalArtifactErrorV1::Contract( - "lexical exact merge cursor escaped its bounded run".to_owned(), - ) - })?; - plan.merge_heap.push(Reverse(PreparedExactMergeCursorV1 { - entry, - run_index: cursor.run_index, - run_offset: next_offset, - })); - } - } - Ok(Some(cursor.entry)) -} - fn sum_prepared_metric( pages: &[PreparedCodeLexicalArtifactPageV1], metric: impl Fn(&PreparedCodeLexicalArtifactPageV1) -> usize, @@ -3177,9 +2808,33 @@ fn prepare_page_batch_admission( pages, )?; let current = progress(connection)?; + let persisted_previous = pages + .first() + .map(|page| cursor_before_page(connection, page.page_ordinal())) + .transpose()? + .flatten(); + // Each page re-derives its chain from its own recorded predecessor, so the + // transitions verify independently; the ordered checks below still report + // the first failure in page order. + let transitions = tracedecay_code_index::parallelism::install(|| { + pages + .par_iter() + .enumerate() + .map(|(index, page)| { + let previous = match index.checked_sub(1) { + None => persisted_previous.as_ref(), + Some(previous) => pages + .get(previous) + .map(VerifiedSealedLexicalPageV1::next_cursor), + }; + page.verify_transition(previous) + }) + .collect::>() + }) + .map_err(|error| CodeLexicalArtifactErrorV1::Io(error.to_string()))?; let mut fresh_start = pages.len(); let mut expected_fresh_ordinal = current.next_page_ordinal; - for (index, page) in pages.iter().enumerate() { + for ((index, page), transition) in pages.iter().enumerate().zip(transitions) { if let Some(previous_page) = index.checked_sub(1).and_then(|index| pages.get(index)) { let expected = previous_page.page_ordinal().checked_add(1).ok_or_else(|| { CodeLexicalArtifactErrorV1::Contract( @@ -3192,17 +2847,7 @@ fn prepare_page_batch_admission( )); } } - let persisted_previous; - let previous = if index == 0 { - persisted_previous = cursor_before_page(connection, page.page_ordinal())?; - persisted_previous.as_ref() - } else { - pages - .get(index - 1) - .map(VerifiedSealedLexicalPageV1::next_cursor) - }; - page.verify_transition(previous) - .map_err(|error| CodeLexicalArtifactErrorV1::Corrupt(error.to_string()))?; + transition.map_err(|error| CodeLexicalArtifactErrorV1::Corrupt(error.to_string()))?; if page.page_ordinal() < current.next_page_ordinal { verify_replayed_page(connection, page)?; continue; @@ -3608,25 +3253,33 @@ fn validate_prepared_page_batch( "prepared lexical pages must continue the exact durable cursor in order".to_owned(), )); } - if usize::try_from(page.chunk_count).map_err(contract_number)? != page.documents.len() + if usize::try_from(page.chunk_count).map_err(contract_number)? < page.documents.len() || usize::try_from(page.import_count).map_err(contract_number)? != page.imports.len() { return Err(CodeLexicalArtifactErrorV1::Corrupt( "prepared lexical page cardinality disagrees with its source receipt".to_owned(), )); } - for document in &page.documents { - if u64::try_from(document.document_id).map_err(contract_number)? != expected_document { - return Err(CodeLexicalArtifactErrorV1::Corrupt( - "prepared lexical document ids are not contiguous".to_owned(), - )); - } - expected_document = expected_document.checked_add(1).ok_or_else(|| { + // A document keeps its source chunk ordinal; chunks the projection + // does not admit leave gaps, never reorderings or out-of-page ids. + let page_end = expected_document + .checked_add(page.chunk_count) + .ok_or_else(|| { CodeLexicalArtifactErrorV1::Contract( "prepared lexical document count overflowed".to_owned(), ) })?; + let mut next_document = expected_document; + for document in &page.documents { + let document_id = u64::try_from(document.document_id).map_err(contract_number)?; + if document_id < next_document || document_id >= page_end { + return Err(CodeLexicalArtifactErrorV1::Corrupt( + "prepared lexical document ids leave their page or repeat".to_owned(), + )); + } + next_document = document_id + 1; } + expected_document = page_end; let next_cursor = decode_cursor(&page.next_cursor)?; if next_cursor.next_page_ordinal() != expected_ordinal.checked_add(1).ok_or_else(|| { @@ -3822,7 +3475,11 @@ fn verify_clone_payload_digests<'body>( let mut statement = transaction.prepare_cached(&sql).map_err(sqlite_error)?; for chunk in digests.chunks(PAYLOAD_DIGEST_CONFLICT_CHECK_CHUNK) { checkpoint(control)?; - let parameters = chunk.iter().copied().map(Some).chain(std::iter::repeat_n( + let keys = chunk + .iter() + .map(|digest| stored_digest_key(digest)) + .collect::, _>>()?; + let parameters = keys.iter().map(Some).chain(std::iter::repeat_n( None, PAYLOAD_DIGEST_CONFLICT_CHECK_CHUNK - chunk.len(), )); @@ -3830,7 +3487,8 @@ fn verify_clone_payload_digests<'body>( .query(rusqlite::params_from_iter(parameters)) .map_err(sqlite_error)?; while let Some(row) = rows.next().map_err(sqlite_error)? { - let digest: String = row.get(0).map_err(sqlite_error)?; + let key: Vec = row.get(0).map_err(sqlite_error)?; + let digest = digest_from_key(&key)?; let stored: Vec = row.get(1).map_err(sqlite_error)?; if let Some(expected) = staged.get(digest.as_str()) && *expected != stored.as_slice() @@ -3855,6 +3513,23 @@ fn append_prepared_clone_bodies( .collect(); if !bodies.is_empty() { verify_clone_payload_digests(transaction, &bodies, control)?; + let payload_keys = bodies + .iter() + .map(|body| stored_digest_key(&body.payload_digest)) + .collect::, _>>()?; + let symbol_keys = bodies + .iter() + .map(|body| stored_symbol_key(&body.symbol_occurrence_id)) + .collect::>(); + let exact_digests = bodies + .iter() + .map(|body| { + body.exact_keys + .iter() + .map(|key| digest_key(&key.digest)) + .collect::, _>>() + }) + .collect::, _>>()?; let mut payload_insert = MultiRowInsertV1::new_with_conflict_clause( transaction, "clone_body_payloads(payload_digest, payload)", @@ -3862,46 +3537,76 @@ fn append_prepared_clone_bodies( " ON CONFLICT(payload_digest) DO NOTHING", sqlite_error, )?; + for (body, key) in bodies.iter().zip(&payload_keys) { + checkpoint(control)?; + payload_insert.push([sql_blob(key), sql_blob(&body.payload)])?; + } + payload_insert.finish()?; + // Occurrences name their payload, and postings their occurrence, by + // the ordinal those rows were just assigned. + let mut payload_ordinal = transaction + .prepare_cached("SELECT ordinal FROM clone_body_payloads WHERE payload_digest = ?1") + .map_err(sqlite_error)?; let mut occurrence_insert = MultiRowInsertV1::new( transaction, - "clone_occurrences(symbol_occurrence_id, payload_digest, path, body_start, body_end, occurrence)", + "clone_occurrences(symbol_key, payload_ordinal, path, body_start, body_end, eligibility)", 6, sqlite_error, )?; - let mut posting_insert = MultiRowInsertV1::new( - transaction, - "clone_exact_postings(class, normalization_revision, digest, symbol_occurrence_id, payload_digest)", - 5, - sqlite_error, - )?; - for body in &bodies { + for ((body, key), symbol_key) in bodies.iter().zip(&payload_keys).zip(&symbol_keys) { checkpoint(control)?; - payload_insert.push([sql_text(&body.payload_digest), sql_blob(&body.payload)])?; + let ordinal: i64 = payload_ordinal + .query_row([key.as_slice()], |row| row.get(0)) + .map_err(sqlite_error)?; occurrence_insert.push([ - sql_text(&body.symbol_occurrence_id), - sql_text(&body.payload_digest), + ToSqlOutput::Borrowed(symbol_key.into()), + sql_integer(ordinal), sql_text(&body.path), sql_integer(i64::try_from(body.body_start).map_err(contract_number)?), sql_integer(i64::try_from(body.body_end).map_err(contract_number)?), - sql_blob(&body.occurrence), + sql_blob(&body.eligibility), ])?; - for key in &body.exact_keys { + } + occurrence_insert.finish()?; + let mut occurrence_ordinal = transaction + .prepare_cached("SELECT ordinal FROM clone_occurrences WHERE symbol_key = ?1") + .map_err(sqlite_error)?; + let mut posting_insert = MultiRowInsertV1::new( + transaction, + "clone_exact_postings(class, normalization_revision, digest, occurrence_ordinal)", + 4, + sqlite_error, + )?; + for ((body, symbol_key), digests) in bodies.iter().zip(&symbol_keys).zip(&exact_digests) { + checkpoint(control)?; + if body.exact_keys.is_empty() { + continue; + } + let ordinal: i64 = occurrence_ordinal + .query_row([symbol_key], |row| row.get(0)) + .map_err(sqlite_error)?; + for (key, digest) in body.exact_keys.iter().zip(digests) { posting_insert.push([ sql_integer(i64::from(key.class as u8)), sql_integer(i64::from(key.normalization_revision)), - sql_text(key.digest.as_str()), - sql_text(&body.symbol_occurrence_id), - sql_text(&body.payload_digest), + sql_blob(digest), + sql_integer(ordinal), ])?; } } - payload_insert.finish()?; - occurrence_insert.finish()?; posting_insert.finish()?; } append_prepared_clone_fingerprints(transaction, pages, control) } +/// The key `clone_body_payloads.payload_digest` stores for `digest`. +fn stored_digest_key(digest: &str) -> Result<[u8; 32], CodeLexicalArtifactErrorV1> { + digest_key( + &ManifestDigest::new(digest.to_owned()) + .map_err(|error| CodeLexicalArtifactErrorV1::Contract(error.to_string()))?, + ) +} + fn append_prepared_clone_fingerprints( transaction: &Transaction<'_>, pages: &[PreparedCodeLexicalArtifactPageV1], @@ -3920,32 +3625,39 @@ fn append_prepared_clone_fingerprints( // journals) most of that tree. Measured on the 1,560-file bench corpus: // 273k postings, 78 MiB final table, 2.1 GiB written through 28 commits // (27x amplification); staged and sorted once at finalization, 0.4 GiB. - let mut insert = transaction - .prepare_cached( - "INSERT INTO clone_fingerprint_postings_pages(language, class, normalization_revision, fingerprint, symbol_occurrence_id, token_position, payload_digest, body_digest) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)", - ) + // Postings name their occurrence by its stored ordinal, which the + // occurrence rows of this batch were just assigned. + let mut ordinal = transaction + .prepare_cached("SELECT ordinal FROM clone_occurrences WHERE symbol_key = ?1") .map_err(sqlite_error)?; + let mut insert = MultiRowInsertV1::new( + transaction, + "clone_fingerprint_postings_pages(language, class, normalization_revision, fingerprint, occurrence_ordinal, token_position)", + 6, + sqlite_error, + )?; for body in pages.iter().flat_map(|page| &page.clone_bodies) { checkpoint(control)?; let Some(stream) = &body.fingerprint_stream else { continue; }; + let occurrence: i64 = ordinal + .query_row([stored_symbol_key(&body.symbol_occurrence_id)], |row| { + row.get(0) + }) + .map_err(sqlite_error)?; for position in &stream.positions { - insert - .execute(params![ - stream.language, - i64::from(stream.class as u8), - i64::from(stream.normalization_revision), - i64::try_from(position.fingerprint).map_err(contract_number)?, - body.symbol_occurrence_id, - i64::from(position.token_position), - body.payload_digest, - stream.body_digest, - ]) - .map_err(sqlite_error)?; + insert.push([ + sql_text(&stream.language), + sql_integer(i64::from(stream.class as u8)), + sql_integer(i64::from(stream.normalization_revision)), + sql_integer(i64::try_from(position.fingerprint).map_err(contract_number)?), + sql_integer(occurrence), + sql_integer(i64::from(position.token_position)), + ])?; } } - Ok(()) + insert.finish() } fn append_prepared_imports( @@ -3953,290 +3665,294 @@ fn append_prepared_imports( page: &PreparedCodeLexicalArtifactPageV1, control: &dyn CodeIndexExecutionControlV1, ) -> Result<(), CodeLexicalArtifactErrorV1> { + // The canonical encoding is the evidence itself, and its integrity digest + // is a pure function of it, so the key is the only stored column. let mut evidence = transaction - .prepare_cached("INSERT INTO import_evidence(canonical, evidence) VALUES (?1, ?1)") - .map_err(sqlite_error)?; - let mut integrity = transaction - .prepare_cached("INSERT INTO import_integrity(canonical, digest) VALUES (?1, ?2)") + .prepare_cached("INSERT INTO import_evidence(canonical) VALUES (?1)") .map_err(sqlite_error)?; for import in &page.imports { checkpoint(control)?; evidence .execute(params![import.canonical.as_slice()]) .map_err(|error| CodeLexicalArtifactErrorV1::Contract(error.to_string()))?; - integrity - .execute(params![ - import.canonical.as_slice(), - import.integrity_digest.as_str() - ]) - .map_err(sqlite_error)?; } Ok(()) } -/// Move the staged fingerprint postings into the keyed -/// `clone_fingerprint_postings` tree in one sorted pass, so the tree is -/// written sequentially once instead of being rewritten under every batch -/// commit, then drop the staging table so its pages are reused by the -/// serving indexes built in the same finalization phase. The keyed table -/// keeps its private-builder insert gate, so the pass holds the mutation -/// authority exactly as a batch append does. +/// `(language, class, normalization revision, fingerprint)` of one sealed +/// fingerprint list. +type FingerprintKeyV1 = (String, i64, i64, i64); + +/// Merge the staged fingerprint postings into one sealed list per +/// `(language, class, normalization revision, fingerprint)` in a single +/// sorted pass, so the keyed tree is written sequentially once, then drop the +/// staging table. The keyed table keeps its private-builder insert gate, so +/// the pass holds the mutation authority exactly as a batch append does. fn derive_clone_fingerprint_postings( transaction: &Transaction<'_>, mutation_gate: &Arc, + control: &dyn CodeIndexExecutionControlV1, ) -> Result<(), CodeLexicalArtifactErrorV1> { let _mutation_authority = BuilderMutationGuardV1::enter(mutation_gate)?; - transaction - .execute_batch( - "INSERT INTO clone_fingerprint_postings(language, class, normalization_revision, fingerprint, symbol_occurrence_id, token_position, payload_digest, body_digest) - SELECT language, class, normalization_revision, fingerprint, symbol_occurrence_id, token_position, payload_digest, body_digest + let mut select = transaction + .prepare( + "SELECT language, class, normalization_revision, fingerprint, occurrence_ordinal, token_position FROM clone_fingerprint_postings_pages - ORDER BY language, class, normalization_revision, fingerprint, symbol_occurrence_id, token_position; - DROP TABLE clone_fingerprint_postings_pages;", + ORDER BY language, class, normalization_revision, fingerprint, occurrence_ordinal, token_position", ) - .map_err(sqlite_error) -} - -pub(super) fn derive_clone_fingerprint_counts( - transaction: &Transaction<'_>, -) -> Result<(), CodeLexicalArtifactErrorV1> { - transaction - .execute_batch( - "INSERT INTO clone_fingerprint_counts(language, class, normalization_revision, fingerprint, posting_count) - SELECT language, class, normalization_revision, fingerprint, COUNT(*) - FROM clone_fingerprint_postings - GROUP BY language, class, normalization_revision, fingerprint;", + .map_err(sqlite_error)?; + let mut insert = transaction + .prepare( + "INSERT INTO clone_fingerprint_postings(language, class, normalization_revision, fingerprint, posting_count, postings) VALUES (?1, ?2, ?3, ?4, ?5, ?6)", ) + .map_err(sqlite_error)?; + let mut seal = |(language, class, revision, fingerprint): &FingerprintKeyV1, + postings: &[(u32, u32)]| + -> Result<(), CodeLexicalArtifactErrorV1> { + insert + .execute(params![ + language, + class, + revision, + fingerprint, + i64::try_from(postings.len()).map_err(contract_number)?, + encode_fingerprint_postings(postings)?, + ]) + .map_err(sqlite_error)?; + Ok(()) + }; + let mut rows = select.query([]).map_err(sqlite_error)?; + let mut current: Option<(FingerprintKeyV1, Vec<(u32, u32)>)> = None; + let mut visited = 0usize; + while let Some(row) = rows.next().map_err(sqlite_error)? { + if visited.is_multiple_of(TERM_INSERT_CONTROL_INTERVAL) { + checkpoint(control)?; + } + visited += 1; + let key = ( + row.get::<_, String>(0).map_err(sqlite_error)?, + row.get::<_, i64>(1).map_err(sqlite_error)?, + row.get::<_, i64>(2).map_err(sqlite_error)?, + row.get::<_, i64>(3).map_err(sqlite_error)?, + ); + let occurrence = + u32::try_from(row.get::<_, i64>(4).map_err(sqlite_error)?).map_err(contract_number)?; + let position = + u32::try_from(row.get::<_, i64>(5).map_err(sqlite_error)?).map_err(contract_number)?; + if let Some((sealed, postings)) = current.take_if(|(current, _)| *current != key) { + seal(&sealed, &postings)?; + } + current + .get_or_insert_with(|| (key, Vec::new())) + .1 + .push((occurrence, position)); + } + if let Some((sealed, postings)) = current { + seal(&sealed, &postings)?; + } + drop(rows); + drop(select); + drop(insert); + transaction + .execute_batch("DROP TABLE clone_fingerprint_postings_pages;") .map_err(sqlite_error) } +/// Append one posting list per `(term, field)` and per `(exact term, field)` +/// covering this batch, keyed by the batch's first page so every batch lands +/// at the tail of its run table. Finalization concatenates the runs of each +/// key in page order; n-gram lists are rebuilt from the stored rows instead. fn append_prepared_postings( transaction: &Transaction<'_>, - layout: LexicalArtifactLayoutV1, pages: &[PreparedCodeLexicalArtifactPageV1], - term_insert_plan: &mut PreparedTermInsertPlanV1<'_>, - exact_insert_plan: &mut PreparedExactInsertPlanV1<'_>, + term_insert_plan: &PreparedTermInsertPlanV1<'_>, + exact_insert_plan: &PreparedExactInsertPlanV1<'_>, control: &dyn CodeIndexExecutionControlV1, ) -> Result<(), CodeLexicalArtifactErrorV1> { - let term_ids = hotpath::measure_block!( - "query.artifact.batch.postings.intern_terms", - intern_terms(transaction, &term_insert_plan.interned_terms, control) + let batch_page = pages + .first() + .map(|page| i64::try_from(page.page_ordinal).map_err(contract_number)) + .transpose()? + .ok_or_else(|| { + CodeLexicalArtifactErrorV1::Contract( + "lexical artifact posting batch has no pages".to_owned(), + ) + })?; + hotpath::measure_block!( + "query.artifact.batch.postings.intern_exact", + intern_exact_terms(transaction, &exact_insert_plan.interned_terms, control) )?; - if layout.interns_exact_terms() { - hotpath::measure_block!( - "query.artifact.batch.postings.intern_exact", - intern_exact_terms(transaction, &exact_insert_plan.interned_terms, control) - )?; - } let mut term_insert = MultiRowInsertV1::new( transaction, - "term_postings(term_id, field, document_id, frequency)", + "term_posting_runs(page_ordinal, term, field, postings)", 4, sqlite_error, )?; - // Plain INSERT, not `INSERT OR IGNORE`: `exact_postings` is - // `PRIMARY KEY(field, term, document_id) WITHOUT ROWID`. Every prepared - // document's `exact_postings` is deduplicated into a `BTreeSet<(field, - // term)>` before this stage (`prepared.rs::prepare_document`), so within - // one document the pair is unique; `document_id` is then unique across - // the whole prepared batch because `validate_prepared_page_batch` - // (this file) rejects any batch whose document ids are not a strictly - // contiguous continuation of the already-committed cursor, and - // `prepare_pages_inner` only ever prepares the fresh suffix of pages - // that cursor has not yet accepted, a resumed or replayed call reuses - // no document id that is already durable. Together those invariants - // make every (field, term, document_id) key in a prepared batch globally - // unique, both within the batch and against every already-committed - // row, so `OR IGNORE` could only mask a real corruption bug. A plain - // INSERT lets that surface as a constraint failure instead of vanishing. - let exact_table_columns = match layout { - LexicalArtifactLayoutV1::V12 - | LexicalArtifactLayoutV1::V13 - | LexicalArtifactLayoutV1::V14 - | LexicalArtifactLayoutV1::V15 - | LexicalArtifactLayoutV1::V16 => "exact_postings(term_id, field, document_id)", - LexicalArtifactLayoutV1::V10 | LexicalArtifactLayoutV1::V11 => { - "exact_postings(field, term, document_id)" - } - }; - let mut exact_insert = - MultiRowInsertV1::new(transaction, exact_table_columns, 3, sqlite_error)?; - let mut ngram_insert = MultiRowInsertV1::new( + // Plain INSERT, not `INSERT OR IGNORE`: every prepared document's exact + // postings are deduplicated into a `BTreeSet<(field, term)>` + // (`prepared.rs::prepare_document`), and `validate_prepared_page_batch` + // admits only the fresh contiguous suffix of pages, so each + // `(batch page, term, field)` run key is globally unique and a conflict + // can only be a real corruption bug. + let mut exact_insert = MultiRowInsertV1::new( transaction, - "ngram_postings(page_ordinal, kind, ngram, documents, cardinality)", - 5, + "exact_posting_runs(page_ordinal, term_id, field, documents)", + 4, sqlite_error, )?; - let expected_term_rows = term_insert_plan.entries.len(); - let mut inserted_term_rows = 0usize; let mut field_totals = BTreeMap::new(); hotpath::measure_block!("query.artifact.batch.postings.term_rows", { - while let Some(entry) = next_term_insert(term_insert_plan)? { - if inserted_term_rows.is_multiple_of(TERM_INSERT_CONTROL_INTERVAL) { + let mut run: Option<((&str, i64), PostingListEncoderV1)> = None; + for (index, entry) in term_insert_plan.entries.iter().enumerate() { + if index.is_multiple_of(TERM_INSERT_CONTROL_INTERVAL) { checkpoint(control)?; } - let (term_id, field, document_id) = entry.columns(layout); - if !term_ids.contains(&term_id) { - return Err(CodeLexicalArtifactErrorV1::Contract( - "lexical artifact term intern omitted a planned posting".to_owned(), - )); + let (term, field, document_id) = entry.key(); + if let Some(((term, field), encoder)) = run.take_if(|(key, _)| *key != (term, field)) { + push_posting_run(&mut term_insert, batch_page, sql_text(term), field, encoder)?; } - term_insert.push([ - sql_integer(term_id), - sql_integer(field), - sql_integer(document_id), - sql_integer(entry.posting.frequency), - ])?; + run.get_or_insert_with(|| ((term, field), PostingListEncoderV1::new(true))) + .1 + .push( + u32::try_from(document_id).map_err(contract_number)?, + u32::try_from(entry.posting.frequency).map_err(contract_number)?, + )?; let total: &mut i64 = field_totals.entry(field).or_default(); *total = total.checked_add(entry.posting.frequency).ok_or_else(|| { CodeLexicalArtifactErrorV1::Contract( "lexical artifact field total overflowed".to_owned(), ) })?; - inserted_term_rows = inserted_term_rows - .checked_add(1) - .ok_or_else(batch_ledger_overflow)?; + } + if let Some(((term, field), encoder)) = run { + push_posting_run(&mut term_insert, batch_page, sql_text(term), field, encoder)?; } term_insert.finish() })?; - if inserted_term_rows != expected_term_rows { - return Err(CodeLexicalArtifactErrorV1::Contract( - "lexical term merge omitted planned postings".to_owned(), - )); - } hotpath::measure_block!( "query.artifact.batch.postings.field_totals", stage_field_totals(transaction, &field_totals) )?; - let expected_exact_rows = exact_insert_plan.entries.len(); - let mut inserted_exact_rows = 0usize; hotpath::measure_block!("query.artifact.batch.postings.exact_rows", { - while let Some(entry) = next_exact_insert(exact_insert_plan)? { - if inserted_exact_rows.is_multiple_of(EXACT_INSERT_CONTROL_INTERVAL) { + let mut run: Option<((i64, i64), PostingListEncoderV1)> = None; + for (index, entry) in exact_insert_plan.entries.iter().enumerate() { + if index.is_multiple_of(EXACT_INSERT_CONTROL_INTERVAL) { checkpoint(control)?; } - match entry.layout { - LexicalArtifactLayoutV1::V10 | LexicalArtifactLayoutV1::V11 => { - exact_insert.push([ - sql_text(entry.field), - sql_blob(entry.term), - sql_integer(entry.document_id), - ])?; - } - LexicalArtifactLayoutV1::V12 - | LexicalArtifactLayoutV1::V13 - | LexicalArtifactLayoutV1::V14 - | LexicalArtifactLayoutV1::V15 - | LexicalArtifactLayoutV1::V16 => { - exact_insert.push([ - sql_integer(entry.term_id), - sql_integer(entry.field_code), - sql_integer(entry.document_id), - ])?; - } + let key = (entry.term_id, entry.field_code); + if let Some(((term_id, field), encoder)) = run.take_if(|(run_key, _)| *run_key != key) { + push_posting_run( + &mut exact_insert, + batch_page, + sql_integer(term_id), + field, + encoder, + )?; } - inserted_exact_rows = inserted_exact_rows - .checked_add(1) - .ok_or_else(batch_ledger_overflow)?; + run.get_or_insert_with(|| (key, PostingListEncoderV1::new(false))) + .1 + .push( + u32::try_from(entry.document_id).map_err(contract_number)?, + 1, + )?; + } + if let Some(((term_id, field), encoder)) = run { + push_posting_run( + &mut exact_insert, + batch_page, + sql_integer(term_id), + field, + encoder, + )?; } exact_insert.finish() })?; - if inserted_exact_rows != expected_exact_rows { - return Err(CodeLexicalArtifactErrorV1::Contract( - "lexical exact merge omitted planned postings".to_owned(), - )); - } - hotpath::measure_block!("query.artifact.batch.postings.ngram_rows", { - for page in pages { - let page_ordinal = i64::try_from(page.page_ordinal).map_err(contract_number)?; - for shard in &page.ngram_shards { - checkpoint(control)?; - ngram_insert.push([ - sql_integer(page_ordinal), - sql_integer(shard.kind), - sql_integer(shard.ngram), - sql_blob(shard.documents.as_slice()), - sql_integer(i64::try_from(shard.cardinality).map_err(contract_number)?), - ])?; - } - } - ngram_insert.finish() - }) + Ok(()) +} + +fn push_posting_run<'a>( + insert: &mut MultiRowInsertV1<'_, 'a>, + batch_page: i64, + term: ToSqlOutput<'a>, + field: i64, + encoder: PostingListEncoderV1, +) -> Result<(), CodeLexicalArtifactErrorV1> { + insert.push([ + sql_integer(batch_page), + term, + sql_integer(field), + ToSqlOutput::Owned(Value::Blob(encoder.finish()?)), + ]) } fn append_prepared_rows( transaction: &Transaction<'_>, - layout: LexicalArtifactLayoutV1, pages: &[PreparedCodeLexicalArtifactPageV1], control: &dyn CodeIndexExecutionControlV1, ) -> Result<(), CodeLexicalArtifactErrorV1> { - let mut row_insert = MultiRowInsertV1::new( + let mut block_insert = MultiRowInsertV1::new( transaction, - "rows(document_id, chunk_id, row)", - 3, + "row_blocks(first_document, payload)", + 2, |error| CodeLexicalArtifactErrorV1::Contract(error.to_string()), )?; - let (integrity_columns, integrity_width) = if layout.stores_document_integrity_bytes() { - ("document_integrity(document_id, digest)", 2) - } else { - ("document_integrity(document_id, chunk_id, digest)", 3) - }; - let mut integrity_insert = MultiRowInsertV1::new( + for page in pages { + for (first_document, payload) in &page.row_blocks { + checkpoint(control)?; + block_insert.push([sql_integer(*first_document), sql_blob(payload)])?; + } + } + block_insert.finish()?; + let mut chunk_insert = MultiRowInsertV1::new( transaction, - integrity_columns, - integrity_width, - sqlite_error, + "row_chunk_pages(document_id, chunk_id)", + 2, + |error| CodeLexicalArtifactErrorV1::Contract(error.to_string()), )?; for page in pages { for document in &page.documents { checkpoint(control)?; - row_insert.push([ + chunk_insert.push([ sql_integer(document.document_id), - sql_text(&document.chunk_id), - sql_blob(&document.row), + ToSqlOutput::Owned(stored_chunk_key(&document.chunk_id)), ])?; - if layout.stores_document_integrity_bytes() { - integrity_insert.push([ - sql_integer(document.document_id), - sql_blob(&document.integrity_digest_bytes), - ])?; - } else { - integrity_insert.push([ - sql_integer(document.document_id), - sql_text(&document.chunk_id), - sql_text(document.integrity_digest.as_str()), - ])?; - } } } - row_insert.finish()?; - integrity_insert.finish() + chunk_insert.finish() } fn insert_prepared_source_page( transaction: &Transaction<'_>, page: &PreparedCodeLexicalArtifactPageV1, ) -> Result<(), CodeLexicalArtifactErrorV1> { + let page_ordinal = i64::try_from(page.page_ordinal).map_err(contract_number)?; transaction .prepare_cached( - "INSERT INTO source_pages(page_ordinal, page_digest, cumulative_digest, chunk_count, payload_bytes, import_count, import_payload_bytes, import_dictionary_digest, ngram_digest, base_sections_receipt, next_cursor) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)", + "INSERT INTO source_pages(page_ordinal, chunk_count, import_count, import_payload_bytes, import_dictionary_digest, ngram_digest, base_sections_receipt) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)", ) .map_err(sqlite_error)? - .execute( - params![ - i64::try_from(page.page_ordinal).map_err(contract_number)?, - page.page_digest.as_str(), - page.cumulative_digest.as_str(), - i64::try_from(page.chunk_count).map_err(contract_number)?, - i64::try_from(page.payload_bytes).map_err(contract_number)?, - i64::try_from(page.import_count).map_err(contract_number)?, - i64::try_from(page.import_payload_bytes).map_err(contract_number)?, - page.import_dictionary_digest.as_str(), - page.ngram_digest.as_str(), - page.base_sections_receipt.as_slice(), - page.next_cursor.as_slice(), - ], + .execute(params![ + page_ordinal, + i64::try_from(page.chunk_count).map_err(contract_number)?, + i64::try_from(page.import_count).map_err(contract_number)?, + i64::try_from(page.import_payload_bytes).map_err(contract_number)?, + page.import_dictionary_digest.as_str(), + page.ngram_digest.as_str(), + page.base_sections_receipt.as_slice(), + ]) + .map_err(sqlite_error)?; + transaction + .prepare_cached( + "INSERT INTO source_page_cursors(page_ordinal, page_digest, cumulative_digest, payload_bytes, next_cursor) VALUES (?1, ?2, ?3, ?4, ?5)", ) + .map_err(sqlite_error)? + .execute(params![ + page_ordinal, + page.page_digest.as_str(), + page.cumulative_digest.as_str(), + i64::try_from(page.payload_bytes).map_err(contract_number)?, + page.next_cursor.as_slice(), + ]) .map_err(sqlite_error)?; Ok(()) } @@ -4263,22 +3979,17 @@ pub(super) fn sqlite_file_size(connection: &Connection) -> Result Result<(), CodeLexicalArtifactErrorV1> { - // Same columns in every interned layout; only the clustered key differs. - // Revision 13 clusters by document so batch appends are sequential and - // the term-leading probe path is a covering index built once at - // finalization (`term_postings_by_term`). - let term_postings_key = if layout.clusters_term_postings_by_document() { - "PRIMARY KEY(document_id, term_id, field)" - } else { - "PRIMARY KEY(term_id, field, document_id)" - }; +fn create_schema(connection: &Connection) -> Result<(), CodeLexicalArtifactErrorV1> { + // Append tables (`*_runs`, `*_pages`, `field_stats_staging`) take batches + // in page order; finalization derives each sealed serving table from its + // staging table and drops it. Incremental auto-vacuum must be chosen + // before the first table exists so those dropped pages leave the file. + // `row_dictionary`, `row_chunks`, and the three posting tables stay empty + // until then. connection - .execute_batch(&format!( + .execute_batch( " + PRAGMA auto_vacuum = INCREMENTAL; CREATE TABLE artifact_state ( singleton INTEGER PRIMARY KEY CHECK(singleton = 1), format_revision INTEGER NOT NULL, @@ -4297,50 +4008,46 @@ fn create_schema( INSERT INTO content_epoch(singleton, epoch) VALUES (1, 0); CREATE TABLE source_pages ( page_ordinal INTEGER PRIMARY KEY, - page_digest TEXT NOT NULL, - cumulative_digest TEXT NOT NULL, chunk_count INTEGER NOT NULL, - payload_bytes INTEGER NOT NULL, import_count INTEGER NOT NULL, import_payload_bytes INTEGER NOT NULL, import_dictionary_digest TEXT NOT NULL, ngram_digest TEXT NOT NULL, - base_sections_receipt BLOB NOT NULL, - next_cursor BLOB NOT NULL + base_sections_receipt BLOB NOT NULL ); - -- Every derived document and import receives its digest in the - -- same private append transaction as its page-level base receipt. - -- External connections cannot invoke that mutation authority. - CREATE TABLE document_integrity ( - document_id INTEGER PRIMARY KEY, - chunk_id TEXT NOT NULL, - digest TEXT NOT NULL + CREATE TABLE source_page_cursors ( + page_ordinal INTEGER PRIMARY KEY, + page_digest TEXT NOT NULL, + cumulative_digest TEXT NOT NULL, + payload_bytes INTEGER NOT NULL, + next_cursor BLOB NOT NULL ); - CREATE TABLE import_integrity ( - canonical BLOB PRIMARY KEY, - digest TEXT NOT NULL - ) WITHOUT ROWID; CREATE TABLE import_evidence ( - canonical BLOB PRIMARY KEY, - evidence BLOB NOT NULL + canonical BLOB NOT NULL PRIMARY KEY ) WITHOUT ROWID; - CREATE TABLE rows ( + CREATE TABLE row_blocks ( + first_document INTEGER PRIMARY KEY, + payload BLOB NOT NULL + ); + CREATE TABLE row_chunk_pages ( document_id INTEGER PRIMARY KEY, - chunk_id TEXT NOT NULL, - row BLOB NOT NULL + chunk_id BLOB NOT NULL ); - CREATE TABLE term_postings ( - term_id INTEGER NOT NULL, - field INTEGER NOT NULL, - document_id INTEGER NOT NULL, - frequency INTEGER NOT NULL, - {term_postings_key} + CREATE TABLE row_chunks ( + chunk_id BLOB NOT NULL PRIMARY KEY, + document_id INTEGER NOT NULL ) WITHOUT ROWID; - CREATE TABLE term_stats ( - term_id INTEGER NOT NULL, + CREATE TABLE term_posting_runs ( + page_ordinal INTEGER NOT NULL, + term TEXT NOT NULL, field INTEGER NOT NULL, - document_frequency INTEGER NOT NULL, - PRIMARY KEY(term_id, field) + postings BLOB NOT NULL, + PRIMARY KEY(page_ordinal, term, field) + ) WITHOUT ROWID; + CREATE TABLE term_postings ( + term TEXT NOT NULL PRIMARY KEY, + in_fuzzy INTEGER NOT NULL CHECK(in_fuzzy IN (0, 1)), + lists BLOB NOT NULL ) WITHOUT ROWID; CREATE TABLE field_stats ( field INTEGER PRIMARY KEY, @@ -4350,166 +4057,115 @@ fn create_schema( field INTEGER PRIMARY KEY, total_length INTEGER NOT NULL ) WITHOUT ROWID; - CREATE TABLE exact_postings ( - field TEXT NOT NULL, - term BLOB NOT NULL, - document_id INTEGER NOT NULL, - PRIMARY KEY(field, term, document_id) - ) WITHOUT ROWID; - CREATE TABLE ngram_postings ( + CREATE TABLE exact_vocabulary ( + term_id INTEGER PRIMARY KEY, + term BLOB NOT NULL + ); + CREATE TABLE exact_posting_runs ( page_ordinal INTEGER NOT NULL, - kind INTEGER NOT NULL, - ngram INTEGER NOT NULL, + term_id INTEGER NOT NULL, + field INTEGER NOT NULL, documents BLOB NOT NULL, - cardinality INTEGER NOT NULL CHECK(cardinality > 0), - PRIMARY KEY(page_ordinal, kind, ngram) + PRIMARY KEY(page_ordinal, term_id, field) ) WITHOUT ROWID; - CREATE TABLE ngram_statistics ( + CREATE TABLE exact_postings ( + term_id INTEGER NOT NULL, + field INTEGER NOT NULL, + documents BLOB NOT NULL, + PRIMARY KEY(term_id, field) + ) WITHOUT ROWID; + CREATE TABLE ngram_postings ( kind INTEGER NOT NULL, ngram INTEGER NOT NULL, document_frequency INTEGER NOT NULL CHECK(document_frequency > 0), + documents BLOB NOT NULL, PRIMARY KEY(kind, ngram) ) WITHOUT ROWID; - CREATE TABLE vocabulary ( - term_id INTEGER PRIMARY KEY, - term TEXT NOT NULL UNIQUE, - in_fuzzy INTEGER NOT NULL CHECK(in_fuzzy IN (0, 1)) + CREATE TABLE row_dictionary ( + entry_id INTEGER PRIMARY KEY, + entry BLOB NOT NULL ); + CREATE TABLE row_dictionary_pages ( + page_ordinal INTEGER NOT NULL, + entry_id INTEGER NOT NULL, + entry BLOB NOT NULL, + PRIMARY KEY(page_ordinal, entry_id) + ) WITHOUT ROWID; CREATE TRIGGER content_epoch_source_pages_insert AFTER INSERT ON source_pages BEGIN UPDATE content_epoch SET epoch = epoch + 1 WHERE singleton = 1; END; - CREATE TRIGGER content_epoch_document_integrity_insert AFTER INSERT ON document_integrity BEGIN UPDATE content_epoch SET epoch = epoch + 1 WHERE singleton = 1; END; - CREATE TRIGGER content_epoch_import_integrity_insert AFTER INSERT ON import_integrity BEGIN UPDATE content_epoch SET epoch = epoch + 1 WHERE singleton = 1; END; + CREATE TRIGGER content_epoch_row_chunk_pages_insert AFTER INSERT ON row_chunk_pages BEGIN UPDATE content_epoch SET epoch = epoch + 1 WHERE singleton = 1; END; CREATE TRIGGER content_epoch_import_evidence_insert AFTER INSERT ON import_evidence BEGIN UPDATE content_epoch SET epoch = epoch + 1 WHERE singleton = 1; END; CREATE TRIGGER immutable_source_pages_update BEFORE UPDATE ON source_pages BEGIN SELECT RAISE(ABORT, 'immutable lexical source pages'); END; CREATE TRIGGER immutable_source_pages_delete BEFORE DELETE ON source_pages BEGIN SELECT RAISE(ABORT, 'immutable lexical source pages'); END; - CREATE TRIGGER immutable_document_integrity_update BEFORE UPDATE ON document_integrity BEGIN SELECT RAISE(ABORT, 'immutable lexical document integrity'); END; - CREATE TRIGGER immutable_document_integrity_delete BEFORE DELETE ON document_integrity BEGIN SELECT RAISE(ABORT, 'immutable lexical document integrity'); END; - CREATE TRIGGER immutable_import_integrity_update BEFORE UPDATE ON import_integrity BEGIN SELECT RAISE(ABORT, 'immutable lexical import integrity'); END; - CREATE TRIGGER immutable_import_integrity_delete BEFORE DELETE ON import_integrity BEGIN SELECT RAISE(ABORT, 'immutable lexical import integrity'); END; + CREATE TRIGGER immutable_source_page_cursors_update BEFORE UPDATE ON source_page_cursors BEGIN SELECT RAISE(ABORT, 'immutable lexical source page cursors'); END; + CREATE TRIGGER immutable_source_page_cursors_delete BEFORE DELETE ON source_page_cursors BEGIN SELECT RAISE(ABORT, 'immutable lexical source page cursors'); END; CREATE TRIGGER immutable_import_evidence_update BEFORE UPDATE ON import_evidence BEGIN SELECT RAISE(ABORT, 'immutable lexical import evidence'); END; CREATE TRIGGER immutable_import_evidence_delete BEFORE DELETE ON import_evidence BEGIN SELECT RAISE(ABORT, 'immutable lexical import evidence'); END; CREATE TRIGGER immutable_ngram_postings_update BEFORE UPDATE ON ngram_postings BEGIN SELECT RAISE(ABORT, 'immutable lexical ngram postings'); END; CREATE TRIGGER immutable_ngram_postings_delete BEFORE DELETE ON ngram_postings BEGIN SELECT RAISE(ABORT, 'immutable lexical ngram postings'); END; + CREATE TRIGGER immutable_exact_vocabulary_update BEFORE UPDATE ON exact_vocabulary BEGIN SELECT RAISE(ABORT, 'immutable lexical exact vocabulary'); END; + CREATE TRIGGER immutable_exact_vocabulary_delete BEFORE DELETE ON exact_vocabulary BEGIN SELECT RAISE(ABORT, 'immutable lexical exact vocabulary'); END; + CREATE TRIGGER immutable_row_dictionary_pages_update BEFORE UPDATE ON row_dictionary_pages BEGIN SELECT RAISE(ABORT, 'immutable lexical row dictionary pages'); END; + CREATE TRIGGER immutable_row_dictionary_pages_delete BEFORE DELETE ON row_dictionary_pages BEGIN SELECT RAISE(ABORT, 'immutable lexical row dictionary pages'); END; CREATE TRIGGER builder_gate_source_pages_insert BEFORE INSERT ON source_pages WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; - CREATE TRIGGER builder_gate_document_integrity_insert BEFORE INSERT ON document_integrity WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; - CREATE TRIGGER builder_gate_import_integrity_insert BEFORE INSERT ON import_integrity WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; + CREATE TRIGGER builder_gate_source_page_cursors_insert BEFORE INSERT ON source_page_cursors WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; CREATE TRIGGER builder_gate_import_evidence_insert BEFORE INSERT ON import_evidence WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; - CREATE TRIGGER builder_gate_rows_insert BEFORE INSERT ON rows WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; - CREATE TRIGGER builder_gate_rows_update BEFORE UPDATE ON rows WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; - CREATE TRIGGER builder_gate_rows_delete BEFORE DELETE ON rows WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; + CREATE TRIGGER builder_gate_row_blocks_insert BEFORE INSERT ON row_blocks WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; + CREATE TRIGGER builder_gate_row_blocks_update BEFORE UPDATE ON row_blocks WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; + CREATE TRIGGER builder_gate_row_blocks_delete BEFORE DELETE ON row_blocks WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; + CREATE TRIGGER builder_gate_row_chunk_pages_insert BEFORE INSERT ON row_chunk_pages WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; + CREATE TRIGGER builder_gate_row_chunk_pages_update BEFORE UPDATE ON row_chunk_pages WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; + CREATE TRIGGER builder_gate_row_chunk_pages_delete BEFORE DELETE ON row_chunk_pages WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; + CREATE TRIGGER builder_gate_row_chunks_insert BEFORE INSERT ON row_chunks WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; + CREATE TRIGGER builder_gate_row_chunks_update BEFORE UPDATE ON row_chunks WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; + CREATE TRIGGER builder_gate_row_chunks_delete BEFORE DELETE ON row_chunks WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; + CREATE TRIGGER builder_gate_term_posting_runs_insert BEFORE INSERT ON term_posting_runs WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; + CREATE TRIGGER builder_gate_term_posting_runs_update BEFORE UPDATE ON term_posting_runs WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; + CREATE TRIGGER builder_gate_term_posting_runs_delete BEFORE DELETE ON term_posting_runs WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; CREATE TRIGGER builder_gate_term_postings_insert BEFORE INSERT ON term_postings WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; CREATE TRIGGER builder_gate_term_postings_update BEFORE UPDATE ON term_postings WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; CREATE TRIGGER builder_gate_term_postings_delete BEFORE DELETE ON term_postings WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; + CREATE TRIGGER builder_gate_exact_posting_runs_insert BEFORE INSERT ON exact_posting_runs WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; + CREATE TRIGGER builder_gate_exact_posting_runs_update BEFORE UPDATE ON exact_posting_runs WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; + CREATE TRIGGER builder_gate_exact_posting_runs_delete BEFORE DELETE ON exact_posting_runs WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; CREATE TRIGGER builder_gate_exact_postings_insert BEFORE INSERT ON exact_postings WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; CREATE TRIGGER builder_gate_exact_postings_update BEFORE UPDATE ON exact_postings WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; CREATE TRIGGER builder_gate_exact_postings_delete BEFORE DELETE ON exact_postings WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; CREATE TRIGGER builder_gate_ngram_postings_insert BEFORE INSERT ON ngram_postings WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; + CREATE TRIGGER builder_gate_exact_vocabulary_insert BEFORE INSERT ON exact_vocabulary WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; + CREATE TRIGGER builder_gate_exact_vocabulary_update BEFORE UPDATE ON exact_vocabulary WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; + CREATE TRIGGER builder_gate_exact_vocabulary_delete BEFORE DELETE ON exact_vocabulary WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; + CREATE TRIGGER builder_gate_row_dictionary_pages_insert BEFORE INSERT ON row_dictionary_pages WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; + CREATE TRIGGER builder_gate_row_dictionary_pages_update BEFORE UPDATE ON row_dictionary_pages WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; + CREATE TRIGGER builder_gate_row_dictionary_pages_delete BEFORE DELETE ON row_dictionary_pages WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; CREATE TRIGGER builder_gate_field_stats_staging_insert BEFORE INSERT ON field_stats_staging WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; CREATE TRIGGER builder_gate_field_stats_staging_update BEFORE UPDATE ON field_stats_staging WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; CREATE TRIGGER builder_gate_field_stats_staging_delete BEFORE DELETE ON field_stats_staging WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; - " - )) + ", + ) .map_err(sqlite_error)?; - if layout.interns_exact_terms() { - connection - .execute_batch( - " - DROP TRIGGER builder_gate_exact_postings_insert; - DROP TRIGGER builder_gate_exact_postings_update; - DROP TRIGGER builder_gate_exact_postings_delete; - DROP TABLE exact_postings; - CREATE TABLE exact_vocabulary ( - term_id INTEGER PRIMARY KEY, - term BLOB NOT NULL - ); - CREATE TABLE exact_postings ( - term_id INTEGER NOT NULL, - field INTEGER NOT NULL, - document_id INTEGER NOT NULL, - PRIMARY KEY(term_id, field, document_id) - ) WITHOUT ROWID; - CREATE TRIGGER builder_gate_exact_vocabulary_insert BEFORE INSERT ON exact_vocabulary WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; - CREATE TRIGGER builder_gate_exact_vocabulary_update BEFORE UPDATE ON exact_vocabulary WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; - CREATE TRIGGER builder_gate_exact_vocabulary_delete BEFORE DELETE ON exact_vocabulary WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; - CREATE TRIGGER immutable_exact_vocabulary_update BEFORE UPDATE ON exact_vocabulary BEGIN SELECT RAISE(ABORT, 'immutable lexical exact vocabulary'); END; - CREATE TRIGGER immutable_exact_vocabulary_delete BEFORE DELETE ON exact_vocabulary BEGIN SELECT RAISE(ABORT, 'immutable lexical exact vocabulary'); END; - CREATE TRIGGER builder_gate_exact_postings_insert BEFORE INSERT ON exact_postings WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; - CREATE TRIGGER builder_gate_exact_postings_update BEFORE UPDATE ON exact_postings WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; - CREATE TRIGGER builder_gate_exact_postings_delete BEFORE DELETE ON exact_postings WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; - ", - ) - .map_err(sqlite_error)?; - } - if layout.stores_document_integrity_bytes() { - // Same triggers as the base table, recreated on the narrower shape. - connection - .execute_batch( - " - DROP TRIGGER content_epoch_document_integrity_insert; - DROP TRIGGER immutable_document_integrity_update; - DROP TRIGGER immutable_document_integrity_delete; - DROP TRIGGER builder_gate_document_integrity_insert; - DROP TABLE document_integrity; - CREATE TABLE document_integrity ( - document_id INTEGER PRIMARY KEY, - digest BLOB NOT NULL - ); - CREATE TRIGGER content_epoch_document_integrity_insert AFTER INSERT ON document_integrity BEGIN UPDATE content_epoch SET epoch = epoch + 1 WHERE singleton = 1; END; - CREATE TRIGGER immutable_document_integrity_update BEFORE UPDATE ON document_integrity BEGIN SELECT RAISE(ABORT, 'immutable lexical document integrity'); END; - CREATE TRIGGER immutable_document_integrity_delete BEFORE DELETE ON document_integrity BEGIN SELECT RAISE(ABORT, 'immutable lexical document integrity'); END; - CREATE TRIGGER builder_gate_document_integrity_insert BEFORE INSERT ON document_integrity WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; - ", - ) - .map_err(sqlite_error)?; - } - if layout.interns_row_dictionary() { - // `row_dictionary` stays empty until finalization derives it; the - // append phase only stages `(page_ordinal, entry_id, entry)` rows, - // which are sequential in page order. - connection - .execute_batch( - " - CREATE TABLE row_dictionary ( - entry_id INTEGER PRIMARY KEY, - entry BLOB NOT NULL - ); - CREATE TABLE row_dictionary_pages ( - page_ordinal INTEGER NOT NULL, - entry_id INTEGER NOT NULL, - entry BLOB NOT NULL, - PRIMARY KEY(page_ordinal, entry_id) - ) WITHOUT ROWID; - CREATE TRIGGER builder_gate_row_dictionary_pages_insert BEFORE INSERT ON row_dictionary_pages WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; - CREATE TRIGGER builder_gate_row_dictionary_pages_update BEFORE UPDATE ON row_dictionary_pages WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; - CREATE TRIGGER builder_gate_row_dictionary_pages_delete BEFORE DELETE ON row_dictionary_pages WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; - CREATE TRIGGER immutable_row_dictionary_pages_update BEFORE UPDATE ON row_dictionary_pages BEGIN SELECT RAISE(ABORT, 'immutable lexical row dictionary pages'); END; - CREATE TRIGGER immutable_row_dictionary_pages_delete BEFORE DELETE ON row_dictionary_pages BEGIN SELECT RAISE(ABORT, 'immutable lexical row dictionary pages'); END; - ", - ) - .map_err(sqlite_error)?; - } - if layout.has_clone_index() { - connection - .execute_batch( - " + connection + .execute_batch( + " CREATE TABLE clone_body_payloads ( - payload_digest TEXT PRIMARY KEY, + ordinal INTEGER PRIMARY KEY, + payload_digest BLOB NOT NULL UNIQUE, payload BLOB NOT NULL - ) WITHOUT ROWID; + ); CREATE TABLE clone_occurrences ( - symbol_occurrence_id TEXT PRIMARY KEY, - payload_digest TEXT NOT NULL, + ordinal INTEGER PRIMARY KEY, + symbol_key BLOB NOT NULL UNIQUE, + payload_ordinal INTEGER NOT NULL, path TEXT NOT NULL, body_start INTEGER NOT NULL, body_end INTEGER NOT NULL, - occurrence BLOB NOT NULL - ) WITHOUT ROWID; + eligibility BLOB NOT NULL + ); CREATE TABLE clone_exact_postings ( class INTEGER NOT NULL, normalization_revision INTEGER NOT NULL, - digest TEXT NOT NULL, - symbol_occurrence_id TEXT NOT NULL, - payload_digest TEXT NOT NULL, - PRIMARY KEY(class, normalization_revision, digest, symbol_occurrence_id) + digest BLOB NOT NULL, + occurrence_ordinal INTEGER NOT NULL, + PRIMARY KEY(class, normalization_revision, digest, occurrence_ordinal) ) WITHOUT ROWID; CREATE TRIGGER builder_gate_clone_body_payloads_insert BEFORE INSERT ON clone_body_payloads WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; CREATE TRIGGER builder_gate_clone_occurrences_insert BEFORE INSERT ON clone_occurrences WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; @@ -4520,42 +4176,22 @@ fn create_schema( CREATE TRIGGER immutable_clone_occurrences_delete BEFORE DELETE ON clone_occurrences BEGIN SELECT RAISE(ABORT, 'immutable clone occurrences'); END; CREATE TRIGGER immutable_clone_exact_postings_update BEFORE UPDATE ON clone_exact_postings BEGIN SELECT RAISE(ABORT, 'immutable clone exact postings'); END; CREATE TRIGGER immutable_clone_exact_postings_delete BEFORE DELETE ON clone_exact_postings BEGIN SELECT RAISE(ABORT, 'immutable clone exact postings'); END; - ", - ) - .map_err(sqlite_error)?; - } - if layout.has_clone_fingerprints() { - connection - .execute_batch( - " - CREATE TABLE clone_fingerprint_counts ( - language TEXT NOT NULL, - class INTEGER NOT NULL, - normalization_revision INTEGER NOT NULL, - fingerprint INTEGER NOT NULL, - posting_count INTEGER NOT NULL, - PRIMARY KEY(language, class, normalization_revision, fingerprint) - ) WITHOUT ROWID; CREATE TABLE clone_fingerprint_postings ( language TEXT NOT NULL, class INTEGER NOT NULL, normalization_revision INTEGER NOT NULL, fingerprint INTEGER NOT NULL, - symbol_occurrence_id TEXT NOT NULL, - token_position INTEGER NOT NULL, - payload_digest TEXT NOT NULL, - body_digest TEXT NOT NULL, - PRIMARY KEY(language, class, normalization_revision, fingerprint, symbol_occurrence_id, token_position) + posting_count INTEGER NOT NULL CHECK(posting_count > 0), + postings BLOB NOT NULL, + PRIMARY KEY(language, class, normalization_revision, fingerprint) ) WITHOUT ROWID; CREATE TABLE clone_fingerprint_postings_pages ( language TEXT NOT NULL, class INTEGER NOT NULL, normalization_revision INTEGER NOT NULL, fingerprint INTEGER NOT NULL, - symbol_occurrence_id TEXT NOT NULL, - token_position INTEGER NOT NULL, - payload_digest TEXT NOT NULL, - body_digest TEXT NOT NULL + occurrence_ordinal INTEGER NOT NULL, + token_position INTEGER NOT NULL ); CREATE TRIGGER builder_gate_clone_fingerprint_postings_insert BEFORE INSERT ON clone_fingerprint_postings WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; CREATE TRIGGER builder_gate_clone_fingerprint_postings_pages_insert BEFORE INSERT ON clone_fingerprint_postings_pages WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; @@ -4565,13 +4201,9 @@ fn create_schema( CREATE TRIGGER immutable_clone_fingerprint_postings_delete BEFORE DELETE ON clone_fingerprint_postings BEGIN SELECT RAISE(ABORT, 'immutable clone fingerprint postings'); END; CREATE TRIGGER immutable_clone_fingerprint_postings_pages_update BEFORE UPDATE ON clone_fingerprint_postings_pages BEGIN SELECT RAISE(ABORT, 'immutable clone fingerprint posting pages'); END; CREATE TRIGGER immutable_clone_fingerprint_postings_pages_delete BEFORE DELETE ON clone_fingerprint_postings_pages BEGIN SELECT RAISE(ABORT, 'immutable clone fingerprint posting pages'); END; - CREATE TRIGGER immutable_clone_fingerprint_counts_update BEFORE UPDATE ON clone_fingerprint_counts BEGIN SELECT RAISE(ABORT, 'immutable clone fingerprint counts'); END; - CREATE TRIGGER immutable_clone_fingerprint_counts_delete BEFORE DELETE ON clone_fingerprint_counts BEGIN SELECT RAISE(ABORT, 'immutable clone fingerprint counts'); END; ", - ) - .map_err(sqlite_error)?; - } - Ok(()) + ) + .map_err(sqlite_error) } fn table_exists(connection: &Connection, table: &str) -> Result { @@ -4599,16 +4231,27 @@ fn verify_builder_mutation_gate_schema( fn verify_layout_dependent_triggers( connection: &Connection, ) -> Result<(), CodeLexicalArtifactErrorV1> { - // Layout-dependent tables are verified only when present: `exact_vocabulary` - // from revision 12, and the staging tables (revision-14 dictionary, field - // statistics) that finalization drops once their sealed table is derived. + // Staging tables are verified only while present: finalization drops + // each once its sealed table is derived. let has_exact_vocabulary = table_exists(connection, "exact_vocabulary")?; let has_row_dictionary_pages = table_exists(connection, "row_dictionary_pages")?; let has_field_stats_staging = table_exists(connection, "field_stats_staging")?; + let has_row_chunk_pages = table_exists(connection, "row_chunk_pages")?; + let has_term_posting_runs = table_exists(connection, "term_posting_runs")?; + let has_exact_posting_runs = table_exists(connection, "exact_posting_runs")?; let has_clone_index = table_exists(connection, "clone_body_payloads")?; let has_clone_fingerprints = table_exists(connection, "clone_fingerprint_postings")?; let has_clone_fingerprint_pages = table_exists(connection, "clone_fingerprint_postings_pages")?; - let gated_layouts: [(bool, &[GateTriggerLayoutV1]); 6] = [ + let has_source_page_cursors = table_exists(connection, "source_page_cursors")?; + let gated_layouts: [(bool, &[GateTriggerLayoutV1]); 10] = [ + ( + has_source_page_cursors, + &SOURCE_PAGE_CURSORS_BUILDER_GATE_TRIGGER_LAYOUT, + ), + ( + has_row_chunk_pages, + &ROW_CHUNK_PAGES_BUILDER_GATE_TRIGGER_LAYOUT, + ), ( has_exact_vocabulary, &EXACT_VOCABULARY_BUILDER_GATE_TRIGGER_LAYOUT, @@ -4621,6 +4264,14 @@ fn verify_layout_dependent_triggers( has_field_stats_staging, &FIELD_STATS_STAGING_BUILDER_GATE_TRIGGER_LAYOUT, ), + ( + has_term_posting_runs, + &TERM_POSTING_RUNS_BUILDER_GATE_TRIGGER_LAYOUT, + ), + ( + has_exact_posting_runs, + &EXACT_POSTING_RUNS_BUILDER_GATE_TRIGGER_LAYOUT, + ), (has_clone_index, &CLONE_BUILDER_GATE_TRIGGER_LAYOUT), ( has_clone_fingerprints, @@ -4631,8 +4282,12 @@ fn verify_layout_dependent_triggers( &CLONE_FINGERPRINT_PAGES_BUILDER_GATE_TRIGGER_LAYOUT, ), ]; - let immutable_layouts: [(bool, &[ImmutableTriggerLayoutV1]); 6] = [ + let immutable_layouts: [(bool, &[ImmutableTriggerLayoutV1]); 7] = [ (true, &IMMUTABLE_TRIGGER_LAYOUT), + ( + has_source_page_cursors, + &SOURCE_PAGE_CURSORS_IMMUTABLE_TRIGGER_LAYOUT, + ), ( has_exact_vocabulary, &EXACT_VOCABULARY_IMMUTABLE_TRIGGER_LAYOUT, @@ -4720,127 +4375,128 @@ fn verify_trigger_schema( Ok(()) } -fn install_base_freeze( - transaction: &Transaction<'_>, - layout: LexicalArtifactLayoutV1, -) -> Result<(), CodeLexicalArtifactErrorV1> { +fn install_base_freeze(transaction: &Transaction<'_>) -> Result<(), CodeLexicalArtifactErrorV1> { transaction .execute_batch( " CREATE TRIGGER frozen_source_pages_insert BEFORE INSERT ON source_pages BEGIN SELECT RAISE(ABORT, 'frozen lexical source pages'); END; - CREATE TRIGGER frozen_document_integrity_insert BEFORE INSERT ON document_integrity BEGIN SELECT RAISE(ABORT, 'frozen lexical document integrity'); END; - CREATE TRIGGER frozen_import_integrity_insert BEFORE INSERT ON import_integrity BEGIN SELECT RAISE(ABORT, 'frozen lexical import integrity'); END; + CREATE TRIGGER frozen_source_page_cursors_insert BEFORE INSERT ON source_page_cursors BEGIN SELECT RAISE(ABORT, 'frozen lexical source pages'); END; CREATE TRIGGER frozen_import_evidence_insert BEFORE INSERT ON import_evidence BEGIN SELECT RAISE(ABORT, 'frozen lexical import evidence'); END; - CREATE TRIGGER frozen_rows_insert BEFORE INSERT ON rows BEGIN SELECT RAISE(ABORT, 'frozen lexical rows'); END; - CREATE TRIGGER frozen_rows_update BEFORE UPDATE ON rows BEGIN SELECT RAISE(ABORT, 'frozen lexical rows'); END; - CREATE TRIGGER frozen_rows_delete BEFORE DELETE ON rows BEGIN SELECT RAISE(ABORT, 'frozen lexical rows'); END; - CREATE TRIGGER frozen_term_postings_insert BEFORE INSERT ON term_postings BEGIN SELECT RAISE(ABORT, 'frozen lexical term postings'); END; - CREATE TRIGGER frozen_term_postings_update BEFORE UPDATE ON term_postings BEGIN SELECT RAISE(ABORT, 'frozen lexical term postings'); END; - CREATE TRIGGER frozen_term_postings_delete BEFORE DELETE ON term_postings BEGIN SELECT RAISE(ABORT, 'frozen lexical term postings'); END; - CREATE TRIGGER frozen_exact_postings_insert BEFORE INSERT ON exact_postings BEGIN SELECT RAISE(ABORT, 'frozen lexical exact postings'); END; - CREATE TRIGGER frozen_exact_postings_update BEFORE UPDATE ON exact_postings BEGIN SELECT RAISE(ABORT, 'frozen lexical exact postings'); END; - CREATE TRIGGER frozen_exact_postings_delete BEFORE DELETE ON exact_postings BEGIN SELECT RAISE(ABORT, 'frozen lexical exact postings'); END; - CREATE TRIGGER frozen_ngram_postings_insert BEFORE INSERT ON ngram_postings BEGIN SELECT RAISE(ABORT, 'frozen lexical ngram postings'); END; - CREATE TRIGGER frozen_ngram_postings_update BEFORE UPDATE ON ngram_postings BEGIN SELECT RAISE(ABORT, 'frozen lexical ngram postings'); END; - CREATE TRIGGER frozen_ngram_postings_delete BEFORE DELETE ON ngram_postings BEGIN SELECT RAISE(ABORT, 'frozen lexical ngram postings'); END; + CREATE TRIGGER frozen_row_blocks_insert BEFORE INSERT ON row_blocks BEGIN SELECT RAISE(ABORT, 'frozen lexical rows'); END; + CREATE TRIGGER frozen_row_blocks_update BEFORE UPDATE ON row_blocks BEGIN SELECT RAISE(ABORT, 'frozen lexical rows'); END; + CREATE TRIGGER frozen_row_blocks_delete BEFORE DELETE ON row_blocks BEGIN SELECT RAISE(ABORT, 'frozen lexical rows'); END; + CREATE TRIGGER frozen_row_chunk_pages_insert BEFORE INSERT ON row_chunk_pages BEGIN SELECT RAISE(ABORT, 'frozen lexical rows'); END; + CREATE TRIGGER frozen_row_chunk_pages_update BEFORE UPDATE ON row_chunk_pages BEGIN SELECT RAISE(ABORT, 'frozen lexical rows'); END; + CREATE TRIGGER frozen_row_chunk_pages_delete BEFORE DELETE ON row_chunk_pages BEGIN SELECT RAISE(ABORT, 'frozen lexical rows'); END; + CREATE TRIGGER frozen_term_posting_runs_insert BEFORE INSERT ON term_posting_runs BEGIN SELECT RAISE(ABORT, 'frozen lexical term posting runs'); END; + CREATE TRIGGER frozen_term_posting_runs_update BEFORE UPDATE ON term_posting_runs BEGIN SELECT RAISE(ABORT, 'frozen lexical term posting runs'); END; + CREATE TRIGGER frozen_term_posting_runs_delete BEFORE DELETE ON term_posting_runs BEGIN SELECT RAISE(ABORT, 'frozen lexical term posting runs'); END; + CREATE TRIGGER frozen_exact_posting_runs_insert BEFORE INSERT ON exact_posting_runs BEGIN SELECT RAISE(ABORT, 'frozen lexical exact posting runs'); END; + CREATE TRIGGER frozen_exact_posting_runs_update BEFORE UPDATE ON exact_posting_runs BEGIN SELECT RAISE(ABORT, 'frozen lexical exact posting runs'); END; + CREATE TRIGGER frozen_exact_posting_runs_delete BEFORE DELETE ON exact_posting_runs BEGIN SELECT RAISE(ABORT, 'frozen lexical exact posting runs'); END; ", ) .map_err(sqlite_error)?; - if layout.interns_exact_terms() { - transaction - .execute_batch( - " - CREATE TRIGGER frozen_exact_vocabulary_insert BEFORE INSERT ON exact_vocabulary BEGIN SELECT RAISE(ABORT, 'frozen lexical exact vocabulary'); END; - CREATE TRIGGER frozen_exact_vocabulary_update BEFORE UPDATE ON exact_vocabulary BEGIN SELECT RAISE(ABORT, 'frozen lexical exact vocabulary'); END; - CREATE TRIGGER frozen_exact_vocabulary_delete BEFORE DELETE ON exact_vocabulary BEGIN SELECT RAISE(ABORT, 'frozen lexical exact vocabulary'); END; - ", - ) - .map_err(sqlite_error)?; - } - if layout.interns_row_dictionary() { - transaction - .execute_batch( - " - CREATE TRIGGER frozen_row_dictionary_pages_insert BEFORE INSERT ON row_dictionary_pages BEGIN SELECT RAISE(ABORT, 'frozen lexical row dictionary pages'); END; - CREATE TRIGGER frozen_row_dictionary_pages_update BEFORE UPDATE ON row_dictionary_pages BEGIN SELECT RAISE(ABORT, 'frozen lexical row dictionary pages'); END; - CREATE TRIGGER frozen_row_dictionary_pages_delete BEFORE DELETE ON row_dictionary_pages BEGIN SELECT RAISE(ABORT, 'frozen lexical row dictionary pages'); END; - ", - ) - .map_err(sqlite_error)?; - } - install_clone_freeze(transaction, layout)?; - Ok(()) -} - -pub(super) fn install_clone_freeze( - transaction: &Transaction<'_>, - layout: LexicalArtifactLayoutV1, -) -> Result<(), CodeLexicalArtifactErrorV1> { - if layout.has_clone_index() { - transaction - .execute_batch( - " - CREATE TRIGGER frozen_clone_body_payloads_insert BEFORE INSERT ON clone_body_payloads BEGIN SELECT RAISE(ABORT, 'frozen clone body payloads'); END; - CREATE TRIGGER frozen_clone_occurrences_insert BEFORE INSERT ON clone_occurrences BEGIN SELECT RAISE(ABORT, 'frozen clone occurrences'); END; - CREATE TRIGGER frozen_clone_exact_postings_insert BEFORE INSERT ON clone_exact_postings BEGIN SELECT RAISE(ABORT, 'frozen clone exact postings'); END; - ", - ) - .map_err(sqlite_error)?; - } - if layout.has_clone_fingerprints() { - transaction - .execute_batch( - " - CREATE TRIGGER frozen_clone_fingerprint_counts_insert BEFORE INSERT ON clone_fingerprint_counts BEGIN SELECT RAISE(ABORT, 'frozen clone fingerprint counts'); END; - CREATE TRIGGER frozen_clone_fingerprint_postings_insert BEFORE INSERT ON clone_fingerprint_postings BEGIN SELECT RAISE(ABORT, 'frozen clone fingerprint postings'); END; - ", - ) - .map_err(sqlite_error)?; - } - Ok(()) + transaction + .execute_batch( + " + CREATE TRIGGER frozen_exact_vocabulary_insert BEFORE INSERT ON exact_vocabulary BEGIN SELECT RAISE(ABORT, 'frozen lexical exact vocabulary'); END; + CREATE TRIGGER frozen_exact_vocabulary_update BEFORE UPDATE ON exact_vocabulary BEGIN SELECT RAISE(ABORT, 'frozen lexical exact vocabulary'); END; + CREATE TRIGGER frozen_exact_vocabulary_delete BEFORE DELETE ON exact_vocabulary BEGIN SELECT RAISE(ABORT, 'frozen lexical exact vocabulary'); END; + ", + ) + .map_err(sqlite_error)?; + transaction + .execute_batch( + " + CREATE TRIGGER frozen_row_dictionary_pages_insert BEFORE INSERT ON row_dictionary_pages BEGIN SELECT RAISE(ABORT, 'frozen lexical row dictionary pages'); END; + CREATE TRIGGER frozen_row_dictionary_pages_update BEFORE UPDATE ON row_dictionary_pages BEGIN SELECT RAISE(ABORT, 'frozen lexical row dictionary pages'); END; + CREATE TRIGGER frozen_row_dictionary_pages_delete BEFORE DELETE ON row_dictionary_pages BEGIN SELECT RAISE(ABORT, 'frozen lexical row dictionary pages'); END; + ", + ) + .map_err(sqlite_error)?; + transaction + .execute_batch( + " + CREATE TRIGGER frozen_clone_body_payloads_insert BEFORE INSERT ON clone_body_payloads BEGIN SELECT RAISE(ABORT, 'frozen clone body payloads'); END; + CREATE TRIGGER frozen_clone_occurrences_insert BEFORE INSERT ON clone_occurrences BEGIN SELECT RAISE(ABORT, 'frozen clone occurrences'); END; + CREATE TRIGGER frozen_clone_exact_postings_insert BEFORE INSERT ON clone_exact_postings BEGIN SELECT RAISE(ABORT, 'frozen clone exact postings'); END; + CREATE TRIGGER frozen_clone_fingerprint_postings_insert BEFORE INSERT ON clone_fingerprint_postings BEGIN SELECT RAISE(ABORT, 'frozen clone fingerprint postings'); END; + ", + ) + .map_err(sqlite_error) } fn authenticated_authority_epoch( transaction: &Transaction<'_>, source: &VerifiedSealedLexicalSourceReceiptV1, + control: &dyn CodeIndexExecutionControlV1, ) -> Result { verify_builder_mutation_gate_schema(transaction)?; - let (pages, documents, import_integrity, import_evidence): (i64, i64, i64, i64) = transaction + let (pages, documents, imports): (i64, i64, i64) = transaction .query_row( - "SELECT (SELECT COUNT(*) FROM source_pages), (SELECT COUNT(*) FROM document_integrity), (SELECT COUNT(*) FROM import_integrity), (SELECT COUNT(*) FROM import_evidence)", + "SELECT (SELECT COUNT(*) FROM source_pages), (SELECT COUNT(*) FROM row_chunk_pages), (SELECT COUNT(*) FROM import_evidence)", [], - |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), ) .map_err(sqlite_error)?; let expected_epoch = pages .checked_add(documents) - .and_then(|count| count.checked_add(import_integrity)) - .and_then(|count| count.checked_add(import_evidence)) + .and_then(|count| count.checked_add(imports)) .ok_or_else(|| { CodeLexicalArtifactErrorV1::Contract( "lexical artifact authority row count overflowed".to_owned(), ) })?; let actual_epoch = content_epoch(transaction)?; + let admitted = admitted_documents(transaction)?; if actual_epoch != expected_epoch || u64::try_from(pages).ok() != Some(source.page_count()) - || u64::try_from(documents).ok() != Some(source.total_chunks()) - || u64::try_from(import_integrity).ok() != Some(source.total_imports()) - || import_integrity != import_evidence + || u64::try_from(documents).ok() != Some(admitted) + || admitted > source.total_chunks() + || u64::try_from(imports).ok() != Some(source.total_imports()) { return Err(CodeLexicalArtifactErrorV1::Corrupt( "lexical artifact authenticated authority disagrees with its source receipt".to_owned(), )); } - if layout_has_clone_index(transaction)? { - verify_clone_rows(transaction, source)?; - } + verify_clone_rows(transaction, source, control)?; Ok(actual_epoch) } -pub(super) fn verify_clone_rows( +/// Documents the staged pages admitted, as their base-section receipts +/// count them: every source chunk except those the projection does not +/// index (annotation uses). +fn admitted_documents(connection: &Connection) -> Result { + let rows_section = BASE_SECTION_NAMES + .iter() + .position(|name| *name == "rows") + .ok_or_else(|| { + CodeLexicalArtifactErrorV1::Contract("lexical rows section is unnamed".to_owned()) + })?; + let mut statement = connection + .prepare( + "SELECT page_ordinal, base_sections_receipt FROM source_pages ORDER BY page_ordinal", + ) + .map_err(sqlite_error)?; + let mut pages = statement.query([]).map_err(sqlite_error)?; + let mut admitted = 0u64; + while let Some(page) = pages.next().map_err(sqlite_error)? { + let page_ordinal = + u64::try_from(page.get::<_, i64>(0).map_err(sqlite_error)?).map_err(contract_number)?; + let receipt: Vec = page.get(1).map_err(sqlite_error)?; + let receipt = decode_page_base_sections_receipt(page_ordinal, &receipt)?; + admitted = admitted + .checked_add(receipt.sections()[rows_section].row_count) + .ok_or_else(source_chain_overflow)?; + } + Ok(admitted) +} + +fn verify_clone_rows( connection: &Connection, source: &VerifiedSealedLexicalSourceReceiptV1, + control: &dyn CodeIndexExecutionControlV1, ) -> Result<(), CodeLexicalArtifactErrorV1> { let (occurrences, missing_payloads, orphan_payloads, dangling_postings): ( i64, @@ -4851,9 +4507,9 @@ pub(super) fn verify_clone_rows( .query_row( "SELECT (SELECT COUNT(*) FROM clone_occurrences), - (SELECT COUNT(*) FROM clone_occurrences AS occurrence LEFT JOIN clone_body_payloads AS payload ON payload.payload_digest = occurrence.payload_digest WHERE payload.payload_digest IS NULL), - (SELECT COUNT(*) FROM clone_body_payloads AS payload LEFT JOIN clone_occurrences AS occurrence ON occurrence.payload_digest = payload.payload_digest WHERE occurrence.symbol_occurrence_id IS NULL), - (SELECT COUNT(*) FROM clone_exact_postings AS posting LEFT JOIN clone_occurrences AS occurrence ON occurrence.symbol_occurrence_id = posting.symbol_occurrence_id LEFT JOIN clone_body_payloads AS payload ON payload.payload_digest = posting.payload_digest WHERE occurrence.symbol_occurrence_id IS NULL OR payload.payload_digest IS NULL OR occurrence.payload_digest != posting.payload_digest)", + (SELECT COUNT(*) FROM clone_occurrences AS occurrence LEFT JOIN clone_body_payloads AS payload ON payload.ordinal = occurrence.payload_ordinal WHERE payload.ordinal IS NULL), + (SELECT COUNT(*) FROM clone_body_payloads WHERE ordinal NOT IN (SELECT payload_ordinal FROM clone_occurrences)), + (SELECT COUNT(*) FROM clone_exact_postings AS posting LEFT JOIN clone_occurrences AS occurrence ON occurrence.ordinal = posting.occurrence_ordinal WHERE occurrence.ordinal IS NULL)", [], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), ) @@ -4867,76 +4523,60 @@ pub(super) fn verify_clone_rows( "clone rows disagree with their source receipt or payload bindings".to_owned(), )); } - if !layout_has_clone_fingerprints(connection)? { - return Ok(()); + // Every fingerprint posting names a stored occurrence, and each list's + // stored count is its length. The read path re-derives each candidate's + // selected positions from its canonical payload before trusting one. + let mut occurrence_ordinals = HashSet::new(); + let mut statement = connection + .prepare("SELECT ordinal FROM clone_occurrences") + .map_err(sqlite_error)?; + let mut rows = statement.query([]).map_err(sqlite_error)?; + while let Some(row) = rows.next().map_err(sqlite_error)? { + occurrence_ordinals.insert(row.get::<_, i64>(0).map_err(sqlite_error)?); } - let (dangling, count_mismatch): (i64, bool) = connection - .query_row( - "WITH actual(language, class, normalization_revision, fingerprint, posting_count) AS ( - SELECT language, class, normalization_revision, fingerprint, COUNT(*) - FROM clone_fingerprint_postings - GROUP BY language, class, normalization_revision, fingerprint - ), - mismatched AS ( - SELECT * FROM actual - EXCEPT - SELECT language, class, normalization_revision, fingerprint, posting_count - FROM clone_fingerprint_counts - UNION ALL - SELECT language, class, normalization_revision, fingerprint, posting_count - FROM clone_fingerprint_counts - EXCEPT - SELECT * FROM actual - ) - SELECT - (SELECT COUNT(*) FROM clone_fingerprint_postings AS posting - LEFT JOIN clone_occurrences AS occurrence ON occurrence.symbol_occurrence_id = posting.symbol_occurrence_id - LEFT JOIN clone_body_payloads AS payload ON payload.payload_digest = posting.payload_digest - WHERE occurrence.symbol_occurrence_id IS NULL - OR payload.payload_digest IS NULL - OR occurrence.payload_digest != posting.payload_digest - OR posting.token_position < 0), - EXISTS(SELECT 1 FROM mismatched)", - [], - |row| Ok((row.get(0)?, row.get(1)?)), - ) + drop(rows); + drop(statement); + let mut statement = connection + .prepare("SELECT posting_count, postings FROM clone_fingerprint_postings") .map_err(sqlite_error)?; - if dangling != 0 || count_mismatch { - return Err(CodeLexicalArtifactErrorV1::Corrupt( - "clone fingerprint postings disagree with their payload bindings or stored counts" - .to_owned(), - )); + let mut rows = statement.query([]).map_err(sqlite_error)?; + let mut visited = 0usize; + while let Some(row) = rows.next().map_err(sqlite_error)? { + if visited.is_multiple_of(TERM_INSERT_CONTROL_INTERVAL) { + checkpoint(control)?; + } + visited += 1; + let count: i64 = row.get(0).map_err(sqlite_error)?; + let encoded = row + .get_ref(1) + .and_then(|value| value.as_blob().map_err(rusqlite::Error::from)) + .map_err(sqlite_corrupt)?; + let postings = decode_fingerprint_postings(encoded)?; + if usize::try_from(count).ok() != Some(postings.len()) + || postings + .iter() + .any(|(occurrence, _)| !occurrence_ordinals.contains(&i64::from(*occurrence))) + { + return Err(CodeLexicalArtifactErrorV1::Corrupt( + "clone fingerprint postings disagree with their occurrences or stored counts" + .to_owned(), + )); + } } Ok(()) } -fn layout_has_clone_index(connection: &Connection) -> Result { - Ok(read_staged_artifact_layout(connection)?.has_clone_index()) -} - -fn layout_has_clone_fingerprints( - connection: &Connection, -) -> Result { - Ok(read_staged_artifact_layout(connection)?.has_clone_fingerprints()) -} - fn advance_pre_digest_work( transaction: &Transaction<'_>, state: &mut PersistedFinalizationStateV1, - layout: LexicalArtifactLayoutV1, + authority: &ServingIndexStepAuthorityV1<'_>, control: &dyn CodeIndexExecutionControlV1, ) -> Result<(), CodeLexicalArtifactErrorV1> { checkpoint(control)?; - // Term-leading layouts derive statistics straight off the clustered key - // and build indexes afterwards. Revision 13 clusters by document, so its - // serving indexes come first: `term_stats` and the fuzzy-vocabulary flag - // then read `term_postings_by_term` in key order instead of sorting the - // whole posting table twice. - let indexes_first = layout.clusters_term_postings_by_document(); match state.phase { PersistedFinalizationPhaseV1::Statistics => { with_cancellable_sqlite_statement(transaction, control, || { - derive_statistics_step(transaction, state.section_ordinal)?; + derive_statistics_step(transaction, state.section_ordinal, control)?; Ok(()) })?; state.section_ordinal = state.section_ordinal.checked_add(1).ok_or_else(|| { @@ -4945,17 +4585,12 @@ fn advance_pre_digest_work( ) })?; if state.section_ordinal == STATISTICS_STEP_COUNT_V11 { - if indexes_first { - enter_digest_phase(state)?; - } else { - state.phase = PersistedFinalizationPhaseV1::Indexes; - state.section_ordinal = 0; - } + enter_digest_phase(state)?; } } PersistedFinalizationPhaseV1::Indexes => { with_cancellable_sqlite_statement(transaction, control, || { - build_serving_index_step(transaction, state.section_ordinal, layout)?; + build_serving_index_step(transaction, state.section_ordinal, authority, control)?; Ok(()) })?; state.section_ordinal = state.section_ordinal.checked_add(1).ok_or_else(|| { @@ -4964,13 +4599,8 @@ fn advance_pre_digest_work( ) })?; if state.section_ordinal == SERVING_INDEX_STEP_COUNT_V11 { - verify_required_artifact_indexes(transaction, layout)?; - if indexes_first { - state.phase = PersistedFinalizationPhaseV1::Statistics; - state.section_ordinal = 0; - } else { - enter_digest_phase(state)?; - } + state.phase = PersistedFinalizationPhaseV1::Statistics; + state.section_ordinal = 0; } } PersistedFinalizationPhaseV1::Digest => { @@ -4994,7 +4624,7 @@ fn enter_digest_phase( } fn with_cancellable_sqlite_statement( - transaction: &Transaction<'_>, + connection: &Connection, control: &dyn CodeIndexExecutionControlV1, operation: impl FnOnce() -> Result, ) -> Result { @@ -5002,7 +4632,7 @@ fn with_cancellable_sqlite_statement( let interruption = Arc::new(AtomicU8::new(0)); let finished = Arc::new(AtomicBool::new(false)); let progress_interruption = Arc::clone(&interruption); - transaction + connection .progress_handler( FINALIZATION_PROGRESS_INTERVAL_OPS, Some(move || progress_interruption.load(Ordering::Acquire) != 0), @@ -5044,7 +4674,7 @@ fn with_cancellable_sqlite_statement( finished.store(true, Ordering::Release); Ok::<_, std::io::Error>((readiness, outcome, monitor.join())) }); - let clear = transaction + let clear = connection .progress_handler(FINALIZATION_PROGRESS_INTERVAL_OPS, None:: bool>) .map_err(sqlite_error); clear?; @@ -5125,161 +4755,527 @@ fn stage_field_totals( Ok(()) } -fn derive_statistics_step( - transaction: &Transaction<'_>, - ordinal: u64, -) -> Result<(), CodeLexicalArtifactErrorV1> { - match ordinal { - 0 => hotpath::measure_block!("query.artifact.finalization.derive_field_stats", { - // Every committed batch already folded its posting lengths into - // the staging totals (`stage_field_totals`): sealing copies at - // most seven rows and drops the staging table with its gates. - transaction.execute_batch( - "INSERT INTO field_stats(field, total_length) SELECT field, total_length FROM field_stats_staging ORDER BY field; - DROP TABLE field_stats_staging; - CREATE TRIGGER frozen_field_stats_insert BEFORE INSERT ON field_stats BEGIN SELECT RAISE(ABORT, 'frozen lexical field statistics'); END; - CREATE TRIGGER frozen_field_stats_update BEFORE UPDATE ON field_stats BEGIN SELECT RAISE(ABORT, 'frozen lexical field statistics'); END; - CREATE TRIGGER frozen_field_stats_delete BEFORE DELETE ON field_stats BEGIN SELECT RAISE(ABORT, 'frozen lexical field statistics'); END;", - ) - }), - 1 => hotpath::measure_block!("query.artifact.finalization.derive_term_stats", { - transaction.execute_batch( - "INSERT INTO term_stats(term_id, field, document_frequency) SELECT term_id, field, COUNT(*) FROM term_postings GROUP BY term_id, field; - CREATE TRIGGER frozen_term_stats_insert BEFORE INSERT ON term_stats BEGIN SELECT RAISE(ABORT, 'frozen lexical term statistics'); END; - CREATE TRIGGER frozen_term_stats_update BEFORE UPDATE ON term_stats BEGIN SELECT RAISE(ABORT, 'frozen lexical term statistics'); END; - CREATE TRIGGER frozen_term_stats_delete BEFORE DELETE ON term_stats BEGIN SELECT RAISE(ABORT, 'frozen lexical term statistics'); END;", - ) - }), - 2 => hotpath::measure_block!("query.artifact.finalization.derive_vocabulary", { - // The preceding statistics step already grouped every posting - // by (term_id, field). Reuse that frozen membership instead of - // scanning the larger posting table again. - let subtoken = field_code(LexicalFieldV1::Subtoken); - transaction - .execute( - "UPDATE vocabulary SET in_fuzzy = 1 WHERE term_id IN (SELECT DISTINCT term_id FROM term_stats WHERE field != ?1)", - [subtoken], - ) - .and_then(|_| { - transaction.execute_batch( - "CREATE TRIGGER frozen_vocabulary_insert BEFORE INSERT ON vocabulary BEGIN SELECT RAISE(ABORT, 'frozen lexical vocabulary'); END; - CREATE TRIGGER frozen_vocabulary_update BEFORE UPDATE ON vocabulary BEGIN SELECT RAISE(ABORT, 'frozen lexical vocabulary'); END; - CREATE TRIGGER frozen_vocabulary_delete BEFORE DELETE ON vocabulary BEGIN SELECT RAISE(ABORT, 'frozen lexical vocabulary'); END;", - ) +fn derive_statistics_step( + transaction: &Transaction<'_>, + ordinal: u64, + control: &dyn CodeIndexExecutionControlV1, +) -> Result<(), CodeLexicalArtifactErrorV1> { + match ordinal { + 0 => hotpath::measure_block!("query.artifact.finalization.derive_field_stats", { + // Every committed batch already folded its posting lengths into + // the staging totals (`stage_field_totals`): sealing copies at + // most nine rows and drops the staging table with its gates. + transaction.execute_batch( + "INSERT INTO field_stats(field, total_length) SELECT field, total_length FROM field_stats_staging ORDER BY field; + DROP TABLE field_stats_staging; + CREATE TRIGGER frozen_field_stats_insert BEFORE INSERT ON field_stats BEGIN SELECT RAISE(ABORT, 'frozen lexical field statistics'); END; + CREATE TRIGGER frozen_field_stats_update BEFORE UPDATE ON field_stats BEGIN SELECT RAISE(ABORT, 'frozen lexical field statistics'); END; + CREATE TRIGGER frozen_field_stats_delete BEFORE DELETE ON field_stats BEGIN SELECT RAISE(ABORT, 'frozen lexical field statistics'); END;", + ) + }), + // Every staging table is gone; return its pages to the filesystem + // before the digest and the sealed size. + 1 => hotpath::measure_block!("query.artifact.finalization.release_staging_pages", { + // The staged per-page cursors bind the building worktree's + // source state; the finalization state keeps the terminal one + // until the seal. + transaction + .execute_batch("DROP TABLE source_page_cursors;") + .map_err(sqlite_error)?; + release_free_pages(transaction, control)?; + Ok(()) + }), + _ => { + return Err(CodeLexicalArtifactErrorV1::Corrupt( + "lexical artifact selected an unknown statistics step".to_owned(), + )); + } + } + .map_err(sqlite_error)?; + Ok(()) +} + +/// Return every free page to the filesystem. The pragma yields one row per +/// released page, so it must be stepped to completion. +fn release_free_pages( + transaction: &Transaction<'_>, + control: &dyn CodeIndexExecutionControlV1, +) -> Result<(), CodeLexicalArtifactErrorV1> { + let mut statement = transaction + .prepare("PRAGMA incremental_vacuum") + .map_err(sqlite_error)?; + let mut released = statement.query([]).map_err(sqlite_error)?; + let mut pages = 0usize; + while released.next().map_err(sqlite_error)?.is_some() { + if pages.is_multiple_of(TERM_INSERT_CONTROL_INTERVAL) { + checkpoint(control)?; + } + pages += 1; + } + Ok(()) +} + +/// What the pre-digest index steps need beyond the transaction: the +/// builder's mutation authority, the generation rows decode under, and the +/// memory the n-gram rebuild may hold at once. +struct ServingIndexStepAuthorityV1<'a> { + mutation_gate: &'a Arc, + generation: &'a CodeGenerationId, + ngram_memory_bytes: usize, +} + +fn build_serving_index_step( + transaction: &Transaction<'_>, + ordinal: u64, + authority: &ServingIndexStepAuthorityV1<'_>, + control: &dyn CodeIndexExecutionControlV1, +) -> Result<(), CodeLexicalArtifactErrorV1> { + let mutation_gate = authority.mutation_gate; + match ordinal { + 0 => hotpath::measure_block!("query.artifact.finalization.index.row_chunks", { + derive_row_dictionary(transaction)?; + let _mutation_authority = BuilderMutationGuardV1::enter(mutation_gate)?; + transaction + .execute_batch( + "INSERT INTO row_chunks(chunk_id, document_id) SELECT chunk_id, document_id FROM row_chunk_pages ORDER BY chunk_id; + DROP TABLE row_chunk_pages; + CREATE TRIGGER frozen_row_chunks_insert BEFORE INSERT ON row_chunks BEGIN SELECT RAISE(ABORT, 'frozen lexical row chunks'); END;", + ) + .map_err(sqlite_error) + }), + 1 => hotpath::measure_block!( + "query.artifact.finalization.merge.term_postings", + derive_term_postings(transaction, mutation_gate, control) + ), + 2 => hotpath::measure_block!( + "query.artifact.finalization.merge.exact_postings", + derive_exact_postings(transaction, mutation_gate, control) + ), + // Last, so its lists reuse the pages the dropped runs freed. + 3 => hotpath::measure_block!( + "query.artifact.finalization.derive.ngram_postings", + derive_ngram_postings(transaction, authority, control) + ), + _ => Err(CodeLexicalArtifactErrorV1::Corrupt( + "lexical artifact selected an unknown serving-index step".to_owned(), + )), + } +} + +/// Merge the term staging runs into one sealed `term_postings` row per term +/// in a single sorted pass: every field's list concatenates that key's runs +/// in page order, re-verified (canonical encoding, ascending documents +/// across runs) on the way. A term is fuzzy-eligible when it occurs in any +/// field but the subtoken field. +fn derive_term_postings( + transaction: &Transaction<'_>, + mutation_gate: &Arc, + control: &dyn CodeIndexExecutionControlV1, +) -> Result<(), CodeLexicalArtifactErrorV1> { + let _mutation_authority = BuilderMutationGuardV1::enter(mutation_gate)?; + let subtoken = field_code(LexicalFieldV1::Subtoken); + let mut select = transaction + .prepare( + "SELECT term, field, postings FROM term_posting_runs ORDER BY term, field, page_ordinal", + ) + .map_err(sqlite_error)?; + let mut insert = transaction + .prepare("INSERT INTO term_postings(term, in_fuzzy, lists) VALUES (?1, ?2, ?3)") + .map_err(sqlite_error)?; + let mut seal = |term: &str, + lists: Vec<(i64, PostingListEncoderV1)>| + -> Result<(), CodeLexicalArtifactErrorV1> { + let in_fuzzy = lists.iter().any(|(field, _)| *field != subtoken); + let lists = lists + .into_iter() + .map(|(field, encoder)| Ok((field, encoder.len(), encoder.finish()?))) + .collect::, CodeLexicalArtifactErrorV1>>()?; + insert + .execute(params![term, in_fuzzy, encode_term_lists(&lists)?]) + .map_err(sqlite_error)?; + Ok(()) + }; + let mut rows = select.query([]).map_err(sqlite_error)?; + let mut current: Option<(String, Vec<(i64, PostingListEncoderV1)>)> = None; + let mut visited = 0usize; + while let Some(row) = rows.next().map_err(sqlite_error)? { + if visited.is_multiple_of(TERM_INSERT_CONTROL_INTERVAL) { + checkpoint(control)?; + } + visited += 1; + let term = row + .get_ref(0) + .and_then(|value| value.as_str().map_err(rusqlite::Error::from)) + .map_err(sqlite_corrupt)?; + let field: i64 = row.get(1).map_err(sqlite_error)?; + let staged = row + .get_ref(2) + .and_then(|value| value.as_blob().map_err(rusqlite::Error::from)) + .map_err(sqlite_corrupt)?; + if let Some((sealed, lists)) = current.take_if(|(current, _)| current != term) { + seal(&sealed, lists)?; + } + let lists = &mut current + .get_or_insert_with(|| (term.to_owned(), Vec::new())) + .1; + if lists.last().is_none_or(|(last, _)| *last != field) { + lists.push((field, PostingListEncoderV1::new(true))); + } + let encoder = lists + .last_mut() + .map(|(_, encoder)| encoder) + .ok_or_else(|| { + CodeLexicalArtifactErrorV1::Contract( + "lexical artifact term merge lost its field list".to_owned(), + ) + })?; + append_staged_run(encoder, staged, true)?; + } + if let Some((sealed, lists)) = current { + seal(&sealed, lists)?; + } + drop(rows); + drop(select); + drop(insert); + checkpoint(control)?; + transaction + .execute_batch( + "DROP TABLE term_posting_runs; + CREATE TRIGGER frozen_term_postings_insert BEFORE INSERT ON term_postings BEGIN SELECT RAISE(ABORT, 'frozen lexical term postings'); END; + CREATE TRIGGER frozen_term_postings_update BEFORE UPDATE ON term_postings BEGIN SELECT RAISE(ABORT, 'frozen lexical term postings'); END; + CREATE TRIGGER frozen_term_postings_delete BEFORE DELETE ON term_postings BEGIN SELECT RAISE(ABORT, 'frozen lexical term postings'); END;", + ) + .map_err(sqlite_error) +} + +/// Merge the exact staging runs into one sealed list per `(term, field)` +/// the same way. +fn derive_exact_postings( + transaction: &Transaction<'_>, + mutation_gate: &Arc, + control: &dyn CodeIndexExecutionControlV1, +) -> Result<(), CodeLexicalArtifactErrorV1> { + let _mutation_authority = BuilderMutationGuardV1::enter(mutation_gate)?; + let mut select = transaction + .prepare( + "SELECT term_id, field, documents FROM exact_posting_runs ORDER BY term_id, field, page_ordinal", + ) + .map_err(sqlite_error)?; + let mut insert = transaction + .prepare("INSERT INTO exact_postings(term_id, field, documents) VALUES (?1, ?2, ?3)") + .map_err(sqlite_error)?; + let mut rows = select.query([]).map_err(sqlite_error)?; + let mut list: Option<((i64, i64), PostingListEncoderV1)> = None; + let mut visited = 0usize; + while let Some(row) = rows.next().map_err(sqlite_error)? { + if visited.is_multiple_of(TERM_INSERT_CONTROL_INTERVAL) { + checkpoint(control)?; + } + visited += 1; + let key: (i64, i64) = ( + row.get(0).map_err(sqlite_error)?, + row.get(1).map_err(sqlite_error)?, + ); + let staged = row + .get_ref(2) + .and_then(|value| value.as_blob().map_err(rusqlite::Error::from)) + .map_err(sqlite_corrupt)?; + if let Some(((term_id, field), encoder)) = list.take_if(|(list_key, _)| *list_key != key) { + insert + .execute(params![term_id, field, encoder.finish()?]) + .map_err(sqlite_error)?; + } + let encoder = &mut list + .get_or_insert_with(|| (key, PostingListEncoderV1::new(false))) + .1; + append_staged_run(encoder, staged, false)?; + } + if let Some(((term_id, field), encoder)) = list { + insert + .execute(params![term_id, field, encoder.finish()?]) + .map_err(sqlite_error)?; + } + drop(rows); + drop(select); + drop(insert); + checkpoint(control)?; + transaction + .execute_batch( + "DROP TABLE exact_posting_runs; + CREATE TRIGGER frozen_exact_postings_insert BEFORE INSERT ON exact_postings BEGIN SELECT RAISE(ABORT, 'frozen lexical exact postings'); END; + CREATE TRIGGER frozen_exact_postings_update BEFORE UPDATE ON exact_postings BEGIN SELECT RAISE(ABORT, 'frozen lexical exact postings'); END; + CREATE TRIGGER frozen_exact_postings_delete BEFORE DELETE ON exact_postings BEGIN SELECT RAISE(ABORT, 'frozen lexical exact postings'); END;", + ) + .map_err(sqlite_error) +} + +/// Append one staged run to its key's list; a run must be non-empty and +/// continue strictly after the documents already merged. +fn append_staged_run( + encoder: &mut PostingListEncoderV1, + staged: &[u8], + frequencies: bool, +) -> Result<(), CodeLexicalArtifactErrorV1> { + let before = encoder.len(); + for posting in PostingListDecoderV1::new(staged, frequencies) { + let (document, frequency) = posting?; + encoder.push(document, frequency).map_err(|_| { + CodeLexicalArtifactErrorV1::Corrupt( + "lexical artifact staged posting runs overlap".to_owned(), + ) + })?; + } + if encoder.len() == before { + return Err(CodeLexicalArtifactErrorV1::Corrupt( + "lexical artifact staged posting run is empty".to_owned(), + )); + } + Ok(()) +} + +/// Per-list bookkeeping charged against the n-gram rebuild's memory: map +/// entry, key, and encoder header. +const NGRAM_LIST_ENTRY_BYTES: usize = 96; +/// Rows decoded between row-dictionary resets, so the memoised dictionary +/// entries of one rebuild pass stay bounded however large the corpus is. +const NGRAM_DICTIONARY_WINDOW_ROWS: usize = 4_096; + +/// One sealed n-gram list under construction, keyed `(kind, ngram)`. +type NgramListV1 = ((i64, i64), PostingListEncoderV1); + +/// One rebuild pass's n-gram lists: keys at or above `lower` (the previous +/// pass's cutoff) and below this pass's own cutoff. When the lists outgrow +/// the memory authority the highest keys are shed and the cutoff drops to +/// them, so a later pass rebuilds exactly those keys. +struct NgramListPassV1 { + lower: Option<(i64, i64)>, + cutoff: Option<(i64, i64)>, + lists: HashMap<(i64, i64), PostingListEncoderV1>, + held: usize, + memory_bytes: usize, +} + +impl NgramListPassV1 { + fn new(lower: Option<(i64, i64)>, memory_bytes: usize) -> Self { + Self { + lower, + cutoff: None, + lists: HashMap::new(), + held: 0, + memory_bytes: memory_bytes.max(1), + } + } + + /// Documents arrive in ascending order, so every list grows by appending. + fn add(&mut self, key: (i64, i64), document: u32) -> Result<(), CodeLexicalArtifactErrorV1> { + if self.lower.is_some_and(|lower| key < lower) + || self.cutoff.is_some_and(|cutoff| key >= cutoff) + { + return Ok(()); + } + let held = &mut self.held; + let list = self.lists.entry(key).or_insert_with(|| { + *held += NGRAM_LIST_ENTRY_BYTES; + PostingListEncoderV1::new(false) + }); + let before = list.retained_bytes(); + list.push(document, 1)?; + self.held += list.retained_bytes() - before; + if self.held > self.memory_bytes { + let mut keys = self.lists.keys().copied().collect::>(); + keys.sort_unstable(); + while self.held > self.memory_bytes / 2 && keys.len() > 1 { + let Some(shed) = keys.pop() else { break }; + if let Some(list) = self.lists.remove(&shed) { + self.held -= list.retained_bytes() + NGRAM_LIST_ENTRY_BYTES; + } + self.cutoff = Some(shed); + } + } + Ok(()) + } + + /// This pass's lists in key order, and where the next pass starts. + fn finish(self) -> (Vec, Option<(i64, i64)>) { + let mut lists = self.lists.into_iter().collect::>(); + lists.sort_unstable_by_key(|(key, _)| *key); + (lists, self.cutoff) + } +} + +/// Row blocks inflated and keyed together by one parallel n-gram step. +const NGRAM_DERIVE_WINDOW_BLOCKS: usize = 64; + +/// A window of stored row blocks awaiting n-gram derivation. Inflating a +/// block and keying a decoded row are independent, so both run on the +/// indexing pool; dictionary decoding (one SQLite connection) and list +/// appends (ascending documents) stay on the calling thread in row order. +#[derive(Default)] +struct NgramDeriveWindowV1 { + blocks: Vec<(i64, Vec)>, +} + +impl NgramDeriveWindowV1 { + fn derive<'connection>( + &mut self, + generation: &CodeGenerationId, + connection: &'connection Connection, + dictionary: &mut ConnectionRowDictionaryV1<'connection>, + visited: &mut usize, + pass: &mut NgramListPassV1, + control: &dyn CodeIndexExecutionControlV1, + ) -> Result<(), CodeLexicalArtifactErrorV1> { + if self.blocks.is_empty() { + return Ok(()); + } + let blocks = std::mem::take(&mut self.blocks); + let inflated = tracedecay_code_index::parallelism::install(|| { + blocks + .par_iter() + .map(|(first_document, payload)| { + tracedecay_code_index::parallelism::with_background_cpu_permit(|| { + decode_row_block(*first_document, payload) + }) }) - }), - _ => { - return Err(CodeLexicalArtifactErrorV1::Corrupt( - "lexical artifact selected an unknown statistics step".to_owned(), - )); + .collect::>() + }) + .map_err(|error| CodeLexicalArtifactErrorV1::Io(error.to_string()))?; + drop(blocks); + let mut decoded = Vec::new(); + for rows in inflated { + checkpoint(control)?; + for stored in rows? { + if *visited > 0 && visited.is_multiple_of(NGRAM_DICTIONARY_WINDOW_ROWS) { + *dictionary = ConnectionRowDictionaryV1::new(connection); + } + *visited += 1; + let row = decode_artifact_row( + generation, + &stored.chunk_id, + &stored.row, + &stored.text, + &*dictionary, + )?; + decoded.push((stored.document_id, row)); + } + } + let keys = tracedecay_code_index::parallelism::install(|| { + decoded + .par_chunks(NGRAM_DERIVE_ROWS_PER_TASK) + .map(|rows| { + tracedecay_code_index::parallelism::with_background_cpu_permit(|| { + rows.iter() + .map(|(_, row)| { + document_ngram_keys( + &normalized_search_text(row), + row.sanitized_text.as_str(), + &row.normalized_text, + control, + ) + }) + .collect::, _>>() + }) + }) + .collect::>() + }) + .map_err(|error| CodeLexicalArtifactErrorV1::Io(error.to_string()))?; + for (rows, keys) in decoded.chunks(NGRAM_DERIVE_ROWS_PER_TASK).zip(keys) { + for ((document, _), keys) in rows.iter().zip(keys?) { + for key in keys { + pass.add(key, *document)?; + } + } } + Ok(()) } - .map_err(sqlite_error)?; - Ok(()) } -fn build_serving_index_step( +/// Decoded rows keyed by one pool task, amortizing its CPU permit. +const NGRAM_DERIVE_ROWS_PER_TASK: usize = 128; + +/// Rebuild every sealed n-gram list from the stored rows, in key order, +/// without any n-gram staging. Each pass walks the rows in document order; +/// one pass suffices unless the lists outgrow `ngram_memory_bytes`, in which +/// case later passes rebuild the keys an earlier pass shed. +fn derive_ngram_postings( transaction: &Transaction<'_>, - ordinal: u64, - layout: LexicalArtifactLayoutV1, + authority: &ServingIndexStepAuthorityV1<'_>, + control: &dyn CodeIndexExecutionControlV1, ) -> Result<(), CodeLexicalArtifactErrorV1> { - // Revision 14 derives the sealed row dictionary before the first serving - // index: dropping the staging table frees its pages for the - // `rows_by_chunk` index built in the same wake. - if ordinal == 0 && layout.interns_row_dictionary() { - hotpath::measure_block!( - "query.artifact.finalization.derive_row_dictionary", - derive_row_dictionary(transaction) + let _mutation_authority = BuilderMutationGuardV1::enter(authority.mutation_gate)?; + let mut insert = transaction + .prepare( + "INSERT INTO ngram_postings(kind, ngram, document_frequency, documents) VALUES (?1, ?2, ?3, ?4)", + ) + .map_err(sqlite_error)?; + let mut lower: Option<(i64, i64)> = None; + loop { + let mut pass = NgramListPassV1::new(lower, authority.ngram_memory_bytes); + let mut statement = transaction + .prepare("SELECT first_document, payload FROM row_blocks ORDER BY first_document") + .map_err(sqlite_error)?; + let mut blocks = statement.query([]).map_err(sqlite_error)?; + let mut dictionary = ConnectionRowDictionaryV1::new(transaction); + let mut visited = 0usize; + let mut window = NgramDeriveWindowV1::default(); + while let Some(block) = blocks.next().map_err(sqlite_error)? { + checkpoint(control)?; + let first_document: i64 = block.get(0).map_err(sqlite_error)?; + let payload = block + .get_ref(1) + .and_then(|value| value.as_blob().map_err(rusqlite::Error::from)) + .map_err(sqlite_corrupt)?; + window.blocks.push((first_document, payload.to_vec())); + if window.blocks.len() >= NGRAM_DERIVE_WINDOW_BLOCKS { + window.derive( + authority.generation, + transaction, + &mut dictionary, + &mut visited, + &mut pass, + control, + )?; + } + } + window.derive( + authority.generation, + transaction, + &mut dictionary, + &mut visited, + &mut pass, + control, )?; - } - match ordinal { - 0 => hotpath::measure_block!( - "query.artifact.finalization.index.rows_by_chunk", - transaction.execute_batch("CREATE UNIQUE INDEX rows_by_chunk ON rows(chunk_id)") - ), - 1 if layout.clusters_term_postings_by_document() => hotpath::measure_block!( - "query.artifact.finalization.index.term_postings_by_term", - transaction.execute_batch( - "CREATE INDEX term_postings_by_term ON term_postings(term_id, field, document_id, frequency)", - ) - ), - 1 => hotpath::measure_block!( - "query.artifact.finalization.index.term_postings_by_document", - transaction.execute_batch( - "CREATE INDEX term_postings_by_document ON term_postings(document_id, term_id, field, frequency)", - ) - ), - 2 => { - let sql = match layout { - LexicalArtifactLayoutV1::V10 | LexicalArtifactLayoutV1::V11 => { - "CREATE INDEX exact_postings_by_document ON exact_postings(document_id, field, term)" - } - LexicalArtifactLayoutV1::V12 - | LexicalArtifactLayoutV1::V13 - | LexicalArtifactLayoutV1::V14 - | LexicalArtifactLayoutV1::V15 - | LexicalArtifactLayoutV1::V16 => { - "CREATE INDEX exact_postings_by_document ON exact_postings(document_id, field, term_id)" - } - }; - hotpath::measure_block!( - "query.artifact.finalization.index.exact_postings_by_document", - transaction.execute_batch(sql) - ) + drop(blocks); + drop(statement); + let (lists, cutoff) = pass.finish(); + for (ordinal, (key, list)) in lists.into_iter().enumerate() { + if ordinal.is_multiple_of(TERM_INSERT_CONTROL_INTERVAL) { + checkpoint(control)?; + } + let document_frequency = i64::try_from(list.len()).map_err(contract_number)?; + let documents = encode_document_set(&decode_ngram_bitmap(&list.finish()?)?)?; + insert + .execute(params![key.0, key.1, document_frequency, documents]) + .map_err(sqlite_error)?; } - // `cardinality` rides in the index purely so the statistics - // aggregation below is covered. Without it the index carries only the - // reordered WITHOUT ROWID key columns, and every `SUM(cardinality)` - // fetch through it is one random main-tree lookup per posting row. - // An N+1 access pattern over a tree dominated by `documents` blobs - // that collapses once the corpus outgrows the bounded page cache - // (measured on a 12M-row/2.9M-group synthetic at production pragmas: - // 121 s warm and 537 s cold non-covering, ~240 s as a sort-backed - // table scan, under 4 s covered in both regimes). Uniqueness of the - // (kind, ngram, page_ordinal) prefix is already guaranteed by the - // table primary key, so the wider UNIQUE declaration loses nothing. - 3 => build_ngram_serving_index(transaction), - 4 => hotpath::measure_block!( - "query.artifact.finalization.ngram_statistics", - transaction.execute_batch( - "INSERT INTO ngram_statistics(kind, ngram, document_frequency) - SELECT kind, ngram, SUM(cardinality) - FROM ngram_postings INDEXED BY ngram_postings_by_ngram - GROUP BY kind, ngram; - CREATE TRIGGER frozen_ngram_statistics_insert BEFORE INSERT ON ngram_statistics BEGIN SELECT RAISE(ABORT, 'frozen lexical ngram statistics'); END; - CREATE TRIGGER frozen_ngram_statistics_update BEFORE UPDATE ON ngram_statistics BEGIN SELECT RAISE(ABORT, 'frozen lexical ngram statistics'); END; - CREATE TRIGGER frozen_ngram_statistics_delete BEFORE DELETE ON ngram_statistics BEGIN SELECT RAISE(ABORT, 'frozen lexical ngram statistics'); END;", - ) - ), - _ => { - return Err(CodeLexicalArtifactErrorV1::Corrupt( - "lexical artifact selected an unknown serving-index step".to_owned(), - )); + match cutoff { + Some(cutoff) => lower = Some(cutoff), + None => break, } } - .map_err(sqlite_error) -} - -fn build_ngram_serving_index(transaction: &Transaction<'_>) -> rusqlite::Result<()> { - #[cfg(feature = "hotpath")] - let started = Instant::now(); - let result = hotpath::measure_block!( - "query.artifact.finalization.index.ngram_postings_by_ngram", - transaction.execute_batch( - "CREATE UNIQUE INDEX ngram_postings_by_ngram ON ngram_postings(kind, ngram, page_ordinal, cardinality)", + drop(insert); + transaction + .execute_batch( + "CREATE TRIGGER frozen_ngram_postings_insert BEFORE INSERT ON ngram_postings BEGIN SELECT RAISE(ABORT, 'frozen lexical ngram postings'); END;", ) - ); - #[cfg(feature = "hotpath")] - hotpath::gauge!("query.artifact.finalization.index.ngram_latest_micros") - .set(started.elapsed().as_micros() as u64); - result + .map_err(sqlite_error) } impl PersistedFinalizationStateV1 { fn new( content_epoch: i64, source: &VerifiedSealedLexicalSourceReceiptV1, - layout: LexicalArtifactLayoutV1, + terminal_cursor: Option>, ) -> Result { if content_epoch < 0 { return Err(CodeLexicalArtifactErrorV1::Corrupt( @@ -5288,13 +5284,8 @@ impl PersistedFinalizationStateV1 { } let (base_section_row_counts, base_section_accumulators) = initial_base_section_receipt_fold()?; - let phase = if layout.clusters_term_postings_by_document() { - PersistedFinalizationPhaseV1::Indexes - } else { - PersistedFinalizationPhaseV1::Statistics - }; Ok(Self { - phase, + phase: PersistedFinalizationPhaseV1::Indexes, section_ordinal: 0, section_row_count: 0, section_last_key: None, @@ -5305,6 +5296,7 @@ impl PersistedFinalizationStateV1 { completed_rows: 0, content_epoch, source_state_digest: source.source_state_digest().clone(), + terminal_cursor, }) } } @@ -5313,7 +5305,6 @@ fn verify_artifact_state_metadata( connection: &Connection, expected_metadata: &CodeLexicalProjectionMetadataV1, expected_digest: &ManifestDigest, - expected_layout: LexicalArtifactLayoutV1, control: &dyn CodeIndexExecutionControlV1, ) -> Result<(), CodeLexicalArtifactErrorV1> { checkpoint(control)?; @@ -5324,21 +5315,21 @@ fn verify_artifact_state_metadata( |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), ) .map_err(|error| artifact_state_row_corrupt("metadata", error))?; - if format_revision != expected_layout.revision() { + if format_revision != CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V1 { return Err(CodeLexicalArtifactErrorV1::Incompatible(format!( "format revision {format_revision} is not supported" ))); } checkpoint(control)?; - let stored_metadata: CodeLexicalProjectionMetadataV1 = serde_json::from_slice(&metadata_bytes) - .map_err(|error| CodeLexicalArtifactErrorV1::Corrupt(error.to_string()))?; - let actual_digest = metadata_digest(&stored_metadata)?; + let actual_digest = stored_metadata_digest(&metadata_bytes)?; if stored_digest != actual_digest.as_str() { return Err(CodeLexicalArtifactErrorV1::Corrupt( "lexical artifact metadata digest does not verify".to_owned(), )); } - if &stored_metadata != expected_metadata || &actual_digest != expected_digest { + if metadata_bytes != content_metadata_bytes(expected_metadata)? + || &actual_digest != expected_digest + { return Err(CodeLexicalArtifactErrorV1::Incompatible( "staging metadata does not match the requested generation".to_owned(), )); @@ -5359,9 +5350,7 @@ fn artifact_state_row_corrupt( )) } -fn read_staged_artifact_layout( - connection: &Connection, -) -> Result { +fn require_staged_revision(connection: &Connection) -> Result<(), CodeLexicalArtifactErrorV1> { let revision: i64 = connection .query_row( "SELECT format_revision FROM artifact_state WHERE singleton = 1", @@ -5374,17 +5363,7 @@ fn read_staged_artifact_layout( "lexical artifact staging revision is outside the supported range".to_owned(), ) })?; - match LexicalArtifactLayoutV1::from_revision(revision)? { - LexicalArtifactLayoutV1::V10 => Err(CodeLexicalArtifactErrorV1::Incompatible( - "revision 10 artifacts are immutable reader inputs and cannot be resumed".to_owned(), - )), - layout @ (LexicalArtifactLayoutV1::V11 - | LexicalArtifactLayoutV1::V12 - | LexicalArtifactLayoutV1::V13 - | LexicalArtifactLayoutV1::V14 - | LexicalArtifactLayoutV1::V15 - | LexicalArtifactLayoutV1::V16) => Ok(layout), - } + require_served_revision(revision) } fn finalization_started(connection: &Connection) -> Result { @@ -5421,9 +5400,14 @@ fn ensure_content_epoch( Ok(()) } +/// The persisted finalization state; `None` before finalization starts and +/// after the seal drops the table. fn load_finalization_state( connection: &Connection, ) -> Result, CodeLexicalArtifactErrorV1> { + if !table_exists(connection, "finalization_state")? { + return Ok(None); + } let bytes: Option> = connection .query_row( "SELECT state FROM finalization_state WHERE singleton = 1", @@ -5458,9 +5442,8 @@ fn store_finalization_state( fn validate_finalization_state( state: &PersistedFinalizationStateV1, - layout: LexicalArtifactLayoutV1, ) -> Result<(), CodeLexicalArtifactErrorV1> { - let section_names = section_names(layout); + let section_names = &SECTION_NAMES; let section_count = u64::try_from(section_names.len()).map_err(contract_number)?; let completed_section_count = if state.phase == PersistedFinalizationPhaseV1::Digest { usize::try_from(state.section_ordinal).map_err(contract_number)? @@ -5536,7 +5519,6 @@ fn validate_finalization_state( fn advance_section_rows( transaction: &Transaction<'_>, section: FinalizationSectionV1, - layout: LexicalArtifactLayoutV1, state: &mut PersistedFinalizationStateV1, maximum_rows: usize, control: &dyn CodeIndexExecutionControlV1, @@ -5548,13 +5530,11 @@ fn advance_section_rows( FinalizationSectionV1::CloneOccurrences | FinalizationSectionV1::CloneExactPostings | FinalizationSectionV1::CloneBodyPayloads - | FinalizationSectionV1::CloneFingerprintCounts | FinalizationSectionV1::CloneFingerprintPostings ) { return advance_clone_section_rows( transaction, section, - layout, state, limit, last_key.as_ref(), @@ -5562,106 +5542,40 @@ fn advance_section_rows( ); } match (section, last_key.as_ref()) { - (FinalizationSectionV1::SourcePages, None) - | (FinalizationSectionV1::DocumentIntegrity, None) - | (FinalizationSectionV1::Rows, None) - | (FinalizationSectionV1::ImportIntegrity, None) - | (FinalizationSectionV1::ImportEvidence, None) - | (FinalizationSectionV1::TermPostings, None) - | (FinalizationSectionV1::ExactPostings, None) - | (FinalizationSectionV1::NgramPostings, None) - | (FinalizationSectionV1::FieldStatistics, None) - | (FinalizationSectionV1::TermStatistics, None) - | (FinalizationSectionV1::Vocabulary, None) => advance_native_section_rows( - transaction, - section, - section.seek_query(layout, false), - params![limit], - state, - control, - ), ( FinalizationSectionV1::SourcePages - | FinalizationSectionV1::DocumentIntegrity - | FinalizationSectionV1::Rows | FinalizationSectionV1::FieldStatistics | FinalizationSectionV1::Vocabulary, - Some(PersistedFinalizationKeyV1::Integer(value)), + None, ) => advance_native_section_rows( transaction, section, - section.seek_query(layout, true), - params![value, limit], + section.walked_query(false)?, + params![limit], state, control, ), ( - FinalizationSectionV1::ImportIntegrity | FinalizationSectionV1::ImportEvidence, - Some(PersistedFinalizationKeyV1::Blob(value)), + FinalizationSectionV1::SourcePages | FinalizationSectionV1::FieldStatistics, + Some(PersistedFinalizationKeyV1::Integer(value)), ) => advance_native_section_rows( transaction, section, - section.seek_query(layout, true), + section.walked_query(true)?, params![value, limit], state, control, ), - ( - FinalizationSectionV1::TermPostings, - Some(PersistedFinalizationKeyV1::IntegerIntegerInteger { - page_ordinal, - kind, - ngram, - }), - ) => advance_native_section_rows( - transaction, - section, - section.seek_query(layout, true), - params![page_ordinal, kind, ngram, limit], - state, - control, - ), - ( - FinalizationSectionV1::ExactPostings, - Some(PersistedFinalizationKeyV1::TextBlobInteger { - field, - term, - document_id, - }), - ) => advance_native_section_rows( - transaction, - section, - section.seek_query(layout, true), - params![field, term, document_id, limit], - state, - control, - ), - ( - FinalizationSectionV1::NgramPostings, - Some(PersistedFinalizationKeyV1::IntegerIntegerInteger { - page_ordinal, - kind, - ngram, - }), - ) => advance_native_section_rows( - transaction, - section, - section.seek_query(layout, true), - params![page_ordinal, kind, ngram, limit], - state, - control, - ), - ( - FinalizationSectionV1::TermStatistics, - Some(PersistedFinalizationKeyV1::IntegerPair { first, second }), - ) => advance_native_section_rows( - transaction, - section, - section.seek_query(layout, true), - params![first, second, limit], - state, - control, - ), + (FinalizationSectionV1::Vocabulary, Some(PersistedFinalizationKeyV1::Text(value))) => { + advance_native_section_rows( + transaction, + section, + section.walked_query(true)?, + params![value, limit], + state, + control, + ) + } _ => Err(CodeLexicalArtifactErrorV1::Corrupt( "persisted lexical artifact finalization key does not match its section".to_owned(), )), @@ -5671,7 +5585,6 @@ fn advance_section_rows( fn advance_clone_section_rows( transaction: &Transaction<'_>, section: FinalizationSectionV1, - layout: LexicalArtifactLayoutV1, state: &mut PersistedFinalizationStateV1, limit: i64, last_key: Option<&PersistedFinalizationKeyV1>, @@ -5682,24 +5595,23 @@ fn advance_clone_section_rows( FinalizationSectionV1::CloneOccurrences | FinalizationSectionV1::CloneExactPostings | FinalizationSectionV1::CloneBodyPayloads - | FinalizationSectionV1::CloneFingerprintCounts | FinalizationSectionV1::CloneFingerprintPostings, None, ) => advance_native_section_rows( transaction, section, - section.seek_query(layout, false), + section.walked_query(false)?, params![limit], state, control, ), ( FinalizationSectionV1::CloneOccurrences | FinalizationSectionV1::CloneBodyPayloads, - Some(PersistedFinalizationKeyV1::Text(value)), + Some(PersistedFinalizationKeyV1::Integer(value)), ) => advance_native_section_rows( transaction, section, - section.seek_query(layout, true), + section.walked_query(true)?, params![value, limit], state, control, @@ -5710,61 +5622,35 @@ fn advance_clone_section_rows( class, normalization_revision, digest, - symbol_occurrence_id, + occurrence_ordinal, }), ) => advance_native_section_rows( transaction, section, - section.seek_query(layout, true), + section.walked_query(true)?, params![ class, normalization_revision, digest, - symbol_occurrence_id, + occurrence_ordinal, limit ], state, control, ), - ( - FinalizationSectionV1::CloneFingerprintCounts, - Some(PersistedFinalizationKeyV1::FingerprintCount { - language, - class, - normalization_revision, - fingerprint, - }), - ) => advance_native_section_rows( - transaction, - section, - section.seek_query(layout, true), - params![language, class, normalization_revision, fingerprint, limit], - state, - control, - ), ( FinalizationSectionV1::CloneFingerprintPostings, - Some(PersistedFinalizationKeyV1::FingerprintPosting { + Some(PersistedFinalizationKeyV1::Fingerprint { language, class, normalization_revision, fingerprint, - symbol_occurrence_id, - token_position, }), ) => advance_native_section_rows( transaction, section, - section.seek_query(layout, true), - params![ - language, - class, - normalization_revision, - fingerprint, - symbol_occurrence_id, - token_position, - limit - ], + section.walked_query(true)?, + params![language, class, normalization_revision, fingerprint, limit], state, control, ), @@ -5807,7 +5693,7 @@ fn advance_native_section_rows( "lexical artifact base-section receipt has a negative page".to_owned(), ) })?; - let receipt: Vec = row.get(9).map_err(sqlite_error)?; + let receipt: Vec = row.get(6).map_err(sqlite_error)?; absorb_page_base_sections_receipt( page_ordinal, &receipt, @@ -5851,55 +5737,32 @@ fn native_row_key( FinalizationSectionV1::CloneOccurrences | FinalizationSectionV1::CloneExactPostings | FinalizationSectionV1::CloneBodyPayloads - | FinalizationSectionV1::CloneFingerprintCounts | FinalizationSectionV1::CloneFingerprintPostings ) { return clone_row_key(section, row); } match section { - FinalizationSectionV1::SourcePages - | FinalizationSectionV1::DocumentIntegrity - | FinalizationSectionV1::Rows => Ok(PersistedFinalizationKeyV1::Integer( + FinalizationSectionV1::SourcePages | FinalizationSectionV1::FieldStatistics => Ok( + PersistedFinalizationKeyV1::Integer(row.get(0).map_err(sqlite_error)?), + ), + FinalizationSectionV1::Vocabulary => Ok(PersistedFinalizationKeyV1::Text( row.get(0).map_err(sqlite_error)?, )), - FinalizationSectionV1::ImportIntegrity | FinalizationSectionV1::ImportEvidence => Ok( - PersistedFinalizationKeyV1::Blob(row.get(0).map_err(sqlite_error)?), - ), - FinalizationSectionV1::TermPostings => { - Ok(PersistedFinalizationKeyV1::IntegerIntegerInteger { - page_ordinal: row.get(0).map_err(sqlite_error)?, - kind: row.get(1).map_err(sqlite_error)?, - ngram: row.get(2).map_err(sqlite_error)?, - }) - } - FinalizationSectionV1::ExactPostings => Ok(PersistedFinalizationKeyV1::TextBlobInteger { - field: row.get(0).map_err(sqlite_error)?, - term: row.get(1).map_err(sqlite_error)?, - document_id: row.get(2).map_err(sqlite_error)?, - }), - FinalizationSectionV1::NgramPostings => { - Ok(PersistedFinalizationKeyV1::IntegerIntegerInteger { - page_ordinal: row.get(0).map_err(sqlite_error)?, - kind: row.get(1).map_err(sqlite_error)?, - ngram: row.get(2).map_err(sqlite_error)?, - }) - } + FinalizationSectionV1::DocumentIntegrity + | FinalizationSectionV1::ImportIntegrity + | FinalizationSectionV1::ImportEvidence + | FinalizationSectionV1::Rows + | FinalizationSectionV1::TermPostings + | FinalizationSectionV1::ExactPostings + | FinalizationSectionV1::NgramPostings => Err(base_section_walk_error()), FinalizationSectionV1::CloneOccurrences | FinalizationSectionV1::CloneExactPostings | FinalizationSectionV1::CloneBodyPayloads - | FinalizationSectionV1::CloneFingerprintCounts | FinalizationSectionV1::CloneFingerprintPostings => { Err(CodeLexicalArtifactErrorV1::Corrupt( "clone row key bypassed its dedicated decoder".to_owned(), )) } - FinalizationSectionV1::FieldStatistics | FinalizationSectionV1::Vocabulary => Ok( - PersistedFinalizationKeyV1::Integer(row.get(0).map_err(sqlite_error)?), - ), - FinalizationSectionV1::TermStatistics => Ok(PersistedFinalizationKeyV1::IntegerPair { - first: row.get(0).map_err(sqlite_error)?, - second: row.get(1).map_err(sqlite_error)?, - }), } } @@ -5911,7 +5774,7 @@ fn clone_row_key( section, FinalizationSectionV1::CloneOccurrences | FinalizationSectionV1::CloneBodyPayloads ) { - return Ok(PersistedFinalizationKeyV1::Text( + return Ok(PersistedFinalizationKeyV1::Integer( row.get(0).map_err(sqlite_error)?, )); } @@ -5920,7 +5783,7 @@ fn clone_row_key( class: row.get(0).map_err(sqlite_error)?, normalization_revision: row.get(1).map_err(sqlite_error)?, digest: row.get(2).map_err(sqlite_error)?, - symbol_occurrence_id: row.get(3).map_err(sqlite_error)?, + occurrence_ordinal: row.get(3).map_err(sqlite_error)?, }); } let language = row.get(0).map_err(sqlite_error)?; @@ -5928,22 +5791,12 @@ fn clone_row_key( let normalization_revision = row.get(2).map_err(sqlite_error)?; let fingerprint = row.get(3).map_err(sqlite_error)?; match section { - FinalizationSectionV1::CloneFingerprintCounts => { - Ok(PersistedFinalizationKeyV1::FingerprintCount { - language, - class, - normalization_revision, - fingerprint, - }) - } FinalizationSectionV1::CloneFingerprintPostings => { - Ok(PersistedFinalizationKeyV1::FingerprintPosting { + Ok(PersistedFinalizationKeyV1::Fingerprint { language, class, normalization_revision, fingerprint, - symbol_occurrence_id: row.get(4).map_err(sqlite_error)?, - token_position: row.get(5).map_err(sqlite_error)?, }) } _ => Err(CodeLexicalArtifactErrorV1::Corrupt( @@ -6037,10 +5890,10 @@ fn verify_staged_source_chain( connection: &Connection, source: &VerifiedSealedLexicalSourceReceiptV1, control: &dyn CodeIndexExecutionControlV1, -) -> Result<(), CodeLexicalArtifactErrorV1> { +) -> Result>, CodeLexicalArtifactErrorV1> { let mut statement = connection .prepare( - "SELECT page_ordinal, cumulative_digest, chunk_count, payload_bytes, import_count, import_payload_bytes, import_dictionary_digest, next_cursor FROM source_pages ORDER BY page_ordinal", + "SELECT p.page_ordinal, c.cumulative_digest, p.chunk_count, c.payload_bytes, p.import_count, p.import_payload_bytes, p.import_dictionary_digest, c.next_cursor FROM source_pages p LEFT JOIN source_page_cursors c ON c.page_ordinal = p.page_ordinal ORDER BY p.page_ordinal", ) .map_err(sqlite_error)?; let mut rows = statement.query([]).map_err(sqlite_error)?; @@ -6054,17 +5907,23 @@ fn verify_staged_source_chain( checkpoint(control)?; let ordinal = u64::try_from(row.get::<_, i64>(0).map_err(sqlite_error)?).map_err(contract_number)?; - let cumulative_digest: String = row.get(1).map_err(sqlite_error)?; + let (Some(cumulative_digest), Some(page_payload), Some(cursor_bytes)) = ( + row.get::<_, Option>(1).map_err(sqlite_error)?, + row.get::<_, Option>(3).map_err(sqlite_error)?, + row.get::<_, Option>>(7).map_err(sqlite_error)?, + ) else { + return Err(CodeLexicalArtifactErrorV1::Corrupt( + "lexical artifact source page has no staged cursor".to_owned(), + )); + }; let page_chunks = u64::try_from(row.get::<_, i64>(2).map_err(sqlite_error)?).map_err(contract_number)?; - let page_payload = - u64::try_from(row.get::<_, i64>(3).map_err(sqlite_error)?).map_err(contract_number)?; + let page_payload = u64::try_from(page_payload).map_err(contract_number)?; let page_imports = u64::try_from(row.get::<_, i64>(4).map_err(sqlite_error)?).map_err(contract_number)?; let page_import_payload = u64::try_from(row.get::<_, i64>(5).map_err(sqlite_error)?).map_err(contract_number)?; let import_digest: String = row.get(6).map_err(sqlite_error)?; - let cursor_bytes: Vec = row.get(7).map_err(sqlite_error)?; let cursor = decode_cursor(&cursor_bytes)?; chunks = chunks .checked_add(page_chunks) @@ -6099,7 +5958,7 @@ fn verify_staged_source_chain( source .verify_completion(terminal.as_ref()) .map_err(|error| CodeLexicalArtifactErrorV1::Corrupt(error.to_string()))?; - Ok(()) + terminal.as_ref().map(encode_cursor).transpose() } fn source_chain_overflow() -> CodeLexicalArtifactErrorV1 { @@ -6126,12 +5985,24 @@ fn verify_final_sections_against_source( sections: &[CodeLexicalArtifactSectionDigestV1], source: &VerifiedSealedLexicalSourceReceiptV1, ) -> Result<(), CodeLexicalArtifactErrorV1> { + // Receipts count admitted documents, bounded by the source's chunks; + // the freeze already matched them against the stored rows. + let admitted_documents = sections + .iter() + .find(|section| section.name == "rows") + .map(|section| section.row_count) + .filter(|rows| *rows <= source.total_chunks()) + .ok_or_else(|| { + CodeLexicalArtifactErrorV1::Corrupt( + "lexical artifact rows exceed the sealed source's chunks".to_owned(), + ) + })?; let expected = [ ("source_pages", source.page_count()), - ("document_integrity", source.total_chunks()), + ("document_integrity", admitted_documents), ("import_integrity", source.total_imports()), ("import_evidence", source.total_imports()), - ("rows", source.total_chunks()), + ("rows", admitted_documents), ]; for (name, expected_rows) in expected { let actual = sections @@ -6158,13 +6029,39 @@ type StoredSourcePageRowV1 = (String, String, i64, i64, i64, i64, String, Vec Result { - let tail: Option<(i64, String, String, Vec)> = connection - .query_row(PROGRESS_TAIL_QUERY, [], |row| { - Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)) - }) - .optional() - .map_err(sqlite_error)?; - let Some((page_ordinal, import_digest, cumulative_digest, cursor_bytes)) = tail else { + if read_receipt(connection)?.is_some() { + return Err(CodeLexicalArtifactErrorV1::Contract( + "a sealed lexical artifact keeps no source progress; publish it".to_owned(), + )); + } + let cursor_bytes = match load_finalization_state(connection)? { + Some(state) => state.terminal_cursor, + None => { + let tail: Option<(i64, Vec)> = connection + .query_row(PROGRESS_TAIL_QUERY, [], |row| { + Ok((row.get(0)?, row.get(1)?)) + }) + .optional() + .map_err(sqlite_error)?; + match tail { + Some((page_ordinal, cursor_bytes)) => { + let next_page_ordinal = u64::try_from(page_ordinal) + .map_err(contract_number)? + .checked_add(1); + if Some(decode_cursor(&cursor_bytes)?.next_page_ordinal()) != next_page_ordinal + { + return Err(CodeLexicalArtifactErrorV1::Corrupt( + "persisted lexical artifact progress disagrees with its exact source cursor" + .to_owned(), + )); + } + Some(cursor_bytes) + } + None => None, + } + } + }; + let Some(cursor_bytes) = cursor_bytes else { return Ok(CodeLexicalArtifactBuildProgressV1 { next_page_ordinal: 0, completed_chunks: 0, @@ -6177,22 +6074,7 @@ fn progress( }); }; let cursor = decode_cursor(&cursor_bytes)?; - let next_page_ordinal = u64::try_from(page_ordinal) - .map_err(contract_number)? - .checked_add(1) - .ok_or_else(|| { - CodeLexicalArtifactErrorV1::Corrupt( - "persisted lexical artifact page ordinal overflowed".to_owned(), - ) - })?; - if cursor.next_page_ordinal() != next_page_ordinal - || cursor.import_dictionary_digest().as_str() != import_digest - || cursor.cumulative_digest().as_str() != cumulative_digest - { - return Err(CodeLexicalArtifactErrorV1::Corrupt( - "persisted lexical artifact progress disagrees with its exact source cursor".to_owned(), - )); - } + let next_page_ordinal = cursor.next_page_ordinal(); Ok(CodeLexicalArtifactBuildProgressV1 { next_page_ordinal, completed_chunks: cursor.emitted_chunks(), @@ -6217,7 +6099,7 @@ fn cursor_before_page( })?; let bytes: Option> = connection .query_row( - "SELECT next_cursor FROM source_pages WHERE page_ordinal = ?1", + "SELECT next_cursor FROM source_page_cursors WHERE page_ordinal = ?1", [i64::try_from(previous).map_err(contract_number)?], |row| row.get(0), ) @@ -6232,7 +6114,7 @@ fn verify_replayed_page( ) -> Result<(), CodeLexicalArtifactErrorV1> { let stored: Option = connection .query_row( - "SELECT page_digest, cumulative_digest, chunk_count, payload_bytes, import_count, import_payload_bytes, import_dictionary_digest, next_cursor FROM source_pages WHERE page_ordinal = ?1", + "SELECT c.page_digest, c.cumulative_digest, p.chunk_count, c.payload_bytes, p.import_count, p.import_payload_bytes, p.import_dictionary_digest, c.next_cursor FROM source_pages p JOIN source_page_cursors c ON c.page_ordinal = p.page_ordinal WHERE p.page_ordinal = ?1", [i64::try_from(page.page_ordinal()).map_err(contract_number)?], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?, row.get(4)?, row.get(5)?, row.get(6)?, row.get(7)?)), ) @@ -6304,69 +6186,29 @@ fn encode_cursor( } fn decode_cursor( - bytes: &[u8], -) -> Result { - VerifiedSealedLexicalCursorV1::restore_persisted(bytes) - .map_err(|error| CodeLexicalArtifactErrorV1::Corrupt(error.to_string())) -} - -pub(super) fn compute_section_digests( - connection: &Connection, - control: &dyn CodeIndexExecutionControlV1, - layout: LexicalArtifactLayoutV1, -) -> Result, CodeLexicalArtifactErrorV1> { - let (source_pages, base_sections) = - digest_source_pages_and_base_receipts(connection, control, layout)?; - let mut sections = Vec::with_capacity(section_names(layout).len()); - sections.push(source_pages); - sections.extend(base_sections); - for section in [ - FinalizationSectionV1::FieldStatistics, - FinalizationSectionV1::TermStatistics, - FinalizationSectionV1::Vocabulary, - ] { - sections.push(digest_query(connection, section, control, layout)?); - } - if layout.has_clone_index() { - for section in [ - FinalizationSectionV1::CloneOccurrences, - FinalizationSectionV1::CloneExactPostings, - FinalizationSectionV1::CloneBodyPayloads, - ] { - sections.push(digest_query(connection, section, control, layout)?); - } - } - if layout.has_clone_fingerprints() { - for section in [ - FinalizationSectionV1::CloneFingerprintCounts, - FinalizationSectionV1::CloneFingerprintPostings, - ] { - sections.push(digest_query(connection, section, control, layout)?); - } - } - Ok(sections) + bytes: &[u8], +) -> Result { + VerifiedSealedLexicalCursorV1::restore_persisted(bytes) + .map_err(|error| CodeLexicalArtifactErrorV1::Corrupt(error.to_string())) } -pub(super) fn compute_clone_section_digests( +pub(super) fn compute_section_digests( connection: &Connection, control: &dyn CodeIndexExecutionControlV1, - layout: LexicalArtifactLayoutV1, ) -> Result, CodeLexicalArtifactErrorV1> { - let mut sections = [ + let (source_pages, base_sections) = digest_source_pages_and_base_receipts(connection, control)?; + let mut sections = Vec::with_capacity(SECTION_NAMES.len()); + sections.push(source_pages); + sections.extend(base_sections); + for section in [ + FinalizationSectionV1::FieldStatistics, + FinalizationSectionV1::Vocabulary, FinalizationSectionV1::CloneOccurrences, FinalizationSectionV1::CloneExactPostings, FinalizationSectionV1::CloneBodyPayloads, - ] - .into_iter() - .map(|section| digest_query(connection, section, control, layout)) - .collect::, _>>()?; - if layout.has_clone_fingerprints() { - for section in [ - FinalizationSectionV1::CloneFingerprintCounts, - FinalizationSectionV1::CloneFingerprintPostings, - ] { - sections.push(digest_query(connection, section, control, layout)?); - } + FinalizationSectionV1::CloneFingerprintPostings, + ] { + sections.push(digest_query(connection, section, control)?); } Ok(sections) } @@ -6374,7 +6216,6 @@ pub(super) fn compute_clone_section_digests( fn digest_source_pages_and_base_receipts( connection: &Connection, control: &dyn CodeIndexExecutionControlV1, - layout: LexicalArtifactLayoutV1, ) -> Result< ( CodeLexicalArtifactSectionDigestV1, @@ -6387,8 +6228,8 @@ fn digest_source_pages_and_base_receipts( let mut accumulator = initial_section_accumulator(section.name())?.to_vec(); let (mut base_row_counts, mut base_accumulators) = initial_base_section_receipt_fold()?; let mut statement = connection - .prepare(section.full_query(layout)) - .map_err(|error| map_section_digest_sql_error(layout, section.name(), error))?; + .prepare(section.full_query().ok_or_else(base_section_walk_error)?) + .map_err(|error| map_section_digest_sql_error(section.name(), error))?; let column_count = statement.column_count(); let mut rows = statement.query([]).map_err(sqlite_error)?; while let Some(row) = rows.next().map_err(sqlite_error)? { @@ -6399,7 +6240,7 @@ fn digest_source_pages_and_base_receipts( "lexical artifact base-section receipt has a negative page".to_owned(), ) })?; - let receipt: Vec = row.get(9).map_err(sqlite_error)?; + let receipt: Vec = row.get(6).map_err(sqlite_error)?; absorb_page_base_sections_receipt( page_ordinal, &receipt, @@ -6429,13 +6270,12 @@ fn digest_query( connection: &Connection, section: FinalizationSectionV1, control: &dyn CodeIndexExecutionControlV1, - layout: LexicalArtifactLayoutV1, ) -> Result { let mut row_count = 0u64; let mut accumulator = initial_section_accumulator(section.name())?.to_vec(); let mut statement = connection - .prepare(section.full_query(layout)) - .map_err(|error| map_section_digest_sql_error(layout, section.name(), error))?; + .prepare(section.full_query().ok_or_else(base_section_walk_error)?) + .map_err(|error| map_section_digest_sql_error(section.name(), error))?; let column_count = statement.column_count(); let mut rows = statement.query([]).map_err(sqlite_error)?; while let Some(row) = rows.next().map_err(sqlite_error)? { @@ -6459,15 +6299,13 @@ fn digest_query( } fn map_section_digest_sql_error( - layout: LexicalArtifactLayoutV1, section: &str, error: rusqlite::Error, ) -> CodeLexicalArtifactErrorV1 { let message = error.to_string(); if message.contains("no such column") || message.contains("no such table") { CodeLexicalArtifactErrorV1::Incompatible(format!( - "lexical artifact revision {} cannot digest {section}: {message}", - layout.revision() + "lexical artifact revision {CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V1} cannot digest {section}: {message}" )) } else { sqlite_error(error) @@ -6529,9 +6367,7 @@ fn verify_source_receipt( receipt: &VerifiedCodeLexicalArtifactV1, source: &VerifiedSealedLexicalSourceReceiptV1, ) -> Result<(), CodeLexicalArtifactErrorV1> { - if receipt.source_state_digest() != source.source_state_digest() - || receipt.source_cumulative_digest() != source.cumulative_digest() - || receipt.page_count() != source.page_count() + if receipt.page_count() != source.page_count() || receipt.total_chunks() != source.total_chunks() || receipt.total_payload_bytes() != source.total_payload_bytes() || receipt.total_imports() != source.total_imports() @@ -6742,40 +6578,19 @@ fn verify_finalized_artifact( checkpoint(control)?; require_integrity(connection, control)?; verify_source_receipt(receipt, source)?; - let progress = progress(connection)?; - source - .verify_completion(progress.next_cursor.as_ref()) - .map_err(|error| CodeLexicalArtifactErrorV1::Corrupt(error.to_string()))?; if receipt.metadata_digest() != expected_metadata_digest { return Err(CodeLexicalArtifactErrorV1::Corrupt( "finalized lexical artifact metadata digest changed".to_owned(), )); } - let sections = compute_section_digests( - connection, - control, - LexicalArtifactLayoutV1::from_revision(receipt.format_revision())?, - )?; + require_served_revision(receipt.format_revision())?; + let sections = compute_section_digests(connection, control)?; if sections != receipt.section_digests() { return Err(CodeLexicalArtifactErrorV1::Corrupt( "finalized lexical artifact section digests do not verify".to_owned(), )); } - let digest = artifact_digest( - receipt.metadata_digest(), - receipt.source_state_digest(), - receipt.source_format_revision(), - receipt.page_count(), - receipt.total_chunks(), - receipt.total_payload_bytes(), - receipt.total_imports(), - receipt.import_payload_bytes(), - receipt.import_dictionary_digest(), - receipt.source_cumulative_digest(), - §ions, - receipt.format_revision(), - )?; - if &digest != receipt.artifact_digest() { + if &receipt_artifact_digest(receipt, §ions)? != receipt.artifact_digest() { return Err(CodeLexicalArtifactErrorV1::Corrupt( "finalized lexical artifact content digest does not verify".to_owned(), )); @@ -6816,9 +6631,8 @@ fn require_integrity( #[cfg(test)] mod tests { - use super::super::format::encode_ngram_bitmap; + use super::super::format::decode_term_lists; use super::*; - use roaring::RoaringBitmap; use rusqlite::StatementStatus; use rusqlite::hooks::{AuthAction, Authorization}; use tracedecay_domain::{ @@ -6842,14 +6656,15 @@ mod tests { fn clone_payload_conflict_checks_cover_full_and_partial_batches() { let mut connection = Connection::open_in_memory().unwrap(); connection.execute_batch( - "CREATE TABLE clone_body_payloads(payload_digest TEXT PRIMARY KEY, payload BLOB NOT NULL)", + "CREATE TABLE clone_body_payloads(payload_digest BLOB PRIMARY KEY, payload BLOB NOT NULL)", ).unwrap(); let transaction = connection.transaction().unwrap(); let bodies: Vec<_> = (0..=PAYLOAD_DIGEST_CONFLICT_CHECK_CHUNK) .map(|index| PreparedCloneBodyV1 { - payload_digest: format!("payload.{index}"), + payload_digest: format!("sha256:{index:064x}"), payload: vec![1], - occurrence: Vec::new(), + eligibility: Vec::new(), + serialized_bytes: 1, symbol_occurrence_id: format!("symbol.{index}"), path: "src/lib.rs".to_owned(), body_start: 0, @@ -6862,7 +6677,10 @@ mod tests { transaction .execute( "INSERT INTO clone_body_payloads VALUES (?1, ?2)", - params![body.payload_digest, body.payload], + params![ + stored_digest_key(&body.payload_digest).unwrap(), + body.payload + ], ) .unwrap(); } @@ -6906,31 +6724,33 @@ mod tests { .expect("lexical retriever"), exact_score_domain: ScoreDomainId::new("score.exact.artifact.v1") .expect("score domain"), + clone_route: None, } } fn create_mutable_test_schema(connection: &Connection) -> BuilderMutationGuardV1 { let gate = register_builder_mutation_gate(connection).expect("register builder mutation gate"); - create_schema(connection, LexicalArtifactLayoutV1::V11).expect("create artifact schema"); + create_schema(connection).expect("create artifact schema"); BuilderMutationGuardV1::enter(&gate).expect("enter test builder mutation authority") } /// Fingerprint postings are appended to an arrival-ordered staging table /// and reach the keyed tree only through the sorted finalization pass: /// the staging table is gated like every other builder table, the pass - /// preserves every row in key order, and nothing of the staging table - /// survives it (so a finalized artifact verifies without it). + /// seals one list per fingerprint holding every posting in key order, and + /// nothing of the staging table survives it (so a finalized artifact + /// verifies without it). #[test] - fn fingerprint_postings_stage_in_arrival_order_and_seal_sorted_once() { + fn fingerprint_postings_stage_in_arrival_order_and_seal_one_list_per_fingerprint() { let mut connection = Connection::open_in_memory().expect("fingerprint database"); let gate = register_builder_mutation_gate(&connection).expect("register builder mutation gate"); - create_schema(&connection, LexicalArtifactLayoutV1::V16).expect("create v16 schema"); + create_schema(&connection).expect("create artifact schema"); verify_builder_mutation_gate_schema(&connection).expect("staging triggers present"); let refused = connection.execute( - "INSERT INTO clone_fingerprint_postings_pages(language, class, normalization_revision, fingerprint, symbol_occurrence_id, token_position, payload_digest, body_digest) VALUES ('rust', 1, 1, 9, 'symbol.b', 0, 'payload.b', 'body.b')", + "INSERT INTO clone_fingerprint_postings_pages(language, class, normalization_revision, fingerprint, occurrence_ordinal, token_position) VALUES ('rust', 1, 1, 9, 2, 0)", [], ); assert!( @@ -6942,18 +6762,19 @@ mod tests { // Arrival order deliberately disagrees with key order. let arrivals = [ - ("rust", 9_i64, "symbol.b", 3_i64), - ("rust", 2, "symbol.b", 1), - ("go", 5, "symbol.a", 0), - ("rust", 2, "symbol.a", 7), + ("rust", 9_i64, 2_i64, 3_i64), + ("rust", 2, 2, 1), + ("go", 5, 1, 0), + ("rust", 2, 1, 7), + ("rust", 2, 1, 4), ]; { let _authority = BuilderMutationGuardV1::enter(&gate).expect("enter authority"); - for (language, fingerprint, symbol, position) in arrivals { + for (language, fingerprint, occurrence, position) in arrivals { connection .execute( - "INSERT INTO clone_fingerprint_postings_pages(language, class, normalization_revision, fingerprint, symbol_occurrence_id, token_position, payload_digest, body_digest) VALUES (?1, 1, 1, ?2, ?3, ?4, 'payload', 'body')", - params![language, fingerprint, symbol, position], + "INSERT INTO clone_fingerprint_postings_pages(language, class, normalization_revision, fingerprint, occurrence_ordinal, token_position) VALUES (?1, 1, 1, ?2, ?3, ?4)", + params![language, fingerprint, occurrence, position], ) .expect("stage fingerprint posting"); } @@ -6968,7 +6789,8 @@ mod tests { assert_eq!(keyed_before, 0, "appends must not touch the keyed tree"); let transaction = connection.transaction().expect("finalization transaction"); - derive_clone_fingerprint_postings(&transaction, &gate).expect("sorted pass"); + derive_clone_fingerprint_postings(&transaction, &gate, &ActiveControl) + .expect("sorted pass"); transaction.commit().expect("commit sorted pass"); assert!( @@ -6979,7 +6801,7 @@ mod tests { .expect("finalized layout verifies without the staging table"); let mut statement = connection .prepare( - "SELECT language, fingerprint, symbol_occurrence_id, token_position FROM clone_fingerprint_postings ORDER BY language, class, normalization_revision, fingerprint, symbol_occurrence_id, token_position", + "SELECT language, fingerprint, posting_count, postings FROM clone_fingerprint_postings ORDER BY language, class, normalization_revision, fingerprint", ) .expect("prepare keyed read"); let sealed = statement @@ -6987,20 +6809,25 @@ mod tests { Ok(( row.get::<_, String>(0)?, row.get::<_, i64>(1)?, - row.get::<_, String>(2)?, - row.get::<_, i64>(3)?, + row.get::<_, i64>(2)?, + row.get::<_, Vec>(3)?, )) }) .expect("read keyed rows") .collect::, _>>() - .expect("collect keyed rows"); + .expect("collect keyed rows") + .into_iter() + .map(|(language, fingerprint, count, postings)| { + let postings = decode_fingerprint_postings(&postings).expect("canonical postings"); + (language, fingerprint, count, postings) + }) + .collect::>(); assert_eq!( sealed, vec![ - ("go".to_owned(), 5, "symbol.a".to_owned(), 0), - ("rust".to_owned(), 2, "symbol.a".to_owned(), 7), - ("rust".to_owned(), 2, "symbol.b".to_owned(), 1), - ("rust".to_owned(), 9, "symbol.b".to_owned(), 3), + ("go".to_owned(), 5, 1, vec![(1, 0)]), + ("rust".to_owned(), 2, 3, vec![(1, 4), (1, 7), (2, 1)]), + ("rust".to_owned(), 9, 1, vec![(2, 3)]), ] ); } @@ -7020,30 +6847,20 @@ mod tests { for batch in batches { let transaction = connection.transaction().expect("batch transaction"); let mut totals = BTreeMap::new(); - for (term_id, field, document_id, frequency) in batch { - transaction - .execute( - "INSERT INTO term_postings(term_id, field, document_id, frequency) VALUES (?1, ?2, ?3, ?4)", - [term_id, field, document_id, frequency], - ) - .expect("posting fixture"); + for (_, field, _, frequency) in batch { *totals.entry(*field).or_insert(0) += frequency; } stage_field_totals(&transaction, &totals).expect("stage field totals"); transaction.commit().expect("commit batch"); } } - let expected = connection - .prepare( - "SELECT field, SUM(frequency) FROM term_postings GROUP BY field ORDER BY field", - ) - .expect("reference sums") - .query_map([], |row| Ok((row.get::<_, i64>(0)?, row.get::<_, i64>(1)?))) - .expect("reference rows") - .collect::>>() - .expect("reference totals"); + let expected: Vec<(i64, i64)> = if empty { + Vec::new() + } else { + vec![(1, 2 + 3 + 4), (3, 7), (7, 11)] + }; let transaction = connection.transaction().expect("statistics transaction"); - derive_statistics_step(&transaction, 0).expect("derive statistics"); + derive_statistics_step(&transaction, 0, &ActiveControl).expect("derive statistics"); let actual = transaction .prepare("SELECT field, total_length FROM field_stats ORDER BY field") .expect("derived sums") @@ -7124,42 +6941,6 @@ mod tests { assert!(matches!(error, CodeLexicalArtifactErrorV1::Incompatible(_))); } - #[test] - fn fuzzy_vocabulary_requires_a_non_subtoken_posting() { - let mut connection = Connection::open_in_memory().expect("vocabulary database"); - let _authority = create_mutable_test_schema(&connection); - connection - .execute_batch( - "INSERT INTO vocabulary(term_id, term, in_fuzzy) VALUES - (1, 'subtoken_only', 0), (2, 'body_only', 0), - (3, 'both_fields', 0), (4, 'unreferenced', 0); - INSERT INTO term_postings(term_id, field, document_id, frequency) VALUES - (1, 7, 1, 3), (1, 7, 2, 5), - (2, 4, 1, 2), (2, 4, 2, 7), - (3, 7, 1, 1), (3, 4, 2, 1);", - ) - .expect("vocabulary fixture"); - let transaction = connection.transaction().expect("statistics transaction"); - derive_statistics_step(&transaction, 1).expect("derive term statistics"); - derive_statistics_step(&transaction, 2).expect("derive fuzzy vocabulary"); - let actual = transaction - .prepare("SELECT term_id, in_fuzzy FROM vocabulary ORDER BY term_id") - .expect("fuzzy membership") - .query_map([], |row| { - Ok((row.get::<_, i64>(0)?, row.get::<_, bool>(1)?)) - }) - .expect("vocabulary rows") - .collect::>>() - .expect("vocabulary membership"); - assert_eq!(actual, [(1, false), (2, true), (3, true), (4, false)]); - assert!( - transaction - .execute("UPDATE vocabulary SET in_fuzzy = 0 WHERE term_id = 2", []) - .is_err(), - "derived membership must remain frozen" - ); - } - #[test] fn canonical_write_limit_refuses_ngram_receipt_past_the_exact_boundary() { let ngram_receipt_bytes = "sha256:".len() + 64; @@ -7195,8 +6976,15 @@ mod tests { connection .authorizer(Some( |context: rusqlite::hooks::AuthContext<'_>| match context.action { + // The vocabulary is the sealed terms and their fuzzy + // flags; their posting lists stay unread. + AuthAction::Read { + table_name: "term_postings", + column_name, + } if column_name != "lists" => Authorization::Allow, AuthAction::Read { table_name, .. } - if BASE_SECTION_NAMES.contains(&table_name) => + if BASE_SECTION_NAMES.contains(&table_name) + || table_name.ends_with("_runs") => { Authorization::Deny } @@ -7205,43 +6993,14 @@ mod tests { )) .expect("deny exhaustive base-table verification reads"); - let sections = - compute_section_digests(&connection, &ActiveControl, LexicalArtifactLayoutV1::V11) - .expect("verify only source-page receipts and derived sections"); + let sections = compute_section_digests(&connection, &ActiveControl) + .expect("verify only source-page receipts and derived sections"); assert_eq!( sections .iter() .map(|section| section.name.as_str()) .collect::>(), - section_names(LexicalArtifactLayoutV1::V11) - ); - } - - #[test] - fn section_digest_sql_dispatches_on_recorded_layout() { - assert!( - !FinalizationSectionV1::Vocabulary - .full_query(LexicalArtifactLayoutV1::V10) - .contains("term_id") - ); - assert!( - FinalizationSectionV1::Vocabulary - .full_query(LexicalArtifactLayoutV1::V11) - .contains("term_id") - ); - assert!( - !FinalizationSectionV1::TermStatistics - .full_query(LexicalArtifactLayoutV1::V10) - .contains("term_id") - ); - assert!( - FinalizationSectionV1::TermStatistics - .full_query(LexicalArtifactLayoutV1::V11) - .contains("term_id") - ); - assert_eq!( - FinalizationSectionV1::Vocabulary.full_query(LexicalArtifactLayoutV1::V10), - "SELECT term FROM vocabulary ORDER BY term" + SECTION_NAMES ); } @@ -7271,52 +7030,56 @@ mod tests { } #[test] - fn ngram_staging_key_preserves_source_page_order_without_a_serving_index() { + fn posting_staging_keeps_source_page_order_without_a_serving_index() { let connection = Connection::open_in_memory().expect("open artifact database"); let _mutation_authority = create_mutable_test_schema(&connection); - for (page_ordinal, ngram, document) in - [(0i64, 90i64, 0u32), (0, 100, 0), (1, 10, 1), (1, 20, 1)] - { - let bitmap = RoaringBitmap::from_iter([document]); - let encoded = encode_ngram_bitmap(LexicalArtifactLayoutV1::V12, &bitmap) - .expect("encode ngram bitmap"); + for (page_ordinal, term) in [(0i64, "omega"), (0, "zeta"), (1, "alpha"), (1, "beta")] { connection .execute( - "INSERT INTO ngram_postings(page_ordinal, kind, ngram, documents, cardinality) VALUES (?1, 1, ?2, ?3, 1)", - params![page_ordinal, ngram, encoded], + "INSERT INTO term_posting_runs(page_ordinal, term, field, postings) VALUES (?1, ?2, 4, X'02')", + params![page_ordinal, term], ) - .expect("seed page-ordered ngram posting"); + .expect("seed page-ordered term run"); } - - let serving_indexes: i64 = connection - .query_row( - "SELECT COUNT(*) FROM pragma_index_list('ngram_postings') WHERE name = 'ngram_postings_by_ngram'", - [], - |row| row.get(0), - ) - .expect("inspect staging indexes"); - assert_eq!( - serving_indexes, 0, - "serving-key maintenance must remain absent during catch-up" + assert!( + !table_exists(&connection, "ngram_posting_pages").expect("table probe"), + "n-gram lists are rebuilt from rows and never staged" ); + for table in ["term_posting_runs", "exact_posting_runs", "row_chunk_pages"] { + let secondary_indexes: i64 = connection + .query_row( + "SELECT COUNT(*) FROM pragma_index_list(?1) WHERE origin != 'pk'", + [table], + |row| row.get(0), + ) + .expect("inspect staging indexes"); + assert_eq!( + secondary_indexes, 0, + "{table}: serving-key maintenance must remain absent during catch-up" + ); + } let mut statement = connection .prepare( - "SELECT page_ordinal, kind, ngram FROM ngram_postings ORDER BY page_ordinal, kind, ngram", + "SELECT page_ordinal, term FROM term_posting_runs ORDER BY page_ordinal, term, field", ) .expect("prepare staging-order scan"); let rows = statement .query_map([], |row| { - Ok(( - row.get::<_, i64>(0)?, - row.get::<_, i64>(1)?, - row.get::<_, i64>(2)?, - )) + Ok((row.get::<_, i64>(0)?, row.get::<_, String>(1)?)) }) .expect("scan staging order") .collect::, _>>() .expect("collect staging order"); - assert_eq!(rows, [(0, 1, 90), (0, 1, 100), (1, 1, 10), (1, 1, 20)]); + assert_eq!( + rows, + [ + (0, "omega".to_owned()), + (0, "zeta".to_owned()), + (1, "alpha".to_owned()), + (1, "beta".to_owned()) + ] + ); assert_eq!( statement.get_status(StatementStatus::Sort), 0, @@ -7324,86 +7087,208 @@ mod tests { ); } + /// A pass that outgrows its memory sheds its highest keys and a later + /// pass rebuilds exactly them: the union of every pass equals one + /// unbounded pass, list for list, in key order. #[test] - fn deferred_ngram_serving_index_is_unique_and_query_selective() { - let mut connection = Connection::open_in_memory().expect("open artifact database"); - let _mutation_authority = create_mutable_test_schema(&connection); - for (page_ordinal, ngram, documents) in [ - (0i64, 10i64, vec![1u32, 2]), - (0, 20, vec![1]), - (1, 10, vec![3]), - ] { - let bitmap = RoaringBitmap::from_iter(documents); - let encoded = encode_ngram_bitmap(LexicalArtifactLayoutV1::V12, &bitmap) - .expect("encode ngram bitmap"); - connection - .execute( - "INSERT INTO ngram_postings(page_ordinal, kind, ngram, documents, cardinality) VALUES (?1, 1, ?2, ?3, ?4)", - params![page_ordinal, ngram, encoded, bitmap.len() as i64], - ) - .expect("seed ngram posting"); - } - - let transaction = connection - .transaction() - .expect("start serving-index transaction"); - build_serving_index_step(&transaction, 3, LexicalArtifactLayoutV1::V11) - .expect("build ngram serving index"); - let statistics_plan = transaction - .prepare( - "EXPLAIN QUERY PLAN - INSERT INTO ngram_statistics(kind, ngram, document_frequency) - SELECT kind, ngram, SUM(cardinality) - FROM ngram_postings INDEXED BY ngram_postings_by_ngram - GROUP BY kind, ngram", - ) - .expect("prepare ngram statistics plan") - .query_map([], |row| row.get::<_, String>(3)) - .expect("query ngram statistics plan") - .collect::, _>>() - .expect("collect ngram statistics plan"); + fn bounded_ngram_passes_rebuild_exactly_the_unbounded_lists() { + let postings = (0u32..600) + .flat_map(|document| { + (0i64..40) + .filter(move |ngram| (document as i64 + ngram) % 3 != 0) + .map(move |ngram| ((ngram % 2, ngram), document)) + }) + .collect::>(); + let run = |memory_bytes: usize| { + let mut sealed = Vec::new(); + let mut passes = 0; + let mut lower = None; + loop { + passes += 1; + let mut pass = NgramListPassV1::new(lower, memory_bytes); + for (key, document) in &postings { + pass.add(*key, *document).expect("ascending documents"); + } + let (lists, cutoff) = pass.finish(); + sealed.extend( + lists + .into_iter() + .map(|(key, list)| (key, list.finish().expect("non-empty list"))), + ); + match cutoff { + Some(cutoff) => lower = Some(cutoff), + None => break, + } + } + (sealed, passes) + }; + let (unbounded, single) = run(usize::MAX); + let (bounded, passes) = run(4 * 1024); + assert_eq!(single, 1); assert!( - statistics_plan - .iter() - .any(|detail| detail.contains("USING COVERING INDEX ngram_postings_by_ngram")), - "n-gram statistics must stream the covering serving index, got {statistics_plan:?}" + passes > 2, + "the bounded rebuild must actually spill: {passes} passes" ); + assert_eq!(bounded, unbounded); assert!( - statistics_plan - .iter() - .all(|detail| !detail.contains("USE TEMP B-TREE")), - "n-gram statistics must not perform another grouped sort, got {statistics_plan:?}" + unbounded.windows(2).all(|pair| pair[0].0 < pair[1].0), + "sealed lists arrive in key order" ); - build_serving_index_step(&transaction, 4, LexicalArtifactLayoutV1::V11) - .expect("build ngram serving statistics"); - transaction.commit().expect("commit ngram serving index"); + } + + fn encoded_postings(frequencies: bool, postings: &[(u32, u32)]) -> Vec { + let mut encoder = PostingListEncoderV1::new(frequencies); + for (document, frequency) in postings { + encoder + .push(*document, *frequency) + .expect("ascending posting"); + } + encoder.finish().expect("non-empty posting list") + } + + fn decoded_postings(frequencies: bool, encoded: &[u8]) -> Vec<(u32, u32)> { + PostingListDecoderV1::new(encoded, frequencies) + .collect::, _>>() + .expect("canonical posting list") + } + + /// Finalization concatenates each key's page-ordered staging runs into + /// exactly one sealed list, drops the staging tables, freezes the + /// sealed tables, and serves every read from the clustered key alone. + #[test] + fn posting_merges_seal_one_list_per_serving_key() { + let mut connection = Connection::open_in_memory().expect("open artifact database"); + let gate = + register_builder_mutation_gate(&connection).expect("register builder mutation gate"); + create_schema(&connection).expect("create artifact schema"); + { + let _authority = BuilderMutationGuardV1::enter(&gate).expect("enter authority"); + for (page_ordinal, term, field, postings) in [ + (0i64, "render", 4i64, vec![(1u32, 1u32), (2, 3)]), + (0, "render", 7, vec![(4, 1)]), + (0, "widget", 7, vec![(2, 1)]), + (2, "render", 4, vec![(7, 2)]), + ] { + connection + .execute( + "INSERT INTO term_posting_runs(page_ordinal, term, field, postings) VALUES (?1, ?2, ?3, ?4)", + params![page_ordinal, term, field, encoded_postings(true, &postings)], + ) + .expect("seed term run"); + } + for (page_ordinal, documents) in [(0i64, vec![(1u32, 1u32)]), (2, vec![(7, 1)])] { + connection + .execute( + "INSERT INTO exact_posting_runs(page_ordinal, term_id, field, documents) VALUES (?1, 9, 1, ?2)", + params![page_ordinal, encoded_postings(false, &documents)], + ) + .expect("seed exact run"); + } + } + + let transaction = connection.transaction().expect("merge transaction"); + let generation = test_metadata().generation; + let authority = ServingIndexStepAuthorityV1 { + mutation_gate: &gate, + generation: &generation, + ngram_memory_bytes: 1024 * 1024, + }; + for ordinal in [1, 2, 3] { + build_serving_index_step(&transaction, ordinal, &authority, &ActiveControl) + .expect("merge posting family"); + } + derive_statistics_step(&transaction, 1, &ActiveControl).expect("release staging pages"); + transaction.commit().expect("commit merges"); - let unique: i64 = connection + for table in ["term_posting_runs", "exact_posting_runs"] { + assert!( + !table_exists(&connection, table).expect("table probe"), + "{table} must not survive finalization" + ); + } + verify_builder_mutation_gate_schema(&connection) + .expect("sealed layout verifies without the staging tables"); + let free_pages: i64 = connection + .query_row("PRAGMA freelist_count", [], |row| row.get(0)) + .expect("read freelist"); + assert_eq!( + free_pages, 0, + "every page the dropped staging tables held leaves the file" + ); + let ngram_lists: i64 = connection + .query_row("SELECT COUNT(*) FROM ngram_postings", [], |row| row.get(0)) + .expect("count ngram lists"); + assert_eq!(ngram_lists, 0, "a corpus without rows seals no n-gram list"); + let terms = connection + .prepare("SELECT term, in_fuzzy, lists FROM term_postings ORDER BY term") + .expect("prepare term read") + .query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, bool>(1)?, + row.get::<_, Vec>(2)?, + )) + }) + .expect("read term lists") + .collect::, _>>() + .expect("collect term lists") + .into_iter() + .map(|(term, in_fuzzy, lists)| { + let lists = decode_term_lists(&lists) + .expect("canonical term lists") + .into_iter() + .map(|(field, frequency, postings)| { + (field, frequency, decoded_postings(true, postings)) + }) + .collect::>(); + (term, in_fuzzy, lists) + }) + .collect::>(); + assert_eq!( + terms, + [ + ( + "render".to_owned(), + true, + vec![(4, 3, vec![(1, 1), (2, 3), (7, 2)]), (7, 1, vec![(4, 1)])] + ), + // A term seen only as a subtoken is not fuzzy-eligible. + ("widget".to_owned(), false, vec![(7, 1, vec![(2, 1)])]), + ] + ); + let exact: Vec = connection .query_row( - "SELECT [unique] FROM pragma_index_list('ngram_postings') WHERE name = 'ngram_postings_by_ngram'", + "SELECT documents FROM exact_postings WHERE term_id = 9 AND field = 1", [], |row| row.get(0), ) - .expect("inspect ngram serving index"); - assert_eq!( - unique, 1, - "the serving index must preserve posting identity" + .expect("read exact list"); + assert_eq!(decoded_postings(false, &exact), [(1, 1), (7, 1)]); + + for table in ["term_postings", "exact_postings", "ngram_postings"] { + let secondary_indexes: i64 = connection + .query_row( + "SELECT COUNT(*) FROM pragma_index_list(?1) WHERE origin != 'pk'", + [table], + |row| row.get(0), + ) + .expect("inspect sealed indexes"); + assert_eq!(secondary_indexes, 0, "{table} carries no duplicate index"); + } + let _authority = BuilderMutationGuardV1::enter(&gate).expect("enter authority"); + assert!( + connection + .execute( + "INSERT INTO term_postings(term, in_fuzzy, lists) VALUES ('late', 1, X'040102')", + [], + ) + .is_err(), + "sealed term postings must be frozen even under builder authority" ); - let columns = connection + let plan = connection .prepare( - "SELECT name FROM pragma_index_xinfo('ngram_postings_by_ngram') WHERE key = 1 ORDER BY seqno", + "EXPLAIN QUERY PLAN SELECT documents, document_frequency FROM ngram_postings WHERE kind = ?1 AND ngram = ?2", ) - .expect("prepare serving-index columns") - .query_map([], |row| row.get::<_, String>(0)) - .expect("query serving-index columns") - .collect::, _>>() - .expect("collect serving-index columns"); - assert_eq!(columns, ["kind", "ngram", "page_ordinal", "cardinality"]); - - let query = "SELECT documents, cardinality FROM ngram_postings \ - WHERE kind = ?1 AND ngram = ?2 ORDER BY page_ordinal"; - let plan = connection - .prepare(&format!("EXPLAIN QUERY PLAN {query}")) .expect("prepare ngram serving plan") .query_map(params![1i64, 10i64], |row| row.get::<_, String>(3)) .expect("query ngram serving plan") @@ -7411,25 +7296,48 @@ mod tests { .expect("collect ngram serving plan"); assert!( plan.iter() - .any(|detail| detail.contains("USING INDEX ngram_postings_by_ngram")), - "phrase candidates must use the deferred serving index, got {plan:?}" + .any(|detail| detail.contains("USING PRIMARY KEY")), + "phrase candidates must seek the clustered key, got {plan:?}" ); - let shard_cardinalities = connection - .prepare(query) - .expect("prepare ngram serving query") - .query_map(params![1i64, 10i64], |row| row.get::<_, i64>(1)) - .expect("query ngram candidates") - .collect::, _>>() - .expect("collect ngram candidates"); - assert_eq!(shard_cardinalities, [2, 1]); - let document_frequency: i64 = connection - .query_row( - "SELECT document_frequency FROM ngram_statistics WHERE kind = 1 AND ngram = 10", - [], - |row| row.get(0), - ) - .expect("query finalized ngram statistics"); - assert_eq!(document_frequency, 3); + } + + #[test] + fn posting_merges_refuse_overlapping_staging() { + for (runs, message) in [ + ( + [(0i64, vec![(1u32, 1u32), (4, 1)]), (1, vec![(3, 1)])], + "overlap", + ), + ([(0, vec![(1, 1)]), (1, vec![(1, 1)])], "overlap"), + ] { + let mut connection = Connection::open_in_memory().expect("open artifact database"); + let gate = register_builder_mutation_gate(&connection).expect("register gate"); + create_schema(&connection).expect("create schema"); + { + let _authority = BuilderMutationGuardV1::enter(&gate).expect("enter authority"); + for (page_ordinal, postings) in &runs { + connection + .execute( + "INSERT INTO term_posting_runs(page_ordinal, term, field, postings) VALUES (?1, 'render', 4, ?2)", + params![page_ordinal, encoded_postings(true, postings)], + ) + .expect("seed term run"); + } + } + let transaction = connection.transaction().expect("merge transaction"); + let generation = test_metadata().generation; + let authority = ServingIndexStepAuthorityV1 { + mutation_gate: &gate, + generation: &generation, + ngram_memory_bytes: 1024 * 1024, + }; + let error = build_serving_index_step(&transaction, 1, &authority, &ActiveControl) + .expect_err("out-of-order staging must fail closed"); + assert!( + matches!(&error, CodeLexicalArtifactErrorV1::Corrupt(detail) if detail.contains(message)), + "unexpected error: {error:?}" + ); + } } #[test] @@ -7474,10 +7382,8 @@ mod tests { let connection = Connection::open_in_memory().expect("open progress database"); connection .execute_batch( - "CREATE TABLE source_pages ( + "CREATE TABLE source_page_cursors ( page_ordinal INTEGER PRIMARY KEY, - import_dictionary_digest TEXT NOT NULL, - cumulative_digest TEXT NOT NULL, next_cursor BLOB NOT NULL );", ) @@ -7485,7 +7391,7 @@ mod tests { for page in 0..4_096i64 { connection .execute( - "INSERT INTO source_pages(page_ordinal, import_dictionary_digest, cumulative_digest, next_cursor) VALUES (?1, 'imports', 'cumulative', X'00')", + "INSERT INTO source_page_cursors(page_ordinal, next_cursor) VALUES (?1, X'00')", [page], ) .expect("seed persisted progress"); @@ -7511,155 +7417,26 @@ mod tests { ); } - #[test] - fn one_document_integrity_row_does_not_visit_unrelated_relational_rows() { - let mut connection = Connection::open_in_memory().expect("open artifact database"); - let _mutation_authority = create_mutable_test_schema(&connection); - let transaction = connection.transaction().expect("start seed transaction"); - for document_id in 0..2_048i64 { - let term = format!("term-{document_id:04}"); - transaction - .execute( - "INSERT INTO vocabulary(term_id, term, in_fuzzy) VALUES (?1, ?2, 0)", - params![document_id + 1, term], - ) - .expect("seed vocabulary term"); - transaction - .execute( - "INSERT INTO term_postings(term_id, field, document_id, frequency) VALUES (?1, 4, ?2, 1)", - params![document_id + 1, document_id], - ) - .expect("seed term posting"); - transaction - .execute( - "INSERT INTO exact_postings(field, term, document_id) VALUES ('symbol', ?1, ?2)", - params![document_id.to_le_bytes().as_slice(), document_id], - ) - .expect("seed exact posting"); - } - transaction.commit().expect("commit seed transaction"); - let transaction = connection - .transaction() - .expect("start index-build transaction"); - for ordinal in [1, 2] { - build_serving_index_step(&transaction, ordinal, LexicalArtifactLayoutV1::V11) - .expect("build document integrity index"); - } - transaction.commit().expect("commit integrity index"); - - for query in [ - "SELECT field, term_id, frequency FROM term_postings INDEXED BY term_postings_by_document WHERE document_id = ?1 ORDER BY term_id, field", - "SELECT field, term FROM exact_postings WHERE document_id = ?1 ORDER BY field, term", - ] { - let mut statement = connection.prepare(query).expect("prepare integrity query"); - { - let mut rows = statement.query([1_024i64]).expect("query one document"); - assert!(rows.next().expect("read matching row").is_some()); - assert!(rows.next().expect("finish matching rows").is_none()); - } - assert_eq!( - statement.get_status(StatementStatus::FullscanStep), - 0, - "one document must not visit unrelated generation rows" - ); - assert_eq!( - statement.get_status(StatementStatus::Sort), - 0, - "one document must not sort unrelated generation rows" - ); - } - } - + /// Finalization adopts the base sections from their page receipts, so a + /// resumed digest wake walks only the source pages and the derived + /// statistics natively. #[test] fn bounded_finalization_resume_seeks_each_native_section_index() { let connection = Connection::open_in_memory().expect("open artifact database"); let _mutation_authority = create_mutable_test_schema(&connection); connection - .execute( - "INSERT INTO source_pages(page_ordinal, page_digest, cumulative_digest, chunk_count, payload_bytes, import_count, import_payload_bytes, import_dictionary_digest, ngram_digest, base_sections_receipt, next_cursor) VALUES (0, 'page', 'cumulative', 1, 1, 1, 1, 'imports', 'ngrams', X'00', X'00')", - [], - ) - .expect("seed source page"); - connection - .execute( - "INSERT INTO document_integrity(document_id, chunk_id, digest) VALUES (0, 'chunk', 'document')", - [], - ) - .expect("seed document integrity"); - connection - .execute( - "INSERT INTO import_integrity(canonical, digest) VALUES (X'01', 'import')", - [], - ) - .expect("seed import integrity"); - connection - .execute( - "INSERT INTO import_evidence(canonical, evidence) VALUES (X'01', X'01')", - [], - ) - .expect("seed import evidence"); - connection - .execute( - "INSERT INTO rows(document_id, chunk_id, row) VALUES (0, 'chunk', X'00')", - [], - ) - .expect("seed row"); - connection - .execute( - "INSERT INTO term_postings(term_id, field, document_id, frequency) VALUES (1, 1, 0, 1)", - [], - ) - .expect("seed term posting"); - connection - .execute( - "INSERT INTO exact_postings(field, term, document_id) VALUES ('field', X'01', 0)", - [], - ) - .expect("seed exact posting"); - let encoded = - encode_ngram_bitmap(LexicalArtifactLayoutV1::V12, &RoaringBitmap::from_iter([0])) - .expect("encode ngram bitmap"); - connection - .execute( - "INSERT INTO ngram_postings(page_ordinal, kind, ngram, documents, cardinality) VALUES (0, 1, 1, ?1, 1)", - [encoded], - ) - .expect("seed ngram posting"); - connection - .execute( - "INSERT INTO field_stats(field, total_length) VALUES (1, 1)", - [], - ) - .expect("seed field statistic"); - connection - .execute( - "INSERT INTO term_stats(term_id, field, document_frequency) VALUES (1, 1, 1)", - [], - ) - .expect("seed term statistic"); - connection - .execute( - "INSERT INTO vocabulary(term_id, term, in_fuzzy) VALUES (1, 'term', 1)", - [], - ) - .expect("seed vocabulary"); - let transaction = connection - .unchecked_transaction() - .expect("start ngram serving-index transaction"); - build_serving_index_step(&transaction, 3, LexicalArtifactLayoutV1::V11) - .expect("build ngram serving index before digest verification"); - transaction.commit().expect("commit ngram serving index"); - - for section in FinalizationSectionV1::ALL.into_iter().filter(|section| { - !matches!( - section, - FinalizationSectionV1::CloneOccurrences - | FinalizationSectionV1::CloneExactPostings - | FinalizationSectionV1::CloneBodyPayloads - | FinalizationSectionV1::CloneFingerprintCounts - | FinalizationSectionV1::CloneFingerprintPostings + .execute_batch( + "INSERT INTO source_pages(page_ordinal, chunk_count, import_count, import_payload_bytes, import_dictionary_digest, ngram_digest, base_sections_receipt) VALUES (0, 1, 1, 1, 'imports', 'ngrams', X'00'); + INSERT INTO field_stats(field, total_length) VALUES (1, 1); + INSERT INTO term_postings(term, in_fuzzy, lists) VALUES ('term', 1, X'01010102');", ) - }) { + .expect("seed natively walked sections"); + + for section in [ + FinalizationSectionV1::SourcePages, + FinalizationSectionV1::FieldStatistics, + FinalizationSectionV1::Vocabulary, + ] { let plan = explain_native_seek_plan(&connection, section) .expect("explain bounded finalization resume query"); assert!( @@ -7681,41 +7458,14 @@ mod tests { connection: &Connection, section: FinalizationSectionV1, ) -> Result, CodeLexicalArtifactErrorV1> { - let query = format!( - "EXPLAIN QUERY PLAN {}", - section.seek_query(LexicalArtifactLayoutV1::V11, true) - ); + let query = format!("EXPLAIN QUERY PLAN {}", section.walked_query(true)?); let mut statement = connection.prepare(&query).map_err(sqlite_error)?; let mut rows = match section { - FinalizationSectionV1::SourcePages - | FinalizationSectionV1::DocumentIntegrity - | FinalizationSectionV1::Rows => statement.query(params![0i64, 1i64]), - FinalizationSectionV1::ImportIntegrity | FinalizationSectionV1::ImportEvidence => { - statement.query(params![vec![0u8], 1i64]) - } - FinalizationSectionV1::TermPostings => statement.query(params![0i64, 0i64, 0i64, 1i64]), - FinalizationSectionV1::TermStatistics => statement.query(params![0i64, 0i64, 1i64]), - FinalizationSectionV1::ExactPostings => { - statement.query(params!["field", vec![0u8], 0i64, 1i64]) - } - FinalizationSectionV1::NgramPostings => { - statement.query(params![0i64, 0i64, 0i64, 1i64]) - } - FinalizationSectionV1::FieldStatistics | FinalizationSectionV1::Vocabulary => { + FinalizationSectionV1::SourcePages | FinalizationSectionV1::FieldStatistics => { statement.query(params![0i64, 1i64]) } - FinalizationSectionV1::CloneOccurrences | FinalizationSectionV1::CloneBodyPayloads => { - statement.query(params!["", 1i64]) - } - FinalizationSectionV1::CloneExactPostings => { - statement.query(params![0i64, 0i64, "", "", 1i64]) - } - FinalizationSectionV1::CloneFingerprintCounts => { - statement.query(params!["", 0i64, 0i64, 0i64, 1i64]) - } - FinalizationSectionV1::CloneFingerprintPostings => { - statement.query(params!["", 0i64, 0i64, 0i64, "", 0i64, 1i64]) - } + FinalizationSectionV1::Vocabulary => statement.query(params!["", 1i64]), + other => panic!("{other:?} is not walked natively"), } .map_err(sqlite_error)?; let mut details = Vec::new(); diff --git a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/clone_census.rs b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/clone_census.rs index 6afb139f0f..729cca6d79 100644 --- a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/clone_census.rs +++ b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/clone_census.rs @@ -1,11 +1,12 @@ -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use rusqlite::Connection; use tracedecay_code_index::clones::{ - CloneBodyEligibilityV1, CloneBodyOccurrenceV1, CloneBodyPayloadV1, CloneBodyRenameStatusV1, - CloneNormalizationClassV1, + CloneBodyEligibilityV1, CloneBodyRenameStatusV1, CloneNormalizationClassV1, }; +use super::clone_codec::{decode_clone_eligibility, decode_clone_payload, digest_from_key}; +use super::format::decode_fingerprint_postings; use super::{CodeLexicalArtifactErrorV1, sqlite_error}; #[derive(Clone, Debug, Default, PartialEq, Eq)] @@ -50,29 +51,26 @@ enum IncompleteRenameCoverageV1 { /// ones the per-occurrence rename counters distinguish. fn validate_stored_clone_payloads( connection: &Connection, -) -> Result, CodeLexicalArtifactErrorV1> { +) -> Result, CodeLexicalArtifactErrorV1> { let mut incomplete_rename = HashMap::new(); let mut statement = connection - .prepare("SELECT payload_digest, payload FROM clone_body_payloads ORDER BY payload_digest") + .prepare( + "SELECT ordinal, payload_digest, payload FROM clone_body_payloads ORDER BY ordinal", + ) .map_err(sqlite_error)?; let mut rows = statement.query([]).map_err(sqlite_error)?; while let Some(row) = rows.next().map_err(sqlite_error)? { - let digest: String = row.get(0).map_err(sqlite_error)?; - let payload_bytes: Vec = row.get(1).map_err(sqlite_error)?; - let payload: CloneBodyPayloadV1 = serde_json::from_slice(&payload_bytes) - .map_err(|error| CodeLexicalArtifactErrorV1::Corrupt(error.to_string()))?; - if payload.payload_digest.as_str() != digest || payload.validate().is_err() { - return Err(CodeLexicalArtifactErrorV1::Corrupt( - "clone census found a payload outside its stored digest".to_owned(), - )); - } + let ordinal: i64 = row.get(0).map_err(sqlite_error)?; + let digest: Vec = row.get(1).map_err(sqlite_error)?; + let payload_bytes: Vec = row.get(2).map_err(sqlite_error)?; + let payload = decode_clone_payload(&payload_bytes, digest_from_key(&digest)?.as_str())?; match payload.rename_coverage { CloneBodyRenameStatusV1::Complete => {} CloneBodyRenameStatusV1::Partial => { - incomplete_rename.insert(digest, IncompleteRenameCoverageV1::Partial); + incomplete_rename.insert(ordinal, IncompleteRenameCoverageV1::Partial); } CloneBodyRenameStatusV1::UnsupportedLanguage => { - incomplete_rename.insert(digest, IncompleteRenameCoverageV1::UnsupportedLanguage); + incomplete_rename.insert(ordinal, IncompleteRenameCoverageV1::UnsupportedLanguage); } } } @@ -81,7 +79,6 @@ fn validate_stored_clone_payloads( pub(super) fn read_clone_index_census( connection: &Connection, - has_fingerprints: bool, hot_posting_threshold: u64, ) -> Result { let mut census = CodeLexicalCloneIndexCensusV1::default(); @@ -90,29 +87,25 @@ pub(super) fn read_clone_index_census( // row; the occurrence total below proves none was dropped by it. let mut statement = connection .prepare( - "SELECT occurrence.payload_digest, occurrence.occurrence + "SELECT occurrence.payload_ordinal, occurrence.eligibility FROM clone_occurrences AS occurrence JOIN clone_body_payloads AS payload - ON payload.payload_digest = occurrence.payload_digest - ORDER BY occurrence.symbol_occurrence_id", + ON payload.ordinal = occurrence.payload_ordinal + ORDER BY occurrence.ordinal", ) .map_err(sqlite_error)?; let mut rows = statement.query([]).map_err(sqlite_error)?; while let Some(row) = rows.next().map_err(sqlite_error)? { - let digest: String = row.get(0).map_err(sqlite_error)?; - let occurrence_bytes: Vec = row.get(1).map_err(sqlite_error)?; - let occurrence: CloneBodyOccurrenceV1 = serde_json::from_slice(&occurrence_bytes) - .map_err(|error| CodeLexicalArtifactErrorV1::Corrupt(error.to_string()))?; - if occurrence.payload_digest.as_str() != digest { - return Err(CodeLexicalArtifactErrorV1::Corrupt( - "clone census found a payload outside its occurrence binding".to_owned(), - )); - } + let payload_ordinal: i64 = row.get(0).map_err(sqlite_error)?; + let eligibility = row + .get_ref(1) + .and_then(|value| value.as_blob().map_err(rusqlite::Error::from)) + .map_err(sqlite_error)?; census.source_bodies = census.source_bodies.saturating_add(1); - match occurrence.eligibility { + match decode_clone_eligibility(eligibility)? { CloneBodyEligibilityV1::Eligible => { census.eligible_source_bodies = census.eligible_source_bodies.saturating_add(1); - match incomplete_rename.get(&digest) { + match incomplete_rename.get(&payload_ordinal) { None => {} Some(IncompleteRenameCoverageV1::Partial) => { census.rename_partial_bodies = @@ -184,63 +177,82 @@ pub(super) fn read_clone_index_census( census.conservative_normalized_bodies = count(conservative)?; census.rename_normalized_bodies = count(rename)?; - if has_fingerprints { - let (fingerprint_bodies, fingerprint_postings, hot_postings, hot_rows): ( - i64, - i64, - i64, - i64, - ) = connection - .query_row( - "SELECT - (SELECT COUNT(DISTINCT symbol_occurrence_id) FROM clone_fingerprint_postings), - (SELECT COUNT(*) FROM clone_fingerprint_postings), - (SELECT COUNT(*) FROM clone_fingerprint_counts WHERE posting_count > ?1), - (SELECT COALESCE(SUM(posting_count), 0) FROM clone_fingerprint_counts WHERE posting_count > ?1)", - [i64::try_from(hot_posting_threshold).map_err(|error| { - CodeLexicalArtifactErrorV1::Contract(error.to_string()) - })?], - |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), - ) + let (fingerprint_postings, hot_postings, hot_rows): (i64, i64, i64) = connection + .query_row( + "SELECT + COALESCE(SUM(posting_count), 0), + COUNT(*) FILTER (WHERE posting_count > ?1), + COALESCE(SUM(posting_count) FILTER (WHERE posting_count > ?1), 0) + FROM clone_fingerprint_postings", + [i64::try_from(hot_posting_threshold) + .map_err(|error| CodeLexicalArtifactErrorV1::Contract(error.to_string()))?], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .map_err(sqlite_error)?; + // Bodies with positional fingerprints are the distinct occurrences their + // lists name. + let mut fingerprinted = HashSet::new(); + let mut statement = connection + .prepare("SELECT postings FROM clone_fingerprint_postings") + .map_err(sqlite_error)?; + let mut rows = statement.query([]).map_err(sqlite_error)?; + while let Some(row) = rows.next().map_err(sqlite_error)? { + let encoded = row + .get_ref(0) + .and_then(|value| value.as_blob().map_err(rusqlite::Error::from)) .map_err(sqlite_error)?; - census.near_fingerprint_bodies = count(fingerprint_bodies)?; - census.near_fingerprint_postings = count(fingerprint_postings)?; - census.hot_postings = count(hot_postings)?; - census.hot_posting_rows = count(hot_rows)?; + fingerprinted.extend( + decode_fingerprint_postings(encoded)? + .into_iter() + .map(|(occurrence, _)| occurrence), + ); } + let fingerprint_bodies = i64::try_from(fingerprinted.len()) + .map_err(|error| CodeLexicalArtifactErrorV1::Contract(error.to_string()))?; + census.near_fingerprint_bodies = count(fingerprint_bodies)?; + census.near_fingerprint_postings = count(fingerprint_postings)?; + census.hot_postings = count(hot_postings)?; + census.hot_posting_rows = count(hot_rows)?; + Ok(census) } #[cfg(test)] mod tests { + use super::super::clone_codec::{digest_key, encode_clone_eligibility, encode_clone_payload}; use super::*; use rusqlite::params; use std::sync::Arc; use tracedecay_code_extraction::{ CloneBodyTokenizationStatusV1, ConservativeCloneTokenV1, ExtractedCloneBodyV1, }; - use tracedecay_domain::{ - CodeGenerationId, NodeKind, ProjectId, RepositoryId, SourceSpan, SymbolOccurrenceId, - }; + use tracedecay_code_index::clones::CloneBodyPayloadV1; + use tracedecay_domain::{NodeKind, SourceSpan}; - /// The three tables the census reads. Triggers and the builder gate belong - /// to the write path, which no census read goes through. + /// The tables the census reads. Triggers and the builder gate belong to + /// the write path, which no census read goes through. fn census_schema(connection: &Connection) { connection .execute_batch( "CREATE TABLE clone_body_payloads( - payload_digest TEXT PRIMARY KEY, + ordinal INTEGER PRIMARY KEY, + payload_digest BLOB NOT NULL UNIQUE, payload BLOB NOT NULL ); CREATE TABLE clone_occurrences( - symbol_occurrence_id TEXT PRIMARY KEY, - payload_digest TEXT NOT NULL, - occurrence BLOB NOT NULL + ordinal INTEGER PRIMARY KEY, + symbol_key BLOB NOT NULL UNIQUE, + payload_ordinal INTEGER NOT NULL, + eligibility BLOB NOT NULL ); CREATE TABLE clone_exact_postings( class INTEGER NOT NULL, - digest TEXT NOT NULL, - symbol_occurrence_id TEXT NOT NULL + digest BLOB NOT NULL, + occurrence_ordinal INTEGER NOT NULL + ); + CREATE TABLE clone_fingerprint_postings( + posting_count INTEGER NOT NULL, + postings BLOB NOT NULL );", ) .expect("census schema"); @@ -277,43 +289,29 @@ mod tests { CloneBodyPayloadV1::from_extracted(&body).expect("canonical clone payload") } - fn store_payload(connection: &Connection, payload: &CloneBodyPayloadV1) { + /// Store `payload` and return its ordinal. + fn store_payload(connection: &Connection, payload: &CloneBodyPayloadV1) -> i64 { connection .execute( "INSERT INTO clone_body_payloads(payload_digest, payload) VALUES (?1, ?2)", params![ - payload.payload_digest.as_str(), - serde_json::to_vec(payload).expect("payload json") + digest_key(&payload.payload_digest).expect("payload digest key"), + encode_clone_payload(payload).expect("payload bytes").0 ], ) .expect("store payload"); + connection.last_insert_rowid() } - fn store_occurrence(connection: &Connection, id: &str, payload: &CloneBodyPayloadV1) { - let occurrence = CloneBodyOccurrenceV1 { - project_id: ProjectId::new("project.clone-census").expect("project"), - repository_id: RepositoryId::new("repository.clone-census").expect("repository"), - worktree_id: None, - source_generation: CodeGenerationId::new("generation.clone-census") - .expect("generation"), - snapshot_digest: payload.body_digest.clone(), - symbol_occurrence_id: SymbolOccurrenceId::new(id).expect("symbol"), - path: "src/lib.rs".to_owned(), - body_span: SourceSpan { - start_byte: 0, - end_byte: 64, - }, - payload_digest: payload.payload_digest.clone(), - eligibility: CloneBodyEligibilityV1::Eligible, - }; + fn store_occurrence(connection: &Connection, id: &str, payload_ordinal: i64) { connection .execute( - "INSERT INTO clone_occurrences(symbol_occurrence_id, payload_digest, occurrence) + "INSERT INTO clone_occurrences(symbol_key, payload_ordinal, eligibility) VALUES (?1, ?2, ?3)", params![ id, - payload.payload_digest.as_str(), - serde_json::to_vec(&occurrence).expect("occurrence json") + payload_ordinal, + encode_clone_eligibility(CloneBodyEligibilityV1::Eligible) ], ) .expect("store occurrence"); @@ -327,13 +325,11 @@ mod tests { fn census_refuses_an_occurrence_whose_payload_row_is_absent() { let connection = Connection::open_in_memory().expect("census database"); census_schema(&connection); - let present = payload(0); - let absent = payload(1); - store_payload(&connection, &present); - store_occurrence(&connection, "symbol.present", &present); - store_occurrence(&connection, "symbol.absent", &absent); + let present = store_payload(&connection, &payload(0)); + store_occurrence(&connection, "symbol.present", present); + store_occurrence(&connection, "symbol.absent", present + 1); - let error = read_clone_index_census(&connection, false, 8) + let error = read_clone_index_census(&connection, 8) .expect_err("an occurrence without its payload row must refuse the census"); assert!( matches!(error, CodeLexicalArtifactErrorV1::Corrupt(_)), @@ -355,15 +351,14 @@ mod tests { let connection = Connection::open_in_memory().expect("census database"); census_schema(&connection); for seed in 0..PAYLOADS { - let payload = payload(seed); - store_payload(&connection, &payload); + let ordinal = store_payload(&connection, &payload(seed)); for index in 0..OCCURRENCES_PER_PAYLOAD { - store_occurrence(&connection, &format!("symbol.{seed}.{index}"), &payload); + store_occurrence(&connection, &format!("symbol.{seed}.{index}"), ordinal); } } let started = std::time::Instant::now(); - let census = read_clone_index_census(&connection, false, 8).expect("census"); + let census = read_clone_index_census(&connection, 8).expect("census"); let elapsed = started.elapsed(); assert_eq!( census.source_bodies, diff --git a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/clone_codec.rs b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/clone_codec.rs new file mode 100644 index 0000000000..0c18784332 --- /dev/null +++ b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/clone_codec.rs @@ -0,0 +1,680 @@ +//! Binary clone records. +//! +//! A payload stores its canonical fields without their digests (a pure +//! function of those fields, re-derived on decode and checked against the +//! row's content address), each syntax kind once in a per-payload table, +//! and its rename stream as its difference from the conservative stream, +//! which it aligns with token for token. The whole record is deflated. +//! An occurrence stores only content: the columns carry its symbol, path, +//! span, and payload, the blob its eligibility, and the opening route +//! supplies project, repository, worktree, generation, and snapshot. + +use std::borrow::Cow; +use std::collections::HashMap; +use std::sync::Arc; + +use tracedecay_code_index::clones::{ + CloneBodyEligibilityV1, CloneBodyOccurrenceV1, CloneBodyPayloadPartsV1, CloneBodyPayloadV1, + CloneBodyRenameIssueV1, CloneBodyRenameStatusV1, CloneBodyTokenizationIssueV1, + CloneBodyTokenizationStatusV1, CodeIndexCloneBodyV1, ConservativeCloneTokenV1, +}; +use tracedecay_domain::{ + CodeGenerationId, ManifestDigest, ProjectId, RepositoryId, SourceSpan, SymbolOccurrenceId, + WorktreeId, +}; + +use super::format::{contract_number, deflate_bytes, encode_varint, inflate_bytes, take_varint}; +use super::row_codec::symbol_id_from_key; +use super::{CodeLexicalArtifactErrorV1, sqlite_error}; + +const CLONE_PAYLOAD_DEFLATE: u8 = 2; +/// Bound on one stored clone payload once inflated; a body's token streams +/// stay far below it. +const CLONE_PAYLOAD_MAX_INFLATED_BYTES: usize = 64 * 1024 * 1024; + +const TOKEN_STRUCTURE_START: u64 = 0; +const TOKEN_SYNTAX: u64 = 1; +const TOKEN_STRUCTURE_END: u64 = 2; + +const RENAME_ABSENT: u8 = 0; +const RENAME_ALIGNED: u8 = 1; +const RENAME_STREAM: u8 = 2; +const RENAME_TOKEN_SAME: u8 = 0; +const RENAME_TOKEN_TEXT: u8 = 1; + +/// The route identity every clone occurrence an opened artifact serves +/// carries: the opener's, never the building worktree's. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct CloneOccurrenceRouteV1 { + pub project_id: ProjectId, + pub repository_id: RepositoryId, + pub worktree_id: Option, + pub source_generation: CodeGenerationId, + pub snapshot_digest: ManifestDigest, +} + +impl CloneOccurrenceRouteV1 { + /// Whether `occurrence` names this route's project, repository, and + /// worktree; generation and snapshot are the route's to assign. + pub(super) fn owns(&self, occurrence: &CloneBodyOccurrenceV1) -> bool { + occurrence.project_id == self.project_id + && occurrence.repository_id == self.repository_id + && occurrence.worktree_id == self.worktree_id + } + + /// Rebuild one stored occurrence and its payload under this route. + pub(super) fn clone_body( + &self, + stored: StoredCloneBodyRowV1, + ) -> Result { + let (occurrence, payload) = self.occurrence_and_payload(stored)?; + Ok(CodeIndexCloneBodyV1 { + payload: Arc::new(payload), + occurrence, + }) + } + + pub(super) fn occurrence_and_payload( + &self, + (stored, payload): StoredCloneBodyRowV1, + ) -> Result<(CloneBodyOccurrenceV1, CloneBodyPayloadV1), CodeLexicalArtifactErrorV1> { + let payload = payload.ok_or_else(|| corrupt("occurrence is missing its payload"))?; + let occurrence = self.occurrence(stored)?; + let payload = decode_clone_payload(&payload, occurrence.payload_digest.as_str())?; + Ok((occurrence, payload)) + } + + /// Rebuild one stored occurrence under this route. + pub(super) fn occurrence( + &self, + stored: StoredCloneOccurrenceV1, + ) -> Result { + let payload_digest = stored + .payload_digest + .ok_or_else(|| corrupt("occurrence is missing its payload"))?; + let corrupt = |error: tracedecay_domain::DomainError| { + CodeLexicalArtifactErrorV1::Corrupt(error.to_string()) + }; + let body_span = SourceSpan { + start_byte: u64::try_from(stored.body_start).map_err(contract_number)?, + end_byte: u64::try_from(stored.body_end).map_err(contract_number)?, + }; + body_span.validate().map_err(corrupt)?; + Ok(CloneBodyOccurrenceV1 { + project_id: self.project_id.clone(), + repository_id: self.repository_id.clone(), + worktree_id: self.worktree_id.clone(), + source_generation: self.source_generation.clone(), + snapshot_digest: self.snapshot_digest.clone(), + symbol_occurrence_id: SymbolOccurrenceId::new(symbol_id_from_key( + (&stored.symbol_key).into(), + )?) + .map_err(corrupt)?, + path: stored.path, + body_span, + payload_digest: digest_from_key(&payload_digest)?, + eligibility: decode_clone_eligibility(&stored.eligibility)?, + }) + } +} + +/// One `clone_occurrences` row's content columns with its payload's +/// digest, selected in the order `symbol_key, payload.payload_digest, path, +/// body_start, body_end, eligibility`. +pub(super) struct StoredCloneOccurrenceV1 { + pub symbol_key: rusqlite::types::Value, + /// `None` when the left-joined payload row is absent. + pub payload_digest: Option>, + pub path: String, + pub body_start: i64, + pub body_end: i64, + pub eligibility: Vec, +} + +impl StoredCloneOccurrenceV1 { + pub(super) fn read(row: &rusqlite::Row<'_>, first: usize) -> rusqlite::Result { + Ok(Self { + symbol_key: row.get(first)?, + payload_digest: row.get(first + 1)?, + path: row.get(first + 2)?, + body_start: row.get(first + 3)?, + body_end: row.get(first + 4)?, + eligibility: row.get(first + 5)?, + }) + } +} + +/// The occurrence columns at `first` of a row that left-joins them, `None` +/// when the join found no occurrence. +pub(super) fn stored_clone_occurrence( + row: &rusqlite::Row<'_>, + first: usize, +) -> Result, CodeLexicalArtifactErrorV1> { + if row.get_ref(first).map_err(sqlite_error)? == rusqlite::types::ValueRef::Null { + return Ok(None); + } + StoredCloneOccurrenceV1::read(row, first) + .map(Some) + .map_err(sqlite_error) +} + +/// An occurrence's columns followed by its left-joined payload. +pub(super) type StoredCloneBodyRowV1 = (StoredCloneOccurrenceV1, Option>); + +pub(super) fn routed_clone_body_row( + row: &rusqlite::Row<'_>, +) -> rusqlite::Result { + Ok((StoredCloneOccurrenceV1::read(row, 0)?, row.get(6)?)) +} + +/// The 32 bytes a `sha256:` manifest digest stores as. +pub(super) fn digest_key(digest: &ManifestDigest) -> Result<[u8; 32], CodeLexicalArtifactErrorV1> { + digest + .as_str() + .strip_prefix("sha256:") + .and_then(|hex_digest| hex::decode(hex_digest).ok()) + .and_then(|bytes| <[u8; 32]>::try_from(bytes).ok()) + .ok_or_else(|| { + CodeLexicalArtifactErrorV1::Contract(format!( + "clone digest {} is not a SHA-256 manifest digest", + digest.as_str() + )) + }) +} + +/// Inverse of [`digest_key`]. +pub(super) fn digest_from_key(key: &[u8]) -> Result { + if key.len() != 32 { + return Err(corrupt("digest is not 32 bytes")); + } + ManifestDigest::from_sha256_bytes(key) + .map_err(|error| CodeLexicalArtifactErrorV1::Corrupt(error.to_string())) +} + +pub(super) fn encode_clone_eligibility(eligibility: CloneBodyEligibilityV1) -> Vec { + let mut encoded = Vec::with_capacity(1); + match eligibility { + CloneBodyEligibilityV1::Eligible => encoded.push(0), + CloneBodyEligibilityV1::ExcludedIncompleteTokenization => encoded.push(1), + CloneBodyEligibilityV1::ExcludedTooSmall { minimum_tokens } => { + encoded.push(2); + encode_varint(u64::from(minimum_tokens), &mut encoded); + } + CloneBodyEligibilityV1::ExcludedTooLarge { + maximum_tokens, + maximum_bytes, + } => { + encoded.push(3); + encode_varint(u64::from(maximum_tokens), &mut encoded); + encode_varint(maximum_bytes, &mut encoded); + } + } + encoded +} + +pub(super) fn decode_clone_eligibility( + mut bytes: &[u8], +) -> Result { + let eligibility = match take_u8(&mut bytes)? { + 0 => CloneBodyEligibilityV1::Eligible, + 1 => CloneBodyEligibilityV1::ExcludedIncompleteTokenization, + 2 => CloneBodyEligibilityV1::ExcludedTooSmall { + minimum_tokens: take_u32(&mut bytes)?, + }, + 3 => CloneBodyEligibilityV1::ExcludedTooLarge { + maximum_tokens: take_u32(&mut bytes)?, + maximum_bytes: take_varint(&mut bytes)?, + }, + _ => return Err(corrupt("eligibility has an unknown tag")), + }; + if !bytes.is_empty() { + return Err(corrupt("eligibility has trailing bytes")); + } + Ok(eligibility) +} + +/// Encode `payload` and return the stored bytes with their inflated length. +pub(super) fn encode_clone_payload( + payload: &CloneBodyPayloadV1, +) -> Result<(Vec, usize), CodeLexicalArtifactErrorV1> { + let mut encoded = Vec::with_capacity(payload.conservative_tokens.len() * 4 + 64); + put_str(&mut encoded, &payload.language); + put_str(&mut encoded, &payload.symbol_kind); + encode_varint(u64::from(payload.token_count), &mut encoded); + encode_varint( + u64::from(payload.conservative_normalization_revision), + &mut encoded, + ); + encoded.push(match payload.tokenization_status { + CloneBodyTokenizationStatusV1::Complete => 0, + CloneBodyTokenizationStatusV1::Partial => 1, + }); + put_len(&mut encoded, payload.tokenization_issues.len())?; + for issue in &payload.tokenization_issues { + encoded.push(match issue { + CloneBodyTokenizationIssueV1::BodyBoundaryUnavailable => 0, + CloneBodyTokenizationIssueV1::BodyExceedsSizeBound => 1, + CloneBodyTokenizationIssueV1::InvalidSourceRange => 2, + CloneBodyTokenizationIssueV1::ParseError => 3, + }); + } + encode_varint( + payload + .rename_normalization_revision + .map_or(0, |revision| u64::from(revision) + 1), + &mut encoded, + ); + encoded.push(match payload.rename_coverage { + CloneBodyRenameStatusV1::Complete => 0, + CloneBodyRenameStatusV1::Partial => 1, + CloneBodyRenameStatusV1::UnsupportedLanguage => 2, + }); + put_len(&mut encoded, payload.rename_issues.len())?; + for issue in &payload.rename_issues { + encoded.push(match issue { + CloneBodyRenameIssueV1::DynamicBinding => 0, + CloneBodyRenameIssueV1::UnsupportedBindingSyntax => 1, + }); + } + + let rename = payload.rename_tokens.as_deref(); + let mut kinds = SyntaxKindTableV1::default(); + for token in payload + .conservative_tokens + .iter() + .chain(rename.into_iter().flatten()) + { + kinds.intern(syntax_kind(token)); + } + put_len(&mut encoded, kinds.names.len())?; + for name in &kinds.names { + put_str(&mut encoded, name); + } + put_tokens(&mut encoded, &payload.conservative_tokens, &kinds)?; + match rename { + None => encoded.push(RENAME_ABSENT), + Some(rename) if aligned(&payload.conservative_tokens, rename) => { + encoded.push(RENAME_ALIGNED); + for (conservative, renamed) in payload.conservative_tokens.iter().zip(rename) { + match (conservative, renamed) { + ( + ConservativeCloneTokenV1::Syntax { text: original, .. }, + ConservativeCloneTokenV1::Syntax { text, .. }, + ) if original != text => { + encoded.push(RENAME_TOKEN_TEXT); + put_str(&mut encoded, text); + } + _ => encoded.push(RENAME_TOKEN_SAME), + } + } + } + Some(rename) => { + encoded.push(RENAME_STREAM); + put_tokens(&mut encoded, rename, &kinds)?; + } + } + Ok(( + deflate_bytes(CLONE_PAYLOAD_DEFLATE, &encoded)?, + encoded.len(), + )) +} + +/// Decode one stored payload and prove it is the payload `expected_digest` +/// names. +pub(super) fn decode_clone_payload( + stored: &[u8], + expected_digest: &str, +) -> Result { + let inflated = inflate_bytes( + CLONE_PAYLOAD_DEFLATE, + stored, + CLONE_PAYLOAD_MAX_INFLATED_BYTES, + )?; + let mut bytes = inflated.as_slice(); + let language = take_string(&mut bytes)?; + let symbol_kind = take_string(&mut bytes)?; + let token_count = take_u32(&mut bytes)?; + let conservative_normalization_revision = take_u16(&mut bytes)?; + let tokenization_status = match take_u8(&mut bytes)? { + 0 => CloneBodyTokenizationStatusV1::Complete, + 1 => CloneBodyTokenizationStatusV1::Partial, + _ => return Err(corrupt("tokenization status has an unknown tag")), + }; + let tokenization_issues = (0..take_len(&mut bytes)?) + .map(|_| match take_u8(&mut bytes)? { + 0 => Ok(CloneBodyTokenizationIssueV1::BodyBoundaryUnavailable), + 1 => Ok(CloneBodyTokenizationIssueV1::BodyExceedsSizeBound), + 2 => Ok(CloneBodyTokenizationIssueV1::InvalidSourceRange), + 3 => Ok(CloneBodyTokenizationIssueV1::ParseError), + _ => Err(corrupt("tokenization issue has an unknown tag")), + }) + .collect::, _>>()?; + let rename_normalization_revision = match take_varint(&mut bytes)? { + 0 => None, + revision => Some( + u16::try_from(revision - 1).map_err(|_| corrupt("rename revision overflows u16"))?, + ), + }; + let rename_coverage = match take_u8(&mut bytes)? { + 0 => CloneBodyRenameStatusV1::Complete, + 1 => CloneBodyRenameStatusV1::Partial, + 2 => CloneBodyRenameStatusV1::UnsupportedLanguage, + _ => return Err(corrupt("rename coverage has an unknown tag")), + }; + let rename_issues = (0..take_len(&mut bytes)?) + .map(|_| match take_u8(&mut bytes)? { + 0 => Ok(CloneBodyRenameIssueV1::DynamicBinding), + 1 => Ok(CloneBodyRenameIssueV1::UnsupportedBindingSyntax), + _ => Err(corrupt("rename issue has an unknown tag")), + }) + .collect::, _>>()?; + let kinds = (0..take_len(&mut bytes)?) + .map(|_| take_string(&mut bytes)) + .collect::, _>>()?; + let conservative_tokens: Arc<[ConservativeCloneTokenV1]> = + take_tokens(&mut bytes, &kinds)?.into(); + let rename_tokens: Option> = match take_u8(&mut bytes)? { + RENAME_ABSENT => None, + RENAME_ALIGNED => Some( + conservative_tokens + .iter() + .map(|token| match (take_u8(&mut bytes)?, token) { + (RENAME_TOKEN_SAME, _) => Ok(token.clone()), + (RENAME_TOKEN_TEXT, ConservativeCloneTokenV1::Syntax { syntax_kind, .. }) => { + Ok(ConservativeCloneTokenV1::Syntax { + syntax_kind: syntax_kind.clone(), + text: take_string(&mut bytes)?, + }) + } + _ => Err(corrupt("rename token difference is not canonical")), + }) + .collect::, _>>()? + .into(), + ), + RENAME_STREAM => Some(take_tokens(&mut bytes, &kinds)?.into()), + _ => return Err(corrupt("rename stream has an unknown tag")), + }; + if !bytes.is_empty() { + return Err(corrupt("payload has trailing bytes")); + } + let payload = CloneBodyPayloadV1::from_parts(CloneBodyPayloadPartsV1 { + language, + symbol_kind, + token_count, + conservative_normalization_revision, + conservative_tokens, + tokenization_status, + tokenization_issues, + rename_normalization_revision, + rename_tokens, + rename_coverage, + rename_issues, + }) + .map_err(CodeLexicalArtifactErrorV1::Corrupt)?; + if payload.payload_digest.as_str() != expected_digest { + return Err(corrupt("payload does not hash to its content address")); + } + Ok(payload) +} + +#[derive(Default)] +struct SyntaxKindTableV1<'a> { + names: Vec<&'a str>, + indexes: HashMap<&'a str, u64>, +} + +impl<'a> SyntaxKindTableV1<'a> { + fn intern(&mut self, name: &'a str) { + let next = self.names.len() as u64; + if let std::collections::hash_map::Entry::Vacant(slot) = self.indexes.entry(name) { + slot.insert(next); + self.names.push(name); + } + } + + fn index(&self, name: &str) -> Result { + self.indexes.get(name).copied().ok_or_else(|| { + CodeLexicalArtifactErrorV1::Contract("clone syntax kind was not interned".to_owned()) + }) + } +} + +fn syntax_kind(token: &ConservativeCloneTokenV1) -> &str { + match token { + ConservativeCloneTokenV1::StructureStart { syntax_kind } + | ConservativeCloneTokenV1::Syntax { syntax_kind, .. } + | ConservativeCloneTokenV1::StructureEnd { syntax_kind } => syntax_kind, + } +} + +/// Whether `rename` differs from `conservative` only in syntax-token text. +fn aligned(conservative: &[ConservativeCloneTokenV1], rename: &[ConservativeCloneTokenV1]) -> bool { + conservative.len() == rename.len() + && conservative + .iter() + .zip(rename) + .all(|(left, right)| match (left, right) { + ( + ConservativeCloneTokenV1::Syntax { + syntax_kind: left, .. + }, + ConservativeCloneTokenV1::Syntax { + syntax_kind: right, .. + }, + ) => left == right, + _ => left == right, + }) +} + +fn put_tokens( + encoded: &mut Vec, + tokens: &[ConservativeCloneTokenV1], + kinds: &SyntaxKindTableV1<'_>, +) -> Result<(), CodeLexicalArtifactErrorV1> { + put_len(encoded, tokens.len())?; + for token in tokens { + let (tag, text) = match token { + ConservativeCloneTokenV1::StructureStart { .. } => (TOKEN_STRUCTURE_START, None), + ConservativeCloneTokenV1::Syntax { text, .. } => (TOKEN_SYNTAX, Some(text)), + ConservativeCloneTokenV1::StructureEnd { .. } => (TOKEN_STRUCTURE_END, None), + }; + encode_varint(kinds.index(syntax_kind(token))? * 3 + tag, encoded); + if let Some(text) = text { + put_str(encoded, text); + } + } + Ok(()) +} + +fn take_tokens( + bytes: &mut &[u8], + kinds: &[String], +) -> Result, CodeLexicalArtifactErrorV1> { + let count = take_len(bytes)?; + // Every token takes at least one byte, so a count above the remaining + // bytes is corrupt rather than an allocation to honor. + if count > bytes.len() { + return Err(corrupt("token count exceeds its record")); + } + let mut tokens = Vec::with_capacity(count); + for _ in 0..count { + let header = take_varint(bytes)?; + let kind = usize::try_from(header / 3) + .ok() + .and_then(|index| kinds.get(index)) + .ok_or_else(|| corrupt("token names an unknown syntax kind"))?; + let syntax_kind = Cow::Owned(kind.clone()); + tokens.push(match header % 3 { + TOKEN_STRUCTURE_START => ConservativeCloneTokenV1::StructureStart { syntax_kind }, + TOKEN_SYNTAX => ConservativeCloneTokenV1::Syntax { + syntax_kind, + text: take_string(bytes)?, + }, + _ => ConservativeCloneTokenV1::StructureEnd { syntax_kind }, + }); + } + Ok(tokens) +} + +fn put_len(encoded: &mut Vec, length: usize) -> Result<(), CodeLexicalArtifactErrorV1> { + encode_varint(u64::try_from(length).map_err(contract_number)?, encoded); + Ok(()) +} + +fn put_str(encoded: &mut Vec, value: &str) { + encode_varint(value.len() as u64, encoded); + encoded.extend_from_slice(value.as_bytes()); +} + +fn take_len(bytes: &mut &[u8]) -> Result { + usize::try_from(take_varint(bytes)?).map_err(|_| corrupt("length overflows usize")) +} + +fn take_u8(bytes: &mut &[u8]) -> Result { + let (&value, rest) = bytes.split_first().ok_or_else(|| corrupt("is truncated"))?; + *bytes = rest; + Ok(value) +} + +fn take_u16(bytes: &mut &[u8]) -> Result { + u16::try_from(take_varint(bytes)?).map_err(|_| corrupt("value overflows u16")) +} + +fn take_u32(bytes: &mut &[u8]) -> Result { + u32::try_from(take_varint(bytes)?).map_err(|_| corrupt("value overflows u32")) +} + +fn take_string(bytes: &mut &[u8]) -> Result { + let length = take_len(bytes)?; + if length > bytes.len() { + return Err(corrupt("string exceeds its record")); + } + let (value, rest) = bytes.split_at(length); + *bytes = rest; + String::from_utf8(value.to_vec()).map_err(|_| corrupt("string is not UTF-8")) +} + +fn corrupt(detail: &str) -> CodeLexicalArtifactErrorV1 { + CodeLexicalArtifactErrorV1::Corrupt(format!("lexical artifact clone record {detail}")) +} + +#[cfg(test)] +mod tests { + use std::borrow::Cow; + use std::sync::Arc; + + use tracedecay_code_index::clones::{ + CloneBodyEligibilityV1, CloneBodyPayloadPartsV1, CloneBodyPayloadV1, + CloneBodyRenameIssueV1, CloneBodyRenameStatusV1, CloneBodyTokenizationIssueV1, + CloneBodyTokenizationStatusV1, ConservativeCloneTokenV1, + }; + + use super::{ + decode_clone_eligibility, decode_clone_payload, encode_clone_eligibility, + encode_clone_payload, + }; + use crate::retrieval::lexical::CodeLexicalArtifactErrorV1; + + fn token(kind: &'static str, text: Option<&str>) -> ConservativeCloneTokenV1 { + match text { + Some(text) => ConservativeCloneTokenV1::Syntax { + syntax_kind: Cow::Borrowed(kind), + text: text.to_owned(), + }, + None => ConservativeCloneTokenV1::StructureStart { + syntax_kind: Cow::Borrowed(kind), + }, + } + } + + fn payload(rename: Option>) -> CloneBodyPayloadV1 { + let conservative = vec![ + token("block", None), + token("identifier", Some("alpha")), + token("+", Some("+")), + token("identifier", Some("beta")), + ConservativeCloneTokenV1::StructureEnd { + syntax_kind: Cow::Borrowed("block"), + }, + ]; + CloneBodyPayloadV1::from_parts(CloneBodyPayloadPartsV1 { + language: "rust".to_owned(), + symbol_kind: "function".to_owned(), + token_count: 3, + conservative_normalization_revision: 1, + conservative_tokens: conservative.into(), + tokenization_status: CloneBodyTokenizationStatusV1::Partial, + tokenization_issues: vec![CloneBodyTokenizationIssueV1::ParseError], + rename_normalization_revision: rename.as_ref().map(|_| 1), + rename_tokens: rename.map(Arc::from), + rename_coverage: CloneBodyRenameStatusV1::Partial, + rename_issues: vec![CloneBodyRenameIssueV1::DynamicBinding], + }) + .expect("payload") + } + + #[test] + fn payloads_round_trip_with_aligned_divergent_and_absent_rename_streams() { + let aligned = vec![ + token("block", None), + token("identifier", Some("$0")), + token("+", Some("+")), + token("identifier", Some("$1")), + ConservativeCloneTokenV1::StructureEnd { + syntax_kind: Cow::Borrowed("block"), + }, + ]; + let divergent = vec![token("call", None), token("identifier", Some("$0"))]; + for payload in [ + payload(Some(aligned)), + payload(Some(divergent)), + payload(None), + ] { + let (stored, inflated) = encode_clone_payload(&payload).expect("encode"); + assert!(inflated > 0); + let decoded = + decode_clone_payload(&stored, payload.payload_digest.as_str()).expect("decode"); + assert_eq!(decoded, payload); + assert_eq!( + encode_clone_payload(&decoded).expect("re-encode").0, + stored, + "the encoding is canonical" + ); + } + } + + #[test] + fn payloads_refuse_another_address_and_damaged_bytes() { + let payload = payload(None); + let (stored, _) = encode_clone_payload(&payload).expect("encode"); + let other = super::super::format::stored_metadata_digest(b"other").expect("digest"); + assert!(matches!( + decode_clone_payload(&stored, other.as_str()), + Err(CodeLexicalArtifactErrorV1::Corrupt(_)) + )); + let mut damaged = stored.clone(); + let last = damaged.len() - 1; + damaged[last] ^= 0xff; + assert!(decode_clone_payload(&damaged, payload.payload_digest.as_str()).is_err()); + } + + #[test] + fn eligibility_round_trips_every_variant() { + for eligibility in [ + CloneBodyEligibilityV1::Eligible, + CloneBodyEligibilityV1::ExcludedIncompleteTokenization, + CloneBodyEligibilityV1::ExcludedTooSmall { minimum_tokens: 30 }, + CloneBodyEligibilityV1::ExcludedTooLarge { + maximum_tokens: 4096, + maximum_bytes: 65_536, + }, + ] { + assert_eq!( + decode_clone_eligibility(&encode_clone_eligibility(eligibility)).expect("decode"), + eligibility + ); + } + assert!(decode_clone_eligibility(&[9]).is_err()); + assert!(decode_clone_eligibility(&[0, 0]).is_err()); + } +} diff --git a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/clone_successor.rs b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/clone_successor.rs deleted file mode 100644 index 520ed70b01..0000000000 --- a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/clone_successor.rs +++ /dev/null @@ -1,812 +0,0 @@ -use std::collections::{HashMap, HashSet}; -use std::io; -use std::path::{Path, PathBuf}; -use std::sync::Arc; - -use rusqlite::{Connection, OptionalExtension, params}; -use tracedecay_code_index::clones::CodeIndexCloneBodyV1; -use tracedecay_code_index::production::{ - CodeIndexExecutionControlV1, VerifiedSealedLexicalCursorV1, VerifiedSealedLexicalPageV1, - VerifiedSealedLexicalSourceReceiptV1, -}; -use tracedecay_private_fs::framed_log::{DirectorySyncPolicy, sync_parent_directory}; -use tracedecay_private_fs::{create_private_file_retained, open_private_file}; - -use super::super::CodeLexicalProjectionMetadataV1; -use super::builder::{ - BuilderMutationGuardV1, compute_clone_section_digests, derive_clone_fingerprint_counts, - install_clone_freeze, register_builder_mutation_gate, sqlite_file_size, verify_clone_rows, -}; -use super::format::{ - RECEIPT_RESERVATION_BYTES, VerifiedCodeLexicalArtifactV1, artifact_digest, - decode_padded_receipt, metadata_digest, new_verified_receipt, padded_receipt, -}; -use super::schema::{CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V1, LexicalArtifactLayoutV1}; -use super::{CodeLexicalArtifactErrorV1, checkpoint, open_builder_connection, sqlite_error}; - -pub struct CodeLexicalCloneSuccessorV1 { - connection: Connection, - mutation_gate: Arc, - staging_path: PathBuf, - prior: VerifiedCodeLexicalArtifactV1, - metadata: CodeLexicalProjectionMetadataV1, -} - -impl CodeLexicalCloneSuccessorV1 { - pub fn open_or_create( - prior_path: impl AsRef, - staging_path: impl AsRef, - prior: VerifiedCodeLexicalArtifactV1, - metadata: CodeLexicalProjectionMetadataV1, - memory_budget_bytes: usize, - ) -> Result { - let staging_path = staging_path.as_ref(); - if staging_path.exists() { - return Self::open(staging_path, prior, metadata, memory_budget_bytes); - } - initialize_successor( - prior_path.as_ref(), - staging_path, - &prior, - &metadata, - memory_budget_bytes, - )?; - Self::open(staging_path, prior, metadata, memory_budget_bytes) - } - - fn open( - staging_path: &Path, - prior: VerifiedCodeLexicalArtifactV1, - metadata: CodeLexicalProjectionMetadataV1, - memory_budget_bytes: usize, - ) -> Result { - let connection = open_builder_connection(staging_path, memory_budget_bytes)?; - ensure_clone_occurrence_indexes(&connection)?; - let mutation_gate = register_builder_mutation_gate(&connection)?; - let (prior_digest, format_revision): (String, i64) = connection - .query_row( - "SELECT prior_artifact_digest, (SELECT format_revision FROM artifact_state WHERE singleton = 1) FROM clone_successor_state WHERE singleton = 1", - [], - |row| Ok((row.get(0)?, row.get(1)?)), - ) - .map_err(|error| { - CodeLexicalArtifactErrorV1::Incompatible(format!( - "clone successor state is unavailable: {error}" - )) - })?; - if prior_digest != prior.artifact_digest().as_str() - || format_revision != i64::from(CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V1) - || metadata_digest(&metadata)? != *prior.metadata_digest() - { - return Err(CodeLexicalArtifactErrorV1::Incompatible( - "clone successor does not match its prior artifact or metadata".to_owned(), - )); - } - Ok(Self { - connection, - mutation_gate, - staging_path: staging_path.to_path_buf(), - prior, - metadata, - }) - } - - pub fn next_cursor( - &self, - ) -> Result, CodeLexicalArtifactErrorV1> { - let (next_page, bytes): (i64, Option>) = self - .connection - .query_row( - "SELECT next_page_ordinal, next_cursor FROM clone_successor_state WHERE singleton = 1", - [], - |row| Ok((row.get(0)?, row.get(1)?)), - ) - .map_err(sqlite_error)?; - let cursor = bytes - .map(|bytes| { - VerifiedSealedLexicalCursorV1::restore_persisted(&bytes) - .map_err(|error| CodeLexicalArtifactErrorV1::Corrupt(error.to_string())) - }) - .transpose()?; - if cursor.as_ref().map_or(next_page != 0, |cursor| { - u64::try_from(next_page).ok() != Some(cursor.next_page_ordinal()) - }) { - return Err(CodeLexicalArtifactErrorV1::Corrupt( - "clone successor page ordinal disagrees with its source cursor".to_owned(), - )); - } - Ok(cursor) - } - - pub fn projection_metadata(&self) -> &CodeLexicalProjectionMetadataV1 { - &self.metadata - } - - /// Append one page through the atomic batch path. - pub fn append_page( - &mut self, - page: &VerifiedSealedLexicalPageV1, - control: &dyn CodeIndexExecutionControlV1, - ) -> Result<(), CodeLexicalArtifactErrorV1> { - self.append_pages(std::slice::from_ref(page), control) - } - - /// Atomically append an ordered, contiguous batch of verified source - /// pages. Every page is checked against the copied source receipt and the - /// persisted successor cursor exactly as a single-page append is; the - /// batch shares one durable SQLite commit, so a crash resumes at the last - /// committed batch boundary instead of paying one `fsync` per page. - pub fn append_pages( - &mut self, - pages: &[VerifiedSealedLexicalPageV1], - control: &dyn CodeIndexExecutionControlV1, - ) -> Result<(), CodeLexicalArtifactErrorV1> { - let Some(last) = pages.last() else { - return Err(CodeLexicalArtifactErrorV1::Contract( - "clone successor page batches must be non-empty".to_owned(), - )); - }; - checkpoint(control)?; - let _authority = BuilderMutationGuardV1::enter(&self.mutation_gate)?; - let transaction = self.connection.transaction().map_err(sqlite_error)?; - let mut next_page: i64 = transaction - .query_row( - "SELECT next_page_ordinal FROM clone_successor_state WHERE singleton = 1", - [], - |row| row.get(0), - ) - .map_err(sqlite_error)?; - for page in pages { - checkpoint(control)?; - if u64::try_from(next_page).ok() != Some(page.page_ordinal()) { - return Err(CodeLexicalArtifactErrorV1::Contract( - "clone successor page is not the next source page".to_owned(), - )); - } - verify_copied_source_page(&transaction, page)?; - append_clone_rows(&transaction, page, control)?; - next_page = i64::try_from(page.page_ordinal().saturating_add(1)) - .map_err(|error| CodeLexicalArtifactErrorV1::Contract(error.to_string()))?; - } - let next_cursor = last - .next_cursor() - .persisted_bytes() - .map_err(|error| CodeLexicalArtifactErrorV1::Contract(error.to_string()))?; - transaction - .execute( - "UPDATE clone_successor_state SET next_page_ordinal = ?1, next_cursor = ?2 WHERE singleton = 1", - params![next_page, next_cursor], - ) - .map_err(sqlite_error)?; - checkpoint(control)?; - transaction.commit().map_err(sqlite_error) - } - - pub fn verify_resumed_page( - &self, - page: &VerifiedSealedLexicalPageV1, - control: &dyn CodeIndexExecutionControlV1, - ) -> Result<(), CodeLexicalArtifactErrorV1> { - checkpoint(control)?; - verify_copied_source_page(&self.connection, page)?; - verify_clone_page_rows(&self.connection, page, control) - } - - pub fn finish( - &mut self, - source: &VerifiedSealedLexicalSourceReceiptV1, - control: &dyn CodeIndexExecutionControlV1, - ) -> Result { - checkpoint(control)?; - verify_source_receipt(&self.prior, source)?; - let next_page: i64 = self - .connection - .query_row( - "SELECT next_page_ordinal FROM clone_successor_state WHERE singleton = 1", - [], - |row| row.get(0), - ) - .map_err(sqlite_error)?; - if u64::try_from(next_page).ok() != Some(source.page_count()) { - return Err(CodeLexicalArtifactErrorV1::Contract( - "clone successor has not consumed every source page".to_owned(), - )); - } - let transaction = self.connection.transaction().map_err(sqlite_error)?; - derive_clone_fingerprint_counts(&transaction)?; - verify_clone_rows(&transaction, source)?; - install_clone_freeze(&transaction, LexicalArtifactLayoutV1::V16)?; - transaction - .execute("DROP TABLE clone_successor_state", []) - .map_err(sqlite_error)?; - let mut sections = self - .prior - .section_digests() - .iter() - .take(11) - .cloned() - .collect::>(); - if sections.len() != 11 { - return Err(CodeLexicalArtifactErrorV1::Corrupt( - "clone successor prior is missing lexical section digests".to_owned(), - )); - } - sections.extend(compute_clone_section_digests( - &transaction, - control, - LexicalArtifactLayoutV1::V16, - )?); - let metadata_digest = metadata_digest(&self.metadata)?; - let digest = artifact_digest( - &metadata_digest, - source.source_state_digest(), - source.format_revision(), - source.page_count(), - source.total_chunks(), - source.total_payload_bytes(), - source.total_imports(), - source.import_payload_bytes(), - source.import_dictionary_digest(), - source.cumulative_digest(), - §ions, - CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V1, - )?; - let file_size = sqlite_file_size(&transaction)?; - let receipt = new_verified_receipt( - self.metadata.clone(), - metadata_digest, - source, - digest, - sections, - file_size, - LexicalArtifactLayoutV1::V16, - ); - let encoded = padded_receipt(&receipt)?; - transaction - .execute( - "UPDATE artifact_state SET format_revision = ?1, receipt = ?2 WHERE singleton = 1", - params![i64::from(CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V1), encoded,], - ) - .map_err(sqlite_error)?; - checkpoint(control)?; - transaction.commit().map_err(sqlite_error)?; - open_private_file(&self.staging_path) - .map_err(|error| CodeLexicalArtifactErrorV1::Io(error.to_string()))? - .sync_all() - .map_err(|error| CodeLexicalArtifactErrorV1::Io(error.to_string()))?; - Ok(receipt) - } -} - -fn initialize_successor( - prior_path: &Path, - staging_path: &Path, - prior: &VerifiedCodeLexicalArtifactV1, - metadata: &CodeLexicalProjectionMetadataV1, - memory_budget_bytes: usize, -) -> Result<(), CodeLexicalArtifactErrorV1> { - let mut source = open_private_file(prior_path) - .map_err(|error| CodeLexicalArtifactErrorV1::Io(error.to_string()))?; - let mut target = create_private_file_retained(staging_path) - .map_err(|error| CodeLexicalArtifactErrorV1::Io(error.into_error().to_string()))?; - io::copy(&mut source, &mut target) - .map_err(|error| CodeLexicalArtifactErrorV1::Io(error.to_string()))?; - target - .sync_all() - .map_err(|error| CodeLexicalArtifactErrorV1::Io(error.to_string()))?; - drop(target); - let connection = open_builder_connection(staging_path, memory_budget_bytes)?; - let stored: Vec = connection - .query_row( - "SELECT receipt FROM artifact_state WHERE singleton = 1", - [], - |row| row.get(0), - ) - .map_err(sqlite_error)?; - if decode_padded_receipt(&stored)?.as_ref() != Some(prior) - || metadata_digest(metadata)? != *prior.metadata_digest() - { - return Err(CodeLexicalArtifactErrorV1::Incompatible( - "clone successor prior artifact does not match its receipt".to_owned(), - )); - } - reset_clone_tables(&connection)?; - connection - .execute_batch( - "CREATE TABLE clone_successor_state ( - singleton INTEGER PRIMARY KEY CHECK(singleton = 1), - prior_artifact_digest TEXT NOT NULL, - next_page_ordinal INTEGER NOT NULL, - next_cursor BLOB - );", - ) - .map_err(sqlite_error)?; - connection - .execute( - "INSERT INTO clone_successor_state(singleton, prior_artifact_digest, next_page_ordinal, next_cursor) VALUES (1, ?1, 0, NULL)", - [prior.artifact_digest().as_str()], - ) - .map_err(sqlite_error)?; - connection - .execute( - "UPDATE artifact_state SET format_revision = ?1, receipt = ?2 WHERE singleton = 1", - params![ - i64::from(CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V1), - vec![0u8; RECEIPT_RESERVATION_BYTES], - ], - ) - .map_err(sqlite_error)?; - connection - .execute("DELETE FROM finalization_state", []) - .map_err(sqlite_error)?; - connection - .execute_batch("PRAGMA optimize;") - .map_err(sqlite_error)?; - drop(connection); - sync_parent_directory(staging_path, DirectorySyncPolicy::Strict) - .map_err(|error| CodeLexicalArtifactErrorV1::Io(error.to_string())) -} - -const RESET_CLONE_TABLES_SQL: &str = - "DROP TRIGGER IF EXISTS frozen_clone_body_payloads_insert; - DROP TRIGGER IF EXISTS frozen_clone_occurrences_insert; - DROP TRIGGER IF EXISTS frozen_clone_exact_postings_insert; - DROP TRIGGER IF EXISTS frozen_clone_fingerprint_counts_insert; - DROP TRIGGER IF EXISTS frozen_clone_fingerprint_postings_insert; - DROP TRIGGER IF EXISTS builder_gate_clone_body_payloads_insert; - DROP TRIGGER IF EXISTS builder_gate_clone_occurrences_insert; - DROP TRIGGER IF EXISTS builder_gate_clone_exact_postings_insert; - DROP TRIGGER IF EXISTS builder_gate_clone_fingerprint_postings_insert; - DROP TRIGGER IF EXISTS immutable_clone_body_payloads_update; - DROP TRIGGER IF EXISTS immutable_clone_body_payloads_delete; - DROP TRIGGER IF EXISTS immutable_clone_occurrences_update; - DROP TRIGGER IF EXISTS immutable_clone_occurrences_delete; - DROP TRIGGER IF EXISTS immutable_clone_exact_postings_update; - DROP TRIGGER IF EXISTS immutable_clone_exact_postings_delete; - DROP TRIGGER IF EXISTS immutable_clone_fingerprint_postings_update; - DROP TRIGGER IF EXISTS immutable_clone_fingerprint_postings_delete; - DROP TRIGGER IF EXISTS immutable_clone_fingerprint_counts_update; - DROP TRIGGER IF EXISTS immutable_clone_fingerprint_counts_delete; - DROP TABLE IF EXISTS clone_fingerprint_counts; - DROP TABLE IF EXISTS clone_fingerprint_postings; - DROP TABLE IF EXISTS clone_exact_postings; - DROP TABLE IF EXISTS clone_occurrences; - DROP TABLE IF EXISTS clone_body_payloads; - CREATE TABLE clone_body_payloads ( - payload_digest TEXT PRIMARY KEY, - payload BLOB NOT NULL - ) WITHOUT ROWID; - CREATE TABLE clone_occurrences ( - symbol_occurrence_id TEXT PRIMARY KEY, - payload_digest TEXT NOT NULL, - path TEXT NOT NULL, - body_start INTEGER NOT NULL, - body_end INTEGER NOT NULL, - occurrence BLOB NOT NULL - ) WITHOUT ROWID; - CREATE TABLE clone_exact_postings ( - class INTEGER NOT NULL, - normalization_revision INTEGER NOT NULL, - digest TEXT NOT NULL, - symbol_occurrence_id TEXT NOT NULL, - payload_digest TEXT NOT NULL, - PRIMARY KEY(class, normalization_revision, digest, symbol_occurrence_id) - ) WITHOUT ROWID; - CREATE TABLE clone_fingerprint_counts ( - language TEXT NOT NULL, - class INTEGER NOT NULL, - normalization_revision INTEGER NOT NULL, - fingerprint INTEGER NOT NULL, - posting_count INTEGER NOT NULL, - PRIMARY KEY(language, class, normalization_revision, fingerprint) - ) WITHOUT ROWID; - CREATE TABLE clone_fingerprint_postings ( - language TEXT NOT NULL, - class INTEGER NOT NULL, - normalization_revision INTEGER NOT NULL, - fingerprint INTEGER NOT NULL, - symbol_occurrence_id TEXT NOT NULL, - token_position INTEGER NOT NULL, - payload_digest TEXT NOT NULL, - body_digest TEXT NOT NULL, - PRIMARY KEY(language, class, normalization_revision, fingerprint, symbol_occurrence_id, token_position) - ) WITHOUT ROWID; - CREATE TRIGGER builder_gate_clone_body_payloads_insert BEFORE INSERT ON clone_body_payloads WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; - CREATE TRIGGER builder_gate_clone_occurrences_insert BEFORE INSERT ON clone_occurrences WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; - CREATE TRIGGER builder_gate_clone_exact_postings_insert BEFORE INSERT ON clone_exact_postings WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; - CREATE TRIGGER builder_gate_clone_fingerprint_postings_insert BEFORE INSERT ON clone_fingerprint_postings WHEN tracedecay_lexical_builder_append_authorized() != 1 BEGIN SELECT RAISE(ABORT, 'private lexical builder mutation required'); END; - CREATE TRIGGER immutable_clone_body_payloads_update BEFORE UPDATE ON clone_body_payloads BEGIN SELECT RAISE(ABORT, 'immutable clone body payloads'); END; - CREATE TRIGGER immutable_clone_body_payloads_delete BEFORE DELETE ON clone_body_payloads BEGIN SELECT RAISE(ABORT, 'immutable clone body payloads'); END; - CREATE TRIGGER immutable_clone_occurrences_update BEFORE UPDATE ON clone_occurrences BEGIN SELECT RAISE(ABORT, 'immutable clone occurrences'); END; - CREATE TRIGGER immutable_clone_occurrences_delete BEFORE DELETE ON clone_occurrences BEGIN SELECT RAISE(ABORT, 'immutable clone occurrences'); END; - CREATE TRIGGER immutable_clone_exact_postings_update BEFORE UPDATE ON clone_exact_postings BEGIN SELECT RAISE(ABORT, 'immutable clone exact postings'); END; - CREATE TRIGGER immutable_clone_exact_postings_delete BEFORE DELETE ON clone_exact_postings BEGIN SELECT RAISE(ABORT, 'immutable clone exact postings'); END; - CREATE TRIGGER immutable_clone_fingerprint_postings_update BEFORE UPDATE ON clone_fingerprint_postings BEGIN SELECT RAISE(ABORT, 'immutable clone fingerprint postings'); END; - CREATE TRIGGER immutable_clone_fingerprint_postings_delete BEFORE DELETE ON clone_fingerprint_postings BEGIN SELECT RAISE(ABORT, 'immutable clone fingerprint postings'); END; - CREATE TRIGGER immutable_clone_fingerprint_counts_update BEFORE UPDATE ON clone_fingerprint_counts BEGIN SELECT RAISE(ABORT, 'immutable clone fingerprint counts'); END; - CREATE TRIGGER immutable_clone_fingerprint_counts_delete BEFORE DELETE ON clone_fingerprint_counts BEGIN SELECT RAISE(ABORT, 'immutable clone fingerprint counts'); END;"; - -fn reset_clone_tables(connection: &Connection) -> Result<(), CodeLexicalArtifactErrorV1> { - connection - .execute_batch(RESET_CLONE_TABLES_SQL) - .map_err(sqlite_error) -} - -/// Lookup indexes for resume verification, which reads postings by -/// occurrence. Both postings tables are keyed from `class`/`language`, so -/// without these every per-occurrence read is a full table scan; replaying N -/// committed pages after a restart then costs N scans of every posting the -/// repository has, and a daemon sat inside that replay for hours. A prior -/// artifact copied from a build that predates the indexes gains them here, -/// and nothing digests or enumerates the index schema. -fn ensure_clone_occurrence_indexes( - connection: &Connection, -) -> Result<(), CodeLexicalArtifactErrorV1> { - connection - .execute_batch( - "CREATE INDEX IF NOT EXISTS clone_exact_postings_by_occurrence ON clone_exact_postings(symbol_occurrence_id); - CREATE INDEX IF NOT EXISTS clone_fingerprint_postings_by_occurrence ON clone_fingerprint_postings(symbol_occurrence_id);", - ) - .map_err(sqlite_error) -} - -fn append_clone_rows( - transaction: &rusqlite::Transaction<'_>, - page: &VerifiedSealedLexicalPageV1, - control: &dyn CodeIndexExecutionControlV1, -) -> Result<(), CodeLexicalArtifactErrorV1> { - for body in page.clone_bodies() { - checkpoint(control)?; - let payload = serde_json::to_vec(&body.payload) - .map_err(|error| CodeLexicalArtifactErrorV1::Contract(error.to_string()))?; - let occurrence = serde_json::to_vec(&body.occurrence) - .map_err(|error| CodeLexicalArtifactErrorV1::Contract(error.to_string()))?; - transaction - .execute( - "INSERT INTO clone_body_payloads(payload_digest, payload) VALUES (?1, ?2) ON CONFLICT(payload_digest) DO NOTHING", - params![body.payload.payload_digest.as_str(), payload], - ) - .map_err(sqlite_error)?; - let stored: Vec = transaction - .query_row( - "SELECT payload FROM clone_body_payloads WHERE payload_digest = ?1", - [body.payload.payload_digest.as_str()], - |row| row.get(0), - ) - .map_err(sqlite_error)?; - if stored - != serde_json::to_vec(&body.payload) - .map_err(|error| CodeLexicalArtifactErrorV1::Contract(error.to_string()))? - { - return Err(CodeLexicalArtifactErrorV1::Corrupt( - "clone successor payload digest collision".to_owned(), - )); - } - transaction - .execute( - "INSERT INTO clone_occurrences(symbol_occurrence_id, payload_digest, path, body_start, body_end, occurrence) VALUES (?1, ?2, ?3, ?4, ?5, ?6)", - params![ - body.occurrence.symbol_occurrence_id.as_str(), - body.occurrence.payload_digest.as_str(), - body.occurrence.path, - i64::try_from(body.occurrence.body_span.start_byte) - .map_err(|error| CodeLexicalArtifactErrorV1::Contract(error.to_string()))?, - i64::try_from(body.occurrence.body_span.end_byte) - .map_err(|error| CodeLexicalArtifactErrorV1::Contract(error.to_string()))?, - occurrence, - ], - ) - .map_err(sqlite_error)?; - for key in body.payload.exact_keys(body.occurrence.eligibility) { - transaction - .execute( - "INSERT INTO clone_exact_postings(class, normalization_revision, digest, symbol_occurrence_id, payload_digest) VALUES (?1, ?2, ?3, ?4, ?5)", - params![ - i64::from(key.class as u8), - i64::from(key.normalization_revision), - key.digest.as_str(), - body.occurrence.symbol_occurrence_id.as_str(), - body.occurrence.payload_digest.as_str(), - ], - ) - .map_err(sqlite_error)?; - } - append_clone_fingerprints(transaction, body)?; - } - Ok(()) -} - -fn append_clone_fingerprints( - transaction: &rusqlite::Transaction<'_>, - body: &CodeIndexCloneBodyV1, -) -> Result<(), CodeLexicalArtifactErrorV1> { - let Some(stream) = body.payload.fingerprint_stream(body.occurrence.eligibility) else { - return Ok(()); - }; - for position in body - .payload - .fingerprint_positions(body.occurrence.eligibility) - .map_err(CodeLexicalArtifactErrorV1::Contract)? - { - transaction - .execute( - "INSERT INTO clone_fingerprint_postings(language, class, normalization_revision, fingerprint, symbol_occurrence_id, token_position, payload_digest, body_digest) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)", - params![ - body.payload.language, - i64::from(stream.class as u8), - i64::from(stream.normalization_revision), - i64::try_from(position.fingerprint) - .map_err(|error| CodeLexicalArtifactErrorV1::Contract(error.to_string()))?, - body.occurrence.symbol_occurrence_id.as_str(), - i64::from(position.token_position), - body.occurrence.payload_digest.as_str(), - body.payload.body_digest.as_str(), - ], - ) - .map_err(sqlite_error)?; - } - Ok(()) -} - -fn verify_copied_source_page( - connection: &Connection, - page: &VerifiedSealedLexicalPageV1, -) -> Result<(), CodeLexicalArtifactErrorV1> { - let stored: Option<(String, String, Vec)> = connection - .query_row( - "SELECT page_digest, cumulative_digest, next_cursor FROM source_pages WHERE page_ordinal = ?1", - [i64::try_from(page.page_ordinal()) - .map_err(|error| CodeLexicalArtifactErrorV1::Contract(error.to_string()))?], - |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), - ) - .optional() - .map_err(sqlite_error)?; - let next_cursor = page - .next_cursor() - .persisted_bytes() - .map_err(|error| CodeLexicalArtifactErrorV1::Contract(error.to_string()))?; - if stored - != Some(( - page.page_digest().as_str().to_owned(), - page.cumulative_digest().as_str().to_owned(), - next_cursor, - )) - { - return Err(CodeLexicalArtifactErrorV1::Corrupt( - "clone successor page does not match the copied lexical source receipt".to_owned(), - )); - } - Ok(()) -} - -type CloneExactRowV1 = (i64, i64, String, String); - -/// Postings for one page's occurrences, read through the occurrence indexes -/// `ensure_clone_occurrence_indexes` installs and bucketed by occurrence. -/// -/// The postings tables are keyed from `class`/`language`; before those -/// indexes existed a per-occurrence read scanned the whole table, and a -/// daemon spent eleven hours replaying committed pages after a restart. -struct ClonePagePostingsV1 { - exact: HashMap>, - fingerprints: HashMap>, -} - -impl ClonePagePostingsV1 { - fn read( - connection: &Connection, - occurrences: &HashSet<&str>, - control: &dyn CodeIndexExecutionControlV1, - ) -> Result { - let mut exact: HashMap> = HashMap::new(); - let mut statement = connection - .prepare( - "SELECT class, normalization_revision, digest, payload_digest FROM clone_exact_postings WHERE symbol_occurrence_id = ?1", - ) - .map_err(sqlite_error)?; - for occurrence in occurrences { - checkpoint(control)?; - let mut rows = statement.query([occurrence]).map_err(sqlite_error)?; - while let Some(row) = rows.next().map_err(sqlite_error)? { - exact.entry((*occurrence).to_owned()).or_default().push(( - row.get(0).map_err(sqlite_error)?, - row.get(1).map_err(sqlite_error)?, - row.get(2).map_err(sqlite_error)?, - row.get(3).map_err(sqlite_error)?, - )); - } - } - drop(statement); - - let mut fingerprints: HashMap> = HashMap::new(); - let mut statement = connection - .prepare( - "SELECT language, class, normalization_revision, fingerprint, token_position, payload_digest, body_digest FROM clone_fingerprint_postings WHERE symbol_occurrence_id = ?1", - ) - .map_err(sqlite_error)?; - for occurrence in occurrences { - checkpoint(control)?; - let mut rows = statement.query([occurrence]).map_err(sqlite_error)?; - while let Some(row) = rows.next().map_err(sqlite_error)? { - fingerprints - .entry((*occurrence).to_owned()) - .or_default() - .push(( - row.get(0).map_err(sqlite_error)?, - row.get(1).map_err(sqlite_error)?, - row.get(2).map_err(sqlite_error)?, - row.get(3).map_err(sqlite_error)?, - row.get(4).map_err(sqlite_error)?, - row.get(5).map_err(sqlite_error)?, - row.get(6).map_err(sqlite_error)?, - )); - } - } - drop(statement); - checkpoint(control)?; - - // Each table's primary key is unique within one occurrence, so sorting - // a bucket reproduces the `ORDER BY` the per-body queries used. - for rows in exact.values_mut() { - rows.sort(); - } - for rows in fingerprints.values_mut() { - rows.sort(); - } - Ok(Self { - exact, - fingerprints, - }) - } - - fn exact_for(&self, occurrence: &str) -> &[CloneExactRowV1] { - self.exact.get(occurrence).map_or(&[], Vec::as_slice) - } - - fn fingerprints_for(&self, occurrence: &str) -> &[CloneFingerprintRowV1] { - self.fingerprints.get(occurrence).map_or(&[], Vec::as_slice) - } -} - -fn verify_clone_page_rows( - connection: &Connection, - page: &VerifiedSealedLexicalPageV1, - control: &dyn CodeIndexExecutionControlV1, -) -> Result<(), CodeLexicalArtifactErrorV1> { - let occurrences = page - .clone_bodies() - .iter() - .map(|body| body.occurrence.symbol_occurrence_id.as_str()) - .collect::>(); - let postings = ClonePagePostingsV1::read(connection, &occurrences, control)?; - for body in page.clone_bodies() { - checkpoint(control)?; - let expected_payload = serde_json::to_vec(&body.payload) - .map_err(|error| CodeLexicalArtifactErrorV1::Contract(error.to_string()))?; - let stored_payload: Option> = connection - .query_row( - "SELECT payload FROM clone_body_payloads WHERE payload_digest = ?1", - [body.payload.payload_digest.as_str()], - |row| row.get(0), - ) - .optional() - .map_err(sqlite_error)?; - if stored_payload.as_deref() != Some(expected_payload.as_slice()) { - return Err(CodeLexicalArtifactErrorV1::Corrupt( - "resumed clone payload differs from its sealed source page".to_owned(), - )); - } - - let expected_occurrence = serde_json::to_vec(&body.occurrence) - .map_err(|error| CodeLexicalArtifactErrorV1::Contract(error.to_string()))?; - let stored_occurrence: Option<(String, String, i64, i64, Vec)> = connection - .query_row( - "SELECT payload_digest, path, body_start, body_end, occurrence FROM clone_occurrences WHERE symbol_occurrence_id = ?1", - [body.occurrence.symbol_occurrence_id.as_str()], - |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?, row.get(4)?)), - ) - .optional() - .map_err(sqlite_error)?; - let expected_span = ( - i64::try_from(body.occurrence.body_span.start_byte) - .map_err(|error| CodeLexicalArtifactErrorV1::Contract(error.to_string()))?, - i64::try_from(body.occurrence.body_span.end_byte) - .map_err(|error| CodeLexicalArtifactErrorV1::Contract(error.to_string()))?, - ); - if stored_occurrence.as_ref().map(|stored| { - ( - stored.0.as_str(), - stored.1.as_str(), - stored.2, - stored.3, - stored.4.as_slice(), - ) - }) != Some(( - body.occurrence.payload_digest.as_str(), - body.occurrence.path.as_str(), - expected_span.0, - expected_span.1, - expected_occurrence.as_slice(), - )) { - return Err(CodeLexicalArtifactErrorV1::Corrupt( - "resumed clone occurrence differs from its sealed source page".to_owned(), - )); - } - - let expected_postings = body - .payload - .exact_keys(body.occurrence.eligibility) - .into_iter() - .map(|key| { - ( - i64::from(key.class as u8), - i64::from(key.normalization_revision), - key.digest.as_str().to_owned(), - body.occurrence.payload_digest.as_str().to_owned(), - ) - }) - .collect::>(); - if postings.exact_for(body.occurrence.symbol_occurrence_id.as_str()) != expected_postings { - return Err(CodeLexicalArtifactErrorV1::Corrupt( - "resumed clone postings differ from their sealed source page".to_owned(), - )); - } - verify_clone_fingerprint_page_rows(&postings, body)?; - } - Ok(()) -} - -type CloneFingerprintRowV1 = (String, i64, i64, i64, i64, String, String); - -fn verify_clone_fingerprint_page_rows( - postings: &ClonePagePostingsV1, - body: &CodeIndexCloneBodyV1, -) -> Result<(), CodeLexicalArtifactErrorV1> { - let mut expected = Vec::new(); - if let Some(stream) = body.payload.fingerprint_stream(body.occurrence.eligibility) { - for position in body - .payload - .fingerprint_positions(body.occurrence.eligibility) - .map_err(CodeLexicalArtifactErrorV1::Contract)? - { - expected.push(( - body.payload.language.clone(), - i64::from(stream.class as u8), - i64::from(stream.normalization_revision), - i64::try_from(position.fingerprint) - .map_err(|error| CodeLexicalArtifactErrorV1::Contract(error.to_string()))?, - i64::from(position.token_position), - body.occurrence.payload_digest.as_str().to_owned(), - body.payload.body_digest.as_str().to_owned(), - )); - } - } - expected.sort(); - let stored = postings.fingerprints_for(body.occurrence.symbol_occurrence_id.as_str()); - if stored != expected { - return Err(CodeLexicalArtifactErrorV1::Corrupt( - "resumed clone fingerprints differ from their sealed source page".to_owned(), - )); - } - Ok(()) -} - -fn verify_source_receipt( - prior: &VerifiedCodeLexicalArtifactV1, - source: &VerifiedSealedLexicalSourceReceiptV1, -) -> Result<(), CodeLexicalArtifactErrorV1> { - if prior.source_state_digest() != source.source_state_digest() - || prior.source_cumulative_digest() != source.cumulative_digest() - || prior.page_count() != source.page_count() - || prior.total_chunks() != source.total_chunks() - || prior.total_payload_bytes() != source.total_payload_bytes() - || prior.total_imports() != source.total_imports() - || prior.import_payload_bytes() != source.import_payload_bytes() - || prior.import_dictionary_digest() != source.import_dictionary_digest() - { - return Err(CodeLexicalArtifactErrorV1::Corrupt( - "clone successor source receipt differs from its lexical predecessor".to_owned(), - )); - } - Ok(()) -} diff --git a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/fingerprints.rs b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/fingerprints.rs index 742d0ff9dc..fd52205b7a 100644 --- a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/fingerprints.rs +++ b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/fingerprints.rs @@ -11,9 +11,9 @@ use tracedecay_code_index::clones::{ use tracedecay_code_index::production::CodeIndexExecutionControlV1; use tracedecay_domain::{ManifestDigest, RetrieverCoverage, SymbolOccurrenceId, canonical_sha256}; -use super::format::{VerifiedCodeLexicalArtifactV1, contract_number}; +use super::clone_codec::{CloneOccurrenceRouteV1, routed_clone_body_row}; +use super::format::{VerifiedCodeLexicalArtifactV1, contract_number, decode_fingerprint_postings}; use super::reader::{CloneArtifactCursorPositionV1, CloneArtifactCursorV1, CloneArtifactPageV1}; -use super::schema::LexicalArtifactLayoutV1; use super::{CodeLexicalArtifactErrorV1, sqlite_error}; pub const CLONE_FINGERPRINT_POSTING_ROW_BUDGET_V1: u64 = 16_384; @@ -128,8 +128,9 @@ struct CandidateAccumulatorV1 { } pub(super) struct CloneFingerprintReadRequestV1<'a> { - pub(super) layout: LexicalArtifactLayoutV1, pub(super) receipt: &'a VerifiedCodeLexicalArtifactV1, + /// The opener's route, which also names the generation cursors bind. + pub(super) route: &'a CloneOccurrenceRouteV1, pub(super) authority_digest: &'a ManifestDigest, pub(super) authority: &'a CloneBodyOccurrenceV1, pub(super) source: &'a CloneBodyPayloadV1, @@ -145,8 +146,8 @@ pub(super) fn read_clone_fingerprint_page( ) -> Result { let started = Instant::now(); let CloneFingerprintReadRequestV1 { - layout, receipt, + route, authority_digest, authority, source, @@ -155,11 +156,6 @@ pub(super) fn read_clone_fingerprint_page( limit, control, } = request; - if !layout.has_clone_fingerprints() { - return Err(CodeLexicalArtifactErrorV1::Incompatible( - "clone fingerprint lookup requires lexical artifact revision 16".to_owned(), - )); - } if limit == 0 || limit > MAX_CLONE_FINGERPRINT_PAGE_BODIES_V1 { return Err(CodeLexicalArtifactErrorV1::Contract(format!( "clone fingerprint page limit must be within 1..={MAX_CLONE_FINGERPRINT_PAGE_BODIES_V1}" @@ -242,7 +238,7 @@ pub(super) fn read_clone_fingerprint_page( let after = match cursor { Some(cursor) if cursor.artifact_digest == *receipt.artifact_digest() - && cursor.generation == *receipt.generation() + && cursor.generation == route.source_generation && cursor.request_digest == request_digest => { match &cursor.after { @@ -284,9 +280,23 @@ pub(super) fn read_clone_fingerprint_page( }; let mut partial_reasons = BTreeSet::new(); let mut ordered_lists = Vec::with_capacity(positions_by_fingerprint.len()); + let mut list_statement = connection + .prepare_cached( + "SELECT postings FROM clone_fingerprint_postings WHERE language = ?1 AND class = ?2 AND normalization_revision = ?3 AND fingerprint = ?4", + ) + .map_err(sqlite_error)?; + let mut occurrence_statement = connection + .prepare_cached( + "SELECT occurrence.symbol_key, payload.payload_digest, occurrence.path, + occurrence.body_start, occurrence.body_end, occurrence.eligibility, payload.payload + FROM clone_occurrences AS occurrence + LEFT JOIN clone_body_payloads AS payload ON payload.ordinal = occurrence.payload_ordinal + WHERE occurrence.ordinal = ?1", + ) + .map_err(sqlite_error)?; let mut count_statement = connection .prepare_cached( - "SELECT posting_count FROM clone_fingerprint_counts WHERE language = ?1 AND class = ?2 AND normalization_revision = ?3 AND fingerprint = ?4", + "SELECT posting_count FROM clone_fingerprint_postings WHERE language = ?1 AND class = ?2 AND normalization_revision = ?3 AND fingerprint = ?4", ) .map_err(sqlite_error)?; for fingerprint in positions_by_fingerprint.keys().copied() { @@ -346,28 +356,28 @@ pub(super) fn read_clone_fingerprint_page( partial_reasons.insert(CloneFingerprintPartialReasonV1::PostingRowBudget); break; } - let mut statement = connection - .prepare_cached( - "SELECT posting.symbol_occurrence_id, posting.token_position, posting.payload_digest, posting.body_digest, occurrence.occurrence, payload.payload - FROM clone_fingerprint_postings AS posting - LEFT JOIN clone_occurrences AS occurrence ON occurrence.symbol_occurrence_id = posting.symbol_occurrence_id - LEFT JOIN clone_body_payloads AS payload ON payload.payload_digest = posting.payload_digest - WHERE posting.language = ?1 AND posting.class = ?2 AND posting.normalization_revision = ?3 AND posting.fingerprint = ?4 - ORDER BY posting.symbol_occurrence_id, posting.token_position - LIMIT ?5", - ) - .map_err(sqlite_error)?; let read_limit = remaining.min(posting_count); - let mut rows = statement - .query(rusqlite::params![ - descriptor.language, - i64::from(descriptor.class as u8), - i64::from(descriptor.normalization_revision), - i64::try_from(fingerprint).map_err(contract_number)?, - i64::try_from(read_limit).map_err(contract_number)?, - ]) + let list: Vec = list_statement + .query_row( + rusqlite::params![ + descriptor.language, + i64::from(descriptor.class as u8), + i64::from(descriptor.normalization_revision), + i64::try_from(fingerprint).map_err(contract_number)?, + ], + |row| row.get(0), + ) .map_err(sqlite_error)?; - while let Some(row) = rows.next().map_err(sqlite_error)? { + let postings = decode_fingerprint_postings(&list)?; + if u64::try_from(postings.len()).ok() != Some(posting_count) { + return Err(CodeLexicalArtifactErrorV1::Corrupt( + "clone fingerprint posting list disagrees with its stored count".to_owned(), + )); + } + for (occurrence_ordinal, candidate_position) in postings + .into_iter() + .take(usize::try_from(read_limit).map_err(contract_number)?) + { accounting.posting_rows_examined = accounting.posting_rows_examined.saturating_add(1); if interrupt( control, @@ -378,42 +388,16 @@ pub(super) fn read_clone_fingerprint_page( stop = true; break; } - let posting_occurrence: String = row.get(0).map_err(sqlite_error)?; - let candidate_position = u32::try_from(row.get::<_, i64>(1).map_err(sqlite_error)?) - .map_err(|_| { + let stored = occurrence_statement + .query_row([occurrence_ordinal], routed_clone_body_row) + .optional() + .map_err(sqlite_error)? + .ok_or_else(|| { CodeLexicalArtifactErrorV1::Corrupt( - "clone fingerprint token position is outside u32".to_owned(), + "clone fingerprint posting is missing its occurrence".to_owned(), ) })?; - let posting_payload: String = row.get(2).map_err(sqlite_error)?; - let posting_body: String = row.get(3).map_err(sqlite_error)?; - let occurrence_bytes: Option> = row.get(4).map_err(sqlite_error)?; - let payload_bytes: Option> = row.get(5).map_err(sqlite_error)?; - let (Some(occurrence_bytes), Some(payload_bytes)) = (occurrence_bytes, payload_bytes) - else { - return Err(CodeLexicalArtifactErrorV1::Corrupt( - "clone fingerprint posting is missing its occurrence or payload".to_owned(), - )); - }; - let occurrence: CloneBodyOccurrenceV1 = serde_json::from_slice(&occurrence_bytes) - .map_err(|error| CodeLexicalArtifactErrorV1::Corrupt(error.to_string()))?; - let payload: CloneBodyPayloadV1 = serde_json::from_slice(&payload_bytes) - .map_err(|error| CodeLexicalArtifactErrorV1::Corrupt(error.to_string()))?; - if occurrence.symbol_occurrence_id.as_str() != posting_occurrence - || occurrence.project_id != authority.project_id - || occurrence.repository_id != authority.repository_id - || occurrence.worktree_id != authority.worktree_id - || occurrence.source_generation != *receipt.generation() - || occurrence.payload_digest.as_str() != posting_payload - || occurrence.payload_digest != payload.payload_digest - || payload.body_digest.as_str() != posting_body - || payload.validate().is_err() - { - return Err(CodeLexicalArtifactErrorV1::Corrupt( - "clone fingerprint posting does not match its payload and occurrence" - .to_owned(), - )); - } + let (occurrence, payload) = route.occurrence_and_payload(stored)?; if occurrence.symbol_occurrence_id == authority.symbol_occurrence_id || payload.language != source.language || (selected_block.is_none() @@ -650,7 +634,7 @@ pub(super) fn read_clone_fingerprint_page( let next_cursor = if has_more { last_compared.map(|(body_digest, payload_digest)| CloneArtifactCursorV1 { artifact_digest: receipt.artifact_digest().clone(), - generation: receipt.generation().clone(), + generation: route.source_generation.clone(), request_digest, after: CloneArtifactCursorPositionV1::Fingerprint { body_digest, diff --git a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/format.rs b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/format.rs index 70bcfe2642..f2338f0c54 100644 --- a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/format.rs +++ b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/format.rs @@ -1,5 +1,6 @@ use std::collections::BTreeMap; +use flate2::{Compress, Compression, Decompress, FlushCompress, FlushDecompress, Status}; use roaring::RoaringBitmap; use rusqlite::{Connection, OptionalExtension}; use serde::{Deserialize, Serialize}; @@ -8,18 +9,15 @@ use tracedecay_code_index::chunks::CodeIndexImportEvidenceV1; use tracedecay_code_index::production::CodeIndexExecutionControlV1; use tracedecay_domain::{ BoundedSanitizedText, CodeGenerationId, CodeSearchChunkAnchorV1, CodeSearchChunkId, - ExactFieldV1, ExactTechnicalTermV1, FileOccurrenceId, LanguageDescriptorRevision, - ManifestDigest, RepositoryId, SourceFreshness, SourceSpan, SymbolOccurrenceId, + ComponentRevision, ExactFieldV1, ExactTechnicalTermV1, FileOccurrenceId, + LanguageDescriptorRevision, ManifestDigest, ScoreDomainId, SourceSpan, SymbolOccurrenceId, }; use super::super::{CodeLexicalProjectionMetadataV1, LexicalFieldV1, ProjectedChunkV1}; use super::CodeLexicalArtifactErrorV1; -use super::schema::{LexicalArtifactLayoutV1, digest_domain_for_revision}; +use super::schema::{CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V1, digest_domain_for_revision}; -pub(super) use super::schema::{ - CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V1, SERVING_INDEX_STEP_COUNT_V11, - STATISTICS_STEP_COUNT_V11, -}; +pub(super) use super::schema::{SERVING_INDEX_STEP_COUNT_V11, STATISTICS_STEP_COUNT_V11}; /// Revision 2 adds durable finalization/integrity state. Revision 1 artifacts /// are branch-only staging files and must fail as incompatible rather than be @@ -46,7 +44,7 @@ pub(super) use super::schema::{ // Revision 15 adds independently digested clone payload, occurrence, and // exact-posting sections without changing lexical document integrity. pub(super) const RECEIPT_RESERVATION_BYTES: usize = 16 * 1024; -pub(super) const SECTION_NAMES: [&str; 16] = [ +pub(super) const SECTION_NAMES: [&str; 14] = [ "source_pages", "document_integrity", "import_integrity", @@ -56,12 +54,10 @@ pub(super) const SECTION_NAMES: [&str; 16] = [ "exact_postings", "ngram_postings", "field_stats", - "term_stats", "vocabulary", "clone_occurrences", "clone_exact_postings", "clone_body_payloads", - "clone_fingerprint_counts", "clone_fingerprint_postings", ]; pub(super) const BASE_SECTION_NAMES: [&str; 7] = [ @@ -74,16 +70,6 @@ pub(super) const BASE_SECTION_NAMES: [&str; 7] = [ "ngram_postings", ]; -pub(super) fn section_names(layout: LexicalArtifactLayoutV1) -> &'static [&'static str] { - if layout.has_clone_fingerprints() { - &SECTION_NAMES - } else if layout.has_clone_index() { - &SECTION_NAMES[..14] - } else { - &SECTION_NAMES[..11] - } -} - #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub(super) struct CodeLexicalArtifactPageBaseSectionsReceiptV1 { @@ -325,430 +311,150 @@ fn validate_page_base_sections( Ok(()) } -pub(super) fn verify_required_artifact_indexes( - connection: &Connection, - layout: LexicalArtifactLayoutV1, -) -> Result<(), CodeLexicalArtifactErrorV1> { - verify_artifact_table_layout(connection, layout)?; - let mut statement = connection - .prepare("SELECT name, desc, coll FROM pragma_index_xinfo(?1) WHERE key = 1 ORDER BY seqno") - .map_err(|error| { - CodeLexicalArtifactErrorV1::Incompatible(format!( - "artifact index schema is unreadable: {error}" - )) - })?; - for (table, index, expected_columns) in layout.required_indexes() { - let partial: Option = connection - .query_row( - "SELECT partial FROM pragma_index_list(?1) WHERE name = ?2", - [table, index], - |row| row.get(0), - ) - .optional() - .map_err(|error| { - CodeLexicalArtifactErrorV1::Incompatible(format!( - "artifact index {index} is unreadable: {error}" - )) - })?; - let columns = statement - .query_map([index], |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, i64>(1)?, - row.get::<_, String>(2)?, - )) - }) - .map_err(|error| { - CodeLexicalArtifactErrorV1::Incompatible(format!( - "artifact index {index} is unreadable: {error}" - )) - })? - .collect::, _>>() - .map_err(|error| { - CodeLexicalArtifactErrorV1::Incompatible(format!( - "artifact index {index} is unreadable: {error}" - )) - })?; - if partial != Some(0) - || !columns - .iter() - .map(|(column, descending, collation)| { - (column.as_str(), *descending, collation.as_str()) - }) - .eq(expected_columns.iter().map(|column| (*column, 0, "BINARY"))) - { - return Err(CodeLexicalArtifactErrorV1::Incompatible(format!( - "artifact index {index} has columns {columns:?}; revision {} requires {expected_columns:?}", - layout.revision() - ))); - } - } - Ok(()) -} +/// `(column, type, NOT NULL, primary-key ordinal)` as `pragma_table_xinfo` +/// reports it. +type ColumnShapeV1 = (&'static str, &'static str, i64, i64); + +/// `(table, WITHOUT ROWID, columns)` for every table a staging or sealed +/// artifact always carries. Staging-only append tables are dropped by +/// finalization and carry no serving contract. +const ARTIFACT_TABLE_LAYOUT: [(&str, bool, &[ColumnShapeV1]); 13] = [ + ( + "source_pages", + false, + &[ + ("page_ordinal", "INTEGER", 0, 1), + ("chunk_count", "INTEGER", 1, 0), + ("import_count", "INTEGER", 1, 0), + ("import_payload_bytes", "INTEGER", 1, 0), + ("import_dictionary_digest", "TEXT", 1, 0), + ("ngram_digest", "TEXT", 1, 0), + ("base_sections_receipt", "BLOB", 1, 0), + ], + ), + ("import_evidence", true, &[("canonical", "BLOB", 1, 1)]), + ( + "row_blocks", + false, + &[ + ("first_document", "INTEGER", 0, 1), + ("payload", "BLOB", 1, 0), + ], + ), + ( + "row_chunks", + true, + &[("chunk_id", "BLOB", 1, 1), ("document_id", "INTEGER", 1, 0)], + ), + ( + "term_postings", + true, + &[ + ("term", "TEXT", 1, 1), + ("in_fuzzy", "INTEGER", 1, 0), + ("lists", "BLOB", 1, 0), + ], + ), + ( + "exact_postings", + true, + &[ + ("term_id", "INTEGER", 1, 1), + ("field", "INTEGER", 1, 2), + ("documents", "BLOB", 1, 0), + ], + ), + ( + "ngram_postings", + true, + &[ + ("kind", "INTEGER", 1, 1), + ("ngram", "INTEGER", 1, 2), + ("document_frequency", "INTEGER", 1, 0), + ("documents", "BLOB", 1, 0), + ], + ), + ( + "exact_vocabulary", + false, + &[("term_id", "INTEGER", 0, 1), ("term", "BLOB", 1, 0)], + ), + ( + "row_dictionary", + false, + &[("entry_id", "INTEGER", 0, 1), ("entry", "BLOB", 1, 0)], + ), + ( + "clone_body_payloads", + false, + &[ + ("ordinal", "INTEGER", 0, 1), + ("payload_digest", "BLOB", 1, 0), + ("payload", "BLOB", 1, 0), + ], + ), + ( + "clone_occurrences", + false, + &[ + ("ordinal", "INTEGER", 0, 1), + ("symbol_key", "BLOB", 1, 0), + ("payload_ordinal", "INTEGER", 1, 0), + ("path", "TEXT", 1, 0), + ("body_start", "INTEGER", 1, 0), + ("body_end", "INTEGER", 1, 0), + ("eligibility", "BLOB", 1, 0), + ], + ), + ( + "clone_exact_postings", + true, + &[ + ("class", "INTEGER", 1, 1), + ("normalization_revision", "INTEGER", 1, 2), + ("digest", "BLOB", 1, 3), + ("occurrence_ordinal", "INTEGER", 1, 4), + ], + ), + ( + "clone_fingerprint_postings", + true, + &[ + ("language", "TEXT", 1, 1), + ("class", "INTEGER", 1, 2), + ("normalization_revision", "INTEGER", 1, 3), + ("fingerprint", "INTEGER", 1, 4), + ("posting_count", "INTEGER", 1, 0), + ("postings", "BLOB", 1, 0), + ], + ), +]; pub(super) fn verify_artifact_table_layout( connection: &Connection, - layout: LexicalArtifactLayoutV1, -) -> Result<(), CodeLexicalArtifactErrorV1> { - let source_columns = connection - .prepare( - "SELECT name, type, [notnull], pk FROM pragma_table_xinfo('source_pages') WHERE hidden = 0 ORDER BY cid", - ) - .and_then(|mut statement| { - statement - .query_map([], |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, String>(1)?, - row.get::<_, i64>(2)?, - row.get::<_, i64>(3)?, - )) - })? - .collect::, _>>() - }) - .map_err(|error| { - CodeLexicalArtifactErrorV1::Incompatible(format!( - "artifact source-page columns are unreadable: {error}" - )) - })?; - let expected_source_columns = [ - ("page_ordinal", "INTEGER", 0, 1), - ("page_digest", "TEXT", 1, 0), - ("cumulative_digest", "TEXT", 1, 0), - ("chunk_count", "INTEGER", 1, 0), - ("payload_bytes", "INTEGER", 1, 0), - ("import_count", "INTEGER", 1, 0), - ("import_payload_bytes", "INTEGER", 1, 0), - ("import_dictionary_digest", "TEXT", 1, 0), - ("ngram_digest", "TEXT", 1, 0), - ("base_sections_receipt", "BLOB", 1, 0), - ("next_cursor", "BLOB", 1, 0), - ]; - if !source_columns - .iter() - .map(|(name, column_type, not_null, primary_key)| { - (name.as_str(), column_type.as_str(), *not_null, *primary_key) - }) - .eq(expected_source_columns) - { - return Err(CodeLexicalArtifactErrorV1::Incompatible(format!( - "artifact source-page table has columns {source_columns:?}; revision {CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V1} requires append receipts" - ))); - } - let without_rowid: Option = connection - .query_row( - "SELECT wr FROM pragma_table_list WHERE schema = 'main' AND name = 'ngram_postings' AND type = 'table'", - [], - |row| row.get(0), - ) - .optional() - .map_err(|error| { - CodeLexicalArtifactErrorV1::Incompatible(format!( - "artifact ngram table schema is unreadable: {error}" - )) - })?; - let mut statement = connection - .prepare( - "SELECT name, type, [notnull], pk FROM pragma_table_xinfo('ngram_postings') WHERE hidden = 0 ORDER BY cid", - ) - .map_err(|error| { - CodeLexicalArtifactErrorV1::Incompatible(format!( - "artifact ngram columns are unreadable: {error}" - )) - })?; - let columns = statement - .query_map([], |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, String>(1)?, - row.get::<_, i64>(2)?, - row.get::<_, i64>(3)?, - )) - }) - .map_err(|error| { - CodeLexicalArtifactErrorV1::Incompatible(format!( - "artifact ngram columns are unreadable: {error}" - )) - })? - .collect::, _>>() - .map_err(|error| { - CodeLexicalArtifactErrorV1::Incompatible(format!( - "artifact ngram columns are unreadable: {error}" - )) - })?; - let expected = [ - ("page_ordinal", "INTEGER", 1, 1), - ("kind", "INTEGER", 1, 2), - ("ngram", "INTEGER", 1, 3), - ("documents", "BLOB", 1, 0), - ("cardinality", "INTEGER", 1, 0), - ]; - if without_rowid != Some(1) - || !columns - .iter() - .map(|(name, column_type, not_null, primary_key)| { - (name.as_str(), column_type.as_str(), *not_null, *primary_key) - }) - .eq(expected) - { - return Err(CodeLexicalArtifactErrorV1::Incompatible(format!( - "artifact ngram table has columns {columns:?} and without-rowid state {without_rowid:?}; revision {CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V1} requires source-page bitmap shards" - ))); - } - let statistics_without_rowid: Option = connection - .query_row( - "SELECT wr FROM pragma_table_list WHERE schema = 'main' AND name = 'ngram_statistics' AND type = 'table'", - [], - |row| row.get(0), - ) - .optional() - .map_err(|error| { - CodeLexicalArtifactErrorV1::Incompatible(format!( - "artifact ngram statistics schema is unreadable: {error}" - )) - })?; - let statistics_columns = connection - .prepare( - "SELECT name, type, [notnull], pk FROM pragma_table_xinfo('ngram_statistics') WHERE hidden = 0 ORDER BY cid", - ) - .and_then(|mut statement| { - statement - .query_map([], |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, String>(1)?, - row.get::<_, i64>(2)?, - row.get::<_, i64>(3)?, - )) - })? - .collect::, _>>() - }) - .map_err(|error| { - CodeLexicalArtifactErrorV1::Incompatible(format!( - "artifact ngram statistics columns are unreadable: {error}" - )) - })?; - let expected_statistics = [ - ("kind", "INTEGER", 1, 1), - ("ngram", "INTEGER", 1, 2), - ("document_frequency", "INTEGER", 1, 0), - ]; - if statistics_without_rowid != Some(1) - || !statistics_columns - .iter() - .map(|(name, column_type, not_null, primary_key)| { - (name.as_str(), column_type.as_str(), *not_null, *primary_key) - }) - .eq(expected_statistics) - { - return Err(CodeLexicalArtifactErrorV1::Incompatible(format!( - "artifact ngram statistics table has columns {statistics_columns:?} and without-rowid state {statistics_without_rowid:?}; revision {} requires finalized selectivity statistics", - layout.revision() - ))); - } - if layout != LexicalArtifactLayoutV1::V10 { - verify_interned_term_layout(connection, layout)?; - } - Ok(()) -} - -fn verify_interned_term_layout( - connection: &Connection, - layout: LexicalArtifactLayoutV1, ) -> Result<(), CodeLexicalArtifactErrorV1> { - let without_rowid: Option = connection - .query_row( - "SELECT wr FROM pragma_table_list WHERE schema = 'main' AND name = 'term_postings' AND type = 'table'", - [], - |row| row.get(0), - ) - .optional() - .map_err(|error| { - CodeLexicalArtifactErrorV1::Incompatible(format!( - "artifact term posting schema is unreadable: {error}" - )) - })?; - let columns = table_columns(connection, "term_postings")?; - // Column order is shared; only the clustered key positions differ - // between the term-leading (11/12) and document-leading (13) layouts. - let (term_key, field_key, document_key) = if layout.clusters_term_postings_by_document() { - (2, 3, 1) - } else { - (1, 2, 3) - }; - let expected = [ - ("term_id", "INTEGER", 1, term_key), - ("field", "INTEGER", 1, field_key), - ("document_id", "INTEGER", 1, document_key), - ("frequency", "INTEGER", 1, 0), - ]; - if without_rowid != Some(1) - || !columns - .iter() - .map(|(name, column_type, not_null, primary_key)| { - (name.as_str(), column_type.as_str(), *not_null, *primary_key) - }) - .eq(expected) - { - return Err(CodeLexicalArtifactErrorV1::Incompatible(format!( - "artifact term posting table has columns {columns:?}; revision {} requires interned term identifiers in its clustered key order", - layout.revision() - ))); - } - let vocabulary = table_columns(connection, "vocabulary")?; - let expected_vocabulary = [ - ("term_id", "INTEGER", 0, 1), - ("term", "TEXT", 1, 0), - ("in_fuzzy", "INTEGER", 1, 0), - ]; - if !vocabulary - .iter() - .map(|(name, column_type, not_null, primary_key)| { - (name.as_str(), column_type.as_str(), *not_null, *primary_key) - }) - .eq(expected_vocabulary) - { - return Err(CodeLexicalArtifactErrorV1::Incompatible(format!( - "artifact vocabulary table has columns {vocabulary:?}; revision 11 requires interned terms" - ))); - } - if layout.interns_exact_terms() { - let exact_without_rowid: Option = connection + for (table, expected_without_rowid, expected_columns) in ARTIFACT_TABLE_LAYOUT { + let without_rowid: Option = connection .query_row( - "SELECT wr FROM pragma_table_list WHERE schema = 'main' AND name = 'exact_postings' AND type = 'table'", - [], + "SELECT wr FROM pragma_table_list WHERE schema = 'main' AND name = ?1 AND type = 'table'", + [table], |row| row.get(0), ) .optional() .map_err(|error| { CodeLexicalArtifactErrorV1::Incompatible(format!( - "artifact exact posting schema is unreadable: {error}" + "artifact {table} schema is unreadable: {error}" )) })?; - let exact_columns = table_columns(connection, "exact_postings")?; - let expected_exact = [ - ("term_id", "INTEGER", 1, 1), - ("field", "INTEGER", 1, 2), - ("document_id", "INTEGER", 1, 3), - ]; - if exact_without_rowid != Some(1) - || !exact_columns - .iter() - .map(|(name, column_type, not_null, primary_key)| { - (name.as_str(), column_type.as_str(), *not_null, *primary_key) - }) - .eq(expected_exact) - { - return Err(CodeLexicalArtifactErrorV1::Incompatible(format!( - "artifact exact posting table has columns {exact_columns:?}; revision 12 requires interned exact term identifiers" - ))); - } - let exact_vocabulary = table_columns(connection, "exact_vocabulary")?; - let expected_exact_vocabulary = [("term_id", "INTEGER", 0, 1), ("term", "BLOB", 1, 0)]; - if !exact_vocabulary - .iter() - .map(|(name, column_type, not_null, primary_key)| { - (name.as_str(), column_type.as_str(), *not_null, *primary_key) - }) - .eq(expected_exact_vocabulary) - { - return Err(CodeLexicalArtifactErrorV1::Incompatible(format!( - "artifact exact vocabulary table has columns {exact_vocabulary:?}; revision 12 requires exact term collision authority" - ))); - } - } - if layout.stores_document_integrity_bytes() { - let document_integrity = table_columns(connection, "document_integrity")?; - let expected_document_integrity = - [("document_id", "INTEGER", 0, 1), ("digest", "BLOB", 1, 0)]; - if !document_integrity - .iter() - .map(|(name, column_type, not_null, primary_key)| { - (name.as_str(), column_type.as_str(), *not_null, *primary_key) - }) - .eq(expected_document_integrity) - { - return Err(CodeLexicalArtifactErrorV1::Incompatible(format!( - "artifact document integrity table has columns {document_integrity:?}; revision 14 requires raw digest bytes keyed by document" - ))); - } - } - if layout.interns_row_dictionary() { - let row_dictionary = table_columns(connection, "row_dictionary")?; - let expected_row_dictionary = [("entry_id", "INTEGER", 0, 1), ("entry", "BLOB", 1, 0)]; - if !row_dictionary - .iter() - .map(|(name, column_type, not_null, primary_key)| { - (name.as_str(), column_type.as_str(), *not_null, *primary_key) - }) - .eq(expected_row_dictionary) + let columns = table_columns(connection, table)?; + if without_rowid != Some(i64::from(expected_without_rowid)) + || !table_column_shapes(&columns).eq(expected_columns.iter().copied()) { return Err(CodeLexicalArtifactErrorV1::Incompatible(format!( - "artifact row dictionary table has columns {row_dictionary:?}; revision 14 requires a per-file/per-symbol row dictionary" + "artifact {table} table has columns {columns:?} and without-rowid state {without_rowid:?}; revision {CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V1} requires {expected_columns:?}" ))); } } - verify_clone_table_layout(connection, layout)?; - Ok(()) -} - -fn verify_clone_table_layout( - connection: &Connection, - layout: LexicalArtifactLayoutV1, -) -> Result<(), CodeLexicalArtifactErrorV1> { - if !layout.has_clone_index() { - return Ok(()); - } - let payloads = table_columns(connection, "clone_body_payloads")?; - let occurrences = table_columns(connection, "clone_occurrences")?; - let postings = table_columns(connection, "clone_exact_postings")?; - if !table_column_shapes(&payloads) - .eq([("payload_digest", "TEXT", 1, 1), ("payload", "BLOB", 1, 0)]) - || !table_column_shapes(&occurrences).eq([ - ("symbol_occurrence_id", "TEXT", 1, 1), - ("payload_digest", "TEXT", 1, 0), - ("path", "TEXT", 1, 0), - ("body_start", "INTEGER", 1, 0), - ("body_end", "INTEGER", 1, 0), - ("occurrence", "BLOB", 1, 0), - ]) - || !table_column_shapes(&postings).eq([ - ("class", "INTEGER", 1, 1), - ("normalization_revision", "INTEGER", 1, 2), - ("digest", "TEXT", 1, 3), - ("symbol_occurrence_id", "TEXT", 1, 4), - ("payload_digest", "TEXT", 1, 0), - ]) - { - return Err(CodeLexicalArtifactErrorV1::Incompatible( - "revision 15 requires clone payload, occurrence, and exact-posting tables".to_owned(), - )); - } - if layout.has_clone_fingerprints() { - let counts = table_columns(connection, "clone_fingerprint_counts")?; - let fingerprints = table_columns(connection, "clone_fingerprint_postings")?; - if !table_column_shapes(&counts).eq([ - ("language", "TEXT", 1, 1), - ("class", "INTEGER", 1, 2), - ("normalization_revision", "INTEGER", 1, 3), - ("fingerprint", "INTEGER", 1, 4), - ("posting_count", "INTEGER", 1, 0), - ]) || !table_column_shapes(&fingerprints).eq([ - ("language", "TEXT", 1, 1), - ("class", "INTEGER", 1, 2), - ("normalization_revision", "INTEGER", 1, 3), - ("fingerprint", "INTEGER", 1, 4), - ("symbol_occurrence_id", "TEXT", 1, 5), - ("token_position", "INTEGER", 1, 6), - ("payload_digest", "TEXT", 1, 0), - ("body_digest", "TEXT", 1, 0), - ]) { - return Err(CodeLexicalArtifactErrorV1::Incompatible( - "revision 16 requires positional clone fingerprint postings and stored counts" - .to_owned(), - )); - } - } Ok(()) } @@ -819,230 +525,479 @@ pub(super) fn ngram_page_digest<'a>( } pub(super) fn encode_ngram_bitmap( - layout: LexicalArtifactLayoutV1, bitmap: &RoaringBitmap, ) -> Result, CodeLexicalArtifactErrorV1> { - match layout { - LexicalArtifactLayoutV1::V10 | LexicalArtifactLayoutV1::V11 => { - encode_ngram_bitmap_v11(bitmap) - } - LexicalArtifactLayoutV1::V12 - | LexicalArtifactLayoutV1::V13 - | LexicalArtifactLayoutV1::V14 - | LexicalArtifactLayoutV1::V15 - | LexicalArtifactLayoutV1::V16 => encode_ngram_delta_varints_v12(bitmap), + let mut encoder = PostingListEncoderV1::new(false); + for document in bitmap { + encoder.push(document, 1)?; } + encoder.finish() } -fn encode_ngram_bitmap_v11(bitmap: &RoaringBitmap) -> Result, CodeLexicalArtifactErrorV1> { - let cardinality = bitmap.len(); - let mut range_count = 0u64; - let mut previous: Option = None; - for document in bitmap.iter() { - if previous.is_none_or(|previous| document != previous.saturating_add(1)) { - range_count = range_count.checked_add(1).ok_or_else(|| { - CodeLexicalArtifactErrorV1::Contract( - "lexical artifact ngram range count overflowed".to_owned(), - ) - })?; - } - previous = Some(document); +pub(super) fn decode_ngram_bitmap( + encoded: &[u8], +) -> Result { + if encoded.is_empty() { + return Err(CodeLexicalArtifactErrorV1::Corrupt( + "lexical artifact document list is empty".to_owned(), + )); } - let list_bytes = cardinality.checked_mul(4).ok_or_else(|| { - CodeLexicalArtifactErrorV1::Contract( - "lexical artifact ngram document-list size overflowed".to_owned(), - ) - })?; - let range_bytes = range_count.checked_mul(8).ok_or_else(|| { - CodeLexicalArtifactErrorV1::Contract( - "lexical artifact ngram range size overflowed".to_owned(), - ) - })?; - let ranges = range_bytes < list_bytes; - let payload_bytes = if ranges { range_bytes } else { list_bytes }; - let capacity = usize::try_from(payload_bytes) - .map_err(|error| CodeLexicalArtifactErrorV1::Contract(error.to_string()))? - .checked_add(16) - .ok_or_else(|| { - CodeLexicalArtifactErrorV1::Contract( - "lexical artifact ngram bitmap size overflowed".to_owned(), + let mut bitmap = RoaringBitmap::new(); + for posting in PostingListDecoderV1::new(encoded, false) { + let (document, _) = posting?; + bitmap.try_push(document).map_err(|_| { + CodeLexicalArtifactErrorV1::Corrupt( + "lexical artifact document list is not strictly ascending".to_owned(), ) })?; - let mut encoded = Vec::with_capacity(capacity); - encoded.extend_from_slice(b"TDN1"); - encoded.push(u8::from(ranges)); - encoded.extend_from_slice(&[0u8; 3]); - encoded.extend_from_slice(&cardinality.to_le_bytes()); - if ranges { - let mut start: Option = None; - let mut previous: Option = None; - for document in bitmap.iter() { - if previous.is_some_and(|previous| document != previous.saturating_add(1)) { - let start_value = start.ok_or_else(|| { - CodeLexicalArtifactErrorV1::Contract( - "lexical artifact ngram range is missing its start".to_owned(), - ) - })?; - let previous_value = previous.ok_or_else(|| { - CodeLexicalArtifactErrorV1::Contract( - "lexical artifact ngram range is missing its end".to_owned(), - ) - })?; - encoded.extend_from_slice(&start_value.to_le_bytes()); - encoded.extend_from_slice(&(previous_value - start_value).to_le_bytes()); - start = Some(document); - } else if start.is_none() { - start = Some(document); + } + Ok(bitmap) +} + +const DOCUMENT_SET_DELTAS: u8 = 0; +const DOCUMENT_SET_BITSET: u8 = 1; + +/// A sealed n-gram document set in the smaller of two tagged encodings: the +/// delta-varint list, or its first document followed by a bitset over the +/// range it spans (bit `i` of byte `i / 8`, least significant first, is +/// document `first + i`). The bitset wins once a list holds more than about +/// one document in eight of that range, which the most common n-grams do. +pub(super) fn encode_document_set( + documents: &RoaringBitmap, +) -> Result, CodeLexicalArtifactErrorV1> { + let deltas = encode_ngram_bitmap(documents)?; + let (Some(first), Some(last)) = (documents.min(), documents.max()) else { + return Err(CodeLexicalArtifactErrorV1::Contract( + "lexical artifact document set is empty".to_owned(), + )); + }; + let mut prefix = Vec::with_capacity(6); + prefix.push(DOCUMENT_SET_BITSET); + encode_varint(u64::from(first), &mut prefix); + let bitset_bytes = usize::try_from(u64::from(last - first) / 8 + 1) + .map_err(|error| CodeLexicalArtifactErrorV1::Contract(error.to_string()))?; + if prefix.len() + bitset_bytes > deltas.len() { + let mut encoded = Vec::with_capacity(1 + deltas.len()); + encoded.push(DOCUMENT_SET_DELTAS); + encoded.extend_from_slice(&deltas); + return Ok(encoded); + } + let offset = prefix.len(); + prefix.resize(offset + bitset_bytes, 0); + for document in documents { + let bit = document - first; + prefix[offset + (bit / 8) as usize] |= 1 << (bit % 8); + } + Ok(prefix) +} + +pub(super) fn decode_document_set( + encoded: &[u8], +) -> Result { + let corrupt = |detail: &str| { + CodeLexicalArtifactErrorV1::Corrupt(format!("lexical artifact document set {detail}")) + }; + match encoded.split_first() { + Some((&DOCUMENT_SET_DELTAS, deltas)) => decode_ngram_bitmap(deltas), + Some((&DOCUMENT_SET_BITSET, mut bitset)) => { + let first = u32::try_from(take_varint(&mut bitset)?) + .map_err(|_| corrupt("start overflows u32"))?; + // Canonical: the range starts and ends on a member. + if bitset.first().is_none_or(|byte| byte & 1 == 0) + || bitset.last().is_some_and(|byte| *byte == 0) + { + return Err(corrupt("bitset is not canonical")); + } + let mut documents = RoaringBitmap::new(); + for (index, byte) in bitset.iter().enumerate() { + for bit in 0..8u32 { + if byte & (1 << bit) == 0 { + continue; + } + let document = u32::try_from(index) + .ok() + .and_then(|index| index.checked_mul(8)) + .and_then(|offset| offset.checked_add(bit)) + .and_then(|offset| first.checked_add(offset)) + .ok_or_else(|| corrupt("bitset overflows u32"))?; + documents + .try_push(document) + .map_err(|_| corrupt("bitset is not ascending"))?; + } } - previous = Some(document); + Ok(documents) } - if let (Some(start), Some(previous)) = (start, previous) { - encoded.extend_from_slice(&start.to_le_bytes()); - encoded.extend_from_slice(&(previous - start).to_le_bytes()); + _ => Err(corrupt("has an unknown encoding tag")), + } +} + +/// Stored bytes as one tag byte, the varint inflated length, and the raw +/// deflate stream of `bytes`. +pub(super) fn deflate_bytes(tag: u8, bytes: &[u8]) -> Result, CodeLexicalArtifactErrorV1> { + let mut compressor = Compress::new(Compression::best(), false); + let mut compressed = Vec::with_capacity(bytes.len() / 2 + 64); + loop { + if compressed.capacity() - compressed.len() < 1024 { + compressed.reserve(bytes.len() / 4 + 1024); } - } else { - for document in bitmap.iter() { - encoded.extend_from_slice(&document.to_le_bytes()); + let consumed = usize::try_from(compressor.total_in()).map_err(contract_number)?; + let status = compressor + .compress_vec(&bytes[consumed..], &mut compressed, FlushCompress::Finish) + .map_err(contract_number)?; + if status == Status::StreamEnd { + break; } } - Ok(encoded) + let mut stored = Vec::with_capacity(compressed.len() + 11); + stored.push(tag); + encode_varint( + u64::try_from(bytes.len()).map_err(contract_number)?, + &mut stored, + ); + stored.extend_from_slice(&compressed); + Ok(stored) } -pub(super) fn decode_ngram_bitmap( - layout: LexicalArtifactLayoutV1, - encoded: &[u8], -) -> Result { - match layout { - LexicalArtifactLayoutV1::V10 | LexicalArtifactLayoutV1::V11 => { - decode_ngram_bitmap_v11(encoded) - } - LexicalArtifactLayoutV1::V12 - | LexicalArtifactLayoutV1::V13 - | LexicalArtifactLayoutV1::V14 - | LexicalArtifactLayoutV1::V15 - | LexicalArtifactLayoutV1::V16 => decode_ngram_delta_varints_v12(encoded), +/// Inverse of [`deflate_bytes`]: refuse another tag, an inflated length above +/// `maximum`, or a stream that does not inflate to exactly its length. +pub(super) fn inflate_bytes( + tag: u8, + stored: &[u8], + maximum: usize, +) -> Result, CodeLexicalArtifactErrorV1> { + let corrupt = |detail: &str| { + CodeLexicalArtifactErrorV1::Corrupt(format!("lexical artifact deflated value {detail}")) + }; + let Some((&stored_tag, mut rest)) = stored.split_first() else { + return Err(corrupt("is empty")); + }; + if stored_tag != tag { + return Err(corrupt("has an unknown encoding tag")); } + let length = usize::try_from(take_varint(&mut rest)?).map_err(|_| corrupt("is too long"))?; + if length > maximum { + return Err(corrupt("exceeds its inflated bound")); + } + let mut decompressor = Decompress::new(false); + let mut inflated = Vec::with_capacity(length); + let status = decompressor + .decompress_vec(rest, &mut inflated, FlushDecompress::Finish) + .map_err(|error| CodeLexicalArtifactErrorV1::Corrupt(error.to_string()))?; + if status != Status::StreamEnd + || inflated.len() != length + || usize::try_from(decompressor.total_in()).ok() != Some(rest.len()) + { + return Err(corrupt("does not inflate to its length")); + } + Ok(inflated) } -fn decode_ngram_bitmap_v11(encoded: &[u8]) -> Result { - let header = encoded.get(..16).ok_or_else(|| { - CodeLexicalArtifactErrorV1::Corrupt( - "lexical artifact ngram bitmap header is truncated".to_owned(), +/// One fingerprint's sealed posting list: its `(occurrence ordinal, token +/// position)` postings sorted by ordinal then position, one group per +/// ordinal (the varint ordinal, absolute for the first group and a non-zero +/// delta after, the varint posting count, then the first position and each +/// later non-zero delta). +pub(super) fn encode_fingerprint_postings( + postings: &[(u32, u32)], +) -> Result, CodeLexicalArtifactErrorV1> { + let unordered = || { + CodeLexicalArtifactErrorV1::Contract( + "clone fingerprint postings are not strictly ordered".to_owned(), ) - })?; - if &header[..4] != b"TDN1" || header[5..8] != [0u8; 3] || header[4] > 1 { - return Err(CodeLexicalArtifactErrorV1::Corrupt( - "lexical artifact ngram bitmap header is invalid".to_owned(), - )); + }; + let mut encoded = Vec::with_capacity(postings.len() * 4); + let mut previous_ordinal: Option = None; + let mut start = 0; + while start < postings.len() { + let ordinal = postings[start].0; + let end = start + + postings[start..] + .iter() + .take_while(|(candidate, _)| *candidate == ordinal) + .count(); + let delta = match previous_ordinal { + None => ordinal, + Some(previous) if ordinal > previous => ordinal - previous, + Some(_) => return Err(unordered()), + }; + previous_ordinal = Some(ordinal); + encode_varint(u64::from(delta), &mut encoded); + encode_varint( + u64::try_from(end - start).map_err(contract_number)?, + &mut encoded, + ); + let mut previous_position = None; + for (_, position) in &postings[start..end] { + let delta = match previous_position { + None => *position, + Some(previous) if *position > previous => position - previous, + Some(_) => return Err(unordered()), + }; + encode_varint(u64::from(delta), &mut encoded); + previous_position = Some(*position); + } + start = end; } - let cardinality = u64::from_le_bytes(header[8..16].try_into().map_err(|_| { + Ok(encoded) +} + +/// Inverse of [`encode_fingerprint_postings`], failing closed on any +/// non-canonical order, empty group, overflow, or truncation. +pub(super) fn decode_fingerprint_postings( + mut encoded: &[u8], +) -> Result, CodeLexicalArtifactErrorV1> { + let corrupt = || { CodeLexicalArtifactErrorV1::Corrupt( - "lexical artifact ngram bitmap cardinality is malformed".to_owned(), + "lexical artifact clone fingerprint postings are malformed".to_owned(), ) - })?); - let width = if header[4] == 1 { 8usize } else { 4usize }; - if !(encoded.len() - 16).is_multiple_of(width) { - return Err(CodeLexicalArtifactErrorV1::Corrupt( - "lexical artifact ngram bitmap payload is truncated".to_owned(), - )); - } - let mut bitmap = RoaringBitmap::new(); - let mut previous: Option = None; - for item in encoded[16..].chunks_exact(width) { - let start = u32::from_le_bytes(item[..4].try_into().map_err(|_| { - CodeLexicalArtifactErrorV1::Corrupt( - "lexical artifact ngram bitmap document is malformed".to_owned(), - ) - })?); - let end = if width == 8 { - let run = u32::from_le_bytes(item[4..8].try_into().map_err(|_| { - CodeLexicalArtifactErrorV1::Corrupt( - "lexical artifact ngram bitmap run is malformed".to_owned(), - ) - })?); - start.checked_add(run).ok_or_else(|| { - CodeLexicalArtifactErrorV1::Corrupt( - "lexical artifact ngram bitmap run overflowed".to_owned(), - ) - })? - } else { - start + }; + let mut postings = Vec::new(); + let mut ordinal: Option = None; + while !encoded.is_empty() { + let delta = u32::try_from(take_varint(&mut encoded)?).map_err(|_| corrupt())?; + let next = match ordinal { + None => delta, + Some(_) if delta == 0 => return Err(corrupt()), + Some(previous) => previous.checked_add(delta).ok_or_else(corrupt)?, }; - if previous.is_some_and(|previous| start <= previous) { - return Err(CodeLexicalArtifactErrorV1::Corrupt( - "lexical artifact ngram bitmap documents are not strictly ordered".to_owned(), + ordinal = Some(next); + let count = take_varint(&mut encoded)?; + if count == 0 || count > encoded.len() as u64 { + return Err(corrupt()); + } + let mut position: Option = None; + for _ in 0..count { + let delta = u32::try_from(take_varint(&mut encoded)?).map_err(|_| corrupt())?; + let value = match position { + None => delta, + Some(_) if delta == 0 => return Err(corrupt()), + Some(previous) => previous.checked_add(delta).ok_or_else(corrupt)?, + }; + position = Some(value); + postings.push((next, value)); + } + } + Ok(postings) +} + +/// One term's sealed `term_postings.lists`: for each field in strictly +/// ascending code order, the varint field code, the varint document +/// frequency, and the length-prefixed frequency posting list. +pub(super) fn encode_term_lists( + lists: &[(i64, u64, Vec)], +) -> Result, CodeLexicalArtifactErrorV1> { + let mut encoded = Vec::with_capacity(lists.iter().map(|(_, _, list)| list.len() + 6).sum()); + let mut previous = None; + for (field, document_frequency, list) in lists { + if previous.is_some_and(|previous| previous >= *field) + || *document_frequency == 0 + || list.is_empty() + { + return Err(CodeLexicalArtifactErrorV1::Contract( + "lexical artifact term lists are not canonical".to_owned(), )); } - bitmap.insert_range(start..=end); - previous = Some(end); + previous = Some(*field); + encode_varint( + u64::try_from(*field).map_err(contract_number)?, + &mut encoded, + ); + encode_varint(*document_frequency, &mut encoded); + encode_varint( + u64::try_from(list.len()).map_err(contract_number)?, + &mut encoded, + ); + encoded.extend_from_slice(list); } - if bitmap.len() != cardinality { - return Err(CodeLexicalArtifactErrorV1::Corrupt( - "lexical artifact ngram bitmap cardinality does not verify".to_owned(), - )); + Ok(encoded) +} + +/// `(field code, document frequency, posting list)` of one field's list. +pub(super) type TermFieldListV1<'a> = (i64, u64, &'a [u8]); + +/// Every field list one `term_postings.lists` value carries. +pub(super) fn decode_term_lists( + mut encoded: &[u8], +) -> Result>, CodeLexicalArtifactErrorV1> { + let corrupt = || { + CodeLexicalArtifactErrorV1::Corrupt("lexical artifact term lists are malformed".to_owned()) + }; + let mut lists = Vec::new(); + let mut previous = None; + while !encoded.is_empty() { + let field = i64::try_from(take_varint(&mut encoded)?).map_err(|_| corrupt())?; + let document_frequency = take_varint(&mut encoded)?; + let length = usize::try_from(take_varint(&mut encoded)?).map_err(|_| corrupt())?; + if previous.is_some_and(|previous| previous >= field) + || document_frequency == 0 + || length == 0 + || length > encoded.len() + { + return Err(corrupt()); + } + previous = Some(field); + let (list, rest) = encoded.split_at(length); + encoded = rest; + lists.push((field, document_frequency, list)); } - Ok(bitmap) + if lists.is_empty() { + return Err(corrupt()); + } + Ok(lists) } -fn encode_ngram_delta_varints_v12( - bitmap: &RoaringBitmap, -) -> Result, CodeLexicalArtifactErrorV1> { - if bitmap.is_empty() { - return Err(CodeLexicalArtifactErrorV1::Contract( - "lexical artifact ngram shard is empty".to_owned(), - )); +/// One sorted posting list: canonical LEB128 varints, the first document +/// absolute and every later one as its non-zero delta. With frequencies each +/// document varint is shifted left one bit and a set low bit announces a +/// following frequency varint (always at least 2), so the dominant frequency +/// of one costs no byte. +pub(super) struct PostingListEncoderV1 { + bytes: Vec, + previous: Option, + len: u64, + frequencies: bool, +} + +impl PostingListEncoderV1 { + pub(super) fn new(frequencies: bool) -> Self { + Self { + bytes: Vec::new(), + previous: None, + len: 0, + frequencies, + } } - let mut encoded = Vec::with_capacity( - usize::try_from(bitmap.len()) - .map_err(|error| CodeLexicalArtifactErrorV1::Contract(error.to_string()))?, - ); - let mut previous: Option = None; - for document in bitmap.iter() { - let value = previous.map_or(document, |prior| document - prior); - encode_u32_varint(value, &mut encoded); - previous = Some(document); + + pub(super) fn push( + &mut self, + document: u32, + frequency: u32, + ) -> Result<(), CodeLexicalArtifactErrorV1> { + let delta = match self.previous { + None => document, + Some(previous) if document > previous => document - previous, + Some(_) => { + return Err(CodeLexicalArtifactErrorV1::Contract( + "lexical artifact posting documents are not strictly ascending".to_owned(), + )); + } + }; + if frequency == 0 || (!self.frequencies && frequency != 1) { + return Err(CodeLexicalArtifactErrorV1::Contract( + "lexical artifact posting frequency is out of range".to_owned(), + )); + } + if self.frequencies { + encode_varint( + (u64::from(delta) << 1) | u64::from(frequency != 1), + &mut self.bytes, + ); + if frequency != 1 { + encode_varint(u64::from(frequency), &mut self.bytes); + } + } else { + encode_varint(u64::from(delta), &mut self.bytes); + } + self.previous = Some(document); + self.len += 1; + Ok(()) + } + + pub(super) fn len(&self) -> u64 { + self.len + } + + /// Heap bytes the encoded list holds, for memory accounting. + pub(super) fn retained_bytes(&self) -> usize { + self.bytes.capacity() + } + + pub(super) fn finish(self) -> Result, CodeLexicalArtifactErrorV1> { + if self.len == 0 { + return Err(CodeLexicalArtifactErrorV1::Contract( + "lexical artifact posting list is empty".to_owned(), + )); + } + Ok(self.bytes) } - Ok(encoded) } -fn decode_ngram_delta_varints_v12( - encoded: &[u8], -) -> Result { - if encoded.is_empty() { - return Err(CodeLexicalArtifactErrorV1::Corrupt( - "lexical artifact ngram delta list is empty".to_owned(), - )); +/// Streams `(document, frequency)` from a [`PostingListEncoderV1`] list, +/// failing closed on any non-canonical, zero-delta, or overflowing entry. +pub(super) struct PostingListDecoderV1<'a> { + bytes: &'a [u8], + previous: Option, + frequencies: bool, +} + +impl<'a> PostingListDecoderV1<'a> { + pub(super) fn new(bytes: &'a [u8], frequencies: bool) -> Self { + Self { + bytes, + previous: None, + frequencies, + } } - let mut bitmap = RoaringBitmap::new(); - let mut offset = 0usize; - let mut previous: Option = None; - while offset < encoded.len() { - let (value, consumed) = decode_u32_varint(&encoded[offset..])?; - offset = offset.checked_add(consumed).ok_or_else(|| { + + fn decode_next(&mut self) -> Result<(u32, u32), CodeLexicalArtifactErrorV1> { + let token = take_varint(&mut self.bytes)?; + let (delta, frequency) = if self.frequencies { + let frequency = if token & 1 == 1 { + let frequency = take_varint(&mut self.bytes)?; + if frequency < 2 { + return Err(CodeLexicalArtifactErrorV1::Corrupt( + "lexical artifact posting frequency is not canonical".to_owned(), + )); + } + frequency + } else { + 1 + }; + (token >> 1, frequency) + } else { + (token, 1) + }; + let corrupt = |_| { CodeLexicalArtifactErrorV1::Corrupt( - "lexical artifact ngram delta offset overflowed".to_owned(), + "lexical artifact posting value overflows u32".to_owned(), ) - })?; - let document = match previous { - None => value, - Some(_) if value == 0 => { + }; + let delta = u32::try_from(delta).map_err(corrupt)?; + let frequency = u32::try_from(frequency).map_err(corrupt)?; + let document = match self.previous { + None => delta, + Some(_) if delta == 0 => { return Err(CodeLexicalArtifactErrorV1::Corrupt( - "lexical artifact ngram delta is zero".to_owned(), + "lexical artifact posting delta is zero".to_owned(), )); } - Some(prior) => prior.checked_add(value).ok_or_else(|| { + Some(previous) => previous.checked_add(delta).ok_or_else(|| { CodeLexicalArtifactErrorV1::Corrupt( - "lexical artifact ngram delta overflowed".to_owned(), + "lexical artifact posting delta overflowed".to_owned(), ) })?, }; - bitmap.insert(document); - previous = Some(document); + self.previous = Some(document); + Ok((document, frequency)) } - Ok(bitmap) } -fn encode_u32_varint(mut value: u32, encoded: &mut Vec) { +impl Iterator for PostingListDecoderV1<'_> { + type Item = Result<(u32, u32), CodeLexicalArtifactErrorV1>; + + fn next(&mut self) -> Option { + if self.bytes.is_empty() { + return None; + } + let decoded = self.decode_next(); + if decoded.is_err() { + self.bytes = &[]; + } + Some(decoded) + } +} + +pub(super) fn encode_varint(mut value: u64, encoded: &mut Vec) { while value >= 0x80 { encoded.push((value as u8 & 0x7f) | 0x80); value >>= 7; @@ -1050,34 +1005,28 @@ fn encode_u32_varint(mut value: u32, encoded: &mut Vec) { encoded.push(value as u8); } -fn decode_u32_varint(encoded: &[u8]) -> Result<(u32, usize), CodeLexicalArtifactErrorV1> { - let mut value = 0u32; - for (ordinal, byte) in encoded.iter().copied().take(5).enumerate() { - let payload = u32::from(byte & 0x7f); - if ordinal == 4 && payload > 0x0f { +pub(super) fn take_varint(encoded: &mut &[u8]) -> Result { + let mut value = 0u64; + for (ordinal, byte) in encoded.iter().copied().take(10).enumerate() { + let payload = u64::from(byte & 0x7f); + if ordinal == 9 && payload > 1 { return Err(CodeLexicalArtifactErrorV1::Corrupt( - "lexical artifact ngram varint overflows u32".to_owned(), + "lexical artifact posting varint overflows u64".to_owned(), )); } value |= payload << (ordinal * 7); if byte & 0x80 == 0 { - let consumed = ordinal + 1; - let canonical = if value == 0 { - 1 - } else { - usize::try_from((value.ilog2() / 7) + 1) - .map_err(|error| CodeLexicalArtifactErrorV1::Corrupt(error.to_string()))? - }; - if consumed != canonical { + if ordinal > 0 && byte == 0 { return Err(CodeLexicalArtifactErrorV1::Corrupt( - "lexical artifact ngram varint is not canonical".to_owned(), + "lexical artifact posting varint is not canonical".to_owned(), )); } - return Ok((value, consumed)); + *encoded = &encoded[ordinal + 1..]; + return Ok(value); } } Err(CodeLexicalArtifactErrorV1::Corrupt( - "lexical artifact ngram varint is truncated".to_owned(), + "lexical artifact posting varint is truncated".to_owned(), )) } @@ -1093,12 +1042,7 @@ pub struct CodeLexicalArtifactSectionDigestV1 { #[serde(deny_unknown_fields)] pub struct VerifiedCodeLexicalArtifactV1 { format_revision: u32, - generation: CodeGenerationId, - repository_id: Option, - freshness: SourceFreshness, metadata_digest: ManifestDigest, - source_state_digest: ManifestDigest, - source_cumulative_digest: ManifestDigest, source_format_revision: u32, page_count: u64, total_chunks: u64, @@ -1120,26 +1064,6 @@ impl VerifiedCodeLexicalArtifactV1 { self.file_size_bytes } - pub fn generation(&self) -> &CodeGenerationId { - &self.generation - } - - pub fn repository_id(&self) -> Option<&RepositoryId> { - self.repository_id.as_ref() - } - - pub fn freshness(&self) -> &SourceFreshness { - &self.freshness - } - - pub fn source_state_digest(&self) -> &ManifestDigest { - &self.source_state_digest - } - - pub fn source_cumulative_digest(&self) -> &ManifestDigest { - &self.source_cumulative_digest - } - pub fn page_count(&self) -> u64 { self.page_count } @@ -1168,7 +1092,7 @@ impl VerifiedCodeLexicalArtifactV1 { &self.section_digests } - pub(super) fn format_revision(&self) -> u32 { + pub fn format_revision(&self) -> u32 { self.format_revision } @@ -1246,6 +1170,13 @@ pub(super) fn manifest_digest( ) -> Result { let bytes = serde_json::to_vec(value) .map_err(|error| CodeLexicalArtifactErrorV1::Contract(error.to_string()))?; + manifest_digest_of_bytes(domain, &bytes) +} + +fn manifest_digest_of_bytes( + domain: &[u8], + bytes: &[u8], +) -> Result { let mut hasher = Sha256::new(); hasher.update(domain); hasher.update( @@ -1258,16 +1189,72 @@ pub(super) fn manifest_digest( .map_err(|error| CodeLexicalArtifactErrorV1::Contract(error.to_string())) } +/// The projection metadata an artifact's bytes depend on. Generation, +/// repository, and freshness name the route that opens the artifact, not its +/// content, so worktrees that index identical trees seal identical files and +/// each opener supplies its own route identity. +#[derive(Serialize)] +struct ArtifactContentMetadataV1<'a> { + logical_paths: &'a BTreeMap, + exact_retriever_revision: &'a ComponentRevision, + lexical_retriever_revision: &'a ComponentRevision, + exact_score_domain: &'a ScoreDomainId, +} + +impl<'a> ArtifactContentMetadataV1<'a> { + fn of(metadata: &'a CodeLexicalProjectionMetadataV1) -> Self { + Self { + logical_paths: &metadata.logical_paths, + exact_retriever_revision: &metadata.exact_retriever_revision, + lexical_retriever_revision: &metadata.lexical_retriever_revision, + exact_score_domain: &metadata.exact_score_domain, + } + } +} + +/// The canonical bytes `artifact_state.metadata` stores for `metadata`. +pub(super) fn content_metadata_bytes( + metadata: &CodeLexicalProjectionMetadataV1, +) -> Result, CodeLexicalArtifactErrorV1> { + serde_json::to_vec(&ArtifactContentMetadataV1::of(metadata)) + .map_err(|error| CodeLexicalArtifactErrorV1::Contract(error.to_string())) +} + pub(super) fn metadata_digest( metadata: &CodeLexicalProjectionMetadataV1, ) -> Result { - manifest_digest(b"tracedecay.code-lexical-artifact-metadata.v1\0", metadata) + stored_metadata_digest(&content_metadata_bytes(metadata)?) +} + +/// The key a lexical artifact built from a source with `source_content_key` +/// under `metadata` is published with. Everything the artifact's bytes +/// depend on is in it (the format, the content metadata, and the source's +/// content), so an artifact published under a key serves any opener whose +/// source and projection produce the same key. +pub fn code_lexical_artifact_content_key( + source_content_key: &ManifestDigest, + metadata: &CodeLexicalProjectionMetadataV1, +) -> Result { + manifest_digest( + b"tracedecay.code-lexical-artifact-content-key.v1\0", + &( + CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V1, + metadata_digest(metadata)?.as_str(), + source_content_key.as_str(), + ), + ) +} + +/// The digest of stored `artifact_state.metadata` bytes. +pub(super) fn stored_metadata_digest( + bytes: &[u8], +) -> Result { + manifest_digest_of_bytes(b"tracedecay.code-lexical-artifact-metadata.v2\0", bytes) } #[allow(clippy::too_many_arguments)] // one committed digest tuple, spelled once pub(super) fn artifact_digest( metadata_digest: &ManifestDigest, - source_state_digest: &ManifestDigest, source_format_revision: u32, page_count: u64, total_chunks: u64, @@ -1275,7 +1262,6 @@ pub(super) fn artifact_digest( total_imports: u64, import_payload_bytes: u64, import_dictionary_digest: &ManifestDigest, - source_cumulative_digest: &ManifestDigest, sections: &[CodeLexicalArtifactSectionDigestV1], format_revision: u32, ) -> Result { @@ -1283,7 +1269,6 @@ pub(super) fn artifact_digest( digest_domain_for_revision(format_revision)?, &( metadata_digest.as_str(), - source_state_digest.as_str(), source_format_revision, page_count, total_chunks, @@ -1291,13 +1276,32 @@ pub(super) fn artifact_digest( total_imports, import_payload_bytes, import_dictionary_digest.as_str(), - source_cumulative_digest.as_str(), sections, format_revision, ), ) } +/// The artifact digest `receipt` binds, recomputed from its own fields over +/// `sections`. +pub(super) fn receipt_artifact_digest( + receipt: &VerifiedCodeLexicalArtifactV1, + sections: &[CodeLexicalArtifactSectionDigestV1], +) -> Result { + artifact_digest( + &receipt.metadata_digest, + receipt.source_format_revision, + receipt.page_count, + receipt.total_chunks, + receipt.total_payload_bytes, + receipt.total_imports, + receipt.import_payload_bytes, + &receipt.import_dictionary_digest, + sections, + receipt.format_revision, + ) +} + pub(super) fn encode_field(field: LexicalFieldV1) -> Result { serde_json::to_string(&field) .map_err(|error| CodeLexicalArtifactErrorV1::Contract(error.to_string())) @@ -1408,23 +1412,32 @@ pub(super) fn decode_padded_receipt_with_control( Ok(Some(receipt)) } +/// Seal `sections` for `source`. The receipt binds only content: the sealed +/// source's state and chunk-chain digests hash the building worktree's +/// generation into every chunk anchor and clone occurrence, so they stay +/// with the build that verified them. Chunk payload bytes count the +/// fixed-width generation id, not its value. pub(super) fn new_verified_receipt( - metadata: CodeLexicalProjectionMetadataV1, metadata_digest: ManifestDigest, source: &tracedecay_code_index::production::VerifiedSealedLexicalSourceReceiptV1, - artifact_digest: ManifestDigest, section_digests: Vec, file_size_bytes: u64, - layout: LexicalArtifactLayoutV1, -) -> VerifiedCodeLexicalArtifactV1 { - VerifiedCodeLexicalArtifactV1 { - format_revision: layout.revision(), - generation: metadata.generation, - repository_id: metadata.repository_id, - freshness: metadata.freshness, +) -> Result { + let artifact_digest = artifact_digest( + &metadata_digest, + source.format_revision(), + source.page_count(), + source.total_chunks(), + source.total_payload_bytes(), + source.total_imports(), + source.import_payload_bytes(), + source.import_dictionary_digest(), + §ion_digests, + CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V1, + )?; + Ok(VerifiedCodeLexicalArtifactV1 { + format_revision: CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V1, metadata_digest, - source_state_digest: source.source_state_digest().clone(), - source_cumulative_digest: source.cumulative_digest().clone(), source_format_revision: source.format_revision(), page_count: source.page_count(), total_chunks: source.total_chunks(), @@ -1435,14 +1448,18 @@ pub(super) fn new_verified_receipt( artifact_digest, section_digests, file_size_bytes, - } + }) } #[cfg(test)] mod tests { use roaring::RoaringBitmap; - use super::{LexicalArtifactLayoutV1, decode_ngram_bitmap, encode_ngram_bitmap}; + use super::{ + DOCUMENT_SET_BITSET, DOCUMENT_SET_DELTAS, PostingListDecoderV1, PostingListEncoderV1, + decode_document_set, decode_fingerprint_postings, decode_ngram_bitmap, decode_term_lists, + encode_document_set, encode_fingerprint_postings, encode_ngram_bitmap, encode_term_lists, + }; #[test] fn v12_ngram_delta_varints_round_trip_sparse_and_dense_shards() { @@ -1453,10 +1470,8 @@ mod tests { (400_000..400_064).collect::>(), ] { let bitmap = RoaringBitmap::from_iter(documents); - let encoded = - encode_ngram_bitmap(LexicalArtifactLayoutV1::V12, &bitmap).expect("encode"); - let decoded = - decode_ngram_bitmap(LexicalArtifactLayoutV1::V12, &encoded).expect("decode"); + let encoded = encode_ngram_bitmap(&bitmap).expect("encode"); + let decoded = decode_ngram_bitmap(&encoded).expect("decode"); assert_eq!(decoded, bitmap); } } @@ -1467,15 +1482,10 @@ mod tests { let dense = RoaringBitmap::from_iter(400_000..400_064); assert_eq!( - encode_ngram_bitmap(LexicalArtifactLayoutV1::V12, &singleton).expect("singleton"), + encode_ngram_bitmap(&singleton).expect("singleton"), vec![0x80, 0xb5, 0x18] ); - assert_eq!( - encode_ngram_bitmap(LexicalArtifactLayoutV1::V12, &dense) - .expect("dense") - .len(), - 66 - ); + assert_eq!(encode_ngram_bitmap(&dense).expect("dense").len(), 66); } #[test] @@ -1487,9 +1497,135 @@ mod tests { vec![0x01, 0x00], ] { assert!( - decode_ngram_bitmap(LexicalArtifactLayoutV1::V12, &malformed).is_err(), + decode_ngram_bitmap(&malformed).is_err(), "accepted malformed delta-varint payload {malformed:?}" ); } } + + #[test] + fn posting_lists_round_trip_frequencies_and_spend_no_byte_on_frequency_one() { + let postings = [(0u32, 1u32), (1, 1), (129, 7), (400_000, 1), (400_001, 300)]; + let mut encoder = PostingListEncoderV1::new(true); + for (document, frequency) in postings { + encoder + .push(document, frequency) + .expect("ascending posting"); + } + assert_eq!(encoder.len(), 5); + let encoded = encoder.finish().expect("encode"); + let decoded = PostingListDecoderV1::new(&encoded, true) + .collect::, _>>() + .expect("decode"); + assert_eq!(decoded, postings); + + let mut ones = PostingListEncoderV1::new(true); + for document in 0..100 { + ones.push(document, 1).expect("ascending posting"); + } + assert_eq!(ones.finish().expect("encode").len(), 100); + + let mut unordered = PostingListEncoderV1::new(true); + unordered.push(5, 1).expect("first posting"); + assert!( + unordered.push(5, 1).is_err(), + "duplicate documents are refused" + ); + let mut plain = PostingListEncoderV1::new(false); + assert!( + plain.push(1, 2).is_err(), + "document sets carry no frequency" + ); + assert!(PostingListEncoderV1::new(true).finish().is_err()); + // Flagged frequency one is not canonical. + assert!( + PostingListDecoderV1::new(&[0x03, 0x01], true) + .collect::, _>>() + .is_err() + ); + } + + #[test] + fn fingerprint_postings_round_trip_and_refuse_non_canonical_order() { + let postings = [(0u32, 4u32), (0, 9), (3, 1), (300, 0), (300, 70_000)]; + let encoded = encode_fingerprint_postings(&postings).expect("encode"); + assert_eq!( + decode_fingerprint_postings(&encoded).expect("decode"), + postings + ); + assert!( + encoded.len() < postings.len() * 4, + "ordinal deltas stay compact" + ); + assert!(encode_fingerprint_postings(&[(2, 1), (1, 1)]).is_err()); + assert!(encode_fingerprint_postings(&[(1, 1), (1, 1)]).is_err()); + for malformed in [ + vec![0x01, 0x00], + vec![0x01, 0x02, 0x05], + vec![0x01, 0x01, 0x00, 0x00, 0x01, 0x00], + ] { + assert!( + decode_fingerprint_postings(&malformed).is_err(), + "accepted malformed fingerprint postings {malformed:?}" + ); + } + } + + #[test] + fn term_lists_round_trip_and_refuse_non_canonical_fields() { + let lists = vec![(1i64, 2u64, vec![0x02, 0x04]), (7, 1, vec![0x0a])]; + let encoded = encode_term_lists(&lists).expect("encode"); + let decoded = decode_term_lists(&encoded).expect("decode"); + assert_eq!( + decoded, + vec![ + (1, 2, [0x02u8, 0x04].as_slice()), + (7, 1, [0x0au8].as_slice()) + ] + ); + assert!(encode_term_lists(&[(7, 1, vec![1]), (7, 1, vec![2])]).is_err()); + assert!(encode_term_lists(&[(1, 0, vec![1])]).is_err()); + for malformed in [ + vec![], + vec![0x07, 0x01, 0x02, 0x0a], + vec![0x07, 0x01, 0x01, 0x0a, 0x01, 0x01, 0x01, 0x0b], + ] { + assert!( + decode_term_lists(&malformed).is_err(), + "accepted malformed term lists {malformed:?}" + ); + } + } + + #[test] + fn document_sets_choose_the_smaller_encoding_and_round_trip() { + let sparse = RoaringBitmap::from_iter([3u32, 90_000, 300_000]); + let dense = (1_000u32..9_000).step_by(2).collect::(); + for (documents, tag) in [ + (&sparse, DOCUMENT_SET_DELTAS), + (&dense, DOCUMENT_SET_BITSET), + ] { + let encoded = encode_document_set(documents).expect("encode"); + assert_eq!(encoded[0], tag); + assert_eq!(&decode_document_set(&encoded).expect("decode"), documents); + } + assert!( + encode_document_set(&dense).expect("encode").len() + < 1 + encode_ngram_bitmap(&dense).expect("deltas").len() / 3, + "a half-dense list is stored as bits, not bytes" + ); + for malformed in [ + vec![], + vec![2, 0x01], + vec![DOCUMENT_SET_BITSET, 0x00, 0x02], + vec![DOCUMENT_SET_BITSET, 0x00, 0x01, 0x00], + vec![DOCUMENT_SET_BITSET, 0x05], + ] { + assert!( + decode_document_set(&malformed).is_err(), + "accepted malformed document set {malformed:?}" + ); + } + assert!(encode_document_set(&RoaringBitmap::new()).is_err()); + } } diff --git a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/postings.rs b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/postings.rs index 7667481d54..0a8bfa42b2 100644 --- a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/postings.rs +++ b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/postings.rs @@ -107,6 +107,34 @@ pub(super) fn query_ngrams(bytes: &[u8]) -> BTreeSet { super::super::packed_query_ngrams(bytes) } +/// Whether a packed n-gram needs the case-preserving kind. Normalized text is +/// the ASCII-lowercased raw text at the same byte offsets, so a raw window +/// without an ASCII uppercase byte is already the normalized window there. +pub(super) fn ngram_is_case_sensitive(ngram: u32) -> bool { + let width = ngram >> 24; + (0..width).any(|index| ((ngram >> (index * 8)) as u8).is_ascii_uppercase()) +} + +/// The kind that holds each query n-gram for a case-sensitive raw match, or +/// `None` when every window is case-insensitive and the normalized query +/// already admits every raw match. +pub(super) fn raw_override_query_ngrams(bytes: &[u8]) -> Option> { + let keys = query_ngrams(bytes) + .into_iter() + .map(|ngram| { + let kind = if ngram_is_case_sensitive(ngram) { + NGRAM_RAW_OVERRIDE + } else { + NGRAM_NORMALIZED + }; + (kind, ngram) + }) + .collect::>(); + keys.iter() + .any(|(kind, _)| *kind == NGRAM_RAW_OVERRIDE) + .then_some(keys) +} + #[cfg(test)] mod tests { use super::{document_ngram_scratch, reserve_ngram_scratch}; diff --git a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/prepared.rs b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/prepared.rs index e74bfe4732..910f3d7566 100644 --- a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/prepared.rs +++ b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/prepared.rs @@ -6,20 +6,22 @@ use tracedecay_code_index::clones::{ CloneExactKeyV1, CloneFingerprintPositionV1, CloneNormalizationClassV1, CodeIndexCloneBodyV1, }; use tracedecay_code_index::production::{CodeIndexExecutionControlV1, VerifiedSealedLexicalPageV1}; -use tracedecay_domain::{ExactFieldV1, ManifestDigest}; +use tracedecay_domain::{ExactFieldV1, ManifestDigest, NodeKind}; use super::super::{ CodeLexicalProjectionMetadataV1, ProjectedChunkV1, canonical_projected_exact_term, exact_field_for_kind, normalized_search_text, }; +use super::clone_codec::{encode_clone_eligibility, encode_clone_payload}; use super::format::{ ArtifactRowV1, BASE_SECTION_NAMES, PageBaseSectionReceiptBuilderV1, contract_number, encode_exact_field, encode_field, encode_ngram_bitmap, encode_page_base_sections_receipt, hash_bytes, ngram_page_digest, }; -use super::postings::{NGRAM_NORMALIZED, NGRAM_RAW_OVERRIDE, document_ngrams}; -use super::row_codec::{RowDictionaryTableV1, encode_artifact_row}; -use super::schema::LexicalArtifactLayoutV1; +use super::postings::{ + NGRAM_NORMALIZED, NGRAM_RAW_OVERRIDE, document_ngrams, ngram_is_case_sensitive, +}; +use super::row_codec::{BlockRowV1, RowDictionaryTableV1, encode_artifact_row, encode_row_blocks}; use super::{ CodeLexicalArtifactErrorV1, NGRAM_AGGREGATION_BYTES_PER_LOGICAL_POSTING_V1, checkpoint, }; @@ -43,11 +45,12 @@ pub struct PreparedCodeLexicalArtifactPageV1 { pub(super) imports: Vec, pub(super) clone_bodies: Vec, pub(super) documents: Vec, + /// The page's rows as stored blocks keyed by their first document. + pub(super) row_blocks: Vec<(i64, Vec)>, pub(super) ngram_shards: Vec, pub(super) ngram_digest: ManifestDigest, pub(super) base_sections_receipt: Vec, - /// Every dictionary entry this page's rows reference (revision 14); - /// empty for layouts whose rows carry their strings inline. + /// Every dictionary entry this page's rows reference. pub(super) row_dictionary: RowDictionaryTableV1, source_retained_bytes: usize, prepared_retained_bytes: usize, @@ -89,6 +92,16 @@ impl PreparedCodeLexicalArtifactPageV1 { self.estimated_write_bytes } + /// Serialized bytes of this page's largest clone body (payload plus + /// occurrence), the scratch one clone row holds while it is written. + pub fn clone_body_peak_bytes(&self) -> usize { + self.clone_bodies + .iter() + .map(|body| body.serialized_bytes) + .max() + .unwrap_or(0) + } + pub fn ledger_charge_bytes(&self) -> Result { self.source_retained_bytes .checked_add(self.prepared_retained_bytes) @@ -110,8 +123,11 @@ pub(super) struct PreparedImportV1 { #[derive(Debug)] pub(super) struct PreparedCloneBodyV1 { pub(super) payload_digest: String, + /// Stored (deflated binary) payload and the occurrence's eligibility. pub(super) payload: Vec, - pub(super) occurrence: Vec, + pub(super) eligibility: Vec, + /// Encoded payload bytes before deflate plus the eligibility bytes. + pub(super) serialized_bytes: usize, pub(super) symbol_occurrence_id: String, pub(super) path: String, pub(super) body_start: u64, @@ -125,7 +141,6 @@ pub(super) struct PreparedCloneFingerprintStreamV1 { pub(super) language: String, pub(super) class: CloneNormalizationClassV1, pub(super) normalization_revision: u16, - pub(super) body_digest: String, pub(super) positions: Vec, } @@ -133,12 +148,13 @@ pub(super) struct PreparedCloneFingerprintStreamV1 { pub(super) struct PreparedDocumentV1 { pub(super) document_id: i64, pub(super) chunk_id: String, + /// Row metadata; its text is stored in the page's row blocks. pub(super) row: Vec, pub(super) term_postings: Vec, pub(super) exact_postings: Vec<(String, Vec)>, - /// Tagged hex form persisted by layouts up to 13. + /// Tagged hex form, which the page receipt and memory accounting read. pub(super) integrity_digest: ManifestDigest, - /// The same digest as the 32 raw bytes revision 14 persists. + /// The same digest as the 32 raw bytes `document_integrity` persists. pub(super) integrity_digest_bytes: [u8; 32], } @@ -158,7 +174,6 @@ pub(super) struct PreparedTermPostingV1 { } pub(super) fn prepare_page( - layout: LexicalArtifactLayoutV1, metadata: &CodeLexicalProjectionMetadataV1, page: &VerifiedSealedLexicalPageV1, previous_cursor: Option>, @@ -176,6 +191,7 @@ pub(super) fn prepare_page( ) })?; let mut documents = Vec::with_capacity(page.chunks().len()); + let mut texts = Vec::with_capacity(page.chunks().len()); let mut row_dictionary = RowDictionaryTableV1::new(); let mut ngram_documents = BTreeMap::<(i64, i64), RoaringBitmap>::new(); let mut logical_ngram_postings = 0usize; @@ -188,6 +204,15 @@ pub(super) fn prepare_page( page.chunks().iter().zip(page.symbol_displays()).enumerate() { checkpoint(control)?; + // Attribute and annotation uses stay graph symbols, but the item they + // annotate already carries their text; they mint no lexical document + // and their source ordinal is left unused. + if display + .as_ref() + .is_some_and(|display| display.kind() == NodeKind::AnnotationUsage.as_str()) + { + continue; + } let document = first_document .checked_add(u64::try_from(offset).map_err(contract_number)?) .ok_or_else(|| { @@ -195,8 +220,7 @@ pub(super) fn prepare_page( "lexical artifact prepared document id overflowed".to_owned(), ) })?; - let (prepared, ngrams) = prepare_document( - layout, + let (prepared, ngrams, text) = prepare_document( metadata, i64::try_from(document).map_err(contract_number)?, admitted.chunk(), @@ -228,6 +252,7 @@ pub(super) fn prepare_page( })?; } documents.push(prepared); + texts.push(text); } let mut imports = Vec::with_capacity(page.imports().len()); for evidence in page.imports() { @@ -243,7 +268,7 @@ pub(super) fn prepare_page( let mut clone_bodies = Vec::with_capacity(page.clone_bodies().len()); for body in page.clone_bodies() { checkpoint(control)?; - clone_bodies.push(prepare_clone_body(layout, metadata, body)?); + clone_bodies.push(prepare_clone_body(metadata, body)?); } let next_cursor = page .next_cursor() @@ -252,7 +277,7 @@ pub(super) fn prepare_page( let mut ngram_shards = Vec::with_capacity(ngram_documents.len()); for ((kind, ngram), documents) in ngram_documents { checkpoint(control)?; - let encoded = encode_ngram_bitmap(layout, &documents)?; + let encoded = encode_ngram_bitmap(&documents)?; ngram_shards.push(PreparedNgramShardV1 { kind, ngram, @@ -272,13 +297,28 @@ pub(super) fn prepare_page( }), )?; let base_sections_receipt = prepare_base_sections_receipt( - layout, page.page_ordinal(), &imports, &documents, + &texts, &ngram_shards, control, )?; + checkpoint(control)?; + let row_blocks = encode_row_blocks( + &documents + .iter() + .zip(&texts) + .map(|(document, text)| BlockRowV1 { + document_id: document.document_id, + chunk_id: &document.chunk_id, + parent_chunk_id: text.parent_chunk_id.as_deref(), + row: &document.row, + text: &text.text, + }) + .collect::>(), + )?; + drop(texts); let aggregation_scratch_bytes = logical_ngram_postings .checked_mul(NGRAM_AGGREGATION_BYTES_PER_LOGICAL_POSTING_V1) .ok_or_else(|| { @@ -307,6 +347,7 @@ pub(super) fn prepare_page( imports, clone_bodies, documents, + row_blocks, ngram_shards, ngram_digest, base_sections_receipt, @@ -326,39 +367,45 @@ pub(super) fn prepare_page( } fn prepare_clone_body( - layout: LexicalArtifactLayoutV1, metadata: &CodeLexicalProjectionMetadataV1, body: &CodeIndexCloneBodyV1, ) -> Result { - let fingerprint_stream = if layout.has_clone_fingerprints() { - body.payload - .fingerprint_stream(body.occurrence.eligibility) - .map(|stream| { - Ok(PreparedCloneFingerprintStreamV1 { - language: body.payload.language.clone(), - class: stream.class, - normalization_revision: stream.normalization_revision, - body_digest: body.payload.body_digest.as_str().to_owned(), - positions: body - .payload - .fingerprint_positions(body.occurrence.eligibility) - .map_err(CodeLexicalArtifactErrorV1::Contract)?, - }) + let fingerprint_stream = body + .payload + .fingerprint_stream(body.occurrence.eligibility) + .map(|stream| { + Ok(PreparedCloneFingerprintStreamV1 { + language: body.payload.language.clone(), + class: stream.class, + normalization_revision: stream.normalization_revision, + positions: body + .payload + .fingerprint_positions(body.occurrence.eligibility) + .map_err(CodeLexicalArtifactErrorV1::Contract)?, }) - .transpose()? - } else { - None - }; - // Stamp the serving generation onto carried Arc-shared clone bodies so the - // artifact authority matches metadata.generation at lookup time. - let mut occurrence = body.occurrence.clone(); - occurrence.source_generation = metadata.generation.clone(); + }) + .transpose()?; + // Only content is stored; the opener's route supplies project, + // repository, worktree, generation, and snapshot. A body from another + // route would be silently re-labelled, so it is refused. + let occurrence = &body.occurrence; + let owned = metadata.clone_route.as_ref().is_some_and(|route| { + route.project_id == occurrence.project_id + && route.worktree_id == occurrence.worktree_id + && metadata.repository_id.as_ref() == Some(&occurrence.repository_id) + }); + if !owned { + return Err(CodeLexicalArtifactErrorV1::Contract( + "clone body belongs to another route than the projection".to_owned(), + )); + } + let (payload, payload_bytes) = encode_clone_payload(&body.payload)?; + let eligibility = encode_clone_eligibility(occurrence.eligibility); Ok(PreparedCloneBodyV1 { payload_digest: body.payload.payload_digest.as_str().to_owned(), - payload: serde_json::to_vec(&body.payload) - .map_err(|error| CodeLexicalArtifactErrorV1::Contract(error.to_string()))?, - occurrence: serde_json::to_vec(&occurrence) - .map_err(|error| CodeLexicalArtifactErrorV1::Contract(error.to_string()))?, + payload, + serialized_bytes: payload_bytes.saturating_add(eligibility.len()), + eligibility, symbol_occurrence_id: occurrence.symbol_occurrence_id.as_str().to_owned(), path: occurrence.path.clone(), body_start: occurrence.body_span.start_byte, @@ -369,10 +416,10 @@ fn prepare_clone_body( } fn prepare_base_sections_receipt( - layout: LexicalArtifactLayoutV1, page_ordinal: u64, imports: &[PreparedImportV1], documents: &[PreparedDocumentV1], + texts: &[PreparedTextV1], ngram_shards: &[PreparedNgramShardV1], control: &dyn CodeIndexExecutionControlV1, ) -> Result, CodeLexicalArtifactErrorV1> { @@ -399,21 +446,17 @@ fn prepare_base_sections_receipt( import_evidence.blob(&import.canonical)?; import_evidence.blob(&import.canonical)?; } - for document in documents { + for (document, text) in documents.iter().zip(texts) { checkpoint(control)?; document_integrity.begin_row()?; document_integrity.integer(document.document_id); - if layout.stores_document_integrity_bytes() { - document_integrity.blob(&document.integrity_digest_bytes)?; - } else { - document_integrity.text(&document.chunk_id)?; - document_integrity.text(document.integrity_digest.as_str())?; - } + document_integrity.blob(&document.integrity_digest_bytes)?; rows.begin_row()?; rows.integer(document.document_id); rows.text(&document.chunk_id)?; rows.blob(&document.row)?; + rows.text(&text.text)?; for posting in &document.term_postings { checkpoint(control)?; @@ -455,15 +498,24 @@ fn prepare_base_sections_receipt( ) } +/// A prepared row's text and parent chunk, held only until the page's row +/// blocks and receipt are built. +struct PreparedTextV1 { + text: String, + parent_chunk_id: Option, +} + +/// One prepared document, its `(kind, n-gram)` keys, and its text. +type PreparedDocumentPartsV1 = (PreparedDocumentV1, Vec<(i64, i64)>, PreparedTextV1); + fn prepare_document( - layout: LexicalArtifactLayoutV1, metadata: &CodeLexicalProjectionMetadataV1, document_id: i64, chunk: &tracedecay_domain::CodeSearchChunkV1, display: Option<&tracedecay_code_index::production::VerifiedSealedLexicalSymbolDisplayV1>, row_dictionary: &mut RowDictionaryTableV1, control: &dyn CodeIndexExecutionControlV1, -) -> Result<(PreparedDocumentV1, Vec<(i64, i64)>), CodeLexicalArtifactErrorV1> { +) -> Result { u32::try_from(document_id).map_err(|_| { CodeLexicalArtifactErrorV1::Contract( "lexical artifact exceeds the posting document-id range".to_owned(), @@ -527,26 +579,29 @@ fn prepare_document( )); } - let search_text = normalized_search_text(&row); - let mut ngram_postings = document_ngrams(search_text.as_bytes(), control)? - .into_iter() - .map(|ngram| (NGRAM_NORMALIZED, i64::from(ngram))) - .collect::>(); - if row.sanitized_text.as_str().as_bytes() != row.normalized_text.as_bytes() { - ngram_postings.extend( - document_ngrams(row.sanitized_text.as_str().as_bytes(), control)? - .into_iter() - .map(|ngram| (NGRAM_RAW_OVERRIDE, i64::from(ngram))), - ); - } + let ngram_postings = document_ngram_keys( + &normalized_search_text(&row), + row.sanitized_text.as_str(), + &row.normalized_text, + control, + )?; let artifact_row = ArtifactRowV1::from(row); let chunk_id = artifact_row.id.as_str().to_owned(); - let row = encode_artifact_row(layout, &artifact_row, row_dictionary)?; + let row = encode_artifact_row(&artifact_row, row_dictionary)?; + let text = PreparedTextV1 { + parent_chunk_id: artifact_row + .anchor + .parent_chunk_id + .as_ref() + .map(|parent| parent.as_str().to_owned()), + text: artifact_row.sanitized_text.as_str().to_owned(), + }; let exact_postings = exact_postings.into_iter().collect::>(); let (integrity_digest, integrity_digest_bytes) = document_integrity_digest( document_id, chunk_id.as_bytes(), &row, + text.text.as_bytes(), &term_postings, &exact_postings, )?; @@ -561,22 +616,50 @@ fn prepare_document( integrity_digest_bytes, }, ngram_postings, + text, )) } +/// The `(kind, n-gram)` keys one admitted document contributes: its +/// normalized search text always, and its raw text where that differs from +/// the normalized form. Preparation and the sealed n-gram lists derive from +/// this one definition. +pub(super) fn document_ngram_keys( + search_text: &str, + sanitized_text: &str, + normalized_text: &str, + control: &dyn CodeIndexExecutionControlV1, +) -> Result, CodeLexicalArtifactErrorV1> { + let mut keys = document_ngrams(search_text.as_bytes(), control)? + .into_iter() + .map(|ngram| (NGRAM_NORMALIZED, i64::from(ngram))) + .collect::>(); + if sanitized_text.as_bytes() != normalized_text.as_bytes() { + keys.extend( + document_ngrams(sanitized_text.as_bytes(), control)? + .into_iter() + .filter(|ngram| ngram_is_case_sensitive(*ngram)) + .map(|ngram| (NGRAM_RAW_OVERRIDE, i64::from(ngram))), + ); + } + Ok(keys) +} + fn document_integrity_digest( document: i64, chunk_id: &[u8], row: &[u8], + text: &[u8], term_postings: &[PreparedTermPostingV1], exact_postings: &[(String, Vec)], ) -> Result<(ManifestDigest, [u8; 32]), CodeLexicalArtifactErrorV1> { let mut hasher = Sha256::new(); - hasher.update(b"tracedecay.code-lexical-artifact-derived-document.v3\0"); + hasher.update(b"tracedecay.code-lexical-artifact-derived-document.v4\0"); hasher.update(document.to_le_bytes()); hash_table(&mut hasher, "row", 1, |hasher, _| { hash_text(hasher, chunk_id)?; - hash_blob(hasher, row) + hash_blob(hasher, row)?; + hash_text(hasher, text) })?; hash_table( &mut hasher, @@ -698,6 +781,13 @@ fn prepared_retained_bytes( }) .and_then(|bytes| bytes.checked_add(page.ngram_digest.as_str().len())) .and_then(|bytes| bytes.checked_add(page.base_sections_receipt.capacity())) + .and_then(|bytes| { + page.row_blocks.iter().try_fold(bytes, |bytes, (_, block)| { + bytes + .checked_add(block.capacity()) + .and_then(|bytes| bytes.checked_add(std::mem::size_of::<(i64, Vec)>())) + }) + }) .ok_or_else(prepared_charge_overflow)?; for import in &page.imports { bytes = bytes @@ -768,7 +858,7 @@ fn prepared_clone_body_retained_bytes( bytes .checked_add(body.payload_digest.capacity()) .and_then(|bytes| bytes.checked_add(body.payload.capacity())) - .and_then(|bytes| bytes.checked_add(body.occurrence.capacity())) + .and_then(|bytes| bytes.checked_add(body.eligibility.capacity())) .and_then(|bytes| bytes.checked_add(body.symbol_occurrence_id.capacity())) .and_then(|bytes| bytes.checked_add(body.path.capacity())) .and_then(|bytes| { @@ -790,7 +880,6 @@ fn prepared_clone_body_retained_bytes( Some(stream) => { bytes .checked_add(stream.language.capacity()) - .and_then(|bytes| bytes.checked_add(stream.body_digest.capacity())) .and_then(|bytes| { bytes.checked_add(stream.positions.capacity().saturating_mul( std::mem::size_of::(), @@ -829,12 +918,18 @@ fn estimated_sqlite_writes( bytes = bytes .checked_add(clone_bytes) .ok_or_else(prepared_write_overflow)?; + for (_, block) in &page.row_blocks { + rows = rows.checked_add(1).ok_or_else(prepared_write_overflow)?; + bytes = bytes + .checked_add(block.len()) + .and_then(|bytes| bytes.checked_add(8)) + .ok_or_else(prepared_write_overflow)?; + } for document in &page.documents { - rows = rows.checked_add(2).ok_or_else(prepared_write_overflow)?; + rows = rows.checked_add(1).ok_or_else(prepared_write_overflow)?; bytes = bytes .checked_add(document.chunk_id.len()) - .and_then(|bytes| bytes.checked_add(document.row.len())) - .and_then(|bytes| bytes.checked_add(document.integrity_digest.as_str().len())) + .and_then(|bytes| bytes.checked_add(8)) .ok_or_else(prepared_write_overflow)?; for posting in &document.term_postings { rows = rows.checked_add(1).ok_or_else(prepared_write_overflow)?; @@ -852,15 +947,6 @@ fn estimated_sqlite_writes( .ok_or_else(prepared_write_overflow)?; } } - rows = rows - .checked_add(page.ngram_shards.len()) - .ok_or_else(prepared_write_overflow)?; - for shard in &page.ngram_shards { - bytes = bytes - .checked_add(shard.documents.len()) - .and_then(|bytes| bytes.checked_add(32)) - .ok_or_else(prepared_write_overflow)?; - } rows = rows .checked_add(page.row_dictionary.len()) .ok_or_else(prepared_write_overflow)?; @@ -891,7 +977,7 @@ fn estimated_clone_body_writes( let bytes = bytes .checked_add(body.payload_digest.len().saturating_mul(2)) .and_then(|bytes| bytes.checked_add(body.payload.len())) - .and_then(|bytes| bytes.checked_add(body.occurrence.len())) + .and_then(|bytes| bytes.checked_add(body.eligibility.len())) .and_then(|bytes| bytes.checked_add(body.symbol_occurrence_id.len())) .and_then(|bytes| bytes.checked_add(body.path.len())) .and_then(|bytes| { @@ -907,9 +993,7 @@ fn estimated_clone_body_writes( stream .language .len() - .saturating_add(stream.body_digest.len()) .saturating_add(body.symbol_occurrence_id.len()) - .saturating_add(body.payload_digest.len()) .saturating_add(32), ), ), diff --git a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/reader.rs b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/reader.rs index 10d7952d02..8f13edb913 100644 --- a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/reader.rs +++ b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/reader.rs @@ -6,7 +6,6 @@ pub use family_report::{CloneExactFamilyArtifactCandidateV1, CloneExactFamilyArt use std::cell::Cell; use std::cmp::Reverse; use std::collections::{BTreeMap, BTreeSet, BinaryHeap}; -use std::fmt::Write as _; use std::fs::File; use std::io::{Read, Seek, SeekFrom}; use std::path::{Path, PathBuf}; @@ -33,6 +32,9 @@ use tracedecay_private_fs::open_private_file; use super::builder::compute_section_digests; use super::clone_census::{CodeLexicalCloneIndexCensusV1, read_clone_index_census}; +use super::clone_codec::{ + CloneOccurrenceRouteV1, digest_key, routed_clone_body_row, stored_clone_occurrence, +}; use super::fingerprints::{ CLONE_FINGERPRINT_HOT_POSTING_THRESHOLD_V1, CloneFingerprintArtifactReadV1, CloneFingerprintReadRequestV1, CloneSelectedBlockArtifactCandidateV1, @@ -40,14 +42,18 @@ use super::fingerprints::{ }; use super::format::{ ArtifactRowV1, CodeLexicalArtifactOccurrenceV1, CodeLexicalImportMembershipWitnessV1, - VerifiedCodeLexicalArtifactV1, artifact_digest, decode_ngram_bitmap, decode_padded_receipt, - encode_exact_field, encode_field, metadata_digest, verify_required_artifact_indexes, + PostingListDecoderV1, VerifiedCodeLexicalArtifactV1, content_metadata_bytes, + decode_document_set, decode_ngram_bitmap, decode_padded_receipt, decode_term_lists, + receipt_artifact_digest, stored_metadata_digest as stored_metadata_digest_of, + verify_artifact_table_layout, +}; +use super::postings::{NGRAM_NORMALIZED, query_ngrams, raw_override_query_ngrams}; +use super::row_codec::{ + ConnectionRowDictionaryV1, ROW_BLOCK_BY_DOCUMENT_SQL, RowBlocksV1, StoredRowV1, + decode_artifact_row, stored_chunk_key, stored_symbol_key, }; -use super::postings::{NGRAM_NORMALIZED, NGRAM_RAW_OVERRIDE, query_ngrams}; -use super::row_codec::{ConnectionRowDictionaryV1, decode_artifact_row}; use super::schema::{ - LexicalArtifactLayoutV1, exact_field_code, field_code, field_from_code, lookup_term_id, - lookup_term_ids, stable_exact_term_id, + exact_field_code, field_from_code, require_served_revision, stable_exact_term_id, }; use super::{ ARTIFACT_SQLITE_CACHE_BYTES, ARTIFACT_SQLITE_CACHE_FLOOR_BYTES, @@ -129,12 +135,10 @@ pub struct CodeLexicalArtifactReaderV1 { path: Arc, metadata: super::super::CodeLexicalProjectionMetadataV1, receipt: VerifiedCodeLexicalArtifactV1, - layout: LexicalArtifactLayoutV1, clone_index_census: Arc, String>>>, retained_owned_bytes: usize, - /// Fuzzy expansion walks every in-fuzzy term. Hash-ordered `term_id` - /// rows make a fresh `ORDER BY term` scan random I/O; share one load - /// across clones and later queries on this reader. + /// Fuzzy expansion walks every in-fuzzy term; share one load across + /// clones and later queries on this reader. fuzzy_vocabulary: Arc>>>, } @@ -156,7 +160,8 @@ pub struct CloneArtifactCursorV1 { #[derive(Clone, Debug, PartialEq, Eq, serde::Deserialize, serde::Serialize)] pub(super) enum CloneArtifactCursorPositionV1 { - Exact(SymbolOccurrenceId), + /// The ordinal of the last occurrence the page returned. + Exact(i64), Fingerprint { body_digest: ManifestDigest, payload_digest: ManifestDigest, @@ -235,7 +240,7 @@ impl std::fmt::Debug for CodeLexicalArtifactReaderV1 { fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { formatter .debug_struct("CodeLexicalArtifactReaderV1") - .field("generation", self.receipt.generation()) + .field("generation", &self.metadata.generation) .field("artifact_digest", self.receipt.artifact_digest()) .field("retained_owned_bytes", &self.retained_owned_bytes) .finish_non_exhaustive() @@ -243,14 +248,6 @@ impl std::fmt::Debug for CodeLexicalArtifactReaderV1 { } impl CodeLexicalArtifactReaderV1 { - pub fn has_clone_index(&self) -> bool { - self.layout.has_clone_index() - } - - pub fn has_clone_fingerprints(&self) -> bool { - self.layout.has_clone_fingerprints() - } - /// Open a published artifact whose trust anchor is its content address: /// the durable head names the artifact file's size and SHA-256 digest, /// the embedded receipt is decoded only after the whole file matches @@ -262,6 +259,7 @@ impl CodeLexicalArtifactReaderV1 { path: impl AsRef, expected_file_digest: &ManifestDigest, expected_file_size_bytes: u64, + authority: &super::super::CodeLexicalProjectionMetadataV1, cache_budget_bytes: usize, control: &dyn CodeIndexExecutionControlV1, ) -> Result { @@ -306,8 +304,8 @@ impl CodeLexicalArtifactReaderV1 { connection .pragma_update(None, "query_only", true) .map_err(sqlite_error)?; - let layout = verify_artifact_state_revision(&connection, control)?; - verify_required_artifact_indexes(&connection, layout) + verify_artifact_state_revision(&connection, control)?; + verify_artifact_table_layout(&connection) })?; let receipt = hotpath::measure_block!("query.artifact.open.head_receipt_restore", { let receipt_bytes: Vec = connection @@ -334,6 +332,7 @@ impl CodeLexicalArtifactReaderV1 { path, connection, &receipt, + authority, cache_budget_bytes, expected_file_size_bytes, control, @@ -352,6 +351,7 @@ impl CodeLexicalArtifactReaderV1 { pub fn open_with_control( path: impl AsRef, expected: &VerifiedCodeLexicalArtifactV1, + authority: &super::super::CodeLexicalProjectionMetadataV1, cache_budget_bytes: usize, control: &dyn CodeIndexExecutionControlV1, ) -> Result { @@ -384,6 +384,7 @@ impl CodeLexicalArtifactReaderV1 { path, connection, expected, + authority, cache_budget_bytes, expected.file_size_bytes(), control, @@ -396,10 +397,15 @@ impl CodeLexicalArtifactReaderV1 { Ok(reader) } + /// `authority` is the opener's projection: the artifact stores only its + /// content part, and the reader serves the opener's generation, + /// repository, freshness, and clone route. + #[allow(clippy::too_many_arguments)] // both open routes' authorities, bound once fn open_connection_with_control( path: &Path, connection: Connection, expected: &VerifiedCodeLexicalArtifactV1, + authority: &super::super::CodeLexicalProjectionMetadataV1, cache_budget_bytes: usize, sealed_file_size_bytes: u64, control: &dyn CodeIndexExecutionControlV1, @@ -410,8 +416,8 @@ impl CodeLexicalArtifactReaderV1 { connection .pragma_update(None, "query_only", true) .map_err(sqlite_error)?; - let layout = verify_artifact_state_revision(&connection, control)?; - verify_required_artifact_indexes(&connection, layout) + verify_artifact_state_revision(&connection, control)?; + verify_artifact_table_layout(&connection) })?; // Read the BLOB length first so the page cache can be configured // before metadata is materialized. The retained metadata copy plus @@ -462,9 +468,12 @@ impl CodeLexicalArtifactReaderV1 { .to_owned(), )); } - let metadata: super::super::CodeLexicalProjectionMetadataV1 = - serde_json::from_slice(&stored_metadata_bytes) - .map_err(|error| CodeLexicalArtifactErrorV1::Corrupt(error.to_string()))?; + if stored_metadata_bytes != content_metadata_bytes(authority)? { + return Err(CodeLexicalArtifactErrorV1::Incompatible( + "lexical artifact content does not match the opening projection".to_owned(), + )); + } + let metadata = authority.clone(); Ok::<_, CodeLexicalArtifactErrorV1>(( page_cache_bytes, stored_metadata_bytes, @@ -499,19 +508,10 @@ impl CodeLexicalArtifactReaderV1 { "lexical artifact receipt does not match its verified seat".to_owned(), )); } - let layout = LexicalArtifactLayoutV1::from_revision(stored.format_revision())?; - let state_layout = verify_artifact_state_revision(&connection, control)?; - if layout != state_layout { - return Err(CodeLexicalArtifactErrorV1::Corrupt( - "lexical artifact receipt revision does not match artifact state".to_owned(), - )); - } - let decoded_metadata_digest = metadata_digest(&metadata)?; - if &decoded_metadata_digest != stored.metadata_digest() + require_served_revision(stored.format_revision())?; + verify_artifact_state_revision(&connection, control)?; + if &stored_metadata_digest_of(&stored_metadata_bytes)? != stored.metadata_digest() || stored_metadata_digest != stored.metadata_digest().as_str() - || &metadata.generation != stored.generation() - || metadata.repository_id.as_ref() != stored.repository_id() - || &metadata.freshness != stored.freshness() { return Err(CodeLexicalArtifactErrorV1::Corrupt( "lexical artifact metadata digest does not verify".to_owned(), @@ -519,7 +519,7 @@ impl CodeLexicalArtifactReaderV1 { } let sections = hotpath::measure_block!( "query.artifact.open.section_digest_verify", - compute_section_digests(&connection, control, layout) + compute_section_digests(&connection, control) )?; if sections != stored.section_digests() { return Err(CodeLexicalArtifactErrorV1::Corrupt( @@ -528,20 +528,7 @@ impl CodeLexicalArtifactReaderV1 { } let digest = hotpath::measure_block!( "query.artifact.open.artifact_digest_verify", - artifact_digest( - stored.metadata_digest(), - stored.source_state_digest(), - stored.source_format_revision(), - stored.page_count(), - stored.total_chunks(), - stored.total_payload_bytes(), - stored.total_imports(), - stored.import_payload_bytes(), - stored.import_dictionary_digest(), - stored.source_cumulative_digest(), - §ions, - stored.format_revision(), - ) + receipt_artifact_digest(&stored, §ions) )?; if &digest != stored.artifact_digest() { return Err(CodeLexicalArtifactErrorV1::Corrupt( @@ -559,7 +546,6 @@ impl CodeLexicalArtifactReaderV1 { path: Arc::new(path.to_path_buf()), metadata, receipt: stored, - layout, clone_index_census: Arc::new(OnceLock::new()), retained_owned_bytes, fuzzy_vocabulary: Arc::new(OnceLock::new()), @@ -578,16 +564,13 @@ impl CodeLexicalArtifactReaderV1 { #[hotpath::skip] pub fn artifact_format_revision(&self) -> u32 { - self.layout.revision() + self.receipt.format_revision() } #[hotpath::skip] pub fn clone_index_census( &self, ) -> Result>, CodeLexicalArtifactErrorV1> { - if !self.layout.has_clone_index() { - return Ok(None); - } let census = self.clone_index_census.get_or_init(|| { let file = open_private_file(self.path.as_ref()) .map_err(map_private_artifact_file_error) @@ -606,13 +589,10 @@ impl CodeLexicalArtifactReaderV1 { .map_err(|error| error.to_string())?; verify_named_path_identity(self.path.as_ref(), &file) .map_err(|error| error.to_string())?; - let census = read_clone_index_census( - &connection, - self.layout.has_clone_fingerprints(), - CLONE_FINGERPRINT_HOT_POSTING_THRESHOLD_V1, - ) - .map(Arc::new) - .map_err(|error| error.to_string())?; + let census = + read_clone_index_census(&connection, CLONE_FINGERPRINT_HOT_POSTING_THRESHOLD_V1) + .map(Arc::new) + .map_err(|error| error.to_string())?; verify_named_path_identity(self.path.as_ref(), &file) .map_err(|error| error.to_string())?; Ok(census) @@ -633,32 +613,42 @@ impl CodeLexicalArtifactReaderV1 { chunk: &CodeSearchChunkId, ) -> Result, CodeLexicalArtifactErrorV1> { let connection = self.lock_connection()?; - let row: Option> = connection + let document: Option = connection .query_row( - "SELECT row FROM rows WHERE chunk_id = ?1", - [chunk.as_str()], + "SELECT document_id FROM row_chunks WHERE chunk_id = ?1", + [stored_chunk_key(chunk.as_str())], |row| row.get(0), ) .optional() .map_err(sqlite_error)?; - row.map(|bytes| { - decode_artifact_row( - self.layout, - self.receipt.generation(), - chunk.as_str(), - &bytes, - &ConnectionRowDictionaryV1::new(&connection), - ) - .map(row_occurrence) - }) - .transpose() + let Some(document) = document else { + return Ok(None); + }; + let stored = RowBlocksV1::new(&connection).row( + u32::try_from(document) + .map_err(|error| CodeLexicalArtifactErrorV1::Corrupt(error.to_string()))?, + )?; + if stored.chunk_id != chunk.as_str() { + return Err(CodeLexicalArtifactErrorV1::Corrupt( + "lexical artifact chunk lookup names another document's row".to_owned(), + )); + } + decode_artifact_row( + &self.metadata.generation, + &stored.chunk_id, + &stored.row, + &stored.text, + &ConnectionRowDictionaryV1::new(&connection), + ) + .map(row_occurrence) + .map(Some) } pub fn occurrence_by_binding( &self, binding: &CodeCandidateBindingV1, ) -> Result, CodeLexicalArtifactErrorV1> { - if &binding.occurrence.generation != self.receipt.generation() { + if binding.occurrence.generation != self.metadata.generation { return Err(CodeLexicalArtifactErrorV1::Contract( "candidate binding belongs to another generation".to_owned(), )); @@ -689,7 +679,7 @@ impl CodeLexicalArtifactReaderV1 { let connection = self.lock_connection()?; let stored: Option> = connection .query_row( - "SELECT evidence FROM import_evidence WHERE canonical = ?1", + "SELECT canonical FROM import_evidence WHERE canonical = ?1", [canonical], |row| row.get(0), ) @@ -731,12 +721,7 @@ impl CodeLexicalArtifactReaderV1 { control: &dyn CodeIndexExecutionControlV1, ) -> Result, CodeLexicalArtifactErrorV1> { checkpoint(control)?; - self.validate_clone_lookup_authority(authority)?; - if !self.layout.has_clone_index() { - return Err(CodeLexicalArtifactErrorV1::Incompatible( - "clone lookup requires lexical artifact revision 15".to_owned(), - )); - } + let route = self.validate_clone_lookup_authority(authority)?; if limit == 0 || limit > MAX_CLONE_EXACT_PAGE_MEMBERS_V1 { return Err(CodeLexicalArtifactErrorV1::Contract(format!( "clone exact page limit must be within 1..={MAX_CLONE_EXACT_PAGE_MEMBERS_V1}" @@ -757,21 +742,24 @@ impl CodeLexicalArtifactReaderV1 { let connection = self.lock_connection()?; let mut statement = connection .prepare_cached( - "SELECT posting.symbol_occurrence_id, posting.payload_digest, occurrence.occurrence, payload.payload \ + "SELECT posting.occurrence_ordinal, \ + occurrence.symbol_key, payload.payload_digest, occurrence.path, \ + occurrence.body_start, occurrence.body_end, occurrence.eligibility, payload.payload \ FROM clone_exact_postings AS posting \ - LEFT JOIN clone_occurrences AS occurrence ON occurrence.symbol_occurrence_id = posting.symbol_occurrence_id \ - LEFT JOIN clone_body_payloads AS payload ON payload.payload_digest = posting.payload_digest \ + LEFT JOIN clone_occurrences AS occurrence ON occurrence.ordinal = posting.occurrence_ordinal \ + LEFT JOIN clone_body_payloads AS payload ON payload.ordinal = occurrence.payload_ordinal \ WHERE posting.class = ?1 AND posting.normalization_revision = ?2 AND posting.digest = ?3 \ - AND posting.symbol_occurrence_id != ?4 AND posting.symbol_occurrence_id > ?5 \ - ORDER BY posting.symbol_occurrence_id LIMIT ?6", + AND posting.occurrence_ordinal > ?5 \ + AND (occurrence.symbol_key IS NULL OR occurrence.symbol_key != ?4) \ + ORDER BY posting.occurrence_ordinal LIMIT ?6", ) .map_err(sqlite_error)?; let mut rows = statement .query(rusqlite::params![ i64::from(key.class as u8), i64::from(key.normalization_revision), - key.digest.as_str(), - authority.symbol_occurrence_id.as_str(), + digest_key(&key.digest)?.as_slice(), + stored_symbol_key(authority.symbol_occurrence_id.as_str()), after, i64::try_from(fetch) .map_err(|error| { CodeLexicalArtifactErrorV1::Contract(error.to_string()) })?, @@ -782,24 +770,24 @@ impl CodeLexicalArtifactReaderV1 { if members.len().is_multiple_of(RETRIEVAL_CANDIDATE_BATCH_SIZE) { checkpoint(control)?; } - members.push(self.verified_clone_member(authority, key, row)?); + members.push(Self::verified_clone_member(&route, key, row)?); } checkpoint(control)?; let next_cursor = (members.len() > limit) .then(|| { - members.get(limit - 1).map(|member| CloneArtifactCursorV1 { - artifact_digest: self.receipt.artifact_digest().clone(), - generation: self.metadata.generation.clone(), - request_digest, - after: CloneArtifactCursorPositionV1::Exact( - member.occurrence.symbol_occurrence_id.clone(), - ), - }) + members + .get(limit - 1) + .map(|(ordinal, _)| CloneArtifactCursorV1 { + artifact_digest: self.receipt.artifact_digest().clone(), + generation: self.metadata.generation.clone(), + request_digest, + after: CloneArtifactCursorPositionV1::Exact(*ordinal), + }) }) .flatten(); members.truncate(limit); Ok(CloneArtifactPageV1 { - members, + members: members.into_iter().map(|(_, member)| member).collect(), next_cursor, }) } @@ -812,14 +800,14 @@ impl CodeLexicalArtifactReaderV1 { limit: usize, control: &dyn CodeIndexExecutionControlV1, ) -> Result { - self.validate_clone_lookup_authority(authority)?; + let route = self.validate_clone_lookup_authority(authority)?; let authority_digest = clone_authority_digest(authority)?; let connection = self.lock_connection()?; read_clone_fingerprint_page( &connection, CloneFingerprintReadRequestV1 { - layout: self.layout, receipt: &self.receipt, + route: &route, authority_digest: &authority_digest, authority, source, @@ -835,56 +823,31 @@ impl CodeLexicalArtifactReaderV1 { &self, symbol: &SymbolOccurrenceId, ) -> Result, CodeLexicalArtifactErrorV1> { - if !self.layout.has_clone_index() { - return Err(CodeLexicalArtifactErrorV1::Incompatible( - "clone lookup requires lexical artifact revision 15".to_owned(), - )); - } + let route = self.clone_route()?; let connection = self.lock_connection()?; let row = connection .query_row( - "SELECT occurrence.occurrence, payload.payload \ + "SELECT occurrence.symbol_key, payload.payload_digest, occurrence.path, \ + occurrence.body_start, occurrence.body_end, occurrence.eligibility, payload.payload \ FROM clone_occurrences AS occurrence \ LEFT JOIN clone_body_payloads AS payload \ - ON payload.payload_digest = occurrence.payload_digest \ - WHERE occurrence.symbol_occurrence_id = ?1", - [symbol.as_str()], - |row| Ok((row.get::<_, Vec>(0)?, row.get::<_, Option>>(1)?)), + ON payload.ordinal = occurrence.payload_ordinal \ + WHERE occurrence.symbol_key = ?1", + [stored_symbol_key(symbol.as_str())], + routed_clone_body_row, ) .optional() .map_err(sqlite_error)?; - let Some((occurrence, payload)) = row else { + let Some(row) = row else { return Ok(None); }; - let payload = payload.ok_or_else(|| { - CodeLexicalArtifactErrorV1::Corrupt( - "clone occurrence is missing its payload".to_owned(), - ) - })?; - let occurrence: CloneBodyOccurrenceV1 = - serde_json::from_slice(&occurrence).map_err(|error| { - CodeLexicalArtifactErrorV1::Corrupt(format!( - "clone occurrence is not canonical JSON: {error}" - )) - })?; - let payload: CloneBodyPayloadV1 = serde_json::from_slice(&payload).map_err(|error| { - CodeLexicalArtifactErrorV1::Corrupt(format!( - "clone body payload is not canonical JSON: {error}" - )) - })?; - self.validate_clone_lookup_authority(&occurrence)?; - if occurrence.symbol_occurrence_id != *symbol - || occurrence.payload_digest != payload.payload_digest - || payload.validate().is_err() - { + let body = route.clone_body(row)?; + if body.occurrence.symbol_occurrence_id != *symbol { return Err(CodeLexicalArtifactErrorV1::Corrupt( "clone body lookup failed canonical validation".to_owned(), )); } - Ok(Some(CodeIndexCloneBodyV1 { - payload: Arc::new(payload), - occurrence, - })) + Ok(Some(body)) } pub fn clone_body_by_source_range( @@ -892,24 +855,21 @@ impl CodeLexicalArtifactReaderV1 { path: &str, span: SourceSpan, ) -> Result, CodeLexicalArtifactErrorV1> { - if !self.layout.has_clone_index() { - return Err(CodeLexicalArtifactErrorV1::Incompatible( - "clone lookup requires lexical artifact revision 15".to_owned(), - )); - } span.validate() .map_err(|error| CodeLexicalArtifactErrorV1::Contract(error.to_string()))?; + let route = self.clone_route()?; let connection = self.lock_connection()?; let row = connection .query_row( - "SELECT occurrence.occurrence, payload.payload \ + "SELECT occurrence.symbol_key, payload.payload_digest, occurrence.path, \ + occurrence.body_start, occurrence.body_end, occurrence.eligibility, payload.payload \ FROM clone_occurrences AS occurrence \ LEFT JOIN clone_body_payloads AS payload \ - ON payload.payload_digest = occurrence.payload_digest \ + ON payload.ordinal = occurrence.payload_ordinal \ WHERE occurrence.path = ?1 AND occurrence.body_start <= ?2 \ AND occurrence.body_end >= ?3 \ ORDER BY occurrence.body_end - occurrence.body_start, \ - occurrence.symbol_occurrence_id \ + occurrence.symbol_key \ LIMIT 1", rusqlite::params![ path, @@ -920,63 +880,60 @@ impl CodeLexicalArtifactReaderV1 { CodeLexicalArtifactErrorV1::Contract(error.to_string()) })?, ], - |row| Ok((row.get::<_, Vec>(0)?, row.get::<_, Option>>(1)?)), + routed_clone_body_row, ) .optional() .map_err(sqlite_error)?; - let Some((occurrence, payload)) = row else { + let Some(row) = row else { return Ok(None); }; - let payload = payload.ok_or_else(|| { - CodeLexicalArtifactErrorV1::Corrupt( - "clone occurrence is missing its payload".to_owned(), - ) - })?; - let occurrence: CloneBodyOccurrenceV1 = - serde_json::from_slice(&occurrence).map_err(|error| { - CodeLexicalArtifactErrorV1::Corrupt(format!( - "clone occurrence is not canonical JSON: {error}" - )) - })?; - let payload: CloneBodyPayloadV1 = serde_json::from_slice(&payload).map_err(|error| { - CodeLexicalArtifactErrorV1::Corrupt(format!( - "clone body payload is not canonical JSON: {error}" - )) - })?; - self.validate_clone_lookup_authority(&occurrence)?; - if occurrence.path != path - || occurrence.body_span.start_byte > span.start_byte - || occurrence.body_span.end_byte < span.end_byte - || occurrence.payload_digest != payload.payload_digest - || payload.validate().is_err() + let body = route.clone_body(row)?; + if body.occurrence.path != path + || body.occurrence.body_span.start_byte > span.start_byte + || body.occurrence.body_span.end_byte < span.end_byte { return Err(CodeLexicalArtifactErrorV1::Corrupt( "source range lookup disagrees with its clone occurrence".to_owned(), )); } - Ok(Some(CodeIndexCloneBodyV1 { - payload: Arc::new(payload), - occurrence, - })) + Ok(Some(body)) + } + + /// The route clone occurrences are served under: the opener's project, + /// repository, worktree, generation, and snapshot. + fn clone_route(&self) -> Result { + match (&self.metadata.clone_route, &self.metadata.repository_id) { + (Some(route), Some(repository_id)) => Ok(CloneOccurrenceRouteV1 { + project_id: route.project_id.clone(), + repository_id: repository_id.clone(), + worktree_id: route.worktree_id.clone(), + source_generation: self.metadata.generation.clone(), + snapshot_digest: route.snapshot_digest.clone(), + }), + _ => Err(CodeLexicalArtifactErrorV1::Missing( + "lexical artifact opener carries no clone route authority".to_owned(), + )), + } } fn validate_clone_lookup_authority( &self, authority: &CloneBodyOccurrenceV1, - ) -> Result<(), CodeLexicalArtifactErrorV1> { - if self.metadata.repository_id.as_ref() != Some(&authority.repository_id) { + ) -> Result { + let route = self.clone_route()?; + if !route.owns(authority) { return Err(CodeLexicalArtifactErrorV1::Missing( - "clone lookup repository authority is unavailable".to_owned(), + "clone lookup route authority is unavailable".to_owned(), )); } - if self.metadata.generation != authority.source_generation { + if route.source_generation != authority.source_generation { return Err(CodeLexicalArtifactErrorV1::Missing(format!( "clone lookup generation {} is stale; the artifact serves {}", authority.source_generation.as_str(), - self.metadata.generation.as_str() + route.source_generation.as_str() ))); } - Ok(()) + Ok(route) } pub fn clone_selected_block_page( @@ -988,14 +945,14 @@ impl CodeLexicalArtifactReaderV1 { limit: usize, control: &dyn CodeIndexExecutionControlV1, ) -> Result { - self.validate_clone_lookup_authority(authority)?; + let route = self.validate_clone_lookup_authority(authority)?; let authority_digest = clone_authority_digest(authority)?; let connection = self.lock_connection()?; let read = read_clone_fingerprint_page( &connection, CloneFingerprintReadRequestV1 { - layout: self.layout, receipt: &self.receipt, + route: &route, authority_digest: &authority_digest, authority, source, @@ -1040,11 +997,11 @@ impl CodeLexicalArtifactReaderV1 { }) } - fn clone_exact_after<'a>( + fn clone_exact_after( &self, - cursor: Option<&'a CloneArtifactCursorV1>, + cursor: Option<&CloneArtifactCursorV1>, request_digest: &ManifestDigest, - ) -> Result<&'a str, CodeLexicalArtifactErrorV1> { + ) -> Result { match cursor { Some(cursor) if cursor.artifact_digest == *self.receipt.artifact_digest() @@ -1052,9 +1009,7 @@ impl CodeLexicalArtifactReaderV1 { && cursor.request_digest == *request_digest => { match &cursor.after { - CloneArtifactCursorPositionV1::Exact(symbol_occurrence_id) => { - Ok(symbol_occurrence_id.as_str()) - } + CloneArtifactCursorPositionV1::Exact(ordinal) => Ok(*ordinal), CloneArtifactCursorPositionV1::Fingerprint { .. } => { Err(CodeLexicalArtifactErrorV1::Contract( "clone cursor position does not match an exact read".to_owned(), @@ -1065,51 +1020,39 @@ impl CodeLexicalArtifactReaderV1 { Some(_) => Err(CodeLexicalArtifactErrorV1::Contract( "clone exact cursor does not match its artifact, key, or authority".to_owned(), )), - None => Ok(""), + None => Ok(0), } } fn verified_clone_member( - &self, - authority: &CloneBodyOccurrenceV1, + route: &CloneOccurrenceRouteV1, key: &CloneExactKeyV1, row: &rusqlite::Row<'_>, - ) -> Result { - let posting_occurrence: String = row.get(0).map_err(sqlite_error)?; - let posting_payload: String = row.get(1).map_err(sqlite_error)?; - let occurrence_bytes: Option> = row.get(2).map_err(sqlite_error)?; - let payload_bytes: Option> = row.get(3).map_err(sqlite_error)?; - let (Some(occurrence_bytes), Some(payload_bytes)) = (occurrence_bytes, payload_bytes) - else { + ) -> Result<(i64, CloneExactArtifactMemberV1), CodeLexicalArtifactErrorV1> { + let ordinal: i64 = row.get(0).map_err(sqlite_error)?; + let Some(stored) = stored_clone_occurrence(row, 1)? else { return Err(CodeLexicalArtifactErrorV1::Corrupt( - "clone exact posting is missing its occurrence or payload".to_owned(), + "clone exact posting is missing its occurrence".to_owned(), )); }; - let occurrence: CloneBodyOccurrenceV1 = serde_json::from_slice(&occurrence_bytes) - .map_err(|error| CodeLexicalArtifactErrorV1::Corrupt(error.to_string()))?; - let payload: CloneBodyPayloadV1 = serde_json::from_slice(&payload_bytes) - .map_err(|error| CodeLexicalArtifactErrorV1::Corrupt(error.to_string()))?; - if occurrence.symbol_occurrence_id.as_str() != posting_occurrence - || occurrence.project_id != authority.project_id - || occurrence.repository_id != authority.repository_id - || occurrence.worktree_id != authority.worktree_id - || occurrence.source_generation != self.metadata.generation - || occurrence.payload_digest.as_str() != posting_payload - || occurrence.payload_digest != payload.payload_digest - || payload.validate().is_err() - || !payload - .exact_keys(occurrence.eligibility) - .iter() - .any(|candidate| candidate == key) + let payload_bytes: Option> = row.get(7).map_err(sqlite_error)?; + let (occurrence, payload) = route.occurrence_and_payload((stored, payload_bytes))?; + if !payload + .exact_keys(occurrence.eligibility) + .iter() + .any(|candidate| candidate == key) { return Err(CodeLexicalArtifactErrorV1::Corrupt( "clone exact posting does not match its payload and occurrence".to_owned(), )); } - Ok(CloneExactArtifactMemberV1 { - payload, - occurrence, - }) + Ok(( + ordinal, + CloneExactArtifactMemberV1 { + payload, + occurrence, + }, + )) } /// Reader queries serialize on this one connection; the wait span makes @@ -1126,7 +1069,7 @@ impl CodeLexicalArtifactReaderV1 { } fn validate_generation(&self, generation: &CodeGenerationId) -> Result<(), RetrievalPortError> { - if generation != self.receipt.generation() { + if generation != &self.metadata.generation { Err(RetrievalPortError::GenerationMismatch) } else { Ok(()) @@ -1141,18 +1084,17 @@ impl LexicalPostingReadPort for CodeLexicalArtifactReaderV1 { request: &LexicalLaneRequest<'_>, ) -> Result>, RetrievalPortError> { self.validate_generation(&request.generation)?; - if self.receipt.freshness().compatibility + if self.metadata.freshness.compatibility != tracedecay_domain::FreshnessCompatibilityV1::Current { crate::hotpath_metrics::Residency::Rebuilding.record("query.lane.lexical.residency"); - return Ok(RetrieverOutcome::Stale(self.receipt.freshness().clone())); + return Ok(RetrieverOutcome::Stale(self.metadata.freshness.clone())); } let connection = self.lock_connection().map_err(map_query_artifact_error)?; let outcome = ArtifactQueryV1::new( &connection, &self.metadata, &self.receipt, - self.layout, &self.fuzzy_vocabulary, )? .lexical_batch(request)?; @@ -1183,12 +1125,12 @@ where request: &ExactLaneRequest, ) -> Result>, RetrievalPortError> { self.reader.validate_generation(&request.generation)?; - if self.reader.receipt.freshness().compatibility + if self.reader.metadata.freshness.compatibility != tracedecay_domain::FreshnessCompatibilityV1::Current { crate::hotpath_metrics::Residency::Rebuilding.record("query.lane.exact.residency"); return Ok(RetrieverOutcome::Stale( - self.reader.receipt.freshness().clone(), + self.reader.metadata.freshness.clone(), )); } let connection = self @@ -1199,7 +1141,6 @@ where &connection, &self.reader.metadata, &self.reader.receipt, - self.reader.layout, &self.reader.fuzzy_vocabulary, )? .exact_batch(request, &self.authority)?; @@ -1217,13 +1158,13 @@ where struct ArtifactQueryV1<'a> { connection: &'a Connection, metadata: &'a super::super::CodeLexicalProjectionMetadataV1, - receipt: &'a VerifiedCodeLexicalArtifactV1, - layout: LexicalArtifactLayoutV1, document_count: usize, metrics: ArtifactQueryMetricsV1, fuzzy_vocabulary: &'a OnceLock>>, - /// Revision-14 dictionary entries resolved during this query. + /// Row dictionary entries resolved during this query. row_dictionary: ConnectionRowDictionaryV1<'a>, + /// The row block this query decoded last. + row_blocks: RowBlocksV1<'a>, } #[derive(Default)] @@ -1233,7 +1174,7 @@ struct ArtifactQueryMetricsV1 { #[cfg(test)] fullscan_steps: Cell, #[cfg(test)] - ngram_decoded_shards: Cell, + ngram_decoded_lists: Cell, #[cfg(test)] ngram_peak_candidates: Cell, } @@ -1284,9 +1225,9 @@ impl ArtifactQueryMetricsV1 { } #[cfg(test)] - fn observe_ngram_shard(&self) { - self.ngram_decoded_shards - .set(self.ngram_decoded_shards.get().saturating_add(1)); + fn observe_ngram_list(&self) { + self.ngram_decoded_lists + .set(self.ngram_decoded_lists.get().saturating_add(1)); } #[cfg(test)] @@ -1296,393 +1237,157 @@ impl ArtifactQueryMetricsV1 { } } -/// A SQLite-owned candidate set. The query is evaluated row-by-row, so Rust -/// never retains one identifier per matching document. Query input is already -/// bounded by the lexical request contract; n-gram intersections additionally -/// have a fixed predicate ceiling below. -#[derive(Clone, Debug)] -struct DocumentQueryV1 { - sql: Option, - parameters: Vec, - maximum_bound_value_bytes: usize, -} - -impl DocumentQueryV1 { - fn empty() -> Self { - Self { - sql: None, - parameters: Vec::new(), - maximum_bound_value_bytes: ARTIFACT_SQLITE_MAX_BOUND_VALUE_BYTES_V1, - } - } - - fn term(field: String, term: String) -> Self { - Self { - sql: Some( - "SELECT document_id FROM term_postings WHERE field = ? AND term = ?".to_owned(), - ), - parameters: vec![Value::Text(field), Value::Text(term)], - maximum_bound_value_bytes: ARTIFACT_SQLITE_MAX_BOUND_VALUE_BYTES_V1, - } - } - - fn term_except(term: String, excluded_field: String) -> Self { - Self { - sql: Some( - "SELECT document_id FROM term_postings WHERE term = ? AND field != ?".to_owned(), - ), - parameters: vec![Value::Text(term), Value::Text(excluded_field)], - maximum_bound_value_bytes: ARTIFACT_SQLITE_MAX_BOUND_VALUE_BYTES_V1, - } - } - - fn exact(field: String, term: Vec) -> Self { - Self { - sql: Some( - "SELECT document_id FROM exact_postings WHERE field = ? AND term = ?".to_owned(), - ), - parameters: vec![Value::Text(field), Value::Blob(term)], - maximum_bound_value_bytes: ARTIFACT_SQLITE_MAX_BOUND_VALUE_BYTES_V1, - } - } - - fn exact_id(field: ExactFieldV1, term: &[u8]) -> Self { - Self { - sql: Some( - "SELECT document_id FROM exact_postings WHERE field = ? AND term_id = ?".to_owned(), - ), - parameters: vec![ - Value::Integer(exact_field_code(field)), - Value::Integer(stable_exact_term_id(term)), - ], - maximum_bound_value_bytes: ARTIFACT_SQLITE_MAX_BOUND_VALUE_BYTES_V1, - } - } - - fn term_id(field: i64, term_id: i64) -> Self { - Self { - sql: Some( - "SELECT document_id FROM term_postings WHERE field = ? AND term_id = ?".to_owned(), - ), - parameters: vec![Value::Integer(field), Value::Integer(term_id)], - maximum_bound_value_bytes: ARTIFACT_SQLITE_MAX_BOUND_VALUE_BYTES_V1, - } - } - - fn term_except_id(term_id: i64, excluded_field: i64) -> Self { - Self { - sql: Some( - "SELECT document_id FROM term_postings WHERE term_id = ? AND field != ?".to_owned(), - ), - parameters: vec![Value::Integer(term_id), Value::Integer(excluded_field)], - maximum_bound_value_bytes: ARTIFACT_SQLITE_MAX_BOUND_VALUE_BYTES_V1, - } - } -} - -/// The query engine, rather than an in-process bitmap, owns duplicate removal -/// and sorted candidate enumeration. SQLite's configured fixed page cache is -/// the only storage used for the set-operation work table. -const ARTIFACT_UNION_COMPOUND_ARMS_V1: usize = 64; - -fn union_document_queries( - queries: impl IntoIterator, -) -> Result { - let queries = queries - .into_iter() - .filter(|query| query.sql.is_some()) - .collect::>(); - if queries.is_empty() { - return Ok(DocumentQueryV1::empty()); - } - - // Keep each compound arm below SQLite's expression limit, while sharing - // equal bind values across arms. A large fuzzy/phrase request commonly - // repeats its encoded field, so retaining one bind slot for every textual - // occurrence needlessly crosses the portable 999-variable ceiling even - // though the request's distinct values remain bounded. Named parameters - // also remain safe when this query is embedded in the frequency probe. - let maximum_bound_value_bytes = queries - .iter() - .map(|query| query.maximum_bound_value_bytes) - .max() - .unwrap_or(ARTIFACT_SQLITE_MAX_BOUND_VALUE_BYTES_V1); - let mut parameters = Vec::new(); - let mut level = queries - .into_iter() - .map(|query| { - let Some(sql) = query.sql else { - return Ok(String::new()); - }; - let sql = rewrite_union_query_parameters(&sql, &query.parameters, &mut parameters)?; - Ok(format!("SELECT document_id FROM ({sql})")) - }) - .collect::, RetrievalPortError>>()? - .into_iter() - .filter(|query| !query.is_empty()) - .collect::>(); - while level.len() > 1 { - level = level - .chunks(ARTIFACT_UNION_COMPOUND_ARMS_V1) - .map(|queries| { - let sql = queries.join(" UNION ALL "); - format!("SELECT document_id FROM ({sql})") - }) - .collect(); - } - let Some(root) = level.pop() else { - return Ok(DocumentQueryV1::empty()); - }; - Ok(DocumentQueryV1 { - sql: Some(format!( - "SELECT DISTINCT document_id FROM ({root}) ORDER BY document_id" - )), - parameters, - maximum_bound_value_bytes, - }) -} - -fn rewrite_union_query_parameters( - sql: &str, - query_parameters: &[Value], - parameters: &mut Vec, -) -> Result { - let mut rewritten = String::with_capacity(sql.len()); - let mut parameter_ordinal = 0usize; - for character in sql.chars() { - if character != '?' { - rewritten.push(character); - continue; - } - let Some(value) = query_parameters.get(parameter_ordinal) else { - return Err(RetrievalPortError::Contract( - "document query SQL has more bind placeholders than values".to_owned(), - )); - }; - let slot = if let Some(slot) = parameters.iter().position(|candidate| candidate == value) { - slot - } else { - parameters.push(value.clone()); - parameters.len() - 1 - }; - rewritten.push_str(":d"); - rewritten.push_str(&slot.to_string()); - parameter_ordinal += 1; - } - if parameter_ordinal != query_parameters.len() { - return Err(RetrievalPortError::Contract( - "document query has values without bind placeholders".to_owned(), - )); - } - Ok(rewritten) -} - +/// Visit a candidate set in ascending document order. The set holds at most +/// one bit per artifact document however many sources contributed; request +/// authority is consulted before each candidate batch and at completion. fn visit_document_ids( - connection: &Connection, - query: &DocumentQueryV1, + documents: &RoaringBitmap, control: &dyn RetrievalExecutionControl, mut visitor: impl FnMut(u32) -> Result<(), RetrievalPortError>, ) -> Result<(), RetrievalPortError> { hotpath::measure_block!("query.stream.visit_documents", { - let Some(sql) = &query.sql else { - return Ok(()); - }; - ensure_sqlite_bind_capacity(0, query.parameters.len())?; - ensure_sqlite_bound_value_bytes( - query.maximum_bound_value_bytes, - &query.parameters, - std::iter::empty(), - )?; - let mut statement = connection.prepare(sql).map_err(map_query_sql_error)?; - let mut rows = statement - .query(params_from_iter(query.parameters.iter())) - .map_err(map_query_sql_error)?; - let mut visited = 0u64; - while let Some(row) = rows.next().map_err(map_query_sql_error)? { - if visited.is_multiple_of(RETRIEVAL_CANDIDATE_BATCH_SIZE as u64) { + for (visited, document) in documents.iter().enumerate() { + if visited.is_multiple_of(RETRIEVAL_CANDIDATE_BATCH_SIZE) { retrieval_checkpoint(control)?; } - let document = row.get::<_, i64>(0).map_err(map_query_sql_error)?; - visitor(u32::try_from(document).map_err(contract_error)?)?; - visited += 1; + visitor(document)?; } retrieval_checkpoint(control)?; - hotpath::gauge!("query.stream.rows_total").inc(visited); + hotpath::gauge!("query.stream.rows_total").inc(documents.len()); Ok(()) }) } -/// Stream each candidate row with all request-relevant term frequencies from -/// one SQLite statement. The correlated posting lookup seeks the maintained -/// document index; it never emits the row BLOB once per matching term. -/// -/// Request authority is consulted before each page-sized batch and at stream -/// completion, bounding abandoned work without a route lookup per candidate. +/// Stream each candidate row with every request-term frequency it carries. +/// The request's posting lists advance once in document order alongside the +/// ascending candidates, so a row costs at most one keyed block read and no +/// posting probe. fn visit_lexical_rows( connection: &Connection, - documents: &DocumentQueryV1, - terms: &BTreeSet, + rows: &RowBlocksV1<'_>, + documents: &RoaringBitmap, + postings: &RequestTermPostingsV1, metrics: &ArtifactQueryMetricsV1, - layout: LexicalArtifactLayoutV1, control: &dyn RetrievalExecutionControl, mut visitor: impl FnMut( u32, - String, - Vec, + StoredRowV1, LexicalTermFrequenciesV1, ) -> Result<(), RetrievalPortError>, ) -> Result<(), RetrievalPortError> { hotpath::measure_block!("query.stream.visit_lexical_rows", { - let Some(document_sql) = documents.sql.as_deref() else { - return Ok(()); - }; - let assigned_ids = match layout { - LexicalArtifactLayoutV1::V10 => BTreeMap::new(), - LexicalArtifactLayoutV1::V11 - | LexicalArtifactLayoutV1::V12 - | LexicalArtifactLayoutV1::V13 - | LexicalArtifactLayoutV1::V14 - | LexicalArtifactLayoutV1::V15 - | LexicalArtifactLayoutV1::V16 => { - lookup_term_ids(connection, terms).map_err(map_query_artifact_error)? - } - }; - let v11_ids = assigned_ids.values().copied().collect::>(); - let dynamic_binds = match layout { - LexicalArtifactLayoutV1::V10 => terms.len(), - LexicalArtifactLayoutV1::V11 - | LexicalArtifactLayoutV1::V12 - | LexicalArtifactLayoutV1::V13 - | LexicalArtifactLayoutV1::V14 - | LexicalArtifactLayoutV1::V15 - | LexicalArtifactLayoutV1::V16 => v11_ids.len(), - }; - ensure_sqlite_bind_capacity(documents.parameters.len(), dynamic_binds)?; - ensure_sqlite_bound_value_bytes( - documents.maximum_bound_value_bytes, - &documents.parameters, - terms.iter().map(String::as_str), - )?; - let mut parameters = - Vec::with_capacity(documents.parameters.len().saturating_add(dynamic_binds)); - let frequencies = match layout { - LexicalArtifactLayoutV1::V10 if terms.is_empty() => "'[]'".to_owned(), - LexicalArtifactLayoutV1::V11 - | LexicalArtifactLayoutV1::V12 - | LexicalArtifactLayoutV1::V13 - | LexicalArtifactLayoutV1::V14 - | LexicalArtifactLayoutV1::V15 - | LexicalArtifactLayoutV1::V16 - if v11_ids.is_empty() => - { - "'[]'".to_owned() - } - LexicalArtifactLayoutV1::V10 => { - let placeholders = std::iter::repeat_n("?", terms.len()) - .collect::>() - .join(", "); - parameters.extend(terms.iter().cloned().map(Value::Text)); - format!( - "COALESCE((SELECT json_group_array(json_array(posting.field, posting.term, posting.frequency)) \ - FROM term_postings AS posting INDEXED BY term_postings_by_document_term \ - WHERE posting.document_id = documents.document_id \ - AND posting.term IN ({placeholders})), '[]')" - ) - } - LexicalArtifactLayoutV1::V11 - | LexicalArtifactLayoutV1::V12 - | LexicalArtifactLayoutV1::V13 - | LexicalArtifactLayoutV1::V14 - | LexicalArtifactLayoutV1::V15 - | LexicalArtifactLayoutV1::V16 => { - let placeholders = std::iter::repeat_n("?", v11_ids.len()) - .collect::>() - .join(", "); - parameters.extend(v11_ids.iter().copied().map(Value::Integer)); - // Revision 13 clusters the table itself by document, so the - // primary key is the document-leading access path there. - let document_access = if layout.clusters_term_postings_by_document() { - "" - } else { - " INDEXED BY term_postings_by_document" - }; - format!( - "COALESCE((SELECT json_group_array(json_array(posting.field, vocabulary.term, posting.frequency)) \ - FROM term_postings AS posting{document_access} \ - JOIN vocabulary ON vocabulary.term_id = posting.term_id \ - WHERE posting.document_id = documents.document_id \ - AND posting.term_id IN ({placeholders})), '[]')" - ) - } - }; - // The frequency expression appears in the SELECT list before the - // document subquery appears in FROM, so its placeholders bind first. - // Keep the value vector in that exact textual order. - parameters.extend(documents.parameters.iter().cloned()); - let sql = format!( - "SELECT documents.document_id, stored.chunk_id, stored.row, {frequencies} \ - FROM ({document_sql}) AS documents \ - JOIN rows AS stored ON stored.document_id = documents.document_id \ - ORDER BY documents.document_id" - ); + let mut cursors = postings.cursors().map_err(map_query_artifact_error)?; metrics.probe(); - let mut statement = connection.prepare(&sql).map_err(map_query_sql_error)?; - let mut rows = statement - .query(params_from_iter(parameters.iter())) - .map_err(map_query_sql_error)?; let mut visited = 0u64; - while let Some(row) = rows.next().map_err(map_query_sql_error)? { + for document in documents { if visited.is_multiple_of(RETRIEVAL_CANDIDATE_BATCH_SIZE as u64) { retrieval_checkpoint(control)?; } - let document = u32::try_from(row.get::<_, i64>(0).map_err(map_query_sql_error)?) - .map_err(contract_error)?; - let chunk_id: String = row.get(1).map_err(map_query_sql_error)?; - let bytes: Vec = row.get(2).map_err(map_query_sql_error)?; - let encoded_frequencies: String = row.get(3).map_err(map_query_sql_error)?; + let stored = rows.row(document).map_err(map_query_artifact_error)?; let mut entries = Vec::new(); - match layout { - LexicalArtifactLayoutV1::V10 => { - let encoded: Vec<(String, String, i64)> = - serde_json::from_str(&encoded_frequencies).map_err(contract_error)?; - entries.reserve(encoded.len()); - for (field, term, frequency) in encoded { - entries.push(( - decode_field(&field)?, - term, - usize::try_from(frequency).map_err(contract_error)?, - )); - } - } - LexicalArtifactLayoutV1::V11 - | LexicalArtifactLayoutV1::V12 - | LexicalArtifactLayoutV1::V13 - | LexicalArtifactLayoutV1::V14 - | LexicalArtifactLayoutV1::V15 - | LexicalArtifactLayoutV1::V16 => { - let encoded: Vec<(i64, String, i64)> = - serde_json::from_str(&encoded_frequencies).map_err(contract_error)?; - entries.reserve(encoded.len()); - for (field, term, frequency) in encoded { - entries.push(( - field_from_code(field).map_err(map_query_artifact_error)?, - term, - usize::try_from(frequency).map_err(contract_error)?, - )); - } + for cursor in &mut cursors { + if let Some(frequency) = cursor + .frequency_at(document) + .map_err(map_query_artifact_error)? + { + entries.push(( + cursor.field, + cursor.term.to_owned(), + usize::try_from(frequency).map_err(contract_error)?, + )); } } - visitor(document, chunk_id, bytes, LexicalTermFrequenciesV1(entries))?; + visitor(document, stored, LexicalTermFrequenciesV1(entries))?; visited = visited.saturating_add(1); } - drop(rows); retrieval_checkpoint(control)?; - metrics.observe_statement(&statement)?; + if !documents.is_empty() { + let statement = connection + .prepare_cached(ROW_BLOCK_BY_DOCUMENT_SQL) + .map_err(map_query_sql_error)?; + metrics.observe_statement(&statement)?; + } metrics.rows(visited); Ok(()) }) } +/// Every sealed `(term, field)` posting list of one request's terms, read +/// once: scoring statistics, candidate sources, and per-row frequencies all +/// come from these lists. +#[derive(Default)] +struct RequestTermPostingsV1 { + by_term: BTreeMap>, +} + +struct TermFieldPostingsV1 { + field: LexicalFieldV1, + postings: Vec, +} + +impl RequestTermPostingsV1 { + fn cursors(&self) -> Result>, CodeLexicalArtifactErrorV1> { + self.by_term + .iter() + .flat_map(|(term, lists)| lists.iter().map(move |list| (term, list))) + .map(|(term, list)| PostingCursorV1::new(term, list)) + .collect() + } + + /// Documents holding `term` in any field `admit` accepts. + fn documents( + &self, + term: &str, + admit: impl Fn(LexicalFieldV1) -> bool, + ) -> Result { + let mut documents = RoaringBitmap::new(); + for list in self.by_term.get(term).into_iter().flatten() { + if !admit(list.field) { + continue; + } + for posting in PostingListDecoderV1::new(&list.postings, true) { + documents.insert(posting?.0); + } + } + Ok(documents) + } +} + +struct PostingCursorV1<'a> { + field: LexicalFieldV1, + term: &'a str, + decoder: PostingListDecoderV1<'a>, + current: Option<(u32, u32)>, +} + +impl<'a> PostingCursorV1<'a> { + fn new( + term: &'a str, + list: &'a TermFieldPostingsV1, + ) -> Result { + let mut decoder = PostingListDecoderV1::new(&list.postings, true); + let current = decoder.next().transpose()?; + Ok(Self { + field: list.field, + term, + decoder, + current, + }) + } + + /// Frequency of this list's term at `document`; callers ask in strictly + /// ascending document order. + fn frequency_at(&mut self, document: u32) -> Result, CodeLexicalArtifactErrorV1> { + while let Some((current, frequency)) = self.current { + if current >= document { + return Ok((current == document).then_some(frequency)); + } + self.current = self.decoder.next().transpose()?; + } + Ok(None) + } +} + const ARTIFACT_NGRAM_INTERSECTION_SCRATCH_V1: usize = 16; /// SQLite distributions are required to support at least 999 variables. Keep @@ -1695,35 +1400,27 @@ const ARTIFACT_SQLITE_MAX_BIND_PARAMETERS_V1: usize = 999; /// one SQLite call, so each individual call stays deterministically bounded. const ARTIFACT_SQLITE_MAX_BOUND_VALUE_BYTES_V1: usize = ARTIFACT_SQLITE_MAX_BIND_PARAMETERS_V1 * MAX_LEXICAL_QUERY_TERM_BYTES_V1; -/// A phrase prefilter may legitimately name more documents than request text -/// can occupy. Keep its one transient JSON1 bridge distinct from the generic -/// query-input bound and below one eighth of the reader cache authority. -const ARTIFACT_NGRAM_CANDIDATE_JSON_BYTES_V1: usize = - CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1 / 8; -/// Bitmap queries may inspect only this many source-page shards per port call. -/// The n-gram prefilter has no per-shard checkpoint, so this fixed work bound -/// is what keeps one prefilter finite before the row stream's per-row -/// cancellation checkpoints take over. A 4 KiB work unit leaves authority for -/// blob decode and intersection. -const ARTIFACT_NGRAM_QUERY_MAX_SHARDS_V1: usize = - CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1 / (4 * 1024); -/// One encoded source-page shard is retained only while it is decoded and +/// One encoded n-gram list is retained only while it is decoded and /// intersected. Keep that transient allocation below one eighth of the cache. -const ARTIFACT_NGRAM_MAX_ENCODED_SHARD_BYTES_V1: usize = +const ARTIFACT_NGRAM_MAX_ENCODED_LIST_BYTES_V1: usize = CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1 / 8; /// The synchronous query may inspect at most one quarter of the reader cache -/// in encoded shard bytes across all selected n-grams. +/// in encoded list bytes across all selected n-grams. const ARTIFACT_NGRAM_QUERY_ENCODED_BYTES_V1: usize = CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1 / 4; /// A sparse Roaring candidate can require containers and two-byte values in /// addition to its identifiers. Eight bytes per admitted identifier is a -/// conservative authority that bounds the first (rarest) full union. +/// conservative authority that bounds the first (rarest) full list. const ARTIFACT_NGRAM_CANDIDATE_BITMAP_BYTES_V1: usize = CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1 / 4; const ARTIFACT_NGRAM_CANDIDATE_BYTES_PER_DOCUMENT_V1: usize = 8; const ARTIFACT_NGRAM_MAX_CANDIDATES_V1: u64 = (ARTIFACT_NGRAM_CANDIDATE_BITMAP_BYTES_V1 / ARTIFACT_NGRAM_CANDIDATE_BYTES_PER_DOCUMENT_V1) as u64; +/// One request's term posting lists are held encoded for the whole row +/// stream; bound them by the same quarter of the reader cache. +const ARTIFACT_TERM_POSTING_QUERY_BYTES_V1: usize = + CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1 / 4; fn ensure_sqlite_bind_capacity( fixed_parameters: usize, dynamic_parameters: usize, @@ -1770,67 +1467,58 @@ fn ensure_sqlite_bound_value_bytes<'a>( /// substring check remains the correctness authority before scoring. fn ngram_document_query( connection: &Connection, - layout: LexicalArtifactLayoutV1, kind: i64, bytes: &[u8], metrics: &ArtifactQueryMetricsV1, -) -> Result { +) -> Result { hotpath::measure_block!("query.artifact.ngram.bitmap_query", { - let ngrams = query_ngrams(bytes).into_iter().collect::>(); + let ngrams = query_ngrams(bytes) + .into_iter() + .map(|ngram| (kind, ngram)) + .collect::>(); if ngrams.is_empty() { - return Ok(DocumentQueryV1::empty()); + return Ok(RoaringBitmap::new()); } - let candidates = ngram_bitmap_candidates(connection, layout, kind, &ngrams, metrics)?; - let encoded = - encode_ngram_candidate_json(&candidates, ARTIFACT_NGRAM_CANDIDATE_JSON_BYTES_V1)?; + let candidates = ngram_bitmap_candidates(connection, &ngrams, metrics)?; #[cfg(feature = "hotpath")] hotpath::gauge!("query.artifact.ngram.query_candidates_total").inc(candidates.len()); - Ok(DocumentQueryV1 { - sql: Some( - "SELECT CAST(value AS INTEGER) AS document_id FROM json_each(?) ORDER BY document_id" - .to_owned(), - ), - parameters: vec![Value::Text(encoded)], - maximum_bound_value_bytes: ARTIFACT_NGRAM_CANDIDATE_JSON_BYTES_V1, - }) + Ok(candidates) }) } #[derive(Clone, Copy)] struct NgramSelectivityV1 { + kind: i64, ngram: u32, cardinality: u64, } -/// One query's ngram shard budget: the shard and encoded-byte allowances every -/// intersection pass charges against, plus (under `hotpath`) the totals each -/// pass adds to so the query reports what it actually consumed. -struct NgramShardBudgetV1 { - remaining_shards: usize, +/// One query's n-gram budget: the encoded-byte allowance every intersection +/// charges against, plus (under `hotpath`) the totals it consumed. +struct NgramListBudgetV1 { remaining_encoded_bytes: usize, #[cfg(feature = "hotpath")] - observed_shards: u64, + observed_lists: u64, #[cfg(feature = "hotpath")] observed_bytes: u64, } -impl NgramShardBudgetV1 { +impl NgramListBudgetV1 { fn for_query() -> Self { Self { - remaining_shards: ARTIFACT_NGRAM_QUERY_MAX_SHARDS_V1, remaining_encoded_bytes: ARTIFACT_NGRAM_QUERY_ENCODED_BYTES_V1, #[cfg(feature = "hotpath")] - observed_shards: 0, + observed_lists: 0, #[cfg(feature = "hotpath")] observed_bytes: 0, } } #[inline(always)] - fn observe_shard(&mut self, encoded_bytes: usize) { + fn observe_list(&mut self, encoded_bytes: usize) { #[cfg(feature = "hotpath")] { - self.observed_shards = self.observed_shards.saturating_add(1); + self.observed_lists = self.observed_lists.saturating_add(1); self.observed_bytes = self.observed_bytes.saturating_add(encoded_bytes as u64); } #[cfg(not(feature = "hotpath"))] @@ -1841,7 +1529,7 @@ impl NgramShardBudgetV1 { fn report(&self) { #[cfg(feature = "hotpath")] { - hotpath::gauge!("query.artifact.ngram.query_shards_total").inc(self.observed_shards); + hotpath::gauge!("query.artifact.ngram.query_lists_total").inc(self.observed_lists); hotpath::gauge!("query.artifact.ngram.query_bytes_total").inc(self.observed_bytes); } } @@ -1849,21 +1537,19 @@ impl NgramShardBudgetV1 { fn ngram_bitmap_candidates( connection: &Connection, - layout: LexicalArtifactLayoutV1, - kind: i64, - ngrams: &[u32], + ngrams: &[(i64, u32)], _metrics: &ArtifactQueryMetricsV1, ) -> Result { - let mut budget = NgramShardBudgetV1::for_query(); + let mut budget = NgramListBudgetV1::for_query(); let mut selectivities = Vec::with_capacity(ngrams.len()); let mut selectivity_statement = connection .prepare_cached( - "SELECT document_frequency FROM ngram_statistics WHERE kind = ?1 AND ngram = ?2", + "SELECT document_frequency FROM ngram_postings WHERE kind = ?1 AND ngram = ?2", ) .map_err(map_query_sql_error)?; - for ngram in ngrams { + for &(kind, ngram) in ngrams { let cardinality = selectivity_statement - .query_row([kind, i64::from(*ngram)], |row| row.get::<_, i64>(0)) + .query_row([kind, i64::from(ngram)], |row| row.get::<_, i64>(0)) .optional() .map_err(map_query_sql_error)?; let Some(cardinality) = cardinality else { @@ -1872,144 +1558,58 @@ fn ngram_bitmap_candidates( let cardinality = u64::try_from(cardinality).map_err(contract_error)?; ensure_ngram_candidate_cardinality(cardinality)?; selectivities.push(NgramSelectivityV1 { - ngram: *ngram, + kind, + ngram, cardinality, }); } drop(selectivity_statement); - selectivities.sort_unstable_by_key(|selectivity| (selectivity.cardinality, selectivity.ngram)); + selectivities.sort_unstable_by_key(|selectivity| { + (selectivity.cardinality, selectivity.kind, selectivity.ngram) + }); selectivities.truncate(ARTIFACT_NGRAM_INTERSECTION_SCRATCH_V1); - if let Some(selectivity) = selectivities.first() { - ensure_ngram_candidate_cardinality(selectivity.cardinality)?; - } - let mut candidates = None::>; - let mut all_pages_statement = connection - .prepare_cached( - "SELECT page_ordinal, documents, cardinality FROM ngram_postings INDEXED BY ngram_postings_by_ngram WHERE kind = ?1 AND ngram = ?2 ORDER BY page_ordinal", - ) - .map_err(map_query_sql_error)?; - let mut candidate_pages_statement = connection + let mut candidates = None::; + let mut list_statement = connection .prepare_cached( - "SELECT posting.page_ordinal, posting.documents, posting.cardinality \ - FROM json_each(?3) AS candidate_page \ - CROSS JOIN ngram_postings AS posting INDEXED BY ngram_postings_by_ngram \ - WHERE posting.kind = ?1 \ - AND posting.ngram = ?2 \ - AND posting.page_ordinal = CAST(candidate_page.value AS INTEGER)", + "SELECT documents, document_frequency FROM ngram_postings WHERE kind = ?1 AND ngram = ?2", ) .map_err(map_query_sql_error)?; for selectivity in selectivities { - let next = if let Some(current) = candidates.as_ref() { - let candidate_pages = encode_ngram_candidate_pages(current)?; - let mut rows = candidate_pages_statement - .query((kind, i64::from(selectivity.ngram), candidate_pages)) - .map_err(map_query_sql_error)?; - let next = - intersect_ngram_shards(&mut rows, Some(current), layout, &mut budget, _metrics)?; - drop(rows); - _metrics.observe_statement(&candidate_pages_statement)?; - next - } else { - let mut rows = all_pages_statement - .query([kind, i64::from(selectivity.ngram)]) - .map_err(map_query_sql_error)?; - let next = intersect_ngram_shards(&mut rows, None, layout, &mut budget, _metrics)?; - drop(rows); - _metrics.observe_statement(&all_pages_statement)?; - next - }; - candidates = Some(next); - if candidates.as_ref().is_none_or(BTreeMap::is_empty) { - break; - } - } - let candidates = candidates.unwrap_or_default().into_values().fold( - RoaringBitmap::new(), - |mut all, shard| { - all |= shard; - all - }, - ); - budget.report(); - Ok(candidates) -} - -fn intersect_ngram_shards( - rows: &mut rusqlite::Rows<'_>, - current: Option<&BTreeMap>, - layout: LexicalArtifactLayoutV1, - budget: &mut NgramShardBudgetV1, - _metrics: &ArtifactQueryMetricsV1, -) -> Result, RetrievalPortError> { - let mut next = BTreeMap::new(); - let mut candidate_count = 0u64; - while let Some(row) = rows.next().map_err(map_query_sql_error)? { - if budget.remaining_shards == 0 { - return Err(RetrievalPortError::BudgetExceeded); - } - let page_ordinal: i64 = row.get(0).map_err(map_query_sql_error)?; - let encoded: Vec = row.get(1).map_err(map_query_sql_error)?; - let cardinality: i64 = row.get(2).map_err(map_query_sql_error)?; - charge_ngram_encoded_shard_bytes( + let (encoded, cardinality): (Vec, i64) = list_statement + .query_row([selectivity.kind, i64::from(selectivity.ngram)], |row| { + Ok((row.get(0)?, row.get(1)?)) + }) + .map_err(map_query_sql_error)?; + charge_ngram_encoded_list_bytes( &mut budget.remaining_encoded_bytes, encoded.len(), - ARTIFACT_NGRAM_MAX_ENCODED_SHARD_BYTES_V1, + ARTIFACT_NGRAM_MAX_ENCODED_LIST_BYTES_V1, )?; - budget.remaining_shards = budget - .remaining_shards - .checked_sub(1) - .ok_or(RetrievalPortError::BudgetExceeded)?; - let mut shard = decode_ngram_bitmap(layout, &encoded).map_err(map_query_artifact_error)?; - if i64::try_from(shard.len()).map_err(contract_error)? != cardinality { + let mut list = decode_document_set(&encoded).map_err(map_query_artifact_error)?; + if i64::try_from(list.len()).map_err(contract_error)? != cardinality { return Err(RetrievalPortError::Contract( - "lexical artifact ngram shard cardinality changed after verification".to_owned(), + "lexical artifact ngram list cardinality changed after verification".to_owned(), )); } - if let Some(current) = current { - let prior = current.get(&page_ordinal).ok_or_else(|| { - RetrievalPortError::Contract( - "lexical artifact returned an ngram shard outside the candidate pages" - .to_owned(), - ) - })?; - shard &= prior; - } - if !shard.is_empty() { - candidate_count = candidate_count - .checked_add(shard.len()) - .ok_or(RetrievalPortError::BudgetExceeded)?; - ensure_ngram_candidate_cardinality(candidate_count)?; - #[cfg(test)] - _metrics.observe_ngram_candidates(candidate_count); - next.insert(page_ordinal, shard); + if let Some(current) = candidates.as_ref() { + list &= current; } + ensure_ngram_candidate_cardinality(list.len())?; #[cfg(test)] - _metrics.observe_ngram_shard(); - budget.observe_shard(encoded.len()); - } - Ok(next) -} - -fn encode_ngram_candidate_pages( - candidates: &BTreeMap, -) -> Result { - let mut encoded = String::with_capacity(candidates.len().saturating_mul(8)); - encoded.push('['); - for (ordinal, page_ordinal) in candidates.keys().enumerate() { - if ordinal > 0 { - encoded.push(','); + { + _metrics.observe_ngram_list(); + _metrics.observe_ngram_candidates(list.len()); } - write!(&mut encoded, "{page_ordinal}").map_err(|_| RetrievalPortError::BudgetExceeded)?; - if encoded.len() > ARTIFACT_NGRAM_CANDIDATE_JSON_BYTES_V1 { - return Err(RetrievalPortError::BudgetExceeded); + budget.observe_list(encoded.len()); + let exhausted = list.is_empty(); + candidates = Some(list); + if exhausted { + break; } } - encoded.push(']'); - if encoded.len() > ARTIFACT_NGRAM_CANDIDATE_JSON_BYTES_V1 { - return Err(RetrievalPortError::BudgetExceeded); - } - Ok(encoded) + budget.report(); + Ok(candidates.unwrap_or_default()) } fn ensure_ngram_candidate_cardinality(cardinality: u64) -> Result<(), RetrievalPortError> { @@ -2020,76 +1620,47 @@ fn ensure_ngram_candidate_cardinality(cardinality: u64) -> Result<(), RetrievalP } } -fn charge_ngram_encoded_shard_bytes( +fn charge_ngram_encoded_list_bytes( remaining_bytes: &mut usize, - shard_bytes: usize, - maximum_shard_bytes: usize, + list_bytes: usize, + maximum_list_bytes: usize, ) -> Result<(), RetrievalPortError> { - if shard_bytes > maximum_shard_bytes { + if list_bytes > maximum_list_bytes { return Err(RetrievalPortError::BudgetExceeded); } *remaining_bytes = remaining_bytes - .checked_sub(shard_bytes) + .checked_sub(list_bytes) .ok_or(RetrievalPortError::BudgetExceeded)?; Ok(()) } -fn encode_ngram_candidate_json( - candidates: &RoaringBitmap, - maximum_bytes: usize, -) -> Result { - if maximum_bytes < 2 { - return Err(RetrievalPortError::BudgetExceeded); - } - let capacity = usize::try_from(candidates.len()) - .map_err(contract_error)? - .checked_mul(11) - .and_then(|bytes| bytes.checked_add(2)) - .ok_or(RetrievalPortError::BudgetExceeded)? - .min(maximum_bytes); - let mut encoded = String::with_capacity(capacity); - encoded.push('['); - for (ordinal, document) in candidates.iter().enumerate() { - let digits = if document == 0 { - 1 - } else { - usize::try_from(document.ilog10()).map_err(contract_error)? + 1 - }; - let additional = digits + usize::from(ordinal != 0); - if encoded - .len() - .checked_add(additional) - .and_then(|bytes| bytes.checked_add(1)) - .is_none_or(|bytes| bytes > maximum_bytes) - { - return Err(RetrievalPortError::BudgetExceeded); - } - if ordinal != 0 { - encoded.push(','); - } - write!(&mut encoded, "{document}").map_err(contract_error)?; - } - encoded.push(']'); - Ok(encoded) -} - impl<'a> ArtifactQueryV1<'a> { fn new( connection: &'a Connection, metadata: &'a super::super::CodeLexicalProjectionMetadataV1, receipt: &'a VerifiedCodeLexicalArtifactV1, - layout: LexicalArtifactLayoutV1, fuzzy_vocabulary: &'a OnceLock>>, ) -> Result { Ok(Self { connection, metadata, - receipt, - layout, - document_count: usize::try_from(receipt.total_chunks()).map_err(contract_error)?, + // Admitted documents, which the sealed `rows` section counts; + // source chunks the projection does not index are not documents. + document_count: receipt + .section_digests() + .iter() + .find(|section| section.name == "rows") + .map(|section| usize::try_from(section.row_count).map_err(contract_error)) + .transpose()? + .ok_or_else(|| { + RetrievalPortError::Contract( + "lexical artifact receipt has no rows section".to_owned(), + ) + })?, metrics: ArtifactQueryMetricsV1::default(), fuzzy_vocabulary, row_dictionary: ConnectionRowDictionaryV1::new(connection), + row_blocks: RowBlocksV1::new(connection), }) } @@ -2107,7 +1678,6 @@ impl<'a> ArtifactQueryV1<'a> { for (_, normalized) in &prepared.phrases { let query = ngram_document_query( self.connection, - self.layout, NGRAM_NORMALIZED, normalized.as_bytes(), &self.metrics, @@ -2119,18 +1689,18 @@ impl<'a> ArtifactQueryV1<'a> { .cloned() .map(|phrase| (phrase, 0usize)) .collect::>(); - let phrase_documents = union_document_queries(phrase_queries.values().cloned())?; + let phrase_documents = phrase_queries + .values() + .fold(RoaringBitmap::new(), |union, documents| union | documents); visit_lexical_rows( self.connection, + &self.row_blocks, &phrase_documents, - &BTreeSet::new(), + &RequestTermPostingsV1::default(), &self.metrics, - self.layout, control, - |_, chunk_id, bytes, _| { - let row = self - .decode_row(&chunk_id, &bytes) - .map_err(map_query_artifact_error)?; + |_, stored, _| { + let row = self.decode_row(&stored).map_err(map_query_artifact_error)?; for (phrase, frequency) in &mut phrase_frequencies { if matches_phrase(&row, phrase) { *frequency += 1; @@ -2151,15 +1721,13 @@ impl<'a> ArtifactQueryV1<'a> { let mut ranked = BinaryHeap::new(); visit_lexical_rows( self.connection, + &self.row_blocks, &documents, - &terms, + &stats.postings, &self.metrics, - self.layout, control, - |document, chunk_id, bytes, frequencies| { - let row = self - .decode_row(&chunk_id, &bytes) - .map_err(map_query_artifact_error)?; + |document, stored, frequencies| { + let row = self.decode_row(&stored).map_err(map_query_artifact_error)?; let score = self.score_row( &row, &prepared, @@ -2198,8 +1766,8 @@ impl<'a> ArtifactQueryV1<'a> { } = entry; let mut candidate = lexical_lane_candidate( &row, - self.receipt.freshness(), - self.receipt.repository_id().cloned(), + &self.metadata.freshness, + self.metadata.repository_id.clone(), RetrieverKind::Lexical, self.metadata.lexical_retriever_revision.clone(), request.score_domain.clone(), @@ -2285,6 +1853,13 @@ impl<'a> ArtifactQueryV1<'a> { retrieval_checkpoint(request.control)?; let selected = ranked.into_sorted_vec(); let truncated = eligible - selected.len() as u64; + // Winners re-read in document order so each row block inflates once. + let mut winner_documents = selected.iter().map(|entry| entry.key.2).collect::>(); + winner_documents.sort_unstable(); + let mut winner_rows = BTreeMap::new(); + for document in winner_documents { + winner_rows.insert(document, self.row(document)?); + } let mut candidates = Vec::with_capacity(selected.len()); let mut evidence_by_occurrence = BTreeMap::new(); for (ordinal, entry) in selected.into_iter().enumerate() { @@ -2300,11 +1875,13 @@ impl<'a> ArtifactQueryV1<'a> { .iter() .map(|literal| request.literals[*literal].clone()) .collect::>(); - let row = self.row(document)?; + let row = winner_rows.remove(&document).ok_or_else(|| { + RetrievalPortError::Contract("exact lane winner row was not read".to_owned()) + })?; let mut candidate = lexical_lane_candidate( &row, - self.receipt.freshness(), - self.receipt.repository_id().cloned(), + &self.metadata.freshness, + self.metadata.repository_id.clone(), RetrieverKind::ExactLiteral, self.metadata.exact_retriever_revision.clone(), self.metadata.exact_score_domain.clone(), @@ -2332,27 +1909,22 @@ impl<'a> ArtifactQueryV1<'a> { fn row(&self, document: u32) -> Result { self.metrics.probe(); - let mut statement = self - .connection - .prepare_cached("SELECT chunk_id, row FROM rows WHERE document_id = ?1") - .map_err(map_query_sql_error)?; - let (chunk_id, bytes): (String, Vec) = statement - .query_row([i64::from(document)], |row| Ok((row.get(0)?, row.get(1)?))) - .map_err(map_query_sql_error)?; - self.decode_row(&chunk_id, &bytes) - .map_err(map_query_artifact_error) + let stored = self + .row_blocks + .row(document) + .map_err(map_query_artifact_error)?; + self.decode_row(&stored).map_err(map_query_artifact_error) } fn decode_row( &self, - chunk_id: &str, - bytes: &[u8], + stored: &StoredRowV1, ) -> Result { decode_artifact_row( - self.layout, - self.receipt.generation(), - chunk_id, - bytes, + &self.metadata.generation, + &stored.chunk_id, + &stored.row, + &stored.text, &self.row_dictionary, ) } @@ -2365,9 +1937,9 @@ impl<'a> ArtifactQueryV1<'a> { request: &LexicalLaneRequest<'_>, fuzzy: &FuzzyExpansionsV1, stats: &LexicalStatsCacheV1, - phrase_queries: &BTreeMap, + phrase_queries: &BTreeMap, pruned: &mut Vec<(String, u64)>, - ) -> Result { + ) -> Result { let mut whole_terms = Vec::new(); for term in request.whole_terms.iter() { whole_terms.push(normalize_lexical(term)); @@ -2388,79 +1960,46 @@ impl<'a> ArtifactQueryV1<'a> { .map(|subtoken| normalize_lexical(subtoken)) .collect::>(); let mut sources = Vec::with_capacity(whole_terms.len() + subtokens.len()); - match self.layout { - LexicalArtifactLayoutV1::V10 => { - let subtoken_field = - encode_field(LexicalFieldV1::Subtoken).map_err(map_query_artifact_error)?; - for term in whole_terms { - let frequency = stats.whole_term_documents(&term); - sources.push(( - frequency, - ( - term.clone(), - DocumentQueryV1::term_except(term, subtoken_field.clone()), - ), - )); - } - for subtoken in subtokens { - let frequency = stats.document_frequency(LexicalFieldV1::Subtoken, &subtoken); - sources.push(( - frequency, - ( - subtoken.clone(), - DocumentQueryV1::term(subtoken_field.clone(), subtoken), - ), - )); - } + for term in whole_terms { + if stats.postings.by_term.contains_key(&term) { + sources.push((stats.whole_term_documents(&term), (term, false))); } - LexicalArtifactLayoutV1::V11 - | LexicalArtifactLayoutV1::V12 - | LexicalArtifactLayoutV1::V13 - | LexicalArtifactLayoutV1::V14 - | LexicalArtifactLayoutV1::V15 - | LexicalArtifactLayoutV1::V16 => { - let subtoken_field = field_code(LexicalFieldV1::Subtoken); - for term in whole_terms { - if let Some(term_id) = - lookup_term_id(self.connection, &term).map_err(map_query_artifact_error)? - { - sources.push(( - stats.whole_term_documents(&term), - ( - term, - DocumentQueryV1::term_except_id(term_id, subtoken_field), - ), - )); - } - } - for subtoken in subtokens { - if let Some(term_id) = lookup_term_id(self.connection, &subtoken) - .map_err(map_query_artifact_error)? - { - sources.push(( - stats.document_frequency(LexicalFieldV1::Subtoken, &subtoken), - (subtoken, DocumentQueryV1::term_id(subtoken_field, term_id)), - )); - } - } + } + for subtoken in subtokens { + if stats.postings.by_term.contains_key(&subtoken) { + sources.push(( + stats.document_frequency(LexicalFieldV1::Subtoken, &subtoken), + (subtoken, true), + )); } } - let mut admitted = phrase_queries.values().cloned().collect::>(); - admitted.extend( - admit_candidate_sources(sources, |frequency, (term, _)| { - pruned.push((term.clone(), frequency as u64)); - }) - .into_iter() - .map(|(_, source)| source), - ); - union_document_queries(admitted) + let mut documents = phrase_queries + .values() + .fold(RoaringBitmap::new(), |union, documents| union | documents); + for (term, subtoken) in admit_candidate_sources(sources, |frequency, (term, _)| { + pruned.push((term.clone(), frequency as u64)); + }) { + documents |= stats + .postings + .documents(&term, |field| { + (field == LexicalFieldV1::Subtoken) == subtoken + }) + .map_err(map_query_artifact_error)?; + } + Ok(documents) } fn exact_documents( &self, request: &ExactLaneRequest, - ) -> Result { - let mut sources = Vec::new(); + ) -> Result { + let mut documents = RoaringBitmap::new(); + let mut statement = self + .connection + .prepare_cached( + "SELECT documents FROM exact_postings WHERE term_id = ?1 AND field = ?2", + ) + .map_err(map_query_sql_error)?; for literal in &request.literals { if matches!( literal.field, @@ -2468,53 +2007,42 @@ impl<'a> ArtifactQueryV1<'a> { | ExactFieldV1::DiagnosticText | ExactFieldV1::CompilerOrRuntimeError ) { - sources.push(ngram_document_query( + documents |= ngram_document_query( self.connection, - self.layout, NGRAM_NORMALIZED, &literal.original_bytes, &self.metrics, - )?); - sources.push(ngram_document_query( - self.connection, - self.layout, - NGRAM_RAW_OVERRIDE, - &literal.original_bytes, - &self.metrics, - )?); - } - match self.layout { - LexicalArtifactLayoutV1::V10 | LexicalArtifactLayoutV1::V11 => { - let field = - encode_exact_field(literal.field).map_err(map_query_artifact_error)?; - sources.push(DocumentQueryV1::exact( - field, - literal.canonical_bytes.clone(), - )); - } - LexicalArtifactLayoutV1::V12 - | LexicalArtifactLayoutV1::V13 - | LexicalArtifactLayoutV1::V14 - | LexicalArtifactLayoutV1::V15 - | LexicalArtifactLayoutV1::V16 => { - sources.push(DocumentQueryV1::exact_id( - literal.field, - &literal.canonical_bytes, - )); + )?; + if let Some(ngrams) = raw_override_query_ngrams(&literal.original_bytes) { + documents |= ngram_bitmap_candidates(self.connection, &ngrams, &self.metrics)?; } } + self.metrics.probe(); + let encoded: Option> = statement + .query_row( + [ + stable_exact_term_id(&literal.canonical_bytes), + exact_field_code(literal.field), + ], + |row| row.get(0), + ) + .optional() + .map_err(map_query_sql_error)?; + if let Some(encoded) = encoded { + documents |= decode_ngram_bitmap(&encoded).map_err(map_query_artifact_error)?; + } } - union_document_queries(sources) + Ok(documents) } fn visit_documents( &self, - query: &DocumentQueryV1, + documents: &RoaringBitmap, control: &dyn RetrievalExecutionControl, visitor: impl FnMut(u32) -> Result<(), RetrievalPortError>, ) -> Result<(), RetrievalPortError> { self.metrics.probe(); - visit_document_ids(self.connection, query, control, visitor) + visit_document_ids(documents, control, visitor) } #[hotpath::measure(label = "query.lane.fuzzy.expand")] @@ -2604,26 +2132,15 @@ impl<'a> ArtifactQueryV1<'a> { Ok(Arc::clone(self.fuzzy_vocabulary.get_or_init(|| loaded))) } - /// Edit-distance expansion does not need term order. `ORDER BY term` - /// against hash-keyed `term_id` rows forces the UNIQUE(term) index plus - /// one random primary-key lookup per vocabulary row. - fn vocabulary_sql(layout: LexicalArtifactLayoutV1) -> &'static str { - match layout { - LexicalArtifactLayoutV1::V10 => "SELECT term FROM vocabulary", - LexicalArtifactLayoutV1::V11 - | LexicalArtifactLayoutV1::V12 - | LexicalArtifactLayoutV1::V13 - | LexicalArtifactLayoutV1::V14 - | LexicalArtifactLayoutV1::V15 - | LexicalArtifactLayoutV1::V16 => "SELECT term FROM vocabulary WHERE in_fuzzy = 1", - } - } + /// One pass over the term-keyed table; its lists sit after both columns + /// read here, so the walk never follows a list's overflow pages. + const VOCABULARY_SQL: &'static str = "SELECT term FROM term_postings WHERE in_fuzzy = 1"; fn load_vocabulary_from_sqlite(&self) -> Result>, RetrievalPortError> { self.metrics.probe(); let mut statement = self .connection - .prepare_cached(Self::vocabulary_sql(self.layout)) + .prepare_cached(Self::VOCABULARY_SQL) .map_err(map_query_sql_error)?; let mut rows = statement.query([]).map_err(map_query_sql_error)?; let mut vocabulary = Vec::new(); @@ -2663,20 +2180,8 @@ impl<'a> ArtifactQueryV1<'a> { .map_err(map_query_sql_error)?; let mut rows = statement.query([]).map_err(map_query_sql_error)?; while let Some(row) = rows.next().map_err(map_query_sql_error)? { - let field = match self.layout { - LexicalArtifactLayoutV1::V10 => { - decode_field(&row.get::<_, String>(0).map_err(map_query_sql_error)?)? - } - LexicalArtifactLayoutV1::V11 - | LexicalArtifactLayoutV1::V12 - | LexicalArtifactLayoutV1::V13 - | LexicalArtifactLayoutV1::V14 - | LexicalArtifactLayoutV1::V15 - | LexicalArtifactLayoutV1::V16 => { - field_from_code(row.get::<_, i64>(0).map_err(map_query_sql_error)?) - .map_err(map_query_artifact_error)? - } - }; + let field = field_from_code(row.get::<_, i64>(0).map_err(map_query_sql_error)?) + .map_err(map_query_artifact_error)?; let total: i64 = row.get(1).map_err(map_query_sql_error)?; field_totals.insert(field, usize::try_from(total).map_err(contract_error)?); } @@ -2685,92 +2190,61 @@ impl<'a> ArtifactQueryV1<'a> { self.metrics .rows(u64::try_from(field_totals.len()).map_err(contract_error)?); let mut document_frequencies = BTreeMap::>::new(); + let mut postings = RequestTermPostingsV1::default(); if !terms.is_empty() { - match self.layout { - LexicalArtifactLayoutV1::V10 => { - let placeholders = std::iter::repeat_n("?", terms.len()) - .collect::>() - .join(", "); - let query = format!( - "SELECT field, term, document_frequency FROM term_stats INDEXED BY term_stats_by_term WHERE term IN ({placeholders})" + for term in terms { + postings.by_term.insert(term.clone(), Vec::new()); + } + let placeholders = std::iter::repeat_n("?", terms.len()) + .collect::>() + .join(", "); + let query = + format!("SELECT term, lists FROM term_postings WHERE term IN ({placeholders})"); + self.metrics.probe(); + let mut statement = self + .connection + .prepare(&query) + .map_err(map_query_sql_error)?; + let mut rows = statement + .query(params_from_iter(terms.iter())) + .map_err(map_query_sql_error)?; + let mut observed_rows = 0u64; + let mut remaining_bytes = ARTIFACT_TERM_POSTING_QUERY_BYTES_V1; + while let Some(row) = rows.next().map_err(map_query_sql_error)? { + let term: String = row.get(0).map_err(map_query_sql_error)?; + let encoded = row + .get_ref(1) + .and_then(|value| value.as_blob().map_err(rusqlite::Error::from)) + .map_err(map_query_sql_error)?; + remaining_bytes = remaining_bytes + .checked_sub(encoded.len()) + .ok_or(RetrievalPortError::BudgetExceeded)?; + let Some(lists) = postings.by_term.get_mut(&term) else { + continue; + }; + for (field, document_frequency, list) in + decode_term_lists(encoded).map_err(map_query_artifact_error)? + { + let field = field_from_code(field).map_err(map_query_artifact_error)?; + document_frequencies.entry(field).or_default().insert( + term.clone(), + usize::try_from(document_frequency).map_err(contract_error)?, ); - self.metrics.probe(); - let mut statement = self - .connection - .prepare(&query) - .map_err(map_query_sql_error)?; - let mut rows = statement - .query(params_from_iter(terms.iter())) - .map_err(map_query_sql_error)?; - let mut observed_rows = 0u64; - while let Some(row) = rows.next().map_err(map_query_sql_error)? { - let field: String = row.get(0).map_err(map_query_sql_error)?; - let term: String = row.get(1).map_err(map_query_sql_error)?; - let frequency: i64 = row.get(2).map_err(map_query_sql_error)?; - document_frequencies - .entry(decode_field(&field)?) - .or_default() - .insert(term, usize::try_from(frequency).map_err(contract_error)?); - observed_rows = observed_rows.saturating_add(1); - } - drop(rows); - self.metrics.observe_statement(&statement)?; - self.metrics.rows(observed_rows); - } - LexicalArtifactLayoutV1::V11 - | LexicalArtifactLayoutV1::V12 - | LexicalArtifactLayoutV1::V13 - | LexicalArtifactLayoutV1::V14 - | LexicalArtifactLayoutV1::V15 - | LexicalArtifactLayoutV1::V16 => { - let assigned = lookup_term_ids(self.connection, terms) - .map_err(map_query_artifact_error)?; - let term_ids = assigned.values().copied().collect::>(); - if !term_ids.is_empty() { - let placeholders = std::iter::repeat_n("?", term_ids.len()) - .collect::>() - .join(", "); - let query = format!( - "SELECT field, term_id, document_frequency FROM term_stats WHERE term_id IN ({placeholders})" - ); - self.metrics.probe(); - let mut statement = self - .connection - .prepare(&query) - .map_err(map_query_sql_error)?; - let mut rows = statement - .query(params_from_iter(term_ids.iter())) - .map_err(map_query_sql_error)?; - let mut observed_rows = 0u64; - let id_to_term = assigned - .iter() - .map(|(term, id)| (*id, term.clone())) - .collect::>(); - while let Some(row) = rows.next().map_err(map_query_sql_error)? { - let field = - field_from_code(row.get::<_, i64>(0).map_err(map_query_sql_error)?) - .map_err(map_query_artifact_error)?; - let term_id: i64 = row.get(1).map_err(map_query_sql_error)?; - let Some(term) = id_to_term.get(&term_id) else { - continue; - }; - let frequency: i64 = row.get(2).map_err(map_query_sql_error)?; - document_frequencies.entry(field).or_default().insert( - term.clone(), - usize::try_from(frequency).map_err(contract_error)?, - ); - observed_rows = observed_rows.saturating_add(1); - } - drop(rows); - self.metrics.observe_statement(&statement)?; - self.metrics.rows(observed_rows); - } + lists.push(TermFieldPostingsV1 { + field, + postings: list.to_vec(), + }); + observed_rows = observed_rows.saturating_add(1); } } + drop(rows); + self.metrics.observe_statement(&statement)?; + self.metrics.rows(observed_rows); } Ok(LexicalStatsCacheV1 { field_totals, document_frequencies, + postings, }) } @@ -2831,6 +2305,7 @@ struct LexicalTermFrequenciesV1(Vec<(LexicalFieldV1, String, usize)>); struct LexicalStatsCacheV1 { field_totals: BTreeMap, document_frequencies: BTreeMap>, + postings: RequestTermPostingsV1, } fn lexical_terms( @@ -3055,10 +2530,6 @@ impl EditDistanceScratchV1 { } } -fn decode_field(encoded: &str) -> Result { - serde_json::from_str(encoded).map_err(contract_error) -} - fn validate_cache_budget(cache_budget_bytes: usize) -> Result<(), CodeLexicalArtifactErrorV1> { if cache_budget_bytes == 0 || cache_budget_bytes > CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1 @@ -3212,7 +2683,7 @@ fn verify_named_path_identity(path: &Path, file: &File) -> Result<(), CodeLexica fn verify_artifact_state_revision( connection: &Connection, control: &dyn CodeIndexExecutionControlV1, -) -> Result { +) -> Result<(), CodeLexicalArtifactErrorV1> { checkpoint(control)?; let revision: i64 = connection .query_row( @@ -3230,9 +2701,8 @@ fn verify_artifact_state_revision( "artifact state format revision is outside the supported range".to_owned(), ) })?; - let layout = LexicalArtifactLayoutV1::from_revision(revision)?; - checkpoint(control)?; - Ok(layout) + require_served_revision(revision)?; + checkpoint(control) } fn sealed_reader_mmap_bytes(file_size_bytes: u64) -> Result { @@ -3361,21 +2831,27 @@ mod tests { use rusqlite::hooks::{AuthAction, Authorization}; use rusqlite::{Connection, OpenFlags, params}; use sha2::{Digest, Sha256}; - use tracedecay_domain::ManifestDigest; + use tracedecay_domain::{ + CodeGenerationId, ComponentRevision, FreshnessCompatibilityV1, ManifestDigest, + ScoreDomainId, SourceFreshness, SourceInstanceKey, SourceNamespace, UtcMicros, + }; use tracedecay_private_fs::open_private_file; - use super::super::format::encode_ngram_bitmap; + use super::super::format::{PostingListEncoderV1, encode_document_set}; + use super::super::row_codec::{BlockRowV1, RowBlocksV1, encode_row_blocks}; use super::{ ARTIFACT_NGRAM_INTERSECTION_SCRATCH_V1, ARTIFACT_NGRAM_MAX_CANDIDATES_V1, ARTIFACT_SQLITE_CACHE_BYTES, ARTIFACT_SQLITE_MAX_BIND_PARAMETERS_V1, ARTIFACT_SQLITE_MAX_BOUND_VALUE_BYTES_V1, ArtifactQueryMetricsV1, CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, CodeLexicalArtifactErrorV1, - CodeLexicalArtifactReaderV1, DocumentQueryV1, LexicalArtifactLayoutV1, NGRAM_NORMALIZED, - charge_ngram_encoded_shard_bytes, configure_reader_window, encode_ngram_candidate_json, - ensure_ngram_candidate_cardinality, map_query_artifact_error, ngram_bitmap_candidates, - ngram_document_query, query_ngrams, retain_bounded, term_frequency, union_document_queries, - visit_document_ids, visit_lexical_rows, + CodeLexicalArtifactReaderV1, LexicalFieldV1, NGRAM_NORMALIZED, RequestTermPostingsV1, + TermFieldPostingsV1, charge_ngram_encoded_list_bytes, configure_reader_window, + ensure_ngram_candidate_cardinality, ensure_sqlite_bind_capacity, + ensure_sqlite_bound_value_bytes, map_query_artifact_error, ngram_bitmap_candidates, + ngram_document_query, query_ngrams, retain_bounded, term_frequency, visit_document_ids, + visit_lexical_rows, }; + use crate::retrieval::lexical::CodeLexicalProjectionMetadataV1; use crate::retrieval::ports::RetrievalExecutionControl; use crate::retrieval::ports::RetrievalPortError; use tracedecay_code_index::production::CodeIndexExecutionControlV1; @@ -3402,6 +2878,32 @@ mod tests { } } + /// An opener projection for reads refused before its content is compared. + fn opener() -> CodeLexicalProjectionMetadataV1 { + CodeLexicalProjectionMetadataV1 { + generation: CodeGenerationId::new("generation.reader.v1").expect("generation"), + repository_id: None, + logical_paths: Default::default(), + freshness: SourceFreshness { + source_namespace: SourceNamespace::new("namespace.reader").expect("namespace"), + source_instance: SourceInstanceKey::new("instance.reader").expect("instance"), + source_watermark: None, + projection_watermark: None, + observed_at: UtcMicros(0), + source_generation: None, + generation_lag: None, + compatibility: FreshnessCompatibilityV1::Unknown, + policy_revision: ComponentRevision::new("policy.reader.v1").expect("policy"), + }, + exact_retriever_revision: ComponentRevision::new("retriever.exact.reader.v1") + .expect("exact retriever"), + lexical_retriever_revision: ComponentRevision::new("retriever.lexical.reader.v1") + .expect("lexical retriever"), + exact_score_domain: ScoreDomainId::new("score.exact.reader.v1").expect("score domain"), + clone_route: None, + } + } + /// A request authority that reports cancellation from its `cancel_at`-th /// consultation onwards, counting every consultation it receives. struct CancelAtObservation { @@ -3570,16 +3072,6 @@ mod tests { } } - fn streamed_documents(connection: &Connection, query: &DocumentQueryV1) -> Vec { - let mut documents = Vec::new(); - visit_document_ids(connection, query, &AlwaysActiveControl, |document| { - documents.push(document); - Ok(()) - }) - .expect("SQLite stream succeeds"); - documents - } - #[test] fn invalid_content_addressed_budget_is_rejected_before_path_touch() { let missing = std::env::temp_dir().join(format!( @@ -3593,6 +3085,7 @@ mod tests { &missing, &digest, 0, + &opener(), 0, &AlwaysActiveControl, ) @@ -3706,6 +3199,7 @@ mod tests { &artifact_path, &digest, size, + &opener(), 1024 * 1024, &AlwaysActiveControl, ) @@ -3781,6 +3275,7 @@ mod tests { &artifact_path, &foreign, size, + &opener(), 1024 * 1024, &AlwaysActiveControl, ) @@ -3820,6 +3315,7 @@ mod tests { &artifact_path, &digest, size, + &opener(), 1024 * 1024, &AlwaysActiveControl, ) @@ -3848,167 +3344,111 @@ mod tests { ); } - #[test] - fn phrase_ngram_stream_selects_the_rarest_fixed_predicates_from_the_whole_phrase() { + fn ngram_fixture(lists: &[(u32, &[u32])]) -> Connection { let connection = Connection::open_in_memory().expect("in-memory SQLite"); connection .execute_batch( "CREATE TABLE ngram_postings ( - page_ordinal INTEGER NOT NULL, - kind INTEGER NOT NULL, - ngram INTEGER NOT NULL, - documents BLOB NOT NULL, - cardinality INTEGER NOT NULL, - PRIMARY KEY(page_ordinal, kind, ngram) - ) WITHOUT ROWID; - CREATE UNIQUE INDEX ngram_postings_by_ngram ON ngram_postings(kind, ngram, page_ordinal, cardinality); - CREATE TABLE ngram_statistics ( kind INTEGER NOT NULL, ngram INTEGER NOT NULL, document_frequency INTEGER NOT NULL, + documents BLOB NOT NULL, PRIMARY KEY(kind, ngram) ) WITHOUT ROWID;", ) .expect("ngram fixture schema"); + for (ngram, documents) in lists { + let bitmap = RoaringBitmap::from_iter(documents.iter().copied()); + connection + .execute( + "INSERT INTO ngram_postings(kind, ngram, document_frequency, documents) VALUES (?1, ?2, ?3, ?4)", + params![ + NGRAM_NORMALIZED, + i64::from(*ngram), + bitmap.len() as i64, + encode_document_set(&bitmap).expect("encode ngram list") + ], + ) + .expect("seed ngram list"); + } + connection + } + + #[test] + fn phrase_ngram_stream_selects_the_rarest_fixed_predicates_from_the_whole_phrase() { let phrase = b"abcdefghijklmnopqrstuvw"; let ngrams = query_ngrams(phrase).into_iter().collect::>(); assert!(ngrams.len() > ARTIFACT_NGRAM_INTERSECTION_SCRATCH_V1); - for (ordinal, ngram) in ngrams.iter().enumerate() { - let documents = if ordinal + 1 < ngrams.len() { - RoaringBitmap::from_iter([1, 2]) - } else { + let lists = ngrams + .iter() + .enumerate() + .map(|(ordinal, ngram)| { // The only selective predicate sorts beyond the fixed // intersection count in packed-ngram order. - RoaringBitmap::from_iter([1]) - }; - let encoded = encode_ngram_bitmap(LexicalArtifactLayoutV1::V11, &documents) - .expect("encode ngram shard"); - connection - .execute( - "INSERT INTO ngram_postings(page_ordinal, kind, ngram, documents, cardinality) VALUES (0, ?1, ?2, ?3, ?4)", - params![NGRAM_NORMALIZED, i64::from(*ngram), encoded, documents.len() as i64], - ) - .expect("complete phrase posting"); - connection - .execute( - "INSERT INTO ngram_statistics(kind, ngram, document_frequency) VALUES (?1, ?2, ?3)", - params![NGRAM_NORMALIZED, i64::from(*ngram), documents.len() as i64], - ) - .expect("complete phrase statistics"); - } + let documents: &[u32] = if ordinal + 1 < ngrams.len() { + &[1, 2] + } else { + &[1] + }; + (*ngram, documents) + }) + .collect::>(); + let connection = ngram_fixture(&lists); let metrics = ArtifactQueryMetricsV1::default(); - let query = ngram_document_query( - &connection, - LexicalArtifactLayoutV1::V11, - NGRAM_NORMALIZED, - phrase, - &metrics, - ) - .expect("build ngram bitmap query"); + let documents = ngram_document_query(&connection, NGRAM_NORMALIZED, phrase, &metrics) + .expect("build ngram bitmap query"); - assert_eq!(query.parameters.len(), 1); - assert_eq!(streamed_documents(&connection, &query), vec![1]); + assert_eq!(documents.iter().collect::>(), vec![1]); assert_eq!( - metrics.ngram_decoded_shards.get(), + metrics.ngram_decoded_lists.get(), ARTIFACT_NGRAM_INTERSECTION_SCRATCH_V1 as u64, - "selectivity must not increase the fixed shard-work bound" + "selectivity must not increase the fixed list-work bound" ); } + fn normalized(ngrams: &[u32]) -> Vec<(i64, u32)> { + ngrams + .iter() + .map(|ngram| (NGRAM_NORMALIZED, *ngram)) + .collect() + } + #[test] - fn ngram_bitmap_query_processes_rare_shards_first_and_short_circuits_common_work() { - let connection = Connection::open_in_memory().expect("in-memory SQLite"); - connection - .execute_batch( - "CREATE TABLE ngram_postings ( - page_ordinal INTEGER NOT NULL, - kind INTEGER NOT NULL, - ngram INTEGER NOT NULL, - documents BLOB NOT NULL, - cardinality INTEGER NOT NULL, - PRIMARY KEY(page_ordinal, kind, ngram) - ) WITHOUT ROWID; - CREATE UNIQUE INDEX ngram_postings_by_ngram ON ngram_postings(kind, ngram, page_ordinal, cardinality); - CREATE TABLE ngram_statistics ( - kind INTEGER NOT NULL, - ngram INTEGER NOT NULL, - document_frequency INTEGER NOT NULL, - PRIMARY KEY(kind, ngram) - ) WITHOUT ROWID;", - ) - .expect("ngram fixture schema"); - for (page_ordinal, ngram, documents) in [ - (0i64, 10u32, vec![1u32, 2]), - (1, 10, vec![3, 4]), - (2, 10, vec![5, 6]), - (0, 20, vec![2]), - (1, 30, vec![3]), - ] { - let bitmap = RoaringBitmap::from_iter(documents); - let encoded = encode_ngram_bitmap(LexicalArtifactLayoutV1::V11, &bitmap) - .expect("encode ngram shard"); - connection - .execute( - "INSERT INTO ngram_postings(page_ordinal, kind, ngram, documents, cardinality) VALUES (?1, ?2, ?3, ?4, ?5)", - params![page_ordinal, NGRAM_NORMALIZED, i64::from(ngram), encoded, bitmap.len() as i64], - ) - .expect("seed ngram shard"); - } - for (ngram, document_frequency) in [(10i64, 6i64), (20, 1), (30, 1)] { - connection - .execute( - "INSERT INTO ngram_statistics(kind, ngram, document_frequency) VALUES (?1, ?2, ?3)", - params![NGRAM_NORMALIZED, ngram, document_frequency], - ) - .expect("seed ngram statistics"); - } + fn ngram_bitmap_query_processes_rare_lists_first_and_short_circuits_common_work() { + let connection = ngram_fixture(&[(10, &[1, 2, 3, 4, 5, 6]), (20, &[2]), (30, &[3])]); let bounded_metrics = ArtifactQueryMetricsV1::default(); - let matching = ngram_bitmap_candidates( - &connection, - LexicalArtifactLayoutV1::V11, - NGRAM_NORMALIZED, - &[10, 20], - &bounded_metrics, - ) - .expect("intersect common and rare shards"); + let matching = + ngram_bitmap_candidates(&connection, &normalized(&[10, 20]), &bounded_metrics) + .expect("intersect common and rare lists"); assert_eq!(matching.iter().collect::>(), [2]); assert_eq!(bounded_metrics.ngram_peak_candidates.get(), 1); - assert_eq!(bounded_metrics.ngram_decoded_shards.get(), 2); + assert_eq!(bounded_metrics.ngram_decoded_lists.get(), 2); assert_eq!(bounded_metrics.observed_fullscan_steps(), 0); let short_circuit_metrics = ArtifactQueryMetricsV1::default(); let empty = ngram_bitmap_candidates( &connection, - LexicalArtifactLayoutV1::V11, - NGRAM_NORMALIZED, - &[10, 20, 30], + &normalized(&[10, 20, 30]), &short_circuit_metrics, ) - .expect("short-circuit disjoint rare shards"); + .expect("short-circuit disjoint rare lists"); assert!(empty.is_empty()); assert_eq!(short_circuit_metrics.ngram_peak_candidates.get(), 1); assert_eq!( - short_circuit_metrics.ngram_decoded_shards.get(), - 1, - "candidate-page pruning must avoid decoding unrelated ngram shards" + short_circuit_metrics.ngram_decoded_lists.get(), + 2, + "disjoint rare lists must end the query before the common list is decoded" ); - } - #[test] - fn ngram_candidate_json_honors_its_distinct_transient_byte_authority() { - let candidates = RoaringBitmap::from_iter([1, 20, 300]); - let exact = "[1,20,300]"; - assert_eq!( - encode_ngram_candidate_json(&candidates, exact.len()) - .expect("exact candidate JSON boundary"), - exact - ); - assert_eq!( - encode_ngram_candidate_json(&candidates, exact.len() - 1), - Err(crate::retrieval::ports::RetrievalPortError::BudgetExceeded) - ); + let missing = ngram_bitmap_candidates( + &connection, + &normalized(&[10, 40]), + &ArtifactQueryMetricsV1::default(), + ) + .expect("absent ngram"); + assert!(missing.is_empty(), "an absent ngram admits no document"); } #[test] @@ -4024,177 +3464,144 @@ mod tests { } #[test] - fn ngram_query_rejects_cumulative_encoded_shards_past_its_authority() { + fn ngram_query_rejects_cumulative_encoded_lists_past_its_authority() { let mut remaining = 40usize; for _ in 0..8 { - charge_ngram_encoded_shard_bytes(&mut remaining, 5, 5) - .expect("individually valid encoded shard"); + charge_ngram_encoded_list_bytes(&mut remaining, 5, 5) + .expect("individually valid encoded list"); } assert_eq!(remaining, 0); assert_eq!( - charge_ngram_encoded_shard_bytes(&mut remaining, 1, 5), + charge_ngram_encoded_list_bytes(&mut remaining, 1, 5), Err(crate::retrieval::ports::RetrievalPortError::BudgetExceeded) ); assert_eq!( remaining, 0, - "a refused shard must not consume the retained query authority" + "a refused list must not consume the retained query authority" + ); + let mut remaining = 40usize; + assert_eq!( + charge_ngram_encoded_list_bytes(&mut remaining, 6, 5), + Err(crate::retrieval::ports::RetrievalPortError::BudgetExceeded), + "one list past the per-list ceiling is refused outright" ); } - #[test] - fn streamed_union_preserves_phrase_and_fuzzy_candidate_membership() { - let connection = Connection::open_in_memory().expect("in-memory SQLite"); - connection - .execute_batch( - "CREATE TABLE term_postings ( - field TEXT NOT NULL, - term TEXT NOT NULL, - document_id INTEGER NOT NULL - );", - ) - .expect("term fixture schema"); - for (field, term, document) in [ - ("body", "render", 1), - ("body", "renderer", 2), - ("subtoken", "render", 3), - ("subtoken", "render", 3), - ] { - connection - .execute( - "INSERT INTO term_postings(field, term, document_id) VALUES (?1, ?2, ?3)", - params![field, term, document], - ) - .expect("term posting"); + fn term_list(field: LexicalFieldV1, postings: &[(u32, u32)]) -> TermFieldPostingsV1 { + let mut encoder = PostingListEncoderV1::new(true); + for (document, frequency) in postings { + encoder + .push(*document, *frequency) + .expect("ascending posting"); + } + TermFieldPostingsV1 { + field, + postings: encoder.finish().expect("non-empty list"), } - let query = union_document_queries([ - DocumentQueryV1::term_except("render".to_owned(), "subtoken".to_owned()), - DocumentQueryV1::term_except("renderer".to_owned(), "subtoken".to_owned()), - DocumentQueryV1::term("subtoken".to_owned(), "render".to_owned()), - ]) - .expect("small document union"); - - assert_eq!(streamed_documents(&connection, &query), vec![1, 2, 3]); - assert_eq!( - streamed_documents( - &connection, - &union_document_queries([DocumentQueryV1::term( - "subtoken".to_owned(), - "render".to_owned(), - )]) - .expect("single document union"), - ), - vec![3], - "one source query must preserve bitmap-like candidate deduplication" - ); } #[test] - fn streamed_union_handles_more_sources_than_sqlite_compound_limit() { - let connection = Connection::open_in_memory().expect("in-memory SQLite"); - connection - .execute_batch( - "CREATE TABLE term_postings ( - field TEXT NOT NULL, - term TEXT NOT NULL, - document_id INTEGER NOT NULL - );", - ) - .expect("term fixture schema"); - let source_count = 513u32; - let sources = (0..source_count) - .map(|document| { - let term = format!("term-{document:04}"); - connection - .execute( - "INSERT INTO term_postings(field, term, document_id) VALUES (?1, ?2, ?3)", - params!["body", term, i64::from(document)], - ) - .expect("term posting"); - DocumentQueryV1::term("body".to_owned(), term) - }) - .collect::>(); - - let query = union_document_queries(sources).expect("large document union"); - assert_eq!( - query.parameters.len(), - source_count as usize + 1, - "repeated field binds share one bounded SQLite parameter slot" + fn whole_term_and_subtoken_sources_split_one_terms_lists_by_field() { + let mut postings = RequestTermPostingsV1::default(); + postings.by_term.insert( + "render".to_owned(), + vec![ + term_list(LexicalFieldV1::BodyText, &[(1, 1)]), + term_list(LexicalFieldV1::Subtoken, &[(3, 2)]), + ], + ); + postings.by_term.insert( + "renderer".to_owned(), + vec![term_list(LexicalFieldV1::BodyText, &[(2, 1)])], ); + let whole = |term| { + postings + .documents(term, |field| field != LexicalFieldV1::Subtoken) + .expect("decode whole-term lists") + }; + let subtoken = postings + .documents("render", |field| field == LexicalFieldV1::Subtoken) + .expect("decode subtoken list"); + assert_eq!(whole("render").iter().collect::>(), [1]); + assert_eq!(subtoken.iter().collect::>(), [3]); assert_eq!( - streamed_documents(&connection, &query), - (0..source_count).collect::>(), - "nested streamed enumeration preserves the exact candidate order beyond SQLite's flat UNION ceiling" + (whole("render") | whole("renderer") | subtoken) + .iter() + .collect::>(), + [1, 2, 3] ); + assert!(whole("absent").is_empty()); } - /// A V10 row/posting fixture where every one of `documents` rows matches - /// the term `alpha` (frequency `document % 3 + 1`) plus one irrelevant - /// posting, returned with the encoded body-text field it was built under. - fn lexical_row_stream_fixture(documents: i64) -> (Connection, String) { + const ALPHA: &str = "alpha"; + + /// `documents` rows whose payload is the little-endian document id, + /// stored in row blocks, and the request postings of `alpha` (every + /// document, frequency `document % 3 + 1`). + fn lexical_row_stream_fixture(documents: u32) -> (Connection, RequestTermPostingsV1) { let connection = Connection::open_in_memory().expect("in-memory SQLite"); connection .execute_batch( - "CREATE TABLE rows ( - document_id INTEGER PRIMARY KEY, - chunk_id TEXT NOT NULL, - row BLOB NOT NULL - ); - CREATE TABLE term_postings ( - field TEXT NOT NULL, - term TEXT NOT NULL, - document_id INTEGER NOT NULL, - frequency INTEGER NOT NULL, - PRIMARY KEY(field, term, document_id) - ) WITHOUT ROWID; - CREATE INDEX term_postings_by_document_term - ON term_postings(document_id, term, field, frequency);", + "CREATE TABLE row_blocks ( + first_document INTEGER PRIMARY KEY, + payload BLOB NOT NULL + );", ) .expect("lexical row fixture schema"); - let field = - super::encode_field(super::LexicalFieldV1::BodyText).expect("encoded lexical field"); - for document in 0..documents { - connection - .execute( - "INSERT INTO rows(document_id, chunk_id, row) VALUES (?1, ?2, ?3)", - params![ - document, - format!("chunk.{document}"), - (document as u32).to_le_bytes().as_slice() - ], - ) - .expect("artifact row"); - connection - .execute( - "INSERT INTO term_postings(field, term, document_id, frequency) VALUES (?1, 'alpha', ?2, ?3)", - params![field, document, document % 3 + 1], - ) - .expect("matching posting"); + let chunk_ids = (0..documents) + .map(|document| format!("chunk.{document}")) + .collect::>(); + let payloads = (0..documents) + .map(|document| document.to_le_bytes()) + .collect::>(); + let rows = (0..documents) + .map(|document| { + let index = usize::try_from(document).expect("document index"); + BlockRowV1 { + document_id: i64::from(document), + chunk_id: &chunk_ids[index], + parent_chunk_id: None, + row: &payloads[index], + text: "fn alpha() {}", + } + }) + .collect::>(); + for (first_document, payload) in encode_row_blocks(&rows).expect("row blocks") { connection .execute( - "INSERT INTO term_postings(field, term, document_id, frequency) VALUES (?1, 'irrelevant', ?2, 99)", - params![field, document], + "INSERT INTO row_blocks(first_document, payload) VALUES (?1, ?2)", + params![first_document, payload], ) - .expect("irrelevant posting"); - } - (connection, field) + .expect("artifact row block"); + } + let mut postings = RequestTermPostingsV1::default(); + postings.by_term.insert( + ALPHA.to_owned(), + vec![term_list( + LexicalFieldV1::BodyText, + &(0..documents) + .map(|document| (document, document % 3 + 1)) + .collect::>(), + )], + ); + (connection, postings) } #[test] fn lexical_candidate_batches_bound_cancellation_probes() { - let (connection, field) = lexical_row_stream_fixture(256); - let documents = DocumentQueryV1::term(field, "alpha".to_owned()); - let terms = BTreeSet::from(["alpha".to_owned()]); + let (connection, postings) = lexical_row_stream_fixture(256); + let documents = RoaringBitmap::from_iter(0..256); let control = CancelAtObservation::new(usize::MAX); let mut visited = 0; visit_lexical_rows( &connection, + &RowBlocksV1::new(&connection), &documents, - &terms, + &postings, &ArtifactQueryMetricsV1::default(), - LexicalArtifactLayoutV1::V10, &control, - |_, _, _, _| { + |_, _, _| { visited += 1; Ok(()) }, @@ -4214,20 +3621,19 @@ mod tests { /// checkpoint changes nothing for an uncancelled request. #[test] fn lexical_row_stream_unwinds_at_the_first_checkpoint_after_cancellation() { - let (connection, field) = lexical_row_stream_fixture(512); - let documents = DocumentQueryV1::term(field, "alpha".to_owned()); - let terms = BTreeSet::from(["alpha".to_owned()]); + let (connection, postings) = lexical_row_stream_fixture(512); + let documents = RoaringBitmap::from_iter(0..512); let control = CancelAtObservation::new(2); let mut visited = 0usize; let error = visit_lexical_rows( &connection, + &RowBlocksV1::new(&connection), &documents, - &terms, + &postings, &ArtifactQueryMetricsV1::default(), - LexicalArtifactLayoutV1::V10, &control, - |_, _, _, _| { + |_, _, _| { visited += 1; Ok(()) }, @@ -4247,12 +3653,12 @@ mod tests { let mut complete = 0usize; visit_lexical_rows( &connection, + &RowBlocksV1::new(&connection), &documents, - &terms, + &postings, &ArtifactQueryMetricsV1::default(), - LexicalArtifactLayoutV1::V10, &AlwaysActiveControl, - |_, _, _, _| { + |_, _, _| { complete += 1; Ok(()) }, @@ -4263,11 +3669,10 @@ mod tests { #[test] fn exact_document_stream_cancels_before_the_next_candidate_batch() { - let (connection, field) = lexical_row_stream_fixture(512); - let documents = DocumentQueryV1::term(field, "alpha".to_owned()); + let documents = RoaringBitmap::from_iter(0..512); let control = CancelAtObservation::new(2); let mut visited = Vec::new(); - let error = visit_document_ids(&connection, &documents, &control, |document| { + let error = visit_document_ids(&documents, &control, |document| { visited.push(document); Ok(()) }) @@ -4278,47 +3683,46 @@ mod tests { } #[test] - fn lexical_row_stream_batches_term_frequencies_in_one_indexed_probe_at_scale() { - let (connection, field) = lexical_row_stream_fixture(2_048); - let documents = DocumentQueryV1::term(field.clone(), "alpha".to_owned()); - let mut terms = (0..250) - .map(|term| format!("absent-{term}")) - .collect::>(); - terms.insert("alpha".to_owned()); - terms.insert("beta".to_owned()); + fn lexical_row_stream_reads_term_frequencies_from_one_list_walk_at_scale() { + let (connection, postings) = lexical_row_stream_fixture(2_048); + // Every third document is a candidate: cursors must skip the + // postings between candidates rather than report neighbours. + let documents = (0..2_048).step_by(3).collect::(); let metrics = ArtifactQueryMetricsV1::default(); let mut visited = 0usize; + let rows = RowBlocksV1::new(&connection); visit_lexical_rows( &connection, + &rows, &documents, - &terms, + &postings, &metrics, - LexicalArtifactLayoutV1::V10, &AlwaysActiveControl, - |document, _chunk_id, row, frequencies| { - assert_eq!(row, document.to_le_bytes()); + |document, stored, frequencies| { + assert_eq!(stored.row, document.to_le_bytes()); + assert_eq!(stored.chunk_id, format!("chunk.{document}")); assert_eq!( - term_frequency(&frequencies, super::LexicalFieldV1::BodyText, "alpha"), + term_frequency(&frequencies, LexicalFieldV1::BodyText, ALPHA), usize::try_from(document).unwrap() % 3 + 1 ); assert_eq!( - term_frequency(&frequencies, super::LexicalFieldV1::BodyText, "beta"), + term_frequency(&frequencies, LexicalFieldV1::Subtoken, ALPHA), 0, - "absent postings stay exact zeroes" + "a field without a list stays an exact zero" ); - assert!( - term_frequency(&frequencies, super::LexicalFieldV1::BodyText, "irrelevant") - == 0, - "the batched probe must not hydrate unrelated terms" + assert_eq!( + term_frequency(&frequencies, LexicalFieldV1::BodyText, "beta"), + 0, + "absent terms stay exact zeroes" ); visited += 1; Ok(()) }, ) - .expect("batched lexical row stream"); + .expect("lexical row stream"); - assert_eq!(visited, 2_048); + assert_eq!(visited, documents.len() as usize); assert_eq!( metrics.probes(), 1, @@ -4327,67 +3731,40 @@ mod tests { assert_eq!( metrics.observed_fullscan_steps(), 0, - "candidate and frequency lookups must stay on maintained indexes at scale" + "row lookups must stay on the row key at scale" + ); + let blocks: i64 = connection + .query_row("SELECT COUNT(*) FROM row_blocks", [], |row| row.get(0)) + .expect("block count"); + assert!( + blocks >= 2_048 / 32, + "rows must be stored in bounded blocks, not one blob: {blocks} blocks" ); } #[test] - fn lexical_row_stream_refuses_more_than_the_portable_sqlite_bind_budget() { - let connection = Connection::open_in_memory().expect("in-memory SQLite"); - let documents = DocumentQueryV1 { - sql: Some("SELECT ? AS document_id".to_owned()), - parameters: vec![rusqlite::types::Value::Integer(1)], - maximum_bound_value_bytes: ARTIFACT_SQLITE_MAX_BOUND_VALUE_BYTES_V1, - }; - let terms = (0..ARTIFACT_SQLITE_MAX_BIND_PARAMETERS_V1) - .map(|term| format!("term-{term}")) - .collect::>(); - - let error = visit_lexical_rows( - &connection, - &documents, - &terms, - &ArtifactQueryMetricsV1::default(), - LexicalArtifactLayoutV1::V10, - &AlwaysActiveControl, - |_, _, _, _| Ok(()), - ) - .expect_err("combined document and term binds must be request-bounded"); - + fn request_term_reads_refuse_more_than_the_portable_sqlite_bind_and_byte_budgets() { assert_eq!( - error, - crate::retrieval::ports::RetrievalPortError::BudgetExceeded + ensure_sqlite_bind_capacity(0, ARTIFACT_SQLITE_MAX_BIND_PARAMETERS_V1), + Ok(()) + ); + assert_eq!( + ensure_sqlite_bind_capacity(0, ARTIFACT_SQLITE_MAX_BIND_PARAMETERS_V1 + 1), + Err(RetrievalPortError::BudgetExceeded) ); - } - - #[test] - fn lexical_row_stream_refuses_aggregate_bound_text_over_budget() { - let connection = Connection::open_in_memory().expect("in-memory SQLite"); - let documents = DocumentQueryV1 { - sql: Some("SELECT 1 AS document_id".to_owned()), - parameters: Vec::new(), - maximum_bound_value_bytes: ARTIFACT_SQLITE_MAX_BOUND_VALUE_BYTES_V1, - }; let per_term_bytes = ARTIFACT_SQLITE_MAX_BOUND_VALUE_BYTES_V1 / ARTIFACT_SQLITE_MAX_BIND_PARAMETERS_V1 + 1; let terms = (0..ARTIFACT_SQLITE_MAX_BIND_PARAMETERS_V1) .map(|term| format!("{term:04}-{}", "x".repeat(per_term_bytes))) .collect::>(); - - let error = visit_lexical_rows( - &connection, - &documents, - &terms, - &ArtifactQueryMetricsV1::default(), - LexicalArtifactLayoutV1::V10, - &AlwaysActiveControl, - |_, _, _, _| Ok(()), - ) - .expect_err("aggregate bound text must stay within a deterministic byte budget"); - assert_eq!( - error, - crate::retrieval::ports::RetrievalPortError::BudgetExceeded + ensure_sqlite_bound_value_bytes( + ARTIFACT_SQLITE_MAX_BOUND_VALUE_BYTES_V1, + &[], + terms.iter().map(String::as_str), + ), + Err(RetrievalPortError::BudgetExceeded), + "aggregate bound text must stay within a deterministic byte budget" ); } @@ -4422,173 +3799,29 @@ mod tests { } #[test] - fn interned_v11_frequency_probe_matches_text_v10_and_uses_document_index() { - let v10 = Connection::open_in_memory().expect("v10 fixture"); - v10.execute_batch( - "CREATE TABLE rows ( - document_id INTEGER PRIMARY KEY, - chunk_id TEXT NOT NULL, - row BLOB NOT NULL - ); - CREATE TABLE term_postings ( - field TEXT NOT NULL, - term TEXT NOT NULL, - document_id INTEGER NOT NULL, - frequency INTEGER NOT NULL, - PRIMARY KEY(field, term, document_id) - ) WITHOUT ROWID; - CREATE INDEX term_postings_by_document_term - ON term_postings(document_id, term, field, frequency);", - ) - .expect("v10 schema"); - let v11 = Connection::open_in_memory().expect("v11 fixture"); - v11.execute_batch( - "CREATE TABLE rows ( - document_id INTEGER PRIMARY KEY, - chunk_id TEXT NOT NULL, - row BLOB NOT NULL - ); - CREATE TABLE vocabulary ( - term_id INTEGER PRIMARY KEY, - term TEXT NOT NULL UNIQUE, - in_fuzzy INTEGER NOT NULL - ); - CREATE TABLE term_postings ( - term_id INTEGER NOT NULL, - field INTEGER NOT NULL, - document_id INTEGER NOT NULL, - frequency INTEGER NOT NULL, - PRIMARY KEY(term_id, field, document_id) - ) WITHOUT ROWID; - CREATE INDEX term_postings_by_document - ON term_postings(document_id, term_id, field, frequency);", - ) - .expect("v11 schema"); - let field = super::encode_field(super::LexicalFieldV1::BodyText).expect("field"); - v11.execute( - "INSERT INTO vocabulary(term_id, term, in_fuzzy) VALUES (1, 'alpha', 1), (2, 'beta', 1)", - [], - ) - .expect("intern terms"); - for document in 0..32i64 { - let row = (document as u32).to_le_bytes(); - let chunk = format!("chunk.{document}"); - v10.execute( - "INSERT INTO rows(document_id, chunk_id, row) VALUES (?1, ?2, ?3)", - params![document, &chunk, row.as_slice()], - ) - .expect("v10 row"); - v11.execute( - "INSERT INTO rows(document_id, chunk_id, row) VALUES (?1, ?2, ?3)", - params![document, &chunk, row.as_slice()], - ) - .expect("v11 row"); - v10.execute( - "INSERT INTO term_postings(field, term, document_id, frequency) VALUES (?1, 'alpha', ?2, ?3)", - params![field, document, document % 3 + 1], - ) - .expect("v10 posting"); - v11.execute( - "INSERT INTO term_postings(term_id, field, document_id, frequency) VALUES (1, 4, ?1, ?2)", - params![document, document % 3 + 1], - ) - .expect("v11 posting"); - } - let v10_query = DocumentQueryV1::term(field, "alpha".to_owned()); - let v11_query = DocumentQueryV1::term_id(4, 1); - let terms = BTreeSet::from(["alpha".to_owned(), "beta".to_owned()]); - let mut v10_hits = Vec::new(); - visit_lexical_rows( - &v10, - &v10_query, - &terms, - &ArtifactQueryMetricsV1::default(), - LexicalArtifactLayoutV1::V10, - &AlwaysActiveControl, - |document, _, _, frequencies| { - v10_hits.push(( - document, - term_frequency(&frequencies, super::LexicalFieldV1::BodyText, "alpha"), - )); - Ok(()) - }, - ) - .expect("v10 stream"); - let mut v11_hits = Vec::new(); - visit_lexical_rows( - &v11, - &v11_query, - &terms, - &ArtifactQueryMetricsV1::default(), - LexicalArtifactLayoutV1::V11, - &AlwaysActiveControl, - |document, _, _, frequencies| { - v11_hits.push(( - document, - term_frequency(&frequencies, super::LexicalFieldV1::BodyText, "alpha"), - )); - Ok(()) - }, - ) - .expect("v11 stream"); - assert_eq!(v10_hits, v11_hits); - let plan = v11 - .prepare( - "EXPLAIN QUERY PLAN SELECT document_id FROM term_postings WHERE field = ? AND term_id = ?", - ) - .expect("prepare term equality plan") - .query_map(params![4i64, 1i64], |row| row.get::<_, String>(3)) - .expect("query term equality plan") - .collect::, _>>() - .expect("collect term equality plan"); - assert!( - plan.iter().any(|detail| { - detail.contains("PRIMARY KEY") || detail.contains("term_postings") - }), - "interned term equality must use the clustered key, got {plan:?}" - ); - let frequency_plan = v11 - .prepare( - "EXPLAIN QUERY PLAN SELECT posting.frequency \ - FROM term_postings AS posting INDEXED BY term_postings_by_document \ - WHERE posting.document_id = 0 AND posting.term_id IN (1)", - ) - .expect("prepare frequency plan") - .query_map([], |row| row.get::<_, String>(3)) - .expect("query frequency plan") - .collect::, _>>() - .expect("collect frequency plan"); - assert!( - frequency_plan - .iter() - .any(|detail| detail.contains("term_postings_by_document")), - "frequency probe must use the document-leading index, got {frequency_plan:?}" - ); - } - - #[test] - fn fuzzy_vocabulary_sql_does_not_order_hash_keyed_rows() { + fn fuzzy_vocabulary_is_one_unsorted_pass_over_the_term_keyed_table() { assert!( - !super::ArtifactQueryV1::vocabulary_sql(LexicalArtifactLayoutV1::V11) + !super::ArtifactQueryV1::VOCABULARY_SQL .to_ascii_uppercase() .contains("ORDER BY"), - "in-fuzzy vocabulary load must not sort hash-keyed term_id rows" + "the in-fuzzy vocabulary load needs no order" ); - let v11 = Connection::open_in_memory().expect("v11 vocab plan db"); - v11.execute_batch( - "CREATE TABLE vocabulary ( - term_id INTEGER PRIMARY KEY, - term TEXT NOT NULL UNIQUE, - in_fuzzy INTEGER NOT NULL - ); - INSERT INTO vocabulary(term_id, term, in_fuzzy) VALUES (1, 'alpha', 1), (2, 'beta', 0);", - ) - .expect("seed vocabulary"); + let connection = Connection::open_in_memory().expect("vocabulary plan db"); + connection + .execute_batch( + "CREATE TABLE term_postings ( + term TEXT NOT NULL PRIMARY KEY, + in_fuzzy INTEGER NOT NULL, + lists BLOB NOT NULL + ) WITHOUT ROWID; + INSERT INTO term_postings(term, in_fuzzy, lists) VALUES ('alpha', 1, x'00'), ('beta', 0, x'00');", + ) + .expect("seed vocabulary"); let sql = format!( "EXPLAIN QUERY PLAN {}", - super::ArtifactQueryV1::vocabulary_sql(LexicalArtifactLayoutV1::V11) + super::ArtifactQueryV1::VOCABULARY_SQL ); - let plan = v11 + let plan = connection .prepare(&sql) .expect("prepare vocab plan") .query_map([], |row| row.get::<_, String>(3)) @@ -4596,9 +3829,9 @@ mod tests { .collect::, _>>() .expect("collect vocab plan"); assert!( - plan.iter().any(|detail| detail.contains("SCAN vocabulary") - && !detail.contains("sqlite_autoindex_vocabulary_1")), - "in-fuzzy load must table-scan, not bounce through UNIQUE(term), got {plan:?}" + plan.iter() + .any(|detail| detail.contains("SCAN term_postings")), + "in-fuzzy load must be one table scan, got {plan:?}" ); } } diff --git a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/reader/family_report.rs b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/reader/family_report.rs index 8ce8024331..5bc67611ee 100644 --- a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/reader/family_report.rs +++ b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/reader/family_report.rs @@ -8,6 +8,8 @@ use tracedecay_domain::{ CodeGenerationId, ManifestDigest, ProjectId, RepositoryId, SymbolOccurrenceId, canonical_sha256, }; +use super::super::clone_codec::{digest_from_key, digest_key}; +use super::super::row_codec::symbol_id_from_key; use super::{CodeLexicalArtifactReaderV1, MAX_CLONE_EXACT_PAGE_MEMBERS_V1}; use crate::retrieval::lexical::projection::artifact::{ CodeLexicalArtifactErrorV1, checkpoint, sqlite_error, @@ -85,11 +87,6 @@ impl CodeLexicalArtifactReaderV1 { "clone family repository authority is unavailable".to_owned(), )); } - if !self.layout.has_clone_index() { - return Err(CodeLexicalArtifactErrorV1::Incompatible( - "clone family lookup requires lexical artifact revision 15".to_owned(), - )); - } if limit == 0 || limit > MAX_CLONE_EXACT_PAGE_MEMBERS_V1 { return Err(CodeLexicalArtifactErrorV1::Contract(format!( "clone family page limit must be within 1..={MAX_CLONE_EXACT_PAGE_MEMBERS_V1}" @@ -141,7 +138,10 @@ impl CodeLexicalArtifactReaderV1 { let after_revision = after .map(|position| position.normalization_revision) .unwrap_or_default(); - let after_digest = after.map(|position| position.digest.as_str()).unwrap_or(""); + let after_digest = after + .map(|position| digest_key(&position.digest)) + .transpose()? + .unwrap_or_default(); let connection = self.lock_connection()?; install_generated_path_function(&connection)?; install_pull_request_path_function(&connection, pull_request_paths)?; @@ -149,14 +149,14 @@ impl CodeLexicalArtifactReaderV1 { .prepare_cached( "WITH families AS ( \ SELECT posting.class, posting.normalization_revision, posting.digest, \ - MIN(posting.symbol_occurrence_id) AS representative, \ + MIN(occurrence.symbol_key) AS representative, \ COUNT(*) AS member_count, \ SUM(occurrence.body_end - occurrence.body_start) \ - MIN(occurrence.body_end - occurrence.body_start) \ AS reviewable_source_bytes \ FROM clone_exact_postings AS posting \ JOIN clone_occurrences AS occurrence \ - ON occurrence.symbol_occurrence_id = posting.symbol_occurrence_id \ + ON occurrence.ordinal = posting.occurrence_ordinal \ WHERE ((:conservative AND posting.class = 1) OR (:rename AND posting.class = 2)) \ AND (:path IS NULL OR occurrence.path = :path \ OR (substr(occurrence.path, 1, length(:path)) = :path \ @@ -193,7 +193,7 @@ impl CodeLexicalArtifactReaderV1 { .map_err(|error| CodeLexicalArtifactErrorV1::Contract(error.to_string()))?, ":after_class": i64::from(after_class), ":after_revision": i64::from(after_revision), - ":after_digest": after_digest, + ":after_digest": after_digest.as_slice(), ":fetch": i64::try_from(fetch) .map_err(|error| CodeLexicalArtifactErrorV1::Contract(error.to_string()))?, }) @@ -217,10 +217,9 @@ impl CodeLexicalArtifactReaderV1 { }; let normalization_revision = u16::try_from(row.get::<_, i64>(1).map_err(sqlite_error)?) .map_err(|error| CodeLexicalArtifactErrorV1::Corrupt(error.to_string()))?; - let digest = ManifestDigest::new(row.get::<_, String>(2).map_err(sqlite_error)?) - .map_err(|error| CodeLexicalArtifactErrorV1::Corrupt(error.to_string()))?; + let digest = digest_from_key(&row.get::<_, Vec>(2).map_err(sqlite_error)?)?; let representative = - SymbolOccurrenceId::new(row.get::<_, String>(3).map_err(sqlite_error)?) + SymbolOccurrenceId::new(symbol_id_from_key(row.get_ref(3).map_err(sqlite_error)?)?) .map_err(|error| CodeLexicalArtifactErrorV1::Corrupt(error.to_string()))?; let member_count = usize::try_from(row.get::<_, i64>(4).map_err(sqlite_error)?) .map_err(|error| CodeLexicalArtifactErrorV1::Corrupt(error.to_string()))?; diff --git a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/row_codec.rs b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/row_codec.rs index 0883c9a173..d6e44fb30d 100644 --- a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/row_codec.rs +++ b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/row_codec.rs @@ -1,51 +1,64 @@ use std::cell::RefCell; use std::collections::{BTreeMap, HashMap}; +use std::ops::Range; use std::sync::Arc; use rusqlite::{Connection, OptionalExtension}; -use serde::{Deserialize, Serialize}; use tracedecay_domain::{ BoundedSanitizedText, CodeGenerationId, CodeSearchChunkAnchorV1, CodeSearchChunkGrainV1, CodeSearchChunkId, ExactTechnicalTermKindV1, ExactTechnicalTermV1, FileOccurrenceId, - LanguageDescriptorRevision, SourceSpan, SymbolOccurrenceId, + LanguageDescriptorRevision, MAX_CHUNK_TEXT_BYTES, SourceSpan, SymbolOccurrenceId, }; use super::super::LexicalFieldV1; use super::CodeLexicalArtifactErrorV1; -use super::format::ArtifactRowV1; -use super::schema::{LexicalArtifactLayoutV1, stable_row_dictionary_id}; +use super::format::{ArtifactRowV1, deflate_bytes, inflate_bytes}; +use super::schema::stable_row_dictionary_id; -const ROW_CODEC_V11_MAGIC: &[u8] = b"TDLR11\0"; -const ROW_CODEC_V14_MAGIC: &[u8] = b"TDLR14\0"; -const ROW_CODEC_V16_MAGIC: &[u8] = b"TDLR16\0"; +/// Rows are stored in blocks of up to `ROW_BLOCK_MAX_ROWS` consecutive +/// documents of one source page, closed early once their uncompressed +/// payload reaches `ROW_BLOCK_TARGET_BYTES`, and deflated as one stream: a +/// read inflates at most one block, and neighbouring chunks of one file +/// share a deflate window (about twice the ratio of per-row deflate). +pub(super) const ROW_BLOCK_MAX_ROWS: usize = 32; +const ROW_BLOCK_TARGET_BYTES: usize = 64 * 1024; +/// Hard bound on one block's inflated payload: a block closes at its target +/// before its last row, and one row holds at most a chunk's text plus its +/// metadata. +const ROW_BLOCK_MAX_INFLATED_BYTES: usize = ROW_BLOCK_TARGET_BYTES + 4 * MAX_CHUNK_TEXT_BYTES; +const ROW_BLOCK_DEFLATE: u8 = 23; +const BLOCK_CHUNK_DIGEST: u8 = 1; +const BLOCK_CHUNK_LITERAL: u8 = 2; +/// A row's text is stored raw, or as the length of the prefix it shares +/// with the raw text of its parent chunk in the same block (a signature +/// chunk is the first line of its symbol's body chunk). +const BLOCK_TEXT_RAW: u8 = 0; +const BLOCK_TEXT_PARENT_PREFIX: u8 = 1; /// Chunk identities the chunker mints: `chunk.v1.` followed by a tagged /// lowercase SHA-256. Revision 14 stores such a parent as its 32 digest bytes /// and any other shape as the literal string. const CANONICAL_CHUNK_ID_PREFIX: &str = "chunk.v1.sha256:"; +/// Symbol identities the extractor mints, stored the same way in symbol +/// dictionary entries. +const CANONICAL_SYMBOL_ID_PREFIX: &str = "symbol.v1.sha256:"; const PARENT_NONE: u8 = 0; const PARENT_CANONICAL_DIGEST: u8 = 1; const PARENT_LITERAL: u8 = 2; const OPTIONAL_ABSENT: u8 = 0; const OPTIONAL_PRESENT: u8 = 1; +/// Symbol-entry presence tags beyond `OPTIONAL_PRESENT`: a canonical symbol +/// id as its 32 digest bytes, and a qualified name stored as the suffix +/// after its file's `"::"`. +const SYMBOL_ID_CANONICAL_DIGEST: u8 = 2; +const QUALIFIED_NAME_IN_FILE: u8 = 2; /// An exact term's symbol authority is almost always the row's own symbol; /// spell that as one byte instead of a second reference. const TERM_SYMBOL_NONE: u8 = 0; const TERM_SYMBOL_ROW: u8 = 1; const TERM_SYMBOL_REFERENCE: u8 = 2; -const LEGACY_FIELD_LENGTH_ORDER: [LexicalFieldV1; 7] = [ - LexicalFieldV1::SymbolName, - LexicalFieldV1::QualifiedName, - LexicalFieldV1::Path, - LexicalFieldV1::BodyText, - LexicalFieldV1::PreambleText, - LexicalFieldV1::ExactTerm, - LexicalFieldV1::Subtoken, -]; - -/// Field order for the revision-16 `field_lengths` presence bitmap. New fields -/// append after the revision-14/15 prefix. +/// Field order for the `field_lengths` presence bitmap. const FIELD_LENGTH_ORDER: [LexicalFieldV1; 9] = [ LexicalFieldV1::SymbolName, LexicalFieldV1::QualifiedName, @@ -63,29 +76,6 @@ const GRAIN_ORDER: &[CodeSearchChunkGrainV1] = CodeSearchChunkGrainV1::ORDER.as_ const EXACT_TERM_KIND_ORDER: &[ExactTechnicalTermKindV1] = ExactTechnicalTermKindV1::ORDER.as_slice(); -/// Compact row payload: drop identities already stored as columns or -/// generation metadata, and reconstruct ASCII-normalized text on read. -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -struct ArtifactRowCompactV11 { - file_occurrence_id: FileOccurrenceId, - symbol_occurrence_id: Option, - parent_chunk_id: Option, - source_span: SourceSpan, - grain: CodeSearchChunkGrainV1, - ordinal: u32, - language_descriptor_revision: LanguageDescriptorRevision, - exact_terms: Vec, - sanitized_text: BoundedSanitizedText, - logical_path: String, - symbol_simple_name: Option, - symbol_qualified_name: Option, - symbol_kind: Option, - symbol_signature: Option, - symbol_documentation: Option, - field_lengths: BTreeMap, -} - /// Dictionary entries a revision-14 row references by content-addressed id: /// one per file (occurrence identity, logical path, descriptor revision) and /// one per symbol display (occurrence identity and parser-attested fields). @@ -108,18 +98,43 @@ pub(super) enum RowDictionaryEntryV1 { Symbol { symbol_occurrence_id: Option, simple_name: Option, - qualified_name: Option, + qualified_name: Option, kind: Option, signature: Option, documentation: Option, }, } +/// A symbol's qualified name. Parser-attested names almost always spell out +/// their file as `"::"`; that prefix is the row's file +/// entry and is not stored twice. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) enum QualifiedNameV1 { + Literal(String), + InFile(String), +} + +impl QualifiedNameV1 { + fn for_path(qualified_name: &str, logical_path: &str) -> Self { + qualified_name + .strip_prefix(logical_path) + .and_then(|suffix| suffix.strip_prefix("::")) + .map_or_else( + || Self::Literal(qualified_name.to_owned()), + |suffix| Self::InFile(suffix.to_owned()), + ) + } + + fn resolve(&self, logical_path: &str) -> String { + match self { + Self::Literal(name) => name.clone(), + Self::InFile(suffix) => format!("{logical_path}::{suffix}"), + } + } +} + impl RowDictionaryEntryV1 { - fn encode( - &self, - include_vocabulary_fields: bool, - ) -> Result, CodeLexicalArtifactErrorV1> { + fn encode(&self) -> Result, CodeLexicalArtifactErrorV1> { let mut out = Vec::with_capacity(256); match self { Self::File { @@ -141,26 +156,35 @@ impl RowDictionaryEntryV1 { documentation, } => { out.push(ENTRY_SYMBOL); - for field in [symbol_occurrence_id, simple_name, qualified_name, kind] { - match field { - None => out.push(OPTIONAL_ABSENT), - Some(value) => { - out.push(OPTIONAL_PRESENT); - put_bytes(&mut out, value.as_bytes())?; - } + match symbol_occurrence_id + .as_deref() + .map(|id| (id, canonical_digest(CANONICAL_SYMBOL_ID_PREFIX, id))) + { + None => out.push(OPTIONAL_ABSENT), + Some((_, Some(digest))) => { + out.push(SYMBOL_ID_CANONICAL_DIGEST); + out.extend_from_slice(&digest); + } + Some((id, None)) => { + out.push(OPTIONAL_PRESENT); + put_bytes(&mut out, id.as_bytes())?; } } - if include_vocabulary_fields { - for field in [signature, documentation] { - match field { - None => out.push(OPTIONAL_ABSENT), - Some(value) => { - out.push(OPTIONAL_PRESENT); - put_bytes(&mut out, value.as_bytes())?; - } - } + put_optional_string(&mut out, simple_name.as_deref())?; + match qualified_name { + None => out.push(OPTIONAL_ABSENT), + Some(QualifiedNameV1::Literal(name)) => { + out.push(OPTIONAL_PRESENT); + put_bytes(&mut out, name.as_bytes())?; + } + Some(QualifiedNameV1::InFile(suffix)) => { + out.push(QUALIFIED_NAME_IN_FILE); + put_bytes(&mut out, suffix.as_bytes())?; } } + for field in [kind, signature, documentation] { + put_optional_string(&mut out, field.as_deref())?; + } } } Ok(out) @@ -174,28 +198,35 @@ impl RowDictionaryEntryV1 { logical_path: cursor.take_string()?, language_descriptor_revision: cursor.take_string()?, }, - ENTRY_SYMBOL => { - let symbol_occurrence_id = cursor.take_optional_string()?; - let simple_name = cursor.take_optional_string()?; - let qualified_name = cursor.take_optional_string()?; - let kind = cursor.take_optional_string()?; - let (signature, documentation) = if cursor.bytes.is_empty() { - (None, None) - } else { - ( - cursor.take_optional_string()?, - cursor.take_optional_string()?, - ) - }; - Self::Symbol { - symbol_occurrence_id, - simple_name, - qualified_name, - kind, - signature, - documentation, - } - } + ENTRY_SYMBOL => Self::Symbol { + symbol_occurrence_id: match cursor.take_u8()? { + OPTIONAL_ABSENT => None, + OPTIONAL_PRESENT => Some(cursor.take_string()?), + SYMBOL_ID_CANONICAL_DIGEST => Some(format!( + "{CANONICAL_SYMBOL_ID_PREFIX}{}", + hex::encode(cursor.take_exact(32)?) + )), + _ => { + return Err(CodeLexicalArtifactErrorV1::Corrupt( + "lexical artifact symbol identity tag is unknown".to_owned(), + )); + } + }, + simple_name: cursor.take_optional_string()?, + qualified_name: match cursor.take_u8()? { + OPTIONAL_ABSENT => None, + OPTIONAL_PRESENT => Some(QualifiedNameV1::Literal(cursor.take_string()?)), + QUALIFIED_NAME_IN_FILE => Some(QualifiedNameV1::InFile(cursor.take_string()?)), + _ => { + return Err(CodeLexicalArtifactErrorV1::Corrupt( + "lexical artifact qualified-name tag is unknown".to_owned(), + )); + } + }, + kind: cursor.take_optional_string()?, + signature: cursor.take_optional_string()?, + documentation: cursor.take_optional_string()?, + }, _ => { return Err(CodeLexicalArtifactErrorV1::Corrupt( "lexical artifact dictionary entry kind is unknown".to_owned(), @@ -211,8 +242,7 @@ impl RowDictionaryEntryV1 { } } -/// Resolves revision-14 dictionary references. Layouts before 14 never -/// consult it, so every call site can hand over its connection. +/// Resolves row dictionary references. pub(super) trait RowDictionaryV1 { fn entry(&self, entry_id: i64) -> Result, CodeLexicalArtifactErrorV1>; @@ -273,187 +303,45 @@ impl RowDictionaryV1 for ConnectionRowDictionaryV1<'_> { } pub(super) fn encode_artifact_row( - layout: LexicalArtifactLayoutV1, row: &ArtifactRowV1, dictionary: &mut RowDictionaryTableV1, ) -> Result, CodeLexicalArtifactErrorV1> { - match layout { - LexicalArtifactLayoutV1::V10 => serde_json::to_vec(row) - .map_err(|error| CodeLexicalArtifactErrorV1::Contract(error.to_string())), - LexicalArtifactLayoutV1::V11 - | LexicalArtifactLayoutV1::V12 - | LexicalArtifactLayoutV1::V13 => encode_compact_v11(row), - // Admission bases (V14/V15) and the fingerprint-complete revision - // (V16) share the current compact row codec. Clone fingerprints stay - // deferred via `has_clone_fingerprints`, not via a weaker row encoding. - LexicalArtifactLayoutV1::V14 - | LexicalArtifactLayoutV1::V15 - | LexicalArtifactLayoutV1::V16 => encode_binary( - row, - dictionary, - ROW_CODEC_V16_MAGIC, - &FIELD_LENGTH_ORDER, - true, - ), - } -} - -fn encode_compact_v11(row: &ArtifactRowV1) -> Result, CodeLexicalArtifactErrorV1> { - let compact = ArtifactRowCompactV11 { - file_occurrence_id: row.anchor.file_occurrence_id.clone(), - symbol_occurrence_id: row.anchor.symbol_occurrence_id.clone(), - parent_chunk_id: row.anchor.parent_chunk_id.clone(), - source_span: row.anchor.source_span, - grain: row.anchor.grain, - ordinal: row.anchor.ordinal, - language_descriptor_revision: row.language_descriptor_revision.clone(), - exact_terms: row.exact_terms.clone(), - sanitized_text: row.sanitized_text.clone(), - logical_path: row.logical_path.clone(), - symbol_simple_name: row.symbol_simple_name.clone(), - symbol_qualified_name: row.symbol_qualified_name.clone(), - symbol_kind: row.symbol_kind.clone(), - symbol_signature: row.symbol_signature.clone(), - symbol_documentation: row.symbol_documentation.clone(), - field_lengths: row.field_lengths.clone(), - }; - let payload = serde_json::to_vec(&compact) - .map_err(|error| CodeLexicalArtifactErrorV1::Contract(error.to_string()))?; - let mut bytes = Vec::with_capacity(ROW_CODEC_V11_MAGIC.len().saturating_add(payload.len())); - bytes.extend_from_slice(ROW_CODEC_V11_MAGIC); - bytes.extend_from_slice(&payload); - Ok(bytes) + encode_binary(row, dictionary) } +/// Decode one row's metadata and restore its text, which the row block +/// stores beside it. pub(super) fn decode_artifact_row( - layout: LexicalArtifactLayoutV1, generation: &CodeGenerationId, chunk_id: &str, bytes: &[u8], + text: &str, dictionary: &dyn RowDictionaryV1, ) -> Result { - match layout { - LexicalArtifactLayoutV1::V10 => decode_json_v10(generation, chunk_id, bytes), - LexicalArtifactLayoutV1::V11 - | LexicalArtifactLayoutV1::V12 - | LexicalArtifactLayoutV1::V13 => decode_compact_v11(generation, chunk_id, bytes), - // Row bytes carry their codec tag. Clone-successor bumps may label an - // artifact V16 before rows are rewritten; dispatch on the tag so - // seated lexical owners stay readable either way. - LexicalArtifactLayoutV1::V14 - | LexicalArtifactLayoutV1::V15 - | LexicalArtifactLayoutV1::V16 => { - if bytes.starts_with(ROW_CODEC_V16_MAGIC) { - decode_binary( - generation, - chunk_id, - bytes, - dictionary, - ROW_CODEC_V16_MAGIC, - &FIELD_LENGTH_ORDER, - true, - ) - } else if bytes.starts_with(ROW_CODEC_V14_MAGIC) { - decode_binary( - generation, - chunk_id, - bytes, - dictionary, - ROW_CODEC_V14_MAGIC, - &LEGACY_FIELD_LENGTH_ORDER, - false, - ) - } else { - Err(CodeLexicalArtifactErrorV1::Corrupt( - "lexical artifact row is missing its binary codec tag".to_owned(), - )) - } - } - } -} - -fn decode_json_v10( - generation: &CodeGenerationId, - chunk_id: &str, - bytes: &[u8], -) -> Result { - let row: ArtifactRowV1 = serde_json::from_slice(bytes) - .map_err(|error| CodeLexicalArtifactErrorV1::Corrupt(error.to_string()))?; - if row.id.as_str() != chunk_id || &row.anchor.generation_id != generation { - return Err(CodeLexicalArtifactErrorV1::Corrupt( - "lexical artifact row identity does not match its stored coordinates".to_owned(), - )); - } - Ok(row) -} - -fn decode_compact_v11( - generation: &CodeGenerationId, - chunk_id: &str, - bytes: &[u8], -) -> Result { - let payload = bytes.strip_prefix(ROW_CODEC_V11_MAGIC).ok_or_else(|| { - CodeLexicalArtifactErrorV1::Corrupt( - "lexical artifact row is missing the compact v11 codec tag".to_owned(), - ) - })?; - let compact: ArtifactRowCompactV11 = serde_json::from_slice(payload) - .map_err(|error| CodeLexicalArtifactErrorV1::Corrupt(error.to_string()))?; - let id = CodeSearchChunkId::new(chunk_id.to_owned()) - .map_err(|error| CodeLexicalArtifactErrorV1::Corrupt(error.to_string()))?; - let normalized_text = compact.sanitized_text.as_str().to_ascii_lowercase(); - Ok(ArtifactRowV1 { - id, - anchor: CodeSearchChunkAnchorV1 { - generation_id: generation.clone(), - file_occurrence_id: compact.file_occurrence_id, - symbol_occurrence_id: compact.symbol_occurrence_id, - parent_chunk_id: compact.parent_chunk_id, - source_span: compact.source_span, - grain: compact.grain, - ordinal: compact.ordinal, - }, - language_descriptor_revision: compact.language_descriptor_revision, - exact_terms: compact.exact_terms, - sanitized_text: compact.sanitized_text, - logical_path: compact.logical_path, - symbol_simple_name: compact.symbol_simple_name, - symbol_qualified_name: compact.symbol_qualified_name, - symbol_kind: compact.symbol_kind, - symbol_signature: compact.symbol_signature, - symbol_documentation: compact.symbol_documentation, - field_lengths: compact.field_lengths, - normalized_text, - }) + decode_binary(generation, chunk_id, bytes, text, dictionary) } // --------------------------------------------------------------------------- // Binary row with a per-file / per-symbol dictionary // --------------------------------------------------------------------------- // -// Legacy magic `TDLR14\0` (still decoded), then in order: +// In order: // ref file entry · opt-ref symbol entry · parent (tag, digest | literal) // varint span start/end · u8 grain · varint ordinal // varint term count × (u8 kind, bytes, varint span start/end, symbol tag [ref]) -// bytes sanitized_text · u8 field bitmap · varint lengths +// u16 field bitmap · varint lengths // +// The sanitized text is not part of the row; its row block stores it. // A `ref` is the little-endian `row_dictionary.entry_id`; an `opt-ref` is one // presence byte followed by the ref when present. `bytes` is a varint length // followed by the bytes. Decoders consume the whole payload and fail closed // on any trailing byte. -// Current codec uses `TDLR16\0`, appends signature and documentation to symbol -// dictionary entries, and widens the field bitmap to `u16`. Admission bases -// (V14/V15) emit this codec while clone fingerprints remain V16-only. fn encode_binary( row: &ArtifactRowV1, dictionary: &mut RowDictionaryTableV1, - magic: &[u8], - field_order: &[LexicalFieldV1], - include_vocabulary_fields: bool, ) -> Result, CodeLexicalArtifactErrorV1> { - let mut out = Vec::with_capacity(96 + row.sanitized_text.as_str().len()); - out.extend_from_slice(magic); + let mut out = Vec::with_capacity(96); put_reference( &mut out, dictionary, @@ -462,7 +350,6 @@ fn encode_binary( logical_path: row.logical_path.clone(), language_descriptor_revision: row.language_descriptor_revision.as_str().to_owned(), }, - include_vocabulary_fields, )?; let symbol = RowDictionaryEntryV1::Symbol { symbol_occurrence_id: row @@ -471,7 +358,10 @@ fn encode_binary( .as_ref() .map(|id| id.as_str().to_owned()), simple_name: row.symbol_simple_name.clone(), - qualified_name: row.symbol_qualified_name.clone(), + qualified_name: row + .symbol_qualified_name + .as_deref() + .map(|name| QualifiedNameV1::for_path(name, &row.logical_path)), kind: row.symbol_kind.clone(), signature: row.symbol_signature.clone(), documentation: row.symbol_documentation.clone(), @@ -484,7 +374,7 @@ fn encode_binary( || row.symbol_documentation.is_some(); if has_symbol { out.push(OPTIONAL_PRESENT); - put_reference(&mut out, dictionary, &symbol, include_vocabulary_fields)?; + put_reference(&mut out, dictionary, &symbol)?; } else { out.push(OPTIONAL_ABSENT); } @@ -533,14 +423,12 @@ fn encode_binary( signature: None, documentation: None, }, - include_vocabulary_fields, )?; } } } - put_bytes(&mut out, row.sanitized_text.as_str().as_bytes())?; let mut bitmap = 0u16; - for (bit, field) in field_order.iter().enumerate() { + for (bit, field) in FIELD_LENGTH_ORDER.iter().enumerate() { if row.field_lengths.contains_key(field) { bitmap |= 1 << bit; } @@ -548,7 +436,7 @@ fn encode_binary( let expected_fields = row .field_lengths .keys() - .filter(|field| field_order.contains(field)) + .filter(|field| FIELD_LENGTH_ORDER.contains(field)) .count(); if expected_fields != bitmap.count_ones() as usize { return Err(CodeLexicalArtifactErrorV1::Contract( @@ -556,12 +444,8 @@ fn encode_binary( .to_owned(), )); } - if include_vocabulary_fields { - out.extend_from_slice(&bitmap.to_le_bytes()); - } else { - out.push(bitmap as u8); - } - for field in field_order { + out.extend_from_slice(&bitmap.to_le_bytes()); + for field in &FIELD_LENGTH_ORDER { if let Some(length) = row.field_lengths.get(field) { put_varint(&mut out, length_u64(*length)?); } @@ -573,17 +457,10 @@ fn decode_binary( generation: &CodeGenerationId, chunk_id: &str, bytes: &[u8], + text: &str, dictionary: &dyn RowDictionaryV1, - magic: &[u8], - field_order: &[LexicalFieldV1], - wide_bitmap: bool, ) -> Result { - let payload = bytes.strip_prefix(magic).ok_or_else(|| { - CodeLexicalArtifactErrorV1::Corrupt( - "lexical artifact row is missing its binary codec tag".to_owned(), - ) - })?; - let mut cursor = RowCursorV1 { bytes: payload }; + let mut cursor = RowCursorV1 { bytes }; let file = dictionary.entry(cursor.take_reference()?)?; let RowDictionaryEntryV1::File { file_occurrence_id, @@ -617,7 +494,7 @@ fn decode_binary( .transpose() .map_err(corrupt)?, simple_name, - qualified_name, + qualified_name.map(|name| name.resolve(&logical_path)), kind, signature, documentation, @@ -693,19 +570,15 @@ fn decode_binary( .map_err(corrupt)?, ); } - let sanitized_text = BoundedSanitizedText::new(&cursor.take_string()?).map_err(corrupt)?; - let bitmap = if wide_bitmap { - cursor.take_u16()? - } else { - u16::from(cursor.take_u8()?) - }; - if bitmap >> field_order.len() != 0 { + let sanitized_text = BoundedSanitizedText::new(text).map_err(corrupt)?; + let bitmap = cursor.take_u16()?; + if bitmap >> FIELD_LENGTH_ORDER.len() != 0 { return Err(CodeLexicalArtifactErrorV1::Corrupt( "lexical artifact row field bitmap names an unknown field".to_owned(), )); } let mut field_lengths = BTreeMap::new(); - for (bit, field) in field_order.iter().enumerate() { + for (bit, field) in FIELD_LENGTH_ORDER.iter().enumerate() { if bitmap & (1 << bit) != 0 { let length = usize::try_from(cursor.take_varint()?).map_err(corrupt)?; field_lengths.insert(*field, length); @@ -746,7 +619,7 @@ fn decode_binary( type SymbolEntryFieldsV1 = ( Option, Option, - Option, + Option, Option, Option, Option, @@ -779,8 +652,387 @@ fn symbol_entry_fields( /// The 32 digest bytes of a chunker-minted chunk id, when re-encoding them /// reproduces the id byte for byte. -fn canonical_chunk_digest(chunk_id: &str) -> Option<[u8; 32]> { - let hex = chunk_id.strip_prefix(CANONICAL_CHUNK_ID_PREFIX)?; +pub(super) fn canonical_chunk_digest(chunk_id: &str) -> Option<[u8; 32]> { + canonical_digest(CANONICAL_CHUNK_ID_PREFIX, chunk_id) +} + +/// The value `row_chunks.chunk_id` stores: 32 digest bytes for a canonical +/// chunk id, the literal text otherwise. +pub(super) fn stored_chunk_key(chunk_id: &str) -> rusqlite::types::Value { + canonical_chunk_digest(chunk_id).map_or_else( + || rusqlite::types::Value::Text(chunk_id.to_owned()), + |digest| rusqlite::types::Value::Blob(digest.to_vec()), + ) +} + +/// The value `clone_occurrences.symbol_key` stores: 32 digest bytes for an +/// extractor-minted symbol id, the literal text otherwise. +pub(super) fn stored_symbol_key(symbol: &str) -> rusqlite::types::Value { + canonical_digest(CANONICAL_SYMBOL_ID_PREFIX, symbol).map_or_else( + || rusqlite::types::Value::Text(symbol.to_owned()), + |digest| rusqlite::types::Value::Blob(digest.to_vec()), + ) +} + +/// Inverse of [`stored_symbol_key`]. +pub(super) fn symbol_id_from_key( + key: rusqlite::types::ValueRef<'_>, +) -> Result { + match key { + rusqlite::types::ValueRef::Blob(digest) if digest.len() == 32 => Ok(format!( + "{CANONICAL_SYMBOL_ID_PREFIX}{}", + hex::encode(digest) + )), + rusqlite::types::ValueRef::Text(text) => std::str::from_utf8(text) + .map(str::to_owned) + .map_err(|error| CodeLexicalArtifactErrorV1::Corrupt(error.to_string())), + _ => Err(CodeLexicalArtifactErrorV1::Corrupt( + "clone occurrence symbol key is malformed".to_owned(), + )), + } +} + +/// One row as a page hands it to [`encode_row_blocks`]. +pub(super) struct BlockRowV1<'a> { + pub(super) document_id: i64, + pub(super) chunk_id: &'a str, + pub(super) parent_chunk_id: Option<&'a str>, + pub(super) row: &'a [u8], + pub(super) text: &'a str, +} + +/// One row restored from its block. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct StoredRowV1 { + pub(super) document_id: u32, + pub(super) chunk_id: String, + pub(super) row: Vec, + pub(super) text: String, +} + +/// Split one page's rows (ascending documents) into stored blocks, each +/// keyed by its first document. +pub(super) fn encode_row_blocks( + rows: &[BlockRowV1<'_>], +) -> Result)>, CodeLexicalArtifactErrorV1> { + let mut blocks = Vec::new(); + let mut start = 0; + while start < rows.len() { + let mut end = start; + let mut bytes = 0usize; + while end < rows.len() && end - start < ROW_BLOCK_MAX_ROWS && bytes < ROW_BLOCK_TARGET_BYTES + { + bytes = bytes + .saturating_add(rows[end].row.len()) + .saturating_add(rows[end].text.len()); + end += 1; + } + blocks.push(( + rows[start].document_id, + encode_row_block(&rows[start..end])?, + )); + start = end; + } + Ok(blocks) +} + +fn encode_row_block(rows: &[BlockRowV1<'_>]) -> Result, CodeLexicalArtifactErrorV1> { + // A row may name only a parent whose own text is stored raw. + let candidate = rows + .iter() + .enumerate() + .map(|(index, row)| { + let parent = row.parent_chunk_id?; + rows.iter() + .position(|other| other.chunk_id == parent) + .filter(|&position| { + position != index + && !row.text.is_empty() + && rows[position].text.starts_with(row.text) + }) + }) + .collect::>(); + let mut payload = Vec::new(); + put_varint(&mut payload, length_u64(rows.len())?); + let mut previous: Option = None; + for (index, row) in rows.iter().enumerate() { + // The first row spells its document out, binding the block to its key. + let gap = match previous { + None => u64::try_from(row.document_id) + .map_err(|error| CodeLexicalArtifactErrorV1::Contract(error.to_string()))?, + Some(previous) => row + .document_id + .checked_sub(previous) + .and_then(|delta| delta.checked_sub(1)) + .and_then(|gap| u64::try_from(gap).ok()) + .ok_or_else(|| { + CodeLexicalArtifactErrorV1::Contract( + "lexical artifact row block documents are not ascending".to_owned(), + ) + })?, + }; + previous = Some(row.document_id); + put_varint(&mut payload, gap); + match canonical_chunk_digest(row.chunk_id) { + Some(digest) => { + payload.push(BLOCK_CHUNK_DIGEST); + payload.extend_from_slice(&digest); + } + None => { + payload.push(BLOCK_CHUNK_LITERAL); + put_bytes(&mut payload, row.chunk_id.as_bytes())?; + } + } + put_bytes(&mut payload, row.row)?; + match candidate[index].filter(|parent| candidate[*parent].is_none()) { + Some(parent) => { + payload.push(BLOCK_TEXT_PARENT_PREFIX); + put_varint(&mut payload, length_u64(parent)?); + put_varint(&mut payload, length_u64(row.text.len())?); + } + None => { + payload.push(BLOCK_TEXT_RAW); + put_bytes(&mut payload, row.text.as_bytes())?; + } + } + } + if payload.len() > ROW_BLOCK_MAX_INFLATED_BYTES { + return Err(CodeLexicalArtifactErrorV1::Contract( + "lexical artifact row block exceeds its inflated bound".to_owned(), + )); + } + deflate_bytes(ROW_BLOCK_DEFLATE, &payload) +} + +enum BlockChunkV1 { + Digest(Range), + Literal(Range), +} + +enum BlockTextV1 { + Raw(Range), + ParentPrefix { parent: usize, length: usize }, +} + +struct BlockEntryV1 { + document_id: u32, + chunk: BlockChunkV1, + row: Range, + text: BlockTextV1, +} + +/// One inflated, structurally verified row block. Rows are materialized one +/// at a time, so a sparse reader pays one inflate and one row per visit. +pub(super) struct RowBlockV1 { + payload: Vec, + entries: Vec, +} + +impl RowBlockV1 { + /// Inflate one stored block (bounded by `ROW_BLOCK_MAX_INFLATED_BYTES`) + /// and index its rows, failing closed on any malformed or trailing byte. + pub(super) fn parse( + first_document: i64, + stored: &[u8], + ) -> Result { + let payload = inflate_bytes(ROW_BLOCK_DEFLATE, stored, ROW_BLOCK_MAX_INFLATED_BYTES)?; + let total = payload.len(); + let mut cursor = RowCursorV1 { bytes: &payload }; + let offset = |cursor: &RowCursorV1<'_>| total - cursor.bytes.len(); + let count = usize::try_from(cursor.take_varint()?).map_err(corrupt)?; + if count == 0 || count > ROW_BLOCK_MAX_ROWS { + return Err(CodeLexicalArtifactErrorV1::Corrupt( + "lexical artifact row block row count is out of range".to_owned(), + )); + } + let mut entries = Vec::with_capacity(count); + let mut document = first_document; + for index in 0..count { + let gap = i64::try_from(cursor.take_varint()?).map_err(corrupt)?; + document = if index == 0 { + if gap != first_document { + return Err(CodeLexicalArtifactErrorV1::Corrupt( + "lexical artifact row block does not start at its key".to_owned(), + )); + } + first_document + } else { + document + .checked_add(1) + .and_then(|next| next.checked_add(gap)) + .ok_or_else(|| corrupt("lexical artifact row block document overflowed"))? + }; + let chunk = match cursor.take_u8()? { + BLOCK_CHUNK_DIGEST => { + let start = offset(&cursor); + cursor.take_exact(32)?; + BlockChunkV1::Digest(start..offset(&cursor)) + } + BLOCK_CHUNK_LITERAL => { + let literal = cursor.take_bytes()?; + let end = offset(&cursor); + BlockChunkV1::Literal(end - literal.len()..end) + } + _ => { + return Err(CodeLexicalArtifactErrorV1::Corrupt( + "lexical artifact row block chunk tag is unknown".to_owned(), + )); + } + }; + let row = cursor.take_bytes()?; + let row = offset(&cursor) - row.len()..offset(&cursor); + let text = match cursor.take_u8()? { + BLOCK_TEXT_RAW => { + let text = cursor.take_bytes()?; + let end = offset(&cursor); + BlockTextV1::Raw(end - text.len()..end) + } + BLOCK_TEXT_PARENT_PREFIX => BlockTextV1::ParentPrefix { + parent: usize::try_from(cursor.take_varint()?).map_err(corrupt)?, + length: usize::try_from(cursor.take_varint()?).map_err(corrupt)?, + }, + _ => { + return Err(CodeLexicalArtifactErrorV1::Corrupt( + "lexical artifact row block text tag is unknown".to_owned(), + )); + } + }; + entries.push(BlockEntryV1 { + document_id: u32::try_from(document).map_err(corrupt)?, + chunk, + row, + text, + }); + } + if !cursor.bytes.is_empty() { + return Err(CodeLexicalArtifactErrorV1::Corrupt( + "lexical artifact row block has trailing bytes".to_owned(), + )); + } + for (index, entry) in entries.iter().enumerate() { + if let BlockTextV1::ParentPrefix { parent, length } = entry.text { + let valid = parent != index + && length > 0 + && matches!( + entries.get(parent).map(|parent| &parent.text), + Some(BlockTextV1::Raw(text)) if length <= text.len() + ); + if !valid { + return Err(CodeLexicalArtifactErrorV1::Corrupt( + "lexical artifact row block prefix names no raw parent row".to_owned(), + )); + } + } + } + Ok(Self { payload, entries }) + } + + /// The row stored for `document`, when this block holds it. + pub(super) fn row( + &self, + document: u32, + ) -> Option> { + self.entries + .binary_search_by_key(&document, |entry| entry.document_id) + .ok() + .map(|index| self.materialize(index)) + } + + pub(super) fn rows(&self) -> Result, CodeLexicalArtifactErrorV1> { + (0..self.entries.len()) + .map(|index| self.materialize(index)) + .collect() + } + + fn materialize(&self, index: usize) -> Result { + let entry = &self.entries[index]; + let chunk_id = match &entry.chunk { + BlockChunkV1::Digest(range) => format!( + "{CANONICAL_CHUNK_ID_PREFIX}{}", + hex::encode(&self.payload[range.clone()]) + ), + BlockChunkV1::Literal(range) => { + String::from_utf8(self.payload[range.clone()].to_vec()).map_err(corrupt)? + } + }; + let text = match entry.text { + BlockTextV1::Raw(ref range) => &self.payload[range.clone()], + BlockTextV1::ParentPrefix { parent, length } => match &self.entries[parent].text { + BlockTextV1::Raw(range) => &self.payload[range.start..range.start + length], + BlockTextV1::ParentPrefix { .. } => { + return Err(CodeLexicalArtifactErrorV1::Corrupt( + "lexical artifact row block prefix names no raw parent row".to_owned(), + )); + } + }, + }; + Ok(StoredRowV1 { + document_id: entry.document_id, + chunk_id, + row: self.payload[entry.row.clone()].to_vec(), + text: std::str::from_utf8(text).map_err(corrupt)?.to_owned(), + }) + } +} + +/// Every row of one stored block. +pub(super) fn decode_row_block( + first_document: i64, + stored: &[u8], +) -> Result, CodeLexicalArtifactErrorV1> { + RowBlockV1::parse(first_document, stored)?.rows() +} + +/// The one block holding a document: the greatest block key at or below it. +pub(super) const ROW_BLOCK_BY_DOCUMENT_SQL: &str = "SELECT first_document, payload FROM row_blocks WHERE first_document <= ?1 ORDER BY first_document DESC LIMIT 1"; + +/// Rows by document over an open artifact connection. Callers visit +/// documents in ascending order, so the one inflated block held here serves +/// every document it contains. +pub(super) struct RowBlocksV1<'a> { + connection: &'a Connection, + block: RefCell>, +} + +impl<'a> RowBlocksV1<'a> { + pub(super) fn new(connection: &'a Connection) -> Self { + Self { + connection, + block: RefCell::new(None), + } + } + + pub(super) fn row(&self, document: u32) -> Result { + if let Some(row) = self + .block + .borrow() + .as_ref() + .and_then(|block| block.row(document)) + { + return row; + } + let mut statement = self + .connection + .prepare_cached(ROW_BLOCK_BY_DOCUMENT_SQL) + .map_err(|error| CodeLexicalArtifactErrorV1::Io(error.to_string()))?; + let block: Option<(i64, Vec)> = statement + .query_row([i64::from(document)], |row| Ok((row.get(0)?, row.get(1)?))) + .optional() + .map_err(|error| CodeLexicalArtifactErrorV1::Io(error.to_string()))?; + let (first_document, payload) = block.ok_or_else(missing_document_row)?; + let block = RowBlockV1::parse(first_document, &payload)?; + let row = block.row(document).ok_or_else(missing_document_row)?; + *self.block.borrow_mut() = Some(block); + row + } +} + +fn missing_document_row() -> CodeLexicalArtifactErrorV1 { + CodeLexicalArtifactErrorV1::Corrupt("lexical artifact document has no stored row".to_owned()) +} + +fn canonical_digest(prefix: &str, id: &str) -> Option<[u8; 32]> { + let hex = id.strip_prefix(prefix)?; let decoded: [u8; 32] = hex::decode(hex).ok()?.try_into().ok()?; (hex::encode(decoded) == hex).then_some(decoded) } @@ -789,9 +1041,8 @@ fn put_reference( out: &mut Vec, dictionary: &mut RowDictionaryTableV1, entry: &RowDictionaryEntryV1, - include_vocabulary_fields: bool, ) -> Result<(), CodeLexicalArtifactErrorV1> { - let encoded = entry.encode(include_vocabulary_fields)?; + let encoded = entry.encode()?; let entry_id = stable_row_dictionary_id(&encoded); match dictionary.get(&entry_id) { Some(existing) if *existing != encoded => { @@ -808,6 +1059,20 @@ fn put_reference( Ok(()) } +fn put_optional_string( + out: &mut Vec, + value: Option<&str>, +) -> Result<(), CodeLexicalArtifactErrorV1> { + match value { + None => out.push(OPTIONAL_ABSENT), + Some(value) => { + out.push(OPTIONAL_PRESENT); + put_bytes(out, value.as_bytes())?; + } + } + Ok(()) +} + fn put_bytes(out: &mut Vec, bytes: &[u8]) -> Result<(), CodeLexicalArtifactErrorV1> { put_varint(out, length_u64(bytes.len())?); out.extend_from_slice(bytes); @@ -951,8 +1216,9 @@ mod tests { use std::sync::Arc; use super::{ - ArtifactRowV1, LexicalArtifactLayoutV1, RowDictionaryEntryV1, RowDictionaryTableV1, - RowDictionaryV1, decode_artifact_row, encode_artifact_row, + ArtifactRowV1, BlockRowV1, QualifiedNameV1, ROW_BLOCK_MAX_ROWS, RowDictionaryEntryV1, + RowDictionaryTableV1, RowDictionaryV1, StoredRowV1, decode_artifact_row, decode_row_block, + encode_artifact_row, encode_row_blocks, }; use crate::retrieval::lexical::LexicalFieldV1; use crate::retrieval::lexical::projection::artifact::CodeLexicalArtifactErrorV1; @@ -1109,17 +1375,14 @@ mod tests { row } - fn round_trip( - layout: LexicalArtifactLayoutV1, - row: &ArtifactRowV1, - ) -> (Vec, RowDictionaryTableV1, ArtifactRowV1) { + fn round_trip(row: &ArtifactRowV1) -> (Vec, RowDictionaryTableV1, ArtifactRowV1) { let mut dictionary = RowDictionaryTableV1::new(); - let encoded = encode_artifact_row(layout, row, &mut dictionary).expect("encode"); + let encoded = encode_artifact_row(row, &mut dictionary).expect("encode"); let decoded = decode_artifact_row( - layout, &row.anchor.generation_id, row.id.as_str(), &encoded, + row.sanitized_text.as_str(), &dictionary, ) .expect("decode"); @@ -1127,121 +1390,28 @@ mod tests { } #[test] - fn compact_v11_round_trip_is_byte_equivalent_to_logical_row() { - let row = sample_row(); - let (encoded, dictionary, decoded) = round_trip(LexicalArtifactLayoutV1::V11, &row); - assert!( - encoded.starts_with(b"TDLR11\0"), - "v11 rows must carry the compact codec tag" - ); - assert!(dictionary.is_empty(), "v11 rows carry their strings inline"); - assert_eq!(decoded, row); - assert!( - encoded.len() < serde_json::to_vec(&row).expect("json").len(), - "compact rows must drop repeated identities" - ); - } - - #[test] - fn compact_decoder_fails_closed_without_the_v11_tag() { - let row = sample_row(); - let json = serde_json::to_vec(&row).expect("json"); - let error = decode_artifact_row( - LexicalArtifactLayoutV1::V11, - &row.anchor.generation_id, - row.id.as_str(), - &json, - &RowDictionaryTableV1::new(), - ) - .expect_err("untagged JSON is not a v11 row"); - assert!(error.to_string().contains("compact v11")); - } - - #[test] - fn binary_v14_and_v15_emit_current_row_codec() { - for layout in [LexicalArtifactLayoutV1::V14, LexicalArtifactLayoutV1::V15] { - for row in [ - sample_row(), - window_row(), - legacy_symbol_row(), - symbol_row(), - ] { - let (encoded, _, decoded) = round_trip(layout, &row); - assert!( - encoded.starts_with(b"TDLR16\0"), - "admission bases must emit the current row codec" - ); - assert_eq!(decoded, row); - } - } - } - - #[test] - fn binary_layouts_restore_legacy_one_byte_bitmap_rows() { - for layout in [ - LexicalArtifactLayoutV1::V14, - LexicalArtifactLayoutV1::V15, - LexicalArtifactLayoutV1::V16, + fn binary_rows_round_trip_without_their_text() { + for row in [ + sample_row(), + window_row(), + legacy_symbol_row(), + symbol_row(), ] { - for row in [sample_row(), window_row(), legacy_symbol_row()] { - let mut dictionary = RowDictionaryTableV1::new(); - let encoded = super::encode_binary( - &row, - &mut dictionary, - super::ROW_CODEC_V14_MAGIC, - &super::LEGACY_FIELD_LENGTH_ORDER, - false, - ) - .expect("encode legacy"); - assert!(encoded.starts_with(b"TDLR14\0")); - let decoded = decode_artifact_row( - layout, - &row.anchor.generation_id, - row.id.as_str(), - &encoded, - &dictionary, - ) - .expect("decode legacy under current layout"); - assert_eq!(decoded, row); - } + let (encoded, _, decoded) = round_trip(&row); + assert_eq!(decoded, row); + assert!( + !encoded + .windows(row.sanitized_text.as_str().len()) + .any(|window| window == row.sanitized_text.as_str().as_bytes()), + "the row block, not the row, stores the text" + ); } } - #[test] - fn binary_v16_round_trips_signature_and_documentation_fields() { - let row = symbol_row(); - let (encoded, _, decoded) = round_trip(LexicalArtifactLayoutV1::V16, &row); - assert!(encoded.starts_with(b"TDLR16\0")); - assert_eq!(decoded, row); - } - - #[test] - fn binary_v16_layout_reads_v14_admission_rows_after_successor_label() { - let row = symbol_row(); - let (encoded, dictionary, _) = round_trip(LexicalArtifactLayoutV1::V14, &row); - assert!(encoded.starts_with(b"TDLR16\0")); - let decoded = decode_artifact_row( - LexicalArtifactLayoutV1::V16, - &row.anchor.generation_id, - row.id.as_str(), - &encoded, - &dictionary, - ) - .expect("V16 layout must read current-codec admission rows"); - assert_eq!(decoded, row); - } - #[test] fn binary_v14_references_one_file_and_one_symbol_entry_per_row() { let row = legacy_symbol_row(); - let (encoded, dictionary, _) = round_trip(LexicalArtifactLayoutV1::V14, &row); - let (compact, _, _) = round_trip(LexicalArtifactLayoutV1::V13, &row); - assert!( - encoded.len() * 4 < compact.len(), - "v14 row {} bytes must be under a quarter of the {} byte v11 payload", - encoded.len(), - compact.len() - ); + let (encoded, dictionary, _) = round_trip(&row); let entries = dictionary .values() .map(|bytes| RowDictionaryEntryV1::decode(bytes).expect("entry")) @@ -1260,7 +1430,10 @@ mod tests { .as_ref() .map(|id| id.as_str().to_owned()), simple_name: row.symbol_simple_name.clone(), - qualified_name: row.symbol_qualified_name.clone(), + qualified_name: row + .symbol_qualified_name + .as_deref() + .map(|name| QualifiedNameV1::for_path(name, &row.logical_path)), kind: row.symbol_kind.clone(), signature: row.symbol_signature.clone(), documentation: row.symbol_documentation.clone(), @@ -1273,7 +1446,7 @@ mod tests { .any(|window| window == parent_hex.as_bytes()), "a canonical parent id is stored as digest bytes, not hex" ); - let (_, window_dictionary, _) = round_trip(LexicalArtifactLayoutV1::V14, &window_row()); + let (_, window_dictionary, _) = round_trip(&window_row()); assert_eq!( window_dictionary.len(), 1, @@ -1281,6 +1454,139 @@ mod tests { ); } + fn block_row<'a>( + document_id: i64, + chunk_id: &'a str, + parent_chunk_id: Option<&'a str>, + text: &'a str, + ) -> BlockRowV1<'a> { + BlockRowV1 { + document_id, + chunk_id, + parent_chunk_id, + row: b"meta", + text, + } + } + + #[test] + fn row_blocks_round_trip_share_parent_prefixes_and_bound_their_rows() { + let body_id = format!("chunk.v1.sha256:{}", "ab".repeat(32)); + let body = "pub fn render(widget: &Widget) -> Frame {\n widget.frame()\n}".repeat(20); + let signature = "pub fn render(widget: &Widget) -> Frame {"; + let texts = (0..40) + .map(|ordinal| format!("let value_{ordinal} = compute(value);\n").repeat(8)) + .collect::>(); + let chunk_ids = (0..40) + .map(|ordinal| format!("chunk.{ordinal}")) + .collect::>(); + let mut rows = vec![ + block_row(10, "chunk.signature", Some(&body_id), signature), + block_row(11, &body_id, None, &body), + block_row(13, "chunk.unrelated", Some("chunk.absent"), "fn other() {}"), + ]; + rows.extend( + texts + .iter() + .zip(&chunk_ids) + .enumerate() + .map(|(ordinal, (text, chunk_id))| { + block_row(14 + ordinal as i64, chunk_id, None, text) + }), + ); + let blocks = encode_row_blocks(&rows).expect("encode blocks"); + assert_eq!(blocks[0].0, 10, "a block is keyed by its first document"); + let decoded = blocks + .iter() + .flat_map(|(first, stored)| { + let rows = decode_row_block(*first, stored).expect("decode block"); + assert!(rows.len() <= ROW_BLOCK_MAX_ROWS); + rows + }) + .collect::>(); + let expected = rows + .iter() + .map(|row| StoredRowV1 { + document_id: u32::try_from(row.document_id).expect("document"), + chunk_id: row.chunk_id.to_owned(), + row: row.row.to_vec(), + text: row.text.to_owned(), + }) + .collect::>(); + assert_eq!(decoded, expected); + let raw_text_bytes = rows.iter().map(|row| row.text.len()).sum::(); + let stored_bytes = blocks.iter().map(|(_, stored)| stored.len()).sum::(); + assert!( + stored_bytes * 8 < raw_text_bytes, + "neighbouring rows must deflate together: {stored_bytes} of {raw_text_bytes} bytes" + ); + + let (first, stored) = &blocks[0]; + let mut damaged = stored.clone(); + let tail = damaged.len() - 4; + damaged[tail] ^= 0xff; + assert!( + decode_row_block(*first, &damaged).is_err(), + "damaged stream" + ); + assert!( + decode_row_block(first + 1, stored).is_err(), + "wrong block key" + ); + let mut trailing = stored.clone(); + trailing.push(0); + assert!( + decode_row_block(*first, &trailing).is_err(), + "trailing byte" + ); + assert!( + encode_row_blocks(&[block_row(5, "a", None, "x"), block_row(5, "b", None, "y")]) + .is_err(), + "documents must ascend" + ); + } + + #[test] + fn symbol_entries_store_file_relative_names_and_digest_identities() { + let row = symbol_row(); + let (_, dictionary, decoded) = round_trip(&row); + assert_eq!(decoded, row); + let symbol = dictionary + .values() + .find(|bytes| bytes.first() == Some(&super::ENTRY_SYMBOL)) + .expect("symbol entry"); + assert!( + !symbol + .windows(row.logical_path.len()) + .any(|window| window == row.logical_path.as_bytes()), + "the qualified name must not repeat the file's logical path" + ); + assert!( + !symbol + .windows(b"symbol.v1.sha256:".len()) + .any(|window| window == b"symbol.v1.sha256:"), + "a canonical symbol id is stored as digest bytes, not hex" + ); + + let mut elsewhere = symbol_row(); + elsewhere.symbol_qualified_name = Some("other/file.rs::probe".to_owned()); + elsewhere.anchor.symbol_occurrence_id = + Some(SymbolOccurrenceId::new("symbol.literal").expect("literal symbol")); + let (_, _, decoded) = round_trip(&elsewhere); + assert_eq!( + decoded, elsewhere, + "names outside the row's file and non-canonical ids survive verbatim" + ); + assert_eq!( + QualifiedNameV1::for_path("src/a.rs::f", "src/a.rs"), + QualifiedNameV1::InFile("f".to_owned()) + ); + assert_eq!( + QualifiedNameV1::for_path("src/a.rsx::f", "src/a.rs"), + QualifiedNameV1::Literal("src/a.rsx::f".to_owned()) + ); + } + #[test] fn binary_v14_keeps_non_canonical_parents_and_foreign_term_symbols_verbatim() { let mut row = legacy_symbol_row(); @@ -1293,7 +1599,7 @@ mod tests { foreign.clone(), ) .expect("foreign whole symbol"); - let (_, dictionary, decoded) = round_trip(LexicalArtifactLayoutV1::V14, &row); + let (_, dictionary, decoded) = round_trip(&row); assert_eq!(decoded, row); assert_eq!( dictionary.len(), @@ -1302,7 +1608,7 @@ mod tests { ); let uppercase_hex = format!("chunk.v1.sha256:{}", "C0".repeat(32)); row.anchor.parent_chunk_id = Some(CodeSearchChunkId::new(uppercase_hex).expect("upper")); - let (_, _, decoded) = round_trip(LexicalArtifactLayoutV1::V14, &row); + let (_, _, decoded) = round_trip(&row); assert_eq!( decoded, row, "uppercase hex is not canonical and must survive verbatim" @@ -1312,17 +1618,17 @@ mod tests { #[test] fn binary_v14_decoder_fails_closed_on_truncation_trailing_bytes_and_missing_entries() { let row = legacy_symbol_row(); - let (encoded, dictionary, _) = round_trip(LexicalArtifactLayoutV1::V14, &row); + let (encoded, dictionary, _) = round_trip(&row); let decode = |bytes: &[u8], dictionary: &RowDictionaryTableV1| { decode_artifact_row( - LexicalArtifactLayoutV1::V14, &row.anchor.generation_id, row.id.as_str(), bytes, + row.sanitized_text.as_str(), dictionary, ) }; - for cut in [7usize, 8, 20, 40, encoded.len() - 1] { + for cut in [0usize, 1, 8, 20, 40, encoded.len() - 1] { assert!( decode(&encoded[..cut], &dictionary).is_err(), "truncated at {cut}" @@ -1331,11 +1637,6 @@ mod tests { let mut trailing = encoded.clone(); trailing.push(0); assert!(decode(&trailing, &dictionary).is_err(), "trailing byte"); - let (compact, _, _) = round_trip(LexicalArtifactLayoutV1::V13, &row); - assert!( - decode(&compact, &dictionary).is_err(), - "v11 payload under v14 layout" - ); let mut missing = dictionary.clone(); let file_id = *missing .iter() diff --git a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/schema.rs b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/schema.rs index e6409d8cfe..725c35d318 100644 --- a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/schema.rs +++ b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/schema.rs @@ -1,6 +1,4 @@ -use std::collections::{BTreeMap, BTreeSet, HashSet}; - -use rusqlite::{Connection, OptionalExtension, Transaction, params}; +use rusqlite::{Transaction, params}; use sha2::{Digest, Sha256}; use super::super::LexicalFieldV1; @@ -9,40 +7,29 @@ use super::{CodeLexicalArtifactErrorV1, checkpoint}; use tracedecay_code_index::production::CodeIndexExecutionControlV1; use tracedecay_domain::{ExactFieldV1, nonnegative_sha256_prefix}; -/// Revision 10 is the last TEXT-term posting layout. Revision 11 interns -/// terms, stores integer field codes, drops redundant serving indexes, and -/// writes compact row payloads. Revision 12 delta-encodes n-gram document -/// lists and interns exact terms. Revision 13 clusters `term_postings` by -/// `(document_id, term_id, field)` so every batch appends to the tail of the -/// tree instead of rewriting leaves across the whole hashed term-id space, -/// and derives the term-leading serving index once at finalization. -/// Revision 14 replaces the JSON row payload with a binary one whose -/// per-file and per-symbol strings (paths, occurrence identities, display -/// names, descriptor revisions) are interned once as `row_dictionary` -/// entries and referenced by content-addressed id. Revision 15 adds -/// content-addressed clone payloads, source-bound occurrences, and exact -/// conservative/rename postings. Revision 16 adds signature and documentation -/// fields without changing the shipped revision-14/15 row codec, positional -/// winnowed fingerprint postings, and stored posting-list counts. Readers -/// accept all shipped layouts; writers emit 16 unless an explicit benchmark -/// revision is selected. -pub(super) const CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V10: u32 = 10; -pub(super) const CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V11: u32 = 11; -pub(super) const CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V12: u32 = 12; -pub(super) const CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V13: u32 = 13; -pub(super) const CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V14: u32 = 14; -pub(super) const CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V15: u32 = 15; -pub(super) const CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V16: u32 = 16; -pub(super) const CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V1: u32 = - CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V16; - -const DIGEST_DOMAIN_V10: &[u8] = b"tracedecay.code-lexical-artifact.v10\0"; -const DIGEST_DOMAIN_V11: &[u8] = b"tracedecay.code-lexical-artifact.v11\0"; -const DIGEST_DOMAIN_V12: &[u8] = b"tracedecay.code-lexical-artifact.v12\0"; -const DIGEST_DOMAIN_V13: &[u8] = b"tracedecay.code-lexical-artifact.v13\0"; -const DIGEST_DOMAIN_V14: &[u8] = b"tracedecay.code-lexical-artifact.v14\0"; -const DIGEST_DOMAIN_V15: &[u8] = b"tracedecay.code-lexical-artifact.v15\0"; -const DIGEST_DOMAIN_V16: &[u8] = b"tracedecay.code-lexical-artifact.v16\0"; +/// Revision 26 is the only layout this build serves: interned exact terms, +/// integer field codes, rows stored as deflated blocks of consecutive +/// documents (per-file and per-symbol strings interned once as +/// `row_dictionary` entries, a signature chunk's text stored as the prefix +/// it shares with its body chunk), one `term_postings` row per term text +/// carrying every field's delta-varint list, one `exact_postings` list per +/// exact term and field, one `ngram_postings` list per n-gram rebuilt from +/// the stored rows (the case-preserving kind holds only windows with an ASCII +/// uppercase byte; every other raw window is its normalized window), and the clone index (binary payloads keyed by 32-byte +/// digest, content-only occurrences keyed by symbol digest, and exact and +/// positional winnowed fingerprint postings, each naming its payload or +/// occurrence by integer ordinal) sealed by the same build. Batches append +/// page-ordered staging that finalization merges and drops, so no secondary +/// index duplicates a posting. Annotation uses mint no document. The sealed +/// file holds content only: route identity (generation, repository, +/// freshness, clone occurrence project/worktree/snapshot) and the sealed +/// source's resume cursors are supplied by the opener or dropped before the +/// seal, so identical trees in different worktrees seal byte-identical +/// files. Every other revision is refused as incompatible and rebuilt from +/// the sealed generation. +pub(super) const CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V1: u32 = 26; + +const DIGEST_DOMAIN: &[u8] = b"tracedecay.code-lexical-artifact.v26\0"; const FIELD_SYMBOL_NAME: i64 = 1; const FIELD_QUALIFIED_NAME: i64 = 2; @@ -54,234 +41,27 @@ const FIELD_SUBTOKEN: i64 = 7; const FIELD_SIGNATURE: i64 = 8; const FIELD_DOCUMENTATION: i64 = 9; -pub(super) const REQUIRED_ARTIFACT_INDEXES_V10: [(&str, &str, &[&str]); 7] = [ - ("rows", "rows_by_chunk", &["chunk_id"]), - ( - "term_postings", - "term_postings_by_term", - &["term", "field", "document_id"], - ), - ( - "term_postings", - "term_postings_by_document", - &["document_id", "field", "term", "frequency"], - ), - ( - "term_postings", - "term_postings_by_document_term", - &["document_id", "term", "field", "frequency"], - ), - ("term_stats", "term_stats_by_term", &["term", "field"]), - ( - "exact_postings", - "exact_postings_by_document", - &["document_id", "field", "term"], - ), - ( - "ngram_postings", - "ngram_postings_by_ngram", - &["kind", "ngram", "page_ordinal", "cardinality"], - ), -]; - -/// Serving indexes retained after EXPLAIN QUERY PLAN on the live read -/// shapes: chunk lookup, one document-leading posting probe, exact -/// document membership, and n-gram page shards. The revision-10 -/// term-leading and duplicate document-term indexes are covered by the -/// interned primary key `(term_id, field, document_id)`. -pub(super) const REQUIRED_ARTIFACT_INDEXES_V11: [(&str, &str, &[&str]); 4] = [ - ("rows", "rows_by_chunk", &["chunk_id"]), - ( - "term_postings", - "term_postings_by_document", - &["document_id", "term_id", "field", "frequency"], - ), - ( - "exact_postings", - "exact_postings_by_document", - &["document_id", "field", "term"], - ), - ( - "ngram_postings", - "ngram_postings_by_ngram", - &["kind", "ngram", "page_ordinal", "cardinality"], - ), -]; - -pub(super) const REQUIRED_ARTIFACT_INDEXES_V12: [(&str, &str, &[&str]); 4] = [ - ("rows", "rows_by_chunk", &["chunk_id"]), - ( - "term_postings", - "term_postings_by_document", - &["document_id", "term_id", "field", "frequency"], - ), - ( - "exact_postings", - "exact_postings_by_document", - &["document_id", "field", "term_id"], - ), - ( - "ngram_postings", - "ngram_postings_by_ngram", - &["kind", "ngram", "page_ordinal", "cardinality"], - ), -]; - -/// Revision 13 keeps the interned exact layout of revision 12 but clusters -/// `term_postings` by document, so the term-leading probe index replaces the -/// document-leading one. -pub(super) const REQUIRED_ARTIFACT_INDEXES_V13: [(&str, &str, &[&str]); 4] = [ - ("rows", "rows_by_chunk", &["chunk_id"]), - ( - "term_postings", - "term_postings_by_term", - &["term_id", "field", "document_id", "frequency"], - ), - ( - "exact_postings", - "exact_postings_by_document", - &["document_id", "field", "term_id"], - ), - ( - "ngram_postings", - "ngram_postings_by_ngram", - &["kind", "ngram", "page_ordinal", "cardinality"], - ), -]; - -/// Statistics wakes stay at three steps (field, term, fuzzy flag). Index -/// wakes are the four serving indexes plus n-gram selectivity. -pub(super) const STATISTICS_STEP_COUNT_V11: u64 = 3; -pub(super) const SERVING_INDEX_STEP_COUNT_V11: u64 = 5; - -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub enum CodeLexicalArtifactWriterRevisionV1 { - V11, - V12, - V13, - V14, - V15, - #[default] - V16, -} - -impl CodeLexicalArtifactWriterRevisionV1 { - pub(super) const fn layout(self) -> LexicalArtifactLayoutV1 { - match self { - Self::V11 => LexicalArtifactLayoutV1::V11, - Self::V12 => LexicalArtifactLayoutV1::V12, - Self::V13 => LexicalArtifactLayoutV1::V13, - Self::V14 => LexicalArtifactLayoutV1::V14, - Self::V15 => LexicalArtifactLayoutV1::V15, - Self::V16 => LexicalArtifactLayoutV1::V16, - } - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(super) enum LexicalArtifactLayoutV1 { - V10, - V11, - V12, - V13, - V14, - V15, - V16, -} - -impl LexicalArtifactLayoutV1 { - pub(super) fn from_revision(revision: u32) -> Result { - match revision { - CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V10 => Ok(Self::V10), - CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V11 => Ok(Self::V11), - CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V12 => Ok(Self::V12), - CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V13 => Ok(Self::V13), - CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V14 => Ok(Self::V14), - CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V15 => Ok(Self::V15), - CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V16 => Ok(Self::V16), - _ => Err(CodeLexicalArtifactErrorV1::Incompatible(format!( - "format revision {revision} is unsupported" - ))), - } - } - - pub(super) fn revision(self) -> u32 { - match self { - Self::V10 => CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V10, - Self::V11 => CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V11, - Self::V12 => CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V12, - Self::V13 => CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V13, - Self::V14 => CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V14, - Self::V15 => CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V15, - Self::V16 => CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V16, - } - } - - pub(super) fn digest_domain(self) -> &'static [u8] { - match self { - Self::V10 => DIGEST_DOMAIN_V10, - Self::V11 => DIGEST_DOMAIN_V11, - Self::V12 => DIGEST_DOMAIN_V12, - Self::V13 => DIGEST_DOMAIN_V13, - Self::V14 => DIGEST_DOMAIN_V14, - Self::V15 => DIGEST_DOMAIN_V15, - Self::V16 => DIGEST_DOMAIN_V16, - } - } - - pub(super) fn required_indexes( - self, - ) -> &'static [(&'static str, &'static str, &'static [&'static str])] { - match self { - Self::V10 => &REQUIRED_ARTIFACT_INDEXES_V10, - Self::V11 => &REQUIRED_ARTIFACT_INDEXES_V11, - Self::V12 => &REQUIRED_ARTIFACT_INDEXES_V12, - // Revision 14 changes only the row payload and its string - // dictionary; the serving indexes are revision 13's. - Self::V13 | Self::V14 | Self::V15 | Self::V16 => &REQUIRED_ARTIFACT_INDEXES_V13, - } - } - - /// Revisions 12 and later intern exact terms through `exact_vocabulary`. - pub(super) fn interns_exact_terms(self) -> bool { - matches!( - self, - Self::V12 | Self::V13 | Self::V14 | Self::V15 | Self::V16 - ) - } - - /// Revisions 13 and later cluster `term_postings` by `(document_id, - /// term_id, field)`; every earlier interned layout clusters by term. - pub(super) fn clusters_term_postings_by_document(self) -> bool { - matches!(self, Self::V13 | Self::V14 | Self::V15 | Self::V16) - } - - /// Revision 14 rows reference `row_dictionary` entries for their per-file - /// and per-symbol strings instead of carrying the text per chunk. - pub(super) fn interns_row_dictionary(self) -> bool { - matches!(self, Self::V14 | Self::V15 | Self::V16) - } - - /// Revision 14 keeps `document_integrity` as `(document_id, digest - /// BLOB)`: the chunk id already lives in `rows` under the same key, and - /// the 32 digest bytes replace their 71-byte tagged hex form. - pub(super) fn stores_document_integrity_bytes(self) -> bool { - matches!(self, Self::V14 | Self::V15 | Self::V16) - } - - pub(super) fn has_clone_index(self) -> bool { - matches!(self, Self::V15 | Self::V16) - } - - pub(super) fn has_clone_fingerprints(self) -> bool { - self == Self::V16 +/// Index wakes: row dictionary with the chunk lookup table, then the three +/// posting merges. Statistics wakes: field totals, then releasing every +/// dropped staging page. +pub(super) const STATISTICS_STEP_COUNT_V11: u64 = 2; +pub(super) const SERVING_INDEX_STEP_COUNT_V11: u64 = 4; + +pub(super) fn require_served_revision(revision: u32) -> Result<(), CodeLexicalArtifactErrorV1> { + if revision == CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V1 { + Ok(()) + } else { + Err(CodeLexicalArtifactErrorV1::Incompatible(format!( + "format revision {revision} is unsupported" + ))) } } pub(super) fn digest_domain_for_revision( revision: u32, ) -> Result<&'static [u8], CodeLexicalArtifactErrorV1> { - Ok(LexicalArtifactLayoutV1::from_revision(revision)?.digest_domain()) + require_served_revision(revision)?; + Ok(DIGEST_DOMAIN) } pub(super) fn field_code(field: LexicalFieldV1) -> i64 { @@ -347,16 +127,6 @@ pub(super) fn exact_field_code_from_encoded( Ok(exact_field_code(field)) } -/// Content-addressed term primary key. Incrementing IDs follow first-seen -/// batch order, so one-page and multi-page commits of the same source would -/// disagree on `vocabulary` / `term_stats` section receipts. -pub(super) fn stable_term_id(term: &str) -> i64 { - stable_prefixed_id( - b"tracedecay.code-lexical-artifact.term-id.v11\0", - term.as_bytes(), - ) -} - pub(super) fn stable_exact_term_id(term: &[u8]) -> i64 { stable_prefixed_id( b"tracedecay.code-lexical-artifact.exact-term-id.v12\0", @@ -384,7 +154,7 @@ pub(super) fn stable_row_dictionary_id(entry: &[u8]) -> i64 { /// Stage every dictionary entry the batch references under its page ordinal. /// `row_dictionary_pages` is clustered by `(page_ordinal, entry_id)`, so a -/// batch appends at the tail like every other revision-13 base table; a +/// batch appends at the tail like every other base table; a /// hash-keyed insert straight into `row_dictionary` would dirty most of that /// tree on every batch (measured: +540 MiB of journal and page rewrites over /// 37 batches on a 24 MB dictionary). Finalization derives the deduplicated @@ -479,94 +249,15 @@ pub(super) fn intern_exact_terms( Ok(()) } -/// Intern the batch's distinct terms and return the ids now present in -/// `vocabulary`, so the posting writer can confirm every planned posting's -/// term was interned with one integer probe per row. `terms` is ascending by -/// term text, the order `vocabulary` was always interned in, and carries -/// the ids the insert plan already content-addressed, so neither the digest -/// nor the walk over every posting is repeated here. -pub(super) fn intern_terms( - transaction: &Transaction<'_>, - terms: &[(&str, i64)], - control: &dyn CodeIndexExecutionControlV1, -) -> Result, CodeLexicalArtifactErrorV1> { - let mut assigned = HashSet::with_capacity(terms.len()); - let mut insert = transaction - .prepare_cached( - "INSERT INTO vocabulary(term_id, term, in_fuzzy) VALUES (?1, ?2, 0) ON CONFLICT(term) DO NOTHING", - ) - .map_err(|error| CodeLexicalArtifactErrorV1::Io(error.to_string()))?; - for (term, term_id) in terms { - checkpoint(control)?; - insert.execute(params![term_id, term]).map_err(|error| { - CodeLexicalArtifactErrorV1::Contract(format!( - "lexical artifact term identifier collided or vocabulary insert failed: {error}" - )) - })?; - assigned.insert(*term_id); - } - Ok(assigned) -} - -pub(super) fn lookup_term_id( - connection: &Connection, - term: &str, -) -> Result, CodeLexicalArtifactErrorV1> { - connection - .query_row( - "SELECT term_id FROM vocabulary WHERE term = ?1", - [term], - |row| row.get(0), - ) - .optional() - .map_err(|error| CodeLexicalArtifactErrorV1::Io(error.to_string())) -} - -pub(super) fn lookup_term_ids( - connection: &Connection, - terms: &BTreeSet, -) -> Result, CodeLexicalArtifactErrorV1> { - let mut assigned = BTreeMap::new(); - if terms.is_empty() { - return Ok(assigned); - } - let placeholders = std::iter::repeat_n("?", terms.len()) - .collect::>() - .join(", "); - let sql = format!("SELECT term, term_id FROM vocabulary WHERE term IN ({placeholders})"); - let mut statement = connection - .prepare(&sql) - .map_err(|error| CodeLexicalArtifactErrorV1::Io(error.to_string()))?; - let mut rows = statement - .query(rusqlite::params_from_iter(terms.iter())) - .map_err(|error| CodeLexicalArtifactErrorV1::Io(error.to_string()))?; - while let Some(row) = rows - .next() - .map_err(|error| CodeLexicalArtifactErrorV1::Io(error.to_string()))? - { - assigned.insert( - row.get(0) - .map_err(|error| CodeLexicalArtifactErrorV1::Io(error.to_string()))?, - row.get(1) - .map_err(|error| CodeLexicalArtifactErrorV1::Io(error.to_string()))?, - ); - } - Ok(assigned) -} - #[cfg(test)] mod tests { use super::{ - CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V10, CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V11, - CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V12, CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V13, - CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V14, CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V15, - CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V16, CodeLexicalArtifactErrorV1, - LexicalArtifactLayoutV1, exact_field_code, field_code, field_from_code, + CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V1, CodeLexicalArtifactErrorV1, field_code, + field_from_code, require_served_revision, }; use crate::retrieval::lexical::LexicalFieldV1; use rusqlite::Connection; use tracedecay_code_index::production::CodeIndexExecutionControlV1; - use tracedecay_domain::ExactFieldV1; struct ActiveControl; @@ -606,66 +297,34 @@ mod tests { } #[test] - fn layout_accepts_open_revisions_and_fails_closed_otherwise() { - assert_eq!( - LexicalArtifactLayoutV1::from_revision(CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V10) - .expect("v10"), - LexicalArtifactLayoutV1::V10 - ); - assert_eq!( - LexicalArtifactLayoutV1::from_revision(CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V11) - .expect("v11"), - LexicalArtifactLayoutV1::V11 - ); - assert_eq!( - LexicalArtifactLayoutV1::from_revision(CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V12) - .expect("v12"), - LexicalArtifactLayoutV1::V12 - ); - assert_eq!( - LexicalArtifactLayoutV1::from_revision(CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V13) - .expect("v13"), - LexicalArtifactLayoutV1::V13 - ); - assert_eq!( - LexicalArtifactLayoutV1::from_revision(CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V14) - .expect("v14"), - LexicalArtifactLayoutV1::V14 - ); - assert_eq!( - LexicalArtifactLayoutV1::from_revision(CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V15) - .expect("v15"), - LexicalArtifactLayoutV1::V15 - ); - assert_eq!( - LexicalArtifactLayoutV1::from_revision(CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V16) - .expect("v16"), - LexicalArtifactLayoutV1::V16 - ); - assert!(LexicalArtifactLayoutV1::from_revision(9).is_err()); - assert!(LexicalArtifactLayoutV1::from_revision(17).is_err()); + fn superseded_revisions_are_rejected() { + require_served_revision(CODE_LEXICAL_ARTIFACT_FORMAT_REVISION_V1) + .expect("the served revision opens"); + for revision in [16, 20, 22, 25, 27] { + assert!(matches!( + require_served_revision(revision), + Err(CodeLexicalArtifactErrorV1::Incompatible(message)) + if message == format!("format revision {revision} is unsupported") + )); + } } #[test] - fn row_dictionary_ids_are_deterministic_and_distinct_from_term_ids() { - assert_eq!( - super::stable_row_dictionary_id(b"src/lib.rs"), - super::stable_row_dictionary_id(b"src/lib.rs") - ); + fn row_dictionary_ids_are_distinct_from_exact_term_ids() { assert_ne!( super::stable_row_dictionary_id(b"src/lib.rs"), super::stable_row_dictionary_id(b"src/lib.rs::main") ); assert_ne!( super::stable_row_dictionary_id(b"return"), - super::stable_term_id("return"), - "dictionary entries and vocabulary terms hash under different domains" + super::stable_exact_term_id(b"return"), + "dictionary entries and exact terms hash under different domains" ); assert!(super::stable_row_dictionary_id(b"src/lib.rs") >= 0); } #[test] - fn field_codes_are_stable_and_bijective() { + fn field_codes_are_bijective() { for field in [ LexicalFieldV1::SymbolName, LexicalFieldV1::QualifiedName, @@ -682,27 +341,5 @@ mod tests { } assert!(field_from_code(0).is_err()); assert!(field_from_code(99).is_err()); - assert_eq!(field_code(LexicalFieldV1::Subtoken), 7); - } - - #[test] - fn exact_field_codes_are_stable() { - for (field, code) in [ - (ExactFieldV1::Identifier, 1), - (ExactFieldV1::QualifiedName, 2), - (ExactFieldV1::Path, 3), - (ExactFieldV1::QuotedPhrase, 4), - (ExactFieldV1::DiagnosticCode, 5), - (ExactFieldV1::DiagnosticText, 6), - (ExactFieldV1::CompilerOrRuntimeError, 7), - (ExactFieldV1::CliFlag, 8), - (ExactFieldV1::ToolName, 9), - (ExactFieldV1::ConfigurationKey, 10), - (ExactFieldV1::CommitIdentifier, 11), - (ExactFieldV1::TaskOrSessionId, 12), - (ExactFieldV1::ProtocolField, 13), - ] { - assert_eq!(exact_field_code(field), code); - } } } diff --git a/crates/tracedecay-query/src/retrieval/lexical/projection/in_memory.rs b/crates/tracedecay-query/src/retrieval/lexical/projection/in_memory.rs deleted file mode 100644 index 7b6afc811b..0000000000 --- a/crates/tracedecay-query/src/retrieval/lexical/projection/in_memory.rs +++ /dev/null @@ -1,1267 +0,0 @@ -//! In-memory lexical projection over one generation's chunks. -//! -//! Evaluation-only: the search-quality evaluator and the query suites build -//! this adapter directly from admitted chunks. Production retrieval reads the -//! durable lexical artifact through [`super::CodeLexicalArtifactReaderV1`]. - -use std::collections::{BTreeMap, BTreeSet}; -use std::sync::Arc; -use std::time::{Duration, Instant}; - -use roaring::RoaringBitmap; -use tracedecay_code_index::production::VerifiedSealedLexicalSymbolDisplayV1; -use tracedecay_domain::{ - CodeGenerationId, CodeSearchChunkV1, ExactFieldV1, ExactTechnicalTermV1, - ExtractionAdmittedChunkV1, FreshnessCompatibilityV1, RetrieverBatch, RetrieverCoverage, - RetrieverKind, RetrieverOutcome, SymbolOccurrenceId, -}; - -use super::super::{ - LexicalFieldV1, LexicalLaneEvidence, LexicalLaneRequest, MAX_FUZZY_TERM_EXPANSIONS_V1, - admit_candidate_sources, candidate_admission_outcome, -}; -use super::{ - CodeLexicalProjectionMetadataV1, ExactMatchRowViewV1, FuzzyExpansionsV1, FuzzyQueryGroupV1, - LexicalRowScoreV1, LiteralProofCacheV1, PreparedLexicalQueryV1, ProjectedChunkV1, - bm25_score_micros, canonical_projected_exact_term, exact_field_for_kind, exact_matches, - field_weight_millis, fuzzy_distance_bound, lexical_lane_binding, lexical_lane_candidate, - matches_phrase, normalize_lexical, normalized_search_text, score_lexical_row, -}; -use crate::retrieval::exact::{ExactAdmissionAuthority, ExactLaneEvidence, ExactLaneRequest}; -use crate::retrieval::ports::{ - ExactTermPostingReadPort, LexicalPostingReadPort, RETRIEVAL_CANDIDATE_BATCH_SIZE, - RetrievalPortError, contract_error, retrieval_checkpoint, -}; - -mod postings; - -use postings::{ByteNgramBudget, ByteNgramPostings, FuzzyTermIndex}; - -const BYTE_NGRAM_POSTINGS_MEMORY_BUDGET_BYTES_V1: usize = 512 * 1024 * 1024; - -/// Wall-clock bound for materializing one lexical generation's postings. -/// First-query `new` / `new_admitted` is O(store); a missing caller deadline -/// must not let that build run unbounded on the daemon query path. -pub const LEXICAL_PROJECTION_BUILD_DEADLINE_MICROS_V1: u64 = 30_000_000; - -/// A set `deadline_micros`, including `Some(0)`, is used as-is. `None` uses the -/// crate 30s fallback. This is not request-over-profile: a caller that has both -/// a lane and a base deadline must pass the tighter value. -pub fn lexical_projection_build_deadline_micros(request_deadline_micros: Option) -> u64 { - request_deadline_micros.unwrap_or(LEXICAL_PROJECTION_BUILD_DEADLINE_MICROS_V1) -} - -fn map_postings_build_error(error: String) -> RetrievalPortError { - if error == postings::LEXICAL_PROJECTION_BUILD_DEADLINE_EXCEEDED - || error.starts_with(postings::LEXICAL_PROJECTION_NGRAM_MEMORY_BUDGET_EXCEEDED) - { - RetrievalPortError::BudgetExceeded - } else { - RetrievalPortError::Contract(error) - } -} - -fn check_projection_build_deadline(deadline: Instant) -> Result<(), RetrievalPortError> { - if Instant::now() >= deadline { - Err(RetrievalPortError::BudgetExceeded) - } else { - Ok(()) - } -} - -/// Immutable adapter over generation-bound code chunks. -/// -/// The value implements the lexical posting port directly. Exact retrieval is -/// enabled independently by deriving an [`CodeExactProjectionAdapterV1`] with -/// the central admission authority; constructing this lexical adapter alone -/// never enables or mints exact proofs. -/// -/// Metadata is shared, not owned: every scoped projection built over one -/// generation reads the same immutable copy instead of cloning its logical -/// path table per scope. -#[derive(Clone, Debug)] -pub struct CodeLexicalProjectionAdapterV1 { - metadata: Arc, - rows: Arc>, - postings: Arc, -} - -#[derive(Clone, Debug)] -struct LexicalGenerationPostingsV1 { - term_documents: BTreeMap>, - exact_documents: BTreeMap, RoaringBitmap>>, - normalized_text: Arc, - raw_text: Arc, - fuzzy_terms: FuzzyTermIndex, - average_field_lengths: BTreeMap, -} - -#[derive(Clone, Debug, Default)] -struct LexicalTermPostingV1 { - documents: RoaringBitmap, - frequencies: Vec<(u32, u32)>, -} - -impl LexicalTermPostingV1 { - fn insert(&mut self, document: u32, frequency: u32) { - self.documents.insert(document); - self.frequencies.push((document, frequency)); - } - - fn frequency(&self, document: u32) -> usize { - self.frequencies - .binary_search_by_key(&document, |(document, _)| *document) - .ok() - .map(|index| self.frequencies[index].1 as usize) - .unwrap_or_default() - } -} - -#[derive(Debug)] -struct LexicalGenerationPostingsBuildV1 { - term_documents: BTreeMap>, - exact_documents: BTreeMap, RoaringBitmap>>, - normalized_text: ByteNgramPostings, - raw_text: ByteNgramPostings, - vocabulary: BTreeSet, - field_lengths: BTreeMap, - ngram_budget: ByteNgramBudget, -} - -impl Default for LexicalGenerationPostingsBuildV1 { - fn default() -> Self { - Self { - term_documents: BTreeMap::new(), - exact_documents: BTreeMap::new(), - normalized_text: ByteNgramPostings::default(), - raw_text: ByteNgramPostings::default(), - vocabulary: BTreeSet::new(), - field_lengths: BTreeMap::new(), - ngram_budget: ByteNgramBudget::new(BYTE_NGRAM_POSTINGS_MEMORY_BUDGET_BYTES_V1), - } - } -} - -impl LexicalGenerationPostingsBuildV1 { - fn insert_row( - &mut self, - document: u32, - row: &ProjectedChunkV1, - fields: &BTreeMap>, - ) -> Result<(), RetrievalPortError> { - for (field, terms) in fields { - *self.field_lengths.entry(*field).or_default() += terms.len(); - let mut frequencies = BTreeMap::<&str, u32>::new(); - for term in terms { - if *field != LexicalFieldV1::Subtoken { - self.vocabulary.insert(term.clone()); - } - frequencies - .entry(term.as_str()) - .and_modify(|frequency| *frequency = frequency.saturating_add(1)) - .or_insert(1); - } - for (term, frequency) in frequencies { - self.term_documents - .entry(*field) - .or_default() - .entry(term.to_owned()) - .or_default() - .insert(document, frequency); - } - } - self.exact_documents - .entry(ExactFieldV1::Path) - .or_default() - .entry(row.logical_path.as_bytes().to_vec()) - .or_default() - .insert(document); - for term in &row.exact_terms { - let canonical = canonical_projected_exact_term(term); - self.exact_documents - .entry(exact_field_for_kind(term.kind())) - .or_default() - .entry(canonical.into_owned()) - .or_default() - .insert(document); - } - let search_text = normalized_search_text(row); - self.normalized_text - .insert_document(document, search_text.as_bytes(), &mut self.ngram_budget) - .map_err(map_postings_build_error) - } - - fn insert_raw_text( - &mut self, - document: u32, - row: &ProjectedChunkV1, - ) -> Result<(), RetrievalPortError> { - self.raw_text - .insert_document( - document, - row.sanitized_text.as_str().as_bytes(), - &mut self.ngram_budget, - ) - .map_err(map_postings_build_error) - } - - fn finish( - self, - document_count: usize, - raw_matches_normalized: bool, - deadline: Option, - ) -> Result { - let divisor = document_count.max(1); - let average_field_lengths = self - .field_lengths - .into_iter() - .map(|(field, total)| (field, total.div_ceil(divisor).max(1))) - .collect(); - let normalized_text = Arc::new(self.normalized_text); - let raw_text = if raw_matches_normalized { - Arc::clone(&normalized_text) - } else { - Arc::new(self.raw_text) - }; - let fuzzy_terms = FuzzyTermIndex::from_terms(self.vocabulary, deadline) - .map_err(map_postings_build_error)?; - Ok(LexicalGenerationPostingsV1 { - term_documents: self.term_documents, - exact_documents: self.exact_documents, - normalized_text, - raw_text, - fuzzy_terms, - average_field_lengths, - }) - } -} - -#[derive(Clone, Debug)] -pub enum CodeLexicalProjectionBuildStepV1 { - Pending { - completed_documents: usize, - total_documents: usize, - }, - Ready(Box), -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum CodeLexicalProjectionBuildPhaseV1 { - Rows, - RawText { next_document: usize }, - Complete, -} - -/// Generation-owned, in-memory lexical projection work that advances by a -/// caller-selected number of document operations and preserves partial state -/// between bounded scheduler windows. -#[derive(Debug)] -pub struct CodeLexicalProjectionBuildV1 { - metadata: Arc, - symbol_displays: Arc>, - chunks: Vec>, - rows: Vec, - postings: Option, - next_document: usize, - raw_matches_normalized: bool, - extraction_admitted: bool, - phase: CodeLexicalProjectionBuildPhaseV1, -} - -impl CodeLexicalProjectionBuildV1 { - pub fn new_admitted( - metadata: impl Into>, - chunks: Vec, - symbol_displays: impl Into< - Arc>, - >, - ) -> Result - where - C: ExtractionAdmittedChunkV1, - { - Self::new_inner( - metadata.into(), - chunks - .into_iter() - .map(ExtractionAdmittedChunkV1::into_admitted_chunk) - .collect(), - symbol_displays.into(), - true, - ) - } - - fn new_inner( - metadata: Arc, - mut chunks: Vec, - symbol_displays: Arc>, - extraction_admitted: bool, - ) -> Result { - metadata.validate()?; - if chunks.len() > u32::MAX as usize { - return Err(RetrievalPortError::Contract( - "lexical projection exceeds the posting document-id range".to_owned(), - )); - } - chunks.sort_by(|left, right| left.id.cmp(&right.id)); - if chunks.windows(2).any(|pair| pair[0].id == pair[1].id) { - return Err(RetrievalPortError::Contract( - "lexical projection chunk identities must be unique".to_owned(), - )); - } - let row_capacity = chunks.len(); - Ok(Self { - metadata, - symbol_displays, - chunks: chunks.into_iter().map(Some).collect(), - rows: Vec::with_capacity(row_capacity), - postings: Some(LexicalGenerationPostingsBuildV1::default()), - next_document: 0, - raw_matches_normalized: true, - extraction_admitted, - phase: CodeLexicalProjectionBuildPhaseV1::Rows, - }) - } - - #[hotpath::measure(label = "query.artifact.projection_advance")] - pub fn advance( - &mut self, - maximum_documents: usize, - ) -> Result { - self.advance_inner(maximum_documents, None) - } - - fn advance_inner( - &mut self, - maximum_documents: usize, - deadline: Option, - ) -> Result { - if maximum_documents == 0 { - return Err(RetrievalPortError::Contract( - "lexical projection build window must admit at least one document".to_owned(), - )); - } - if self.phase == CodeLexicalProjectionBuildPhaseV1::Complete { - return Err(RetrievalPortError::Contract( - "lexical projection build is already complete".to_owned(), - )); - } - let mut remaining = maximum_documents; - while remaining > 0 { - if let Some(deadline) = deadline { - check_projection_build_deadline(deadline)?; - } - match self.phase { - CodeLexicalProjectionBuildPhaseV1::Rows => { - if self.next_document == self.chunks.len() { - self.phase = if self.raw_matches_normalized { - CodeLexicalProjectionBuildPhaseV1::Complete - } else { - CodeLexicalProjectionBuildPhaseV1::RawText { next_document: 0 } - }; - continue; - } - let document = self.next_document; - let chunk = self.chunks[document].take().ok_or_else(|| { - RetrievalPortError::Contract( - "lexical projection row was advanced more than once".to_owned(), - ) - })?; - chunk.validate().map_err(contract_error)?; - if !self.extraction_admitted - && chunk - .exact_terms - .iter() - .any(ExactTechnicalTermV1::requires_extraction_authority) - { - return Err(RetrievalPortError::Contract( - "raw exact terms require parser-backed extraction admission".to_owned(), - )); - } - // Serving generation is metadata.generation. Chunk anchors may - // retain extraction provenance from Arc-shared parent pages; - // stamp the projected row with the serving id (same as artifact prepare). - let logical_path = self - .metadata - .logical_paths - .get(&chunk.anchor.file_occurrence_id) - .cloned() - .ok_or_else(|| { - RetrievalPortError::Contract(format!( - "lexical projection is missing the logical path for {}", - chunk.anchor.file_occurrence_id - )) - })?; - let symbol_display = chunk - .anchor - .symbol_occurrence_id - .as_ref() - .and_then(|symbol| self.symbol_displays.get(symbol)); - let (mut row, fields) = - ProjectedChunkV1::from_ref(&chunk, logical_path, symbol_display); - row.anchor.generation_id = self.metadata.generation.clone(); - self.raw_matches_normalized &= - row.sanitized_text.as_str().as_bytes() == row.normalized_text.as_bytes(); - self.postings - .as_mut() - .ok_or_else(|| { - RetrievalPortError::Contract( - "lexical projection build state is missing".to_owned(), - ) - })? - .insert_row(document as u32, &row, &fields)?; - self.rows.push(row); - self.next_document += 1; - remaining -= 1; - } - CodeLexicalProjectionBuildPhaseV1::RawText { next_document } => { - if next_document == self.rows.len() { - self.phase = CodeLexicalProjectionBuildPhaseV1::Complete; - continue; - } - self.postings - .as_mut() - .ok_or_else(|| { - RetrievalPortError::Contract( - "lexical projection build state is missing".to_owned(), - ) - })? - .insert_raw_text(next_document as u32, &self.rows[next_document])?; - self.phase = CodeLexicalProjectionBuildPhaseV1::RawText { - next_document: next_document + 1, - }; - remaining -= 1; - } - CodeLexicalProjectionBuildPhaseV1::Complete => break, - } - } - if matches!( - self.phase, - CodeLexicalProjectionBuildPhaseV1::Rows - if self.next_document == self.chunks.len() - ) { - self.phase = if self.raw_matches_normalized { - CodeLexicalProjectionBuildPhaseV1::Complete - } else { - CodeLexicalProjectionBuildPhaseV1::RawText { next_document: 0 } - }; - } - if matches!( - self.phase, - CodeLexicalProjectionBuildPhaseV1::RawText { next_document } - if next_document == self.rows.len() - ) { - self.phase = CodeLexicalProjectionBuildPhaseV1::Complete; - } - if self.phase != CodeLexicalProjectionBuildPhaseV1::Complete { - crate::hotpath_metrics::Residency::Cold.record("query.artifact.residency"); - hotpath::gauge!("query.artifact.rows").set(self.next_document); - return Ok(CodeLexicalProjectionBuildStepV1::Pending { - completed_documents: self.next_document, - total_documents: self.chunks.len(), - }); - } - let postings = self - .postings - .take() - .ok_or_else(|| { - RetrievalPortError::Contract("lexical projection build state is missing".to_owned()) - })? - .finish(self.rows.len(), self.raw_matches_normalized, deadline)?; - crate::hotpath_metrics::Residency::Warm.record("query.artifact.residency"); - hotpath::gauge!("query.artifact.rows").set(self.rows.len()); - Ok(CodeLexicalProjectionBuildStepV1::Ready(Box::new( - CodeLexicalProjectionAdapterV1 { - metadata: self.metadata.clone(), - rows: Arc::new(std::mem::take(&mut self.rows)), - postings: Arc::new(postings), - }, - ))) - } -} - -impl LexicalGenerationPostingsV1 { - fn retained_owned_bytes(&self) -> usize { - let term_bytes = self - .term_documents - .values() - .fold(0usize, |bytes, postings| { - postings.iter().fold(bytes, |bytes, (term, posting)| { - bytes - .saturating_add(term.capacity()) - .saturating_add( - (posting.documents.len() as usize) - .saturating_mul(std::mem::size_of::()), - ) - .saturating_add( - posting - .frequencies - .capacity() - .saturating_mul(std::mem::size_of::<(u32, u32)>()), - ) - }) - }); - let exact_bytes = self - .exact_documents - .values() - .fold(0usize, |bytes, postings| { - postings.iter().fold(bytes, |bytes, (term, documents)| { - bytes.saturating_add(term.capacity()).saturating_add( - (documents.len() as usize).saturating_mul(std::mem::size_of::()), - ) - }) - }); - let raw_text_bytes = if Arc::ptr_eq(&self.normalized_text, &self.raw_text) { - 0 - } else { - self.raw_text.retained_owned_bytes() - }; - term_bytes - .saturating_add(exact_bytes) - .saturating_add(self.normalized_text.retained_owned_bytes()) - .saturating_add(raw_text_bytes) - .saturating_add(self.fuzzy_terms.retained_owned_bytes()) - .saturating_add( - self.average_field_lengths - .len() - .saturating_mul(std::mem::size_of::<(LexicalFieldV1, usize)>()), - ) - } - - fn document_frequency(&self, field: LexicalFieldV1, term: &str) -> usize { - self.term_documents - .get(&field) - .and_then(|postings| postings.get(term)) - .map(|posting| posting.documents.len() as usize) - .unwrap_or_default() - } - - fn term_frequency(&self, field: LexicalFieldV1, term: &str, document: u32) -> usize { - self.term_documents - .get(&field) - .and_then(|postings| postings.get(term)) - .map(|posting| posting.frequency(document)) - .unwrap_or_default() - } - - fn average_field_length(&self, field: LexicalFieldV1) -> usize { - self.average_field_lengths.get(&field).copied().unwrap_or(1) - } - - fn lexical_documents( - &self, - request: &LexicalLaneRequest<'_>, - fuzzy: &FuzzyExpansionsV1, - phrase_candidates: &BTreeMap, - pruned: &mut Vec<(String, u64)>, - ) -> RoaringBitmap { - let mut sources = Vec::new(); - for term in request.whole_terms.iter() { - let (frequency, documents) = self.whole_term_documents(&normalize_lexical(term)); - sources.push((frequency, (term.clone(), documents))); - if let Some(expansions) = fuzzy.by_query.get(term) { - for expansion in expansions { - let (frequency, documents) = self.whole_term_documents(expansion); - sources.push((frequency, (expansion.clone(), documents))); - } - } - } - sources.extend( - request - .proximities - .iter() - .flat_map(|proximity| &proximity.terms) - .map(|term| { - let (frequency, documents) = - self.whole_term_documents(&normalize_lexical(term)); - (frequency, (term.clone(), documents)) - }), - ); - if let Some(postings) = self.term_documents.get(&LexicalFieldV1::Subtoken) { - for subtoken in request.subtokens.iter() { - if let Some(posting) = postings.get(&normalize_lexical(subtoken)) { - sources.push(( - posting.documents.len() as usize, - (subtoken.clone(), posting.documents.clone()), - )); - } - } - } - let mut documents = RoaringBitmap::new(); - for (_, source) in admit_candidate_sources(sources, |frequency, (term, _)| { - pruned.push((term.clone(), frequency as u64)); - }) { - documents |= source; - } - // Reuse the per-phrase n-gram candidate sets computed once by the - // caller. Union is idempotent, so unioning the deduplicated normalized - // phrases yields exactly the same document set as re-intersecting the - // n-gram postings for every raw phrase here. - for candidates in phrase_candidates.values() { - documents |= candidates; - } - documents - } - - /// The n-gram candidate-document set for one already-normalized phrase. - /// Computed once per phrase and shared by both the phrase document-frequency - /// tally and the lexical document set. - fn phrase_candidate_documents(&self, normalized_phrase: &str) -> RoaringBitmap { - self.normalized_text - .candidate_documents(normalized_phrase.as_bytes()) - } - - fn exact_candidate_documents(&self, request: &ExactLaneRequest) -> RoaringBitmap { - let mut documents = RoaringBitmap::new(); - for literal in &request.literals { - if matches!( - literal.field, - ExactFieldV1::QuotedPhrase - | ExactFieldV1::DiagnosticText - | ExactFieldV1::CompilerOrRuntimeError - ) { - documents |= self.raw_text.candidate_documents(&literal.original_bytes); - } - if let Some(posting) = self - .exact_documents - .get(&literal.field) - .and_then(|postings| postings.get(&literal.canonical_bytes)) - { - documents |= posting; - } - } - documents - } - - fn phrase_document_frequency( - &self, - rows: &[ProjectedChunkV1], - phrase: &str, - candidates: &RoaringBitmap, - ) -> usize { - candidates - .iter() - .filter(|document| matches_phrase(&rows[*document as usize], phrase)) - .count() - } - - /// A whole-term candidate source: the term's documents across every - /// non-subtoken field, keyed by the summed per-field document frequency - /// the artifact reader also admits by. - fn whole_term_documents(&self, term: &str) -> (usize, RoaringBitmap) { - let mut documents = RoaringBitmap::new(); - let mut frequency = 0usize; - for (field, postings) in &self.term_documents { - if *field == LexicalFieldV1::Subtoken { - continue; - } - if let Some(posting) = postings.get(term) { - frequency = frequency.saturating_add(posting.documents.len() as usize); - documents |= &posting.documents; - } - } - (frequency, documents) - } -} - -impl CodeLexicalProjectionAdapterV1 { - /// Count heap payload bytes owned exclusively by this immutable projection. - /// Shared sanitized chunk text is deliberately excluded because its Arc - /// backing remains owned by the sealed generation; derived normalized text, - /// posting keys/frequencies, n-grams, exact keys, and the fuzzy FST count. - pub fn retained_owned_bytes(&self) -> usize { - let row_bytes = self.rows.iter().fold(0usize, |bytes, row| { - let exact_term_bytes = row.exact_terms.iter().fold(0usize, |bytes, term| { - bytes - .saturating_add(term.original_bytes().len()) - .saturating_add(term.canonical_bytes().len()) - }); - bytes - .saturating_add(std::mem::size_of::()) - .saturating_add(row.id.as_str().len()) - .saturating_add(row.anchor.generation_id.as_str().len()) - .saturating_add(row.anchor.file_occurrence_id.as_str().len()) - .saturating_add( - row.anchor - .symbol_occurrence_id - .as_ref() - .map(|symbol| symbol.as_str().len()) - .unwrap_or_default(), - ) - .saturating_add(row.logical_path.capacity()) - .saturating_add(row.normalized_text.capacity()) - .saturating_add(row.symbol_signature.as_ref().map_or(0, String::capacity)) - .saturating_add( - row.symbol_documentation - .as_ref() - .map_or(0, String::capacity), - ) - .saturating_add(exact_term_bytes) - .saturating_add( - row.field_lengths - .len() - .saturating_mul(std::mem::size_of::<(LexicalFieldV1, usize)>()), - ) - }); - let metadata_path_bytes = - self.metadata - .logical_paths - .iter() - .fold(0usize, |bytes, (file, path)| { - bytes - .saturating_add(file.as_str().len()) - .saturating_add(path.capacity()) - }); - row_bytes - .saturating_add(metadata_path_bytes) - .saturating_add(self.postings.retained_owned_bytes()) - } - - pub fn new( - metadata: impl Into>, - chunks: Vec, - ) -> Result { - Self::new_inner( - metadata.into(), - chunks, - Arc::new(BTreeMap::new()), - false, - None, - ) - } - - pub fn new_admitted( - metadata: impl Into>, - chunks: Vec, - symbol_displays: impl Into< - Arc>, - >, - ) -> Result - where - C: ExtractionAdmittedChunkV1, - { - Self::new_admitted_with_deadline(metadata.into(), chunks, symbol_displays.into(), None) - } - - fn new_admitted_with_deadline( - metadata: Arc, - chunks: Vec, - symbol_displays: Arc>, - deadline_micros: Option, - ) -> Result - where - C: ExtractionAdmittedChunkV1, - { - Self::new_inner( - metadata, - chunks - .into_iter() - .map(ExtractionAdmittedChunkV1::into_admitted_chunk) - .collect(), - symbol_displays, - true, - deadline_micros, - ) - } - - fn new_inner( - metadata: Arc, - chunks: Vec, - symbol_displays: Arc>, - extraction_admitted: bool, - deadline_micros: Option, - ) -> Result { - let deadline = Instant::now() - + Duration::from_micros(lexical_projection_build_deadline_micros(deadline_micros)); - check_projection_build_deadline(deadline)?; - let mut build = CodeLexicalProjectionBuildV1::new_inner( - metadata, - chunks, - symbol_displays, - extraction_admitted, - )?; - match build.advance_inner(usize::MAX, Some(deadline))? { - CodeLexicalProjectionBuildStepV1::Ready(projection) => Ok(*projection), - CodeLexicalProjectionBuildStepV1::Pending { .. } => Err(RetrievalPortError::Contract( - "unbounded lexical projection build did not complete".to_owned(), - )), - } - } - - pub fn exact_adapter(&self, authority: A) -> CodeExactProjectionAdapterV1 - where - A: ExactAdmissionAuthority, - { - CodeExactProjectionAdapterV1 { - projection: self.clone(), - authority, - } - } - - fn validate_generation(&self, generation: &CodeGenerationId) -> Result<(), RetrievalPortError> { - if generation != &self.metadata.generation { - return Err(RetrievalPortError::GenerationMismatch); - } - Ok(()) - } - - fn stale_outcome(&self) -> Option> { - (self.metadata.freshness.compatibility != FreshnessCompatibilityV1::Current).then(|| { - crate::hotpath_metrics::Residency::Rebuilding.record("query.lane.lexical.residency"); - RetrieverOutcome::Stale(self.metadata.freshness.clone()) - }) - } - - #[hotpath::measure(label = "query.lane.lexical.generate")] - fn lexical_batch( - &self, - request: &LexicalLaneRequest<'_>, - ) -> Result>, RetrievalPortError> { - let fuzzy = self.fuzzy_expansions(request)?; - let prepared = PreparedLexicalQueryV1::new(request); - // Intersect the n-gram postings for each normalized phrase exactly once, - // then reuse the candidate set for both the document-frequency tally and - // the lexical document set below (previously each phrase was intersected - // twice per query). - let phrase_candidates: BTreeMap = prepared - .phrases - .iter() - .map(|(_, normalized)| { - let candidates = self.postings.phrase_candidate_documents(normalized); - (normalized.clone(), candidates) - }) - .collect(); - let phrase_document_frequencies = phrase_candidates - .iter() - .map(|(phrase, candidates)| { - let frequency = self - .postings - .phrase_document_frequency(&self.rows, phrase, candidates); - (phrase.clone(), frequency) - }) - .collect::>(); - let mut pruned = Vec::new(); - let documents = - self.postings - .lexical_documents(request, &fuzzy, &phrase_candidates, &mut pruned); - let mut pairs = Vec::new(); - let mut excluded = self.rows.len() as u64 - documents.len(); - for (ordinal, document) in documents.into_iter().enumerate() { - if ordinal.is_multiple_of(RETRIEVAL_CANDIDATE_BATCH_SIZE) { - retrieval_checkpoint(request.control)?; - } - let row = &self.rows[document as usize]; - let score = self.score_row( - document, - row, - &prepared, - &fuzzy, - &phrase_document_frequencies, - ); - if score.field_scores.is_empty() { - excluded += 1; - continue; - } - let candidate = lexical_lane_candidate( - row, - &self.metadata.freshness, - self.metadata.repository_id.clone(), - RetrieverKind::Lexical, - self.metadata.lexical_retriever_revision.clone(), - request.score_domain.clone(), - None, - )?; - let evidence = LexicalLaneEvidence { - binding: lexical_lane_binding(row, &candidate, score.matched_kinds), - field_scores_micros: score.field_scores, - matched_whole_terms: score.matched_whole_terms, - matched_subtokens: score.matched_subtokens, - matched_phrases: score.matched_phrases, - matched_proximities: score.matched_proximities, - spelling_variants: score.spelling_variants, - typo_recovery_applied: score.typo_recovery_applied, - echo_penalty_applied: score.echo_penalty_applied, - }; - pairs.push((candidate, evidence)); - } - retrieval_checkpoint(request.control)?; - pairs.sort_by(|left, right| { - left.0 - .source_occurrence_id - .cmp(&right.0.source_occurrence_id) - }); - let mut candidates = Vec::with_capacity(pairs.len()); - let mut evidence_by_occurrence = BTreeMap::new(); - for (ordinal, (mut candidate, evidence)) in pairs.into_iter().enumerate() { - if ordinal.is_multiple_of(RETRIEVAL_CANDIDATE_BATCH_SIZE) { - retrieval_checkpoint(request.control)?; - } - candidate.ordinal_rank = ordinal as u32; - evidence_by_occurrence.insert(candidate.source_occurrence_id.clone(), evidence); - candidates.push(candidate); - } - retrieval_checkpoint(request.control)?; - hotpath::gauge!("query.lane.lexical.candidates").set(candidates.len()); - hotpath::gauge!("query.lane.lexical.examined").set(self.rows.len()); - Ok(candidate_admission_outcome( - RetrieverBatch { - coverage: RetrieverCoverage { - examined: self.rows.len() as u64, - eligible: candidates.len() as u64, - excluded, - capped: 0, - unknown: 0, - }, - candidates, - evidence_by_occurrence, - continuation: None, - }, - pruned, - )) - } - - #[hotpath::measure(label = "query.lane.fuzzy.expand")] - fn fuzzy_expansions( - &self, - request: &LexicalLaneRequest<'_>, - ) -> Result { - if request.fuzzy_budget == 0 { - return Ok(FuzzyExpansionsV1::default()); - } - let limit = request.fuzzy_budget.min(MAX_FUZZY_TERM_EXPANSIONS_V1) as usize; - let mut group_by_query = BTreeMap::::new(); - let mut groups = Vec::::new(); - for (query_ordinal, query) in request.whole_terms.iter().enumerate() { - let normalized_query = normalize_lexical(query); - let bound = fuzzy_distance_bound(&normalized_query); - if bound == 0 { - continue; - } - if let Some(group) = group_by_query.get(&normalized_query).copied() { - groups[group].queries.insert(query.clone()); - continue; - } - let group = groups.len(); - group_by_query.insert(normalized_query.clone(), group); - groups.push(FuzzyQueryGroupV1 { - first_ordinal: query_ordinal, - normalized_query, - queries: BTreeSet::from([query.clone()]), - bound, - seen: BTreeSet::new(), - }); - } - groups.sort_by_key(|group| group.first_ordinal); - let maximum_distance = groups.iter().map(|group| group.bound).max().unwrap_or(0); - let mut selected = Vec::<(usize, String)>::with_capacity(limit); - 'distance: for distance in 1..=maximum_distance { - for (group_index, group) in groups.iter_mut().enumerate() { - if distance > group.bound { - continue; - } - let remaining = limit.saturating_sub(selected.len()); - if remaining == 0 { - break 'distance; - } - let slice = self - .postings - .fuzzy_terms - .terms_at_distance( - &group.normalized_query, - distance, - remaining, - &mut group.seen, - ) - .map_err(RetrievalPortError::Contract)?; - selected.extend(slice.terms.into_iter().map(|term| (group_index, term))); - } - } - let mut by_query: BTreeMap> = BTreeMap::new(); - let expansion_count = selected.len(); - for (group_index, term) in selected { - for query in &groups[group_index].queries { - by_query - .entry(query.clone()) - .or_default() - .insert(term.clone()); - } - } - hotpath::gauge!("query.lane.fuzzy.expansions").set(expansion_count); - Ok(FuzzyExpansionsV1 { by_query }) - } - - fn score_row( - &self, - document: u32, - row: &ProjectedChunkV1, - prepared: &PreparedLexicalQueryV1<'_>, - fuzzy: &FuzzyExpansionsV1, - phrase_document_frequencies: &BTreeMap, - ) -> LexicalRowScoreV1 { - crate::hotpath_metrics::measure_frequent("query.lane.lexical.score_row", || { - score_lexical_row( - row, - &row.exact_terms, - prepared, - fuzzy, - phrase_document_frequencies, - |field, term| self.postings.term_frequency(field, term, document), - |field, term| self.postings.document_frequency(field, term), - |field, term_frequency, document_frequency| { - bm25_score_micros( - self.rows.len(), - document_frequency, - term_frequency, - row.field_lengths.get(&field).copied().unwrap_or(0).max(1), - self.postings.average_field_length(field), - field_weight_millis(field), - ) - }, - ) - }) - } -} - -impl LexicalPostingReadPort for CodeLexicalProjectionAdapterV1 { - fn read_lexical_postings( - &self, - request: &LexicalLaneRequest<'_>, - ) -> Result>, RetrievalPortError> { - self.validate_generation(&request.generation)?; - if let Some(outcome) = self.stale_outcome() { - return Ok(outcome); - } - self.lexical_batch(request) - } -} - -/// Exact-reader view over the same immutable lexical projection. -/// -/// This type cannot exist without an [`ExactAdmissionAuthority`], and every -/// emitted proof comes from that authority's `admit` method. -#[derive(Clone, Debug)] -pub struct CodeExactProjectionAdapterV1 { - projection: CodeLexicalProjectionAdapterV1, - authority: A, -} - -impl ExactTermPostingReadPort for CodeExactProjectionAdapterV1 -where - A: ExactAdmissionAuthority, -{ - fn read_exact_postings( - &self, - request: &ExactLaneRequest, - ) -> Result>, RetrievalPortError> { - retrieval_checkpoint(request.control)?; - self.projection.validate_generation(&request.generation)?; - if let Some(outcome) = self.projection.stale_outcome() { - return Ok(outcome); - } - let documents = self.projection.postings.exact_candidate_documents(request); - let mut pairs = Vec::new(); - let mut excluded = self.projection.rows.len() as u64 - documents.len(); - let mut proofs = LiteralProofCacheV1::new(request.literals.len()); - for (ordinal, document) in documents.into_iter().enumerate() { - if ordinal.is_multiple_of(RETRIEVAL_CANDIDATE_BATCH_SIZE) { - retrieval_checkpoint(request.control)?; - } - let row = &self.projection.rows[document as usize]; - let (matched_literals, matched_kinds) = exact_matches(row.exact_match_view(), request); - if matched_literals.is_empty() { - excluded += 1; - continue; - } - let (_, proof) = proofs - .first_admitted(&matched_literals, request, &self.authority)? - .ok_or_else(|| { - RetrievalPortError::Contract( - "central authority rejected every projected exact match".to_owned(), - ) - })?; - let matched_literals = matched_literals - .iter() - .map(|ordinal| request.literals[*ordinal].clone()) - .collect::>(); - let candidate = lexical_lane_candidate( - row, - &self.projection.metadata.freshness, - self.projection.metadata.repository_id.clone(), - RetrieverKind::ExactLiteral, - self.projection.metadata.exact_retriever_revision.clone(), - self.projection.metadata.exact_score_domain.clone(), - Some(proof.clone()), - )?; - let evidence = ExactLaneEvidence { - binding: lexical_lane_binding(row, &candidate, matched_kinds), - matched_literals, - admission_proof: proof, - }; - pairs.push((candidate, evidence)); - } - retrieval_checkpoint(request.control)?; - pairs.sort_by(|left, right| { - left.0 - .source_occurrence_id - .cmp(&right.0.source_occurrence_id) - }); - let mut candidates = Vec::with_capacity(pairs.len()); - let mut evidence_by_occurrence = BTreeMap::new(); - for (ordinal, (mut candidate, evidence)) in pairs.into_iter().enumerate() { - if ordinal.is_multiple_of(RETRIEVAL_CANDIDATE_BATCH_SIZE) { - retrieval_checkpoint(request.control)?; - } - candidate.ordinal_rank = ordinal as u32; - evidence_by_occurrence.insert(candidate.source_occurrence_id.clone(), evidence); - candidates.push(candidate); - } - retrieval_checkpoint(request.control)?; - Ok(RetrieverOutcome::Complete(RetrieverBatch { - coverage: RetrieverCoverage { - examined: self.projection.rows.len() as u64, - eligible: candidates.len() as u64, - excluded, - capped: 0, - unknown: 0, - }, - candidates, - evidence_by_occurrence, - continuation: None, - })) - } -} - -impl ProjectedChunkV1 { - fn exact_match_view(&self) -> ExactMatchRowViewV1<'_> { - ExactMatchRowViewV1 { - sanitized_text: self.sanitized_text.as_str(), - logical_path: &self.logical_path, - exact_terms: &self.exact_terms, - } - } -} - -#[cfg(test)] -mod deadline_budget_tests { - use super::*; - use tracedecay_domain::{ - ComponentRevision, ScoreDomainId, SourceFreshness, SourceInstanceKey, SourceNamespace, - UtcMicros, - }; - - fn dummy_metadata() -> CodeLexicalProjectionMetadataV1 { - CodeLexicalProjectionMetadataV1 { - generation: CodeGenerationId::new("generation.deadline.v1").expect("generation"), - repository_id: None, - logical_paths: BTreeMap::new(), - freshness: SourceFreshness { - source_namespace: SourceNamespace::new("ns.deadline").expect("namespace"), - source_instance: SourceInstanceKey::new("instance.deadline").expect("instance"), - source_watermark: None, - projection_watermark: None, - observed_at: UtcMicros(0), - source_generation: None, - generation_lag: None, - compatibility: FreshnessCompatibilityV1::Unknown, - policy_revision: ComponentRevision::new("policy.deadline.v1").expect("policy"), - }, - exact_retriever_revision: ComponentRevision::new("retriever.exact.v1").expect("exact"), - lexical_retriever_revision: ComponentRevision::new("retriever.lexical.v1") - .expect("lexical"), - exact_score_domain: ScoreDomainId::new(crate::retrieval::QUERY_EXACT_SCORE_DOMAIN_V1) - .expect("score"), - } - } - - #[test] - fn zero_deadline_is_immediate_budget_exceeded() { - let error = CodeLexicalProjectionAdapterV1::new_inner( - Arc::new(dummy_metadata()), - Vec::::new(), - Arc::new(BTreeMap::new()), - true, - Some(0), - ) - .expect_err("Some(0) must expire before validate"); - assert!( - matches!(error, RetrievalPortError::BudgetExceeded), - "Some(0) is a set deadline, not the crate fallback: {error:?}" - ); - } - - #[test] - fn projected_rows_stamp_serving_generation_over_extraction_provenance() { - use tracedecay_domain::{ - BoundedSanitizedText, ChunkerRevision, CodeSearchChunkAnchorV1, CodeSearchChunkGrainV1, - CodeSearchChunkId, ContentDigest, FileOccurrenceId, LanguageDescriptorRevision, - PolicyRevisionId, SanitizerRevision, SensitivityDecision, SensitivityLevelV1, - SourceSpan, - }; - - let serving = CodeGenerationId::new("generation.serving.v1").expect("serving"); - let parent = CodeGenerationId::new("generation.parent.v1").expect("parent"); - let file = FileOccurrenceId::new("file.carry.v1").expect("file"); - let mut metadata = dummy_metadata(); - metadata.generation = serving.clone(); - metadata - .logical_paths - .insert(file.clone(), "src/carry.rs".to_owned()); - - let digest = |byte: char| { - ContentDigest::try_from(format!("sha256:{}", byte.to_string().repeat(64))) - .expect("digest") - }; - let chunk = CodeSearchChunkV1 { - id: CodeSearchChunkId::new("chunk.carry.v1").expect("chunk"), - anchor: CodeSearchChunkAnchorV1 { - generation_id: parent, - file_occurrence_id: file, - symbol_occurrence_id: None, - parent_chunk_id: None, - source_span: SourceSpan { - start_byte: 0, - end_byte: 4, - }, - grain: CodeSearchChunkGrainV1::FileWindow, - ordinal: 0, - }, - content_digest: digest('c'), - language_descriptor_revision: LanguageDescriptorRevision::new("language.rust.v1") - .expect("language"), - chunker_revision: ChunkerRevision::new("chunker.v1").expect("chunker"), - sanitizer_revision: SanitizerRevision::new("sanitizer.v1").expect("sanitizer"), - sensitivity: SensitivityDecision { - level: SensitivityLevelV1::Public, - policy_revision: PolicyRevisionId::new("policy.v1").expect("policy"), - }, - exact_terms: Vec::new(), - subtokens: Vec::new(), - sanitized_text: BoundedSanitizedText::new("code").expect("text"), - }; - - let adapter = CodeLexicalProjectionAdapterV1::new_inner( - Arc::new(metadata), - vec![chunk], - Arc::new(BTreeMap::new()), - true, - None, - ) - .expect("project carried chunk"); - assert_eq!( - adapter.rows[0].anchor.generation_id, serving, - "in-memory projection must stamp serving generation like artifact prepare" - ); - } - - #[test] - fn ngram_resident_budget_refusal_is_typed_budget_exceeded() { - let error = format!( - "{}: maximum 0 bytes", - postings::LEXICAL_PROJECTION_NGRAM_MEMORY_BUDGET_EXCEEDED - ); - - assert!(matches!( - map_postings_build_error(error), - RetrievalPortError::BudgetExceeded - )); - } -} diff --git a/crates/tracedecay-query/src/retrieval/lexical/projection/in_memory/postings.rs b/crates/tracedecay-query/src/retrieval/lexical/projection/in_memory/postings.rs deleted file mode 100644 index 343989df23..0000000000 --- a/crates/tracedecay-query/src/retrieval/lexical/projection/in_memory/postings.rs +++ /dev/null @@ -1,564 +0,0 @@ -use std::collections::BTreeSet; -use std::time::Instant; - -use fst::{IntoStreamer, Set, Streamer, automaton::Levenshtein}; -use roaring::RoaringBitmap; - -const NGRAM_PAGE_BACKING_BYTES: usize = 1024 * 1024; -const NGRAM_PAGE_ENTRY_CAPACITY: usize = NGRAM_PAGE_BACKING_BYTES / std::mem::size_of::(); -const NGRAM_MINIMUM_PAGE_ENTRY_CAPACITY: usize = 1024; -const NGRAM_MINIMUM_PAGE_BYTES: usize = - NGRAM_MINIMUM_PAGE_ENTRY_CAPACITY * std::mem::size_of::(); -const NGRAM_MAXIMUM_PAGE_COUNT: usize = 512; -pub(super) const LEXICAL_PROJECTION_BUILD_DEADLINE_EXCEEDED: &str = - "lexical projection exceeded its build deadline"; -pub(super) const LEXICAL_PROJECTION_NGRAM_MEMORY_BUDGET_EXCEEDED: &str = - "lexical projection n-gram posting memory budget exceeded"; - -#[derive(Debug, Default)] -pub(super) struct ByteNgramPostings { - pages: Vec, - entry_count: usize, -} - -#[derive(Debug)] -struct PackedNgramPage { - entries: Vec, - // Full pages are immutable sorted runs. Only the single trailing page is - // scanned; this avoids a second O(generation) merge allocation at finish. - sorted: bool, -} - -impl PackedNgramPage { - fn try_new(capacity: usize) -> Result { - debug_assert!((1..=NGRAM_PAGE_ENTRY_CAPACITY).contains(&capacity)); - let mut entries = Vec::new(); - entries.try_reserve_exact(capacity).map_err(|error| { - format!("lexical projection could not allocate a bounded n-gram page: {error}") - })?; - debug_assert_eq!(entries.capacity(), capacity); - Ok(Self { - entries, - sorted: false, - }) - } - - fn push(&mut self, entry: u64) { - debug_assert!(self.entries.len() < self.entries.capacity()); - self.entries.push(entry); - if self.entries.len() == self.entries.capacity() { - self.entries.sort_unstable(); - self.sorted = true; - } - } - - fn remaining_capacity(&self) -> usize { - self.entries.capacity().saturating_sub(self.entries.len()) - } - - fn add_documents(&self, ngram: u32, documents: &mut RoaringBitmap) { - if self.sorted { - let lower = self - .entries - .partition_point(|entry| unpack_ngram(*entry) < ngram); - let upper = self.entries[lower..] - .partition_point(|entry| unpack_ngram(*entry) == ngram) - .saturating_add(lower); - for entry in &self.entries[lower..upper] { - documents.insert(unpack_document(*entry)); - } - } else { - for entry in self - .entries - .iter() - .filter(|entry| unpack_ngram(**entry) == ngram) - { - documents.insert(unpack_document(*entry)); - } - } - } -} - -impl ByteNgramPostings { - pub(super) fn insert_document( - &mut self, - document: u32, - bytes: &[u8], - budget: &mut ByteNgramBudget, - ) -> Result<(), String> { - let scratch_entries = (1..=bytes.len().min(3)).try_fold(0usize, |entries, width| { - entries - .checked_add(bytes.len().saturating_sub(width).saturating_add(1)) - .ok_or_else(|| budget.exceeded()) - })?; - let scratch_bytes = scratch_entries - .checked_mul(std::mem::size_of::()) - .ok_or_else(|| budget.exceeded())?; - budget.ensure_peak(scratch_bytes, 0)?; - - let mut unique = Vec::new(); - unique.try_reserve_exact(scratch_entries).map_err(|error| { - format!("lexical projection could not allocate bounded n-gram scratch: {error}") - })?; - debug_assert_eq!(unique.capacity(), scratch_entries); - for width in 1..=bytes.len().min(3) { - unique.extend(bytes.windows(width).map(super::super::pack_byte_ngram)); - } - unique.sort_unstable(); - unique.dedup(); - if unique.is_empty() { - return Ok(()); - } - - let available_entries = self - .pages - .last() - .map(PackedNgramPage::remaining_capacity) - .unwrap_or_default(); - let mut entries_to_allocate = unique.len().saturating_sub(available_entries); - let mut next_page_capacity = - self.pages - .last() - .map_or(NGRAM_MINIMUM_PAGE_ENTRY_CAPACITY, |page| { - page.entries - .capacity() - .saturating_mul(2) - .min(NGRAM_PAGE_ENTRY_CAPACITY) - }); - let mut required_pages = 0usize; - let mut page_bytes = 0usize; - while entries_to_allocate > 0 { - let capacity = entries_to_allocate - .max(next_page_capacity) - .min(NGRAM_PAGE_ENTRY_CAPACITY); - page_bytes = page_bytes - .checked_add( - capacity - .checked_mul(std::mem::size_of::()) - .ok_or_else(|| budget.exceeded())?, - ) - .ok_or_else(|| budget.exceeded())?; - required_pages = required_pages - .checked_add(1) - .ok_or_else(|| budget.exceeded())?; - entries_to_allocate = entries_to_allocate.saturating_sub(capacity); - next_page_capacity = capacity.saturating_mul(2).min(NGRAM_PAGE_ENTRY_CAPACITY); - } - let descriptor_capacity = if self.pages.capacity() == 0 { - budget.page_descriptor_capacity() - } else { - 0 - }; - if self.pages.len().saturating_add(required_pages) - > self.pages.capacity().max(descriptor_capacity) - { - return Err(budget.exceeded()); - } - let descriptor_bytes = descriptor_capacity - .checked_mul(std::mem::size_of::()) - .ok_or_else(|| budget.exceeded())?; - let retained_bytes = descriptor_bytes - .checked_add(page_bytes) - .ok_or_else(|| budget.exceeded())?; - budget.reserve_retained_at_peak(scratch_bytes, retained_bytes)?; - - if descriptor_capacity > 0 - && let Err(error) = self.pages.try_reserve_exact(descriptor_capacity) - { - budget.release_retained(retained_bytes); - return Err(format!( - "lexical projection could not allocate bounded n-gram page metadata: {error}" - )); - } - debug_assert!(descriptor_capacity == 0 || self.pages.capacity() == descriptor_capacity); - let mut entries_to_allocate = unique.len().saturating_sub(available_entries); - let mut next_page_capacity = - self.pages - .last() - .map_or(NGRAM_MINIMUM_PAGE_ENTRY_CAPACITY, |page| { - page.entries - .capacity() - .saturating_mul(2) - .min(NGRAM_PAGE_ENTRY_CAPACITY) - }); - let mut allocated_page_bytes = 0usize; - while entries_to_allocate > 0 { - let capacity = entries_to_allocate - .max(next_page_capacity) - .min(NGRAM_PAGE_ENTRY_CAPACITY); - let allocated_bytes = capacity.saturating_mul(std::mem::size_of::()); - match PackedNgramPage::try_new(capacity) { - Ok(page) => self.pages.push(page), - Err(error) => { - budget.release_retained(page_bytes.saturating_sub(allocated_page_bytes)); - return Err(error); - } - } - allocated_page_bytes = allocated_page_bytes.saturating_add(allocated_bytes); - entries_to_allocate = entries_to_allocate.saturating_sub(capacity); - next_page_capacity = capacity.saturating_mul(2).min(NGRAM_PAGE_ENTRY_CAPACITY); - } - - let mut page_index = self - .pages - .iter() - .position(|page| page.remaining_capacity() > 0) - .ok_or_else(|| "lexical n-gram page reservation was incomplete".to_owned())?; - for ngram in unique { - loop { - let page = self - .pages - .get_mut(page_index) - .ok_or_else(|| "lexical n-gram page reservation was incomplete".to_owned())?; - if page.remaining_capacity() > 0 { - page.push(pack_posting(ngram, document)); - break; - } - page_index = page_index - .checked_add(1) - .ok_or_else(|| "lexical n-gram page index overflowed".to_owned())?; - } - self.entry_count = self.entry_count.saturating_add(1); - } - Ok(()) - } - - #[cfg(test)] - pub(super) fn from_documents<'a>( - documents: impl IntoIterator, - budget: &mut ByteNgramBudget, - deadline: Option, - ) -> Result { - let mut postings = Self::default(); - for (document, bytes) in documents.into_iter().enumerate() { - if deadline.is_some_and(|deadline| Instant::now() >= deadline) { - return Err(LEXICAL_PROJECTION_BUILD_DEADLINE_EXCEEDED.to_owned()); - } - let document = u32::try_from(document) - .map_err(|_| "posting document id exceeds u32".to_owned())?; - postings.insert_document(document, bytes, budget)?; - } - Ok(postings) - } - - pub(super) fn candidate_documents(&self, needle: &[u8]) -> RoaringBitmap { - let mut ngrams = super::super::packed_query_ngrams(needle).into_iter(); - let Some(first) = ngrams.next() else { - return RoaringBitmap::new(); - }; - let mut documents = self.documents_for_ngram(first); - for ngram in ngrams { - documents &= self.documents_for_ngram(ngram); - if documents.is_empty() { - break; - } - } - documents - } - - pub(super) fn retained_owned_bytes(&self) -> usize { - self.pages.iter().fold( - self.pages - .capacity() - .saturating_mul(std::mem::size_of::()), - |bytes, page| { - bytes.saturating_add( - page.entries - .capacity() - .saturating_mul(std::mem::size_of::()), - ) - }, - ) - } - - fn documents_for_ngram(&self, ngram: u32) -> RoaringBitmap { - let mut documents = RoaringBitmap::new(); - for page in &self.pages { - page.add_documents(ngram, &mut documents); - } - documents - } - - #[cfg(test)] - fn page_count(&self) -> usize { - self.pages.len() - } -} - -#[derive(Clone, Copy, Debug)] -pub(super) struct ByteNgramBudget { - consumed_bytes: usize, - maximum_bytes: usize, -} - -impl ByteNgramBudget { - #[hotpath::skip] - pub(super) const fn new(maximum_bytes: usize) -> Self { - Self { - consumed_bytes: 0, - maximum_bytes, - } - } - - fn ensure_peak(&self, temporary_bytes: usize, retained_bytes: usize) -> Result<(), String> { - let consumed = self - .consumed_bytes - .checked_add(temporary_bytes) - .and_then(|bytes| bytes.checked_add(retained_bytes)) - .ok_or_else(|| self.exceeded())?; - if consumed > self.maximum_bytes { - return Err(self.exceeded()); - } - Ok(()) - } - - fn reserve_retained_at_peak( - &mut self, - temporary_bytes: usize, - retained_bytes: usize, - ) -> Result<(), String> { - self.ensure_peak(temporary_bytes, retained_bytes)?; - self.consumed_bytes = self - .consumed_bytes - .checked_add(retained_bytes) - .ok_or_else(|| self.exceeded())?; - Ok(()) - } - - fn release_retained(&mut self, bytes: usize) { - self.consumed_bytes = self.consumed_bytes.saturating_sub(bytes); - } - - fn page_descriptor_capacity(&self) -> usize { - self.maximum_bytes - .checked_div(NGRAM_MINIMUM_PAGE_BYTES) - .unwrap_or_default() - .clamp(1, NGRAM_MAXIMUM_PAGE_COUNT) - } - - #[cfg(test)] - #[hotpath::skip] - const fn consumed_bytes(&self) -> usize { - self.consumed_bytes - } - - fn exceeded(&self) -> String { - format!( - "{}: maximum {} bytes", - LEXICAL_PROJECTION_NGRAM_MEMORY_BUDGET_EXCEEDED, self.maximum_bytes - ) - } -} - -fn pack_posting(ngram: u32, document: u32) -> u64 { - (u64::from(ngram) << 32) | u64::from(document) -} - -fn unpack_ngram(posting: u64) -> u32 { - (posting >> 32) as u32 -} - -fn unpack_document(posting: u64) -> u32 { - posting as u32 -} - -#[derive(Clone, Debug)] -pub(super) struct FuzzyTermIndex { - terms: Set>, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub(super) struct FuzzySearchSlice { - pub(super) terms: Vec, - #[cfg(test)] - pub(super) examined: usize, -} - -impl FuzzyTermIndex { - pub(super) fn from_terms(terms: I, deadline: Option) -> Result - where - I: IntoIterator, - S: AsRef, - { - let mut canonical_terms = BTreeSet::new(); - for term in terms { - if deadline.is_some_and(|deadline| Instant::now() >= deadline) { - return Err(LEXICAL_PROJECTION_BUILD_DEADLINE_EXCEEDED.to_owned()); - } - canonical_terms.insert(term.as_ref().to_owned()); - } - if deadline.is_some_and(|deadline| Instant::now() >= deadline) { - return Err(LEXICAL_PROJECTION_BUILD_DEADLINE_EXCEEDED.to_owned()); - } - let terms = Set::from_iter(canonical_terms.into_iter().map(String::into_bytes)) - .map_err(|error| error.to_string())?; - if deadline.is_some_and(|deadline| Instant::now() >= deadline) { - return Err(LEXICAL_PROJECTION_BUILD_DEADLINE_EXCEEDED.to_owned()); - } - Ok(Self { terms }) - } - - pub(super) fn terms_at_distance( - &self, - query: &str, - distance: usize, - limit: usize, - seen: &mut BTreeSet, - ) -> Result { - let automaton = - Levenshtein::new(query, distance as u32).map_err(|error| error.to_string())?; - let mut stream = self.terms.search(automaton).into_stream(); - let mut terms = Vec::with_capacity(limit); - #[cfg(test)] - let mut examined = 0; - while terms.len() < limit { - let Some(term) = stream.next() else { - break; - }; - #[cfg(test)] - { - examined += 1; - } - let term = std::str::from_utf8(term).map_err(|error| error.to_string())?; - if term != query && seen.insert(term.to_owned()) { - terms.push(term.to_owned()); - } - } - Ok(FuzzySearchSlice { - terms, - #[cfg(test)] - examined, - }) - } - - pub(super) fn retained_owned_bytes(&self) -> usize { - self.terms.as_fst().size() - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn ngram_postings_prune_without_false_negatives() { - let mut budget = ByteNgramBudget::new(2 * 1024 * 1024); - let postings = ByteNgramPostings::from_documents( - [ - b"alpha connection refused".as_slice(), - b"beta connection restored".as_slice(), - b"gamma request completed".as_slice(), - ], - &mut budget, - None, - ) - .expect("bounded postings"); - - assert_eq!( - postings - .candidate_documents(b"connection refused") - .iter() - .collect::>(), - vec![0] - ); - assert_eq!( - postings - .candidate_documents(b"connection") - .iter() - .collect::>(), - vec![0, 1] - ); - assert!( - postings - .candidate_documents(b"missing diagnostic") - .is_empty() - ); - } - - #[test] - fn ngram_postings_reject_over_budget_generations() { - let mut budget = ByteNgramBudget::new(80); - let error = ByteNgramPostings::from_documents([b"abcdef".as_slice()], &mut budget, None) - .expect_err("posting memory must be bounded"); - assert!(error.contains("n-gram posting memory budget")); - } - - #[test] - fn ngram_postings_charge_flat_page_capacity_and_preserve_candidates() { - let documents = (0..70_000_u32) - .map(|document| { - [ - (document & 0xff) as u8, - ((document >> 8) & 0xff) as u8, - ((document >> 16) & 0xff) as u8, - ] - }) - .collect::>(); - let mut budget = ByteNgramBudget::new(8 * 1024 * 1024); - let postings = ByteNgramPostings::from_documents( - documents.iter().map(|document| document.as_slice()), - &mut budget, - None, - ) - .expect("flat pages fit the resident budget"); - - assert!(postings.page_count() > 1, "fixture must cross a page"); - assert_eq!(postings.retained_owned_bytes(), budget.consumed_bytes()); - for document in [0_u32, 65_535, 69_999] { - assert_eq!( - postings - .candidate_documents(&documents[document as usize]) - .iter() - .collect::>(), - vec![document] - ); - } - } - - #[test] - fn ngram_postings_refuse_before_allocating_a_page_or_unique_scratch() { - let bytes = vec![b'a'; 256 * 1024]; - let mut postings = ByteNgramPostings::default(); - let mut budget = ByteNgramBudget::new(NGRAM_PAGE_BACKING_BYTES); - - let error = postings - .insert_document(0, &bytes, &mut budget) - .expect_err("page plus document scratch must exceed the budget"); - - assert!(error.contains("n-gram posting memory budget")); - assert_eq!(postings.retained_owned_bytes(), 0); - assert_eq!(budget.consumed_bytes(), 0); - } - - #[test] - fn ngram_postings_reject_an_already_expired_build_deadline() { - let mut budget = ByteNgramBudget::new(1024 * 1024); - let error = ByteNgramPostings::from_documents( - [b"abcdef".as_slice()], - &mut budget, - Some(Instant::now()), - ) - .expect_err("expired deadline must fail closed"); - assert_eq!(error, LEXICAL_PROJECTION_BUILD_DEADLINE_EXCEEDED); - } - - #[test] - fn fuzzy_index_rejects_an_already_expired_build_deadline() { - let error = FuzzyTermIndex::from_terms(["alpha"], Some(Instant::now())) - .expect_err("expired deadline must cover fuzzy index construction"); - assert_eq!(error, LEXICAL_PROJECTION_BUILD_DEADLINE_EXCEEDED); - } - - #[test] - fn fuzzy_enumeration_stops_at_the_remaining_budget() { - let terms = ('!'..='~').map(|character| format!("aaaa{character}aaaaa")); - let index = FuzzyTermIndex::from_terms(terms, None).expect("valid FST"); - let mut seen = BTreeSet::new(); - let slice = index - .terms_at_distance("aaaaaaaaaa", 1, 3, &mut seen) - .expect("bounded fuzzy search"); - - assert_eq!(slice.terms.len(), 3); - assert!(slice.examined <= 4); - } -} diff --git a/crates/tracedecay-query/src/retrieval/task_session.rs b/crates/tracedecay-query/src/retrieval/task_session.rs index e8a0471f8f..8081b55f44 100644 --- a/crates/tracedecay-query/src/retrieval/task_session.rs +++ b/crates/tracedecay-query/src/retrieval/task_session.rs @@ -16,7 +16,8 @@ use tracedecay_domain::{ WorkAttemptIdentityV1, canonical_sha256, }; use tracedecay_temporal_query::TemporalCandidateExport; -use tracedecay_temporal_query::ports::{TemporalExecutionSnapshot, TemporalRetrievalScope}; +use tracedecay_temporal_query::ports::TemporalRetrievalScope; +use tracedecay_temporal_query::snapshot::TemporalExecutionSnapshot; use super::evidence_lanes::{EvidenceLaneExecutionControlV1, LaneEvidenceBinding, execute_lane}; use super::ports::{RetrievalPortError, contract_error}; diff --git a/crates/tracedecay-query/src/retrieval/tests/evidence_lanes.rs b/crates/tracedecay-query/src/retrieval/tests/evidence_lanes.rs index 0d10bfee0c..dba1ac3599 100644 --- a/crates/tracedecay-query/src/retrieval/tests/evidence_lanes.rs +++ b/crates/tracedecay-query/src/retrieval/tests/evidence_lanes.rs @@ -15,12 +15,14 @@ use tracedecay_domain::{ TemporalModeV1, UtcMicros, WorkAttemptIdentityV1, WorkGraphVersionV1, WorkProductEventSequenceV1, WorkProductSourceWatermarkV1, }; -use tracedecay_temporal_query::ports::{ - BindingDigest, KernelVersions, TemporalExecutionSnapshot, TemporalParticipantAuthorization, - TemporalParticipantGeneration, TemporalParticipantManifest, TemporalSnapshotRequest, - TemporalSourceAccess, TemporalWatermarks, -}; +use tracedecay_temporal_query::execution::BindingDigest; +use tracedecay_temporal_query::ports::TemporalSnapshotRequest; use tracedecay_temporal_query::resolution::ValidatedAuthorization; +use tracedecay_temporal_query::snapshot::{ + KernelVersions, TemporalExecutionSnapshot, TemporalParticipantAuthorization, + TemporalParticipantGeneration, TemporalParticipantManifest, TemporalSourceAccess, + TemporalWatermarks, +}; use crate::retrieval::evidence_lanes::score_diagnostic; use crate::retrieval::evidence_lanes::{ diff --git a/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs b/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs index ab2a5b8549..c68be55afc 100644 --- a/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs +++ b/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs @@ -2,7 +2,6 @@ use std::borrow::Cow; use std::collections::{BTreeMap, BTreeSet}; use std::fmt; use std::fmt::Write as _; -use std::io::Cursor; use std::num::NonZeroUsize; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; @@ -10,42 +9,36 @@ use std::sync::{Arc, Mutex}; use std::time::Instant; use sha2::{Digest, Sha256}; -use tracedecay_code_index::chunks::{ - DeterministicCodeChunker, ExtractionAdmittedCodeSearchChunkV1, content_digest, -}; +use tracedecay_code_index::chunks::content_digest; use tracedecay_code_index::clones::CloneNormalizationClassV1; -use tracedecay_code_index::extract::{LanguageExtractor, NeverCancelled, TreeSitterExtractor}; -use tracedecay_code_index::intake::{CodeIndexIntake, SanitizedCodeIntake}; use tracedecay_code_index::languages::{LanguageRegistry, StaticLanguageRegistry}; use tracedecay_code_index::production::{ CodeIndexAtomicPublicationPort, CodeIndexBuildRequestV1, CodeIndexCapturedFileV1, CodeIndexExecutionControlV1, CodeIndexGenerationScopeV1, CodeIndexInterruptionV1, CodeIndexProductionConfigV1, CodeIndexProductionErrorV1, CodeIndexProductionOwnerV1, CodeIndexPublicationStoreErrorV1, CodeIndexPublishedGenerationV1, - CodeIndexRepositoryParseIdentityV1, VerifiedSealedLexicalCursorV1, - VerifiedSealedLexicalPageBatchBoundsV1, VerifiedSealedLexicalPageBatchReadV1, - VerifiedSealedLexicalPageReadV1, VerifiedSealedLexicalPageSourceV1, - VerifiedSealedLexicalPageV1, VerifiedSealedLexicalSourceReceiptV1, - VerifiedSealedLexicalSymbolDisplayV1, + CodeIndexRepositoryParseIdentityV1, SealedGenerationSegmentPublicationV1, + VerifiedSealedLexicalCursorV1, VerifiedSealedLexicalPageBatchBoundsV1, + VerifiedSealedLexicalPageBatchReadV1, VerifiedSealedLexicalPageReadV1, + VerifiedSealedLexicalPageSourceV1, VerifiedSealedLexicalPageV1, + VerifiedSealedLexicalSourceReceiptV1, }; use tracedecay_code_index::projection::{ ChunkProjectionDecisionV1, CodeChunkProjectionSink, ProjectionReceiptBuilderV1, ProjectionSinkErrorV1, ProjectionSinkReceiptV1, }; use tracedecay_domain::{ - BoundedSanitizedText, ChunkerRevision, CodeGenerationId, CodeSearchChunkAnchorV1, - CodeSearchChunkGrainV1, CodeSearchChunkId, CodeSearchChunkV1, ComponentRevision, ContentDigest, + ChunkerRevision, CodeGenerationId, CompactCandidate, ComponentRevision, EphemeralSanitizedQueryViewV1, ExactAdmissionProof, ExactAdmissionRuleRevision, - ExactAdmissionValidator, ExactFieldV1, ExactTechnicalTermKindV1, ExactTechnicalTermV1, - FileOccurrenceId, FreshnessCompatibilityV1, LanguageDescriptorRevision, ManifestDigest, - PolicyRevisionId, PrincipalId, PrivacyDomainId, ProjectId, ProjectionBatchRequestV1, - ProjectionKeyV1, ProjectionKindV1, ProjectionOperationV1, ProjectionOutcomeV1, - QueryNormalizationRevision, RepositoryDirtyStateV1, RepositoryId, RetrievalBudget, - RetrievalError, RetrievalRequest, RetrievalScope, RetrievalSnapshot, RetrieverCoverage, - RetrieverOutcome, SanitizationReceiptId, SanitizedCodeFileV1, SanitizedCodeSnapshotV1, - SanitizerRevision, ScoreDomainId, SensitivityDecision, SensitivityLevelV1, SingleRootScopeV1, - SnapshotFileDispositionV1, SourceFreshness, SourceInstanceKey, SourceNamespace, SourceSpan, - SymbolOccurrenceId, TemporalModeV1, UtcMicros, ValidatedCodeFileV1, VectorWatermark, + ExactAdmissionValidator, ExactFieldV1, ExactTechnicalTermKindV1, FileOccurrenceId, + FreshnessCompatibilityV1, ManifestDigest, PolicyRevisionId, PrincipalId, PrivacyDomainId, + ProjectId, ProjectionBatchRequestV1, ProjectionKeyV1, ProjectionKindV1, ProjectionOperationV1, + ProjectionOutcomeV1, QueryNormalizationRevision, RepositoryDirtyStateV1, RepositoryId, + RetrievalBudget, RetrievalError, RetrievalRequest, RetrievalScope, RetrievalSnapshot, + RetrieverBatch, RetrieverCoverage, RetrieverOutcome, SanitizationReceiptId, + SanitizedCodeFileV1, SanitizedCodeSnapshotV1, SanitizerRevision, ScoreDomainId, + SensitivityLevelV1, SingleRootScopeV1, SnapshotFileDispositionV1, SourceFreshness, + SourceInstanceKey, SourceNamespace, TemporalModeV1, UtcMicros, VectorWatermark, }; use tracedecay_query::retrieval::exact::{ CentralExactAdmissionAuthorityV1, ExactAdmissionAuthority, ExactLane, ExactLaneRequest, @@ -58,11 +51,9 @@ use tracedecay_query::retrieval::lexical::{ CloneFingerprintPartialReasonV1, CloneNearMatchExtentV1, CloneSelectedBlockContainmentClassV1, CloneSelectedBlockV1, CodeLexicalArtifactBatchLimitV1, CodeLexicalArtifactBuilderV1, CodeLexicalArtifactErrorV1, CodeLexicalArtifactFinalizationStepV1, CodeLexicalArtifactReaderV1, - CodeLexicalArtifactWriterRevisionV1, CodeLexicalCloneSuccessorV1, - CodeLexicalProjectionAdapterV1, CodeLexicalProjectionBuildStepV1, CodeLexicalProjectionBuildV1, - CodeLexicalProjectionMetadataV1, LexicalFieldFilterV1, LexicalFieldV1, LexicalLane, - LexicalLaneRequest, LexicalLaneRetriever, LexicalProximityV1, LexicalSpellingVariantV1, - MAX_CLONE_EXACT_PAGE_MEMBERS_V1, MAX_FUZZY_TERM_EXPANSIONS_V1, + CodeLexicalCloneRouteV1, CodeLexicalProjectionMetadataV1, LexicalFieldFilterV1, LexicalFieldV1, + LexicalLane, LexicalLaneEvidence, LexicalLaneRequest, LexicalLaneRetriever, LexicalProximityV1, + LexicalSpellingVariantV1, MAX_CLONE_EXACT_PAGE_MEMBERS_V1, MAX_FUZZY_TERM_EXPANSIONS_V1, MAX_LEXICAL_CANDIDATE_DOCUMENTS_V1, MAX_LEXICAL_QUERY_TERM_BYTES_V1, VerifiedCodeLexicalArtifactV1, }; @@ -401,25 +392,32 @@ impl CodeIndexExecutionControlV1 for BudgetExhaustedAtObservation { } } +/// A partitioned sealed generation held in memory: the manifest, its content +/// address, and every published segment under its digest. #[derive(Clone)] -struct RealLexicalSourceFixture { - sealed: Vec, +pub(crate) struct RealLexicalSourceFixture { + manifest: Vec, + segments: Arc>>, state_digest: ManifestDigest, - metadata: CodeLexicalProjectionMetadataV1, + pub(crate) metadata: CodeLexicalProjectionMetadataV1, } impl RealLexicalSourceFixture { - fn open_source( - &self, - maximum_page_chunks: usize, - ) -> VerifiedSealedLexicalPageSourceV1>> { - VerifiedSealedLexicalPageSourceV1::open( - Cursor::new(self.sealed.clone()), - u64::try_from(self.sealed.len()).expect("sealed length"), + fn open_source(&self, maximum_page_chunks: usize) -> VerifiedSealedLexicalPageSourceV1 { + let segments = Arc::clone(&self.segments); + VerifiedSealedLexicalPageSourceV1::open_partitioned_sealed( + &self.manifest, self.state_digest.clone(), + move |digest, _, buffer, _control| { + let bytes = segments.get(digest.as_str()).ok_or_else(|| { + CodeIndexProductionErrorV1::Contract("fixture segment is missing".to_owned()) + })?; + buffer.clear(); + buffer.extend_from_slice(bytes); + Ok(()) + }, maximum_page_chunks, 1024 * 1024, - &ArtifactControl { cancelled: false }, ) .expect("verified sealed lexical source") } @@ -429,34 +427,6 @@ fn real_lexical_source_fixture() -> RealLexicalSourceFixture { real_lexical_source_fixture_with_files(1) } -/// The in-memory projection over every admitted chunk of `generation`, -/// carrying the generation's own extracted qualified names, the same -/// authority the sealed-page artifact path reads per chunk. -fn generation_backed_projection( - metadata: CodeLexicalProjectionMetadataV1, - generation: &CodeIndexPublishedGenerationV1, -) -> CodeLexicalProjectionAdapterV1 { - let chunks = generation - .admitted_chunks() - .expect("published generation admitted chunks") - .iter() - .cloned() - .collect::>(); - let symbol_displays = generation - .symbols() - .symbols - .iter() - .map(|symbol| { - ( - symbol.occurrence.clone(), - VerifiedSealedLexicalSymbolDisplayV1::from(symbol.as_ref()), - ) - }) - .collect::>(); - CodeLexicalProjectionAdapterV1::new_admitted(metadata, chunks, symbol_displays) - .expect("generation-backed lexical projection") -} - /// One real production corpus with `file_count` TypeScript files. The first /// file keeps the original single-file identity; the rest share its token /// shape (identical per-field token counts) under distinct symbols so BM25 @@ -489,7 +459,7 @@ fn real_lexical_source_fixture_with_files(file_count: usize) -> RealLexicalSourc real_lexical_source_fixture_from_sources(sources) } -fn real_lexical_source_fixture_from_sources( +pub(crate) fn real_lexical_source_fixture_from_sources( source_inputs: Vec<(String, String, Vec)>, ) -> RealLexicalSourceFixture { assert!(!source_inputs.is_empty(), "fixture needs at least one file"); @@ -579,16 +549,32 @@ fn real_lexical_source_fixture_from_sources( let generation = owner .build_and_publish(request, &ArtifactControl { cancelled: false }) .expect("production generation"); - let sealed = generation - .encode_sealed() + let mut segments = BTreeMap::new(); + let mut evidence_pack = Vec::new(); + let manifest = generation + .encode_partitioned_sealed(|publication| { + match publication { + SealedGenerationSegmentPublicationV1::File { digest, bytes } => { + segments.insert(digest.as_str().to_owned(), bytes.to_vec()); + } + SealedGenerationSegmentPublicationV1::GenerationEvidencePage { bytes, .. } => { + evidence_pack.extend_from_slice(bytes); + } + SealedGenerationSegmentPublicationV1::GenerationEvidenceCommit { + segment_digest, + .. + } => { + segments.insert( + segment_digest.as_str().to_owned(), + std::mem::take(&mut evidence_pack), + ); + } + } + Ok(()) + }) .expect("sealed production generation"); - let envelope: serde_json::Value = - serde_json::from_slice(&sealed).expect("sealed generation envelope"); - let state_digest = id::( - envelope["state_digest"] - .as_str() - .expect("sealed state digest"), - ); + let state_digest = + ManifestDigest::from_sha256_bytes(&Sha256::digest(&manifest)).expect("manifest digest"); let logical_paths = generation .snapshot() .files @@ -603,9 +589,15 @@ fn real_lexical_source_fixture_from_sources( exact_retriever_revision: id::("retriever.exact.v1"), lexical_retriever_revision: id::("retriever.lexical.v1"), exact_score_domain: id::(QUERY_EXACT_SCORE_DOMAIN_V1), + clone_route: Some(CodeLexicalCloneRouteV1 { + project_id: generation.manifest().project_id.clone(), + worktree_id: generation.snapshot().worktree.clone(), + snapshot_digest: generation.manifest().snapshot_digest.clone(), + }), }; RealLexicalSourceFixture { - sealed, + manifest, + segments: Arc::new(segments), state_digest, metadata, } @@ -628,16 +620,6 @@ fn real_verified_pages_with_maximum_page_chunks( (fixture, pages, receipt) } -fn page_symbol_displays( - pages: &[VerifiedSealedLexicalPageV1], -) -> BTreeMap { - pages - .iter() - .flat_map(|page| page.symbol_displays().iter().flatten()) - .map(|display| (display.occurrence().clone(), display.clone())) - .collect() -} - fn drain_verified_pages( fixture: &RealLexicalSourceFixture, maximum_page_chunks: usize, @@ -681,6 +663,7 @@ fn build_clone_artifact( let reader = CodeLexicalArtifactReaderV1::open_with_control( &path, &verified, + &fixture.metadata, CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, &control, ) @@ -688,6 +671,112 @@ fn build_clone_artifact( (directory, pages, reader) } +/// A production lexical artifact sealed from a real corpus and reopened for +/// serving, with the metadata it was sealed under. +pub(crate) struct SealedArtifactFixture { + _directory: tempfile::TempDir, + pub(crate) metadata: CodeLexicalProjectionMetadataV1, + pub(crate) reader: CodeLexicalArtifactReaderV1, +} + +impl SealedArtifactFixture { + pub(crate) fn lane(&self) -> LexicalLane { + LexicalLane::new(self.reader.clone()) + } + + /// [`lexical_request`] bound to this artifact's generation. + pub(crate) fn request( + &self, + query: &str, + whole_terms: &[&str], + subtokens: &[&str], + phrases: &[&str], + fuzzy_budget: u32, + max_candidates: u32, + ) -> LexicalLaneRequest<'static> { + let mut request = lexical_request( + query, + whole_terms, + subtokens, + phrases, + fuzzy_budget, + max_candidates, + ); + request.generation = self.metadata.generation.clone(); + request + } +} + +/// Seal `fixture`'s verified pages under `metadata` and reopen the artifact. +pub(crate) fn sealed_artifact( + fixture: &RealLexicalSourceFixture, + metadata: CodeLexicalProjectionMetadataV1, +) -> SealedArtifactFixture { + let directory = tempfile::tempdir().expect("artifact directory"); + let path = directory.path().join("lexical.sqlite"); + let control = ArtifactControl { cancelled: false }; + let mut builder = + CodeLexicalArtifactBuilderV1::create(&path, metadata.clone()).expect("create artifact"); + let verified = builder + .rebuild_and_finalize(&mut fixture.open_source(128), &control) + .expect("build from parser-attested pages"); + drop(builder); + let reader = CodeLexicalArtifactReaderV1::open_with_control( + &path, + &verified, + &metadata, + CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, + &control, + ) + .expect("reopen sealed artifact"); + SealedArtifactFixture { + _directory: directory, + metadata, + reader, + } +} + +/// Rust files `src/fixture_{ordinal}.rs` with file ids `file.fixture.{ordinal}`. +pub(crate) fn rust_source_fixture(sources: &[&str]) -> RealLexicalSourceFixture { + real_lexical_source_fixture_from_sources( + sources + .iter() + .enumerate() + .map(|(ordinal, source)| { + ( + format!("file.fixture.{ordinal}"), + format!("src/fixture_{ordinal}.rs"), + source.as_bytes().to_vec(), + ) + }) + .collect(), + ) +} + +/// [`rust_source_fixture`] sealed into one reopened artifact. +pub(crate) fn rust_artifact(sources: &[&str]) -> SealedArtifactFixture { + let fixture = rust_source_fixture(sources); + sealed_artifact(&fixture, fixture.metadata.clone()) +} + +/// The fixture file ordinals a batch's candidates came from. +pub(crate) fn candidate_files(candidates: &[CompactCandidate]) -> BTreeSet { + candidates + .iter() + .map(|candidate| { + candidate + .file_occurrence_id + .as_ref() + .expect("code candidate names its file") + .as_str() + .strip_prefix("file.fixture.") + .expect("fixture file id") + .parse() + .expect("fixture file ordinal") + }) + .collect() +} + fn real_verified_pages() -> ( RealLexicalSourceFixture, Vec, @@ -848,17 +937,17 @@ fn stored_base_section_receipts(path: &Path) -> Vec> { /// Fixture-only source driver retained for legacy regression setup. Production /// finalization receives the source receipt and never owns a source reader. trait TestArtifactSourceStaging { - fn rebuild_and_finalize( + fn rebuild_and_finalize( &mut self, - source: &mut VerifiedSealedLexicalPageSourceV1, + source: &mut VerifiedSealedLexicalPageSourceV1, control: &dyn CodeIndexExecutionControlV1, ) -> Result; } impl TestArtifactSourceStaging for CodeLexicalArtifactBuilderV1 { - fn rebuild_and_finalize( + fn rebuild_and_finalize( &mut self, - source: &mut VerifiedSealedLexicalPageSourceV1, + source: &mut VerifiedSealedLexicalPageSourceV1, control: &dyn CodeIndexExecutionControlV1, ) -> Result { let bounds = VerifiedSealedLexicalPageBatchBoundsV1::new(16, 32 * 1024 * 1024) @@ -978,253 +1067,16 @@ pub(crate) fn projection_metadata( exact_retriever_revision: id::("retriever.exact.v1"), lexical_retriever_revision: id::("retriever.lexical.v1"), exact_score_domain: id::(QUERY_EXACT_SCORE_DOMAIN_V1), + clone_route: Some(CodeLexicalCloneRouteV1 { + project_id: id("project.fixture"), + worktree_id: None, + snapshot_digest: digest_id('d'), + }), } } -pub(crate) fn chunk( - generation: &CodeGenerationId, - ordinal: u32, - grain: CodeSearchChunkGrainV1, - text: &str, - terms: &[(ExactTechnicalTermKindV1, &str)], - subtokens: &[&str], -) -> CodeSearchChunkV1 { - let symbol = matches!( - grain, - CodeSearchChunkGrainV1::SymbolSignature - | CodeSearchChunkGrainV1::SymbolBody - | CodeSearchChunkGrainV1::SymbolMember - ) - .then(|| id::(&format!("symbol.{ordinal}"))); - let mut exact_terms: Vec = terms - .iter() - .map(|(kind, term)| { - let start = text - .find(term) - .unwrap_or_else(|| panic!("term {term:?} is present in {text:?}")); - let span = SourceSpan { - start_byte: start as u64, - end_byte: (start + term.len()) as u64, - }; - if *kind == ExactTechnicalTermKindV1::WholeSymbol { - ExactTechnicalTermV1::untrusted_whole_symbol_candidate( - term.as_bytes().to_vec(), - span, - symbol.clone().expect("symbol grain"), - ) - } else if matches!( - kind, - ExactTechnicalTermKindV1::CompilerErrorText - | ExactTechnicalTermKindV1::RuntimeErrorText - ) { - ExactTechnicalTermV1::untrusted_contextual_text_candidate( - *kind, - term.as_bytes().to_vec(), - span, - ) - } else { - ExactTechnicalTermV1::technical(*kind, term.as_bytes().to_vec(), span) - } - .expect("valid exact-term fixture") - }) - .collect(); - exact_terms.sort_by(|left, right| { - ( - left.span().start_byte, - left.span().end_byte, - left.kind(), - left.canonical_bytes(), - left.original_bytes(), - ) - .cmp(&( - right.span().start_byte, - right.span().end_byte, - right.kind(), - right.canonical_bytes(), - right.original_bytes(), - )) - }); - CodeSearchChunkV1 { - id: id::(&format!("chunk.{ordinal}")), - anchor: CodeSearchChunkAnchorV1 { - generation_id: generation.clone(), - file_occurrence_id: id::(&format!("file.{ordinal}")), - symbol_occurrence_id: symbol, - parent_chunk_id: None, - source_span: SourceSpan { - start_byte: 0, - end_byte: text.len() as u64, - }, - grain, - ordinal, - }, - content_digest: digest_id::( - char::from_digit((ordinal % 10) + 1, 16).expect("hex digit"), - ), - language_descriptor_revision: id::("language.rust.v1"), - chunker_revision: id::("chunker.v1"), - sanitizer_revision: id("sanitizer.v1"), - sensitivity: SensitivityDecision { - level: SensitivityLevelV1::Internal, - policy_revision: id::("policy.fixture.v1"), - }, - exact_terms, - subtokens: subtokens.iter().map(|value| (*value).to_owned()).collect(), - sanitized_text: BoundedSanitizedText::new(text).expect("bounded fixture text"), - } -} - -fn admitted_rust_chunk( - generation: &CodeGenerationId, - ordinal: u32, - source: &str, - grain: CodeSearchChunkGrainV1, - symbol_name: &str, -) -> ExtractionAdmittedCodeSearchChunkV1 { - let registry = StaticLanguageRegistry::new(); - let descriptor = registry - .descriptor(&id("rust")) - .expect("rust descriptor") - .clone(); - let sanitizer_revision = id::("sanitizer.v1"); - let file = SanitizedCodeFileV1 { - file_occurrence_id: id(&format!("file.admitted.{ordinal}")), - logical_path: format!("src/admitted_{ordinal}.rs"), - language: Some(id("rust")), - content_digest: content_digest(source.as_bytes()), - disposition: SnapshotFileDispositionV1::Present, - }; - let intake = - SanitizedCodeIntake::new(registry, sanitizer_revision.clone(), UtcMicros(1_000_000)); - let snapshot = intake - .admit(SanitizedCodeSnapshotV1 { - repository: id("repo.fixture"), - worktree: None, - reference: None, - source_revision: None, - sanitizer_revision: sanitizer_revision.clone(), - sanitization_receipts: vec![id::("receipt.fixture")], - content_identity: content_digest(source.as_bytes()), - captured_at: UtcMicros(1_000_000), - files: vec![file.clone()], - }) - .expect("snapshot admission"); - let file = intake - .bind_file( - &snapshot, - &id::("project.fixture"), - ValidatedCodeFileV1 { - generation_id: generation.clone(), - file, - snapshot_digest: snapshot.snapshot().intake_digest.clone(), - sanitized_bytes: source.as_bytes().to_vec(), - }, - ) - .expect("file admission"); - let batch = TreeSitterExtractor::new() - .extract(&file, &descriptor, &NeverCancelled) - .expect("extract rust fixture"); - let chunker = DeterministicCodeChunker::new( - generation.clone(), - id("repo.fixture"), - sanitizer_revision, - id("policy.fixture.v1"), - id("chunker.v1"), - tracedecay_code_extraction::LanguageRegistry::new(), - ); - let (artifacts, authority) = chunker - .index_file_with_authority_from_extraction( - &file, - &batch, - &descriptor, - SensitivityLevelV1::Public, - &NeverCancelled, - ) - .expect("chunk with exact authority"); - let chunk = artifacts - .chunks - .chunks - .into_iter() - .find(|chunk| { - chunk.anchor.grain == grain - && chunk.exact_terms.iter().any(|term| { - term.kind() == ExactTechnicalTermKindV1::WholeSymbol - && term.original_bytes() == symbol_name.as_bytes() - }) - }) - .expect("requested parser-minted symbol chunk"); - authority.admit(chunk).expect("exact extraction admission") -} - -#[test] -fn retained_lexical_projection_preserves_progress_across_bounded_windows() { - let generation = id::("generation.1"); - let chunks = (0..3) - .map(|ordinal| { - let symbol = format!("retained_symbol_{ordinal}"); - admitted_rust_chunk( - &generation, - ordinal, - &format!("pub fn {symbol}() -> usize {{ {ordinal} }}\n"), - CodeSearchChunkGrainV1::SymbolSignature, - &symbol, - ) - }) - .collect::>(); - let one_shot = CodeLexicalProjectionAdapterV1::new_admitted( - projection_metadata(&generation, FreshnessCompatibilityV1::Current), - chunks.clone(), - BTreeMap::new(), - ) - .expect("one-shot retained lexical projection"); - let mut build = CodeLexicalProjectionBuildV1::new_admitted( - projection_metadata(&generation, FreshnessCompatibilityV1::Current), - chunks, - BTreeMap::new(), - ) - .expect("start retained lexical projection"); - - assert!(matches!( - build.advance(1).expect("first bounded window"), - CodeLexicalProjectionBuildStepV1::Pending { - completed_documents: 1, - total_documents: 3, - } - )); - assert!(matches!( - build.advance(1).expect("second bounded window"), - CodeLexicalProjectionBuildStepV1::Pending { - completed_documents: 2, - total_documents: 3, - } - )); - let projection = loop { - match build.advance(1).expect("finish bounded projection") { - CodeLexicalProjectionBuildStepV1::Pending { .. } => {} - CodeLexicalProjectionBuildStepV1::Ready(projection) => break *projection, - } - }; - let request = lexical_request("retained_symbol_2", &["retained_symbol_2"], &[], &[], 0, 8); - let outcome = LexicalLane::new(projection) - .retrieve_lexical(&request) - .expect("query completed retained projection"); - let one_shot_outcome = LexicalLane::new(one_shot) - .retrieve_lexical(&request) - .expect("query completed one-shot projection"); - assert_eq!(outcome, one_shot_outcome); - let RetrieverOutcome::Complete(batch) = outcome else { - panic!("completed retained projection must serve lexical query"); - }; - assert!( - batch - .candidates - .iter() - .any(|candidate| candidate.file_occurrence_id.as_ref() == Some(&id("file.admitted.2"))) - ); -} - #[test] -fn disk_artifact_resume_reopen_and_lexical_results_match_one_shot_projection() { +fn disk_artifact_resume_and_reopen_serve_lexical_results() { let (fixture, pages, source_receipt) = real_verified_pages(); let metadata = fixture.metadata.clone(); let generation = metadata.generation.clone(); @@ -1232,12 +1084,6 @@ fn disk_artifact_resume_reopen_and_lexical_results_match_one_shot_projection() { .iter() .flat_map(|page| page.chunks().iter().cloned()) .collect::>(); - let one_shot = CodeLexicalProjectionAdapterV1::new_admitted( - metadata.clone(), - chunks.clone(), - page_symbol_displays(&pages), - ) - .expect("one-shot lexical projection"); let import_evidence = pages .iter() .flat_map(|page| page.imports()) @@ -1264,7 +1110,7 @@ fn disk_artifact_resume_reopen_and_lexical_results_match_one_shot_projection() { let mut resumed = CodeLexicalArtifactBuilderV1::open_or_resume_with_memory_budget_and_control( &artifact_path, - metadata, + metadata.clone(), CODE_LEXICAL_ARTIFACT_BUILD_MEMORY_BUDGET_BYTES_V1, &control, ) @@ -1288,6 +1134,7 @@ fn disk_artifact_resume_reopen_and_lexical_results_match_one_shot_projection() { &artifact_path, &artifact_digest, u64::try_from(artifact_bytes.len()).expect("artifact length fits u64"), + &metadata, CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, &control, ) @@ -1331,17 +1178,24 @@ fn disk_artifact_resume_reopen_and_lexical_results_match_one_shot_projection() { 8, ); request.generation = generation; - let artifact = LexicalLane::new(reader) - .retrieve_lexical(&request) - .expect("artifact lexical query"); - let expected = LexicalLane::new(one_shot) - .retrieve_lexical(&request) - .expect("one-shot lexical query"); - assert_eq!(artifact, expected); + let artifact = complete( + LexicalLane::new(reader) + .retrieve_lexical(&request) + .expect("artifact lexical query"), + ); + assert!( + artifact + .evidence_by_occurrence + .values() + .any(|evidence| evidence + .matched_phrases + .contains(&"return value".to_owned())), + "the resumed artifact serves the source's phrase" + ); } #[test] -fn v16_clone_payloads_are_content_addressed_and_postings_page() { +fn clone_payloads_are_content_addressed_and_postings_page() { let body = "one(); two(); three(); four(); five(); six(); seven(); eight(); nine(); ten();"; let fixture = real_lexical_source_fixture_from_sources(vec![ ( @@ -1390,183 +1244,17 @@ fn v16_clone_payloads_are_content_addressed_and_postings_page() { let authority = clone_bodies[0].occurrence.clone(); let directory = tempfile::tempdir().expect("artifact tempdir"); - let legacy_path = directory.path().join("lexical-artifact-v14.sqlite"); - let artifact_path = directory.path().join("lexical-artifact-v16.sqlite"); + let artifact_path = directory.path().join("lexical-artifact.sqlite"); let control = ArtifactControl { cancelled: false }; - let legacy_verified = { - let mut builder = CodeLexicalArtifactBuilderV1::create_with_format_revision( - &legacy_path, - fixture.metadata.clone(), - CodeLexicalArtifactWriterRevisionV1::V14, - ) - .expect("create V14 artifact"); - for page in &pages { - builder - .append_page(page, &control) - .expect("append V14 page"); - } - finish_staged_artifact(&mut builder, &receipt, &control) - }; - let legacy_reader = CodeLexicalArtifactReaderV1::open_with_control( - &legacy_path, - &legacy_verified, - CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, - &control, - ) - .expect("open V14 artifact"); - assert!(matches!( - legacy_reader.clone_exact_page(&authority, &key, None, 1, &control,), - Err(CodeLexicalArtifactErrorV1::Incompatible(_)) - )); let verified = { let mut builder = CodeLexicalArtifactBuilderV1::create(&artifact_path, fixture.metadata.clone()) - .expect("create V16 artifact"); + .expect("create artifact"); for page in &pages { builder.append_page(page, &control).expect("append page"); } finish_staged_artifact(&mut builder, &receipt, &control) }; - assert_eq!( - legacy_verified.section_digests(), - &verified.section_digests()[..legacy_verified.section_digests().len()], - "V16 clone sections must not rewrite lexical section identities" - ); - let successor_path = directory - .path() - .join("lexical-artifact-successor-v16.sqlite"); - let mut successor = CodeLexicalCloneSuccessorV1::open_or_create( - &legacy_path, - &successor_path, - legacy_verified.clone(), - fixture.metadata.clone(), - CODE_LEXICAL_ARTIFACT_BUILD_MEMORY_BUDGET_BYTES_V1, - ) - .expect("create clone-only successor"); - successor - .append_page(&pages[0], &control) - .expect("append first clone page"); - drop(successor); - // A successor staged before occurrence indexes existed must still verify - // on resume. Dropping them here is that shipped shape. - rusqlite::Connection::open(&successor_path) - .expect("open successor before index backfill") - .execute_batch( - "DROP INDEX IF EXISTS clone_exact_postings_by_occurrence; - DROP INDEX IF EXISTS clone_fingerprint_postings_by_occurrence;", - ) - .expect("drop occurrence indexes"); - let mut successor = CodeLexicalCloneSuccessorV1::open_or_create( - &legacy_path, - &successor_path, - legacy_verified, - fixture.metadata.clone(), - CODE_LEXICAL_ARTIFACT_BUILD_MEMORY_BUDGET_BYTES_V1, - ) - .expect("resume clone-only successor"); - // The resume reads the same rows with or without the indexes, so assert - // the backfill itself as well as the verification it is there to speed up. - let backfilled = rusqlite::Connection::open(&successor_path) - .expect("open successor after index backfill") - .query_row( - "SELECT count(*) FROM sqlite_master WHERE type = 'index' AND name IN ('clone_exact_postings_by_occurrence', 'clone_fingerprint_postings_by_occurrence')", - [], - |row| row.get::<_, i64>(0), - ) - .expect("count occurrence indexes"); - assert_eq!( - backfilled, 2, - "opening a successor staged before the occurrence indexes must install both" - ); - successor - .verify_resumed_page(&pages[0], &control) - .expect("resumed clone page verifies through the occurrence index"); - assert_eq!( - successor - .next_cursor() - .expect("successor cursor") - .expect("accepted page cursor"), - pages[0].next_cursor().clone() - ); - for page in &pages[1..] { - successor - .append_page(page, &control) - .expect("append remaining clone page"); - } - let successor_verified = successor - .finish(&receipt, &control) - .expect("finish clone-only successor"); - assert_eq!( - successor_verified.section_digests(), - verified.section_digests() - ); - assert_eq!( - successor_verified.artifact_digest(), - verified.artifact_digest() - ); - CodeLexicalArtifactReaderV1::open_with_control( - &successor_path, - &successor_verified, - CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, - &control, - ) - .expect("open clone-only successor"); - let v15_path = directory.path().join("lexical-artifact-v15.sqlite"); - let v15_verified = { - let mut builder = CodeLexicalArtifactBuilderV1::create_with_format_revision( - &v15_path, - fixture.metadata.clone(), - CodeLexicalArtifactWriterRevisionV1::V15, - ) - .expect("create V15 artifact"); - for page in &pages { - builder - .append_page(page, &control) - .expect("append V15 page"); - } - finish_staged_artifact(&mut builder, &receipt, &control) - }; - let v15_reader = CodeLexicalArtifactReaderV1::open_with_control( - &v15_path, - &v15_verified, - CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, - &control, - ) - .expect("open V15 serving owner"); - let v16_from_v15_path = directory - .path() - .join("lexical-artifact-v16-from-v15.sqlite"); - let mut v16_from_v15 = CodeLexicalCloneSuccessorV1::open_or_create( - &v15_path, - &v16_from_v15_path, - v15_verified, - fixture.metadata.clone(), - CODE_LEXICAL_ARTIFACT_BUILD_MEMORY_BUDGET_BYTES_V1, - ) - .expect("create V16 successor from V15"); - v16_from_v15 - .append_page(&pages[0], &control) - .expect("append V16 successor page"); - assert_eq!( - v15_reader - .clone_exact_page(&authority, &key, None, 1, &control) - .expect("V15 owner serves during successor work") - .members - .len(), - 1 - ); - for page in &pages[1..] { - v16_from_v15 - .append_page(page, &control) - .expect("append remaining V16 successor page"); - } - let v16_from_v15_verified = v16_from_v15 - .finish(&receipt, &control) - .expect("finish V16 successor from V15"); - assert_eq!( - v16_from_v15_verified.artifact_digest(), - verified.artifact_digest() - ); let connection = rusqlite::Connection::open(&artifact_path).expect("inspect V16 artifact"); assert_eq!( connection @@ -1582,22 +1270,28 @@ fn v16_clone_payloads_are_content_addressed_and_postings_page() { .expect("occurrence count"), 4 ); - let (fingerprint_rows, counted_rows): (i64, i64) = connection + let (fingerprint_lists, counted_postings, untagged_payloads): (i64, i64, i64) = connection .query_row( "SELECT (SELECT COUNT(*) FROM clone_fingerprint_postings), - (SELECT COALESCE(SUM(posting_count), 0) FROM clone_fingerprint_counts)", + (SELECT COALESCE(SUM(posting_count), 0) FROM clone_fingerprint_postings), + (SELECT COUNT(*) FROM clone_body_payloads WHERE substr(payload, 1, 1) != x'02')", [], - |row| Ok((row.get(0)?, row.get(1)?)), + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), ) .expect("fingerprint counts"); - assert!(fingerprint_rows > 0); - assert_eq!(counted_rows, fingerprint_rows); + assert!(fingerprint_lists > 0); + assert!( + counted_postings >= fingerprint_lists, + "one sealed list per fingerprint holds every posting" + ); + assert_eq!(untagged_payloads, 0, "clone payloads are stored deflated"); drop(connection); let reader = CodeLexicalArtifactReaderV1::open_with_control( &artifact_path, &verified, + &fixture.metadata, CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, &control, ) @@ -1708,7 +1402,7 @@ fn v16_clone_payloads_are_content_addressed_and_postings_page() { 1, &control, ), - Err(CodeLexicalArtifactErrorV1::Contract(_)) + Err(CodeLexicalArtifactErrorV1::Missing(_)) )); let second = reader .clone_exact_page(&authority, &key, first.next_cursor.as_ref(), 1, &control) @@ -1761,6 +1455,7 @@ fn v16_clone_payloads_are_content_addressed_and_postings_page() { let reduced_reader = CodeLexicalArtifactReaderV1::open_with_control( &reduced_path, &reduced_verified, + &reduced.metadata, CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, &control, ) @@ -1787,6 +1482,7 @@ fn v16_clone_payloads_are_content_addressed_and_postings_page() { CodeLexicalArtifactReaderV1::open_with_control( &artifact_path, &verified, + &fixture.metadata, CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, &control, ), @@ -1801,6 +1497,7 @@ fn v16_clone_payloads_are_content_addressed_and_postings_page() { CodeLexicalArtifactReaderV1::open_with_control( &artifact_path, &verified, + &fixture.metadata, CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, &control, ), @@ -1931,7 +1628,7 @@ fn fingerprint_candidates_reject_incompatible_bodies_and_page_byte_identically() altered_scope.project_id = id::("project.other"); assert!(matches!( reader.clone_fingerprint_page(&altered_scope, &source.payload, Some(cursor), 1, &control,), - Err(CodeLexicalArtifactErrorV1::Contract(_)) + Err(CodeLexicalArtifactErrorV1::Missing(_)) )); let mut stale = source.occurrence.clone(); stale.source_generation = id::("generation.stale"); @@ -2377,29 +2074,17 @@ fn hot_only_fingerprints_are_partial_while_exact_digest_reads_still_work() { /// with the typed cancellation error and stops consulting the control at that /// checkpoint, far short of the candidate set, while the same request under /// an active control completes, agrees byte-for-byte between the sealed -/// artifact and the in-memory projection, and is stable across runs. +/// artifact reopened for serving, and is stable across runs. #[test] -fn lexical_scan_cancellation_unwinds_artifact_and_in_memory_sources_before_completion() { +fn lexical_scan_cancellation_unwinds_the_artifact_before_completion() { let fixture = real_lexical_source_fixture_with_files(24); let (pages, source_receipt) = drain_verified_pages(&fixture, 128); let metadata = fixture.metadata.clone(); - let chunks = pages - .iter() - .flat_map(|page| page.chunks().iter().cloned()) - .collect::>(); - let in_memory = LexicalLane::new( - CodeLexicalProjectionAdapterV1::new_admitted( - metadata.clone(), - chunks, - page_symbol_displays(&pages), - ) - .expect("in-memory lexical projection"), - ); let directory = tempfile::tempdir().expect("artifact tempdir"); let artifact_path = directory.path().join("cancellable-lexical.sqlite"); let control = ArtifactControl { cancelled: false }; - let mut builder = - CodeLexicalArtifactBuilderV1::create(&artifact_path, metadata).expect("create artifact"); + let mut builder = CodeLexicalArtifactBuilderV1::create(&artifact_path, metadata.clone()) + .expect("create artifact"); for page in &pages { builder.append_page(page, &control).expect("append page"); } @@ -2408,6 +2093,7 @@ fn lexical_scan_cancellation_unwinds_artifact_and_in_memory_sources_before_compl CodeLexicalArtifactReaderV1::open_with_control( &artifact_path, &verified, + &metadata, CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, &control, ) @@ -2423,16 +2109,12 @@ fn lexical_scan_cancellation_unwinds_artifact_and_in_memory_sources_before_compl request.control = control; request } - let generation = verified.generation(); + let generation = &metadata.generation; let request = widget_request(generation, &ACTIVE_CONTROL); let artifact_complete = artifact .retrieve_lexical(&request) .expect("uncancelled artifact scan completes"); - let in_memory_complete = in_memory - .retrieve_lexical(&request) - .expect("uncancelled in-memory scan completes"); - assert_eq!(artifact_complete, in_memory_complete); let candidates = complete(artifact_complete.clone()).candidates.len(); assert!( candidates >= 24, @@ -2447,34 +2129,27 @@ fn lexical_scan_cancellation_unwinds_artifact_and_in_memory_sources_before_compl ); let cancel_at = 6; - let lanes: [(&dyn LexicalLaneRetriever, &str); 2] = - [(&artifact, "artifact"), (&in_memory, "in-memory")]; - for (lane, source) in lanes { - let cancelled = CancelAtObservation::new(cancel_at); - assert_eq!( - lane.retrieve_lexical(&widget_request(generation, &cancelled)), - Err(RetrievalPortError::Cancelled), - "{source}: a cancelled scan unwinds with the typed cancellation error" - ); - assert_eq!( - cancelled.observations(), - cancel_at, - "{source}: the scan stops at the cancelling checkpoint instead of visiting the \ - remaining {candidates} candidates" - ); - } + let cancelled = CancelAtObservation::new(cancel_at); + assert_eq!( + artifact.retrieve_lexical(&widget_request(generation, &cancelled)), + Err(RetrievalPortError::Cancelled), + "a cancelled scan unwinds with the typed cancellation error" + ); + assert_eq!( + cancelled.observations(), + cancel_at, + "the scan stops at the cancelling checkpoint instead of visiting the \ + remaining {candidates} candidates" + ); } #[test] -fn extracted_qualified_names_match_in_memory_and_reopened_artifacts() { +fn extracted_qualified_names_search_reopened_artifacts() { let fixture = real_lexical_source_fixture_from_sources(vec![( "file.qualified".to_owned(), "src/qualified.rs".to_owned(), b"pub struct VectorWatermark;\nimpl VectorWatermark { pub fn merge_max(&self) {} }\npub struct UnrelatedContainer;\nimpl UnrelatedContainer { pub fn merge_max(&self) {} }\n".to_vec(), )]); - let generation = CodeIndexPublishedGenerationV1::decode_sealed(&fixture.sealed) - .expect("restore canonical generation"); - let memory = generation_backed_projection(fixture.metadata.clone(), &generation); let directory = tempfile::tempdir().expect("artifact directory"); let path = directory.path().join("qualified.sqlite"); let control = ArtifactControl { cancelled: false }; @@ -2487,6 +2162,7 @@ fn extracted_qualified_names_match_in_memory_and_reopened_artifacts() { let reader = CodeLexicalArtifactReaderV1::open_with_control( &path, &verified, + &fixture.metadata, CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, &control, ) @@ -2516,12 +2192,6 @@ fn extracted_qualified_names_match_in_memory_and_reopened_artifacts() { .read_lexical_postings(&request) .expect("artifact query"), ); - let in_memory = complete( - memory - .read_lexical_postings(&request) - .expect("memory query"), - ); - assert_eq!(disk, in_memory, "{query} must use the same search fields"); if let Some(expected_name) = expected_name { assert!(!disk.candidates.is_empty(), "missing {query}"); for candidate in &disk.candidates { @@ -2562,12 +2232,48 @@ fn extracted_qualified_names_match_in_memory_and_reopened_artifacts() { } } -#[test] -fn vocabulary_fields_phrase_and_proximity_match_in_memory_and_reopened_artifacts() { - let fixture = real_lexical_source_fixture_from_sources(vec![( - "file.vocabulary".to_owned(), - "src/http-cache/client-store.rs".to_owned(), - b"pub struct CachedResponse;\n\ +/// Each candidate as its canonical symbol name and the fields that scored it, +/// read back through the artifact's own occurrence row. +fn scored_fields( + reader: &CodeLexicalArtifactReaderV1, + batch: &RetrieverBatch, +) -> Vec<(String, Vec)> { + batch + .candidates + .iter() + .map(|candidate| { + let evidence = &batch.evidence_by_occurrence[&candidate.source_occurrence_id]; + let occurrence = reader + .occurrence_by_chunk( + evidence + .binding + .occurrence + .chunk + .as_ref() + .expect("chunk binding"), + ) + .expect("read canonical occurrence") + .expect("matched occurrence"); + let fields = evidence + .field_scores_micros + .iter() + .filter(|(_, score)| *score > 0) + .map(|(field, _)| *field) + .collect(); + ( + occurrence.qualified_name.unwrap_or(occurrence.logical_path), + fields, + ) + }) + .collect() +} + +#[test] +fn vocabulary_fields_phrase_and_proximity_search_reopened_artifacts() { + let fixture = real_lexical_source_fixture_from_sources(vec![( + "file.vocabulary".to_owned(), + "src/http-cache/client-store.rs".to_owned(), + b"pub struct CachedResponse;\n\ /// Loads the durable cache entry for the request owner.\n\ pub fn loadCachedResponse(retry_budget: u32) -> CachedResponse {\n\ let _ = retry_budget;\n\ @@ -2575,12 +2281,6 @@ fn vocabulary_fields_phrase_and_proximity_match_in_memory_and_reopened_artifacts }\n" .to_vec(), )]); - let generation = CodeIndexPublishedGenerationV1::decode_sealed(&fixture.sealed) - .expect("restore canonical generation"); - let memory = LexicalLane::new(generation_backed_projection( - fixture.metadata.clone(), - &generation, - )); let directory = tempfile::tempdir().expect("artifact directory"); let path = directory.path().join("vocabulary.sqlite"); let control = ArtifactControl { cancelled: false }; @@ -2590,65 +2290,111 @@ fn vocabulary_fields_phrase_and_proximity_match_in_memory_and_reopened_artifacts .rebuild_and_finalize(&mut fixture.open_source(128), &control) .expect("build from parser-attested pages"); drop(builder); - let artifact = LexicalLane::new( - CodeLexicalArtifactReaderV1::open_with_control( - &path, - &verified, - CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, - &control, - ) - .expect("reopen lexical fields"), - ); + let reader = CodeLexicalArtifactReaderV1::open_with_control( + &path, + &verified, + &fixture.metadata, + CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, + &control, + ) + .expect("reopen lexical fields"); + let artifact = LexicalLane::new(reader.clone()); let run = |query: &str, whole_terms: &[&str], phrases: &[&str], - field: LexicalFieldV1, + filter: Option, proximities: Vec| { let mut request = lexical_request(query, whole_terms, &[], phrases, 0, 32); request.generation = fixture.metadata.generation.clone(); - request.field_filters = Cow::Owned(vec![LexicalFieldFilterV1 { - field, - include: true, - }]); + request.field_filters = Cow::Owned( + filter + .into_iter() + .map(|field| LexicalFieldFilterV1 { + field, + include: true, + }) + .collect(), + ); request.proximities = Cow::Owned(proximities); - let disk = artifact - .retrieve_lexical(&request) - .expect("artifact lexical query"); - let in_memory = memory - .retrieve_lexical(&request) - .expect("in-memory lexical query"); - assert_eq!(disk, in_memory, "{query} must agree across readers"); - complete(disk) + complete( + artifact + .retrieve_lexical(&request) + .expect("artifact lexical query"), + ) }; - for (query, field) in [ - ("cached", LexicalFieldV1::SymbolName), - ("client", LexicalFieldV1::Path), - ("budget", LexicalFieldV1::Signature), - ("response", LexicalFieldV1::QualifiedName), + const STRUCT: &str = "src/http-cache/client-store.rs::CachedResponse"; + const LOADER: &str = "src/http-cache/client-store.rs::loadCachedResponse"; + let both = [STRUCT, STRUCT, LOADER, LOADER]; + let scored = |names: &[&str], field| { + names + .iter() + .map(|name| ((*name).to_owned(), vec![field])) + .collect::>() + }; + // Each query scores in `field` only, and the same query under `other` + // scores there instead, so ignoring the filter would widen both readings. + for (query, field, drawn, other, drawn_elsewhere) in [ + ( + "cached", + LexicalFieldV1::SymbolName, + &both[..], + LexicalFieldV1::QualifiedName, + &both[..], + ), + ( + "client", + LexicalFieldV1::Path, + &both[..], + LexicalFieldV1::QualifiedName, + &both[..], + ), + ( + "budget", + LexicalFieldV1::Signature, + &[LOADER][..], + LexicalFieldV1::SymbolName, + &[][..], + ), + ( + "response", + LexicalFieldV1::QualifiedName, + &both[..], + LexicalFieldV1::SymbolName, + &both[..], + ), ] { - assert!( - !run(query, &[query], &[], field, Vec::new()) - .candidates - .is_empty(), - "{query} must recover from its field vocabulary" + assert_eq!( + scored_fields(&reader, &run(query, &[query], &[], Some(field), Vec::new())), + scored(drawn, field), + "{query} under {field:?}" + ); + assert_eq!( + scored_fields(&reader, &run(query, &[query], &[], Some(other), Vec::new())), + scored(drawn_elsewhere, other), + "{query} under {other:?}" ); } + assert_eq!( + scored_fields(&reader, &run("client", &["client"], &[], None, Vec::new()))[0], + ( + STRUCT.to_owned(), + vec![LexicalFieldV1::QualifiedName, LexicalFieldV1::Path] + ), + "without a filter the path term also scores the qualified name" + ); let mut typo = lexical_request("budgt", &["budgt"], &[], &[], 1, 32); typo.generation = fixture.metadata.generation.clone(); typo.field_filters = Cow::Owned(vec![LexicalFieldFilterV1 { field: LexicalFieldV1::Signature, include: true, }]); - let disk_typo = artifact - .retrieve_lexical(&typo) - .expect("artifact typo query"); - let memory_typo = memory - .retrieve_lexical(&typo) - .expect("in-memory typo query"); - assert_eq!(disk_typo, memory_typo); - let disk_typo = complete(disk_typo); + let disk_typo = complete( + artifact + .retrieve_lexical(&typo) + .expect("artifact typo query"), + ); assert_eq!( disk_typo.evidence_by_occurrence[&disk_typo.candidates[0].source_occurrence_id] .spelling_variants, @@ -2661,7 +2407,7 @@ fn vocabulary_fields_phrase_and_proximity_match_in_memory_and_reopened_artifacts "durable cache", &[], &["durable cache"], - LexicalFieldV1::Documentation, + Some(LexicalFieldV1::Documentation), Vec::new(), ); assert_eq!(phrase.candidates.len(), 1); @@ -2675,7 +2421,7 @@ fn vocabulary_fields_phrase_and_proximity_match_in_memory_and_reopened_artifacts "durable owner", &[], &[], - LexicalFieldV1::Documentation, + Some(LexicalFieldV1::Documentation), vec![proximity], ) .candidates @@ -2691,7 +2437,7 @@ fn vocabulary_fields_phrase_and_proximity_match_in_memory_and_reopened_artifacts "durable owner", &[], &[], - LexicalFieldV1::Documentation, + Some(LexicalFieldV1::Documentation), vec![too_narrow], ) .candidates @@ -2701,48 +2447,53 @@ fn vocabulary_fields_phrase_and_proximity_match_in_memory_and_reopened_artifacts } #[test] -fn disk_artifact_batch_stores_one_ngram_bitmap_shard_per_distinct_key() { +fn disk_artifact_seals_one_ngram_list_per_distinct_key_without_staging() { let (fixture, pages, source_receipt) = real_verified_pages(); let metadata = fixture.metadata.clone(); let generation = metadata.generation.clone(); - let chunks = pages - .iter() - .flat_map(|page| page.chunks().iter().cloned()) - .collect::>(); - let one_shot = CodeLexicalProjectionAdapterV1::new_admitted( - metadata.clone(), - chunks, - page_symbol_displays(&pages), - ) - .expect("one-shot lexical projection"); let directory = tempfile::tempdir().expect("artifact tempdir"); let artifact_path = directory.path().join("ngram-bitmap-shards.sqlite"); let control = ArtifactControl { cancelled: false }; - let mut builder = - CodeLexicalArtifactBuilderV1::create(&artifact_path, metadata).expect("create artifact"); + let mut builder = CodeLexicalArtifactBuilderV1::create(&artifact_path, metadata.clone()) + .expect("create artifact"); builder .append_pages(&pages, &control) .expect("commit one durable source batch"); - let connection = rusqlite::Connection::open(&artifact_path).expect("inspect ngram shards"); - let (stored_rows, distinct_keys): (i64, i64) = connection + let connection = rusqlite::Connection::open(&artifact_path).expect("inspect staging"); + let staged_ngram_tables: i64 = connection .query_row( - "SELECT COUNT(*), COUNT(DISTINCT printf('%d:%d', kind, ngram)) FROM ngram_postings", + "SELECT COUNT(*) FROM sqlite_schema WHERE type = 'table' AND name LIKE 'ngram_posting%' AND name != 'ngram_postings'", [], - |row| Ok((row.get(0)?, row.get(1)?)), + |row| row.get(0), ) - .expect("count durable ngram keys"); - assert!(stored_rows > 0, "the fixture must produce ngram candidates"); + .expect("inspect ngram staging"); assert_eq!( - stored_rows, distinct_keys, - "one atomic source batch must store one bitmap shard per distinct (kind, ngram), not one row per matching document" + staged_ngram_tables, 0, + "n-gram lists are rebuilt from the stored rows, never staged per batch" ); drop(connection); let verified = finish_staged_artifact(&mut builder, &source_receipt, &control); + let connection = rusqlite::Connection::open(&artifact_path).expect("inspect ngram lists"); + let (stored_rows, distinct_keys, postings): (i64, i64, i64) = connection + .query_row( + "SELECT COUNT(*), COUNT(DISTINCT printf('%d:%d', kind, ngram)), SUM(document_frequency) FROM ngram_postings", + [], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .expect("count sealed ngram keys"); + assert!(stored_rows > 0, "the fixture must produce ngram candidates"); + assert_eq!( + stored_rows, distinct_keys, + "one sealed list per distinct (kind, ngram), not one row per matching document" + ); + assert!(postings > stored_rows, "lists hold several documents each"); + drop(connection); let reader = CodeLexicalArtifactReaderV1::open_with_control( &artifact_path, &verified, + &metadata, CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, &control, ) @@ -2756,13 +2507,19 @@ fn disk_artifact_batch_stores_one_ngram_bitmap_shard_per_distinct_key() { 8, ); request.generation = generation; - assert_eq!( + let served = complete( LexicalLane::new(reader) .retrieve_lexical(&request) .expect("bitmap artifact lexical query"), - LexicalLane::new(one_shot) - .retrieve_lexical(&request) - .expect("one-shot lexical query") + ); + assert!( + served + .evidence_by_occurrence + .values() + .any(|evidence| evidence + .matched_phrases + .contains(&"return value".to_owned())), + "the sealed n-gram lists serve the source's phrase" ); } @@ -2807,8 +2564,9 @@ fn content_addressed_reader_rejects_atomic_same_size_replacement() { let artifact_path = directory.path().join("content-addressed.sqlite"); let replacement_path = directory.path().join("replacement.sqlite"); let control = ArtifactControl { cancelled: false }; - let mut builder = CodeLexicalArtifactBuilderV1::create(&artifact_path, fixture.metadata) - .expect("create artifact"); + let mut builder = + CodeLexicalArtifactBuilderV1::create(&artifact_path, fixture.metadata.clone()) + .expect("create artifact"); for page in &pages { builder.append_page(page, &control).expect("append page"); } @@ -2842,6 +2600,7 @@ fn content_addressed_reader_rejects_atomic_same_size_replacement() { &artifact_path, &original_digest, u64::try_from(original_bytes.len()).expect("artifact length fits u64"), + &fixture.metadata, CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, &replacement_control, ), @@ -2853,14 +2612,102 @@ fn content_addressed_reader_rejects_atomic_same_size_replacement() { ); } +/// Route identity is the opener's: two builds of one sealed content under +/// different generations, freshness, snapshots, and batch sizes seal +/// byte-identical files, either opener's reader serves its own route over +/// the same bytes, and a projection whose content differs is refused. +#[test] +fn artifacts_of_identical_content_are_byte_identical_across_routes() { + let (fixture, pages, source_receipt) = real_verified_pages_with_maximum_page_chunks(1); + assert!(pages.len() > 3, "the fixture spans several pages"); + let control = ArtifactControl { cancelled: false }; + let directory = tempfile::tempdir().expect("artifact tempdir"); + let mut route_b = fixture.metadata.clone(); + let generation = fixture.metadata.generation.as_str(); + route_b.generation = id(&format!( + "{}{}", + &generation[..generation.len() - 1], + if generation.ends_with('0') { '1' } else { '0' } + )); + route_b.freshness.source_instance = id("instance.route-b"); + route_b + .clone_route + .as_mut() + .expect("fixture clone route") + .snapshot_digest = digest_id('b'); + let build = |name: &str, metadata: &CodeLexicalProjectionMetadataV1, batch: usize| { + let path = directory.path().join(name); + let mut builder = + CodeLexicalArtifactBuilderV1::create(&path, metadata.clone()).expect("create"); + for batch in pages.chunks(batch) { + builder.append_pages(batch, &control).expect("append pages"); + } + let verified = finish_staged_artifact(&mut builder, &source_receipt, &control); + drop(builder); + (path, verified) + }; + let (path_a, verified_a) = build("route-a.sqlite", &fixture.metadata, 1); + let (path_b, verified_b) = build("route-b.sqlite", &route_b, 3); + assert_eq!(verified_a, verified_b, "the receipt binds content only"); + assert!( + std::fs::read(&path_a).expect("read a") == std::fs::read(&path_b).expect("read b"), + "identical content seals byte-identical files" + ); + + let reader = CodeLexicalArtifactReaderV1::open_with_control( + &path_a, + &verified_a, + &route_b, + CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, + &control, + ) + .expect("route b opens route a's bytes"); + let body = pages + .iter() + .flat_map(VerifiedSealedLexicalPageV1::clone_bodies) + .next() + .expect("fixture clone body"); + let served = reader + .clone_body(&body.occurrence.symbol_occurrence_id) + .expect("clone lookup") + .expect("stored clone body"); + assert_eq!(served.occurrence.source_generation, route_b.generation); + assert_eq!(served.occurrence.snapshot_digest, digest_id('b')); + assert_eq!(served.occurrence.project_id, body.occurrence.project_id); + assert_eq!(served.occurrence.path, body.occurrence.path); + assert_eq!(*served.payload, *body.payload); + let mut from_route_a = body.occurrence.clone(); + from_route_a.source_generation = fixture.metadata.generation.clone(); + assert!(matches!( + reader.clone_fingerprint_page(&from_route_a, &body.payload, None, 1, &control), + Err(CodeLexicalArtifactErrorV1::Missing(_)) + )); + + let mut other_content = route_b.clone(); + other_content + .logical_paths + .insert(id("file.route-only"), "src/route_only.rs".to_owned()); + assert!(matches!( + CodeLexicalArtifactReaderV1::open_with_control( + &path_a, + &verified_a, + &other_content, + CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, + &control, + ), + Err(CodeLexicalArtifactErrorV1::Incompatible(_)) + )); +} + #[test] fn reader_rejects_unsupported_open_revisions_and_accepts_current() { let (fixture, pages, source_receipt) = real_verified_pages(); let directory = tempfile::tempdir().expect("artifact tempdir"); let artifact_path = directory.path().join("open-revision.sqlite"); let control = ArtifactControl { cancelled: false }; - let mut builder = CodeLexicalArtifactBuilderV1::create(&artifact_path, fixture.metadata) - .expect("create artifact"); + let mut builder = + CodeLexicalArtifactBuilderV1::create(&artifact_path, fixture.metadata.clone()) + .expect("create artifact"); for page in &pages { builder.append_page(page, &control).expect("append page"); } @@ -2868,12 +2715,13 @@ fn reader_rejects_unsupported_open_revisions_and_accepts_current() { CodeLexicalArtifactReaderV1::open_with_control( &artifact_path, &verified, + &fixture.metadata, CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, &control, ) .expect("the current revision must open"); - for revision in [9i64, 17] { + for revision in [25i64, 27] { let connection = rusqlite::Connection::open(&artifact_path).expect("open artifact mutation"); connection @@ -2888,6 +2736,7 @@ fn reader_rejects_unsupported_open_revisions_and_accepts_current() { CodeLexicalArtifactReaderV1::open_with_control( &artifact_path, &verified, + &fixture.metadata, CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, &control, ), @@ -2899,7 +2748,7 @@ fn reader_rejects_unsupported_open_revisions_and_accepts_current() { } #[test] -fn absent_and_common_terms_match_in_memory_and_reopened_artifacts() { +fn absent_terms_leave_common_term_artifact_candidates_unchanged() { let files = 128; let functions_per_file = MAX_LEXICAL_CANDIDATE_DOCUMENTS_V1 / files + 1; let fixture = real_lexical_source_fixture_from_sources( @@ -2918,15 +2767,14 @@ fn absent_and_common_terms_match_in_memory_and_reopened_artifacts() { }) .collect(), ); - let generation = CodeIndexPublishedGenerationV1::decode_sealed(&fixture.sealed) - .expect("restore canonical generation"); - let memory = LexicalLane::new(generation_backed_projection( - fixture.metadata.clone(), - &generation, - )); - let mut common = lexical_request("shared_candidate", &["shared_candidate"], &[], &[], 0, 8); - common.generation = fixture.metadata.generation.clone(); - let baseline = complete(memory.retrieve_lexical(&common).expect("common term query")); + let artifact = sealed_artifact(&fixture, fixture.metadata.clone()); + let common = artifact.request("shared_candidate", &["shared_candidate"], &[], &[], 0, 8); + let baseline = complete( + artifact + .lane() + .retrieve_lexical(&common) + .expect("common term query"), + ); assert_eq!(baseline.candidates.len(), 8); assert!( baseline.coverage.eligible > MAX_LEXICAL_CANDIDATE_DOCUMENTS_V1 as u64, @@ -2940,258 +2788,220 @@ fn absent_and_common_terms_match_in_memory_and_reopened_artifacts() { 0, 8, ); - mixed.generation = fixture.metadata.generation.clone(); - let expected = complete(memory.retrieve_lexical(&mixed).expect("mixed term query")); - assert_eq!(expected.candidates, baseline.candidates); - assert_eq!(expected.coverage.eligible, baseline.coverage.eligible); + mixed.generation = artifact.metadata.generation.clone(); + let mixed = complete( + artifact + .lane() + .retrieve_lexical(&mixed) + .expect("mixed term query"), + ); + assert_eq!( + mixed.candidates, baseline.candidates, + "an absent term must not change the common term's candidates" + ); + assert_eq!(mixed.coverage.eligible, baseline.coverage.eligible); +} +#[test] +fn case_sensitive_quoted_literals_match_reopened_artifacts() { + let sources = [ + "pub fn fooBarValue() -> u32 { 1 }\n", + "pub fn foobarvalue() -> u32 { 2 }\n", + "pub struct FooBar;\nimpl FooBar { pub fn value() -> u32 { 3 } }\n", + "pub const Q: u32 = 4;\n", + ]; + let fixture = real_lexical_source_fixture_from_sources( + sources + .iter() + .enumerate() + .map(|(file, source)| { + ( + format!("file.case.{file}"), + format!("src/case_{file}.rs"), + source.as_bytes().to_vec(), + ) + }) + .collect(), + ); let directory = tempfile::tempdir().expect("artifact directory"); let control = ArtifactControl { cancelled: false }; - for revision in [ - CodeLexicalArtifactWriterRevisionV1::V11, - CodeLexicalArtifactWriterRevisionV1::V12, - CodeLexicalArtifactWriterRevisionV1::V13, - CodeLexicalArtifactWriterRevisionV1::V14, + let path = directory.path().join("case.sqlite"); + let mut builder = CodeLexicalArtifactBuilderV1::create(&path, fixture.metadata.clone()) + .expect("create artifact"); + let verified = builder + .rebuild_and_finalize(&mut fixture.open_source(128), &control) + .expect("build canonical pages"); + drop(builder); + let reader = CodeLexicalArtifactReaderV1::open_with_control( + &path, + &verified, + &fixture.metadata, + CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, + &control, + ) + .expect("reopen artifact"); + let authority = || { + CentralExactAdmissionAuthorityV1::new(id::("exact-rules.v1")) + }; + for (query, expected_matches) in [ + (r#""fooBarValue""#, true), + (r#""FooBar""#, true), + (r#""impl FooBar {""#, true), + (r#""Q""#, true), + (r#""oBa""#, true), + (r#""foobar""#, true), + (r#""FOOBAR""#, false), ] { - let path = directory.path().join(format!("common-{revision:?}.sqlite")); - let mut builder = CodeLexicalArtifactBuilderV1::create_with_format_revision( - &path, - fixture.metadata.clone(), - revision, - ) - .expect("create versioned artifact"); - let verified = builder - .rebuild_and_finalize(&mut fixture.open_source(128), &control) - .expect("build canonical pages"); - drop(builder); - let reader = CodeLexicalArtifactReaderV1::open_with_control( - &path, - &verified, - CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, - &control, - ) - .expect("reopen artifact"); - let actual = complete( - LexicalLane::new(reader) - .retrieve_lexical(&mixed) - .expect("mixed artifact query"), + let base = base_request(query, 8); + let view = query_view(query); + let request = ExactLaneRequest { + control: &ACTIVE_CONTROL, + literals: authority().parse_literals(&view, &base), + base, + query_view: &view, + generation: fixture.metadata.generation.clone(), + budget: budget(8), + }; + let batch = complete( + reader + .exact_adapter(authority()) + .read_exact_postings(&request) + .expect("artifact exact query"), ); assert_eq!( - actual, expected, - "{revision:?} must preserve canonical candidate parity" + !batch.candidates.is_empty(), + expected_matches, + "{query}: the artifact must admit exactly the case-sensitive raw matches" ); } } #[test] -fn writer_revision_toggle_preserves_v11_through_v14_lexical_results() { - let (fixture, pages, source_receipt) = real_verified_pages(); - let directory = tempfile::tempdir().expect("artifact tempdir"); - let v11_path = directory.path().join("writer-v11.sqlite"); - let v12_path = directory.path().join("writer-v12.sqlite"); - let v13_path = directory.path().join("writer-v13.sqlite"); - let control = ArtifactControl { cancelled: false }; - let mut v11_builder = CodeLexicalArtifactBuilderV1::create_with_format_revision( - &v11_path, - fixture.metadata.clone(), - CodeLexicalArtifactWriterRevisionV1::V11, - ) - .expect("create revision 11 artifact"); - for page in &pages { - v11_builder - .append_page(page, &control) - .expect("append v11 page"); - } - let v11 = finish_staged_artifact(&mut v11_builder, &source_receipt, &control); - let connection = rusqlite::Connection::open(&v11_path).expect("inspect v11 artifact"); - let revision: i64 = connection - .query_row( - "SELECT format_revision FROM artifact_state WHERE singleton = 1", - [], - |row| row.get(0), - ) - .expect("read v11 revision"); - assert_eq!(revision, 11); - let legacy_ngram_rows: i64 = connection - .query_row( - "SELECT COUNT(*) FROM ngram_postings WHERE substr(documents, 1, 4) = x'54444e31'", - [], - |row| row.get(0), - ) - .expect("count v11 ngram rows"); - assert!(legacy_ngram_rows > 0); - let exact_term_column: i64 = connection - .query_row( - "SELECT COUNT(*) FROM pragma_table_xinfo('exact_postings') WHERE name = 'term' AND type = 'BLOB'", - [], - |row| row.get(0), - ) - .expect("read v11 exact schema"); - assert_eq!(exact_term_column, 1); - drop(connection); - let v11_reader = CodeLexicalArtifactReaderV1::open_with_control( - &v11_path, - &v11, - CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, - &control, - ) - .expect("reopen revision 11 artifact"); - - let mut v12_builder = CodeLexicalArtifactBuilderV1::create_with_format_revision( - &v12_path, - fixture.metadata.clone(), - CodeLexicalArtifactWriterRevisionV1::V12, - ) - .expect("create revision 12 artifact"); - for page in &pages { - v12_builder - .append_page(page, &control) - .expect("append v12 page"); - } - let v12 = finish_staged_artifact(&mut v12_builder, &source_receipt, &control); - let v12_reader = CodeLexicalArtifactReaderV1::open_with_control( - &v12_path, - &v12, - CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, - &control, - ) - .expect("reopen revision 12 artifact"); - - let mut v13_builder = CodeLexicalArtifactBuilderV1::create_with_format_revision( - &v13_path, - fixture.metadata, - CodeLexicalArtifactWriterRevisionV1::V13, - ) - .expect("create revision 13 artifact"); - for page in &pages { - v13_builder - .append_page(page, &control) - .expect("append v13 page"); - } - let v13 = finish_staged_artifact(&mut v13_builder, &source_receipt, &control); - let connection = rusqlite::Connection::open(&v13_path).expect("inspect v13 artifact"); - let revision: i64 = connection - .query_row( - "SELECT format_revision FROM artifact_state WHERE singleton = 1", - [], - |row| row.get(0), - ) - .expect("read v13 revision"); - assert_eq!(revision, 13); - // Revision 13 clusters postings by document and serves term probes from - // the finalized term-leading covering index. - let document_leading_key: i64 = connection - .query_row( - "SELECT pk FROM pragma_table_xinfo('term_postings') WHERE name = 'document_id'", - [], - |row| row.get(0), - ) - .expect("read v13 term posting key"); - assert_eq!(document_leading_key, 1); - let term_probe_plan: String = connection - .query_row( - "EXPLAIN QUERY PLAN SELECT document_id FROM term_postings WHERE field = 1 AND term_id = 2", - [], - |row| row.get(3), - ) - .expect("explain v13 term probe"); +fn annotation_uses_mint_no_lexical_artifact_documents() { + let fixture = real_lexical_source_fixture_from_sources(vec![( + "file.annotated.001".to_owned(), + "src/annotated.rs".to_owned(), + b"#[derive(Debug, Clone)]\npub struct AnnotatedProbe;\n\n#[inline]\n#[must_use]\npub fn annotated_probe() -> u32 {\n 1\n}\n" + .to_vec(), + )]); + let (pages, _) = drain_verified_pages(&fixture, 128); + let source_chunks: usize = pages.iter().map(|page| page.chunks().len()).sum(); + let annotation_chunks = pages + .iter() + .flat_map(|page| page.symbol_displays()) + .flatten() + .filter(|display| display.kind() == "annotation_usage") + .count(); assert!( - term_probe_plan.contains("term_postings_by_term"), - "v13 term probe must use the covering term index, got {term_probe_plan}" + annotation_chunks > 0, + "the fixture's attributes must reach the lexical source as annotation-use chunks" ); - drop(connection); - let v13_reader = CodeLexicalArtifactReaderV1::open_with_control( - &v13_path, - &v13, - CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, - &control, - ) - .expect("reopen revision 13 artifact"); - let mut request = lexical_request("widget return", &["widget"], &[], &["return"], 2, 8); - request.generation = v11.generation().clone(); - let v11_result = v11_reader - .read_lexical_postings(&request) - .expect("read v11 lexical postings"); - request.generation = v12.generation().clone(); - let v12_result = v12_reader - .read_lexical_postings(&request) - .expect("read v12 lexical postings"); - assert_eq!(v12_result, v11_result); - request.generation = v13.generation().clone(); - let v13_result = v13_reader - .read_lexical_postings(&request) - .expect("read v13 lexical postings"); - assert_eq!(v13_result, v11_result); - - let v14_path = directory.path().join("writer-v14.sqlite"); - let mut v14_builder = CodeLexicalArtifactBuilderV1::create_with_format_revision( - &v14_path, - v13_reader.metadata().clone(), - CodeLexicalArtifactWriterRevisionV1::V14, - ) - .expect("create revision 14 artifact"); - for page in &pages { - v14_builder - .append_page(page, &control) - .expect("append v14 page"); - } - let v14 = finish_staged_artifact(&mut v14_builder, &source_receipt, &control); - let connection = rusqlite::Connection::open(&v14_path).expect("inspect v14 artifact"); - let revision: i64 = connection - .query_row( - "SELECT format_revision FROM artifact_state WHERE singleton = 1", - [], - |row| row.get(0), - ) - .expect("read v14 revision"); - assert_eq!(revision, 14); - // Revision 14 rows reference interned strings; the per-page staging - // table is dropped once the sealed dictionary is derived. - let (interned, staging_tables): (i64, i64) = connection - .query_row( - "SELECT (SELECT COUNT(*) FROM row_dictionary), \ - (SELECT COUNT(*) FROM sqlite_schema WHERE type = 'table' AND name = 'row_dictionary_pages')", - [], - |row| Ok((row.get(0)?, row.get(1)?)), - ) - .expect("inspect v14 dictionary"); - assert!(interned > 0, "v14 must intern a row dictionary"); + let directory = tempfile::tempdir().expect("artifact directory"); + let path = directory.path().join("annotated.sqlite"); + let control = ArtifactControl { cancelled: false }; + let mut builder = CodeLexicalArtifactBuilderV1::create(&path, fixture.metadata.clone()) + .expect("create artifact"); + let verified = builder + .rebuild_and_finalize(&mut fixture.open_source(128), &control) + .expect("build artifact"); + drop(builder); + let stored_rows: i64 = rusqlite::Connection::open(&path) + .expect("inspect artifact") + .query_row("SELECT COUNT(*) FROM row_chunks", [], |row| row.get(0)) + .expect("count rows"); assert_eq!( - staging_tables, 0, - "v14 must drop its staging table at finalization" - ); - let v13_row_bytes: i64 = rusqlite::Connection::open(&v13_path) - .expect("reopen v13 for row bytes") - .query_row("SELECT SUM(length(row)) FROM rows", [], |row| row.get(0)) - .expect("v13 row bytes"); - let v14_row_bytes: i64 = connection - .query_row("SELECT SUM(length(row)) FROM rows", [], |row| row.get(0)) - .expect("v14 row bytes"); - assert!( - v14_row_bytes * 2 < v13_row_bytes, - "v14 row payloads ({v14_row_bytes} B) must be under half of v13 ({v13_row_bytes} B)" + usize::try_from(stored_rows).expect("row count"), + source_chunks - annotation_chunks, + "every source chunk except annotation uses is one lexical document" ); - drop(connection); - let v14_reader = CodeLexicalArtifactReaderV1::open_with_control( - &v14_path, - &v14, + assert_eq!(verified.total_chunks() as usize, source_chunks); + let reader = CodeLexicalArtifactReaderV1::open_with_control( + &path, + &verified, + &fixture.metadata, CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, &control, ) - .expect("reopen revision 14 artifact"); - request.generation = v14.generation().clone(); - let v14_result = v14_reader - .read_lexical_postings(&request) - .expect("read v14 lexical postings"); - assert_eq!(v14_result, v11_result); + .expect("open artifact"); + let artifact = LexicalLane::new(reader.clone()); + let probe = |name: &str, fields: &[LexicalFieldV1], terms: &[&str]| { + ( + format!("src/annotated.rs::{name}"), + fields.to_vec(), + terms + .iter() + .map(|term| (*term).to_owned()) + .collect::>(), + ) + }; + let body = [LexicalFieldV1::BodyText]; + for (query, terms, drawn) in [ + ( + "must_use inline", + &["must_use", "inline"][..], + vec![ + probe("annotated_probe", &body, &["inline", "must_use"]), + probe("annotated_probe", &body, &["inline"]), + ], + ), + ( + "derive Debug", + &["derive", "debug"][..], + vec![ + probe("AnnotatedProbe", &body, &["debug", "derive"]), + probe("AnnotatedProbe", &body, &["debug", "derive"]), + ], + ), + ( + "annotated_probe", + &["annotated_probe"][..], + vec![ + probe( + "annotated_probe", + &[ + LexicalFieldV1::SymbolName, + LexicalFieldV1::BodyText, + LexicalFieldV1::ExactTerm, + ], + &["annotated_probe"], + ), + probe( + "annotated_probe", + &[LexicalFieldV1::SymbolName, LexicalFieldV1::Signature], + &["annotated_probe"], + ), + ], + ), + ] { + let mut request = lexical_request(query, terms, &[], &[], 0, 8); + request.generation = fixture.metadata.generation.clone(); + let expected = complete(artifact.retrieve_lexical(&request).expect("artifact query")); + let matched = scored_fields(&reader, &expected) + .into_iter() + .zip(&expected.candidates) + .map(|((name, fields), candidate)| { + let terms = &expected.evidence_by_occurrence[&candidate.source_occurrence_id] + .matched_whole_terms; + (name, fields, terms.clone()) + }) + .collect::>(); + assert_eq!( + matched, drawn, + "{query}: attribute text stays searchable through the item it annotates" + ); + assert_eq!( + expected.coverage.examined, + (source_chunks - annotation_chunks) as u64 + ); + } } /// Historical revision-10 artifact sealed by the pre-interning writer -/// (`tests/fixtures/lexical-artifact-v10.sqlite`). Readers must serve it; -/// a raw `term_id` SQL error is not an upgrade path. +/// (`tests/fixtures/lexical-artifact-v10.sqlite`). Readers refuse it as +/// incompatible, which withdraws the descriptor so the artifact is rebuilt. #[test] -fn reader_serves_historical_v10_writer_artifact() { +fn reader_refuses_historical_v10_writer_artifact_as_incompatible() { let checked_in = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/lexical-artifact-v10.sqlite"); let control = ArtifactControl { cancelled: false }; @@ -3217,25 +3027,19 @@ fn reader_serves_historical_v10_writer_artifact() { let digest = ManifestDigest::new(format!("sha256:{}", hex::encode(Sha256::digest(&bytes)))) .expect("v10 fixture digest"); - let reader = CodeLexicalArtifactReaderV1::open_content_addressed( + let Err(error) = CodeLexicalArtifactReaderV1::open_content_addressed( &artifact_path, &digest, file_size_bytes, + &projection_metadata(&id("generation.v10"), FreshnessCompatibilityV1::Current), CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, &control, - ) - .expect("readers accept sealed revision 10"); - let mut request = lexical_request("widget", &["widget"], &[], &[], 0, 8); - request.generation = reader.metadata().generation.clone(); - let RetrieverOutcome::Complete(batch) = reader - .read_lexical_postings(&request) - .expect("v10 lexical serving") - else { - panic!("v10 lexical read must complete, not stale or rebuild"); + ) else { + panic!("a revision-10 artifact must not be served"); }; assert!( - batch.coverage.eligible > 0, - "served v10 artifact must return widget candidates" + matches!(error, CodeLexicalArtifactErrorV1::Incompatible(_)), + "unexpected error: {error:?}" ); } @@ -3246,8 +3050,9 @@ fn sealed_current_artifact_uses_compact_postings_and_reports_dbstat() { let artifact_path = directory.path().join("current-plans.sqlite"); let control = ArtifactControl { cancelled: false }; let started = Instant::now(); - let mut builder = CodeLexicalArtifactBuilderV1::create(&artifact_path, fixture.metadata) - .expect("create artifact"); + let mut builder = + CodeLexicalArtifactBuilderV1::create(&artifact_path, fixture.metadata.clone()) + .expect("create artifact"); for page in &pages { builder.append_page(page, &control).expect("append page"); } @@ -3264,17 +3069,18 @@ fn sealed_current_artifact_uses_compact_postings_and_reports_dbstat() { |row| row.get(0), ) .expect("read current format revision"); - assert_eq!(format_revision, 16); - let uncompressed_ngram_rows: i64 = connection + assert_eq!(format_revision, 26); + let (ngram_lists, ngram_postings, untagged_ngram_lists): (i64, i64, i64) = connection .query_row( - "SELECT COUNT(*) FROM ngram_postings WHERE substr(documents, 1, 4) = x'54444e31' OR length(documents) > cardinality + 4", + "SELECT COUNT(*), SUM(document_frequency), SUM(substr(documents, 1, 1) NOT IN (x'00', x'01')) FROM ngram_postings", [], - |row| row.get(0), + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), ) - .expect("count non-delta ngram rows"); + .expect("inspect sealed ngram lists"); + assert!(ngram_lists > 0 && ngram_postings >= ngram_lists); assert_eq!( - uncompressed_ngram_rows, 0, - "current ngram shards must use canonical delta varints" + untagged_ngram_lists, 0, + "every sealed ngram list is a tagged delta-varint list or bitset" ); let exact_columns = connection .prepare( @@ -3290,66 +3096,56 @@ fn sealed_current_artifact_uses_compact_postings_and_reports_dbstat() { [ ("term_id".to_owned(), "INTEGER".to_owned()), ("field".to_owned(), "INTEGER".to_owned()), - ("document_id".to_owned(), "INTEGER".to_owned()), + ("documents".to_owned(), "BLOB".to_owned()), ] ); let term_plan = connection - .prepare( - "EXPLAIN QUERY PLAN SELECT document_id FROM term_postings WHERE field = ?1 AND term_id = ?2", - ) + .prepare("EXPLAIN QUERY PLAN SELECT lists FROM term_postings WHERE term = ?1") .expect("prepare term plan") - .query_map(rusqlite::params![4i64, 1i64], |row| row.get::<_, String>(3)) + .query_map(rusqlite::params!["widget"], |row| row.get::<_, String>(3)) .expect("query term plan") .collect::, _>>() .expect("collect term plan"); assert!( term_plan .iter() - .any(|detail| detail.contains("term_postings_by_term")), - "term equality must use the covering term index, got {term_plan:?}" - ); - let frequency_plan = connection - .prepare( - "EXPLAIN QUERY PLAN SELECT posting.field, posting.frequency \ - FROM term_postings AS posting \ - WHERE posting.document_id = 0 AND posting.term_id IN (1)", - ) - .expect("prepare frequency plan") - .query_map([], |row| row.get::<_, String>(3)) - .expect("query frequency plan") - .collect::, _>>() - .expect("collect frequency plan"); - assert!( - frequency_plan - .iter() - .any(|detail| detail.contains("PRIMARY KEY")), - "frequency probe must use the document-leading primary key, got {frequency_plan:?}" + .any(|detail| detail.contains("USING PRIMARY KEY")), + "a term's lists must be one clustered-key seek, got {term_plan:?}" ); - let missing_dropped: i64 = connection + let redundant_structures: i64 = connection .query_row( - "SELECT COUNT(*) FROM sqlite_schema WHERE type = 'index' AND name IN \ - ('term_postings_by_document', 'term_postings_by_document_term', 'term_stats_by_term')", + "SELECT COUNT(*) FROM sqlite_schema WHERE (type = 'index' AND name NOT LIKE 'sqlite_autoindex_clone_%') \ + OR (type = 'table' AND name IN ('rows', 'vocabulary', 'term_stats', 'ngram_statistics', 'document_integrity', 'import_integrity', 'term_posting_runs', 'exact_posting_runs', 'ngram_posting_pages', 'row_chunk_pages'))", [], |row| row.get(0), ) - .expect("count dropped indexes"); + .expect("count redundant structures"); + assert_eq!( + redundant_structures, 0, + "the current revision keeps one physical order per family, no secondary index, and no derivable tables" + ); + let freelist_pages: i64 = connection + .query_row("PRAGMA freelist_count", [], |row| row.get(0)) + .expect("read freelist"); assert_eq!( - missing_dropped, 0, - "the current revision must not keep redundant indexes" + freelist_pages, 0, + "finalization returns every dropped staging page to the filesystem" ); - let binary_rows: i64 = connection + let (blocks, documents, untagged_blocks): (i64, i64, i64) = connection .query_row( - "SELECT COUNT(*) FROM rows WHERE substr(row, 1, 7) = x'54444c52313600'", + "SELECT (SELECT COUNT(*) FROM row_blocks), (SELECT COUNT(*) FROM row_chunks), \ + (SELECT COUNT(*) FROM row_blocks WHERE substr(payload, 1, 1) != x'17')", [], - |row| row.get(0), + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), ) - .expect("count binary rows"); - let total_rows: i64 = connection - .query_row("SELECT COUNT(*) FROM rows", [], |row| row.get(0)) - .expect("count rows"); + .expect("count row blocks"); assert_eq!( - binary_rows, total_rows, - "every revision-16 row carries the binary tag" + untagged_blocks, 0, + "every row block carries the current tag" + ); + assert!( + blocks < documents && blocks * 32 >= documents, + "rows are grouped into blocks of at most 32: {blocks} blocks for {documents} rows" ); let (interned_strings, staging_tables): (i64, i64) = connection .query_row( @@ -3382,8 +3178,10 @@ fn sealed_current_artifact_uses_compact_postings_and_reports_dbstat() { "dbstat must account interned postings: {sizes:?}" ); assert!( - !sizes.keys().any(|name| name == "term_postings_by_document"), - "dbstat must not retain the superseded document-leading index: {sizes:?}" + !sizes + .keys() + .any(|name| name.starts_with("term_postings_by") || name.ends_with("_runs")), + "dbstat must not retain a secondary posting order or staging run: {sizes:?}" ); assert!( sizes.contains_key("exact_vocabulary"), @@ -3394,13 +3192,13 @@ fn sealed_current_artifact_uses_compact_postings_and_reports_dbstat() { "dbstat must account the sealed row dictionary and no staging table: {sizes:?}" ); eprintln!( - "lexical v14 dbstat file_bytes={file_bytes} build_ms={build_ms} pages={} digest={} sizes={sizes:?}", + "lexical v23 dbstat file_bytes={file_bytes} build_ms={build_ms} pages={} digest={} sizes={sizes:?}", verified.page_count(), verified.artifact_digest().as_str(), ); } else { eprintln!( - "lexical v14 size file_bytes={file_bytes} build_ms={build_ms} pages={} digest={} (dbstat unavailable)", + "lexical v23 size file_bytes={file_bytes} build_ms={build_ms} pages={} digest={} (dbstat unavailable)", verified.page_count(), verified.artifact_digest().as_str(), ); @@ -3411,10 +3209,11 @@ fn sealed_current_artifact_uses_compact_postings_and_reports_dbstat() { let reader = CodeLexicalArtifactReaderV1::open_with_control( &artifact_path, &verified, + &fixture.metadata, CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, &control, ) - .expect("open v13 artifact"); + .expect("open current artifact"); let mut request = lexical_request( "rendre return value", &["rendre"], @@ -3423,45 +3222,47 @@ fn sealed_current_artifact_uses_compact_postings_and_reports_dbstat() { 2, 8, ); - request.generation = verified.generation().clone(); + request.generation = reader.metadata().generation.clone(); let lane = LexicalLane::new(reader); let mut latencies = Vec::new(); for _ in 0..16 { let started = Instant::now(); - let _ = lane.retrieve_lexical(&request).expect("v13 lexical query"); + let _ = lane + .retrieve_lexical(&request) + .expect("current lexical query"); latencies.push(started.elapsed().as_micros()); } latencies.sort_unstable(); let p50 = latencies[latencies.len() / 2]; let p95 = latencies[(latencies.len() * 95) / 100]; - eprintln!("lexical v13 query_us p50={p50} p95={p95} samples={latencies:?}"); + eprintln!("lexical v19 query_us p50={p50} p95={p95} samples={latencies:?}"); assert!(p50 > 0 || file_bytes > 0); } #[test] -fn reader_rejects_current_artifact_missing_required_term_statistics_index() { +fn reader_rejects_current_artifact_missing_its_chunk_lookup_table() { let (fixture, pages, source_receipt) = real_verified_pages(); let directory = tempfile::tempdir().expect("artifact tempdir"); - let artifact_path = directory - .path() - .join("missing-term-statistics-index.sqlite"); + let artifact_path = directory.path().join("missing-chunk-lookup-index.sqlite"); let control = ArtifactControl { cancelled: false }; - let mut builder = CodeLexicalArtifactBuilderV1::create(&artifact_path, fixture.metadata) - .expect("create artifact"); + let mut builder = + CodeLexicalArtifactBuilderV1::create(&artifact_path, fixture.metadata.clone()) + .expect("create artifact"); for page in &pages { builder.append_page(page, &control).expect("append page"); } let verified = finish_staged_artifact(&mut builder, &source_receipt, &control); let connection = rusqlite::Connection::open(&artifact_path).expect("open artifact mutation"); connection - .execute_batch("DROP INDEX term_postings_by_term;") - .expect("remove required term-leading posting index"); + .execute_batch("DROP TABLE row_chunks;") + .expect("remove required chunk lookup table"); drop(connection); assert!(matches!( CodeLexicalArtifactReaderV1::open_with_control( &artifact_path, &verified, + &fixture.metadata, CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, &control, ), @@ -3492,7 +3293,13 @@ fn disk_artifact_defers_statistics_and_serving_indexes_until_freeze() { .collect::>() .expect("read index inventory"); assert_eq!(staging_indexes, Vec::::new()); - for table in ["field_stats", "term_stats"] { + for table in [ + "field_stats", + "term_postings", + "exact_postings", + "ngram_postings", + "row_chunks", + ] { let rows: i64 = connection .query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |row| { row.get(0) @@ -3500,18 +3307,23 @@ fn disk_artifact_defers_statistics_and_serving_indexes_until_freeze() { .expect("count deferred statistic rows"); assert_eq!(rows, 0, "{table} must be derived after the base freeze"); } - let vocabulary_rows: i64 = connection - .query_row("SELECT COUNT(*) FROM vocabulary", [], |row| row.get(0)) - .expect("count interned vocabulary"); - assert!( - vocabulary_rows > 0, - "revision 11 interns terms during append, before statistics freeze" - ); + for table in [ + "term_posting_runs", + "exact_posting_runs", + "row_chunk_pages", + "row_blocks", + ] { + let rows: i64 = connection + .query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |row| { + row.get(0) + }) + .expect("count staged rows"); + assert!(rows > 0, "{table} is written during append"); + } let authority_rows: i64 = connection .query_row( "SELECT (SELECT COUNT(*) FROM source_pages) + \ - (SELECT COUNT(*) FROM document_integrity) + \ - (SELECT COUNT(*) FROM import_integrity) + \ + (SELECT COUNT(*) FROM row_chunk_pages) + \ (SELECT COUNT(*) FROM import_evidence)", [], |row| row.get(0), @@ -3537,7 +3349,7 @@ fn disk_artifact_defers_statistics_and_serving_indexes_until_freeze() { assert!( connection .execute( - "UPDATE rows SET row = row WHERE document_id = (SELECT MIN(document_id) FROM rows)", + "UPDATE row_blocks SET payload = payload WHERE first_document = (SELECT MIN(first_document) FROM row_blocks)", [], ) .is_err(), @@ -3557,19 +3369,11 @@ fn disk_artifact_defers_statistics_and_serving_indexes_until_freeze() { .expect("query final index inventory") .collect::>() .expect("read final index inventory"); - assert_eq!( - serving_indexes, - [ - "exact_postings_by_document", - "ngram_postings_by_ngram", - "rows_by_chunk", - "term_postings_by_term", - ] - ); + assert_eq!(serving_indexes, Vec::::new()); // `field_stats` is sealed from running totals the append phase carried, - // and the fuzzy flag is derived from `term_stats`; both must agree - // exactly (no extra, missing, or differing rows) with a fresh scan of - // the postings they summarize. + // each list's document frequency and each term's fuzzy flag from the + // merge; all must agree exactly with a fresh decode of the postings + // they summarize. let field_stats_rows: i64 = connection .query_row("SELECT COUNT(*) FROM field_stats", [], |row| row.get(0)) .expect("count field statistics"); @@ -3577,33 +3381,43 @@ fn disk_artifact_defers_statistics_and_serving_indexes_until_freeze() { field_stats_rows > 0, "the fixture must index at least one field" ); - let field_stats_divergence: i64 = connection - .query_row( - "SELECT (SELECT COUNT(*) FROM (SELECT field, total_length FROM field_stats EXCEPT SELECT field, SUM(frequency) FROM term_postings GROUP BY field)) \ - + (SELECT COUNT(*) FROM (SELECT field, SUM(frequency) FROM term_postings GROUP BY field EXCEPT SELECT field, total_length FROM field_stats))", - [], - |row| row.get(0), - ) - .expect("compare field statistics"); - let term_stats_divergence: i64 = connection - .query_row( - "SELECT (SELECT COUNT(*) FROM (SELECT term_id, field, document_frequency FROM term_stats EXCEPT SELECT term_id, field, COUNT(*) FROM term_postings GROUP BY term_id, field)) \ - + (SELECT COUNT(*) FROM (SELECT term_id, field, COUNT(*) FROM term_postings GROUP BY term_id, field EXCEPT SELECT term_id, field, document_frequency FROM term_stats))", - [], - |row| row.get(0), - ) - .expect("compare term statistics"); - // Field code 7 is the subtoken field of every shipped layout. - let fuzzy_flag_divergence: i64 = connection - .query_row( - "SELECT COUNT(*) FROM vocabulary WHERE in_fuzzy != EXISTS(SELECT 1 FROM term_postings WHERE term_postings.term_id = vocabulary.term_id AND term_postings.field != 7)", - [], - |row| row.get(0), - ) - .expect("compare fuzzy vocabulary flags"); + let mut decoded_field_totals = BTreeMap::::new(); + let mut term_stats_divergence = 0i64; + let mut fuzzy_flag_divergence = 0i64; + for (in_fuzzy, lists) in connection + .prepare("SELECT in_fuzzy, lists FROM term_postings") + .expect("prepare sealed term lists") + .query_map([], |row| { + Ok((row.get::<_, bool>(0)?, row.get::<_, Vec>(1)?)) + }) + .expect("read sealed term lists") + .collect::, _>>() + .expect("collect sealed term lists") + { + let lists = decode_term_lists_oracle(&lists); + // Field code 7 is the subtoken field. + fuzzy_flag_divergence += + i64::from(in_fuzzy != lists.iter().any(|(field, _, _)| *field != 7)); + for (field, document_frequency, postings) in lists { + let decoded = decode_frequency_posting_list(&postings); + term_stats_divergence += i64::from(decoded.len() as i64 != document_frequency); + *decoded_field_totals.entry(field).or_default() += decoded + .iter() + .map(|(_, frequency)| i64::from(*frequency)) + .sum::(); + } + } + let sealed_field_totals = connection + .prepare("SELECT field, total_length FROM field_stats ORDER BY field") + .expect("prepare field statistics") + .query_map([], |row| Ok((row.get::<_, i64>(0)?, row.get::<_, i64>(1)?))) + .expect("read field statistics") + .collect::, _>>() + .expect("collect field statistics"); + let field_stats_divergence = i64::from(sealed_field_totals != decoded_field_totals); let staging_tables: i64 = connection .query_row( - "SELECT COUNT(*) FROM sqlite_schema WHERE type = 'table' AND name IN ('field_stats_staging', 'row_dictionary_pages')", + "SELECT COUNT(*) FROM sqlite_schema WHERE type = 'table' AND name IN ('field_stats_staging', 'row_dictionary_pages', 'row_chunk_pages', 'term_posting_runs', 'exact_posting_runs', 'ngram_posting_pages')", [], |row| row.get(0), ) @@ -3628,8 +3442,7 @@ fn disk_artifact_production_wake_commits_one_restartable_setwise_step() { builder.append_page(page, &control).expect("append page"); } - // Revision 13 clusters postings by document, so its serving indexes are - // built before the statistics that read them in key order. + // Posting merges run before the statistics that read the sealed keys. assert!(matches!( builder .advance_finalization(&source_receipt, 4_096, &control) @@ -3643,7 +3456,7 @@ fn disk_artifact_production_wake_commits_one_restartable_setwise_step() { assert!(matches!( builder .advance_finalization(&source_receipt, 4_096, &control) - .expect("build only the chunk index"), + .expect("build only the chunk lookup"), CodeLexicalArtifactFinalizationStepV1::Pending { .. } )); assert_eq!( @@ -3659,7 +3472,7 @@ fn disk_artifact_production_wake_commits_one_restartable_setwise_step() { CODE_LEXICAL_ARTIFACT_BUILD_MEMORY_BUDGET_BYTES_V1, &control, ) - .expect("restart after committed chunk index"); + .expect("restart after committed chunk lookup"); let cancellation = CancelOnBackgroundObservation::new(); assert!(matches!( resumed.advance_finalization(&source_receipt, 4_096, &cancellation), @@ -3668,23 +3481,21 @@ fn disk_artifact_production_wake_commits_one_restartable_setwise_step() { )) )); assert_eq!( - persisted_finalization_position(&artifact_path), - ("indexes".to_owned(), 1), - "cancellation inside the next SQLite statement must not advance its durable state" - ); - let connection = rusqlite::Connection::open(&artifact_path).expect("inspect cancelled step"); - let committed_indexes: Vec = connection - .prepare( - "SELECT name FROM sqlite_schema WHERE type = 'index' AND name NOT LIKE 'sqlite_autoindex_%' ORDER BY name", - ) - .expect("prepare index inventory") - .query_map([], |row| row.get(0)) - .expect("query index inventory") - .collect::>() - .expect("read index inventory"); - assert_eq!( - committed_indexes, - ["rows_by_chunk"], + persisted_finalization_position(&artifact_path), + ("indexes".to_owned(), 1), + "cancellation inside the next SQLite statement must not advance its durable state" + ); + let connection = rusqlite::Connection::open(&artifact_path).expect("inspect cancelled step"); + let (chunk_lookups, term_runs, sealed_terms): (i64, i64, i64) = connection + .query_row( + "SELECT (SELECT COUNT(*) FROM row_chunks), (SELECT COUNT(*) FROM term_posting_runs), \ + (SELECT COUNT(*) FROM term_postings)", + [], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .expect("inspect committed steps"); + assert!( + chunk_lookups > 0 && term_runs > 0 && sealed_terms == 0, "the prior committed step survives cancellation and the interrupted step rolls back atomically" ); drop(connection); @@ -3696,10 +3507,10 @@ fn disk_artifact_production_wake_commits_one_restartable_setwise_step() { CODE_LEXICAL_ARTIFACT_BUILD_MEMORY_BUDGET_BYTES_V1, &control, ) - .expect("restart after cancelled term index"); + .expect("restart after cancelled term merge"); resumed .advance_finalization(&source_receipt, 4_096, &control) - .expect("retry only the term index"); + .expect("retry only the term merge"); assert_eq!( persisted_finalization_position(&artifact_path), ("indexes".to_owned(), 2), @@ -3707,15 +3518,15 @@ fn disk_artifact_production_wake_commits_one_restartable_setwise_step() { ); drop(resumed); - // Remaining index steps (exact, ngram, ngram statistics), then the three - // statistics steps, each committed by exactly one restarted wake. + // The remaining index steps (exact merge, n-gram rebuild from rows), then + // the two statistics steps (field totals, releasing every dropped + // staging page), each committed by exactly one restarted wake. No step + // builds a secondary index. let expected_positions = [ - ("indexes", 3, 3), - ("indexes", 4, 4), - ("statistics", 0, 4), - ("statistics", 1, 4), - ("statistics", 2, 4), - ("digest", 0, 4), + ("indexes", 3, 0), + ("statistics", 0, 0), + ("statistics", 1, 0), + ("digest", 0, 0), ]; for (phase, ordinal, expected_indexes) in expected_positions { let mut resumed = @@ -3747,29 +3558,150 @@ fn disk_artifact_production_wake_commits_one_restartable_setwise_step() { indexes, expected_indexes, "each restarted production wake commits at most one serving index" ); - if (phase, ordinal) == ("statistics", 0) { - let ngram_statistics: i64 = connection - .query_row("SELECT COUNT(*) FROM ngram_statistics", [], |row| { - row.get(0) - }) - .expect("count derived ngram statistics"); + if (phase, ordinal) == ("digest", 0) { + let (sealed_lists, staging_tables, freelist): (i64, i64, i64) = connection + .query_row( + "SELECT (SELECT COUNT(*) FROM term_postings) + (SELECT COUNT(*) FROM exact_postings) + (SELECT COUNT(*) FROM ngram_postings), \ + (SELECT COUNT(*) FROM sqlite_schema WHERE type = 'table' AND name IN ('term_posting_runs', 'exact_posting_runs', 'ngram_posting_pages')), \ + (SELECT freelist_count FROM pragma_freelist_count)", + [], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .expect("inspect merged postings"); assert!( - ngram_statistics > 0, - "the final index-phase wake derives ngram statistics from committed postings" + sealed_lists > 0, + "the index phase merges committed staging into sealed lists" + ); + assert_eq!(staging_tables, 0, "every merged staging table is dropped"); + assert_eq!( + freelist, 0, + "the final pre-digest wake releases their pages" ); } if (phase, ordinal) == ("digest", 0) { - let term_statistics: i64 = connection - .query_row("SELECT COUNT(*) FROM term_stats", [], |row| row.get(0)) - .expect("count derived term statistics"); + let fuzzy_terms: i64 = connection + .query_row( + "SELECT COUNT(*) FROM term_postings WHERE in_fuzzy = 1", + [], + |row| row.get(0), + ) + .expect("count derived fuzzy vocabulary"); assert!( - term_statistics > 0, - "the statistics phase derives term statistics from the committed term index" + fuzzy_terms > 0, + "the term merge derives the fuzzy vocabulary from the sealed term lists" ); } } } +/// Independent oracle for the sealed term-list format: LEB128 varints, each +/// document delta shifted left one bit whose low bit announces a following +/// frequency varint (frequency one otherwise). +fn decode_frequency_posting_list(mut encoded: &[u8]) -> Vec<(u32, u32)> { + fn take(encoded: &mut &[u8]) -> u64 { + let mut value = 0u64; + let mut shift = 0; + loop { + let (byte, rest) = encoded.split_first().expect("truncated posting varint"); + *encoded = rest; + value |= u64::from(byte & 0x7f) << shift; + if byte & 0x80 == 0 { + return value; + } + shift += 7; + } + } + let mut postings = Vec::new(); + let mut previous: Option = None; + while !encoded.is_empty() { + let token = take(&mut encoded); + let frequency = if token & 1 == 1 { + u32::try_from(take(&mut encoded)).expect("frequency fits u32") + } else { + 1 + }; + let delta = u32::try_from(token >> 1).expect("delta fits u32"); + let document = previous.map_or(delta, |previous| previous + delta); + postings.push((document, frequency)); + previous = Some(document); + } + postings +} + +/// Independent oracle for one sealed `term_postings.lists` value: per field, +/// LEB128 field code, document frequency, and length, then the list bytes. +fn decode_term_lists_oracle(mut encoded: &[u8]) -> Vec<(i64, i64, Vec)> { + fn take(encoded: &mut &[u8]) -> u64 { + let mut value = 0u64; + let mut shift = 0; + loop { + let (byte, rest) = encoded.split_first().expect("truncated term-list varint"); + *encoded = rest; + value |= u64::from(byte & 0x7f) << shift; + if byte & 0x80 == 0 { + return value; + } + shift += 7; + } + } + let mut lists = Vec::new(); + while !encoded.is_empty() { + let field = i64::try_from(take(&mut encoded)).expect("field code"); + let document_frequency = i64::try_from(take(&mut encoded)).expect("document frequency"); + let length = usize::try_from(take(&mut encoded)).expect("list length"); + let (list, rest) = encoded.split_at(length); + encoded = rest; + lists.push((field, document_frequency, list.to_vec())); + } + lists +} + +/// Postings staged in one run table, counted by decoding every run with the +/// independent format oracle. +fn staged_posting_count(path: &Path, table: &str, frequencies: bool) -> usize { + let connection = rusqlite::Connection::open(path).expect("open staged runs"); + let column = if frequencies { "postings" } else { "documents" }; + connection + .prepare(&format!("SELECT {column} FROM {table}")) + .expect("prepare staged runs") + .query_map([], |row| row.get::<_, Vec>(0)) + .expect("read staged runs") + .map(|run| { + let run = run.expect("staged run"); + if frequencies { + decode_frequency_posting_list(&run).len() + } else { + decode_document_list(&run).len() + } + }) + .sum() +} + +/// Independent oracle for sealed document sets without frequencies: LEB128 +/// document deltas, the first absolute. +fn decode_document_list(mut encoded: &[u8]) -> Vec { + let mut documents = Vec::new(); + let mut previous: Option = None; + while !encoded.is_empty() { + let mut delta = 0u64; + let mut shift = 0; + loop { + let (byte, rest) = encoded.split_first().expect("truncated document varint"); + encoded = rest; + delta |= u64::from(byte & 0x7f) << shift; + if byte & 0x80 == 0 { + break; + } + shift += 7; + } + let delta = u32::try_from(delta).expect("delta fits u32"); + let document = previous.map_or(delta, |previous| previous + delta); + documents.push(document); + previous = Some(document); + } + documents +} + fn persisted_finalization_position(path: &Path) -> (String, u64) { let connection = rusqlite::Connection::open(path).expect("open finalization state"); let state: Vec = connection @@ -3852,13 +3784,13 @@ fn disk_artifact_term_insert_execution_is_monotone_by_primary_key() { .execute_batch( "CREATE TABLE term_insert_trace ( sequence INTEGER PRIMARY KEY AUTOINCREMENT, - term_id INTEGER NOT NULL, - field INTEGER NOT NULL, - document_id INTEGER NOT NULL + page_ordinal INTEGER NOT NULL, + term TEXT NOT NULL, + field INTEGER NOT NULL ); - CREATE TRIGGER trace_term_insert AFTER INSERT ON term_postings BEGIN - INSERT INTO term_insert_trace(term_id, field, document_id) - VALUES (NEW.term_id, NEW.field, NEW.document_id); + CREATE TRIGGER trace_term_insert AFTER INSERT ON term_posting_runs BEGIN + INSERT INTO term_insert_trace(page_ordinal, term, field) + VALUES (NEW.page_ordinal, NEW.term, NEW.field); END;", ) .expect("install term insert observer"); @@ -3868,22 +3800,25 @@ fn disk_artifact_term_insert_execution_is_monotone_by_primary_key() { .append_pages(&pages, &ArtifactControl { cancelled: false }) .expect("append observed term postings"); let trace = rusqlite::Connection::open(&artifact_path).expect("read term insert observer"); - // Revision 13 clusters `term_postings` by `(document_id, term_id, field)`, + // Batches stage `term_posting_runs` keyed `(page_ordinal, term, field)`, // so that is the order a monotone insert stream must follow. let keys = trace - .prepare("SELECT document_id, term_id, field FROM term_insert_trace ORDER BY sequence") + .prepare("SELECT page_ordinal, term, field FROM term_insert_trace ORDER BY sequence") .expect("prepare term insert trace") .query_map([], |row| { Ok(( row.get::<_, i64>(0)?, - row.get::<_, i64>(1)?, + row.get::<_, String>(1)?, row.get::<_, i64>(2)?, )) }) .expect("query term insert trace") .collect::, _>>() .expect("read term insert trace"); - assert!(keys.len() > 1, "fixture must emit multiple term postings"); + assert!( + keys.len() > 1, + "fixture must emit multiple term posting runs" + ); let resets = keys.windows(2).filter(|pair| pair[1] < pair[0]).count(); assert_eq!( resets, 0, @@ -3893,7 +3828,7 @@ fn disk_artifact_term_insert_execution_is_monotone_by_primary_key() { #[test] fn disk_artifact_posting_insert_plans_obey_exact_memory_boundary_before_mutation() { - const TERM_INSERT_PLAN_BYTES_PER_REF: usize = 4 * std::mem::size_of::(); + const TERM_INSERT_PLAN_BYTES_PER_REF: usize = 5 * std::mem::size_of::(); const TERM_INSERT_SORT_RUN_ROWS: usize = 4_096; const EXACT_INSERT_PLAN_BYTES_PER_REF: usize = 8 * std::mem::size_of::(); const EXACT_INSERT_SORT_RUN_ROWS: usize = TERM_INSERT_SORT_RUN_ROWS; @@ -3916,35 +3851,23 @@ fn disk_artifact_posting_insert_plans_obey_exact_memory_boundary_before_mutation probe .append_prepared_pages(&prepared, &control) .expect("append posting plans probe"); - let term_rows = rusqlite::Connection::open(&probe_path) - .expect("open posting plans probe for term rows") - .query_row("SELECT COUNT(*) FROM term_postings", [], |row| { - row.get::<_, i64>(0) - }) - .expect("count prepared term rows"); - let term_rows = usize::try_from(term_rows).expect("term row count"); + let term_rows = staged_posting_count(&probe_path, "term_posting_runs", true); assert!(term_rows > 0, "fixture must emit term postings"); - let exact_rows = rusqlite::Connection::open(&probe_path) - .expect("open exact plan probe") - .query_row("SELECT COUNT(*) FROM exact_postings", [], |row| { - row.get::<_, i64>(0) - }) - .expect("count prepared exact rows"); - let exact_rows = usize::try_from(exact_rows).expect("exact row count"); + let exact_rows = staged_posting_count(&probe_path, "exact_posting_runs", false); assert!(exact_rows > 0, "fixture must emit exact postings"); let entry_ledger = term_rows .checked_mul(TERM_INSERT_PLAN_BYTES_PER_REF) .expect("term plan ledger charge"); let merge_heap_ledger = term_rows .div_ceil(TERM_INSERT_SORT_RUN_ROWS) - .checked_mul(std::mem::size_of::<(i64, i64, i64, usize, usize, usize)>()) + .checked_mul(std::mem::size_of::<(&str, i64, i64, usize, usize, usize)>()) .expect("term merge heap ledger charge"); let exact_entry_ledger = exact_rows .checked_mul(EXACT_INSERT_PLAN_BYTES_PER_REF) .expect("exact plan ledger charge"); let exact_merge_heap_ledger = exact_rows .div_ceil(EXACT_INSERT_SORT_RUN_ROWS) - .checked_mul(std::mem::size_of::<[usize; 10]>()) + .checked_mul(std::mem::size_of::<(i64, i64, i64, usize, usize)>()) .expect("exact merge heap ledger charge"); let plan_ledger = entry_ledger .checked_add(merge_heap_ledger) @@ -3981,15 +3904,11 @@ fn disk_artifact_posting_insert_plans_obey_exact_memory_boundary_before_mutation 0 ); assert_eq!(staged_row_cardinality(&refused_path), (0, 0)); - let refused_term_rows: i64 = rusqlite::Connection::open(&refused_path) - .expect("open refused posting plans artifact") - .query_row("SELECT COUNT(*) FROM term_postings", [], |row| row.get(0)) - .expect("count refused term rows"); - assert_eq!(refused_term_rows, 0); - let refused_exact_rows: i64 = rusqlite::Connection::open(&refused_path) - .expect("open refused posting plans artifact for exact rows") - .query_row("SELECT COUNT(*) FROM exact_postings", [], |row| row.get(0)) - .expect("count refused exact rows"); + assert_eq!( + staged_posting_count(&refused_path, "term_posting_runs", true), + 0 + ); + let refused_exact_rows = staged_posting_count(&refused_path, "exact_posting_runs", false); assert_eq!( refused_exact_rows, 0, "memory refusal must not write exact postings" @@ -4086,13 +4005,7 @@ fn disk_artifact_term_run_sort_observes_cancellation_before_transaction_entry() probe .append_prepared_pages(&prepared, &control) .expect("append term-run probe"); - let term_rows = rusqlite::Connection::open(&probe_path) - .expect("open term-run probe") - .query_row("SELECT COUNT(*) FROM term_postings", [], |row| { - row.get::<_, i64>(0) - }) - .expect("count term-run rows"); - let term_rows = usize::try_from(term_rows).expect("term-run row count"); + let term_rows = staged_posting_count(&probe_path, "term_posting_runs", true); assert!( term_rows > TERM_SORT_RUN_ROWS, "fixture must require at least two bounded sort runs: {term_rows}" @@ -4665,7 +4578,8 @@ fn disk_artifact_finalization_resumes_after_restart_without_source_replay() { // The restart must continue from SQLite state. Clearing this fixture's // only raw copy makes a source replay impossible in the finalization path. - fixture.sealed.clear(); + fixture.manifest.clear(); + fixture.segments = Arc::default(); let mut resumed = CodeLexicalArtifactBuilderV1::open_or_resume_with_memory_budget_and_control( &artifact_path, metadata, @@ -4812,11 +4726,11 @@ fn disk_artifact_revision_four_is_incompatible_before_new_index_queries() { } #[test] -fn disk_artifact_resume_rejects_current_revision_with_wrong_term_index_shape() { +fn disk_artifact_resume_rejects_current_revision_with_wrong_chunk_lookup_shape() { let (fixture, pages, source_receipt) = real_verified_pages(); let metadata = fixture.metadata.clone(); let directory = tempfile::tempdir().expect("artifact tempdir"); - let artifact_path = directory.path().join("wrong-term-index-shape.sqlite"); + let artifact_path = directory.path().join("wrong-chunk-index-shape.sqlite"); let control = ArtifactControl { cancelled: false }; let mut builder = CodeLexicalArtifactBuilderV1::create(&artifact_path, metadata.clone()).expect("create"); @@ -4831,7 +4745,7 @@ fn disk_artifact_resume_rejects_current_revision_with_wrong_term_index_shape() { .expect("freeze current artifact"), CodeLexicalArtifactFinalizationStepV1::Pending { .. } )); - for _ in 0..8 { + for _ in 0..6 { assert!(matches!( builder .advance_finalization(&source_receipt, 4_096, &control) @@ -4841,13 +4755,17 @@ fn disk_artifact_resume_rejects_current_revision_with_wrong_term_index_shape() { } drop(builder); - let connection = rusqlite::Connection::open(&artifact_path).expect("open index mutation"); + let connection = rusqlite::Connection::open(&artifact_path).expect("open lookup mutation"); connection .execute_batch( - "DROP INDEX term_postings_by_term; - CREATE INDEX term_postings_by_term ON term_postings(term_id, document_id, field, frequency);", + "DROP TABLE row_chunks; + CREATE TABLE row_chunks ( + document_id INTEGER NOT NULL, + chunk_id BLOB NOT NULL, + PRIMARY KEY(document_id, chunk_id) + ) WITHOUT ROWID;", ) - .expect("replace term-leading posting index with wrong column order"); + .expect("replace the chunk lookup table with the wrong key order"); drop(connection); assert!(matches!( @@ -4887,7 +4805,7 @@ fn disk_artifact_finalization_refuses_inter_wake_mutation() { assert!( connection .execute( - "UPDATE rows SET row = row WHERE document_id = (SELECT MIN(document_id) FROM rows)", + "UPDATE row_blocks SET payload = payload WHERE first_document = (SELECT MIN(first_document) FROM row_blocks)", [], ) .is_err(), @@ -4895,12 +4813,13 @@ fn disk_artifact_finalization_refuses_inter_wake_mutation() { ); drop(connection); - finish_staged_artifact(&mut builder, &source_receipt, &control); + let sealed = finish_staged_artifact(&mut builder, &source_receipt, &control); assert_eq!( - builder.progress().expect("source progress after refusal"), - staged, + staged.next_page_ordinal, + sealed.page_count(), "a changed artifact must not self-attest through later bounded wakes" ); + assert_eq!(staged.completed_chunks, sealed.total_chunks()); } #[test] @@ -4909,8 +4828,9 @@ fn disk_artifact_rejects_noncanonical_receipt_reservation_tail() { let directory = tempfile::tempdir().expect("artifact tempdir"); let artifact_path = directory.path().join("noncanonical-receipt.sqlite"); let control = ArtifactControl { cancelled: false }; - let mut builder = CodeLexicalArtifactBuilderV1::create(&artifact_path, fixture.metadata) - .expect("create artifact"); + let mut builder = + CodeLexicalArtifactBuilderV1::create(&artifact_path, fixture.metadata.clone()) + .expect("create artifact"); for page in &pages { builder .append_page(page, &control) @@ -4947,6 +4867,7 @@ fn disk_artifact_rejects_noncanonical_receipt_reservation_tail() { CodeLexicalArtifactReaderV1::open_with_control( &artifact_path, &verified, + &fixture.metadata, CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, &control, ), @@ -5004,15 +4925,23 @@ fn disk_artifact_seal_is_terminal_and_refuses_page_replay() { builder.append_page(page, &control).expect("append page"); } let verified = finish_staged_artifact(&mut builder, &source_receipt, &control); - let progress_before = builder.progress().expect("sealed progress"); + assert!( + matches!( + builder.progress(), + Err(CodeLexicalArtifactErrorV1::Contract(_)) + ), + "a sealed artifact keeps no source cursor to resume" + ); assert!(matches!( builder.append_page(&pages[0], &control), Err(CodeLexicalArtifactErrorV1::Contract(_)) )); assert_eq!( - builder.progress().expect("progress after rejected replay"), - progress_before, - "a sealed artifact must reject an append without changing source progress" + builder + .sealed_receipt() + .expect("sealed receipt after rejected replay"), + Some(verified.clone()), + "a sealed artifact must reject an append without changing its seal" ); assert_eq!( builder @@ -5030,8 +4959,8 @@ fn disk_artifact_preseal_gate_denies_external_derived_mutation() { let directory = tempfile::tempdir().expect("artifact tempdir"); let artifact_path = directory.path().join("preseal-derived-mutation.sqlite"); let control = ArtifactControl { cancelled: false }; - let mut builder = - CodeLexicalArtifactBuilderV1::create(&artifact_path, metadata).expect("create artifact"); + let mut builder = CodeLexicalArtifactBuilderV1::create(&artifact_path, metadata.clone()) + .expect("create artifact"); for page in &pages { builder.append_page(page, &control).expect("append page"); } @@ -5039,14 +4968,17 @@ fn disk_artifact_preseal_gate_denies_external_derived_mutation() { let connection = rusqlite::Connection::open(&artifact_path).expect("open artifact mutation"); let original_row: Vec = connection .query_row( - "SELECT row FROM rows ORDER BY document_id LIMIT 1", + "SELECT payload FROM row_blocks ORDER BY first_document LIMIT 1", [], |row| row.get(0), ) - .expect("artifact row"); - let original_term_postings: i64 = connection - .query_row("SELECT COUNT(*) FROM term_postings", [], |row| row.get(0)) - .expect("term posting count"); + .expect("artifact row block"); + let original_term_postings = i64::try_from(staged_posting_count( + &artifact_path, + "term_posting_runs", + true, + )) + .expect("term posting count"); let original_imports: i64 = connection .query_row("SELECT COUNT(*) FROM import_evidence", [], |row| row.get(0)) .expect("import evidence count"); @@ -5055,12 +4987,12 @@ fn disk_artifact_preseal_gate_denies_external_derived_mutation() { let mut mutated_row = original_row.clone(); mutated_row.push(b' '); let row_mutation = connection.execute( - "UPDATE rows SET row = ?1 WHERE document_id = (SELECT MIN(document_id) FROM rows)", + "UPDATE row_blocks SET payload = ?1 WHERE first_document = (SELECT MIN(first_document) FROM row_blocks)", [mutated_row], ); - let posting_mutation = connection.execute("DELETE FROM term_postings", []); + let posting_mutation = connection.execute("DELETE FROM term_posting_runs", []); let row_insertion = connection.execute( - "INSERT INTO rows(document_id, chunk_id, row) VALUES (?1, 'external-conflict', X'7b7d')", + "INSERT INTO row_chunk_pages(document_id, chunk_id) VALUES (?1, 'external-conflict')", [i64::MAX], ); assert!( @@ -5085,6 +5017,7 @@ fn disk_artifact_preseal_gate_denies_external_derived_mutation() { CodeLexicalArtifactReaderV1::open_with_control( &artifact_path, &verified, + &metadata, CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, &control, ) @@ -5093,14 +5026,23 @@ fn disk_artifact_preseal_gate_denies_external_derived_mutation() { rusqlite::Connection::open(&artifact_path).expect("inspect finalized artifact"); let rebuilt_row: Vec = connection .query_row( - "SELECT row FROM rows ORDER BY document_id LIMIT 1", + "SELECT payload FROM row_blocks ORDER BY first_document LIMIT 1", [], |row| row.get(0), ) - .expect("finalized artifact row"); + .expect("finalized artifact row block"); let rebuilt_term_postings: i64 = connection - .query_row("SELECT COUNT(*) FROM term_postings", [], |row| row.get(0)) - .expect("finalized term posting count"); + .prepare("SELECT lists FROM term_postings") + .expect("prepare finalized term lists") + .query_map([], |row| row.get::<_, Vec>(0)) + .expect("read finalized term lists") + .map(|lists| { + decode_term_lists_oracle(&lists.expect("finalized term lists")) + .iter() + .map(|(_, document_frequency, _)| document_frequency) + .sum::() + }) + .sum(); let rebuilt_imports: i64 = connection .query_row("SELECT COUNT(*) FROM import_evidence", [], |row| row.get(0)) .expect("finalized import evidence count"); @@ -5222,8 +5164,8 @@ fn disk_artifact_cancellation_rolls_back_import_append_and_reopen_verification() let directory = tempfile::tempdir().expect("artifact tempdir"); let artifact_path = directory.path().join("cancelled-verification.sqlite"); let control = ArtifactControl { cancelled: false }; - let mut builder = - CodeLexicalArtifactBuilderV1::create(&artifact_path, metadata).expect("create artifact"); + let mut builder = CodeLexicalArtifactBuilderV1::create(&artifact_path, metadata.clone()) + .expect("create artifact"); for page in pages .iter() .take_while(|page| page.page_ordinal() < import_page.page_ordinal()) @@ -5277,6 +5219,7 @@ fn disk_artifact_cancellation_rolls_back_import_append_and_reopen_verification() CodeLexicalArtifactReaderV1::open_with_control( &artifact_path, &verified, + &metadata, CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, &reopen_cancellation, ), @@ -5285,6 +5228,7 @@ fn disk_artifact_cancellation_rolls_back_import_append_and_reopen_verification() CodeLexicalArtifactReaderV1::open_with_control( &artifact_path, &verified, + &metadata, CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, &control, ) @@ -5297,9 +5241,23 @@ fn disk_artifact_cancellation_rolls_back_import_append_and_reopen_verification() /// exact activation error it protects against. fn staged_row_cardinality(artifact_path: &Path) -> (u64, u64) { let connection = rusqlite::Connection::open(artifact_path).expect("inspect staging artifact"); + // Appends stage chunk ids in `row_chunk_pages`; sealing moves them into + // `row_chunks`. + let staging: bool = connection + .query_row( + "SELECT EXISTS(SELECT 1 FROM sqlite_schema WHERE type = 'table' AND name = 'row_chunk_pages')", + [], + |row| row.get(0), + ) + .expect("probe chunk staging"); + let table = if staging { + "row_chunk_pages" + } else { + "row_chunks" + }; let (rows, distinct): (i64, i64) = connection .query_row( - "SELECT COUNT(*), COUNT(DISTINCT chunk_id) FROM rows", + &format!("SELECT COUNT(*), COUNT(DISTINCT chunk_id) FROM {table}"), [], |row| Ok((row.get(0)?, row.get(1)?)), ) @@ -5411,7 +5369,7 @@ fn disk_artifact_budget_refusal_precedes_progress_and_accepts_boundary() { ); let connection = rusqlite::Connection::open(&refused_path).expect("inspect refusal state"); let rows: i64 = connection - .query_row("SELECT COUNT(*) FROM rows", [], |row| row.get(0)) + .query_row("SELECT COUNT(*) FROM row_chunk_pages", [], |row| row.get(0)) .expect("row count after refusal"); assert_eq!(rows, 0, "preflight refusal must precede row staging"); drop(connection); @@ -5461,8 +5419,8 @@ fn disk_artifact_rows_advance_once_across_retry_replay_and_cancellation() { let control = ArtifactControl { cancelled: false }; let directory = tempfile::tempdir().expect("artifact tempdir"); let artifact_path = directory.path().join("once-advance.sqlite"); - let mut builder = - CodeLexicalArtifactBuilderV1::create(&artifact_path, metadata).expect("create artifact"); + let mut builder = CodeLexicalArtifactBuilderV1::create(&artifact_path, metadata.clone()) + .expect("create artifact"); let mut appended_chunks = 0u64; for page in &pages { @@ -5521,6 +5479,7 @@ fn disk_artifact_rows_advance_once_across_retry_replay_and_cancellation() { CodeLexicalArtifactReaderV1::open_with_control( &artifact_path, &verified, + &metadata, CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, &control, ) @@ -5627,6 +5586,7 @@ fn disk_artifact_bounded_work_budget_exhaustion_resumes_activation() { CodeLexicalArtifactReaderV1::open_with_control( &artifact_path, &verified, + &metadata, CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, &exhausted_open, ), @@ -5635,6 +5595,7 @@ fn disk_artifact_bounded_work_budget_exhaustion_resumes_activation() { let reader = CodeLexicalArtifactReaderV1::open_with_control( &artifact_path, &verified, + &metadata, CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, &control, ) @@ -5746,8 +5707,8 @@ fn artifact_exact_reader_prefers_admitted_matches_over_denied_best() { let (pages, _) = drain_verified_pages(&fixture, 128); let directory = tempfile::tempdir().expect("artifact tempdir"); let artifact_path = directory.path().join("denied-best.sqlite"); - let mut builder = - CodeLexicalArtifactBuilderV1::create(&artifact_path, metadata).expect("create artifact"); + let mut builder = CodeLexicalArtifactBuilderV1::create(&artifact_path, metadata.clone()) + .expect("create artifact"); for page in &pages { builder.append_page(page, &control).expect("append page"); } @@ -5758,6 +5719,7 @@ fn artifact_exact_reader_prefers_admitted_matches_over_denied_best() { let reader = CodeLexicalArtifactReaderV1::open_with_control( &artifact_path, &verified, + &metadata, CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, &control, ) @@ -5831,8 +5793,8 @@ fn exact_candidate_scan_stops_before_the_next_batch_after_cancellation() { let (pages, _) = drain_verified_pages(&fixture, 128); let directory = tempfile::tempdir().expect("artifact tempdir"); let path = directory.path().join("cancel-exact.sqlite"); - let mut builder = - CodeLexicalArtifactBuilderV1::create(&path, metadata).expect("create real artifact"); + let mut builder = CodeLexicalArtifactBuilderV1::create(&path, metadata.clone()) + .expect("create real artifact"); for page in &pages { builder .append_page(page, &build_control) @@ -5845,6 +5807,7 @@ fn exact_candidate_scan_stops_before_the_next_batch_after_cancellation() { let reader = CodeLexicalArtifactReaderV1::open_with_control( &path, &verified, + &metadata, CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, &build_control, ) @@ -5883,73 +5846,6 @@ fn exact_candidate_scan_stops_before_the_next_batch_after_cancellation() { ); } -#[test] -fn in_memory_rebuilds_observe_cancellation_at_phase_and_batch_boundaries() { - let fixture = real_lexical_source_fixture_with_files(256); - let (pages, _) = drain_verified_pages(&fixture, 128); - let projection = CodeLexicalProjectionAdapterV1::new_admitted( - fixture.metadata.clone(), - pages - .iter() - .flat_map(|page| page.chunks().iter().cloned()) - .collect::>(), - page_symbol_displays(&pages), - ) - .expect("real admitted in-memory projection"); - - // Empty rebuilds must still consult the phase boundary. For the wide - // fixture, cancellation occurs after enough observations to enter a later - // rebuild batch; entry and final checks alone cannot trigger it. - for (term, cancel_at, has_matches) in [("absentzzxyz", 2, false), ("widget", 10, true)] { - let control = CancelAtObservation::new(cancel_at); - let mut request = lexical_request(term, &[term], &[], &[], 0, 1024); - request.generation = fixture.metadata.generation.clone(); - let baseline = complete( - projection - .read_lexical_postings(&request) - .expect("active lexical read"), - ); - assert_eq!(baseline.candidates.len() > 128, has_matches); - request.control = &control; - assert_eq!( - projection.read_lexical_postings(&request), - Err(RetrievalPortError::Cancelled) - ); - assert_eq!(control.observations(), cancel_at); - } - - let authority = CentralExactAdmissionAuthorityV1::new(id("exact-rules.v1")); - let exact = projection.exact_adapter(authority.clone()); - for (query, cancel_at, has_matches) in [ - (r#""absentzzxyz""#, 3, false), - (r#""return value""#, 9, true), - ] { - let control = CancelAtObservation::new(cancel_at); - let view = query_view(query); - let base = base_request(query, 1024); - let mut request = ExactLaneRequest { - literals: authority.parse_literals(&view, &base), - base, - query_view: &view, - generation: fixture.metadata.generation.clone(), - budget: budget(1024), - control: &ACTIVE_CONTROL, - }; - let baseline = complete( - exact - .read_exact_postings(&request) - .expect("active exact read"), - ); - assert_eq!(baseline.candidates.len() > 128, has_matches); - request.control = &control; - assert_eq!( - exact.read_exact_postings(&request), - Err(RetrievalPortError::Cancelled) - ); - assert_eq!(control.observations(), cancel_at); - } -} - #[test] fn disk_artifact_ledger_charges_stay_page_local_across_corpus_scaling() { let control = ArtifactControl { cancelled: false }; @@ -6011,20 +5907,10 @@ fn disk_artifact_reader_selects_bounded_top_k_with_lane_tie_order_and_coverage() let generation = metadata.generation.clone(); let control = ArtifactControl { cancelled: false }; let (pages, _) = drain_verified_pages(&fixture, 128); - let chunks = pages - .iter() - .flat_map(|page| page.chunks().iter().cloned()) - .collect::>(); - let one_shot = CodeLexicalProjectionAdapterV1::new_admitted( - metadata.clone(), - chunks, - page_symbol_displays(&pages), - ) - .expect("one-shot lexical projection"); let directory = tempfile::tempdir().expect("artifact tempdir"); let artifact_path = directory.path().join("top-k.sqlite"); - let mut builder = - CodeLexicalArtifactBuilderV1::create(&artifact_path, metadata).expect("create artifact"); + let mut builder = CodeLexicalArtifactBuilderV1::create(&artifact_path, metadata.clone()) + .expect("create artifact"); for page in &pages { builder.append_page(page, &control).expect("append page"); } @@ -6035,6 +5921,7 @@ fn disk_artifact_reader_selects_bounded_top_k_with_lane_tie_order_and_coverage() let reader = CodeLexicalArtifactReaderV1::open_with_control( &artifact_path, &verified, + &metadata, CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, &control, ) @@ -6071,17 +5958,6 @@ fn disk_artifact_reader_selects_bounded_top_k_with_lane_tie_order_and_coverage() .retrieve_lexical(&request) .expect("artifact lexical lane"), ); - let memory_lane = complete( - LexicalLane::new(one_shot.clone()) - .retrieve_lexical(&request) - .expect("one-shot lexical lane"), - ); - assert_eq!( - artifact_lane, memory_lane, - "the K=7 lexical lane batch (candidates, evidence, coverage, continuation) must \ - match the one-shot projection exactly; a pre-capped port must not surface as \ - eligible=K/capped=0/exhausted" - ); assert_eq!( artifact_lane.coverage.capped, artifact_lane.coverage.eligible - 7, @@ -6109,7 +5985,7 @@ fn disk_artifact_reader_selects_bounded_top_k_with_lane_tie_order_and_coverage() "the port's bounded selection already uses the lane's canonical tie order" ); - // Exact-lane parity under the same K=7: every document matches the + // The exact lane under the same K=7: every document matches the // quoted literal once, so the cut again runs through a tie. let authority = CentralExactAdmissionAuthorityV1::new(id::("exact-rules.v1")); @@ -6143,17 +6019,6 @@ fn disk_artifact_reader_selects_bounded_top_k_with_lane_tie_order_and_coverage() .retrieve_exact(&exact_request) .expect("artifact exact lane"), ); - let memory_exact = complete( - ExactLane::new(authority.clone(), one_shot.exact_adapter(authority)) - .retrieve_exact(&exact_request) - .expect("one-shot exact lane"), - ); - assert_eq!( - artifact_exact, memory_exact, - "the K=7 exact lane batch (candidates, evidence, coverage, continuation) must \ - match the one-shot projection exactly; a pre-capped port must not surface as \ - eligible=K/capped=0/exhausted" - ); assert_eq!( artifact_exact.coverage.capped, artifact_exact.coverage.eligible - 7, @@ -6169,67 +6034,6 @@ fn disk_artifact_reader_selects_bounded_top_k_with_lane_tie_order_and_coverage() ); } -#[test] -fn retained_lexical_projection_bounds_marginal_owned_byte_growth_for_repeated_tokens() { - let generation = id::("generation.1"); - let small_repeated = "retained_token ".repeat(1_000); - let large_repeated = "retained_token ".repeat(3_000); - let small_source = format!( - "pub fn retained_symbol() -> usize {{ let retained_token = 1; {small_repeated} retained_token }}\n" - ); - let large_source = format!( - "pub fn retained_symbol() -> usize {{ let retained_token = 1; {large_repeated} retained_token }}\n" - ); - let small_projection = CodeLexicalProjectionAdapterV1::new_admitted( - projection_metadata(&generation, FreshnessCompatibilityV1::Current), - vec![admitted_rust_chunk( - &generation, - 0, - &small_source, - CodeSearchChunkGrainV1::SymbolBody, - "retained_symbol", - )], - BTreeMap::new(), - ) - .expect("build small repeated-token projection"); - let large_projection = CodeLexicalProjectionAdapterV1::new_admitted( - projection_metadata(&generation, FreshnessCompatibilityV1::Current), - vec![admitted_rust_chunk( - &generation, - 0, - &large_source, - CodeSearchChunkGrainV1::SymbolBody, - "retained_symbol", - )], - BTreeMap::new(), - ) - .expect("build large repeated-token projection"); - - let small_retained = small_projection.retained_owned_bytes(); - let large_retained = large_projection.retained_owned_bytes(); - let marginal_retained = large_retained - .checked_sub(small_retained) - .expect("large projection must not retain fewer owned bytes than small projection"); - let marginal_source = large_source - .len() - .checked_sub(small_source.len()) - .expect("large source must not be smaller than small source"); - assert!( - marginal_retained <= marginal_source * 2, - "projection retained {marginal_retained} marginal owned bytes for {marginal_source} marginal source bytes" - ); - - let request = lexical_request("retained_token", &["retained_token"], &[], &[], 0, 8); - let RetrieverOutcome::Complete(batch) = LexicalLane::new(large_projection) - .retrieve_lexical(&request) - .expect("query repeated-token projection") - else { - panic!("repeated-token projection must be current"); - }; - assert_eq!(batch.candidates.len(), 1); - assert!(batch.candidates[0].raw_score.micros() > 0); -} - pub(crate) fn lexical_request( query: &str, whole_terms: &[&str], @@ -6263,39 +6067,6 @@ pub(crate) fn complete(outcome: RetrieverOutcome) -> T { } } -#[test] -fn matching_symbol_occurrence_does_not_admit_raw_or_json_exact_terms() { - let generation = id::("generation.1"); - let raw = chunk( - &generation, - 1, - CodeSearchChunkGrainV1::SymbolSignature, - "fn forged_symbol", - &[(ExactTechnicalTermKindV1::WholeSymbol, "forged_symbol")], - &["forged", "symbol"], - ); - assert_eq!( - raw.exact_terms[0].symbol_occurrence_id(), - raw.anchor.symbol_occurrence_id.as_ref() - ); - let metadata = projection_metadata(&generation, FreshnessCompatibilityV1::Current); - assert!( - CodeLexicalProjectionAdapterV1::new(metadata.clone(), vec![raw.clone()]).is_err(), - "public raw-parts construction cannot admit WholeSymbol evidence" - ); - - let decoded: CodeSearchChunkV1 = - serde_json::from_slice(&serde_json::to_vec(&raw).unwrap()).unwrap(); - assert_eq!( - decoded.exact_terms[0].symbol_occurrence_id(), - decoded.anchor.symbol_occurrence_id.as_ref() - ); - assert!( - CodeLexicalProjectionAdapterV1::new(metadata, vec![decoded]).is_err(), - "JSON chunks remain untrusted even when occurrence ids match" - ); -} - #[test] fn central_exact_authority_classifies_every_protected_term() { let authority = @@ -6341,39 +6112,14 @@ fn central_exact_authority_classifies_every_protected_term() { } #[test] -fn exact_projection_emits_only_authority_minted_proofs() { - let generation = id::("generation.1"); - let text = "std::collections::HashMap src/main.rs E0308 --release cargo tracedecay.data.dir commit:deadbee"; - let source = chunk( - &generation, - 1, - CodeSearchChunkGrainV1::SymbolBody, - text, - &[ - ( - ExactTechnicalTermKindV1::QualifiedName, - "std::collections::HashMap", - ), - (ExactTechnicalTermKindV1::Path, "src/main.rs"), - (ExactTechnicalTermKindV1::CompilerErrorCode, "E0308"), - (ExactTechnicalTermKindV1::CliFlag, "--release"), - (ExactTechnicalTermKindV1::ToolName, "cargo"), - ( - ExactTechnicalTermKindV1::ConfigurationKey, - "tracedecay.data.dir", - ), - (ExactTechnicalTermKindV1::CommitIdentifier, "commit:deadbee"), - ], - &["reserve", "stock"], - ); +fn exact_artifact_emits_only_authority_minted_proofs() { + let artifact = rust_artifact(&[ + "use std::collections::HashMap;\n/// Fails with \"connection refused\" when built without --release.\npub fn connect(map: HashMap) -> usize { map.len() }\n", + "pub fn unrelated() -> u32 { 7 }\n", + ]); let authority = CentralExactAdmissionAuthorityV1::new(id::("exact-rules.v1")); - let projection = CodeLexicalProjectionAdapterV1::new( - projection_metadata(&generation, FreshnessCompatibilityV1::Current), - vec![source], - ) - .expect("projection builds"); - let query = r#"std::collections::HashMap src/main.rs E0308 --release cargo tracedecay.data.dir commit:deadbee"#; + let query = r#"std::collections::HashMap "connection refused" --release"#; let base = base_request(query, 16); let query_view = query_view(query); let request = ExactLaneRequest { @@ -6381,99 +6127,87 @@ fn exact_projection_emits_only_authority_minted_proofs() { literals: authority.parse_literals(&query_view, &base), base, query_view: &query_view, - generation, + generation: artifact.metadata.generation.clone(), budget: budget(16), }; - let lane = ExactLane::new(authority.clone(), projection.exact_adapter(authority)); + let lane = ExactLane::new(authority.clone(), artifact.reader.exact_adapter(authority)); let batch = complete( lane.retrieve_exact(&request) - .expect("exact projection query succeeds"), + .expect("exact artifact query succeeds"), ); - assert_eq!(batch.candidates.len(), 1); - assert_eq!(batch.coverage.examined, 1); - assert_eq!(batch.coverage.eligible, 1); - assert_eq!(batch.coverage.excluded, 0); - let candidate = &batch.candidates[0]; - let proof = candidate - .exact_admission_proof - .as_ref() - .expect("exact candidate carries an authority proof"); - proof - .validate_for_request(&request.base) - .expect("proof remains request-bound"); - let evidence = &batch.evidence_by_occurrence[&candidate.source_occurrence_id]; - assert_eq!(evidence.matched_literals.len(), 7); + assert!(!batch.candidates.is_empty(), "the literals exist in file 0"); + assert_eq!(candidate_files(&batch.candidates), BTreeSet::from([0])); + for candidate in &batch.candidates { + candidate + .exact_admission_proof + .as_ref() + .expect("exact candidate carries an authority proof") + .validate_for_request(&request.base) + .expect("proof remains request-bound"); + assert!( + !batch.evidence_by_occurrence[&candidate.source_occurrence_id] + .matched_literals + .is_empty() + ); + } } #[test] fn fielded_bm25_keeps_whole_identifiers_and_subtokens_distinct() { - let generation = id::("generation.1"); - let chunks = vec![ - admitted_rust_chunk( - &generation, - 1, - "pub fn reserve_stock() {}\n", - CodeSearchChunkGrainV1::SymbolSignature, - "reserve_stock", - ), - admitted_rust_chunk( - &generation, - 2, - "pub fn reserve() { let stock_inventory = 1; }\n", - CodeSearchChunkGrainV1::SymbolBody, - "reserve", - ), - ]; - let projection = CodeLexicalProjectionAdapterV1::new_admitted( - projection_metadata(&generation, FreshnessCompatibilityV1::Current), - chunks, - BTreeMap::new(), - ) - .expect("projection builds"); - let whole_request = lexical_request("reserve_stock", &["reserve_stock"], &[], &[], 0, 8); + let artifact = rust_artifact(&[ + "pub fn reserve_stock() {}\n", + "pub fn reserve() { let stock_inventory = 1; }\n", + ]); + let lane = artifact.lane(); let whole = complete( - LexicalLane::new(projection.clone()) - .retrieve_lexical(&whole_request) - .expect("whole-term retrieval succeeds"), + lane.retrieve_lexical(&artifact.request( + "reserve_stock", + &["reserve_stock"], + &[], + &[], + 0, + 8, + )) + .expect("whole-term retrieval succeeds"), ); - - assert_eq!(whole.candidates.len(), 1); - let evidence = &whole.evidence_by_occurrence[&whole.candidates[0].source_occurrence_id]; - assert!( + assert_eq!(candidate_files(&whole.candidates), BTreeSet::from([0])); + assert!(whole.evidence_by_occurrence.values().all(|evidence| { evidence .matched_whole_terms .contains(&"reserve_stock".to_owned()) - ); - assert!(evidence.matched_subtokens.is_empty()); - assert!( + && evidence.matched_subtokens.is_empty() + })); + assert!(whole.evidence_by_occurrence.values().any(|evidence| { evidence .field_scores_micros .iter() .any(|(field, _)| *field == LexicalFieldV1::SymbolName) - ); + })); - let whole_subtoken_text = lexical_request("reserve", &["reserve"], &[], &[], 0, 8); let whole_only = complete( - LexicalLane::new(projection.clone()) - .retrieve_lexical(&whole_subtoken_text) + lane.retrieve_lexical(&artifact.request("reserve", &["reserve"], &[], &[], 0, 8)) .expect("whole-term/subtoken boundary retrieval succeeds"), ); - assert_eq!( - whole_only.candidates.len(), - 1, + assert!(candidate_files(&whole_only.candidates).contains(&1)); + assert!( + whole_only + .evidence_by_occurrence + .values() + .all(|evidence| evidence.matched_subtokens.is_empty()), "a whole-term query must not consume the distinct subtoken field" ); - let subtoken_request = lexical_request("reserve", &[], &["reserve"], &[], 0, 8); let subtokens = complete( - LexicalLane::new(projection) - .retrieve_lexical(&subtoken_request) + lane.retrieve_lexical(&artifact.request("reserve", &[], &["reserve"], &[], 0, 8)) .expect("subtoken retrieval succeeds"), ); - assert_eq!(subtokens.candidates.len(), 2); + assert_eq!( + candidate_files(&subtokens.candidates), + BTreeSet::from([0, 1]) + ); assert!(subtokens.evidence_by_occurrence.values().all(|evidence| { evidence.matched_whole_terms.is_empty() && evidence.matched_subtokens == vec!["reserve".to_owned()] @@ -6482,71 +6216,53 @@ fn fielded_bm25_keeps_whole_identifiers_and_subtokens_distinct() { #[test] fn lexical_phrase_and_bounded_fuzzy_recovery_are_deterministic() { - let generation = id::("generation.1"); - let chunks = vec![ - admitted_rust_chunk( - &generation, - 1, - "pub fn reserve() { // reserve stock inventory\n}\n", - CodeSearchChunkGrainV1::SymbolBody, - "reserve", - ), - admitted_rust_chunk( - &generation, - 2, - "pub fn reserve_stock() {}\n", - CodeSearchChunkGrainV1::SymbolSignature, - "reserve_stock", - ), - ]; - let projection = CodeLexicalProjectionAdapterV1::new_admitted( - projection_metadata(&generation, FreshnessCompatibilityV1::Current), - chunks, - BTreeMap::new(), - ) - .expect("projection builds"); - let phrase_request = lexical_request(r#""reserve stock""#, &[], &[], &["reserve stock"], 0, 8); + let artifact = rust_artifact(&[ + "pub fn reserve() { // reserve stock inventory\n}\n", + "pub fn reserve_stock() {}\n", + ]); + let lane = artifact.lane(); let phrase = complete( - LexicalLane::new(projection.clone()) - .retrieve_lexical(&phrase_request) - .expect("phrase retrieval succeeds"), - ); - assert_eq!(phrase.candidates.len(), 1); - assert_eq!( - phrase.evidence_by_occurrence[&phrase.candidates[0].source_occurrence_id].matched_phrases, - vec!["reserve stock".to_owned()] + lane.retrieve_lexical(&artifact.request( + r#""reserve stock""#, + &[], + &[], + &["reserve stock"], + 0, + 8, + )) + .expect("phrase retrieval succeeds"), ); + assert!(!phrase.candidates.is_empty()); + assert!(phrase.candidates.iter().all(|candidate| { + phrase.evidence_by_occurrence[&candidate.source_occurrence_id].matched_phrases + == vec!["reserve stock".to_owned()] + })); - let disabled = lexical_request("resreve_stock", &["resreve_stock"], &[], &[], 0, 8); + let disabled = artifact.request("resreve_stock", &["resreve_stock"], &[], &[], 0, 8); assert!( complete( - LexicalLane::new(projection.clone()) - .retrieve_lexical(&disabled) + lane.retrieve_lexical(&disabled) .expect("disabled fuzzy retrieval succeeds"), ) .candidates .is_empty() ); - let fuzzy = lexical_request("resreve_stock", &["resreve_stock"], &[], &[], 1, 8); + let fuzzy = artifact.request("resreve_stock", &["resreve_stock"], &[], &[], 1, 8); let first = complete( - LexicalLane::new(projection.clone()) - .retrieve_lexical(&fuzzy) + lane.retrieve_lexical(&fuzzy) .expect("fuzzy retrieval succeeds"), ); let second = complete( - LexicalLane::new(projection) - .retrieve_lexical(&fuzzy) + lane.retrieve_lexical(&fuzzy) .expect("fuzzy replay succeeds"), ); assert_eq!(first, second); - assert_eq!(first.candidates.len(), 1); - assert!( - first.evidence_by_occurrence[&first.candidates[0].source_occurrence_id] - .typo_recovery_applied - ); + assert_eq!(candidate_files(&first.candidates), BTreeSet::from([1])); + let evidence = &first.evidence_by_occurrence[&first.candidates[0].source_occurrence_id]; + assert!(evidence.typo_recovery_applied); assert_eq!( - first.evidence_by_occurrence[&first.candidates[0].source_occurrence_id].spelling_variants, + evidence.spelling_variants, [LexicalSpellingVariantV1 { query: "resreve_stock".to_owned(), alternative: "reserve_stock".to_owned(), @@ -6570,59 +6286,28 @@ fn lexical_phrase_and_bounded_fuzzy_recovery_are_deterministic() { #[test] fn lexical_phrase_candidate_set_and_frequency_are_reused_without_drift() { - // Equivalence guard for finding 14: the per-phrase n-gram candidate set is - // now intersected once and reused for both the document-frequency tally and - // the lexical document set. Two documents contain the phrase and one does - // not; the reused candidate set must still return exactly the two - // phrase-bearing documents, deterministically. - let generation = id::("generation.1"); - let chunks = vec![ - admitted_rust_chunk( - &generation, - 1, - "pub fn reserve() {\n // reserve stock inventory ledger\n}\n", - CodeSearchChunkGrainV1::SymbolBody, - "reserve", - ), - admitted_rust_chunk( - &generation, - 2, - "pub fn hold() {\n // reserve stock inventory ledger\n}\n", - CodeSearchChunkGrainV1::SymbolBody, - "hold", - ), - admitted_rust_chunk( - &generation, - 3, - "pub fn unrelated() {\n // nothing relevant lives here\n}\n", - CodeSearchChunkGrainV1::SymbolBody, - "unrelated", - ), - ]; - let projection = CodeLexicalProjectionAdapterV1::new_admitted( - projection_metadata(&generation, FreshnessCompatibilityV1::Current), - chunks, - BTreeMap::new(), - ) - .expect("projection builds"); - - let phrase_request = lexical_request(r#""reserve stock""#, &[], &[], &["reserve stock"], 0, 8); + // The per-phrase n-gram candidate set is intersected once and reused for + // both the document-frequency tally and the lexical document set. Two + // files contain the phrase and one does not; the reused candidate set + // must return exactly the phrase-bearing files, deterministically. + let artifact = rust_artifact(&[ + "pub fn reserve() {\n // reserve stock inventory ledger\n}\n", + "pub fn hold() {\n // reserve stock inventory ledger\n}\n", + "pub fn unrelated() {\n // nothing relevant lives here\n}\n", + ]); + let lane = artifact.lane(); + let request = artifact.request(r#""reserve stock""#, &[], &[], &["reserve stock"], 0, 8); let first = complete( - LexicalLane::new(projection.clone()) - .retrieve_lexical(&phrase_request) + lane.retrieve_lexical(&request) .expect("phrase retrieval succeeds"), ); let second = complete( - LexicalLane::new(projection) - .retrieve_lexical(&phrase_request) + lane.retrieve_lexical(&request) .expect("phrase retrieval replays"), ); - // Reusing the shared candidate set is deterministic and drift-free. assert_eq!(first, second); - // Exactly the two phrase-bearing documents are returned; the unrelated - // document is excluded. - assert_eq!(first.candidates.len(), 2); + assert_eq!(candidate_files(&first.candidates), BTreeSet::from([0, 1])); for candidate in &first.candidates { assert_eq!( first.evidence_by_occurrence[&candidate.source_occurrence_id].matched_phrases, @@ -6633,101 +6318,67 @@ fn lexical_phrase_candidate_set_and_frequency_are_reused_without_drift() { #[test] fn duplicate_whole_terms_do_not_consume_the_global_fuzzy_budget() { - let generation = id::("generation.1"); - let chunks = vec![ - admitted_rust_chunk( - &generation, - 1, - "pub fn reserve() {}\n", - CodeSearchChunkGrainV1::SymbolSignature, - "reserve", - ), - admitted_rust_chunk( - &generation, - 2, - "pub fn reserved() {}\n", - CodeSearchChunkGrainV1::SymbolSignature, - "reserved", - ), - admitted_rust_chunk( - &generation, - 3, - "pub fn other() {}\n", - CodeSearchChunkGrainV1::SymbolSignature, - "other", - ), - ]; - let projection = CodeLexicalProjectionAdapterV1::new_admitted( - projection_metadata(&generation, FreshnessCompatibilityV1::Current), - chunks, - BTreeMap::new(), - ) - .expect("projection builds"); - let request = lexical_request( - "reservd reservd otherr", - &["reservd", "reservd", "otherr"], - &[], - &[], - 3, - 8, - ); - + let artifact = rust_artifact(&[ + "pub fn reserve() {}\n", + "pub fn reserved() {}\n", + "pub fn other() {}\n", + ]); let batch = complete( - LexicalLane::new(projection) - .retrieve_lexical(&request) + artifact + .lane() + .retrieve_lexical(&artifact.request( + "reservd reservd otherr", + &["reservd", "reservd", "otherr"], + &[], + &[], + 3, + 8, + )) .expect("fuzzy retrieval succeeds"), ); - assert_eq!(batch.candidates.len(), 3); + assert_eq!( + candidate_files(&batch.candidates), + BTreeSet::from([0, 1, 2]), + "the duplicate term must leave budget for the distinct typo" + ); assert!( batch .evidence_by_occurrence .values() - .any(|evidence| { evidence.matched_whole_terms.contains(&"otherr".to_owned()) }) + .any(|evidence| evidence.matched_whole_terms.contains(&"otherr".to_owned())) ); } #[test] -fn lexical_projection_reports_freshness_coverage_and_page_cutoff() { - let generation = id::("generation.1"); - let chunks: Vec = (1..=3) - .map(|ordinal| { - admitted_rust_chunk( - &generation, - ordinal, - "pub fn target() {}\n", - CodeSearchChunkGrainV1::SymbolSignature, - "target", - ) - }) - .collect(); - let current = CodeLexicalProjectionAdapterV1::new_admitted( - projection_metadata(&generation, FreshnessCompatibilityV1::Current), - chunks.clone(), - BTreeMap::new(), - ) - .expect("current projection builds"); - let request = lexical_request("target", &["target"], &[], &[], 0, 2); +fn lexical_artifact_reports_freshness_coverage_and_page_cutoff() { + let sources = ["pub fn target() {}\n"; 3]; + let fixture = rust_source_fixture(&sources); + let current = sealed_artifact(&fixture, fixture.metadata.clone()); + let request = current.request("target", &["target"], &[], &[], 0, 2); let page = complete( - LexicalLane::new(current) + current + .lane() .retrieve_lexical(&request) .expect("page retrieval succeeds"), ); assert_eq!(page.candidates.len(), 2); - assert_eq!(page.coverage.examined, 3); - assert_eq!(page.coverage.eligible, 3); - assert_eq!(page.coverage.capped, 1); + assert!( + page.coverage.eligible >= 3, + "every file matches: {:?}", + page.coverage + ); + assert_eq!(page.coverage.examined, page.coverage.eligible); + assert_eq!(page.coverage.capped, page.coverage.eligible - 2); assert!(!page.continuation.expect("continuation").exhausted); - let stale = CodeLexicalProjectionAdapterV1::new_admitted( - projection_metadata(&generation, FreshnessCompatibilityV1::Stale), - chunks, - BTreeMap::new(), - ) - .expect("stale projection remains inspectable"); - let outcome = LexicalLane::new(stale) + let mut stale_metadata = fixture.metadata.clone(); + stale_metadata.freshness = freshness(FreshnessCompatibilityV1::Stale); + let stale = sealed_artifact(&fixture, stale_metadata); + let outcome = stale + .lane() .retrieve_lexical(&request) .expect("staleness is a typed outcome"); assert!(matches!(outcome, RetrieverOutcome::Stale(_))); @@ -6735,43 +6386,25 @@ fn lexical_projection_reports_freshness_coverage_and_page_cutoff() { #[test] fn lexical_source_occurrence_identity_is_generation_exact() { - let first_generation = id::("generation.1"); - let second_generation = id::("generation.2"); - let first_projection = CodeLexicalProjectionAdapterV1::new_admitted( - projection_metadata(&first_generation, FreshnessCompatibilityV1::Current), - vec![admitted_rust_chunk( - &first_generation, - 1, - "pub fn target() {}\n", - CodeSearchChunkGrainV1::SymbolSignature, - "target", - )], - BTreeMap::new(), - ) - .expect("first projection builds"); - let second_projection = CodeLexicalProjectionAdapterV1::new_admitted( - projection_metadata(&second_generation, FreshnessCompatibilityV1::Current), - vec![admitted_rust_chunk( - &second_generation, - 1, - "pub fn target() {}\n", - CodeSearchChunkGrainV1::SymbolSignature, - "target", - )], - BTreeMap::new(), - ) - .expect("second projection builds"); - let first_request = lexical_request("target", &["target"], &[], &[], 0, 8); - let mut second_request = lexical_request("target", &["target"], &[], &[], 0, 8); - second_request.generation = second_generation; + let target = "pub fn target() {}\n"; + let first = rust_artifact(&[target]); + let second = rust_artifact(&[target, "pub fn unrelated() {}\n"]); + assert_ne!( + first.metadata.generation, second.metadata.generation, + "a changed corpus seals a distinct generation" + ); + let first_request = first.request("target", &["target"], &[], &[], 0, 8); + let second_request = second.request("target", &["target"], &[], &[], 0, 8); - let first = complete( - LexicalLane::new(first_projection) + let first_batch = complete( + first + .lane() .retrieve_lexical(&first_request) .expect("first retrieval succeeds"), ); - let second = complete( - LexicalLane::new(second_projection) + let second_batch = complete( + second + .lane() .retrieve_lexical(&second_request) .expect("second retrieval succeeds"), ); @@ -6781,15 +6414,16 @@ fn lexical_source_occurrence_identity_is_generation_exact() { // shareable anchor across generations. Generation exactness lives // in the source occurrence instead. assert_eq!( - first.candidates[0].anchor_id, second.candidates[0].anchor_id, + first_batch.candidates[0].anchor_id, second_batch.candidates[0].anchor_id, "an unchanged symbol occurrence keeps one anchor across generations" ); assert_ne!( - first.candidates[0].source_occurrence_id, second.candidates[0].source_occurrence_id, + first_batch.candidates[0].source_occurrence_id, + second_batch.candidates[0].source_occurrence_id, "the logical chunk is stable but each generation has a distinct occurrence" ); assert!( - first.candidates[0] + first_batch.candidates[0] .source_occurrence_id .as_str() .contains(first_request.generation.as_str()), diff --git a/crates/tracedecay-query/tests/search_quality_suite/scaling.rs b/crates/tracedecay-query/tests/search_quality_suite/scaling.rs index a39d594461..1d6ea4be5b 100644 --- a/crates/tracedecay-query/tests/search_quality_suite/scaling.rs +++ b/crates/tracedecay-query/tests/search_quality_suite/scaling.rs @@ -1,13 +1,11 @@ -use tracedecay_domain::{ - CodeGenerationId, CodeSearchChunkGrainV1, FileOccurrenceId, FreshnessCompatibilityV1, - RetrievalFailure, RetrieverOutcome, -}; +use tracedecay_domain::{RetrievalFailure, RetrieverOutcome}; use tracedecay_query::retrieval::lexical::{ - CodeLexicalProjectionAdapterV1, LexicalLane, LexicalLaneRetriever, - MAX_LEXICAL_CANDIDATE_DOCUMENTS_V1, + LexicalLaneRetriever, MAX_LEXICAL_CANDIDATE_DOCUMENTS_V1, }; -use crate::candidate_producers::{chunk, complete, id, lexical_request, projection_metadata}; +use crate::candidate_producers::{ + complete, real_lexical_source_fixture_from_sources, sealed_artifact, +}; /// A term shared by more documents than the lane may hydrate per request /// generates no candidates of its own once a rarer term is present: only the @@ -17,42 +15,49 @@ use crate::candidate_producers::{chunk, complete, id, lexical_request, projectio /// common term is still admitted and the retrieval is complete. #[test] fn common_term_candidates_are_bounded_by_the_rarest_source() { - let documents = (MAX_LEXICAL_CANDIDATE_DOCUMENTS_V1 + 1) as u32; - let generation = id::("generation.1"); - let target = documents / 2; - let chunks = (0..documents) - .map(|ordinal| { - let text = if ordinal == target { - format!("fn function_{ordinal}() {{ let needle = shared_flag; }}") - } else { - format!("fn function_{ordinal}() {{ let shared_flag = true; }}") - }; - chunk( - &generation, - ordinal, - CodeSearchChunkGrainV1::SymbolBody, - &text, - &[], - &[], - ) - }) - .collect(); - let mut metadata = projection_metadata(&generation, FreshnessCompatibilityV1::Current); - metadata.logical_paths.extend((0..documents).map(|ordinal| { - ( - id::(&format!("file.{ordinal}")), - format!("src/file-{ordinal}.rs"), - ) - })); - let lane = LexicalLane::new( - CodeLexicalProjectionAdapterV1::new(metadata, chunks).expect("postings build"), + let files = 128; + let functions_per_file = MAX_LEXICAL_CANDIDATE_DOCUMENTS_V1 / files + 1; + let fixture = real_lexical_source_fixture_from_sources( + (0..files) + .map(|file| { + let source = (0..functions_per_file) + .map(|function| { + if file == files / 2 && function == 0 { + format!( + "pub fn function_{function}() {{ let needle = shared_flag; }}\n" + ) + } else { + format!("pub fn function_{function}() {{ let shared_flag = true; }}\n") + } + }) + .collect::(); + ( + format!("file.scaling.{file:03}"), + format!("src/scaling_{file:03}.rs"), + source.into_bytes(), + ) + }) + .collect(), + ); + let artifact = sealed_artifact(&fixture, fixture.metadata.clone()); + let lane = artifact.lane(); + + let common_only = complete( + lane.retrieve_lexical(&artifact.request("shared_flag", &["shared_flag"], &[], &[], 0, 8)) + .expect("common-only query"), + ); + assert_eq!(common_only.candidates.len(), 8); + let documents = common_only.coverage.eligible; + assert!( + documents > MAX_LEXICAL_CANDIDATE_DOCUMENTS_V1 as u64, + "the common term must exceed the document-frequency budget: {documents}" ); let RetrieverOutcome::Partial { value: mixed, reason, } = lane - .retrieve_lexical(&lexical_request( + .retrieve_lexical(&artifact.request( "needle shared_flag", &["needle", "shared_flag"], &[], @@ -67,29 +72,14 @@ fn common_term_candidates_are_bounded_by_the_rarest_source() { assert_eq!( reason, RetrievalFailure::CandidateSourcesPruned { - term_sources: vec![("shared_flag".to_owned(), u64::from(documents))], + term_sources: vec![("shared_flag".to_owned(), documents)], document_frequency_budget: MAX_LEXICAL_CANDIDATE_DOCUMENTS_V1 as u64, } ); assert_eq!( - mixed.candidates.len(), - 1, - "only the rare term's document may be hydrated: {:?}", + (mixed.candidates.len(), mixed.coverage.eligible), + (2, 2), + "only the two documents holding the rare term may be hydrated: {:?}", mixed.coverage ); - assert_eq!(mixed.coverage.eligible, 1); - - let common_only = complete( - lane.retrieve_lexical(&lexical_request( - "shared_flag", - &["shared_flag"], - &[], - &[], - 0, - 8, - )) - .expect("common-only query"), - ); - assert_eq!(common_only.candidates.len(), 8); - assert_eq!(common_only.coverage.eligible, u64::from(documents)); } diff --git a/crates/tracedecay-query/tests/search_quality_suite/single_root.rs b/crates/tracedecay-query/tests/search_quality_suite/single_root.rs index 9e87387a20..4aa54b1d55 100644 --- a/crates/tracedecay-query/tests/search_quality_suite/single_root.rs +++ b/crates/tracedecay-query/tests/search_quality_suite/single_root.rs @@ -1,14 +1,13 @@ -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::sync::Arc; use tracedecay_domain::{ - CalibrationProfileId, CodeGenerationId, CodeSearchChunkGrainV1, CompactCandidate, - ComponentRevision, DiversityPolicy, EdgeAuthorityV1, EphemeralSanitizedQueryViewV1, - ExactAdmissionRuleRevision, ExactClass, ExactTechnicalTermKindV1, FixedPointScore, - FreshnessCompatibilityV1, FusionProfile, HydrationReceipt, QueryNormalizationRevision, - RankedCandidate, RelationEdgeKindV1, RetrievalAnchorId, RetrievalCursorKeyId, RetrieverBatch, - RetrieverCoverage, RetrieverKind, RetrieverOutcome, SanitizerRevision, - ScoreDomainCalibrationV1, SourceSpan, SymbolOccurrenceId, UtcMicros, + CalibrationProfileId, CodeGenerationId, CompactCandidate, ComponentRevision, DiversityPolicy, + EdgeAuthorityV1, EphemeralSanitizedQueryViewV1, ExactAdmissionRuleRevision, ExactClass, + FixedPointScore, FusionProfile, HydrationReceipt, QueryNormalizationRevision, RankedCandidate, + RelationEdgeKindV1, RetrievalAnchorId, RetrievalCursorKeyId, RetrieverBatch, RetrieverCoverage, + RetrieverKind, RetrieverOutcome, SanitizerRevision, ScoreDomainCalibrationV1, SourceSpan, + SymbolOccurrenceId, UtcMicros, }; use tracedecay_query::retrieval::exact::{ CentralExactAdmissionAuthorityV1, ExactAdmissionAuthority, ExactLane, ExactLaneRequest, @@ -25,9 +24,7 @@ use tracedecay_query::retrieval::hydrate::{ CanonicalLateHydration, HydrationAuthorizationV1, HydrationPreflightOutcomeV1, HydrationReadOutcomeV1, HydrationWorkPermitV1, LateHydrationSource, }; -use tracedecay_query::retrieval::lexical::{ - CodeLexicalProjectionAdapterV1, LexicalLane, LexicalLaneRetriever, -}; +use tracedecay_query::retrieval::lexical::{LexicalLane, LexicalLaneRetriever}; use tracedecay_query::retrieval::ports::{ CodeCandidateBindingV1, CodeOccurrenceRefV1, GraphEvidenceReadPort, RetrievalExecutionControl, RetrievalPortError, @@ -37,8 +34,7 @@ use tracedecay_query::retrieval::{ }; use crate::candidate_producers::{ - FixtureRetrievalExecutionControl, base_request, budget, chunk, complete, id, lexical_request, - projection_metadata, + FixtureRetrievalExecutionControl, base_request, budget, complete, id, rust_artifact, }; #[derive(Clone, Copy)] @@ -277,47 +273,15 @@ fn graph_batch( } fn fixture(disposition: GraphDisposition) -> SingleRootFixture { - let generation = id::("generation.1"); let request = base_request("--release", 16); - let chunks = vec![ - chunk( - &generation, - 1, - CodeSearchChunkGrainV1::SymbolBody, - "build with --release", - &[(ExactTechnicalTermKindV1::CliFlag, "--release")], - &["build", "release"], - ), - chunk( - &generation, - 2, - CodeSearchChunkGrainV1::SymbolSignature, - "fn target_alpha", - &[], - &["target", "alpha"], - ), - chunk( - &generation, - 3, - CodeSearchChunkGrainV1::SymbolSignature, - "fn target_beta", - &[], - &["target", "beta"], - ), - chunk( - &generation, - 4, - CodeSearchChunkGrainV1::SymbolSignature, - "fn target_gamma", - &[], - &["target", "gamma"], - ), - ]; - let projection = CodeLexicalProjectionAdapterV1::new( - projection_metadata(&generation, FreshnessCompatibilityV1::Current), - chunks, - ) - .expect("single-root projection builds"); + let artifact = rust_artifact(&[ + "pub fn build() -> &'static str {\n \"build with --release\"\n}\n", + "pub fn target_alpha() {}\n", + "pub fn target_beta() {}\n", + "pub fn target_gamma() {}\n", + ]); + let generation = artifact.metadata.generation.clone(); + let projection = artifact.reader.clone(); let authority = CentralExactAdmissionAuthorityV1::new(id::("exact-rules.v1")); @@ -334,13 +298,23 @@ fn fixture(disposition: GraphDisposition) -> SingleRootFixture { .retrieve_exact(&exact_request) .expect("exact lane completes"); - let mut lexical_request = lexical_request("--release", &[], &["target"], &[], 0, 16); + let mut lexical_request = artifact.request("--release", &[], &["target"], &[], 0, 16); lexical_request.base = request.clone(); let lexical_outcome = LexicalLane::new(projection) .retrieve_lexical(&lexical_request) .expect("lexical lane completes"); let lexical_batch = complete(lexical_outcome.clone()); - assert_eq!(lexical_batch.candidates.len(), 3); + // Every row a target file seals shares that symbol's anchor, so fusion + // sees exactly three approximate candidates. + assert_eq!( + lexical_batch + .candidates + .iter() + .map(|candidate| &candidate.anchor_id) + .collect::>() + .len(), + 3 + ); let graph_request = graph_request(&request, &generation); let reply = match disposition { diff --git a/crates/tracedecay-runtime-core/src/branch.rs b/crates/tracedecay-runtime-core/src/branch.rs index bb20f49034..da124c90b7 100644 --- a/crates/tracedecay-runtime-core/src/branch.rs +++ b/crates/tracedecay-runtime-core/src/branch.rs @@ -5,6 +5,7 @@ use std::path::Path; use tracedecay_domain::errors::{Result, TraceDecayError}; +use tracedecay_private_fs::FileLease; #[cfg(any(test, feature = "test-helpers"))] use std::collections::HashMap; @@ -80,13 +81,11 @@ mod tracking; pub use admin::{ BranchAdminAction, BranchAdminOutcome, BranchAdminReport, PreparedBranchAdminMutation, SingleStoreBranchRetirementV1, prepare_branch_admin_mutation, - remove_tracked_branch_store_checked, }; pub use tracking::{ BranchAddOutcome, BranchTrackingPreparation, PreparedBranchRollbackOutcome, PreparedBranchTracking, finalize_prepared_branch_tracking, find_nearest_tracked_ancestor, - is_branch_ref_present, local_branch_exists, prepare_branch_tracking_in_layout, - rollback_prepared_branch_tracking, + local_branch_exists, prepare_branch_tracking_in_layout, rollback_prepared_branch_tracking, }; pub(crate) use tracking::{now_unix_secs, parse_unix_secs}; @@ -182,7 +181,7 @@ impl BranchMemo { } /// Acquires the shared branch-add lock. -pub fn try_acquire_branch_add_lock(tracedecay_dir: &Path) -> Result { +pub fn try_acquire_branch_add_lock(tracedecay_dir: &Path) -> Result { std::fs::create_dir_all(tracedecay_dir)?; let lock_path = tracedecay_dir.join(".branch-add.lock"); let file = std::fs::OpenOptions::new() @@ -195,20 +194,20 @@ pub fn try_acquire_branch_add_lock(tracedecay_dir: &Path) -> Result Result { +pub fn acquire_branch_lock_blocking(tracedecay_dir: &Path) -> Result { acquire_branch_add_lock_blocking_with(tracedecay_dir, try_acquire_branch_add_lock) } fn acquire_branch_add_lock_blocking_with( tracedecay_dir: &Path, - acquire: fn(&Path) -> Result, -) -> Result { + acquire: fn(&Path) -> Result, +) -> Result { let mut last_contention = None; for _ in 0..BRANCH_LOCK_RETRY_ATTEMPTS { match acquire(tracedecay_dir) { diff --git a/crates/tracedecay-runtime-core/src/branch/admin.rs b/crates/tracedecay-runtime-core/src/branch/admin.rs index 6c815c98b2..cb5203786e 100644 --- a/crates/tracedecay-runtime-core/src/branch/admin.rs +++ b/crates/tracedecay-runtime-core/src/branch/admin.rs @@ -1,12 +1,12 @@ -//! Destructive branch-store administration. +//! Branch-tracking administration. use std::path::{Path, PathBuf}; use crate::branch_meta::BranchMeta; +use tracedecay_private_fs::FileLease; -/// Destructive branch-store operation accepted by the daemon-owned -/// administrative path. The tagged representation is also the wire contract -/// used by the CLI. +/// Branch-tracking operation accepted by the daemon-owned administrative +/// path. The tagged representation is also the wire contract used by the CLI. #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] #[serde(tag = "action", rename_all = "snake_case")] pub enum BranchAdminAction { @@ -15,7 +15,7 @@ pub enum BranchAdminAction { Gc, } -/// Typed outcome returned to the CLI after a destructive branch operation. +/// Typed outcome returned to the CLI after a branch administration operation. #[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] #[serde(rename_all = "snake_case")] pub enum BranchAdminOutcome { @@ -30,15 +30,13 @@ pub struct BranchAdminReport { pub outcome: BranchAdminOutcome, #[serde(default)] pub removed_branches: Vec, - #[serde(default)] - pub removed_orphan_dbs: Vec, #[serde(default, skip_serializing_if = "Option::is_none")] pub default_branch: Option, } /// Exact single-store provenance selected for cleanup alongside a metadata -/// removal. Older or legacy branch entries without sealed graph provenance are -/// deliberately absent: destructive Git/worktree cleanup must never guess. +/// removal. Branch entries without sealed graph provenance are deliberately +/// absent: destructive Git/worktree cleanup must never guess. #[derive(Debug, Clone, PartialEq, Eq)] pub struct SingleStoreBranchRetirementV1 { pub branch: String, @@ -46,25 +44,20 @@ pub struct SingleStoreBranchRetirementV1 { } /// A branch metadata mutation selected while holding the shared branch lock. -/// The daemon reserves [`Self::database_paths`] through the store runtime -/// registry's destructive maintenance path before committing. +/// Every branch is served by the single project store, so the metadata entry +/// is the only store state a removal retires. pub struct PreparedBranchAdminMutation { project_root: PathBuf, tracedecay_dir: PathBuf, metadata_before: Option, metadata_after: Option, - database_paths: Vec, gc_branches: Vec, single_store_retirements: Vec, report: BranchAdminReport, - _branch_lock: std::fs::File, + _branch_lock: FileLease, } impl PreparedBranchAdminMutation { - pub fn database_paths(&self) -> &[PathBuf] { - &self.database_paths - } - pub fn report(&self) -> &BranchAdminReport { &self.report } @@ -73,54 +66,21 @@ impl PreparedBranchAdminMutation { &self.single_store_retirements } - #[cfg(test)] - fn commit(self) -> tracedecay_domain::errors::Result { - self.commit_with_hook(|_| Ok(())) - } - - /// CAS-publishes the prepared metadata mutation when no database file was - /// selected for deletion. Branches served by the single project store - /// retire this way: the metadata entry is the only state to remove. - pub fn finish_without_database_deletion( - self, - ) -> tracedecay_domain::errors::Result { - if !self.database_paths.is_empty() { - return Err(tracedecay_domain::errors::TraceDecayError::Config { - message: "branch database deletion requires daemon store administration" - .to_string(), - }); - } - self.commit_with_hook(|_| Ok(())) - } - - /// CAS-publishes the exact prepared branch metadata, then unlinks every - /// selected DB/WAL/SHM family. The caller must hold the canonical runtime - /// destructive reservation until this returns. - pub fn commit_destructive(self) -> tracedecay_domain::errors::Result { - self.commit_with_hook(|_| Ok(())) - } - - #[hotpath::measure(label = "runtime_core.branch.commit_destructive")] - fn commit_with_hook( - self, - mut hook: H, - ) -> tracedecay_domain::errors::Result - where - H: FnMut(BranchAdminCommitBoundary) -> tracedecay_domain::errors::Result<()>, - { + /// CAS-publishes the exact prepared branch metadata. + #[hotpath::measure(label = "runtime_core.branch.commit_admin_mutation")] + pub fn commit(self) -> tracedecay_domain::errors::Result { if self.report.outcome != BranchAdminOutcome::Removed { return Ok(self.report); } let (_, current_metadata) = load_branch_meta_exact(&self.tracedecay_dir)?; if current_metadata != self.metadata_before { return Err(tracedecay_domain::errors::TraceDecayError::Config { - message: - "branch metadata changed after deletion selection; destructive CAS refused" - .to_owned(), + message: "branch metadata changed after selection; branch admin CAS refused" + .to_owned(), }); } for branch in &self.gc_branches { - if super::is_branch_ref_present(&self.project_root, branch) { + if super::local_branch_exists(&self.project_root, branch) { return Err(tracedecay_domain::errors::TraceDecayError::Config { message: format!( "branch ref '{branch}' reappeared before GC metadata publication; deletion refused" @@ -128,7 +88,6 @@ impl PreparedBranchAdminMutation { }); } } - hook(BranchAdminCommitBoundary::BeforeMetadataCas)?; if self.metadata_before != self.metadata_after { let after = self.metadata_after.as_deref().ok_or_else(|| { tracedecay_domain::errors::TraceDecayError::Config { @@ -147,123 +106,74 @@ impl PreparedBranchAdminMutation { }, )?; } - hook(BranchAdminCommitBoundary::AfterMetadataCas)?; - for path in &self.database_paths { - remove_branch_db_files_checked(path)?; - } Ok(self.report) } } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum BranchAdminCommitBoundary { - BeforeMetadataCas, - AfterMetadataCas, -} - -/// Selects a destructive branch mutation while holding the same lock used by -/// branch add. This function does not mutate metadata or unlink any file. +/// Selects a branch metadata mutation while holding the same lock used by +/// branch add. This function does not mutate metadata. #[hotpath::measure(label = "runtime_core.branch.prepare_admin_mutation")] pub fn prepare_branch_admin_mutation( project_root: &Path, tracedecay_dir: &Path, action: BranchAdminAction, branch_gc_days: u64, - orphan_db_gc_days: u64, ) -> tracedecay_domain::errors::Result { let branch_lock = acquire_branch_add_lock_blocking(tracedecay_dir)?; let (mut meta, metadata_before) = load_branch_meta_exact(tracedecay_dir)?; let default_branch = meta.as_ref().map(|meta| meta.default_branch.clone()); - let mut database_paths = Vec::new(); let mut removed_branches = Vec::new(); - let mut removed_orphan_dbs = Vec::new(); let mut gc_branches = Vec::new(); let mut single_store_retirements = Vec::new(); let mut outcome = BranchAdminOutcome::NoChanges; match action { BranchAdminAction::Remove { branch } => { - let Some(branch_meta) = meta.as_mut() else { - outcome = BranchAdminOutcome::NoTracking; - return Ok(PreparedBranchAdminMutation { - project_root: project_root.to_path_buf(), - tracedecay_dir: tracedecay_dir.to_path_buf(), - metadata_before: metadata_before.clone(), - metadata_after: metadata_before.clone(), - database_paths, - gc_branches, - single_store_retirements, - report: BranchAdminReport { - outcome, - removed_branches, - removed_orphan_dbs, - default_branch, - }, - _branch_lock: branch_lock, - }); - }; - if branch == branch_meta.default_branch { - return Err(tracedecay_domain::errors::TraceDecayError::Config { - message: format!("cannot remove default branch '{branch}'"), - }); - } - if let Some(entry) = branch_meta.remove_branch(&branch) { - // Branches served by the single project store retire - // metadata-only; only a legacy private copy is deletable. - if !entry.served_by_project_store() { - database_paths.push(tracedecay_dir.join(entry.db_file)); - } else if let Some(source) = entry.graph_source { - single_store_retirements.push(SingleStoreBranchRetirementV1 { - branch: branch.clone(), - source, + if let Some(branch_meta) = meta.as_mut() { + if branch == branch_meta.default_branch { + return Err(tracedecay_domain::errors::TraceDecayError::Config { + message: format!("cannot remove default branch '{branch}'"), }); } - removed_branches.push(branch.clone()); - outcome = BranchAdminOutcome::Removed; + if let Some(entry) = branch_meta.remove_branch(&branch) { + if let Some(source) = entry.graph_source { + single_store_retirements.push(SingleStoreBranchRetirementV1 { + branch: branch.clone(), + source, + }); + } + removed_branches.push(branch); + outcome = BranchAdminOutcome::Removed; + } else { + outcome = BranchAdminOutcome::NotTracked; + } } else { - outcome = BranchAdminOutcome::NotTracked; + outcome = BranchAdminOutcome::NoTracking; } } BranchAdminAction::RemoveAll => { - let Some(branch_meta) = meta.as_mut() else { - outcome = BranchAdminOutcome::NoTracking; - return Ok(PreparedBranchAdminMutation { - project_root: project_root.to_path_buf(), - tracedecay_dir: tracedecay_dir.to_path_buf(), - metadata_before: metadata_before.clone(), - metadata_after: metadata_before.clone(), - database_paths, - gc_branches, - single_store_retirements, - report: BranchAdminReport { - outcome, - removed_branches, - removed_orphan_dbs, - default_branch, - }, - _branch_lock: branch_lock, - }); - }; - let mut removed = branch_meta.remove_all_branches(); - removed.sort_by(|left, right| left.0.cmp(&right.0)); - for (branch, entry) in removed { - removed_branches.push(branch.clone()); - if !entry.served_by_project_store() { - database_paths.push(tracedecay_dir.join(entry.db_file)); - } else if let Some(source) = entry.graph_source { - single_store_retirements.push(SingleStoreBranchRetirementV1 { - branch: branch.clone(), - source, - }); + if let Some(branch_meta) = meta.as_mut() { + let mut removed = branch_meta.remove_all_branches(); + removed.sort_by(|left, right| left.0.cmp(&right.0)); + for (branch, entry) in removed { + if let Some(source) = entry.graph_source { + single_store_retirements.push(SingleStoreBranchRetirementV1 { + branch: branch.clone(), + source, + }); + } + removed_branches.push(branch); } - } - if !removed_branches.is_empty() { - outcome = BranchAdminOutcome::Removed; + if !removed_branches.is_empty() { + outcome = BranchAdminOutcome::Removed; + } + } else { + outcome = BranchAdminOutcome::NoTracking; } } BranchAdminAction::Gc => { - let now = super::now_unix_secs(); if let Some(branch_meta) = meta.as_mut() { + let now = super::now_unix_secs(); let branch_grace = branch_gc_days.saturating_mul(86_400); let default = branch_meta.default_branch.clone(); let mut candidates = branch_meta @@ -271,55 +181,33 @@ pub fn prepare_branch_admin_mutation( .iter() .filter(|(name, entry)| **name != default && !entry.gc_protected) .filter(|(name, entry)| { - !super::is_branch_ref_present(project_root, name) + !super::local_branch_exists(project_root, name) && now.saturating_sub(super::parse_unix_secs(&entry.last_synced_at)) >= branch_grace }) - .map(|(name, entry)| { - // Metadata is always collectable; only a legacy - // private store is a physical deletion candidate. - let private_store = (!entry.served_by_project_store()) - .then(|| tracedecay_dir.join(&entry.db_file)); - (name.clone(), private_store, entry.graph_source.clone()) - }) + .map(|(name, entry)| (name.clone(), entry.graph_source.clone())) .collect::>(); candidates.sort_by(|left, right| left.0.cmp(&right.0)); - for (name, private_store, source) in candidates { + for (name, source) in candidates { branch_meta.remove_branch(&name); gc_branches.push(name.clone()); removed_branches.push(name.clone()); - if let Some(path) = private_store { - database_paths.push(path); - } else if let Some(source) = source { + if let Some(source) = source { single_store_retirements.push(SingleStoreBranchRetirementV1 { branch: name, source, }); } } - } - let referenced = meta - .as_ref() - .map(|meta| { - meta.branches - .values() - .map(|entry| tracedecay_dir.join(&entry.db_file)) - .collect::>() - }) - .unwrap_or_default(); - removed_orphan_dbs = - select_orphan_dbs(tracedecay_dir, &referenced, orphan_db_gc_days, now); - database_paths.extend(removed_orphan_dbs.iter().cloned()); - if !removed_branches.is_empty() || !database_paths.is_empty() { - outcome = BranchAdminOutcome::Removed; - } else if meta.is_none() { + if !removed_branches.is_empty() { + outcome = BranchAdminOutcome::Removed; + } + } else { outcome = BranchAdminOutcome::NoTracking; } } } - database_paths.sort(); - database_paths.dedup(); let metadata_after = if removed_branches.is_empty() { metadata_before.clone() } else { @@ -336,75 +224,17 @@ pub fn prepare_branch_admin_mutation( tracedecay_dir: tracedecay_dir.to_path_buf(), metadata_before, metadata_after, - database_paths, gc_branches, single_store_retirements, report: BranchAdminReport { outcome, removed_branches, - removed_orphan_dbs, default_branch, }, _branch_lock: branch_lock, }) } -/// Retires branch metadata that branch-add published but could not sync. -/// Metadata is the only mutation: the branch never owned a database of its -/// own, and any legacy private store is left for canonical orphan collection. -/// Takes the branch-add lock for its own load-verify-save window. -#[cfg(test)] -pub(super) fn rollback_published_branch_tracking( - tracedecay_dir: &Path, - branch_name: &str, - db_file: &str, -) -> tracedecay_domain::errors::Result<()> { - let _branch_lock = acquire_branch_add_lock_blocking(tracedecay_dir)?; - let (meta, metadata_before) = load_branch_meta_exact(tracedecay_dir)?; - let mut meta = meta.ok_or_else(|| tracedecay_domain::errors::TraceDecayError::Config { - message: format!("cannot roll back branch '{branch_name}': branch metadata is missing"), - })?; - if meta - .branches - .get(branch_name) - .is_none_or(|entry| entry.db_file != db_file) - { - return Err(tracedecay_domain::errors::TraceDecayError::Config { - message: format!( - "cannot roll back branch '{branch_name}': published database path changed" - ), - }); - } - meta.remove_branch(branch_name); - let metadata_after = Some(crate::branch_meta::serialize_branch_meta(&meta)?); - let (_, current_metadata) = load_branch_meta_exact(tracedecay_dir)?; - if current_metadata != metadata_before { - return Err(tracedecay_domain::errors::TraceDecayError::Config { - message: format!("cannot roll back branch '{branch_name}': branch metadata changed"), - }); - } - let after = - metadata_after.ok_or_else(|| tracedecay_domain::errors::TraceDecayError::Config { - message: format!("cannot roll back branch '{branch_name}': metadata disappeared"), - })?; - crate::branch_meta::save_branch_meta_serialized(tracedecay_dir, &after).map_err(|error| { - tracedecay_domain::errors::TraceDecayError::Config { - message: format!("cannot retire failed branch '{branch_name}': {error}"), - } - }) -} - -/// Strict removal entry point used by daemon-owned administrative operations. -pub fn remove_tracked_branch_store_checked( - _tracedecay_dir: &Path, - _branch: &str, -) -> tracedecay_domain::errors::Result { - Err(tracedecay_domain::errors::TraceDecayError::Config { - message: "branch database deletion requires daemon store administration; use tracedecay_admin_branch through the managed daemon" - .to_string(), - }) -} - fn load_branch_meta_exact( tracedecay_dir: &Path, ) -> tracedecay_domain::errors::Result<(Option, Option)> { @@ -450,65 +280,6 @@ fn load_branch_meta_exact( use super::acquire_branch_lock_blocking as acquire_branch_add_lock_blocking; -fn branch_db_family_paths(db_path: &Path) -> [PathBuf; 3] { - let mut wal = db_path.to_path_buf(); - wal.set_extension("db-wal"); - let mut shm = db_path.to_path_buf(); - shm.set_extension("db-shm"); - [db_path.to_path_buf(), wal, shm] -} - -pub(super) fn remove_branch_db_files_checked( - db_path: &Path, -) -> tracedecay_domain::errors::Result<()> { - for path in branch_db_family_paths(db_path) { - match std::fs::remove_file(&path) { - Ok(()) => {} - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} - Err(error) => { - return Err(tracedecay_domain::errors::TraceDecayError::Config { - message: format!( - "failed to delete branch store file '{}': {error}", - path.display() - ), - }); - } - } - } - Ok(()) -} - -pub(super) fn select_orphan_dbs( - tracedecay_dir: &Path, - referenced: &std::collections::HashSet, - orphan_db_gc_days: u64, - now: u64, -) -> Vec { - let mut selected = Vec::new(); - let branches_dir = tracedecay_dir.join("branches"); - let Ok(entries) = std::fs::read_dir(&branches_dir) else { - return selected; - }; - let orphan_grace = orphan_db_gc_days.saturating_mul(86_400); - for entry in entries.flatten() { - let path = entry.path(); - if path.extension().and_then(|e| e.to_str()) != Some("db") || referenced.contains(&path) { - continue; - } - let mtime_secs = entry - .metadata() - .ok() - .and_then(|m| m.modified().ok()) - .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()) - .map_or(0, |d| d.as_secs()); - if now.saturating_sub(mtime_secs) >= orphan_grace { - selected.push(path); - } - } - selected.sort(); - selected -} - #[cfg(test)] #[allow(clippy::unwrap_used, clippy::expect_used)] mod tests; diff --git a/crates/tracedecay-runtime-core/src/branch/admin/tests.rs b/crates/tracedecay-runtime-core/src/branch/admin/tests.rs index ebab491960..48dae28035 100644 --- a/crates/tracedecay-runtime-core/src/branch/admin/tests.rs +++ b/crates/tracedecay-runtime-core/src/branch/admin/tests.rs @@ -20,18 +20,17 @@ fn fixture() -> (tempfile::TempDir, PathBuf, PathBuf) { let project_root = temp.path().join("repo"); let tracedecay_dir = temp.path().join("store"); std::fs::create_dir_all(&project_root).unwrap(); + std::fs::create_dir_all(&tracedecay_dir).unwrap(); run_git(&project_root, &["init", "-b", "main"]); run_git(&project_root, &["config", "user.email", "test@example.com"]); run_git(&project_root, &["config", "user.name", "TraceDecay Test"]); std::fs::write(project_root.join("fixture"), b"fixture").unwrap(); run_git(&project_root, &["add", "fixture"]); run_git(&project_root, &["commit", "-m", "fixture"]); - std::fs::create_dir_all(tracedecay_dir.join("branches")).unwrap(); std::fs::write(tracedecay_dir.join(crate::config::DB_FILENAME), b"main").unwrap(); let mut meta = crate::branch_meta::BranchMeta::new("main"); - meta.add_branch("feature", "branches/feature.db", "main"); + meta.add_branch("feature", "main"); crate::branch_meta::save_branch_meta(&tracedecay_dir, &meta).unwrap(); - std::fs::write(tracedecay_dir.join("branches/feature.db"), b"feature").unwrap(); (temp, project_root, tracedecay_dir) } @@ -43,20 +42,13 @@ fn prepare_remove(project_root: &Path, tracedecay_dir: &Path) -> PreparedBranchA branch: "feature".to_string(), }, 14, - 7, ) .unwrap() } -fn failpoint(message: &str) -> tracedecay_domain::errors::Result<()> { - Err(tracedecay_domain::errors::TraceDecayError::Config { - message: message.to_string(), - }) -} - fn add_sealed_single_store_branch(tracedecay_dir: &Path, branch: &str) { let mut meta = crate::branch_meta::load_branch_meta(tracedecay_dir).unwrap(); - meta.add_branch(branch, crate::config::DB_FILENAME, "main"); + meta.add_branch(branch, "main"); crate::branch_meta::save_branch_meta(tracedecay_dir, &meta).unwrap(); let source = crate::branch_meta::BranchGraphSourceDraftV1 { project_id: "project".to_owned(), @@ -75,15 +67,11 @@ fn add_sealed_single_store_branch(tracedecay_dir: &Path, branch: &str) { } #[test] -fn selection_is_read_only_and_commit_unlinks_exact_family() { +fn selection_is_read_only_and_commit_retires_metadata_only() { let (_temp, project_root, tracedecay_dir) = fixture(); - let db = tracedecay_dir.join("branches/feature.db"); - std::fs::write(db.with_extension("db-wal"), b"wal").unwrap(); - std::fs::write(db.with_extension("db-shm"), b"shm").unwrap(); + let main_db = tracedecay_dir.join(crate::config::DB_FILENAME); let prepared = prepare_remove(&project_root, &tracedecay_dir); - assert_eq!(prepared.database_paths(), std::slice::from_ref(&db)); - assert!(db.exists()); assert!( crate::branch_meta::load_branch_meta(&tracedecay_dir) .unwrap() @@ -92,55 +80,7 @@ fn selection_is_read_only_and_commit_unlinks_exact_family() { let report = prepared.commit().unwrap(); assert_eq!(report.outcome, BranchAdminOutcome::Removed); - assert!(!db.exists()); - assert!(!db.with_extension("db-wal").exists()); - assert!(!db.with_extension("db-shm").exists()); - assert!( - !crate::branch_meta::load_branch_meta(&tracedecay_dir) - .unwrap() - .is_tracked("feature") - ); -} - -#[test] -fn crash_before_metadata_cas_preserves_route_and_files() { - let (_temp, project_root, tracedecay_dir) = fixture(); - let db = tracedecay_dir.join("branches/feature.db"); - let error = prepare_remove(&project_root, &tracedecay_dir) - .commit_with_hook(|boundary| { - if boundary == BranchAdminCommitBoundary::BeforeMetadataCas { - failpoint("crash before metadata CAS") - } else { - Ok(()) - } - }) - .unwrap_err(); - - assert!(error.to_string().contains("crash before metadata CAS")); - assert!(db.exists()); - assert!( - crate::branch_meta::load_branch_meta(&tracedecay_dir) - .unwrap() - .is_tracked("feature") - ); -} - -#[test] -fn crash_after_metadata_cas_leaves_only_unreferenced_files() { - let (_temp, project_root, tracedecay_dir) = fixture(); - let db = tracedecay_dir.join("branches/feature.db"); - let error = prepare_remove(&project_root, &tracedecay_dir) - .commit_with_hook(|boundary| { - if boundary == BranchAdminCommitBoundary::AfterMetadataCas { - failpoint("crash after metadata CAS") - } else { - Ok(()) - } - }) - .unwrap_err(); - - assert!(error.to_string().contains("crash after metadata CAS")); - assert!(db.exists()); + assert!(main_db.exists(), "the project store must survive removal"); assert!( !crate::branch_meta::load_branch_meta(&tracedecay_dir) .unwrap() @@ -149,69 +89,40 @@ fn crash_after_metadata_cas_leaves_only_unreferenced_files() { } #[test] -fn metadata_cas_rejects_changed_store_path_without_unlink() { +fn metadata_cas_rejects_a_concurrent_metadata_change() { let (_temp, project_root, tracedecay_dir) = fixture(); - let db = tracedecay_dir.join("branches/feature.db"); let prepared = prepare_remove(&project_root, &tracedecay_dir); let mut changed = crate::branch_meta::load_branch_meta(&tracedecay_dir).unwrap(); - changed.branches.get_mut("feature").unwrap().db_file = "branches/recreated.db".to_owned(); + changed.branches.get_mut("feature").unwrap().last_synced_at = "foreign".to_owned(); crate::branch_meta::save_branch_meta(&tracedecay_dir, &changed).unwrap(); let error = prepared.commit().unwrap_err(); - assert!(error.to_string().contains("destructive CAS refused")); - assert!(db.exists()); + assert!(error.to_string().contains("CAS refused")); assert_eq!( crate::branch_meta::load_branch_meta(&tracedecay_dir) .unwrap() .branches["feature"] - .db_file, - "branches/recreated.db" + .last_synced_at, + "foreign" ); } #[test] fn gc_ref_reappearance_is_refused_before_metadata_cas() { let (_temp, project_root, tracedecay_dir) = fixture(); - let db = tracedecay_dir.join("branches/feature.db"); let mut meta = crate::branch_meta::load_branch_meta(&tracedecay_dir).unwrap(); meta.branches.get_mut("feature").unwrap().last_synced_at = "0".to_string(); crate::branch_meta::save_branch_meta(&tracedecay_dir, &meta).unwrap(); - let prepared = prepare_branch_admin_mutation( - &project_root, - &tracedecay_dir, - BranchAdminAction::Gc, - 0, - u64::MAX, - ) - .unwrap(); + let prepared = + prepare_branch_admin_mutation(&project_root, &tracedecay_dir, BranchAdminAction::Gc, 0) + .unwrap(); assert_eq!(prepared.report().removed_branches, vec!["feature"]); run_git(&project_root, &["branch", "feature"]); let error = prepared.commit().unwrap_err(); assert!(error.to_string().contains("reappeared")); - assert!(db.exists()); - assert!( - crate::branch_meta::load_branch_meta(&tracedecay_dir) - .unwrap() - .is_tracked("feature") - ); -} - -#[test] -fn nonempty_metadata_only_finish_fails_closed_without_deleting() { - let (_temp, project_root, tracedecay_dir) = fixture(); - let db = tracedecay_dir.join("branches/feature.db"); - let error = prepare_remove(&project_root, &tracedecay_dir) - .finish_without_database_deletion() - .unwrap_err(); - assert!( - error - .to_string() - .contains("requires daemon store administration") - ); - assert!(db.exists()); assert!( crate::branch_meta::load_branch_meta(&tracedecay_dir) .unwrap() @@ -229,7 +140,6 @@ fn branch_admin_never_selects_default_branch_for_removal() { branch: "main".to_string(), }, 14, - 7, ) .err() .expect("default branch removal must fail closed"); @@ -238,7 +148,7 @@ fn branch_admin_never_selects_default_branch_for_removal() { } #[test] -fn branch_admin_refuses_corrupt_metadata_without_selecting_stores() { +fn branch_admin_refuses_corrupt_metadata() { let (_temp, project_root, tracedecay_dir) = fixture(); std::fs::write( tracedecay_dir.join(crate::storage::BRANCH_META_FILENAME), @@ -247,48 +157,11 @@ fn branch_admin_refuses_corrupt_metadata_without_selecting_stores() { .unwrap(); let error = - prepare_branch_admin_mutation(&project_root, &tracedecay_dir, BranchAdminAction::Gc, 0, 0) + prepare_branch_admin_mutation(&project_root, &tracedecay_dir, BranchAdminAction::Gc, 0) .err() .expect("corrupt branch metadata must fail closed"); assert!(error.to_string().contains("corrupt or unreadable metadata")); - assert!(tracedecay_dir.join("branches/feature.db").exists()); -} - -/// A branch tracked on the single project store retires metadata-only: no -/// physical deletion candidate may reference the shared main database. -#[test] -fn removing_a_single_store_branch_never_deletes_the_project_store() { - let (_temp, project_root, tracedecay_dir) = fixture(); - let mut meta = crate::branch_meta::load_branch_meta(&tracedecay_dir).unwrap(); - meta.add_branch("topic", crate::config::DB_FILENAME, "main"); - crate::branch_meta::save_branch_meta(&tracedecay_dir, &meta).unwrap(); - let main_db = tracedecay_dir.join(crate::config::DB_FILENAME); - - let prepared = prepare_branch_admin_mutation( - &project_root, - &tracedecay_dir, - BranchAdminAction::Remove { - branch: "topic".to_string(), - }, - 14, - 7, - ) - .unwrap(); - - assert!( - prepared.database_paths().is_empty(), - "single-store branch removal must not select any database for deletion" - ); - // The daemon routes empty selections through the metadata-only commit. - let report = prepared.finish_without_database_deletion().unwrap(); - assert_eq!(report.outcome, BranchAdminOutcome::Removed); - assert!(main_db.exists(), "the project store must survive removal"); - assert!( - !crate::branch_meta::load_branch_meta(&tracedecay_dir) - .unwrap() - .is_tracked("topic") - ); } #[test] @@ -302,7 +175,6 @@ fn remove_all_carries_exact_single_store_provenance_for_daemon_retirement() { &tracedecay_dir, BranchAdminAction::RemoveAll, 14, - 7, ) .unwrap(); @@ -317,45 +189,36 @@ fn remove_all_carries_exact_single_store_provenance_for_daemon_retirement() { ); } -/// GC of a dead single-store branch collects its metadata while the shared -/// main database survives; a dead legacy private copy is still physically -/// collected in the same pass (Plan 38 keep-list). +/// GC of dead branches collects their metadata while the shared main +/// database survives; live and protected branches are retained. #[test] -fn gc_collects_single_store_metadata_and_legacy_stores_but_keeps_the_project_store() { +fn gc_collects_dead_branch_metadata_but_keeps_the_project_store() { let (_temp, project_root, tracedecay_dir) = fixture(); - let legacy_db = tracedecay_dir.join("branches/feature.db"); let main_db = tracedecay_dir.join(crate::config::DB_FILENAME); + run_git(&project_root, &["branch", "live"]); let mut meta = crate::branch_meta::load_branch_meta(&tracedecay_dir).unwrap(); - meta.add_branch("topic", crate::config::DB_FILENAME, "main"); - meta.branches.get_mut("topic").unwrap().last_synced_at = "0".to_string(); - meta.branches.get_mut("feature").unwrap().last_synced_at = "0".to_string(); + meta.add_branch("topic", "main"); + meta.add_branch("live", "main"); + meta.add_branch("pinned", "main"); + for branch in ["feature", "topic", "live", "pinned"] { + meta.branches.get_mut(branch).unwrap().last_synced_at = "0".to_string(); + } + meta.branches.get_mut("pinned").unwrap().gc_protected = true; crate::branch_meta::save_branch_meta(&tracedecay_dir, &meta).unwrap(); - let prepared = prepare_branch_admin_mutation( - &project_root, - &tracedecay_dir, - BranchAdminAction::Gc, - 0, - u64::MAX, - ) - .unwrap(); + let prepared = + prepare_branch_admin_mutation(&project_root, &tracedecay_dir, BranchAdminAction::Gc, 0) + .unwrap(); assert_eq!(prepared.report().removed_branches, vec!["feature", "topic"]); - assert_eq!( - prepared.database_paths(), - std::slice::from_ref(&legacy_db), - "only the legacy private store may be a physical deletion candidate" - ); let report = prepared.commit().unwrap(); assert_eq!(report.outcome, BranchAdminOutcome::Removed); - assert!( - !legacy_db.exists(), - "legacy private store must be collected" - ); assert!(main_db.exists(), "the project store must survive GC"); let persisted = crate::branch_meta::load_branch_meta(&tracedecay_dir).unwrap(); assert!(!persisted.is_tracked("topic")); assert!(!persisted.is_tracked("feature")); + assert!(persisted.is_tracked("live")); + assert!(persisted.is_tracked("pinned")); } #[test] @@ -369,14 +232,9 @@ fn gc_carries_only_exact_sealed_single_store_provenance_for_retirement() { .last_synced_at = "0".to_owned(); crate::branch_meta::save_branch_meta(&tracedecay_dir, &meta).unwrap(); - let prepared = prepare_branch_admin_mutation( - &project_root, - &tracedecay_dir, - BranchAdminAction::Gc, - 0, - u64::MAX, - ) - .unwrap(); + let prepared = + prepare_branch_admin_mutation(&project_root, &tracedecay_dir, BranchAdminAction::Gc, 0) + .unwrap(); assert_eq!( prepared diff --git a/crates/tracedecay-runtime-core/src/branch/tracking.rs b/crates/tracedecay-runtime-core/src/branch/tracking.rs index d6babfe8cf..10187a414c 100644 --- a/crates/tracedecay-runtime-core/src/branch/tracking.rs +++ b/crates/tracedecay-runtime-core/src/branch/tracking.rs @@ -291,15 +291,14 @@ pub async fn prepare_branch_tracking_in_layout( } })?; ( - branch_meta::BranchMeta::for_legacy_single_db(tracedecay_dir, &default), + branch_meta::BranchMeta::new_for_dir(tracedecay_dir, &default), true, ) } }; - let pruned_missing_branches = prune_missing_branch_dbs(tracedecay_dir, &mut meta); if meta.is_tracked(branch_name) { - if metadata_was_missing || pruned_missing_branches { + if metadata_was_missing { branch_meta::save_branch_meta(tracedecay_dir, &meta)?; } return Ok(BranchTrackingPreparation::AlreadyTracked); @@ -319,8 +318,7 @@ pub async fn prepare_branch_tracking_in_layout( // The branch is served by the single project graph store; the metadata // entry records lineage and the branch's graph-publication slot. Save // before the caller syncs so the fenced publication finds the entry. - let db_file = crate::config::db_filename(tracedecay_dir).to_owned(); - meta.add_branch(branch_name, &db_file, &parent); + meta.add_branch(branch_name, &parent); let entry = meta.branches.get(branch_name).cloned().ok_or_else(|| { tracedecay_domain::errors::TraceDecayError::Config { message: format!( @@ -388,92 +386,9 @@ async fn default_branch_bootstrap_persists_canonical_metadata() { let default = meta.branches.get("main").unwrap(); assert_eq!(default.db_file, crate::config::db_filename(&data_dir)); assert!(default.parent.is_none()); - assert_eq!(default.created_at, "0"); - assert_eq!(default.last_synced_at, "0"); + assert!(default.created_at.parse::().unwrap() > 0); + assert_eq!(default.last_synced_at, default.created_at); assert!(!meta_path.with_extension("json.tmp").exists()); - assert!(!data_dir.join("branches").exists()); -} - -#[cfg(test)] -#[tokio::test] -async fn already_tracked_branch_persists_pruned_missing_database_entries() { - let temp = tempfile::tempdir().unwrap(); - let project_root = temp.path().join("repo"); - std::fs::create_dir_all(&project_root).unwrap(); - let data_dir = temp.path().join("profile-shard"); - std::fs::create_dir_all(&data_dir).unwrap(); - std::fs::write(data_dir.join(crate::config::DB_FILENAME), b"graph").unwrap(); - - let mut meta = crate::branch_meta::BranchMeta::new("main"); - meta.add_branch("stale", "branches/missing.db", "main"); - crate::branch_meta::save_branch_meta(&data_dir, &meta).unwrap(); - - let outcome = prepare_branch_tracking_in_layout(&project_root, "main", &data_dir) - .await - .unwrap(); - - assert!(matches!(outcome, BranchTrackingPreparation::AlreadyTracked)); - let persisted = crate::branch_meta::load_branch_meta(&data_dir).unwrap(); - assert!(!persisted.is_tracked("stale")); -} - -#[cfg(test)] -#[test] -fn rollback_keeps_database_when_metadata_removal_cannot_be_saved() { - let temp = tempfile::tempdir().unwrap(); - let data_dir = temp.path(); - let branches_dir = data_dir.join("branches"); - std::fs::create_dir_all(&branches_dir).unwrap(); - let db_path = branches_dir.join("feature.db"); - std::fs::write(&db_path, b"graph").unwrap(); - - let mut meta = crate::branch_meta::BranchMeta::new("main"); - meta.add_branch("feature", "branches/feature.db", "main"); - crate::branch_meta::save_branch_meta(data_dir, &meta).unwrap(); - std::fs::create_dir(data_dir.join("branch-meta.json.tmp")).unwrap(); - - let error = rollback_branch_tracking(data_dir, "feature", "branches/feature.db") - .expect_err("blocked metadata publication must fail rollback"); - - assert!(db_path.exists()); - let persisted = crate::branch_meta::load_branch_meta(data_dir).unwrap(); - assert!(persisted.is_tracked("feature")); - assert!( - error.to_string().contains("cannot retire failed branch"), - "unexpected rollback error: {error}" - ); -} - -#[cfg(test)] -#[test] -fn rollback_retires_metadata_and_leaves_database_family_for_collection() { - let temp = tempfile::tempdir().unwrap(); - let data_dir = temp.path(); - let branches_dir = data_dir.join("branches"); - std::fs::create_dir_all(&branches_dir).unwrap(); - let db_path = branches_dir.join("feature.db"); - for path in [ - db_path.clone(), - db_path.with_extension("db-wal"), - db_path.with_extension("db-shm"), - ] { - std::fs::write(path, b"sqlite").unwrap(); - } - - let mut meta = crate::branch_meta::BranchMeta::new("main"); - meta.add_branch("feature", "branches/feature.db", "main"); - crate::branch_meta::save_branch_meta(data_dir, &meta).unwrap(); - - rollback_branch_tracking(data_dir, "feature", "branches/feature.db").unwrap(); - - assert!(db_path.exists()); - assert!(db_path.with_extension("db-wal").exists()); - assert!(db_path.with_extension("db-shm").exists()); - assert!( - !crate::branch_meta::load_branch_meta(data_dir) - .unwrap() - .is_tracked("feature") - ); } pub fn finalize_prepared_branch_tracking(tracedecay_dir: &Path, prepared: &PreparedBranchTracking) { @@ -501,45 +416,6 @@ pub fn rollback_prepared_branch_tracking( Ok(PreparedBranchRollbackOutcome::RolledBack) } -#[cfg(test)] -fn rollback_branch_tracking( - tracedecay_dir: &Path, - branch_name: &str, - db_file: &str, -) -> tracedecay_domain::errors::Result<()> { - super::admin::rollback_published_branch_tracking(tracedecay_dir, branch_name, db_file) -} - -fn prune_missing_branch_dbs( - tracedecay_dir: &Path, - meta: &mut crate::branch_meta::BranchMeta, -) -> bool { - let missing: Vec = meta - .branches - .iter() - .filter_map(|(name, entry)| { - if name == &meta.default_branch { - return None; - } - let path = tracedecay_dir.join(&entry.db_file); - (!path.exists()).then(|| name.clone()) - }) - .collect(); - let changed = !missing.is_empty(); - for name in missing { - meta.remove_branch(&name); - } - changed -} - -/// Returns true if `branch` currently exists as a local `refs/heads/*` ref. -/// -/// Thin alias over [`local_branch_exists`] under the name the branch-store GC -/// design refers to; keeping both avoids churning existing call sites. -pub fn is_branch_ref_present(project_root: &Path, branch: &str) -> bool { - local_branch_exists(project_root, branch) -} - /// Parses a `last_synced_at` / `created_at` unix-seconds string defensively. /// Returns 0 (epoch, i.e. maximally stale) when unparseable so a corrupt /// timestamp never protects a dead store from collection. diff --git a/crates/tracedecay-runtime-core/src/branch/tracking/tests.rs b/crates/tracedecay-runtime-core/src/branch/tracking/tests.rs index f886b0e1b4..e6e380ad6f 100644 --- a/crates/tracedecay-runtime-core/src/branch/tracking/tests.rs +++ b/crates/tracedecay-runtime-core/src/branch/tracking/tests.rs @@ -29,12 +29,7 @@ async fn tracking_a_new_branch_publishes_metadata_without_creating_a_database() crate::config::db_filename(&td), "single-store tracking must reference the canonical main database" ); - assert!(entry.served_by_project_store()); assert_eq!(entry.parent.as_deref(), Some("main")); - assert!( - !td.join("branches").exists(), - "tracking must not create a per-branch database" - ); assert_eq!( rollback_prepared_branch_tracking(&td, &prepared).unwrap(), diff --git a/crates/tracedecay-runtime-core/src/branch_meta.rs b/crates/tracedecay-runtime-core/src/branch_meta.rs index a89b69d42c..884ba8dae9 100644 --- a/crates/tracedecay-runtime-core/src/branch_meta.rs +++ b/crates/tracedecay-runtime-core/src/branch_meta.rs @@ -14,11 +14,9 @@ use crate::storage::{BRANCH_META_FILENAME, PrivateStoreIo}; /// Metadata for a single tracked branch. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct BranchEntry { - /// Relative path to the database serving this branch. Branches tracked on - /// the single project graph store reference the canonical main database - /// (`tracedecay.db`), the same shape the default branch has always used. - /// Legacy private branch copies reference `branches/.db`; those - /// files are retained only for garbage collection and never serve. + /// Relative path to the database serving this branch. Every branch is + /// served by the single project graph store, so this is always the + /// canonical main database (`tracedecay.db`). pub db_file: String, /// Nearest tracked ancestor at tracking time (None for the default /// branch). @@ -30,7 +28,6 @@ pub struct BranchEntry { pub last_synced_at: String, /// Whether automatic branch-store GC must retain this entry even when it /// has no matching git ref. - #[serde(default)] pub gc_protected: bool, /// Exact source identity of the graph published by the last successful /// sync. Older metadata omits this evidence and is not branch-query @@ -39,17 +36,6 @@ pub struct BranchEntry { pub graph_source: Option, } -impl BranchEntry { - /// True when this branch is served by the single project graph store. - /// - /// Only legacy entries reference a private `branches/.db` copy; - /// physical deletion inventories must be limited to those. - #[must_use] - pub fn served_by_project_store(&self) -> bool { - self.db_file == crate::config::DB_FILENAME - } -} - #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct BranchGraphSourceV1 { @@ -149,27 +135,16 @@ impl BranchMeta { Self::with_db_file(default_branch, crate::config::db_filename(data_dir)) } - /// Synthesizes metadata for a legacy store that only has the canonical - /// main database. The timestamps are deliberately unknown (`0`) so the - /// same input produces byte-identical metadata across interrupted retries. - pub fn for_legacy_single_db(data_dir: &Path, default_branch: &str) -> Self { - Self::with_db_file_and_timestamp(default_branch, crate::config::db_filename(data_dir), "0") - } - fn with_db_file(default_branch: &str, db_file: &str) -> Self { let now = now_unix_str(); - Self::with_db_file_and_timestamp(default_branch, db_file, &now) - } - - fn with_db_file_and_timestamp(default_branch: &str, db_file: &str, timestamp: &str) -> Self { let mut branches = HashMap::new(); branches.insert( default_branch.to_string(), BranchEntry { db_file: db_file.to_string(), parent: None, - created_at: timestamp.to_string(), - last_synced_at: timestamp.to_string(), + created_at: now.clone(), + last_synced_at: now, gc_protected: false, graph_source: None, }, @@ -180,13 +155,13 @@ impl BranchMeta { } } - /// Adds a new tracked branch entry. - pub fn add_branch(&mut self, name: &str, db_file: &str, parent: &str) { + /// Adds a new tracked branch entry served by the project graph store. + pub fn add_branch(&mut self, name: &str, parent: &str) { let now = now_unix_str(); self.branches.insert( name.to_string(), BranchEntry { - db_file: db_file.to_string(), + db_file: crate::config::DB_FILENAME.to_string(), parent: Some(parent.to_string()), created_at: now.clone(), last_synced_at: now, @@ -257,13 +232,6 @@ impl BranchMeta { self.default_branch ) })?; - let canonical_main = crate::config::DB_FILENAME; - if default.db_file != canonical_main { - return Err(format!( - "default branch '{}' must reference canonical main database '{canonical_main}', found '{}'", - self.default_branch, default.db_file - )); - } if default.parent.is_some() { return Err(format!( "default branch '{}' must not have a parent", @@ -271,27 +239,20 @@ impl BranchMeta { )); } - let mut db_files = BTreeMap::new(); + let canonical_main = crate::config::DB_FILENAME; for (name, entry) in &self.branches { if name.is_empty() { return Err("branch names must not be empty".to_string()); } - validate_db_file(name, entry, name == &self.default_branch)?; - if entry.parent.as_deref() == Some(name.as_str()) { - return Err(format!("branch '{name}' must not be its own parent")); - } - // The canonical main database is shared by every branch served - // from the single project store; only private legacy copies must - // be uniquely owned. - if entry.served_by_project_store() { - continue; - } - if let Some(previous) = db_files.insert(entry.db_file.as_str(), name.as_str()) { + if entry.db_file != canonical_main { return Err(format!( - "branches '{previous}' and '{name}' reference the same database '{}'", + "branch '{name}' must reference the project graph store '{canonical_main}', found '{}'", entry.db_file )); } + if entry.parent.as_deref() == Some(name.as_str()) { + return Err(format!("branch '{name}' must not be its own parent")); + } } Ok(()) } @@ -310,35 +271,6 @@ where .serialize(serializer) } -fn validate_db_file(name: &str, entry: &BranchEntry, is_default: bool) -> Result<(), String> { - let relative = Path::new(&entry.db_file); - if relative.as_os_str().is_empty() - || relative.is_absolute() - || relative - .components() - .any(|component| !matches!(component, std::path::Component::Normal(_))) - { - return Err(format!( - "branch '{name}' database path '{}' is not a normalized store-relative path", - entry.db_file - )); - } - if !is_default - && !entry.served_by_project_store() - && (!relative.starts_with("branches") - || !relative - .extension() - .is_some_and(|extension| extension.eq_ignore_ascii_case("db"))) - { - return Err(format!( - "non-default branch '{name}' database path '{}' must be the canonical main database \ - or a legacy store under 'branches/' with a .db extension", - entry.db_file - )); - } - Ok(()) -} - /// Parses `branch-meta.json` content into [`BranchMeta`]. /// /// This is the canonical definition of "corrupt branch metadata": anything @@ -583,7 +515,7 @@ mod tests { #[test] fn add_and_remove_branch() { let mut meta = BranchMeta::new("main"); - meta.add_branch("feature/foo", "branches/feature_foo.db", "main"); + meta.add_branch("feature/foo", "main"); assert!(meta.is_tracked("feature/foo")); assert!(!meta.is_query_eligible("feature/foo")); assert!(meta.is_query_eligible("main")); @@ -611,10 +543,10 @@ mod tests { #[test] fn parse_rejects_semantically_invalid_branch_metadata() { for content in [ - r#"{"default_branch":"main","branches":{"main":{"db_file":"branches/main.db","created_at":"0","last_synced_at":"0"}}}"#, - r#"{"default_branch":"main","branches":{"main":{"db_file":"tracedecay.db","parent":"main","created_at":"0","last_synced_at":"0"}}}"#, - r#"{"default_branch":"main","branches":{"main":{"db_file":"tracedecay.db","created_at":"0","last_synced_at":"0"},"escape":{"db_file":"../escape.db","created_at":"0","last_synced_at":"0"}}}"#, - r#"{"default_branch":"main","branches":{"main":{"db_file":"tracedecay.db","created_at":"0","last_synced_at":"0"},"left":{"db_file":"branches/shared.db","created_at":"0","last_synced_at":"0"},"right":{"db_file":"branches/shared.db","created_at":"0","last_synced_at":"0"}}}"#, + r#"{"default_branch":"main","branches":{"main":{"db_file":"branches/main.db","created_at":"0","last_synced_at":"0","gc_protected":false}}}"#, + r#"{"default_branch":"main","branches":{"main":{"db_file":"tracedecay.db","parent":"main","created_at":"0","last_synced_at":"0","gc_protected":false}}}"#, + r#"{"default_branch":"main","branches":{"main":{"db_file":"tracedecay.db","created_at":"0","last_synced_at":"0","gc_protected":false},"escape":{"db_file":"../escape.db","created_at":"0","last_synced_at":"0","gc_protected":false}}}"#, + r#"{"default_branch":"main","branches":{"main":{"db_file":"tracedecay.db","created_at":"0","last_synced_at":"0","gc_protected":false},"legacy":{"db_file":"branches/legacy.db","parent":"main","created_at":"0","last_synced_at":"0","gc_protected":false}}}"#, ] { assert!( parse(content).is_err(), @@ -628,41 +560,13 @@ mod tests { // The single-store tracking shape: every branch references the // canonical main database while keeping its own lineage and // graph-source provenance. - let content = r#"{"default_branch":"main","branches":{"main":{"db_file":"tracedecay.db","created_at":"0","last_synced_at":"0"},"feature/one":{"db_file":"tracedecay.db","parent":"main","created_at":"0","last_synced_at":"0"},"feature/two":{"db_file":"tracedecay.db","parent":"main","created_at":"0","last_synced_at":"0"}}}"#; + let content = r#"{"default_branch":"main","branches":{"main":{"db_file":"tracedecay.db","created_at":"0","last_synced_at":"0","gc_protected":false},"feature/one":{"db_file":"tracedecay.db","parent":"main","created_at":"0","last_synced_at":"0","gc_protected":false},"feature/two":{"db_file":"tracedecay.db","parent":"main","created_at":"0","last_synced_at":"0","gc_protected":true}}}"#; let meta = parse(content).expect("single-store tracking metadata must parse"); - assert!(meta.branches["feature/one"].served_by_project_store()); - assert!(meta.branches["feature/two"].served_by_project_store()); + assert!(meta.is_tracked("feature/one")); + assert!(meta.branches["feature/two"].gc_protected); assert!(!meta.is_tracked("feature/three")); - let legacy = parse( - r#"{"default_branch":"main","branches":{"main":{"db_file":"tracedecay.db","created_at":"0","last_synced_at":"0"},"legacy":{"db_file":"branches/legacy.db","created_at":"0","last_synced_at":"0"}}}"#, - ) - .expect("legacy private stores must keep parsing for collection"); - assert!(!legacy.branches["legacy"].served_by_project_store()); - } - - #[test] - fn parse_accepts_case_insensitive_branch_database_extensions() { - let mut meta = BranchMeta::new("main"); - meta.add_branch("legacy", "branches/legacy.DB", "main"); - - let content = serde_json::to_string(&meta).unwrap(); - - assert!(parse(&content).is_ok()); - } - - #[test] - fn legacy_single_db_metadata_is_byte_stable() { - let first = BranchMeta::for_legacy_single_db(Path::new("/profile/project"), "trunk"); - let second = BranchMeta::for_legacy_single_db(Path::new("/profile/project"), "trunk"); - - assert_eq!(first.branches["trunk"].created_at, "0"); - assert_eq!(first.branches["trunk"].last_synced_at, "0"); - assert_eq!( - serde_json::to_vec_pretty(&first).unwrap(), - serde_json::to_vec_pretty(&second).unwrap() - ); } #[cfg(unix)] @@ -682,20 +586,11 @@ mod tests { assert!(load_branch_meta(&data_dir).is_none()); } - #[test] - fn parse_old_entry_defaults_gc_protected_to_false() { - let meta = parse( - r#"{"default_branch":"main","branches":{"main":{"db_file":"tracedecay.db","created_at":"1","last_synced_at":"1"}}}"#, - ) - .unwrap(); - assert!(!meta.branches["main"].gc_protected); - } - #[test] fn update_synced_timestamp_advances_tracked_branch() { let dir = tempfile::tempdir().unwrap(); let mut meta = BranchMeta::new("main"); - meta.add_branch("feature/foo", "branches/feature_foo.db", "main"); + meta.add_branch("feature/foo", "main"); // Backdate so the advance is observable regardless of same-second timing. meta.branches.get_mut("feature/foo").unwrap().last_synced_at = "1000".to_string(); save_branch_meta(dir.path(), &meta).unwrap(); @@ -714,7 +609,7 @@ mod tests { fn update_synced_timestamp_holds_shared_branch_lock_during_load_modify_save() { let dir = tempfile::tempdir().unwrap(); let mut meta = BranchMeta::new("main"); - meta.add_branch("feature/foo", "branches/feature_foo.db", "main"); + meta.add_branch("feature/foo", "main"); save_branch_meta(dir.path(), &meta).unwrap(); let mut observed_contention = false; @@ -851,8 +746,8 @@ mod tests { fn concurrent_graph_source_publications_allocate_distinct_epochs() { let dir = tempfile::tempdir().unwrap(); let mut meta = BranchMeta::new_for_dir(dir.path(), "main"); - meta.add_branch("feature/one", crate::config::DB_FILENAME, "main"); - meta.add_branch("feature/two", crate::config::DB_FILENAME, "main"); + meta.add_branch("feature/one", "main"); + meta.add_branch("feature/two", "main"); save_branch_meta(dir.path(), &meta).unwrap(); let barrier = std::sync::Arc::new(std::sync::Barrier::new(2)); diff --git a/crates/tracedecay-runtime-core/src/config.rs b/crates/tracedecay-runtime-core/src/config.rs index 0762e128f8..7497775407 100644 --- a/crates/tracedecay-runtime-core/src/config.rs +++ b/crates/tracedecay-runtime-core/src/config.rs @@ -60,20 +60,6 @@ pub fn db_filename(data_dir: &Path) -> &'static str { DB_FILENAME } -/// Full path to the repo-local graph database marker path. -/// -/// Normal runtime graph storage resolves through [`crate::storage::StoreLayout`] -/// into the user profile shard; this helper is only for explicit marker checks -/// and migration cleanup. -pub fn get_project_db_path(project_root: &Path) -> PathBuf { - get_tracedecay_dir(project_root).join(DB_FILENAME) -} - -/// Returns true when the old repo-local `TraceDecay` graph DB exists at this root. -pub fn has_project_database(project_root: &Path) -> bool { - project_root.join(TRACEDECAY_DIR).join(DB_FILENAME).exists() -} - /// User-level data directory. Runtime storage is always rooted at /// `~/.tracedecay` unless `TRACEDECAY_DATA_DIR` explicitly overrides it. pub fn user_data_dir() -> Option { @@ -190,8 +176,8 @@ fn canonicalize_data_dir(path: PathBuf) -> PathBuf { /// 1. **Explicit path** (`--path`/`-p`, tool `path` argument): used verbatim, /// no discovery, and failure to open is fatal, never silently fall back. /// 2. **CWD walk-up** (this function via `resolve_path_with_discovery`): -/// nearest ancestor of the working directory containing an initialised -/// project database (see [`get_project_db_path`]). +/// nearest ancestor of the working directory hosting a path-local profile +/// store or, at a worktree root, a repository identity marker. /// /// `serve` forwards this routing metadata to the managed daemon. MCP /// `initialize` roots and registry aliases are resolved there; the proxy never @@ -227,8 +213,7 @@ pub fn is_initialized_project_root(dir: &Path) -> bool { } fn directory_hosts_initialized_project(dir: &Path, at_worktree_root: bool) -> bool { - has_project_database(dir) - || crate::storage::has_path_local_profile_store(dir) + crate::storage::has_path_local_profile_store(dir) || (at_worktree_root && crate::storage::has_repository_identity_marker(dir)) } diff --git a/crates/tracedecay-runtime-core/src/db/access.rs b/crates/tracedecay-runtime-core/src/db/access.rs index 56c8e826aa..4b84d651b1 100644 --- a/crates/tracedecay-runtime-core/src/db/access.rs +++ b/crates/tracedecay-runtime-core/src/db/access.rs @@ -30,17 +30,12 @@ pub use bootstrap::windows_hard_link_count; pub use lease::enter_maintenance_database_scope; #[cfg(not(test))] pub use lease::enter_owned_maintenance_database_scope; -#[cfg(test)] -use lease::fallback_scoped_runtime_role; -use lease::{acquire_process_lease, exact_scoped_runtime_role, scoped_runtime_role}; +use lease::{acquire_process_lease, exact_scoped_runtime_role}; pub use lease::{enter_daemon_database_scope, probe_writer_owner}; use owner_io::{ authority_token, epoch_ms, publish_record_atomically, read_record_strict, writer_owner, }; -use path_layout::{ - canonical_profile_root, database_profile_root, is_legacy_repository_database, - platform_identity_key, -}; +use path_layout::{canonical_profile_root, database_profile_root, platform_identity_key}; pub use tracedecay_private_fs::is_lock_contended; static PROCESS_LEASES: LazyLock>> = @@ -181,7 +176,6 @@ struct DatabaseIdentity { database_path: PathBuf, database_key: PathBuf, profile_root: PathBuf, - allows_ambient_profile_scope: bool, } #[derive(Debug)] @@ -247,9 +241,6 @@ impl DatabaseAuthority { if maintenance_active { return Self::acquire_identity(identity, DatabaseAuthorityRole::Maintenance, intent); } - if let Some(role) = scoped_runtime_role(&identity, intent)? { - return Self::acquire_identity(identity, role, intent); - } Err(access_error( intent, &identity.database_path, @@ -297,9 +288,6 @@ impl DatabaseAuthority { { return Self::acquire_identity(identity, DatabaseAuthorityRole::Test, intent); } - if let Some(role) = scoped_runtime_role(&identity, intent)? { - return Self::acquire_identity(identity, role, intent); - } Err(access_error( intent, &identity.database_path, @@ -339,16 +327,14 @@ impl DatabaseAuthority { return Ok(()); } - let active = match exact_scoped_runtime_role(&self.inner.identity.profile_root, intent)? { - Some(role) => role, - None => scoped_runtime_role(&self.inner.identity, intent)?.ok_or_else(|| { + let active = exact_scoped_runtime_role(&self.inner.identity.profile_root, intent)? + .ok_or_else(|| { access_error( intent, &self.inner.identity.database_path, "database write requires an active daemon or exclusive maintenance scope", ) - })?, - }; + })?; if active == self.inner.role { return Ok(()); } @@ -483,7 +469,6 @@ impl DatabaseIdentity { let database_key = platform_identity_key(&database_path); let profile_root = database_profile_root(&database_path, parent); Ok(Self { - allows_ambient_profile_scope: is_legacy_repository_database(&database_path), database_path, database_key, profile_root: platform_identity_key(&profile_root), @@ -561,7 +546,9 @@ fn foreign_daemon_authority_held(profile_root: &Path) -> bool { }; match file.try_lock().map_err(std::io::Error::from) { Ok(()) => { - let _ = file.unlock(); + if let Err(error) = file.unlock() { + tracing::warn!(%error, "daemon authority probe lock could not be released"); + } false } Err(error) if is_lock_contended(&error) => true, diff --git a/crates/tracedecay-runtime-core/src/db/access/lease.rs b/crates/tracedecay-runtime-core/src/db/access/lease.rs index ef32c3889e..0cf9931fd4 100644 --- a/crates/tracedecay-runtime-core/src/db/access/lease.rs +++ b/crates/tracedecay-runtime-core/src/db/access/lease.rs @@ -25,35 +25,6 @@ pub(super) fn exact_scoped_runtime_role( } } -pub(super) fn scoped_runtime_role( - identity: &DatabaseIdentity, - intent: &str, -) -> Result> { - if !identity.allows_ambient_profile_scope { - return Ok(None); - } - let maintenance = MAINTENANCE_SCOPES - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - let daemon = DAEMON_SCOPES - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - fallback_scoped_runtime_role(maintenance.len(), daemon.len()) - .map_err(|message| access_error(intent, &identity.profile_root, message)) -} - -pub(super) fn fallback_scoped_runtime_role( - maintenance_count: usize, - daemon_count: usize, -) -> std::result::Result, &'static str> { - match (maintenance_count, daemon_count) { - (1, 0) => Ok(Some(DatabaseAuthorityRole::Maintenance)), - (0, 1) => Ok(Some(DatabaseAuthorityRole::Daemon)), - (0, 0) => Ok(None), - _ => Err("database path is ambiguous across active profile authorities"), - } -} - pub fn enter_daemon_database_scope( profile_root: &Path, election_epoch: u64, diff --git a/crates/tracedecay-runtime-core/src/db/access/path_layout.rs b/crates/tracedecay-runtime-core/src/db/access/path_layout.rs index ef31e4539c..8edf3e4951 100644 --- a/crates/tracedecay-runtime-core/src/db/access/path_layout.rs +++ b/crates/tracedecay-runtime-core/src/db/access/path_layout.rs @@ -41,23 +41,21 @@ pub(super) fn database_profile_root(database_path: &Path, fallback_parent: &Path fn profile_project_root(database_path: &Path) -> Option<&Path> { let parent = database_path.parent()?; - // Branch graphs live one level below their project data root and share - // its profile authority scope. Consolidation staging shares it only for - // the two session snapshots consolidation itself creates; every other - // file under `.consolidation-input/` keeps its independent database - // identity rather than inheriting profile maintenance authority. + // Consolidation staging shares the project data root's profile authority + // scope only for the two session snapshots consolidation itself creates; + // every other file under `.consolidation-input/` keeps its independent + // database identity rather than inheriting profile maintenance authority. let staged_session_snapshot = parent .file_name() .is_some_and(|name| name == ".consolidation-input") && database_path .file_name() .is_some_and(|name| name == "source-sessions.db" || name == "target-sessions.db"); - let data_root = - if staged_session_snapshot || parent.file_name().is_some_and(|name| name == "branches") { - parent.parent()? - } else { - parent - }; + let data_root = if staged_session_snapshot { + parent.parent()? + } else { + parent + }; let shard_root = data_root.parent()?; if shard_root .file_name() @@ -96,28 +94,3 @@ fn profile_remote_node_root(database_path: &Path) -> Option<&Path> { } remote_root.parent() } - -pub(super) fn is_legacy_repository_database(database_path: &Path) -> bool { - let Some(parent) = database_path.parent() else { - return false; - }; - let is_branch_database = parent.file_name().is_some_and(|name| name == "branches"); - if !is_branch_database - && database_path.file_name().is_some_and(|name| { - name == "global.db" || name == "user-memory.db" || name == "user-sessions.db" - }) - { - return false; - } - let data_root = if is_branch_database { - let Some(data_root) = parent.parent() else { - return false; - }; - data_root - } else { - parent - }; - data_root - .file_name() - .is_some_and(|name| name == ".tracedecay") -} diff --git a/crates/tracedecay-runtime-core/src/db/access/tests.rs b/crates/tracedecay-runtime-core/src/db/access/tests.rs index 36b6ede9f1..440bfaf10d 100644 --- a/crates/tracedecay-runtime-core/src/db/access/tests.rs +++ b/crates/tracedecay-runtime-core/src/db/access/tests.rs @@ -127,7 +127,7 @@ fn profile_project_databases_share_the_profile_scope() { let temp = tempfile::tempdir().unwrap(); let profile = temp.path().join("profile"); let first = profile.join("projects/first/graph.db"); - let second = profile.join("projects/second/branches/main.db"); + let second = profile.join("projects/second/tracedecay.db"); std::fs::create_dir_all(first.parent().unwrap()).unwrap(); std::fs::create_dir_all(second.parent().unwrap()).unwrap(); @@ -273,21 +273,6 @@ fn writer_owner_intent_is_bounded_and_single_line() { assert!(owner.intent.len() <= 256); } -#[test] -fn fallback_scope_is_unambiguous_only_with_one_profile_owner() { - assert_eq!(fallback_scoped_runtime_role(0, 0).unwrap(), None); - assert_eq!( - fallback_scoped_runtime_role(1, 0).unwrap(), - Some(DatabaseAuthorityRole::Maintenance) - ); - assert_eq!( - fallback_scoped_runtime_role(0, 1).unwrap(), - Some(DatabaseAuthorityRole::Daemon) - ); - assert!(fallback_scoped_runtime_role(1, 1).is_err()); - assert!(fallback_scoped_runtime_role(2, 0).is_err()); -} - /// Reproduces the macOS `/var` -> `/private/var` scope-key shape on any unix /// host: a daemon enters database scope naming a profile root that does not /// exist yet, through a directory that is an alias for somewhere else. diff --git a/crates/tracedecay-runtime-core/src/db/evidence_assembly.rs b/crates/tracedecay-runtime-core/src/db/evidence_assembly.rs deleted file mode 100644 index d6871fc4fa..0000000000 --- a/crates/tracedecay-runtime-core/src/db/evidence_assembly.rs +++ /dev/null @@ -1,239 +0,0 @@ -use crate::db::engine::Executor; -use tracedecay_domain::errors::{Result, TraceDecayError}; - -pub(super) const EVIDENCE_ASSEMBLY_SCHEMA: &str = r" - CREATE TABLE IF NOT EXISTS evidence_source_occurrences ( - occurrence_id TEXT PRIMARY KEY CHECK(length(occurrence_id) > 0), - owner_digest TEXT NOT NULL CHECK(length(owner_digest) > 0), - timeline_digest TEXT NOT NULL CHECK(length(timeline_digest) > 0), - source_anchor_id TEXT NOT NULL CHECK(length(source_anchor_id) > 0), - source_order INTEGER NOT NULL CHECK(source_order >= 0), - record_digest TEXT NOT NULL CHECK(length(record_digest) > 0), - record_json TEXT NOT NULL CHECK(json_valid(record_json)) - ); - CREATE INDEX IF NOT EXISTS idx_evidence_occurrences_anchor - ON evidence_source_occurrences(owner_digest, source_anchor_id); - CREATE INDEX IF NOT EXISTS idx_evidence_occurrences_timeline - ON evidence_source_occurrences(owner_digest, timeline_digest, source_order); - - CREATE TABLE IF NOT EXISTS evidence_occurrence_sets ( - occurrence_set_id TEXT PRIMARY KEY CHECK(length(occurrence_set_id) > 0), - owner_digest TEXT NOT NULL CHECK(length(owner_digest) > 0), - record_digest TEXT NOT NULL CHECK(length(record_digest) > 0), - record_json TEXT NOT NULL CHECK(json_valid(record_json)) - ); - CREATE TABLE IF NOT EXISTS evidence_occurrence_set_members ( - occurrence_set_id TEXT NOT NULL, - canonical_ordinal INTEGER NOT NULL CHECK(canonical_ordinal >= 0), - occurrence_id TEXT NOT NULL, - PRIMARY KEY(occurrence_set_id, canonical_ordinal), - UNIQUE(occurrence_set_id, occurrence_id), - FOREIGN KEY(occurrence_set_id) - REFERENCES evidence_occurrence_sets(occurrence_set_id), - FOREIGN KEY(occurrence_id) - REFERENCES evidence_source_occurrences(occurrence_id) - ); - - CREATE TABLE IF NOT EXISTS evidence_spans ( - span_id TEXT PRIMARY KEY CHECK(length(span_id) > 0), - owner_digest TEXT NOT NULL CHECK(length(owner_digest) > 0), - occurrence_set_id TEXT NOT NULL, - anchor_id TEXT NOT NULL UNIQUE CHECK(length(anchor_id) > 0), - producer_kind TEXT NOT NULL CHECK(length(producer_kind) > 0), - record_digest TEXT NOT NULL CHECK(length(record_digest) > 0), - record_json TEXT NOT NULL CHECK(json_valid(record_json)), - FOREIGN KEY(occurrence_set_id) - REFERENCES evidence_occurrence_sets(occurrence_set_id) - ); - CREATE TABLE IF NOT EXISTS evidence_span_members ( - span_id TEXT NOT NULL, - assembly_ordinal INTEGER NOT NULL CHECK(assembly_ordinal >= 0), - run_ordinal INTEGER NOT NULL CHECK(run_ordinal >= 0), - run_member_ordinal INTEGER NOT NULL CHECK(run_member_ordinal >= 0), - occurrence_id TEXT NOT NULL, - PRIMARY KEY(span_id, assembly_ordinal), - UNIQUE(span_id, occurrence_id), - FOREIGN KEY(span_id) REFERENCES evidence_spans(span_id), - FOREIGN KEY(occurrence_id) - REFERENCES evidence_source_occurrences(occurrence_id) - ); - - CREATE TABLE IF NOT EXISTS evidence_span_projection_receipts ( - projection_receipt_id TEXT PRIMARY KEY CHECK(length(projection_receipt_id) > 0), - span_id TEXT NOT NULL, - record_digest TEXT NOT NULL CHECK(length(record_digest) > 0), - record_json TEXT NOT NULL CHECK(json_valid(record_json)), - UNIQUE(span_id, projection_receipt_id), - FOREIGN KEY(span_id) REFERENCES evidence_spans(span_id) - ); - - CREATE TABLE IF NOT EXISTS evidence_retriever_contributions ( - contribution_id TEXT PRIMARY KEY CHECK(length(contribution_id) > 0), - owner_digest TEXT NOT NULL CHECK(length(owner_digest) > 0), - span_id TEXT NOT NULL, - anchor_id TEXT NOT NULL UNIQUE CHECK(length(anchor_id) > 0), - record_digest TEXT NOT NULL CHECK(length(record_digest) > 0), - record_json TEXT NOT NULL CHECK(json_valid(record_json)), - FOREIGN KEY(span_id) REFERENCES evidence_spans(span_id) - ); - - CREATE TABLE IF NOT EXISTS evidence_derived_anchors ( - anchor_id TEXT PRIMARY KEY CHECK(length(anchor_id) > 0), - owner_digest TEXT NOT NULL CHECK(length(owner_digest) > 0), - target_kind TEXT NOT NULL CHECK( - target_kind IN ('source_occurrence', 'evidence_span', 'retriever_contribution') - ), - target_id TEXT NOT NULL CHECK(length(target_id) > 0), - anchor_json TEXT NOT NULL CHECK(json_valid(anchor_json)), - UNIQUE(owner_digest, target_kind, target_id) - ); - - CREATE TABLE IF NOT EXISTS evidence_assembly_receipts ( - publication_receipt_id TEXT PRIMARY KEY CHECK(length(publication_receipt_id) > 0), - owner_digest TEXT NOT NULL CHECK(length(owner_digest) > 0), - privacy_domain_id TEXT NOT NULL CHECK(length(privacy_domain_id) > 0), - key_epoch INTEGER NOT NULL CHECK(key_epoch > 0), - idempotency_key TEXT NOT NULL CHECK(length(idempotency_key) > 0), - assembly_digest TEXT NOT NULL CHECK(length(assembly_digest) > 0), - occurrence_set_id TEXT NOT NULL, - span_id TEXT NOT NULL, - contribution_id TEXT NOT NULL, - projection_receipt_id TEXT NOT NULL, - receipt_json TEXT NOT NULL CHECK(json_valid(receipt_json)), - UNIQUE(owner_digest, privacy_domain_id, key_epoch, idempotency_key), - FOREIGN KEY(occurrence_set_id) - REFERENCES evidence_occurrence_sets(occurrence_set_id), - FOREIGN KEY(span_id) REFERENCES evidence_spans(span_id), - FOREIGN KEY(contribution_id) - REFERENCES evidence_retriever_contributions(contribution_id), - FOREIGN KEY(projection_receipt_id) - REFERENCES evidence_span_projection_receipts(projection_receipt_id) - ); -"; - -pub(super) const EVIDENCE_ASSEMBLY_IMMUTABILITY: &str = r" - CREATE TRIGGER IF NOT EXISTS evidence_source_occurrences_immutable_update - BEFORE UPDATE ON evidence_source_occurrences BEGIN - SELECT RAISE(ABORT, 'evidence source occurrences are immutable'); - END; - CREATE TRIGGER IF NOT EXISTS evidence_source_occurrences_immutable_delete - BEFORE DELETE ON evidence_source_occurrences BEGIN - SELECT RAISE(ABORT, 'evidence source occurrences are immutable'); - END; - CREATE TRIGGER IF NOT EXISTS evidence_occurrence_sets_immutable_update - BEFORE UPDATE ON evidence_occurrence_sets BEGIN - SELECT RAISE(ABORT, 'evidence occurrence sets are immutable'); - END; - CREATE TRIGGER IF NOT EXISTS evidence_occurrence_sets_immutable_delete - BEFORE DELETE ON evidence_occurrence_sets BEGIN - SELECT RAISE(ABORT, 'evidence occurrence sets are immutable'); - END; - CREATE TRIGGER IF NOT EXISTS evidence_occurrence_set_members_immutable_update - BEFORE UPDATE ON evidence_occurrence_set_members BEGIN - SELECT RAISE(ABORT, 'evidence occurrence set membership is immutable'); - END; - CREATE TRIGGER IF NOT EXISTS evidence_occurrence_set_members_immutable_delete - BEFORE DELETE ON evidence_occurrence_set_members BEGIN - SELECT RAISE(ABORT, 'evidence occurrence set membership is immutable'); - END; - CREATE TRIGGER IF NOT EXISTS evidence_spans_immutable_update - BEFORE UPDATE ON evidence_spans BEGIN - SELECT RAISE(ABORT, 'evidence spans are immutable'); - END; - CREATE TRIGGER IF NOT EXISTS evidence_spans_immutable_delete - BEFORE DELETE ON evidence_spans BEGIN - SELECT RAISE(ABORT, 'evidence spans are immutable'); - END; - CREATE TRIGGER IF NOT EXISTS evidence_span_members_immutable_update - BEFORE UPDATE ON evidence_span_members BEGIN - SELECT RAISE(ABORT, 'evidence span membership is immutable'); - END; - CREATE TRIGGER IF NOT EXISTS evidence_span_members_immutable_delete - BEFORE DELETE ON evidence_span_members BEGIN - SELECT RAISE(ABORT, 'evidence span membership is immutable'); - END; - CREATE TRIGGER IF NOT EXISTS evidence_span_projection_receipts_immutable_update - BEFORE UPDATE ON evidence_span_projection_receipts BEGIN - SELECT RAISE(ABORT, 'evidence projection receipts are immutable'); - END; - CREATE TRIGGER IF NOT EXISTS evidence_span_projection_receipts_immutable_delete - BEFORE DELETE ON evidence_span_projection_receipts BEGIN - SELECT RAISE(ABORT, 'evidence projection receipts are immutable'); - END; - CREATE TRIGGER IF NOT EXISTS evidence_retriever_contributions_immutable_update - BEFORE UPDATE ON evidence_retriever_contributions BEGIN - SELECT RAISE(ABORT, 'retriever contributions are immutable'); - END; - CREATE TRIGGER IF NOT EXISTS evidence_retriever_contributions_immutable_delete - BEFORE DELETE ON evidence_retriever_contributions BEGIN - SELECT RAISE(ABORT, 'retriever contributions are immutable'); - END; - CREATE TRIGGER IF NOT EXISTS evidence_derived_anchors_immutable_update - BEFORE UPDATE ON evidence_derived_anchors BEGIN - SELECT RAISE(ABORT, 'evidence derived anchors are immutable'); - END; - CREATE TRIGGER IF NOT EXISTS evidence_derived_anchors_immutable_delete - BEFORE DELETE ON evidence_derived_anchors BEGIN - SELECT RAISE(ABORT, 'evidence derived anchors are immutable'); - END; - CREATE TRIGGER IF NOT EXISTS evidence_assembly_receipts_immutable_update - BEFORE UPDATE ON evidence_assembly_receipts BEGIN - SELECT RAISE(ABORT, 'evidence assembly receipts are immutable'); - END; - CREATE TRIGGER IF NOT EXISTS evidence_assembly_receipts_immutable_delete - BEFORE DELETE ON evidence_assembly_receipts BEGIN - SELECT RAISE(ABORT, 'evidence assembly receipts are immutable'); - END; -"; - -pub(crate) async fn install_evidence_assembly_schema( - conn: &(impl Executor + Sync), - operation: &str, -) -> Result<()> { - super::retrieval_anchor_schema::install_retrieval_anchor_schema(conn, operation).await?; - conn.execute_batch(EVIDENCE_ASSEMBLY_SCHEMA) - .await - .map_err(|error| TraceDecayError::Database { - message: format!("{operation}: failed to install evidence assembly schema: {error}"), - operation: operation.to_owned(), - })?; - conn.execute_batch(EVIDENCE_ASSEMBLY_IMMUTABILITY) - .await - .map_err(|error| TraceDecayError::Database { - message: format!("{operation}: failed to install evidence immutability: {error}"), - operation: operation.to_owned(), - })?; - conn.execute_batch( - "INSERT OR IGNORE INTO retrieval_anchor_reverse_lineage ( - source_anchor_id, owner_json, derivative_kind, derivative_id, direct_evidence - ) - SELECT occurrence.source_anchor_id, anchor.owner_json, - 'span', span.span_id, 1 - FROM evidence_spans AS span - JOIN evidence_span_members AS member - ON member.span_id = span.span_id - JOIN evidence_source_occurrences AS occurrence - ON occurrence.occurrence_id = member.occurrence_id - JOIN retrieval_anchors AS anchor - ON anchor.anchor_id = occurrence.source_anchor_id; - - INSERT OR IGNORE INTO retrieval_anchor_reverse_lineage ( - source_anchor_id, owner_json, derivative_kind, derivative_id, direct_evidence - ) - SELECT occurrence.source_anchor_id, anchor.owner_json, - 'contribution', contribution.contribution_id, 1 - FROM evidence_retriever_contributions AS contribution - JOIN evidence_span_members AS member - ON member.span_id = contribution.span_id - JOIN evidence_source_occurrences AS occurrence - ON occurrence.occurrence_id = member.occurrence_id - JOIN retrieval_anchors AS anchor - ON anchor.anchor_id = occurrence.source_anchor_id;", - ) - .await - .map_err(|error| TraceDecayError::Database { - message: format!("{operation}: failed to replay evidence anchor lineage: {error}"), - operation: operation.to_owned(), - })?; - Ok(()) -} diff --git a/crates/tracedecay-runtime-core/src/db/external_source.rs b/crates/tracedecay-runtime-core/src/db/external_source.rs index 255a1a81a7..2cfa00c14b 100644 --- a/crates/tracedecay-runtime-core/src/db/external_source.rs +++ b/crates/tracedecay-runtime-core/src/db/external_source.rs @@ -1,17 +1,10 @@ -//! Additive external-source state schema owned by the canonical database, -//! and the store-sized rewrite that retires its payload-copying predecessors. +//! Additive external-source state schema owned by the canonical database. -use crate::db::engine::{Executor, QueryExecutor, params}; +use crate::db::engine::Executor; use tracedecay_domain::errors::{Result, TraceDecayError}; -use tracedecay_rusqlite_runtime::repository::{ - RETIRED_MUTATION_COPY_CHUNK_ROWS, RETIRED_MUTATION_COPY_TABLES, -}; /// Installs the external-source state shape. Cheap idempotent DDL only, so it /// belongs inside a caller's leased schema transaction. -/// -/// Retiring a store's payload-copying predecessors is store-sized work and -/// runs separately through [`migrate_retired_mutation_copy_tables`]. pub async fn install_external_source_schema( connection: &impl Executor, operation: &str, @@ -24,473 +17,3 @@ pub async fn install_external_source_schema( operation: operation.to_owned(), }) } - -/// Rewrites every retired payload-copying table into its digest-referencing -/// successor, in bounded chunks, and drops it once it is empty. -/// -/// This is store-sized work: on a store with 189k `external_source_commit_ -/// receipts_v1` rows carrying about a gigabyte of `receipt_json`, the whole -/// rewrite measured around ten minutes. Inside the leased schema transaction -/// that made every open of a large store fail its per-statement execution -/// limit, so it runs here instead, after admission, on the writer, as -/// bounded work that never blocks admission or ordinary retrieval. -/// -/// The retired table's remaining contents are the durable progress: each -/// chunk moves its rows and removes them from the retired table in one -/// transaction, so an interrupted migration resumes exactly where it stopped, -/// no row is moved twice, the successors are keyed and the moves are -/// `INSERT OR IGNORE`, and none is lost. A store already at the current -/// shape carries none of these tables and pays one catalog probe each. -pub async fn migrate_retired_mutation_copy_tables(conn: &crate::db::Database) -> Result<()> { - const OPERATION: &str = "migrate retired external source mutation copies"; - for (retired_table, chunk_statements) in RETIRED_MUTATION_COPY_TABLES { - if !table_exists(&conn.read_connection(), retired_table) - .await - .map_err(|error| { - migration_failure( - format!("failed to probe for retired table {retired_table}"), - error, - ) - })? - { - continue; - } - loop { - let transaction = conn.begin_bulk_write_transaction(OPERATION).await?; - let Some(ceiling) = retired_chunk_ceiling(&transaction, retired_table).await? else { - // Empty: every row has moved, so only the retired table's - // own identity is left to retire. - transaction - .execute_batch(&format!("DROP TABLE {retired_table}")) - .await - .map_err(|error| { - migration_failure(format!("failed to drop {retired_table}"), error) - })?; - transaction.commit().await?; - break; - }; - for statement in *chunk_statements { - transaction - .execute(statement, params![ceiling]) - .await - .map_err(|error| { - migration_failure( - format!( - "failed to move {retired_table} rows through rowid {ceiling} \ - into their digest-referencing successor" - ), - error, - ) - })?; - } - transaction.commit().await?; - } - } - Ok(()) -} - -/// Retires the payload-copying predecessors inside a caller's transaction. -/// -/// The released project store carries them and the shape this binary admits -/// does not, so the one-time convergence of such a store has to finish the -/// move before admission rather than after it. Each chunk is still bounded by -/// the same statements [`migrate_retired_mutation_copy_tables`] uses; what a -/// caller gives up is resumability, which a one-shot upgrade transaction does -/// not have anyway. -pub(super) async fn retire_mutation_copies_in_transaction( - conn: &(impl Executor + Sync), -) -> Result<()> { - for (retired_table, chunk_statements) in RETIRED_MUTATION_COPY_TABLES { - if !table_exists(conn, retired_table).await? { - continue; - } - while let Some(ceiling) = retired_chunk_ceiling(conn, retired_table).await? { - for statement in *chunk_statements { - conn.execute(statement, params![ceiling]) - .await - .map_err(|error| { - migration_failure( - format!("failed to move {retired_table} rows through rowid {ceiling}"), - error, - ) - })?; - } - } - conn.execute_batch(&format!("DROP TABLE {retired_table}")) - .await - .map_err(|error| migration_failure(format!("failed to drop {retired_table}"), error))?; - } - Ok(()) -} - -fn migration_failure(message: String, error: impl std::fmt::Display) -> TraceDecayError { - TraceDecayError::Database { - message: format!("{message}: {error}"), - operation: "migrate retired external source mutation copies".to_owned(), - } -} - -/// The inclusive `rowid` of the last row in the next chunk, or `None` once the -/// retired table is empty. -async fn retired_chunk_ceiling( - connection: &impl QueryExecutor, - table: &str, -) -> Result> { - let mut rows = connection - .query( - &format!( - "SELECT MAX(rowid) FROM ( - SELECT rowid FROM {table} - ORDER BY rowid - LIMIT {RETIRED_MUTATION_COPY_CHUNK_ROWS} - )" - ), - (), - ) - .await - .map_err(|error| { - migration_failure(format!("failed to read the next {table} chunk"), error) - })?; - let Some(row) = rows.next().await.map_err(|error| { - migration_failure(format!("failed to step the next {table} chunk"), error) - })? - else { - return Err(migration_failure( - format!("chunk aggregate over {table} returned no row"), - "aggregate queries always return one row", - )); - }; - row.get::>(0).map_err(|error| { - migration_failure(format!("failed to decode the next {table} chunk"), error) - }) -} - -async fn table_exists(connection: &impl QueryExecutor, table: &str) -> Result { - let mut rows = connection - .query( - "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?1", - (table,), - ) - .await?; - Ok(rows.next().await?.is_some()) -} - -#[cfg(test)] -mod tests { - use super::{ - RETIRED_MUTATION_COPY_CHUNK_ROWS, install_external_source_schema, - migrate_retired_mutation_copy_tables, - }; - use crate::db::engine::QueryExecutor; - use crate::db::{Database, DatabaseAuthority, TestDatabaseRuntimeMode}; - - /// The retired shape exactly as an older binary left it, beside the - /// current schema the fixture runtime already installed. - const SEED_RETIRED_SHAPE: &str = " - CREATE TABLE external_source_objects_v1 ( - binding_id TEXT NOT NULL, native_object_digest TEXT NOT NULL, - partition_digest TEXT NOT NULL, mutation_digest TEXT NOT NULL, - mutation_json TEXT NOT NULL, - PRIMARY KEY (binding_id, native_object_digest)); - CREATE TABLE external_source_projected_objects_v1 ( - binding_id TEXT NOT NULL, native_object_digest TEXT NOT NULL, - mutation_json TEXT NOT NULL, - PRIMARY KEY (binding_id, native_object_digest)); - CREATE TABLE external_source_projection_effects_v1 ( - binding_id TEXT NOT NULL, projection_digest TEXT NOT NULL, - effect_index INTEGER NOT NULL, native_object_digest TEXT NOT NULL, - effect_json TEXT NOT NULL, mutation_json TEXT NOT NULL, - PRIMARY KEY (binding_id, projection_digest, effect_index)); - CREATE TABLE external_source_commit_receipts_v1 ( - binding_id TEXT NOT NULL, idempotency_key TEXT NOT NULL, - request_digest TEXT NOT NULL, definition_revision INTEGER NOT NULL, - binding_revision INTEGER NOT NULL, predecessor_frontier_digest TEXT NOT NULL, - successor_frontier_digest TEXT NOT NULL, receipt_digest TEXT NOT NULL, - receipt_json TEXT NOT NULL, - PRIMARY KEY (binding_id, idempotency_key));"; - - /// One retired receipt whose encoding carries every value the slim shape - /// keeps: two mutation digests, a source frontier, and a null prior one. - const SEED_RETIRED_ROWS: &str = " - INSERT INTO external_source_objects_v1 VALUES - ('b', 'sha256:obj', 'sha256:part', 'sha256:mut1', '{\"mutation_digest\":\"sha256:mut1\"}'); - INSERT INTO external_source_projected_objects_v1 VALUES - ('b', 'sha256:obj', '{\"x\":1,\"mutation_digest\":\"sha256:mut1\"}'); - INSERT INTO external_source_projection_effects_v1 VALUES - ('b', 'sha256:proj', 0, 'sha256:obj', '{\"effect\":true}', - '{\"mutation_digest\":\"sha256:mut1\"}'); - INSERT INTO external_source_commit_receipts_v1 VALUES - ('b', 'sha256:key', 'sha256:req', 1, 1, 'root', 'sha256:front1', 'sha256:rcpt', - '{\"idempotency_key\":\"sha256:key\",\"prior_source_frontier\":null,' - || '\"source_frontier\":{\"binding\":{\"id\":\"b\"},\"partitions\":{},\"digest\":\"sha256:front1\"},' - || '\"mutations\":[{\"x\":1,\"mutation_digest\":\"sha256:mut1\"},{\"x\":2,\"mutation_digest\":\"sha256:mut2\"}],' - || '\"receipt_digest\":\"sha256:rcpt\"}');"; - - /// Seeds the retired object table in batches that each fit inside an - /// ordinary write, so the fixture itself never depends on the limit this - /// test is about. - async fn seed_retired_objects(db: &Database, rows: i64) { - const BATCH: i64 = 250_000; - let mut seeded = 0; - while seeded < rows { - let batch = BATCH.min(rows - seeded); - commit_batch( - db, - "seed retired objects", - &format!( - "INSERT INTO external_source_objects_v1 - WITH RECURSIVE row_index(index_value) AS ( - SELECT {start} UNION ALL - SELECT index_value + 1 FROM row_index WHERE index_value < {end} - ) - SELECT 'b', 'sha256:obj' || index_value, 'sha256:part', - 'sha256:mut' || index_value, - '{{\"mutation_digest\":\"sha256:mut' || index_value || '\"}}' - FROM row_index", - start = seeded + 1, - end = seeded + batch, - ), - ) - .await; - seeded += batch; - } - } - - async fn count(connection: &impl QueryExecutor, sql: &str) -> i64 { - let mut rows = connection.query(sql, ()).await.unwrap(); - rows.next().await.unwrap().unwrap().get::(0).unwrap() - } - - async fn open_store(path: &std::path::Path) -> Database { - let authority = DatabaseAuthority::acquire_test(path, "external source migration").unwrap(); - let (db, _) = - Database::publish_test_runtime(path, &authority, TestDatabaseRuntimeMode::Initialize) - .await - .unwrap(); - // Leaking the authority keeps the published runtime's write scope for - // the whole test; it is dropped with the process. - std::mem::forget(authority); - db - } - - async fn commit_batch(db: &Database, operation: &'static str, sql: &str) { - let writer = db.begin_write_transaction(operation).await.unwrap(); - writer.execute_batch(sql).await.unwrap(); - writer.commit().await.unwrap(); - } - - /// A store still carrying the tables that duplicated `mutation_json` is - /// moved to the digest-referencing shape in place: every current row - /// survives with the digest its own encoding carried, the retired tables - /// are dropped, and a second run is a no-op. - #[tokio::test] - async fn migration_moves_retired_mutation_copies_by_reference() { - let temp = tempfile::tempdir().unwrap(); - let db = open_store(&temp.path().join("graph.db")).await; - commit_batch( - &db, - "seed retired shape", - &format!("{SEED_RETIRED_SHAPE}{SEED_RETIRED_ROWS}"), - ) - .await; - - migrate_retired_mutation_copy_tables(&db).await.unwrap(); - - let reader = db.read_connection(); - assert_eq!( - count( - &reader, - "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name IN ( - 'external_source_objects_v1', 'external_source_projected_objects_v1', - 'external_source_projection_effects_v1', 'external_source_commit_receipts_v1')", - ) - .await, - 0, - "every retired table is dropped" - ); - // The receipt is slim: mutation digests where mutations were, the - // frontier digest where the frontier was, and the frontier itself - // stored once under its digest. - assert_eq!( - count( - &reader, - "SELECT COUNT(*) FROM external_source_commit_receipts_v2 - WHERE binding_id = 'b' - AND json_extract(receipt_json, '$.source_frontier') = 'sha256:front1' - AND json_type(receipt_json, '$.prior_source_frontier') = 'null' - AND json_extract(receipt_json, '$.mutations[0]') = 'sha256:mut1' - AND json_extract(receipt_json, '$.mutations[1]') = 'sha256:mut2' - AND json_array_length(receipt_json, '$.mutations') = 2 - AND json_extract(receipt_json, '$.receipt_digest') = 'sha256:rcpt'" - ) - .await, - 1 - ); - assert_eq!( - count( - &reader, - "SELECT COUNT(*) FROM external_source_frontiers_v1 - WHERE binding_id = 'b' AND frontier_digest = 'sha256:front1' - AND json_extract(frontier_json, '$.binding.id') = 'b'" - ) - .await, - 1 - ); - assert_eq!( - count( - &reader, - "SELECT COUNT(*) FROM external_source_objects_v2 - WHERE binding_id = 'b' AND mutation_digest = 'sha256:mut1'" - ) - .await, - 1 - ); - assert_eq!( - count( - &reader, - "SELECT COUNT(*) FROM external_source_projected_objects_v2 - WHERE binding_id = 'b' AND mutation_digest = 'sha256:mut1'" - ) - .await, - 1, - "the projected object's digest is read out of its retired encoding" - ); - assert_eq!( - count( - &reader, - "SELECT COUNT(*) FROM external_source_projection_effects_v2 - WHERE binding_id = 'b' AND mutation_digest = 'sha256:mut1' - AND effect_json = '{\"effect\":true}'" - ) - .await, - 1 - ); - - migrate_retired_mutation_copy_tables(&db).await.unwrap(); - assert_eq!( - count(&reader, "SELECT COUNT(*) FROM external_source_objects_v2").await, - 1, - "re-running on a migrated store changes nothing" - ); - } - - /// Installing the schema is cheap idempotent DDL and must leave a retired - /// table alone: the store-sized rewrite is not the installer's work, and - /// running it inside the caller's leased schema transaction is what took - /// the daemon down on a large store. - #[tokio::test] - async fn install_leaves_the_store_sized_rewrite_to_the_migration() { - let temp = tempfile::tempdir().unwrap(); - let db = open_store(&temp.path().join("graph.db")).await; - commit_batch( - &db, - "seed retired shape", - &format!("{SEED_RETIRED_SHAPE}{SEED_RETIRED_ROWS}"), - ) - .await; - - let writer = db - .begin_write_transaction("reinstall schema") - .await - .unwrap(); - install_external_source_schema(&writer, "install test") - .await - .unwrap(); - writer.commit().await.unwrap(); - - let reader = db.read_connection(); - assert_eq!( - count( - &reader, - "SELECT COUNT(*) FROM external_source_commit_receipts_v1" - ) - .await, - 1, - "install must not rewrite retired rows" - ); - assert_eq!( - count( - &reader, - "SELECT COUNT(*) FROM external_source_commit_receipts_v2" - ) - .await, - 0 - ); - } - - /// A migration killed between chunks leaves the rows it moved in the - /// successor and the rest in the retired table. Resuming from that exact - /// on-disk state must move every remaining row once: the successor's row - /// count and its digest content match the whole seeded set, with no row - /// duplicated and none lost. - #[tokio::test] - async fn migration_resumes_from_a_partially_moved_table() { - let temp = tempfile::tempdir().unwrap(); - let db = open_store(&temp.path().join("graph.db")).await; - // Two chunks plus a remainder, so resumption is exercised across a - // chunk boundary rather than inside a single pass. - let seeded = RETIRED_MUTATION_COPY_CHUNK_ROWS * 2 + 17; - commit_batch(&db, "seed retired shape", SEED_RETIRED_SHAPE).await; - seed_retired_objects(&db, seeded).await; - - // Exactly what a kill after the first committed chunk leaves behind. - commit_batch( - &db, - "replay one committed chunk", - &format!( - "INSERT OR IGNORE INTO external_source_objects_v2 ( - binding_id, native_object_digest, partition_digest, mutation_digest - ) - SELECT binding_id, native_object_digest, partition_digest, mutation_digest - FROM external_source_objects_v1 - WHERE rowid <= {RETIRED_MUTATION_COPY_CHUNK_ROWS}; - DELETE FROM external_source_objects_v1 - WHERE rowid <= {RETIRED_MUTATION_COPY_CHUNK_ROWS};" - ), - ) - .await; - let reader = db.read_connection(); - assert_eq!( - count(&reader, "SELECT COUNT(*) FROM external_source_objects_v1").await, - seeded - RETIRED_MUTATION_COPY_CHUNK_ROWS, - "the partially moved store must still hold the unmoved rows" - ); - - migrate_retired_mutation_copy_tables(&db).await.unwrap(); - - assert_eq!( - count(&reader, "SELECT COUNT(*) FROM external_source_objects_v2").await, - seeded, - "resumption moves every remaining row exactly once" - ); - // Content digest over the whole successor, so a duplicated or lost - // row fails here even when the count happens to agree. - let mut rows = reader - .query( - "SELECT COUNT(*), COUNT(DISTINCT native_object_digest), - SUM(CAST(replace(mutation_digest, 'sha256:mut', '') AS INTEGER)) - FROM external_source_objects_v2", - (), - ) - .await - .unwrap(); - let row = rows.next().await.unwrap().unwrap(); - assert_eq!(row.get::(0).unwrap(), seeded); - assert_eq!(row.get::(1).unwrap(), seeded, "no row moved twice"); - assert_eq!( - row.get::(2).unwrap(), - seeded * (seeded + 1) / 2, - "every seeded digest survives exactly once" - ); - assert_eq!( - count( - &reader, - "SELECT COUNT(*) FROM sqlite_master - WHERE type = 'table' AND name = 'external_source_objects_v1'" - ) - .await, - 0, - "the retired table is dropped only once it is empty" - ); - } -} diff --git a/crates/tracedecay-runtime-core/src/db/memory_v2/mod.rs b/crates/tracedecay-runtime-core/src/db/memory_v2/mod.rs index 72d59d8d95..4be0fc756d 100644 --- a/crates/tracedecay-runtime-core/src/db/memory_v2/mod.rs +++ b/crates/tracedecay-runtime-core/src/db/memory_v2/mod.rs @@ -16,9 +16,7 @@ mod tests; #[cfg(test)] mod types; -pub(in crate::db) use schema::{ - FINAL_SCHEMA_BATCHES, PAYLOAD_DIGEST_OBJECTS, PAYLOAD_DIGESTS_SCHEMA, create_schema, -}; +pub(in crate::db) use schema::{FINAL_SCHEMA_BATCHES, create_schema}; #[cfg(test)] use types::OwnerKey; diff --git a/crates/tracedecay-runtime-core/src/db/memory_v2/schema/mod.rs b/crates/tracedecay-runtime-core/src/db/memory_v2/schema/mod.rs index 0ef0d1f1d5..863aa32aa5 100644 --- a/crates/tracedecay-runtime-core/src/db/memory_v2/schema/mod.rs +++ b/crates/tracedecay-runtime-core/src/db/memory_v2/schema/mod.rs @@ -6,7 +6,6 @@ mod final_authority; mod payload_digests; pub(in crate::db) use baseline::create_schema; -pub(in crate::db) use payload_digests::{PAYLOAD_DIGEST_OBJECTS, PAYLOAD_DIGESTS_SCHEMA}; pub(in crate::db) const FINAL_SCHEMA_BATCHES: &[&str] = &[ baseline::BASELINE_SCHEMA, payload_digests::PAYLOAD_DIGESTS_SCHEMA, diff --git a/crates/tracedecay-runtime-core/src/db/memory_v2/schema/payload_digests.rs b/crates/tracedecay-runtime-core/src/db/memory_v2/schema/payload_digests.rs index c98b642a4d..b059f6ecdd 100644 --- a/crates/tracedecay-runtime-core/src/db/memory_v2/schema/payload_digests.rs +++ b/crates/tracedecay-runtime-core/src/db/memory_v2/schema/payload_digests.rs @@ -14,9 +14,8 @@ use tracedecay_domain::errors::Result; use super::super::{MemoryV2Executor, db_error}; /// Companion table plus the lookup index and the immutability / cascade -/// triggers. Everything is `IF NOT EXISTS` so the v34 → v35 step can resume -/// after an interrupted run. -pub(in crate::db) const PAYLOAD_DIGESTS_SCHEMA: &str = +/// triggers. +pub(super) const PAYLOAD_DIGESTS_SCHEMA: &str = "CREATE TABLE IF NOT EXISTS memory_v2_assertion_payload_digests ( payload_rowid INTEGER PRIMARY KEY, assertion_id TEXT NOT NULL, @@ -46,16 +45,6 @@ pub(in crate::db) const PAYLOAD_DIGESTS_SCHEMA: &str = WHERE payload_rowid = OLD.rowid; END;"; -/// Names of the objects `PAYLOAD_DIGESTS_SCHEMA` creates, in the order the -/// final-shape inventory reports them. The v34 → v35 step admits a store -/// whose inventory is exactly the final shape minus these. -pub(in crate::db) const PAYLOAD_DIGEST_OBJECTS: &[&str] = &[ - "memory_v2_assertion_payload_digests", - "memory_v2_assertion_payload_digests_lookup", - "memory_v2_assertion_payload_digests_no_update", - "memory_v2_payloads_digest_delete", -]; - pub(super) async fn install_payload_digests( conn: &impl MemoryV2Executor, operation: &str, diff --git a/crates/tracedecay-runtime-core/src/db/migrations.rs b/crates/tracedecay-runtime-core/src/db/migrations.rs index 8982a31fd2..dff3b3fc83 100644 --- a/crates/tracedecay-runtime-core/src/db/migrations.rs +++ b/crates/tracedecay-runtime-core/src/db/migrations.rs @@ -1,19 +1,17 @@ //! Schema creation for the tracedecay database. //! -//! Fresh stores use the current relational shape. Known released v34/v35 -//! stores converge in place, retaining their durable rows and exact historical -//! staging objects; unknown shapes remain typed refusals. +//! Fresh stores are created directly at the current relational shape. Any +//! other stamp or shape is a typed reset-required refusal; nothing upgrades in +//! place. use std::future::Future; use std::pin::Pin; use crate::db::connection::DatabaseEngineWriteConnection; -use crate::db::engine::{Connection, Executor, QueryExecutor, params}; +use crate::db::engine::{Connection, Executor, QueryExecutor}; use tracedecay_domain::errors::{Result, TraceDecayError}; -use tracedecay_rusqlite_runtime::runtime_ledger; mod final_shape; -mod released_shape; pub use final_shape::{ expected_final_schema_fingerprint, fingerprint_schema_objects, @@ -42,34 +40,23 @@ const ROOT_SCHEMA: &str = "CREATE TABLE IF NOT EXISTS metadata ( CREATE INDEX IF NOT EXISTS idx_read_cache_session ON read_cache(session_id, created_at);"; -/// The schema stamp this binary creates after released-store convergence. +/// The schema stamp this binary creates. /// /// Code topology lives only in the verified Grafeo generation. Exact memory /// content, provenance, trust, retention, and feedback live only in the /// canonical `memory_v2_*` tables; holographic memory vectors are re-derived -/// from that content and never persisted. v36 dropped the semantic-vector -/// staging family (`semantic_vector_*`) from fresh stores with dense code -/// retrieval. Existing exact released staging objects remain preserved. -pub const SCHEMA_VERSION: u32 = 36; - -/// Verifies that a rusqlite connection sees the final relational shape this -/// binary admits, including the one shipped-v35 trigger shape it repairs on a -/// writer open. This is query-only and shares the daemon's schema authority. +/// from that content and never persisted. v38 stores compact writer-ledger +/// receipts, keeps full external-source receipts only while retained, and +/// stores each external-source mutation without what its row already says. +pub const SCHEMA_VERSION: u32 = 38; + +/// Verifies that a rusqlite connection sees the exact final relational shape +/// this binary creates. This is query-only and shares the daemon's schema +/// authority. pub fn verify_admissible_final_shape_rusqlite(conn: &rusqlite::Connection) -> Result<()> { final_shape::require_admissible_final_shape_rusqlite(conn) } -/// The released stamp preceding persisted payload digests. Live v35 stores -/// already carry those digests and only need relational convergence. -pub const PAYLOAD_DIGEST_STEP_SOURCE_VERSION: u32 = 34; - -/// Metadata key journaling the payload-digest backfill receipt the v34 step -/// writes; named for the stamp that introduced the digests. -pub const PAYLOAD_DIGEST_BACKFILL_RECEIPT_KEY: &str = "memory_v2.payload_digest_backfill.v35"; - -/// Payload rows fingerprinted per short backfill write. -const PAYLOAD_DIGEST_BACKFILL_CHUNK_ROWS: usize = 512; - /// Reads the current schema version from `PRAGMA user_version`. async fn get_version(conn: &impl QueryExecutor) -> Result { let mut rows = @@ -197,7 +184,6 @@ async fn create_schema_transaction(conn: &(impl Executor + Sync)) -> Result<()> })?; super::memory_v2::create_schema(conn, "create_schema").await?; - super::evidence_assembly::install_evidence_assembly_schema(conn, "create_schema").await?; super::external_source::install_external_source_schema(conn, "create_schema").await?; conn.execute_batch(tracedecay_store::GENERATION_DIAGNOSTICS_SCHEMA_DDL) .await @@ -228,67 +214,6 @@ async fn create_schema_transaction(conn: &(impl Executor + Sync)) -> Result<()> Ok(()) } -/// Installs the runtime-writer ledger into a canonical store that predates it -/// and folds a retired `idempotency_v1` table into the current one with the -/// same bounded statements the registered stores converge with. Idempotent: a -/// store already at the current ledger shape is untouched. -async fn install_runtime_writer_ledger( - conn: &(impl Executor + Sync), - operation: &str, -) -> Result<()> { - let failure = |message: String| TraceDecayError::Database { - message, - operation: operation.to_owned(), - }; - conn.execute_batch(runtime_ledger::RUNTIME_LEDGER_SCHEMA) - .await - .map_err(|e| { - failure(format!( - "failed to create runtime-writer ledger schema: {e}" - )) - })?; - if !runtime_writer_ledger_retired_table_present(conn, operation).await? { - return Ok(()); - } - loop { - let copied = conn - .execute(runtime_ledger::COPY_RETIRED_IDEMPOTENCY_LEDGER_PAGE_SQL, ()) - .await - .map_err(|e| failure(format!("failed to copy retired idempotency page: {e}")))?; - let retired = conn - .execute( - runtime_ledger::DELETE_CONVERGED_IDEMPOTENCY_LEDGER_PAGE_SQL, - (), - ) - .await - .map_err(|e| failure(format!("failed to retire converged idempotency page: {e}")))?; - let remaining = sqlite_master_probe( - conn, - "SELECT 1 FROM td_runtime_writer_idempotency_v1 LIMIT 1", - operation, - ) - .await?; - if !remaining { - break; - } - if copied == 0 && retired == 0 { - // A key present in both tables with different receipts: neither - // side may be chosen silently. - return Err(failure( - "retired runtime-writer idempotency receipts diverge from the current ledger" - .to_owned(), - )); - } - } - conn.execute_batch(runtime_ledger::DROP_RETIRED_IDEMPOTENCY_LEDGER_SQL) - .await - .map_err(|e| { - failure(format!( - "failed to drop the retired idempotency ledger: {e}" - )) - }) -} - /// Runs a `SELECT 1 ... LIMIT 1`-shaped probe against `sqlite_master` and /// reports whether it matched anything. async fn sqlite_master_probe( @@ -313,41 +238,6 @@ async fn sqlite_master_probe( .is_some()) } -async fn runtime_writer_ledger_retired_table_present( - conn: &impl QueryExecutor, - operation: &str, -) -> Result { - sqlite_master_probe( - conn, - runtime_ledger::RETIRED_IDEMPOTENCY_LEDGER_PRESENT_SQL, - operation, - ) - .await -} - -/// Reports whether an existing store still lacks any ledger object or carries -/// the retired idempotency table, i.e. whether the sanctioned additive ledger -/// install has work to do before the exact-shape check. -async fn runtime_writer_ledger_pending(conn: &impl QueryExecutor, operation: &str) -> Result { - sqlite_master_probe( - conn, - "SELECT 1 WHERE EXISTS ( - SELECT 1 FROM sqlite_master - WHERE type = 'table' AND name = 'td_runtime_writer_idempotency_v1' - ) OR ( - SELECT count(*) FROM sqlite_master - WHERE type = 'table' AND name IN ( - 'td_runtime_writer_checkpoint_v1', - 'td_runtime_writer_idempotency_v2', - 'td_runtime_writer_outbox_v1', - 'td_runtime_writer_inbox_v1' - ) - ) < 4", - operation, - ) - .await -} - /// Reports whether the file already carries user schema objects. /// /// A brand-new file has `user_version = 0` and no objects at all. That is not a @@ -422,11 +312,10 @@ async fn retired_sqlite_projection_object(conn: &impl QueryExecutor) -> Result Result<()> { let Some(object) = retired_sqlite_projection_object(conn).await? else { - return final_shape::require_admissible_released_staging(conn).await; + return Ok(()); }; let current = get_version(conn).await?; Err(TraceDecayError::reset_required( @@ -451,9 +340,7 @@ fn unsupported_schema_version(current: u32) -> TraceDecayError { } /// Verifies an opened store carries the schema this binary creates, creating it -/// when the file is still empty. -/// -/// Known released stores converge before admission; unknown shapes remain refused. +/// when the file is still empty. Every other stamp or shape is refused. /// /// The schema ladder is awaited through a `dyn Future` so its concrete future /// type stops at this phase boundary: with the `hotpath` wrappers compiled in, @@ -468,23 +355,6 @@ pub async fn ensure_schema_current(database: &crate::db::Database) -> Result<()> ladder.await } -/// Applies either sanctioned writer-side repair before read-only admission: -/// the v34 -> v35 payload-digest step, or the exact shipped-v35 alias-trigger -/// replacement. Every other stamp or shape remains for -/// [`verify_final_schema_connection`] to refuse. -pub(crate) async fn step_schema_if_pending(conn: &Connection) -> Result { - let current = get_version(conn).await?; - if matches!(current, PAYLOAD_DIGEST_STEP_SOURCE_VERSION | 35) { - converge_released_project_schema_connection(conn).await?; - if current == PAYLOAD_DIGEST_STEP_SOURCE_VERSION { - step_payload_digests(conn).await?; - } - return Ok(true); - } - let ledger_installed = install_runtime_writer_ledger_connection(conn).await?; - Ok(repair_shipped_v35_alias_trigger_connection(conn).await? || ledger_installed) -} - async fn ensure_schema_current_engine_connection( conn: &DatabaseEngineWriteConnection, ) -> Result<()> { @@ -492,400 +362,15 @@ async fn ensure_schema_current_engine_connection( if current == 0 && !store_has_objects(conn).await? { return create_schema_engine_connection(conn).await; } - if matches!(current, PAYLOAD_DIGEST_STEP_SOURCE_VERSION | 35) { - converge_released_project_schema_engine_connection(conn).await?; - if current == PAYLOAD_DIGEST_STEP_SOURCE_VERSION { - step_payload_digests(conn).await?; - } - } - install_runtime_writer_ledger_engine_connection(conn).await?; - repair_shipped_v35_alias_trigger_engine_connection(conn).await?; verify_final_schema_connection(conn).await } -const RELEASED_SCHEMA_OPERATION: &str = "converge released project schema"; - -/// Converges a released store to the current shape in one transaction, so an -/// interrupted upgrade leaves the released shape rather than a half-rebuilt -/// table. Runs before the payload-digest step, whose admission check requires -/// the current shape everywhere but the digest objects. -async fn converge_released_project_schema_engine_connection( - conn: &DatabaseEngineWriteConnection, -) -> Result<()> { - let transaction = conn - .authorized_long_lease_transaction() - .await - .map_err(|error| released_schema_failure(format!("failed to acquire lock: {error}")))?; - match released_shape::converge_released_project_schema(&transaction).await { - Ok(()) => transaction - .commit() - .await - .map_err(|error| released_schema_failure(format!("failed to commit: {error}"))), - Err(error) => match transaction.rollback().await { - Ok(()) => Err(error), - Err(rollback_error) => Err(trigger_repair_rollback_failure(error, rollback_error)), - }, - } -} - -async fn converge_released_project_schema_connection(conn: &Connection) -> Result<()> { - let transaction = conn - .authorized_long_lease_transaction() - .await - .map_err(|error| released_schema_failure(format!("failed to acquire lock: {error}")))?; - match released_shape::converge_released_project_schema(&transaction).await { - Ok(()) => transaction - .commit() - .await - .map_err(|error| released_schema_failure(format!("failed to commit: {error}"))), - Err(error) => match transaction.rollback().await { - Ok(()) => Err(error), - Err(rollback_error) => Err(trigger_repair_rollback_failure(error, rollback_error)), - }, - } -} - -fn released_schema_failure(message: String) -> TraceDecayError { - TraceDecayError::Database { - message, - operation: RELEASED_SCHEMA_OPERATION.to_owned(), - } -} - -const LEDGER_INSTALL_OPERATION: &str = "install runtime-writer ledger"; - -/// Sanctioned additive step for a current-version store that predates the -/// ledger being part of the canonical shape, or that still carries the retired -/// idempotency table. Only ledger objects are touched; the exact-shape check -/// that follows still refuses every other drift. -async fn install_runtime_writer_ledger_engine_connection( - conn: &DatabaseEngineWriteConnection, -) -> Result<()> { - if get_version(conn).await? != SCHEMA_VERSION - || !runtime_writer_ledger_pending(conn, LEDGER_INSTALL_OPERATION).await? - { - return Ok(()); - } - let transaction = conn - .authorized_long_lease_transaction() - .await - .map_err(|error| ledger_install_failure(format!("failed to acquire lock: {error}")))?; - match install_runtime_writer_ledger(&transaction, LEDGER_INSTALL_OPERATION).await { - Ok(()) => transaction - .commit() - .await - .map_err(|error| ledger_install_failure(format!("failed to commit: {error}"))), - Err(error) => match transaction.rollback().await { - Ok(()) => Err(error), - Err(rollback_error) => Err(trigger_repair_rollback_failure(error, rollback_error)), - }, - } -} - -async fn install_runtime_writer_ledger_connection(conn: &Connection) -> Result { - if get_version(conn).await? != SCHEMA_VERSION - || !runtime_writer_ledger_pending(conn, LEDGER_INSTALL_OPERATION).await? - { - return Ok(false); - } - let transaction = conn - .authorized_long_lease_transaction() - .await - .map_err(|error| ledger_install_failure(format!("failed to acquire lock: {error}")))?; - match install_runtime_writer_ledger(&transaction, LEDGER_INSTALL_OPERATION).await { - Ok(()) => transaction - .commit() - .await - .map(|()| true) - .map_err(|error| ledger_install_failure(format!("failed to commit: {error}"))), - Err(error) => match transaction.rollback().await { - Ok(()) => Err(error), - Err(rollback_error) => Err(trigger_repair_rollback_failure(error, rollback_error)), - }, - } -} - -fn ledger_install_failure(message: String) -> TraceDecayError { - TraceDecayError::Database { - message, - operation: LEDGER_INSTALL_OPERATION.to_owned(), - } -} - -async fn repair_shipped_v35_alias_trigger(conn: &(impl Executor + Sync)) -> Result { - if !final_shape::require_exact_final_shape_or_shipped_v35_alias_trigger(conn).await? { - return Ok(false); - } - crate::db::retrieval_anchor_schema::install_retrieval_anchor_schema( - conn, - "repair shipped v35 retrieval-anchor alias trigger", - ) - .await?; - final_shape::require_exact_final_shape(conn).await?; - Ok(true) -} - -async fn repair_shipped_v35_alias_trigger_engine_connection( - conn: &DatabaseEngineWriteConnection, -) -> Result<()> { - if get_version(conn).await? != SCHEMA_VERSION - || !final_shape::require_exact_final_shape_or_shipped_v35_alias_trigger(conn).await? - { - return Ok(()); - } - let transaction = conn - .authorized_long_lease_transaction() - .await - .map_err(|error| TraceDecayError::Database { - message: format!("failed to acquire shipped-v35 trigger repair lock: {error}"), - operation: "repair shipped v35 retrieval-anchor alias trigger".to_owned(), - })?; - let result = repair_shipped_v35_alias_trigger(&transaction).await; - match result { - Ok(true) => transaction - .commit() - .await - .map_err(|error| TraceDecayError::Database { - message: format!("failed to commit shipped-v35 trigger repair: {error}"), - operation: "repair shipped v35 retrieval-anchor alias trigger".to_owned(), - }), - Ok(false) => transaction - .rollback() - .await - .map_err(|error| TraceDecayError::Database { - message: format!( - "failed to roll back redundant shipped-v35 trigger repair: {error}" - ), - operation: "repair shipped v35 retrieval-anchor alias trigger".to_owned(), - }), - Err(error) => match transaction.rollback().await { - Ok(()) => Err(error), - Err(rollback_error) => Err(trigger_repair_rollback_failure(error, rollback_error)), - }, - } -} - -async fn repair_shipped_v35_alias_trigger_connection(conn: &Connection) -> Result { - if get_version(conn).await? != SCHEMA_VERSION - || !final_shape::require_exact_final_shape_or_shipped_v35_alias_trigger(conn).await? - { - return Ok(false); - } - let transaction = conn - .authorized_long_lease_transaction() - .await - .map_err(|error| TraceDecayError::Database { - message: format!("failed to acquire shipped-v35 trigger repair lock: {error}"), - operation: "repair shipped v35 retrieval-anchor alias trigger".to_owned(), - })?; - let result = repair_shipped_v35_alias_trigger(&transaction).await; - match result { - Ok(true) => { - transaction - .commit() - .await - .map(|()| true) - .map_err(|error| TraceDecayError::Database { - message: format!("failed to commit shipped-v35 trigger repair: {error}"), - operation: "repair shipped v35 retrieval-anchor alias trigger".to_owned(), - }) - } - Ok(false) => transaction - .rollback() - .await - .map(|()| false) - .map_err(|error| TraceDecayError::Database { - message: format!( - "failed to roll back redundant shipped-v35 trigger repair: {error}" - ), - operation: "repair shipped v35 retrieval-anchor alias trigger".to_owned(), - }), - Err(error) => match transaction.rollback().await { - Ok(()) => Err(error), - Err(rollback_error) => Err(trigger_repair_rollback_failure(error, rollback_error)), - }, - } -} - -fn trigger_repair_rollback_failure( - error: TraceDecayError, - rollback_error: impl std::fmt::Display, -) -> TraceDecayError { - match error { - TraceDecayError::ResetRequired { authority, reason } => TraceDecayError::ResetRequired { - authority, - reason: format!("{reason}; trigger-repair rollback also failed: {rollback_error}"), - }, - TraceDecayError::Database { message, operation } => TraceDecayError::Database { - message: format!("{message}; trigger-repair rollback also failed: {rollback_error}"), - operation, - }, - error => TraceDecayError::Database { - message: format!("{error}; trigger-repair rollback also failed: {rollback_error}"), - operation: "repair shipped v35 retrieval-anchor alias trigger".to_owned(), - }, - } -} - -/// Steps a v34 store to v35: creates the payload-digest objects (idempotent) -/// and fingerprints every existing payload in bounded chunks, each its own -/// short write, so the writer is released between chunks and an interrupted -/// run resumes from the rows still missing a digest. The stamp moves only -/// after the last chunk and the receipt are durable. -async fn step_payload_digests(conn: &C) -> Result<()> { - const OPERATION: &str = "step_payload_digests"; - final_shape::require_final_shape_except_payload_digests(conn).await?; - conn.execute_batch(super::memory_v2::PAYLOAD_DIGESTS_SCHEMA) - .await - .map_err(|error| TraceDecayError::Database { - message: format!("failed to create payload digest objects: {error}"), - operation: OPERATION.to_owned(), - })?; - let mut cursor: i64 = 0; - let mut backfilled: u64 = 0; - loop { - let chunk = payload_digest_backfill_chunk(conn, cursor).await?; - let Some(last_rowid) = chunk.last().map(|row| row.rowid) else { - break; - }; - for row in &chunk { - let digest = payload_content_digest(&row.content); - conn.execute( - "INSERT OR IGNORE INTO memory_v2_assertion_payload_digests( - payload_rowid, assertion_id, fact_id, owner_kind, project_id, content_digest - ) VALUES(?1, ?2, ?3, ?4, ?5, ?6)", - params![ - row.rowid, - row.assertion_id.as_str(), - row.fact_id.as_str(), - row.owner_kind.as_str(), - row.project_id.as_str(), - digest.as_str(), - ], - ) - .await - .map_err(|error| TraceDecayError::Database { - message: format!("failed to backfill payload digest: {error}"), - operation: OPERATION.to_owned(), - })?; - backfilled += 1; - } - cursor = last_rowid; - } - let receipt = serde_json::json!({ - "from_version": PAYLOAD_DIGEST_STEP_SOURCE_VERSION, - "to_version": SCHEMA_VERSION, - "backfilled_rows": backfilled, - "chunk_rows": PAYLOAD_DIGEST_BACKFILL_CHUNK_ROWS, - }); - conn.execute( - "INSERT OR REPLACE INTO metadata (key, value) VALUES (?1, ?2)", - params![PAYLOAD_DIGEST_BACKFILL_RECEIPT_KEY, receipt.to_string()], - ) - .await - .map_err(|error| TraceDecayError::Database { - message: format!("failed to journal payload digest backfill receipt: {error}"), - operation: OPERATION.to_owned(), - })?; - set_version(conn, SCHEMA_VERSION).await -} - -struct PayloadDigestBackfillRow { - rowid: i64, - assertion_id: String, - fact_id: String, - owner_kind: String, - project_id: String, - content: String, -} - -async fn payload_digest_backfill_chunk( - conn: &impl QueryExecutor, - after_rowid: i64, -) -> Result> { - const OPERATION: &str = "step_payload_digests"; - let map = |error: String| TraceDecayError::Database { - message: error, - operation: OPERATION.to_owned(), - }; - let mut rows = conn - .query( - "SELECT payloads.rowid, payloads.assertion_id, payloads.fact_id, - payloads.owner_kind, payloads.project_id, payloads.content - FROM memory_v2_assertion_payloads AS payloads - LEFT JOIN memory_v2_assertion_payload_digests AS digests - ON digests.payload_rowid = payloads.rowid - WHERE digests.payload_rowid IS NULL AND payloads.rowid > ?1 - ORDER BY payloads.rowid ASC - LIMIT ?2", - params![ - after_rowid, - i64::try_from(PAYLOAD_DIGEST_BACKFILL_CHUNK_ROWS).unwrap_or(i64::MAX) - ], - ) - .await - .map_err(|error| { - map(format!( - "failed to read payload digest backfill chunk: {error}" - )) - })?; - let mut chunk = Vec::with_capacity(PAYLOAD_DIGEST_BACKFILL_CHUNK_ROWS); - while let Some(row) = rows.next().await.map_err(|error| { - map(format!( - "failed to read payload digest backfill row: {error}" - )) - })? { - let column = |index: i32| -> Result { - row.get::(index) - .map_err(|error| map(format!("failed to read backfill column {index}: {error}"))) - }; - chunk.push(PayloadDigestBackfillRow { - rowid: row - .get::(0) - .map_err(|error| map(format!("failed to read backfill rowid: {error}")))?, - assertion_id: column(1)?, - fact_id: column(2)?, - owner_kind: column(3)?, - project_id: column(4)?, - content: column(5)?, - }); - } - Ok(chunk) -} - -/// Byte-for-byte the digest `tracedecay_session_memory::fact_store::crud::content_digest` derives for -/// a payload's `content`: `sha256:` plus lowercase hex. -fn payload_content_digest(content: &str) -> String { - use sha2::Digest as _; - tracedecay_domain::canonical_text::encode_tagged_lowercase_hex( - "sha256:", - &sha2::Sha256::digest(content.as_bytes()), - ) -} - /// Verifies that an already-existing store has the one exact final shape this /// binary accepts. This query-only authority intentionally cannot initialize a /// fresh file, so read-only mounts cannot change persisted state. pub(crate) async fn verify_final_schema_connection(conn: &impl QueryExecutor) -> Result<()> { require_no_retired_sqlite_projection_object(conn).await?; let current = get_version(conn).await?; - if matches!(current, PAYLOAD_DIGEST_STEP_SOURCE_VERSION | 35) { - // A read-only mount may not step the store; the message names the - // writer-side remedy instead of the fresh-start reset. - let step = if current == PAYLOAD_DIGEST_STEP_SOURCE_VERSION { - "payload digest" - } else { - "released schema" - }; - return Err(TraceDecayError::Database { - message: format!( - "database schema v{current} needs convergence to v{SCHEMA_VERSION}: \ - the {step} step is pending and runs the next time a writer \ - opens this store; retry after that open instead of resetting the store" - ), - operation: "verify_final_schema".to_owned(), - }); - } if current != SCHEMA_VERSION { return Err(unsupported_schema_version(current)); } @@ -899,13 +384,6 @@ pub(crate) async fn ensure_schema_current_connection(conn: &Connection) -> Resul if current == 0 && !store_has_objects(conn).await? { return create_schema_connection(conn).await; } - if matches!(current, PAYLOAD_DIGEST_STEP_SOURCE_VERSION | 35) { - converge_released_project_schema_connection(conn).await?; - if current == PAYLOAD_DIGEST_STEP_SOURCE_VERSION { - step_payload_digests(conn).await?; - } - } - repair_shipped_v35_alias_trigger_connection(conn).await?; verify_final_schema_connection(conn).await } diff --git a/crates/tracedecay-runtime-core/src/db/migrations/final_shape.rs b/crates/tracedecay-runtime-core/src/db/migrations/final_shape.rs index 989de7e9c0..a83d78bbda 100644 --- a/crates/tracedecay-runtime-core/src/db/migrations/final_shape.rs +++ b/crates/tracedecay-runtime-core/src/db/migrations/final_shape.rs @@ -24,30 +24,6 @@ type SchemaInventory = BTreeMap; static EXPECTED_FINAL_SHAPE: LazyLock> = LazyLock::new(build_expected_final_shape); -pub(super) const SHIPPED_V35_ALIAS_UPDATE_TRIGGER: &str = " - CREATE TRIGGER retrieval_anchor_aliases_immutable_update - BEFORE UPDATE ON retrieval_anchor_aliases BEGIN - SELECT RAISE(ABORT, 'retrieval anchor aliases are immutable'); - END; -"; - -static SHIPPED_V35_ALIAS_UPDATE_OBJECT: LazyLock> = - LazyLock::new(build_shipped_v35_alias_update_object); - -fn build_shipped_v35_alias_update_object() -> std::result::Result { - let connection = rusqlite::Connection::open_in_memory() - .map_err(|error| format!("failed to open shipped-v35 trigger fixture: {error}"))?; - connection - .execute_batch("CREATE TABLE retrieval_anchor_aliases(anchor_id TEXT);") - .map_err(|error| format!("failed to create shipped-v35 trigger table: {error}"))?; - connection - .execute_batch(SHIPPED_V35_ALIAS_UPDATE_TRIGGER) - .map_err(|error| format!("failed to create shipped-v35 trigger: {error}"))?; - read_rusqlite_inventory(&connection)? - .remove("retrieval_anchor_aliases_immutable_update") - .ok_or_else(|| "shipped-v35 trigger fixture did not create its trigger".to_owned()) -} - fn build_expected_final_shape() -> std::result::Result { let connection = rusqlite::Connection::open_in_memory() .map_err(|error| format!("failed to open canonical in-memory schema: {error}"))?; @@ -73,8 +49,6 @@ fn build_expected_final_shape() -> std::result::Result } for schema in [ tracedecay_store::GENERATION_DIAGNOSTICS_SCHEMA_DDL, - crate::db::evidence_assembly::EVIDENCE_ASSEMBLY_SCHEMA, - crate::db::evidence_assembly::EVIDENCE_ASSEMBLY_IMMUTABILITY, tracedecay_rusqlite_runtime::repository::EXTERNAL_SOURCE_SCHEMA_V1, tracedecay_rusqlite_runtime::repository::GRAPH_PUBLICATION_SCHEMA_V1, tracedecay_rusqlite_runtime::handoff::HANDOFF_OPEN_SCHEMA_V1, @@ -210,20 +184,6 @@ fn write_schema_section(out: &mut String, title: &str, objects: &[(&str, &Schema } } -/// The DDL this binary creates for one schema object, or `None` when the -/// object is not part of the final shape. -/// -/// This is the single expected-shape authority, so a convergence step can ask -/// it whether a store's stored DDL is the current one instead of carrying its -/// own copy of either shape. -pub(super) fn expected_object_sql(name: &str) -> Result> { - Ok(EXPECTED_FINAL_SHAPE - .as_ref() - .map_err(|error| database_error(error.clone()))? - .get(name) - .map(|object| object.sql.as_str())) -} - /// Fingerprint of the exact final shape this binary creates and admits. pub fn expected_final_schema_fingerprint() -> Result { let inventory = EXPECTED_FINAL_SHAPE @@ -237,12 +197,8 @@ pub fn expected_final_schema_fingerprint() -> Result { pub(super) fn require_admissible_final_shape_rusqlite( connection: &rusqlite::Connection, ) -> Result<()> { - let mut actual = read_rusqlite_inventory(connection).map_err(database_error)?; - admit_released_staging_objects(&mut actual)?; - let shipped = SHIPPED_V35_ALIAS_UPDATE_OBJECT - .as_ref() - .map_err(|error| database_error(error.clone()))?; - require_final_shape_inventory(&actual, Some(shipped)).map(|_| ()) + let actual = read_rusqlite_inventory(connection).map_err(database_error)?; + require_final_shape_inventory(&actual) } async fn read_inventory(conn: &impl QueryExecutor) -> Result { @@ -294,53 +250,6 @@ async fn read_inventory(conn: &impl QueryExecutor) -> Result { Ok(inventory) } -/// The tagged beta.25..beta.37 and live a2e7694c51 staging inventories are retained, -/// including publication receipts and identity guards. It has no current -/// writer or retrieval path; fresh stores never install it. Only this exact -/// inventory may accompany the current relational authority. -fn admit_released_staging_objects(actual: &mut SchemaInventory) -> Result<()> { - let released: Vec<_> = actual - .iter() - .filter(|(name, object)| { - name.starts_with("semantic_vector_") || object.table.starts_with("semantic_vector_") - }) - .collect(); - let Some((first_name, _)) = released.first() else { - return Ok(()); - }; - let parts: Vec<_> = released - .iter() - .flat_map(|(name, object)| { - [ - name.as_bytes(), - object.object_type.as_bytes(), - object.table.as_bytes(), - object.sql.as_bytes(), - ] - }) - .collect(); - // Tagged DDL and the a2e7694c51 Mac dogfood DDL differ only in the - // expected_chunk_count CHECK. Neither inventory comes from today's schema. - let digest = canonical_framed_sha256(b"tracedecay.released-staging-shape.v1", &parts); - if !matches!( - digest.as_str(), - "ca0d1cd46378c80005b081b095095a1d9e1d0afb804547fe43f60787f7e3fba1" - | "418ac9a0900844ba66f869c1f113086d6d726e6dafa5963b8fa5a2730a09596f" - ) { - return Err(reset_required(format!( - "database schema has an incompatible released staging inventory at '{first_name}'" - ))); - } - actual.retain(|name, object| { - !name.starts_with("semantic_vector_") && !object.table.starts_with("semantic_vector_") - }); - Ok(()) -} - -pub(super) async fn require_admissible_released_staging(conn: &impl QueryExecutor) -> Result<()> { - admit_released_staging_objects(&mut read_inventory(conn).await?) -} - fn database_error(message: String) -> TraceDecayError { TraceDecayError::Database { message, @@ -359,30 +268,15 @@ fn reset_required(reason: impl Into) -> TraceDecayError { ) } -/// Admits a store for the v34 -> v35 payload-digest step: apart from the -/// digest objects themselves (absent, or already created by an interrupted -/// earlier step) its inventory must be exactly the final shape. -pub(super) async fn require_final_shape_except_payload_digests( - conn: &impl QueryExecutor, -) -> Result<()> { - let mut actual = read_inventory(conn).await?; - admit_released_staging_objects(&mut actual)?; +pub(super) async fn require_exact_final_shape(conn: &impl QueryExecutor) -> Result<()> { + require_final_shape_inventory(&read_inventory(conn).await?) +} + +fn require_final_shape_inventory(actual: &SchemaInventory) -> Result<()> { let expected = EXPECTED_FINAL_SHAPE .as_ref() .map_err(|error| database_error(error.clone()))?; - let digest_objects = crate::db::memory_v2::PAYLOAD_DIGEST_OBJECTS; for (name, expected_object) in expected { - if digest_objects.contains(&name.as_str()) { - if let Some(actual_object) = actual.get(name) - && actual_object != expected_object - { - return Err(reset_required(format!( - "database schema has incompatible {} '{name}' from an earlier payload-digest step", - expected_object.object_type - ))); - } - continue; - } let Some(actual_object) = actual.get(name) else { return Err(reset_required(format!( "database schema is missing required {} '{name}'", @@ -408,71 +302,6 @@ pub(super) async fn require_final_shape_except_payload_digests( Ok(()) } -pub(super) async fn require_exact_final_shape(conn: &impl QueryExecutor) -> Result<()> { - let mut actual = read_inventory(conn).await?; - admit_released_staging_objects(&mut actual)?; - require_final_shape_inventory(&actual, None)?; - Ok(()) -} - -/// Admits only the exact current shape or the exact shape emitted by the -/// shipped v35 binary before alias-target correction was supported. -/// -/// The returned flag identifies the one known trigger replacement the writer -/// may perform. Every other missing, additional, or byte-different schema -/// object remains reset-required. -pub(super) async fn require_exact_final_shape_or_shipped_v35_alias_trigger( - conn: &impl QueryExecutor, -) -> Result { - let mut actual = read_inventory(conn).await?; - admit_released_staging_objects(&mut actual)?; - let shipped = SHIPPED_V35_ALIAS_UPDATE_OBJECT - .as_ref() - .map_err(|error| database_error(error.clone()))?; - require_final_shape_inventory(&actual, Some(shipped)) -} - -fn require_final_shape_inventory( - actual: &SchemaInventory, - shipped_v35_alias_trigger: Option<&SchemaObject>, -) -> Result { - const TRIGGER: &str = "retrieval_anchor_aliases_immutable_update"; - - let expected = EXPECTED_FINAL_SHAPE - .as_ref() - .map_err(|error| database_error(error.clone()))?; - let mut shipped_trigger_found = false; - - for (name, expected_object) in expected { - let Some(actual_object) = actual.get(name) else { - return Err(reset_required(format!( - "database schema is missing required {} '{name}'", - expected_object.object_type - ))); - }; - if actual_object != expected_object { - if name == TRIGGER && shipped_v35_alias_trigger == Some(actual_object) { - shipped_trigger_found = true; - } else { - return Err(reset_required(format!( - "database schema has incompatible {} '{name}'", - expected_object.object_type - ))); - } - } - } - if let Some((name, object)) = actual - .iter() - .find(|(name, _object)| !expected.contains_key(*name)) - { - return Err(reset_required(format!( - "database schema contains unexpected {} '{name}'", - object.object_type - ))); - } - Ok(shipped_trigger_found) -} - #[cfg(test)] mod render_tests { use std::collections::BTreeSet; diff --git a/crates/tracedecay-runtime-core/src/db/migrations/released_shape.rs b/crates/tracedecay-runtime-core/src/db/migrations/released_shape.rs deleted file mode 100644 index 86141d8352..0000000000 --- a/crates/tracedecay-runtime-core/src/db/migrations/released_shape.rs +++ /dev/null @@ -1,259 +0,0 @@ -//! Convergence for a v34-stamped project store whose inventory is free of -//! retired projections. -//! -//! Every release from v0.1.0-beta.25 through v0.1.0-beta.37 created one -//! byte-identical `tracedecay.db` stamped `user_version` 34, the exact SQL -//! lives in `tests/fixtures/project-store-released-v34.sql`, whose header -//! carries the tag-to-inventory table. The current contract differs from it in -//! objects that hold no data of their own (two absent indexes, a renamed -//! external-source mutation family, the runtime-writer ledger) and in two -//! diagnostics tables that gained a `publication_revision` column and a wider -//! primary key. Those differences are convergeable and are converged here. -//! -//! Released dense-staging objects remain untouched under exact inventory -//! admission. Removing dense retrieval must not discard unrelated durable -//! project data or the historical publication receipts sharing its store. - -use tracedecay_domain::errors::{Result, TraceDecayError}; - -use crate::db::engine::{Executor, QueryExecutor, params}; - -const OPERATION: &str = "converge released project schema"; - -/// One table the released shape carries in a form `SQLite` cannot alter in -/// place: widening a primary key, adding a `NOT NULL` column with no default, -/// and relaxing a `CHECK` all require a rebuild. -/// -/// A table is rebuilt when its stored DDL differs from the one the current -/// contract expects, so this list needs no record of which release changed -/// what. [`super::final_shape`] stays the single authority on the expected -/// shape. -struct ReleasedTableRebuild { - table: &'static str, - /// The columns the released table carried, written verbatim into the - /// canonical table so the rebuild is a copy rather than a re-derivation. - released_columns: &'static str, - /// The column the canonical shape added, with the value every released - /// row takes. `None` when only a constraint changed. - added_column: Option<(&'static str, &'static str)>, -} - -/// A canonical DDL batch and every released table it recreates. -/// -/// The batch owns the table's indexes and triggers too, which the rebuild's -/// `DROP TABLE` removes, so each group drops all of its tables before -/// replaying the batch once. -struct ReleasedRebuildGroup { - canonical: &'static str, - tables: &'static [ReleasedTableRebuild], -} - -/// Diagnostics rows published before revisions existed are the first revision -/// of their generation. -const RELEASED_V34_REBUILDS: &[ReleasedRebuildGroup] = &[ReleasedRebuildGroup { - canonical: tracedecay_store::GENERATION_DIAGNOSTICS_SCHEMA_DDL, - tables: &[ - ReleasedTableRebuild { - table: "diagnostic_generation_publications", - released_columns: "generation_id, record_state, state_generation, published_at", - added_column: Some(("publication_revision", "1")), - }, - ReleasedTableRebuild { - table: "generation_diagnostics", - released_columns: "diagnostic_anchor, generation_id, repository, worktree, \ - reference, source_revision, file_occurrence_id, \ - content_digest, symbol_occurrence_id, span_start, span_end, \ - code, severity, message, message_digest, producer_kind, \ - producer, analyzer_revision, configuration_revision, \ - sanitization_receipt, evidence_class, collected_at, \ - record_state, state_generation, persisted_at", - added_column: Some(("publication_revision", "1")), - }, - ], -}]; - -fn failure(message: String) -> TraceDecayError { - TraceDecayError::Database { - message, - operation: OPERATION.to_owned(), - } -} - -/// Converges a store stamped with the released version to the shape this -/// binary creates, carrying every row forward. -/// -/// A store still carrying a retired projection is refused first, inside the -/// caller's transaction, so the refusal leaves the store byte-identical. -/// Runs before the payload-digest step, whose own admission check requires the -/// current shape everywhere but the digest objects. Idempotent by -/// construction: the rebuilds are selected by the released column being -/// absent, the schema installs are `CREATE ... IF NOT EXISTS`, and the row -/// moves are the same resumable statements the registered stores converge -/// with. -pub(super) async fn converge_released_project_schema(conn: &(impl Executor + Sync)) -> Result<()> { - super::require_no_retired_sqlite_projection_object(conn).await?; - if super::get_version(conn).await? == 35 { - // This live stamp already has the final relational schema; only the - // known alias guard may differ. Do not repair arbitrary v35 drift. - super::final_shape::require_exact_final_shape_or_shipped_v35_alias_trigger(conn).await?; - } - for group in RELEASED_V34_REBUILDS { - rebuild_released_group(conn, group).await?; - } - // Replaces the released alias-immutability trigger, which guarded every - // update instead of only the fields that must not change. - crate::db::retrieval_anchor_schema::install_retrieval_anchor_schema(conn, OPERATION).await?; - crate::db::external_source::install_external_source_schema(conn, OPERATION).await?; - crate::db::external_source::retire_mutation_copies_in_transaction(conn).await?; - super::install_runtime_writer_ledger(conn, OPERATION).await?; - if super::get_version(conn).await? == super::PAYLOAD_DIGEST_STEP_SOURCE_VERSION { - super::final_shape::require_final_shape_except_payload_digests(conn).await - } else { - super::final_shape::require_exact_final_shape(conn).await?; - super::set_version(conn, super::SCHEMA_VERSION).await - } -} - -/// Rebuilds every table in one group whose stored DDL is not the one the -/// current contract expects. -/// -/// The released rows are copied aside, the tables dropped with their indexes -/// and triggers, the canonical batch replayed, and the rows written back -/// through the canonical column list. A table's own triggers are dropped again -/// before the rows return: a census trigger that fired per copied row would -/// count work its authority already records. Replaying the batch afterwards -/// restores them, which is sound because every statement in these batches -/// creates its object only if it is absent. -/// -/// A store this binary created has no pending table and pays one catalog probe -/// per listed table. -async fn rebuild_released_group( - conn: &(impl Executor + Sync), - group: &ReleasedRebuildGroup, -) -> Result<()> { - let mut pending = Vec::new(); - for rebuild in group.tables { - if released_table_pending(conn, rebuild.table).await? { - pending.push(rebuild); - } - } - if pending.is_empty() { - // Released stores installed diagnostics lazily; both tables may be absent. - return batch(conn, group.canonical).await; - } - let mut triggers = Vec::new(); - for rebuild in &pending { - triggers.extend(table_triggers(conn, rebuild.table).await?); - } - // Children of a rebuilt table are valid again before this transaction - // commits, which is when deferred enforcement checks them. - batch(conn, "PRAGMA defer_foreign_keys = ON;").await?; - for rebuild in &pending { - let scratch = scratch_table(rebuild.table); - batch( - conn, - &format!( - "CREATE TABLE {scratch} AS SELECT * FROM {table}; - DROP TABLE {table};", - table = rebuild.table - ), - ) - .await?; - } - batch(conn, group.canonical).await?; - for trigger in &triggers { - batch(conn, &format!("DROP TRIGGER IF EXISTS {trigger};")).await?; - } - for rebuild in &pending { - let scratch = scratch_table(rebuild.table); - let columns = rebuild.released_columns; - let (added, value) = match rebuild.added_column { - Some((added, value)) => (format!("{added}, "), format!("{value}, ")), - None => (String::new(), String::new()), - }; - batch( - conn, - &format!( - "INSERT INTO {table}({added}{columns}) - SELECT {value}{columns} FROM {scratch}; - DROP TABLE {scratch};", - table = rebuild.table - ), - ) - .await?; - } - batch(conn, group.canonical).await -} - -/// Reports whether a table exists carrying DDL other than the one this binary -/// creates. -async fn released_table_pending(conn: &impl QueryExecutor, table: &str) -> Result { - let Some(expected) = super::final_shape::expected_object_sql(table)? else { - return Err(failure(format!( - "'{table}' is not part of the shape this binary creates" - ))); - }; - Ok(stored_object_sql(conn, table) - .await? - .is_some_and(|stored| stored != expected)) -} - -/// Names the copy a rebuild reads its released rows out of. The copy lives and -/// dies inside the caller's transaction, so an interrupted convergence leaves -/// neither it nor a half-rebuilt table behind. -fn scratch_table(table: &str) -> String { - format!("{table}_released_v34") -} - -async fn batch(conn: &impl Executor, sql: &str) -> Result<()> { - conn.execute_batch(sql) - .await - .map_err(|error| failure(format!("failed to converge released schema: {error}"))) -} - -async fn stored_object_sql(conn: &impl QueryExecutor, name: &str) -> Result> { - let mut rows = conn - .query( - "SELECT COALESCE(sql, '') FROM sqlite_master WHERE name = ?1", - params![name], - ) - .await - .map_err(|error| failure(format!("failed to read the stored DDL of {name}: {error}")))?; - let Some(row) = rows.next().await.map_err(|error| { - failure(format!( - "failed to decode the stored DDL of {name}: {error}" - )) - })? - else { - return Ok(None); - }; - row.get::(0).map(Some).map_err(|error| { - failure(format!( - "failed to decode the stored DDL of {name}: {error}" - )) - }) -} - -/// Every trigger defined on one table, in catalog order. -async fn table_triggers(conn: &impl QueryExecutor, table: &str) -> Result> { - let mut rows = conn - .query( - "SELECT name FROM sqlite_master - WHERE type = 'trigger' AND tbl_name = ?1 ORDER BY name", - params![table], - ) - .await - .map_err(|error| failure(format!("failed to list the triggers of {table}: {error}")))?; - let mut triggers = Vec::new(); - while let Some(row) = rows - .next() - .await - .map_err(|error| failure(format!("failed to read the triggers of {table}: {error}")))? - { - triggers - .push(row.get::(0).map_err(|error| { - failure(format!("failed to decode a {table} trigger: {error}")) - })?); - } - Ok(triggers) -} diff --git a/crates/tracedecay-runtime-core/src/db/migrations/tests.rs b/crates/tracedecay-runtime-core/src/db/migrations/tests.rs index 4f24404cbf..b611b90718 100644 --- a/crates/tracedecay-runtime-core/src/db/migrations/tests.rs +++ b/crates/tracedecay-runtime-core/src/db/migrations/tests.rs @@ -5,11 +5,10 @@ use tracedecay_rusqlite_runtime::exact_sql::{ }; use crate::db::engine::{Connection, TestConnection}; -use crate::db::{Database, DatabaseAuthority, TestDatabaseRuntimeMode}; use super::{ - PAYLOAD_DIGEST_BACKFILL_RECEIPT_KEY, PAYLOAD_DIGEST_STEP_SOURCE_VERSION, SCHEMA_VERSION, - create_schema_connection, ensure_schema_current_connection, verify_final_schema_connection, + SCHEMA_VERSION, create_schema_connection, ensure_schema_current_connection, + verify_final_schema_connection, }; use crate::db::engine::params; @@ -112,16 +111,6 @@ async fn string_column(conn: &Connection, sql: &str) -> Vec { values } -async fn scalar_string(conn: &Connection, sql: &str) -> String { - let mut rows = conn.query(sql, ()).await.expect("failed to query string"); - rows.next() - .await - .expect("failed to read string row") - .expect("string query should return a row") - .get(0) - .expect("failed to read string value") -} - async fn column_exists(conn: &Connection, table: &str, column: &str) -> bool { let mut rows = conn .query(&format!("PRAGMA table_info({table})"), ()) @@ -140,250 +129,11 @@ async fn column_exists(conn: &Connection, table: &str, column: &str) -> bool { // Tests // --------------------------------------------------------------------------- -#[tokio::test] -async fn a_shipped_v35_alias_trigger_is_repaired_without_losing_rows() { - let (conn, dir) = create_schema_db().await; - let path = dir.path().join("test.db"); - conn.execute( - "INSERT INTO retrieval_anchors( - anchor_id, anchor_json, owner_json, projection_generation - ) VALUES ('anchor.fixture', '{}', '{}', 'generation.fixture')", - (), - ) - .await - .unwrap(); - conn.execute( - "INSERT INTO retrieval_anchor_aliases( - owner_json, alias_kind, locator_digest, anchor_id - ) VALUES ('{}', 'native', 'digest.fixture', 'anchor.fixture')", - (), - ) - .await - .unwrap(); - conn.execute_batch("DROP TRIGGER retrieval_anchor_aliases_immutable_update;") - .await - .unwrap(); - conn.execute_batch(super::final_shape::SHIPPED_V35_ALIAS_UPDATE_TRIGGER) - .await - .unwrap(); - drop(conn); - - let authority = DatabaseAuthority::acquire_test(&path, "shipped-v35 trigger repair fixture") - .expect("acquire production-open authority"); - let (database, _initialized) = - Database::publish_test_runtime(&path, &authority, TestDatabaseRuntimeMode::Existing) - .await - .expect("the production writer should repair the exact shipped-v35 trigger"); - drop(database); - drop(authority); - - let conn = TestConnection::open(&path); - - assert_eq!( - scalar_string( - &conn, - "SELECT anchor_id FROM retrieval_anchor_aliases WHERE locator_digest = 'digest.fixture'" - ) - .await, - "anchor.fixture" - ); - let trigger = scalar_string( - &conn, - "SELECT sql FROM sqlite_master - WHERE type = 'trigger' AND name = 'retrieval_anchor_aliases_immutable_update'", - ) - .await; - assert!(trigger.contains("retrieval anchor alias requires exact supersession")); - verify_final_schema_connection(&conn) - .await - .expect("the repaired store must carry the exact final shape"); - let preserved_alias = scalar_string( - &conn, - "SELECT anchor_id FROM retrieval_anchor_aliases WHERE locator_digest = 'digest.fixture'", - ) - .await; - let canonical_trigger = trigger; - drop(conn); - - let authority = DatabaseAuthority::acquire_test(&path, "canonical v35 reopen fixture") - .expect("reacquire production-open authority"); - let (database, _initialized) = - Database::publish_test_runtime(&path, &authority, TestDatabaseRuntimeMode::Existing) - .await - .expect("a canonical store should reopen without mutation"); - drop(database); - drop(authority); - let conn = TestConnection::open(&path); - assert_eq!( - scalar_string( - &conn, - "SELECT anchor_id FROM retrieval_anchor_aliases WHERE locator_digest = 'digest.fixture'", - ) - .await, - preserved_alias - ); - assert_eq!( - scalar_string( - &conn, - "SELECT sql FROM sqlite_master - WHERE type = 'trigger' AND name = 'retrieval_anchor_aliases_immutable_update'", - ) - .await, - canonical_trigger - ); - conn.execute( - "INSERT INTO retrieval_anchors( - anchor_id, anchor_json, owner_json, projection_generation - ) VALUES ('anchor.corrected', '{}', '{}', 'generation.fixture')", - (), - ) - .await - .unwrap(); - assert!( - conn.execute( - "UPDATE retrieval_anchor_aliases - SET anchor_id = 'anchor.corrected' - WHERE locator_digest = 'digest.fixture'", - (), - ) - .await - .is_err() - ); - conn.execute( - "INSERT INTO retrieval_anchor_dispositions( - disposition_id, anchor_id, owner_json, state, superseded_by, - reason_class, effective_at, record_json - ) VALUES ( - 'dis.fixture', 'anchor.fixture', '{}', 'superseded', - 'anchor.corrected', 'correction', 1, '{}' - )", - (), - ) - .await - .unwrap(); - assert_eq!( - conn.execute( - "UPDATE retrieval_anchor_aliases - SET anchor_id = 'anchor.corrected' - WHERE locator_digest = 'digest.fixture'", - (), - ) - .await - .unwrap(), - 1 - ); -} - -#[tokio::test] -async fn a_shipped_v35_alias_trigger_with_another_incompatibility_is_refused_unchanged() { - let (conn, dir) = create_schema_db().await; - let path = dir.path().join("test.db"); - conn.execute_batch("DROP TRIGGER retrieval_anchor_aliases_immutable_update;") - .await - .unwrap(); - conn.execute_batch(super::final_shape::SHIPPED_V35_ALIAS_UPDATE_TRIGGER) - .await - .unwrap(); - conn.execute_batch("CREATE TABLE unexpected_v35_object(id INTEGER PRIMARY KEY);") - .await - .unwrap(); - let shipped_trigger = scalar_string( - &conn, - "SELECT sql FROM sqlite_master - WHERE type = 'trigger' AND name = 'retrieval_anchor_aliases_immutable_update'", - ) - .await; - drop(conn); - - let authority = DatabaseAuthority::acquire_test(&path, "incompatible v35 fixture") - .expect("acquire production-open authority"); - let error = - match Database::publish_test_runtime(&path, &authority, TestDatabaseRuntimeMode::Existing) - .await - { - Ok(_) => panic!("a second incompatibility must prevent the known trigger repair"), - Err(error) => error, - }; - - assert_eq!( - error - .reset_required_context() - .map(|(authority, _reason)| authority), - Some("SQLite store") - ); - drop(authority); - let conn = TestConnection::open(&path); - assert_eq!( - scalar_string( - &conn, - "SELECT sql FROM sqlite_master - WHERE type = 'trigger' AND name = 'retrieval_anchor_aliases_immutable_update'", - ) - .await, - shipped_trigger - ); - assert!(table_exists(&conn, "unexpected_v35_object").await); - drop(conn); - drop(dir); - - let (conn, dir) = create_schema_db().await; - let path = dir.path().join("test.db"); - conn.execute_batch( - "DROP TRIGGER retrieval_anchor_aliases_immutable_update; - CREATE TRIGGER retrieval_anchor_aliases_immutable_update - BEFORE UPDATE ON retrieval_anchor_aliases BEGIN - SELECT RAISE(ABORT, 'unrecognized alias trigger'); - END;", - ) - .await - .unwrap(); - let unknown_trigger = scalar_string( - &conn, - "SELECT sql FROM sqlite_master - WHERE type = 'trigger' AND name = 'retrieval_anchor_aliases_immutable_update'", - ) - .await; - drop(conn); - - let authority = DatabaseAuthority::acquire_test(&path, "unknown v35 trigger fixture") - .expect("acquire production-open authority"); - let error = - match Database::publish_test_runtime(&path, &authority, TestDatabaseRuntimeMode::Existing) - .await - { - Ok(_) => panic!("an unknown trigger body must remain reset-required"), - Err(error) => error, - }; - assert_eq!( - error - .reset_required_context() - .map(|(authority, _reason)| authority), - Some("SQLite store") - ); - drop(authority); - let conn = TestConnection::open(&path); - assert_eq!( - scalar_string( - &conn, - "SELECT sql FROM sqlite_master - WHERE type = 'trigger' AND name = 'retrieval_anchor_aliases_immutable_update'", - ) - .await, - unknown_trigger - ); -} - -/// Released v34 and live v35 stores have explicit convergence paths; -/// unrelated stamps remain refused without mutation. +/// Every stamp but the current one, including the released v34 and v35 +/// stores, is refused without mutation: nothing upgrades in place. #[tokio::test] async fn a_store_at_another_schema_version_is_refused_with_a_fresh_start_remedy() { - for stamped in [ - 1_u32, - 18, - 24, - PAYLOAD_DIGEST_STEP_SOURCE_VERSION - 1, - SCHEMA_VERSION + 1, - ] { + for stamped in [1_u32, 18, 24, 33, 34, 35, SCHEMA_VERSION + 1] { let (conn, _dir) = create_schema_db().await; set_user_version(&conn, stamped).await; @@ -540,9 +290,8 @@ async fn a_current_stamp_with_retired_memory_projection_objects_is_reset_require } /// v36 retired the semantic-vector staging family with dense code retrieval. -/// A leftover object of that family is refused on the current stamp and on -/// the shipped v35 stamp alike, by the writer ladder and by the read-only -/// verifier, and the refusal names the object instead of the stamp. +/// A leftover object of that family on the current stamp is refused by the +/// writer ladder and by the read-only verifier, and the refusal names it. #[tokio::test] async fn a_retired_semantic_vector_staging_object_is_reset_required_on_every_stamp() { for (retired, ddl) in [ @@ -558,7 +307,7 @@ async fn a_retired_semantic_vector_staging_object_is_reset_required_on_every_sta END;", ), ] { - for stamped in [SCHEMA_VERSION, SCHEMA_VERSION - 1] { + for stamped in [SCHEMA_VERSION] { let (conn, _dir) = create_schema_db().await; conn.execute_batch(ddl).await.unwrap(); set_user_version(&conn, stamped).await; @@ -842,33 +591,20 @@ async fn fresh_creation_installs_every_stage_of_the_final_shape() { } // --------------------------------------------------------------------------- -// v34 -> v35: persisted payload content digests (#834) +// Persisted payload content digests (#834) // --------------------------------------------------------------------------- -const PAYLOAD_DIGEST_OBJECT_DROPS: &str = "DROP TRIGGER memory_v2_payloads_digest_delete; - DROP TRIGGER memory_v2_assertion_payload_digests_no_update; - DROP INDEX memory_v2_assertion_payload_digests_lookup; - DROP TABLE memory_v2_assertion_payload_digests;"; - -const V34_FIXTURE_CONTENTS: [&str; 3] = [ +const PAYLOAD_FIXTURE_CONTENTS: [&str; 3] = [ "Unicode café naïve 東京 🚀", "JSON escaped quote \" and backslash \\ with\ttab and\nnewline", "trailing whitespace ", ]; -fn expected_payload_digest(content: &str) -> String { - use sha2::Digest as _; - tracedecay_domain::canonical_text::encode_tagged_lowercase_hex( - "sha256:", - &sha2::Sha256::digest(content.as_bytes()), - ) -} - /// Seeds one profile fact with a single asserted payload through the raw -/// authority tables, the way a pre-#834 binary left them. +/// authority tables. async fn seed_payload(conn: &Connection, ordinal: usize, content: &str) { - let fact_id = format!("fact.v34.{ordinal}"); - let assertion_id = format!("assertion.v34.{ordinal}"); + let fact_id = format!("fact.payload.{ordinal}"); + let assertion_id = format!("assertion.payload.{ordinal}"); conn.execute( "INSERT INTO memory_v2_facts ( fact_id, owner_kind, project_id, owner_json, identity_json, created_at @@ -876,7 +612,7 @@ async fn seed_payload(conn: &Connection, ordinal: usize, content: &str) { params![fact_id.as_str(), ordinal as i64], ) .await - .expect("seed v34 fact"); + .expect("seed fact"); conn.execute( "INSERT INTO memory_v2_assertions ( assertion_id, fact_id, owner_kind, project_id, owner_json, @@ -886,7 +622,7 @@ async fn seed_payload(conn: &Connection, ordinal: usize, content: &str) { params![assertion_id.as_str(), fact_id.as_str(), ordinal as i64], ) .await - .expect("seed v34 assertion"); + .expect("seed assertion"); conn.execute( "INSERT INTO memory_v2_assertion_payloads ( assertion_id, fact_id, owner_kind, project_id, payload_json, content @@ -894,26 +630,7 @@ async fn seed_payload(conn: &Connection, ordinal: usize, content: &str) { params![assertion_id.as_str(), fact_id.as_str(), content], ) .await - .expect("seed v34 payload"); -} - -/// A v35 store whose payloads were written before the digest objects -/// existed: the objects are dropped and the stamp rewound, exactly the shape -/// a pre-#834 binary leaves behind. -async fn create_v34_db_with_payloads() -> (TestConnection, TempDir) { - let (conn, dir) = create_schema_db().await; - for (ordinal, content) in V34_FIXTURE_CONTENTS.iter().enumerate() { - seed_payload(&conn, ordinal, content).await; - } - conn.execute_batch(PAYLOAD_DIGEST_OBJECT_DROPS) - .await - .expect("drop payload digest objects"); - set_user_version(&conn, PAYLOAD_DIGEST_STEP_SOURCE_VERSION).await; - assert!( - !table_exists(&conn, "memory_v2_assertion_payload_digests").await, - "fixture must start without the digest table" - ); - (conn, dir) + .expect("seed payload"); } async fn digest_rows(conn: &Connection) -> Vec<(String, String)> { @@ -935,135 +652,10 @@ async fn digest_rows(conn: &Connection) -> Vec<(String, String)> { values } -fn expected_digest_rows() -> Vec<(String, String)> { - V34_FIXTURE_CONTENTS - .iter() - .enumerate() - .map(|(ordinal, content)| { - ( - format!("fact.v34.{ordinal}"), - expected_payload_digest(content), - ) - }) - .collect() -} - -#[tokio::test] -async fn a_v34_store_is_stepped_to_the_current_stamp_with_a_digest_for_every_payload() { - let (conn, _dir) = create_v34_db_with_payloads().await; - - ensure_schema_current_connection(&conn) - .await - .expect("a v34 store must step forward in place"); - - assert_eq!(get_user_version(&conn).await, SCHEMA_VERSION); - assert_eq!(digest_rows(&conn).await, expected_digest_rows()); - let receipts = string_column( - &conn, - &format!("SELECT value FROM metadata WHERE key = '{PAYLOAD_DIGEST_BACKFILL_RECEIPT_KEY}'"), - ) - .await; - let receipt: serde_json::Value = - serde_json::from_str(receipts.first().expect("step must journal a receipt")) - .expect("receipt is JSON"); - assert_eq!(receipt["from_version"], PAYLOAD_DIGEST_STEP_SOURCE_VERSION); - assert_eq!(receipt["to_version"], SCHEMA_VERSION); - assert_eq!(receipt["backfilled_rows"], V34_FIXTURE_CONTENTS.len()); - - ensure_schema_current_connection(&conn) - .await - .expect("a stepped store is the exact final shape"); - assert_eq!(digest_rows(&conn).await.len(), V34_FIXTURE_CONTENTS.len()); -} - -#[tokio::test] -async fn an_interrupted_payload_digest_step_resumes_from_the_rows_still_missing() { - let (conn, _dir) = create_v34_db_with_payloads().await; - // A previous run created the objects and fingerprinted the first payload - // before losing the writer; the stamp never moved. - conn.execute_batch(crate::db::memory_v2::PAYLOAD_DIGESTS_SCHEMA) - .await - .expect("recreate digest objects as an interrupted step left them"); - conn.execute( - "INSERT INTO memory_v2_assertion_payload_digests ( - payload_rowid, assertion_id, fact_id, owner_kind, project_id, content_digest - ) - SELECT rowid, assertion_id, fact_id, owner_kind, project_id, ?1 - FROM memory_v2_assertion_payloads WHERE fact_id = 'fact.v34.0'", - params![expected_payload_digest(V34_FIXTURE_CONTENTS[0]).as_str()], - ) - .await - .expect("seed the partial backfill"); - assert_eq!( - get_user_version(&conn).await, - PAYLOAD_DIGEST_STEP_SOURCE_VERSION - ); - - ensure_schema_current_connection(&conn) - .await - .expect("an interrupted step must resume"); - - assert_eq!(get_user_version(&conn).await, SCHEMA_VERSION); - assert_eq!(digest_rows(&conn).await, expected_digest_rows()); -} - -#[tokio::test] -async fn a_v34_store_is_refused_read_only_with_the_step_pending_remedy() { - let (conn, _dir) = create_v34_db_with_payloads().await; - - let error = verify_final_schema_connection(&conn) - .await - .expect_err("a read-only verifier must not step the store"); - - let message = error.to_string(); - assert!( - message.contains("payload digest step is pending"), - "read-only refusal must name the pending step: {message}" - ); - assert!( - error.reset_required_context().is_none(), - "a steppable store must not be reported as reset-required: {message}" - ); - assert_eq!( - get_user_version(&conn).await, - PAYLOAD_DIGEST_STEP_SOURCE_VERSION - ); - assert!( - !table_exists(&conn, "memory_v2_assertion_payload_digests").await, - "read-only verification must not create the digest objects" - ); -} - -#[tokio::test] -async fn a_v34_stamp_on_a_store_that_is_not_v34_shaped_is_reset_required() { - let (conn, _dir) = create_schema_db().await; - conn.execute_batch(PAYLOAD_DIGEST_OBJECT_DROPS) - .await - .expect("drop payload digest objects"); - conn.execute_batch("DROP TABLE memory_v2_assertion_supersession;") - .await - .expect("drop an unrelated final-shape table"); - set_user_version(&conn, PAYLOAD_DIGEST_STEP_SOURCE_VERSION).await; - - let error = ensure_schema_current_connection(&conn) - .await - .expect_err("the step admits only the exact pre-digest shape"); - assert_eq!( - error - .reset_required_context() - .map(|(authority, _reason)| authority), - Some("SQLite store") - ); - assert_eq!( - get_user_version(&conn).await, - PAYLOAD_DIGEST_STEP_SOURCE_VERSION - ); -} - #[tokio::test] async fn deleting_a_payload_drops_its_digest_and_digests_never_update() { let (conn, _dir) = create_schema_db().await; - for (ordinal, content) in V34_FIXTURE_CONTENTS.iter().enumerate() { + for (ordinal, content) in PAYLOAD_FIXTURE_CONTENTS.iter().enumerate() { seed_payload(&conn, ordinal, content).await; } conn.execute_batch( @@ -1076,7 +668,10 @@ async fn deleting_a_payload_drops_its_digest_and_digests_never_update() { ) .await .expect("seed digest rows"); - assert_eq!(digest_rows(&conn).await.len(), V34_FIXTURE_CONTENTS.len()); + assert_eq!( + digest_rows(&conn).await.len(), + PAYLOAD_FIXTURE_CONTENTS.len() + ); let update = conn .execute_batch( @@ -1091,9 +686,11 @@ async fn deleting_a_payload_drops_its_digest_and_digests_never_update() { .contains("immutable") ); - conn.execute_batch("DELETE FROM memory_v2_assertion_payloads WHERE fact_id = 'fact.v34.1';") - .await - .expect("delete a payload"); + conn.execute_batch( + "DELETE FROM memory_v2_assertion_payloads WHERE fact_id = 'fact.payload.1';", + ) + .await + .expect("delete a payload"); let remaining: Vec = digest_rows(&conn) .await .into_iter() @@ -1101,7 +698,7 @@ async fn deleting_a_payload_drops_its_digest_and_digests_never_update() { .collect(); assert_eq!( remaining, - vec!["fact.v34.0".to_owned(), "fact.v34.2".to_owned()] + vec!["fact.payload.0".to_owned(), "fact.payload.2".to_owned()] ); } diff --git a/crates/tracedecay-runtime-core/src/db/migrations/tests/final_shape.rs b/crates/tracedecay-runtime-core/src/db/migrations/tests/final_shape.rs index d7a57392af..8a65a3c4dc 100644 --- a/crates/tracedecay-runtime-core/src/db/migrations/tests/final_shape.rs +++ b/crates/tracedecay-runtime-core/src/db/migrations/tests/final_shape.rs @@ -10,10 +10,7 @@ use tempfile::TempDir; use crate::db::engine::TestConnection; use crate::db::{Database, DatabaseAuthority, TestDatabaseRuntimeMode}; -use super::super::{ - PAYLOAD_DIGEST_STEP_SOURCE_VERSION, SCHEMA_VERSION, create_schema_connection, - verify_final_schema_connection, -}; +use super::super::{SCHEMA_VERSION, create_schema_connection}; #[derive(Debug, PartialEq, Eq)] struct StoreSnapshot { @@ -117,229 +114,6 @@ async fn assert_reset_required_without_repair(path: &Path, mutation: &str) -> St reason.to_owned() } -/// The canonical project store exactly as every release from v0.1.0-beta.25 -/// through v0.1.0-beta.37 wrote it. The fixture header carries the -/// tag-to-inventory table; it is assembled from the tagged DDL rather than -/// from the current contract, because a released shape derived from the -/// current contract agrees with whatever this binary expects and so cannot -/// detect an admission that refuses what shipped. -const RELEASED_V34_PROJECT_STORE_SQL: &str = - include_str!("../../../../tests/fixtures/project-store-released-v34.sql"); - -/// Writes the released project store into an empty file, in the WAL mode -/// every shipped binary ran, and stamps the `user_version` a shipped binary -/// left, which is what selects the released admission path. -fn released_project_store(directory: &TempDir, schema: &str) -> PathBuf { - let path = directory.path().join("released-v34.db"); - let connection = rusqlite::Connection::open(&path).expect("create released store fixture"); - connection - .pragma_update(None, "journal_mode", "WAL") - .expect("run the released store in WAL mode"); - connection - .execute_batch(schema) - .expect("install the released project schema"); - connection - .execute_batch(&format!( - "PRAGMA user_version = {PAYLOAD_DIGEST_STEP_SOURCE_VERSION};" - )) - .expect("stamp the released schema version"); - path -} - -/// Rows a real profile carries in the tables whose shape changed after the -/// last release: a renamed external-source mutation table and two diagnostics -/// tables that gained a column and a wider primary key. -fn seed_released_project_rows(path: &Path) { - tamper( - path, - "INSERT INTO metadata(key, value) VALUES('mac.profile', 'beta.32'); - INSERT INTO semantic_vector_stage_census_authority VALUES('released-shard', 7); - INSERT INTO external_source_objects_v1( - binding_id, native_object_digest, partition_digest, mutation_digest, - mutation_json) - VALUES('binding-one', 'object-digest-one', 'partition-one', 'mutation-one', '{}'); - INSERT INTO diagnostic_generation_publications( - generation_id, record_state, state_generation, published_at) - VALUES('generation-one', 'current', NULL, 1700); - INSERT INTO generation_diagnostics( - diagnostic_anchor, generation_id, repository, worktree, reference, - source_revision, file_occurrence_id, content_digest, - symbol_occurrence_id, span_start, span_end, code, severity, message, - message_digest, producer_kind, producer, analyzer_revision, - configuration_revision, sanitization_receipt, evidence_class, - collected_at, record_state, state_generation, persisted_at) - VALUES('anchor-one', 'generation-one', '/Users/mac/project', NULL, NULL, - NULL, 'file-one', 'content-one', NULL, 1, 9, 'E0001', 'error', - 'keep my diagnostics', 'message-one', 'compiler', 'rustc', 'rev-one', - 'config-one', NULL, 'observed', 1700, 'current', NULL, 1700);", - ); -} - -/// Every seeded row, as a stable comparable snapshot. The external-source -/// mutations are read through whichever of the retired or current table the -/// store carries: the rename replaced a copied `mutation_json` payload with -/// the digest that already identified it, so the identifying columns are what -/// both shapes share and what the move must preserve. -fn seeded_project_rows(path: &Path) -> Vec { - let connection = - rusqlite::Connection::open_with_flags(path, rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY) - .expect("open released store fixture read-only"); - let mutations = if object_sql(path, "table", "external_source_objects_v2").is_some() { - "external_source_objects_v2" - } else { - "external_source_objects_v1" - }; - let mut rows = Vec::new(); - for sql in [ - "SELECT json_array(assertion_id, fact_id, payload_json, content) FROM memory_v2_assertion_payloads ORDER BY assertion_id".to_owned(), - "SELECT json_array(shard_id, revision) FROM semantic_vector_stage_census_authority ORDER BY shard_id".to_owned(), - // Only the seeded key: the step journals its own backfill receipt - // here, which is a migration record rather than retained content. - "SELECT json_array(key, value) FROM metadata - WHERE key = 'mac.profile'" - .to_owned(), - format!( - "SELECT json_array(binding_id, native_object_digest, partition_digest, - mutation_digest) - FROM {mutations} ORDER BY binding_id, native_object_digest" - ), - "SELECT json_array(generation_id, record_state, state_generation, published_at) - FROM diagnostic_generation_publications ORDER BY generation_id" - .to_owned(), - "SELECT json_array(diagnostic_anchor, generation_id, repository, file_occurrence_id, - content_digest, span_start, span_end, code, severity, message, - message_digest, producer_kind, producer, analyzer_revision, - configuration_revision, evidence_class, collected_at, record_state, - state_generation, persisted_at) - FROM generation_diagnostics ORDER BY diagnostic_anchor" - .to_owned(), - ] { - let mut statement = connection - .prepare(&sql) - .expect("prepare seeded project row snapshot"); - let snapshot = statement - .query_map((), |row| row.get::<_, String>(0)) - .expect("query seeded project rows") - .collect::, _>>() - .expect("read seeded project rows"); - rows.extend(snapshot); - } - rows -} - -/// Every release from v0.1.0-beta.25 through v0.1.0-beta.37 wrote one -/// byte-identical project store. Opening one of those stores must migrate it, -/// not ask its owner to discard the memory, diagnostics, and external-source -/// mutations it holds. -#[tokio::test] -async fn released_project_store_migrates_and_retains_every_row() { - let directory = tempfile::tempdir().expect("create released fixture directory"); - let path = released_project_store(&directory, RELEASED_V34_PROJECT_STORE_SQL); - seed_released_project_rows(&path); - let connection = TestConnection::open(&path); - super::seed_payload(&connection, 0, "Preserve café 東京 and \nwhitespace ").await; - drop(connection); - let read_only = verify_final_schema_connection(&TestConnection::open(&path)) - .await - .expect_err("a released store must await writer convergence"); - assert!(read_only.reset_required_context().is_none(), "{read_only}"); - let seeded = seeded_project_rows(&path); - assert_eq!(seeded.len(), 6, "the fixture must seed rows to retain"); - - admit_existing(&path, "a released project store must migrate, not reset").await; - - assert_eq!( - store_snapshot(&path).user_version, - i64::from(SCHEMA_VERSION), - "a migrated store must carry this binary's stamp" - ); - verify_final_schema_connection(&TestConnection::open(&path)) - .await - .expect("read-only admission accepts the converged released store"); - assert_eq!( - seeded_project_rows(&path), - seeded, - "every seeded row must survive the migration byte-exact" - ); - - let after_first_open = store_snapshot(&path); - admit_existing(&path, "a migrated store must reopen unchanged").await; - assert_eq!( - store_snapshot(&path), - after_first_open, - "the second open must be a no-op" - ); -} - -#[tokio::test] -async fn released_project_store_without_diagnostics_converges() { - let directory = tempfile::tempdir().unwrap(); - let path = released_project_store(&directory, RELEASED_V34_PROJECT_STORE_SQL); - // The Mac beta.32 profile never published diagnostics, so neither table exists. - tamper( - &path, - "DROP TABLE generation_diagnostics; DROP TABLE diagnostic_generation_publications;", - ); - admit_existing( - &path, - "lazily absent released diagnostics must be installed", - ) - .await; - verify_final_schema_connection(&TestConnection::open(&path)) - .await - .unwrap(); -} - -#[tokio::test] -async fn released_project_schema_drift_is_refused_without_mutation() { - for mutation in [ - "ALTER TABLE semantic_vector_stage_census_authority ADD COLUMN tamper TEXT;", - "DROP TRIGGER semantic_vector_stage_census_after_stage_insert;", - "CREATE TABLE unrecognized_authority(id INTEGER PRIMARY KEY);", - ] { - let directory = tempfile::tempdir().unwrap(); - let path = released_project_store(&directory, RELEASED_V34_PROJECT_STORE_SQL); - seed_released_project_rows(&path); - tamper(&path, mutation); - assert_reset_required_without_repair(&path, mutation).await; - } -} - -#[tokio::test] -async fn live_v35_project_store_converges_and_reopens_without_reset() { - let directory = tempfile::tempdir().unwrap(); - // Exact DDL delta in a2e7694c51's repository/semantic_vector_staging_schema.sql. - let live_schema = RELEASED_V34_PROJECT_STORE_SQL.replace( - "expected_chunk_count INTEGER NOT NULL\n CHECK (expected_chunk_count >= 0 AND expected_chunk_count <= 100000)", - "expected_chunk_count INTEGER NOT NULL CHECK (expected_chunk_count >= 0)", - ); - assert_ne!(live_schema, RELEASED_V34_PROJECT_STORE_SQL); - let path = released_project_store(&directory, &live_schema); - seed_released_project_rows(&path); - admit_existing(&path, "converge the released fixture").await; - // The previous Mac dogfood binary wrote this payload-digest-complete stamp. - tamper(&path, "PRAGMA user_version = 35;"); - let seeded = seeded_project_rows(&path); - let read_only = verify_final_schema_connection(&TestConnection::open(&path)) - .await - .expect_err("a prior stamp needs a writer"); - assert!(read_only.reset_required_context().is_none(), "{read_only}"); - admit_existing(&path, "a live v35 store must migrate without reset").await; - assert_eq!(seeded_project_rows(&path), seeded); - assert_eq!( - store_snapshot(&path).user_version, - i64::from(SCHEMA_VERSION) - ); - let converged = store_snapshot(&path); - admit_existing(&path, "reopen converged v35").await; - assert_eq!(store_snapshot(&path), converged); - tamper( - &path, - "PRAGMA user_version = 35; CREATE TABLE unknown_v35_authority(id INTEGER);", - ); - assert_reset_required_without_repair(&path, "unknown v35 shape").await; -} - #[tokio::test] async fn current_final_store_is_admitted_without_mutation() { let (_directory, path) = fresh_current_store().await; @@ -398,16 +172,17 @@ fn ledger_tables(path: &Path) -> Vec { .expect("read ledger tables") } -/// The runtime writer creates its ledger lazily inside the canonical store, so -/// a store's first lifetime used to leave a shape its next open refused. The -/// ledger is part of the exact shape now: a store that predates it gains it -/// on open, one that already carries it is admitted unchanged, and one still -/// holding the retired idempotency table has it folded into the current one. +/// The runtime writer ledger is part of the exact shape a fresh store is +/// created with. A store missing it, or still holding the retired +/// idempotency table, is refused unchanged rather than upgraded. #[tokio::test] async fn runtime_writer_ledger_is_part_of_the_final_shape() { let (_directory, path) = fresh_current_store().await; - let expected_ledger = ledger_tables(&path); - assert_eq!(expected_ledger.len(), 4, "fresh store carries the ledger"); + assert_eq!( + ledger_tables(&path).len(), + 4, + "fresh store carries the ledger" + ); let before = store_snapshot(&path); admit_existing(&path, "store carrying the ledger must be admitted").await; assert_eq!( @@ -416,57 +191,20 @@ async fn runtime_writer_ledger_is_part_of_the_final_shape() { "ledger-carrying admission stays query-only" ); - tamper( - &path, - "DROP TABLE td_runtime_writer_checkpoint_v1; - DROP TABLE td_runtime_writer_idempotency_v2; - DROP TABLE td_runtime_writer_outbox_v1; - DROP TABLE td_runtime_writer_inbox_v1;", - ); - assert!(ledger_tables(&path).is_empty()); - admit_existing(&path, "store predating the ledger must be admitted").await; - assert_eq!( - ledger_tables(&path), - expected_ledger, - "open installs the ledger" - ); - assert_eq!(store_snapshot(&path).schema_bytes, before.schema_bytes); + tamper(&path, "DROP TABLE td_runtime_writer_checkpoint_v1;"); + assert_reset_required_without_repair(&path, "store missing a ledger table").await; + let (_directory, path) = fresh_current_store().await; tamper( &path, - "DROP TABLE td_runtime_writer_idempotency_v2; - CREATE TABLE td_runtime_writer_idempotency_v1 ( - shard_json TEXT NOT NULL, incarnation INTEGER NOT NULL, - authority_epoch INTEGER NOT NULL, idempotency_key TEXT NOT NULL, - request_digest TEXT NOT NULL, original_receipt_json TEXT NOT NULL, - transaction_scope_json TEXT NOT NULL, operation_id TEXT NOT NULL, - durability_json TEXT NOT NULL, committed_at_micros INTEGER NOT NULL, - PRIMARY KEY (shard_json, incarnation, authority_epoch, idempotency_key) - ) WITHOUT ROWID; - INSERT INTO td_runtime_writer_idempotency_v1 VALUES - ('{}', 1, 1, 'key-1', 'digest', '{}', '{}', 'op-1', '{}', 42);", + "CREATE TABLE td_runtime_writer_idempotency_v1 (key TEXT PRIMARY KEY);", ); - admit_existing( - &path, - "store with the retired idempotency ledger must be admitted", - ) - .await; - assert_eq!( - ledger_tables(&path), - expected_ledger, - "open folds the retired ledger" + let reason = + assert_reset_required_without_repair(&path, "store holding the retired ledger").await; + assert!( + reason.contains("td_runtime_writer_idempotency_v1"), + "{reason}" ); - let connection = - rusqlite::Connection::open_with_flags(&path, rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY) - .expect("open final-shape fixture read-only"); - let migrated: (String, i64) = connection - .query_row( - "SELECT idempotency_key, committed_at_micros FROM td_runtime_writer_idempotency_v2", - (), - |row| Ok((row.get(0)?, row.get(1)?)), - ) - .expect("retired receipt survives the fold"); - assert_eq!(migrated, ("key-1".to_owned(), 42)); } #[tokio::test] diff --git a/crates/tracedecay-runtime-core/src/db/mod.rs b/crates/tracedecay-runtime-core/src/db/mod.rs index e0354c3fc1..8ce0629c48 100644 --- a/crates/tracedecay-runtime-core/src/db/mod.rs +++ b/crates/tracedecay-runtime-core/src/db/mod.rs @@ -3,7 +3,6 @@ mod connection; pub mod engine; mod row_codec; pub use row_codec::{decode_stored_json, encode_stored_json, optional_text_column, text_column}; -mod evidence_assembly; mod external_source; mod file_identity; mod graph_publication; @@ -59,7 +58,7 @@ pub use connection::{ RegisteredTestRuntimeFixtureV1, RegisteredTestRuntimeRetirementControlV1, TestDatabaseRuntimeMode, TestDatabaseRuntimeScope, TestRuntimeProfileIdentityV1, }; -pub use external_source::{install_external_source_schema, migrate_retired_mutation_copy_tables}; +pub use external_source::install_external_source_schema; pub use file_identity::{ SqliteFileIdentityError, SqliteFileIdentityErrorCategory, SqliteFileIdentityOperation, sqlite_generation_identity, @@ -76,5 +75,4 @@ pub use sql::{ pub use tracedecay_store::{ AnchorDerivativeKindV1, AnchorDispositionAppendOutcomeV1, AnchorDispositionReasonClassV1, AnchorDispositionStateV1, RetrievalAnchorDerivativeV1, RetrievalAnchorDispositionRecordV1, - RetrievalAnchorOwnerV1, }; diff --git a/crates/tracedecay-runtime-core/src/db/retrieval_anchor_authority.rs b/crates/tracedecay-runtime-core/src/db/retrieval_anchor_authority.rs index d976e2a23c..84ff328f4a 100644 --- a/crates/tracedecay-runtime-core/src/db/retrieval_anchor_authority.rs +++ b/crates/tracedecay-runtime-core/src/db/retrieval_anchor_authority.rs @@ -4,8 +4,8 @@ use tracedecay_domain::{FactOwnerV1, RetrievalAnchorId, UtcMicros}; use tracedecay_store::{ AnchorDerivativeKindV1, AnchorDispositionAppendOutcomeV1, AnchorDispositionStateV1, RetrievalAnchorDerivativeV1, RetrievalAnchorDispositionRecordV1, - RetrievalAnchorDispositionStore, RetrievalAnchorOwnerV1, RetrievalAnchorStoreError, - RetrievalAnchorStoreResult, RetrievalAnchorTombstoneV1, + RetrievalAnchorDispositionStore, RetrievalAnchorStoreError, RetrievalAnchorStoreResult, + RetrievalAnchorTombstoneV1, }; use crate::db::engine::{Executor, QueryExecutor, params}; @@ -84,7 +84,7 @@ pub(crate) async fn resolve_anchor_derivatives( anchor_id: &RetrievalAnchorId, ) -> Result> where - O: serde::Serialize + Clone + Into, + O: serde::Serialize + Clone + Into, { let owner_json = owner_json(owner)?; if !AnchorDispositionStateV1::serves_derivatives( @@ -421,7 +421,7 @@ impl super::Database { anchor_id: &RetrievalAnchorId, ) -> Result> where - O: serde::Serialize + Clone + Into, + O: serde::Serialize + Clone + Into, { let connection = self.read_connection(); resolve_anchor_derivatives(&connection, owner, anchor_id).await @@ -504,7 +504,7 @@ impl RetrievalAnchorDispositionStore for super::Database { fn current_disposition( &self, anchor_id: &RetrievalAnchorId, - owner: &RetrievalAnchorOwnerV1, + owner: &FactOwnerV1, ) -> impl std::future::Future< Output = RetrievalAnchorStoreResult>, > + Send { @@ -519,7 +519,7 @@ impl RetrievalAnchorDispositionStore for super::Database { fn tombstone( &self, anchor_id: &RetrievalAnchorId, - owner: &RetrievalAnchorOwnerV1, + owner: &FactOwnerV1, ) -> impl std::future::Future< Output = RetrievalAnchorStoreResult>, > + Send { @@ -554,7 +554,7 @@ impl RetrievalAnchorDispositionStore for super::Database { fn derivatives( &self, anchor_id: &RetrievalAnchorId, - owner: &RetrievalAnchorOwnerV1, + owner: &FactOwnerV1, ) -> impl std::future::Future< Output = RetrievalAnchorStoreResult>, > + Send { diff --git a/crates/tracedecay-runtime-core/src/db/retrieval_anchor_schema.rs b/crates/tracedecay-runtime-core/src/db/retrieval_anchor_schema.rs index 5c6659bedd..0cfdbf8313 100644 --- a/crates/tracedecay-runtime-core/src/db/retrieval_anchor_schema.rs +++ b/crates/tracedecay-runtime-core/src/db/retrieval_anchor_schema.rs @@ -10,11 +10,6 @@ use std::collections::BTreeSet; use crate::db::engine::{Executor, params}; use tracedecay_domain::errors::{Result, TraceDecayError}; -const ALIASES_TABLE: &str = "retrieval_anchor_aliases"; -const LEGACY_ALIASES_TABLE: &str = "retrieval_anchor_aliases_owner_unbound_v1"; -const DISPOSITIONS_TABLE: &str = "retrieval_anchor_dispositions"; -const LEGACY_DISPOSITIONS_TABLE: &str = "retrieval_anchor_dispositions_terminal_v0"; - /// The canonical anchor DDL lives in `tracedecay-store` because the concrete /// executors in the rusqlite runtime crate write the same table and must see /// the same constraints; installing a private copy here is how a fixture ends @@ -188,20 +183,6 @@ fn schema_error(operation: &str, message: impl Into) -> TraceDecayError } } -async fn table_exists(conn: &(impl Executor + Sync), table: &str, operation: &str) -> Result { - let mut rows = conn - .query( - "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?1", - params![table], - ) - .await - .map_err(|error| database_error(operation, error))?; - rows.next() - .await - .map(|row| row.is_some()) - .map_err(|error| database_error(operation, error)) -} - async fn table_columns( conn: &(impl Executor + Sync), table: &str, @@ -225,59 +206,6 @@ async fn table_columns( Ok(columns) } -async fn aliases_have_owner_bound_foreign_key( - conn: &(impl Executor + Sync), - operation: &str, -) -> Result { - let mut rows = conn - .query( - "SELECT \"from\", \"to\" - FROM pragma_foreign_key_list('retrieval_anchor_aliases') - WHERE \"table\" = 'retrieval_anchors' - ORDER BY id, seq", - (), - ) - .await - .map_err(|error| database_error(operation, error))?; - let mut columns = Vec::new(); - while let Some(row) = rows - .next() - .await - .map_err(|error| database_error(operation, error))? - { - columns.push(( - row.get::(0) - .map_err(|error| database_error(operation, error))?, - row.get::(1) - .map_err(|error| database_error(operation, error))?, - )); - } - Ok(columns - == [ - ("anchor_id".to_owned(), "anchor_id".to_owned()), - ("owner_json".to_owned(), "owner_json".to_owned()), - ]) -} - -async fn validate_alias_table_columns( - conn: &(impl Executor + Sync), - table: &str, - operation: &str, -) -> Result<()> { - let expected = ["owner_json", "alias_kind", "locator_digest", "anchor_id"] - .into_iter() - .map(str::to_owned) - .collect::>(); - let columns = table_columns(conn, table, operation).await?; - if columns == expected { - return Ok(()); - } - Err(schema_error( - operation, - format!("{table} has unsupported columns: {columns:?}"), - )) -} - async fn validate_anchor_table_columns( conn: &(impl Executor + Sync), operation: &str, @@ -301,474 +229,11 @@ async fn validate_anchor_table_columns( )) } -async fn validate_legacy_alias_ownership( - conn: &(impl Executor + Sync), - operation: &str, -) -> Result<()> { - let mut rows = conn - .query( - "SELECT aliases.anchor_id - FROM retrieval_anchor_aliases_owner_unbound_v1 AS aliases - LEFT JOIN retrieval_anchors AS anchors - ON anchors.anchor_id = aliases.anchor_id - AND anchors.owner_json = aliases.owner_json - WHERE anchors.anchor_id IS NULL - LIMIT 1", - (), - ) - .await - .map_err(|error| database_error(operation, error))?; - if rows - .next() - .await - .map_err(|error| database_error(operation, error))? - .is_some() - { - return Err(schema_error( - operation, - "legacy retrieval-anchor alias has no anchor with the same owner", - )); - } - Ok(()) -} - -/// The two directions an owner-unbound alias row can collide with a canonical -/// one: same owner key under a different anchor, or same anchor key under a -/// different owner. -/// -/// These live outside the loop below because an array literal iterated in -/// place leaves a `std::array::IntoIter` alive across the query `await`, and -/// that iterator's `MaybeDangling`/`ManuallyDrop`/`MaybeUninit` layers are -/// re-entered for every auto-trait obligation raised on the enclosing future, -/// at the deepest point of the schema-install chain that daemon project-open -/// awaits. Borrowing a promoted slice keeps a plain `slice::Iter` there. -const ALIAS_COPY_CONFLICT_QUERIES: &[&str] = &[ - "SELECT 1 - FROM retrieval_anchor_aliases_owner_unbound_v1 AS legacy - JOIN retrieval_anchor_aliases AS current - ON current.owner_json = legacy.owner_json - AND current.alias_kind = legacy.alias_kind - AND current.locator_digest = legacy.locator_digest - WHERE current.anchor_id <> legacy.anchor_id - LIMIT 1", - "SELECT 1 - FROM retrieval_anchor_aliases_owner_unbound_v1 AS legacy - JOIN retrieval_anchor_aliases AS current - ON current.anchor_id = legacy.anchor_id - AND current.alias_kind = legacy.alias_kind - AND current.locator_digest = legacy.locator_digest - WHERE current.owner_json <> legacy.owner_json - LIMIT 1", -]; - -async fn validate_alias_copy_conflicts( - conn: &(impl Executor + Sync), - operation: &str, -) -> Result<()> { - for &sql in ALIAS_COPY_CONFLICT_QUERIES { - let mut rows = conn - .query(sql, ()) - .await - .map_err(|error| database_error(operation, error))?; - if rows - .next() - .await - .map_err(|error| database_error(operation, error))? - .is_some() - { - return Err(schema_error( - operation, - "retrieval-anchor alias migration conflicts with canonical aliases", - )); - } - } - Ok(()) -} - -async fn restore_legacy_aliases(conn: &(impl Executor + Sync), operation: &str) -> Result<()> { - if !table_exists(conn, LEGACY_ALIASES_TABLE, operation).await? { - return Ok(()); - } - validate_alias_table_columns(conn, LEGACY_ALIASES_TABLE, operation).await?; - validate_legacy_alias_ownership(conn, operation).await?; - validate_alias_copy_conflicts(conn, operation).await?; - conn.execute_batch( - "INSERT OR IGNORE INTO retrieval_anchor_aliases ( - owner_json, alias_kind, locator_digest, anchor_id - ) - SELECT owner_json, alias_kind, locator_digest, anchor_id - FROM retrieval_anchor_aliases_owner_unbound_v1;", - ) - .await - .map_err(|error| database_error(operation, error))?; - - let mut rows = conn - .query( - "SELECT 1 - FROM retrieval_anchor_aliases_owner_unbound_v1 AS legacy - LEFT JOIN retrieval_anchor_aliases AS current - ON current.owner_json = legacy.owner_json - AND current.alias_kind = legacy.alias_kind - AND current.locator_digest = legacy.locator_digest - AND current.anchor_id = legacy.anchor_id - WHERE current.anchor_id IS NULL - LIMIT 1", - (), - ) - .await - .map_err(|error| database_error(operation, error))?; - if rows - .next() - .await - .map_err(|error| database_error(operation, error))? - .is_some() - { - return Err(schema_error( - operation, - "retrieval-anchor alias migration did not preserve every legacy row", - )); - } - drop(rows); - conn.execute_batch("DROP TABLE retrieval_anchor_aliases_owner_unbound_v1;") - .await - .map_err(|error| database_error(operation, error)) -} - -#[hotpath::measure(label = "runtime_core.db.anchor_alias_upgrade")] -async fn upgrade_aliases_if_needed(conn: &(impl Executor + Sync), operation: &str) -> Result<()> { - let aliases_exist = table_exists(conn, ALIASES_TABLE, operation).await?; - let legacy_exists = table_exists(conn, LEGACY_ALIASES_TABLE, operation).await?; - if aliases_exist && !aliases_have_owner_bound_foreign_key(conn, operation).await? { - if legacy_exists { - return Err(schema_error( - operation, - "both legacy and noncanonical retrieval-anchor alias tables exist", - )); - } - validate_alias_table_columns(conn, ALIASES_TABLE, operation).await?; - conn.execute_batch( - "DROP TRIGGER IF EXISTS retrieval_anchor_aliases_immutable_update; - DROP TRIGGER IF EXISTS retrieval_anchor_aliases_immutable_delete; - DROP TRIGGER IF EXISTS retrieval_anchor_aliases_no_update; - DROP TRIGGER IF EXISTS retrieval_anchor_aliases_no_delete; - ALTER TABLE retrieval_anchor_aliases - RENAME TO retrieval_anchor_aliases_owner_unbound_v1;", - ) - .await - .map_err(|error| database_error(operation, error))?; - } - - conn.execute_batch(ALIASES_SCHEMA) - .await - .map_err(|error| database_error(operation, error))?; - restore_legacy_aliases(conn, operation).await -} - -async fn dispositions_support_terminal_states( - conn: &(impl Executor + Sync), - operation: &str, -) -> Result { - let mut rows = conn - .query( - "SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?1", - params![DISPOSITIONS_TABLE], - ) - .await - .map_err(|error| database_error(operation, error))?; - let Some(row) = rows - .next() - .await - .map_err(|error| database_error(operation, error))? - else { - return Ok(false); - }; - let sql = row - .get::(0) - .map_err(|error| database_error(operation, error))?; - Ok([ - "'active'", - "'superseded'", - "'redacted'", - "'expired'", - "'quarantined'", - "'deleted'", - "'unavailable'", - ] - .iter() - .all(|state| sql.contains(state))) -} - -async fn dispositions_have_owner_bound_identity( - conn: &(impl Executor + Sync), - operation: &str, -) -> Result { - let mut rows = conn - .query( - "SELECT name FROM pragma_index_list(?1) WHERE \"unique\" = 1 ORDER BY seq", - params![DISPOSITIONS_TABLE], - ) - .await - .map_err(|error| database_error(operation, error))?; - let mut indexes = Vec::new(); - while let Some(row) = rows - .next() - .await - .map_err(|error| database_error(operation, error))? - { - indexes.push( - row.get::(0) - .map_err(|error| database_error(operation, error))?, - ); - } - drop(rows); - for index in indexes { - let mut columns = conn - .query( - "SELECT name FROM pragma_index_info(?1) ORDER BY seqno", - params![index], - ) - .await - .map_err(|error| database_error(operation, error))?; - let mut names = Vec::new(); - while let Some(row) = columns - .next() - .await - .map_err(|error| database_error(operation, error))? - { - names.push( - row.get::(0) - .map_err(|error| database_error(operation, error))?, - ); - } - if names.len() == 2 && names[0] == "owner_json" && names[1] == "disposition_id" { - return Ok(true); - } - } - Ok(false) -} - -async fn validate_disposition_table_columns( - conn: &(impl Executor + Sync), - table: &str, - operation: &str, -) -> Result<()> { - let expected = [ - "sequence", - "disposition_id", - "anchor_id", - "owner_json", - "state", - "superseded_by", - "reason_class", - "effective_at", - "record_json", - ] - .into_iter() - .map(str::to_owned) - .collect::>(); - let columns = table_columns(conn, table, operation).await?; - if columns == expected { - return Ok(()); - } - Err(schema_error( - operation, - format!("{table} has unsupported columns: {columns:?}"), - )) -} - -async fn restore_legacy_dispositions(conn: &(impl Executor + Sync), operation: &str) -> Result<()> { - if !table_exists(conn, LEGACY_DISPOSITIONS_TABLE, operation).await? { - return Ok(()); - } - validate_disposition_table_columns(conn, LEGACY_DISPOSITIONS_TABLE, operation).await?; - validate_disposition_rows(conn, LEGACY_DISPOSITIONS_TABLE, operation).await?; - conn.execute_batch( - "INSERT OR IGNORE INTO retrieval_anchor_dispositions ( - sequence, disposition_id, anchor_id, owner_json, state, superseded_by, - reason_class, effective_at, record_json - ) - SELECT sequence, disposition_id, anchor_id, owner_json, state, superseded_by, - reason_class, effective_at, record_json - FROM retrieval_anchor_dispositions_terminal_v0;", - ) - .await - .map_err(|error| database_error(operation, error))?; - - let mut rows = conn - .query( - "SELECT 1 - FROM retrieval_anchor_dispositions_terminal_v0 AS legacy - LEFT JOIN retrieval_anchor_dispositions AS current - ON current.sequence = legacy.sequence - AND current.disposition_id = legacy.disposition_id - AND current.anchor_id = legacy.anchor_id - AND current.owner_json = legacy.owner_json - AND current.state = legacy.state - AND current.superseded_by IS legacy.superseded_by - AND current.reason_class = legacy.reason_class - AND current.effective_at = legacy.effective_at - AND current.record_json = legacy.record_json - WHERE current.sequence IS NULL - LIMIT 1", - (), - ) - .await - .map_err(|error| database_error(operation, error))?; - if rows - .next() - .await - .map_err(|error| database_error(operation, error))? - .is_some() - { - return Err(schema_error( - operation, - "retrieval-anchor disposition migration did not preserve every legacy row", - )); - } - drop(rows); - conn.execute_batch("DROP TABLE retrieval_anchor_dispositions_terminal_v0;") - .await - .map_err(|error| database_error(operation, error)) -} - -async fn validate_disposition_rows( - conn: &(impl Executor + Sync), - table: &str, - operation: &str, -) -> Result<()> { - let sql = match table { - DISPOSITIONS_TABLE => { - "SELECT disposition_id, anchor_id, owner_json, state, superseded_by, - reason_class, effective_at, record_json - FROM retrieval_anchor_dispositions - ORDER BY sequence" - } - LEGACY_DISPOSITIONS_TABLE => { - "SELECT disposition_id, anchor_id, owner_json, state, superseded_by, - reason_class, effective_at, record_json - FROM retrieval_anchor_dispositions_terminal_v0 - ORDER BY sequence" - } - _ => { - return Err(schema_error( - operation, - "unsupported retrieval-anchor disposition table", - )); - } - }; - let mut rows = conn - .query(sql, ()) - .await - .map_err(|error| database_error(operation, error))?; - while let Some(row) = rows - .next() - .await - .map_err(|error| database_error(operation, error))? - { - let disposition_id = row - .get::(0) - .map_err(|error| database_error(operation, error))?; - let anchor_id = row - .get::(1) - .map_err(|error| database_error(operation, error))?; - let owner_json = row - .get::(2) - .map_err(|error| database_error(operation, error))?; - let state = row - .get::(3) - .map_err(|error| database_error(operation, error))?; - let superseded_by = row - .get::>(4) - .map_err(|error| database_error(operation, error))?; - let reason_class = row - .get::(5) - .map_err(|error| database_error(operation, error))?; - let effective_at = row - .get::(6) - .map_err(|error| database_error(operation, error))?; - let record_json = row - .get::(7) - .map_err(|error| database_error(operation, error))?; - let record = serde_json::from_str::( - &record_json, - ) - .map_err(|error| database_error(operation, error))?; - record - .validate() - .map_err(|error| database_error(operation, error))?; - let canonical_owner = serde_json::to_string(record.owner()) - .map_err(|error| database_error(operation, error))?; - if record.disposition_id() != disposition_id - || record.anchor_id().as_str() != anchor_id - || canonical_owner != owner_json - || record.state().as_str() != state - || record - .superseded_by() - .map(tracedecay_domain::RetrievalAnchorId::as_str) - != superseded_by.as_deref() - || record.reason_class().as_str() != reason_class - || record.effective_at().0 != effective_at - { - return Err(schema_error( - operation, - "legacy retrieval-anchor disposition record does not match its indexed columns", - )); - } - } - Ok(()) -} - -#[hotpath::measure(label = "runtime_core.db.anchor_disposition_upgrade")] -async fn upgrade_dispositions_if_needed( - conn: &(impl Executor + Sync), - operation: &str, -) -> Result<()> { - let current_exists = table_exists(conn, DISPOSITIONS_TABLE, operation).await?; - let legacy_exists = table_exists(conn, LEGACY_DISPOSITIONS_TABLE, operation).await?; - if current_exists { - validate_disposition_table_columns(conn, DISPOSITIONS_TABLE, operation).await?; - } - if current_exists - && (!dispositions_support_terminal_states(conn, operation).await? - || !dispositions_have_owner_bound_identity(conn, operation).await?) - { - if legacy_exists { - return Err(schema_error( - operation, - "both legacy and noncanonical retrieval-anchor disposition tables exist", - )); - } - validate_disposition_rows(conn, DISPOSITIONS_TABLE, operation).await?; - conn.execute_batch( - "DROP TRIGGER IF EXISTS retrieval_anchor_dispositions_immutable_update; - DROP TRIGGER IF EXISTS retrieval_anchor_dispositions_immutable_delete; - DROP INDEX IF EXISTS idx_retrieval_anchor_dispositions_current; - ALTER TABLE retrieval_anchor_dispositions - RENAME TO retrieval_anchor_dispositions_terminal_v0;", - ) - .await - .map_err(|error| database_error(operation, error))?; - } - - conn.execute_batch(AUTHORITY_SCHEMA) - .await - .map_err(|error| database_error(operation, error))?; - restore_legacy_dispositions(conn, operation).await -} - /// Installs the physical schema for immutable, owner-bound retrieval anchors. /// /// The caller owns its local binding table (for example observation-to-anchor /// or fact-evidence-to-anchor) and should invoke this before creating a table /// with a composite foreign key to `retrieval_anchors(anchor_id, owner_json)`. -/// Existing one-column alias foreign keys are upgraded with a resumable, -/// validated copy; conflicting or ownerless rows are retained and reported -/// rather than discarded. -/// -/// The two upgrade phases are awaited through a box. Each is a deep tree of -/// nested `async fn` validators, and async lowering would otherwise expand -/// those trees into the future of every migration, admission, and store-open -/// path that installs this schema. #[hotpath::measure(label = "runtime_core.db.anchor_schema_install")] pub async fn install_retrieval_anchor_schema( conn: &(impl Executor + Sync), @@ -778,17 +243,12 @@ pub async fn install_retrieval_anchor_schema( .await .map_err(|error| database_error(operation, error))?; validate_anchor_table_columns(conn, operation).await?; - Box::pin(upgrade_aliases_if_needed(conn, operation)).await?; - Box::pin(upgrade_dispositions_if_needed(conn, operation)).await?; - conn.execute_batch( - "DROP TRIGGER IF EXISTS retrieval_anchors_no_update; - DROP TRIGGER IF EXISTS retrieval_anchors_no_delete; - DROP TRIGGER IF EXISTS retrieval_anchor_aliases_no_update; - DROP TRIGGER IF EXISTS retrieval_anchor_aliases_no_delete; - DROP TRIGGER IF EXISTS retrieval_anchor_aliases_immutable_update;", - ) - .await - .map_err(|error| database_error(operation, error))?; + conn.execute_batch(ALIASES_SCHEMA) + .await + .map_err(|error| database_error(operation, error))?; + conn.execute_batch(AUTHORITY_SCHEMA) + .await + .map_err(|error| database_error(operation, error))?; conn.execute_batch(RETRIEVAL_ANCHOR_IMMUTABILITY_TRIGGERS_SQL) .await .map_err(|error| database_error(operation, error)) @@ -796,15 +256,9 @@ pub async fn install_retrieval_anchor_schema( #[cfg(test)] mod tests { - use tracedecay_domain::{FactOwnerV1, ProjectId, RetrievalAnchorId, UtcMicros}; - use tracedecay_store::{ - AnchorDispositionReasonClassV1, AnchorDispositionStateV1, - RetrievalAnchorDispositionRecordV1, - }; - use crate::db::engine::{Executor, QueryExecutor, TestConnection, params}; - use super::{ANCHORS_SCHEMA, install_retrieval_anchor_schema}; + use super::install_retrieval_anchor_schema; async fn connection() -> (tempfile::TempDir, TestConnection) { let directory = tempfile::tempdir().expect("create retrieval-anchor schema fixture"); @@ -827,29 +281,6 @@ mod tests { .expect("insert anchor"); } - /// Async lowering embeds an awaited future inside its caller, so the alias - /// and disposition upgrade phases would expand into every migration, - /// admission, and store-open future that installs this schema. Both phases - /// stay behind pointers at this boundary; the probe fails if either is - /// inlined back into the install future. - /// - /// Measured under `--features hotpath`, where each measured `async fn` - /// embeds its body a second time: 2,144 B with both phases boxed, 7,664 B - /// with either one inlined. - #[test] - fn schema_install_holds_its_upgrade_phases_behind_pointers() { - const CEILING: usize = 4 * 1024; - let directory = tempfile::tempdir().expect("create anchor future probe fixture"); - let connection = TestConnection::open(&directory.path().join("anchors.db")); - let install = install_retrieval_anchor_schema(&connection, "probe anchor schema future"); - let size = std::mem::size_of_val(&install); - drop(install); - assert!( - size <= CEILING, - "install_retrieval_anchor_schema future is {size} B; ceiling {CEILING} B" - ); - } - #[tokio::test] async fn installs_owner_bound_aliases_and_immutable_records() { let (_directory, conn) = connection().await; @@ -902,18 +333,6 @@ mod tests { install_retrieval_anchor_schema(&conn, "install alias transition fixture") .await .unwrap(); - conn.execute_batch( - "DROP TRIGGER retrieval_anchor_aliases_immutable_update; - CREATE TRIGGER retrieval_anchor_aliases_immutable_update - BEFORE UPDATE ON retrieval_anchor_aliases BEGIN - SELECT RAISE(ABORT, 'retrieval anchor aliases are immutable'); - END;", - ) - .await - .unwrap(); - install_retrieval_anchor_schema(&conn, "upgrade shipped alias trigger") - .await - .unwrap(); conn.execute( "INSERT INTO retrieval_anchors(anchor_id, anchor_json, owner_json, projection_generation) VALUES ('old', '{}', '{}', 'g'), ('new', '{}', '{}', 'g'), ('other', '{}', '{}', 'g')", (), @@ -1018,233 +437,4 @@ mod tests { 2 ); } - - #[tokio::test] - async fn upgrades_legacy_aliases_without_losing_rows() { - let (_directory, conn) = connection().await; - conn.execute_batch( - "CREATE TABLE retrieval_anchors ( - anchor_id TEXT PRIMARY KEY, - anchor_json TEXT NOT NULL, - owner_json TEXT NOT NULL, - projection_generation TEXT NOT NULL - ); - CREATE TABLE retrieval_anchor_aliases ( - owner_json TEXT NOT NULL, - alias_kind TEXT NOT NULL, - locator_digest TEXT NOT NULL, - anchor_id TEXT NOT NULL, - PRIMARY KEY(owner_json, alias_kind, locator_digest), - UNIQUE(anchor_id, alias_kind, locator_digest), - FOREIGN KEY(anchor_id) REFERENCES retrieval_anchors(anchor_id) - );", - ) - .await - .expect("create legacy schema"); - insert_anchor(&conn, "{\"owner\":\"one\"}").await; - conn.execute( - "INSERT INTO retrieval_anchor_aliases ( - owner_json, alias_kind, locator_digest, anchor_id - ) VALUES (?1, 'fixture', 'digest-1', 'anchor-1')", - params!["{\"owner\":\"one\"}"], - ) - .await - .expect("insert legacy alias"); - - install_retrieval_anchor_schema(&conn, "upgrade retrieval-anchor schema") - .await - .expect("upgrade schema"); - install_retrieval_anchor_schema(&conn, "upgrade retrieval-anchor schema") - .await - .expect("replay upgrade"); - - let mut rows = conn - .query("SELECT count(*) FROM retrieval_anchor_aliases", ()) - .await - .expect("count aliases"); - let count = rows - .next() - .await - .expect("read alias count") - .expect("alias count row") - .get::(0) - .expect("decode alias count"); - assert_eq!(count, 1); - assert!( - conn.execute( - "INSERT INTO retrieval_anchor_aliases ( - owner_json, alias_kind, locator_digest, anchor_id - ) VALUES (?1, 'fixture', 'digest-2', 'anchor-1')", - params!["{\"owner\":\"other\"}"], - ) - .await - .is_err() - ); - } - - #[tokio::test] - async fn widens_legacy_dispositions_without_losing_rows() { - let (_directory, conn) = connection().await; - let owner = FactOwnerV1::Project { - project_id: ProjectId::new("project.fixture").expect("project id"), - }; - let owner_json = serde_json::to_string(&owner).expect("serialize owner"); - let disposition = RetrievalAnchorDispositionRecordV1::new( - "disposition-1", - RetrievalAnchorId::new("anchor-1").expect("anchor id"), - owner, - AnchorDispositionStateV1::Active, - None, - AnchorDispositionReasonClassV1::Correction, - UtcMicros(1), - ) - .expect("legacy disposition"); - let disposition_json = - serde_json::to_string(&disposition).expect("serialize legacy disposition"); - conn.execute_batch(ANCHORS_SCHEMA) - .await - .expect("install anchor schema"); - insert_anchor(&conn, &owner_json).await; - conn.execute_batch( - "CREATE TABLE retrieval_anchor_dispositions ( - sequence INTEGER PRIMARY KEY AUTOINCREMENT, - disposition_id TEXT NOT NULL UNIQUE, - anchor_id TEXT NOT NULL, - owner_json TEXT NOT NULL, - state TEXT NOT NULL CHECK( - state IN ('active', 'superseded', 'deleted', 'unavailable') - ), - superseded_by TEXT, - reason_class TEXT NOT NULL, - effective_at INTEGER NOT NULL, - record_json TEXT NOT NULL - );", - ) - .await - .expect("create legacy disposition schema"); - conn.execute( - "INSERT INTO retrieval_anchor_dispositions ( - disposition_id, anchor_id, owner_json, state, superseded_by, - reason_class, effective_at, record_json - ) VALUES ( - 'disposition-1', 'anchor-1', ?1, 'active', NULL, - 'correction', 1, ?2 - )", - params![owner_json, disposition_json], - ) - .await - .expect("insert legacy disposition"); - - install_retrieval_anchor_schema(&conn, "upgrade retrieval-anchor dispositions") - .await - .expect("upgrade disposition schema"); - install_retrieval_anchor_schema(&conn, "upgrade retrieval-anchor dispositions") - .await - .expect("replay disposition upgrade"); - conn.execute( - "INSERT INTO retrieval_anchor_dispositions ( - disposition_id, anchor_id, owner_json, state, superseded_by, - reason_class, effective_at, record_json - ) VALUES ( - 'disposition-2', 'anchor-1', ?1, 'redacted', NULL, - 'redaction', 2, '{}' - )", - params![serde_json::to_string(disposition.owner()).expect("serialize owner")], - ) - .await - .expect("insert evolved disposition"); - - let mut rows = conn - .query("SELECT count(*) FROM retrieval_anchor_dispositions", ()) - .await - .expect("count dispositions"); - assert_eq!( - rows.next() - .await - .expect("read disposition count") - .expect("disposition count row") - .get::(0) - .expect("decode disposition count"), - 2 - ); - } - - #[tokio::test] - async fn refuses_mismatched_legacy_disposition_before_renaming_it() { - let (_directory, conn) = connection().await; - let owner = FactOwnerV1::Project { - project_id: ProjectId::new("project.fixture").expect("project id"), - }; - let owner_json = serde_json::to_string(&owner).expect("serialize owner"); - let disposition = RetrievalAnchorDispositionRecordV1::new( - "disposition-1", - RetrievalAnchorId::new("anchor-1").expect("anchor id"), - owner, - AnchorDispositionStateV1::Deleted, - None, - AnchorDispositionReasonClassV1::UserRequest, - UtcMicros(1), - ) - .expect("legacy disposition"); - conn.execute_batch(ANCHORS_SCHEMA) - .await - .expect("install anchor schema"); - insert_anchor(&conn, &owner_json).await; - conn.execute_batch( - "CREATE TABLE retrieval_anchor_dispositions ( - sequence INTEGER PRIMARY KEY AUTOINCREMENT, - disposition_id TEXT NOT NULL UNIQUE, - anchor_id TEXT NOT NULL, - owner_json TEXT NOT NULL, - state TEXT NOT NULL CHECK( - state IN ('active', 'superseded', 'deleted', 'unavailable') - ), - superseded_by TEXT, - reason_class TEXT NOT NULL, - effective_at INTEGER NOT NULL, - record_json TEXT NOT NULL - );", - ) - .await - .expect("create legacy disposition schema"); - conn.execute( - "INSERT INTO retrieval_anchor_dispositions ( - disposition_id, anchor_id, owner_json, state, superseded_by, - reason_class, effective_at, record_json - ) VALUES ( - 'disposition-1', 'anchor-1', ?1, 'active', NULL, - 'user_request', 1, ?2 - )", - params![ - owner_json, - serde_json::to_string(&disposition).expect("serialize disposition") - ], - ) - .await - .expect("insert mismatched disposition"); - - assert!( - install_retrieval_anchor_schema(&conn, "reject invalid disposition migration") - .await - .is_err() - ); - assert!( - super::table_exists( - &conn, - super::DISPOSITIONS_TABLE, - "inspect rejected migration" - ) - .await - .expect("inspect current table") - ); - assert!( - !super::table_exists( - &conn, - super::LEGACY_DISPOSITIONS_TABLE, - "inspect rejected migration" - ) - .await - .expect("inspect legacy table") - ); - } } diff --git a/crates/tracedecay-runtime-core/src/lifecycle_lease.rs b/crates/tracedecay-runtime-core/src/lifecycle_lease.rs index 05b95672bd..977d4b88f7 100644 --- a/crates/tracedecay-runtime-core/src/lifecycle_lease.rs +++ b/crates/tracedecay-runtime-core/src/lifecycle_lease.rs @@ -7,8 +7,10 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use crate::db::is_lock_contended; use tracedecay_domain::errors::{Result, TraceDecayError}; +use tracedecay_private_fs::FileLease; const LIFECYCLE_LOCK_FILENAME: &str = "lifecycle.lock"; +const LIFECYCLE_LEASE_LABEL: &str = "lifecycle"; const EXCLUSIVE_LEASE_POLL_INTERVAL: Duration = Duration::from_millis(25); static LEASE_NONCE: AtomicU64 = AtomicU64::new(0); static PROCESS_LEASE_TOKENS: LazyLock>> = @@ -16,7 +18,7 @@ static PROCESS_LEASE_TOKENS: LazyLock>> = #[derive(Debug)] enum LeaseHold { - File(File), + File(FileLease), Inherited, } @@ -128,12 +130,9 @@ impl Drop for LifecycleLease { if let Some(token) = self.token.as_deref() { unregister_process_token(token); } - if let LeaseHold::File(file) = &self.hold { - #[cfg(windows)] - if self.exclusive { - remove_owner_sidecar_if_current(&self.lock_path, self.token.as_deref()); - } - let _ = file.unlock(); + #[cfg(windows)] + if self.exclusive && matches!(self.hold, LeaseHold::File(_)) { + remove_owner_sidecar_if_current(&self.lock_path, self.token.as_deref()); } } } @@ -191,7 +190,7 @@ pub fn acquire_shared_blocking(operation: &str) -> Result { file.lock_shared() .map_err(|error| lock_error(&path, operation, &error))?; Ok(LifecycleLease { - hold: LeaseHold::File(file), + hold: LeaseHold::File(FileLease::held(file, LIFECYCLE_LEASE_LABEL)), token: None, lock_path: path, exclusive: false, @@ -223,7 +222,7 @@ fn acquire_shared_or_inherited_at(path: &Path, operation: &str) -> Result Ok(LifecycleLease { - hold: LeaseHold::File(file), + hold: LeaseHold::File(FileLease::held(file, LIFECYCLE_LEASE_LABEL)), token: None, lock_path: path.to_path_buf(), exclusive: false, @@ -383,7 +382,7 @@ fn acquire_shared_at(path: &Path, operation: &str) -> Result { let mut file = open_lock_file(path)?; match file.try_lock_shared().map_err(std::io::Error::from) { Ok(()) => Ok(LifecycleLease { - hold: LeaseHold::File(file), + hold: LeaseHold::File(FileLease::held(file, LIFECYCLE_LEASE_LABEL)), token: None, lock_path: path.to_path_buf(), exclusive: false, @@ -400,7 +399,7 @@ fn try_acquire_shared_at(path: &Path, operation: &str) -> Result Ok(SharedLeaseAttempt::Acquired(LifecycleLease { - hold: LeaseHold::File(file), + hold: LeaseHold::File(FileLease::held(file, LIFECYCLE_LEASE_LABEL)), token: None, lock_path: path.to_path_buf(), exclusive: false, @@ -410,7 +409,8 @@ fn try_acquire_shared_at(path: &Path, operation: &str) -> Result Result { +fn own_exclusive(file: File, path: &Path, operation: &str) -> Result { + let mut file = FileLease::held(file, LIFECYCLE_LEASE_LABEL); let token = lease_token(); let pid = std::process::id(); #[cfg(not(windows))] diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry.rs index b218d83607..8871d21947 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry.rs @@ -46,9 +46,9 @@ pub use attachment::{ PhysicalRuntimeAttachment, PhysicalRuntimeSnapshot, PhysicalWriterRuntimeSnapshot, PublishedShardRuntime, }; -pub use capacity::StoreRuntimeRegistryConfig; #[cfg(test)] -pub(crate) use capacity::{DEFAULT_PROJECT_CODE_OPEN_RUNTIMES, MAX_PROJECT_CODE_OPEN_RUNTIMES}; +pub(crate) use capacity::MAX_PROJECT_CODE_OPEN_RUNTIMES; +pub use capacity::StoreRuntimeRegistryConfig; pub use close::ClosedStoreRuntime; pub use destructive::{DestructiveMaintenanceReservation, DestructiveMaintenanceTarget}; pub use graph::{ @@ -1173,24 +1173,6 @@ impl StoreRuntimeClientLease { } } -impl tracedecay_store::StorageRuntimeReadPort for StoreRuntimeClientLease { - fn dispatch_read<'a>( - &'a self, - request: tracedecay_store::RuntimeReadRequestV1, - probe: &'a dyn tracedecay_store::RuntimeRequestProbeV1, - ) -> tracedecay_store::StorageRuntimePortFutureV1<'a, tracedecay_store::RuntimeReadOutcomeV1> - { - Box::pin(async move { - StoreRuntimeClientLease::dispatch_read(self, request, probe).map_err(|_| { - tracedecay_store::StorageRuntimeErrorV1::Infrastructure { - operation: "dispatch registered runtime read".to_owned(), - } - .into() - }) - }) - } -} - impl fmt::Debug for StoreRuntimeClientLease { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { formatter diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/capacity.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/capacity.rs index 67ae5300e2..ef7801b1ce 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry/capacity.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/capacity.rs @@ -138,39 +138,56 @@ impl StoreRuntimeRegistry { let Some(candidate) = candidate else { return Ok(CapacityReservation::Exhausted); }; + Ok(Self::reserve_eviction(state, candidate)?.map_or( + CapacityReservation::Exhausted, + CapacityReservation::Eviction, + )) + } + + /// Fences one exact `Ready` entry as `Evicting`; `None` when the entry is + /// no longer `Ready`. + pub(super) fn reserve_eviction( + state: &mut RegistryState, + key: StoreRuntimeKey, + ) -> Result, StoreRuntimeRegistryFailure> { let Some(attempt) = state.next_eviction_attempt.checked_add(1) else { return Err(StoreRuntimeRegistryFailure::EvictionAttemptExhausted); }; state.next_eviction_attempt = attempt; - let Some(RegistryEntry::Ready(ready)) = state.entries.remove(&candidate) else { - return Ok(CapacityReservation::Exhausted); + let ready = match state.entries.remove(&key) { + Some(RegistryEntry::Ready(ready)) => ready, + Some(entry) => { + state.entries.insert(key, entry); + return Ok(None); + } + None => return Ok(None), }; if let Err(error) = ready .owner .runtime() .transition(RuntimeMaintenanceStateV1::Draining) { - state.entries.insert(candidate, RegistryEntry::Ready(ready)); + state.entries.insert(key, RegistryEntry::Ready(ready)); return Err(StoreRuntimeRegistryFailure::RuntimeLifecycleFailed { message: error.to_string(), }); } let owner = ready.owner; state.entries.insert( - candidate.clone(), + key.clone(), RegistryEntry::Evicting(EvictingRuntime { attempt, owner: owner.clone(), }), ); - Ok(CapacityReservation::Eviction(EvictionReservation { - key: candidate, + Ok(Some(EvictionReservation { + key, attempt, owner, })) } - pub(super) fn complete_project_code_eviction( + pub(super) fn complete_eviction( &self, reservation: EvictionReservation, ) -> Result<(), StoreRuntimeRegistryFailure> { @@ -214,6 +231,14 @@ impl StoreRuntimeRegistry { ); return outcome; } + if reservation.key.is_profile() + && state + .profile_authorities + .get(reservation.key.shard_id()) + .is_some_and(|binding| binding == reservation.owner.binding()) + { + state.profile_authorities.remove(reservation.key.shard_id()); + } drop(state); drop(evicting); hotpath::gauge!("runtime_core.registry.runtimes_ready").dec(1.0); diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/close.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/close.rs index 952051b72b..8d22e06052 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry/close.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/close.rs @@ -1,5 +1,7 @@ +use std::collections::BTreeSet; use std::path::Path; use std::sync::Arc; +use std::time::Duration; use tracedecay_store::{RuntimeMaintenanceStateV1, StoreRuntimeBindingV1, VerifiedStoreLocatorV1}; @@ -9,6 +11,7 @@ use super::{ StoreRuntimeRegistry, StoreRuntimeRegistryFailure, }; use crate::db::DatabaseAuthority; +use crate::shard_runtime::shard::ShardRuntimeEvictionBlocker; /// Proof that one exact runtime reached `Closed` after all physical `SQLite` /// handles joined and before its registry entry was removed. @@ -46,6 +49,77 @@ struct CloseReservation { } impl StoreRuntimeRegistry { + /// Closes every mounted runtime that no lease, queued work, profile pin, + /// or graph lease still holds, so each writer runs its shutdown TRUNCATE + /// checkpoint. The daemon process exits with this registry reachable, so + /// no destructor closes these attachments otherwise. Held runtimes stay + /// mounted and are logged with their blockers. Returns the number of + /// runtimes closed. + #[hotpath::measure(label = "runtime_core.registry.close_idle_for_shutdown", future = true)] + pub async fn close_idle_for_shutdown(&self) -> Result { + let (reservations, reserve_failure) = { + let mut state = self.lock_state(); + let mut idle = Vec::new(); + for (key, entry) in &state.entries { + let RegistryEntry::Ready(ready) = entry else { + continue; + }; + let mut blockers = ready + .owner + .runtime() + .eviction_eligibility(Duration::ZERO) + .blockers; + blockers.retain(|blocker| { + !matches!(blocker, ShardRuntimeEvictionBlocker::PinnedProfile) + }); + let profile_pins = state.profile_pin_tokens.get(key).map_or(0, BTreeSet::len); + let graph_leases = state + .graph_publications + .get(key) + .map_or(0, |retained| retained.lease_tokens.len()); + if blockers.is_empty() && profile_pins == 0 && graph_leases == 0 { + idle.push(key.clone()); + } else { + tracing::warn!( + path = %ready.owner.locator().path().display(), + ?blockers, + profile_pins, + graph_leases, + "store runtime still held at shutdown; its WAL is not truncated" + ); + } + } + let mut reservations = Vec::with_capacity(idle.len()); + let mut reserve_failure = None; + for key in idle { + match Self::reserve_eviction(&mut state, key) { + Ok(reservation) => reservations.extend(reservation), + Err(failure) => { + reserve_failure = Some(failure); + break; + } + } + } + (reservations, reserve_failure) + }; + let registry = self.clone(); + tokio::task::spawn_blocking(move || { + let closed = reservations.len(); + let mut first_failure = reserve_failure; + for reservation in reservations { + if let Err(failure) = registry.complete_eviction(reservation) { + first_failure.get_or_insert(failure); + } + } + first_failure.map_or(Ok(closed), Err) + }) + .await + .map_err(|error| StoreRuntimeRegistryFailure::PhysicalRuntimeFailed { + operation: "join shutdown close of idle registered runtimes", + message: error.to_string(), + })? + } + #[hotpath::measure(label = "runtime_core.registry.close_path")] pub async fn close_path( &self, diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/open.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/open.rs index d10eb0ca51..0f48bd1758 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry/open.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/open.rs @@ -305,7 +305,7 @@ impl StoreRuntimeRegistry { }; if let Some(eviction) = eviction - && let Err(failure) = self.complete_project_code_eviction(eviction) + && let Err(failure) = self.complete_eviction(eviction) { self.fail_reserved_open(&key, attempt, &updates, failure.clone()); return StoreRuntimeOpenBegin::Rejected(failure); diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/ports.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/ports.rs index dfc8483187..992a626a13 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry/ports.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/ports.rs @@ -157,20 +157,12 @@ async fn publish_lifecycle_runtime( } if request.mode == StoreRuntimeOpenMode::Existing && runtime_core_final_schema_applies(&request.binding.shard_id.scope) - { - if let Err(error) = - step_final_schema_before_existing_publication(&request, attachment.as_physical()).await - { - attachment.abort(request.locator.is_prospective()); - return Err(error); - } - if let Err(error) = + && let Err(error) = verify_final_schema_before_existing_publication(&request, attachment.as_physical()) .await - { - attachment.abort(request.locator.is_prospective()); - return Err(error); - } + { + attachment.abort(request.locator.is_prospective()); + return Err(error); } if let Err(error) = runtime.transition(RuntimeMaintenanceStateV1::Ready) { attachment.abort(request.locator.is_prospective()); @@ -400,36 +392,6 @@ async fn authorized_schema_connection( Ok(crate::db::engine::Connection::attach(handle)) } -/// Steps an existing runtime-core store that is exactly one sanctioned step -/// behind the final shape before admission verifies it. Only a request that -/// carries an active write authority may step; a read-only mount falls -/// through to the verifier, which names this writer-side remedy. -async fn step_final_schema_before_existing_publication( - request: &ShardRuntimeBuildRequest, - attachment: &dyn PhysicalRuntimeAttachment, -) -> Result<(), StoreRuntimeRegistryFailure> { - const OPERATION: &str = "step final schema for existing SQLite runtime"; - let Some(authority) = request.database_authority.as_ref() else { - return Ok(()); - }; - if authority.require_active_write_scope(OPERATION).is_err() { - return Ok(()); - } - let connection = authorized_schema_connection(request, attachment, OPERATION).await?; - crate::db::migrations::step_schema_if_pending(&connection) - .await - .map(|_stepped| ()) - .map_err(|error| match error { - tracedecay_domain::errors::TraceDecayError::ResetRequired { authority, reason } => { - StoreRuntimeRegistryFailure::ResetRequired { authority, reason } - } - error => StoreRuntimeRegistryFailure::PhysicalRuntimeFailed { - operation: OPERATION, - message: error.to_string(), - }, - }) -} - async fn install_final_schema_before_publication( request: &ShardRuntimeBuildRequest, attachment: &dyn PhysicalRuntimeAttachment, diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/tests.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/tests.rs index 370be02eba..fe2e4a7c3a 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry/tests.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/tests.rs @@ -13,12 +13,13 @@ use super::*; use support::*; #[test] -fn budget_defaults_to_four_caps_at_eight_and_rejects_zero() { +fn budget_accepts_the_cap_and_rejects_zero_or_above_cap() { assert_eq!( - StoreRuntimeRegistryConfig::default().project_code_open_runtime_budget(), - DEFAULT_PROJECT_CODE_OPEN_RUNTIMES + StoreRuntimeRegistryConfig::new(MAX_PROJECT_CODE_OPEN_RUNTIMES) + .unwrap() + .project_code_open_runtime_budget(), + MAX_PROJECT_CODE_OPEN_RUNTIMES ); - assert!(StoreRuntimeRegistryConfig::new(MAX_PROJECT_CODE_OPEN_RUNTIMES).is_ok()); for invalid in [0, MAX_PROJECT_CODE_OPEN_RUNTIMES + 1] { assert!(matches!( StoreRuntimeRegistryConfig::new(invalid), diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/tests/production_routes.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/tests/production_routes.rs index d36ebc7d7c..c6e95c2d4c 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry/tests/production_routes.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/tests/production_routes.rs @@ -16,7 +16,7 @@ use tracedecay_store::{ CodeShardScopeV1, ConsistencyModeV1, OperationPriorityV1, RuntimeCancellationIdV1, RuntimeCancellationIdentityV1, RuntimeDeadlineIdV1, RuntimeDeadlineV1, RuntimeReadOperationV1, RuntimeReadRequestV1, RuntimeReadResultV1, RuntimeRequestControlV1, RuntimeRequestProbeV1, - StorageRuntimeReadPort, StoreShardIdV1, StoreShardScopeV1, VerifiedStoreLocatorV1, + StoreShardIdV1, StoreShardScopeV1, VerifiedStoreLocatorV1, }; use super::super::*; @@ -216,8 +216,8 @@ async fn assert_health_route(handle: &StoreRuntimeClientLease, writer_expected: ); let (request, probe) = health_request(handle.binding()); - let outcome = StorageRuntimeReadPort::read(handle, request, &probe) - .await + let outcome = handle + .dispatch_read(request, &probe) .expect("health data port must be mounted"); assert!(matches!( outcome.value(), @@ -534,9 +534,9 @@ async fn distinct_logical_shards_cannot_publish_two_writers_for_one_database() { )); } -/// Rewinds a final-shape graph store to the pre-digest shape a v34 binary -/// left behind: the payload-digest objects are dropped and the stamp moved -/// back one step. +/// Rewinds a final-shape graph store to the pre-digest shape a released v34 +/// binary left behind: the payload-digest objects are dropped and the stamp +/// moved back one step. async fn rewind_to_pre_digest_shape(path: &Path) { let connection = crate::db::engine::TestConnection::open(path); connection @@ -558,8 +558,11 @@ fn user_version(path: &Path) -> u32 { .unwrap() } +/// A store at an older released schema is never upgraded in place, even by a +/// write-authorized open: admission fails with the typed fresh-start refusal +/// and leaves the stamp untouched. #[tokio::test] -async fn existing_store_one_step_behind_is_stepped_by_write_authorized_admission() { +async fn existing_store_at_an_older_schema_is_refused_even_with_write_authority() { let root = TempDir::new().unwrap(); let resolver = Arc::new(FileResolver::default()); resolver.push(seed_final_graph_db(&root, "profile.db").await); @@ -577,64 +580,34 @@ async fn existing_store_one_step_behind_is_stepped_by_write_authorized_admission other => panic!("profile was not pinned: {other:?}"), }; let authority = - crate::db::DatabaseAuthority::acquire_test(&project_path, "step an existing v34 store") + crate::db::DatabaseAuthority::acquire_test(&project_path, "open an older store") .expect("test database authority"); - let project = open_published( - ®istry, - StoreRuntimeOpenRequest::new_authorized( - project_shard("project.one-step-behind"), + let outcome = registry + .open(StoreRuntimeOpenRequest::new_authorized( + project_shard("project.older-schema"), incarnation(), Some(pin), authority, - ), - ) - .await; - - assert_health_route(&project, true).await; - assert_eq!( - user_version(&project_path), - crate::db::migrations::SCHEMA_VERSION, - "write-authorized admission must step the store to the final shape" - ); -} - -#[tokio::test] -async fn existing_store_one_step_behind_is_refused_without_write_authority() { - let root = TempDir::new().unwrap(); - let resolver = Arc::new(FileResolver::default()); - resolver.push(seed_final_graph_db(&root, "profile.db").await); - let project_path = seed_final_graph_db(&root, "project.db").await; - rewind_to_pre_digest_shape(&project_path).await; - resolver.push(project_path.clone()); - let registry = StoreRuntimeRegistry::new(resolver, Arc::new(LifecycleShardRuntimePublisher)); - let _profile = open_published( - ®istry, - StoreRuntimeOpenRequest::new(profile_shard(), incarnation(), None), - ) - .await; - let pin = match registry.profile_authority_pin(&profile_shard()) { - ProfileAuthorityPinResult::Pinned(pin) => pin, - other => panic!("profile was not pinned: {other:?}"), - }; - - let outcome = registry - .open(project_request("project.one-step-behind", &pin)) + )) .await; match outcome { - StoreRuntimeOpenResult::Failed(StoreRuntimeRegistryFailure::PhysicalRuntimeFailed { - message, - .. - }) => assert!( - message.contains("payload digest step is pending"), - "an unauthorized open must name the pending writer-side step: {message}" - ), - other => panic!("an open without write authority must not step the store: {other:?}"), + StoreRuntimeOpenResult::Failed(StoreRuntimeRegistryFailure::ResetRequired { + authority, + reason, + }) => { + assert_eq!(authority, "SQLite store"); + assert!( + reason.contains("cannot be upgraded in place"), + "the refusal must name the fresh-start remedy: {reason}" + ); + } + other => panic!("an older store must be refused, not upgraded: {other:?}"), } assert_eq!( user_version(&project_path), 34, - "an open without write authority must leave the stamp alone" + "a refused open must leave the stamp alone" ); } diff --git a/crates/tracedecay-runtime-core/src/sqlite_read_snapshot.rs b/crates/tracedecay-runtime-core/src/sqlite_read_snapshot.rs index 0ad4aa7831..b6d5c3d0fc 100644 --- a/crates/tracedecay-runtime-core/src/sqlite_read_snapshot.rs +++ b/crates/tracedecay-runtime-core/src/sqlite_read_snapshot.rs @@ -15,8 +15,11 @@ use rusqlite::backup::StepResult; use rusqlite::{Connection, OpenFlags}; use sha2::{Digest, Sha256}; use tracedecay_domain::canonical_text::encode_lowercase_hex; +use tracedecay_private_fs::FileLease; use tracedecay_private_fs::framed_log::rename_noreplace; +use crate::storage::retry_transient_file_op; + #[path = "sqlite_snapshot_connection.rs"] mod connection; #[path = "sqlite_snapshot_control.rs"] @@ -660,7 +663,7 @@ enum SnapshotSourcePolicy { struct ScratchDirectory { path: PathBuf, - owner_lock: Option, + owner_lock: Option, } impl Drop for ScratchDirectory { @@ -1102,6 +1105,7 @@ fn create_scratch_directory( ensure_private_root(root, expected_uid)?; let cleanup_lock = open_private_lock(&root.join(".cleanup.lock"), true)?; cleanup_lock.lock()?; + let cleanup_lock = FileLease::held(cleanup_lock, "sqlite_read_snapshot.cleanup"); cleanup_stale_directories(root)?; for _ in 0..100 { let id = NEXT_SNAPSHOT.fetch_add(1, Ordering::Relaxed); @@ -1110,7 +1114,8 @@ fn create_scratch_directory( Ok(()) => { let owner_lock = open_private_lock(&path.join(".owner.lock"), true)?; owner_lock.lock()?; - cleanup_lock.unlock()?; + let owner_lock = FileLease::held(owner_lock, "sqlite_read_snapshot.owner"); + cleanup_lock.release()?; return Ok(ScratchDirectory { path, owner_lock: Some(owner_lock), @@ -1312,27 +1317,33 @@ fn cleanup_stale_directories(root: &Path) -> io::Result<()> { if !name.to_string_lossy().starts_with("read-") { continue; } - let path = entry.path(); // An owner releases its directory without the cleanup lock, so an // entry listed above can be gone by now. Gone is the state this // sweep wants; only a failure to reach a present entry is an error. - match fs::symlink_metadata(&path) { - Ok(metadata) if metadata.is_dir() => {} - Ok(_) => continue, - Err(error) if error.kind() == io::ErrorKind::NotFound => continue, - Err(error) => return Err(error), - } - let removable = match open_private_lock(&path.join(".owner.lock"), false) { - Ok(lock) => lock.try_lock().map_err(std::io::Error::from).is_ok(), - Err(error) if error.kind() == io::ErrorKind::NotFound => true, + // Windows reports an entry mid-release as access denied until its + // last handle closes, which the transient-file retry waits out. + retry_transient_file_op(|| cleanup_stale_directory(&entry.path()))?; + } + Ok(()) +} + +fn cleanup_stale_directory(path: &Path) -> io::Result<()> { + match fs::symlink_metadata(path) { + Ok(metadata) if metadata.is_dir() => {} + Ok(_) => return Ok(()), + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(error), + } + let removable = match open_private_lock(&path.join(".owner.lock"), false) { + Ok(lock) => lock.try_lock().map_err(std::io::Error::from).is_ok(), + Err(error) if error.kind() == io::ErrorKind::NotFound => true, + Err(error) => return Err(error), + }; + if removable { + match fs::remove_dir_all(path) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => {} Err(error) => return Err(error), - }; - if removable { - match fs::remove_dir_all(&path) { - Ok(()) => {} - Err(error) if error.kind() == io::ErrorKind::NotFound => {} - Err(error) => return Err(error), - } } } Ok(()) diff --git a/crates/tracedecay-runtime-core/src/sqlite_read_snapshot_backup_tests.rs b/crates/tracedecay-runtime-core/src/sqlite_read_snapshot_backup_tests.rs index 2c4ed6eafb..8c1e45a6c4 100644 --- a/crates/tracedecay-runtime-core/src/sqlite_read_snapshot_backup_tests.rs +++ b/crates/tracedecay-runtime-core/src/sqlite_read_snapshot_backup_tests.rs @@ -813,12 +813,16 @@ async fn windows_live_wal_writer_survives_copy_mode_backup() { [], ) .unwrap(); - let illegal = temp.path().join("illegal-copy.db"); - let error = fs::copy(&source, &illegal).expect_err("copying a live Windows store must fail"); - assert!( - matches!(error.raw_os_error(), Some(32 | 33)), - "expected sharing/lock violation, got {error}" - ); + // SQLite shares its Windows handles for reading and locks only the byte + // range past its pending byte, so a raw copy of a small live store + // succeeds. It is still not a backup: it misses every WAL-resident row. + let raw = temp.path().join("raw-copy.db"); + fs::copy(&source, &raw).unwrap(); + let raw_ids: String = Connection::open(&raw) + .unwrap() + .query_row("SELECT group_concat(id) FROM durable", [], |row| row.get(0)) + .unwrap(); + assert_eq!(raw_ids, "0"); assert_eq!(snapshot_ids(&destination), [0, 1]); drop(writer); } diff --git a/crates/tracedecay-runtime-core/src/storage.rs b/crates/tracedecay-runtime-core/src/storage.rs index 1115caa3bc..cffaee5945 100644 --- a/crates/tracedecay-runtime-core/src/storage.rs +++ b/crates/tracedecay-runtime-core/src/storage.rs @@ -8,7 +8,6 @@ use serde::{Deserialize, Serialize}; use crate::config; -pub const ENROLLMENT_FILENAME: &str = "enrollment.json"; pub const STORE_MANIFEST_FILENAME: &str = "store_manifest.json"; pub const PROFILE_IDENTITY_FILENAME: &str = "profile-identity.json"; /// File name of the profile-scoped exclusive daemon-authority lock. Single @@ -153,7 +152,6 @@ pub fn has_sqlite_database_header(path: &Path) -> io::Result { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum StorageMode { - ProjectLocal, ProfileSharded, } @@ -191,21 +189,18 @@ pub struct StoreLayout { pub project_root: PathBuf, pub data_root: PathBuf, pub graph_db_path: PathBuf, - pub config_path: PathBuf, pub branch_meta_path: PathBuf, pub sessions_db_path: PathBuf, pub response_handle_root: PathBuf, pub lcm_payload_root: PathBuf, pub dashboard_root: PathBuf, pub manifest_path: Option, - pub dirty_path: PathBuf, pub sync_lock_path: PathBuf, pub branch_add_lock_path: PathBuf, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ProjectStorageStatus { - RepoLocal, ProfileSharded, ManifestReconstructable, Stale, @@ -222,7 +217,6 @@ pub struct ProjectStorageLocation { impl ProjectStorageStatus { pub fn label(self) -> &'static str { match self { - Self::RepoLocal => "repo-local", Self::ProfileSharded => "profile-sharded", Self::ManifestReconstructable => "manifest-reconstructable", Self::Stale => "stale", @@ -230,7 +224,7 @@ impl ProjectStorageStatus { } pub fn is_live(self) -> bool { - matches!(self, Self::RepoLocal | Self::ProfileSharded) + self == Self::ProfileSharded } } @@ -252,20 +246,17 @@ fn classify_layout_storage(project_root: &Path, layout: StoreLayout) -> ProjectS .manifest_path .as_ref() .is_some_and(|path| path.is_file()); - let status = match layout.storage_mode { - StorageMode::ProjectLocal if graph_exists => ProjectStorageStatus::RepoLocal, - StorageMode::ProfileSharded if graph_exists => ProjectStorageStatus::ProfileSharded, - StorageMode::ProfileSharded if manifest_exists => { - ProjectStorageStatus::ManifestReconstructable - } - _ => ProjectStorageStatus::Stale, + let status = if graph_exists { + ProjectStorageStatus::ProfileSharded + } else if manifest_exists { + ProjectStorageStatus::ManifestReconstructable + } else { + ProjectStorageStatus::Stale }; - let marker_root = (layout.storage_mode == StorageMode::ProfileSharded) - .then(|| project_root.join(config::TRACEDECAY_DIR)); ProjectStorageLocation { project_root: project_root.to_path_buf(), data_root: layout.data_root, - marker_root, + marker_root: Some(project_root.join(config::TRACEDECAY_DIR)), status, } } @@ -441,7 +432,6 @@ pub enum ProfileShardNonCanonicalReasonV1 { ManifestSchemaMismatch, ManifestProjectIdMismatch, ManifestStoreKindMismatch, - ManifestStorageModeMismatch, ManifestSessionsDbPathMismatch, ManifestDataRootUnavailable, ManifestDataRootMismatch, @@ -458,7 +448,6 @@ impl ProfileShardNonCanonicalReasonV1 { Self::ManifestSchemaMismatch => "manifest_schema_mismatch", Self::ManifestProjectIdMismatch => "manifest_project_id_mismatch", Self::ManifestStoreKindMismatch => "manifest_store_kind_mismatch", - Self::ManifestStorageModeMismatch => "manifest_storage_mode_mismatch", Self::ManifestSessionsDbPathMismatch => "manifest_sessions_db_path_mismatch", Self::ManifestDataRootUnavailable => "manifest_data_root_unavailable", Self::ManifestDataRootMismatch => "manifest_data_root_mismatch", @@ -476,7 +465,6 @@ pub struct PrivateStoreIo; mod identity; mod layout; -mod legacy_layouts; mod manifest; mod paths_and_io; mod profile_identity; @@ -484,8 +472,8 @@ mod profile_identity; #[cfg(any(test, feature = "test-helpers", feature = "test-transport"))] pub use identity::pin_fixture_repository_identity; pub use identity::{ - has_repository_identity_marker, legacy_enrollment_marker_path, read_legacy_enrollment_marker, - read_repository_identity_marker, repository_identity_path, write_repository_identity_marker, + has_repository_identity_marker, read_repository_identity_marker, repository_identity_path, + write_repository_identity_marker, }; pub(crate) use layout::has_path_local_profile_store; pub use layout::{ @@ -495,7 +483,6 @@ pub use layout::{ resolve_layout, resolve_layout_for_current_profile, resolve_lcm_payload_root, resolve_persisted_layout, resolve_project_session_db_path, resolve_response_handle_root, }; -pub use legacy_layouts::matching_legacy_profile_layouts; pub use manifest::{read_store_manifest, write_store_manifest, write_store_manifest_to_path}; pub use paths_and_io::{ acquire_sidecar_lock_blocking, append_lock_path, reject_symlink_components, @@ -509,7 +496,6 @@ pub use profile_identity::{ #[cfg(test)] use paths_and_io::open_lock_file; -use paths_and_io::validate_enrollment_marker; include!("storage/tests.rs"); include!("storage/identity_tests.rs"); diff --git a/crates/tracedecay-runtime-core/src/storage/identity.rs b/crates/tracedecay-runtime-core/src/storage/identity.rs index 0bd90097a0..b8665bcfa3 100644 --- a/crates/tracedecay-runtime-core/src/storage/identity.rs +++ b/crates/tracedecay-runtime-core/src/storage/identity.rs @@ -1,48 +1,13 @@ use std::fs; use std::path::{Path, PathBuf}; -use crate::config::TRACEDECAY_DIR; use tracedecay_domain::errors::{Result, TraceDecayError}; use super::{ - ENROLLMENT_FILENAME, EnrollmentMarker, PrivateStoreIo, REPOSITORY_IDENTITY_FILENAME, - REPOSITORY_IDENTITY_SCHEMA_VERSION, RepositoryIdentityMarker, validate_enrollment_marker, - validate_project_id, + PrivateStoreIo, REPOSITORY_IDENTITY_FILENAME, REPOSITORY_IDENTITY_SCHEMA_VERSION, + RepositoryIdentityMarker, validate_project_id, }; -/// Location of the retired repo-local enrollment marker. -/// -/// `TraceDecay` never creates files inside a project's working tree. This path -/// exists only so legacy identity can be adopted (read once, ingested into -/// the home-profile registry) and so cleanup flows can recognize the debris. -/// Users may delete the file at any time. -pub fn legacy_enrollment_marker_path(project_root: &Path) -> PathBuf { - project_root.join(TRACEDECAY_DIR).join(ENROLLMENT_FILENAME) -} - -/// Reads the retired repo-local enrollment marker, if the user still has one. -/// -/// Read-only legacy adoption source: registry-aware resolution ingests the -/// identity it names exactly once (when the project is not otherwise -/// resolvable) and never consults the file again. Nothing writes it. -pub fn read_legacy_enrollment_marker(project_root: &Path) -> Result> { - let path = legacy_enrollment_marker_path(project_root); - if !path.is_file() { - return Ok(None); - } - let text = fs::read_to_string(&path).map_err(|e| TraceDecayError::Config { - message: format!("failed to read enrollment marker '{}': {e}", path.display()), - })?; - let marker = serde_json::from_str(&text).map_err(|e| TraceDecayError::Config { - message: format!( - "failed to parse enrollment marker '{}': {e}", - path.display() - ), - })?; - validate_enrollment_marker(&marker, &path)?; - Ok(Some(marker)) -} - /// The repository-wide identity marker shared by every checkout of a /// repository, including detached linked worktrees. /// diff --git a/crates/tracedecay-runtime-core/src/storage/layout.rs b/crates/tracedecay-runtime-core/src/storage/layout.rs index fcccfe400a..06a071327d 100644 --- a/crates/tracedecay-runtime-core/src/storage/layout.rs +++ b/crates/tracedecay-runtime-core/src/storage/layout.rs @@ -136,15 +136,6 @@ pub fn profile_sharded_layout( profile_root: &Path, marker: &EnrollmentMarker, ) -> Result { - if marker.storage_mode != StorageMode::ProfileSharded { - return Err(TraceDecayError::Config { - message: format!( - "enrollment marker for '{}' uses storage_mode={:?}, not profile_sharded", - project_root.display(), - marker.storage_mode - ), - }); - } validate_project_id(&marker.project_id).map_err(|message| TraceDecayError::Config { message: format!( "invalid enrollment marker for '{}': {message}", diff --git a/crates/tracedecay-runtime-core/src/storage/legacy_layouts.rs b/crates/tracedecay-runtime-core/src/storage/legacy_layouts.rs deleted file mode 100644 index 79f4ad860f..0000000000 --- a/crates/tracedecay-runtime-core/src/storage/legacy_layouts.rs +++ /dev/null @@ -1,199 +0,0 @@ -use std::collections::HashMap; -use std::fs; -use std::path::{Path, PathBuf}; - -use crate::git_discovery::{ - GitDiscoveryUnknown, GitRepositoryIdentityOutcome, discover_repository_identity_cli_first, -}; -use crate::worktree; -use tracedecay_domain::errors::{Result, TraceDecayError}; - -use super::{ - EnrollmentMarker, STORE_MANIFEST_FILENAME, STORE_MANIFEST_SCHEMA_VERSION, StorageMode, - StoreKind, StoreLayout, profile_sharded_layout, read_store_manifest, validate_project_id, -}; - -/// Finds pre-repository-identity profile stores that were keyed by an older -/// path-derived project id but still name this exact local checkout, or one of -/// its linked worktrees, in their manifest. Remote URLs are deliberately not -/// considered: two clones of one remote are different local identities. -pub fn matching_legacy_profile_layouts( - project_root: &Path, - profile_root: &Path, - excluded_project_id: Option<&str>, -) -> Result<(Vec, bool, bool)> { - matching_legacy_profile_layouts_with_git_identity_resolver( - project_root, - profile_root, - excluded_project_id, - worktree::is_detached_linked_worktree, - discover_repository_identity_cli_first, - ) -} - -fn matching_legacy_profile_layouts_with_git_identity_resolver( - project_root: &Path, - profile_root: &Path, - excluded_project_id: Option<&str>, - mut is_detached_linked_worktree: D, - mut git_identity: G, -) -> Result<(Vec, bool, bool)> -where - D: FnMut(&Path) -> bool, - G: FnMut(&Path) -> GitRepositoryIdentityOutcome, -{ - let projects_root = profile_root.join("projects"); - let Ok(entries) = fs::read_dir(&projects_root) else { - return Ok((Vec::new(), false, false)); - }; - let mut manifest_paths = entries - .flatten() - .map(|entry| entry.path().join(STORE_MANIFEST_FILENAME)) - .filter(|path| path.is_file()) - .collect::>(); - manifest_paths.sort(); - - let mut exact_manifests = Vec::new(); - let mut non_exact_manifests = Vec::new(); - let mut selected_manifest_matches_exact_root = false; - for manifest_path in manifest_paths { - let Ok(manifest) = read_store_manifest(&manifest_path) else { - continue; - }; - let exact_root = same_local_path(&manifest.project_root, project_root); - if manifest.project_id.is_some() && manifest.project_id.as_deref() == excluded_project_id { - selected_manifest_matches_exact_root |= exact_root; - continue; - } - if exact_root { - exact_manifests.push((manifest_path, manifest)); - continue; - } - non_exact_manifests.push((manifest_path, manifest)); - } - - let candidates_match_exact_root = !exact_manifests.is_empty(); - let matching_manifests = if exact_manifests.is_empty() { - let project_git_common_dir = if is_detached_linked_worktree(project_root) { - None - } else { - match git_identity(project_root) { - GitRepositoryIdentityOutcome::Resolved(identity) => Some(identity.common_dir), - GitRepositoryIdentityOutcome::NotRepository => None, - GitRepositoryIdentityOutcome::Unknown(reason) => { - return Err(unknown_git_identity(project_root, reason)); - } - } - }; - let mut legacy_git_common_dirs = HashMap::>::new(); - non_exact_manifests - .into_iter() - .filter(|(_, manifest)| { - project_git_common_dir.as_deref().is_some_and(|current| { - legacy_git_common_dirs - .entry(manifest.project_root.clone()) - .or_insert_with(|| { - manifest - .project_root - .is_dir() - .then(|| match git_identity(&manifest.project_root) { - GitRepositoryIdentityOutcome::Resolved(identity) => { - Some(identity.common_dir) - } - GitRepositoryIdentityOutcome::NotRepository => None, - // Skip an unreadable sibling rather than - // adopting it. The current checkout's - // Unknown already failed closed above. - GitRepositoryIdentityOutcome::Unknown(_) => None, - }) - .flatten() - }) - .as_deref() - .is_some_and(|legacy| same_local_path(legacy, current)) - }) - }) - .collect() - } else { - exact_manifests - }; - let mut layouts = Vec::new(); - for (manifest_path, manifest) in matching_manifests { - let project_id = manifest - .project_id - .as_deref() - .ok_or_else(|| invalid_legacy_manifest(&manifest_path, "project_id is missing"))?; - validate_project_id(project_id) - .map_err(|message| invalid_legacy_manifest(&manifest_path, message))?; - if manifest.schema_version != STORE_MANIFEST_SCHEMA_VERSION - || manifest.store_kind != StoreKind::CodeProject - || manifest.storage_mode != StorageMode::ProfileSharded - { - return Err(invalid_legacy_manifest( - &manifest_path, - "unsupported schema, store kind, or storage mode", - )); - } - - let layout = profile_sharded_layout( - project_root, - profile_root, - &EnrollmentMarker { - project_id: project_id.to_string(), - storage_mode: StorageMode::ProfileSharded, - }, - )?; - let manifest_data_root = manifest - .data_root - .canonicalize() - .unwrap_or_else(|_| manifest.data_root.clone()); - let layout_data_root = layout - .data_root - .canonicalize() - .unwrap_or_else(|_| layout.data_root.clone()); - if manifest_path.parent() != Some(manifest.data_root.as_path()) - || manifest_data_root != layout_data_root - || manifest.data_root.join(&manifest.graph_db_relpath) != layout.graph_db_path - || manifest.data_root.join(&manifest.sessions_db_relpath) != layout.sessions_db_path - || manifest.data_root.join(&manifest.branch_meta_relpath) != layout.branch_meta_path - { - return Err(invalid_legacy_manifest( - &manifest_path, - "manifest paths do not match the profile shard layout", - )); - } - layouts.push(layout); - } - Ok(( - layouts, - selected_manifest_matches_exact_root, - candidates_match_exact_root, - )) -} - -fn same_local_path(left: &Path, right: &Path) -> bool { - if left == right { - return true; - } - match (left.canonicalize(), right.canonicalize()) { - (Ok(left), Ok(right)) => left == right, - _ => false, - } -} - -fn unknown_git_identity(path: &Path, reason: GitDiscoveryUnknown) -> TraceDecayError { - TraceDecayError::Config { - message: format!( - "cannot adopt a legacy profile store for '{}': git repository identity is unknown ({reason})", - path.display() - ), - } -} - -fn invalid_legacy_manifest(path: &Path, detail: impl std::fmt::Display) -> TraceDecayError { - TraceDecayError::Config { - message: format!( - "legacy profile store manifest '{}' cannot be adopted safely: {detail}", - path.display() - ), - } -} diff --git a/crates/tracedecay-runtime-core/src/storage/manifest.rs b/crates/tracedecay-runtime-core/src/storage/manifest.rs index 90ac50e682..bddadb2c33 100644 --- a/crates/tracedecay-runtime-core/src/storage/manifest.rs +++ b/crates/tracedecay-runtime-core/src/storage/manifest.rs @@ -5,8 +5,8 @@ use tracedecay_domain::errors::{Result, TraceDecayError}; use super::{ PrivateStoreIo, ProfileShardValidationError, SESSIONS_DB_FILENAME, STORE_MANIFEST_FILENAME, - STORE_MANIFEST_SCHEMA_VERSION, StorageMode, StoreKind, StoreLayout, StoreManifest, - ValidatedProfileShard, has_sqlite_database_header, profile_sharded_data_root, + STORE_MANIFEST_SCHEMA_VERSION, StoreKind, StoreLayout, StoreManifest, ValidatedProfileShard, + has_sqlite_database_header, profile_sharded_data_root, }; pub fn write_store_manifest(layout: &StoreLayout) -> Result { @@ -179,9 +179,6 @@ fn validate_profile_shard_manifest( if manifest.store_kind != StoreKind::CodeProject { return Err(invalid(Reason::ManifestStoreKindMismatch)); } - if manifest.storage_mode != StorageMode::ProfileSharded { - return Err(invalid(Reason::ManifestStorageModeMismatch)); - } if manifest.sessions_db_relpath != Path::new(SESSIONS_DB_FILENAME) { return Err(invalid(Reason::ManifestSessionsDbPathMismatch)); } diff --git a/crates/tracedecay-runtime-core/src/storage/paths_and_io.rs b/crates/tracedecay-runtime-core/src/storage/paths_and_io.rs index 77f46b52e1..992fd64329 100644 --- a/crates/tracedecay-runtime-core/src/storage/paths_and_io.rs +++ b/crates/tracedecay-runtime-core/src/storage/paths_and_io.rs @@ -3,6 +3,7 @@ use std::fs; use std::io::{self, Write}; use std::path::{Component, Path, PathBuf}; +use tracedecay_private_fs::FileLease; use tracedecay_private_fs::framed_log::{DirectorySyncPolicy, set_owner_private_file_mode}; use crate::config; @@ -11,10 +12,10 @@ use tracedecay_domain::errors::{Result, TraceDecayError}; #[cfg(windows)] use super::DURABLE_REMOVAL_TOMBSTONE_PREFIX; use super::{ - ActiveProjectContext, BRANCH_META_FILENAME, DurableAtomicWritePhase, EnrollmentMarker, - GraphScopeId, PrivateStoreIo, ProjectIdentity, ProjectPath, QueryTarget, - RESPONSE_HANDLES_DIRECTORY, SESSIONS_DB_FILENAME, STORE_MANIFEST_SCHEMA_VERSION, StorageMode, - StoreArtifactPath, StoreKind, StoreLayout, StoreManifest, inject_durable_atomic_write_fault, + ActiveProjectContext, BRANCH_META_FILENAME, DurableAtomicWritePhase, GraphScopeId, + PrivateStoreIo, ProjectIdentity, ProjectPath, QueryTarget, RESPONSE_HANDLES_DIRECTORY, + SESSIONS_DB_FILENAME, STORE_MANIFEST_SCHEMA_VERSION, StorageMode, StoreArtifactPath, StoreKind, + StoreLayout, StoreManifest, inject_durable_atomic_write_fault, inject_durable_namespace_sync_fault, }; @@ -749,6 +750,8 @@ pub fn append_lock_path(path: &Path) -> PathBuf { // data region being written. This rationale lives here once; call sites point // back to it rather than restating it. +const SIDECAR_LEASE_LABEL: &str = "storage.sidecar"; + pub(super) fn open_lock_file(lock_path: &Path, private: bool) -> io::Result { if let Some(parent) = lock_path.parent() { fs::create_dir_all(parent)?; @@ -776,10 +779,10 @@ pub(super) fn open_lock_file(lock_path: &Path, private: bool) -> io::Result io::Result> { +pub fn try_acquire_sidecar_lock(lock_path: &Path) -> io::Result> { let file = open_lock_file(lock_path, false)?; match file.try_lock().map_err(std::io::Error::from) { - Ok(()) => Ok(Some(file)), + Ok(()) => Ok(Some(FileLease::held(file, SIDECAR_LEASE_LABEL))), // `is_lock_contended` covers Windows, where contention surfaces as // ERROR_LOCK_VIOLATION rather than a `WouldBlock` error kind. Err(err) if crate::db::is_lock_contended(&err) => Ok(None), @@ -790,14 +793,14 @@ pub fn try_acquire_sidecar_lock(lock_path: &Path) -> io::Result /// Blocking sidecar lock acquisition. Returns the held lock file once the /// exclusive lock is granted. See the sidecar-lock module note above for the /// read+write-handle rationale. -pub fn acquire_sidecar_lock_blocking(lock_path: &Path) -> io::Result { +pub fn acquire_sidecar_lock_blocking(lock_path: &Path) -> io::Result { acquire_lock_file_blocking(lock_path, false) } -fn acquire_lock_file_blocking(lock_path: &Path, private: bool) -> io::Result { +fn acquire_lock_file_blocking(lock_path: &Path, private: bool) -> io::Result { let file = open_lock_file(lock_path, private)?; file.lock()?; - Ok(file) + Ok(FileLease::held(file, SIDECAR_LEASE_LABEL)) } /// Appends `line` (newline-terminated) to `path` under the shared sidecar @@ -815,7 +818,7 @@ pub(crate) fn append_line_locked(path: &Path, line: &str, private: bool) -> io:: } else { append_line_plain(path, line) }; - let unlock_result = lock_file.unlock(); + let unlock_result = lock_file.release(); write_result?; unlock_result?; if private { @@ -894,14 +897,12 @@ impl StoreLayout { manifest_filename: Option<&str>, ) -> Self { let graph_db_path = data_root.join(config::db_filename(&data_root)); - let config_path = data_root.join("config.json"); let branch_meta_path = data_root.join(BRANCH_META_FILENAME); let sessions_db_path = data_root.join(SESSIONS_DB_FILENAME); let response_handle_root = data_root.join(RESPONSE_HANDLES_DIRECTORY); let lcm_payload_root = data_root.join("lcm-payloads"); let dashboard_root = data_root.join("dashboard"); let manifest_path = manifest_filename.map(|filename| data_root.join(filename)); - let dirty_path = data_root.join("dirty"); let sync_lock_path = data_root.join("sync.lock"); let branch_add_lock_path = data_root.join(".branch-add.lock"); Self { @@ -911,26 +912,18 @@ impl StoreLayout { project_root, data_root, graph_db_path, - config_path, branch_meta_path, sessions_db_path, response_handle_root, lcm_payload_root, dashboard_root, manifest_path, - dirty_path, sync_lock_path, branch_add_lock_path, } } } -pub(super) fn validate_enrollment_marker(marker: &EnrollmentMarker, path: &Path) -> Result<()> { - validate_project_id(&marker.project_id).map_err(|message| TraceDecayError::Config { - message: format!("invalid enrollment marker '{}': {message}", path.display()), - }) -} - pub fn validate_project_id(project_id: &str) -> std::result::Result<(), &'static str> { if project_id.is_empty() { return Err("project_id must not be empty"); diff --git a/crates/tracedecay-runtime-core/tests/fixtures/project-store-released-v34.sql b/crates/tracedecay-runtime-core/tests/fixtures/project-store-released-v34.sql deleted file mode 100644 index 9977eddd8d..0000000000 --- a/crates/tracedecay-runtime-core/tests/fixtures/project-store-released-v34.sql +++ /dev/null @@ -1,1582 +0,0 @@ --- The canonical project store (`tracedecay.db`) exactly as every release from --- v0.1.0-beta.25 through v0.1.0-beta.37 created it. --- --- Assembled verbatim from the tagged DDL constants that --- `migrations::create_schema` and `final_shape::build_expected_final_shape` --- compose, so this is the shape a shipped binary wrote rather than a shape --- derived from the current contract. Deriving the released shape from the --- current contract is what let two admission regressions ship green. --- --- tag user_version objects inventory digest --- v0.1.0-beta.25 .. v0.1.0-beta.37 34 183 0126b4dd550109a6 --- (working tree) 35 190 ebae4fc200dd4e9f --- --- The digest is sha256 over `name|sql` lines of `sqlite_master`. All 12 tags --- produce one byte-identical inventory; the objects that differ from the --- current contract are enumerated in the admission table this fixture's test --- carries. A store loading this file must also carry `PRAGMA user_version = --- 34`, which the test sets, because that stamp is what selects the --- released admission path. - -CREATE TABLE IF NOT EXISTS metadata ( - key TEXT PRIMARY KEY, - value TEXT NOT NULL - ); - - CREATE TABLE IF NOT EXISTS read_cache ( - project_id TEXT NOT NULL, - session_id TEXT NOT NULL, - file_path TEXT NOT NULL, - mtime_ns INTEGER NOT NULL, - mode TEXT NOT NULL, - args_hash TEXT NOT NULL, - digest TEXT NOT NULL, - body BLOB NOT NULL, - token_count INTEGER NOT NULL, - created_at INTEGER NOT NULL, - PRIMARY KEY (project_id, session_id, file_path, mode, args_hash) - ); - - CREATE INDEX IF NOT EXISTS idx_read_cache_session - ON read_cache(session_id, created_at); - -CREATE TABLE IF NOT EXISTS retrieval_anchors ( - anchor_id TEXT PRIMARY KEY CHECK(length(anchor_id) > 0), - anchor_json TEXT NOT NULL CHECK(json_valid(anchor_json)), - owner_json TEXT NOT NULL CHECK(json_valid(owner_json)), - projection_generation TEXT NOT NULL CHECK(length(projection_generation) > 0) - ); - -- SQLite requires an exact unique parent key for the composite owner-bound - -- alias and evidence foreign keys, even though anchor_id is itself unique. - CREATE UNIQUE INDEX IF NOT EXISTS idx_retrieval_anchors_owner - ON retrieval_anchors(anchor_id, owner_json); - -CREATE TABLE IF NOT EXISTS retrieval_anchor_aliases ( - owner_json TEXT NOT NULL CHECK(json_valid(owner_json)), - alias_kind TEXT NOT NULL CHECK(length(alias_kind) > 0), - locator_digest TEXT NOT NULL CHECK(length(locator_digest) > 0), - anchor_id TEXT NOT NULL, - PRIMARY KEY(owner_json, alias_kind, locator_digest), - UNIQUE(anchor_id, alias_kind, locator_digest), - FOREIGN KEY(anchor_id, owner_json) - REFERENCES retrieval_anchors(anchor_id, owner_json) - ); - -CREATE TABLE IF NOT EXISTS retrieval_anchor_dispositions ( - sequence INTEGER PRIMARY KEY AUTOINCREMENT, - disposition_id TEXT NOT NULL CHECK(length(disposition_id) > 0), - anchor_id TEXT NOT NULL, - owner_json TEXT NOT NULL CHECK(json_valid(owner_json)), - state TEXT NOT NULL - CHECK(state IN ( - 'active', 'superseded', 'redacted', 'expired', 'quarantined', - 'deleted', 'unavailable' - )), - superseded_by TEXT, - reason_class TEXT NOT NULL CHECK(reason_class IN ( - 'user_request', 'retention', 'redaction', 'quarantine', - 'correction', 'legal_hold', 'source_unavailable' - )), - effective_at INTEGER NOT NULL, - record_json TEXT NOT NULL CHECK(json_valid(record_json)), - UNIQUE(owner_json, disposition_id), - FOREIGN KEY(anchor_id, owner_json) - REFERENCES retrieval_anchors(anchor_id, owner_json), - FOREIGN KEY(superseded_by, owner_json) - REFERENCES retrieval_anchors(anchor_id, owner_json), - CHECK( - (state = 'superseded' AND superseded_by IS NOT NULL) - OR (state <> 'superseded' AND superseded_by IS NULL) - ) - ); - CREATE INDEX IF NOT EXISTS idx_retrieval_anchor_dispositions_current - ON retrieval_anchor_dispositions(anchor_id, owner_json, sequence DESC); - - CREATE TABLE IF NOT EXISTS retrieval_anchor_reverse_lineage ( - source_anchor_id TEXT NOT NULL, - owner_json TEXT NOT NULL CHECK(json_valid(owner_json)), - derivative_kind TEXT NOT NULL - CHECK(derivative_kind IN ('span', 'contribution', 'finding')), - derivative_id TEXT NOT NULL CHECK(length(derivative_id) > 0), - direct_evidence INTEGER NOT NULL CHECK(direct_evidence IN (0, 1)), - PRIMARY KEY( - source_anchor_id, owner_json, derivative_kind, derivative_id - ), - FOREIGN KEY(source_anchor_id, owner_json) - REFERENCES retrieval_anchors(anchor_id, owner_json) - ); - CREATE INDEX IF NOT EXISTS idx_retrieval_anchor_reverse_derivative - ON retrieval_anchor_reverse_lineage( - owner_json, derivative_kind, derivative_id, direct_evidence - ); - - CREATE TABLE IF NOT EXISTS retrieval_anchor_derivative_tombstones ( - source_anchor_id TEXT NOT NULL, - owner_json TEXT NOT NULL CHECK(json_valid(owner_json)), - derivative_kind TEXT NOT NULL - CHECK(derivative_kind IN ('span', 'contribution', 'finding')), - derivative_id TEXT NOT NULL CHECK(length(derivative_id) > 0), - disposition_id TEXT NOT NULL, - effective_at INTEGER NOT NULL, - PRIMARY KEY( - source_anchor_id, owner_json, derivative_kind, derivative_id, - disposition_id - ), - FOREIGN KEY( - source_anchor_id, owner_json, derivative_kind, derivative_id - ) REFERENCES retrieval_anchor_reverse_lineage( - source_anchor_id, owner_json, derivative_kind, derivative_id - ) - ); - -CREATE TRIGGER IF NOT EXISTS retrieval_anchors_immutable_update - BEFORE UPDATE ON retrieval_anchors BEGIN - SELECT RAISE(ABORT, 'retrieval anchors are immutable'); - END; - CREATE TRIGGER IF NOT EXISTS retrieval_anchors_immutable_delete - BEFORE DELETE ON retrieval_anchors BEGIN - SELECT RAISE(ABORT, 'retrieval anchors are immutable'); - END; - CREATE TRIGGER IF NOT EXISTS retrieval_anchor_aliases_immutable_update - BEFORE UPDATE ON retrieval_anchor_aliases BEGIN - SELECT RAISE(ABORT, 'retrieval anchor aliases are immutable'); - END; - CREATE TRIGGER IF NOT EXISTS retrieval_anchor_aliases_immutable_delete - BEFORE DELETE ON retrieval_anchor_aliases BEGIN - SELECT RAISE(ABORT, 'retrieval anchor aliases are immutable'); - END; - CREATE TRIGGER IF NOT EXISTS retrieval_anchor_dispositions_immutable_update - BEFORE UPDATE ON retrieval_anchor_dispositions BEGIN - SELECT RAISE(ABORT, 'retrieval anchor dispositions are immutable'); - END; - CREATE TRIGGER IF NOT EXISTS retrieval_anchor_dispositions_immutable_delete - BEFORE DELETE ON retrieval_anchor_dispositions BEGIN - SELECT RAISE(ABORT, 'retrieval anchor dispositions are immutable'); - END; - CREATE TRIGGER IF NOT EXISTS retrieval_anchor_reverse_lineage_immutable_update - BEFORE UPDATE ON retrieval_anchor_reverse_lineage BEGIN - SELECT RAISE(ABORT, 'retrieval anchor reverse lineage is immutable'); - END; - CREATE TRIGGER IF NOT EXISTS retrieval_anchor_reverse_lineage_immutable_delete - BEFORE DELETE ON retrieval_anchor_reverse_lineage BEGIN - SELECT RAISE(ABORT, 'retrieval anchor reverse lineage is immutable'); - END; - CREATE TRIGGER IF NOT EXISTS retrieval_anchor_derivative_tombstones_immutable_update - BEFORE UPDATE ON retrieval_anchor_derivative_tombstones BEGIN - SELECT RAISE(ABORT, 'retrieval anchor derivative tombstones are immutable'); - END; - CREATE TRIGGER IF NOT EXISTS retrieval_anchor_derivative_tombstones_immutable_delete - BEFORE DELETE ON retrieval_anchor_derivative_tombstones BEGIN - SELECT RAISE(ABORT, 'retrieval anchor derivative tombstones are immutable'); - END; - -CREATE TABLE IF NOT EXISTS memory_v2_facts ( - fact_id TEXT NOT NULL, - owner_kind TEXT NOT NULL CHECK(owner_kind IN ('profile', 'project')), - project_id TEXT NOT NULL, - owner_json TEXT NOT NULL CHECK(json_valid(owner_json)), - identity_json TEXT NOT NULL CHECK(json_valid(identity_json)), - created_at INTEGER NOT NULL, - PRIMARY KEY(fact_id, owner_kind, project_id), - UNIQUE(fact_id, owner_json), - CHECK( - (owner_kind = 'profile' AND project_id = '') OR - (owner_kind = 'project' AND project_id <> '') - ) - ); - - CREATE TABLE IF NOT EXISTS memory_v2_assertions ( - assertion_id TEXT NOT NULL, - fact_id TEXT NOT NULL, - owner_kind TEXT NOT NULL, - project_id TEXT NOT NULL, - owner_json TEXT NOT NULL CHECK(json_valid(owner_json)), - assertion_header_json TEXT NOT NULL CHECK(json_valid(assertion_header_json)), - kind_json TEXT NOT NULL CHECK(json_valid(kind_json)), - payload_reference_json TEXT NOT NULL CHECK(json_valid(payload_reference_json)), - receipt_json TEXT NOT NULL CHECK(json_valid(receipt_json)), - asserted_at INTEGER NOT NULL, - actor_id TEXT, - PRIMARY KEY(assertion_id, fact_id, owner_kind, project_id), - UNIQUE(assertion_id, owner_json), - FOREIGN KEY(fact_id, owner_kind, project_id) - REFERENCES memory_v2_facts(fact_id, owner_kind, project_id) - ); - - CREATE TABLE IF NOT EXISTS memory_v2_assertion_supersession ( - assertion_id TEXT NOT NULL, - fact_id TEXT NOT NULL, - owner_kind TEXT NOT NULL, - project_id TEXT NOT NULL, - superseded_assertion_id TEXT NOT NULL, - ordinal INTEGER NOT NULL CHECK(ordinal >= 0), - PRIMARY KEY(assertion_id, fact_id, owner_kind, project_id, ordinal), - UNIQUE(assertion_id, fact_id, owner_kind, project_id, superseded_assertion_id), - FOREIGN KEY(assertion_id, fact_id, owner_kind, project_id) - REFERENCES memory_v2_assertions(assertion_id, fact_id, owner_kind, project_id), - FOREIGN KEY(superseded_assertion_id, fact_id, owner_kind, project_id) - REFERENCES memory_v2_assertions(assertion_id, fact_id, owner_kind, project_id) - ); - - CREATE TABLE IF NOT EXISTS memory_v2_assertion_payloads ( - rowid INTEGER PRIMARY KEY AUTOINCREMENT, - assertion_id TEXT NOT NULL, - fact_id TEXT NOT NULL, - owner_kind TEXT NOT NULL, - project_id TEXT NOT NULL, - payload_json TEXT NOT NULL CHECK(json_valid(payload_json)), - content TEXT NOT NULL, - UNIQUE(assertion_id, fact_id, owner_kind, project_id), - FOREIGN KEY(assertion_id, fact_id, owner_kind, project_id) - REFERENCES memory_v2_assertions(assertion_id, fact_id, owner_kind, project_id) - ); - - CREATE VIRTUAL TABLE IF NOT EXISTS memory_v2_assertion_payloads_fts USING fts5( - content, - content='memory_v2_assertion_payloads', - content_rowid='rowid' - ); - CREATE TRIGGER IF NOT EXISTS memory_v2_payloads_fts_insert - AFTER INSERT ON memory_v2_assertion_payloads BEGIN - INSERT INTO memory_v2_assertion_payloads_fts(rowid, content) - VALUES(NEW.rowid, NEW.content); - END; - CREATE TRIGGER IF NOT EXISTS memory_v2_payloads_fts_delete - AFTER DELETE ON memory_v2_assertion_payloads BEGIN - INSERT INTO memory_v2_assertion_payloads_fts( - memory_v2_assertion_payloads_fts, rowid, content - ) VALUES('delete', OLD.rowid, OLD.content); - END; - CREATE TRIGGER IF NOT EXISTS memory_v2_payloads_no_update - BEFORE UPDATE ON memory_v2_assertion_payloads BEGIN - SELECT RAISE(ABORT, 'memory_v2 assertion payloads are immutable'); - END; - - CREATE TABLE IF NOT EXISTS memory_v2_assertion_payload_purges ( - assertion_id TEXT NOT NULL, - fact_id TEXT NOT NULL, - owner_kind TEXT NOT NULL, - project_id TEXT NOT NULL, - payload_reference_json TEXT NOT NULL CHECK(json_valid(payload_reference_json)), - detector_revision TEXT NOT NULL CHECK(length(detector_revision) > 0), - purge_reason TEXT NOT NULL CHECK(purge_reason = 'detector_flagged'), - PRIMARY KEY(assertion_id, fact_id, owner_kind, project_id), - FOREIGN KEY(assertion_id, fact_id, owner_kind, project_id) - REFERENCES memory_v2_assertions(assertion_id, fact_id, owner_kind, project_id) - ); - CREATE TRIGGER IF NOT EXISTS memory_v2_assertion_payload_purges_no_update - BEFORE UPDATE ON memory_v2_assertion_payload_purges BEGIN - SELECT RAISE(ABORT, 'memory_v2 assertion payload purge receipts are immutable'); - END; - CREATE TRIGGER IF NOT EXISTS memory_v2_assertion_payload_purges_no_delete - BEFORE DELETE ON memory_v2_assertion_payload_purges BEGIN - SELECT RAISE(ABORT, 'memory_v2 assertion payload purge receipts are immutable'); - END; - - CREATE TABLE IF NOT EXISTS memory_v2_evidence ( - evidence_id TEXT NOT NULL, - fact_id TEXT NOT NULL, - owner_kind TEXT NOT NULL, - project_id TEXT NOT NULL, - owner_json TEXT NOT NULL CHECK(json_valid(owner_json)), - anchor_id TEXT NOT NULL, - evidence_json TEXT NOT NULL CHECK(json_valid(evidence_json)), - PRIMARY KEY(evidence_id, fact_id, owner_kind, project_id), - FOREIGN KEY(fact_id, owner_kind, project_id) - REFERENCES memory_v2_facts(fact_id, owner_kind, project_id), - FOREIGN KEY(anchor_id, owner_json) - REFERENCES retrieval_anchors(anchor_id, owner_json) - ); - - CREATE TABLE IF NOT EXISTS memory_v2_assertion_evidence ( - assertion_id TEXT NOT NULL, - evidence_id TEXT NOT NULL, - fact_id TEXT NOT NULL, - owner_kind TEXT NOT NULL, - project_id TEXT NOT NULL, - ordinal INTEGER NOT NULL CHECK(ordinal >= 0), - PRIMARY KEY(assertion_id, fact_id, owner_kind, project_id, ordinal), - UNIQUE(assertion_id, fact_id, owner_kind, project_id, evidence_id), - FOREIGN KEY(assertion_id, fact_id, owner_kind, project_id) - REFERENCES memory_v2_assertions(assertion_id, fact_id, owner_kind, project_id), - FOREIGN KEY(evidence_id, fact_id, owner_kind, project_id) - REFERENCES memory_v2_evidence(evidence_id, fact_id, owner_kind, project_id) - ); - - CREATE TABLE IF NOT EXISTS memory_v2_lineage_events ( - event_sequence INTEGER PRIMARY KEY AUTOINCREMENT, - event_id TEXT NOT NULL, - fact_id TEXT NOT NULL, - owner_kind TEXT NOT NULL, - project_id TEXT NOT NULL, - event_json TEXT NOT NULL CHECK(json_valid(event_json)), - occurred_at INTEGER NOT NULL, - recorded_at INTEGER NOT NULL, - UNIQUE(event_id, fact_id, owner_kind, project_id), - FOREIGN KEY(fact_id, owner_kind, project_id) - REFERENCES memory_v2_facts(fact_id, owner_kind, project_id) - ); - - CREATE TABLE IF NOT EXISTS memory_v2_current_facts ( - fact_id TEXT NOT NULL, - owner_kind TEXT NOT NULL, - project_id TEXT NOT NULL, - payload_access TEXT NOT NULL CHECK(payload_access IN ( - 'eligible', 'redacted', 'quarantined', 'retention_expired', - 'deleted', 'unavailable', 'ambiguous' - )), - trust_score REAL CHECK( - trust_score IS NULL OR (trust_score >= 0.0 AND trust_score <= 1.0) - ), - active_assertion_id TEXT, - last_event_id TEXT NOT NULL, - updated_at INTEGER NOT NULL, - retrieval_count INTEGER NOT NULL DEFAULT 0 CHECK(retrieval_count >= 0), - access_count INTEGER NOT NULL DEFAULT 0 CHECK(access_count >= 0), - helpful_count INTEGER NOT NULL DEFAULT 0 CHECK(helpful_count >= 0), - unhelpful_count INTEGER NOT NULL DEFAULT 0 CHECK(unhelpful_count >= 0), - last_retrieved_at INTEGER, - last_recalled_at INTEGER, - last_feedback_at INTEGER, - PRIMARY KEY(fact_id, owner_kind, project_id), - FOREIGN KEY(fact_id, owner_kind, project_id) - REFERENCES memory_v2_facts(fact_id, owner_kind, project_id), - FOREIGN KEY(active_assertion_id, fact_id, owner_kind, project_id) - REFERENCES memory_v2_assertions(assertion_id, fact_id, owner_kind, project_id), - FOREIGN KEY(last_event_id, fact_id, owner_kind, project_id) - REFERENCES memory_v2_lineage_events(event_id, fact_id, owner_kind, project_id) - ); - - CREATE TABLE IF NOT EXISTS memory_v2_automatic_fact_receipts ( - apply_id TEXT NOT NULL, - owner_kind TEXT NOT NULL CHECK(owner_kind IN ('profile', 'project')), - project_id TEXT NOT NULL, - owner_json TEXT NOT NULL CHECK(json_valid(owner_json)), - idempotency_key TEXT NOT NULL, - request_digest TEXT NOT NULL, - request_json TEXT NOT NULL CHECK(json_valid(request_json)), - evidence_json TEXT NOT NULL CHECK(json_valid(evidence_json)), - state TEXT NOT NULL CHECK(state IN ('applied', 'quarantined')), - quarantine_reason TEXT, - applied_fact_id TEXT, - applied_assertion_id TEXT, - applied_event_id TEXT, - recorded_at INTEGER NOT NULL, - PRIMARY KEY(apply_id, owner_kind, project_id), - UNIQUE(owner_kind, project_id, idempotency_key), - UNIQUE(owner_kind, project_id, request_digest), - CHECK( - (owner_kind = 'profile' AND project_id = '') OR - (owner_kind = 'project' AND project_id <> '') - ), - FOREIGN KEY(applied_fact_id, owner_kind, project_id) - REFERENCES memory_v2_facts(fact_id, owner_kind, project_id), - FOREIGN KEY(applied_assertion_id, applied_fact_id, owner_kind, project_id) - REFERENCES memory_v2_assertions(assertion_id, fact_id, owner_kind, project_id), - FOREIGN KEY(applied_event_id, applied_fact_id, owner_kind, project_id) - REFERENCES memory_v2_lineage_events(event_id, fact_id, owner_kind, project_id), - CHECK( - (state = 'applied' - AND quarantine_reason IS NULL - AND applied_fact_id IS NOT NULL - AND applied_event_id IS NOT NULL) OR - (state = 'quarantined' - AND quarantine_reason IS NOT NULL - AND applied_fact_id IS NULL - AND applied_assertion_id IS NULL - AND applied_event_id IS NULL) - ) - ); - CREATE INDEX IF NOT EXISTS idx_memory_v2_assertions_fact - ON memory_v2_assertions(fact_id, owner_kind, project_id, asserted_at); - CREATE INDEX IF NOT EXISTS idx_memory_v2_events_fact - ON memory_v2_lineage_events(fact_id, owner_kind, project_id, event_sequence); - CREATE INDEX IF NOT EXISTS idx_memory_v2_events_as_of - ON memory_v2_lineage_events( - fact_id, owner_kind, project_id, occurred_at, event_id - ); - CREATE INDEX IF NOT EXISTS idx_memory_v2_current_page - ON memory_v2_current_facts(owner_kind, project_id, fact_id); - CREATE INDEX IF NOT EXISTS idx_memory_v2_evidence_anchor - ON memory_v2_evidence(anchor_id, owner_json); - CREATE INDEX IF NOT EXISTS idx_memory_v2_automatic_fact_receipt_list - ON memory_v2_automatic_fact_receipts( - owner_kind, project_id, state, recorded_at, apply_id - ); - - CREATE TRIGGER IF NOT EXISTS memory_v2_facts_no_update - BEFORE UPDATE ON memory_v2_facts BEGIN - SELECT RAISE(ABORT, 'memory_v2 fact identities are immutable'); - END; - CREATE TRIGGER IF NOT EXISTS memory_v2_facts_no_delete - BEFORE DELETE ON memory_v2_facts BEGIN - SELECT RAISE(ABORT, 'memory_v2 fact identities are immutable'); - END; - CREATE TRIGGER IF NOT EXISTS memory_v2_assertions_no_update - BEFORE UPDATE ON memory_v2_assertions BEGIN - SELECT RAISE(ABORT, 'memory_v2 assertions are immutable'); - END; - CREATE TRIGGER IF NOT EXISTS memory_v2_assertions_no_delete - BEFORE DELETE ON memory_v2_assertions BEGIN - SELECT RAISE(ABORT, 'memory_v2 assertions are immutable'); - END; - CREATE TRIGGER IF NOT EXISTS memory_v2_supersession_no_update - BEFORE UPDATE ON memory_v2_assertion_supersession BEGIN - SELECT RAISE(ABORT, 'memory_v2 assertion supersession is immutable'); - END; - CREATE TRIGGER IF NOT EXISTS memory_v2_supersession_no_delete - BEFORE DELETE ON memory_v2_assertion_supersession BEGIN - SELECT RAISE(ABORT, 'memory_v2 assertion supersession is immutable'); - END; - CREATE TRIGGER IF NOT EXISTS memory_v2_evidence_no_update - BEFORE UPDATE ON memory_v2_evidence BEGIN - SELECT RAISE(ABORT, 'memory_v2 evidence is immutable'); - END; - CREATE TRIGGER IF NOT EXISTS memory_v2_evidence_no_delete - BEFORE DELETE ON memory_v2_evidence BEGIN - SELECT RAISE(ABORT, 'memory_v2 evidence is immutable'); - END; - CREATE TRIGGER IF NOT EXISTS memory_v2_assertion_evidence_no_update - BEFORE UPDATE ON memory_v2_assertion_evidence BEGIN - SELECT RAISE(ABORT, 'memory_v2 assertion evidence is immutable'); - END; - CREATE TRIGGER IF NOT EXISTS memory_v2_assertion_evidence_no_delete - BEFORE DELETE ON memory_v2_assertion_evidence BEGIN - SELECT RAISE(ABORT, 'memory_v2 assertion evidence is immutable'); - END; - CREATE TRIGGER IF NOT EXISTS memory_v2_events_no_update - BEFORE UPDATE ON memory_v2_lineage_events BEGIN - SELECT RAISE(ABORT, 'memory_v2 lineage events are immutable'); - END; - CREATE TRIGGER IF NOT EXISTS memory_v2_events_no_delete - BEFORE DELETE ON memory_v2_lineage_events BEGIN - SELECT RAISE(ABORT, 'memory_v2 lineage events are immutable'); - END; - CREATE TRIGGER IF NOT EXISTS memory_v2_automatic_fact_receipts_no_update - BEFORE UPDATE ON memory_v2_automatic_fact_receipts BEGIN - SELECT RAISE(ABORT, 'memory_v2 automatic fact receipts are immutable'); - END; - CREATE TRIGGER IF NOT EXISTS memory_v2_automatic_fact_receipts_no_delete - BEFORE DELETE ON memory_v2_automatic_fact_receipts BEGIN - SELECT RAISE(ABORT, 'memory_v2 automatic fact receipts are immutable'); - END; - -CREATE TABLE IF NOT EXISTS memory_v2_operation_receipts ( - owner_kind TEXT NOT NULL CHECK(owner_kind IN ('profile', 'project')), - project_id TEXT NOT NULL, - operation_id TEXT NOT NULL CHECK(length(operation_id) > 0), - operation_kind TEXT NOT NULL CHECK(operation_kind IN ( - 'add', 'update', 'remove', 'feedback', 'retrieval', - 'curation', 'merge', 'automatic_fact_apply' - )), - request_digest TEXT NOT NULL CHECK(length(request_digest) > 0), - fact_id TEXT, - event_id TEXT, - receipt_json TEXT NOT NULL CHECK(json_valid(receipt_json)), - recorded_at INTEGER NOT NULL, - PRIMARY KEY(owner_kind, project_id, operation_id), - FOREIGN KEY(fact_id, owner_kind, project_id) - REFERENCES memory_v2_facts(fact_id, owner_kind, project_id), - FOREIGN KEY(event_id, fact_id, owner_kind, project_id) - REFERENCES memory_v2_lineage_events(event_id, fact_id, owner_kind, project_id), - CHECK(event_id IS NULL OR fact_id IS NOT NULL), - CHECK( - (owner_kind = 'profile' AND project_id = '') OR - (owner_kind = 'project' AND project_id <> '') - ) - ); - - CREATE TABLE IF NOT EXISTS memory_v2_feedback_history ( - owner_kind TEXT NOT NULL CHECK(owner_kind IN ('profile', 'project')), - project_id TEXT NOT NULL, - fact_id TEXT NOT NULL, - event_id TEXT NOT NULL, - action TEXT NOT NULL CHECK(action IN ('helpful', 'unhelpful')), - old_trust REAL NOT NULL CHECK(old_trust >= 0.0 AND old_trust <= 1.0), - new_trust REAL NOT NULL CHECK(new_trust >= 0.0 AND new_trust <= 1.0), - occurred_at INTEGER NOT NULL, - source TEXT, - note TEXT, - details_availability TEXT NOT NULL CHECK( - details_availability IN ('available', 'redacted', 'unknown') - ), - PRIMARY KEY(owner_kind, project_id, fact_id, event_id), - FOREIGN KEY(fact_id, owner_kind, project_id) - REFERENCES memory_v2_facts(fact_id, owner_kind, project_id), - FOREIGN KEY(event_id, fact_id, owner_kind, project_id) - REFERENCES memory_v2_lineage_events(event_id, fact_id, owner_kind, project_id), - CHECK( - (owner_kind = 'profile' AND project_id = '') OR - (owner_kind = 'project' AND project_id <> '') - ), - CHECK( - details_availability = 'available' OR (source IS NULL AND note IS NULL) - ) - ); - - CREATE INDEX IF NOT EXISTS idx_memory_v2_operation_receipts_fact - ON memory_v2_operation_receipts( - fact_id, owner_kind, project_id, recorded_at - ); - CREATE INDEX IF NOT EXISTS idx_memory_v2_operation_receipts_automation_run - ON memory_v2_operation_receipts( - owner_kind, project_id, operation_kind, - json_extract(receipt_json, '$.automation_run_id'), - recorded_at, operation_id - ); - CREATE INDEX IF NOT EXISTS idx_memory_v2_automatic_fact_receipts_automation_run - ON memory_v2_automatic_fact_receipts( - owner_kind, project_id, - json_extract(request_json, '$.automation_run_id'), - recorded_at, apply_id - ); - CREATE INDEX IF NOT EXISTS idx_memory_v2_feedback_history_fact - ON memory_v2_feedback_history( - owner_kind, project_id, fact_id, occurred_at, event_id - ); - CREATE TRIGGER IF NOT EXISTS memory_v2_operation_receipts_no_update - BEFORE UPDATE ON memory_v2_operation_receipts BEGIN - SELECT RAISE(ABORT, 'memory_v2 operation receipts are immutable'); - END; - CREATE TRIGGER IF NOT EXISTS memory_v2_operation_receipts_no_delete - BEFORE DELETE ON memory_v2_operation_receipts BEGIN - SELECT RAISE(ABORT, 'memory_v2 operation receipts are immutable'); - END; - CREATE TRIGGER IF NOT EXISTS memory_v2_operation_receipts_no_payload - BEFORE INSERT ON memory_v2_operation_receipts - WHEN EXISTS ( - SELECT 1 FROM json_tree(NEW.receipt_json) - WHERE lower(CAST(key AS TEXT)) IN ( - 'content', 'payload', 'payload_json', 'metadata', - 'vector', 'vectors', 'embedding', 'embeddings', - 'vector_watermark', 'vector_watermark_json' - ) - ) BEGIN - SELECT RAISE(ABORT, 'memory_v2 operation receipts cannot retain payload data'); - END; - CREATE TRIGGER IF NOT EXISTS memory_v2_feedback_history_only_redaction - BEFORE UPDATE ON memory_v2_feedback_history - WHEN NOT ( - NEW.owner_kind IS OLD.owner_kind - AND NEW.project_id IS OLD.project_id - AND NEW.fact_id IS OLD.fact_id - AND NEW.event_id IS OLD.event_id - AND NEW.action IS OLD.action - AND NEW.old_trust IS OLD.old_trust - AND NEW.new_trust IS OLD.new_trust - AND NEW.occurred_at IS OLD.occurred_at - AND NEW.source IS NULL - AND NEW.note IS NULL - AND ( - (OLD.details_availability = 'available' - AND NEW.details_availability = 'redacted') - OR ( - OLD.source IS NULL AND OLD.note IS NULL - AND NEW.details_availability IS OLD.details_availability - ) - ) - ) BEGIN - SELECT RAISE(ABORT, 'memory_v2 feedback history permits only detail redaction'); - END; - CREATE TRIGGER IF NOT EXISTS memory_v2_feedback_history_no_delete - BEFORE DELETE ON memory_v2_feedback_history BEGIN - SELECT RAISE(ABORT, 'memory_v2 feedback history records are immutable'); - END; - -CREATE TRIGGER IF NOT EXISTS memory_v2_automatic_fact_receipts_require_keys - BEFORE INSERT ON memory_v2_automatic_fact_receipts - WHEN NEW.idempotency_key IS NULL OR length(NEW.idempotency_key) = 0 - OR NEW.request_digest IS NULL OR length(NEW.request_digest) = 0 - BEGIN - SELECT RAISE(ABORT, 'memory_v2 automatic fact receipts require idempotency and request digests'); - END; - -CREATE INDEX IF NOT EXISTS idx_memory_v2_current_search - ON memory_v2_current_facts( - owner_kind, project_id, updated_at DESC, fact_id - ); - -CREATE TABLE IF NOT EXISTS generation_diagnostics ( - diagnostic_anchor TEXT PRIMARY KEY, - generation_id TEXT NOT NULL, - repository TEXT NOT NULL, - worktree TEXT, - reference TEXT, - source_revision TEXT, - file_occurrence_id TEXT NOT NULL, - content_digest TEXT NOT NULL, - symbol_occurrence_id TEXT, - span_start INTEGER NOT NULL, - span_end INTEGER NOT NULL, - code TEXT NOT NULL, - severity TEXT NOT NULL, - message TEXT NOT NULL, - message_digest TEXT NOT NULL, - producer_kind TEXT NOT NULL, - producer TEXT NOT NULL, - analyzer_revision TEXT NOT NULL, - configuration_revision TEXT NOT NULL, - sanitization_receipt TEXT, - evidence_class TEXT NOT NULL, - collected_at INTEGER NOT NULL, - record_state TEXT NOT NULL DEFAULT 'current', - state_generation TEXT, - persisted_at INTEGER NOT NULL DEFAULT 0 - ); - - CREATE INDEX IF NOT EXISTS idx_generation_diagnostics_generation_state - ON generation_diagnostics (generation_id, record_state); - - CREATE INDEX IF NOT EXISTS idx_generation_diagnostics_generation_state_anchor - ON generation_diagnostics (generation_id, record_state, diagnostic_anchor); - - CREATE INDEX IF NOT EXISTS idx_generation_diagnostics_file - ON generation_diagnostics (file_occurrence_id, generation_id); - - CREATE INDEX IF NOT EXISTS idx_generation_diagnostics_file_generation_state_anchor - ON generation_diagnostics ( - file_occurrence_id, generation_id, record_state, diagnostic_anchor - ); - - CREATE TABLE IF NOT EXISTS diagnostic_generation_publications ( - generation_id TEXT PRIMARY KEY, - record_state TEXT NOT NULL, - state_generation TEXT, - published_at INTEGER NOT NULL - ); - - CREATE UNIQUE INDEX IF NOT EXISTS idx_diagnostic_generation_current - ON diagnostic_generation_publications (record_state) - WHERE record_state = 'current'; - -CREATE TABLE IF NOT EXISTS evidence_source_occurrences ( - occurrence_id TEXT PRIMARY KEY CHECK(length(occurrence_id) > 0), - owner_digest TEXT NOT NULL CHECK(length(owner_digest) > 0), - timeline_digest TEXT NOT NULL CHECK(length(timeline_digest) > 0), - source_anchor_id TEXT NOT NULL CHECK(length(source_anchor_id) > 0), - source_order INTEGER NOT NULL CHECK(source_order >= 0), - record_digest TEXT NOT NULL CHECK(length(record_digest) > 0), - record_json TEXT NOT NULL CHECK(json_valid(record_json)) - ); - CREATE INDEX IF NOT EXISTS idx_evidence_occurrences_anchor - ON evidence_source_occurrences(owner_digest, source_anchor_id); - CREATE INDEX IF NOT EXISTS idx_evidence_occurrences_timeline - ON evidence_source_occurrences(owner_digest, timeline_digest, source_order); - - CREATE TABLE IF NOT EXISTS evidence_occurrence_sets ( - occurrence_set_id TEXT PRIMARY KEY CHECK(length(occurrence_set_id) > 0), - owner_digest TEXT NOT NULL CHECK(length(owner_digest) > 0), - record_digest TEXT NOT NULL CHECK(length(record_digest) > 0), - record_json TEXT NOT NULL CHECK(json_valid(record_json)) - ); - CREATE TABLE IF NOT EXISTS evidence_occurrence_set_members ( - occurrence_set_id TEXT NOT NULL, - canonical_ordinal INTEGER NOT NULL CHECK(canonical_ordinal >= 0), - occurrence_id TEXT NOT NULL, - PRIMARY KEY(occurrence_set_id, canonical_ordinal), - UNIQUE(occurrence_set_id, occurrence_id), - FOREIGN KEY(occurrence_set_id) - REFERENCES evidence_occurrence_sets(occurrence_set_id), - FOREIGN KEY(occurrence_id) - REFERENCES evidence_source_occurrences(occurrence_id) - ); - - CREATE TABLE IF NOT EXISTS evidence_spans ( - span_id TEXT PRIMARY KEY CHECK(length(span_id) > 0), - owner_digest TEXT NOT NULL CHECK(length(owner_digest) > 0), - occurrence_set_id TEXT NOT NULL, - anchor_id TEXT NOT NULL UNIQUE CHECK(length(anchor_id) > 0), - producer_kind TEXT NOT NULL CHECK(length(producer_kind) > 0), - record_digest TEXT NOT NULL CHECK(length(record_digest) > 0), - record_json TEXT NOT NULL CHECK(json_valid(record_json)), - FOREIGN KEY(occurrence_set_id) - REFERENCES evidence_occurrence_sets(occurrence_set_id) - ); - CREATE TABLE IF NOT EXISTS evidence_span_members ( - span_id TEXT NOT NULL, - assembly_ordinal INTEGER NOT NULL CHECK(assembly_ordinal >= 0), - run_ordinal INTEGER NOT NULL CHECK(run_ordinal >= 0), - run_member_ordinal INTEGER NOT NULL CHECK(run_member_ordinal >= 0), - occurrence_id TEXT NOT NULL, - PRIMARY KEY(span_id, assembly_ordinal), - UNIQUE(span_id, occurrence_id), - FOREIGN KEY(span_id) REFERENCES evidence_spans(span_id), - FOREIGN KEY(occurrence_id) - REFERENCES evidence_source_occurrences(occurrence_id) - ); - - CREATE TABLE IF NOT EXISTS evidence_span_projection_receipts ( - projection_receipt_id TEXT PRIMARY KEY CHECK(length(projection_receipt_id) > 0), - span_id TEXT NOT NULL, - record_digest TEXT NOT NULL CHECK(length(record_digest) > 0), - record_json TEXT NOT NULL CHECK(json_valid(record_json)), - UNIQUE(span_id, projection_receipt_id), - FOREIGN KEY(span_id) REFERENCES evidence_spans(span_id) - ); - - CREATE TABLE IF NOT EXISTS evidence_retriever_contributions ( - contribution_id TEXT PRIMARY KEY CHECK(length(contribution_id) > 0), - owner_digest TEXT NOT NULL CHECK(length(owner_digest) > 0), - span_id TEXT NOT NULL, - anchor_id TEXT NOT NULL UNIQUE CHECK(length(anchor_id) > 0), - record_digest TEXT NOT NULL CHECK(length(record_digest) > 0), - record_json TEXT NOT NULL CHECK(json_valid(record_json)), - FOREIGN KEY(span_id) REFERENCES evidence_spans(span_id) - ); - - CREATE TABLE IF NOT EXISTS evidence_derived_anchors ( - anchor_id TEXT PRIMARY KEY CHECK(length(anchor_id) > 0), - owner_digest TEXT NOT NULL CHECK(length(owner_digest) > 0), - target_kind TEXT NOT NULL CHECK( - target_kind IN ('source_occurrence', 'evidence_span', 'retriever_contribution') - ), - target_id TEXT NOT NULL CHECK(length(target_id) > 0), - anchor_json TEXT NOT NULL CHECK(json_valid(anchor_json)), - UNIQUE(owner_digest, target_kind, target_id) - ); - - CREATE TABLE IF NOT EXISTS evidence_assembly_receipts ( - publication_receipt_id TEXT PRIMARY KEY CHECK(length(publication_receipt_id) > 0), - owner_digest TEXT NOT NULL CHECK(length(owner_digest) > 0), - privacy_domain_id TEXT NOT NULL CHECK(length(privacy_domain_id) > 0), - key_epoch INTEGER NOT NULL CHECK(key_epoch > 0), - idempotency_key TEXT NOT NULL CHECK(length(idempotency_key) > 0), - assembly_digest TEXT NOT NULL CHECK(length(assembly_digest) > 0), - occurrence_set_id TEXT NOT NULL, - span_id TEXT NOT NULL, - contribution_id TEXT NOT NULL, - projection_receipt_id TEXT NOT NULL, - receipt_json TEXT NOT NULL CHECK(json_valid(receipt_json)), - UNIQUE(owner_digest, privacy_domain_id, key_epoch, idempotency_key), - FOREIGN KEY(occurrence_set_id) - REFERENCES evidence_occurrence_sets(occurrence_set_id), - FOREIGN KEY(span_id) REFERENCES evidence_spans(span_id), - FOREIGN KEY(contribution_id) - REFERENCES evidence_retriever_contributions(contribution_id), - FOREIGN KEY(projection_receipt_id) - REFERENCES evidence_span_projection_receipts(projection_receipt_id) - ); - -CREATE TRIGGER IF NOT EXISTS evidence_source_occurrences_immutable_update - BEFORE UPDATE ON evidence_source_occurrences BEGIN - SELECT RAISE(ABORT, 'evidence source occurrences are immutable'); - END; - CREATE TRIGGER IF NOT EXISTS evidence_source_occurrences_immutable_delete - BEFORE DELETE ON evidence_source_occurrences BEGIN - SELECT RAISE(ABORT, 'evidence source occurrences are immutable'); - END; - CREATE TRIGGER IF NOT EXISTS evidence_occurrence_sets_immutable_update - BEFORE UPDATE ON evidence_occurrence_sets BEGIN - SELECT RAISE(ABORT, 'evidence occurrence sets are immutable'); - END; - CREATE TRIGGER IF NOT EXISTS evidence_occurrence_sets_immutable_delete - BEFORE DELETE ON evidence_occurrence_sets BEGIN - SELECT RAISE(ABORT, 'evidence occurrence sets are immutable'); - END; - CREATE TRIGGER IF NOT EXISTS evidence_occurrence_set_members_immutable_update - BEFORE UPDATE ON evidence_occurrence_set_members BEGIN - SELECT RAISE(ABORT, 'evidence occurrence set membership is immutable'); - END; - CREATE TRIGGER IF NOT EXISTS evidence_occurrence_set_members_immutable_delete - BEFORE DELETE ON evidence_occurrence_set_members BEGIN - SELECT RAISE(ABORT, 'evidence occurrence set membership is immutable'); - END; - CREATE TRIGGER IF NOT EXISTS evidence_spans_immutable_update - BEFORE UPDATE ON evidence_spans BEGIN - SELECT RAISE(ABORT, 'evidence spans are immutable'); - END; - CREATE TRIGGER IF NOT EXISTS evidence_spans_immutable_delete - BEFORE DELETE ON evidence_spans BEGIN - SELECT RAISE(ABORT, 'evidence spans are immutable'); - END; - CREATE TRIGGER IF NOT EXISTS evidence_span_members_immutable_update - BEFORE UPDATE ON evidence_span_members BEGIN - SELECT RAISE(ABORT, 'evidence span membership is immutable'); - END; - CREATE TRIGGER IF NOT EXISTS evidence_span_members_immutable_delete - BEFORE DELETE ON evidence_span_members BEGIN - SELECT RAISE(ABORT, 'evidence span membership is immutable'); - END; - CREATE TRIGGER IF NOT EXISTS evidence_span_projection_receipts_immutable_update - BEFORE UPDATE ON evidence_span_projection_receipts BEGIN - SELECT RAISE(ABORT, 'evidence projection receipts are immutable'); - END; - CREATE TRIGGER IF NOT EXISTS evidence_span_projection_receipts_immutable_delete - BEFORE DELETE ON evidence_span_projection_receipts BEGIN - SELECT RAISE(ABORT, 'evidence projection receipts are immutable'); - END; - CREATE TRIGGER IF NOT EXISTS evidence_retriever_contributions_immutable_update - BEFORE UPDATE ON evidence_retriever_contributions BEGIN - SELECT RAISE(ABORT, 'retriever contributions are immutable'); - END; - CREATE TRIGGER IF NOT EXISTS evidence_retriever_contributions_immutable_delete - BEFORE DELETE ON evidence_retriever_contributions BEGIN - SELECT RAISE(ABORT, 'retriever contributions are immutable'); - END; - CREATE TRIGGER IF NOT EXISTS evidence_derived_anchors_immutable_update - BEFORE UPDATE ON evidence_derived_anchors BEGIN - SELECT RAISE(ABORT, 'evidence derived anchors are immutable'); - END; - CREATE TRIGGER IF NOT EXISTS evidence_derived_anchors_immutable_delete - BEFORE DELETE ON evidence_derived_anchors BEGIN - SELECT RAISE(ABORT, 'evidence derived anchors are immutable'); - END; - CREATE TRIGGER IF NOT EXISTS evidence_assembly_receipts_immutable_update - BEFORE UPDATE ON evidence_assembly_receipts BEGIN - SELECT RAISE(ABORT, 'evidence assembly receipts are immutable'); - END; - CREATE TRIGGER IF NOT EXISTS evidence_assembly_receipts_immutable_delete - BEFORE DELETE ON evidence_assembly_receipts BEGIN - SELECT RAISE(ABORT, 'evidence assembly receipts are immutable'); - END; - -CREATE TABLE IF NOT EXISTS external_source_states_v1 ( - binding_id TEXT PRIMARY KEY, - source_id TEXT NOT NULL, - owner_kind TEXT NOT NULL CHECK (owner_kind IN ('project', 'profile')), - owner_id TEXT NOT NULL, - definition_revision INTEGER NOT NULL CHECK (definition_revision > 0), - definition_digest TEXT NOT NULL, - binding_revision INTEGER NOT NULL CHECK (binding_revision > 0), - binding_digest TEXT NOT NULL, - source_frontier_digest TEXT NOT NULL, - source_frontier_json TEXT NOT NULL, - projection_frontier_digest TEXT, - latest_source_receipt_digest TEXT NOT NULL, - latest_projection_receipt_digest TEXT -); -CREATE INDEX IF NOT EXISTS idx_external_source_states_owner_v1 - ON external_source_states_v1(owner_kind, owner_id, source_id); -CREATE TABLE IF NOT EXISTS external_source_definition_revisions_v1 ( - source_id TEXT NOT NULL, - definition_revision INTEGER NOT NULL CHECK (definition_revision > 0), - definition_digest TEXT NOT NULL, - definition_json TEXT NOT NULL, - PRIMARY KEY (source_id, definition_revision) -); -CREATE TABLE IF NOT EXISTS external_source_binding_revisions_v1 ( - binding_id TEXT NOT NULL, - binding_revision INTEGER NOT NULL CHECK (binding_revision > 0), - definition_revision INTEGER NOT NULL CHECK (definition_revision > 0), - binding_digest TEXT NOT NULL, - binding_json TEXT NOT NULL, - PRIMARY KEY (binding_id, binding_revision) -); -CREATE TABLE IF NOT EXISTS external_source_authority_receipts_v1 ( - binding_id TEXT NOT NULL, - idempotency_key TEXT NOT NULL, - request_digest TEXT NOT NULL, - definition_digest TEXT NOT NULL, - binding_digest TEXT NOT NULL, - receipt_json TEXT NOT NULL, - PRIMARY KEY (binding_id, idempotency_key) -); -CREATE TABLE IF NOT EXISTS external_source_commit_receipts_v1 ( - binding_id TEXT NOT NULL, - idempotency_key TEXT NOT NULL, - request_digest TEXT NOT NULL, - definition_revision INTEGER NOT NULL CHECK (definition_revision > 0), - binding_revision INTEGER NOT NULL CHECK (binding_revision > 0), - predecessor_frontier_digest TEXT NOT NULL, - successor_frontier_digest TEXT NOT NULL, - receipt_digest TEXT NOT NULL, - receipt_json TEXT NOT NULL, - PRIMARY KEY (binding_id, idempotency_key), - UNIQUE (binding_id, receipt_digest), - UNIQUE (binding_id, successor_frontier_digest) -); -CREATE TABLE IF NOT EXISTS external_source_mutations_v1 ( - binding_id TEXT NOT NULL, - mutation_digest TEXT NOT NULL, - native_object_digest TEXT NOT NULL, - revision_digest TEXT NOT NULL, - source_receipt_digest TEXT NOT NULL, - mutation_json TEXT NOT NULL, - PRIMARY KEY (binding_id, mutation_digest), - UNIQUE (binding_id, native_object_digest, revision_digest) -); -CREATE TABLE IF NOT EXISTS external_source_lineage_v1 ( - binding_id TEXT NOT NULL, - lineage_digest TEXT NOT NULL, - source_receipt_digest TEXT NOT NULL, - lineage_json TEXT NOT NULL, - PRIMARY KEY (binding_id, lineage_digest) -); -CREATE TABLE IF NOT EXISTS external_source_objects_v1 ( - binding_id TEXT NOT NULL, - native_object_digest TEXT NOT NULL, - partition_digest TEXT NOT NULL, - mutation_digest TEXT NOT NULL, - mutation_json TEXT NOT NULL, - PRIMARY KEY (binding_id, native_object_digest) -); -CREATE TABLE IF NOT EXISTS external_source_pending_projections_v1 ( - binding_id TEXT NOT NULL, - predecessor_frontier_digest TEXT NOT NULL, - successor_frontier_digest TEXT NOT NULL, - successor_sequence INTEGER NOT NULL CHECK (successor_sequence > 0), - source_receipt_digest TEXT NOT NULL, - PRIMARY KEY (binding_id, predecessor_frontier_digest), - UNIQUE (binding_id, successor_frontier_digest), - UNIQUE (binding_id, source_receipt_digest) -); -CREATE TABLE IF NOT EXISTS external_source_projection_publications_v1 ( - binding_id TEXT NOT NULL, - projection_digest TEXT NOT NULL, - source_receipt_digest TEXT NOT NULL, - predecessor_frontier_digest TEXT NOT NULL, - successor_frontier_digest TEXT NOT NULL, - receipt_json TEXT NOT NULL, - PRIMARY KEY (binding_id, projection_digest), - UNIQUE (binding_id, source_receipt_digest), - UNIQUE (binding_id, successor_frontier_digest) -); -CREATE TABLE IF NOT EXISTS external_source_projection_effects_v1 ( - binding_id TEXT NOT NULL, - projection_digest TEXT NOT NULL, - effect_index INTEGER NOT NULL CHECK (effect_index >= 0), - native_object_digest TEXT NOT NULL, - effect_json TEXT NOT NULL, - mutation_json TEXT NOT NULL, - PRIMARY KEY (binding_id, projection_digest, effect_index) -); -CREATE TABLE IF NOT EXISTS external_source_projection_lineage_v1 ( - binding_id TEXT NOT NULL, - projection_digest TEXT NOT NULL, - lineage_index INTEGER NOT NULL CHECK (lineage_index >= 0), - lineage_digest TEXT NOT NULL, - lineage_json TEXT NOT NULL, - PRIMARY KEY (binding_id, projection_digest, lineage_index) -); -CREATE TABLE IF NOT EXISTS external_source_projected_objects_v1 ( - binding_id TEXT NOT NULL, - native_object_digest TEXT NOT NULL, - mutation_json TEXT NOT NULL, - PRIMARY KEY (binding_id, native_object_digest) -); -CREATE TABLE IF NOT EXISTS external_source_acquisition_queue_v1 ( - binding_id TEXT PRIMARY KEY, - state_digest TEXT NOT NULL, - not_before_micros INTEGER, - state_json TEXT NOT NULL -); -CREATE INDEX IF NOT EXISTS idx_external_source_acquisition_ready_v1 - ON external_source_acquisition_queue_v1(not_before_micros, binding_id) - WHERE not_before_micros IS NOT NULL; - -CREATE TABLE IF NOT EXISTS graph_publication_replay_v1 ( - sequence INTEGER PRIMARY KEY AUTOINCREMENT, - shard_id TEXT NOT NULL, - namespace TEXT NOT NULL, - projection TEXT NOT NULL, - generation TEXT NOT NULL, - idempotency_key TEXT NOT NULL, - input_digest TEXT NOT NULL, - dependency_generation_closure_digest TEXT NOT NULL, - direct_dependency_bytes INTEGER NOT NULL - CHECK (direct_dependency_bytes >= 2 - AND direct_dependency_bytes <= 1048576), - expected_prior_head TEXT, - expected_recovered_digest TEXT NOT NULL, - canonical_replay_source_digest TEXT NOT NULL, - canonical_replay_source BLOB NOT NULL - CHECK (length(canonical_replay_source) > 0 - AND length(canonical_replay_source) <= 4194304 - AND length(canonical_replay_source) - + direct_dependency_bytes <= 4194304), - UNIQUE (shard_id, namespace, projection, generation), - UNIQUE (shard_id, namespace, projection, idempotency_key), - UNIQUE (sequence, shard_id, namespace, projection, generation) -) STRICT; - -CREATE INDEX IF NOT EXISTS idx_graph_publication_replay_projection_sequence - ON graph_publication_replay_v1(shard_id, namespace, projection, sequence); - -CREATE TABLE IF NOT EXISTS graph_publication_replay_dependencies_v1 ( - owner_replay_sequence INTEGER NOT NULL - REFERENCES graph_publication_replay_v1(sequence) ON DELETE CASCADE, - ordinal INTEGER NOT NULL CHECK (ordinal >= 0), - dependency_replay_sequence INTEGER NOT NULL, - shard_id TEXT NOT NULL, - namespace TEXT NOT NULL, - projection TEXT NOT NULL, - generation TEXT NOT NULL, - PRIMARY KEY (owner_replay_sequence, ordinal), - UNIQUE (owner_replay_sequence, shard_id, namespace, projection), - FOREIGN KEY ( - dependency_replay_sequence, shard_id, namespace, projection, generation - ) REFERENCES graph_publication_replay_v1( - sequence, shard_id, namespace, projection, generation - ) ON DELETE RESTRICT -) STRICT; - -CREATE INDEX IF NOT EXISTS idx_graph_publication_dependency_replay - ON graph_publication_replay_dependencies_v1(dependency_replay_sequence); - -CREATE TABLE IF NOT EXISTS graph_publication_replay_tombstones_v1 ( - replay_sequence INTEGER PRIMARY KEY, - shard_id TEXT NOT NULL, - namespace TEXT NOT NULL, - projection TEXT NOT NULL, - generation TEXT NOT NULL, - idempotency_key TEXT NOT NULL, - input_digest TEXT NOT NULL, - dependency_generation_closure_digest TEXT NOT NULL, - direct_dependency_bytes INTEGER NOT NULL - CHECK (direct_dependency_bytes >= 2 - AND direct_dependency_bytes <= 1048576), - expected_prior_head TEXT, - expected_recovered_digest TEXT NOT NULL, - canonical_replay_source_digest TEXT NOT NULL, - UNIQUE (shard_id, namespace, projection, generation), - UNIQUE (shard_id, namespace, projection, idempotency_key) -) STRICT; - -CREATE INDEX IF NOT EXISTS idx_graph_publication_tombstone_projection - ON graph_publication_replay_tombstones_v1(shard_id, namespace, projection); - -CREATE TABLE IF NOT EXISTS graph_publication_replay_tombstone_dependencies_v1 ( - tombstone_replay_sequence INTEGER NOT NULL - REFERENCES graph_publication_replay_tombstones_v1(replay_sequence) - ON DELETE CASCADE, - ordinal INTEGER NOT NULL CHECK (ordinal >= 0), - shard_id TEXT NOT NULL, - namespace TEXT NOT NULL, - projection TEXT NOT NULL, - generation TEXT NOT NULL, - PRIMARY KEY (tombstone_replay_sequence, ordinal), - UNIQUE (tombstone_replay_sequence, shard_id, namespace, projection) -) STRICT; - -CREATE TABLE IF NOT EXISTS graph_verified_heads_v1 ( - shard_id TEXT NOT NULL, - namespace TEXT NOT NULL, - projection TEXT NOT NULL, - replay_sequence INTEGER NOT NULL UNIQUE - REFERENCES graph_publication_replay_v1(sequence) ON DELETE RESTRICT, - recovered_digest TEXT NOT NULL, - PRIMARY KEY (shard_id, namespace, projection) -) STRICT; - -CREATE TABLE IF NOT EXISTS semantic_vector_stages ( - stage_id INTEGER PRIMARY KEY AUTOINCREMENT, - shard_id TEXT NOT NULL, - namespace TEXT NOT NULL, - projection TEXT NOT NULL, - build_id TEXT NOT NULL, - plan_digest TEXT NOT NULL, - semantic_generation_id TEXT NOT NULL, - base_generation TEXT, - publication_generation TEXT NOT NULL, - publication_idempotency_key TEXT NOT NULL, - source_scope TEXT NOT NULL, - source_generation TEXT NOT NULL, - source_dependency TEXT NOT NULL CHECK (json_valid(source_dependency)), - source_manifest_digest TEXT NOT NULL, - embedding_projection_digest TEXT NOT NULL, - embedding_dimension INTEGER NOT NULL - CHECK (embedding_dimension > 0 AND embedding_dimension <= 4096), - model_artifact_digest TEXT NOT NULL, - projection_manifest_digest TEXT NOT NULL, - privacy_domain_digest TEXT NOT NULL, - privacy_key_epoch INTEGER NOT NULL CHECK (privacy_key_epoch > 0), - expected_chunk_manifest_digest TEXT NOT NULL, - expected_chunk_count INTEGER NOT NULL - CHECK (expected_chunk_count >= 0 AND expected_chunk_count <= 100000), - expected_prior_verified_head TEXT, - writer_binding TEXT NOT NULL CHECK (json_valid(writer_binding)), - code_scope_hash TEXT NOT NULL - CHECK (length(code_scope_hash) = 64 - AND code_scope_hash NOT GLOB '*[^0-9a-f]*'), - plan_json TEXT NOT NULL CHECK (json_valid(plan_json)), - state TEXT NOT NULL CHECK (state IN ('pending', 'ready_to_publish', 'published', 'cancelled')), - next_ordinal INTEGER NOT NULL CHECK (next_ordinal >= 0), - checkpoint_digest TEXT NOT NULL, - recorded_chunk_count INTEGER NOT NULL - CHECK (recorded_chunk_count >= 0 - AND recorded_chunk_count <= expected_chunk_count), - applied_ordinal INTEGER CHECK (applied_ordinal >= 0), - applied_receipt_digest TEXT, - applied_checkpoint_digest TEXT, - applied_graph_batch_digest TEXT, - expected_recovered_digest TEXT, - publication_intent_digest TEXT, - CHECK ( - (applied_ordinal IS NULL - AND applied_receipt_digest IS NULL - AND applied_checkpoint_digest IS NULL - AND applied_graph_batch_digest IS NULL) - OR - (applied_ordinal IS NOT NULL - AND applied_receipt_digest IS NOT NULL - AND applied_checkpoint_digest IS NOT NULL - AND applied_graph_batch_digest IS NOT NULL) - ), - CHECK ( - (state IN ('ready_to_publish', 'published') - AND expected_recovered_digest IS NOT NULL - AND publication_intent_digest IS NOT NULL) - OR - (state NOT IN ('ready_to_publish', 'published') - AND expected_recovered_digest IS NULL - AND publication_intent_digest IS NULL) - ), - UNIQUE (shard_id, namespace, projection, build_id), - UNIQUE (shard_id, namespace, projection, plan_digest) -) STRICT; - -CREATE UNIQUE INDEX IF NOT EXISTS idx_semantic_vector_one_pending_stage - ON semantic_vector_stages(shard_id, namespace, projection) - WHERE state IN ('pending', 'ready_to_publish'); - --- Cancelled attempts stay durable for audit but release their publication --- identity so the same semantic generation can be rebuilt under a new plan. -CREATE UNIQUE INDEX IF NOT EXISTS idx_semantic_vector_live_semantic_generation - ON semantic_vector_stages(shard_id, namespace, projection, semantic_generation_id) - WHERE state != 'cancelled'; - -CREATE UNIQUE INDEX IF NOT EXISTS idx_semantic_vector_live_publication_generation - ON semantic_vector_stages(shard_id, namespace, projection, publication_generation) - WHERE state != 'cancelled'; - -CREATE UNIQUE INDEX IF NOT EXISTS idx_semantic_vector_live_publication_idempotency - ON semantic_vector_stages(shard_id, namespace, projection, publication_idempotency_key) - WHERE state != 'cancelled'; - -CREATE INDEX IF NOT EXISTS idx_semantic_vector_live_base_generation - ON semantic_vector_stages(shard_id, base_generation) - WHERE state IN ('pending', 'ready_to_publish', 'published') - AND base_generation IS NOT NULL; - -CREATE INDEX IF NOT EXISTS idx_semantic_vector_live_source_generation - ON semantic_vector_stages(shard_id, source_generation) - WHERE state IN ('pending', 'ready_to_publish', 'published'); - -CREATE INDEX IF NOT EXISTS idx_semantic_vector_live_source_scope - ON semantic_vector_stages(shard_id, source_scope) - WHERE state IN ('pending', 'ready_to_publish', 'published'); - -CREATE INDEX IF NOT EXISTS idx_semantic_vector_code_scope_binding - ON semantic_vector_stages(shard_id, code_scope_hash, source_scope) - WHERE state IN ('pending', 'ready_to_publish', 'published'); - -CREATE INDEX IF NOT EXISTS idx_semantic_vector_published_project_generation - ON semantic_vector_stages(shard_id, semantic_generation_id) - WHERE state = 'published'; - -CREATE INDEX IF NOT EXISTS idx_semantic_vector_project_census - ON semantic_vector_stages(shard_id, stage_id); - -CREATE INDEX IF NOT EXISTS idx_semantic_vector_projection_census - ON semantic_vector_stages(shard_id, namespace, projection, stage_id); - -CREATE TABLE IF NOT EXISTS semantic_vector_stage_census_authority ( - shard_id TEXT PRIMARY KEY, - revision INTEGER NOT NULL CHECK (revision > 0) -) STRICT; - -CREATE TABLE IF NOT EXISTS semantic_vector_stage_adoption_authority ( - shard_id TEXT PRIMARY KEY, - revision INTEGER NOT NULL CHECK (revision > 0) -) STRICT; - -CREATE TABLE IF NOT EXISTS semantic_vector_source_scope_bindings ( - shard_id TEXT NOT NULL, - code_scope_hash TEXT NOT NULL - CHECK (length(code_scope_hash) = 64 - AND code_scope_hash NOT GLOB '*[^0-9a-f]*'), - source_scope TEXT NOT NULL CHECK (json_valid(source_scope)), - PRIMARY KEY (shard_id, code_scope_hash), - UNIQUE (shard_id, source_scope) -) WITHOUT ROWID, STRICT; - -CREATE TRIGGER IF NOT EXISTS semantic_vector_stage_census_after_scope_binding_insert -AFTER INSERT ON semantic_vector_source_scope_bindings -BEGIN - INSERT INTO semantic_vector_stage_census_authority(shard_id,revision) - VALUES(NEW.shard_id,1) - ON CONFLICT(shard_id) DO UPDATE SET revision=revision+1; -END; - -CREATE TRIGGER IF NOT EXISTS semantic_vector_stage_census_after_scope_binding_delete -AFTER DELETE ON semantic_vector_source_scope_bindings -BEGIN - INSERT INTO semantic_vector_stage_census_authority(shard_id,revision) - VALUES(OLD.shard_id,1) - ON CONFLICT(shard_id) DO UPDATE SET revision=revision+1; -END; - -CREATE TRIGGER IF NOT EXISTS semantic_vector_source_scope_binding_immutable -BEFORE UPDATE ON semantic_vector_source_scope_bindings -BEGIN - SELECT RAISE(ABORT, 'semantic vector source-scope binding is immutable'); -END; - -CREATE TRIGGER IF NOT EXISTS semantic_vector_stage_census_after_stage_insert -AFTER INSERT ON semantic_vector_stages -BEGIN - INSERT INTO semantic_vector_stage_census_authority(shard_id,revision) - VALUES(NEW.shard_id,1) - ON CONFLICT(shard_id) DO UPDATE SET revision=revision+1; - INSERT INTO semantic_vector_stage_adoption_authority(shard_id,revision) - VALUES(NEW.shard_id,1) - ON CONFLICT(shard_id) DO UPDATE SET revision=revision+1; -END; - -CREATE TRIGGER IF NOT EXISTS semantic_vector_stage_census_after_stage_update -AFTER UPDATE ON semantic_vector_stages -BEGIN - INSERT INTO semantic_vector_stage_census_authority(shard_id,revision) - VALUES(NEW.shard_id,1) - ON CONFLICT(shard_id) DO UPDATE SET revision=revision+1; -END; - -CREATE TRIGGER IF NOT EXISTS semantic_vector_stage_adoption_after_state_update -AFTER UPDATE OF state ON semantic_vector_stages -WHEN OLD.state != NEW.state -BEGIN - INSERT INTO semantic_vector_stage_adoption_authority(shard_id,revision) - VALUES(NEW.shard_id,1) - ON CONFLICT(shard_id) DO UPDATE SET revision=revision+1; -END; - -CREATE TRIGGER IF NOT EXISTS semantic_vector_stage_census_after_stage_delete -AFTER DELETE ON semantic_vector_stages -BEGIN - INSERT INTO semantic_vector_stage_census_authority(shard_id,revision) - VALUES(OLD.shard_id,1) - ON CONFLICT(shard_id) DO UPDATE SET revision=revision+1; - INSERT INTO semantic_vector_stage_adoption_authority(shard_id,revision) - VALUES(OLD.shard_id,1) - ON CONFLICT(shard_id) DO UPDATE SET revision=revision+1; -END; - -CREATE TABLE IF NOT EXISTS semantic_vector_retirement_cleanup ( - cleanup_id INTEGER PRIMARY KEY AUTOINCREMENT, - shard_id TEXT NOT NULL, - namespace TEXT NOT NULL, - projection TEXT NOT NULL, - semantic_generation_id TEXT NOT NULL, - publication_generation TEXT NOT NULL, - publication_idempotency_key TEXT NOT NULL, - retirement_json TEXT NOT NULL CHECK (json_valid(retirement_json)), - UNIQUE (shard_id, namespace, projection, semantic_generation_id), - UNIQUE (shard_id, namespace, projection, publication_generation), - UNIQUE (shard_id, namespace, projection, publication_idempotency_key) -) STRICT; - -CREATE INDEX IF NOT EXISTS idx_semantic_vector_pending_retirement_cleanup - ON semantic_vector_retirement_cleanup(shard_id, cleanup_id); - -CREATE TRIGGER IF NOT EXISTS semantic_vector_stage_publication_identity_guard -BEFORE INSERT ON semantic_vector_stages -WHEN EXISTS ( - SELECT 1 FROM graph_publication_replay_v1 - WHERE shard_id=NEW.shard_id - AND namespace=NEW.namespace - AND projection=NEW.projection - AND ( - generation=NEW.publication_generation - OR idempotency_key=NEW.publication_idempotency_key - ) - UNION ALL - SELECT 1 FROM graph_publication_replay_tombstones_v1 - WHERE shard_id=NEW.shard_id - AND namespace=NEW.namespace - AND projection=NEW.projection - AND ( - generation=NEW.publication_generation - OR idempotency_key=NEW.publication_idempotency_key - ) -) -BEGIN - SELECT RAISE(ABORT, 'semantic vector publication identity is already retained'); -END; - -CREATE TRIGGER IF NOT EXISTS semantic_vector_replay_stage_identity_guard -BEFORE INSERT ON graph_publication_replay_v1 -WHEN EXISTS ( - SELECT 1 FROM semantic_vector_stages - WHERE shard_id=NEW.shard_id - AND namespace=NEW.namespace - AND projection=NEW.projection - AND ( - publication_generation=NEW.generation - OR publication_idempotency_key=NEW.idempotency_key - ) - AND NOT ( - state='ready_to_publish' - AND - publication_generation=NEW.generation - AND publication_idempotency_key=NEW.idempotency_key - ) -) -BEGIN - SELECT RAISE(ABORT, 'graph replay conflicts with a semantic vector publication identity'); -END; - -CREATE TABLE IF NOT EXISTS semantic_vector_stage_batches ( - batch_id INTEGER PRIMARY KEY AUTOINCREMENT, - stage_id INTEGER NOT NULL - REFERENCES semantic_vector_stages(stage_id) ON DELETE RESTRICT, - ordinal INTEGER NOT NULL CHECK (ordinal >= 0), - expected_checkpoint_digest TEXT NOT NULL, - input_digest TEXT NOT NULL, - output_digest TEXT NOT NULL, - receipt_digest TEXT NOT NULL, - checkpoint_digest TEXT NOT NULL, - chunk_count INTEGER NOT NULL CHECK (chunk_count >= 0 AND chunk_count <= 512), - receipt_json TEXT NOT NULL CHECK (json_valid(receipt_json)), - UNIQUE (stage_id, ordinal), - UNIQUE (stage_id, receipt_digest) -) STRICT; - -CREATE TRIGGER IF NOT EXISTS semantic_vector_stage_census_after_batch_insert -AFTER INSERT ON semantic_vector_stage_batches -BEGIN - UPDATE semantic_vector_stage_census_authority - SET revision=revision+1 - WHERE shard_id=( - SELECT shard_id FROM semantic_vector_stages WHERE stage_id=NEW.stage_id - ); -END; - -CREATE TRIGGER IF NOT EXISTS semantic_vector_stage_census_after_batch_update -AFTER UPDATE ON semantic_vector_stage_batches -BEGIN - UPDATE semantic_vector_stage_census_authority - SET revision=revision+1 - WHERE shard_id=( - SELECT shard_id FROM semantic_vector_stages WHERE stage_id=NEW.stage_id - ); -END; - -CREATE TRIGGER IF NOT EXISTS semantic_vector_stage_census_after_batch_delete -AFTER DELETE ON semantic_vector_stage_batches -BEGIN - UPDATE semantic_vector_stage_census_authority - SET revision=revision+1 - WHERE shard_id=( - SELECT shard_id FROM semantic_vector_stages WHERE stage_id=OLD.stage_id - ); -END; - -CREATE TABLE IF NOT EXISTS semantic_vector_stage_chunk_receipts ( - stage_id INTEGER NOT NULL - REFERENCES semantic_vector_stages(stage_id) ON DELETE RESTRICT, - batch_id INTEGER NOT NULL - REFERENCES semantic_vector_stage_batches(batch_id) ON DELETE RESTRICT, - effect_ordinal INTEGER NOT NULL CHECK (effect_ordinal >= 0), - chunk_id TEXT NOT NULL, - chunk_digest TEXT NOT NULL, - operation TEXT NOT NULL CHECK (operation IN ('embed', 'reuse', 'tombstone')), - output_digest TEXT, - CHECK ( - (operation = 'embed' AND output_digest IS NOT NULL) - OR (operation IN ('reuse', 'tombstone') AND output_digest IS NULL) - ), - PRIMARY KEY (batch_id, effect_ordinal), - UNIQUE (stage_id, chunk_id) -) WITHOUT ROWID, STRICT; - -CREATE TRIGGER IF NOT EXISTS semantic_vector_stage_census_after_chunk_insert -AFTER INSERT ON semantic_vector_stage_chunk_receipts -BEGIN - UPDATE semantic_vector_stage_census_authority - SET revision=revision+1 - WHERE shard_id=( - SELECT shard_id FROM semantic_vector_stages WHERE stage_id=NEW.stage_id - ); -END; - -CREATE TRIGGER IF NOT EXISTS semantic_vector_stage_census_after_chunk_update -AFTER UPDATE ON semantic_vector_stage_chunk_receipts -BEGIN - UPDATE semantic_vector_stage_census_authority - SET revision=revision+1 - WHERE shard_id=( - SELECT shard_id FROM semantic_vector_stages WHERE stage_id=NEW.stage_id - ); -END; - -CREATE TRIGGER IF NOT EXISTS semantic_vector_stage_census_after_chunk_delete -AFTER DELETE ON semantic_vector_stage_chunk_receipts -BEGIN - UPDATE semantic_vector_stage_census_authority - SET revision=revision+1 - WHERE shard_id=( - SELECT shard_id FROM semantic_vector_stages WHERE stage_id=OLD.stage_id - ); -END; - -CREATE TABLE IF NOT EXISTS semantic_vector_stage_graph_effects ( - outbox_sequence INTEGER PRIMARY KEY AUTOINCREMENT, - batch_id INTEGER NOT NULL UNIQUE - REFERENCES semantic_vector_stage_batches(batch_id) ON DELETE RESTRICT, - state TEXT NOT NULL CHECK (state IN ('pending', 'applied', 'failed', 'cancelled')), - terminal_digest TEXT, - CHECK ( - (state = 'pending' AND terminal_digest IS NULL) - OR (state = 'cancelled' AND terminal_digest IS NULL) - OR (state IN ('applied', 'failed') AND terminal_digest IS NOT NULL) - ) -) STRICT; - -CREATE TRIGGER IF NOT EXISTS semantic_vector_stage_census_after_effect_insert -AFTER INSERT ON semantic_vector_stage_graph_effects -BEGIN - UPDATE semantic_vector_stage_census_authority - SET revision=revision+1 - WHERE shard_id=( - SELECT s.shard_id - FROM semantic_vector_stage_batches b - JOIN semantic_vector_stages s ON s.stage_id=b.stage_id - WHERE b.batch_id=NEW.batch_id - ); -END; - -CREATE TRIGGER IF NOT EXISTS semantic_vector_stage_census_after_effect_update -AFTER UPDATE ON semantic_vector_stage_graph_effects -BEGIN - UPDATE semantic_vector_stage_census_authority - SET revision=revision+1 - WHERE shard_id=( - SELECT s.shard_id - FROM semantic_vector_stage_batches b - JOIN semantic_vector_stages s ON s.stage_id=b.stage_id - WHERE b.batch_id=NEW.batch_id - ); -END; - -CREATE TRIGGER IF NOT EXISTS semantic_vector_stage_census_after_effect_delete -AFTER DELETE ON semantic_vector_stage_graph_effects -BEGIN - UPDATE semantic_vector_stage_census_authority - SET revision=revision+1 - WHERE shard_id=( - SELECT s.shard_id - FROM semantic_vector_stage_batches b - JOIN semantic_vector_stages s ON s.stage_id=b.stage_id - WHERE b.batch_id=OLD.batch_id - ); -END; - -CREATE INDEX IF NOT EXISTS idx_semantic_vector_pending_effects - ON semantic_vector_stage_graph_effects(state, outbox_sequence); - -CREATE TABLE IF NOT EXISTS handoff_open_grants_v1 ( - token_digest TEXT NOT NULL PRIMARY KEY, - issued_request_id TEXT NOT NULL UNIQUE, - grant_payload TEXT NOT NULL, - issued_at INTEGER NOT NULL, - expires_at INTEGER NOT NULL CHECK (expires_at > issued_at), - consumed_request_id TEXT, - consumed_input_digest TEXT, - consumption_payload TEXT, - CHECK ( - (consumed_request_id IS NULL - AND consumed_input_digest IS NULL - AND consumption_payload IS NULL) - OR - (consumed_request_id IS NOT NULL - AND consumed_input_digest IS NOT NULL - AND consumption_payload IS NOT NULL) - ) -) STRICT; - -CREATE TABLE IF NOT EXISTS td_runtime_writer_checkpoint_v1 ( - shard_json TEXT NOT NULL, - incarnation INTEGER NOT NULL CHECK (incarnation > 0), - authority_epoch INTEGER NOT NULL CHECK (authority_epoch > 0), - commit_sequence INTEGER NOT NULL CHECK (commit_sequence > 0), - watermark_json TEXT NOT NULL, - transaction_scope_json TEXT NOT NULL, - original_receipt_json TEXT NOT NULL, - operation_id TEXT NOT NULL, - durability_json TEXT NOT NULL, - committed_at_micros INTEGER NOT NULL, - PRIMARY KEY (shard_json, incarnation) -) WITHOUT ROWID; - -CREATE TABLE IF NOT EXISTS td_runtime_writer_idempotency_v1 ( - shard_json TEXT NOT NULL, - incarnation INTEGER NOT NULL CHECK (incarnation > 0), - authority_epoch INTEGER NOT NULL CHECK (authority_epoch > 0), - idempotency_key TEXT NOT NULL, - request_digest TEXT NOT NULL, - original_receipt_json TEXT NOT NULL, - transaction_scope_json TEXT NOT NULL, - operation_id TEXT NOT NULL, - durability_json TEXT NOT NULL, - committed_at_micros INTEGER NOT NULL, - PRIMARY KEY (shard_json, incarnation, authority_epoch, idempotency_key) -) WITHOUT ROWID; - -CREATE TABLE IF NOT EXISTS td_runtime_writer_outbox_v1 ( - source_shard_json TEXT NOT NULL, - source_incarnation INTEGER NOT NULL CHECK (source_incarnation > 0), - source_authority_epoch INTEGER NOT NULL CHECK (source_authority_epoch > 0), - effect_id TEXT NOT NULL, - ordering_key TEXT NOT NULL, - source_sequence INTEGER NOT NULL CHECK (source_sequence >= 0), - state TEXT NOT NULL CHECK ( - state IN ('pending', 'dispatched', 'effect_unknown', 'acknowledged') - ), - entry_json TEXT NOT NULL, - source_receipt_json TEXT NOT NULL, - transaction_scope_json TEXT NOT NULL, - operation_id TEXT NOT NULL, - durability_json TEXT NOT NULL, - updated_at_micros INTEGER NOT NULL, - PRIMARY KEY (source_shard_json, source_incarnation, source_authority_epoch, effect_id) -) WITHOUT ROWID; - -CREATE INDEX IF NOT EXISTS td_runtime_writer_outbox_ordering_v1 -ON td_runtime_writer_outbox_v1 ( - source_shard_json, - source_incarnation, - source_authority_epoch, - ordering_key, - source_sequence, - effect_id -); - -CREATE UNIQUE INDEX IF NOT EXISTS td_runtime_writer_outbox_effect_v1 -ON td_runtime_writer_outbox_v1 (source_shard_json, effect_id); - -CREATE INDEX IF NOT EXISTS td_runtime_writer_outbox_state_v1 -ON td_runtime_writer_outbox_v1 ( - source_shard_json, - source_incarnation, - source_authority_epoch, - state, - updated_at_micros -); - -CREATE TABLE IF NOT EXISTS td_runtime_writer_inbox_v1 ( - target_shard_json TEXT NOT NULL, - target_incarnation INTEGER NOT NULL CHECK (target_incarnation > 0), - target_authority_epoch INTEGER NOT NULL CHECK (target_authority_epoch > 0), - effect_id TEXT NOT NULL, - ordering_key TEXT NOT NULL, - source_sequence INTEGER NOT NULL CHECK (source_sequence >= 0), - target_sequence INTEGER NOT NULL CHECK (target_sequence > 0), - identity_json TEXT NOT NULL, - receipt_json TEXT NOT NULL, - committed_at_micros INTEGER NOT NULL, - PRIMARY KEY (target_shard_json, target_incarnation, target_authority_epoch, effect_id) -) WITHOUT ROWID; - -CREATE INDEX IF NOT EXISTS td_runtime_writer_inbox_ordering_v1 -ON td_runtime_writer_inbox_v1 ( - target_shard_json, - target_incarnation, - target_authority_epoch, - ordering_key, - source_sequence, - effect_id -); - -CREATE UNIQUE INDEX IF NOT EXISTS td_runtime_writer_inbox_effect_v1 -ON td_runtime_writer_inbox_v1 (target_shard_json, effect_id); - diff --git a/crates/tracedecay-rusqlite-runtime/src/checkpoint/controller.rs b/crates/tracedecay-rusqlite-runtime/src/checkpoint/controller.rs index 0766bbbbc5..834511681a 100644 --- a/crates/tracedecay-rusqlite-runtime/src/checkpoint/controller.rs +++ b/crates/tracedecay-rusqlite-runtime/src/checkpoint/controller.rs @@ -149,6 +149,26 @@ impl WriterCheckpointController { ) } + /// Returns the whole WAL to the database as the writer stops. + /// + /// Only the writer's own shutdown may call this: its admission is closed, + /// its queues are empty, and the attachment released every reader before + /// joining it, so the writer holds the exclusivity a maintenance permit + /// would otherwise prove. A reader outside this attachment can still keep + /// the checkpoint pending; the WAL then stays for the next open. + pub(crate) fn truncate_at_shutdown( + &mut self, + ) -> Result> { + let sample = self.driver.sample_wal().map_err(CheckpointError::Driver)?; + let decision = self.run_checkpoint( + CheckpointMode::Truncate, + self.pressure(sample.bytes), + sample.bytes, + CheckpointBlockers::default(), + )?; + Ok(CheckpointResult::Decision { sample, decision }) + } + fn run_exclusive( &mut self, mode: CheckpointMode, diff --git a/crates/tracedecay-rusqlite-runtime/src/exact_sql/tests/lease.rs b/crates/tracedecay-rusqlite-runtime/src/exact_sql/tests/lease.rs index 6c0375eccc..fd5160708f 100644 --- a/crates/tracedecay-rusqlite-runtime/src/exact_sql/tests/lease.rs +++ b/crates/tracedecay-rusqlite-runtime/src/exact_sql/tests/lease.rs @@ -169,138 +169,3 @@ fn active_transaction_hits_absolute_lease_and_releases_writer() { .execute_batch("CREATE TABLE after_absolute_expiry (value INTEGER)".to_owned()) .unwrap(); } - -/// Seeds `rows` of the shape a payload-copying predecessor left behind, in -/// batches that each fit inside one execution, so the fixture itself never -/// depends on the limit the test is about. -fn seed_migration_source(channel: &ExactSqlHandle, rows: i64) { - const BATCH: i64 = 25_000; - let mut seeded = 0; - while seeded < rows { - let batch = BATCH.min(rows - seeded); - channel - .execute_batch(format!( - "INSERT INTO source (key, payload) - WITH RECURSIVE row_index(index_value) AS ( - SELECT {start} UNION ALL - SELECT index_value + 1 FROM row_index WHERE index_value < {end} - ) - SELECT index_value, - '{{\"mutation_digest\":\"sha256:mut' || index_value - || '\",\"partition_digest\":\"sha256:part\"}}' - FROM row_index", - start = seeded + 1, - end = seeded + batch, - )) - .unwrap(); - seeded += batch; - } -} - -/// Rows the store-sized fixture seeds. -/// -/// Sized for the retired projected-object copy's repeated JSON extraction: -/// the whole-table form exceeds the shortened test execution limit, while a -/// production-sized chunk of the same statement leaves headroom. The fixture -/// seed uses smaller statements so seeding does not consume that limit. -const STORE_SIZED_ROWS: i64 = 3_000_000; - -/// Why a store-sized migration cannot run as one statement inside a caller's -/// leased transaction, which is what took a daemon down on a large store: a -/// schema stage rewrote whole tables and rebuilt an index on every open, each -/// as a single statement. On a real store each one outran its execution -/// deadline, `SQLite` interrupted it, and the open failed, every open, since -/// the rewrite never got far enough to retire anything. -/// -/// The same move is driven both ways over one fixture. As one statement the -/// limit must refuse it, which is what makes chunking a migration load -/// bearing rather than decorative. At the chunk size a migration actually -/// writes, that same statement must finish with the limit far from reach, -/// the progress guard bounds statement work, not the later commit/fsync, -/// and the chunk must still commit so the durable move is proven. Headroom -/// is timed on the statement alone so an unpaid WAL from the store-sized -/// seed cannot masquerade as chunk cost. Whether the chunk loop then -/// finishes a whole table is settled where the migrations live. -#[test] -fn a_store_sized_statement_is_refused_and_a_migration_chunk_has_headroom() { - // The non-null predicate is part of the real projected-object migration; - // dropping its second JSON extraction made this store-sized copy complete - // inside the test deadline on a fast host. - const MOVE: &str = "INSERT OR IGNORE INTO moved (key, mutation_digest) - SELECT key, - json_extract(payload, '$.mutation_digest') - FROM source WHERE key <= ? - AND json_extract(payload, '$.mutation_digest') IS NOT NULL"; - - let fixture = fixture('a', 'a'); - let channel = ExactSqlHandle::attach(&fixture.writer, &fixture.readers).unwrap(); - channel - .execute_batch( - "CREATE TABLE source (key INTEGER PRIMARY KEY, payload TEXT NOT NULL); - CREATE TABLE moved ( - key INTEGER PRIMARY KEY, - mutation_digest TEXT NOT NULL - );" - .to_owned(), - ) - .unwrap(); - seed_migration_source(&channel, STORE_SIZED_ROWS); - - let transaction = channel.begin_immediate().unwrap(); - let started = Instant::now(); - let error = transaction - .execute(statement( - MOVE, - vec![ExactSqlValue::Integer(STORE_SIZED_ROWS)], - )) - .expect_err("a store-sized rewrite cannot fit in one leased execution"); - let refused_after = started.elapsed(); - assert!( - matches!(&error, ExactSqlError::Sqlite { message, .. } if message.contains("interrupted")), - "expected the execution limit to interrupt the whole-table rewrite, got: {error}" - ); - assert!( - refused_after < EXACT_SQL_EXECUTION_LIMIT * 2, - "the limit must refuse the statement at its deadline, not long after: {refused_after:?}" - ); - transaction.rollback().unwrap(); - assert_eq!( - moved_rows(&channel), - 0, - "the refused statement must move nothing" - ); - - let chunk = channel.begin_immediate().unwrap(); - let started = Instant::now(); - chunk - .execute(statement( - MOVE, - vec![ExactSqlValue::Integer( - crate::repository::RETIRED_MUTATION_COPY_CHUNK_ROWS, - )], - )) - .expect("a bounded chunk of the same move fits inside one execution"); - let chunk_took = started.elapsed(); - chunk.commit().unwrap(); - assert_eq!( - moved_rows(&channel), - crate::repository::RETIRED_MUTATION_COPY_CHUNK_ROWS - ); - assert!( - chunk_took * 4 < EXACT_SQL_EXECUTION_LIMIT, - "a migration chunk must leave the limit room to spare on a slower host: {chunk_took:?}" - ); -} - -fn moved_rows(channel: &ExactSqlHandle) -> i64 { - let rows = channel - .query( - statement("SELECT count(*) FROM moved", vec![]), - Duration::from_secs(5), - ) - .unwrap(); - match rows.rows[0].values[0] { - ExactSqlValue::Integer(count) => count, - ref other => panic!("count(*) must be an integer, got {other:?}"), - } -} diff --git a/crates/tracedecay-rusqlite-runtime/src/ledger.rs b/crates/tracedecay-rusqlite-runtime/src/ledger.rs index 10a34d6a70..1fb9e0a97e 100644 --- a/crates/tracedecay-rusqlite-runtime/src/ledger.rs +++ b/crates/tracedecay-rusqlite-runtime/src/ledger.rs @@ -24,12 +24,8 @@ pub(crate) use idempotency::{LedgerDisposition, lookup_receipt}; pub(crate) use inbox::lookup as lookup_inbox; #[cfg(test)] pub(crate) use outbox::outbox_entry; -pub use schema::{ - COPY_RETIRED_IDEMPOTENCY_LEDGER_PAGE_SQL, DELETE_CONVERGED_IDEMPOTENCY_LEDGER_PAGE_SQL, - DROP_RETIRED_IDEMPOTENCY_LEDGER_SQL, RETIRED_IDEMPOTENCY_LEDGER_PRESENT_SQL, - RUNTIME_LEDGER_SCHEMA, -}; -pub(crate) use schema::{initialize_schema, retired_idempotency_ledger_present}; +pub use schema::RUNTIME_LEDGER_SCHEMA; +pub(crate) use schema::initialize_schema; #[cfg(test)] mod tests; diff --git a/crates/tracedecay-rusqlite-runtime/src/ledger/commit.rs b/crates/tracedecay-rusqlite-runtime/src/ledger/commit.rs index aafef25f1c..b2744cde82 100644 --- a/crates/tracedecay-rusqlite-runtime/src/ledger/commit.rs +++ b/crates/tracedecay-rusqlite-runtime/src/ledger/commit.rs @@ -6,7 +6,7 @@ use tracedecay_store::{ use super::{ LedgerDisposition, LedgerError, checkpoint, idempotency, inbox, outbox, prune, - sqlite::{LedgerTransaction, Submission, encode_json}, + sqlite::{LedgerTransaction, Submission}, }; enum RuntimeBookkeeping<'a> { @@ -30,9 +30,37 @@ pub(crate) fn record_commit( outbox_entry .map(RuntimeBookkeeping::Outbox) .unwrap_or(RuntimeBookkeeping::None), + ReplayAuthority::Ledger, ) } +/// Where a resubmission of an already-committed operation is answered. +#[derive(Clone, Copy, PartialEq, Eq)] +enum ReplayAuthority { + /// The ledger retains the original receipt and replays it. + Ledger, + /// The repository executor settles a resubmission from its own durable + /// state, so a ledger copy of every receipt would only duplicate it: + /// external-source receipts are keyed by the same idempotency identity, + /// and an observation already stored is classified as an exact + /// duplicate. Cursor advances stay on the ledger: its replay is what tells + /// a losing concurrent owner that the frontier was not its commit. + Repository, +} + +impl ReplayAuthority { + fn for_payload(payload: &RepositoryWritePayloadV1) -> Self { + match payload { + RepositoryWritePayloadV1::ExternalSource(_) + | RepositoryWritePayloadV1::ExternalSourceBatch(_) + | RepositoryWritePayloadV1::ExternalSourceProjection(_) + | RepositoryWritePayloadV1::Observation(_) + | RepositoryWritePayloadV1::ObservationBatch(_) => Self::Repository, + _ => Self::Ledger, + } + } +} + pub(crate) fn record_runtime_commit( transaction: &impl LedgerTransaction, metadata: &StoreOperationMetadataV1, @@ -47,7 +75,13 @@ pub(crate) fn record_runtime_commit( } _ => RuntimeBookkeeping::None, }; - record_with_bookkeeping(transaction, metadata, transaction_scope, bookkeeping) + record_with_bookkeeping( + transaction, + metadata, + transaction_scope, + bookkeeping, + ReplayAuthority::for_payload(payload), + ) } #[hotpath::measure(label = "rusqlite.ledger.record_commit")] @@ -56,11 +90,14 @@ fn record_with_bookkeeping( metadata: &StoreOperationMetadataV1, transaction_scope: &RuntimeTransactionScopeV1, bookkeeping: RuntimeBookkeeping<'_>, + replay: ReplayAuthority, ) -> Result { let submission = Submission::new(metadata, transaction_scope)?; - match idempotency::disposition(transaction, &submission)? { - LedgerDisposition::New => {} - existing => return Ok(existing), + if replay == ReplayAuthority::Ledger { + match idempotency::disposition(transaction, &submission)? { + LedgerDisposition::New => {} + existing => return Ok(existing), + } } let checkpoint = checkpoint::next(transaction, &submission)?; @@ -73,7 +110,6 @@ fn record_with_bookkeeping( commit_sequence: checkpoint.watermark.commit_sequence, committed_at: metadata.admitted_at, }; - let receipt_json = encode_json(&receipt, "original_receipt_json")?; checkpoint::persist(transaction, &submission, &checkpoint, &receipt)?; // The persisted checkpoint is the validated authority for which of this // incarnation's records are now unreachable, so the prune runs after @@ -81,7 +117,9 @@ fn record_with_bookkeeping( // so a backlog converges; the record inserted below sits at the persisted // epoch and is never eligible. prune::prune_superseded(transaction, &submission, &checkpoint)?; - idempotency::insert(transaction, &submission, &receipt, &receipt_json)?; + if replay == ReplayAuthority::Ledger { + idempotency::insert(transaction, &submission, &receipt)?; + } match bookkeeping { RuntimeBookkeeping::None => {} RuntimeBookkeeping::Outbox(entry) => { diff --git a/crates/tracedecay-rusqlite-runtime/src/ledger/idempotency.rs b/crates/tracedecay-rusqlite-runtime/src/ledger/idempotency.rs index 56237ea7c7..b2aae39bb8 100644 --- a/crates/tracedecay-rusqlite-runtime/src/ledger/idempotency.rs +++ b/crates/tracedecay-rusqlite-runtime/src/ledger/idempotency.rs @@ -1,35 +1,31 @@ use rusqlite::{Row, params}; +use tracedecay_domain::UtcMicros; use tracedecay_store::{ - CommandDigestV1, DurabilityClassV1, IdempotencyIdentityV1, RuntimeTransactionScopeV1, - StoreCommitReceiptV1, StoreIdempotencyKeyV1, StoreOperationIdV1, StoreRuntimeBindingV1, + CommandDigestV1, CommitSequenceV1, DurabilityClassV1, IdempotencyIdentityV1, + OperationPriorityV1, RuntimeBatchCompatibilityV1, RuntimeTransactionIdV1, + RuntimeTransactionScopeV1, StoreCommitReceiptV1, StoreIdempotencyKeyV1, StoreOperationIdV1, + StoreRuntimeBindingV1, }; use super::{ LedgerError, - sqlite::{BindingKey, LedgerTransaction, Submission, decode_json, sqlite_u64}, + sqlite::{BindingKey, LedgerTransaction, Submission, decode_json, encode_json, sqlite_u64}, }; const IDEMPOTENCY_TABLE: &str = "td_runtime_writer_idempotency_v2"; const SELECT_IDEMPOTENCY: &str = r#" -SELECT request_digest, original_receipt_json, transaction_scope_json, - operation_id, durability_json, committed_at_micros +SELECT request_digest, operation_id, transaction_id, commit_sequence, + durability_json, priority_json, opened_at_micros, committed_at_micros FROM td_runtime_writer_idempotency_v2 WHERE shard_json = ?1 AND incarnation = ?2 AND authority_epoch = ?3 AND idempotency_key = ?4 "#; -const SELECT_RETIRED_IDEMPOTENCY: &str = r#" -SELECT request_digest, original_receipt_json, transaction_scope_json, - operation_id, durability_json, committed_at_micros -FROM td_runtime_writer_idempotency_v1 -WHERE shard_json = ?1 AND incarnation = ?2 AND authority_epoch = ?3 - AND idempotency_key = ?4 -"#; const INSERT_IDEMPOTENCY: &str = r#" INSERT OR IGNORE INTO td_runtime_writer_idempotency_v2 ( shard_json, incarnation, authority_epoch, idempotency_key, request_digest, - original_receipt_json, transaction_scope_json, operation_id, durability_json, - committed_at_micros -) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10) + operation_id, transaction_id, commit_sequence, durability_json, priority_json, + opened_at_micros, committed_at_micros +) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12) "#; #[derive(Debug)] @@ -40,7 +36,6 @@ pub(crate) enum LedgerDisposition { Conflict(StoreCommitReceiptV1), } -#[derive(PartialEq)] struct IdempotencyRecord { request_digest: CommandDigestV1, receipt: StoreCommitReceiptV1, @@ -84,28 +79,8 @@ pub(crate) fn lookup_receipt( transaction: &impl LedgerTransaction, binding: &StoreRuntimeBindingV1, idempotency: &IdempotencyIdentityV1, - include_retired: bool, ) -> Result, LedgerError> { - let current = load(transaction, binding, &idempotency.key)?; - let retired = if include_retired { - load_from( - transaction, - binding, - &idempotency.key, - SELECT_RETIRED_IDEMPOTENCY, - "td_runtime_writer_idempotency_v1", - )? - } else { - None - }; - match (current, retired) { - (Some(current), Some(retired)) if current != retired => Err(LedgerError::Corrupt { - table: IDEMPOTENCY_TABLE, - field: "retired/current idempotency disagreement", - }), - (Some(record), _) | (None, Some(record)) => Ok(Some(record.receipt)), - (None, None) => Ok(None), - } + Ok(load(transaction, binding, &idempotency.key)?.map(|record| record.receipt)) } #[hotpath::measure(label = "rusqlite.ledger.idempotency_insert")] @@ -113,7 +88,6 @@ pub(super) fn insert( transaction: &impl LedgerTransaction, submission: &Submission<'_>, receipt: &StoreCommitReceiptV1, - receipt_json: &str, ) -> Result<(), LedgerError> { let changed = transaction.execute( INSERT_IDEMPOTENCY, @@ -123,10 +97,15 @@ pub(super) fn insert( submission.authority_epoch_sql, submission.metadata.idempotency.key.as_str(), submission.metadata.idempotency.command_digest.as_str(), - receipt_json, - &submission.transaction_scope_json, submission.metadata.operation_id.as_str(), + submission.transaction_scope.transaction_id.as_str(), + sqlite_u64(receipt.commit_sequence.0, "commit sequence")?, &submission.durability_json, + encode_json( + &submission.transaction_scope.compatibility.priority, + "priority_json" + )?, + submission.transaction_scope.opened_at.0, receipt.committed_at.0, ], )?; @@ -140,26 +119,10 @@ fn load( transaction: &impl LedgerTransaction, binding: &StoreRuntimeBindingV1, key: &StoreIdempotencyKeyV1, -) -> Result, LedgerError> { - load_from( - transaction, - binding, - key, - SELECT_IDEMPOTENCY, - IDEMPOTENCY_TABLE, - ) -} - -fn load_from( - transaction: &impl LedgerTransaction, - binding: &StoreRuntimeBindingV1, - key: &StoreIdempotencyKeyV1, - sql: &str, - table: &'static str, ) -> Result, LedgerError> { let binding_key = BindingKey::from_binding(binding)?; let authority_epoch = sqlite_u64(binding.authority_epoch.get(), "authority epoch")?; - let mut statement = transaction.prepare(sql)?; + let mut statement = transaction.prepare(SELECT_IDEMPOTENCY)?; let mut rows = statement.query(params![ &binding_key.shard_json, binding_key.incarnation_sql, @@ -169,57 +132,62 @@ fn load_from( let Some(row) = rows.next()? else { return Ok(None); }; - let record = decode_row(row, binding, key, table)?; - if rows.next()?.is_some() { - return Err(LedgerError::Corrupt { - table, - field: "duplicate idempotency identity", - }); + decode_row(row, binding, key).map(Some) +} + +fn corrupt(field: &'static str) -> LedgerError { + LedgerError::Corrupt { + table: IDEMPOTENCY_TABLE, + field, } - Ok(Some(record)) } fn decode_row( row: &Row<'_>, binding: &StoreRuntimeBindingV1, key: &StoreIdempotencyKeyV1, - table: &'static str, ) -> Result { let request_digest = - CommandDigestV1::new(row.get::<_, String>(0)?).map_err(|_| LedgerError::Corrupt { - table, - field: "request_digest", - })?; - let receipt: StoreCommitReceiptV1 = - decode_json(&row.get::<_, String>(1)?, table, "original_receipt_json")?; - let transaction_scope: RuntimeTransactionScopeV1 = - decode_json(&row.get::<_, String>(2)?, table, "transaction_scope_json")?; + CommandDigestV1::new(row.get::<_, String>(0)?).map_err(|_| corrupt("request_digest"))?; let operation_id = - StoreOperationIdV1::new(row.get::<_, String>(3)?).map_err(|_| LedgerError::Corrupt { - table, - field: "operation_id", - })?; - let durability: DurabilityClassV1 = - decode_json(&row.get::<_, String>(4)?, table, "durability_json")?; - let committed_at_micros: i64 = row.get(5)?; - let receipt_binding = StoreRuntimeBindingV1::new( - receipt.shard_id.clone(), - receipt.incarnation, - receipt.authority_epoch, - ); - if receipt.validate().is_err() - || receipt_binding != *binding - || receipt.idempotency.key != *key - || receipt.idempotency.command_digest != request_digest - || receipt.operation_id != operation_id - || receipt.committed_at.0 != committed_at_micros - || transaction_scope.compatibility.binding != receipt_binding - || transaction_scope.compatibility.durability != durability - { - return Err(LedgerError::Corrupt { - table, - field: "original receipt binding", - }); + StoreOperationIdV1::new(row.get::<_, String>(1)?).map_err(|_| corrupt("operation_id"))?; + let transaction_id = RuntimeTransactionIdV1::new(row.get::<_, String>(2)?) + .map_err(|_| corrupt("transaction_id"))?; + let commit_sequence = + u64::try_from(row.get::<_, i64>(3)?).map_err(|_| corrupt("commit_sequence"))?; + let durability: DurabilityClassV1 = decode_json( + &row.get::<_, String>(4)?, + IDEMPOTENCY_TABLE, + "durability_json", + )?; + let priority: OperationPriorityV1 = decode_json( + &row.get::<_, String>(5)?, + IDEMPOTENCY_TABLE, + "priority_json", + )?; + let receipt = StoreCommitReceiptV1 { + operation_id, + idempotency: IdempotencyIdentityV1 { + key: key.clone(), + command_digest: request_digest.clone(), + }, + shard_id: binding.shard_id.clone(), + incarnation: binding.incarnation, + authority_epoch: binding.authority_epoch, + commit_sequence: CommitSequenceV1(commit_sequence), + committed_at: UtcMicros(row.get(7)?), + }; + let transaction_scope = RuntimeTransactionScopeV1 { + transaction_id, + compatibility: RuntimeBatchCompatibilityV1 { + binding: binding.clone(), + durability, + priority, + }, + opened_at: UtcMicros(row.get(6)?), + }; + if receipt.validate().is_err() || transaction_scope.validate().is_err() { + return Err(corrupt("original receipt binding")); } Ok(IdempotencyRecord { request_digest, diff --git a/crates/tracedecay-rusqlite-runtime/src/ledger/schema.rs b/crates/tracedecay-rusqlite-runtime/src/ledger/schema.rs index 4a7cc67a99..043cedaae8 100644 --- a/crates/tracedecay-rusqlite-runtime/src/ledger/schema.rs +++ b/crates/tracedecay-rusqlite-runtime/src/ledger/schema.rs @@ -15,25 +15,25 @@ CREATE TABLE IF NOT EXISTS td_runtime_writer_checkpoint_v1 ( PRIMARY KEY (shard_json, incarnation) ) WITHOUT ROWID; --- A rowid table with a unique key index, not WITHOUT ROWID: every row carries --- ~1.5 KB of receipt and scope JSON, which exceeds the local payload an index --- b-tree leaf may hold, so the previous WITHOUT ROWID shape spilled each row --- to an overflow page and, under random SHA-256 keys, averaged under one cell --- per page (one store: 423k rows, 1.98 GB, 68% unused). The rowid b-tree packs --- payload sequentially and the key index carries only the key. +-- The receipt and transaction scope are rebuilt from these columns and the +-- key: every other field they carry is the row's binding. A row of about +-- 0.5 KB fits an index b-tree leaf, so the key is the table and no separate +-- unique index repeats it. CREATE TABLE IF NOT EXISTS td_runtime_writer_idempotency_v2 ( shard_json TEXT NOT NULL, incarnation INTEGER NOT NULL CHECK (incarnation > 0), authority_epoch INTEGER NOT NULL CHECK (authority_epoch > 0), idempotency_key TEXT NOT NULL, request_digest TEXT NOT NULL, - original_receipt_json TEXT NOT NULL, - transaction_scope_json TEXT NOT NULL, operation_id TEXT NOT NULL, + transaction_id TEXT NOT NULL, + commit_sequence INTEGER NOT NULL CHECK (commit_sequence > 0), durability_json TEXT NOT NULL, + priority_json TEXT NOT NULL, + opened_at_micros INTEGER NOT NULL, committed_at_micros INTEGER NOT NULL, - UNIQUE (shard_json, incarnation, authority_epoch, idempotency_key) -); + PRIMARY KEY (shard_json, incarnation, authority_epoch, idempotency_key) +) WITHOUT ROWID; CREATE TABLE IF NOT EXISTS td_runtime_writer_outbox_v1 ( source_shard_json TEXT NOT NULL, @@ -104,83 +104,8 @@ CREATE UNIQUE INDEX IF NOT EXISTS td_runtime_writer_inbox_effect_v1 ON td_runtime_writer_inbox_v1 (target_shard_json, effect_id); "#; -pub const RETIRED_IDEMPOTENCY_LEDGER_PRESENT_SQL: &str = r#" -SELECT 1 FROM sqlite_master -WHERE type = 'table' AND name = 'td_runtime_writer_idempotency_v1' -"#; - -/// Copies one bounded page whose key has not reached the current authority. -/// A key already present with different content remains in V1 and makes the -/// convergence caller fail closed instead of choosing either receipt. -pub const COPY_RETIRED_IDEMPOTENCY_LEDGER_PAGE_SQL: &str = r#" -WITH retired_page AS MATERIALIZED ( - SELECT shard_json, incarnation, authority_epoch, idempotency_key, - request_digest, original_receipt_json, transaction_scope_json, - operation_id, durability_json, committed_at_micros - FROM td_runtime_writer_idempotency_v1 - ORDER BY shard_json, incarnation, authority_epoch, idempotency_key - LIMIT 1024 -) -INSERT INTO td_runtime_writer_idempotency_v2 ( - shard_json, incarnation, authority_epoch, idempotency_key, request_digest, - original_receipt_json, transaction_scope_json, operation_id, durability_json, - committed_at_micros -) -SELECT retired.shard_json, retired.incarnation, retired.authority_epoch, - retired.idempotency_key, retired.request_digest, - retired.original_receipt_json, retired.transaction_scope_json, - retired.operation_id, retired.durability_json, retired.committed_at_micros -FROM retired_page AS retired -WHERE NOT EXISTS ( - SELECT 1 FROM td_runtime_writer_idempotency_v2 AS current - WHERE current.shard_json = retired.shard_json - AND current.incarnation = retired.incarnation - AND current.authority_epoch = retired.authority_epoch - AND current.idempotency_key = retired.idempotency_key -) -"#; - -/// Retires only rows now represented byte-for-byte by the current authority. -pub const DELETE_CONVERGED_IDEMPOTENCY_LEDGER_PAGE_SQL: &str = r#" -WITH retired_page AS MATERIALIZED ( - SELECT shard_json, incarnation, authority_epoch, idempotency_key, - request_digest, original_receipt_json, transaction_scope_json, - operation_id, durability_json, committed_at_micros - FROM td_runtime_writer_idempotency_v1 - ORDER BY shard_json, incarnation, authority_epoch, idempotency_key - LIMIT 1024 -) -DELETE FROM td_runtime_writer_idempotency_v1 -WHERE (shard_json, incarnation, authority_epoch, idempotency_key) IN ( - SELECT retired.shard_json, retired.incarnation, retired.authority_epoch, - retired.idempotency_key - FROM retired_page AS retired - JOIN td_runtime_writer_idempotency_v2 AS current - ON current.shard_json = retired.shard_json - AND current.incarnation = retired.incarnation - AND current.authority_epoch = retired.authority_epoch - AND current.idempotency_key = retired.idempotency_key - AND current.request_digest = retired.request_digest - AND current.original_receipt_json = retired.original_receipt_json - AND current.transaction_scope_json = retired.transaction_scope_json - AND current.operation_id = retired.operation_id - AND current.durability_json = retired.durability_json - AND current.committed_at_micros = retired.committed_at_micros -) -"#; - -pub const DROP_RETIRED_IDEMPOTENCY_LEDGER_SQL: &str = "DROP TABLE td_runtime_writer_idempotency_v1"; - pub(crate) fn initialize_schema(transaction: &impl LedgerTransaction) -> Result<(), LedgerError> { transaction .execute_batch(RUNTIME_LEDGER_SCHEMA) .map_err(Into::into) } - -pub(crate) fn retired_idempotency_ledger_present( - transaction: &impl LedgerTransaction, -) -> Result { - Ok(transaction - .prepare(RETIRED_IDEMPOTENCY_LEDGER_PRESENT_SQL)? - .exists([])?) -} diff --git a/crates/tracedecay-rusqlite-runtime/src/ledger/sqlite.rs b/crates/tracedecay-rusqlite-runtime/src/ledger/sqlite.rs index bfe4454c47..85e323c885 100644 --- a/crates/tracedecay-rusqlite-runtime/src/ledger/sqlite.rs +++ b/crates/tracedecay-rusqlite-runtime/src/ledger/sqlite.rs @@ -2,9 +2,9 @@ use std::ops::Deref; use rusqlite::{Savepoint, Statement, Transaction}; use tracedecay_store::{ - DurabilityClassV1, RuntimeTransactionScopeV1, ShardWatermarkV1, StoreCommitReceiptV1, - StoreIncarnationV1, StoreOperationMetadataV1, StoreRuntimeBindingV1, StoreShardIdV1, - TransactionalOutboxEntryV1, + DurabilityClassV1, OperationPriorityV1, RuntimeTransactionScopeV1, ShardWatermarkV1, + StoreCommitReceiptV1, StoreIncarnationV1, StoreOperationMetadataV1, StoreRuntimeBindingV1, + StoreShardIdV1, TransactionalOutboxEntryV1, }; use super::LedgerError; @@ -65,6 +65,7 @@ impl_canonical_json!( StoreShardIdV1, RuntimeTransactionScopeV1, DurabilityClassV1, + OperationPriorityV1, ShardWatermarkV1, StoreCommitReceiptV1, TransactionalOutboxEntryV1, diff --git a/crates/tracedecay-rusqlite-runtime/src/ledger/tests.rs b/crates/tracedecay-rusqlite-runtime/src/ledger/tests.rs index b10940eb2b..e00bb153e0 100644 --- a/crates/tracedecay-rusqlite-runtime/src/ledger/tests.rs +++ b/crates/tracedecay-rusqlite-runtime/src/ledger/tests.rs @@ -52,69 +52,6 @@ fn ledger_records_share_the_callers_transaction_boundary() { ); } -/// Ordinary writer initialization installs the current target without doing -/// store-sized work, while the retained receipt remains available to lookup. -#[test] -fn initialize_schema_leaves_retired_idempotency_for_background_convergence() { - let mut connection = Connection::open_in_memory().unwrap(); - let metadata = metadata("operation.migrated", "key.migrated", 'a'); - let binding = binding(&metadata); - // Seed the retired shape exactly as an older binary created it, then - // commit one receipt into it through the current ledger code by - // temporarily giving the old table the current name. - let transaction = connection.transaction().unwrap(); - initialize_schema(&transaction).unwrap(); - let receipt = commit(&transaction, &metadata); - transaction - .execute_batch( - "CREATE TABLE td_runtime_writer_idempotency_v1 ( - shard_json TEXT NOT NULL, - incarnation INTEGER NOT NULL, - authority_epoch INTEGER NOT NULL, - idempotency_key TEXT NOT NULL, - request_digest TEXT NOT NULL, - original_receipt_json TEXT NOT NULL, - transaction_scope_json TEXT NOT NULL, - operation_id TEXT NOT NULL, - durability_json TEXT NOT NULL, - committed_at_micros INTEGER NOT NULL, - PRIMARY KEY (shard_json, incarnation, authority_epoch, idempotency_key) - ) WITHOUT ROWID; - INSERT INTO td_runtime_writer_idempotency_v1 - SELECT * FROM td_runtime_writer_idempotency_v2; - DROP TABLE td_runtime_writer_idempotency_v2;", - ) - .unwrap(); - transaction.commit().unwrap(); - - let transaction = connection.transaction().unwrap(); - initialize_schema(&transaction).unwrap(); - let retired_present: i64 = transaction - .query_row( - "SELECT COUNT(*) FROM sqlite_master - WHERE type = 'table' AND name = 'td_runtime_writer_idempotency_v1'", - [], - |row| row.get(0), - ) - .unwrap(); - assert_eq!(retired_present, 1, "ordinary initialization retains V1"); - let migrated: i64 = transaction - .query_row( - "SELECT COUNT(*) FROM td_runtime_writer_idempotency_v2", - [], - |row| row.get(0), - ) - .unwrap(); - assert_eq!(migrated, 0, "ordinary initialization does not copy history"); - assert_eq!( - lookup_receipt(&transaction, &binding, &metadata.idempotency, true).unwrap(), - Some(receipt), - "the writer can replay from V1 while convergence is pending" - ); - initialize_schema(&transaction).unwrap(); - assert!(current_watermark(&transaction, &binding).unwrap().is_some()); -} - #[test] fn commit_uses_one_replay_and_conflict_disposition() { let mut connection = Connection::open_in_memory().unwrap(); @@ -148,14 +85,14 @@ fn malformed_canonical_json_fails_closed() { transaction.commit().unwrap(); connection .execute( - "UPDATE td_runtime_writer_idempotency_v2 SET original_receipt_json = '{}'", + "UPDATE td_runtime_writer_idempotency_v2 SET transaction_id = ''", [], ) .unwrap(); let transaction = connection.transaction().unwrap(); assert!(matches!( - lookup_receipt(&transaction, &binding, &metadata.idempotency, false), + lookup_receipt(&transaction, &binding, &metadata.idempotency), Err(LedgerError::Corrupt { .. }) )); } diff --git a/crates/tracedecay-rusqlite-runtime/src/lib.rs b/crates/tracedecay-rusqlite-runtime/src/lib.rs index 84c54da106..3ca5d42780 100644 --- a/crates/tracedecay-rusqlite-runtime/src/lib.rs +++ b/crates/tracedecay-rusqlite-runtime/src/lib.rs @@ -16,19 +16,14 @@ pub mod exact_sql; pub mod handoff; mod hotpath_observe; mod ledger; -/// Canonical schema and bounded convergence statements for the runtime writer -/// ledger installed in registered SQLite stores. +/// Canonical schema for the runtime writer ledger installed in registered +/// SQLite stores. pub mod runtime_ledger { - pub use crate::ledger::{ - COPY_RETIRED_IDEMPOTENCY_LEDGER_PAGE_SQL, DELETE_CONVERGED_IDEMPOTENCY_LEDGER_PAGE_SQL, - DROP_RETIRED_IDEMPOTENCY_LEDGER_SQL, RETIRED_IDEMPOTENCY_LEDGER_PRESENT_SQL, - RUNTIME_LEDGER_SCHEMA, - }; + pub use crate::ledger::RUNTIME_LEDGER_SCHEMA; } pub mod maintenance; mod operation; mod persistence; -pub mod read_consistency; pub mod reader; pub mod remote; pub mod repository; diff --git a/crates/tracedecay-rusqlite-runtime/src/persistence.rs b/crates/tracedecay-rusqlite-runtime/src/persistence.rs index a472e14c1d..b72c7578fe 100644 --- a/crates/tracedecay-rusqlite-runtime/src/persistence.rs +++ b/crates/tracedecay-rusqlite-runtime/src/persistence.rs @@ -15,16 +15,12 @@ use crate::{ pub(crate) struct RuntimeWriterPersistence { executor: E, - may_contain_retired_idempotency: bool, } impl RuntimeWriterPersistence { #[hotpath::skip] pub(crate) const fn new(executor: E) -> Self { - Self { - executor, - may_contain_retired_idempotency: true, - } + Self { executor } } } @@ -39,11 +35,7 @@ where idempotency: &IdempotencyIdentityV1, ) -> Result, StorageRuntimeErrorV1> { ledger::initialize_schema(transaction).map_err(map_ledger_error)?; - let include_retired = self.may_contain_retired_idempotency - && ledger::retired_idempotency_ledger_present(transaction).map_err(map_ledger_error)?; - self.may_contain_retired_idempotency = include_retired; - ledger::lookup_receipt(transaction, binding, idempotency, include_retired) - .map_err(map_ledger_error) + ledger::lookup_receipt(transaction, binding, idempotency).map_err(map_ledger_error) } fn apply_and_record( @@ -216,7 +208,6 @@ mod tests { &savepoint, &binding, &request.envelope().metadata.idempotency, - false, ) .unwrap(), Some(receipt.clone()) diff --git a/crates/tracedecay-rusqlite-runtime/src/read_consistency/mod.rs b/crates/tracedecay-rusqlite-runtime/src/read_consistency/mod.rs deleted file mode 100644 index d71432b710..0000000000 --- a/crates/tracedecay-rusqlite-runtime/src/read_consistency/mod.rs +++ /dev/null @@ -1,7 +0,0 @@ -//! Driver-free watermark and retained-snapshot observation contracts. - -mod ports; - -pub use crate::watermark::CommitWatermarkSubscription; -pub(crate) use crate::watermark::{CommitWatermarkPublicationError, CommittedWatermarkPublisher}; -pub use ports::{CommitWatermarkSource, WatermarkSourceState}; diff --git a/crates/tracedecay-rusqlite-runtime/src/read_consistency/ports.rs b/crates/tracedecay-rusqlite-runtime/src/read_consistency/ports.rs deleted file mode 100644 index 99709320f0..0000000000 --- a/crates/tracedecay-rusqlite-runtime/src/read_consistency/ports.rs +++ /dev/null @@ -1,29 +0,0 @@ -use std::future::Future; -use std::pin::Pin; - -use tracedecay_store::{ShardWatermarkV1, StoreShardIdV1, UnavailableReasonV1}; - -pub type WatermarkFuture<'a> = Pin + Send + 'a>>; - -/// Published writer state. Infrastructure remains represented by the existing -/// driver-neutral unavailability reasons rather than an invented ledger error. -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum WatermarkSourceState { - Available(ShardWatermarkV1), - Unavailable(UnavailableReasonV1), -} - -/// Narrow subscription to successful writer commits. -/// -/// `wait_for_change` must complete immediately if the source has already moved -/// past `after`, and must be cancellation-safe when its future is dropped. This -/// closes the current/subscribe race without exposing the private commit ledger. -pub trait CommitWatermarkSource: Send + Sync { - fn current(&self, shard_id: &StoreShardIdV1) -> WatermarkSourceState; - - fn wait_for_change<'a>( - &'a self, - shard_id: &'a StoreShardIdV1, - after: &'a ShardWatermarkV1, - ) -> WatermarkFuture<'a>; -} diff --git a/crates/tracedecay-rusqlite-runtime/src/reader/locator.rs b/crates/tracedecay-rusqlite-runtime/src/reader/locator.rs index a6b66a66fc..6b05770918 100644 --- a/crates/tracedecay-rusqlite-runtime/src/reader/locator.rs +++ b/crates/tracedecay-rusqlite-runtime/src/reader/locator.rs @@ -7,7 +7,7 @@ use std::{ use tracedecay_store::{StoreRuntimeBindingV1, VerifiedStoreLocatorV1}; -use crate::connection::{OpenedDatabaseFile, OpenedDatabaseFileError}; +use crate::connection::{ConnectionPolicyError, OpenedDatabaseFile, OpenedDatabaseFileError}; /// An existing file whose canonical identity was verified by the daemon. /// @@ -94,8 +94,9 @@ pub enum ReaderStartError { LocatorPathIsNotFile, ThreadSpawn(std::io::Error), StartupChannelClosed, - OpenFailed, - ReadOnlySetupFailed, + OpenFailed(ConnectionPolicyError), + ReadOnlySetupFailed(ConnectionPolicyError), + SchemaProbeFailed(rusqlite::Error), OpenedDatabaseIdentity(OpenedDatabaseFileError), OpenedDatabaseIdentityMismatch { expected: u64, actual: u64 }, } @@ -116,9 +117,17 @@ impl fmt::Display for ReaderStartError { Self::StartupChannelClosed => { f.write_str("SQLite reader exited before reporting startup") } - Self::OpenFailed => f.write_str("failed to open verified SQLite store read-only"), - Self::ReadOnlySetupFailed => { - f.write_str("failed to establish query-only SQLite reader") + Self::OpenFailed(error) => { + write!(f, "failed to open verified SQLite store read-only: {error}") + } + Self::ReadOnlySetupFailed(error) => { + write!(f, "failed to establish query-only SQLite reader: {error}") + } + Self::SchemaProbeFailed(error) => { + write!( + f, + "SQLite reader could not read the schema during startup: {error}" + ) } Self::OpenedDatabaseIdentity(error) => { write!(f, "failed to identify opened SQLite reader file: {error}") @@ -136,6 +145,8 @@ impl Error for ReaderStartError { match self { Self::InvalidReaderBudget(error) => Some(error), Self::ThreadSpawn(error) => Some(error), + Self::OpenFailed(error) | Self::ReadOnlySetupFailed(error) => Some(error), + Self::SchemaProbeFailed(error) => Some(error), Self::OpenedDatabaseIdentity(error) => Some(error), _ => None, } diff --git a/crates/tracedecay-rusqlite-runtime/src/reader/worker.rs b/crates/tracedecay-rusqlite-runtime/src/reader/worker.rs index 36dbbad28e..767f2b9c76 100644 --- a/crates/tracedecay-rusqlite-runtime/src/reader/worker.rs +++ b/crates/tracedecay-rusqlite-runtime/src/reader/worker.rs @@ -408,11 +408,11 @@ pub(crate) fn spawn( let connection = match connection::open(&worker_open_path, ConnectionMode::Reader) { Ok(connection) => connection, Err(error) if error.is_open_failure() => { - let _ = started.send(Err(ReaderStartError::OpenFailed)); + let _ = started.send(Err(ReaderStartError::OpenFailed(error))); return; } - Err(_) => { - let _ = started.send(Err(ReaderStartError::ReadOnlySetupFailed)); + Err(error) => { + let _ = started.send(Err(ReaderStartError::ReadOnlySetupFailed(error))); return; } }; @@ -429,13 +429,12 @@ pub(crate) fn spawn( }; // Opening and policy verification do not enter the database schema. // Complete WAL-index recovery before this worker can race admitted writes. - if connection - .query_row("SELECT count(*) FROM sqlite_schema", [], |row| { + if let Err(error) = + connection.query_row("SELECT count(*) FROM sqlite_schema", [], |row| { row.get::<_, i64>(0) }) - .is_err() { - let _ = started.send(Err(ReaderStartError::ReadOnlySetupFailed)); + let _ = started.send(Err(ReaderStartError::SchemaProbeFailed(error))); return; } let _keep_pinned_database_alive = locator; diff --git a/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs b/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs index c6afd4f92f..7e887471fc 100644 --- a/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs +++ b/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs @@ -1105,6 +1105,11 @@ mod tests { attachment.drain().unwrap(); attachment.close_and_join().unwrap(); + assert_eq!( + fs::metadata(&wal_path).map_or(0, |metadata| metadata.len()), + 0, + "a graceful close must return the whole WAL to the database" + ); { let state = attachment.lock_state(); assert!(state.closed); diff --git a/crates/tracedecay-rusqlite-runtime/src/repository/diagnostics.rs b/crates/tracedecay-rusqlite-runtime/src/repository/diagnostics.rs index dbad7088f0..0c0ff59f76 100644 --- a/crates/tracedecay-rusqlite-runtime/src/repository/diagnostics.rs +++ b/crates/tracedecay-rusqlite-runtime/src/repository/diagnostics.rs @@ -5,9 +5,8 @@ use tracedecay_domain::{ RetrievalAnchorId, SourceSpan, UtcMicros, }; use tracedecay_store::{ - DIAGNOSTIC_STATE_CLEARED, DIAGNOSTIC_STATE_CURRENT, DIAGNOSTIC_STATE_SUPERSEDED, - DiagnosticGenerationSupersessionV1, DiagnosticReadOperationV1, DiagnosticReadResultV1, - DiagnosticRecordStateKindV1, SanitizedCleanDiagnosticSnapshotV1, + DIAGNOSTIC_STATE_CLEARED, DIAGNOSTIC_STATE_CURRENT, DiagnosticReadOperationV1, + DiagnosticReadResultV1, DiagnosticRecordStateKindV1, SanitizedCleanDiagnosticSnapshotV1, diagnostic_evidence_class_name, diagnostic_producer_kind_name, diagnostic_severity_name, diagnostic_snapshot_observation_eq, diagnostic_state_columns, parse_diagnostic_evidence_class, parse_diagnostic_producer_kind, parse_diagnostic_severity, @@ -19,7 +18,6 @@ use super::support::{conversion, invalid, u64_to_i64}; // this executor and the root `DiagnosticsStore` cannot drift apart across a // migration. These aliases keep the SQL below readable. const CURRENT: &str = DIAGNOSTIC_STATE_CURRENT; -const SUPERSEDED: &str = DIAGNOSTIC_STATE_SUPERSEDED; const CLEARED: &str = DIAGNOSTIC_STATE_CLEARED; #[derive(Clone, Default)] @@ -123,48 +121,6 @@ impl DiagnosticExecutor { Ok(()) } - /// Transitions every current record of `request.prior_generation()` into - /// the superseded state, back-pointing at the successor generation, and - /// moves the prior generation's publication row with it. - /// - /// This mirrors `DiagnosticsStore::supersede_generation` exactly: the same - /// two `UPDATE`s over the same predicates, the same `state_generation` - /// back-pointer, and the same refusal to let a generation supersede itself - /// (enforced by [`DiagnosticGenerationSupersessionV1`] before admission, - /// and re-checked here so a hand-built request cannot bypass it). Returns - /// the number of diagnostic rows transitioned. - /// - /// Clearing (the publication path above) and supersession are distinct - /// lanes and must stay so: clearing marks records a newer clean generation - /// replaced wholesale, while supersession preserves a walkable chain from - /// a prior finding to its logical successor. - pub fn execute_supersession( - &mut self, - savepoint: &Savepoint<'_>, - request: &DiagnosticGenerationSupersessionV1, - ) -> rusqlite::Result { - request.validate().map_err(invalid)?; - let prior = request.prior_generation().as_str(); - let successor = request.successor_generation().as_str(); - let transitioned = savepoint.execute( - "UPDATE generation_diagnostics - SET record_state = ?1, state_generation = ?2 - WHERE record_state = ?3 AND generation_id = ?4 - AND publication_revision = ( - SELECT publication_revision FROM diagnostic_generation_publications - WHERE generation_id = ?4 AND record_state = ?3 - )", - params![SUPERSEDED, successor, CURRENT, prior], - )?; - savepoint.execute( - "UPDATE diagnostic_generation_publications - SET record_state = ?1, state_generation = ?2 - WHERE record_state = ?3 AND generation_id = ?4", - params![SUPERSEDED, successor, CURRENT, prior], - )?; - Ok(transitioned as u64) - } - pub fn execute_read( &mut self, snapshot: &Transaction<'_>, @@ -230,105 +186,10 @@ impl DiagnosticExecutor { let record = read_record_by_anchor(snapshot, anchor)?; Ok(DiagnosticReadResultV1::Record(Box::new(record))) } - // Stale findings stay queryable but never re-enter active - // publication, so this lane selects the exact complement of the - // current set rather than naming the two stale states. - DiagnosticReadOperationV1::Stale(generation) => read_records( - snapshot, - "WHERE generation_id = ?1 AND record_state != 'current' - AND publication_revision = (SELECT MAX(publication_revision) - FROM diagnostic_generation_publications WHERE generation_id = ?1) - ORDER BY diagnostic_anchor", - [generation.as_str()], - ) - .map(DiagnosticReadResultV1::Records), - DiagnosticReadOperationV1::SupersessionChain(anchor) => { - read_supersession_chain(snapshot, anchor).map(DiagnosticReadResultV1::Records) - } - } - } -} - -/// Walks the supersession chain from `anchor`, oldest first and including the -/// starting record. -/// -/// Each step follows the record's `Superseded { successor_generation }` edge to -/// the record in the successor generation carrying the same logical finding key, -/// repository, producer, code, file occurrence, span, and message digest. -/// The walk stops at a current, cleared, or missing successor. An anchor -/// already visited also stops the walk, so a cyclic `state_generation` graph -/// cannot spin here. -fn read_supersession_chain( - connection: &rusqlite::Connection, - anchor: &RetrievalAnchorId, -) -> rusqlite::Result> { - let mut chain = Vec::new(); - let Some(start) = read_record_by_anchor(connection, anchor)? else { - return Ok(chain); - }; - chain.push(start); - loop { - let Some(last) = chain.last() else { - return Ok(chain); - }; - let DiagnosticRecordStateV1::Superseded { - successor_generation, - } = &last.state - else { - return Ok(chain); - }; - let Some(successor) = read_logical_successor(connection, last, successor_generation)? - else { - return Ok(chain); - }; - if chain - .iter() - .any(|seen| seen.diagnostic_anchor == successor.diagnostic_anchor) - { - return Ok(chain); } - chain.push(successor); } } -fn read_logical_successor( - connection: &rusqlite::Connection, - prior: &GenerationDiagnosticV1, - successor_generation: &CodeGenerationId, -) -> rusqlite::Result> { - let sql = format!( - "{SELECT_RECORDS} WHERE generation_id = ?1 AND publication_revision = (\ - SELECT MAX(publication_revision) FROM diagnostic_generation_publications \ - WHERE generation_id = ?1) AND repository = ?2 \ - AND producer = ?3 AND code = ?4 AND file_occurrence_id = ?5 \ - AND span_start = ?6 AND span_end = ?7 AND message_digest = ?8 \ - ORDER BY diagnostic_anchor" - ); - let mut statement = connection.prepare_cached(&sql)?; - let mut records = statement - .query_map( - params![ - successor_generation.as_str(), - prior.repository.as_str(), - prior.provenance.producer.as_str(), - prior.code, - prior.file_occurrence_id.as_str(), - u64_to_i64(prior.span.start_byte, "diagnostic span start")?, - u64_to_i64(prior.span.end_byte, "diagnostic span end")?, - prior.message_digest.as_str(), - ], - record_from_row, - )? - .collect::>>()?; - if records.len() > 1 { - return Err(conversion(format!( - "ambiguous logical successor for {} in {successor_generation}", - prior.diagnostic_anchor - ))); - } - Ok(records.pop()) -} - fn insert_record( savepoint: &Savepoint<'_>, publication_revision: i64, @@ -432,12 +293,7 @@ fn record_from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result Some(CodeGenerationId::new(value).map_err(conversion)?), - (Some(_), None) => { - return Err(conversion(match kind { - DiagnosticRecordStateKindV1::Cleared => "cleared diagnostic has no generation", - _ => "superseded diagnostic has no generation", - })); - } + (Some(_), None) => return Err(conversion("cleared diagnostic has no generation")), (None, _) => None, }; let state = kind diff --git a/crates/tracedecay-rusqlite-runtime/src/repository/evidence_assembly/anchor_state.rs b/crates/tracedecay-rusqlite-runtime/src/repository/evidence_assembly/anchor_state.rs deleted file mode 100644 index 3c04c7f785..0000000000 --- a/crates/tracedecay-rusqlite-runtime/src/repository/evidence_assembly/anchor_state.rs +++ /dev/null @@ -1,268 +0,0 @@ -//! Retrieval-anchor liveness as evidence assembly needs to see it. -//! -//! The disposition tables these read are appended to by the root authority -//! in `crates/tracedecay-runtime-core/src/db/retrieval_anchor_authority.rs` -//! as well, so this module only ever reads them. - -use std::collections::{BTreeSet, HashMap, hash_map::Entry}; - -use rusqlite::{OptionalExtension, params, params_from_iter}; -use tracedecay_domain::RetrievalAnchorRecordV3; -use tracedecay_store::{EvidenceSourceOccurrenceRecordV1, RetrievalAnchorOwnerV1}; - -use super::super::support::{decode, encode, invalid}; - -/// The largest `anchor_id IN (...)` batch a single prepared statement binds. -/// -/// A drilldown page carries at most 256 occurrences, each contributing an -/// occurrence anchor and a source anchor, so the deduplicated set never -/// approaches SQLite's default variable ceiling, but chunking keeps the -/// batched liveness load correct if a caller ever exceeds it. -const ANCHOR_LIVENESS_BATCH: usize = 500; - -pub(super) fn evidence_anchor_is_current( - connection: &rusqlite::Connection, - anchor: &RetrievalAnchorRecordV3, -) -> rusqlite::Result { - let owner_json = encode(anchor.owner())?; - let Some((anchor_json, projection_generation)) = connection - .query_row( - "SELECT anchor_json, projection_generation - FROM retrieval_anchors - WHERE anchor_id = ?1 AND owner_json = ?2", - params![anchor.anchor_id().as_str(), owner_json], - |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), - ) - .optional()? - else { - return Ok(false); - }; - if anchor_json != encode(anchor)? - || projection_generation != anchor.projection_generation().as_str() - { - return Err(invalid("evidence retrieval anchor persistence mismatch")); - } - let state = latest_disposition_state(connection, anchor.anchor_id().as_str(), &owner_json)?; - Ok(state.as_deref().is_none_or(|state| state == "active")) -} - -/// Reads the newest disposition recorded for an anchor, if it has one. -/// -/// `None` means the anchor was never disposed, which every caller treats the -/// same as an explicitly active disposition. -fn latest_disposition_state( - connection: &rusqlite::Connection, - anchor_id: &str, - owner_json: &str, -) -> rusqlite::Result> { - connection - .query_row( - "SELECT state FROM retrieval_anchor_dispositions - WHERE anchor_id = ?1 AND owner_json = ?2 - ORDER BY sequence DESC LIMIT 1", - params![anchor_id, owner_json], - |row| row.get::<_, String>(0), - ) - .optional() -} - -/// Confirms the exact source anchor an occurrence names is present and active, -/// returning the anchor's stored `owner_json` so a caller in the same -/// transaction can reuse it instead of reading the row a second time. -pub(super) fn require_source_anchor_current( - connection: &rusqlite::Connection, - occurrence: &EvidenceSourceOccurrenceRecordV1, -) -> rusqlite::Result { - let owner_json = connection - .query_row( - "SELECT owner_json FROM retrieval_anchors WHERE anchor_id = ?1", - [occurrence.exact_source_anchor.as_str()], - |row| row.get::<_, String>(0), - ) - .optional()? - .ok_or_else(|| invalid("evidence source anchor unavailable"))?; - let source_owner: RetrievalAnchorOwnerV1 = decode(owner_json.clone())?; - if !source_owner_matches_assembly(&source_owner, &occurrence.owner) { - return Err(invalid("evidence source anchor owner mismatch")); - } - let state = latest_disposition_state( - connection, - occurrence.exact_source_anchor.as_str(), - &owner_json, - )?; - if state.as_deref().is_none_or(|state| state == "active") { - Ok(owner_json) - } else { - Err(invalid("evidence source anchor is disposed")) - } -} - -/// One `retrieval_anchors` row as the liveness checks need to see it. -struct AnchorRow { - owner_json: String, - anchor_json: String, - projection_generation: String, -} - -/// A batch-loaded view of anchor rows and their latest dispositions, so a page -/// of occurrences can be checked for liveness without a per-occurrence pair of -/// round trips. -/// -/// The cached checks reproduce [`evidence_anchor_is_current`] and -/// [`require_source_anchor_current`] exactly, reading the same columns and -/// returning the same `Ok`/`Err` outcomes, they only replace the individual -/// `SELECT`s with two `anchor_id IN (...)` loads made up front. -pub(super) struct AnchorLivenessCache { - anchors: HashMap, - /// `(anchor_id, owner_json)` to the newest disposition state recorded for - /// it, mirroring [`latest_disposition_state`]'s `ORDER BY sequence DESC`. - dispositions: HashMap<(String, String), String>, -} - -/// Loads every anchor row and latest disposition for `anchor_ids` in two -/// batched statements, regardless of how many occurrences reference them. -pub(super) fn load_anchor_liveness<'a, I>( - connection: &rusqlite::Connection, - anchor_ids: I, -) -> rusqlite::Result -where - I: IntoIterator, -{ - let mut anchors: HashMap = HashMap::new(); - let mut latest: HashMap<(String, String), (i64, String)> = HashMap::new(); - let unique: BTreeSet<&str> = anchor_ids.into_iter().collect(); - let ids: Vec<&str> = unique.into_iter().collect(); - for chunk in ids.chunks(ANCHOR_LIVENESS_BATCH) { - let placeholders = (1..=chunk.len()) - .map(|index| format!("?{index}")) - .collect::>() - .join(", "); - - let mut anchor_statement = connection.prepare(&format!( - "SELECT anchor_id, owner_json, anchor_json, projection_generation - FROM retrieval_anchors - WHERE anchor_id IN ({placeholders})", - ))?; - let anchor_rows = - anchor_statement.query_map(params_from_iter(chunk.iter().copied()), |row| { - Ok(( - row.get::<_, String>(0)?, - AnchorRow { - owner_json: row.get::<_, String>(1)?, - anchor_json: row.get::<_, String>(2)?, - projection_generation: row.get::<_, String>(3)?, - }, - )) - })?; - for row in anchor_rows { - let (anchor_id, anchor) = row?; - anchors.insert(anchor_id, anchor); - } - - let mut disposition_statement = connection.prepare(&format!( - "SELECT anchor_id, owner_json, state, sequence - FROM retrieval_anchor_dispositions - WHERE anchor_id IN ({placeholders})", - ))?; - let disposition_rows = - disposition_statement.query_map(params_from_iter(chunk.iter().copied()), |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, String>(1)?, - row.get::<_, String>(2)?, - row.get::<_, i64>(3)?, - )) - })?; - for row in disposition_rows { - let (anchor_id, owner_json, state, sequence) = row?; - match latest.entry((anchor_id, owner_json)) { - Entry::Occupied(mut occupied) => { - if sequence >= occupied.get().0 { - *occupied.get_mut() = (sequence, state); - } - } - Entry::Vacant(vacant) => { - vacant.insert((sequence, state)); - } - } - } - } - let dispositions = latest - .into_iter() - .map(|(key, (_, state))| (key, state)) - .collect(); - Ok(AnchorLivenessCache { - anchors, - dispositions, - }) -} - -impl AnchorLivenessCache { - /// The batched equivalent of the free [`evidence_anchor_is_current`]. - pub(super) fn evidence_anchor_is_current( - &self, - anchor: &RetrievalAnchorRecordV3, - ) -> rusqlite::Result { - let owner_json = encode(anchor.owner())?; - // A missing row, or one filed under a different owner, is exactly the - // `WHERE anchor_id = ?1 AND owner_json = ?2` miss the row query returns. - let Some(row) = self - .anchors - .get(anchor.anchor_id().as_str()) - .filter(|row| row.owner_json == owner_json) - else { - return Ok(false); - }; - if row.anchor_json != encode(anchor)? - || row.projection_generation != anchor.projection_generation().as_str() - { - return Err(invalid("evidence retrieval anchor persistence mismatch")); - } - let state = self - .dispositions - .get(&(anchor.anchor_id().as_str().to_owned(), owner_json)); - Ok(state - .map(String::as_str) - .is_none_or(|state| state == "active")) - } - - /// The batched equivalent of the free [`require_source_anchor_current`]. - pub(super) fn require_source_anchor_current( - &self, - occurrence: &EvidenceSourceOccurrenceRecordV1, - ) -> rusqlite::Result<()> { - let row = self - .anchors - .get(occurrence.exact_source_anchor.as_str()) - .ok_or_else(|| invalid("evidence source anchor unavailable"))?; - let source_owner: RetrievalAnchorOwnerV1 = decode(row.owner_json.clone())?; - if !source_owner_matches_assembly(&source_owner, &occurrence.owner) { - return Err(invalid("evidence source anchor owner mismatch")); - } - let state = self.dispositions.get(&( - occurrence.exact_source_anchor.as_str().to_owned(), - row.owner_json.clone(), - )); - if state - .map(String::as_str) - .is_none_or(|state| state == "active") - { - Ok(()) - } else { - Err(invalid("evidence source anchor is disposed")) - } - } -} - -fn source_owner_matches_assembly( - source: &RetrievalAnchorOwnerV1, - assembly: &tracedecay_domain::AnchorOwnerBindingV1, -) -> bool { - match source { - RetrievalAnchorOwnerV1::V3(owner) => owner == assembly, - // A V2 owner has no authoritative profile/privacy-domain identity. - // Decoding remains supported, but it cannot establish V3 evidence - // ownership without a separate exact migration binding. - RetrievalAnchorOwnerV1::V2(_) => false, - } -} diff --git a/crates/tracedecay-rusqlite-runtime/src/repository/evidence_assembly/mod.rs b/crates/tracedecay-rusqlite-runtime/src/repository/evidence_assembly/mod.rs deleted file mode 100644 index ef4575493e..0000000000 --- a/crates/tracedecay-rusqlite-runtime/src/repository/evidence_assembly/mod.rs +++ /dev/null @@ -1,241 +0,0 @@ -//! Publishing and reading one evidence assembly. -//! -//! The executor owns the transaction shape; the siblings own the pieces it -//! composes, [`writes`] the replay-safe table inserts, [`reads`] the two read -//! operations, and [`anchor_state`] the retrieval-anchor liveness both consult. - -use rusqlite::{OptionalExtension, Savepoint, Transaction, params}; -use tracedecay_store::{ - EvidenceAssemblyReadOperationV1, EvidenceAssemblyReadResultV1, EvidenceAssemblyWriteV1, -}; - -use super::support::{canonical_digest, decode, encode, invalid, u64_to_i64}; - -mod anchor_state; -mod reads; -mod writes; - -use anchor_state::require_source_anchor_current; -use writes::{ - insert_anchor, insert_derived_anchor, insert_immutable, insert_membership, - insert_span_membership, publish_reverse_lineage, -}; - -#[derive(Clone, Default)] -pub struct EvidenceAssemblyExecutor; - -impl EvidenceAssemblyExecutor { - pub fn execute_write( - &mut self, - savepoint: &Savepoint<'_>, - write: &EvidenceAssemblyWriteV1, - ) -> rusqlite::Result<()> { - write.validate().map_err(invalid)?; - let owner_digest = canonical_digest(&write.owner)?; - let evidence_owner_digest = canonical_digest(&write.owner.owner)?; - if let Some((assembly_digest, receipt_json)) = savepoint - .query_row( - "SELECT assembly_digest, receipt_json - FROM evidence_assembly_receipts - WHERE owner_digest = ?1 AND privacy_domain_id = ?2 - AND key_epoch = ?3 AND idempotency_key = ?4", - params![ - owner_digest, - write.owner.owner.privacy_domain_id().as_str(), - u64_to_i64(write.owner.key_epoch, "evidence assembly key epoch")?, - write.idempotency_key.as_digest().as_str(), - ], - |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), - ) - .optional()? - { - let existing = - decode::(receipt_json)?; - existing.validate().map_err(invalid)?; - return if assembly_digest == write.receipt.assembly_digest.as_str() - && existing == write.receipt - { - Ok(()) - } else { - Err(invalid("evidence assembly replay conflict")) - }; - } - - let mut source_owner_jsons = Vec::with_capacity(write.occurrences.len()); - for occurrence in &write.occurrences { - let source_owner_json = require_source_anchor_current(savepoint, occurrence)?; - source_owner_jsons.push(source_owner_json); - insert_anchor(savepoint, &occurrence.occurrence_anchor)?; - insert_immutable( - savepoint, - "evidence_source_occurrences", - "occurrence_id", - occurrence.occurrence_id.as_str(), - canonical_digest(occurrence)?, - encode(occurrence)?, - &[ - ("owner_digest", evidence_owner_digest.clone()), - ( - "timeline_digest", - occurrence.timeline.digest().map_err(invalid)?.to_string(), - ), - ( - "source_anchor_id", - occurrence.exact_source_anchor.as_str().to_owned(), - ), - ("source_order", occurrence.source_order.to_string()), - ], - )?; - } - - insert_immutable( - savepoint, - "evidence_occurrence_sets", - "occurrence_set_id", - write.occurrence_set.occurrence_set_id.as_str(), - canonical_digest(&write.occurrence_set)?, - encode(&write.occurrence_set)?, - &[("owner_digest", evidence_owner_digest.clone())], - )?; - for (ordinal, occurrence_id) in write.occurrence_set.members.iter().enumerate() { - insert_membership( - savepoint, - "evidence_occurrence_set_members", - "occurrence_set_id", - write.occurrence_set.occurrence_set_id.as_str(), - "canonical_ordinal", - ordinal, - occurrence_id.as_str(), - )?; - } - - insert_anchor(savepoint, &write.span.anchor)?; - insert_immutable( - savepoint, - "evidence_spans", - "span_id", - write.span.span_id.as_str(), - canonical_digest(&write.span)?, - encode(&write.span)?, - &[ - ("owner_digest", evidence_owner_digest.clone()), - ( - "occurrence_set_id", - write.occurrence_set.occurrence_set_id.as_str().to_owned(), - ), - ( - "anchor_id", - write.span.anchor.anchor_id().as_str().to_owned(), - ), - ("producer_kind", "v3".to_owned()), - ], - )?; - let mut assembly_ordinal = 0; - for (run_ordinal, run) in write.span.runs.iter().enumerate() { - for (member_ordinal, occurrence_id) in run.occurrence_ids.iter().enumerate() { - insert_span_membership( - savepoint, - write.span.span_id.as_str(), - assembly_ordinal, - run_ordinal, - member_ordinal, - occurrence_id.as_str(), - )?; - assembly_ordinal = assembly_ordinal - .checked_add(1) - .ok_or_else(|| invalid("evidence span assembly ordinal overflow"))?; - } - } - - insert_immutable( - savepoint, - "evidence_span_projection_receipts", - "projection_receipt_id", - write.projection_receipt.projection_receipt_id.as_str(), - canonical_digest(&write.projection_receipt)?, - encode(&write.projection_receipt)?, - &[("span_id", write.span.span_id.as_str().to_owned())], - )?; - - insert_anchor(savepoint, &write.contribution.anchor)?; - insert_immutable( - savepoint, - "evidence_retriever_contributions", - "contribution_id", - write.contribution.contribution_id.as_str(), - canonical_digest(&write.contribution)?, - encode(&write.contribution)?, - &[ - ("owner_digest", owner_digest.clone()), - ("span_id", write.span.span_id.as_str().to_owned()), - ( - "anchor_id", - write.contribution.anchor.anchor_id().as_str().to_owned(), - ), - ], - )?; - - for anchor in [&write.span.anchor, &write.contribution.anchor] - .into_iter() - .chain( - write - .occurrences - .iter() - .map(|occurrence| &occurrence.occurrence_anchor), - ) - { - insert_derived_anchor(savepoint, anchor, &evidence_owner_digest)?; - } - - publish_reverse_lineage(savepoint, write, &source_owner_jsons)?; - savepoint.execute( - "INSERT INTO evidence_assembly_receipts ( - publication_receipt_id, owner_digest, privacy_domain_id, key_epoch, - idempotency_key, assembly_digest, occurrence_set_id, span_id, - contribution_id, projection_receipt_id, receipt_json - ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)", - params![ - write.receipt.publication_receipt_id.as_str(), - owner_digest, - write.owner.owner.privacy_domain_id().as_str(), - u64_to_i64(write.owner.key_epoch, "evidence assembly key epoch")?, - write.idempotency_key.as_digest().as_str(), - write.receipt.assembly_digest.as_str(), - write.occurrence_set.occurrence_set_id.as_str(), - write.span.span_id.as_str(), - write.contribution.contribution_id.as_str(), - write.projection_receipt.projection_receipt_id.as_str(), - encode(&write.receipt)?, - ], - )?; - Ok(()) - } - - pub fn execute_read( - &mut self, - snapshot: &Transaction<'_>, - operation: &EvidenceAssemblyReadOperationV1, - ) -> rusqlite::Result { - match operation { - EvidenceAssemblyReadOperationV1::PublicationByIdempotency { - owner, - idempotency_key, - } => reads::publication_by_idempotency(snapshot, owner, idempotency_key), - EvidenceAssemblyReadOperationV1::ContributionPage { - owner, - contribution_id, - start_ordinal, - page_size, - } => reads::contribution_page( - snapshot, - owner, - contribution_id, - *start_ordinal, - *page_size, - ), - } - } -} - -#[cfg(any(test, feature = "test-transport"))] -pub mod tests; diff --git a/crates/tracedecay-rusqlite-runtime/src/repository/evidence_assembly/reads.rs b/crates/tracedecay-rusqlite-runtime/src/repository/evidence_assembly/reads.rs deleted file mode 100644 index 33e76e092d..0000000000 --- a/crates/tracedecay-rusqlite-runtime/src/repository/evidence_assembly/reads.rs +++ /dev/null @@ -1,358 +0,0 @@ -//! The two evidence assembly read operations and the persistence checks -//! they make before serving a result. - -use rusqlite::{OptionalExtension, Transaction, params}; -use tracedecay_domain::RetrieverContributionIdV1; -use tracedecay_store::{ - CanonicalSourceOccurrenceSetRecordV1, EvidenceAssemblyDrilldownPageV1, - EvidenceAssemblyIdempotencyKeyV1, EvidenceAssemblyOwnerV1, - EvidenceAssemblyPublicationReceiptV1, EvidenceAssemblyReadResultV1, - EvidenceSourceOccurrenceRecordV1, RetrieverContributionRecordV1, -}; - -use super::super::support::{canonical_digest, decode, invalid, u64_to_i64, usize_to_i64}; -use super::anchor_state::{self, evidence_anchor_is_current}; - -pub(super) fn publication_by_idempotency( - snapshot: &Transaction<'_>, - owner: &EvidenceAssemblyOwnerV1, - idempotency_key: &EvidenceAssemblyIdempotencyKeyV1, -) -> rusqlite::Result { - let receipt = snapshot - .query_row( - "SELECT publication_receipt_id, assembly_digest, occurrence_set_id, - span_id, contribution_id, projection_receipt_id, receipt_json - FROM evidence_assembly_receipts - WHERE owner_digest = ?1 AND privacy_domain_id = ?2 - AND key_epoch = ?3 AND idempotency_key = ?4", - params![ - canonical_digest(owner)?, - owner.owner.privacy_domain_id().as_str(), - u64_to_i64(owner.key_epoch, "evidence assembly key epoch")?, - idempotency_key.as_digest().as_str(), - ], - |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, String>(1)?, - row.get::<_, String>(2)?, - row.get::<_, String>(3)?, - row.get::<_, String>(4)?, - row.get::<_, String>(5)?, - row.get::<_, String>(6)?, - )) - }, - ) - .optional()? - .map( - |( - publication_receipt_id, - assembly_digest, - occurrence_set_id, - span_id, - contribution_id, - projection_receipt_id, - record, - )| { - let receipt: EvidenceAssemblyPublicationReceiptV1 = decode(record)?; - receipt.validate().map_err(invalid)?; - let expected_id = - tracedecay_store::derive_evidence_assembly_publication_receipt_id_v1( - &receipt.identity_projection(idempotency_key), - ) - .map_err(invalid)?; - if &receipt.owner != owner - || receipt.publication_receipt_id != expected_id - || receipt.publication_receipt_id.as_str() != publication_receipt_id - || receipt.assembly_digest.as_str() != assembly_digest - || receipt.occurrence_set_id.as_str() != occurrence_set_id - || receipt.span_id.as_str() != span_id - || receipt.contribution_id.as_str() != contribution_id - || receipt.projection_receipt_id.as_str() != projection_receipt_id - { - return Err(invalid("evidence publication receipt identity")); - } - Ok(receipt) - }, - ) - .transpose()?; - Ok(EvidenceAssemblyReadResultV1::Publication(receipt)) -} - -pub(super) fn contribution_page( - snapshot: &Transaction<'_>, - owner: &EvidenceAssemblyOwnerV1, - contribution_id: &RetrieverContributionIdV1, - start_ordinal: u64, - page_size: u64, -) -> rusqlite::Result { - if page_size == 0 || page_size > 256 { - return Err(invalid("evidence drilldown page size")); - } - let owner_digest = canonical_digest(owner)?; - let evidence_owner_digest = canonical_digest(&owner.owner)?; - let Some(contribution) = snapshot - .query_row( - "SELECT span_id, anchor_id, record_digest, record_json - FROM evidence_retriever_contributions - WHERE contribution_id = ?1 AND owner_digest = ?2", - params![contribution_id.as_str(), owner_digest], - |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, String>(1)?, - row.get::<_, String>(2)?, - row.get::<_, String>(3)?, - )) - }, - ) - .optional()? - .map(|(span_id, anchor_id, record_digest, record_json)| { - let contribution: RetrieverContributionRecordV1 = decode(record_json)?; - contribution.validate().map_err(invalid)?; - if &contribution.contribution_id != contribution_id - || contribution.span_id.as_str() != span_id - || contribution.anchor.anchor_id().as_str() != anchor_id - || canonical_digest(&contribution)? != record_digest - { - return Err(invalid( - "evidence retriever contribution persistence mismatch", - )); - } - Ok(contribution) - }) - .transpose()? - else { - return Ok(EvidenceAssemblyReadResultV1::ContributionPage(None)); - }; - if &contribution.owner != owner { - return Ok(EvidenceAssemblyReadResultV1::ContributionPage(None)); - } - if !evidence_anchor_is_current(snapshot, &contribution.anchor)? { - return Ok(EvidenceAssemblyReadResultV1::ContributionPage(None)); - } - let span: tracedecay_store::EvidenceSpanRecordV1 = snapshot - .query_row( - "SELECT owner_digest, occurrence_set_id, anchor_id, producer_kind, - record_digest, record_json - FROM evidence_spans WHERE span_id = ?1", - [contribution.span_id.as_str()], - |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, String>(1)?, - row.get::<_, String>(2)?, - row.get::<_, String>(3)?, - row.get::<_, String>(4)?, - row.get::<_, String>(5)?, - )) - }, - ) - .and_then( - |( - stored_owner, - occurrence_set_id, - anchor_id, - producer_kind, - record_digest, - record_json, - )| { - let span: tracedecay_store::EvidenceSpanRecordV1 = decode(record_json)?; - span.validate().map_err(invalid)?; - if stored_owner.as_str() != evidence_owner_digest.as_str() - || span.occurrence_set_id.as_str() != occurrence_set_id - || span.anchor.anchor_id().as_str() != anchor_id - || producer_kind != "v3" - || canonical_digest(&span)? != record_digest - { - return Err(invalid("evidence span persistence mismatch")); - } - Ok(span) - }, - )?; - if span.owner != owner.owner - || span.span_id != contribution.span_id - || span.occurrence_set_id != contribution.occurrence_set_id - || &contribution.span_anchor_id != span.anchor.anchor_id() - || contribution.exact_source_anchors != span.exact_source_anchors - { - return Err(invalid("evidence drilldown cross-record binding")); - } - validate_occurrence_set(snapshot, owner, &span)?; - validate_span_members(snapshot, &span)?; - if !evidence_anchor_is_current(snapshot, &span.anchor)? { - return Ok(EvidenceAssemblyReadResultV1::ContributionPage(None)); - } - let end = start_ordinal.saturating_add(page_size); - let mut statement = snapshot.prepare_cached( - "SELECT member.occurrence_id, occurrence.owner_digest, - occurrence.timeline_digest, occurrence.source_anchor_id, - occurrence.source_order, occurrence.record_digest, - occurrence.record_json - FROM evidence_span_members AS member - JOIN evidence_source_occurrences AS occurrence - ON occurrence.occurrence_id = member.occurrence_id - WHERE member.span_id = ?1 - AND member.assembly_ordinal >= ?2 - AND member.assembly_ordinal < ?3 - ORDER BY member.assembly_ordinal", - )?; - let occurrences = statement - .query_map( - params![ - span.span_id.as_str(), - u64_to_i64(start_ordinal, "evidence drilldown start")?, - u64_to_i64(end, "evidence drilldown end")?, - ], - |row| { - let occurrence_id = row.get::<_, String>(0)?; - let stored_owner = row.get::<_, String>(1)?; - let timeline_digest = row.get::<_, String>(2)?; - let source_anchor_id = row.get::<_, String>(3)?; - let source_order = row.get::<_, i64>(4)?; - let record_digest = row.get::<_, String>(5)?; - let occurrence: EvidenceSourceOccurrenceRecordV1 = - row.get::<_, String>(6).and_then(decode)?; - occurrence.validate().map_err(invalid)?; - if occurrence.occurrence_id.as_str() != occurrence_id - || occurrence.owner != owner.owner - || stored_owner.as_str() != evidence_owner_digest.as_str() - || occurrence.timeline.digest().map_err(invalid)?.as_str() != timeline_digest - || occurrence.exact_source_anchor.as_str() != source_anchor_id - || u64_to_i64(occurrence.source_order, "evidence source occurrence order")? - != source_order - || canonical_digest(&occurrence)? != record_digest - { - return Err(invalid("evidence drilldown occurrence binding")); - } - Ok(occurrence) - }, - )? - .collect::>>()?; - let consumed = - start_ordinal.saturating_add(u64::try_from(occurrences.len()).unwrap_or(u64::MAX)); - let liveness = anchor_state::load_anchor_liveness( - snapshot, - occurrences.iter().flat_map(|occurrence| { - [ - occurrence.occurrence_anchor.anchor_id().as_str(), - occurrence.exact_source_anchor.as_str(), - ] - }), - )?; - for occurrence in &occurrences { - if !liveness.evidence_anchor_is_current(&occurrence.occurrence_anchor)? { - return Ok(EvidenceAssemblyReadResultV1::ContributionPage(None)); - } - liveness.require_source_anchor_current(occurrence)?; - } - let total = u64::try_from(span.ordered_occurrence_ids().len()).unwrap_or(u64::MAX); - Ok(EvidenceAssemblyReadResultV1::ContributionPage(Some( - EvidenceAssemblyDrilldownPageV1 { - occurrence_set_id: contribution.occurrence_set_id.clone(), - contribution, - span, - occurrences, - next_ordinal: (consumed < total).then_some(consumed), - }, - ))) -} - -fn validate_occurrence_set( - connection: &rusqlite::Connection, - owner: &tracedecay_store::EvidenceAssemblyOwnerV1, - span: &tracedecay_store::EvidenceSpanRecordV1, -) -> rusqlite::Result<()> { - let (owner_digest, record_digest, record_json) = connection.query_row( - "SELECT owner_digest, record_digest, record_json - FROM evidence_occurrence_sets WHERE occurrence_set_id = ?1", - [span.occurrence_set_id.as_str()], - |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, String>(1)?, - row.get::<_, String>(2)?, - )) - }, - )?; - let occurrence_set: CanonicalSourceOccurrenceSetRecordV1 = decode(record_json)?; - occurrence_set.validate().map_err(invalid)?; - if occurrence_set.occurrence_set_id != span.occurrence_set_id - || occurrence_set.owner != owner.owner - || owner_digest != canonical_digest(&owner.owner)? - || record_digest != canonical_digest(&occurrence_set)? - { - return Err(invalid("evidence occurrence set persistence mismatch")); - } - let mut statement = connection.prepare_cached( - "SELECT canonical_ordinal, occurrence_id - FROM evidence_occurrence_set_members - WHERE occurrence_set_id = ?1 - ORDER BY canonical_ordinal", - )?; - let members = statement - .query_map([span.occurrence_set_id.as_str()], |row| { - Ok((row.get::<_, i64>(0)?, row.get::<_, String>(1)?)) - })? - .collect::>>()?; - if members.len() != occurrence_set.members.len() { - return Err(invalid("evidence occurrence set membership mismatch")); - } - for (ordinal, ((stored_ordinal, stored_id), expected_id)) in - members.iter().zip(&occurrence_set.members).enumerate() - { - if *stored_ordinal != usize_to_i64(ordinal, "evidence canonical occurrence ordinal")? - || stored_id != expected_id.as_str() - { - return Err(invalid("evidence occurrence set membership mismatch")); - } - } - Ok(()) -} - -fn validate_span_members( - connection: &rusqlite::Connection, - span: &tracedecay_store::EvidenceSpanRecordV1, -) -> rusqlite::Result<()> { - let mut statement = connection.prepare_cached( - "SELECT assembly_ordinal, run_ordinal, run_member_ordinal, occurrence_id - FROM evidence_span_members - WHERE span_id = ?1 - ORDER BY assembly_ordinal", - )?; - let members = statement - .query_map([span.span_id.as_str()], |row| { - Ok(( - row.get::<_, i64>(0)?, - row.get::<_, i64>(1)?, - row.get::<_, i64>(2)?, - row.get::<_, String>(3)?, - )) - })? - .collect::>>()?; - let expected = - span.runs - .iter() - .enumerate() - .flat_map(|(run_ordinal, run)| { - run.occurrence_ids.iter().enumerate().map( - move |(run_member_ordinal, occurrence_id)| { - (run_ordinal, run_member_ordinal, occurrence_id) - }, - ) - }) - .collect::>(); - if members.len() != expected.len() { - return Err(invalid("evidence span membership mismatch")); - } - for (assembly_ordinal, (member, expected)) in members.iter().zip(expected).enumerate() { - if member.0 != usize_to_i64(assembly_ordinal, "evidence assembly ordinal")? - || member.1 != usize_to_i64(expected.0, "evidence run ordinal")? - || member.2 != usize_to_i64(expected.1, "evidence run member ordinal")? - || member.3 != expected.2.as_str() - { - return Err(invalid("evidence span membership mismatch")); - } - } - Ok(()) -} diff --git a/crates/tracedecay-rusqlite-runtime/src/repository/evidence_assembly/tests.rs b/crates/tracedecay-rusqlite-runtime/src/repository/evidence_assembly/tests.rs deleted file mode 100644 index 4eaeaa3a1a..0000000000 --- a/crates/tracedecay-rusqlite-runtime/src/repository/evidence_assembly/tests.rs +++ /dev/null @@ -1,1000 +0,0 @@ -use super::*; -use tracedecay_domain::{ - AccessPolicyDigest, AnchorDurabilityClass, AnchorLineageRefV3, AnchorOwnerBindingV1, - AnchorProvenanceRelationV2, AnchorSourceGenerationV3, CoverageReportV1, - EvidenceAssemblyPublicationReceiptIdV1, EvidenceClass, ManifestDigest, - ObservationOrderingDomainV1, ObservationScopeV1, ObservationSourceGenerationV1, - ObservationSourceIdentityV1, ObservationSourceRangeV1, PayloadAccessState, - PrivacyDomainBoundLocatorDigest, PrivacyDomainId, ProjectId, ProjectionGenerationId, - ProviderId, ResolutionAuthorizationV1, RetentionClass, RetrievalAnchorId, - RetrievalAnchorRecordV3, RetrievalAnchorRecordV3Parts, RetrievalAnchorTargetV3, - SanitizationReceiptId, SanitizationReceiptRefV1, ScopeResolutionId, SessionId, UserProfileId, - UtcMicros, VectorWatermark, -}; -use tracedecay_store::{ - CanonicalSourceOccurrenceSetIdentityProjectionV1, CanonicalSourceOccurrenceSetRecordV1, - EvidenceAssemblyIdempotencyKeyV1, EvidenceAssemblyOwnerV1, - EvidenceAssemblyPublicationReceiptV1, EvidenceSourceOccurrenceRecordV1, - EvidenceSourceTimelineV1, EvidenceSpanCatalogBindingV1, EvidenceSpanHorizonV1, - EvidenceSpanIdentityProjectionV1, EvidenceSpanMemberReceiptBindingV1, - EvidenceSpanProjectionReceiptIdentityProjectionV1, EvidenceSpanProjectionReceiptV1, - EvidenceSpanRecordV1, EvidenceSpanRunV1, PrivacyBoundRequestDigestV1, - PrivacyBoundRequestEnvelopeV1, RetrieverContributionIdentityProjectionV1, - RetrieverContributionRecordV1, RetrieverIdentityV1, RetrieverWatermarkBindingV1, - SanitizedObservationByteRangeV1, SourceCapabilityCatalogBindingV1, - SourceOccurrenceCoordinateV1, SourceOccurrenceIdentityProjectionV1, SourceOccurrenceKindV1, - SourceOccurrenceSanitizationV1, VerifiedSourceOrderingProofV1, - derive_canonical_source_occurrence_set_id_v1, - derive_evidence_assembly_publication_receipt_id_v1, derive_evidence_span_id_v1, - derive_evidence_span_projection_receipt_id_v1, derive_retriever_contribution_id_v1, - derive_source_occurrence_id_v1, -}; -#[cfg(test)] -use tracedecay_store::{ - RetrievalAnchorReadOperationV1, RetrievalAnchorReadResultV1, StoredRetrievalAnchorRecordV1, -}; - -const DIGEST: &str = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; - -fn owner(project_id: ProjectId) -> EvidenceAssemblyOwnerV1 { - EvidenceAssemblyOwnerV1 { - owner: AnchorOwnerBindingV1::for_project( - UserProfileId::new("profile.fixture").unwrap(), - project_id, - PrivacyDomainId::new("privacy.fixture").unwrap(), - ) - .unwrap(), - scope_digest: ManifestDigest::new(DIGEST).unwrap(), - key_epoch: 1, - } -} - -fn timeline(project_id: ProjectId) -> EvidenceSourceTimelineV1 { - EvidenceSourceTimelineV1 { - source: ObservationSourceIdentityV1::for_provider( - ProviderId::new("provider.fixture").unwrap(), - SessionId::new("session.fixture").unwrap(), - ) - .unwrap(), - scope: ObservationScopeV1::Project { project_id }, - source_generation: ObservationSourceGenerationV1::new(1).unwrap(), - ordering_domain: ObservationOrderingDomainV1::DaemonSequence, - } -} - -fn catalog_binding() -> SourceCapabilityCatalogBindingV1 { - SourceCapabilityCatalogBindingV1 { - connector_id: "connector.fixture".to_owned(), - root_id: "root.fixture".to_owned(), - capability_id: tracedecay_domain::CapabilityId::new("capability.fixture").unwrap(), - catalog_digest: ManifestDigest::new(DIGEST).unwrap(), - integration_manifest_digest: ManifestDigest::new(DIGEST).unwrap(), - configuration_digest: ManifestDigest::new(DIGEST).unwrap(), - authorization_scope_digest: ManifestDigest::new(DIGEST).unwrap(), - projector_revision: tracedecay_domain::ComponentVersion::new("projector.fixture").unwrap(), - source_watermark: ManifestDigest::new(DIGEST).unwrap(), - } -} - -fn sanitization() -> SourceOccurrenceSanitizationV1 { - SourceOccurrenceSanitizationV1::new( - SanitizationReceiptRefV1::new( - SanitizationReceiptId::new("receipt.capture.fixture").unwrap(), - tracedecay_domain::ComponentVersion::new("sanitizer.fixture").unwrap(), - ) - .unwrap(), - SanitizationReceiptRefV1::new( - SanitizationReceiptId::new("receipt.projection.fixture").unwrap(), - tracedecay_domain::ComponentVersion::new("sanitizer.fixture").unwrap(), - ) - .unwrap(), - ) - .unwrap() -} - -fn anchor( - target: RetrievalAnchorTargetV3, - owner: &EvidenceAssemblyOwnerV1, - sources: Vec, -) -> RetrievalAnchorRecordV3 { - let source_anchors = sources - .into_iter() - .enumerate() - .map(|(ordinal, source)| { - AnchorLineageRefV3::new( - u64::try_from(ordinal).unwrap(), - AnchorProvenanceRelationV2::DerivedFrom, - source, - owner.owner.clone(), - ) - .unwrap() - }) - .collect(); - RetrievalAnchorRecordV3::new(RetrievalAnchorRecordV3Parts { - target, - owner: owner.owner.clone(), - aliases: vec![], - occurred_at: None, - ingested_at: UtcMicros(1), - evidence_class: EvidenceClass::Observed, - source_generation: AnchorSourceGenerationV3::Unknown, - projection_generation: ProjectionGenerationId::new("projection.fixture").unwrap(), - projection_watermark: VectorWatermark::default(), - coverage: CoverageReportV1::default(), - source_observations: vec![], - source_anchors, - authorization: ResolutionAuthorizationV1 { - resolved_scope_id: ScopeResolutionId::new("scope.fixture").unwrap(), - privacy_domain_id: PrivacyDomainId::new("privacy.fixture").unwrap(), - access_policy_digest: AccessPolicyDigest::new(DIGEST).unwrap(), - capability_id: tracedecay_domain::CapabilityId::new("capability.fixture").unwrap(), - canonical_request_digest: PrivacyDomainBoundLocatorDigest::new(DIGEST).unwrap(), - }, - payload_access: PayloadAccessState::Eligible, - retention_class: RetentionClass::new("retention.fixture").unwrap(), - durability: AnchorDurabilityClass::DurableEvidence, - }) - .unwrap() -} - -pub fn write_fixture_for_project( - component_version: &str, - project_id: ProjectId, -) -> (RetrievalAnchorRecordV3, EvidenceAssemblyWriteV1) { - let owner = owner(project_id.clone()); - let timeline = timeline(project_id); - let source_anchor = anchor( - RetrievalAnchorTargetV3::Entity(tracedecay_domain::EntityRef { - id: tracedecay_domain::EntityId::new("entity.source.fixture".to_owned()).unwrap(), - kind: tracedecay_domain::EntityKind::Document, - }), - &owner, - Vec::new(), - ); - let source = source_anchor.anchor_id().clone(); - let coordinate = SourceOccurrenceCoordinateV1::ObservationProjection { - canonical_observation_id: tracedecay_domain::CanonicalObservationIdV1::new(format!( - "sha256:{}", - "33".repeat(32) - )) - .unwrap(), - source_range: ObservationSourceRangeV1::new(7, 8).unwrap(), - projection_output_ordinal: 0, - sanitized_byte_range: SanitizedObservationByteRangeV1::new(0, 8).unwrap(), - }; - let projector_version = tracedecay_domain::ComponentVersion::new("projector.fixture").unwrap(); - let occurrence_id = derive_source_occurrence_id_v1(&SourceOccurrenceIdentityProjectionV1 { - owner: &owner.owner, - timeline: &timeline, - exact_source_anchor: &source, - source_order: 7, - coordinate: &coordinate, - occurrence_kind: SourceOccurrenceKindV1::Message, - relations: &[], - projector_version: &projector_version, - }) - .unwrap(); - let occurrence_anchor = anchor( - RetrievalAnchorTargetV3::ExactSourceOccurrence(occurrence_id.clone()), - &owner, - vec![source.clone()], - ); - let occurrence = EvidenceSourceOccurrenceRecordV1 { - occurrence_id: occurrence_id.clone(), - owner: owner.owner.clone(), - timeline, - exact_source_anchor: source.clone(), - occurrence_anchor: occurrence_anchor.clone(), - source_order: 7, - coordinate, - occurrence_kind: SourceOccurrenceKindV1::Message, - relations: Vec::new(), - projector_version: projector_version.clone(), - sanitization: sanitization(), - knowledge_time: UtcMicros(1), - valid_time: Some(UtcMicros(1)), - }; - let occurrence_set_id = derive_canonical_source_occurrence_set_id_v1( - &CanonicalSourceOccurrenceSetIdentityProjectionV1 { - owner: &owner.owner, - canonical_members: std::slice::from_ref(&occurrence_id), - }, - ) - .unwrap(); - let run = EvidenceSpanRunV1 { - assembly_ordinal: 0, - timeline: occurrence.timeline.clone(), - ordering_proof: VerifiedSourceOrderingProofV1::verify( - occurrence.timeline.clone(), - catalog_binding(), - catalog_binding(), - vec![occurrence_id.clone()], - vec![7], - ) - .unwrap(), - timeline_digest: occurrence.timeline.digest().unwrap(), - first_source_order: 7, - last_source_order: 7, - occurrence_ids: vec![occurrence_id.clone()], - }; - let horizon = EvidenceSpanHorizonV1 { - knowledge_through: UtcMicros(1), - valid_through: Some(UtcMicros(1)), - contains_unknown_valid_time: false, - }; - let span_catalog_binding = EvidenceSpanCatalogBindingV1::SourceCapability { - binding: catalog_binding(), - }; - let span_id = derive_evidence_span_id_v1(&EvidenceSpanIdentityProjectionV1 { - owner: &owner.owner, - occurrence_set_id: &occurrence_set_id, - ordered_runs: std::slice::from_ref(&run), - exact_source_anchors: std::slice::from_ref(&source), - projector_version: &projector_version, - horizon: &horizon, - catalog_binding: &span_catalog_binding, - }) - .unwrap(); - let span_anchor = anchor( - RetrievalAnchorTargetV3::ExactEvidenceSpan(span_id.clone()), - &owner, - vec![occurrence_anchor.anchor_id().clone()], - ); - let span = EvidenceSpanRecordV1 { - span_id: span_id.clone(), - anchor: span_anchor.clone(), - owner: owner.owner.clone(), - occurrence_set_id: occurrence_set_id.clone(), - runs: vec![run], - exact_source_anchors: vec![source.clone()], - projector_version, - horizon: horizon.clone(), - catalog_binding: span_catalog_binding, - }; - let member_receipts = vec![EvidenceSpanMemberReceiptBindingV1 { - occurrence_id: occurrence_id.clone(), - sanitization: sanitization(), - }]; - let projection_receipt_id = derive_evidence_span_projection_receipt_id_v1( - &EvidenceSpanProjectionReceiptIdentityProjectionV1 { - span_id: &span_id, - projector_snapshot: "projector.snapshot.fixture", - projection_generation: &ProjectionGenerationId::new("projection.fixture").unwrap(), - projection_watermark: &VectorWatermark::default(), - source_watermark: &ManifestDigest::new(DIGEST).unwrap(), - member_receipts: &member_receipts, - ordered_occurrence_ids: std::slice::from_ref(&occurrence_id), - exact_source_anchors: std::slice::from_ref(&source), - }, - ) - .unwrap(); - let request_digest = PrivacyBoundRequestDigestV1::derive( - owner.owner.privacy_domain_id().clone(), - owner.key_epoch, - b"fixture-privacy-key", - &PrivacyBoundRequestEnvelopeV1 { - use_case_id: tracedecay_domain::UseCaseId::new("use-case.fixture").unwrap(), - scope_resolution_id: ScopeResolutionId::new("scope.fixture").unwrap(), - temporal_mode: tracedecay_domain::TemporalModeV1::Current, - horizon: horizon.clone(), - requested_capabilities: vec![ - tracedecay_domain::CapabilityId::new("capability.fixture").unwrap(), - ], - }, - ) - .unwrap(); - let retriever = RetrieverIdentityV1 { - capability_id: tracedecay_domain::CapabilityId::new("capability.fixture").unwrap(), - component_version: tracedecay_domain::ComponentVersion::new(component_version).unwrap(), - }; - let watermarks = RetrieverWatermarkBindingV1 { - source_watermark: ManifestDigest::new(DIGEST).unwrap(), - projection_watermark: VectorWatermark::default(), - index_watermark: None, - summary_watermark: None, - }; - let contribution_id = - derive_retriever_contribution_id_v1(&RetrieverContributionIdentityProjectionV1 { - owner: &owner, - retriever: &retriever, - catalog_binding: &catalog_binding(), - request_digest: &request_digest, - scope_resolution_id: &ScopeResolutionId::new("scope.fixture").unwrap(), - temporal_mode: tracedecay_domain::TemporalModeV1::Current, - watermarks: &watermarks, - horizon: &horizon, - occurrence_set_id: &occurrence_set_id, - span_id: &span_id, - span_anchor_id: span_anchor.anchor_id(), - exact_source_anchors: std::slice::from_ref(&source), - coverage: &CoverageReportV1::default(), - }) - .unwrap(); - let contribution_anchor = anchor( - RetrievalAnchorTargetV3::RetrieverContribution(contribution_id.clone()), - &owner, - vec![span_anchor.anchor_id().clone()], - ); - let mut write = EvidenceAssemblyWriteV1 { - owner: owner.clone(), - idempotency_key: EvidenceAssemblyIdempotencyKeyV1::new( - ManifestDigest::new(format!("sha256:{}", "cc".repeat(32))).unwrap(), - ) - .unwrap(), - occurrences: vec![occurrence], - occurrence_set: CanonicalSourceOccurrenceSetRecordV1 { - occurrence_set_id: occurrence_set_id.clone(), - owner: owner.owner.clone(), - members: vec![occurrence_id.clone()], - }, - span, - projection_receipt: EvidenceSpanProjectionReceiptV1 { - projection_receipt_id: projection_receipt_id.clone(), - span_id: span_id.clone(), - projector_snapshot: "projector.snapshot.fixture".to_owned(), - projection_generation: ProjectionGenerationId::new("projection.fixture").unwrap(), - projection_watermark: VectorWatermark::default(), - source_watermark: ManifestDigest::new(DIGEST).unwrap(), - member_receipts, - ordered_occurrence_ids: vec![occurrence_id.clone()], - exact_source_anchors: vec![source.clone()], - }, - contribution: RetrieverContributionRecordV1 { - contribution_id: contribution_id.clone(), - anchor: contribution_anchor.clone(), - owner: owner.clone(), - retriever, - catalog_binding: catalog_binding(), - request_digest, - scope_resolution_id: ScopeResolutionId::new("scope.fixture").unwrap(), - temporal_mode: tracedecay_domain::TemporalModeV1::Current, - watermarks, - horizon, - occurrence_set_id: occurrence_set_id.clone(), - span_id: span_id.clone(), - span_anchor_id: span_anchor.anchor_id().clone(), - exact_source_anchors: vec![source.clone()], - coverage: CoverageReportV1::default(), - created_at: UtcMicros(2), - }, - receipt: EvidenceAssemblyPublicationReceiptV1 { - publication_receipt_id: EvidenceAssemblyPublicationReceiptIdV1::new( - "publication.fixture", - ) - .unwrap(), - owner, - assembly_digest: ManifestDigest::new(DIGEST).unwrap(), - occurrence_set_id, - span_id, - span_anchor_id: span_anchor.anchor_id().clone(), - contribution_id, - contribution_anchor_id: contribution_anchor.anchor_id().clone(), - projection_receipt_id, - ordered_occurrence_ids: vec![occurrence_id], - exact_source_anchors: vec![source], - }, - }; - write.receipt.assembly_digest = write.compute_assembly_digest().unwrap(); - write.receipt.publication_receipt_id = derive_evidence_assembly_publication_receipt_id_v1( - &write.receipt.identity_projection(&write.idempotency_key), - ) - .unwrap(); - write.validate().unwrap(); - (source_anchor, write) -} - -#[cfg(test)] -pub(crate) fn write_fixture(component_version: &str) -> EvidenceAssemblyWriteV1 { - write_fixture_for_project( - component_version, - ProjectId::new("project.fixture").unwrap(), - ) - .1 -} - -#[cfg(test)] -fn install(connection: &rusqlite::Connection) { - // The anchors table is installed from the canonical production DDL, not - // a relaxed local copy: this executor writes anchors in production, so - // a fixture without the real CHECK and UNIQUE clauses would accept rows - // the live table rejects. - connection - .execute_batch(tracedecay_store::RETRIEVAL_ANCHORS_SCHEMA_DDL) - .unwrap(); - connection - .execute_batch( - "CREATE TABLE retrieval_anchor_dispositions ( - sequence INTEGER PRIMARY KEY AUTOINCREMENT, disposition_id TEXT UNIQUE, - anchor_id TEXT, owner_json TEXT, state TEXT, superseded_by TEXT, - reason_class TEXT, effective_at INTEGER, record_json TEXT - ); - CREATE TABLE retrieval_anchor_reverse_lineage ( - source_anchor_id TEXT, owner_json TEXT, derivative_kind TEXT, - derivative_id TEXT, direct_evidence INTEGER, - PRIMARY KEY(source_anchor_id, owner_json, derivative_kind, derivative_id) - ); - CREATE TABLE evidence_source_occurrences ( - occurrence_id TEXT PRIMARY KEY, owner_digest TEXT, timeline_digest TEXT, - source_anchor_id TEXT, source_order INTEGER, record_digest TEXT, record_json TEXT - ); - CREATE TABLE evidence_occurrence_sets ( - occurrence_set_id TEXT PRIMARY KEY, owner_digest TEXT, - record_digest TEXT, record_json TEXT - ); - CREATE TABLE evidence_occurrence_set_members ( - occurrence_set_id TEXT, canonical_ordinal INTEGER, occurrence_id TEXT, - PRIMARY KEY(occurrence_set_id, canonical_ordinal) - ); - CREATE TABLE evidence_spans ( - span_id TEXT PRIMARY KEY, owner_digest TEXT, occurrence_set_id TEXT, - anchor_id TEXT, producer_kind TEXT, record_digest TEXT, record_json TEXT - ); - CREATE TABLE evidence_span_members ( - span_id TEXT, assembly_ordinal INTEGER, run_ordinal INTEGER, - run_member_ordinal INTEGER, occurrence_id TEXT, - PRIMARY KEY(span_id, assembly_ordinal) - ); - CREATE TABLE evidence_span_projection_receipts ( - projection_receipt_id TEXT PRIMARY KEY, span_id TEXT, - record_digest TEXT, record_json TEXT - ); - CREATE TABLE evidence_retriever_contributions ( - contribution_id TEXT PRIMARY KEY, owner_digest TEXT, span_id TEXT, - anchor_id TEXT, record_digest TEXT, record_json TEXT - ); - CREATE TABLE evidence_derived_anchors ( - anchor_id TEXT PRIMARY KEY, owner_digest TEXT, target_kind TEXT, - target_id TEXT, anchor_json TEXT - ); - CREATE TABLE evidence_assembly_receipts ( - publication_receipt_id TEXT PRIMARY KEY, owner_digest TEXT, - privacy_domain_id TEXT, key_epoch INTEGER, idempotency_key TEXT, - assembly_digest TEXT, occurrence_set_id TEXT, span_id TEXT, - contribution_id TEXT, projection_receipt_id TEXT, receipt_json TEXT, - UNIQUE(owner_digest, privacy_domain_id, key_epoch, idempotency_key) - );", - ) - .unwrap(); -} - -#[cfg(test)] -fn evidence_table_counts(connection: &rusqlite::Connection) -> Vec { - [ - "retrieval_anchors", - "retrieval_anchor_reverse_lineage", - "evidence_source_occurrences", - "evidence_occurrence_sets", - "evidence_occurrence_set_members", - "evidence_spans", - "evidence_span_members", - "evidence_span_projection_receipts", - "evidence_retriever_contributions", - "evidence_derived_anchors", - "evidence_assembly_receipts", - ] - .into_iter() - .map(|table| { - connection - .query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |row| { - row.get::<_, i64>(0) - }) - .unwrap() - }) - .collect() -} - -#[test] -fn publish_replay_conflict_and_drilldown_are_atomic() { - let mut connection = rusqlite::Connection::open_in_memory().unwrap(); - install(&connection); - let write = write_fixture("1"); - connection - .execute( - "INSERT INTO retrieval_anchors ( - anchor_id, anchor_json, owner_json, projection_generation - ) VALUES (?1, '{}', ?2, 'source.fixture')", - params![ - write.occurrences[0].exact_source_anchor.as_str(), - encode(&write.owner.owner).unwrap(), - ], - ) - .unwrap(); - let mut executor = EvidenceAssemblyExecutor; - for _ in 0..2 { - let mut transaction = connection.transaction().unwrap(); - let savepoint = transaction.savepoint().unwrap(); - executor.execute_write(&savepoint, &write).unwrap(); - savepoint.commit().unwrap(); - transaction.commit().unwrap(); - } - let snapshot = connection.transaction().unwrap(); - let page = executor - .execute_read( - &snapshot, - &EvidenceAssemblyReadOperationV1::ContributionPage { - owner: write.owner.clone(), - contribution_id: write.contribution.contribution_id.clone(), - start_ordinal: 0, - page_size: 1, - }, - ) - .unwrap(); - assert!(matches!( - page, - EvidenceAssemblyReadResultV1::ContributionPage(Some(ref page)) - if page.occurrences.len() == 1 && page.next_ordinal.is_none() - )); - let mut wrong_owner = write.owner.clone(); - wrong_owner.scope_digest = ManifestDigest::new(format!("sha256:{}", "bb".repeat(32))).unwrap(); - assert_eq!( - executor - .execute_read( - &snapshot, - &EvidenceAssemblyReadOperationV1::ContributionPage { - owner: wrong_owner, - contribution_id: write.contribution.contribution_id.clone(), - start_ordinal: 0, - page_size: 1, - }, - ) - .unwrap(), - EvidenceAssemblyReadResultV1::ContributionPage(None) - ); - let mut anchor_executor = super::super::RetrievalAnchorExecutor; - assert!(matches!( - anchor_executor - .execute_read( - &snapshot, - &RetrievalAnchorReadOperationV1::AnchorById { - anchor_id: write.contribution.anchor.anchor_id().clone(), - owner: write.owner.owner.clone().into(), - }, - ) - .unwrap(), - RetrievalAnchorReadResultV1::Anchor(Some(StoredRetrievalAnchorRecordV1::V3(record))) - if record == write.contribution.anchor - )); - snapshot.commit().unwrap(); - connection - .execute( - "UPDATE retrieval_anchors SET projection_generation = 'tampered' - WHERE anchor_id = ?1", - [write.contribution.anchor.anchor_id().as_str()], - ) - .unwrap(); - let snapshot = connection.transaction().unwrap(); - assert!( - anchor_executor - .execute_read( - &snapshot, - &RetrievalAnchorReadOperationV1::AnchorById { - anchor_id: write.contribution.anchor.anchor_id().clone(), - owner: write.owner.owner.clone().into(), - }, - ) - .is_err() - ); - snapshot.commit().unwrap(); - - let counts_before_conflict = evidence_table_counts(&connection); - let conflict = write_fixture("2"); - let mut transaction = connection.transaction().unwrap(); - { - let mut savepoint = transaction.savepoint().unwrap(); - assert!(executor.execute_write(&savepoint, &conflict).is_err()); - savepoint.rollback().unwrap(); - } - transaction.rollback().unwrap(); - assert_eq!( - evidence_table_counts(&connection), - counts_before_conflict, - "a replay conflict must not partially mutate any evidence table" - ); -} - -#[test] -fn canonical_identity_validation_rejects_tampered_material() { - let write = write_fixture("1"); - let replay = write_fixture("1"); - assert_eq!(write, replay); - - let changed = write_fixture("2"); - assert_ne!( - write.contribution.contribution_id, - changed.contribution.contribution_id - ); - assert_ne!( - write.receipt.publication_receipt_id, - changed.receipt.publication_receipt_id - ); - - let mut tampered = write; - tampered.contribution.retriever.component_version = - tracedecay_domain::ComponentVersion::new("2").unwrap(); - assert!(tampered.validate().is_err()); -} - -#[test] -fn typed_catalog_order_horizon_privacy_watermark_and_owner_tampering_is_rejected() { - let baseline = write_fixture("1"); - - let mut catalog = baseline.clone(); - catalog.span.runs[0] - .ordering_proof - .catalog_binding - .catalog_digest = ManifestDigest::new(format!("sha256:{}", "ab".repeat(32))).unwrap(); - assert!(catalog.validate().is_err()); - - let mut ordering = baseline.clone(); - ordering.span.runs[0].ordering_proof.source_orders[0] = 8; - assert!(ordering.validate().is_err()); - - let mut horizon = baseline.clone(); - horizon.span.horizon.knowledge_through = UtcMicros(0); - assert!(matches!( - horizon.span.horizon.validate_members(&horizon.occurrences), - Err(tracedecay_store::EvidenceAssemblyStoreError::HorizonMismatch) - )); - assert!(horizon.validate().is_err()); - - let mut privacy = baseline.clone(); - privacy.contribution.request_digest.key_epoch = - privacy.contribution.owner.key_epoch.saturating_add(1); - assert!(matches!( - privacy.validate(), - Err(tracedecay_store::EvidenceAssemblyStoreError::RequestPrivacyBindingMismatch) - )); - - let mut watermark = baseline.clone(); - watermark.contribution.watermarks.source_watermark = - ManifestDigest::new(format!("sha256:{}", "bc".repeat(32))).unwrap(); - assert!(watermark.validate().is_err()); - - let mut owner = baseline; - owner.occurrences[0].owner = AnchorOwnerBindingV1::for_project( - UserProfileId::new("profile.fixture").unwrap(), - ProjectId::new("project.other").unwrap(), - PrivacyDomainId::new("privacy.fixture").unwrap(), - ) - .unwrap(); - assert!(owner.validate().is_err()); -} - -#[test] -fn drilldown_and_receipt_reads_reject_physical_index_tampering() { - let mut connection = rusqlite::Connection::open_in_memory().unwrap(); - install(&connection); - let write = write_fixture("1"); - connection - .execute( - "INSERT INTO retrieval_anchors ( - anchor_id, anchor_json, owner_json, projection_generation - ) VALUES (?1, '{}', ?2, 'source.fixture')", - params![ - write.occurrences[0].exact_source_anchor.as_str(), - encode(&write.owner.owner).unwrap(), - ], - ) - .unwrap(); - let mut executor = EvidenceAssemblyExecutor; - let mut transaction = connection.transaction().unwrap(); - let savepoint = transaction.savepoint().unwrap(); - executor.execute_write(&savepoint, &write).unwrap(); - savepoint.commit().unwrap(); - transaction.commit().unwrap(); - - connection - .execute( - "UPDATE evidence_occurrence_set_members - SET canonical_ordinal = 7 WHERE occurrence_set_id = ?1", - [write.occurrence_set.occurrence_set_id.as_str()], - ) - .unwrap(); - let snapshot = connection.transaction().unwrap(); - assert!( - executor - .execute_read( - &snapshot, - &EvidenceAssemblyReadOperationV1::ContributionPage { - owner: write.owner.clone(), - contribution_id: write.contribution.contribution_id.clone(), - start_ordinal: 0, - page_size: 1, - }, - ) - .is_err() - ); - snapshot.commit().unwrap(); - - connection - .execute( - "UPDATE evidence_assembly_receipts - SET span_id = 'span.tampered' WHERE publication_receipt_id = ?1", - [write.receipt.publication_receipt_id.as_str()], - ) - .unwrap(); - let snapshot = connection.transaction().unwrap(); - assert!( - executor - .execute_read( - &snapshot, - &EvidenceAssemblyReadOperationV1::PublicationByIdempotency { - owner: write.owner.clone(), - idempotency_key: write.idempotency_key.clone(), - }, - ) - .is_err() - ); -} - -#[test] -fn publication_rejects_cross_project_source_anchor_without_partial_rows() { - let mut connection = rusqlite::Connection::open_in_memory().unwrap(); - install(&connection); - connection - .execute( - "INSERT INTO retrieval_anchors ( - anchor_id, anchor_json, owner_json, projection_generation - ) VALUES ('retrieval.source.fixture', '{}', ?1, 'source.fixture')", - [encode( - &AnchorOwnerBindingV1::for_project( - UserProfileId::new("profile.fixture").unwrap(), - ProjectId::new("project.other").unwrap(), - PrivacyDomainId::new("privacy.fixture").unwrap(), - ) - .unwrap(), - ) - .unwrap()], - ) - .unwrap(); - let write = write_fixture("1"); - let mut transaction = connection.transaction().unwrap(); - { - let mut savepoint = transaction.savepoint().unwrap(); - assert!( - EvidenceAssemblyExecutor - .execute_write(&savepoint, &write) - .is_err() - ); - savepoint.rollback().unwrap(); - } - transaction.rollback().unwrap(); - assert_eq!( - connection - .query_row( - "SELECT COUNT(*) FROM evidence_assembly_receipts", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 0 - ); - assert_eq!( - connection - .query_row( - "SELECT COUNT(*) FROM evidence_source_occurrences", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 0 - ); -} - -#[test] -fn publication_rejects_unresolved_v2_owner_without_partial_rows() { - let mut connection = rusqlite::Connection::open_in_memory().unwrap(); - install(&connection); - connection - .execute( - "INSERT INTO retrieval_anchors ( - anchor_id, anchor_json, owner_json, projection_generation - ) VALUES ('retrieval.source.fixture', '{}', ?1, 'source.fixture')", - [encode(&tracedecay_domain::FactOwnerV1::Project { - project_id: ProjectId::new("project.fixture").unwrap(), - }) - .unwrap()], - ) - .unwrap(); - let write = write_fixture("1"); - let mut transaction = connection.transaction().unwrap(); - { - let mut savepoint = transaction.savepoint().unwrap(); - assert!( - EvidenceAssemblyExecutor - .execute_write(&savepoint, &write) - .is_err() - ); - savepoint.rollback().unwrap(); - } - transaction.rollback().unwrap(); - assert_eq!( - connection - .query_row( - "SELECT COUNT(*) FROM evidence_assembly_receipts", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 0 - ); - assert_eq!( - connection - .query_row( - "SELECT COUNT(*) FROM evidence_source_occurrences", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 0 - ); -} - -#[test] -fn batched_anchor_liveness_matches_row_at_a_time() { - fn dispose( - connection: &rusqlite::Connection, - anchor_id: &str, - owner_json: &str, - disposition_id: &str, - state: &str, - ) { - connection - .execute( - "INSERT INTO retrieval_anchor_dispositions - (disposition_id, anchor_id, owner_json, state) - VALUES (?1, ?2, ?3, ?4)", - params![disposition_id, anchor_id, owner_json, state], - ) - .unwrap(); - } - - let mut connection = rusqlite::Connection::open_in_memory().unwrap(); - install(&connection); - let write = write_fixture("1"); - let owner_json = encode(&write.owner.owner).unwrap(); - connection - .execute( - "INSERT INTO retrieval_anchors ( - anchor_id, anchor_json, owner_json, projection_generation - ) VALUES (?1, '{}', ?2, 'source.fixture')", - params![ - write.occurrences[0].exact_source_anchor.as_str(), - owner_json - ], - ) - .unwrap(); - { - let mut transaction = connection.transaction().unwrap(); - let savepoint = transaction.savepoint().unwrap(); - EvidenceAssemblyExecutor - .execute_write(&savepoint, &write) - .unwrap(); - savepoint.commit().unwrap(); - transaction.commit().unwrap(); - } - let occurrence = write.occurrences[0].clone(); - - // Compares the batched cache against the row-at-a-time free functions it - // replaced, asserting they agree, and hands back the shared outcome. - let compare = |connection: &rusqlite::Connection| { - let cache = super::anchor_state::load_anchor_liveness( - connection, - [ - occurrence.occurrence_anchor.anchor_id().as_str(), - occurrence.exact_source_anchor.as_str(), - ], - ) - .unwrap(); - - let free_current = super::anchor_state::evidence_anchor_is_current( - connection, - &occurrence.occurrence_anchor, - ) - .map_err(|error| error.to_string()); - let cached_current = cache - .evidence_anchor_is_current(&occurrence.occurrence_anchor) - .map_err(|error| error.to_string()); - assert_eq!(free_current, cached_current); - - let free_source = - super::anchor_state::require_source_anchor_current(connection, &occurrence) - .map(|_| ()) - .map_err(|error| error.to_string()); - let cached_source = cache - .require_source_anchor_current(&occurrence) - .map_err(|error| error.to_string()); - assert_eq!(free_source, cached_source); - - (free_current, free_source) - }; - - // Active: both anchors resolve as current. - assert_eq!(compare(&connection), (Ok(true), Ok(()))); - - // A disposed occurrence anchor makes the drilldown page read as absent. - dispose( - &connection, - occurrence.occurrence_anchor.anchor_id().as_str(), - &owner_json, - "disposition.occurrence.revoked", - "revoked", - ); - assert_eq!(compare(&connection).0, Ok(false)); - - // A newer active disposition supersedes the revocation (latest by - // sequence), while a disposed source anchor is rejected. - dispose( - &connection, - occurrence.occurrence_anchor.anchor_id().as_str(), - &owner_json, - "disposition.occurrence.reactivated", - "active", - ); - dispose( - &connection, - occurrence.exact_source_anchor.as_str(), - &owner_json, - "disposition.source.revoked", - "revoked", - ); - let (current, source) = compare(&connection); - assert_eq!(current, Ok(true)); - assert!(source.is_err()); -} - -#[test] -fn reverse_lineage_reuses_source_owner_json() { - let mut connection = rusqlite::Connection::open_in_memory().unwrap(); - install(&connection); - let write = write_fixture("1"); - let owner_json = encode(&write.owner.owner).unwrap(); - connection - .execute( - "INSERT INTO retrieval_anchors ( - anchor_id, anchor_json, owner_json, projection_generation - ) VALUES (?1, '{}', ?2, 'source.fixture')", - params![ - write.occurrences[0].exact_source_anchor.as_str(), - owner_json - ], - ) - .unwrap(); - let mut transaction = connection.transaction().unwrap(); - let savepoint = transaction.savepoint().unwrap(); - EvidenceAssemblyExecutor - .execute_write(&savepoint, &write) - .unwrap(); - savepoint.commit().unwrap(); - transaction.commit().unwrap(); - - let source_anchor_id = write.occurrences[0].exact_source_anchor.as_str().to_owned(); - let stored_owner_json: String = connection - .query_row( - "SELECT owner_json FROM retrieval_anchors WHERE anchor_id = ?1", - [source_anchor_id.as_str()], - |row| row.get::<_, String>(0), - ) - .unwrap(); - let lineage_owner_jsons: Vec = connection - .prepare( - "SELECT owner_json FROM retrieval_anchor_reverse_lineage - WHERE source_anchor_id = ?1", - ) - .unwrap() - .query_map([source_anchor_id.as_str()], |row| row.get::<_, String>(0)) - .unwrap() - .collect::>>() - .unwrap(); - assert_eq!( - lineage_owner_jsons.len(), - 2, - "one reverse-lineage row per derivative kind (span, contribution)" - ); - assert!( - lineage_owner_jsons - .iter() - .all(|json| *json == stored_owner_json), - "threaded owner_json must equal the source anchor's stored owner_json" - ); -} diff --git a/crates/tracedecay-rusqlite-runtime/src/repository/evidence_assembly/writes.rs b/crates/tracedecay-rusqlite-runtime/src/repository/evidence_assembly/writes.rs deleted file mode 100644 index c4ad629f0a..0000000000 --- a/crates/tracedecay-rusqlite-runtime/src/repository/evidence_assembly/writes.rs +++ /dev/null @@ -1,187 +0,0 @@ -//! The evidence assembly write path's table-by-table inserts. -//! -//! Every one of these is a replay-safe write: see -//! [`idempotent_insert`](super::super::support::idempotent_insert) for the -//! contract they all share. - -use tracedecay_domain::{RetrievalAnchorRecordV3, RetrievalAnchorTargetV3}; -use tracedecay_store::EvidenceAssemblyWriteV1; - -use super::super::support::{encode, idempotent_insert, invalid, usize_to_i64}; - -pub(super) fn insert_anchor( - connection: &rusqlite::Connection, - anchor: &RetrievalAnchorRecordV3, -) -> rusqlite::Result<()> { - anchor.validate().map_err(invalid)?; - idempotent_insert( - connection, - "retrieval_anchors", - &[("anchor_id", anchor.anchor_id().as_str().into())], - &[ - ("anchor_json", encode(anchor)?.into()), - ("owner_json", encode(anchor.owner())?.into()), - ( - "projection_generation", - anchor.projection_generation().as_str().into(), - ), - ], - "retrieval anchor replay conflict", - ) -} - -/// Writes one row of an immutable record table, which is any table keyed by a -/// single id and carrying the canonical `record_digest`/`record_json` pair plus -/// whatever columns it denormalizes out of that record for indexing. -pub(super) fn insert_immutable( - connection: &rusqlite::Connection, - table: &'static str, - id_column: &'static str, - id: &str, - record_digest: String, - record_json: String, - extra: &[(&'static str, String)], -) -> rusqlite::Result<()> { - let mut values = vec![ - ("record_digest", record_digest.into()), - ("record_json", record_json.into()), - ]; - values.extend( - extra - .iter() - .map(|(column, value)| (*column, value.clone().into())), - ); - idempotent_insert( - connection, - table, - &[(id_column, id.into())], - &values, - &format!("{table} immutable replay conflict"), - ) -} - -pub(super) fn insert_membership( - connection: &rusqlite::Connection, - table: &'static str, - parent_column: &'static str, - parent_id: &str, - ordinal_column: &'static str, - ordinal: usize, - occurrence_id: &str, -) -> rusqlite::Result<()> { - idempotent_insert( - connection, - table, - &[ - (parent_column, parent_id.into()), - ( - ordinal_column, - usize_to_i64(ordinal, "evidence membership ordinal")?.into(), - ), - ], - &[("occurrence_id", occurrence_id.into())], - &format!("{table} immutable replay conflict"), - ) -} - -pub(super) fn insert_span_membership( - connection: &rusqlite::Connection, - span_id: &str, - assembly_ordinal: usize, - run_ordinal: usize, - run_member_ordinal: usize, - occurrence_id: &str, -) -> rusqlite::Result<()> { - idempotent_insert( - connection, - "evidence_span_members", - &[ - ("span_id", span_id.into()), - ( - "assembly_ordinal", - usize_to_i64(assembly_ordinal, "evidence assembly ordinal")?.into(), - ), - ], - &[ - ( - "run_ordinal", - usize_to_i64(run_ordinal, "evidence run ordinal")?.into(), - ), - ( - "run_member_ordinal", - usize_to_i64(run_member_ordinal, "evidence run member ordinal")?.into(), - ), - ("occurrence_id", occurrence_id.into()), - ], - "evidence span membership replay conflict", - ) -} - -/// Records reverse lineage for every occurrence's source anchor. -/// -/// `source_owner_jsons` carries the `owner_json` each source anchor was already -/// read under in `execute_write` (via `require_source_anchor_current`), parallel -/// to `write.occurrences`, so this pass reuses those values instead of reading -/// each `retrieval_anchors` row a second time. -pub(super) fn publish_reverse_lineage( - connection: &rusqlite::Connection, - write: &EvidenceAssemblyWriteV1, - source_owner_jsons: &[String], -) -> rusqlite::Result<()> { - for (occurrence, owner_json) in write.occurrences.iter().zip(source_owner_jsons) { - for (kind, derivative_id) in [ - ("span", write.span.span_id.as_str()), - ("contribution", write.contribution.contribution_id.as_str()), - ] { - idempotent_insert( - connection, - "retrieval_anchor_reverse_lineage", - &[ - ( - "source_anchor_id", - occurrence.exact_source_anchor.as_str().into(), - ), - ("owner_json", owner_json.clone().into()), - ("derivative_kind", kind.into()), - ("derivative_id", derivative_id.into()), - ], - &[("direct_evidence", 1_i64.into())], - "evidence reverse lineage replay conflict", - )?; - } - } - Ok(()) -} - -pub(super) fn insert_derived_anchor( - connection: &rusqlite::Connection, - anchor: &RetrievalAnchorRecordV3, - owner_digest: &str, -) -> rusqlite::Result<()> { - let (target_kind, target_id) = evidence_target(anchor)?; - idempotent_insert( - connection, - "evidence_derived_anchors", - &[("anchor_id", anchor.anchor_id().as_str().into())], - &[ - ("owner_digest", owner_digest.into()), - ("target_kind", target_kind.into()), - ("target_id", target_id.into()), - ("anchor_json", encode(anchor)?.into()), - ], - "evidence derived anchor replay conflict", - ) -} - -fn evidence_target(anchor: &RetrievalAnchorRecordV3) -> rusqlite::Result<(&'static str, &str)> { - match anchor.target() { - RetrievalAnchorTargetV3::ExactSourceOccurrence(id) => { - Ok(("source_occurrence", id.as_str())) - } - RetrievalAnchorTargetV3::ExactEvidenceSpan(id) => Ok(("evidence_span", id.as_str())), - RetrievalAnchorTargetV3::RetrieverContribution(id) => { - Ok(("retriever_contribution", id.as_str())) - } - _ => Err(invalid("non-evidence target in evidence assembly")), - } -} diff --git a/crates/tracedecay-rusqlite-runtime/src/repository/external_source.rs b/crates/tracedecay-rusqlite-runtime/src/repository/external_source.rs index ef798b6081..8ec4b01395 100644 --- a/crates/tracedecay-rusqlite-runtime/src/repository/external_source.rs +++ b/crates/tracedecay-rusqlite-runtime/src/repository/external_source.rs @@ -1,14 +1,14 @@ //! Canonical SQLite projection for owner-bound external source state. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use rusqlite::{OptionalExtension, Savepoint, Transaction, params}; use tracedecay_domain::{SourceBindingIdentityV1, SourceBindingOwnerV1, SourceDeletionSemanticsV1}; use tracedecay_store::{ ExternalSourceReadOperationV1, ExternalSourceReadResultV1, SourceAcquisitionQueueCasV1, SourceAcquisitionQueueStateV1, SourceAuthorityPublicationReceiptV1, - SourceAuthorityPublicationV1, SourceCommitApplyOutcomeV1, SourceCommitReceiptV1, - SourceCommitV1, SourceObjectMutationV1, SourcePendingProjectionV1, + SourceAuthorityPublicationV1, SourceCommitApplyOutcomeV1, SourceCommitReceiptSummaryV1, + SourceCommitReceiptV1, SourceCommitV1, SourceObjectMutationV1, SourcePendingProjectionV1, SourceProjectionApplyOutcomeV1, SourceProjectionCommitV1, SourceStoreStateV1, apply_source_authority_publication_owned, apply_source_commit_owned, apply_source_projection_owned, build_source_projection, @@ -16,211 +16,20 @@ use tracedecay_store::{ use super::support::{decode, encode, invalid, same_json}; -/// Rows moved per retired-table migration write. -/// -/// The costliest production rewrite measured about 1.5 ms per row. Three -/// thousand rows take about 4.5 seconds, leaving headroom inside the unchanged -/// 30-second statement limit while amortizing each transaction over a batch. -pub const RETIRED_MUTATION_COPY_CHUNK_ROWS: i64 = 3_000; - -// Immutable histories stay append-only until the canonical retention policy -// explicitly covers external-source receipts. Current-state reads and writes -// use only primary-key/index probes and normalized current rows. +// Only current state keeps full documents. Every commit keeps its replay +// identity (key, request, receipt, and mutation digests) in an insert-only +// row; its full receipt lives in `external_source_retained_receipts_v1` only +// while it is the binding's current receipt or awaits projection, and every +// frontier no retained document names is deleted. Only the current +// projection publication, with its effects and lineage, is kept. // // A mutation's JSON lives once, in `external_source_mutations_v1`, keyed by -// its digest. The current-object, projected-object, and projection-effect -// tables reference it by digest and join for the payload: the earlier shape -// stored the same ~2 KB encoding in all four tables, which on one store was -// 2.4 GB of byte-identical copies beside the 1 GB history. Commit and -// projection receipts likewise persist slim (see `slim.rs`): their mutations -// and aggregate frontiers are digests into the history and -// `external_source_frontiers_v1`, and effects hydrate from the effects table. -/// Retired tables that carried their own copy of payloads the history tables -/// already hold, paired with the statements that move one bounded chunk of -/// rows into the digest-referencing successors and then remove that chunk -/// from the retired table. `json_extract` reads every digest out of the -/// retired row's own encoding, so no row needs another table migrated first. -/// -/// `?1` is the chunk's inclusive `rowid` ceiling in every statement, so the -/// moves and the removal in one chunk describe exactly the same rows. The -/// removal is last: a chunk that commits has moved its rows, and a chunk that -/// does not commit has moved none. -pub const RETIRED_MUTATION_COPY_TABLES: &[(&str, &[&str])] = &[ - ( - "external_source_objects_v1", - &[ - "INSERT OR IGNORE INTO external_source_objects_v2 ( - binding_id, native_object_digest, partition_digest, mutation_digest - ) - SELECT binding_id, native_object_digest, partition_digest, mutation_digest - FROM external_source_objects_v1 - WHERE rowid <= ?1", - "DELETE FROM external_source_objects_v1 WHERE rowid <= ?1", - ], - ), - ( - "external_source_projected_objects_v1", - &[ - "INSERT OR IGNORE INTO external_source_projected_objects_v2 ( - binding_id, native_object_digest, mutation_digest - ) - SELECT binding_id, native_object_digest, - json_extract(mutation_json, '$.mutation_digest') - FROM external_source_projected_objects_v1 - WHERE rowid <= ?1 - AND json_extract(mutation_json, '$.mutation_digest') IS NOT NULL", - "DELETE FROM external_source_projected_objects_v1 WHERE rowid <= ?1", - ], - ), - ( - "external_source_projection_effects_v1", - &[ - "INSERT OR IGNORE INTO external_source_projection_effects_v2 ( - binding_id, projection_digest, effect_index, - native_object_digest, mutation_digest, effect_json - ) - SELECT binding_id, projection_digest, effect_index, native_object_digest, - json_extract(mutation_json, '$.mutation_digest'), effect_json - FROM external_source_projection_effects_v1 - WHERE rowid <= ?1 - AND json_extract(mutation_json, '$.mutation_digest') IS NOT NULL", - "DELETE FROM external_source_projection_effects_v1 WHERE rowid <= ?1", - ], - ), - // Receipts embedded their mutations and aggregate frontiers. The slim - // shape keeps mutation digests in place of mutations, frontier digests in - // place of frontiers (the payloads move to `external_source_frontiers_v1`), - // and, for projections, an empty effects list that hydrates from the - // effects table. Every replacement value is read out of the retired row's - // own encoding. - ( - "external_source_commit_receipts_v1", - &[ - "INSERT OR IGNORE INTO external_source_frontiers_v1 ( - binding_id, frontier_digest, frontier_json - ) - SELECT binding_id, - json_extract(receipt_json, '$.source_frontier.digest'), - json_extract(receipt_json, '$.source_frontier') - FROM external_source_commit_receipts_v1 - WHERE rowid <= ?1 - AND json_extract(receipt_json, '$.source_frontier.digest') IS NOT NULL", - "INSERT OR IGNORE INTO external_source_frontiers_v1 ( - binding_id, frontier_digest, frontier_json - ) - SELECT binding_id, - json_extract(receipt_json, '$.prior_source_frontier.digest'), - json_extract(receipt_json, '$.prior_source_frontier') - FROM external_source_commit_receipts_v1 - WHERE rowid <= ?1 - AND json_extract(receipt_json, '$.prior_source_frontier.digest') IS NOT NULL", - "INSERT OR IGNORE INTO external_source_commit_receipts_v2 ( - binding_id, idempotency_key, request_digest, definition_revision, - binding_revision, predecessor_frontier_digest, successor_frontier_digest, - receipt_digest, receipt_json - ) - SELECT binding_id, idempotency_key, request_digest, definition_revision, - binding_revision, predecessor_frontier_digest, successor_frontier_digest, - receipt_digest, - json_set( - receipt_json, - '$.mutations', json(( - SELECT json_group_array( - json_extract(mutation.value, '$.mutation_digest') - ) - FROM json_each(receipt_json, '$.mutations') AS mutation - )), - '$.source_frontier', - json_extract(receipt_json, '$.source_frontier.digest'), - '$.prior_source_frontier', - json_extract(receipt_json, '$.prior_source_frontier.digest') - ) - FROM external_source_commit_receipts_v1 - WHERE rowid <= ?1", - "DELETE FROM external_source_commit_receipts_v1 WHERE rowid <= ?1", - ], - ), - ( - "external_source_projection_publications_v1", - &[ - "INSERT OR IGNORE INTO external_source_frontiers_v1 ( - binding_id, frontier_digest, frontier_json - ) - SELECT binding_id, - json_extract(receipt_json, '$.source_frontier.digest'), - json_extract(receipt_json, '$.source_frontier') - FROM external_source_projection_publications_v1 - WHERE rowid <= ?1 - AND json_extract(receipt_json, '$.source_frontier.digest') IS NOT NULL", - "INSERT OR IGNORE INTO external_source_frontiers_v1 ( - binding_id, frontier_digest, frontier_json - ) - SELECT binding_id, - json_extract(receipt_json, '$.expected_projection_frontier.digest'), - json_extract(receipt_json, '$.expected_projection_frontier') - FROM external_source_projection_publications_v1 - WHERE rowid <= ?1 - AND json_extract( - receipt_json, '$.expected_projection_frontier.digest' - ) IS NOT NULL", - "INSERT OR IGNORE INTO external_source_projection_publications_v2 ( - binding_id, projection_digest, source_receipt_digest, - predecessor_frontier_digest, successor_frontier_digest, receipt_json - ) - SELECT binding_id, projection_digest, source_receipt_digest, - predecessor_frontier_digest, successor_frontier_digest, - json_set( - receipt_json, - '$.mutations', json(( - SELECT json_group_array( - json_extract(mutation.value, '$.mutation_digest') - ) - FROM json_each(receipt_json, '$.mutations') AS mutation - )), - '$.effects', json('[]'), - '$.source_frontier', - json_extract(receipt_json, '$.source_frontier.digest'), - '$.expected_projection_frontier', - json_extract(receipt_json, '$.expected_projection_frontier.digest') - ) - FROM external_source_projection_publications_v1 - WHERE rowid <= ?1", - "DELETE FROM external_source_projection_publications_v1 WHERE rowid <= ?1", - ], - ), -]; - -/// Refuses a history read while the store is still being moved off the -/// payload-copying predecessors. -/// -/// Retiring those tables moves rows in chunks, so between chunks the current -/// tables hold only part of a store's history. Composing an answer from that -/// would understate what the store knows, and blaming an absent row would -/// name the wrong cause. Both become this one typed state, which clears when -/// the last chunk drops the last predecessor, the same presence the -/// migration itself uses as its progress marker. -fn refuse_while_history_migrates(connection: &rusqlite::Connection) -> rusqlite::Result<()> { - let placeholders = RETIRED_MUTATION_COPY_TABLES - .iter() - .map(|(table, _)| format!("'{table}'")) - .collect::>() - .join(", "); - let migrating = connection - .prepare_cached(&format!( - "SELECT name FROM sqlite_master - WHERE type = 'table' AND name IN ({placeholders}) - ORDER BY name LIMIT 1" - ))? - .query_row([], |row| row.get::<_, String>(0)) - .optional()?; - match migrating { - Some(table) => Err(invalid(format!( - "external source history is still migrating out of {table}" - ))), - None => Ok(()), - } -} - +// its digest, and omits what its row and binding already say (see +// `slim.rs`). The current-object, projected-object, and projection-effect +// tables reference it by digest and join for the payload. Retained receipts +// persist slim too: their mutations and aggregate frontiers are digests into +// the history and `external_source_frontiers_v1`, and effects hydrate from +// the effects table. pub const EXTERNAL_SOURCE_SCHEMA_V1: &str = " CREATE TABLE IF NOT EXISTS external_source_states_v1 ( binding_id TEXT PRIMARY KEY, @@ -273,22 +82,27 @@ CREATE TABLE IF NOT EXISTS external_source_commit_receipts_v2 ( binding_id TEXT NOT NULL, idempotency_key TEXT NOT NULL, request_digest TEXT NOT NULL, - definition_revision INTEGER NOT NULL CHECK (definition_revision > 0), - binding_revision INTEGER NOT NULL CHECK (binding_revision > 0), + receipt_digest TEXT NOT NULL, + mutation_digests_json TEXT NOT NULL, + PRIMARY KEY (binding_id, idempotency_key) +) WITHOUT ROWID; +CREATE TABLE IF NOT EXISTS external_source_retained_receipts_v1 ( + binding_id TEXT NOT NULL, + receipt_digest TEXT NOT NULL, predecessor_frontier_digest TEXT NOT NULL, successor_frontier_digest TEXT NOT NULL, - receipt_digest TEXT NOT NULL, receipt_json TEXT NOT NULL, - PRIMARY KEY (binding_id, idempotency_key), - UNIQUE (binding_id, receipt_digest), - UNIQUE (binding_id, successor_frontier_digest) -); + PRIMARY KEY (binding_id, receipt_digest) +) WITHOUT ROWID; +CREATE INDEX IF NOT EXISTS idx_external_source_retained_receipts_predecessor_v1 + ON external_source_retained_receipts_v1(binding_id, predecessor_frontier_digest); +CREATE INDEX IF NOT EXISTS idx_external_source_retained_receipts_successor_v1 + ON external_source_retained_receipts_v1(binding_id, successor_frontier_digest); CREATE TABLE IF NOT EXISTS external_source_mutations_v1 ( binding_id TEXT NOT NULL, mutation_digest TEXT NOT NULL, native_object_digest TEXT NOT NULL, revision_digest TEXT NOT NULL, - source_receipt_digest TEXT NOT NULL, mutation_json TEXT NOT NULL, PRIMARY KEY (binding_id, mutation_digest), UNIQUE (binding_id, native_object_digest, revision_digest) @@ -306,7 +120,7 @@ CREATE TABLE IF NOT EXISTS external_source_objects_v2 ( partition_digest TEXT NOT NULL, mutation_digest TEXT NOT NULL, PRIMARY KEY (binding_id, native_object_digest) -); +) WITHOUT ROWID; CREATE TABLE IF NOT EXISTS external_source_pending_projections_v1 ( binding_id TEXT NOT NULL, predecessor_frontier_digest TEXT NOT NULL, @@ -328,6 +142,8 @@ CREATE TABLE IF NOT EXISTS external_source_projection_publications_v2 ( UNIQUE (binding_id, source_receipt_digest), UNIQUE (binding_id, successor_frontier_digest) ); +CREATE INDEX IF NOT EXISTS idx_external_source_projection_publications_predecessor_v1 + ON external_source_projection_publications_v2(binding_id, predecessor_frontier_digest); CREATE TABLE IF NOT EXISTS external_source_projection_effects_v2 ( binding_id TEXT NOT NULL, projection_digest TEXT NOT NULL, @@ -350,7 +166,7 @@ CREATE TABLE IF NOT EXISTS external_source_projected_objects_v2 ( native_object_digest TEXT NOT NULL, mutation_digest TEXT NOT NULL, PRIMARY KEY (binding_id, native_object_digest) -); +) WITHOUT ROWID; CREATE TABLE IF NOT EXISTS external_source_acquisition_queue_v1 ( binding_id TEXT PRIMARY KEY, state_digest TEXT NOT NULL, @@ -403,7 +219,7 @@ impl ExternalSourceExecutor { commit.validate().map_err(invalid)?; let binding = commit.binding().immutable_identity().map_err(invalid)?; if let Some(receipt) = - load_commit_receipt_by_idempotency(savepoint, &binding, commit.idempotency_key())? + load_commit_receipt_summary(savepoint, &binding, commit.idempotency_key())? { return if receipt.request_digest() == commit.request_digest() { Ok(()) @@ -556,7 +372,6 @@ impl ExternalSourceExecutor { match operation { ExternalSourceReadOperationV1::State { binding } => { binding.validate().map_err(invalid)?; - refuse_while_history_migrates(snapshot)?; load_state(snapshot, binding) .map(|state| ExternalSourceReadResultV1::State(state.map(Box::new))) } @@ -566,12 +381,10 @@ impl ExternalSourceExecutor { } => { binding.validate().map_err(invalid)?; idempotency_key.validate().map_err(invalid)?; - refuse_while_history_migrates(snapshot)?; - load_commit_receipt_by_idempotency(snapshot, binding, idempotency_key) + load_commit_receipt_summary(snapshot, binding, idempotency_key) .map(|receipt| ExternalSourceReadResultV1::CommitReceipt(receipt.map(Box::new))) } ExternalSourceReadOperationV1::NextPendingProjection { binding } => { - refuse_while_history_migrates(snapshot)?; let pending = match binding { Some(binding) => { binding.validate().map_err(invalid)?; @@ -773,16 +586,9 @@ fn load_state( .map(|digest| load_projection_receipt_by_digest(connection, binding, digest)) .transpose()? .flatten(); - let observed = load_current_mutations( - connection, - "external_source_objects_v2", - binding.binding_id.as_str(), - )?; - let projected = load_current_mutations( - connection, - "external_source_projected_objects_v2", - binding.binding_id.as_str(), - )?; + let observed = load_current_mutations(connection, "external_source_objects_v2", binding)?; + let projected = + load_current_mutations(connection, "external_source_projected_objects_v2", binding)?; let state = SourceStoreStateV1::restore( definition, stored_binding, @@ -832,13 +638,14 @@ fn load_binding( fn load_current_mutations( connection: &rusqlite::Connection, table: &str, - binding_id: &str, + binding: &SourceBindingIdentityV1, ) -> rusqlite::Result> { // LEFT JOIN so a current row whose digest names no history row surfaces // as corruption instead of silently vanishing from the current state. let sql = match table { "external_source_objects_v2" => { - "SELECT history.mutation_json + "SELECT history.mutation_json, history.native_object_digest, + history.revision_digest, current.mutation_digest FROM external_source_objects_v2 AS current LEFT JOIN external_source_mutations_v1 AS history ON history.binding_id = current.binding_id @@ -846,7 +653,8 @@ fn load_current_mutations( WHERE current.binding_id = ?1" } "external_source_projected_objects_v2" => { - "SELECT history.mutation_json + "SELECT history.mutation_json, history.native_object_digest, + history.revision_digest, current.mutation_digest FROM external_source_projected_objects_v2 AS current LEFT JOIN external_source_mutations_v1 AS history ON history.binding_id = current.binding_id @@ -857,12 +665,22 @@ fn load_current_mutations( }; let mut statement = connection.prepare_cached(sql)?; statement - .query_map([binding_id], |row| { - let encoded: Option = row.get(0)?; - let encoded = encoded.ok_or_else(|| { - invalid("external source current object names a mutation absent from history") - })?; - decode(encoded) + .query_map([binding.binding_id.as_str()], |row| { + let absent = + || invalid("external source current object names a mutation absent from history"); + let slim: String = row.get::<_, Option>(0)?.ok_or_else(absent)?; + let native_object: String = row.get::<_, Option>(1)?.ok_or_else(absent)?; + let revision: String = row.get::<_, Option>(2)?.ok_or_else(absent)?; + let mutation_digest: String = row.get(3)?; + slim::hydrate_mutation( + &slim, + binding, + slim::MutationRowKeys { + native_object: &native_object, + revision: &revision, + mutation_digest: &mutation_digest, + }, + ) })? .collect() } @@ -887,62 +705,61 @@ fn persist_source_commit( frontiers, } = slim::slim_commit_receipt(receipt)?; persist_frontiers(savepoint, binding.binding_id.as_str(), &frontiers)?; + let summary = SourceCommitReceiptSummaryV1::of(receipt); // `INSERT OR IGNORE` reports zero changed rows only when a conflict was // swallowed; only then can the stored row differ from this write, so the // read-back proof is needed only on that path. let changed = savepoint.execute( "INSERT OR IGNORE INTO external_source_commit_receipts_v2 ( - binding_id, idempotency_key, request_digest, - definition_revision, binding_revision, - predecessor_frontier_digest, successor_frontier_digest, - receipt_digest, receipt_json - ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)", + binding_id, idempotency_key, request_digest, receipt_digest, + mutation_digests_json + ) VALUES (?1, ?2, ?3, ?4, ?5)", params![ binding.binding_id.as_str(), receipt.idempotency_key().as_str(), receipt.request_digest().as_str(), - i64::try_from(receipt.definition_revision()).map_err(|_| invalid( - "external source definition revision exceeds SQLite INTEGER" - ))?, - i64::try_from(receipt.binding_revision()) - .map_err(|_| invalid("external source binding revision exceeds SQLite INTEGER"))?, + receipt.receipt_digest().as_str(), + encode(summary.mutation_digests())?, + ], + )?; + if changed == 0 + && load_commit_receipt_summary(savepoint, &binding, receipt.idempotency_key())?.as_ref() + != Some(&summary) + { + return Err(invalid("external source commit receipt collision")); + } + savepoint.execute( + "INSERT OR IGNORE INTO external_source_retained_receipts_v1 ( + binding_id, receipt_digest, predecessor_frontier_digest, + successor_frontier_digest, receipt_json + ) VALUES (?1, ?2, ?3, ?4, ?5)", + params![ + binding.binding_id.as_str(), + receipt.receipt_digest().as_str(), predecessor, successor, - receipt.receipt_digest().as_str(), receipt_json, ], )?; - if changed == 0 { - verify_encoded_row( - savepoint, - "SELECT receipt_json FROM external_source_commit_receipts_v2 - WHERE binding_id = ?1 AND idempotency_key = ?2", - binding.binding_id.as_str(), - receipt.idempotency_key().as_str(), - &receipt_json, - "external source commit receipt collision", - )?; - } for mutation in receipt.mutations() { // The collision validation already encoded every commit mutation; the // receipt carries those same mutations through, so a miss here only // means the encoding was not pre-computed and is re-derived. let mutation_json = match mutation_encodings.remove(mutation.mutation_digest().as_str()) { Some(encoded) => encoded, - None => encode(mutation)?, + None => slim::slim_mutation(mutation, &binding)?, }; let native_object = mutation.observation().native_object(); let changed = savepoint.execute( "INSERT OR IGNORE INTO external_source_mutations_v1 ( binding_id, mutation_digest, native_object_digest, - revision_digest, source_receipt_digest, mutation_json - ) VALUES (?1, ?2, ?3, ?4, ?5, ?6)", + revision_digest, mutation_json + ) VALUES (?1, ?2, ?3, ?4, ?5)", params![ binding.binding_id.as_str(), mutation.mutation_digest().as_str(), native_object.digest().as_str(), mutation.observation().revision().digest().as_str(), - receipt.receipt_digest().as_str(), mutation_json, ], )?; @@ -1033,7 +850,111 @@ fn persist_source_commit( return Err(invalid("external source pending projection fork collision")); } } - upsert_current_state(savepoint, state) + let displaced_receipt: Option = savepoint + .query_row( + "SELECT latest_source_receipt_digest FROM external_source_states_v1 + WHERE binding_id = ?1", + [binding.binding_id.as_str()], + |row| row.get(0), + ) + .optional()?; + upsert_current_state(savepoint, state)?; + retire_superseded_history( + savepoint, + binding.binding_id.as_str(), + displaced_receipt.as_deref().as_slice(), + &[], + ) +} + +/// Deletes the documents one commit stopped needing. Replay keeps working +/// from the insert-only receipt summaries; the deleted receipts, frontiers, +/// publications, effects, and projection lineage are named by nothing that +/// remains. +/// +/// Only the commit's own candidates are examined: `receipts` are the source +/// receipts it displaced as latest or whose pending projection it consumed, +/// `projections` the projection it displaced. A receipt is retired once it is +/// neither the latest nor awaiting projection; a frontier once no remaining +/// retained receipt or publication names it. Every check is an indexed point +/// lookup, so a commit's cost does not grow with the pending backlog: the +/// receipts a backlog still needs were examined when they became eligible and +/// need no rescan now. +#[hotpath::measure(label = "rusqlite.external_source.retire_superseded_history")] +fn retire_superseded_history( + savepoint: &Savepoint<'_>, + binding_id: &str, + receipts: &[&str], + projections: &[&str], +) -> rusqlite::Result<()> { + let mut frontiers = BTreeSet::new(); + let mut collect_frontiers = |rows: rusqlite::Rows<'_>| -> rusqlite::Result<()> { + for row in rows.mapped(|row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))) { + let (predecessor, successor) = row?; + frontiers.insert(predecessor); + frontiers.insert(successor); + } + Ok(()) + }; + for receipt in receipts { + let mut retire = savepoint.prepare_cached( + "DELETE FROM external_source_retained_receipts_v1 + WHERE binding_id = ?1 + AND receipt_digest = ?2 + AND receipt_digest <> ( + SELECT latest_source_receipt_digest FROM external_source_states_v1 + WHERE binding_id = ?1 + ) + AND NOT EXISTS ( + SELECT 1 FROM external_source_pending_projections_v1 + WHERE binding_id = ?1 AND source_receipt_digest = ?2 + ) + RETURNING predecessor_frontier_digest, successor_frontier_digest", + )?; + collect_frontiers(retire.query(params![binding_id, receipt])?)?; + } + for projection in projections { + for table in [ + "external_source_projection_effects_v2", + "external_source_projection_lineage_v1", + ] { + savepoint.execute( + &format!("DELETE FROM {table} WHERE binding_id = ?1 AND projection_digest = ?2"), + params![binding_id, projection], + )?; + } + let mut retire = savepoint.prepare_cached( + "DELETE FROM external_source_projection_publications_v2 + WHERE binding_id = ?1 AND projection_digest = ?2 + RETURNING predecessor_frontier_digest, successor_frontier_digest", + )?; + collect_frontiers(retire.query(params![binding_id, projection])?)?; + } + for frontier in &frontiers { + savepoint.execute( + "DELETE FROM external_source_frontiers_v1 + WHERE binding_id = ?1 + AND frontier_digest = ?2 + AND NOT EXISTS ( + SELECT 1 FROM external_source_retained_receipts_v1 + WHERE binding_id = ?1 AND predecessor_frontier_digest = ?2 + ) + AND NOT EXISTS ( + SELECT 1 FROM external_source_retained_receipts_v1 + WHERE binding_id = ?1 AND successor_frontier_digest = ?2 + ) + AND NOT EXISTS ( + SELECT 1 FROM external_source_projection_publications_v2 + WHERE binding_id = ?1 AND predecessor_frontier_digest = ?2 + ) + AND NOT EXISTS ( + SELECT 1 FROM external_source_projection_publications_v2 + WHERE binding_id = ?1 AND successor_frontier_digest = ?2 + )", + params![binding_id, frontier], + )?; + } + Ok(()) } #[hotpath::measure(label = "rusqlite.external_source.persist_projection")] @@ -1164,6 +1085,15 @@ fn persist_projection( "external source pending projection compare-and-set failed", )); } + let displaced_projection: Option = savepoint + .query_row( + "SELECT latest_projection_receipt_digest FROM external_source_states_v1 + WHERE binding_id = ?1", + [binding.binding_id.as_str()], + |row| row.get(0), + ) + .optional()? + .flatten(); savepoint.execute( "UPDATE external_source_states_v1 SET projection_frontier_digest = ?1, @@ -1175,7 +1105,14 @@ fn persist_projection( binding.binding_id.as_str(), ], )?; - Ok(()) + let displaced_projection = + displaced_projection.filter(|displaced| displaced != projection.receipt_digest().as_str()); + retire_superseded_history( + savepoint, + binding.binding_id.as_str(), + &[source_receipt.receipt_digest().as_str()], + displaced_projection.as_deref().as_slice(), + ) } #[hotpath::measure(label = "rusqlite.external_source.persist_authority")] @@ -1366,7 +1303,7 @@ fn validate_revision_collisions( for mutation in mutations { encodings.insert( mutation.mutation_digest().as_str().to_owned(), - encode(mutation)?, + slim::slim_mutation(mutation, binding)?, ); } for chunk in mutations.chunks(REVISION_COLLISION_PROBE_CHUNK) { @@ -1445,7 +1382,7 @@ fn persist_frontiers( mod reads; mod slim; use reads::{ - load_authority_receipt, load_commit_receipt_by_digest, load_commit_receipt_by_idempotency, + load_authority_receipt, load_commit_receipt_by_digest, load_commit_receipt_summary, load_next_pending_projection, load_next_pending_projection_any, load_projection_receipt, load_projection_receipt_by_digest, verify_encoded_row, }; diff --git a/crates/tracedecay-rusqlite-runtime/src/repository/external_source/reads.rs b/crates/tracedecay-rusqlite-runtime/src/repository/external_source/reads.rs index 8b39a9cb13..fa692b9187 100644 --- a/crates/tracedecay-rusqlite-runtime/src/repository/external_source/reads.rs +++ b/crates/tracedecay-rusqlite-runtime/src/repository/external_source/reads.rs @@ -77,11 +77,7 @@ pub(super) fn load_next_pending_projection( == SourceDeletionSemanticsV1::CompleteSnapshotAbsence && receipt.snapshot_completion().is_some() { - load_current_mutations( - connection, - "external_source_projected_objects_v2", - binding.binding_id.as_str(), - )? + load_current_mutations(connection, "external_source_projected_objects_v2", binding)? } else { Vec::new() }; @@ -125,23 +121,44 @@ pub(super) fn load_next_pending_projection_any( load_next_pending_projection(connection, &identity) } -#[hotpath::measure(label = "rusqlite.external_source.load_commit_receipt_by_idempotency")] -pub(super) fn load_commit_receipt_by_idempotency( +/// The replay identity a key committed, whether or not its full receipt is +/// still retained. +#[hotpath::measure(label = "rusqlite.external_source.load_commit_receipt_summary")] +pub(super) fn load_commit_receipt_summary( connection: &rusqlite::Connection, binding: &SourceBindingIdentityV1, key: &tracedecay_domain::ManifestDigest, -) -> rusqlite::Result> { - load_slim_optional( - connection, - "SELECT receipt_json FROM external_source_commit_receipts_v2 - WHERE binding_id = ?1 AND idempotency_key = ?2", - binding.binding_id.as_str(), - key.as_str(), - )? - .map(|slim| super::slim::hydrate_commit_receipt(connection, binding.binding_id.as_str(), &slim)) - .transpose() +) -> rusqlite::Result> { + let row = connection + .prepare_cached( + "SELECT request_digest, receipt_digest, mutation_digests_json + FROM external_source_commit_receipts_v2 + WHERE binding_id = ?1 AND idempotency_key = ?2", + )? + .query_row(params![binding.binding_id.as_str(), key.as_str()], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + )) + }) + .optional()?; + let Some((request_digest, receipt_digest, mutation_digests)) = row else { + return Ok(None); + }; + SourceCommitReceiptSummaryV1::new( + binding.clone(), + key.clone(), + tracedecay_domain::ManifestDigest::new(request_digest).map_err(invalid)?, + tracedecay_domain::ManifestDigest::new(receipt_digest).map_err(invalid)?, + decode(mutation_digests)?, + ) + .map(Some) + .map_err(invalid) } +/// A receipt still retained in full: the binding's current receipt or one +/// awaiting projection. #[hotpath::measure(label = "rusqlite.external_source.load_commit_receipt_by_digest")] pub(super) fn load_commit_receipt_by_digest( connection: &rusqlite::Connection, @@ -150,12 +167,12 @@ pub(super) fn load_commit_receipt_by_digest( ) -> rusqlite::Result> { load_slim_optional( connection, - "SELECT receipt_json FROM external_source_commit_receipts_v2 + "SELECT receipt_json FROM external_source_retained_receipts_v1 WHERE binding_id = ?1 AND receipt_digest = ?2", binding.binding_id.as_str(), digest, )? - .map(|slim| super::slim::hydrate_commit_receipt(connection, binding.binding_id.as_str(), &slim)) + .map(|slim| super::slim::hydrate_commit_receipt(connection, binding, &slim)) .transpose() } @@ -196,9 +213,7 @@ pub(super) fn load_projection_receipt_by_digest( binding.binding_id.as_str(), digest, )? - .map(|slim| { - super::slim::hydrate_projection_receipt(connection, binding.binding_id.as_str(), &slim) - }) + .map(|slim| super::slim::hydrate_projection_receipt(connection, binding, &slim)) .transpose() } diff --git a/crates/tracedecay-rusqlite-runtime/src/repository/external_source/slim.rs b/crates/tracedecay-rusqlite-runtime/src/repository/external_source/slim.rs index 55bc60e7cb..f9966d4cdf 100644 --- a/crates/tracedecay-rusqlite-runtime/src/repository/external_source/slim.rs +++ b/crates/tracedecay-rusqlite-runtime/src/repository/external_source/slim.rs @@ -11,10 +11,18 @@ //! through `serde_json::Value` so the domain types keep their private fields //! and their own digest validation decides whether the rebuilt receipt is the //! one that was committed. +//! +//! A history row's mutation likewise drops every value its row and binding +//! already carry: the binding identity, the native-object, revision, and +//! mutation digests, and the evidence fields that repeat the observation. A +//! value is dropped only when it equals what hydration restores, so a +//! mutation that disagrees with its row keeps its own value and fails its +//! domain validation instead of silently adopting the row's. use rusqlite::{OptionalExtension, params}; -use serde_json::Value; -use tracedecay_store::{SourceCommitReceiptV1, SourceProjectionCommitV1}; +use serde_json::{Map, Value}; +use tracedecay_domain::SourceBindingIdentityV1; +use tracedecay_store::{SourceCommitReceiptV1, SourceObjectMutationV1, SourceProjectionCommitV1}; use super::super::support::invalid; @@ -25,6 +33,135 @@ const MUTATION_DIGEST: &str = "mutation_digest"; const FRONTIER_DIGEST: &str = "digest"; const COMMIT_FRONTIERS: [&str; 2] = ["prior_source_frontier", "source_frontier"]; const PROJECTION_FRONTIERS: [&str; 2] = ["expected_projection_frontier", "source_frontier"]; +const OBSERVATION: &str = "observation"; +const EVIDENCE: &str = "evidence"; +const BINDING: &str = "binding"; +const NATIVE_OBJECT: &str = "native_object"; +const REVISION: &str = "revision"; +/// Evidence fields that restate the mutation's observation. +const EVIDENCE_OBSERVATION_FIELDS: [&str; 3] = [NATIVE_OBJECT, REVISION, "sanitized_digest"]; + +/// The digests a mutation history row stores as columns. +pub(super) struct MutationRowKeys<'a> { + pub(super) native_object: &'a str, + pub(super) revision: &'a str, + pub(super) mutation_digest: &'a str, +} + +pub(super) fn slim_mutation( + mutation: &SourceObjectMutationV1, + binding: &SourceBindingIdentityV1, +) -> rusqlite::Result { + let mut value = serde_json::to_value(mutation).map_err(|error| invalid(error.to_string()))?; + let binding = serde_json::to_value(binding).map_err(|error| invalid(error.to_string()))?; + let root = object(&mut value)?; + let observation = root + .get(OBSERVATION) + .cloned() + .ok_or_else(|| invalid("external source mutation has no observation"))?; + let evidence = nested(root, EVIDENCE)?; + drop_if_equal(evidence, BINDING, &binding); + for field in EVIDENCE_OBSERVATION_FIELDS { + if let Some(expected) = observation.get(field) { + drop_if_equal(evidence, field, expected); + } + } + let observation = nested(root, OBSERVATION)?; + drop_if_equal( + observation, + NATIVE_OBJECT, + &Value::String( + mutation + .observation() + .native_object() + .digest() + .as_str() + .to_owned(), + ), + ); + drop_if_equal( + observation, + REVISION, + &Value::String( + mutation + .observation() + .revision() + .digest() + .as_str() + .to_owned(), + ), + ); + drop_if_equal( + root, + MUTATION_DIGEST, + &Value::String(mutation.mutation_digest().as_str().to_owned()), + ); + serde_json::to_string(&value).map_err(|error| invalid(error.to_string())) +} + +pub(super) fn hydrate_mutation( + slim: &str, + binding: &SourceBindingIdentityV1, + keys: MutationRowKeys<'_>, +) -> rusqlite::Result { + serde_json::from_value(hydrate_mutation_value(slim, binding, keys)?) + .map_err(|error| invalid(error.to_string())) +} + +fn hydrate_mutation_value( + slim: &str, + binding: &SourceBindingIdentityV1, + keys: MutationRowKeys<'_>, +) -> rusqlite::Result { + let mut value: Value = + serde_json::from_str(slim).map_err(|error| invalid(error.to_string()))?; + let binding = serde_json::to_value(binding).map_err(|error| invalid(error.to_string()))?; + let root = object(&mut value)?; + restore( + root, + MUTATION_DIGEST, + Value::String(keys.mutation_digest.to_owned()), + ); + let observation = nested(root, OBSERVATION)?; + restore( + observation, + NATIVE_OBJECT, + Value::String(keys.native_object.to_owned()), + ); + restore( + observation, + REVISION, + Value::String(keys.revision.to_owned()), + ); + let observation = observation.clone(); + let evidence = nested(root, EVIDENCE)?; + restore(evidence, BINDING, binding); + for field in EVIDENCE_OBSERVATION_FIELDS { + if let Some(value) = observation.get(field) { + restore(evidence, field, value.clone()); + } + } + Ok(value) +} + +fn nested<'a>( + root: &'a mut Map, + field: &str, +) -> rusqlite::Result<&'a mut Map> { + root.get_mut(field) + .and_then(Value::as_object_mut) + .ok_or_else(|| invalid("external source mutation encoding is missing an object field")) +} + +fn drop_if_equal(object: &mut Map, field: &str, expected: &Value) { + if object.get(field) == Some(expected) { + object.remove(field); + } +} + +fn restore(object: &mut Map, field: &str, value: Value) { + object.entry(field).or_insert(value); +} /// One receipt reduced to digests, plus the frontier payloads it referenced, /// for the caller to store beside it. @@ -63,25 +200,27 @@ pub(super) fn slim_projection_receipt( pub(super) fn hydrate_commit_receipt( connection: &rusqlite::Connection, - binding_id: &str, + binding: &SourceBindingIdentityV1, slim: &str, ) -> rusqlite::Result { + let binding_id = binding.binding_id.as_str(); let mut value: Value = serde_json::from_str(slim).map_err(|error| invalid(error.to_string()))?; attach_frontiers(connection, binding_id, &mut value, &COMMIT_FRONTIERS)?; - attach_mutations(connection, binding_id, &mut value)?; + attach_mutations(connection, binding, &mut value)?; serde_json::from_value(value).map_err(|error| invalid(error.to_string())) } pub(super) fn hydrate_projection_receipt( connection: &rusqlite::Connection, - binding_id: &str, + binding: &SourceBindingIdentityV1, slim: &str, ) -> rusqlite::Result { + let binding_id = binding.binding_id.as_str(); let mut value: Value = serde_json::from_str(slim).map_err(|error| invalid(error.to_string()))?; attach_frontiers(connection, binding_id, &mut value, &PROJECTION_FRONTIERS)?; - attach_mutations(connection, binding_id, &mut value)?; + attach_mutations(connection, binding, &mut value)?; let projection_digest = object(&mut value)? .get(RECEIPT_DIGEST) .and_then(Value::as_str) @@ -180,7 +319,7 @@ fn attach_frontiers( fn attach_mutations( connection: &rusqlite::Connection, - binding_id: &str, + binding: &SourceBindingIdentityV1, value: &mut Value, ) -> rusqlite::Result<()> { let object = object(value)?; @@ -188,7 +327,8 @@ fn attach_mutations( return Err(invalid("external source receipt has no mutation list")); }; let mut statement = connection.prepare_cached( - "SELECT mutation_json FROM external_source_mutations_v1 + "SELECT mutation_json, native_object_digest, revision_digest + FROM external_source_mutations_v1 WHERE binding_id = ?1 AND mutation_digest = ?2", )?; let mut mutations = Vec::with_capacity(digests.len()); @@ -198,13 +338,23 @@ fn attach_mutations( "external source receipt mutation reference is not a digest", )); }; - let encoded: Option = statement - .query_row(params![binding_id, digest], |row| row.get(0)) + let row: Option<(String, String, String)> = statement + .query_row(params![binding.binding_id.as_str(), digest], |row| { + Ok((row.get(0)?, row.get(1)?, row.get(2)?)) + }) .optional()?; - let encoded = encoded.ok_or_else(|| { + let (slim, native_object, revision) = row.ok_or_else(|| { invalid("external source receipt names a mutation absent from history") })?; - mutations.push(serde_json::from_str(&encoded).map_err(|error| invalid(error.to_string()))?); + mutations.push(hydrate_mutation_value( + &slim, + binding, + MutationRowKeys { + native_object: &native_object, + revision: &revision, + mutation_digest: digest, + }, + )?); } object.insert(MUTATIONS.to_owned(), Value::Array(mutations)); Ok(()) diff --git a/crates/tracedecay-rusqlite-runtime/src/repository/external_source/tests.rs b/crates/tracedecay-rusqlite-runtime/src/repository/external_source/tests.rs index f503561957..71eb58d3da 100644 --- a/crates/tracedecay-rusqlite-runtime/src/repository/external_source/tests.rs +++ b/crates/tracedecay-rusqlite-runtime/src/repository/external_source/tests.rs @@ -603,7 +603,7 @@ fn ten_thousand_receipts_do_not_make_current_read_or_write_scan_history() { ); let mut lookup = connection .prepare( - "SELECT receipt_json FROM external_source_commit_receipts_v2 + "SELECT request_digest FROM external_source_commit_receipts_v2 WHERE binding_id = ?1 AND idempotency_key = ?2", ) .unwrap(); @@ -839,6 +839,91 @@ fn separate_projection_write_rolls_back_effect_and_checkpoint_together() { ); } +#[test] +fn projected_history_shrinks_to_replay_summaries() { + let mut connection = rusqlite::Connection::open_in_memory().unwrap(); + connection.execute_batch(EXTERNAL_SOURCE_SCHEMA_V1).unwrap(); + let (first, binding) = fixture(); + let projector = ComponentVersion::new("external-source-projector-v1").unwrap(); + let write = |connection: &mut rusqlite::Connection, commit: &SourceCommitV1| { + let mut transaction = connection.transaction().unwrap(); + let savepoint = transaction.savepoint().unwrap(); + ExternalSourceExecutor::default() + .execute_write(&savepoint, commit) + .unwrap(); + savepoint.commit().unwrap(); + transaction.commit().unwrap(); + }; + let project = |connection: &mut rusqlite::Connection| { + let pending = load_next_pending_projection(connection, &binding) + .unwrap() + .unwrap(); + let projection = build_source_projection(&pending, projector.clone()).unwrap(); + let mut transaction = connection.transaction().unwrap(); + let savepoint = transaction.savepoint().unwrap(); + ExternalSourceExecutor::default() + .execute_projection_write(&savepoint, &projection) + .unwrap(); + savepoint.commit().unwrap(); + transaction.commit().unwrap(); + }; + write(&mut connection, &first); + project(&mut connection); + for sequence in 2..=20 { + let state = load_state(&connection, &binding).unwrap().unwrap(); + write(&mut connection, &numbered_empty_successor(&state, sequence)); + project(&mut connection); + } + let count = |connection: &rusqlite::Connection, sql: &str| -> i64 { + connection.query_row(sql, [], |row| row.get(0)).unwrap() + }; + assert_eq!( + count( + &connection, + "SELECT COUNT(*) FROM external_source_commit_receipts_v2" + ), + 20 + ); + assert_eq!( + count( + &connection, + "SELECT COUNT(*) FROM external_source_retained_receipts_v1" + ), + 1, + "only the current receipt stays hydratable once every commit is projected" + ); + assert_eq!( + count( + &connection, + "SELECT COUNT(*) FROM external_source_projection_publications_v2" + ), + 1 + ); + assert!( + count( + &connection, + "SELECT COUNT(*) FROM external_source_frontiers_v1" + ) <= 3 + ); + + let summary = load_commit_receipt_summary(&connection, &binding, first.idempotency_key()) + .unwrap() + .expect("a superseded commit keeps its replay identity"); + assert_eq!(summary.request_digest(), first.request_digest()); + assert!(summary.committed(&first.mutations()[0])); + write(&mut connection, &first); + let current = load_state(&connection, &binding).unwrap().unwrap(); + assert!(current.projection().is_some()); + assert_eq!( + count( + &connection, + "SELECT COUNT(*) FROM external_source_commit_receipts_v2" + ), + 20, + "replaying a superseded commit settles from its summary without a new receipt" + ); +} + #[test] fn commit_replay_and_restart_read_share_one_durable_state() { let temporary = tempfile::tempdir().unwrap(); @@ -926,6 +1011,24 @@ fn commit_replay_and_restart_read_share_one_durable_state() { .unwrap(); assert!(!durable_json.contains("secret")); assert!(!durable_json.contains("https://")); + let stored_mutation: String = transaction + .query_row( + "SELECT mutation_json FROM external_source_mutations_v1 WHERE binding_id = ?1", + [binding.binding_id.as_str()], + |row| row.get(0), + ) + .unwrap(); + let mutation = &commit.mutations()[0]; + for repeated in [ + binding.binding_id.as_str(), + mutation.mutation_digest().as_str(), + mutation.observation().native_object().digest().as_str(), + ] { + assert!( + !stored_mutation.contains(repeated), + "a history row omits what its columns and binding carry: {stored_mutation}" + ); + } } #[test] @@ -1465,85 +1568,3 @@ fn reopened_executor_fully_validates_historical_current_rows() { "a reopened writer must not inherit any prior process verification" ); } - -/// A store still being moved off its payload-copying predecessors holds only -/// part of its history in the current tables. A read there must say so: an -/// answer composed from the moved subset would understate what the store -/// knows, and "receipt is missing" would name the wrong cause for a row the -/// migration simply has not reached. The state clears when the migration -/// drops the last predecessor, and reads unrelated to that history keep -/// answering throughout. -#[test] -fn history_reads_report_the_migration_instead_of_a_partial_answer() { - let temporary = tempfile::tempdir().unwrap(); - let path = temporary.path().join("external-source-migrating.sqlite"); - let mut connection = rusqlite::Connection::open(&path).unwrap(); - connection.execute_batch(EXTERNAL_SOURCE_SCHEMA_V1).unwrap(); - let (commit, binding) = fixture(); - let mut transaction = connection.transaction().unwrap(); - let savepoint = transaction.savepoint().unwrap(); - ExternalSourceExecutor::default() - .execute_write(&savepoint, &commit) - .unwrap(); - savepoint.commit().unwrap(); - transaction.commit().unwrap(); - - let read_state = |connection: &mut rusqlite::Connection| { - let snapshot = connection.transaction().unwrap(); - let result = ExternalSourceExecutor::default().execute_read( - &snapshot, - &ExternalSourceReadOperationV1::State { - binding: binding.clone(), - }, - ); - snapshot.finish().unwrap(); - result - }; - let read_pending_count = |connection: &mut rusqlite::Connection| { - let snapshot = connection.transaction().unwrap(); - let result = ExternalSourceExecutor::default().execute_read( - &snapshot, - &ExternalSourceReadOperationV1::AcquisitionPendingCount, - ); - snapshot.finish().unwrap(); - result - }; - - assert!( - matches!( - read_state(&mut connection), - Ok(ExternalSourceReadResultV1::State(Some(_))) - ), - "a converged store must answer its state read" - ); - - // Exactly the shape a migration in progress leaves: the predecessor is - // still on disk with the rows that have not moved. - let (retired_table, _) = RETIRED_MUTATION_COPY_TABLES[0]; - connection - .execute_batch(&format!( - "CREATE TABLE {retired_table} ( - binding_id TEXT NOT NULL, native_object_digest TEXT NOT NULL, - partition_digest TEXT NOT NULL, mutation_digest TEXT NOT NULL, - mutation_json TEXT NOT NULL, - PRIMARY KEY (binding_id, native_object_digest));" - )) - .unwrap(); - - let error = read_state(&mut connection) - .expect_err("a half-moved history must not be answered as if it were whole"); - assert!( - error.to_string().contains(retired_table), - "the refusal must name the migration that is still running: {error}" - ); - assert!( - read_pending_count(&mut connection).is_ok(), - "a read that does not touch the migrating history must keep answering" - ); - - connection - .execute_batch(&format!("DROP TABLE {retired_table}")) - .unwrap(); - read_state(&mut connection) - .expect("the state read answers again once the migration has retired its predecessor"); -} diff --git a/crates/tracedecay-rusqlite-runtime/src/repository/fact/writes.rs b/crates/tracedecay-rusqlite-runtime/src/repository/fact/writes.rs index 2204f42e48..4feea17d09 100644 --- a/crates/tracedecay-rusqlite-runtime/src/repository/fact/writes.rs +++ b/crates/tracedecay-rusqlite-runtime/src/repository/fact/writes.rs @@ -7,7 +7,7 @@ use rusqlite::{OptionalExtension, Savepoint, params}; use tracedecay_domain::{ FactCurationActionV1, FactEventId, FactId, FactIdentityMaterialV1, FactLineageEventKindV1, - FactLineageEventV1, FactOwnerV1, PayloadAccessState, RetrievalAnchorRecordV2, + FactLineageEventV1, FactOwnerV1, PayloadAccessState, RetrievalAnchorRecord, }; use tracedecay_store::FactWriteBatch; @@ -87,7 +87,7 @@ pub(super) fn ensure_fact( pub(super) fn insert_anchor( savepoint: &Savepoint<'_>, owner: &OwnerColumns, - anchor: &RetrievalAnchorRecordV2, + anchor: &RetrievalAnchorRecord, ) -> rusqlite::Result<()> { let encoded = encode(anchor)?; let stored = savepoint diff --git a/crates/tracedecay-rusqlite-runtime/src/repository/mod.rs b/crates/tracedecay-rusqlite-runtime/src/repository/mod.rs index 00a4485f00..705b8150ef 100644 --- a/crates/tracedecay-rusqlite-runtime/src/repository/mod.rs +++ b/crates/tracedecay-rusqlite-runtime/src/repository/mod.rs @@ -14,16 +14,11 @@ //! are mounted. Every payload and read operation an application actually //! constructs today routes through here: facts, observations and cursor //! advances, diagnostics, evidence assembly, external sources, retrieval-anchor -//! dispositions and derivatives. Three surfaces are wired and tested here; -//! their live writers remain elsewhere: -//! -//! - the profile/configuration family -//! ([`RepositoryWritePayloadV1::Configuration`] and every -//! [`RepositoryReadOperationV1::Profile`] operation), whose live writer is -//! still `crates/tracedecay-global-db/src/configuration/store.rs`; -//! - [`RepositoryWritePayloadV1::DiagnosticSupersession`] and the -//! `Stale`/`SupersessionChain` diagnostic reads, whose live engine is still -//! `src/diagnostics_store.rs`; +//! dispositions and derivatives. The profile/configuration family +//! ([`RepositoryWritePayloadV1::Configuration`] and every +//! [`RepositoryReadOperationV1::Profile`] operation) is wired and tested here, +//! but its live writer is still +//! `crates/tracedecay-global-db/src/configuration/store.rs`. //! //! `Code` operations cross the graph-db boundary, while `Effects` operations //! are owned by the writer ledger; both dispatch arms here reject them. @@ -31,7 +26,6 @@ mod attachment; mod configuration; mod diagnostics; -pub(crate) mod evidence_assembly; mod external_source; mod fact; mod graph_publication; @@ -56,14 +50,7 @@ pub use attachment::{ }; pub use configuration::ConfigurationExecutor; pub use diagnostics::DiagnosticExecutor; -pub use evidence_assembly::EvidenceAssemblyExecutor; -#[cfg(feature = "test-transport")] -#[doc(hidden)] -pub use evidence_assembly::tests::write_fixture_for_project; -pub use external_source::{ - EXTERNAL_SOURCE_SCHEMA_V1, ExternalSourceExecutor, RETIRED_MUTATION_COPY_CHUNK_ROWS, - RETIRED_MUTATION_COPY_TABLES, -}; +pub use external_source::{EXTERNAL_SOURCE_SCHEMA_V1, ExternalSourceExecutor}; pub use fact::FactExecutor; pub use graph_publication::{GRAPH_PUBLICATION_SCHEMA_V1, GraphPublicationExactSqlStorage}; pub use observation::cursor_authority as observation_cursor_authority; @@ -139,14 +126,6 @@ impl StorageOperationExecutor for ConcreteRepositoryWriteExecutor { RepositoryWritePayloadV1::Diagnostics(snapshot) => { self.project.execute_diagnostic_write(savepoint, snapshot)?; } - RepositoryWritePayloadV1::DiagnosticSupersession(request) => { - self.project - .execute_diagnostic_supersession(savepoint, request)?; - } - RepositoryWritePayloadV1::EvidenceAssembly(write) => { - self.project - .execute_evidence_assembly_write(savepoint, write)?; - } RepositoryWritePayloadV1::ExternalSource(commit) => { self.project .execute_external_source_write(savepoint, commit)?; diff --git a/crates/tracedecay-rusqlite-runtime/src/repository/observation/authority.rs b/crates/tracedecay-rusqlite-runtime/src/repository/observation/authority.rs index 7b205e9888..dcbb4470b3 100644 --- a/crates/tracedecay-rusqlite-runtime/src/repository/observation/authority.rs +++ b/crates/tracedecay-rusqlite-runtime/src/repository/observation/authority.rs @@ -6,9 +6,9 @@ use rusqlite::{OptionalExtension, params}; use tracedecay_domain::{ - AnchorSourceGenerationV2, DurableObservationV1, EvidenceAvailabilityV1, FactOwnerV1, + AnchorSourceGeneration, DurableObservationV1, EvidenceAvailabilityV1, FactOwnerV1, GenerationBoundRepositoryProvenanceV1, ObservationSourceCursorV1, RepositoryProvenanceV1, - RetrievalAnchorRecordV2, RetrievalAnchorRecordV2Parts, RetrievalAnchorTargetV2, + RetrievalAnchorRecord, RetrievalAnchorRecordParts, RetrievalAnchorTarget, prove_cline_native_source_transition, }; use tracedecay_store::{ @@ -92,7 +92,7 @@ pub(super) fn cursor_advance_receipt_matches( pub(super) fn persist_retrieval_anchor( connection: &rusqlite::Connection, - anchor: &RetrievalAnchorRecordV2, + anchor: &RetrievalAnchorRecord, ) -> rusqlite::Result<()> { let anchor_json = encode(anchor)?; let owner_json = encode(anchor.owner())?; @@ -135,7 +135,7 @@ pub(super) fn persist_retrieval_anchor( fn verify_retrieval_anchor( connection: &rusqlite::Connection, - anchor: &RetrievalAnchorRecordV2, + anchor: &RetrievalAnchorRecord, ) -> rusqlite::Result<()> { let owner_json = encode(anchor.owner())?; let stored = connection @@ -155,7 +155,7 @@ fn verify_retrieval_anchor( let Some((stored_anchor_json, stored_owner_json, stored_projection_generation)) = stored else { return Err(invalid("retrieval anchor identity collision")); }; - let stored_anchor: RetrievalAnchorRecordV2 = decode(stored_anchor_json)?; + let stored_anchor: RetrievalAnchorRecord = decode(stored_anchor_json)?; if !stored_anchor.is_semantic_replay_of(anchor) || stored_owner_json != owner_json || stored_projection_generation != anchor.projection_generation().as_str() @@ -202,7 +202,7 @@ fn verify_retrieval_anchor( // immutable historical anchor replay requires the exact supersession receipt. fn cline_alias_transition_is_valid( connection: &rusqlite::Connection, - anchor: &RetrievalAnchorRecordV2, + anchor: &RetrievalAnchorRecord, current_anchor_id: &str, ) -> rusqlite::Result { let Some(current_json) = connection @@ -215,10 +215,10 @@ fn cline_alias_transition_is_valid( else { return Ok(false); }; - let current: RetrievalAnchorRecordV2 = decode(current_json)?; + let current: RetrievalAnchorRecord = decode(current_json)?; let ( - RetrievalAnchorTargetV2::ExactObservation(anchor_observation_id), - RetrievalAnchorTargetV2::ExactObservation(current_observation_id), + RetrievalAnchorTarget::ExactObservation(anchor_observation_id), + RetrievalAnchorTarget::ExactObservation(current_observation_id), ) = (anchor.target(), current.target()) else { return Ok(false); @@ -280,7 +280,7 @@ fn cline_alias_transition_is_valid( let disposition: RetrievalAnchorDispositionRecordV1 = decode(disposition_json)?; disposition.validate().map_err(invalid)?; Ok(disposition.anchor_id() == anchor.anchor_id() - && disposition.owner().v2() == Some(&FactOwnerV1::from(anchor.owner().clone())) + && *disposition.owner() == FactOwnerV1::from(anchor.owner().clone()) && disposition.state() == AnchorDispositionStateV1::Superseded && disposition.reason_class() == AnchorDispositionReasonClassV1::Correction && disposition.superseded_by() == Some(current.anchor_id())) @@ -499,7 +499,7 @@ pub(super) fn verify_observation_authority( fn repository_replay_anchor( attachment: &RepositoryProvenanceAttachmentV1, retained_json: &str, -) -> rusqlite::Result> { +) -> rusqlite::Result> { let retained: EvidenceAvailabilityV1 = decode(retained_json.to_owned())?; let (Some(old), Some(new), Some(anchor)) = ( @@ -534,7 +534,7 @@ fn repository_replay_anchor( if !same_json(retained_json, &encode(&normalized)?) { return Ok(None); } - let RetrievalAnchorTargetV2::RepositoryCapture { + let RetrievalAnchorTarget::RepositoryCapture { repository_id, receipt, .. @@ -542,8 +542,8 @@ fn repository_replay_anchor( else { return Ok(None); }; - RetrievalAnchorRecordV2::new(RetrievalAnchorRecordV2Parts { - target: RetrievalAnchorTargetV2::RepositoryCapture { + RetrievalAnchorRecord::new(RetrievalAnchorRecordParts { + target: RetrievalAnchorTarget::RepositoryCapture { repository_id: repository_id.clone(), capture_id: old.capture_id().clone(), receipt: receipt.clone(), @@ -553,7 +553,7 @@ fn repository_replay_anchor( occurred_at: anchor.occurred_at(), ingested_at: anchor.ingested_at(), evidence_class: anchor.evidence_class(), - source_generation: AnchorSourceGenerationV2::RepositoryCapture(old.capture_id().clone()), + source_generation: AnchorSourceGeneration::RepositoryCapture(old.capture_id().clone()), projection_generation: anchor.projection_generation().clone(), projection_watermark: anchor.projection_watermark().clone(), coverage: anchor.coverage().clone(), diff --git a/crates/tracedecay-rusqlite-runtime/src/repository/observation/cursor_authority.rs b/crates/tracedecay-rusqlite-runtime/src/repository/observation/cursor_authority.rs index 670f7a76fc..8cd61cc485 100644 --- a/crates/tracedecay-rusqlite-runtime/src/repository/observation/cursor_authority.rs +++ b/crates/tracedecay-rusqlite-runtime/src/repository/observation/cursor_authority.rs @@ -36,6 +36,24 @@ pub const COMMIT_SOURCE_CURSOR_SQL: &str = ON CONFLICT(source_json, scope_json) DO UPDATE SET cursor_json = excluded.cursor_json"; +/// Deletes the advance rows the durable cursor strictly supersedes: params +/// `(source_json, scope_json)`, run after the cursor moves in the same +/// transaction. The row supporting the current frontier and rows beyond it +/// stay; the predicate is the one `source_cursor_advances_immutable_delete_v1` +/// admits, so a prune can never trip that trigger. +pub const PRUNE_SUPERSEDED_CURSOR_ADVANCES_SQL: &str = "DELETE FROM source_cursor_advances + WHERE source_json = ?1 AND scope_json = ?2 + AND EXISTS ( + SELECT 1 FROM source_cursors AS cursor + WHERE cursor.source_json = source_cursor_advances.source_json + AND cursor.scope_json = source_cursor_advances.scope_json + AND (json_extract(cursor.cursor_json, '$.generation') + IS NOT json_extract(source_cursor_advances.coverage_json, '$.generation') + OR (COALESCE(json_extract(cursor.cursor_json, '$.ordering_domain'), 'file_bytes') + = json_extract(source_cursor_advances.coverage_json, '$.ordering_domain') + AND json_extract(cursor.cursor_json, '$.byte_offset') + > json_extract(source_cursor_advances.coverage_json, '$.range.end'))))"; + /// Whether one [`READ_CURSOR_ADVANCE_SQL`] row is exactly this advance's /// row, the same reason and the same (possibly absent) sanitization receipt /// id. Any other row retained under the coverage key is a cursor-advance diff --git a/crates/tracedecay-rusqlite-runtime/src/repository/observation/mod.rs b/crates/tracedecay-rusqlite-runtime/src/repository/observation/mod.rs index 0ff4734612..564eea5fd5 100644 --- a/crates/tracedecay-rusqlite-runtime/src/repository/observation/mod.rs +++ b/crates/tracedecay-rusqlite-runtime/src/repository/observation/mod.rs @@ -32,7 +32,8 @@ use authority::{ persist_sanitization_receipt, read_cursor, verify_observation_authority, }; use cursor_authority::{ - COMMIT_SOURCE_CURSOR_SQL, READ_CURSOR_ADVANCE_SQL, RECORD_CURSOR_ADVANCE_SQL, + COMMIT_SOURCE_CURSOR_SQL, PRUNE_SUPERSEDED_CURSOR_ADVANCES_SQL, READ_CURSOR_ADVANCE_SQL, + RECORD_CURSOR_ADVANCE_SQL, }; use rows::{ decode_nonnegative, decode_observation_row, encoded_observation_row, observation_row_projection, @@ -161,6 +162,10 @@ impl ObservationExecutor { COMMIT_SOURCE_CURSOR_SQL, params![source_json, scope_json, committed_cursor_json], )?; + savepoint.execute( + PRUNE_SUPERSEDED_CURSOR_ADVANCES_SQL, + params![source_json, scope_json], + )?; savepoint.execute( "INSERT INTO projection_queue (observation_id, observation_sequence) VALUES (?1, ?2)", @@ -222,6 +227,10 @@ impl ObservationExecutor { COMMIT_SOURCE_CURSOR_SQL, params![source_json, scope_json, encode(advance.next_cursor())?], )?; + savepoint.execute( + PRUNE_SUPERSEDED_CURSOR_ADVANCES_SQL, + params![source_json, scope_json], + )?; Ok(()) } diff --git a/crates/tracedecay-rusqlite-runtime/src/repository/observation/rows.rs b/crates/tracedecay-rusqlite-runtime/src/repository/observation/rows.rs index 09f3b7a3b7..9c14499e83 100644 --- a/crates/tracedecay-rusqlite-runtime/src/repository/observation/rows.rs +++ b/crates/tracedecay-rusqlite-runtime/src/repository/observation/rows.rs @@ -6,7 +6,7 @@ use tracedecay_domain::{ EvidenceAvailabilityV1, GenerationBoundRepositoryProvenanceV1, ObservationSourceCursorV1, - ProjectionGenerationId, RetrievalAnchorRecordV2, + ProjectionGenerationId, RetrievalAnchorRecord, }; use tracedecay_store::{ ObservationCommitReceipt, RepositoryProvenanceAttachmentV1, StoredObservationRowV1, @@ -120,7 +120,7 @@ pub(super) fn decode_observation_row( { return Err(invalid("observation committed cursor binding mismatch")); } - let retrieval_anchor: RetrievalAnchorRecordV2 = decode( + let retrieval_anchor: RetrievalAnchorRecord = decode( retrieval_anchor.ok_or_else(|| invalid("observation retrieval anchor is missing"))?, )?; let projection_generation = ProjectionGenerationId::new( @@ -129,7 +129,7 @@ pub(super) fn decode_observation_row( ) .map_err(invalid)?; let repository_anchor = repository_anchor - .map(decode::) + .map(decode::) .transpose()?; let expected_repository_owner = repository_anchor .as_ref() diff --git a/crates/tracedecay-rusqlite-runtime/src/repository/observation/tests.rs b/crates/tracedecay-rusqlite-runtime/src/repository/observation/tests.rs index 6079a44a5f..7dbe5bd0c0 100644 --- a/crates/tracedecay-rusqlite-runtime/src/repository/observation/tests.rs +++ b/crates/tracedecay-rusqlite-runtime/src/repository/observation/tests.rs @@ -1,7 +1,7 @@ use rusqlite::Connection; use serde_json::json; use tracedecay_domain::{ - AnchorDurabilityClass, AnchorSourceGenerationV2, CanonicalObservationEnvelopeV1, + AnchorDurabilityClass, AnchorSourceGeneration, CanonicalObservationEnvelopeV1, CanonicalObservationEvidenceV1, CanonicalObservationFactV1, CanonicalObservationRelationsV1, ComponentVersion, CoverageReportV1, EvidenceAvailabilityV1, EvidenceClass, FactOwnerV1, GenerationBoundRepositoryProvenanceV1, ObservationId, ObservationIdentityMaterialV1, @@ -10,7 +10,7 @@ use tracedecay_domain::{ PayloadAccessState, PayloadReferenceV1, PrivacyDomainBoundLocatorDigest, ProjectId, ProjectionGenerationId, ProviderId, ProviderUsageContractDimensionV1, RefId, RepositoryEvidenceV1, RepositoryId, RepositoryProvenanceV1, RepositoryRemoteIdentityV1, - RetentionClass, RetrievalAnchorRecordV2, RetrievalAnchorRecordV2Parts, RetrievalAnchorTargetV2, + RetentionClass, RetrievalAnchorRecord, RetrievalAnchorRecordParts, RetrievalAnchorTarget, SanitizationReceiptId, SanitizationReceiptRefV1, SanitizationReceiptV1, SanitizerDispositionV1, SensitivityV1, SessionId, UtcMicros, VectorWatermark, }; @@ -20,7 +20,7 @@ use tracedecay_store::{ ObservationCursorAdvance, ObservationReadOperationV1, ObservationReadResultV1, ObservationWrite, RetrievalAnchorDispositionRecordV1, SESSION_MESSAGE_PROJECTOR_VERSION, StorageRuntimeErrorV1, build_observation_resolution_authorization_v1, - build_observation_retrieval_anchor_v2, + build_observation_retrieval_anchor, }; use crate::operation::StorageOperationError; @@ -124,7 +124,7 @@ fn anchored_at(write: ObservationWrite, ingested_at: UtcMicros) -> AnchoredObser let authorization = build_observation_resolution_authorization_v1(write.observation(), "runtime.fixture.v1") .unwrap(); - let anchor = build_observation_retrieval_anchor_v2( + let anchor = build_observation_retrieval_anchor( write.observation(), projection_generation.clone(), ingested_at, @@ -234,8 +234,8 @@ fn repository_write_for( Some(write.observation().observation_id().clone()), ) .unwrap(); - let anchor = RetrievalAnchorRecordV2::new(RetrievalAnchorRecordV2Parts { - target: RetrievalAnchorTargetV2::RepositoryCapture { + let anchor = RetrievalAnchorRecord::new(RetrievalAnchorRecordParts { + target: RetrievalAnchorTarget::RepositoryCapture { repository_id: binding.capture().repository_id().clone(), capture_id: binding.capture_id().clone(), receipt: write.observation().receipt().receipt().clone(), @@ -245,9 +245,7 @@ fn repository_write_for( occurred_at: None, ingested_at: UtcMicros(clock), evidence_class, - source_generation: AnchorSourceGenerationV2::RepositoryCapture( - binding.capture_id().clone(), - ), + source_generation: AnchorSourceGeneration::RepositoryCapture(binding.capture_id().clone()), projection_generation: write.projection_generation().clone(), projection_watermark: VectorWatermark::default(), coverage: CoverageReportV1::default(), @@ -623,7 +621,7 @@ fn replay_with_different_anchor_fails_without_mutating_authority_rows() { let authorization = build_observation_resolution_authorization_v1(write.observation(), "runtime.fixture.v1") .unwrap(); - let conflicting_anchor = build_observation_retrieval_anchor_v2( + let conflicting_anchor = build_observation_retrieval_anchor( write.observation(), conflicting_generation.clone(), UtcMicros(1), @@ -835,6 +833,63 @@ fn source_cursor_advance_keeps_the_first_owner_once_the_frontier_is_reached() { )); } +fn advance_ledger_ends(connection: &Connection) -> Vec { + let mut statement = connection + .prepare( + "SELECT json_extract(coverage_json, '$.range.end') + FROM source_cursor_advances ORDER BY 1", + ) + .unwrap(); + statement + .query_map([], |row| row.get(0)) + .unwrap() + .collect::>() + .unwrap() +} + +#[test] +fn cursor_commits_keep_only_the_advance_supporting_the_frontier() { + let mut connection = connection(); + let write = anchored_observation_write("fixture", "receipt.fixture"); + execute(&mut connection, &write).unwrap(); + let identity = write.observation().identity(); + let mut cursor = write.next_cursor().clone(); + for end in 2_u64..=6 { + let advance = ObservationCursorAdvance::for_ordering( + write.observation().source().clone(), + write.observation().scope().clone(), + identity.generation(), + identity.ordering_domain(), + Some(cursor.clone()), + ObservationSourceRangeV1::new(end - 1, end).unwrap(), + ObservationCoverageReason::OutOfScope, + ) + .unwrap(); + execute_cursor_advance(&mut connection, &advance).unwrap(); + cursor = advance.next_cursor().clone(); + assert_eq!( + advance_ledger_ends(&connection), + vec![i64::try_from(end).unwrap()] + ); + } + assert_eq!(cursor.position(), 6); + + let next = anchored(observation_write_for_record( + "next", + "receipt.next", + 1, + 6, + 7, + Some(cursor), + "record.next", + )); + execute(&mut connection, &next).unwrap(); + assert!( + advance_ledger_ends(&connection).is_empty(), + "an observation past the frontier settles the last advance" + ); +} + #[test] fn canonical_cursor_advance_receipt_remains_typed_after_authority_lookup() { let mut connection = connection(); diff --git a/crates/tracedecay-rusqlite-runtime/src/repository/project.rs b/crates/tracedecay-rusqlite-runtime/src/repository/project.rs index 73228fffc5..6fbb5f4568 100644 --- a/crates/tracedecay-rusqlite-runtime/src/repository/project.rs +++ b/crates/tracedecay-rusqlite-runtime/src/repository/project.rs @@ -1,10 +1,10 @@ use rusqlite::{Savepoint, Transaction}; use tracedecay_store::{ - AnchoredObservationWrite, DiagnosticGenerationSupersessionV1, EvidenceAssemblyWriteV1, - FactWriteBatch, ObservationCursorAdvance, ProjectReadOperationV1, ProjectReadResultV1, - RemoteObservationReplayWriteV1, RemoteWriterFenceInstallV1, RetrievalAnchorDerivativeV1, - RetrievalAnchorDispositionRecordV1, SanitizedCleanDiagnosticSnapshotV1, - SourceAcquisitionQueueCasV1, SourceCommitV1, SourceProjectionCommitV1, + AnchoredObservationWrite, FactWriteBatch, ObservationCursorAdvance, ProjectReadOperationV1, + ProjectReadResultV1, RemoteObservationReplayWriteV1, RemoteWriterFenceInstallV1, + RetrievalAnchorDerivativeV1, RetrievalAnchorDispositionRecordV1, + SanitizedCleanDiagnosticSnapshotV1, SourceAcquisitionQueueCasV1, SourceCommitV1, + SourceProjectionCommitV1, }; use crate::operation::StorageOperationError; @@ -13,8 +13,8 @@ use super::remote::{ install_writer_fence, persist_remote_observation_event, verify_and_seed_writer_fence, }; use super::{ - DiagnosticExecutor, EvidenceAssemblyExecutor, ExternalSourceExecutor, FactExecutor, - ObservationExecutor, RetrievalAnchorExecutor, + DiagnosticExecutor, ExternalSourceExecutor, FactExecutor, ObservationExecutor, + RetrievalAnchorExecutor, }; #[derive(Clone, Default)] @@ -22,7 +22,6 @@ pub struct ProjectExecutor { fact: FactExecutor, observation: ObservationExecutor, diagnostics: DiagnosticExecutor, - evidence_assembly: EvidenceAssemblyExecutor, external_source: ExternalSourceExecutor, retrieval_anchor: RetrievalAnchorExecutor, } @@ -102,28 +101,6 @@ impl ProjectExecutor { self.diagnostics.execute_write(savepoint, snapshot) } - /// Supersedes one prior diagnostic generation. The transitioned row count - /// is intentionally dropped here: the repository write dispatch is - /// uniformly `Result<()>`, and the count is recoverable by reading the - /// stale lane for the prior generation. - pub fn execute_diagnostic_supersession( - &mut self, - savepoint: &Savepoint<'_>, - request: &DiagnosticGenerationSupersessionV1, - ) -> rusqlite::Result<()> { - self.diagnostics - .execute_supersession(savepoint, request) - .map(|_| ()) - } - - pub fn execute_evidence_assembly_write( - &mut self, - savepoint: &Savepoint<'_>, - write: &EvidenceAssemblyWriteV1, - ) -> rusqlite::Result<()> { - self.evidence_assembly.execute_write(savepoint, write) - } - pub fn execute_external_source_write( &mut self, savepoint: &Savepoint<'_>, @@ -205,10 +182,6 @@ impl ProjectExecutor { .diagnostics .execute_read(snapshot, operation) .map(ProjectReadResultV1::Diagnostics), - ProjectReadOperationV1::EvidenceAssembly(operation) => self - .evidence_assembly - .execute_read(snapshot, operation) - .map(ProjectReadResultV1::EvidenceAssembly), ProjectReadOperationV1::RetrievalAnchor(operation) => self .retrieval_anchor .execute_read(snapshot, operation) diff --git a/crates/tracedecay-rusqlite-runtime/src/repository/retrieval_anchor.rs b/crates/tracedecay-rusqlite-runtime/src/repository/retrieval_anchor.rs index 0b8a26bab9..259bdd6ec7 100644 --- a/crates/tracedecay-rusqlite-runtime/src/repository/retrieval_anchor.rs +++ b/crates/tracedecay-rusqlite-runtime/src/repository/retrieval_anchor.rs @@ -1,9 +1,9 @@ use rusqlite::{OptionalExtension, Savepoint, Transaction, params}; -use tracedecay_domain::RetrievalAnchorId; +use tracedecay_domain::{FactOwnerV1, RetrievalAnchorId, RetrievalAnchorRecord}; use tracedecay_store::{ AnchorDerivativeKindV1, AnchorDispositionStateV1, RetrievalAnchorDerivativeV1, - RetrievalAnchorDispositionRecordV1, RetrievalAnchorOwnerV1, RetrievalAnchorReadOperationV1, - RetrievalAnchorReadResultV1, RetrievalAnchorTombstoneV1, StoredRetrievalAnchorRecordV1, + RetrievalAnchorDispositionRecordV1, RetrievalAnchorReadOperationV1, + RetrievalAnchorReadResultV1, RetrievalAnchorTombstoneV1, }; use super::support::{decode, encode, idempotent_insert, invalid}; @@ -163,8 +163,8 @@ impl RetrievalAnchorExecutor { fn read_anchor( connection: &rusqlite::Connection, anchor_id: &RetrievalAnchorId, - owner: &RetrievalAnchorOwnerV1, -) -> rusqlite::Result> { + owner: &FactOwnerV1, +) -> rusqlite::Result> { let owner_json = encode(owner)?; if !AnchorDispositionStateV1::serves_derivatives(current_state( connection, @@ -183,10 +183,10 @@ fn read_anchor( ) .optional()? .map(|(record_json, projection_generation)| { - let record: StoredRetrievalAnchorRecordV1 = decode(record_json)?; + let record: RetrievalAnchorRecord = decode(record_json)?; record.validate().map_err(invalid)?; if record.anchor_id() != anchor_id - || record.owner() != *owner + || FactOwnerV1::from(record.owner().clone()) != *owner || record.projection_generation().as_str() != projection_generation { return Err(invalid("retrieval anchor record identity mismatch")); @@ -199,7 +199,7 @@ fn read_anchor( fn current_state( connection: &rusqlite::Connection, anchor_id: &RetrievalAnchorId, - owner: &RetrievalAnchorOwnerV1, + owner: &FactOwnerV1, owner_json: &str, ) -> rusqlite::Result> { current_record(connection, anchor_id, owner, owner_json) @@ -209,7 +209,7 @@ fn current_state( fn current_record( connection: &rusqlite::Connection, anchor_id: &RetrievalAnchorId, - owner: &RetrievalAnchorOwnerV1, + owner: &FactOwnerV1, owner_json: &str, ) -> rusqlite::Result> { connection @@ -261,7 +261,7 @@ fn current_record( fn read_derivatives( connection: &rusqlite::Connection, anchor_id: &RetrievalAnchorId, - owner: &RetrievalAnchorOwnerV1, + owner: &FactOwnerV1, ) -> rusqlite::Result> { let owner_json = encode(owner)?; if !AnchorDispositionStateV1::serves_derivatives(current_state( @@ -442,7 +442,7 @@ mod tests { &snapshot, &RetrievalAnchorReadOperationV1::Derivatives { anchor_id: source.clone(), - owner: owner().into(), + owner: owner(), }, ) .unwrap(); @@ -455,7 +455,7 @@ mod tests { &snapshot, &RetrievalAnchorReadOperationV1::Tombstone { anchor_id: source, - owner: owner().into(), + owner: owner(), }, ) .unwrap(); @@ -469,7 +469,7 @@ mod tests { &snapshot, &RetrievalAnchorReadOperationV1::AnchorById { anchor_id: anchor("retrieval.source.fixture"), - owner: owner().into(), + owner: owner(), }, ) .unwrap(), @@ -592,7 +592,7 @@ mod tests { &snapshot, &RetrievalAnchorReadOperationV1::Derivatives { anchor_id: source, - owner: owner().into(), + owner: owner(), }, ) .unwrap(), @@ -613,7 +613,7 @@ mod tests { &snapshot, &RetrievalAnchorReadOperationV1::AnchorById { anchor_id: source, - owner: owner().into(), + owner: owner(), }, ) .is_err() @@ -658,7 +658,7 @@ mod tests { &snapshot, &RetrievalAnchorReadOperationV1::CurrentDisposition { anchor_id: source, - owner: owner().into(), + owner: owner(), }, ) .is_err() diff --git a/crates/tracedecay-rusqlite-runtime/src/repository/support.rs b/crates/tracedecay-rusqlite-runtime/src/repository/support.rs index c338429bb9..256c48fca6 100644 --- a/crates/tracedecay-rusqlite-runtime/src/repository/support.rs +++ b/crates/tracedecay-rusqlite-runtime/src/repository/support.rs @@ -30,13 +30,6 @@ pub(super) fn same_json(stored: &str, expected: &str) -> bool { } } -pub(super) fn canonical_digest(value: &T) -> rusqlite::Result { - let value = serde_json::to_value(value).map_err(|error| conversion(error.to_string()))?; - tracedecay_domain::canonical_sha256(&value) - .map(|digest| digest.as_str().to_owned()) - .map_err(|error| conversion(error.to_string())) -} - pub(super) fn conversion(error: impl Display) -> rusqlite::Error { rusqlite::Error::FromSqlConversionFailure(0, Type::Text, error.to_string().into()) } diff --git a/crates/tracedecay-rusqlite-runtime/src/watermark/mod.rs b/crates/tracedecay-rusqlite-runtime/src/watermark/mod.rs index 2c79345d77..2c4f714844 100644 --- a/crates/tracedecay-rusqlite-runtime/src/watermark/mod.rs +++ b/crates/tracedecay-rusqlite-runtime/src/watermark/mod.rs @@ -5,9 +5,7 @@ mod publisher; -pub use publisher::{ - CommitWatermarkPublicationError, CommitWatermarkSubscription, CommittedWatermarkPublisher, -}; +pub use publisher::{CommitWatermarkPublicationError, CommittedWatermarkPublisher}; #[cfg(test)] mod tests; diff --git a/crates/tracedecay-rusqlite-runtime/src/watermark/publisher.rs b/crates/tracedecay-rusqlite-runtime/src/watermark/publisher.rs index a75a7e7dfe..5a0c04bf96 100644 --- a/crates/tracedecay-rusqlite-runtime/src/watermark/publisher.rs +++ b/crates/tracedecay-rusqlite-runtime/src/watermark/publisher.rs @@ -1,18 +1,12 @@ use std::collections::BTreeMap; use std::error::Error; use std::fmt; -use std::future::Future; -use std::pin::Pin; -use std::sync::Arc; use tokio::sync::watch; use tracedecay_store::{ - CommitSequenceV1, ShardWatermarkV1, StoreCommitReceiptV1, StoreRuntimeBindingV1, - StoreShardIdV1, UnavailableReasonV1, + CommitSequenceV1, ShardWatermarkV1, StoreCommitReceiptV1, StoreRuntimeBindingV1, StoreShardIdV1, }; -use crate::read_consistency::{CommitWatermarkSource, WatermarkSourceState}; - #[derive(Clone, Debug, PartialEq, Eq)] pub enum CommitWatermarkPublicationError { DuplicateShard(Box), @@ -75,10 +69,9 @@ struct Channels { /// The small capability a writer calls only after its transaction commits. /// /// Publication is strictly monotonic and fenced to the bindings supplied at -/// construction. Keeping this capability distinct from the subscription makes -/// it impossible for readers or telemetry to advance commit truth. +/// construction. pub struct CommittedWatermarkPublisher { - channels: Arc, + channels: Channels, } impl CommittedWatermarkPublisher { @@ -113,16 +106,10 @@ impl CommittedWatermarkPublisher { } } Ok(Self { - channels: Arc::new(Channels { by_shard }), + channels: Channels { by_shard }, }) } - pub fn subscribe(&self) -> CommitWatermarkSubscription { - CommitWatermarkSubscription { - channels: Arc::clone(&self.channels), - } - } - pub(crate) fn current(&self, shard_id: &StoreShardIdV1) -> Option { self.channels .by_shard @@ -180,51 +167,3 @@ impl CommittedWatermarkPublisher { outcome } } - -/// Read-only view over committed writer notifications. -#[derive(Clone)] -pub struct CommitWatermarkSubscription { - channels: Arc, -} - -impl CommitWatermarkSource for CommitWatermarkSubscription { - fn current(&self, shard_id: &StoreShardIdV1) -> WatermarkSourceState { - self.channels - .by_shard - .get(shard_id) - .map(|sender| WatermarkSourceState::Available(sender.borrow().clone())) - .unwrap_or(WatermarkSourceState::Unavailable( - UnavailableReasonV1::MissingAuthority, - )) - } - - fn wait_for_change<'a>( - &'a self, - shard_id: &'a StoreShardIdV1, - after: &'a ShardWatermarkV1, - ) -> Pin + Send + 'a>> { - let receiver = self - .channels - .by_shard - .get(shard_id) - .map(watch::Sender::subscribe); - Box::pin(async move { - let Some(mut receiver) = receiver else { - return WatermarkSourceState::Unavailable(UnavailableReasonV1::MissingAuthority); - }; - loop { - let current = receiver.borrow_and_update().clone(); - if !current.same_history_as(after) - || current.commit_sequence > after.commit_sequence - { - return WatermarkSourceState::Available(current); - } - if receiver.changed().await.is_err() { - return WatermarkSourceState::Unavailable( - UnavailableReasonV1::MissingAuthority, - ); - } - } - }) - } -} diff --git a/crates/tracedecay-rusqlite-runtime/src/watermark/tests.rs b/crates/tracedecay-rusqlite-runtime/src/watermark/tests.rs index d163875e9e..e87841ad2b 100644 --- a/crates/tracedecay-rusqlite-runtime/src/watermark/tests.rs +++ b/crates/tracedecay-rusqlite-runtime/src/watermark/tests.rs @@ -1,12 +1,9 @@ -use std::future::Future; - use tracedecay_store::{ BrainId, CommitSequenceV1, ProjectId, StoreAuthorityEpochV1, StoreIncarnationV1, StoreRuntimeBindingV1, StoreShardIdV1, UserProfileId, }; use super::*; -use crate::read_consistency::{CommitWatermarkSource, WatermarkSourceState}; use tracedecay_domain::test_fixtures::id; @@ -31,56 +28,6 @@ fn watermark(binding: &StoreRuntimeBindingV1, sequence: u64) -> tracedecay_store } } -fn run(future: impl Future) -> T { - tokio::runtime::Builder::new_current_thread() - .enable_time() - .build() - .unwrap() - .block_on(future) -} - -#[test] -fn notification_before_subscribe_is_visible() { - run(async { - let binding = binding("project.before"); - let publisher = CommittedWatermarkPublisher::new(binding.clone()); - publisher - .publish_committed_watermark(watermark(&binding, 1)) - .unwrap(); - let source = publisher.subscribe(); - - assert_eq!( - source - .wait_for_change(&binding.shard_id, &watermark(&binding, 0)) - .await, - WatermarkSourceState::Available(watermark(&binding, 1)) - ); - }); -} - -#[test] -fn notification_after_subscribe_and_missed_notifications_yield_latest() { - run(async { - let binding = binding("project.after"); - let publisher = CommittedWatermarkPublisher::new(binding.clone()); - let source = publisher.subscribe(); - let initial = watermark(&binding, 0); - let waiting = source.wait_for_change(&binding.shard_id, &initial); - - publisher - .publish_committed_watermark(watermark(&binding, 1)) - .unwrap(); - publisher - .publish_committed_watermark(watermark(&binding, 2)) - .unwrap(); - - assert_eq!( - waiting.await, - WatermarkSourceState::Available(watermark(&binding, 2)) - ); - }); -} - #[test] fn wrong_epoch_and_non_monotonic_publications_are_rejected() { let binding = binding("project.fenced"); @@ -108,8 +55,8 @@ fn wrong_epoch_and_non_monotonic_publications_are_rejected() { "Display must describe the fence: {rendered}" ); assert_eq!( - publisher.subscribe().current(&binding.shard_id), - WatermarkSourceState::Available(watermark(&binding, 3)) + publisher.current(&binding.shard_id), + Some(watermark(&binding, 3)) ); } @@ -125,14 +72,13 @@ fn one_source_tracks_multiple_shards_without_crossing_histories() { publisher .publish_committed_watermark(watermark(&first, 2)) .unwrap(); - let source = publisher.subscribe(); assert_eq!( - source.current(&first.shard_id), - WatermarkSourceState::Available(watermark(&first, 2)) + publisher.current(&first.shard_id), + Some(watermark(&first, 2)) ); assert_eq!( - source.current(&second.shard_id), - WatermarkSourceState::Available(watermark(&second, 5)) + publisher.current(&second.shard_id), + Some(watermark(&second, 5)) ); } diff --git a/crates/tracedecay-rusqlite-runtime/src/work_product/authorization.rs b/crates/tracedecay-rusqlite-runtime/src/work_product/authorization.rs index 3be007ac76..2353d79b39 100644 --- a/crates/tracedecay-rusqlite-runtime/src/work_product/authorization.rs +++ b/crates/tracedecay-rusqlite-runtime/src/work_product/authorization.rs @@ -18,8 +18,9 @@ //! different question than the caller asked. use tracedecay_contracts::{ - AuthorizedWorkProductScopeV1, RequestContext, WorkProductOwnerAuthorizationErrorV1, - WorkProductOwnerAuthorizationPortV1, WorkProductSelectionScopeV1, WorkRelationScopeV1, + AuthorizedWorkProductScopeV1, RequestContext, WorkProductAuthorizedRelationScopeV1, + WorkProductOwnerAuthorizationErrorV1, WorkProductOwnerAuthorizationPortV1, + WorkProductSelectionScopeV1, }; use tracedecay_domain::UtcMicros; @@ -58,10 +59,10 @@ fn selection_is_within_resolved_scope( WorkProductSelectionScopeV1::Relations { relation_scopes } => { !relation_scopes.is_empty() && relation_scopes.iter().all(|scope| match scope { - WorkRelationScopeV1::Project { project_id } => { + WorkProductAuthorizedRelationScopeV1::Project { project_id } => { *project_id == resolved.project_id } - WorkRelationScopeV1::Repository { + WorkProductAuthorizedRelationScopeV1::Repository { project_id, repository_id, } => { diff --git a/crates/tracedecay-rusqlite-runtime/src/work_product/history.rs b/crates/tracedecay-rusqlite-runtime/src/work_product/history.rs index 86059833d6..c7c8457d0b 100644 --- a/crates/tracedecay-rusqlite-runtime/src/work_product/history.rs +++ b/crates/tracedecay-rusqlite-runtime/src/work_product/history.rs @@ -31,8 +31,8 @@ use tracedecay_contracts::{ OpaqueCursor, WorkHistoryCoverageV1, WorkHistoryReadPortV1, WorkHistoryRequestV1, - WorkHistoryV1, WorkProductApplicationErrorV1, WorkProductPortContextV1, - WorkProductSelectionScopeV1, WorkRelationScopeV1, + WorkHistoryV1, WorkProductApplicationErrorV1, WorkProductAuthorizedRelationScopeV1, + WorkProductPortContextV1, WorkProductSelectionScopeV1, }; use super::{covered_prefix, load_journal}; @@ -108,7 +108,9 @@ impl WorkHistoryReadPortV1 for WorkSqliteStorage { /// /// This mirrors the set the application re-derives when it checks the answer, /// so an event that would fail that check is never returned in the first place. -fn selected_relation_scopes(selection: &WorkProductSelectionScopeV1) -> Vec { +fn selected_relation_scopes( + selection: &WorkProductSelectionScopeV1, +) -> Vec { selection .relation_scopes() .map_or_else(Vec::new, |relations| relations.iter().cloned().collect()) diff --git a/crates/tracedecay-rusqlite-runtime/src/writer.rs b/crates/tracedecay-rusqlite-runtime/src/writer.rs index 94a1a3feae..c844fc9370 100644 --- a/crates/tracedecay-rusqlite-runtime/src/writer.rs +++ b/crates/tracedecay-rusqlite-runtime/src/writer.rs @@ -46,7 +46,7 @@ use crate::{ maintenance::ExclusiveMaintenancePermit, persistence::RuntimeWriterPersistence, telemetry::{WriterTelemetry, WriterTelemetrySnapshot}, - watermark::{CommitWatermarkSubscription, CommittedWatermarkPublisher}, + watermark::CommittedWatermarkPublisher, }; struct UnrestrictedRuntimeWriteAuthority; @@ -540,7 +540,6 @@ pub struct PersistentWriter { join: Option>, admission: Admission, telemetry: WriterTelemetry, - watermark_source: CommitWatermarkSubscription, checkpoint_status: watch::Receiver, checkpoint_pressure: watch::Receiver, opened_file_identity: Option, @@ -617,7 +616,6 @@ impl PersistentWriter { let expected_file_identity = locator.expected_file_identity(); let opened_database = locator.opened_database; let watermark_publisher = CommittedWatermarkPublisher::new(binding.clone()); - let watermark_source = watermark_publisher.subscribe(); let (sender, receiver) = mpsc::channel(capacity); // Exact-SQL transactions are serialized by the writer actor. Keep // the same bounded admission depth as ordinary writes so a second @@ -676,7 +674,6 @@ impl PersistentWriter { join: Some(join), admission, telemetry, - watermark_source, checkpoint_status, checkpoint_pressure, opened_file_identity, @@ -720,11 +717,6 @@ impl PersistentWriter { self.telemetry.snapshot() } - /// Returns a read-only view of this writer's committed watermark. - pub fn commit_watermark_source(&self) -> CommitWatermarkSubscription { - self.watermark_source.clone() - } - pub fn checkpoint_handle(&self) -> CheckpointHandle { CheckpointHandle { binding: self.binding.clone(), diff --git a/crates/tracedecay-rusqlite-runtime/src/writer/tests/authority.rs b/crates/tracedecay-rusqlite-runtime/src/writer/tests/authority.rs index b78bab603a..900613b55a 100644 --- a/crates/tracedecay-rusqlite-runtime/src/writer/tests/authority.rs +++ b/crates/tracedecay-rusqlite-runtime/src/writer/tests/authority.rs @@ -38,10 +38,7 @@ fn queued_fact_write_rechecks_authority_before_opening_a_transaction() { } #[test] -fn queued_evidence_and_anchor_writes_recheck_authority_before_sql_dispatch() { - let evidence = RepositoryWritePayloadV1::EvidenceAssembly(Box::new( - crate::repository::evidence_assembly::tests::write_fixture("authority.test"), - )); +fn queued_anchor_writes_recheck_authority_before_sql_dispatch() { let anchor = RepositoryWritePayloadV1::RetrievalAnchorDisposition(Box::new( RetrievalAnchorDispositionRecordV1::new( "disposition.authority.fixture", @@ -57,50 +54,45 @@ fn queued_evidence_and_anchor_writes_recheck_authority_before_sql_dispatch() { .unwrap(), )); - for (label, payload, digest_byte) in [ - ("evidence", evidence, 'e'), - ("retrieval_anchor", anchor, 'r'), - ] { - let database = TestDatabase::new(); - let request = project_fixture_request( - &format!("operation.authority.{label}"), - &format!("key.authority.{label}"), - digest_byte, - payload, - ); - let applied = Arc::new(AtomicU64::new(0)); - let writer = start(&database, &request, Arc::clone(&applied)); - let authority = Arc::new(RevokeAfterAdmissionAuthority { - admitted: AtomicBool::new(false), - }); - let probe = Arc::new(Probe::new(&request, None)); - let runtime = tokio::runtime::Builder::new_current_thread() - .build() - .unwrap(); + let database = TestDatabase::new(); + let request = project_fixture_request( + "operation.authority.retrieval_anchor", + "key.authority.retrieval_anchor", + 'r', + anchor, + ); + let applied = Arc::new(AtomicU64::new(0)); + let writer = start(&database, &request, Arc::clone(&applied)); + let authority = Arc::new(RevokeAfterAdmissionAuthority { + admitted: AtomicBool::new(false), + }); + let probe = Arc::new(Probe::new(&request, None)); + let runtime = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); - let outcome = runtime - .block_on(writer.submit_authorized(request, probe, authority)) - .unwrap(); + let outcome = runtime + .block_on(writer.submit_authorized(request, probe, authority)) + .unwrap(); - assert_eq!( - outcome, - RuntimeSubmitOutcomeV1::Unavailable { - reason: UnavailableReasonV1::MissingAuthority, - }, - "{label} write bypassed the actor authority recheck" - ); - assert_eq!(applied.load(Ordering::SeqCst), 0); - let table_count: i64 = Connection::open(&database.0) - .unwrap() - .query_row( - "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'writer_test'", - [], - |row| row.get(0), - ) - .unwrap(); - assert_eq!(table_count, 0); - writer.shutdown_and_join().unwrap(); - } + assert_eq!( + outcome, + RuntimeSubmitOutcomeV1::Unavailable { + reason: UnavailableReasonV1::MissingAuthority, + }, + "retrieval anchor write bypassed the actor authority recheck" + ); + assert_eq!(applied.load(Ordering::SeqCst), 0); + let table_count: i64 = Connection::open(&database.0) + .unwrap() + .query_row( + "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'writer_test'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(table_count, 0); + writer.shutdown_and_join().unwrap(); } #[test] diff --git a/crates/tracedecay-rusqlite-runtime/src/writer/transaction.rs b/crates/tracedecay-rusqlite-runtime/src/writer/transaction.rs index c3a324f893..71b5663eb4 100644 --- a/crates/tracedecay-rusqlite-runtime/src/writer/transaction.rs +++ b/crates/tracedecay-rusqlite-runtime/src/writer/transaction.rs @@ -16,11 +16,11 @@ use crate::{ RuntimeWriteAuthorityStage, admission::QueueItem, connection, - read_consistency::{CommitWatermarkPublicationError, CommittedWatermarkPublisher}, telemetry::{ LockWorkScope, WriterBatchMetrics, WriterLockWorkSnapshot, WriterTelemetry, WriterTransactionMetrics, WriterTransactionOutcome, take_observed_vm, }, + watermark::{CommitWatermarkPublicationError, CommittedWatermarkPublisher}, }; use super::{ @@ -627,10 +627,7 @@ mod tests { use tracedecay_store::{CommitSequenceV1, StoreCommitReceiptV1}; use super::*; - use crate::{ - read_consistency::{CommitWatermarkSource, WatermarkSourceState}, - test_support::{binding, metadata}, - }; + use crate::test_support::{binding, metadata}; fn receipt(sequence: u64) -> (StoreRuntimeBindingV1, StoreCommitReceiptV1) { let metadata = metadata("operation.publish", "key.publish", 'a'); @@ -658,11 +655,9 @@ mod tests { ) .unwrap(); - let WatermarkSourceState::Available(observed) = - publisher.subscribe().current(&binding.shard_id) - else { - panic!("committed watermark must be available"); - }; + let observed = publisher + .current(&binding.shard_id) + .expect("committed watermark must be available"); assert_eq!(observed.commit_sequence, receipt.commit_sequence); assert_eq!(observed.shard_id, receipt.shard_id); assert_eq!(observed.incarnation, receipt.incarnation); @@ -677,11 +672,9 @@ mod tests { publish_results([&result], &publisher).unwrap(); - let WatermarkSourceState::Available(observed) = - publisher.subscribe().current(&binding.shard_id) - else { - panic!("initial watermark must be available"); - }; + let observed = publisher + .current(&binding.shard_id) + .expect("initial watermark must be available"); assert_eq!(observed.commit_sequence, CommitSequenceV1(0)); } } diff --git a/crates/tracedecay-rusqlite-runtime/src/writer/worker/mod.rs b/crates/tracedecay-rusqlite-runtime/src/writer/worker/mod.rs index c02b79219b..dca7df8d5a 100644 --- a/crates/tracedecay-rusqlite-runtime/src/writer/worker/mod.rs +++ b/crates/tracedecay-rusqlite-runtime/src/writer/worker/mod.rs @@ -40,10 +40,10 @@ use crate::{ exact_sql::{ WriterCommand as ExactSqlWriterCommand, reject_writer_command, run_writer_command, }, - read_consistency::CommittedWatermarkPublisher, telemetry::{ LockWorkScope, WalCheckpointSample, WriterTelemetry, duration_micros, take_observed_vm, }, + watermark::CommittedWatermarkPublisher, }; use super::{ @@ -859,9 +859,18 @@ impl Worker { } prefer_auxiliary = true; } - if self.state.load(Ordering::Acquire) != WriterState::Faulted as u8 { - self.state - .store(WriterState::Closed as u8, Ordering::Release); + if self.state.load(Ordering::Acquire) == WriterState::Faulted as u8 { + return; + } + match checkpoint.truncate_at_shutdown() { + Ok(result) => { + self.publish_checkpoint_result(result); + self.state + .store(WriterState::Closed as u8, Ordering::Release); + } + Err(_) => self + .state + .store(WriterState::Faulted as u8, Ordering::Release), } } diff --git a/crates/tracedecay-rusqlite-runtime/tests/rusqlite_suite/runtime_actor/faults.rs b/crates/tracedecay-rusqlite-runtime/tests/rusqlite_suite/runtime_actor/faults.rs index 749587a3ae..be53147501 100644 --- a/crates/tracedecay-rusqlite-runtime/tests/rusqlite_suite/runtime_actor/faults.rs +++ b/crates/tracedecay-rusqlite-runtime/tests/rusqlite_suite/runtime_actor/faults.rs @@ -75,7 +75,7 @@ fn binding_mismatches_are_typed_and_corrupt_replay_faults_closed() { database .connect() .execute( - "UPDATE td_runtime_writer_idempotency_v2 SET original_receipt_json = '{}'", + "UPDATE td_runtime_writer_idempotency_v2 SET transaction_id = ''", [], ) .unwrap(); diff --git a/crates/tracedecay-rusqlite-runtime/tests/rusqlite_suite/work_product_graph_authority.rs b/crates/tracedecay-rusqlite-runtime/tests/rusqlite_suite/work_product_graph_authority.rs index 2d16d00d07..76c01a0866 100644 --- a/crates/tracedecay-rusqlite-runtime/tests/rusqlite_suite/work_product_graph_authority.rs +++ b/crates/tracedecay-rusqlite-runtime/tests/rusqlite_suite/work_product_graph_authority.rs @@ -20,9 +20,9 @@ use tracedecay_contracts::{ AddWorkTaskRequestV1, CancellationContext, CapabilityGrantSnapshot, CreateWorkProductRequestV1, Deadline, DisclosureClass, RequestContext, RequestId, ResolvedScope, WorkGraphReadRequestV1, WorkGraphReadV1, WorkGraphSelectionCoverageV1, WorkProductApplicationErrorV1, - WorkProductBindingV1, WorkProductExpectedAuthorityV1, WorkProductMutationIdentityV1, - WorkProductMutationServiceV1, WorkProductReadServiceV1, WorkProductRevisionPinsV1, - WorkProductSelectionScopeV1, WorkRelationScopeV1, + WorkProductAuthorizedRelationScopeV1, WorkProductBindingV1, WorkProductExpectedAuthorityV1, + WorkProductMutationIdentityV1, WorkProductMutationServiceV1, WorkProductReadServiceV1, + WorkProductRevisionPinsV1, WorkProductSelectionScopeV1, }; use tracedecay_domain::{ AcceptanceCriterionId, ActorId, CatalogGenerationId, ConfigurationRevisionId, InitiativeId, @@ -55,10 +55,12 @@ fn binding() -> WorkProductBindingV1 { } fn repository_selection() -> WorkProductSelectionScopeV1 { - WorkProductSelectionScopeV1::relations(BTreeSet::from([WorkRelationScopeV1::Repository { - project_id: id(PROJECT), - repository_id: id(REPOSITORY), - }])) + WorkProductSelectionScopeV1::relations(BTreeSet::from([ + WorkProductAuthorizedRelationScopeV1::Repository { + project_id: id(PROJECT), + repository_id: id(REPOSITORY), + }, + ])) .unwrap() } @@ -526,11 +528,11 @@ fn a_selection_naming_another_project_is_refused_rather_than_narrowed() { .expect("create the work product"); let foreign = WorkProductSelectionScopeV1::relations(BTreeSet::from([ - WorkRelationScopeV1::Repository { + WorkProductAuthorizedRelationScopeV1::Repository { project_id: id(PROJECT), repository_id: id(REPOSITORY), }, - WorkRelationScopeV1::Project { + WorkProductAuthorizedRelationScopeV1::Project { project_id: id::("project.someone-else"), }, ])) diff --git a/crates/tracedecay-rusqlite-runtime/tests/rusqlite_suite/work_product_query_authority.rs b/crates/tracedecay-rusqlite-runtime/tests/rusqlite_suite/work_product_query_authority.rs index 048597c289..2849bc5b61 100644 --- a/crates/tracedecay-rusqlite-runtime/tests/rusqlite_suite/work_product_query_authority.rs +++ b/crates/tracedecay-rusqlite-runtime/tests/rusqlite_suite/work_product_query_authority.rs @@ -31,10 +31,11 @@ use tracedecay_contracts::{ ResolvedScope, SelectedWorkEvidenceV1, VerifiedWorkGraphVersionV1, WorkEvidenceExpandRequestV1, WorkEvidenceSelectRequestV1, WorkGraphReadRequestV1, WorkGraphReadV1, WorkGraphSelectionCoverageV1, WorkHistoryCoverageV1, WorkHistoryRequestV1, - WorkHistoryServiceV1, WorkHistoryV1, WorkProductApplicationErrorV1, WorkProductBindingV1, - WorkProductEvidenceServiceV1, WorkProductExpectedAuthorityV1, WorkProductMutationIdentityV1, - WorkProductMutationReceiptV1, WorkProductMutationServiceV1, WorkProductReadServiceV1, - WorkProductRevisionPinsV1, WorkProductSelectionScopeV1, WorkRelationScopeV1, + WorkHistoryServiceV1, WorkHistoryV1, WorkProductApplicationErrorV1, + WorkProductAuthorizedRelationScopeV1, WorkProductBindingV1, WorkProductEvidenceServiceV1, + WorkProductExpectedAuthorityV1, WorkProductMutationIdentityV1, WorkProductMutationReceiptV1, + WorkProductMutationServiceV1, WorkProductReadServiceV1, WorkProductRevisionPinsV1, + WorkProductSelectionScopeV1, }; use tracedecay_domain::{ AcceptanceCriterionId, ActorId, CatalogGenerationId, ConfigurationRevisionId, InitiativeId, @@ -68,10 +69,12 @@ fn binding() -> WorkProductBindingV1 { } fn repository_selection() -> WorkProductSelectionScopeV1 { - WorkProductSelectionScopeV1::relations(BTreeSet::from([WorkRelationScopeV1::Repository { - project_id: id(PROJECT), - repository_id: id(REPOSITORY), - }])) + WorkProductSelectionScopeV1::relations(BTreeSet::from([ + WorkProductAuthorizedRelationScopeV1::Repository { + project_id: id(PROJECT), + repository_id: id(REPOSITORY), + }, + ])) .unwrap() } diff --git a/crates/tracedecay-sdk/src/codegen.rs b/crates/tracedecay-sdk/src/codegen.rs index 6983f06b42..85093d4f3f 100644 --- a/crates/tracedecay-sdk/src/codegen.rs +++ b/crates/tracedecay-sdk/src/codegen.rs @@ -1484,7 +1484,18 @@ mod tests { .iter() .filter(|operation| operation.operation_id.starts_with("operation.workflow.")) .collect::>(); - assert!(!workflows.is_empty()); + let register = workflows + .iter() + .find(|operation| operation.operation_id == "operation.workflow.register_definition") + .expect("workflow registration operation"); + assert_eq!( + http_route(register), + Some("/application/workflow/register-definition") + ); + assert_eq!( + register.binding, + "binding.http.workflow.register_definition" + ); assert!(workflows.iter().all(|operation| { http_route(operation).is_some_and(|route| route.starts_with("/application/workflow/")) && operation.binding.starts_with("binding.http.workflow.") @@ -1522,7 +1533,15 @@ mod tests { }) .collect::>(); - assert!(!configuration.is_empty()); + let get = configuration + .iter() + .find(|operation| operation.operation_id == "operation.application.configuration_get") + .expect("configuration get operation"); + assert_eq!( + http_route(get), + Some("/application/configuration/configuration_get") + ); + assert_eq!(get.binding, "binding.http.configuration_get.v1"); assert!(unavailable.iter().all(|operation| { !operation .operation_id diff --git a/crates/tracedecay-sdk/tests/sdk_suite/production_daemon.rs b/crates/tracedecay-sdk/tests/sdk_suite/production_daemon.rs index 28b3d9d933..543d0c78c9 100644 --- a/crates/tracedecay-sdk/tests/sdk_suite/production_daemon.rs +++ b/crates/tracedecay-sdk/tests/sdk_suite/production_daemon.rs @@ -344,7 +344,25 @@ fn enrolled_remote_client_rejects_an_untrusted_private_authority_and_isolates_en .unwrap() .enroll(&request, enrollment_credential) .expect("the SDK must join /remote/ and trust the configured private root"); - assert!(enrolled.result.is_ok()); + let envelope = enrolled.result.as_ref().expect("enrollment is admitted"); + assert_eq!( + envelope.request_id.as_str(), + "request.remote-sdk-production" + ); + let record = envelope + .outcome + .payload() + .expect("admitted enrollment carries its credential record"); + assert_eq!( + record.enrollment_id.as_str(), + "enrollment.remote-sdk-production" + ); + assert_eq!(record.brain_id, grant.brain_id); + assert_eq!(record.node_id, grant.node_id); + assert_eq!(record.expires_at, grant.expires_at); + assert_eq!(record.revoked_at, None); + assert_eq!(record.capabilities, grant.capabilities); + assert_eq!(record.scope, grant.scope); let local_route_response = tls_http11_request( first_remote, diff --git a/crates/tracedecay-search-eval/Cargo.toml b/crates/tracedecay-search-eval/Cargo.toml index e168d20b3c..25d044c29f 100644 --- a/crates/tracedecay-search-eval/Cargo.toml +++ b/crates/tracedecay-search-eval/Cargo.toml @@ -22,7 +22,7 @@ tracedecay-capture = { path = "../tracedecay-capture", version = "0.1.0" } tracedecay-code-index = { path = "../tracedecay-code-index", version = "0.1.0", default-features = false, features = ["eval-helpers"] } tracedecay-domain = { path = "../tracedecay-domain", version = "0.1.0" } tracedecay-private-fs = { path = "../tracedecay-private-fs", version = "0.1.0" } -tracedecay-query = { path = "../tracedecay-query", version = "0.1.0", features = ["search-eval"] } +tracedecay-query = { path = "../tracedecay-query", version = "0.1.0" } tracedecay-runtime-core = { path = "../tracedecay-runtime-core", version = "0.1.0" } tracedecay-sessions = { path = "../tracedecay-sessions", version = "0.1.0" } diff --git a/crates/tracedecay-search-eval/src/candidate_output.rs b/crates/tracedecay-search-eval/src/candidate_output.rs index 3ce4a90842..771bee3246 100644 --- a/crates/tracedecay-search-eval/src/candidate_output.rs +++ b/crates/tracedecay-search-eval/src/candidate_output.rs @@ -1,8 +1,9 @@ //! Production-bound exact/lexical/graph candidate-output generator. //! -//! Builds one published code generation from checked-in sanitized corpus -//! fixtures, then runs the shared `CompositionKernel` over the real exact, -//! lexical, and graph production lanes. +//! Publishes one code generation per queried scope set from checked-in +//! sanitized corpus fixtures, seals each into the production lexical artifact, +//! then runs the shared `CompositionKernel` over the real exact, lexical, and +//! graph production lanes. //! //! Outputs deterministic checked-in `train` / `validation` candidate records //! plus current/10x resource samples and ranking receipts. Cancellation is @@ -18,16 +19,19 @@ use std::sync::{Arc, Mutex}; use std::time::Instant; use serde::Serialize; +use sha2::{Digest, Sha256}; +use tempfile::TempDir; -use tracedecay_code_index::chunks::{ExtractionAdmittedCodeSearchChunkV1, content_digest}; +use tracedecay_code_index::chunks::content_digest; use tracedecay_code_index::graph_projection::CodeGraphEvidenceReader; use tracedecay_code_index::languages::{LanguageRegistry, StaticLanguageRegistry}; use tracedecay_code_index::production::{ CodeIndexAtomicPublicationPort, CodeIndexBuildRequestV1, CodeIndexCapturedFileV1, - CodeIndexGenerationScopeV1, CodeIndexProductionConfigV1, CodeIndexProductionOwnerV1, - CodeIndexPublicationStoreErrorV1, CodeIndexPublishedGenerationV1, + CodeIndexGenerationScopeV1, CodeIndexProductionConfigV1, CodeIndexProductionErrorV1, + CodeIndexProductionOwnerV1, CodeIndexPublicationStoreErrorV1, CodeIndexPublishedGenerationV1, CodeIndexRepositoryParseIdentityV1, DAEMON_CODE_INDEX_CHUNKER_REVISION, - VerifiedSealedLexicalSymbolDisplayV1, + SealedGenerationSegmentPublicationV1, VerifiedSealedLexicalPageReadV1, + VerifiedSealedLexicalPageSourceV1, VerifiedSealedLexicalSymbolDisplayV1, }; use tracedecay_code_index::projection::{ ChunkProjectionDecisionV1, CodeChunkProjectionSink, ProjectionReceiptBuilderV1, @@ -46,28 +50,32 @@ use tracedecay_domain::{ ProjectionBatchRequestV1, ProjectionKeyV1, ProjectionKindV1, ProjectionOperationV1, ProjectionOutcomeV1, PublicRetrieverStatus, QueryFallbackSubpayload, QueryNormalizationRevision, RelationEdgeKindV1, RepositoryDirtyStateV1, RepositoryId, - RetrievalFailure, RetrievalRequest, RetrievalScope, RetrievalSnapshot, RetrieverKind, - RetrieverOutcome, SanitizationReceiptId, SanitizedCodeFileV1, SanitizedCodeSnapshotV1, - SanitizerRevision, SingleRootScopeV1, SnapshotFileDispositionV1, SymbolOccurrenceId, + RetrievalFailure, RetrievalRequest, RetrievalScope, RetrievalSnapshot, RetrieverBatch, + RetrieverCoverage, RetrieverKind, RetrieverOutcome, SanitizationReceiptId, SanitizedCodeFileV1, + SanitizedCodeSnapshotV1, SanitizerRevision, SingleRootScopeV1, SnapshotFileDispositionV1, TemporalModeV1, UtcMicros, VectorWatermark, }; use tracedecay_query::native_git::NativeHistoricalBlobReaderV1; use tracedecay_query::retrieval::exact::{ - CentralExactAdmissionAuthorityV1, ExactAdmissionAuthority, ExactLane, ExactLaneRequest, - ExactLaneRetriever, + CentralExactAdmissionAuthorityV1, ExactAdmissionAuthority, ExactLane, ExactLaneEvidence, + ExactLaneRequest, ExactLaneRetriever, }; use tracedecay_query::retrieval::fusion::{ - CompositionKernel, CompositionLaneInput, CompositionOutputV1, FusionStageInput, + CompositionKernel, CompositionLaneInput, CompositionOutputV1, FusionStageError, + FusionStageInput, }; use tracedecay_query::retrieval::graph::{ - GraphLane, GraphLaneRequest, GraphLaneRetriever, production_code_index_freshness, + GraphLane, GraphLaneEvidence, GraphLaneRequest, GraphLaneRetriever, + production_code_index_freshness, }; use tracedecay_query::retrieval::lexical::{ - CodeLexicalProjectionAdapterV1, CodeLexicalProjectionMetadataV1, LexicalLane, - LexicalLaneRequest, LexicalLaneRetriever, LexicalRouteOutcomeV1, LexicalRoutePlanV1, - LexicalRoutingV1, lexical_query_parts, merge_lexical_routes, + CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, CodeLexicalArtifactBuilderV1, + CodeLexicalArtifactFinalizationStepV1, CodeLexicalArtifactReaderV1, CodeLexicalCloneRouteV1, + CodeLexicalProjectionMetadataV1, LexicalLane, LexicalLaneEvidence, LexicalLaneRequest, + LexicalLaneRetriever, LexicalRouteOutcomeV1, LexicalRoutePlanV1, LexicalRoutingV1, + lexical_query_parts, merge_lexical_routes, }; -use tracedecay_query::retrieval::ports::CodeCandidateBindingV1; +use tracedecay_query::retrieval::ports::{CodeCandidateBindingV1, RETRIEVAL_CANDIDATE_BATCH_SIZE}; use tracedecay_query::search_quality::candidate_output::{ CandidateOutputError, CandidateWorkloadV1, CorpusDocumentV1, EVALUATION_CACHE_STATE, EVALUATION_SEED, GenerateCandidateOutputsResultV1, HistoricalQueryExecutionV1, @@ -197,15 +205,27 @@ struct OccurrenceMapEntry { display_anchors: Vec, } -/// Retrieval adapters keyed by canonical allowed-scope key (sorted, deduped), -/// as produced by [`canonical_scope_key`]. -type ScopedLexicalProjections = BTreeMap, CodeLexicalProjectionAdapterV1>; -type ScopedGraphEvidence = BTreeMap, CodeGraphEvidenceReader>; +/// Page byte bound for draining a sealed generation into its lexical artifact, +/// the daemon's text-artifact page bound. +const LEXICAL_ARTIFACT_PAGE_BYTES: usize = 4 * 1024 * 1024; -struct PublishedCorpus { +/// One queried scope set's corpus: the generation published over exactly the +/// files those scopes admit, its sealed lexical artifact, and its graph +/// evidence. Each scope set is its own repository snapshot, so lexical +/// statistics and graph edges never cross into files a query cannot see. +struct ScopedCorpus { generation: Arc, - lexical_projections: ScopedLexicalProjections, - graph_projections: ScopedGraphEvidence, + lexical: CodeLexicalArtifactReaderV1, + graph: CodeGraphEvidenceReader, + /// Owns the sealed artifact file `lexical` serves. + _artifact_directory: TempDir, +} + +struct PublishedCorpus { + /// Keyed by canonical allowed-scope key, as produced by + /// [`canonical_scope_key`]; `None` when the scopes admit no indexable + /// source file. + scopes: BTreeMap, Option>, occurrence_map: BTreeMap, repo_root: PathBuf, source_commit: GitOidV1, @@ -237,26 +257,110 @@ fn canonical_scope_key(scopes: &[String]) -> Vec { key } -/// Build every scoped retrieval projection the workload's queries need. -/// -/// Preparation is measured on its own span so query evaluation timing can -/// neither absorb nor hide it. Cost is O(chunks + scope memberships): the -/// corpus is classified once through a reverse scope map rather than once per -/// distinct scope set. -#[hotpath::measure(label = "search_eval.corpus.query_projections")] -fn build_query_projections( +/// Seal `generation` into the production lexical artifact and reopen it for +/// serving: the same partitioned encoding, verified page source, builder, and +/// reader the daemon runs, over an in-memory segment store and a private +/// temporary directory. +#[hotpath::measure(label = "search_eval.corpus.lexical_artifact")] +fn seal_lexical_artifact( generation: &CodeIndexPublishedGenerationV1, - file_scopes: &BTreeMap, - symbol_displays: Arc>, - queries: &[WorkloadQueryV1], -) -> Result<(ScopedLexicalProjections, ScopedGraphEvidence), CandidateOutputError> { + metadata: &CodeLexicalProjectionMetadataV1, +) -> Result<(TempDir, CodeLexicalArtifactReaderV1), CandidateOutputError> { + let contract = + |error: &dyn std::fmt::Display| CandidateOutputError::Contract(error.to_string()); + let mut segments = BTreeMap::new(); + let mut evidence_pack = Vec::new(); + let manifest = generation + .encode_partitioned_sealed(|publication| { + match publication { + SealedGenerationSegmentPublicationV1::File { digest, bytes } => { + segments.insert(digest.as_str().to_owned(), bytes.to_vec()); + } + SealedGenerationSegmentPublicationV1::GenerationEvidencePage { bytes, .. } => { + evidence_pack.extend_from_slice(bytes); + } + SealedGenerationSegmentPublicationV1::GenerationEvidenceCommit { + segment_digest, + .. + } => { + segments.insert( + segment_digest.as_str().to_owned(), + std::mem::take(&mut evidence_pack), + ); + } + } + Ok(()) + }) + .map_err(|error| contract(&error))?; + let state_digest = ManifestDigest::from_sha256_bytes(&Sha256::digest(&manifest)) + .map_err(|error| contract(&error))?; + let segments = Arc::new(segments); + let mut source = VerifiedSealedLexicalPageSourceV1::open_partitioned_sealed( + &manifest, + state_digest, + move |digest, _, buffer, _control| { + let bytes = segments.get(digest.as_str()).ok_or_else(|| { + CodeIndexProductionErrorV1::Contract( + "sealed evaluation segment is missing".to_owned(), + ) + })?; + buffer.clear(); + buffer.extend_from_slice(bytes); + Ok(()) + }, + RETRIEVAL_CANDIDATE_BATCH_SIZE, + LEXICAL_ARTIFACT_PAGE_BYTES, + ) + .map_err(|error| contract(&error))?; + let directory = tempfile::tempdir().map_err(|error| contract(&error))?; + let path = directory.path().join("lexical-artifact.sqlite"); + let mut builder = CodeLexicalArtifactBuilderV1::create(&path, metadata.clone()) + .map_err(|error| contract(&error))?; + let receipt = loop { + match source + .next_page(&ActiveControl) + .map_err(|error| contract(&error))? + { + VerifiedSealedLexicalPageReadV1::Page(page) => { + builder + .append_page(&page, &ActiveControl) + .map_err(|error| contract(&error))?; + } + VerifiedSealedLexicalPageReadV1::Complete(receipt) => break receipt, + } + }; + let verified = loop { + match builder + .advance_finalization(&receipt, 4_096, &ActiveControl) + .map_err(|error| contract(&error))? + { + CodeLexicalArtifactFinalizationStepV1::Pending { .. } => {} + CodeLexicalArtifactFinalizationStepV1::Ready(verified) => break *verified, + } + }; + drop(builder); + let reader = CodeLexicalArtifactReaderV1::open_with_control( + &path, + &verified, + metadata, + CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, + &ActiveControl, + ) + .map_err(|error| contract(&error))?; + Ok((directory, reader)) +} + +/// The lexical artifact and graph evidence over every file of `generation`. +fn scoped_retrieval( + generation: Arc, +) -> Result { let generation_id = generation.manifest().generation_id.clone(); let freshness = production_code_index_freshness( generation.manifest().seal.sealed_at, id::("policy.candidate.v1")?, ) .map_err(|error| CandidateOutputError::Contract(error.to_string()))?; - let metadata = Arc::new(CodeLexicalProjectionMetadataV1 { + let metadata = CodeLexicalProjectionMetadataV1 { generation: generation_id.clone(), repository_id: Some(generation.snapshot().repository.clone()), logical_paths: generation @@ -273,80 +377,27 @@ fn build_query_projections( tracedecay_query::retrieval::QUERY_LEXICAL_RETRIEVER_REVISION_V1, )?, exact_score_domain: id(tracedecay_query::retrieval::QUERY_EXACT_SCORE_DOMAIN_V1)?, - }); - // Canonical scope keys, deduplicated once; the position is the bucket id. - let scope_keys = queries - .iter() - .map(|query| canonical_scope_key(&query.allowed_scopes)) - .collect::>() - .into_iter() - .collect::>(); - // Reverse map from a file scope to every scope key admitting it, so one - // corpus pass places each chunk in all of its buckets instead of scanning - // the corpus once per distinct scope set. - let mut interested_buckets: BTreeMap<&str, Vec> = BTreeMap::new(); - for (bucket, scope_key) in scope_keys.iter().enumerate() { - for scope in scope_key { - interested_buckets - .entry(scope.as_str()) - .or_default() - .push(bucket); - } - } - let buckets_for = |file_occurrence_id: &str| { - file_scopes - .get(file_occurrence_id) - .and_then(|scope| interested_buckets.get(scope.as_str())) - .into_iter() - .flatten() - .copied() + clone_route: Some(CodeLexicalCloneRouteV1 { + project_id: generation.manifest().project_id.clone(), + worktree_id: generation.snapshot().worktree.clone(), + snapshot_digest: generation.manifest().snapshot_digest.clone(), + }), }; - // Corpus order within each bucket is the order the admitted sweep and the - // chunk manifest already carry, exactly what a per-scope filter yielded. - let admitted = generation - .admitted_chunks() - .map_err(|error| CandidateOutputError::Contract(error.to_string()))?; - let mut lexical_chunks: Vec> = - vec![Vec::new(); scope_keys.len()]; - for chunk in admitted.iter() { - for bucket in buckets_for(chunk.chunk().anchor.file_occurrence_id.as_str()) { - lexical_chunks[bucket].push(chunk.clone()); - } - } - drop(admitted); - let mut graph_chunks: Vec>> = vec![Vec::new(); scope_keys.len()]; - for chunk in generation.chunks().chunks() { - for bucket in buckets_for(chunk.anchor.file_occurrence_id.as_str()) { - graph_chunks[bucket].push(Arc::clone(chunk)); - } - } - let mut lexical = BTreeMap::new(); - let mut graph = BTreeMap::new(); - for ((scope_key, chunks), graph_chunks) in - scope_keys.into_iter().zip(lexical_chunks).zip(graph_chunks) - { - lexical.insert( - scope_key.clone(), - CodeLexicalProjectionAdapterV1::new_admitted( - Arc::clone(&metadata), - chunks, - Arc::clone(&symbol_displays), - ) - .map_err(|error| CandidateOutputError::Contract(error.to_string()))?, - ); - graph.insert( - scope_key, - CodeGraphEvidenceReader::new_for_evaluation( - generation_id.clone(), - Some(generation.snapshot().repository.clone()), - freshness.clone(), - generation.edges(), - &graph_chunks, - ) - .map_err(|error| CandidateOutputError::Contract(error.to_string()))?, - ); - } - Ok((lexical, graph)) + let (artifact_directory, lexical) = seal_lexical_artifact(&generation, &metadata)?; + let graph = CodeGraphEvidenceReader::new_for_evaluation( + generation_id, + Some(generation.snapshot().repository.clone()), + freshness, + generation.edges(), + generation.chunks().chunks(), + ) + .map_err(|error| CandidateOutputError::Contract(error.to_string()))?; + Ok(ScopedCorpus { + generation, + lexical, + graph, + _artifact_directory: artifact_directory, + }) } /// The corpora published for one candidate-generation call, memoized by scale. @@ -725,8 +776,68 @@ fn compose_production_query( profile: &ProfileSpecV1, query: &WorkloadQueryV1, ) -> Result { - let generation_id = published.generation.manifest().generation_id.clone(); - let request = retrieval_request(&profile.profile_id, published)?; + let scope_key = canonical_scope_key(&query.allowed_scopes); + let scoped = published.scopes.get(&scope_key).ok_or_else(|| { + CandidateOutputError::Contract(format!( + "missing scoped corpus for query {}", + query.query_id + )) + })?; + let lanes = match scoped { + Some(scoped) => production_lanes(scoped, profile, query)?, + None => unindexed_scope_lanes()?, + }; + let kernel = CompositionKernel::new(id::( + tracedecay_query::retrieval::QUERY_RANKING_REVISION_V1, + )?); + kernel + .compose( + &FusionStageInput { + profile: fusion_profile(profile)?, + lanes, + }, + &evaluated_diversity_policy()?, + ) + .map_err(|error| CandidateOutputError::Contract(error.to_string())) +} + +/// A scope set with no indexable source file publishes no generation: its +/// exact and lexical lanes complete over zero documents, and with no seeds the +/// graph lane is unavailable exactly as it is for any seedless query. +fn unindexed_scope_lanes() -> Result, CandidateOutputError> { + fn empty() -> RetrieverOutcome> { + RetrieverOutcome::Complete(RetrieverBatch { + candidates: Vec::new(), + evidence_by_occurrence: BTreeMap::new(), + coverage: RetrieverCoverage::default(), + continuation: None, + }) + } + let contract = |error: FusionStageError| CandidateOutputError::Contract(error.to_string()); + Ok(vec![ + CompositionLaneInput::new(RetrieverKind::ExactLiteral, empty::()) + .map_err(contract)?, + CompositionLaneInput::new(RetrieverKind::Lexical, empty::()) + .map_err(contract)?, + CompositionLaneInput::new( + RetrieverKind::Graph, + RetrieverOutcome::>::Unavailable( + RetrievalFailure::AuthorityUnavailable { + detail: "no graph seeds from exact/lexical".to_owned(), + }, + ), + ) + .map_err(contract)?, + ]) +} + +fn production_lanes( + scoped: &ScopedCorpus, + profile: &ProfileSpecV1, + query: &WorkloadQueryV1, +) -> Result, CandidateOutputError> { + let generation_id = scoped.generation.manifest().generation_id.clone(); + let request = retrieval_request(&profile.profile_id, &scoped.generation)?; let query_view = EphemeralSanitizedQueryViewV1::sanitize( &query.query, id::(tracedecay_query::retrieval::QUERY_SANITIZER_REVISION_V1)?, @@ -736,37 +847,15 @@ fn compose_production_query( ) .map_err(|error| CandidateOutputError::Contract(error.to_string()))?; - let scope_key = canonical_scope_key(&query.allowed_scopes); - let lexical_projection = published - .lexical_projections - .get(&scope_key) - .cloned() - .ok_or_else(|| { - CandidateOutputError::Contract(format!( - "missing lexical projection for query {}", - query.query_id - )) - })?; let authority = CentralExactAdmissionAuthorityV1::new(id::( tracedecay_query::retrieval::QUERY_EXACT_RULE_REVISION_V1, )?); let exact_lane = ExactLane::new( authority.clone(), - lexical_projection.exact_adapter(authority.clone()), - ); - let lexical_lane = LexicalLane::new(lexical_projection); - let graph_lane = GraphLane::new( - published - .graph_projections - .get(&scope_key) - .cloned() - .ok_or_else(|| { - CandidateOutputError::Contract(format!( - "missing graph projection for query {}", - query.query_id - )) - })?, + scoped.lexical.exact_adapter(authority.clone()), ); + let lexical_lane = LexicalLane::new(scoped.lexical.clone()); + let graph_lane = GraphLane::new(scoped.graph.clone()); let budget = retrieval_budget(); let exact_request = ExactLaneRequest { @@ -839,26 +928,14 @@ fn compose_production_query( .map_err(|error| CandidateOutputError::Contract(error.to_string()))? }; - let kernel = CompositionKernel::new(id::( - tracedecay_query::retrieval::QUERY_RANKING_REVISION_V1, - )?); - let lanes = vec![ + Ok(vec![ CompositionLaneInput::new(RetrieverKind::ExactLiteral, exact_outcome) .map_err(|error| CandidateOutputError::Contract(error.to_string()))?, CompositionLaneInput::new(RetrieverKind::Lexical, lexical_outcome) .map_err(|error| CandidateOutputError::Contract(error.to_string()))?, CompositionLaneInput::new(RetrieverKind::Graph, graph_outcome) .map_err(|error| CandidateOutputError::Contract(error.to_string()))?, - ]; - kernel - .compose( - &FusionStageInput { - profile: fusion_profile(profile)?, - lanes, - }, - &evaluated_diversity_policy()?, - ) - .map_err(|error| CandidateOutputError::Contract(error.to_string())) + ]) } fn query_lane_coverage( @@ -1128,19 +1205,142 @@ fn publish_corpus_with_scale( copies: usize, admitted_scope: AdmittedCorpusScopeFn, ) -> Result { - if copies == 0 { - return Err(CandidateOutputError::Contract( - "corpus scale must be positive".to_owned(), - )); - } + let expected_chunks = match copies { + 1 => workload.execution_contract.exact_eligible_chunks_current, + 10 => workload.execution_contract.exact_eligible_chunks_10x, + _ => { + return Err(CandidateOutputError::Contract( + "evaluation corpus scale must be current or exact 10x".to_owned(), + )); + } + }; let corpus_digest = compute_corpus_digest(repo_root, workload)?; + let scope_keys = workload + .queries + .iter() + .map(|query| canonical_scope_key(&query.allowed_scopes)) + .collect::>(); + let mut scopes = BTreeMap::new(); + let mut occurrence_map = BTreeMap::new(); + // Chunk and admitted-chunk counts per corpus file. Chunking is file-local, + // so a file several scope sets admit is counted once. + let mut file_chunks: BTreeMap = BTreeMap::new(); + for scope_key in scope_keys { + let documents = workload + .corpus + .iter() + .filter(|document| scope_key.contains(&document.scope)) + .collect::>(); + let Some((generation, file_to_document)) = + publish_scope_generation(repo_root, &corpus_digest, &scope_key, &documents, copies)? + else { + scopes.insert(scope_key, None); + continue; + }; + let mut scope_file_chunks: BTreeMap = BTreeMap::new(); + for chunk in generation.chunks().chunks() { + scope_file_chunks + .entry(chunk.anchor.file_occurrence_id.clone()) + .or_default() + .0 += 1; + } + let admitted = generation + .admitted_chunks() + .map_err(|error| CandidateOutputError::Contract(error.to_string()))?; + for chunk in admitted.iter() { + scope_file_chunks + .entry(chunk.chunk().anchor.file_occurrence_id.clone()) + .or_default() + .1 += 1; + } + drop(admitted); + for (file, counts) in scope_file_chunks { + file_chunks.entry(file).or_insert(counts); + } + let symbol_displays: BTreeMap<_, _> = generation + .symbols() + .symbols + .iter() + .map(|symbol| { + ( + symbol.occurrence.clone(), + VerifiedSealedLexicalSymbolDisplayV1::from(symbol.as_ref()), + ) + }) + .collect(); + for chunk in generation.chunks().chunks() { + let Some(document) = file_to_document.get(chunk.anchor.file_occurrence_id.as_str()) + else { + continue; + }; + let qualified_name = chunk + .anchor + .symbol_occurrence_id + .as_ref() + .and_then(|symbol| symbol_displays.get(symbol)) + .map(VerifiedSealedLexicalSymbolDisplayV1::qualified_name); + let display_anchors = display_anchors_for_chunk(chunk, document, qualified_name); + let entry = || OccurrenceMapEntry { + document_id: document.document_id.clone(), + scope: document.scope.clone(), + display_anchors: display_anchors.clone(), + }; + if let Some(symbol) = &chunk.anchor.symbol_occurrence_id { + occurrence_map.insert(format!("code-symbol:{}", symbol.as_str()), entry()); + occurrence_map.insert(format!("code-graph:{}", symbol.as_str()), entry()); + } + occurrence_map.insert(format!("code-chunk:{}", chunk.id.as_str()), entry()); + } + scopes.insert( + scope_key, + Some(hotpath::measure_block!( + "search_eval.corpus.scope_retrieval", + scoped_retrieval(generation) + )?), + ); + } + let observed_chunks: u64 = file_chunks.values().map(|(chunks, _)| chunks).sum(); + if observed_chunks != expected_chunks { + return Err(CandidateOutputError::Contract(format!( + "eligible chunk count mismatch for {copies}x corpus: declared {expected_chunks}, observed {observed_chunks}" + ))); + } + Ok(PublishedCorpus { + scopes, + occurrence_map, + repo_root: repo_root.to_path_buf(), + source_commit: GitOidV1::new(workload.source_repository_commit.clone()) + .map_err(|error| CandidateOutputError::Contract(error.to_string()))?, + corpus: workload.corpus.clone(), + corpus_digest, + eligible_chunks: file_chunks.values().map(|(_, admitted)| admitted).sum(), + admitted_scope, + }) +} + +/// A published generation and the corpus document behind each file path. +type ScopeGeneration = ( + Arc, + BTreeMap, +); + +/// Publish one generation over `documents` (each copied `copies` times) with +/// the daemon's production code-index owner, returning it with the corpus +/// document each file occurrence came from. `None` when no document is in an +/// indexable language: production publishes no generation for such a tree. +fn publish_scope_generation( + repo_root: &Path, + corpus_digest: &str, + scope_key: &[String], + documents: &[&CorpusDocumentV1], + copies: usize, +) -> Result, CandidateOutputError> { let language_registry = StaticLanguageRegistry::new(); let mut files = Vec::new(); let mut captured = Vec::new(); let mut file_to_document = BTreeMap::new(); - let mut file_scopes = BTreeMap::new(); for copy in 0..copies { - for document in &workload.corpus { + for &document in documents { let absolute = repo_root.join(&document.path); let bytes = fs::read(&absolute).map_err(|source| CandidateOutputError::Read { path: absolute.clone(), @@ -1154,10 +1354,6 @@ fn publish_corpus_with_scale( let file_occurrence_id = id::(&format!("file.{}{}", document.document_id, copy_suffix))?; file_to_document.insert(file_occurrence_id.as_str().to_owned(), document.clone()); - file_scopes.insert( - file_occurrence_id.as_str().to_owned(), - document.scope.clone(), - ); let language = id::(&document.language)?; let indexable = language_registry.descriptor(&language).is_some(); files.push(SanitizedCodeFileV1 { @@ -1180,6 +1376,9 @@ fn publish_corpus_with_scale( } } } + if captured.is_empty() { + return Ok(None); + } files.sort_by(|left, right| { (&left.logical_path, &left.file_occurrence_id) .cmp(&(&right.logical_path, &right.file_occurrence_id)) @@ -1193,8 +1392,9 @@ fn publish_corpus_with_scale( sanitizer_revision: id::("sanitizer.candidate.v1")?, sanitization_receipts: vec![id::("receipt.candidate.v1")?], content_identity: id(&canonical_sha256(&( - "tracedecay.search-eval.scaled-corpus.v1", - &corpus_digest, + "tracedecay.search-eval.scoped-corpus.v1", + corpus_digest, + scope_key, copies, ))?)?, captured_at: UtcMicros(1_000_000), @@ -1238,97 +1438,7 @@ fn publish_corpus_with_scale( .map_err(|error| CandidateOutputError::Contract(format!("open production owner: {error}")))? .build_and_publish(request, &ActiveControl) .map_err(|error| CandidateOutputError::Contract(format!("publish generation: {error}")))?; - let expected_chunks = match copies { - 1 => workload.execution_contract.exact_eligible_chunks_current, - 10 => workload.execution_contract.exact_eligible_chunks_10x, - _ => { - return Err(CandidateOutputError::Contract( - "evaluation corpus scale must be current or exact 10x".to_owned(), - )); - } - }; - let observed_chunks = generation.chunks().chunks().len() as u64; - if observed_chunks != expected_chunks { - return Err(CandidateOutputError::Contract(format!( - "eligible chunk count mismatch for {copies}x corpus: declared {expected_chunks}, observed {observed_chunks}" - ))); - } - let symbol_displays: Arc> = Arc::new( - generation - .symbols() - .symbols - .iter() - .map(|symbol| { - ( - symbol.occurrence.clone(), - VerifiedSealedLexicalSymbolDisplayV1::from(symbol.as_ref()), - ) - }) - .collect(), - ); - let mut occurrence_map = BTreeMap::new(); - for chunk in generation.chunks().chunks() { - let Some(document) = file_to_document.get(chunk.anchor.file_occurrence_id.as_str()) else { - continue; - }; - let qualified_name = chunk - .anchor - .symbol_occurrence_id - .as_ref() - .and_then(|symbol| symbol_displays.get(symbol)) - .map(VerifiedSealedLexicalSymbolDisplayV1::qualified_name); - let display_anchors = display_anchors_for_chunk(chunk, document, qualified_name); - if let Some(symbol) = &chunk.anchor.symbol_occurrence_id { - occurrence_map.insert( - format!("code-symbol:{}", symbol.as_str()), - OccurrenceMapEntry { - document_id: document.document_id.clone(), - scope: document.scope.clone(), - display_anchors: display_anchors.clone(), - }, - ); - occurrence_map.insert( - format!("code-graph:{}", symbol.as_str()), - OccurrenceMapEntry { - document_id: document.document_id.clone(), - scope: document.scope.clone(), - display_anchors: display_anchors.clone(), - }, - ); - } - occurrence_map.insert( - format!("code-chunk:{}", chunk.id.as_str()), - OccurrenceMapEntry { - document_id: document.document_id.clone(), - scope: document.scope.clone(), - display_anchors, - }, - ); - } - - let eligible_chunks = generation - .admitted_chunks() - .map_err(|error| CandidateOutputError::Contract(error.to_string()))? - .len() as u64; - let (lexical_projections, graph_projections) = build_query_projections( - &generation, - &file_scopes, - symbol_displays, - &workload.queries, - )?; - Ok(PublishedCorpus { - generation, - lexical_projections, - graph_projections, - occurrence_map, - repo_root: repo_root.to_path_buf(), - source_commit: GitOidV1::new(workload.source_repository_commit.clone()) - .map_err(|error| CandidateOutputError::Contract(error.to_string()))?, - corpus: workload.corpus.clone(), - corpus_digest, - eligible_chunks, - admitted_scope, - }) + Ok(Some((generation, file_to_document))) } fn display_anchors_for_chunk( @@ -1472,9 +1582,9 @@ fn graph_seeds_from_outcomes( fn retrieval_request( profile_id: &str, - published: &PublishedCorpus, + generation: &CodeIndexPublishedGenerationV1, ) -> Result { - let manifest = published.generation.manifest(); + let manifest = generation.manifest(); let freshness_digest = canonical_sha256(&( "tracedecay.search-eval.freshness.v1", &manifest.generation_id, @@ -2024,7 +2134,12 @@ pub(crate) mod tests { let published = publish_corpus(fixture.root(), &workload, fixture_admitted_scope) .expect("publish corpus"); - for chunk in published.generation.chunks().chunks() { + for chunk in published + .scopes + .values() + .flatten() + .flat_map(|scoped| scoped.generation.chunks().chunks()) + { assert_eq!( chunk.chunker_revision.as_str(), DAEMON_CODE_INDEX_CHUNKER_REVISION, @@ -2223,10 +2338,16 @@ pub(crate) mod tests { .expect("current corpus"); let ten_x = publish_corpus_with_scale(fixture_root, &workload, 10, fixture_admitted_scope) .expect("10x corpus"); - assert_ne!( - current.generation.manifest().generation_id, - ten_x.generation.manifest().generation_id - ); + for (scope_key, scoped) in ¤t.scopes { + let (Some(scoped), Some(ten_x_scoped)) = (scoped, &ten_x.scopes[scope_key]) else { + assert!(scoped.is_none() && ten_x.scopes[scope_key].is_none()); + continue; + }; + assert_ne!( + scoped.generation.manifest().generation_id, + ten_x_scoped.generation.manifest().generation_id + ); + } assert_eq!( ten_x.eligible_chunks, current.eligible_chunks.saturating_mul(10) diff --git a/crates/tracedecay-session-memory/Cargo.toml b/crates/tracedecay-session-memory/Cargo.toml index c38e5accef..d1c4824600 100644 --- a/crates/tracedecay-session-memory/Cargo.toml +++ b/crates/tracedecay-session-memory/Cargo.toml @@ -25,6 +25,7 @@ hex = "0.4" regex = "1.12.3" hotpath.workspace = true memmap2 = "0.9" +schemars.workspace = true serde = { version = "1", features = ["derive"] } serde_json = "1" sha2 = "0.11" @@ -52,7 +53,6 @@ tracedecay-tool-catalog = { path = "../tracedecay-tool-catalog", version = "0.1. # Test-only helper surfaces reached across crate boundaries by this crate's # test targets. Feature unification supplies them to the test build only. tokio = { version = "1", features = ["macros", "rt", "rt-multi-thread", "sync", "time"] } -rusqlite = { version = "0.40.1", default-features = false } tracedecay-global-db = { path = "../tracedecay-global-db", version = "0.1.0", features = ["test-helpers"] } tracedecay-lcm = { path = "../tracedecay-lcm", version = "0.1.0", features = ["test-helpers"] } tracedecay-runtime-core = { path = "../tracedecay-runtime-core", version = "0.1.0", features = ["test-helpers"] } diff --git a/crates/tracedecay-session-memory/src/anchor_resolution.rs b/crates/tracedecay-session-memory/src/anchor_resolution.rs index 562a8e4292..a7a22929e9 100644 --- a/crates/tracedecay-session-memory/src/anchor_resolution.rs +++ b/crates/tracedecay-session-memory/src/anchor_resolution.rs @@ -15,7 +15,7 @@ use serde::Serialize; use tracedecay_domain::{ AnchorResolutionStateV2, AuthorizedAnchorResolution, CoverageReportV1, DomainError, FactOwnerV1, FrozenWatermarkResolutionV1, PayloadAccessState, ResolutionAuthorizationV1, - RetrievalAnchorId, RetrievalAnchorRecordV2, VectorWatermark, canonical_sha256, + RetrievalAnchorId, RetrievalAnchorRecord, VectorWatermark, canonical_sha256, }; use tracedecay_store::ObservedEvidenceAnchorResolution; @@ -41,7 +41,7 @@ struct UnresolvedAnchorDigestV1<'a> { #[derive(Clone, Debug, PartialEq, Eq)] pub struct EvidenceAnchorResolutionReport { resolution: AuthorizedAnchorResolution, - record: Option, + record: Option, } impl EvidenceAnchorResolutionReport { @@ -149,7 +149,7 @@ impl EvidenceAnchorResolutionReport { /// The single authoritative retained record, when the store resolved one. /// The record is immutable metadata; its declared `payload_access` says /// whether the retained payload may be accessed. - pub fn record(&self) -> Option<&RetrievalAnchorRecordV2> { + pub fn record(&self) -> Option<&RetrievalAnchorRecord> { self.record.as_ref() } } @@ -170,11 +170,11 @@ mod tests { use std::collections::BTreeMap; use tracedecay_domain::{ - AccessPolicyDigest, AnchorDurabilityClass, AnchorSourceGenerationV2, + AccessPolicyDigest, AnchorDurabilityClass, AnchorSourceGeneration, CanonicalObservationIdV1, CapabilityId, EvidenceClass, ManifestDigest, ObservationScopeV1, PrivacyDomainBoundLocatorDigest, PrivacyDomainId, ProjectionGenerationId, RetentionClass, - RetrievalAnchorRecordV2Parts, RetrievalAnchorTargetV2, ScopeResolutionId, ShardId, - UtcMicros, WatermarkDriftV1, + RetrievalAnchorRecordParts, RetrievalAnchorTarget, ScopeResolutionId, ShardId, UtcMicros, + WatermarkDriftV1, }; use super::*; @@ -201,16 +201,16 @@ mod tests { } } - fn record_with_access(payload_access: PayloadAccessState) -> RetrievalAnchorRecordV2 { + fn record_with_access(payload_access: PayloadAccessState) -> RetrievalAnchorRecord { let observation_id = CanonicalObservationIdV1::new(SHA256_FIXTURE).unwrap(); - RetrievalAnchorRecordV2::new(RetrievalAnchorRecordV2Parts { - target: RetrievalAnchorTargetV2::ExactObservation(observation_id.clone()), + RetrievalAnchorRecord::new(RetrievalAnchorRecordParts { + target: RetrievalAnchorTarget::ExactObservation(observation_id.clone()), owner: ObservationScopeV1::Profile, aliases: vec![], occurred_at: None, ingested_at: UtcMicros(1), evidence_class: EvidenceClass::Observed, - source_generation: AnchorSourceGenerationV2::Observation( + source_generation: AnchorSourceGeneration::Observation( tracedecay_domain::ObservationSourceGenerationV1::new(1).unwrap(), ), projection_generation: ProjectionGenerationId::new("projection.fixture.v1").unwrap(), diff --git a/crates/tracedecay-session-memory/src/context/mod.rs b/crates/tracedecay-session-memory/src/context/mod.rs index 0e004477a7..f59ce9d91b 100644 --- a/crates/tracedecay-session-memory/src/context/mod.rs +++ b/crates/tracedecay-session-memory/src/context/mod.rs @@ -1,8 +1,8 @@ //! Request-context value types and the bounded read cache. //! -//! The code-index-backed source-read helpers (`source_read`, `read_modes`, -//! `markdown_sections`) stayed in `tracedecay-application`; its `context` -//! module re-exports this one alongside them. +//! The code-index-backed source-read helpers (`source_read`, `read_modes`) +//! live in `tracedecay-graph-query`; its `context` module composes with this +//! one. pub mod read_cache; mod registered_scope; @@ -13,6 +13,7 @@ use tracedecay_contracts::now_micros; use tracedecay_domain::{ AccessPolicyDigest, ProjectId, RepositoryId, WorktreeId, sha256_hex_suffix, }; +use tracedecay_runtime_core::cancellation::CancellationToken; pub use registered_scope::RegisteredScopeResolver; @@ -302,12 +303,6 @@ impl PolicyDigest { } } -/// The monotonic deadline and cooperative cancellation token moved into -/// `tracedecay_runtime_core::cancellation`: the kernel bounds its store-runtime -/// probes with them. Re-exported so every historical -/// `application::context::` path keeps resolving. -pub use tracedecay_runtime_core::cancellation::{CancellationToken, MonotonicDeadline}; - #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct RequestBudgets { max_results: u64, diff --git a/crates/tracedecay-session-memory/src/context/read_cache.rs b/crates/tracedecay-session-memory/src/context/read_cache.rs index 3ed8a75eba..897613f0e8 100644 --- a/crates/tracedecay-session-memory/src/context/read_cache.rs +++ b/crates/tracedecay-session-memory/src/context/read_cache.rs @@ -1,4 +1,4 @@ -//! Cross-session response cache for `tracedecay_read`. +//! Cross-session response cache for mode-aware source reads. //! //! Cached entries are keyed by `(project_id, file_path, mode, args_hash)` and //! survive across MCP sessions. The `mtime_ns` column on each row is the diff --git a/crates/tracedecay-session-memory/src/external_source_store.rs b/crates/tracedecay-session-memory/src/external_source_store.rs index b7231eada1..c086fc19dd 100644 --- a/crates/tracedecay-session-memory/src/external_source_store.rs +++ b/crates/tracedecay-session-memory/src/external_source_store.rs @@ -24,9 +24,10 @@ use tracedecay_store::{ ExternalSourceReadOperationV1, ExternalSourceReadResultV1, RepositoryOperationEnvelopeV1, RepositoryReadOperationV1, RepositoryReadResultV1, RepositoryWritePayloadV1, RuntimeReadCoverageV1, RuntimeReadOperationV1, RuntimeReadResultV1, RuntimeSubmitOutcomeV1, - SourceCommitApplyOutcomeV1, SourceCommitReceiptV1, SourceCommitV1, SourceObjectMutationV1, - SourceObjectTransitionV1, SourceObservationEvidenceV1, SourcePendingProjectionV1, - SourceProjectionCommitV1, SourceStoreStateV1, apply_source_commit, build_source_projection, + SourceCommitApplyOutcomeV1, SourceCommitReceiptSummaryV1, SourceCommitV1, + SourceObjectMutationV1, SourceObjectTransitionV1, SourceObservationEvidenceV1, + SourcePendingProjectionV1, SourceProjectionCommitV1, SourceStoreStateV1, apply_source_commit, + build_source_projection, }; use tracedecay_contracts::request_identity::{ @@ -68,8 +69,8 @@ const HOST_EXTERNAL_SOURCE_PROJECTOR: &str = "projector.host-observation.externa #[derive(Clone, Debug, PartialEq, Eq)] pub enum RuntimeSourceCaptureOutcomeV1 { - Projected(SourceCommitReceiptV1), - ProjectionPending(SourceCommitReceiptV1), + Projected(SourceCommitReceiptSummaryV1), + ProjectionPending(SourceCommitReceiptSummaryV1), } #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] @@ -423,11 +424,15 @@ impl RuntimeExternalSourceStore { } match apply_source_commit(current, commit.clone()).map_err(invalid)? { SourceCommitApplyOutcomeV1::Committed(state) => { - settled[slot] = Some((binding_identity.clone(), state.receipt().clone())); + settled[slot] = Some(( + binding_identity.clone(), + SourceCommitReceiptSummaryV1::of(state.receipt()), + )); states.insert(binding_identity.clone(), Some(*state)); } SourceCommitApplyOutcomeV1::ExactDuplicate(receipt) => { - settled[slot] = Some((binding_identity, *receipt)); + settled[slot] = + Some((binding_identity, SourceCommitReceiptSummaryV1::of(&receipt))); continue; } } @@ -612,11 +617,12 @@ impl RuntimeExternalSourceStore { .read_receipt(binding.clone(), key) .await? .ok_or(RuntimeExternalSourceErrorV1::IdempotencyConflict)?; - if !retained - .mutations() - .iter() - .any(|mutation| mutation.observation() == previous) - { + let committed_previous = current + .and_then(|state| state.latest_mutation(object.native_object())) + .is_some_and(|mutation| { + mutation.observation() == previous && retained.committed(mutation) + }); + if !committed_previous { return Err(RuntimeExternalSourceErrorV1::IdempotencyConflict); } Ok(Some(previous.revision().clone())) @@ -687,12 +693,14 @@ impl RuntimeExternalSourceStore { let Some(retained) = self.read_receipt(binding.clone(), key).await? else { return Ok(false); }; - return Ok(retained.mutations().iter().any(|mutation| { - mutation.observation() == latest - && mutation.predecessor() == Some(object.revision()) - && mutation.transition() == SourceObjectTransitionV1::Successor - && state.latest_mutation(object.native_object()) == Some(mutation) - })); + return Ok(state + .latest_mutation(object.native_object()) + .is_some_and(|mutation| { + mutation.observation() == latest + && mutation.predecessor() == Some(object.revision()) + && mutation.transition() == SourceObjectTransitionV1::Successor + && retained.committed(mutation) + })); } Ok(false) } @@ -750,7 +758,7 @@ impl RuntimeExternalSourceStore { &self, binding: tracedecay_domain::SourceBindingIdentityV1, idempotency_key: ManifestDigest, - ) -> Result, RuntimeExternalSourceErrorV1> { + ) -> Result, RuntimeExternalSourceErrorV1> { let operation = ExternalSourceReadOperationV1::CommitReceipt { binding, idempotency_key, diff --git a/crates/tracedecay-session-memory/src/external_source_store/tests.rs b/crates/tracedecay-session-memory/src/external_source_store/tests.rs index 349e7c33d2..b6f735e3e2 100644 --- a/crates/tracedecay-session-memory/src/external_source_store/tests.rs +++ b/crates/tracedecay-session-memory/src/external_source_store/tests.rs @@ -76,7 +76,7 @@ impl Fixture { "cline-retained-test", ) .unwrap(); - let anchor = tracedecay_store::build_observation_retrieval_anchor_v2( + let anchor = tracedecay_store::build_observation_retrieval_anchor( &observation, generation.clone(), UtcMicros(1), @@ -460,70 +460,3 @@ async fn cline_cutover_requires_the_durable_retained_receipt_and_exact_revision( "a different retained payload revision cannot authorize the successor" ); } - -/// A scoped observation reset destroys the stream the host-observation -/// journal attested. Re-admitting the same observation id must be a rebuild -/// (one pass, no conflict), not a retryable reuse of the prior command. -#[tokio::test] -async fn scoped_reset_readmits_the_same_host_observation_without_conflict() { - let directory = TempDir::new().unwrap(); - let path = directory.path().join("sessions.db"); - let first = observation( - ClineTranscriptStream::UiMessages, - false, - 4, - "native-reset", - false, - ); - { - let fixture = Fixture::open_at( - path.clone(), - TestDatabaseRuntimeMode::Initialize, - TempDir::new().unwrap(), - ) - .await; - let receipt = fixture.persist(first.clone()).await; - fixture - .retained - .capture_host_observation(&receipt) - .await - .expect("first capture must persist the host-observation receipt"); - } - - { - let mut connection = rusqlite::Connection::open(&path).unwrap(); - connection - .execute( - "DELETE FROM global_schema_migrations WHERE migration = ?1", - [tracedecay_global_db::observation::OBSERVATION_NATIVE_SOURCE_SCHEME_MIGRATION], - ) - .unwrap(); - let report = - tracedecay_global_db::observation::reset_refused_observation_authority(&mut connection) - .expect("a store whose scheme marker was removed is refused and resettable"); - assert!( - report.cleared_external_source_rows > 0, - "the reset must retire the host-observation journal: {report:?}" - ); - let leftover: i64 = connection - .query_row( - "SELECT COUNT(*) FROM external_source_commit_receipts_v2", - [], - |row| row.get(0), - ) - .unwrap(); - assert_eq!(leftover, 0, "no prior receipt may survive the scoped reset"); - } - - let fixture = Fixture::open_at(path, TestDatabaseRuntimeMode::Existing, directory).await; - let receipt = fixture.persist(first).await; - let outcome = fixture.retained.capture_host_observation(&receipt).await; - assert!( - !matches!( - outcome, - Err(RuntimeExternalSourceErrorV1::IdempotencyConflict) - ), - "re-admission after reset must not conflict with the retired journal: {outcome:?}" - ); - outcome.expect("re-admission must converge in one pass"); -} diff --git a/crates/tracedecay-session-memory/src/fact_store/automation_run_receipts/tests.rs b/crates/tracedecay-session-memory/src/fact_store/automation_run_receipts/tests.rs index 4e0e91f519..05d6f4e0a1 100644 --- a/crates/tracedecay-session-memory/src/fact_store/automation_run_receipts/tests.rs +++ b/crates/tracedecay-session-memory/src/fact_store/automation_run_receipts/tests.rs @@ -305,7 +305,6 @@ async fn postcommit_recovery_returns_exact_nonempty_automatic_receipts_determini 1 ); assert!(recovered.curation_receipt().is_none()); - assert!(!recovered.is_empty()); } #[tokio::test] diff --git a/crates/tracedecay-session-memory/src/fact_store/crud/commit.rs b/crates/tracedecay-session-memory/src/fact_store/crud/commit.rs index 4e5afb8072..18acff7127 100644 --- a/crates/tracedecay-session-memory/src/fact_store/crud/commit.rs +++ b/crates/tracedecay-session-memory/src/fact_store/crud/commit.rs @@ -17,7 +17,7 @@ use super::{ use serde::Serialize; use tracedecay_domain::{ FactAssertionId, FactAssertionKindV1, FactAssertionV1, FactEventId, FactId, FactLineageEventV1, - FactOwnerV1, RetrievalAnchorId, RetrievalAnchorRecordV2, UtcMicros, + FactOwnerV1, RetrievalAnchorId, RetrievalAnchorRecord, UtcMicros, }; use tracedecay_runtime_core::db::DatabaseMemoryTransaction as Transaction; use tracedecay_runtime_core::db::engine::{params, params_from_iter}; @@ -359,7 +359,7 @@ async fn ensure_referenced_anchors( async fn insert_or_verify_anchor( transaction: &Transaction<'_>, owner: &OwnerKey, - anchor: &RetrievalAnchorRecordV2, + anchor: &RetrievalAnchorRecord, ) -> FactStoreResult<()> { if anchor_exists(transaction, anchor.anchor_id()).await? { if anchor_matches(transaction, owner, anchor).await? { @@ -418,7 +418,7 @@ async fn anchor_exists( pub(super) async fn anchor_matches( transaction: &Transaction<'_>, owner: &OwnerKey, - anchor: &RetrievalAnchorRecordV2, + anchor: &RetrievalAnchorRecord, ) -> FactStoreResult { let mut rows = transaction .query( diff --git a/crates/tracedecay-session-memory/src/fact_store/crud/feedback.rs b/crates/tracedecay-session-memory/src/fact_store/crud/feedback.rs index a3966c72da..f67048096b 100644 --- a/crates/tracedecay-session-memory/src/fact_store/crud/feedback.rs +++ b/crates/tracedecay-session-memory/src/fact_store/crud/feedback.rs @@ -31,7 +31,7 @@ use super::{ use serde_json::{Value, json}; use tracedecay_domain::{ ActorId, Confidence, FactCurationActionV1, FactEventId, FactId, FactLineageEventKindV1, - FactLineageEventV1, FactOwnerV1, ProvenanceId, RetrievalAnchorRecordV2, UtcMicros, + FactLineageEventV1, FactOwnerV1, ProvenanceId, RetrievalAnchorRecord, UtcMicros, }; use tracedecay_privacy::sanitize_provider_metadata_text; use tracedecay_runtime_core::db::DatabaseMemoryTransaction as Transaction; @@ -577,7 +577,7 @@ async fn inspect_project_memory_fact_inner_tx( if let Some(read_control) = read_control { ensure_project_memory_read_active(read_control)?; } - let anchor = from_json::( + let anchor = from_json::( &row_string(&row, 0, PROJECT_MEMORY_READ_OPERATION)?, PROJECT_MEMORY_READ_OPERATION, )?; diff --git a/crates/tracedecay-session-memory/src/fact_store/crud/mod.rs b/crates/tracedecay-session-memory/src/fact_store/crud/mod.rs index 090597fcf6..7c4a3a202d 100644 --- a/crates/tracedecay-session-memory/src/fact_store/crud/mod.rs +++ b/crates/tracedecay-session-memory/src/fact_store/crud/mod.rs @@ -1,6 +1,4 @@ //! Canonical fact CRUD, commit, feedback, and automatic fact application. -//! -//! Re-exports below preserve every `crud::*` path used outside this module. use sha2::{Digest, Sha256}; use tracedecay_domain::LocatorDigest; diff --git a/crates/tracedecay-session-memory/src/fact_store/crud/queries.rs b/crates/tracedecay-session-memory/src/fact_store/crud/queries.rs index 73afbc2b7e..b16c938a29 100644 --- a/crates/tracedecay-session-memory/src/fact_store/crud/queries.rs +++ b/crates/tracedecay-session-memory/src/fact_store/crud/queries.rs @@ -12,7 +12,7 @@ use super::{Projection, anchor_matches, commit_fact_tx}; use tracedecay_domain::{ Confidence, CoverageUniverseKnowledgeV1, FactAssertionId, FactEventId, FactId, FactLineageEventKindV1, FactLineageEventV1, FactOwnerV1, FactPayloadV1, PayloadAccessState, - RetrievalAnchorRecordV2, ShardDispositionV1, UtcMicros, + RetrievalAnchorRecord, ShardDispositionV1, UtcMicros, }; use tracedecay_runtime_core::db::DatabaseMemoryTransaction as Transaction; use tracedecay_runtime_core::db::build_qmark_placeholders; @@ -847,7 +847,7 @@ async fn query_fact_coverage_tx( .map_err(|error| storage_error(QUERY_OPERATION, error))? { let anchor_id = row_string(&row, 0, QUERY_OPERATION)?; - let anchor = from_json::( + let anchor = from_json::( &row_string(&row, 1, QUERY_OPERATION)?, QUERY_OPERATION, )?; @@ -875,7 +875,7 @@ async fn query_fact_coverage_tx( fn classify_fact_coverage( effective_access: PayloadAccessState, - anchor: Option<&RetrievalAnchorRecordV2>, + anchor: Option<&RetrievalAnchorRecord>, ) -> FactQueryCoverageV1 { let (visible, hidden, unknown, mut redacted, frontier_count) = match anchor { None => (0, 0, 1, 0, 1), @@ -909,7 +909,7 @@ fn classify_fact_coverage( }; let anchor_access = anchor.map_or( PayloadAccessState::Eligible, - RetrievalAnchorRecordV2::payload_access, + RetrievalAnchorRecord::payload_access, ); if effective_access == PayloadAccessState::Redacted || anchor_access == PayloadAccessState::Redacted @@ -928,7 +928,7 @@ fn classify_fact_coverage( pub(in crate::fact_store) async fn get_retrieval_anchor_tx( snapshot: &Transaction<'_>, query: &RetrievalAnchorQuery, -) -> FactStoreResult> { +) -> FactStoreResult> { let owner = OwnerKey::new(query.owner())?; let mut rows = snapshot .query( @@ -953,7 +953,7 @@ pub(in crate::fact_store) async fn get_retrieval_anchor_tx( else { return Ok(None); }; - let anchor = from_json::( + let anchor = from_json::( &row_string(&row, 0, QUERY_OPERATION)?, QUERY_OPERATION, )?; diff --git a/crates/tracedecay-session-memory/src/fact_store/mod.rs b/crates/tracedecay-session-memory/src/fact_store/mod.rs index 58151f8716..9cb58ccf13 100644 --- a/crates/tracedecay-session-memory/src/fact_store/mod.rs +++ b/crates/tracedecay-session-memory/src/fact_store/mod.rs @@ -3,7 +3,7 @@ use tracedecay_runtime_core::db::Database; use tracedecay_domain::RunId; -use tracedecay_domain::{FactLineageEventV1, FactOwnerV1, ProvenanceId, RetrievalAnchorRecordV2}; +use tracedecay_domain::{FactLineageEventV1, FactOwnerV1, ProvenanceId, RetrievalAnchorRecord}; use tracedecay_store::ProjectMemoryAutomationRunReceiptsV1; use tracedecay_store::{ CurrentFactsQuery, FactAsOfQuery, FactAsOfResponseV1, FactCommitOutcome, FactCurrentQuery, @@ -313,7 +313,7 @@ impl FactStore for DatabaseFactStore<'_> { async fn get_retrieval_anchor( &self, query: RetrievalAnchorQuery, - ) -> FactStoreResult> { + ) -> FactStoreResult> { let snapshot = self .db .begin_memory_read_transaction(QUERY_OPERATION) @@ -968,7 +968,7 @@ impl FactStore for ProjectFactStore<'_> { ) -> FactStoreResult; fn get_retrieval_anchor( query: RetrievalAnchorQuery, - ) -> FactStoreResult>; + ) -> FactStoreResult>; } } diff --git a/crates/tracedecay-session-memory/src/memory/anchors.rs b/crates/tracedecay-session-memory/src/memory/anchors.rs index 2cd76f3a74..884cca14f6 100644 --- a/crates/tracedecay-session-memory/src/memory/anchors.rs +++ b/crates/tracedecay-session-memory/src/memory/anchors.rs @@ -5,7 +5,7 @@ use std::future::Future; use thiserror::Error; -use tracedecay_domain::{DomainError, FactOwnerV1, RetrievalAnchorId, RetrievalAnchorRecordV2}; +use tracedecay_domain::{DomainError, FactOwnerV1, RetrievalAnchorId, RetrievalAnchorRecord}; use tracedecay_store::FactStore; use crate::anchor_resolution::{EvidenceAnchorReportResolver, EvidenceAnchorResolutionReport}; @@ -17,11 +17,11 @@ use super::error::MemoryApplicationError; /// a fact shard. It deliberately reuses the canonical retrieval-anchor model. #[derive(Clone, Debug)] pub struct ResolvedEvidenceAnchor { - record: RetrievalAnchorRecordV2, + record: RetrievalAnchorRecord, } impl ResolvedEvidenceAnchor { - pub fn new(record: RetrievalAnchorRecordV2) -> Result { + pub fn new(record: RetrievalAnchorRecord) -> Result { record.validate()?; Ok(Self { record }) } @@ -30,11 +30,11 @@ impl ResolvedEvidenceAnchor { self.record.anchor_id() } - pub fn record(&self) -> &RetrievalAnchorRecordV2 { + pub fn record(&self) -> &RetrievalAnchorRecord { &self.record } - pub fn into_record(self) -> RetrievalAnchorRecordV2 { + pub fn into_record(self) -> RetrievalAnchorRecord { self.record } } @@ -70,7 +70,7 @@ impl MemoryApplication { &self, resolver: &R, anchor_id: RetrievalAnchorId, - ) -> Result { + ) -> Result { anchor_id .validate() .map_err(MemoryApplicationError::InvalidEvidenceAnchor)?; diff --git a/crates/tracedecay-session-memory/src/memory/canonical.rs b/crates/tracedecay-session-memory/src/memory/canonical.rs index 63305c8dde..bd98733312 100644 --- a/crates/tracedecay-session-memory/src/memory/canonical.rs +++ b/crates/tracedecay-session-memory/src/memory/canonical.rs @@ -10,7 +10,7 @@ use tracedecay_contracts::memory::{ MemoryOptionalFactPortResult, MemoryReadCoverage, MemoryReadResult, MemoryRetrievalAnchorQuery, MemoryUseCaseError, RetrievalAnchorPort, }; -use tracedecay_domain::{FactId, FactLineageEventV1, FactOwnerV1, RetrievalAnchorRecordV2}; +use tracedecay_domain::{FactId, FactLineageEventV1, FactOwnerV1, RetrievalAnchorRecord}; use tracedecay_store::{ CurrentFactsQuery, FactAsOfQuery, FactAsOfResponseV1, FactCommitOutcome, FactContradictionStateV1 as StoreFactContradictionStateV1, FactCurrentQuery, @@ -136,7 +136,7 @@ impl RetrievalAnchorPort for FactStoreAdapter<'_, A> { async fn get_retrieval_anchor( &self, query: Self::Query, - ) -> Result, Self::Error> { + ) -> Result, Self::Error> { self.0.get_retrieval_anchor(query).await } } @@ -228,7 +228,7 @@ impl MemoryApplication { pub async fn get_retrieval_anchor( &self, query: RetrievalAnchorQuery, - ) -> Result, MemoryApplicationError> { + ) -> Result, MemoryApplicationError> { let owner = query.owner().clone(); let anchor_id = query.anchor_id().clone(); canonical_application(&self.owner, &self.authority)? diff --git a/crates/tracedecay-session-memory/src/memory/mod.rs b/crates/tracedecay-session-memory/src/memory/mod.rs index 14ea9853a9..5c25b1a971 100644 --- a/crates/tracedecay-session-memory/src/memory/mod.rs +++ b/crates/tracedecay-session-memory/src/memory/mod.rs @@ -45,7 +45,7 @@ pub use project_memory::{ #[cfg(test)] use tracedecay_domain::{ - DomainError, FactId, FactLineageEventV1, ProvenanceId, RetrievalAnchorRecordV2, + DomainError, FactId, FactLineageEventV1, ProvenanceId, RetrievalAnchorRecord, }; #[cfg(test)] use tracedecay_store::{ diff --git a/crates/tracedecay-session-memory/src/memory/project_memory/add.rs b/crates/tracedecay-session-memory/src/memory/project_memory/add.rs index aa28cd5ae1..200b0fa10f 100644 --- a/crates/tracedecay-session-memory/src/memory/project_memory/add.rs +++ b/crates/tracedecay-session-memory/src/memory/project_memory/add.rs @@ -1,6 +1,7 @@ //! Canonical project-memory add preflight and execution. use crate::memory::trust::DEFAULT_TRUST; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use serde_json::Value; use tracedecay_domain::{ @@ -22,7 +23,7 @@ use super::validate_project_memory_add_outcome; /// Transport adapters own their wire DTOs. This single use-case request owns /// the boundary between unsanitized user intent and the canonical store /// command, so callers cannot accidentally bypass payload sanitization. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] pub struct ProjectMemoryFactAddRequest { pub content: String, @@ -30,6 +31,7 @@ pub struct ProjectMemoryFactAddRequest { pub source_label: Option, pub tags: Vec, pub entities: Vec, + #[schemars(with = "Option")] pub trust: Option, pub metadata: Value, } diff --git a/crates/tracedecay-session-memory/src/memory/tests.rs b/crates/tracedecay-session-memory/src/memory/tests.rs index 9c4547cc74..0b7ab70a83 100644 --- a/crates/tracedecay-session-memory/src/memory/tests.rs +++ b/crates/tracedecay-session-memory/src/memory/tests.rs @@ -4,12 +4,12 @@ use std::sync::{ }; use tracedecay_domain::{ - AccessPolicyDigest, AnchorDurabilityClass, AnchorSourceGenerationV2, CapabilityId, Confidence, + AccessPolicyDigest, AnchorDurabilityClass, AnchorSourceGeneration, CapabilityId, Confidence, CoverageReportV1, EntityId, EntityKind, EntityRef, EvidenceClass, FactAssertionId, FactEventId, FactIdentityMaterialV1, FactIdentitySourceV1, FactLineageEventKindV1, ObservationScopeV1, PayloadAccessState, PrivacyDomainBoundLocatorDigest, PrivacyDomainId, ProjectId, ProjectionGenerationId, ResolutionAuthorizationV1, RetentionClass, RetrievalAnchorId, - RetrievalAnchorRecordV2Parts, RetrievalAnchorTargetV2, ScopeResolutionId, UtcMicros, + RetrievalAnchorRecordParts, RetrievalAnchorTarget, ScopeResolutionId, UtcMicros, VectorWatermark, }; use tracedecay_store::{ @@ -172,7 +172,7 @@ impl FactStore for FakeAuthority { async fn get_retrieval_anchor( &self, query: RetrievalAnchorQuery, - ) -> FactStoreResult> { + ) -> FactStoreResult> { self.anchor_queries .lock() .unwrap() @@ -632,13 +632,13 @@ fn stored_fact(owner: FactOwnerV1, operation: &str, projected_as_of: UtcMicros) .unwrap() } -fn profile_anchor() -> RetrievalAnchorRecordV2 { +fn profile_anchor() -> RetrievalAnchorRecord { const DIGEST_A: &str = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; const DIGEST_B: &str = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; - RetrievalAnchorRecordV2::new(RetrievalAnchorRecordV2Parts { - target: RetrievalAnchorTargetV2::Entity(EntityRef { + RetrievalAnchorRecord::new(RetrievalAnchorRecordParts { + target: RetrievalAnchorTarget::Entity(EntityRef { id: EntityId::new("entity.memory.external").unwrap(), kind: EntityKind::Document, }), @@ -647,7 +647,7 @@ fn profile_anchor() -> RetrievalAnchorRecordV2 { occurred_at: None, ingested_at: UtcMicros(1), evidence_class: EvidenceClass::Observed, - source_generation: AnchorSourceGenerationV2::Unknown, + source_generation: AnchorSourceGeneration::Unknown, projection_generation: ProjectionGenerationId::new("projection.memory.external").unwrap(), projection_watermark: VectorWatermark::default(), coverage: CoverageReportV1::default(), diff --git a/crates/tracedecay-session-memory/src/monitor_ring.rs b/crates/tracedecay-session-memory/src/monitor_ring.rs index dd256c168c..6eadee826a 100644 --- a/crates/tracedecay-session-memory/src/monitor_ring.rs +++ b/crates/tracedecay-session-memory/src/monitor_ring.rs @@ -126,7 +126,7 @@ fn write_entry_inner( file.set_len(FILE_SIZE as u64)?; } - let mut mmap = unsafe { memmap2::MmapMut::map_mut(&file)? }; + let mut mmap = unsafe { memmap2::MmapMut::map_mut(&*file)? }; let write_idx = u64::from_le_bytes( mmap[OFF_WRITE_IDX..OFF_WRITE_IDX + 8] @@ -151,7 +151,7 @@ fn write_entry_inner( mmap[OFF_WRITE_IDX..OFF_WRITE_IDX + 8].copy_from_slice(&new_idx.to_le_bytes()); mmap.flush()?; - file.unlock()?; + file.release()?; Ok(()) } diff --git a/crates/tracedecay-session-memory/src/provider_pricing.rs b/crates/tracedecay-session-memory/src/provider_pricing.rs index b9439ef902..9668c7f739 100644 --- a/crates/tracedecay-session-memory/src/provider_pricing.rs +++ b/crates/tracedecay-session-memory/src/provider_pricing.rs @@ -224,7 +224,7 @@ pub fn load_table() -> &'static PriceTable { }) } -/// JSON payload for `GET /api/plugins/savings/pricing`. +/// Bundled pricing provenance and per-model rates as JSON. pub fn pricing_payload() -> Value { let table = load_table(); let mut models = Map::new(); diff --git a/crates/tracedecay-session-memory/src/provider_usage.rs b/crates/tracedecay-session-memory/src/provider_usage.rs index 68587dc29a..18a09e9981 100644 --- a/crates/tracedecay-session-memory/src/provider_usage.rs +++ b/crates/tracedecay-session-memory/src/provider_usage.rs @@ -1,5 +1,6 @@ use std::collections::{BTreeMap, HashMap}; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use tracedecay_domain::{ CanonicalUnknownStateV1, ObservationScopeV1, ProviderUsageCounterSemanticsV1, @@ -26,7 +27,7 @@ pub fn provider_usage_range_start(range: &str) -> Result { }) } -#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "snake_case")] pub enum ProviderUsageCoverageV1 { Complete, @@ -64,7 +65,7 @@ pub struct ProviderUsageIssueV1 { pub session_id: Option, } -#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[derive(Clone, Debug, Default, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] pub struct AggregatedProviderUsageCountersV1 { pub input_tokens: Option, pub output_tokens: Option, @@ -126,6 +127,52 @@ pub struct ProviderUsageAggregateV1 { pub upper_observation_sequence: Option, } +/// Provider-reported usage attributed to one `(provider, session_id)`, summed +/// from the reduced deltas of one aggregate. +#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] +pub struct ProviderUsageSessionTotalsV1 { + pub usage_events: u64, + pub counters: AggregatedProviderUsageCountersV1, + /// `false` when the reduction recorded an issue against this session, so + /// the sums are a lower bound on what the provider wrote. + pub complete: bool, +} + +/// Groups one aggregate's deltas by `(provider, session_id)`. A session that +/// only appears in `issues` is present with zero events and unknown counters: +/// the provider wrote usage for it that could not be reduced. +pub fn provider_usage_by_session( + aggregate: &ProviderUsageAggregateV1, +) -> BTreeMap<(String, String), ProviderUsageSessionTotalsV1> { + let mut sums: BTreeMap<(String, String), (u64, CounterSum, bool)> = BTreeMap::new(); + for delta in &aggregate.deltas { + let entry = sums + .entry((delta.provider.clone(), delta.session_id.clone())) + .or_insert_with(|| (0, CounterSum::default(), true)); + entry.0 = entry.0.saturating_add(1); + entry.1.add_aggregated(&delta.counters); + } + for issue in &aggregate.issues { + if let (Some(provider), Some(session_id)) = (&issue.provider, &issue.session_id) { + sums.entry((provider.clone(), session_id.clone())) + .or_insert_with(|| (0, CounterSum::default(), true)) + .2 = false; + } + } + sums.into_iter() + .map(|(key, (usage_events, counters, complete))| { + ( + key, + ProviderUsageSessionTotalsV1 { + usage_events, + counters: counters.finish(), + complete, + }, + ) + }) + .collect() +} + #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] pub struct ProviderUsageModelCostV1 { pub provider: String, @@ -424,6 +471,15 @@ impl CounterSum { self.total.add(counters.total); } + fn add_aggregated(&mut self, counters: &AggregatedProviderUsageCountersV1) { + self.input.add(counters.input_tokens); + self.output.add(counters.output_tokens); + self.cache_read.add(counters.cache_read_tokens); + self.cache_write.add(counters.cache_write_tokens); + self.reasoning.add(counters.reasoning_tokens); + self.total.add(counters.total_tokens); + } + fn finish(self) -> AggregatedProviderUsageCountersV1 { AggregatedProviderUsageCountersV1 { input_tokens: self.input.finish(), diff --git a/crates/tracedecay-session-memory/src/provider_usage/tests.rs b/crates/tracedecay-session-memory/src/provider_usage/tests.rs index 6c7ac30bb8..f8dc882507 100644 --- a/crates/tracedecay-session-memory/src/provider_usage/tests.rs +++ b/crates/tracedecay-session-memory/src/provider_usage/tests.rs @@ -9,7 +9,8 @@ use tracedecay_domain::{ use super::{ AggregatedProviderUsageCountersV1, ProviderUsageCoverageV1, ProviderUsageIssueKindV1, - ProviderUsageScanV1, ScanStep, price_provider_usage, reduce_provider_usage, + ProviderUsageScanV1, ScanStep, price_provider_usage, provider_usage_by_session, + reduce_provider_usage, }; use crate::provider_pricing::{ModelPrice, PriceTable}; @@ -428,6 +429,72 @@ fn cumulative_decrease_is_a_typed_reset_and_never_underflows() { ); } +#[test] +fn per_session_totals_sum_own_deltas_and_flag_issue_sessions_without_zero_fill() { + let aggregate = reduce_provider_usage(&[ + observation( + 1, + 0, + "codex", + "parent", + ProviderUsageCounterSemanticsV1::Delta, + counters(100, 10), + ), + observation( + 2, + 0, + "codex", + "child", + ProviderUsageCounterSemanticsV1::Delta, + counters(7, 3), + ), + observation( + 3, + 0, + "codex", + "parent", + ProviderUsageCounterSemanticsV1::Delta, + counters(20, 5), + ), + observation( + 4, + 0, + "claude", + "broken", + ProviderUsageCounterSemanticsV1::Delta, + ProviderUsageCountersV1::Unknown { + reason: tracedecay_domain::CanonicalUnknownStateV1::Malformed, + }, + ), + ]); + + let by_session = provider_usage_by_session(&aggregate); + + let parent = &by_session[&("codex".to_owned(), "parent".to_owned())]; + assert_eq!(parent.usage_events, 2); + assert_eq!(parent.counters, totals(120, 15)); + assert!(parent.complete); + + let child = &by_session[&("codex".to_owned(), "child".to_owned())]; + assert_eq!(child.usage_events, 1); + assert_eq!(child.counters, totals(7, 3)); + assert!(child.complete); + + let broken = &by_session[&("claude".to_owned(), "broken".to_owned())]; + assert_eq!(broken.usage_events, 0); + assert_eq!( + broken.counters, + AggregatedProviderUsageCountersV1::unknown() + ); + assert!( + !broken.complete, + "an unreducible session is flagged, not zeroed" + ); + + assert_eq!(by_session.len(), 3); + assert_eq!(aggregate.totals, totals(127, 18)); +} + #[test] fn malformed_or_unknown_counters_make_coverage_partial_without_zero_fill() { let aggregate = reduce_provider_usage(&[ diff --git a/crates/tracedecay-session-memory/src/session/lcm/authority.rs b/crates/tracedecay-session-memory/src/session/lcm/authority.rs index 0fd104e37b..4f39e895d7 100644 --- a/crates/tracedecay-session-memory/src/session/lcm/authority.rs +++ b/crates/tracedecay-session-memory/src/session/lcm/authority.rs @@ -14,11 +14,11 @@ use tracedecay_contracts::{ }; use tracedecay_domain::ManifestDigest; -use crate::context::CancellationToken; use crate::session::SessionRequestBinding; use tracedecay_lcm::{ LcmCompressionResponse, LcmPreflightRequest, LcmPreflightResponse, LcmStatus, }; +use tracedecay_runtime_core::cancellation::CancellationToken; use tracedecay_tool_catalog::{CapabilityId, UseCaseId}; pub const LCM_DAEMON_COMMAND_CAPABILITY: &str = "capability.application.lcm-daemon-command"; diff --git a/crates/tracedecay-session-memory/src/session/retrieval.rs b/crates/tracedecay-session-memory/src/session/retrieval.rs index 4dbafefc4f..c09d8a2706 100644 --- a/crates/tracedecay-session-memory/src/session/retrieval.rs +++ b/crates/tracedecay-session-memory/src/session/retrieval.rs @@ -13,10 +13,10 @@ use tracedecay_domain::{ }; use tracedecay_temporal_query::context::{ContextBudget, ContextError, VersionedTokenEstimator}; use tracedecay_temporal_query::cursor::CursorError; +use tracedecay_temporal_query::execution::{ExecutionControl, ExecutionLimits}; use tracedecay_temporal_query::hydration::HydrationError; use tracedecay_temporal_query::ports::{ - ExecutionControl, ExecutionLimits, TemporalAuthorizedRoot, TemporalCandidateFilterV1, - TemporalPortError, TemporalRetrievalScope, + TemporalAuthorizedRoot, TemporalCandidateFilterV1, TemporalPortError, TemporalRetrievalScope, }; use tracedecay_temporal_query::ranking::DiversityLimits; use tracedecay_temporal_query::resolution::SummaryLineageRejection; diff --git a/crates/tracedecay-session-memory/src/session/tests/application.rs b/crates/tracedecay-session-memory/src/session/tests/application.rs index 03bf490b67..f9842f50bc 100644 --- a/crates/tracedecay-session-memory/src/session/tests/application.rs +++ b/crates/tracedecay-session-memory/src/session/tests/application.rs @@ -16,8 +16,8 @@ use tracedecay_tool_catalog::{CapabilityId, UseCaseId}; use super::harness::{EXTERNAL_PAYLOAD, INLINE_PAYLOAD, PROJECT_ID, RegisteredTemporalHarness}; use crate::context::{ - BranchId, CancellationToken, CapabilityDigest, ConfigurationDigest, PolicyDigest, ProfileId, - RequestBudgets, ResolvedGitRoute, ResolvedSessionIdentity, SessionRootId, SessionStoreId, + BranchId, CapabilityDigest, ConfigurationDigest, PolicyDigest, ProfileId, RequestBudgets, + ResolvedGitRoute, ResolvedSessionIdentity, SessionRootId, SessionStoreId, application_observed_at, session_application_grant_digest, }; use crate::session::{ @@ -26,6 +26,7 @@ use crate::session::{ SessionRetrievalScope, SessionRetrievalService, SessionScopeAuthorizationRequest, SessionScopeAuthorizer, SessionTemporalQuery, }; +use tracedecay_runtime_core::cancellation::CancellationToken; const DIGEST: [u8; 32] = [0x5a; 32]; diff --git a/crates/tracedecay-session-memory/src/session/tests/harness.rs b/crates/tracedecay-session-memory/src/session/tests/harness.rs index 22a102949b..279e0db029 100644 --- a/crates/tracedecay-session-memory/src/session/tests/harness.rs +++ b/crates/tracedecay-session-memory/src/session/tests/harness.rs @@ -15,7 +15,7 @@ use tracedecay_domain::{ sha256_hex_suffix, }; use tracedecay_store::{ - build_observation_resolution_authorization_v1, build_observation_retrieval_anchor_v2, + build_observation_resolution_authorization_v1, build_observation_retrieval_anchor, }; use tracedecay_global_db::RegisteredGlobalDbLeaseV1; @@ -24,7 +24,7 @@ use tracedecay_lcm::payload::{upsert_payload_metadata, write_external_payload}; use tracedecay_lcm::types::LcmImmutableSummaryPublication; use tracedecay_lcm::{LcmSourceRef, LcmSummaryNodeDraft}; use tracedecay_runtime_core::db::engine::params; -use tracedecay_temporal_query::ports::ExecutionControl; +use tracedecay_temporal_query::execution::ExecutionControl; pub(super) const PROJECT_ID: &str = "project.tracedecay"; pub(super) const INLINE_PAYLOAD: &str = "non-empty inline occurrence payload"; @@ -242,14 +242,13 @@ impl RegisteredTemporalHarness { INSERT INTO lcm_raw_messages ( provider, message_id, session_id, role, ordinal, timestamp, content, content_hash, storage_kind, payload_ref, - snippet_text, index_text, metadata_json, legacy_source, legacy_truncated + placeholder_text, metadata_json ) VALUES ( 'claude', 'message.temporal.legacy', 'session.temporal.legacy', - 'user', 1, 1, 'sk-proj-private-canary', + 'user', 1, 1, NULL, 'sha256:quarantined', 'inline', NULL, - 'quarantined legacy record', 'quarantined legacy record', - '{\"payload_access\":\"quarantined\",\"migration\":\"legacy-unsanitized\"}', - 1, 0 + 'quarantined legacy record', + '{\"payload_access\":\"quarantined\",\"migration\":\"legacy-unsanitized\",\"legacy_preview\":\"sk-proj-private-canary\"}' );", ) .await @@ -445,7 +444,7 @@ impl RegisteredTemporalHarness { let authorization = build_observation_resolution_authorization_v1(observation, "application-fixture") .unwrap(); - let anchor = build_observation_retrieval_anchor_v2( + let anchor = build_observation_retrieval_anchor( observation, projection, UtcMicros(1), @@ -573,11 +572,10 @@ impl RegisteredTemporalHarness { session_id, generation, occurrence_id, source_observation_id, source_provider, projection_output_ordinal, retrieval_anchor_id, message_id, role, knowledge_at, valid_time_json, evidence_json, - sanitized_content_digest, sanitized_content_bytes, - snippet_text, index_text + sanitized_content_digest, sanitized_content_bytes, index_text ) VALUES ( ?1, 1, ?2, ?3, ?4, 0, ?5, ?6, 'assistant', ?7, ?8, ?9, - ?10, ?11, ?12, ?12 + ?10, ?11, ?12 )", params![ observation.source().session_id().as_str(), @@ -616,11 +614,9 @@ impl RegisteredTemporalHarness { .execute( "INSERT INTO lcm_raw_messages ( provider, message_id, session_id, role, ordinal, timestamp, - content, content_hash, storage_kind, payload_ref, - snippet_text, index_text, legacy_source, legacy_truncated, metadata_json + content, content_hash, storage_kind, payload_ref, metadata_json ) VALUES ( - ?1, ?2, ?3, 'assistant', ?4, ?4, ?5, ?6, - 'inline', NULL, ?5, ?5, 0, 0, ?7 + ?1, ?2, ?3, 'assistant', ?4, ?4, ?5, ?6, 'inline', NULL, ?7 )", params![ observation.source().provider().as_str(), @@ -667,10 +663,10 @@ impl RegisteredTemporalHarness { "INSERT INTO lcm_raw_messages ( provider, message_id, session_id, role, ordinal, timestamp, content, content_hash, storage_kind, payload_ref, - snippet_text, index_text, legacy_source, legacy_truncated, metadata_json + placeholder_text, metadata_json ) VALUES ( 'provider.application', 'message-2', 'session.temporal.application', - 'assistant', 2, 2, NULL, ?1, 'external', ?2, ?3, ?3, 0, 0, ?4 + 'assistant', 2, 2, NULL, ?1, 'external', ?2, ?3, ?4 )", params![ payload.content_hash.as_str(), diff --git a/crates/tracedecay-session-memory/src/session/tests/privacy.rs b/crates/tracedecay-session-memory/src/session/tests/privacy.rs index 3b90456610..4d11b8f399 100644 --- a/crates/tracedecay-session-memory/src/session/tests/privacy.rs +++ b/crates/tracedecay-session-memory/src/session/tests/privacy.rs @@ -25,8 +25,8 @@ use super::harness::{ SAFE_PRIVACY_PAYLOAD, }; use crate::context::{ - BranchId, CancellationToken, CapabilityDigest, ConfigurationDigest, PolicyDigest, ProfileId, - RequestBudgets, ResolvedGitRoute, ResolvedSessionIdentity, SessionRootId, SessionStoreId, + BranchId, CapabilityDigest, ConfigurationDigest, PolicyDigest, ProfileId, RequestBudgets, + ResolvedGitRoute, ResolvedSessionIdentity, SessionRootId, SessionStoreId, application_observed_at, session_application_grant_digest, }; use crate::session::{ @@ -35,6 +35,7 @@ use crate::session::{ SessionRetrievalOutcome, SessionRetrievalService, SessionScopeAuthorizationRequest, SessionScopeAuthorizer, SessionTemporalQuery, }; +use tracedecay_runtime_core::cancellation::CancellationToken; const DIGEST: [u8; 32] = [0x5a; 32]; diff --git a/crates/tracedecay-session-memory/src/session/types.rs b/crates/tracedecay-session-memory/src/session/types.rs index a3b9a5dfd1..beffae072b 100644 --- a/crates/tracedecay-session-memory/src/session/types.rs +++ b/crates/tracedecay-session-memory/src/session/types.rs @@ -13,9 +13,10 @@ pub use tracedecay_sessions::{ }; use crate::context::{ - CancellationToken, CapabilityDigest, ConfigurationDigest, PolicyDigest, RequestBudgets, - ResolvedSessionIdentity, SessionOwner, session_application_grant_digest, + CapabilityDigest, ConfigurationDigest, PolicyDigest, RequestBudgets, ResolvedSessionIdentity, + SessionOwner, session_application_grant_digest, }; +use tracedecay_runtime_core::cancellation::CancellationToken; /// Typed authority for the already-resolved session store/root. /// @@ -661,10 +662,10 @@ mod tests { use super::*; use crate::context::{ - BranchId, CancellationToken, CapabilityDigest, ConfigurationDigest, PolicyDigest, - ProfileId, RequestBudgets, ResolvedGitRoute, ResolvedSessionIdentity, SessionRootId, - SessionStoreId, + BranchId, CapabilityDigest, ConfigurationDigest, PolicyDigest, ProfileId, RequestBudgets, + ResolvedGitRoute, ResolvedSessionIdentity, SessionRootId, SessionStoreId, }; + use tracedecay_runtime_core::cancellation::CancellationToken; const DIGEST: [u8; 32] = [0xa5; 32]; diff --git a/crates/tracedecay-session-memory/src/user_config.rs b/crates/tracedecay-session-memory/src/user_config.rs index a69bcd4ad1..9c88e57d56 100644 --- a/crates/tracedecay-session-memory/src/user_config.rs +++ b/crates/tracedecay-session-memory/src/user_config.rs @@ -67,7 +67,7 @@ pub struct UserConfig { pub agent_dashboard_enabled: BTreeMap, /// Debounce duration for the embedded MCP file watcher (e.g. "2s", "15s", "1m"). - #[serde(default = "default_watcher_debounce", alias = "daemon_debounce")] + #[serde(default = "default_watcher_debounce")] pub watcher_debounce: String, /// Cached country flags from the worldwide counter. @@ -506,7 +506,7 @@ impl UserConfig { revision_id, }) })(); - let _ = lock_file.unlock(); + drop(lock_file); result } @@ -542,7 +542,7 @@ impl UserConfig { })?; let result = Self::write_locked(&path, &contents, recover); - let _ = lock_file.unlock(); + drop(lock_file); result } diff --git a/crates/tracedecay-session-memory/tests/hotpath_coverage.rs b/crates/tracedecay-session-memory/tests/hotpath_coverage.rs index 7d49e1e2f0..a22ab1b6f4 100644 --- a/crates/tracedecay-session-memory/tests/hotpath_coverage.rs +++ b/crates/tracedecay-session-memory/tests/hotpath_coverage.rs @@ -4,8 +4,9 @@ //! alone must not create a report without a process-boundary guard. use tracedecay_domain::{FactCategoryV1, FactOwnerV1, ProjectId}; +use tracedecay_runtime_core::cancellation::CancellationToken; use tracedecay_session_memory::context::{ - CancellationToken, CapabilityDigest, ConfigurationDigest, PolicyDigest, RequestBudgets, + CapabilityDigest, ConfigurationDigest, PolicyDigest, RequestBudgets, session_application_grant_digest, }; use tracedecay_session_memory::memory::{ProjectMemoryFactAddRequest, automatic_fact_add_command}; diff --git a/crates/tracedecay-session-runtime/Cargo.toml b/crates/tracedecay-session-runtime/Cargo.toml index 8929130559..ab811477f5 100644 --- a/crates/tracedecay-session-runtime/Cargo.toml +++ b/crates/tracedecay-session-runtime/Cargo.toml @@ -36,6 +36,7 @@ tracing = "0.1" tracedecay-contracts = { path = "../tracedecay-contracts", version = "0.1.0" } tracedecay-code-index = { path = "../tracedecay-code-index", version = "0.1.0", default-features = false } tracedecay-code-index-runtime = { path = "../tracedecay-code-index-runtime", version = "0.1.0", default-features = false } +tracedecay-configuration = { path = "../tracedecay-configuration", version = "0.1.0" } tracedecay-daemon-protocol = { path = "../tracedecay-daemon-protocol", version = "0.1.0" } tracedecay-domain = { path = "../tracedecay-domain", version = "0.1.0" } tracedecay-global-db = { path = "../tracedecay-global-db", version = "0.1.0" } @@ -57,6 +58,7 @@ tracedecay-application = { path = "../tracedecay-application", version = "0.1.0" hex = "0.4" tempfile = "3" tokio = { version = "1", features = ["full", "test-util"] } +tracedecay-configuration = { path = "../tracedecay-configuration", version = "0.1.0", features = ["test-helpers"] } tracedecay-daemon-identity = { path = "../tracedecay-daemon-identity", version = "0.1.0" } tracedecay-global-db = { path = "../tracedecay-global-db", version = "0.1.0", features = ["test-helpers"] } tracedecay-session-temporal-store = { path = "../tracedecay-session-temporal-store", version = "0.1.0", features = ["test-helpers"] } diff --git a/crates/tracedecay-session-runtime/src/lcm_authority.rs b/crates/tracedecay-session-runtime/src/lcm_authority.rs index 0648714ecc..3acfefe883 100644 --- a/crates/tracedecay-session-runtime/src/lcm_authority.rs +++ b/crates/tracedecay-session-runtime/src/lcm_authority.rs @@ -15,9 +15,10 @@ use tracedecay_lcm::{ LcmCompressionRequest, LcmCompressionResponse, LcmError, LcmGcConfig, LcmPreflightRequest, LcmPreflightResponse, LcmStatus, LcmSummarizerMode, }; +use tracedecay_runtime_core::cancellation::CancellationToken; use tracedecay_session_memory::context::{ - CancellationToken, RequestInterruption, application_observed_at, - application_request_interruption, run_application_request_interruptible, + RequestInterruption, application_observed_at, application_request_interruption, + run_application_request_interruptible, }; use tracedecay_session_memory::session::lcm::{ LcmAuthorityFuture, LcmAuthorityInvocation, LcmAuthorityOperation, LcmAuthorityOutcome, @@ -27,6 +28,7 @@ use tracedecay_session_memory::session::lcm::{ }; use tracedecay_global_db::RegisteredGlobalDbLeaseV1; +use tracedecay_session_temporal_store::SessionTemporalAccess; mod mount; mod receipt; @@ -95,8 +97,12 @@ impl LcmDaemonStore for RegisteredLcmDaemonStore { fn doctor(&self, _query: LcmDoctorQuery) -> StoreFuture<'_, serde_json::Value> { Box::pin(async move { - serde_json::to_value(self.database.session_temporal_doctor_health().await) - .map_err(|error| LcmError::Db(error.to_string())) + serde_json::to_value( + SessionTemporalAccess::new(&*self.database) + .session_temporal_doctor_health() + .await, + ) + .map_err(|error| LcmError::Db(error.to_string())) }) } } diff --git a/crates/tracedecay-session-runtime/src/lcm_authority/mount.rs b/crates/tracedecay-session-runtime/src/lcm_authority/mount.rs index 36d3a7c41f..a1e6f950d5 100644 --- a/crates/tracedecay-session-runtime/src/lcm_authority/mount.rs +++ b/crates/tracedecay-session-runtime/src/lcm_authority/mount.rs @@ -11,9 +11,10 @@ use tracedecay_contracts::{ DisclosureClass, RequestContext, RequestId, }; use tracedecay_domain::{ActorId, UtcMicros}; +use tracedecay_runtime_core::cancellation::CancellationToken; use tracedecay_session_memory::context::{ - CancellationToken, CapabilityDigest, ConfigurationDigest, PolicyDigest, RequestBudgets, - ResolvedSessionIdentity, application_observed_at, session_application_grant_digest, + CapabilityDigest, ConfigurationDigest, PolicyDigest, RequestBudgets, ResolvedSessionIdentity, + application_observed_at, session_application_grant_digest, }; use tracedecay_session_memory::session::SessionRequestBinding; use tracedecay_session_memory::session::lcm::{ diff --git a/crates/tracedecay-session-runtime/src/lcm_authority/tests.rs b/crates/tracedecay-session-runtime/src/lcm_authority/tests.rs index 4bbcfe3030..1d1fb57249 100644 --- a/crates/tracedecay-session-runtime/src/lcm_authority/tests.rs +++ b/crates/tracedecay-session-runtime/src/lcm_authority/tests.rs @@ -76,7 +76,6 @@ impl LcmDaemonStore for FakeStore { replay_token_estimate: 0, replay_over_budget: false, compression_attempts: 0, - fallback_used: false, context_recovery_hint: None, retry_status: None, relation_projection_status: @@ -635,10 +634,9 @@ async fn unsupported_pressure_preflight_does_not_create_session_or_raw_messages( assert_eq!(response.outcome, LcmAuthorityOutcome::Ready); for table in [ "lcm_raw_messages", - "lcm_summary_nodes", - "lcm_summary_sources", "lcm_lifecycle_state", "session_summary_nodes", + "session_summary_sources", ] { assert_eq!( table_row_count(&database, table).await, diff --git a/crates/tracedecay-session-runtime/src/lcm_effects.rs b/crates/tracedecay-session-runtime/src/lcm_effects.rs index 0f98356be4..e08ccb4c13 100644 --- a/crates/tracedecay-session-runtime/src/lcm_effects.rs +++ b/crates/tracedecay-session-runtime/src/lcm_effects.rs @@ -1,12 +1,13 @@ use std::time::Duration; use tracedecay_contracts::{CancellationSignal, Deadline}; -use tracedecay_temporal_query::ports::ExecutionControl; +use tracedecay_temporal_query::execution::ExecutionControl; use tracedecay_global_db::RegisteredGlobalDbLeaseV1; use tracedecay_lcm::{LcmCompressionRequest, LcmCompressionResponse, LcmError, LcmSummarizerMode}; #[cfg(any(test, feature = "test-helpers"))] use tracedecay_lcm::{LcmSessionBoundaryRequest, LcmSessionBoundaryResponse}; +use tracedecay_session_temporal_store::SessionTemporalAccess; pub(super) const LCM_EFFECT_CEILING: Duration = tracedecay_daemon_protocol::DEFAULT_DAEMON_OPERATION_DEADLINE; @@ -339,7 +340,7 @@ impl DaemonLcmEffectService { let recovered = self .control .execute(&execution, async { - self.db + SessionTemporalAccess::new(&*self.db) .recover_pending_session_relation_projection_page( RELATION_PAGE_LIMIT, tracedecay_session_temporal_store::store::execution_control_graph_cancellation( diff --git a/crates/tracedecay-session-runtime/src/lcm_effects/tests.rs b/crates/tracedecay-session-runtime/src/lcm_effects/tests.rs index 2543dd158f..33c442676b 100644 --- a/crates/tracedecay-session-runtime/src/lcm_effects/tests.rs +++ b/crates/tracedecay-session-runtime/src/lcm_effects/tests.rs @@ -1,12 +1,27 @@ use super::*; use serde_json::Value; -use tracedecay_domain::SessionId; +use tracedecay_domain::ProjectId; +use tracedecay_domain::configuration::{LcmSummarizerExecutableV1, LcmSummarizerExecutablesV1}; +use tracedecay_domain::{ + CanonicalObservationEnvelopeV1, ComponentVersion, DurableObservationV1, + ObservationIdentityMaterialV1, ObservationScopeV1, ObservationSourceCursorV1, + ObservationSourceGenerationV1, ObservationSourceIdentityV1, PayloadReferenceV1, + ProjectionGenerationId, RetentionClass, SanitizationReceiptId, SanitizationReceiptRefV1, + SanitizationReceiptV1, SanitizerDispositionV1, SensitivityV1, SessionId, UtcMicros, +}; use tracedecay_global_db::RegisteredGlobalDb; -use tracedecay_global_db::tests::harness::RegisteredGlobalDbHarness; +use tracedecay_global_db::RegisteredGlobalDbLeaseV1; +use tracedecay_global_db::tests::harness::{ + RegisteredGlobalDbHarness, RegisteredGlobalDbTestRuntime, +}; use tracedecay_lcm::{LcmRelationProjectionStatus, LcmSourceRef, LcmSummarizerMode}; use tracedecay_runtime_core::db::engine::params; use tracedecay_sessions::runtime::{SessionMessageRecord, SessionRecord}; -use tracedecay_store::ParseOffset; +use tracedecay_store::{ + AnchoredObservationWrite, ObservationProjectionStore, ObservationStore, ObservationWrite, + ParseOffset, build_observation_resolution_authorization_v1, build_observation_retrieval_anchor, + derive_canonical_projection, +}; mod compression_ownership; @@ -98,8 +113,90 @@ fn retained_guard( } } +/// A registered project sessions shard whose summarizer binding the test +/// publishes explicitly through `lcm.summarizer_executables.v1`. Provider +/// executables are never resolved from the environment or `PATH`, so a test +/// that expects a fake summarizer to run must pin it here. +struct ProjectSummarizerFixture { + runtime: RegisteredGlobalDbTestRuntime, + project_id: ProjectId, + root: tempfile::TempDir, +} + +impl ProjectSummarizerFixture { + /// The project identity is the `project_key` every fixture session + /// carries, so observations scoped to this project reconcile with the + /// sessions the tests upsert directly. + async fn open() -> Self { + let root = tempfile::tempdir().unwrap(); + let project_id = ProjectId::new("project.lcm-effects".to_owned()).unwrap(); + let runtime = RegisteredGlobalDbTestRuntime::project( + root.path().join("profile"), + root.path().join("project"), + project_id.clone(), + ) + .await + .unwrap(); + Self { + runtime, + project_id, + root, + } + } + + fn db(&self) -> RegisteredGlobalDbLeaseV1 { + self.runtime.project_database_arc().unwrap() + } + + fn pin(&self, executables: LcmSummarizerExecutablesV1) { + tracedecay_configuration::test_support::pin_lcm_summarizer_executables( + self.project_id.clone(), + &self.root.path().join("project"), + executables, + ) + .unwrap(); + } + + fn pin_cursor_agent(&self, cursor_agent: &std::path::Path) { + self.pin(LcmSummarizerExecutablesV1 { + cursor_agent: LcmSummarizerExecutableV1::configured(cursor_agent.to_path_buf()) + .unwrap(), + codex: LcmSummarizerExecutableV1::Unconfigured, + }); + } + + fn pin_codex(&self, codex: &std::path::Path) { + self.pin(LcmSummarizerExecutablesV1 { + cursor_agent: LcmSummarizerExecutableV1::Unconfigured, + codex: LcmSummarizerExecutableV1::configured(codex.to_path_buf()).unwrap(), + }); + } + + /// Reopens the same project shard, as a daemon restart would. + async fn restart(self) -> Self { + let Self { + runtime, + project_id, + root, + } = self; + drop(runtime); + let runtime = RegisteredGlobalDbTestRuntime::project( + root.path().join("profile"), + root.path().join("project"), + project_id.clone(), + ) + .await + .unwrap(); + Self { + runtime, + project_id, + root, + } + } +} + /// Runs a future under the canonical user-data-dir env lock so provider -/// binary env overrides cannot race parallel tests. +/// tuning env overrides cannot race parallel tests. fn run_with_test_env_lock(future: impl std::future::Future) -> T { let _lock = tracedecay_runtime_core::config::lock_user_data_dir_test_env(); tokio::runtime::Builder::new_multi_thread() @@ -133,8 +230,7 @@ async fn retained_relation_recovery_preserves_typed_cancellation() { let control = LcmEffectControl::new(None, Some(&cancellation)); let execution = control.execution_control(); - let error = harness - .registered + let error = SessionTemporalAccess::new(&*harness.registered) .recover_pending_session_relation_projection_page( 1, tracedecay_session_temporal_store::store::execution_control_graph_cancellation( @@ -221,7 +317,10 @@ async fn compression_producer_apply_read_and_rollback_stay_one_authority() { content, "canonical historical message 1 with durable context" ); - assert!(!summary.summary_text.is_empty()); + assert_eq!( + summary.summary_text, + "fixture summary preserving canonical historical context" + ); assert_eq!( response.relation_projection_status, LcmRelationProjectionStatus::Applied @@ -230,7 +329,7 @@ async fn compression_producer_apply_read_and_rollback_stay_one_authority() { let session_id = SessionId::new("compress-session").unwrap(); let relation_ids = [summary.node_id.clone()]; let read_control = execution_control(); - let (_, relations) = db + let (_, relations) = SessionTemporalAccess::new(&*db) .active_session_summary_relations( &session_id, &relation_ids, @@ -244,14 +343,15 @@ async fn compression_producer_apply_read_and_rollback_stay_one_authority() { assert_eq!(relations.len(), 1); assert_eq!(relations[0].sources.len(), summary.source_refs.len()); assert_eq!( - db.recover_pending_session_relation_projections( - 1, - tracedecay_session_temporal_store::store::execution_control_graph_cancellation( - &read_control, - ), - ) - .await - .unwrap(), + SessionTemporalAccess::new(&*db) + .recover_pending_session_relation_projections( + 1, + tracedecay_session_temporal_store::store::execution_control_graph_cancellation( + &read_control, + ), + ) + .await + .unwrap(), 0, "compress applies the graph projection in the same journey" ); @@ -262,7 +362,7 @@ async fn compression_producer_apply_read_and_rollback_stay_one_authority() { let harness = harness.restart().await; let restarted = harness.registered.clone(); let restart_control = execution_control(); - let (_, restarted_relations) = restarted + let (_, restarted_relations) = SessionTemporalAccess::new(&*restarted) .active_session_summary_relations( &session_id, &relation_ids, @@ -305,7 +405,7 @@ async fn preflight_reads_canonical_state_without_creating_or_ingesting_a_session assert!(response.replay_messages.is_empty()); let snapshot = db.read_snapshot().await.unwrap(); - for table in ["sessions", "session_messages", "lcm_raw_messages"] { + for table in ["sessions", "lcm_raw_messages"] { let mut rows = snapshot .query(&format!("SELECT COUNT(*) FROM {table}"), ()) .await @@ -329,11 +429,12 @@ async fn native_summary_evidence_requires_exact_cursor_text_and_claude_pair_iden assert!(db.upsert_session(&session(provider, session_id)).await); } let cursor_text = "exact Cursor Composer compacted text"; - let cursor_metadata = canonical_envelope( + let cursor_summary = canonical_record(canonical_envelope( "cursor", "cursor-native-session", "cursor-summary", None, + (10, 0), vec![ serde_json::json!({ "kind": "message", @@ -345,17 +446,8 @@ async fn native_summary_evidence_requires_exact_cursor_text_and_claude_pair_iden "summary": cursor_text }), ], - ); - insert_summary_evidence( - (&db, "cursor"), - "cursor-native-session", - "cursor-summary", - 10, - cursor_text, - "message", - &cursor_metadata, - ) - .await; + )); + ingest_canonical(&db, "cursor-native-session", &[], &[&cursor_summary]).await; insert_summary_evidence( (&db, "codex"), "codex-native-session", @@ -384,11 +476,12 @@ async fn native_summary_evidence_requires_exact_cursor_text_and_claude_pair_iden .await; let claude_text = "exact Claude compact summary wrapper and body"; - let claude_summary_metadata = canonical_envelope( + let claude_summary = canonical_record(canonical_envelope( "claude", "claude-native-session", "claude-summary", Some("claude-boundary"), + (11, 0), vec![ serde_json::json!({ "kind": "message", @@ -403,17 +496,8 @@ async fn native_summary_evidence_requires_exact_cursor_text_and_claude_pair_iden } }), ], - ); - insert_summary_evidence( - (&db, "claude"), - "claude-native-session", - "claude-summary", - 11, - claude_text, - "message", - &claude_summary_metadata, - ) - .await; + )); + ingest_canonical(&db, "claude-native-session", &[], &[&claude_summary]).await; let cursor = super::super::lcm_summarization::native_summary_evidence( &db, @@ -475,11 +559,12 @@ async fn native_summary_evidence_requires_exact_cursor_text_and_claude_pair_iden assert_eq!(codex.text, "exact Codex plaintext summary"); assert_eq!(codex.route, "codex_native_compaction"); - let boundary_metadata = canonical_envelope( + let boundary = canonical_record(canonical_envelope( "claude", "claude-native-session", "claude-boundary", None, + (12, 1), vec![ serde_json::json!({ "kind": "boundary", @@ -494,97 +579,8 @@ async fn native_summary_evidence_requires_exact_cursor_text_and_claude_pair_iden } }), ], - ); - insert_summary_evidence( - (&db, "claude"), - "claude-native-session", - "claude-boundary", - 10, - "Claude compaction boundary", - "compaction", - &boundary_metadata, - ) - .await; - let claude = super::super::lcm_summarization::native_summary_evidence( - &db, - "claude", - "claude-native-session", - None, - ) - .await - .unwrap() - .unwrap(); - assert_eq!(claude.text, claude_text); - assert_eq!(claude.route, "claude_native_compaction"); -} - -#[tokio::test] -async fn claude_native_compaction_recognizes_production_boundary_id() { - let harness = RegisteredGlobalDbHarness::open("lcm-claude-prod-boundary").await; - let db = harness.registered.clone(); - assert!( - db.upsert_session(&session("claude", "claude-native-session")) - .await - ); - let claude_text = "production Claude compact pair body"; - let summary_metadata = canonical_envelope( - "claude", - "claude-native-session", - "aaaaaaaa-0000-4000-8000-000000000001", - Some("ffffffff-0000-4000-8000-000000000001"), - vec![ - serde_json::json!({ - "kind": "message", - "role": "user", - "content": claude_text - }), - serde_json::json!({ - "kind": "compaction", - "summary": { - "isCompactSummary": true, - "isVisibleInTranscriptOnly": true - } - }), - ], - ); - insert_summary_evidence( - (&db, "claude"), - "claude-native-session", - "aaaaaaaa-0000-4000-8000-000000000001", - 11, - claude_text, - "message", - &summary_metadata, - ) - .await; - let boundary_envelope = canonical_envelope( - "claude", - "claude-native-session", - "ffffffff-0000-4000-8000-000000000001", - Some("pre-compact-parent"), - vec![serde_json::json!({ - "kind": "compaction", - "summary": { - "preservedSegment": { - "anchorUuid": "aaaaaaaa-0000-4000-8000-000000000001" - } - } - })], - ); - insert_summary_evidence( - (&db, "claude"), - "claude-native-session", - "compact_boundary:ffffffff-0000-4000-8000-000000000001", - 10, - "Claude compaction boundary", - "compact_boundary", - &serde_json::json!({ - "source": "claude_compact_boundary", - "trigger": "manual", - "canonical_envelope": boundary_envelope - }), - ) - .await; + )); + ingest_canonical(&db, "claude-native-session", &[], &[&boundary]).await; let claude = super::super::lcm_summarization::native_summary_evidence( &db, "claude", @@ -613,62 +609,45 @@ async fn ingest_claude_compact_pair(db: &RegisteredGlobalDb, session_id: &str, l record.role = leading_role.to_string(); messages.push(record); } - let mut boundary = message(session_id, 3); - boundary.provider = "claude".to_string(); - boundary.message_id = boundary_id.clone(); - boundary.role = "system".to_string(); - boundary.kind = Some("compaction".to_string()); - boundary.metadata_json = Some( - canonical_envelope( - "claude", - session_id, - &boundary_id, - None, - vec![ - serde_json::json!({ - "kind": "boundary", - "boundary_kind": "compaction_boundary" - }), - serde_json::json!({ - "kind": "compaction", - "summary": {"preservedSegment": {"anchorUuid": summary_id}} - }), - ], - ) - .to_string(), - ); - messages.push(boundary); - let mut summary = message(session_id, 4); - summary.provider = "claude".to_string(); - summary.message_id = summary_id.clone(); - summary.role = "user".to_string(); - summary.text = "authoritative Claude compaction".to_string(); - summary.metadata_json = Some( - canonical_envelope( - "claude", - session_id, - &summary_id, - Some(&boundary_id), - vec![serde_json::json!({ + let boundary = canonical_record(canonical_envelope( + "claude", + session_id, + &boundary_id, + None, + (3, 0), + vec![ + serde_json::json!({ + "kind": "boundary", + "boundary_kind": "compaction_boundary" + }), + serde_json::json!({ + "kind": "compaction", + "summary": {"preservedSegment": {"anchorUuid": summary_id}} + }), + ], + )); + let summary = canonical_record(canonical_envelope( + "claude", + session_id, + &summary_id, + Some(&boundary_id), + (4, 1), + vec![ + serde_json::json!({ + "kind": "message", + "role": "user", + "content": "authoritative Claude compaction" + }), + serde_json::json!({ "kind": "compaction", "summary": { "isCompactSummary": true, "isVisibleInTranscriptOnly": true } - })], - ) - .to_string(), - ); - messages.push(summary); - assert!( - db.upsert_transcript_batch( - &session("claude", session_id), - &messages, - &format!("/tmp/{session_id}.jsonl"), - ParseOffset::default(), - ) - .await - ); + }), + ], + )); + ingest_canonical(db, session_id, &messages, &[&boundary, &summary]).await; } /// An absent predecessor interval must never reach a published summary as @@ -821,7 +800,7 @@ async fn transcript_ingest_persists_native_compaction_raw_range() { let mut rows = snapshot .query( "SELECT summary_text, json_extract(metadata_json, '$.summary_route') - FROM lcm_summary_nodes + FROM session_summary_nodes WHERE provider = 'codex' AND session_id = ?1", params![session_id], ) @@ -850,115 +829,83 @@ async fn successive_claude_compactions_bind_to_the_previous_native_boundary_afte record.message_id = format!("claude-before-first-{ordinal}"); messages.push(record); } + let compact_pair = + |boundary_id: &str, summary_id: &str, (ordinal, position): (u64, u64), text: &str| { + let boundary = canonical_record(canonical_envelope( + "claude", + session_id, + boundary_id, + None, + (ordinal, position), + vec![ + serde_json::json!({ + "kind": "boundary", + "boundary_kind": "compaction_boundary" + }), + serde_json::json!({ + "kind": "compaction", + "summary": {"preservedSegment": {"anchorUuid": summary_id}} + }), + ], + )); + let summary = canonical_record(canonical_envelope( + "claude", + session_id, + summary_id, + Some(boundary_id), + (ordinal + 1, position + 1), + vec![ + serde_json::json!({ + "kind": "message", + "role": "assistant", + "content": text + }), + serde_json::json!({ + "kind": "compaction", + "summary": { + "isCompactSummary": true, + "isVisibleInTranscriptOnly": true + } + }), + ], + )); + (boundary, summary) + }; let first_summary_id = "claude-first-summary"; - let first_boundary_id = "claude-first-boundary"; - let mut first_boundary = message(session_id, 4); - first_boundary.provider = "claude".to_string(); - first_boundary.message_id = first_boundary_id.to_string(); - first_boundary.kind = Some("compaction".to_string()); - first_boundary.metadata_json = Some( - canonical_envelope( - "claude", - session_id, - first_boundary_id, - None, - vec![ - serde_json::json!({ - "kind": "boundary", - "boundary_kind": "compaction_boundary" - }), - serde_json::json!({ - "kind": "compaction", - "summary": {"preservedSegment": {"anchorUuid": first_summary_id}} - }), - ], - ) - .to_string(), + let (first_boundary, first_summary) = compact_pair( + "claude-first-boundary", + first_summary_id, + (4, 0), + "first authoritative Claude compaction", ); - messages.push(first_boundary); - let mut first_summary = message(session_id, 5); - first_summary.provider = "claude".to_string(); - first_summary.message_id = first_summary_id.to_string(); - first_summary.text = "first authoritative Claude compaction".to_string(); - first_summary.metadata_json = Some( - canonical_envelope( - "claude", - session_id, - first_summary_id, - Some(first_boundary_id), - vec![serde_json::json!({ - "kind": "compaction", - "summary": { - "isCompactSummary": true, - "isVisibleInTranscriptOnly": true - } - })], - ) - .to_string(), - ); - messages.push(first_summary); + ingest_canonical( + &db, + session_id, + &messages, + &[&first_boundary, &first_summary], + ) + .await; + let mut between = Vec::new(); for ordinal in 6..=520 { let mut record = message(session_id, ordinal); record.provider = "claude".to_string(); record.message_id = format!("claude-between-{ordinal}"); - messages.push(record); + between.push(record); } let second_summary_id = "claude-second-summary"; - let second_boundary_id = "claude-second-boundary"; - let mut second_boundary = message(session_id, 521); - second_boundary.provider = "claude".to_string(); - second_boundary.message_id = second_boundary_id.to_string(); - second_boundary.kind = Some("compaction".to_string()); - second_boundary.metadata_json = Some( - canonical_envelope( - "claude", - session_id, - second_boundary_id, - None, - vec![ - serde_json::json!({ - "kind": "boundary", - "boundary_kind": "compaction_boundary" - }), - serde_json::json!({ - "kind": "compaction", - "summary": {"preservedSegment": {"anchorUuid": second_summary_id}} - }), - ], - ) - .to_string(), - ); - messages.push(second_boundary); - let mut second_summary = message(session_id, 522); - second_summary.provider = "claude".to_string(); - second_summary.message_id = second_summary_id.to_string(); - second_summary.text = "second authoritative Claude compaction".to_string(); - second_summary.metadata_json = Some( - canonical_envelope( - "claude", - session_id, - second_summary_id, - Some(second_boundary_id), - vec![serde_json::json!({ - "kind": "compaction", - "summary": { - "isCompactSummary": true, - "isVisibleInTranscriptOnly": true - } - })], - ) - .to_string(), - ); - messages.push(second_summary); - assert!( - db.upsert_transcript_batch( - &session("claude", session_id), - &messages, - "/tmp/claude-successive-native-ranges.jsonl", - ParseOffset::default(), - ) - .await + let (second_boundary, second_summary) = compact_pair( + "claude-second-boundary", + second_summary_id, + (521, 2), + "second authoritative Claude compaction", ); + ingest_canonical( + &db, + session_id, + &between, + &[&second_boundary, &second_summary], + ) + .await; let snapshot = db.read_snapshot().await.unwrap(); let mut rows = snapshot @@ -991,9 +938,11 @@ async fn successive_claude_compactions_bind_to_the_previous_native_boundary_afte .find(|(_, message_id, _)| message_id == second_summary_id) .unwrap() .0; + // Compression pins `system` rows (the compaction boundaries) apart from + // the conversational backlog a native summary is compared against. let first_sources = raw .iter() - .filter(|(store_id, _, _)| *store_id < first_summary_store_id) + .filter(|(store_id, _, role)| *store_id < first_summary_store_id && role != "system") .map( |(store_id, _, role)| tracedecay_lcm::LcmSummarySourceMessage { store_id: *store_id, @@ -1038,8 +987,10 @@ async fn successive_claude_compactions_bind_to_the_previous_native_boundary_afte let restarted = harness.registered.clone(); let second_sources = raw .iter() - .filter(|(store_id, _, _)| { - *store_id >= first_summary_store_id && *store_id < second_summary_store_id + .filter(|(store_id, _, role)| { + *store_id >= first_summary_store_id + && *store_id < second_summary_store_id + && role != "system" }) .map( |(store_id, _, role)| tracedecay_lcm::LcmSummarySourceMessage { @@ -1081,8 +1032,8 @@ async fn successive_claude_compactions_bind_to_the_previous_native_boundary_afte #[test] fn native_compaction_requires_exact_selected_raw_membership() { run_with_test_env_lock(async { - let harness = RegisteredGlobalDbHarness::open("lcm-native-membership").await; - let db = harness.registered.clone(); + let fixture = ProjectSummarizerFixture::open().await; + let db = fixture.db(); let session_id = "codex-native-membership-session"; let mut messages = Vec::new(); for ordinal in 1..=4 { @@ -1146,11 +1097,8 @@ done .unwrap(); use std::os::unix::fs::PermissionsExt as _; std::fs::set_permissions(&codex_bin, std::fs::Permissions::from_mode(0o700)).unwrap(); - let codex_bin_env = codex_bin.to_string_lossy().into_owned(); - let _env = TestEnvironment::set([ - ("TRACEDECAY_CODEX_BIN", codex_bin_env.as_str()), - ("TRACEDECAY_CODEX_SUMMARY_TIMEOUT_SECS", "5"), - ]); + fixture.pin_codex(&codex_bin); + let _env = TestEnvironment::set([("TRACEDECAY_CODEX_SUMMARY_TIMEOUT_SECS", "5")]); let converged = super::super::lcm_summary_convergence::run_summary_convergence_page(db.clone(), 1) @@ -1160,8 +1108,8 @@ done let snapshot = db.read_snapshot().await.unwrap(); let mut rows = snapshot .query( - "SELECT node_id, summary_text - FROM lcm_summary_nodes + "SELECT summary_id, summary_text + FROM session_summary_nodes WHERE provider = 'codex' AND session_id = ?1", params![session_id], ) @@ -1176,8 +1124,8 @@ done drop(rows); let mut sources = snapshot .query( - "SELECT source_id FROM lcm_summary_sources - WHERE node_id = ?1 AND source_kind = 'raw_message' + "SELECT source_id FROM session_summary_sources + WHERE summary_id = ?1 AND source_kind = 'raw_message' ORDER BY ordinal", params![node_id], ) @@ -1717,14 +1665,24 @@ async fn partial_revision_invalidation_hides_replay_and_yields_to_a_due_peer() { .unwrap() .unwrap(); let transaction = db.begin_write_transaction().await.unwrap(); + transaction + .execute_batch( + "INSERT INTO retrieval_anchors ( + anchor_id, anchor_json, owner_json, projection_generation + ) VALUES ('synthetic-fairness-anchor', '{}', '{}', 'test');", + ) + .await + .unwrap(); transaction .execute( - "INSERT INTO lcm_summary_nodes( - node_id, provider, conversation_id, session_id, depth, - summary_text, summary_hash, summary_token_count, source_token_count + "INSERT INTO session_summary_nodes( + summary_id, session_id, provider, conversation_id, depth, + summary_anchor_id, summary_text, summary_hash, summary_token_count, + source_token_count, source_horizon_json, created_at ) VALUES ( - 'synthetic-fairness-dependent', 'cursor', ?1, ?1, 0, - 'second dependent summary', 'synthetic-fairness-hash', 3, 4 + 'synthetic-fairness-dependent', ?1, 'cursor', ?1, 0, + 'synthetic-fairness-anchor', 'second dependent summary', + 'synthetic-fairness-hash', 3, 4, '{}', 1 )", params![large_session], ) @@ -1732,7 +1690,7 @@ async fn partial_revision_invalidation_hides_replay_and_yields_to_a_due_peer() { .unwrap(); transaction .execute( - "INSERT INTO lcm_summary_sources(node_id, source_kind, source_id, ordinal) + "INSERT INTO session_summary_sources(summary_id, source_kind, source_id, ordinal) VALUES ('synthetic-fairness-dependent', 'raw_message', CAST(?1 AS TEXT), 0)", params![first_store_id], ) @@ -1924,12 +1882,13 @@ async fn malformed_relation_receipt_is_permanent_without_starving_summary_work() .unwrap(); transaction.commit().await.unwrap(); - db.recover_pending_session_relation_projection_page( - 16, - std::sync::Arc::new(tracedecay_graph_db::NeverCancelled), - ) - .await - .unwrap(); + SessionTemporalAccess::new(&*db) + .recover_pending_session_relation_projection_page( + 16, + std::sync::Arc::new(tracedecay_graph_db::NeverCancelled), + ) + .await + .unwrap(); let snapshot = db.read_snapshot().await.unwrap(); let mut rows = snapshot .query( @@ -1973,58 +1932,72 @@ async fn malformed_relation_receipt_is_permanent_without_starving_summary_work() ); } -#[tokio::test] -async fn mega_session_convergence_bounds_protection_and_compression_pages() { - const RAW_ROWS: i64 = tracedecay_lcm::LCM_SCAN_PAGE_ROWS + 1; - let harness = RegisteredGlobalDbHarness::open("lcm-summary-convergence-mega").await; - let db = harness.registered.clone(); - let storage_root = db.db_path().parent().unwrap(); - let session_id = "mega-convergence-session"; - assert!(db.upsert_session(&session("cursor", session_id)).await); - for ordinal in 1..=RAW_ROWS { - let mut record = message(session_id, ordinal); - record.message_id = format!("{session_id}-message-{ordinal}"); - record.text = format!("{ordinal:04}:{}", "bounded retained context ".repeat(32)); - db.lcm_ingest_raw_message(storage_root, &record) - .await - .unwrap(); - } +#[test] +fn mega_session_convergence_bounds_protection_and_compression_pages() { + run_with_test_env_lock(async { + // Summarization must never reach the operator's installed agent CLI: + // this profile shard has no `lcm.summarizer_executables.v1` binding, + // so every provider stays unconfigured and nothing is launched. + let temporary = tempfile::tempdir().unwrap(); + let workspace_env = temporary.path().to_string_lossy().into_owned(); + let _env = TestEnvironment::set([( + "TRACEDECAY_CURSOR_SUMMARY_WORKSPACE", + workspace_env.as_str(), + )]); + const RAW_ROWS: i64 = tracedecay_lcm::LCM_SCAN_PAGE_ROWS + 1; + let harness = RegisteredGlobalDbHarness::open("lcm-summary-convergence-mega").await; + let db = harness.registered.clone(); + let storage_root = db.db_path().parent().unwrap(); + let session_id = "mega-convergence-session"; + assert!(db.upsert_session(&session("cursor", session_id)).await); + for ordinal in 1..=RAW_ROWS { + let mut record = message(session_id, ordinal); + record.message_id = format!("{session_id}-message-{ordinal}"); + record.text = format!("{ordinal:04}:{}", "bounded retained context ".repeat(32)); + db.lcm_ingest_raw_message(storage_root, &record) + .await + .unwrap(); + } - let first = super::super::lcm_summary_convergence::run_summary_convergence_page(db.clone(), 1) - .await - .unwrap(); - assert_eq!(first.sessions.len(), 1); - assert_eq!( - first.sessions[0].disposition, - super::super::lcm_summary_convergence::LcmSummaryConvergenceDisposition::Preparing - ); - assert_eq!( - first.sessions[0].protection_rows_scanned, - tracedecay_lcm::LCM_SCAN_PAGE_ROWS as usize - ); - assert_eq!(first.sessions[0].compression_rows_scanned, 0); - assert!( - first.sessions[0].protection_bytes_scanned - <= tracedecay_lcm::LCM_SCAN_PAGE_MAX_BYTES as u64 - ); + let first = + super::super::lcm_summary_convergence::run_summary_convergence_page(db.clone(), 1) + .await + .unwrap(); + assert_eq!(first.sessions.len(), 1); + assert_eq!( + first.sessions[0].disposition, + super::super::lcm_summary_convergence::LcmSummaryConvergenceDisposition::Preparing + ); + assert_eq!( + first.sessions[0].protection_rows_scanned, + tracedecay_lcm::LCM_SCAN_PAGE_ROWS as usize + ); + assert_eq!(first.sessions[0].compression_rows_scanned, 0); + assert!( + first.sessions[0].protection_bytes_scanned + <= tracedecay_lcm::LCM_SCAN_PAGE_MAX_BYTES as u64 + ); - let second = super::super::lcm_summary_convergence::run_summary_convergence_page(db.clone(), 1) - .await - .unwrap(); - assert!( - second.sessions[0].compression_rows_scanned <= tracedecay_lcm::LCM_SCAN_PAGE_ROWS as usize - ); - assert!( - second.sessions[0].compression_bytes_scanned - <= tracedecay_lcm::LCM_SCAN_PAGE_MAX_BYTES as u64 - ); - assert_eq!( - db.lcm_status("cursor", Some(session_id)) - .await - .unwrap() - .raw_message_count, - RAW_ROWS - ); + let second = + super::super::lcm_summary_convergence::run_summary_convergence_page(db.clone(), 1) + .await + .unwrap(); + assert!( + second.sessions[0].compression_rows_scanned + <= tracedecay_lcm::LCM_SCAN_PAGE_ROWS as usize + ); + assert!( + second.sessions[0].compression_bytes_scanned + <= tracedecay_lcm::LCM_SCAN_PAGE_MAX_BYTES as u64 + ); + assert_eq!( + db.lcm_status("cursor", Some(session_id)) + .await + .unwrap() + .raw_message_count, + RAW_ROWS + ); + }); } #[cfg(unix)] @@ -2032,8 +2005,8 @@ async fn mega_session_convergence_bounds_protection_and_compression_pages() { fn retained_pages_never_reuse_unbound_session_wide_native_text() { run_with_test_env_lock(async { const RAW_ROWS: i64 = tracedecay_lcm::LCM_SCAN_PAGE_ROWS + 1; - let harness = RegisteredGlobalDbHarness::open("lcm-page-bound-summary").await; - let db = harness.registered.clone(); + let fixture = ProjectSummarizerFixture::open().await; + let db = fixture.db(); let storage_root = db.db_path().parent().unwrap(); let session_id = "page-bound-summary-session"; assert!(db.upsert_session(&session("cursor", session_id)).await); @@ -2045,26 +2018,23 @@ fn retained_pages_never_reuse_unbound_session_wide_native_text() { .unwrap(); } let native_text = "one native summary for the entire retained session"; - let metadata = canonical_envelope( - "cursor", - session_id, - "session-wide-native-summary", - None, - vec![serde_json::json!({ - "kind": "compaction", - "summary": native_text - })], + let native_summary = canonical_record_for_scope( + canonical_envelope( + "cursor", + session_id, + "session-wide-native-summary", + None, + (u64::try_from(RAW_ROWS + 1).unwrap(), 0), + vec![serde_json::json!({ + "kind": "compaction", + "summary": native_text + })], + ), + ObservationScopeV1::Project { + project_id: fixture.project_id.clone(), + }, ); - insert_summary_evidence( - (&db, "cursor"), - session_id, - "session-wide-native-summary", - RAW_ROWS + 1, - native_text, - "message", - &metadata, - ) - .await; + ingest_canonical(&db, session_id, &[], &[&native_summary]).await; let temporary = tempfile::tempdir().unwrap(); let cursor_bin = temporary.path().join("cursor-agent"); @@ -2080,10 +2050,9 @@ fn retained_pages_never_reuse_unbound_session_wide_native_text() { .unwrap(); use std::os::unix::fs::PermissionsExt as _; std::fs::set_permissions(&cursor_bin, std::fs::Permissions::from_mode(0o700)).unwrap(); - let cursor_bin_env = cursor_bin.to_string_lossy().into_owned(); + fixture.pin_cursor_agent(&cursor_bin); let workspace_env = temporary.path().to_string_lossy().into_owned(); let _env = TestEnvironment::set([ - ("TRACEDECAY_CURSOR_AGENT_BIN", cursor_bin_env.as_str()), ( "TRACEDECAY_CURSOR_SUMMARY_WORKSPACE", workspace_env.as_str(), @@ -2109,9 +2078,9 @@ fn retained_pages_never_reuse_unbound_session_wide_native_text() { let mut rows = snapshot .query( "SELECT summary_text - FROM lcm_summary_nodes + FROM session_summary_nodes WHERE provider = 'cursor' AND session_id = ?1 AND depth = 0 - ORDER BY created_at, node_id + ORDER BY created_at, summary_id LIMIT 2", params![session_id], ) @@ -2146,8 +2115,8 @@ fn retained_pages_never_reuse_unbound_session_wide_native_text() { #[test] fn protected_in_place_revision_stales_old_summary_before_reconvergence() { run_with_test_env_lock(async { - let harness = RegisteredGlobalDbHarness::open("lcm-revised-raw-summary").await; - let db = harness.registered.clone(); + let fixture = ProjectSummarizerFixture::open().await; + let db = fixture.db(); let storage_root = db.db_path().parent().unwrap(); let session_id = "revised-raw-summary-session"; assert!(db.upsert_session(&session("cursor", session_id)).await); @@ -2186,7 +2155,7 @@ fn protected_in_place_revision_stales_old_summary_before_reconvergence() { let snapshot = db.read_snapshot().await.unwrap(); let mut rows = snapshot .query( - "SELECT node_id FROM lcm_summary_nodes + "SELECT summary_id FROM session_summary_nodes WHERE provider = 'cursor' AND session_id = ?1", params![session_id], ) @@ -2209,10 +2178,9 @@ fn protected_in_place_revision_stales_old_summary_before_reconvergence() { .unwrap(); use std::os::unix::fs::PermissionsExt as _; std::fs::set_permissions(&cursor_bin, std::fs::Permissions::from_mode(0o700)).unwrap(); - let cursor_bin_env = cursor_bin.to_string_lossy().into_owned(); + fixture.pin_cursor_agent(&cursor_bin); let workspace_env = temporary.path().to_string_lossy().into_owned(); let _env = TestEnvironment::set([ - ("TRACEDECAY_CURSOR_AGENT_BIN", cursor_bin_env.as_str()), ( "TRACEDECAY_CURSOR_SUMMARY_WORKSPACE", workspace_env.as_str(), @@ -2300,8 +2268,8 @@ fn protected_in_place_revision_stales_old_summary_before_reconvergence() { #[test] fn disjoint_published_summary_revisions_both_reconverge_across_restart() { run_with_test_env_lock(async { - let harness = RegisteredGlobalDbHarness::open("lcm-disjoint-summary-revisions").await; - let db = harness.registered.clone(); + let fixture = ProjectSummarizerFixture::open().await; + let db = fixture.db(); let storage_root = db.db_path().parent().unwrap(); let session_id = "disjoint-summary-revisions"; assert!(db.upsert_session(&session("cursor", session_id)).await); @@ -2354,12 +2322,12 @@ fn disjoint_published_summary_revisions_both_reconverge_across_restart() { let snapshot = db.read_snapshot().await.unwrap(); let mut rows = snapshot .query( - "SELECT node.node_id, MIN(CAST(source.source_id AS INTEGER)) - FROM lcm_summary_nodes AS node - JOIN lcm_summary_sources AS source ON source.node_id = node.node_id + "SELECT node.summary_id, MIN(CAST(source.source_id AS INTEGER)) + FROM session_summary_nodes AS node + JOIN session_summary_sources AS source ON source.summary_id = node.summary_id WHERE node.provider = 'cursor' AND node.session_id = ?1 AND node.depth = 0 AND source.source_kind = 'raw_message' - GROUP BY node.node_id + GROUP BY node.summary_id ORDER BY MIN(CAST(source.source_id AS INTEGER)) LIMIT 2", params![session_id], @@ -2410,10 +2378,9 @@ fn disjoint_published_summary_revisions_both_reconverge_across_restart() { .unwrap(); use std::os::unix::fs::PermissionsExt as _; std::fs::set_permissions(&cursor_bin, std::fs::Permissions::from_mode(0o700)).unwrap(); - let cursor_bin_env = cursor_bin.to_string_lossy().into_owned(); + fixture.pin_cursor_agent(&cursor_bin); let workspace_env = temporary.path().to_string_lossy().into_owned(); let _env = TestEnvironment::set([ - ("TRACEDECAY_CURSOR_AGENT_BIN", cursor_bin_env.as_str()), ( "TRACEDECAY_CURSOR_SUMMARY_WORKSPACE", workspace_env.as_str(), @@ -2432,8 +2399,8 @@ fn disjoint_published_summary_revisions_both_reconverge_across_restart() { )); drop(db); - let harness = harness.restart().await; - let restarted = harness.registered.clone(); + let fixture = fixture.restart().await; + let restarted = fixture.db(); for _ in 0..8 { super::super::lcm_summary_convergence::run_summary_convergence_page( restarted.clone(), @@ -2485,8 +2452,8 @@ fn disjoint_published_summary_revisions_both_reconverge_across_restart() { ON availability.session_id = generation.session_id AND availability.generation = generation.generation AND availability.availability = 'available' - JOIN lcm_summary_sources AS source - ON source.node_id = availability.summary_id + JOIN session_summary_sources AS source + ON source.summary_id = availability.summary_id AND source.source_kind = 'raw_message' WHERE generation.session_id = ?1 AND generation.state = 'active' @@ -2520,8 +2487,8 @@ fn disjoint_published_summary_revisions_both_reconverge_across_restart() { #[test] fn retained_summary_rejects_a_role_revision_during_model_generation() { run_with_test_env_lock(async { - let harness = RegisteredGlobalDbHarness::open("lcm-summary-revision-barrier").await; - let db = harness.registered.clone(); + let fixture = ProjectSummarizerFixture::open().await; + let db = fixture.db(); let storage_root = db.db_path().parent().unwrap().to_path_buf(); let session_id = "summary-revision-barrier-session"; assert!(db.upsert_session(&session("cursor", session_id)).await); @@ -2548,10 +2515,9 @@ fn retained_summary_rejects_a_role_revision_during_model_generation() { .unwrap(); use std::os::unix::fs::PermissionsExt as _; std::fs::set_permissions(&cursor_bin, std::fs::Permissions::from_mode(0o700)).unwrap(); - let cursor_bin_env = cursor_bin.to_string_lossy().into_owned(); + fixture.pin_cursor_agent(&cursor_bin); let workspace_env = temporary.path().to_string_lossy().into_owned(); let _env = TestEnvironment::set([ - ("TRACEDECAY_CURSOR_AGENT_BIN", cursor_bin_env.as_str()), ( "TRACEDECAY_CURSOR_SUMMARY_WORKSPACE", workspace_env.as_str(), @@ -2585,7 +2551,7 @@ fn retained_summary_rejects_a_role_revision_during_model_generation() { let snapshot = db.read_snapshot().await.unwrap(); let mut rows = snapshot .query( - "SELECT summary_text FROM lcm_summary_nodes + "SELECT summary_text FROM session_summary_nodes WHERE provider = 'cursor' AND session_id = ?1", params![session_id], ) @@ -2765,30 +2731,12 @@ async fn large_byte_session_stops_each_retained_pass_at_the_existing_budget() { let mut record = message(session_id, ordinal); record.message_id = format!("{session_id}-message-{ordinal}"); record.text = format!("{ordinal:04}:{body}"); - transaction - .execute( - "INSERT INTO session_messages ( - provider, message_id, session_id, role, timestamp, ordinal, text - ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)", - params![ - record.provider.as_str(), - record.message_id.as_str(), - record.session_id.as_str(), - record.role.as_str(), - record.timestamp, - record.ordinal, - record.text.as_str(), - ], - ) - .await - .unwrap(); transaction .execute( "INSERT INTO lcm_raw_messages ( provider, message_id, session_id, role, ordinal, timestamp, - content, content_hash, storage_kind, snippet_text, index_text, - metadata_json - ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, '', ?2, 'inline', '', '', '{}')", + content, content_hash, storage_kind, metadata_json + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?2, 'inline', '{}')", params![ record.provider.as_str(), record.message_id.as_str(), @@ -2796,6 +2744,7 @@ async fn large_byte_session_stops_each_retained_pass_at_the_existing_budget() { record.role.as_str(), record.ordinal, record.timestamp, + record.text.as_str(), ], ) .await @@ -2848,9 +2797,11 @@ fn concurrent_raw_revision_cannot_be_overwritten_by_staged_protection() { let text = format!("source-a-{ordinal}:{}", "x".repeat(768 * 1024)); transaction .execute( - "INSERT INTO session_messages ( - provider, message_id, session_id, role, timestamp, ordinal, text, kind - ) VALUES ('cursor', ?1, ?2, 'tool', ?3, ?3, ?4, 'tool_result')", + "INSERT INTO lcm_raw_messages ( + provider, message_id, session_id, role, ordinal, timestamp, + content, content_hash, storage_kind, metadata_json, kind + ) VALUES ('cursor', ?1, ?2, 'tool', ?3, ?3, ?4, ?1, + 'inline', '{}', 'tool_result')", params![ format!("barrier-message-{ordinal}"), session_id, @@ -2860,18 +2811,6 @@ fn concurrent_raw_revision_cannot_be_overwritten_by_staged_protection() { ) .await .unwrap(); - transaction - .execute( - "INSERT INTO lcm_raw_messages ( - provider, message_id, session_id, role, ordinal, timestamp, - content, content_hash, storage_kind, snippet_text, index_text, - metadata_json - ) VALUES ('cursor', ?1, ?2, 'tool', ?3, ?3, '', ?1, - 'inline', '', '', '{}')", - params![format!("barrier-message-{ordinal}"), session_id, ordinal], - ) - .await - .unwrap(); } transaction.commit().await.unwrap(); @@ -2965,11 +2904,15 @@ fn concurrent_raw_revision_cannot_be_overwritten_by_staged_protection() { }); } +/// A canonical record at transcript `ordinal`, read from source bytes +/// `[position, position + 1)`. A source's records are projected in position +/// order from position 0, the contiguity its cursor requires. fn canonical_envelope( provider: &str, session_id: &str, message_id: &str, parent_message_id: Option<&str>, + (ordinal, position): (u64, u64), facts: Vec, ) -> Value { let mut relations = serde_json::json!({ @@ -2988,11 +2931,137 @@ fn canonical_envelope( "facts": facts, "evidence": { "ordering_domain": "file_bytes", - "range": {"start": 1, "end": 2} + "range": {"start": position, "end": position + 1}, + "native_sequence": ordinal } }) } +/// A canonical Claude or Cursor record as production stores it: the message +/// row is the observation's projection, and the envelope lives only in the +/// observation row. +struct CanonicalRecord { + observation: DurableObservationV1, + message: SessionMessageRecord, +} + +fn canonical_record(envelope: Value) -> CanonicalRecord { + canonical_record_for_scope(envelope, ObservationScopeV1::Profile) +} + +/// A canonical record whose observation is owned by `scope`; the scope must +/// match the shard the record is ingested into (profile-wide or one project). +fn canonical_record_for_scope(envelope: Value, scope: ObservationScopeV1) -> CanonicalRecord { + let typed: CanonicalObservationEnvelopeV1 = serde_json::from_value(envelope.clone()).unwrap(); + let record_id = typed.stable_record_id().as_str(); + let receipt = SanitizationReceiptV1::new( + SanitizationReceiptRefV1::new( + SanitizationReceiptId::new(format!("receipt.lcm-effects.{record_id}")).unwrap(), + ComponentVersion::new("sanitizer.lcm-effects-fixture.v1").unwrap(), + ) + .unwrap(), + SanitizerDispositionV1::Accepted, + SensitivityV1::NonSensitive, + Some(PayloadReferenceV1::for_payload(&envelope).unwrap()), + ) + .unwrap(); + let observation = DurableObservationV1::new( + ObservationIdentityMaterialV1::for_native_record( + ObservationSourceIdentityV1::for_provider( + typed.provider().clone(), + typed.relations().session_id().clone(), + ) + .unwrap(), + scope, + ObservationSourceGenerationV1::new(1).unwrap(), + typed.evidence().range(), + typed.evidence().ordering_domain(), + typed.stable_record_id().clone(), + ) + .unwrap(), + receipt, + RetentionClass::new("retention.lcm-effects-fixture").unwrap(), + envelope, + ) + .unwrap(); + let message = derive_canonical_projection(&observation) + .unwrap() + .messages() + .next() + .expect("canonical record projects a message") + .message() + .clone(); + CanonicalRecord { + observation, + message, + } +} + +/// Ingests `messages` followed by each record's projected row through the +/// transcript path, then persists and projects each record's observation so +/// the row's envelope authority exists exactly as capture leaves it. +async fn ingest_canonical( + db: &RegisteredGlobalDb, + session_id: &str, + messages: &[SessionMessageRecord], + records: &[&CanonicalRecord], +) { + let provider = records[0].message.provider.as_str(); + let mut batch = messages.to_vec(); + batch.extend(records.iter().map(|record| record.message.clone())); + assert!( + db.upsert_transcript_batch( + &session(provider, session_id), + &batch, + &format!("/tmp/{session_id}.jsonl"), + ParseOffset::default(), + ) + .await + ); + let store = db.observation_store(); + for record in records { + let observation = &record.observation; + let previous = store + .get_source_cursor(observation.source(), observation.scope()) + .await + .unwrap(); + let next = ObservationSourceCursorV1::for_ordering( + observation.source().clone(), + observation.scope().clone(), + observation.identity().generation(), + observation.identity().ordering_domain(), + observation.identity().position().end(), + ) + .unwrap(); + let write = ObservationWrite::new(observation.clone(), previous, next).unwrap(); + let generation = ProjectionGenerationId::new(format!( + "projection.lcm-effects.{}", + record.message.message_id + )) + .unwrap(); + let authorization = build_observation_resolution_authorization_v1( + write.observation(), + tracedecay_store::OBSERVATION_CAPTURE_AUTHORITY_V1, + ) + .unwrap(); + let anchor = build_observation_retrieval_anchor( + write.observation(), + generation.clone(), + UtcMicros(1), + authorization, + ) + .unwrap(); + store + .persist_observation(AnchoredObservationWrite::new(write, anchor, generation).unwrap()) + .await + .unwrap(); + store + .project_observation(observation.observation_id()) + .await + .unwrap(); + } +} + async fn insert_summary_evidence( source: (&RegisteredGlobalDb, &str), session_id: &str, @@ -3040,9 +3109,10 @@ async fn insert_summary_evidence( let transaction = db.begin_write_transaction().await.unwrap(); transaction .execute( - "INSERT INTO session_messages ( - provider, message_id, session_id, role, ordinal, text, kind, metadata_json - ) VALUES (?1, ?2, ?3, 'system', ?4, ?5, ?6, ?7)", + "INSERT INTO lcm_raw_messages ( + provider, message_id, session_id, role, ordinal, content, content_hash, + storage_kind, kind, metadata_json + ) VALUES (?1, ?2, ?3, 'system', ?4, ?5, ?2, 'inline', ?6, ?7)", tracedecay_runtime_core::db::engine::params![ provider, message_id, @@ -3095,8 +3165,8 @@ async fn ingest_codex_compaction_evidence( #[test] fn codex_and_cursor_daemon_adapters_commit_exact_authoritative_summaries() { run_with_test_env_lock(async { - let harness = RegisteredGlobalDbHarness::open("lcm-provider-summary-adapters").await; - let db = harness.registered.clone(); + let fixture = ProjectSummarizerFixture::open().await; + let db = fixture.db(); let temporary = tempfile::tempdir().unwrap(); let cursor_bin = temporary.path().join("cursor-agent"); let codex_bin = temporary.path().join("codex"); @@ -3125,17 +3195,17 @@ done for path in [&cursor_bin, &codex_bin] { std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o700)).unwrap(); } - let cursor_bin_env = cursor_bin.to_string_lossy().into_owned(); - let codex_bin_env = codex_bin.to_string_lossy().into_owned(); + fixture.pin(LcmSummarizerExecutablesV1 { + cursor_agent: LcmSummarizerExecutableV1::configured(cursor_bin.clone()).unwrap(), + codex: LcmSummarizerExecutableV1::configured(codex_bin.clone()).unwrap(), + }); let workspace_env = temporary.path().to_string_lossy().into_owned(); let env = TestEnvironment::set([ - ("TRACEDECAY_CURSOR_AGENT_BIN", cursor_bin_env.as_str()), ( "TRACEDECAY_CURSOR_SUMMARY_WORKSPACE", workspace_env.as_str(), ), ("TRACEDECAY_CURSOR_SUMMARY_TIMEOUT_SECS", "5"), - ("TRACEDECAY_CODEX_BIN", codex_bin_env.as_str()), ("TRACEDECAY_CODEX_SUMMARY_TIMEOUT_SECS", "5"), ]); @@ -3160,7 +3230,6 @@ done assert_eq!(response.status, "ok"); assert_eq!(response.summary_nodes_created, 1); assert_eq!(response.summary_nodes[0].summary_text, expected); - assert!(!response.fallback_used); assert_eq!( response.relation_projection_status, LcmRelationProjectionStatus::Applied diff --git a/crates/tracedecay-session-runtime/src/lcm_summarization.rs b/crates/tracedecay-session-runtime/src/lcm_summarization.rs index 85abcdee44..9c49ceb297 100644 --- a/crates/tracedecay-session-runtime/src/lcm_summarization.rs +++ b/crates/tracedecay-session-runtime/src/lcm_summarization.rs @@ -2,6 +2,7 @@ use std::time::Duration; use serde_json::Value; use tracedecay_domain::CanonicalObservationEnvelopeV1; +use tracedecay_domain::configuration::LcmSummarizerExecutablesV1; use tracedecay_global_db::RegisteredGlobalDb; use tracedecay_lcm::raw::{LcmPredecessorRangeState, predecessor_range_state}; @@ -10,14 +11,25 @@ use tracedecay_runtime_core::db::{ DatabaseEngineReadSnapshot, engine::{QueryExecutor, params}, }; +use tracedecay_store::StoreShardScopeV1; mod cursor_agent; mod provider_capabilities; +// The summarizer fixtures are `#!/bin/sh` executables found through a +// `:`-joined PATH. +#[cfg(all(test, unix))] +mod summarizer_executable_tests; +#[cfg(test)] +use provider_capabilities::{CODEX_APP_SERVER_UNCONFIGURED, CURSOR_AGENT_UNCONFIGURED}; use provider_capabilities::{ NativeSummaryCandidate, authoritative_summarizer, native_summary_recognizers, }; +/// Reason reported when a project shard has no published configuration pin, +/// so its summarizer binding cannot be read at all. +const SUMMARIZER_CONFIGURATION_UNAVAILABLE: &str = "summarizer_configuration_unavailable"; + pub(super) struct AuthoritativeSummary { pub(super) text: String, pub(super) route: String, @@ -46,11 +58,12 @@ pub(super) async fn resolve_authoritative_summary( { return Ok(summary); } - generate_provider_summary(provider, request, timeout).await + generate_provider_summary(database, provider, request, timeout).await } #[hotpath::measure(label = "daemon.lcm.summarize", future = true)] async fn generate_provider_summary( + database: &RegisteredGlobalDb, provider: &str, request: &LcmSummaryRequest, timeout: Duration, @@ -60,8 +73,43 @@ async fn generate_provider_summary( "authoritative_summarizer_unavailable", )); }; + // The binding is read before the summarizer runs, so an unconfigured or + // unreadable setting is a typed pending reason and never a spawn. + let executables = summarizer_executables(database)?; // Provider summarizers run on a blocking thread and need an owned request. - summarizer.summarize(request.clone(), timeout).await + summarizer + .summarize(request.clone(), timeout, &executables) + .await +} + +/// The summarizer executables configured for the shard `database` serves. +/// +/// Project shards read the daemon-published pin for their registered project. +/// Profile-wide shards have no project configuration authority, so every +/// provider is unconfigured there and their sessions stay pending. +fn summarizer_executables( + database: &RegisteredGlobalDb, +) -> Result { + match &database.binding().shard_id.scope { + StoreShardScopeV1::Project { project_id } + | StoreShardScopeV1::ProjectSessions { project_id } + | StoreShardScopeV1::Code { project_id, .. } => { + tracedecay_configuration::lcm_summarizer_executables_for_project(project_id).map_err( + |error| { + tracing::debug!( + project_id = project_id.as_str(), + %error, + "LCM summarizer binding is unavailable for this project shard" + ); + SummaryResolutionError::Unavailable(SUMMARIZER_CONFIGURATION_UNAVAILABLE) + }, + ) + } + StoreShardScopeV1::Profile + | StoreShardScopeV1::ProfileMemory + | StoreShardScopeV1::ProfileSessions + | StoreShardScopeV1::RemoteNode { .. } => Ok(LcmSummarizerExecutablesV1::unconfigured()), + } } /// Finds evidence that the host itself already produced an authoritative @@ -85,46 +133,44 @@ pub(super) async fn native_summary_evidence( .map_err(|error| LcmError::Db(error.to_string()))?; let (candidate_sql, candidate_params) = if let Some(required) = required_source { ( - "SELECT message.message_id, message.text, message.kind, message.metadata_json, - source_range.from_store_id, source_range.to_store_id, raw.store_id + format!( + "SELECT message.message_id, COALESCE(message.content, message.placeholder_text, ''), message.kind, + message.metadata_json, source_range.from_store_id, source_range.to_store_id, + message.store_id, {MESSAGE_ENVELOPE_COLUMN} FROM lcm_raw_predecessor_ranges AS source_range - JOIN lcm_raw_messages AS raw - ON raw.provider = source_range.provider - AND raw.message_id = source_range.message_id - AND raw.session_id = source_range.session_id - JOIN session_messages AS message - ON message.provider = raw.provider - AND message.message_id = raw.message_id - AND message.session_id = raw.session_id + JOIN lcm_raw_messages AS message + ON message.provider = source_range.provider + AND message.message_id = source_range.message_id + AND message.session_id = source_range.session_id WHERE source_range.provider = ?1 AND source_range.session_id = ?2 AND source_range.to_store_id = ?3 - AND length(trim(message.text)) > 0 - ORDER BY raw.store_id, raw.message_id - LIMIT 2", + AND length(trim(COALESCE(message.content, message.placeholder_text, ''))) > 0 + ORDER BY message.store_id, message.message_id + LIMIT 2" + ), params![provider, session_id, required.source_range.to_store_id,], ) } else { ( - "SELECT message.message_id, message.text, message.kind, message.metadata_json, - source_range.from_store_id, source_range.to_store_id, raw.store_id - FROM session_messages AS message - LEFT JOIN lcm_raw_messages AS raw - ON raw.provider = message.provider - AND raw.message_id = message.message_id - AND raw.session_id = message.session_id + format!( + "SELECT message.message_id, COALESCE(message.content, message.placeholder_text, ''), message.kind, + message.metadata_json, source_range.from_store_id, source_range.to_store_id, + message.store_id, {MESSAGE_ENVELOPE_COLUMN} + FROM lcm_raw_messages AS message LEFT JOIN lcm_raw_predecessor_ranges AS source_range ON source_range.provider = message.provider AND source_range.message_id = message.message_id AND source_range.session_id = message.session_id WHERE message.provider = ?1 AND message.session_id = ?2 - AND length(trim(message.text)) > 0 + AND length(trim(COALESCE(message.content, message.placeholder_text, ''))) > 0 ORDER BY message.ordinal DESC, message.message_id DESC - LIMIT 512", + LIMIT 512" + ), params![provider, session_id], ) }; let mut rows = snapshot - .query(candidate_sql, candidate_params) + .query(&candidate_sql, candidate_params) .await .map_err(|error| LcmError::Db(error.to_string()))?; let mut candidates = Vec::new(); @@ -148,25 +194,19 @@ pub(super) async fn native_summary_evidence( .map_err(|error| LcmError::Db(error.to_string()))?, row.get::>(6) .map_err(|error| LcmError::Db(error.to_string()))?, + row.get::>(7) + .map_err(|error| LcmError::Db(error.to_string()))?, )); } drop(rows); let recognizers = native_summary_recognizers(provider); let mut previous_native_store_id = None; let mut matched = None; - for (message_id, text, kind, metadata_json, range_from, range_to, store_id) in + for (message_id, text, kind, metadata_json, range_from, range_to, store_id, envelope_json) in candidates.into_iter().rev() { - let Some(metadata) = metadata_json - .as_deref() - .and_then(|metadata| serde_json::from_str::(metadata).ok()) - else { - continue; - }; - let envelope = match decode_canonical_observation_metadata(metadata.clone())? { - CanonicalObservationMetadata::Envelope(envelope) => Some(envelope), - CanonicalObservationMetadata::Unrecognized => None, - }; + let metadata = parse_message_metadata(metadata_json.as_deref()); + let envelope = decode_message_envelope(envelope_json.as_deref())?; let candidate = NativeSummaryCandidate { provider, message_id: &message_id, @@ -254,14 +294,14 @@ async fn native_store_is_recognized( ) -> Result { let mut rows = snapshot .query( - "SELECT message.message_id, message.text, message.kind, message.metadata_json - FROM lcm_raw_messages AS raw - JOIN session_messages AS message - ON message.provider = raw.provider - AND message.message_id = raw.message_id - AND message.session_id = raw.session_id - WHERE raw.provider = ?1 AND raw.session_id = ?2 AND raw.store_id = ?3 - LIMIT 1", + &format!( + "SELECT message.message_id, COALESCE(message.content, message.placeholder_text, ''), message.kind, + message.metadata_json, {MESSAGE_ENVELOPE_COLUMN} + FROM lcm_raw_messages AS message + WHERE message.provider = ?1 AND message.session_id = ?2 + AND message.store_id = ?3 + LIMIT 1" + ), params![provider, session_id, store_id], ) .await @@ -282,18 +322,17 @@ async fn native_store_is_recognized( let kind = row .get::>(2) .map_err(|error| LcmError::Db(error.to_string()))?; - let metadata = row - .get::>(3) - .map_err(|error| LcmError::Db(error.to_string()))? - .and_then(|metadata| serde_json::from_str::(&metadata).ok()); + let metadata = parse_message_metadata( + row.get::>(3) + .map_err(|error| LcmError::Db(error.to_string()))? + .as_deref(), + ); + let envelope = decode_message_envelope( + row.get::>(4) + .map_err(|error| LcmError::Db(error.to_string()))? + .as_deref(), + )?; drop(rows); - let Some(metadata) = metadata else { - return Ok(false); - }; - let envelope = match decode_canonical_observation_metadata(metadata.clone())? { - CanonicalObservationMetadata::Envelope(envelope) => Some(envelope), - CanonicalObservationMetadata::Unrecognized => None, - }; let candidate = NativeSummaryCandidate { provider, message_id: &message_id, @@ -310,34 +349,43 @@ async fn native_store_is_recognized( Ok(false) } -#[derive(Debug, PartialEq, Eq)] -pub(super) enum CanonicalObservationMetadata { - Envelope(Box), - Unrecognized, +/// The canonical envelope of the newest observation projected into the +/// `message` row. Message metadata does not embed it: the observation row is +/// its only copy. Rows no observation projected (direct transcript ingest) +/// select NULL. +pub(super) const MESSAGE_ENVELOPE_COLUMN: &str = + "(SELECT json_extract(observation.observation_json, '$.payload') + FROM observation_projection_provenance AS provenance + JOIN observations AS observation + ON observation.observation_id = provenance.observation_id + WHERE provenance.output_provider = message.provider + AND provenance.output_message_id = message.message_id + ORDER BY observation.sequence DESC + LIMIT 1)"; + +/// Stored message metadata, or `Null` when the row has none or it is not JSON. +fn parse_message_metadata(metadata: Option<&str>) -> Value { + metadata + .and_then(|metadata| serde_json::from_str(metadata).ok()) + .unwrap_or(Value::Null) } -/// Decode persisted observation metadata. -/// -/// A nested `canonical_envelope` is the persisted pairing authority: if that -/// key is present it must decode, and a broken envelope is a typed error -/// rather than a silent fallthrough onto the stripped metadata. Providers -/// that never persist the nested key still decode the whole object, and a -/// missing or non-envelope object is [`CanonicalObservationMetadata::Unrecognized`]. -pub(super) fn decode_canonical_observation_metadata( - mut metadata: Value, -) -> Result { - let Some(object) = metadata.as_object_mut() else { - return Ok(CanonicalObservationMetadata::Unrecognized); - }; - object.remove("ingest_protection"); - if let Some(envelope) = object.remove("canonical_envelope") { - return serde_json::from_value(envelope) - .map(|envelope| CanonicalObservationMetadata::Envelope(Box::new(envelope))) - .map_err(|error| LcmError::Db(format!("canonical_envelope decode failed: {error}"))); - } - Ok(serde_json::from_value(metadata) - .map(|envelope| CanonicalObservationMetadata::Envelope(Box::new(envelope))) - .unwrap_or(CanonicalObservationMetadata::Unrecognized)) +/// A projected row's observation payload is a validated canonical envelope, so +/// one that does not decode is corruption, never an unrecognized row. +pub(super) fn decode_message_envelope( + envelope: Option<&str>, +) -> Result>, LcmError> { + envelope + .map(|envelope| { + serde_json::from_str(envelope) + .map(Box::new) + .map_err(|error| { + LcmError::Db(format!( + "message observation envelope decode failed: {error}" + )) + }) + }) + .transpose() } async fn native_source_membership_is_exact( @@ -401,31 +449,23 @@ impl From for SummaryResolutionError { } #[cfg(test)] -mod decode_canonical_observation_metadata_tests { - use super::{CanonicalObservationMetadata, decode_canonical_observation_metadata}; - use serde_json::json; +mod decode_message_envelope_tests { + use super::decode_message_envelope; #[test] - fn nested_envelope_decode_failure_is_typed() { - let error = decode_canonical_observation_metadata(json!({ - "canonical_envelope": {"not": "an envelope"} - })) - .expect_err("broken nested envelope must not fall through"); + fn corrupt_observation_envelope_is_typed() { + let error = decode_message_envelope(Some(r#"{"not": "an envelope"}"#)) + .expect_err("a corrupt observation payload must not read as unrecognized"); assert!( error .to_string() - .contains("canonical_envelope decode failed"), + .contains("message observation envelope decode failed"), "typed envelope failure: {error}" ); } #[test] - fn missing_nested_envelope_stays_unrecognized() { - let decoded = decode_canonical_observation_metadata(json!({"source": "codex"})) - .expect("absent nested envelope is not a decode error"); - assert!(matches!( - decoded, - CanonicalObservationMetadata::Unrecognized - )); + fn unprojected_row_has_no_envelope() { + assert!(decode_message_envelope(None).unwrap().is_none()); } } diff --git a/crates/tracedecay-session-runtime/src/lcm_summarization/cursor_agent.rs b/crates/tracedecay-session-runtime/src/lcm_summarization/cursor_agent.rs index bd9a727dba..5980c8f5f0 100644 --- a/crates/tracedecay-session-runtime/src/lcm_summarization/cursor_agent.rs +++ b/crates/tracedecay-session-runtime/src/lcm_summarization/cursor_agent.rs @@ -1,43 +1,41 @@ //! Cursor CLI adapter used by daemon LCM compress to request an on-demand //! authoritative summary. Pressure-only hook compaction stays read-only and //! does not call this path. +//! +//! The executable is configuration data supplied by the caller from the +//! `lcm.summarizer_executables.v1` setting. This module never consults `PATH` +//! or the process environment to find it. use std::fmt::Write as _; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use std::time::{Duration, Instant, SystemTime}; use tracedecay_domain::errors::{Result, TraceDecayError}; use tracedecay_lcm::LcmSummaryRequest; -use tracedecay_sessions::runtime::codex_app_server::strip_reasoning_tags; +use tracedecay_sessions::runtime::hosts::codex_app_server::strip_reasoning_tags; const CURSOR_SUMMARY_CHILD_ENV: &str = "TRACEDECAY_CURSOR_SUMMARY_CHILD"; #[derive(Debug, Clone, PartialEq, Eq)] pub(super) struct CursorAgentSummaryConfig { - pub(super) cursor_agent_bin: String, + pub(super) cursor_agent_bin: PathBuf, pub(super) model: Option, pub(super) timeout: Duration, pub(super) workspace: Option, } -impl Default for CursorAgentSummaryConfig { - fn default() -> Self { - Self { - cursor_agent_bin: "cursor-agent".to_string(), +impl CursorAgentSummaryConfig { + /// Tuning for one configured executable. Model, timeout, and workspace + /// are operator tuning knobs read from the environment; the executable + /// itself is never resolved that way. + pub(super) fn for_executable(cursor_agent_bin: &Path) -> Self { + let mut config = Self { + cursor_agent_bin: cursor_agent_bin.to_path_buf(), model: None, timeout: Duration::from_secs(90), workspace: None, - } - } -} - -impl CursorAgentSummaryConfig { - pub(super) fn from_env() -> Self { - let mut config = Self::default(); - if let Some(bin) = non_empty_env("TRACEDECAY_CURSOR_AGENT_BIN") { - config.cursor_agent_bin = bin; - } + }; if let Some(model) = non_empty_env("TRACEDECAY_CURSOR_SUMMARY_MODEL") { config.model = Some(model); } @@ -96,7 +94,10 @@ pub(super) fn summarize_with_cursor_agent( .stderr(Stdio::piped()); let mut child = command.spawn().map_err(|err| TraceDecayError::Config { - message: format!("failed to start `{}`: {err}", config.cursor_agent_bin), + message: format!( + "failed to start `{}`: {err}", + config.cursor_agent_bin.display() + ), })?; let deadline = Instant::now() + config.timeout; loop { @@ -107,7 +108,10 @@ pub(super) fn summarize_with_cursor_agent( let _ = child.kill(); let _ = child.wait(); return Err(TraceDecayError::Config { - message: format!("timed out waiting for `{}`", config.cursor_agent_bin), + message: format!( + "timed out waiting for `{}`", + config.cursor_agent_bin.display() + ), }); } std::thread::sleep(Duration::from_millis(50)); @@ -121,12 +125,13 @@ pub(super) fn summarize_with_cursor_agent( message: if stderr.is_empty() { format!( "`{}` exited with status {}", - config.cursor_agent_bin, output.status + config.cursor_agent_bin.display(), + output.status ) } else { format!( "`{}` exited with status {}: {}", - config.cursor_agent_bin, + config.cursor_agent_bin.display(), output.status, stderr.chars().take(2000).collect::() ) diff --git a/crates/tracedecay-session-runtime/src/lcm_summarization/provider_capabilities.rs b/crates/tracedecay-session-runtime/src/lcm_summarization/provider_capabilities.rs index 29ac6182e7..0a6b3de69c 100644 --- a/crates/tracedecay-session-runtime/src/lcm_summarization/provider_capabilities.rs +++ b/crates/tracedecay-session-runtime/src/lcm_summarization/provider_capabilities.rs @@ -11,6 +11,7 @@ use std::pin::Pin; use std::time::Duration; use serde_json::Value; +use tracedecay_domain::configuration::{LcmSummarizerExecutableV1, LcmSummarizerExecutablesV1}; use tracedecay_domain::{CanonicalObservationEnvelopeV1, CanonicalObservationFactV1}; use tracedecay_lcm::{LcmError, LcmSummaryRequest}; @@ -20,9 +21,12 @@ use tracedecay_runtime_core::db::{ }; use super::cursor_agent::{CursorAgentSummaryConfig, summarize_with_cursor_agent}; -use super::{AuthoritativeSummary, LcmPredecessorRangeState, SummaryResolutionError}; +use super::{ + AuthoritativeSummary, LcmPredecessorRangeState, MESSAGE_ENVELOPE_COLUMN, + SummaryResolutionError, decode_message_envelope, +}; -/// One `session_messages` row offered to the recognizers. +/// One stored message row offered to the recognizers. /// /// Both views of the row are carried because providers disagree about what /// their native summary looks like on disk: Codex records raw provider @@ -208,28 +212,15 @@ async fn claude_summary_pair_is_exact( if summary_id.as_str() != summary_message_id { return Ok(false); } - // Production Claude ingest stores the boundary as `compact_boundary:{uuid}` - // while the summary parent remains the raw uuid. Accept either spelling. - let production_boundary_id = format!("compact_boundary:{}", boundary_id.as_str()); let mut rows = snapshot .query( - "SELECT metadata_json - FROM session_messages - WHERE provider = ?1 AND session_id = ?2 - AND message_id IN (?3, ?4) - AND kind IN ('compact_boundary', 'compaction') - ORDER BY - CASE WHEN json_extract(metadata_json, '$.canonical_envelope') IS NOT NULL - THEN 0 ELSE 1 END, - CASE WHEN message_id LIKE 'compact_boundary:%' THEN 0 ELSE 1 END, - message_id - LIMIT 1", - params![ - provider, - session_id.as_str(), - boundary_id.as_str(), - production_boundary_id.as_str(), - ], + &format!( + "SELECT {MESSAGE_ENVELOPE_COLUMN} + FROM lcm_raw_messages AS message + WHERE provider = ?1 AND session_id = ?2 AND message_id = ?3 + AND kind = 'compaction'" + ), + params![provider, session_id.as_str(), boundary_id.as_str()], ) .await .map_err(|error| LcmError::Db(error.to_string()))?; @@ -240,19 +231,14 @@ async fn claude_summary_pair_is_exact( else { return Ok(false); }; - let metadata = row - .get::>(0) - .map_err(|error| LcmError::Db(error.to_string()))?; - let Some(metadata) = metadata - .as_deref() - .and_then(|metadata| serde_json::from_str::(metadata).ok()) + let Some(boundary) = decode_message_envelope( + row.get::>(0) + .map_err(|error| LcmError::Db(error.to_string()))? + .as_deref(), + )? else { return Ok(false); }; - let boundary = match super::decode_canonical_observation_metadata(metadata)? { - super::CanonicalObservationMetadata::Envelope(envelope) => envelope, - super::CanonicalObservationMetadata::Unrecognized => return Ok(false), - }; let anchor = boundary.facts().iter().find_map(|fact| match fact { CanonicalObservationFactV1::Compaction { summary: Some(metadata), @@ -269,14 +255,23 @@ type AuthoritativeSummaryFuture = Pin> + Send>>; /// One provider's ability to be asked for a summary it has not already stored. +/// +/// `executables` is the configured binding for this shard. A summarizer whose +/// provider is unconfigured returns its typed `*_unconfigured` reason without +/// spawning anything; there is no ambient executable lookup behind it. pub(super) trait AuthoritativeSummarizerV1: Sync { fn summarize( &self, request: LcmSummaryRequest, timeout: Duration, + executables: &LcmSummarizerExecutablesV1, ) -> AuthoritativeSummaryFuture; } +/// Reason reported while a provider's summarizer executable is unconfigured. +pub(super) const CURSOR_AGENT_UNCONFIGURED: &str = "cursor_agent_unconfigured"; +pub(super) const CODEX_APP_SERVER_UNCONFIGURED: &str = "codex_app_server_unconfigured"; + struct CursorAgentSummarizerV1; impl AuthoritativeSummarizerV1 for CursorAgentSummarizerV1 { @@ -284,16 +279,24 @@ impl AuthoritativeSummarizerV1 for CursorAgentSummarizerV1 { &self, request: LcmSummaryRequest, timeout: Duration, + executables: &LcmSummarizerExecutablesV1, ) -> AuthoritativeSummaryFuture { - Box::pin(cursor_agent_summary(request, timeout)) + let executable = executables.cursor_agent.clone(); + Box::pin(cursor_agent_summary(request, timeout, executable)) } } async fn cursor_agent_summary( request: LcmSummaryRequest, timeout: Duration, + executable: LcmSummarizerExecutableV1, ) -> Result { - let mut config = CursorAgentSummaryConfig::from_env(); + let Some(cursor_agent_bin) = executable.canonical_path() else { + return Err(SummaryResolutionError::Unavailable( + CURSOR_AGENT_UNCONFIGURED, + )); + }; + let mut config = CursorAgentSummaryConfig::for_executable(cursor_agent_bin); config.timeout = config.timeout.min(timeout); let source_range = request.source_range.clone(); let text = tokio::task::spawn_blocking(move || summarize_with_cursor_agent(&request, &config)) @@ -314,21 +317,29 @@ impl AuthoritativeSummarizerV1 for CodexAppServerSummarizerV1 { &self, request: LcmSummaryRequest, timeout: Duration, + executables: &LcmSummarizerExecutablesV1, ) -> AuthoritativeSummaryFuture { - Box::pin(codex_app_server_summary(request, timeout)) + let executable = executables.codex.clone(); + Box::pin(codex_app_server_summary(request, timeout, executable)) } } async fn codex_app_server_summary( request: LcmSummaryRequest, timeout: Duration, + executable: LcmSummarizerExecutableV1, ) -> Result { + let Some(codex_bin) = executable.canonical_path() else { + return Err(SummaryResolutionError::Unavailable( + CODEX_APP_SERVER_UNCONFIGURED, + )); + }; let mut config = - tracedecay_sessions::runtime::codex_app_server::CodexAppServerSummaryConfig::from_env(); + tracedecay_sessions::runtime::hosts::codex_app_server::CodexAppServerSummaryConfig::for_executable(codex_bin); config.timeout = config.timeout.min(timeout); let source_range = request.source_range.clone(); let result = tokio::task::spawn_blocking(move || { - tracedecay_sessions::runtime::codex_app_server::summarize_with_codex_app_server( + tracedecay_sessions::runtime::hosts::codex_app_server::summarize_with_codex_app_server( &request, &config, ) }) diff --git a/crates/tracedecay-session-runtime/src/lcm_summarization/summarizer_executable_tests.rs b/crates/tracedecay-session-runtime/src/lcm_summarization/summarizer_executable_tests.rs new file mode 100644 index 0000000000..2eb1e2f867 --- /dev/null +++ b/crates/tracedecay-session-runtime/src/lcm_summarization/summarizer_executable_tests.rs @@ -0,0 +1,250 @@ +//! Guards that on-demand summarization launches a host CLI only through the +//! configured `lcm.summarizer_executables.v1` binding. +//! +//! A trap `cursor-agent`/`codex` sits first on `PATH` and records every launch. +//! With the isolated profile and no configured executable the request must +//! settle on the typed unconfigured reason and the trap must stay silent. + +use std::ffi::OsString; +use std::path::PathBuf; +use std::sync::{Mutex, MutexGuard}; +use std::time::Duration; + +use tracedecay_domain::ProjectId; +use tracedecay_domain::configuration::{LcmSummarizerExecutableV1, LcmSummarizerExecutablesV1}; +use tracedecay_global_db::tests::harness::{ + RegisteredGlobalDbHarness, RegisteredGlobalDbTestRuntime, +}; +use tracedecay_lcm::{LcmSummaryRequest, LcmSummarySourceMessage, LcmSummarySourceRange}; + +use super::{ + CODEX_APP_SERVER_UNCONFIGURED, CURSOR_AGENT_UNCONFIGURED, SUMMARIZER_CONFIGURATION_UNAVAILABLE, + SummaryResolutionError, generate_provider_summary, summarizer_executables, +}; + +/// `PATH` is process-wide, so the guards below take turns owning it. +static PATH_OWNER: Mutex<()> = Mutex::new(()); + +/// Puts trap executables first on `PATH` for the guard's lifetime. Each trap +/// appends its name to `launches` so a spawn is observable after the fact. +struct TrapPath { + previous: Option, + launches: PathBuf, + _directory: tempfile::TempDir, + _owner: MutexGuard<'static, ()>, +} + +impl TrapPath { + fn install(names: &[&str]) -> Self { + let owner = PATH_OWNER + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let directory = tempfile::tempdir().unwrap(); + let launches = directory.path().join("launches.log"); + for name in names { + let trap = directory.path().join(name); + std::fs::write( + &trap, + format!( + "#!/bin/sh\nprintf '%s\\n' \"$0\" >> '{}'\nexit 0\n", + launches.display() + ), + ) + .unwrap(); + let mut permissions = std::fs::metadata(&trap).unwrap().permissions(); + std::os::unix::fs::PermissionsExt::set_mode(&mut permissions, 0o700); + std::fs::set_permissions(&trap, permissions).unwrap(); + } + let previous = std::env::var_os("PATH"); + let mut path = directory.path().as_os_str().to_owned(); + if let Some(rest) = &previous { + path.push(":"); + path.push(rest); + } + // SAFETY: tests serialize process-environment access through the + // shared TraceDecay environment lock; the guard restores PATH on drop. + unsafe { std::env::set_var("PATH", &path) }; + Self { + previous, + launches, + _directory: directory, + _owner: owner, + } + } + + fn launches(&self) -> String { + std::fs::read_to_string(&self.launches).unwrap_or_default() + } +} + +impl Drop for TrapPath { + fn drop(&mut self) { + // SAFETY: see `install`. + unsafe { + match self.previous.take() { + Some(previous) => std::env::set_var("PATH", previous), + None => std::env::remove_var("PATH"), + } + } + } +} + +fn summary_request(provider: &str) -> LcmSummaryRequest { + LcmSummaryRequest { + provider: provider.to_owned(), + session_id: "guard-session".to_owned(), + focus_topic: None, + prompt: "summarize".to_owned(), + source_range: LcmSummarySourceRange { + from_store_id: 1, + to_store_id: 2, + }, + source_messages: vec![LcmSummarySourceMessage { + store_id: 1, + role: "user".to_owned(), + content: "hello".to_owned(), + }], + extraction_request: None, + } +} + +fn unavailable_reason(error: SummaryResolutionError) -> &'static str { + match error { + SummaryResolutionError::Unavailable(reason) => reason, + SummaryResolutionError::Storage(error) => { + panic!("expected a typed unavailable reason, got storage error {error}") + } + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn unconfigured_summarizers_report_typed_state_and_spawn_nothing() { + let trap = TrapPath::install(&["cursor-agent", "codex"]); + let harness = RegisteredGlobalDbHarness::open("lcm-summarizer-executable-guard").await; + let database = harness.registered.clone(); + + // A profile-sessions shard has no project configuration: every provider + // is unconfigured there by construction. + assert_eq!( + summarizer_executables(&database).ok(), + Some(LcmSummarizerExecutablesV1::unconfigured()) + ); + + let cursor = generate_provider_summary( + &database, + "cursor", + &summary_request("cursor"), + Duration::from_secs(5), + ) + .await + .err() + .map(unavailable_reason); + assert_eq!(cursor, Some(CURSOR_AGENT_UNCONFIGURED)); + + let codex = generate_provider_summary( + &database, + "codex", + &summary_request("codex"), + Duration::from_secs(5), + ) + .await + .err() + .map(unavailable_reason); + assert_eq!(codex, Some(CODEX_APP_SERVER_UNCONFIGURED)); + + assert_eq!( + trap.launches(), + "", + "an unconfigured summarizer must never reach a PATH-resolved host CLI" + ); + // The traps are reachable through PATH, so silence above is the setting + // refusing, not a missing binary. + let resolved = std::process::Command::new("cursor-agent").status().unwrap(); + assert!(resolved.success()); + assert!(trap.launches().contains("cursor-agent")); +} + +#[tokio::test(flavor = "multi_thread")] +async fn project_shard_without_a_published_pin_is_unavailable_not_ambient() { + let trap = TrapPath::install(&["cursor-agent"]); + let root = tempfile::tempdir().unwrap(); + let project_id = ProjectId::new("project.lcm-summarizer-guard".to_owned()).unwrap(); + let runtime = RegisteredGlobalDbTestRuntime::project( + root.path().join("profile"), + root.path().join("project"), + project_id, + ) + .await + .unwrap(); + let database = runtime.project_database_arc().unwrap(); + + let reason = generate_provider_summary( + &database, + "cursor", + &summary_request("cursor"), + Duration::from_secs(5), + ) + .await + .err() + .map(unavailable_reason); + assert_eq!(reason, Some(SUMMARIZER_CONFIGURATION_UNAVAILABLE)); + assert_eq!(trap.launches(), ""); +} + +#[tokio::test(flavor = "multi_thread")] +async fn configured_executable_is_launched_instead_of_the_path_binary() { + let trap = TrapPath::install(&["cursor-agent"]); + let root = tempfile::tempdir().unwrap(); + let project_root = root.path().join("project"); + let project_id = ProjectId::new("project.lcm-summarizer-configured".to_owned()).unwrap(); + let runtime = RegisteredGlobalDbTestRuntime::project( + root.path().join("profile"), + &project_root, + project_id.clone(), + ) + .await + .unwrap(); + let database = runtime.project_database_arc().unwrap(); + + let configured = root.path().join("bin").join("cursor-agent"); + std::fs::create_dir_all(configured.parent().unwrap()).unwrap(); + std::fs::write( + &configured, + "#!/bin/sh\nprintf '%s\\n' 'configured summary text'\n", + ) + .unwrap(); + let mut permissions = std::fs::metadata(&configured).unwrap().permissions(); + std::os::unix::fs::PermissionsExt::set_mode(&mut permissions, 0o700); + std::fs::set_permissions(&configured, permissions).unwrap(); + tracedecay_configuration::test_support::pin_lcm_summarizer_executables( + project_id, + &project_root, + LcmSummarizerExecutablesV1 { + cursor_agent: LcmSummarizerExecutableV1::configured(configured.clone()).unwrap(), + codex: LcmSummarizerExecutableV1::Unconfigured, + }, + ) + .unwrap(); + + let summary = generate_provider_summary( + &database, + "cursor", + &summary_request("cursor"), + Duration::from_secs(5), + ) + .await + .ok() + .map(|summary| (summary.text, summary.route)); + assert_eq!( + summary, + Some(( + "configured summary text".to_owned(), + "cursor_agent".to_owned() + )) + ); + assert_eq!( + trap.launches(), + "", + "the PATH binary must stay untouched while a configured executable exists" + ); +} diff --git a/crates/tracedecay-session-runtime/src/retained/lcm.rs b/crates/tracedecay-session-runtime/src/retained/lcm.rs index e94ac9eb6f..32a155533f 100644 --- a/crates/tracedecay-session-runtime/src/retained/lcm.rs +++ b/crates/tracedecay-session-runtime/src/retained/lcm.rs @@ -9,7 +9,7 @@ use tracedecay_contracts::retained_surfaces::{ LcmDoctorResultV1, LcmExpandQueryRequestV1, LcmExpandRequestV1, LcmGrepRequestV1, LcmLifecycleStatusV1, LcmLoadSessionRequestV1, LcmPayloadCoverageStateV1, LcmPayloadCoverageV1, LcmPayloadGcStatusV1, LcmPayloadStatusV1, LcmRedactionStatusV1, LcmStatusRequestV1, - LcmStatusResultV1, LcmStatusV1, LcmStoreStatusV1, LcmStoreTokenCoverageV1, LcmTemporalModeV1, + LcmStatusResultV1, LcmStatusV1, LcmStoreStatusV1, LcmStoreTokenCoverageV1, MessageRelationshipScopeV1, MessageTypeFilterV1, RetainedOutcomeStatusV1, RetainedSurfaceOperation, RetainedSurfaceResultV1, RetainedTimeFilterV1, RetrievalWorkerStatusV1, @@ -19,7 +19,7 @@ use tracedecay_contracts::{ RetainedLcmRequestV1, RetainedSurfaceExecutionContextV1, RetainedSurfaceExecutionErrorV1, RetainedSurfaceExecutionFutureV1, }; -use tracedecay_domain::{SessionId, TemporalModeV1, UtcMicros}; +use tracedecay_domain::SessionId; use tracedecay_lcm::LcmStatus; use tracedecay_lcm::types::LcmPayloadCoverageState; use tracedecay_session_memory::session::lcm::{ @@ -841,7 +841,6 @@ fn lcm_status(value: LcmStatus) -> LcmStatusV1 { redaction: LcmRedactionStatusV1 { enabled: value.redaction.enabled, lossy_records: value.redaction.lossy_records, - legacy_truncated_count: value.redaction.legacy_truncated_count, }, } } @@ -978,25 +977,6 @@ pub(super) fn unsigned_i64( .map_err(|_| RetainedSurfaceExecutionErrorV1::InvalidRequest) } -pub(super) fn temporal_mode( - mode: Option, - as_of: Option, - default: TemporalModeV1, -) -> Result { - match mode { - None => Ok(default), - Some(LcmTemporalModeV1::Current) => Ok(TemporalModeV1::Current), - Some(LcmTemporalModeV1::Evolution) => Ok(TemporalModeV1::Evolution), - Some(LcmTemporalModeV1::Forensic) => Ok(TemporalModeV1::Forensic), - Some(LcmTemporalModeV1::AsOf) => Ok(TemporalModeV1::AsOf { - cutoff: UtcMicros( - i64::try_from(as_of.ok_or(RetainedSurfaceExecutionErrorV1::InvalidRequest)?) - .map_err(|_| RetainedSurfaceExecutionErrorV1::InvalidRequest)?, - ), - }), - } -} - pub(super) fn relationship_scope(value: Option) -> SessionSearchScope { SessionSearchScope::from(value.unwrap_or(MessageRelationshipScopeV1::All)) } diff --git a/crates/tracedecay-session-runtime/src/retained/lcm/output.rs b/crates/tracedecay-session-runtime/src/retained/lcm/output.rs index 9c571b80d2..6234c2637c 100644 --- a/crates/tracedecay-session-runtime/src/retained/lcm/output.rs +++ b/crates/tracedecay-session-runtime/src/retained/lcm/output.rs @@ -99,8 +99,6 @@ pub(super) fn sliced_message( content_hash: None, storage_kind: LcmStorageKindV1::CanonicalOccurrence, payload_ref: None, - legacy_source: false, - legacy_truncated: false, metadata_json: result.message.metadata_json, } } @@ -267,7 +265,6 @@ pub(super) fn expansion(value: LcmExpandResponse) -> LcmExpansionV1 { .collect(), payload_ref: value.payload_ref, from_current_session: value.from_current_session, - externalized_note: value.externalized_note, source_pagination: value.source_pagination.map(|page| LcmSourcePaginationV1 { source_limit: page.source_limit, returned_sources: page.returned_sources, @@ -524,8 +521,6 @@ fn raw_message(value: LcmRawMessage) -> LcmRawMessageV1 { content_hash: value.content_hash, storage_kind: storage_kind(value.storage_kind), payload_ref: value.payload_ref, - legacy_source: value.legacy_source, - legacy_truncated: value.legacy_truncated, metadata_json: value.metadata_json, } } @@ -542,8 +537,6 @@ fn raw_message_metadata(value: LcmRawMessageMetadata) -> LcmRawMessageMetadataV1 content_hash: value.content_hash, storage_kind: storage_kind(value.storage_kind), payload_ref: value.payload_ref, - legacy_source: value.legacy_source, - legacy_truncated: value.legacy_truncated, metadata_json: value.metadata_json, } } diff --git a/crates/tracedecay-session-runtime/src/retained/lcm/retrieval.rs b/crates/tracedecay-session-runtime/src/retained/lcm/retrieval.rs index 7af90825ee..39e7c5cff0 100644 --- a/crates/tracedecay-session-runtime/src/retained/lcm/retrieval.rs +++ b/crates/tracedecay-session-runtime/src/retained/lcm/retrieval.rs @@ -30,7 +30,7 @@ use tracedecay_temporal_query::ranking::DiversityLimits; use super::output; use super::{ cursor, message_type, optional_provider, optional_usize, relationship_scope, required, - session_id, specific_provider, temporal_mode, time_filter, trimmed, unsigned_i64, + session_id, specific_provider, time_filter, trimmed, unsigned_i64, }; use crate::retained::session_retrieval_unavailable_detail; use crate::session_retrieval::{ @@ -99,11 +99,7 @@ pub(super) async fn execute_load_session( provider, "", cursor(request.cursor.as_deref())?, - temporal_mode( - request.temporal_mode, - request.as_of_micros, - TemporalModeV1::Forensic, - )?, + request.temporal_mode.unwrap_or(TemporalModeV1::Forensic), bounded_limit(request.limit, 50)?, default_context_budget(), SessionRetrievalScope::Session(session_id.clone()), @@ -202,11 +198,7 @@ pub(super) async fn execute_grep( provider, query_text, cursor(request.cursor.as_deref())?, - temporal_mode( - request.temporal_mode, - request.as_of_micros, - TemporalModeV1::Current, - )?, + request.temporal_mode.unwrap_or(TemporalModeV1::Current), bounded_limit(request.limit, 10)?, default_context_budget(), retrieval_scope, @@ -412,7 +404,7 @@ pub(super) async fn execute_describe( evidence_outcome( context, RetainedSurfaceOperation::LcmDescribe, - RetainedSurfaceResultV1::LcmDescribe(result), + RetainedSurfaceResultV1::LcmDescribe(Box::new(result)), ) } diff --git a/crates/tracedecay-session-runtime/src/retained/profile.rs b/crates/tracedecay-session-runtime/src/retained/profile.rs index 83bb9dfad6..eab37d6266 100644 --- a/crates/tracedecay-session-runtime/src/retained/profile.rs +++ b/crates/tracedecay-session-runtime/src/retained/profile.rs @@ -319,6 +319,9 @@ pub async fn execute_profile_retained_application( request_id, scope, outcome, + touched_files: Vec::new(), + code_graph: None, + analytics: None, }), Err(problem) => Err(application_problem_envelope( operation.result_contract().clone(), @@ -402,7 +405,7 @@ mod tests { use tracedecay_store::{ AnchoredObservationWrite, ObservationProjectionStore, ObservationStore, ObservationWrite, SessionTemporalSnapshotRequestV1, build_observation_resolution_authorization_v1, - build_observation_retrieval_anchor_v2, + build_observation_retrieval_anchor, }; use super::*; @@ -557,7 +560,7 @@ mod tests { tracedecay_store::OBSERVATION_CAPTURE_AUTHORITY_V1, ) .expect("resolution authorization"); - let anchor = build_observation_retrieval_anchor_v2( + let anchor = build_observation_retrieval_anchor( write.observation(), projection_generation.clone(), UtcMicros(1), @@ -594,7 +597,7 @@ mod tests { .materialize_pending_session_refresh_for_test(&session) .await .expect("materialize canonical temporal occurrence"); - let snapshot = database + let snapshot = tracedecay_session_temporal_store::SessionTemporalAccess::new(database) .freeze_session_temporal_snapshot_result(SessionTemporalSnapshotRequestV1::new(session)) .await .expect("activate canonical temporal snapshot"); @@ -1000,7 +1003,6 @@ mod tests { until: None, relation: None, limit: None, - format: None, }), RequestId::new("request.profile-retained-unsupported-sessions-for") .expect("request identity"), diff --git a/crates/tracedecay-session-runtime/src/retained/session.rs b/crates/tracedecay-session-runtime/src/retained/session.rs index 0ce0a8dad4..0325f45057 100644 --- a/crates/tracedecay-session-runtime/src/retained/session.rs +++ b/crates/tracedecay-session-runtime/src/retained/session.rs @@ -1,4 +1,4 @@ -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::sync::Arc; use std::time::Duration; @@ -30,10 +30,9 @@ use tracedecay_sessions::runtime::{ SessionRecord, SessionSearchScope, }; use tracedecay_temporal_query::context::ContextBudget; -use tracedecay_temporal_query::ports::{ - TemporalCandidateFilterV1, TemporalCandidatePopulationCount, -}; +use tracedecay_temporal_query::ports::TemporalCandidateFilterV1; use tracedecay_temporal_query::ranking::DiversityLimits; +use tracedecay_temporal_query::snapshot::TemporalCandidatePopulationCount; use super::session_refresh::{ MountedSessionRefreshAuthorityV1, RetainedSessionRefreshPortV1, @@ -469,7 +468,7 @@ struct MessageSearchInput { provider: ProviderScope, project_key: Option, include_subagents: bool, - catch_up: bool, + require_fresh: bool, cursor: Option, parent_session_id: Option, since: Option, @@ -507,14 +506,8 @@ impl MessageSearchInput { if workflow_agent.is_some() && workflow_run.is_none() { return Err(RetainedSurfaceExecutionErrorV1::InvalidRequest); } - let since = time_filter( - request.since.as_ref().or(request.time_from.as_ref()), - SearchTimeBound::Start, - )?; - let until = time_filter( - request.until.as_ref().or(request.time_to.as_ref()), - SearchTimeBound::End, - )?; + let since = time_filter(request.since.as_ref(), SearchTimeBound::Start)?; + let until = time_filter(request.until.as_ref(), SearchTimeBound::End)?; if since.zip(until).is_some_and(|(since, until)| since > until) { return Err(RetainedSurfaceExecutionErrorV1::InvalidRequest); } @@ -530,7 +523,7 @@ impl MessageSearchInput { provider, project_key: optional_string(request.project_key.as_deref())?, include_subagents, - catch_up: request.catch_up.unwrap_or(false), + require_fresh: request.require_fresh.unwrap_or(false), cursor: optional_string(request.cursor.as_deref())?, parent_session_id: optional_string(request.parent_session_id.as_deref())?, since, @@ -590,7 +583,7 @@ impl MessageSearchInput { .map(|query| { query .with_retrieval_scope(SessionRetrievalScope::AllSessionsInAuthorizedRoot) - .with_freshness_policy(if self.catch_up { + .with_freshness_policy(if self.require_fresh { SessionFreshnessPolicy::RequireFresh } else { SessionFreshnessPolicy::AllowStored @@ -628,7 +621,7 @@ impl MessageSearchInput { } => { result.status = RetainedOutcomeStatusV1::Stale; result.outcome = RetainedOutcomeStatusV1::Stale; - result.refresh_required = self.catch_up; + result.refresh_required = self.require_fresh; apply_temporal(&mut result, temporal, freshness); } SessionRetrievalServiceOutcome::Partial { @@ -640,7 +633,7 @@ impl MessageSearchInput { result.outcome = RetainedOutcomeStatusV1::Partial; result.omitted = Some(omitted); result.refresh_required = - self.catch_up && !matches!(freshness, SessionDataFreshness::Fresh); + self.require_fresh && !matches!(freshness, SessionDataFreshness::Fresh); apply_page(&mut result, page, freshness)?; } SessionRetrievalServiceOutcome::Redacted => { @@ -707,10 +700,7 @@ impl MessageSearchInput { fn base_result(&self, store_scope: SessionRetrievalStoreScope) -> MessageSearchResultV1 { MessageSearchResultV1 { - catch_up: self.catch_up, - catch_up_failures: Vec::new(), - catch_up_performed: false, - catch_up_provider: self.provider.response_label().to_owned(), + require_fresh: self.require_fresh, count: Some(0), goals: self.goals, include_subagents: self.include_subagents, @@ -737,14 +727,7 @@ impl MessageSearchInput { git_filter_applied: (!self.git.is_empty()).then_some(true), message: None, omitted: None, - project_scope: None, - registry_truncated: None, - roots: None, - searched_project_count: None, - selected_project_root: None, service_status: None, - skipped: None, - skipped_project_count: None, store_scope: Some( match store_scope { SessionRetrievalStoreScope::Project => "project", @@ -776,21 +759,9 @@ fn ensure_project_message_scope( ) -> Result<(), RetainedSurfaceExecutionErrorV1> { ensure_mounted_project_context(context, authorities)?; if request - .project_scope - .as_deref() - .is_some_and(|scope| scope != "project") - || request - .project_id - .as_deref() - .is_some_and(|project_id| project_id != authorities.project_id.as_str()) - || request - .project_path - .as_deref() - .is_some_and(|path| Path::new(path) != authorities.project_root.as_path()) - || request - .project_selector - .as_ref() - .is_some_and(|selector| selector.project_id != authorities.project_id) + .project_selector + .as_ref() + .is_some_and(|selector| selector.project_id != authorities.project_id) { return Err(RetainedSurfaceExecutionErrorV1::NotFoundOrNotAuthorized); } @@ -800,12 +771,11 @@ fn ensure_project_message_scope( fn ensure_profile_message_scope( request: &MessageSearchRequestV1, ) -> Result<(), RetainedSurfaceExecutionErrorV1> { - (request.project_scope.is_none() - && request.project_id.is_none() - && request.project_path.is_none() - && request.project_selector.is_none()) - .then_some(()) - .ok_or(RetainedSurfaceExecutionErrorV1::NotFoundOrNotAuthorized) + request + .project_selector + .is_none() + .then_some(()) + .ok_or(RetainedSurfaceExecutionErrorV1::NotFoundOrNotAuthorized) } fn ensure_session_refresh_identity( @@ -926,9 +896,6 @@ fn apply_page( page: SessionRetrievalPageView, freshness: SessionDataFreshness, ) -> Result<(), RetainedSurfaceExecutionErrorV1> { - result - .selected_project_root - .clone_from(&page.temporal.authorized_root); result.count = Some(page.results.len()); result.results = Some( page.results @@ -945,9 +912,6 @@ fn apply_temporal( temporal_view: SessionTemporalMetadataView, freshness: SessionDataFreshness, ) { - result - .selected_project_root - .clone_from(&temporal_view.authorized_root); result.temporal = Some(temporal(temporal_view, freshness)); } @@ -1084,7 +1048,7 @@ mod refusal_tests { ApplicationProblemKind, LegalAction, RetryDirective, retained_surface_execution_problem, }; use tracedecay_domain::CursorManifestLimitKindV1; - use tracedecay_temporal_query::ports::TemporalCandidatePopulationCount; + use tracedecay_temporal_query::snapshot::TemporalCandidatePopulationCount; use crate::session_retrieval::{ SessionRetrievalCoverageOmissionView, SessionTemporalMetadataView, diff --git a/crates/tracedecay-session-runtime/src/retained/session_refresh.rs b/crates/tracedecay-session-runtime/src/retained/session_refresh.rs index 99924db830..4860495828 100644 --- a/crates/tracedecay-session-runtime/src/retained/session_refresh.rs +++ b/crates/tracedecay-session-runtime/src/retained/session_refresh.rs @@ -10,18 +10,16 @@ use sha2::{Digest, Sha256}; use tracedecay_contracts::retained_surfaces::{ RetainedSurfaceExecutionErrorV1, SessionRefreshActionRequestV1, SessionRefreshActionV1, SessionRefreshGrainV1, SessionRefreshRequestV1, SessionRefreshScopeV1, - SessionRefreshTemporalModeV1, }; use tracedecay_contracts::{ CancellationContext, CancellationSignal, CapabilityGrantId, CapabilityGrantSnapshot, Deadline, DisclosureClass, RequestContext, retained_surface_application_operation, }; -use tracedecay_domain::{ - ManifestDigest, RetrievalGrainV1, SessionId, TemporalModeV1, UserProfileId, UtcMicros, -}; +use tracedecay_domain::{ManifestDigest, RetrievalGrainV1, SessionId, UserProfileId}; +use tracedecay_runtime_core::cancellation::CancellationToken; use tracedecay_session_memory::context::{ - BranchId, CancellationToken, CapabilityDigest, ConfigurationDigest, PolicyDigest, ProfileId, - RequestBudgets, ResolvedGitRoute, ResolvedSessionIdentity, SessionRootId, SessionStoreId, + BranchId, CapabilityDigest, ConfigurationDigest, PolicyDigest, ProfileId, RequestBudgets, + ResolvedGitRoute, ResolvedSessionIdentity, SessionRootId, SessionStoreId, session_application_grant_digest, }; use tracedecay_session_memory::session::{SessionRefreshTarget, SessionRequestBinding}; @@ -230,17 +228,6 @@ fn admitted_identity( fn admitted_target( request: &SessionRefreshActionRequestV1, ) -> Result { - let temporal_mode = match request.target.temporal_mode { - SessionRefreshTemporalModeV1::Current => TemporalModeV1::Current, - SessionRefreshTemporalModeV1::AsOf { cutoff } => TemporalModeV1::AsOf { - cutoff: UtcMicros( - i64::try_from(cutoff) - .map_err(|_| RetainedSurfaceExecutionErrorV1::InvalidRequest)?, - ), - }, - SessionRefreshTemporalModeV1::Evolution => TemporalModeV1::Evolution, - SessionRefreshTemporalModeV1::Forensic => TemporalModeV1::Forensic, - }; let grain = match request.target.grain { SessionRefreshGrainV1::Occurrence => RetrievalGrainV1::Occurrence, SessionRefreshGrainV1::LogicalMessage => RetrievalGrainV1::LogicalMessage, @@ -259,7 +246,7 @@ fn admitted_target( SessionId::new(request.session.id.clone()) .map_err(|_| RetainedSurfaceExecutionErrorV1::InvalidRequest)?, Some(request.source.scope.clone()), - temporal_mode, + request.target.temporal_mode, grain, frontier, ) diff --git a/crates/tracedecay-session-runtime/src/retained/wire.rs b/crates/tracedecay-session-runtime/src/retained/wire.rs index fc7601f5cf..b4fd87ac1b 100644 --- a/crates/tracedecay-session-runtime/src/retained/wire.rs +++ b/crates/tracedecay-session-runtime/src/retained/wire.rs @@ -1,7 +1,7 @@ //! One retained projection of domain coverage and hydration onto wire results. use tracedecay_contracts::retained_surfaces::{ - ClosedUtcIntervalV1, SessionCoverageIntervalV1, SessionCoverageModeV1, SessionCoverageReasonV1, + ClosedUtcIntervalV1, SessionCoverageIntervalV1, SessionCoverageReasonV1, SessionCoverageRequestV1, SessionCoverageStateV1, SessionSourceCoverageV1 as WireSourceCoverageV1, TemporalCoverageV1, TemporalWatermarksV1, ValidCoverageIntervalV1, @@ -9,8 +9,7 @@ use tracedecay_contracts::retained_surfaces::{ use tracedecay_domain::{ ClosedUtcIntervalV1 as DomainClosedUtcIntervalV1, SessionSourceCoverageIntervalV1, SessionSourceCoverageReasonV1, SessionSourceCoverageStateV1, SessionSourceCoverageV1, - TemporalCoverageCountsV1, TemporalModeV1, - ValidCoverageIntervalV1 as DomainValidCoverageIntervalV1, + TemporalCoverageCountsV1, ValidCoverageIntervalV1 as DomainValidCoverageIntervalV1, }; use crate::session_retrieval::SessionTemporalWatermarksView; @@ -43,7 +42,7 @@ pub(super) fn source_coverage(value: SessionSourceCoverageV1) -> WireSourceCover committed_frontier: value.committed_frontier().value(), target_watermark: value.target_watermark().value(), request: SessionCoverageRequestV1 { - mode: coverage_mode(value.request().mode()), + mode: value.request().mode(), }, covered_intervals: value .covered_intervals() @@ -81,15 +80,6 @@ fn closed_interval(value: DomainClosedUtcIntervalV1) -> ClosedUtcIntervalV1 { } } -const fn coverage_mode(value: TemporalModeV1) -> SessionCoverageModeV1 { - match value { - TemporalModeV1::Current => SessionCoverageModeV1::Current, - TemporalModeV1::AsOf { cutoff } => SessionCoverageModeV1::AsOf { cutoff: cutoff.0 }, - TemporalModeV1::Evolution => SessionCoverageModeV1::Evolution, - TemporalModeV1::Forensic => SessionCoverageModeV1::Forensic, - } -} - const fn coverage_state(value: SessionSourceCoverageStateV1) -> SessionCoverageStateV1 { match value { SessionSourceCoverageStateV1::Fresh => SessionCoverageStateV1::Fresh, diff --git a/crates/tracedecay-session-runtime/src/session_retrieval.rs b/crates/tracedecay-session-runtime/src/session_retrieval.rs index 2de0cd6590..fa86435a77 100644 --- a/crates/tracedecay-session-runtime/src/session_retrieval.rs +++ b/crates/tracedecay-session-runtime/src/session_retrieval.rs @@ -34,11 +34,13 @@ use tracedecay_session_temporal_store::{ }; use tracedecay_sessions::runtime::SessionMessageSearchResult; use tracedecay_temporal_query::context::{ContextError, TokenPolicy, VersionedTokenEstimator}; +use tracedecay_temporal_query::execution::ExecutionLimits; use tracedecay_temporal_query::hydration::HydrationError; -use tracedecay_temporal_query::ports::{ - ExecutionLimits, TemporalCandidatePopulationCount, TemporalExecutionSnapshot, TemporalPortError, -}; +use tracedecay_temporal_query::ports::TemporalPortError; use tracedecay_temporal_query::ranking::RankedCandidate; +use tracedecay_temporal_query::snapshot::{ + TemporalCandidatePopulationCount, TemporalExecutionSnapshot, +}; use tracedecay_temporal_query::{ TemporalHydratedResult, TemporalKernelError, TemporalKernelResult, }; diff --git a/crates/tracedecay-session-runtime/src/session_retrieval/contract.rs b/crates/tracedecay-session-runtime/src/session_retrieval/contract.rs index 1d83dca639..8b068fdecc 100644 --- a/crates/tracedecay-session-runtime/src/session_retrieval/contract.rs +++ b/crates/tracedecay-session-runtime/src/session_retrieval/contract.rs @@ -13,11 +13,10 @@ use tracedecay_domain::{ }; use tracedecay_lcm::contracts::LcmRetrievalOutcome; use tracedecay_temporal_query::ports::{ - TemporalCandidateFilterV1, TemporalCandidatePopulationCount, TemporalMessageTypeFilterV1, - TemporalSessionScopeFilterV1, + TemporalCandidateFilterV1, TemporalMessageTypeFilterV1, TemporalSessionScopeFilterV1, }; +use tracedecay_temporal_query::snapshot::TemporalCandidatePopulationCount; -use tracedecay_global_db::WorkflowScopeFilter; use tracedecay_lcm::{ LcmContentSlice, LcmDescribeResponse, LcmDescribeTarget, LcmExpandResponse, LcmExpandTarget, }; @@ -25,6 +24,7 @@ use tracedecay_session_memory::session::{ SessionDataFreshness, SessionRetrievalBudgetAccountingV1, SessionRetrievalBudgetStageV1, SessionTemporalQuery, }; +use tracedecay_sessions::WorkflowScopeFilter; use tracedecay_sessions::runtime::git_correlation::GitScopeFilter; use tracedecay_sessions::runtime::{ SessionMessageSearchResult, SessionMessageType, SessionSearchScope, SessionSearchTimeRange, diff --git a/crates/tracedecay-session-runtime/src/session_retrieval/lcm.rs b/crates/tracedecay-session-runtime/src/session_retrieval/lcm.rs index b6a315d8b1..077b65f36d 100644 --- a/crates/tracedecay-session-runtime/src/session_retrieval/lcm.rs +++ b/crates/tracedecay-session-runtime/src/session_retrieval/lcm.rs @@ -764,7 +764,7 @@ fn describe_hydration_state(state: HydrationStateV1) -> LcmDescribeServiceOutcom HydrationStateV1::Unauthorized => LcmDescribeServiceOutcome::Denied, HydrationStateV1::Available | HydrationStateV1::RetainedButUnavailable - | HydrationStateV1::UnverifiableLegacy => { + | HydrationStateV1::Unverifiable => { LcmDescribeServiceOutcome::Unavailable(SessionRetrievalUnavailable::without_worker( SessionRetrievalUnavailableReason::HydrationUnavailable, )) @@ -782,7 +782,7 @@ fn expand_hydration_state(state: HydrationStateV1) -> LcmExpandServiceOutcome { HydrationStateV1::Unauthorized => LcmExpandServiceOutcome::Denied, HydrationStateV1::Available | HydrationStateV1::RetainedButUnavailable - | HydrationStateV1::UnverifiableLegacy => { + | HydrationStateV1::Unverifiable => { LcmExpandServiceOutcome::Unavailable(SessionRetrievalUnavailable::without_worker( SessionRetrievalUnavailableReason::HydrationUnavailable, )) diff --git a/crates/tracedecay-session-runtime/src/session_retrieval/tests.rs b/crates/tracedecay-session-runtime/src/session_retrieval/tests.rs index 421551c090..ae3607cdc9 100644 --- a/crates/tracedecay-session-runtime/src/session_retrieval/tests.rs +++ b/crates/tracedecay-session-runtime/src/session_retrieval/tests.rs @@ -23,18 +23,18 @@ use tracedecay_domain::{ SessionId, TemporalModeV1, UtcMicros, derive_exact_observation_anchor_id, }; use tracedecay_lcm::contracts::{LcmDataFreshness, LcmRetrievalOutcome}; +use tracedecay_session_temporal_store::SessionTemporalAccess; use tracedecay_store::{ AnchoredObservationWrite, ObservationProjectionStore, ObservationStore, ObservationWrite, SessionRecord, SessionTemporalSnapshotRequestV1, build_observation_resolution_authorization_v1, - build_observation_retrieval_anchor_v2, + build_observation_retrieval_anchor, }; use tracedecay_temporal_query::context::{CompactContext, ContextBudget}; -use tracedecay_temporal_query::ports::{ - BindingDigest, KernelVersions, TemporalAuthorizedRoot, TemporalSnapshotRequest, - TemporalWatermarks, -}; +use tracedecay_temporal_query::execution::BindingDigest; +use tracedecay_temporal_query::ports::{TemporalAuthorizedRoot, TemporalSnapshotRequest}; use tracedecay_temporal_query::ranking::{DiversityLimits, RankedCandidate, RetrieverContribution}; use tracedecay_temporal_query::resolution::ValidatedAuthorization; +use tracedecay_temporal_query::snapshot::{KernelVersions, TemporalWatermarks}; use tracedecay_temporal_query::{TemporalHydratedResult, TemporalKernelResult}; use tracedecay_tool_catalog::{CapabilityId, SchemaId, UseCaseId}; @@ -269,7 +269,7 @@ async fn seed_real_page_fixture_in_session( tracedecay_store::OBSERVATION_CAPTURE_AUTHORITY_V1, ) .expect("resolution authorization"); - let anchor = build_observation_retrieval_anchor_v2( + let anchor = build_observation_retrieval_anchor( write.observation(), projection_generation.clone(), UtcMicros(1), @@ -298,7 +298,7 @@ async fn seed_real_page_fixture_in_session( ) .await .expect("materialize canonical temporal occurrence"); - database + SessionTemporalAccess::new(database) .freeze_session_temporal_snapshot_result(SessionTemporalSnapshotRequestV1::new( SessionId::new(session_id.clone()).expect("frozen session"), )) @@ -1604,11 +1604,7 @@ async fn project_retrieval_mounts_each_branch_of_a_shared_graph_store() { ) .unwrap(); let mut branches = tracedecay_runtime_core::branch_meta::BranchMeta::new("master"); - branches.add_branch( - "refs/heads/feature", - tracedecay_runtime_core::config::DB_FILENAME, - "master", - ); + branches.add_branch("refs/heads/feature", "master"); tracedecay_runtime_core::branch_meta::save_branch_meta(&layout.data_root, &branches).unwrap(); let registry = runtime.profile_database(); tracedecay_global_db::register_project_store(registry, &project, &layout) diff --git a/crates/tracedecay-session-runtime/src/session_sync.rs b/crates/tracedecay-session-runtime/src/session_sync.rs index 57125baf99..c9dd93dfdb 100644 --- a/crates/tracedecay-session-runtime/src/session_sync.rs +++ b/crates/tracedecay-session-runtime/src/session_sync.rs @@ -20,6 +20,7 @@ use tracedecay_domain::{BrainId, ProjectId, SessionId, UserProfileId, UtcMicros} use tracedecay_global_db::GlobalDbGitCorrelationStore; use tracedecay_global_db::RegisteredGlobalDbLeaseV1; use tracedecay_runtime_core::background_cpu::ProcessBackgroundCpuV1; +use tracedecay_session_temporal_store::SessionTemporalAccess; use tracedecay_sessions::admission::{SESSION_INGEST_DISABLED_REASON_V1, session_ingest_disabled}; use tracedecay_sessions::serving::{ SessionProjectionServingState, SessionProjectionServingStatusPort, @@ -761,10 +762,11 @@ impl DaemonSessionSyncService { .max_begin_requests_per_pass; let active_scan_slots = page_limit / 2; + let temporal = SessionTemporalAccess::new(&**database); let mut active_after: Option = None; loop { let page = { - let discovery = database.pending_session_temporal_refresh_page_result( + let discovery = temporal.pending_session_temporal_refresh_page_result( page_limit, active_scan_slots, active_after.as_ref(), diff --git a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/history.rs b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/history.rs index 69058bbe15..d80a46488c 100644 --- a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/history.rs +++ b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/history.rs @@ -78,7 +78,7 @@ pub struct ProjectSessionHistoricalIngestor { project_id: tracedecay_domain::ProjectId, transcript_source_home: Option, cancellation: ObservationCancellation, - codex_discovery: Arc, + codex_discovery: Arc, background_cpu: Arc, codex_consumer: String, codex_registered: AtomicBool, @@ -92,7 +92,7 @@ impl ProjectSessionHistoricalIngestor { project_root: PathBuf, project_id: tracedecay_domain::ProjectId, transcript_source_home: Option, - codex_discovery: Arc, + codex_discovery: Arc, background_cpu: Arc, ) -> Self { let source_home = transcript_source_home @@ -186,7 +186,7 @@ pub struct ProfileSessionHistoricalIngestor { profile_identity: Arc, transcript_source_home: Option, cancellation: ObservationCancellation, - codex_discovery: Arc, + codex_discovery: Arc, background_cpu: Arc, session_review: SessionReviewPort, codex_consumer: String, @@ -200,7 +200,7 @@ impl ProfileSessionHistoricalIngestor { registry_database: RegisteredGlobalDbLeaseV1, profile_identity: Arc, transcript_source_home: Option, - codex_discovery: Arc, + codex_discovery: Arc, background_cpu: Arc, session_review: SessionReviewPort, ) -> Self { diff --git a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/projector.rs b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/projector.rs index 632898ca24..64737ca81d 100644 --- a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/projector.rs +++ b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/projector.rs @@ -9,7 +9,9 @@ use tracedecay_store::{ }; use tracedecay_global_db::RegisteredGlobalDbLeaseV1; -use tracedecay_session_temporal_store::{SessionRefreshRecoveryV1, SessionRefreshRestartStateV1}; +use tracedecay_session_temporal_store::{ + SessionRefreshRecoveryV1, SessionRefreshRestartStateV1, SessionTemporalAccess, +}; #[derive(Clone, Copy, Debug)] pub struct SessionTemporalRefreshPolicy { @@ -109,7 +111,7 @@ impl SessionTemporalRefreshProjector for CanonicalSessionTemporalProjector { recovery: SessionRefreshRecoveryV1, ) -> SessionTemporalRefreshProjectionFuture<'a> { Box::pin(async move { - match database + match SessionTemporalAccess::new(&**database) .materialize_session_temporal_refresh_batch_result(&recovery) .await { diff --git a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/registry.rs b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/registry.rs index 8357b708ac..6755c6a404 100644 --- a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/registry.rs +++ b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/registry.rs @@ -142,7 +142,7 @@ pub struct SessionTemporalRefreshSchedulerRegistry { shutdown_guard: tokio::sync::Mutex<()>, project_lifecycle: tokio::sync::Mutex<()>, retired_project_owners: std::sync::Mutex>, - codex_discovery: Arc, + codex_discovery: Arc, /// The process background CPU authority mounted by /// [`Self::configure_codex_preparation_resources`]; retained so historical /// ingest compositions built through this registry inject the same @@ -163,7 +163,7 @@ impl Default for SessionTemporalRefreshSchedulerRegistry { project_lifecycle: tokio::sync::Mutex::new(()), retired_project_owners: std::sync::Mutex::new(HashSet::new()), codex_discovery: Arc::new( - tracedecay_sessions::runtime::codex::CodexDiscoveryHub::default(), + tracedecay_sessions::runtime::hosts::codex::CodexDiscoveryHub::default(), ), background_cpu: std::sync::OnceLock::new(), historical_ingest_admission: Arc::new(tokio::sync::Semaphore::new( @@ -245,7 +245,9 @@ impl SessionTemporalRefreshSchedulerRegistry { self.background_cpu.get().map(Arc::clone) } - pub fn codex_discovery(&self) -> Arc { + pub fn codex_discovery( + &self, + ) -> Arc { Arc::clone(&self.codex_discovery) } @@ -570,18 +572,6 @@ impl SessionTemporalRefreshSchedulerRegistry { } } - #[hotpath::skip] - pub async fn owns_project_database_paths( - &self, - database_paths: &HashSet, - ) -> bool { - self.project - .lock() - .await - .keys() - .any(|owner| database_paths.contains(&owner.graph_db_path)) - } - #[hotpath::skip] pub async fn cancel_historical_ingest(&self) { let project = self.project.lock().await; diff --git a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/wake.rs b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/wake.rs index aa243d5e05..d40bce8c2b 100644 --- a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/wake.rs +++ b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/wake.rs @@ -4,7 +4,7 @@ use std::sync::PoisonError; use std::sync::atomic::{AtomicBool, Ordering}; use tracedecay_domain::SessionId; use tracedecay_store::SessionRefreshBeginOrJoinRequestV1; -use tracedecay_temporal_query::ports::ExecutionControl; +use tracedecay_temporal_query::execution::ExecutionControl; use super::history::SessionHistoricalIngestOutcome; use tracedecay_session_temporal_store::SessionRefreshRecoveryV1; diff --git a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/worker.rs b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/worker.rs index e63f47dccf..a472b09e2a 100644 --- a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/worker.rs +++ b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/worker.rs @@ -27,7 +27,8 @@ use super::wake::{ use tracedecay_global_db::{RegisteredGlobalDb, RegisteredGlobalDbLeaseV1}; use tracedecay_runtime_core::db::engine::Error as EngineError; use tracedecay_session_temporal_store::{ - SessionRefreshRecoveryV1, SessionRefreshRestartStateV1, SessionTemporalStore, + SessionRefreshRecoveryV1, SessionRefreshRestartStateV1, SessionTemporalAccess, + SessionTemporalStore, }; const HISTORY_IDLE_RECHECK_INTERVAL: Duration = Duration::from_mins(1); @@ -723,7 +724,7 @@ pub async fn begin_admitted_session_refreshes( } let active_after = state.projection_discovery_after(); let active_scan_slots = state.projection_discovery_active_slots(limit); - let page = match database + let page = match SessionTemporalAccess::new(database) .pending_session_temporal_refresh_page_result( limit, active_scan_slots, diff --git a/crates/tracedecay-session-temporal-store/src/cursor_keys.rs b/crates/tracedecay-session-temporal-store/src/cursor_keys.rs index e8f02f91e6..97f6b49e8a 100644 --- a/crates/tracedecay-session-temporal-store/src/cursor_keys.rs +++ b/crates/tracedecay-session-temporal-store/src/cursor_keys.rs @@ -17,8 +17,8 @@ use tracedecay_runtime_core::db::{DatabaseEngineReadSnapshot, engine::params}; use tracedecay_temporal_query::cursor::{CURSOR_CLOCK_SKEW_MICROS, CURSOR_LIFETIME_MICROS}; use tracedecay_temporal_query::ports::{ CursorKeyError, CursorSignature, InMemoryCursorAuthenticator, SessionCursorAuthenticator, - TemporalExecutionSnapshot, }; +use tracedecay_temporal_query::snapshot::TemporalExecutionSnapshot; const LOAD_OPERATION: &str = "load snapshot cursor authentication key"; const PROVISION_OPERATION: &str = "provision active session cursor authentication key"; diff --git a/crates/tracedecay-session-temporal-store/src/doctor_health.rs b/crates/tracedecay-session-temporal-store/src/doctor_health.rs index 9bdf989170..de7e10aaa5 100644 --- a/crates/tracedecay-session-temporal-store/src/doctor_health.rs +++ b/crates/tracedecay-session-temporal-store/src/doctor_health.rs @@ -120,13 +120,8 @@ fn session_temporal_store_fingerprint( }) } -const REQUIRED_BASE_TABLES: &[&str] = &[ - "lcm_summary_nodes", - "lcm_summary_sources", - "observations", - "retrieval_anchors", - "sanitization_receipts", -]; +const REQUIRED_BASE_TABLES: &[&str] = + &["observations", "retrieval_anchors", "sanitization_receipts"]; const REQUIRED_FTS_SHADOW_TABLES: &[&str] = &[ "session_occurrences_fts_docsize", @@ -164,8 +159,11 @@ const REQUIRED_INDEXES: &[&str] = &[ "idx_session_refresh_operations_state", "idx_session_refresh_receipts_session", "idx_session_summary_availability_generation", + "idx_session_summary_nodes_depth_tokens", "idx_session_summary_nodes_root_created_order", "idx_session_summary_nodes_session_created", + "idx_session_summary_nodes_session_depth_time", + "idx_session_summary_sources_source", "idx_session_temporal_generations_one_active", "idx_session_temporal_generations_session_state", "idx_session_temporal_observation_effects_session", @@ -177,38 +175,34 @@ const REQUIRED_TRIGGERS: &[(&str, &str)] = &[ "session_occurrences_fts_insert_v1", "CREATE TRIGGER session_occurrences_fts_insert_v1 AFTER INSERT ON session_occurrences BEGIN - INSERT INTO session_occurrences_fts(rowid, index_text, snippet_text) - VALUES (NEW.rowid, NEW.index_text, NEW.snippet_text); + INSERT INTO session_occurrences_fts(rowid, index_text) + VALUES (NEW.rowid, NEW.index_text); END", ), ( "session_occurrences_fts_delete_v1", "CREATE TRIGGER session_occurrences_fts_delete_v1 AFTER DELETE ON session_occurrences BEGIN - INSERT INTO session_occurrences_fts( - session_occurrences_fts, rowid, index_text, snippet_text - ) - VALUES ('delete', OLD.rowid, OLD.index_text, OLD.snippet_text); + INSERT INTO session_occurrences_fts(session_occurrences_fts, rowid, index_text) + VALUES ('delete', OLD.rowid, OLD.index_text); END", ), ( "session_occurrences_fts_update_v1", "CREATE TRIGGER session_occurrences_fts_update_v1 - AFTER UPDATE OF index_text, snippet_text ON session_occurrences BEGIN - INSERT INTO session_occurrences_fts( - session_occurrences_fts, rowid, index_text, snippet_text - ) - VALUES ('delete', OLD.rowid, OLD.index_text, OLD.snippet_text); - INSERT INTO session_occurrences_fts(rowid, index_text, snippet_text) - VALUES (NEW.rowid, NEW.index_text, NEW.snippet_text); + AFTER UPDATE OF index_text ON session_occurrences BEGIN + INSERT INTO session_occurrences_fts(session_occurrences_fts, rowid, index_text) + VALUES ('delete', OLD.rowid, OLD.index_text); + INSERT INTO session_occurrences_fts(rowid, index_text) + VALUES (NEW.rowid, NEW.index_text); END", ), ( "session_summary_nodes_fts_insert_v1", "CREATE TRIGGER session_summary_nodes_fts_insert_v1 AFTER INSERT ON session_summary_nodes BEGIN - INSERT INTO session_summary_nodes_fts(rowid, summary_text, index_text) - VALUES (NEW.rowid, NEW.summary_text, NEW.index_text); + INSERT INTO session_summary_nodes_fts(rowid, summary_text) + VALUES (NEW.rowid, NEW.summary_text); END", ), ( @@ -216,21 +210,21 @@ const REQUIRED_TRIGGERS: &[(&str, &str)] = &[ "CREATE TRIGGER session_summary_nodes_fts_delete_v1 AFTER DELETE ON session_summary_nodes BEGIN INSERT INTO session_summary_nodes_fts( - session_summary_nodes_fts, rowid, summary_text, index_text + session_summary_nodes_fts, rowid, summary_text ) - VALUES ('delete', OLD.rowid, OLD.summary_text, OLD.index_text); + VALUES ('delete', OLD.rowid, OLD.summary_text); END", ), ( "session_summary_nodes_fts_update_v1", "CREATE TRIGGER session_summary_nodes_fts_update_v1 - AFTER UPDATE OF summary_text, index_text ON session_summary_nodes BEGIN + AFTER UPDATE OF summary_text ON session_summary_nodes BEGIN INSERT INTO session_summary_nodes_fts( - session_summary_nodes_fts, rowid, summary_text, index_text + session_summary_nodes_fts, rowid, summary_text ) - VALUES ('delete', OLD.rowid, OLD.summary_text, OLD.index_text); - INSERT INTO session_summary_nodes_fts(rowid, summary_text, index_text) - VALUES (NEW.rowid, NEW.summary_text, NEW.index_text); + VALUES ('delete', OLD.rowid, OLD.summary_text); + INSERT INTO session_summary_nodes_fts(rowid, summary_text) + VALUES (NEW.rowid, NEW.summary_text); END", ), ]; @@ -379,15 +373,14 @@ const STUCK_RECEIPT_TAIL: &str = "LEFT JOIN session_refresh_receipts AS receipt OR receipt.failure_code IS NOT candidate.failure_code ))"; -const COMPATIBILITY_DRIFT_TAIL: &str = "LEFT JOIN lcm_summary_nodes AS compatibility - ON compatibility.node_id = candidate.summary_id - WHERE compatibility.node_id IS NULL - OR candidate.publication_json IS NULL - OR json_extract(candidate.publication_json, '$.summary_hash') IS NULL - OR compatibility.session_id <> candidate.session_id - OR compatibility.summary_text <> candidate.summary_text - OR compatibility.summary_hash - <> json_extract(candidate.publication_json, '$.summary_hash')"; +/// The promoted summary columns and the frozen publication manifest describe +/// the same publication; a row where they disagree was not written by the +/// publication path. +const COMPATIBILITY_DRIFT_TAIL: &str = "WHERE candidate.publication_json IS NULL + OR json_extract(candidate.publication_json, '$.summary_hash') IS NOT candidate.summary_hash + OR json_extract(candidate.publication_json, '$.provider') IS NOT candidate.provider + OR json_extract(candidate.publication_json, '$.session_id') IS NOT candidate.session_id + OR json_extract(candidate.publication_json, '$.depth') IS NOT candidate.depth"; macro_rules! row_health_check { ( @@ -676,9 +669,9 @@ const CHECKS: &[HealthCheck] = &[ ), row_health_check!( CompatibilityDrift, - &["lcm_summary_nodes", "session_summary_nodes"], + &["session_summary_nodes"], "session_summary_nodes", - ", summary_id, session_id, summary_text, publication_json", + ", summary_id, session_id, provider, depth, summary_hash, publication_json", "COUNT(*)", COMPATIBILITY_DRIFT_TAIL ), diff --git a/crates/tracedecay-session-temporal-store/src/execution.rs b/crates/tracedecay-session-temporal-store/src/execution.rs index 3e64c9059f..a1f13a0db3 100644 --- a/crates/tracedecay-session-temporal-store/src/execution.rs +++ b/crates/tracedecay-session-temporal-store/src/execution.rs @@ -25,8 +25,9 @@ use tracedecay_query::retrieval::evidence_lanes::{ TaskSessionLaneEvidenceV1, }; use tracedecay_temporal_query::context::{ContextBudget, VersionedTokenEstimator}; +use tracedecay_temporal_query::execution::ExecutionLimits; use tracedecay_temporal_query::ports::{ - BudgetObservation, ExecutionLimits, ReadBudgetAccounting, TemporalSnapshotRequest, + BudgetObservation, ReadBudgetAccounting, TemporalSnapshotRequest, }; use tracedecay_temporal_query::ranking::DiversityLimits; use tracedecay_temporal_query::{TemporalKernelError, TemporalKernelResult}; @@ -129,7 +130,7 @@ impl AuthorizedTemporalExecutionRequest { pub fn into_kernel_request( self, - snapshot: tracedecay_temporal_query::ports::TemporalExecutionSnapshot, + snapshot: tracedecay_temporal_query::snapshot::TemporalExecutionSnapshot, ) -> tracedecay_temporal_query::TemporalKernelRequest { tracedecay_temporal_query::TemporalKernelRequest { snapshot, diff --git a/crates/tracedecay-session-temporal-store/src/expand.rs b/crates/tracedecay-session-temporal-store/src/expand.rs index 340f1f626b..a7a112d1ca 100644 --- a/crates/tracedecay-session-temporal-store/src/expand.rs +++ b/crates/tracedecay-session-temporal-store/src/expand.rs @@ -20,7 +20,8 @@ use tracedecay_store::{ SessionTemporalCapabilityV1, SessionTemporalRetrievalRequestV1, SessionTemporalSnapshotRequestV1, SessionTemporalSnapshotV1, }; -use tracedecay_temporal_query::ports::{ExecutionControl, TemporalPortError}; +use tracedecay_temporal_query::execution::ExecutionControl; +use tracedecay_temporal_query::ports::TemporalPortError; use super::query::{decode_generation_i64, now_micros, storage, storage_message}; use super::relations::{SessionRelationError, SummarySourceRef}; diff --git a/crates/tracedecay-session-temporal-store/src/hydration.rs b/crates/tracedecay-session-temporal-store/src/hydration.rs index 8b924d96c4..27b7cdae11 100644 --- a/crates/tracedecay-session-temporal-store/src/hydration.rs +++ b/crates/tracedecay-session-temporal-store/src/hydration.rs @@ -17,21 +17,19 @@ use zeroize::Zeroizing; use crate::relations::{ SessionRelationError, SessionRelationGraphStore, SessionRelationScope, SummarySourceVisitKind, }; -use crate::support::{ - derive_projection, record_hydration_emitted_bytes, record_hydration_verified_bytes, -}; +use crate::support::{record_hydration_emitted_bytes, record_hydration_verified_bytes}; use tracedecay_lcm::payload::{ PayloadStreamError, VerifiedPayloadStream, open_verified_payload_stream, }; use tracedecay_lcm::{LcmStorageKind, raw}; +use tracedecay_store::{derive_canonical_projection, message_metadata_with_envelope}; +use tracedecay_temporal_query::execution::ExecutionControl; use tracedecay_temporal_query::hydration::{ HydrationAuthorization, HydrationDenial, HydrationError, HydrationFuture, HydrationGrant, HydrationSink, TemporalHydrationPort, }; -use tracedecay_temporal_query::ports::{ - ExecutionControl, TemporalExecutionSnapshot, TemporalPortError, TemporalRetrievalScope, - TemporalSourceAccess, -}; +use tracedecay_temporal_query::ports::{TemporalPortError, TemporalRetrievalScope}; +use tracedecay_temporal_query::snapshot::{TemporalExecutionSnapshot, TemporalSourceAccess}; use super::operations::CanonicalPublicationManifest; use super::sql::TemporalSqlRead; @@ -42,6 +40,18 @@ const MAX_SUMMARY_SOURCE_RELATIONS: usize = 256; /// Window a file-backed payload is proven through; emission uses the grant's /// chunk size instead, so neither pass holds more than one window. const PAYLOAD_PROOF_WINDOW_BYTES: usize = 64 * 1024; +/// The occurrence's source observation envelope. Message rows store only the +/// metadata the envelope lacks, so the record's full metadata joins it back. +/// Stored message metadata without the raw authority's ingest-protection +/// receipts, which are storage bookkeeping rather than message metadata. +const SERVED_MESSAGE_METADATA_COLUMN: &str = "CASE WHEN json_valid(message.metadata_json) + THEN NULLIF(json_remove(message.metadata_json, '$.ingest_protection'), '{}') + ELSE message.metadata_json END"; + +const OCCURRENCE_ENVELOPE_COLUMN: &str = + "(SELECT json_extract(observation.observation_json, '$.payload') + FROM observations AS observation + WHERE observation.observation_id = occurrence.source_observation_id)"; mod external; use external::resolve_external_manifest; @@ -497,16 +507,18 @@ pub(super) async fn session_message_from_hydrated_bytes( return Err(HydrationError::Unavailable); } read.query( - "SELECT occurrence.message_id, occurrence.role, + &format!( + "SELECT occurrence.message_id, occurrence.role, occurrence.projection_output_ordinal, source.provider, occurrence.session_id, message.timestamp, message.kind, message.model, message.tool_names, message.source_path, message.source_offset, - message.metadata_json, message.role, message.session_id + {SERVED_MESSAGE_METADATA_COLUMN}, message.role, message.session_id, + {OCCURRENCE_ENVELOPE_COLUMN} FROM session_occurrences AS occurrence JOIN sessions AS source ON source.session_id = occurrence.session_id - LEFT JOIN session_messages AS message + LEFT JOIN lcm_raw_messages AS message ON message.provider = source.provider AND message.message_id = occurrence.message_id AND message.session_id = occurrence.session_id @@ -516,7 +528,8 @@ pub(super) async fn session_message_from_hydrated_bytes( AND source.project_key = ?4 AND source.provider = ?5 ORDER BY occurrence.occurrence_id - LIMIT 2", + LIMIT 2" + ), params![ session_id.as_str(), generation, @@ -529,12 +542,14 @@ pub(super) async fn session_message_from_hydrated_bytes( } TemporalRetrievalScope::AllSessionsInAuthorizedRoot => { read.query( - "SELECT occurrence.message_id, occurrence.role, + &format!( + "SELECT occurrence.message_id, occurrence.role, occurrence.projection_output_ordinal, source.provider, occurrence.session_id, message.timestamp, message.kind, message.model, message.tool_names, message.source_path, message.source_offset, - message.metadata_json, message.role, message.session_id + {SERVED_MESSAGE_METADATA_COLUMN}, message.role, message.session_id, + {OCCURRENCE_ENVELOPE_COLUMN} FROM session_occurrences AS occurrence JOIN session_temporal_generations AS generation ON generation.session_id = occurrence.session_id @@ -542,7 +557,7 @@ pub(super) async fn session_message_from_hydrated_bytes( AND generation.state = 'active' JOIN sessions AS source ON source.session_id = occurrence.session_id - LEFT JOIN session_messages AS message + LEFT JOIN lcm_raw_messages AS message ON message.provider = source.provider AND message.message_id = occurrence.message_id AND message.session_id = occurrence.session_id @@ -551,7 +566,8 @@ pub(super) async fn session_message_from_hydrated_bytes( AND occurrence.session_id = ?3 AND source.provider = ?4 ORDER BY occurrence.session_id, occurrence.occurrence_id - LIMIT 2", + LIMIT 2" + ), params![ anchor_id.as_str(), project_key, @@ -579,9 +595,20 @@ pub(super) async fn session_message_from_hydrated_bytes( let tool_names = row.get(8).ok(); let source_path = row.get(9).ok(); let source_offset = row.get(10).ok(); - let metadata_json = row.get(11).ok(); + let stored_metadata: Option = row.get(11).ok(); let compatibility_role: Option = row.get(12).ok(); let compatibility_session: Option = row.get(13).ok(); + let envelope: Option = row.get(14).ok(); + let metadata_json = match (&compatibility_role, envelope) { + (Some(_), Some(envelope)) => Some( + message_metadata_with_envelope( + stored_metadata.as_deref(), + &serde_json::from_str(&envelope).map_err(hydration_failure)?, + ) + .map_err(hydration_failure)?, + ), + _ => stored_metadata, + }; if compatibility_role .as_deref() .is_some_and(|compatibility_role| compatibility_role != role) @@ -615,7 +642,7 @@ fn canonical_projected_message( output_ordinal: i64, ) -> Option { let output_ordinal = u32::try_from(output_ordinal).ok()?; - let projection = derive_projection(observation).ok()?; + let projection = derive_canonical_projection(observation).ok()?; projection .messages() .find(|output| { @@ -875,7 +902,7 @@ async fn resolve_current( Ok(anchor) => anchor, Err(_) => { return Ok(HydrationResolution::Unavailable( - HydrationStateV1::UnverifiableLegacy, + HydrationStateV1::Unverifiable, )); } }; @@ -884,7 +911,7 @@ async fn resolve_current( || serde_json::to_string(anchor.owner()).ok().as_deref() != Some(owner_json.as_str()) { return Ok(HydrationResolution::Unavailable( - HydrationStateV1::UnverifiableLegacy, + HydrationStateV1::Unverifiable, )); } if anchor.authorization().validate().is_err() @@ -932,7 +959,7 @@ async fn resolve_current( return Ok(resolution); } Ok(HydrationResolution::Unavailable( - HydrationStateV1::UnverifiableLegacy, + HydrationStateV1::Unverifiable, )) } @@ -1021,7 +1048,7 @@ async fn resolve_occurrence( .any(|observation_id| observation_id.as_str() == source_observation_id) { return Ok(Some(HydrationResolution::Unavailable( - HydrationStateV1::UnverifiableLegacy, + HydrationStateV1::Unverifiable, ))); } if let Some(state) = participant_access_state(snapshot, &session_id, &provider) { @@ -1134,14 +1161,14 @@ async fn resolve_summary( } if publication_json.is_empty() { return Ok(Some(HydrationResolution::Unavailable( - HydrationStateV1::UnverifiableLegacy, + HydrationStateV1::Unverifiable, ))); } let manifest: CanonicalPublicationManifest = match serde_json::from_str(&publication_json) { Ok(manifest) => manifest, Err(_) => { return Ok(Some(HydrationResolution::Unavailable( - HydrationStateV1::UnverifiableLegacy, + HydrationStateV1::Unverifiable, ))); } }; @@ -1376,7 +1403,6 @@ fn source_access_hydration_state(access: TemporalSourceAccess) -> Option Some(HydrationStateV1::RetentionExpired), TemporalSourceAccess::Deleted => Some(HydrationStateV1::Deleted), TemporalSourceAccess::Redacted => Some(HydrationStateV1::Redacted), - TemporalSourceAccess::LegacyUnauthorized => Some(HydrationStateV1::Unauthorized), } } @@ -1458,16 +1484,17 @@ mod tests { }; use tracedecay_store::{ AnchoredObservationWrite, ObservationStore, ObservationWrite, - build_observation_resolution_authorization_v1, build_observation_retrieval_anchor_v2, + build_observation_resolution_authorization_v1, build_observation_retrieval_anchor, }; use super::*; use tracedecay_global_db::tests::harness::{HostAdmissionScope, HostAdmissionTestRuntimeV1}; + use tracedecay_temporal_query::execution::{BindingDigest, ExecutionLimits}; use tracedecay_temporal_query::ports::{ - BindingDigest, ExecutionLimits, KernelVersions, TemporalAuthorizedRoot, TemporalPortError, - TemporalSnapshotRequest, TemporalWatermarks, + TemporalAuthorizedRoot, TemporalPortError, TemporalSnapshotRequest, }; use tracedecay_temporal_query::resolution::ValidatedAuthorization; + use tracedecay_temporal_query::snapshot::{KernelVersions, TemporalWatermarks}; struct RegisteredHydrationRead { read: DatabaseEngineReadSnapshot, @@ -1628,11 +1655,10 @@ mod tests { session_id, generation, occurrence_id, source_observation_id, source_provider, projection_output_ordinal, retrieval_anchor_id, message_id, role, knowledge_at, valid_time_json, evidence_json, - sanitized_content_digest, sanitized_content_bytes, - snippet_text, index_text + sanitized_content_digest, sanitized_content_bytes, index_text ) VALUES ( ?1, 1, 'occurrence-1', ?2, ?3, 0, ?4, ?5, - 'assistant', 1, '{\"kind\":\"unknown\"}', '{}', ?6, ?7, ?8, ?8 + 'assistant', 1, '{\"kind\":\"unknown\"}', '{}', ?6, ?7, ?8 )", params![ session_id, @@ -1726,11 +1752,10 @@ mod tests { &writer, "INSERT INTO lcm_raw_messages ( provider, message_id, session_id, role, ordinal, timestamp, - content, content_hash, storage_kind, payload_ref, - snippet_text, index_text, legacy_source, legacy_truncated + content, content_hash, storage_kind, payload_ref ) VALUES ( ?1, 'message-1', 'session-2', 'assistant', 1, 1, - ?2, ?3, 'inline', NULL, ?2, ?2, 0, 0 + ?2, ?3, 'inline', NULL )", params![ provider, @@ -1746,11 +1771,10 @@ mod tests { session_id, generation, occurrence_id, source_observation_id, source_provider, projection_output_ordinal, retrieval_anchor_id, message_id, role, knowledge_at, valid_time_json, evidence_json, - sanitized_content_digest, sanitized_content_bytes, - snippet_text, index_text + sanitized_content_digest, sanitized_content_bytes, index_text ) VALUES ( 'session-2', 1, 'occurrence-1', ?1, ?2, 0, ?3, 'message-1', - 'assistant', 1, '{\"kind\":\"unknown\"}', '{}', ?4, ?5, ?6, ?6 + 'assistant', 1, '{\"kind\":\"unknown\"}', '{}', ?4, ?5, ?6 )", params![ observation.observation_id().as_str(), @@ -1842,10 +1866,10 @@ mod tests { "INSERT INTO lcm_raw_messages ( provider, message_id, session_id, role, ordinal, timestamp, content, content_hash, storage_kind, payload_ref, - snippet_text, index_text, legacy_source, legacy_truncated + placeholder_text ) VALUES ( ?1, 'message-1', 'session-1', 'assistant', 1, 1, - NULL, ?2, 'external', ?3, ?4, ?4, 0, 0 + NULL, ?2, 'external', ?3, ?4 )", params![ provider, @@ -1892,11 +1916,12 @@ mod tests { Executor::execute( &writer, "INSERT INTO session_summary_nodes ( - summary_id, session_id, summary_anchor_id, summary_text, - index_text, source_horizon_json, publication_json, created_at + summary_id, session_id, provider, conversation_id, depth, + summary_anchor_id, summary_text, summary_hash, summary_token_count, + source_token_count, source_horizon_json, publication_json, created_at ) VALUES ( - 'summary-authority', 'session-1', ?1, 'authority', - 'authority', '{}', ?2, 1 + 'summary-authority', 'session-1', 'test', 'session-1', 0, ?1, + 'authority', 'hash', 1, 1, '{}', ?2, 1 )", params![authority_anchor.anchor_id().as_str(), authority_publication], ) @@ -1921,13 +1946,11 @@ mod tests { session_id, generation, occurrence_id, source_observation_id, source_provider, projection_output_ordinal, retrieval_anchor_id, message_id, role, knowledge_at, valid_time_json, evidence_json, - sanitized_content_digest, sanitized_content_bytes, - snippet_text, index_text + sanitized_content_digest, sanitized_content_bytes, index_text ) VALUES ( 'session-1', 1, 'occurrence-1', ?1, ?2, 0, ?3, 'message-1', 'assistant', 1, '{\"kind\":\"unknown\"}', '{}', - ?4, ?5, - 'non-empty occurrence payload', 'non-empty occurrence payload' + ?4, ?5, 'non-empty occurrence payload' )", { let canonical = @@ -1978,9 +2001,11 @@ mod tests { Executor::execute( &writer, "INSERT INTO session_summary_nodes ( - summary_id, session_id, summary_anchor_id, summary_text, - index_text, source_horizon_json, publication_json, created_at - ) VALUES ('summary-1', 'session-1', ?1, ?2, ?2, '{}', ?3, 1)", + summary_id, session_id, provider, conversation_id, depth, + summary_anchor_id, summary_text, summary_hash, summary_token_count, + source_token_count, source_horizon_json, publication_json, created_at + ) VALUES ('summary-1', 'session-1', 'test', 'session-1', 0, ?1, ?2, 'hash', + 1, 1, '{}', ?3, 1)", params![ summary_anchor.anchor_id().as_str(), summary_payload, @@ -2413,7 +2438,7 @@ mod tests { let authorization = build_observation_resolution_authorization_v1(write.observation(), "snapshot-test") .expect("authorization"); - let anchor = build_observation_retrieval_anchor_v2( + let anchor = build_observation_retrieval_anchor( write.observation(), projection.clone(), UtcMicros(1), @@ -2538,7 +2563,7 @@ mod tests { matches!( authorization, Ok(HydrationAuthorization::Denied(ref denial)) - if denial.state() == HydrationStateV1::UnverifiableLegacy + if denial.state() == HydrationStateV1::Unverifiable ), "{authorization:?}" ); @@ -2572,12 +2597,10 @@ mod tests { .expect("registered profile writer"), "INSERT INTO lcm_raw_messages ( provider, message_id, session_id, role, ordinal, timestamp, - content, content_hash, storage_kind, payload_ref, - snippet_text, index_text, legacy_source, legacy_truncated + content, content_hash, storage_kind, payload_ref ) VALUES ( ?1, 'message-1', 'session-1', 'assistant', 1, 1, - 'raw-content-canary', 'invalid-content-hash', 'inline', NULL, - 'raw-content-canary', 'raw-content-canary', 0, 0 + 'raw-content-canary', 'invalid-content-hash', 'inline', NULL )", [provider], ) @@ -2732,7 +2755,7 @@ mod tests { .authorize(&snapshot, occurrence_anchor.anchor_id()) .await, Ok(HydrationAuthorization::Denied(ref denial)) - if denial.state() == HydrationStateV1::UnverifiableLegacy + if denial.state() == HydrationStateV1::Unverifiable )); let mut denied_output = Vec::new(); assert_eq!( @@ -2755,9 +2778,9 @@ mod tests { /// An occurrence whose `message_id` was projected from the stable record id /// (because the canonical envelope carries no `relations.message_id`) must - /// hydrate; refusing it as `UnverifiableLegacy` drops real + /// hydrate; refusing it as `Unverifiable` drops real /// `lcm_grep`/`lcm_expand` matches into - /// `omissions: reason=unverifiable_legacy`. + /// `omissions: reason=unverifiable`. #[tokio::test] async fn occurrence_keyed_on_stable_record_id_resolves_when_relations_message_id_absent() { let dir = tempdir().expect("temporary directory"); diff --git a/crates/tracedecay-session-temporal-store/src/hydration/external.rs b/crates/tracedecay-session-temporal-store/src/hydration/external.rs index 64c10ea686..c604405d36 100644 --- a/crates/tracedecay-session-temporal-store/src/hydration/external.rs +++ b/crates/tracedecay-session-temporal-store/src/hydration/external.rs @@ -256,5 +256,5 @@ pub(crate) async fn resolve_external_target( } fn unverifiable() -> HydrationResolution { - HydrationResolution::Unavailable(HydrationStateV1::UnverifiableLegacy) + HydrationResolution::Unavailable(HydrationStateV1::Unverifiable) } diff --git a/crates/tracedecay-session-temporal-store/src/hydration/file_stream_tests.rs b/crates/tracedecay-session-temporal-store/src/hydration/file_stream_tests.rs index 15c8371552..5315868313 100644 --- a/crates/tracedecay-session-temporal-store/src/hydration/file_stream_tests.rs +++ b/crates/tracedecay-session-temporal-store/src/hydration/file_stream_tests.rs @@ -23,11 +23,14 @@ use tempfile::tempdir; use tracedecay_domain::{RetrievalAnchorId, RetrievalGrainV1, SessionId, TemporalModeV1}; use tracedecay_global_db::tests::harness::{HostAdmissionScope, HostAdmissionTestRuntimeV1}; use tracedecay_runtime_core::db::DatabaseEngineReadSnapshot; +use tracedecay_temporal_query::execution::{BindingDigest, ExecutionControl, ExecutionLimits}; use tracedecay_temporal_query::ports::{ - BindingDigest, ExecutionControl, ExecutionLimits, KernelVersions, ReadBudgetAccounting, - TemporalExecutionSnapshot, TemporalPortError, TemporalSnapshotRequest, TemporalWatermarks, + ReadBudgetAccounting, TemporalPortError, TemporalSnapshotRequest, }; use tracedecay_temporal_query::resolution::ValidatedAuthorization; +use tracedecay_temporal_query::snapshot::{ + KernelVersions, TemporalExecutionSnapshot, TemporalWatermarks, +}; use super::{ BackendFuture, BoundedPayload, HydrationAuthorization, HydrationError, HydrationResolution, @@ -255,9 +258,11 @@ impl TemporalHydrationBackend for ExternalPayloadBackend<'_> { use tracedecay_domain::test_fixtures::repeated_sha256_text as digest; fn snapshot(control: ExecutionControl) -> TemporalExecutionSnapshot { - let limits = ExecutionLimits::default(); - assert_eq!(limits.hydration_payload_bytes, MAX_PAYLOAD_BYTES); - assert_eq!(limits.hydration_chunk_bytes, CHUNK_BYTES); + let limits = ExecutionLimits { + hydration_payload_bytes: MAX_PAYLOAD_BYTES, + hydration_chunk_bytes: CHUNK_BYTES, + ..ExecutionLimits::default() + }; TemporalExecutionSnapshot::new_authorized( TemporalSnapshotRequest::new( SessionId::new("session-1").expect("session"), diff --git a/crates/tracedecay-session-temporal-store/src/hydration/graph_relation_tests.rs b/crates/tracedecay-session-temporal-store/src/hydration/graph_relation_tests.rs index 5940addc7c..ba60d9ed29 100644 --- a/crates/tracedecay-session-temporal-store/src/hydration/graph_relation_tests.rs +++ b/crates/tracedecay-session-temporal-store/src/hydration/graph_relation_tests.rs @@ -1,7 +1,8 @@ use tracedecay_domain::{ProjectId, RetrievalAnchorId, SessionId}; use tracedecay_runtime_core::db::engine::{Executor, TestConnection}; +use tracedecay_temporal_query::execution::ExecutionControl; use tracedecay_temporal_query::hydration::HydrationError; -use tracedecay_temporal_query::ports::{ExecutionControl, TemporalPortError}; +use tracedecay_temporal_query::ports::TemporalPortError; use super::{TemporalSqlRead, summary_has_provider_evidence}; use crate::relations::{ diff --git a/crates/tracedecay-session-temporal-store/src/lib.rs b/crates/tracedecay-session-temporal-store/src/lib.rs index 2a85858386..16b630e2e6 100644 --- a/crates/tracedecay-session-temporal-store/src/lib.rs +++ b/crates/tracedecay-session-temporal-store/src/lib.rs @@ -11,7 +11,6 @@ mod handle; mod schema_constants; mod support; -pub use support::derive_projection; #[cfg(test)] mod test_registered_impls; #[cfg(test)] @@ -84,13 +83,12 @@ use tracedecay_sessions::runtime::git_correlation::{ use tracedecay_store::{SessionMessageRecord, SessionRecord}; use tracedecay_temporal_query::context::VersionedTokenEstimator; use tracedecay_temporal_query::cursor::{CursorError, StableSortKey, encode_cursor, verify_cursor}; +use tracedecay_temporal_query::execution::{BindingDigest, ExecutionControl}; use tracedecay_temporal_query::hydrate_temporal_candidate_selection; use tracedecay_temporal_query::hydration::hydrate_selected; -use tracedecay_temporal_query::ports::{ - BindingDigest, ExecutionControl, KernelVersions, TemporalExecutionSnapshot, - TemporalRetrievalScope, -}; +use tracedecay_temporal_query::ports::TemporalRetrievalScope; use tracedecay_temporal_query::resolution::ValidatedAuthorization; +use tracedecay_temporal_query::snapshot::{KernelVersions, TemporalExecutionSnapshot}; use tracedecay_temporal_query::{execute_temporal_candidate_export, execute_temporal_kernel}; pub use self::cursor_keys::{ @@ -158,13 +156,6 @@ impl SessionTemporalAccess<'_, D> { "verified Git-evidence projection has not been published".to_owned(), )); } - // A pre-index head cannot be scoped through the graph either; its - // next publication re-projects it. - GitEvidenceGraphHead::Legacy { generation } => { - return Err(GitCorrelationError::Unavailable(format!( - "verified Git-evidence generation `{generation}` predates the indexed projector" - ))); - } }; let session_ids = match maximum { Some(maximum) => view.session_ids_for_scope_bounded(filter, maximum), @@ -671,9 +662,7 @@ impl<'db, D: SessionTemporalRegisteredDb + Sync> HydrationStateV1::Unauthorized => SessionTemporalExecutionError::Denied, HydrationStateV1::Available | HydrationStateV1::RetainedButUnavailable - | HydrationStateV1::UnverifiableLegacy => { - SessionTemporalExecutionError::Unavailable - } + | HydrationStateV1::Unverifiable => SessionTemporalExecutionError::Unavailable, }); } }; diff --git a/crates/tracedecay-session-temporal-store/src/operations/generation.rs b/crates/tracedecay-session-temporal-store/src/operations/generation.rs index 397badba36..e14431134e 100644 --- a/crates/tracedecay-session-temporal-store/src/operations/generation.rs +++ b/crates/tracedecay-session-temporal-store/src/operations/generation.rs @@ -82,10 +82,10 @@ pub async fn invalidate_raw_summary_revision( .map_err(|_| lineage_limit(session_id, "raw_revision_invalidation_budget_exhausted"))?; let mut rows = conn .query( - "SELECT node_id - FROM lcm_summary_sources - WHERE source_kind = ?1 AND source_id = ?2 AND node_id > ?3 - ORDER BY node_id + "SELECT summary_id + FROM session_summary_sources + WHERE source_kind = ?1 AND source_id = ?2 AND summary_id > ?3 + ORDER BY summary_id LIMIT ?4", params![ source_kind.as_str(), @@ -147,8 +147,8 @@ pub async fn invalidate_raw_summary_revision( let mut source_rows = conn .query( "SELECT MIN(CAST(source_id AS INTEGER)) - FROM lcm_summary_sources - WHERE node_id = ?1 AND source_kind = 'raw_message'", + FROM session_summary_sources + WHERE summary_id = ?1 AND source_kind = 'raw_message'", params![summary_id], ) .await?; @@ -666,8 +666,8 @@ mod tests { generation INTEGER NOT NULL, state TEXT NOT NULL ); - CREATE TABLE lcm_summary_sources ( - node_id TEXT NOT NULL, + CREATE TABLE session_summary_sources ( + summary_id TEXT NOT NULL, source_kind TEXT NOT NULL, source_id TEXT NOT NULL ); @@ -708,9 +708,9 @@ mod tests { WITH RECURSIVE ids(value) AS ( VALUES(1) UNION ALL SELECT value + 1 FROM ids WHERE value < 4098 ) - INSERT INTO lcm_summary_sources(node_id, source_kind, source_id) + INSERT INTO session_summary_sources(summary_id, source_kind, source_id) SELECT printf('summary-%05d', value), 'raw_message', '100' FROM ids; - INSERT INTO lcm_summary_sources(node_id, source_kind, source_id) + INSERT INTO session_summary_sources(summary_id, source_kind, source_id) VALUES ('summary-00001', 'raw_message', '5'); WITH RECURSIVE ids(value) AS ( VALUES(1) UNION ALL SELECT value + 1 FROM ids WHERE value < 4098 @@ -768,13 +768,13 @@ mod tests { generation INTEGER NOT NULL, state TEXT NOT NULL ); - CREATE TABLE lcm_summary_sources ( - node_id TEXT NOT NULL, + CREATE TABLE session_summary_sources ( + summary_id TEXT NOT NULL, source_kind TEXT NOT NULL, source_id TEXT NOT NULL ); - CREATE TABLE lcm_summary_nodes ( - node_id TEXT PRIMARY KEY, + CREATE TABLE session_summary_nodes ( + summary_id TEXT PRIMARY KEY, provider TEXT NOT NULL, conversation_id TEXT NOT NULL, session_id TEXT NOT NULL, @@ -820,15 +820,15 @@ mod tests { INSERT INTO lcm_summary_convergence_dirty_raw( provider, session_id, store_id, rewind_frontier_store_id ) VALUES ('cursor', 'diamond', 100, 99); - INSERT INTO lcm_summary_sources(node_id, source_kind, source_id) VALUES + INSERT INTO session_summary_sources(summary_id, source_kind, source_id) VALUES ('a', 'raw_message', '100'), ('d', 'raw_message', '100'), ('b', 'summary_node', 'a'), ('d', 'summary_node', 'b'), ('e', 'summary_node', 'd'), ('z', 'raw_message', '200'); - INSERT INTO lcm_summary_nodes( - node_id, provider, conversation_id, session_id, depth, + INSERT INTO session_summary_nodes( + summary_id, provider, conversation_id, session_id, depth, summary_text, summary_hash, summary_token_count, source_token_count, created_at ) VALUES ('a', 'cursor', 'diamond', 'diamond', 0, 'a', 'a-hash', 1, 1, 1), diff --git a/crates/tracedecay-session-temporal-store/src/operations/message_anchor.rs b/crates/tracedecay-session-temporal-store/src/operations/message_anchor.rs index b155665056..3b8971de44 100644 --- a/crates/tracedecay-session-temporal-store/src/operations/message_anchor.rs +++ b/crates/tracedecay-session-temporal-store/src/operations/message_anchor.rs @@ -34,7 +34,8 @@ use tracedecay_store::derive_canonical_projection; use super::sources::unavailable; /// Resolved canonical source binding: anchor id, whether the publication still -/// has to write a compatibility anchor row, and the source's knowledge time. +/// has to write an unobserved raw-message anchor row, and the source's +/// knowledge time. pub(super) type ResolvedMessageAnchor = (String, bool, i64); /// One materialized occurrence row of a requested message. @@ -51,8 +52,8 @@ struct MaterializedOccurrence { /// source order) for one session, reading the shared authorities once. /// /// A message absent from the returned map has no canonical anchor in this -/// store at all, the only case in which the publication falls back to a -/// legacy compatibility anchor. A refusal raised by one message's own +/// store at all (a raw row with no durable observation behind it), the only +/// case in which the publication writes an unobserved raw-message anchor. A refusal raised by one message's own /// evidence names that message; a refusal the shared observation scan raises /// before any message matched (missing or undecodable observation authority) /// names the first still-unresolved message in source order, which is the @@ -480,13 +481,13 @@ mod tests { fn fixture_anchor( observation: &DurableObservationV1, - ) -> tracedecay_domain::RetrievalAnchorRecordV2 { + ) -> tracedecay_domain::RetrievalAnchorRecord { let authorization = tracedecay_store::build_observation_resolution_authorization_v1( observation, "message-anchor-test", ) .expect("anchor authorization"); - tracedecay_store::build_observation_retrieval_anchor_v2( + tracedecay_store::build_observation_retrieval_anchor( observation, ProjectionGenerationId::new("projection.message-anchor-test.v1") .expect("projection generation"), @@ -507,13 +508,11 @@ mod tests { conn.execute_batch(&format!( "INSERT INTO lcm_raw_messages ( provider, message_id, session_id, store_id, role, ordinal, timestamp, - content, content_hash, storage_kind, payload_ref, snippet_text, - index_text, legacy_source, legacy_truncated, metadata_json + content, content_hash, storage_kind, payload_ref, metadata_json ) VALUES ( 'codex', 'message.source', 'session.message-anchor', 41, 'assistant', 0, {timestamp_sql}, 'source body', - 'sha256:source-body', 'inline', NULL, 'source body', 'source body', - 0, 0, NULL + 'sha256:source-body', 'inline', NULL, NULL );", )) .await @@ -524,7 +523,7 @@ mod tests { conn: &impl crate::handle::SessionTemporalExec, observation_json: &str, observation: &DurableObservationV1, - anchor: &tracedecay_domain::RetrievalAnchorRecordV2, + anchor: &tracedecay_domain::RetrievalAnchorRecord, owner_json: &str, ) { seed_canonical_binding_at(conn, observation_json, observation, anchor, owner_json, 1).await; @@ -534,7 +533,7 @@ mod tests { conn: &impl crate::handle::SessionTemporalExec, observation_json: &str, observation: &DurableObservationV1, - anchor: &tracedecay_domain::RetrievalAnchorRecordV2, + anchor: &tracedecay_domain::RetrievalAnchorRecord, owner_json: &str, sequence: i64, ) { @@ -634,12 +633,10 @@ mod tests { conn.execute( "INSERT INTO lcm_raw_messages ( provider, message_id, session_id, store_id, role, ordinal, timestamp, - content, content_hash, storage_kind, payload_ref, snippet_text, - index_text, legacy_source, legacy_truncated, metadata_json + content, content_hash, storage_kind, payload_ref, metadata_json ) VALUES ( 'codex', ?1, 'session.message-anchor', ?2, 'assistant', ?3, 1715000001, - 'source body', 'sha256:source-body', 'inline', NULL, 'source body', - 'source body', 0, 0, NULL + 'source body', 'sha256:source-body', 'inline', NULL, NULL )", params![format!("message.source.{index}"), 41 + index, index], ) @@ -653,7 +650,7 @@ mod tests { async fn materialize_occurrence( conn: &impl crate::handle::SessionTemporalExec, observation: &DurableObservationV1, - anchor: &tracedecay_domain::RetrievalAnchorRecordV2, + anchor: &tracedecay_domain::RetrievalAnchorRecord, message_id: &str, ) { // The generation lifecycle guards admit only building -> ready -> active. @@ -675,11 +672,11 @@ mod tests { session_id, generation, occurrence_id, source_observation_id, source_provider, projection_output_ordinal, retrieval_anchor_id, message_id, role, knowledge_at, valid_time_json, evidence_json, sanitized_content_digest, - sanitized_content_bytes, snippet_text, index_text + sanitized_content_bytes, index_text ) VALUES ( 'session.message-anchor', 1, ?1, ?2, 'codex', 0, ?3, ?1, 'assistant', 1715000002, '{\"kind\":\"unknown\"}', '{}', - '0000000000000000000000000000000000000000000000000000000000000000', 0, '', '' + '0000000000000000000000000000000000000000000000000000000000000000', 0, '' )", params![ message_id, @@ -786,25 +783,25 @@ mod tests { .expect("summary node count value") } - async fn legacy_anchor_count(conn: &impl crate::handle::SessionTemporalExec) -> i64 { + async fn unobserved_raw_anchor_count(conn: &impl crate::handle::SessionTemporalExec) -> i64 { let mut rows = conn .query( "SELECT COUNT(*) FROM retrieval_anchors - WHERE json_extract(anchor_json, '$.kind') = 'legacy_lcm_raw_message'", + WHERE json_extract(anchor_json, '$.kind') = 'lcm_unobserved_raw_message'", (), ) .await - .expect("legacy anchor count"); + .expect("unobserved raw anchor count"); rows.next() .await - .expect("legacy anchor row") - .expect("legacy anchor count row") + .expect("unobserved raw anchor row") + .expect("unobserved raw anchor count row") .get(0) - .expect("legacy anchor count value") + .expect("unobserved raw anchor count value") } #[tokio::test] - async fn malformed_canonical_observation_never_falls_back_to_a_legacy_anchor() { + async fn malformed_canonical_observation_never_falls_back_to_an_unobserved_anchor() { let directory = tempdir().expect("temporary directory"); let runtime = HostAdmissionTestRuntimeV1::profile(directory.path()) .await @@ -831,7 +828,7 @@ mod tests { let result = publish(&conn).await; - assert_eq!(legacy_anchor_count(&conn).await, 0); + assert_eq!(unobserved_raw_anchor_count(&conn).await, 0); assert!(matches!( result, Err(LcmError::SummarySourceUnavailable { ref reason, .. }) @@ -868,7 +865,7 @@ mod tests { let result = publish(&conn).await; - assert_eq!(legacy_anchor_count(&conn).await, 0); + assert_eq!(unobserved_raw_anchor_count(&conn).await, 0); assert!(matches!( result, Err(LcmError::SummarySourceUnavailable { ref reason, .. }) @@ -877,7 +874,7 @@ mod tests { } #[tokio::test] - async fn ownership_mismatched_canonical_binding_never_falls_back_to_a_legacy_anchor() { + async fn ownership_mismatched_canonical_binding_never_falls_back_to_an_unobserved_anchor() { let directory = tempdir().expect("temporary directory"); let runtime = HostAdmissionTestRuntimeV1::profile(directory.path()) .await @@ -902,7 +899,7 @@ mod tests { let result = publish(&conn).await; - assert_eq!(legacy_anchor_count(&conn).await, 0); + assert_eq!(unobserved_raw_anchor_count(&conn).await, 0); assert!(matches!( result, Err(LcmError::SummarySourceNotOwnedBySession) @@ -910,7 +907,7 @@ mod tests { } #[tokio::test] - async fn non_exact_canonical_binding_never_falls_back_to_a_legacy_anchor() { + async fn non_exact_canonical_binding_never_falls_back_to_an_unobserved_anchor() { let directory = tempdir().expect("temporary directory"); let runtime = HostAdmissionTestRuntimeV1::profile(directory.path()) .await @@ -937,7 +934,7 @@ mod tests { let result = publish(&conn).await; - assert_eq!(legacy_anchor_count(&conn).await, 0); + assert_eq!(unobserved_raw_anchor_count(&conn).await, 0); assert!(matches!( result, Err(LcmError::SummarySourceNotOwnedBySession) @@ -945,7 +942,7 @@ mod tests { } #[tokio::test] - async fn missing_canonical_anchor_binding_never_falls_back_to_a_legacy_anchor() { + async fn missing_canonical_anchor_binding_never_falls_back_to_an_unobserved_anchor() { let directory = tempdir().expect("temporary directory"); let runtime = HostAdmissionTestRuntimeV1::profile(directory.path()) .await @@ -967,7 +964,7 @@ mod tests { let result = publish(&conn).await; - assert_eq!(legacy_anchor_count(&conn).await, 0); + assert_eq!(unobserved_raw_anchor_count(&conn).await, 0); assert!(matches!( result, Err(LcmError::SummarySourceUnavailable { ref reason, .. }) @@ -976,7 +973,7 @@ mod tests { } #[tokio::test] - async fn unavailable_session_owner_never_inserts_a_legacy_anchor() { + async fn unavailable_session_owner_never_inserts_an_unobserved_anchor() { let directory = tempdir().expect("temporary directory"); let runtime = HostAdmissionTestRuntimeV1::profile(directory.path()) .await @@ -997,7 +994,7 @@ mod tests { let result = publish(&conn).await; - assert_eq!(legacy_anchor_count(&conn).await, 0); + assert_eq!(unobserved_raw_anchor_count(&conn).await, 0); assert!(matches!( result, Err(LcmError::SummarySourceNotOwnedBySession) @@ -1005,7 +1002,7 @@ mod tests { } #[tokio::test] - async fn malformed_raw_timestamp_never_inserts_a_zero_time_legacy_anchor() { + async fn malformed_raw_timestamp_never_inserts_a_zero_time_unobserved_anchor() { let directory = tempdir().expect("temporary directory"); let runtime = HostAdmissionTestRuntimeV1::profile(directory.path()) .await @@ -1019,7 +1016,7 @@ mod tests { let result = publish(&conn).await; - assert_eq!(legacy_anchor_count(&conn).await, 0); + assert_eq!(unobserved_raw_anchor_count(&conn).await, 0); assert!(matches!( result, Err(LcmError::SummarySourceUnavailable { ref reason, .. }) @@ -1046,11 +1043,14 @@ mod tests { // is testing something the store cannot produce. conn.execute( "INSERT INTO session_summary_nodes ( - summary_id, session_id, summary_anchor_id, summary_text, - index_text, source_horizon_json, publication_json, created_at + summary_id, session_id, provider, conversation_id, depth, summary_anchor_id, + summary_text, summary_hash, summary_token_count, source_token_count, + source_horizon_json, publication_json, created_at ) - SELECT 'summary.message-anchor.malformed', session_id, summary_anchor_id, - summary_text, index_text, '{}', publication_json, created_at + SELECT 'summary.message-anchor.malformed', session_id, provider, + conversation_id, depth, summary_anchor_id, summary_text, summary_hash, + summary_token_count, source_token_count, '{}', publication_json, + created_at FROM session_summary_nodes WHERE summary_id = 'summary.message-anchor'", (), @@ -1091,6 +1091,82 @@ mod tests { )); } + /// A summary built only from other summaries inherits the typed anchor of + /// its first typed child summary rather than falling back to an untyped + /// session anchor. + #[tokio::test] + async fn summary_of_summaries_inherits_its_childs_typed_anchor() { + let directory = tempdir().expect("temporary directory"); + let runtime = HostAdmissionTestRuntimeV1::profile(directory.path()) + .await + .expect("registered profile runtime"); + let conn = runtime + .registered_database(HostAdmissionScope::Profile) + .expect("profile database") + .writer_connection() + .expect("profile writer"); + seed_raw_source(&conn, "1715000001").await; + let observation = + fixture_observation("codex", "session.message-anchor", "message.source", 1); + let anchor = fixture_anchor(&observation); + seed_canonical_binding( + &conn, + &serde_json::to_string(&observation).expect("observation json"), + &observation, + &anchor, + &anchor.owner_column_json().expect("owner json"), + ) + .await; + publish(&conn).await.expect("leaf summary publication"); + let parent = super::super::publication::publish_immutable_summary( + &conn, + parent_publication(), + &empty_relation_projection(), + ) + .await + .expect("parent summary publication"); + assert_eq!(parent.summary.node_id, "summary.message-anchor.parent"); + + let anchor_for = |summary_id: &'static str| { + let conn = &conn; + async move { + let mut rows = conn + .query( + "SELECT anchor.anchor_json + FROM session_summary_nodes AS summary + JOIN retrieval_anchors AS anchor + ON anchor.anchor_id = summary.summary_anchor_id + WHERE summary.summary_id = ?1", + params![summary_id], + ) + .await + .expect("summary anchor"); + let row = rows + .next() + .await + .expect("summary anchor row") + .expect("published summary has an anchor"); + row.get::(0).expect("summary anchor json") + } + }; + let leaf: tracedecay_domain::RetrievalAnchorRecord = + serde_json::from_str(&anchor_for("summary.message-anchor").await) + .expect("the leaf summary anchor is typed"); + let parent_json = anchor_for("summary.message-anchor.parent").await; + let parent_anchor: tracedecay_domain::RetrievalAnchorRecord = + serde_json::from_str(&parent_json).unwrap_or_else(|error| { + panic!( + "a summary of typed summaries must get a typed anchor: {error}: {parent_json}" + ) + }); + assert_eq!(parent_anchor.owner(), leaf.owner()); + assert_eq!( + parent_anchor.source_observations(), + leaf.source_observations() + ); + assert_ne!(parent_anchor.anchor_id(), leaf.anchor_id()); + } + /// `K` raw sources published before any refresh resolve through one scan /// of the session's `N` observation effects, and the same bindings come /// back once the refresh has materialized some or all of the occurrences. @@ -1156,7 +1232,7 @@ mod tests { let prepared_bindings = |sources: &[super::super::PreparedSource]| { sources .iter() - .map(|source| (source.canonical.id.clone(), source.compatibility_anchor)) + .map(|source| (source.canonical.id.clone(), source.unobserved_raw_anchor)) .collect::>() }; @@ -1223,7 +1299,7 @@ mod tests { /// A mixed publication refuses on the first source (in source order) that /// fails its own check, with that source's typed refusal, and leaves - /// nothing published: no summary node and no legacy anchor. + /// nothing published: no summary node and no unobserved raw anchor. #[tokio::test] async fn mixed_source_publication_refuses_on_the_first_failing_source() { let directory = tempdir().expect("temporary directory"); @@ -1235,7 +1311,7 @@ mod tests { .expect("profile database") .writer_connection() .expect("profile writer"); - // 41: canonical anchor; 42: no canonical evidence (legacy fallback); + // 41: canonical anchor; 42: no canonical evidence (unobserved raw row); // 43: ambiguous (two exact anchors project it); 44: retention-expired; // 45: foreign session. seed_raw_sources(&conn, 4).await; @@ -1285,12 +1361,10 @@ mod tests { VALUES ('codex', 'session.foreign', 'user', '/foreign'); INSERT INTO lcm_raw_messages ( provider, message_id, session_id, store_id, role, ordinal, timestamp, - content, content_hash, storage_kind, payload_ref, snippet_text, - index_text, legacy_source, legacy_truncated, metadata_json + content, content_hash, storage_kind, payload_ref, metadata_json ) VALUES ( 'codex', 'message.foreign', 'session.foreign', 45, 'assistant', 0, 1715000001, - 'foreign body', 'sha256:foreign-body', 'inline', NULL, 'foreign body', - 'foreign body', 0, 0, NULL + 'foreign body', 'sha256:foreign-body', 'inline', NULL, NULL );", ) .await @@ -1308,12 +1382,12 @@ mod tests { Err(LcmError::SummarySourceUnavailable { ref source_id, ref reason }) if source_id == "44" && reason == "retention_expired" )); - assert_eq!(legacy_anchor_count(&conn).await, 0); + assert_eq!(unobserved_raw_anchor_count(&conn).await, 0); assert_eq!(summary_node_count(&conn).await, 0); // The ambiguous source (43) is refused by its own evidence even though - // the canonical (41) and legacy-fallback (42) sources ahead of it are - // fine; the fallback anchor for 42 is never written. + // the canonical (41) and unobserved (42) sources ahead of it are + // fine; the unobserved anchor for 42 is never written. let result = super::super::publication::publish_immutable_summary( &conn, publication_over(&[41, 42, 43]), @@ -1325,19 +1399,19 @@ mod tests { Err(LcmError::SummarySourceUnavailable { ref source_id, ref reason }) if source_id == "message.source.2" && reason == "ambiguous_anchor" )); - assert_eq!(legacy_anchor_count(&conn).await, 0); + assert_eq!(unobserved_raw_anchor_count(&conn).await, 0); assert_eq!(summary_node_count(&conn).await, 0); // Without the failing sources the same publication commits: 41 keeps - // its canonical anchor and 42 falls back to exactly one legacy anchor. + // its canonical anchor and 42 gets exactly one unobserved raw anchor. super::super::publication::publish_immutable_summary( &conn, publication_over(&[41, 42]), &empty_relation_projection(), ) .await - .expect("publication over canonical and legacy sources"); - assert_eq!(legacy_anchor_count(&conn).await, 1); + .expect("publication over canonical and unobserved sources"); + assert_eq!(unobserved_raw_anchor_count(&conn).await, 1); assert_eq!(summary_node_count(&conn).await, 1); } } diff --git a/crates/tracedecay-session-temporal-store/src/operations/mod.rs b/crates/tracedecay-session-temporal-store/src/operations/mod.rs index d554ccf8a6..d1f513385f 100644 --- a/crates/tracedecay-session-temporal-store/src/operations/mod.rs +++ b/crates/tracedecay-session-temporal-store/src/operations/mod.rs @@ -2,7 +2,6 @@ mod generation; mod message_anchor; mod publication; mod sources; -mod summary_projection; use std::collections::BTreeMap; @@ -37,11 +36,17 @@ pub(super) struct PreparedPayload { #[derive(Clone, Debug)] pub(super) struct PreparedSource { pub canonical: CanonicalSourceBinding, - pub compatibility_anchor: bool, + pub unobserved_raw_anchor: bool, pub timestamp: i64, pub payload: Option, } +/// The frozen publication manifest stored in `session_summary_nodes.publication_json`. +/// +/// The summary body, expand hint, and provider metadata are stored once, in +/// the row's real columns; they are skipped on serialization and +/// [`load_manifest`] fills them back from those columns so in-memory callers +/// see one complete manifest. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub(super) struct CanonicalPublicationManifest { pub version: u32, @@ -49,6 +54,7 @@ pub(super) struct CanonicalPublicationManifest { pub conversation_id: String, pub session_id: String, pub depth: i64, + #[serde(skip)] pub summary_text: String, pub summary_hash: String, pub source_refs: Vec, @@ -57,7 +63,9 @@ pub(super) struct CanonicalPublicationManifest { pub summary_token_count: i64, pub source_time_start: Option, pub source_time_end: Option, + #[serde(skip)] pub expand_hint: Option, + #[serde(skip)] pub metadata_json: Option, pub source_horizon_json: String, pub owner_json: String, @@ -216,7 +224,7 @@ pub(super) async fn load_manifest( ) -> Result, LcmError> { let mut rows = conn .query( - "SELECT publication_json, created_at + "SELECT publication_json, created_at, summary_text, expand_hint, metadata_json FROM session_summary_nodes WHERE summary_id = ?1", params![summary_id], ) @@ -225,8 +233,12 @@ pub(super) async fn load_manifest( return Ok(None); }; let raw: String = row.get(0)?; - let manifest = serde_json::from_str(&raw).map_err(|_| LcmError::ImmutableSummaryConflict { - summary_id: summary_id.to_string(), - })?; + let mut manifest: CanonicalPublicationManifest = + serde_json::from_str(&raw).map_err(|_| LcmError::ImmutableSummaryConflict { + summary_id: summary_id.to_string(), + })?; + manifest.summary_text = row.get(2)?; + manifest.expand_hint = row.get(3)?; + manifest.metadata_json = row.get(4)?; Ok(Some((manifest, row.get(1)?))) } diff --git a/crates/tracedecay-session-temporal-store/src/operations/publication.rs b/crates/tracedecay-session-temporal-store/src/operations/publication.rs index 005844de28..45c33f7e3d 100644 --- a/crates/tracedecay-session-temporal-store/src/operations/publication.rs +++ b/crates/tracedecay-session-temporal-store/src/operations/publication.rs @@ -1,24 +1,21 @@ use std::sync::Mutex; -use serde_json::{Value, json}; -use tracedecay_domain::{ - EntityKind, RetrievalAnchorId, RetrievalAnchorRecord, RetrievalAnchorTargetV2, -}; +use serde_json::Value; +use tracedecay_domain::RetrievalAnchorId; use tracedecay_runtime_core::db::engine::params; use tracedecay_lcm::retrieval_content::projected_content_hash; use tracedecay_lcm::{ dag::LcmSummaryPublicationPort, types::{ - LcmError, LcmImmutableSummaryPublication, LcmSummaryNode, LcmSummaryPublicationDisposition, - LcmSummaryPublicationReceipt, + LcmError, LcmImmutableSummaryPublication, LcmSourceRef, LcmSummaryNode, + LcmSummaryPublicationDisposition, LcmSummaryPublicationReceipt, }, }; use super::{ - CanonicalPublicationManifest, FrozenPublicationReceipt, PUBLICATION_ROUTE, SANITIZER_VERSION, - generation, load_manifest, logical_identity_digest, receipt_id, sources, summary_projection, - unixepoch, + CanonicalPublicationManifest, FrozenPublicationReceipt, SANITIZER_VERSION, generation, + load_manifest, logical_identity_digest, receipt_id, sources, unixepoch, }; use crate::relations::{SessionRelationProjection, SummaryRelationNode, SummarySourceRef}; @@ -183,13 +180,20 @@ pub async fn publish_immutable_summary( ); let source_horizon = sources::source_horizon_json(&sources, draft.source_time_end); let owner_json = sources::session_owner_json(conn, &draft.provider, &draft.session_id).await?; - sources::insert_compatibility_source_anchors(conn, &sources, &owner_json).await?; - let typed_summary_anchor = - sources::build_summary_anchor(conn, summary_id, &sources, created_at).await?; - let summary_anchor_id = typed_summary_anchor.as_ref().map_or_else( - || format!("anchor_summary_{}", projected_content_hash(summary_id)), - |anchor| anchor.anchor_id().as_str().to_string(), - ); + sources::insert_unobserved_raw_anchors(conn, &sources, &owner_json).await?; + let canonical_sources = sources + .iter() + .map(|source| source.canonical.clone()) + .collect::>(); + let summary_anchor = sources::derive_summary_anchor( + conn, + summary_id, + &canonical_sources, + &owner_json, + &source_horizon, + created_at, + ) + .await?; let receipt_id = receipt_id(summary_id, &summary_hash); let manifest = CanonicalPublicationManifest::from_publication( draft, @@ -197,7 +201,7 @@ pub async fn publish_immutable_summary( &sources, source_horizon.clone(), owner_json.clone(), - summary_anchor_id.clone(), + summary_anchor.anchor_id.clone(), receipt_id.clone(), publication.predecessor_summary_id.clone(), logical_identity, @@ -205,23 +209,12 @@ pub async fn publish_immutable_summary( let publication_json = serde_json::to_string(&manifest) .map_err(|error| LcmError::Db(format!("encode summary publication manifest: {error}")))?; - sources::insert_summary_anchor( - conn, - &summary_anchor_id, - summary_id, - &owner_json, - &source_horizon, - created_at, - typed_summary_anchor.as_ref(), - ) - .await?; + sources::insert_anchor(conn, &summary_anchor, summary_id).await?; insert_canonical_node( conn, summary_id, - draft.session_id.as_str(), - &summary_anchor_id, - draft.summary_text.as_str(), - &source_horizon, + &manifest, + &summary_anchor.anchor_id, &publication_json, created_at, ) @@ -250,11 +243,6 @@ pub async fn publish_immutable_summary( insert_sanitization_receipt(conn, &receipt_id, &summary_hash, &frozen_receipt).await?; sources::insert_payload_manifests(conn, &manifest).await?; - // The durable summary projection is deliberately last: a projection - // failure rolls back all canonical rows and lets the outer payload - // rollback guard remove files. - summary_projection::project_canonical_summary(conn, summary_id, &manifest, created_at).await?; - Ok(LcmSummaryPublicationReceipt { summary: summary_node(summary_id, &manifest, created_at), disposition: LcmSummaryPublicationDisposition::Published, @@ -289,33 +277,57 @@ fn validate_publication_shape( } } -#[allow(clippy::too_many_arguments)] +/// Writes the one summary row and its ordered lineage. The summary body, +/// expand hint, and metadata live only in these columns; `publication_json` +/// carries the rest of the frozen manifest. async fn insert_canonical_node( conn: &impl crate::handle::SessionTemporalExec, summary_id: &str, - session_id: &str, + manifest: &CanonicalPublicationManifest, summary_anchor_id: &str, - summary_text: &str, - source_horizon_json: &str, publication_json: &str, created_at: i64, ) -> Result<(), LcmError> { conn.execute( "INSERT INTO session_summary_nodes ( - summary_id, session_id, summary_anchor_id, summary_text, index_text, + summary_id, session_id, provider, conversation_id, depth, summary_anchor_id, + summary_text, summary_hash, summary_token_count, source_token_count, + source_time_start, source_time_end, expand_hint, metadata_json, source_horizon_json, publication_json, created_at - ) VALUES (?1, ?2, ?3, ?4, ?4, ?5, ?6, ?7)", + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17)", params![ summary_id, - session_id, + manifest.session_id.as_str(), + manifest.provider.as_str(), + manifest.conversation_id.as_str(), + manifest.depth, summary_anchor_id, - summary_text, - source_horizon_json, + manifest.summary_text.as_str(), + manifest.summary_hash.as_str(), + manifest.summary_token_count, + manifest.source_token_count, + manifest.source_time_start, + manifest.source_time_end, + manifest.expand_hint.as_deref(), + manifest.metadata_json.as_deref(), + manifest.source_horizon_json.as_str(), publication_json, created_at, ], ) .await?; + for (ordinal, source) in manifest.source_refs.iter().enumerate() { + let (kind, id) = match source { + LcmSourceRef::RawMessage { store_id } => ("raw_message", store_id.to_string()), + LcmSourceRef::SummaryNode { node_id } => ("summary_node", node_id.clone()), + }; + conn.execute( + "INSERT INTO session_summary_sources (summary_id, ordinal, source_kind, source_id) + VALUES (?1, ?2, ?3, ?4)", + params![summary_id, ordinal as i64, kind, id.as_str()], + ) + .await?; + } Ok(()) } @@ -362,7 +374,7 @@ async fn verify_canonical_node( .map_err(|error| LcmError::Db(format!("encode summary manifest: {error}")))?; let mut rows = conn .query( - "SELECT session_id, summary_anchor_id, summary_text, index_text, + "SELECT session_id, summary_anchor_id, summary_text, provider, depth, summary_hash, source_horizon_json, publication_json, created_at FROM session_summary_nodes WHERE summary_id = ?1", params![summary_id], @@ -374,10 +386,12 @@ async fn verify_canonical_node( if row.get::(0)? != manifest.session_id || row.get::(1)? != manifest.summary_anchor_id || row.get::(2)? != manifest.summary_text - || row.get::(3)? != manifest.summary_text - || row.get::(4)? != manifest.source_horizon_json - || row.get::(5)? != expected_json - || row.get::(6)? != created_at + || row.get::(3)? != manifest.provider + || row.get::(4)? != manifest.depth + || row.get::(5)? != manifest.summary_hash + || row.get::(6)? != manifest.source_horizon_json + || row.get::(7)? != expected_json + || row.get::(8)? != created_at { return Err(conflict(summary_id)); } @@ -390,45 +404,18 @@ async fn verify_summary_anchor( manifest: &CanonicalPublicationManifest, created_at: i64, ) -> Result<(), LcmError> { - let expected_anchor_json = json!({ - "kind": "immutable_session_summary", - "anchor_id": manifest.summary_anchor_id, - "summary_id": summary_id, - "owner": serde_json::from_str::(&manifest.owner_json).unwrap_or(Value::Null), - "source_horizon": serde_json::from_str::(&manifest.source_horizon_json) - .unwrap_or(Value::Null), - "ingested_at": created_at, - "payload_access": "eligible", - "retention_class": "retention.session-summary", - }) - .to_string(); - let mut rows = conn - .query( - "SELECT anchor_json, owner_json, projection_generation - FROM retrieval_anchors WHERE anchor_id = ?1", - params![manifest.summary_anchor_id.as_str()], - ) - .await?; - let Some(row) = rows.next().await? else { - return Err(conflict(summary_id)); - }; - let actual_anchor_json = row.get::(0)?; - let actual_owner_json = row.get::(1)?; - let typed_match = serde_json::from_str::(&actual_anchor_json) - .ok() - .is_some_and(|anchor| { - anchor.anchor_id().as_str() == manifest.summary_anchor_id - && anchor.owner_column_matches(actual_owner_json.as_str()) - && matches!( - anchor.target(), - RetrievalAnchorTargetV2::Entity(entity) - if entity.kind == EntityKind::SessionSummary - && entity.id.as_str() == summary_id - ) - }); - let legacy_match = - actual_anchor_json == expected_anchor_json && actual_owner_json == manifest.owner_json; - if (!legacy_match && !typed_match) || row.get::(2)? != PUBLICATION_ROUTE { + let expected = sources::derive_summary_anchor( + conn, + summary_id, + &manifest.canonical_sources, + &manifest.owner_json, + &manifest.source_horizon_json, + created_at, + ) + .await?; + if expected.anchor_id != manifest.summary_anchor_id + || !sources::stored_anchor_matches(conn, &expected).await? + { return Err(LcmError::SummarySourceNotOwnedBySession); } Ok(()) diff --git a/crates/tracedecay-session-temporal-store/src/operations/sources.rs b/crates/tracedecay-session-temporal-store/src/operations/sources.rs index 4ef9d132bd..739924085c 100644 --- a/crates/tracedecay-session-temporal-store/src/operations/sources.rs +++ b/crates/tracedecay-session-temporal-store/src/operations/sources.rs @@ -2,9 +2,9 @@ use std::collections::{BTreeMap, BTreeSet}; use serde_json::{Value, json}; use tracedecay_domain::{ - AnchorDurabilityClass, AnchorSourceGenerationV2, EntityId, EntityKind, EntityRef, - EvidenceClass, PayloadAccessState, ProjectionGenerationId, RetentionClass, - RetrievalAnchorRecord, RetrievalAnchorRecordV2Parts, RetrievalAnchorTargetV2, UtcMicros, + AnchorDurabilityClass, AnchorSourceGeneration, EntityId, EntityKind, EntityRef, EvidenceClass, + PayloadAccessState, ProjectionGenerationId, RetentionClass, RetrievalAnchorRecord, + RetrievalAnchorRecordParts, RetrievalAnchorTarget, UtcMicros, }; use tracedecay_runtime_core::db::engine::params; @@ -32,7 +32,8 @@ struct LoadedSummarySource { source_horizon_json: String, publication_json: String, summary_anchor_id: String, - owner_json: String, + anchor_json: String, + anchor_owner_json: String, } /// A raw source that exists, is owned by the publishing session, and is @@ -198,7 +199,8 @@ async fn summary_nodes_by_id( let mut rows = conn .query( "SELECT node.summary_id, node.session_id, node.source_horizon_json, - node.publication_json, node.summary_anchor_id, anchor.owner_json + node.publication_json, node.summary_anchor_id, anchor.anchor_json, + anchor.owner_json FROM session_summary_nodes node JOIN retrieval_anchors anchor ON anchor.anchor_id = node.summary_anchor_id WHERE node.summary_id IN (SELECT value FROM json_each(?1))", @@ -213,7 +215,8 @@ async fn summary_nodes_by_id( source_horizon_json: row.get(2)?, publication_json: row.get(3)?, summary_anchor_id: row.get(4)?, - owner_json: row.get(5)?, + anchor_json: row.get(5)?, + anchor_owner_json: row.get(6)?, }); } Ok(nodes) @@ -250,8 +253,9 @@ fn validate_raw_source( } /// Binds a validated raw source to its canonical anchor; `None` is the typed -/// "no canonical anchor in this store" outcome, the only case that writes a -/// legacy compatibility anchor. +/// "no canonical anchor in this store" outcome of a raw row with no durable +/// observation behind it (an active replay message persisted by compression), +/// the only case that writes an unobserved raw-message anchor. async fn prepare_raw_source( conn: &impl crate::handle::SessionTemporalExec, raw: ValidatedRawSource, @@ -270,7 +274,7 @@ async fn prepare_raw_source( } else { None }; - let (canonical_id, compatibility_anchor, timestamp) = match canonical_anchor { + let (canonical_id, unobserved_raw_anchor, timestamp) = match canonical_anchor { Some(canonical) => canonical.clone(), None => { let source_timestamp = raw @@ -278,7 +282,7 @@ async fn prepare_raw_source( .map(normalize_timestamp) .ok_or_else(|| unavailable(&raw.store_id.to_string(), "unverifiable_timestamp"))?; ( - compatibility_anchor_id( + unobserved_raw_anchor_id( &raw.provider, &raw.session_id, raw.store_id, @@ -294,7 +298,7 @@ async fn prepare_raw_source( kind: "anchor".to_string(), id: canonical_id, }, - compatibility_anchor, + unobserved_raw_anchor, timestamp, payload, }) @@ -312,10 +316,22 @@ fn validate_summary_source<'a>( .map_err(|_| LcmError::ImmutableSummaryConflict { summary_id: node_id.to_string(), })?; + // A typed child anchor is owned by its source observations; an untyped + // one by the publishing session, as its manifest records. + let anchor_owner_matches = + match serde_json::from_str::(&node.anchor_json) { + Ok(typed) => { + typed.anchor_id().as_str() == node.summary_anchor_id + && typed + .owner_column_json() + .is_ok_and(|owner| owner == node.anchor_owner_json) + } + Err(_) => node.anchor_owner_json == manifest.owner_json, + }; if manifest.session_id != draft.session_id || manifest.provider != draft.provider || manifest.summary_anchor_id != node.summary_anchor_id - || manifest.owner_json != node.owner_json + || !anchor_owner_matches || manifest.depth >= draft.depth { return Err(LcmError::SummarySourceNotOwnedBySession); @@ -350,7 +366,7 @@ fn prepare_summary_source( kind: "summary".to_string(), id: node_id.to_string(), }, - compatibility_anchor: false, + unobserved_raw_anchor: false, timestamp, payload: None, }) @@ -484,7 +500,7 @@ pub(super) async fn session_owner_json( Ok(owner_json_for(provider, session_id, &project_key)) } -fn compatibility_anchor_id( +fn unobserved_raw_anchor_id( provider: &str, session_id: &str, store_id: i64, @@ -526,88 +542,92 @@ pub(super) fn source_horizon_json( .to_string() } -pub(super) async fn insert_compatibility_source_anchors( +pub(super) async fn insert_unobserved_raw_anchors( conn: &impl crate::handle::SessionTemporalExec, sources: &[PreparedSource], owner_json: &str, ) -> Result<(), LcmError> { let mut seen = BTreeSet::new(); - for source in sources.iter().filter(|source| source.compatibility_anchor) { + for source in sources.iter().filter(|source| source.unobserved_raw_anchor) { if !seen.insert(source.canonical.id.as_str()) { continue; } - let anchor_json = json!({ - "kind": "legacy_lcm_raw_message", - "anchor_id": source.canonical.id, - "owner": serde_json::from_str::(owner_json).unwrap_or(Value::Null), - "ingested_at": source.timestamp, - "payload_access": "eligible", - "retention_class": "retention.legacy-lcm", - }) - .to_string(); - conn.execute( - "INSERT OR IGNORE INTO retrieval_anchors ( - anchor_id, anchor_json, owner_json, projection_generation - ) VALUES (?1, ?2, ?3, ?4)", - params![ - source.canonical.id.as_str(), - anchor_json.as_str(), - owner_json, - PUBLICATION_ROUTE, - ], - ) - .await?; - verify_anchor( - conn, - &source.canonical.id, - &anchor_json, - owner_json, - &source.canonical.id, - ) - .await?; + let anchor = StoredAnchor { + anchor_id: source.canonical.id.clone(), + anchor_json: json!({ + "kind": "lcm_unobserved_raw_message", + "anchor_id": source.canonical.id, + "owner": serde_json::from_str::(owner_json).unwrap_or(Value::Null), + "ingested_at": source.timestamp, + "payload_access": "eligible", + "retention_class": "retention.lcm-raw-message", + }) + .to_string(), + owner_json: owner_json.to_string(), + }; + insert_anchor(conn, &anchor, &source.canonical.id).await?; } Ok(()) } -pub(super) async fn build_summary_anchor( +/// One `retrieval_anchors` row exactly as publication writes it. +pub(super) struct StoredAnchor { + pub anchor_id: String, + pub anchor_json: String, + pub owner_json: String, +} + +/// Derives a summary's retrieval anchor from its canonical sources. +/// +/// The first source (in source order) with a typed observation-backed anchor +/// makes the summary anchor a typed [`RetrievalAnchorRecord`] inheriting its +/// owner, watermark, coverage, observations, and authorization. A child +/// summary source contributes its own summary anchor, so a summary of typed +/// summaries is typed too. Sources with no typed anchor (unobserved raw rows, +/// untyped child summaries) carry none of that authority, so such a summary +/// gets a session-owned anchor instead. Anchors are immutable, so publication +/// and exact replay derive the same row. +pub(super) async fn derive_summary_anchor( conn: &impl crate::handle::SessionTemporalExec, summary_id: &str, - sources: &[PreparedSource], + sources: &[CanonicalSourceBinding], + owner_json: &str, + source_horizon_json: &str, created_at: i64, -) -> Result, LcmError> { - let mut retained_source = None; - for source in sources { - let mut rows = conn - .query( - "SELECT anchor_json FROM retrieval_anchors WHERE anchor_id = ?1", - params![source.canonical.id.as_str()], - ) - .await?; - let Some(row) = rows.next().await? else { - continue; - }; - let encoded = row.get::(0)?; - if let Ok(anchor) = serde_json::from_str::(&encoded) { - retained_source = Some(anchor); - break; - } - } - let Some(source) = retained_source else { - return Ok(None); +) -> Result { + let Some(source) = first_typed_source_anchor(conn, sources).await? else { + let anchor_id = format!("anchor_summary_{}", projected_content_hash(summary_id)); + let anchor_json = json!({ + "kind": "immutable_session_summary", + "anchor_id": anchor_id, + "summary_id": summary_id, + "owner": serde_json::from_str::(owner_json).unwrap_or(Value::Null), + "source_horizon": serde_json::from_str::(source_horizon_json) + .unwrap_or(Value::Null), + "ingested_at": created_at, + "payload_access": "eligible", + "retention_class": "retention.session-summary", + }) + .to_string(); + return Ok(StoredAnchor { + anchor_id, + anchor_json, + owner_json: owner_json.to_string(), + }); }; - let target = RetrievalAnchorTargetV2::Entity(EntityRef { + let target = RetrievalAnchorTarget::Entity(EntityRef { id: EntityId::new(summary_id.to_string()) .map_err(|error| LcmError::Db(error.to_string()))?, kind: EntityKind::SessionSummary, }); - RetrievalAnchorRecord::new(RetrievalAnchorRecordV2Parts { + let anchor = RetrievalAnchorRecord::new(RetrievalAnchorRecordParts { target, owner: source.owner().clone(), aliases: Vec::new(), occurred_at: None, ingested_at: UtcMicros(created_at), evidence_class: EvidenceClass::DerivedExact, - source_generation: AnchorSourceGenerationV2::Unknown, + source_generation: AnchorSourceGeneration::Unknown, projection_generation: ProjectionGenerationId::new(PUBLICATION_ROUTE) .map_err(|error| LcmError::Db(error.to_string()))?, projection_watermark: source.projection_watermark().clone(), @@ -620,89 +640,91 @@ pub(super) async fn build_summary_anchor( .map_err(|error| LcmError::Db(error.to_string()))?, durability: AnchorDurabilityClass::DurableEvidence, }) - .map(Some) - .map_err(|error| LcmError::Db(error.to_string())) + .map_err(|error| LcmError::Db(error.to_string()))?; + Ok(StoredAnchor { + anchor_id: anchor.anchor_id().as_str().to_string(), + anchor_json: serde_json::to_string(&anchor) + .map_err(|error| LcmError::Db(format!("encode summary anchor: {error}")))?, + owner_json: anchor + .owner_column_json() + .map_err(|error| LcmError::Db(format!("encode summary anchor owner: {error}")))?, + }) } -pub(super) async fn insert_summary_anchor( +async fn first_typed_source_anchor( conn: &impl crate::handle::SessionTemporalExec, - anchor_id: &str, - summary_id: &str, - owner_json: &str, - source_horizon_json: &str, - created_at: i64, - typed_anchor: Option<&RetrievalAnchorRecord>, + sources: &[CanonicalSourceBinding], +) -> Result, LcmError> { + let encoded_sources = + serde_json::to_string(sources).map_err(|error| LcmError::Db(error.to_string()))?; + let mut rows = conn + .query( + "SELECT source.key, anchor.anchor_json + FROM json_each(?1) AS source + LEFT JOIN session_summary_nodes AS summary + ON json_extract(source.value, '$.kind') = 'summary' + AND summary.summary_id = json_extract(source.value, '$.id') + JOIN retrieval_anchors AS anchor + ON anchor.anchor_id = CASE json_extract(source.value, '$.kind') + WHEN 'summary' THEN summary.summary_anchor_id + ELSE json_extract(source.value, '$.id') + END + ORDER BY source.key", + params![encoded_sources], + ) + .await?; + while let Some(row) = rows.next().await? { + if let Ok(anchor) = serde_json::from_str::(&row.get::(1)?) { + return Ok(Some(anchor)); + } + } + Ok(None) +} + +pub(super) async fn insert_anchor( + conn: &impl crate::handle::SessionTemporalExec, + anchor: &StoredAnchor, + conflict_id: &str, ) -> Result<(), LcmError> { - let stored_owner_json = match typed_anchor { - Some(anchor) => anchor - .owner_column_json() - .map_err(|error| LcmError::Db(format!("encode summary anchor owner: {error}")))?, - None => owner_json.to_string(), - }; - let anchor_json = match typed_anchor { - Some(anchor) => serde_json::to_string(anchor) - .map_err(|error| LcmError::Db(format!("encode summary anchor: {error}")))?, - None => json!({ - "kind": "immutable_session_summary", - "anchor_id": anchor_id, - "summary_id": summary_id, - "owner": serde_json::from_str::(owner_json).unwrap_or(Value::Null), - "source_horizon": serde_json::from_str::(source_horizon_json) - .unwrap_or(Value::Null), - "ingested_at": created_at, - "payload_access": "eligible", - "retention_class": "retention.session-summary", - }) - .to_string(), - }; conn.execute( "INSERT OR IGNORE INTO retrieval_anchors ( anchor_id, anchor_json, owner_json, projection_generation ) VALUES (?1, ?2, ?3, ?4)", params![ - anchor_id, - anchor_json.as_str(), - stored_owner_json.as_str(), + anchor.anchor_id.as_str(), + anchor.anchor_json.as_str(), + anchor.owner_json.as_str(), PUBLICATION_ROUTE ], ) .await?; - verify_anchor( - conn, - anchor_id, - &anchor_json, - &stored_owner_json, - summary_id, - ) - .await + if !stored_anchor_matches(conn, anchor).await? { + return Err(LcmError::ImmutableSummaryConflict { + summary_id: conflict_id.to_string(), + }); + } + Ok(()) } -async fn verify_anchor( +/// Whether the stored row for `anchor.anchor_id` is byte-identical to the +/// row publication writes; a missing row is a mismatch. +pub(super) async fn stored_anchor_matches( conn: &impl crate::handle::SessionTemporalExec, - anchor_id: &str, - anchor_json: &str, - owner_json: &str, - conflict_id: &str, -) -> Result<(), LcmError> { + anchor: &StoredAnchor, +) -> Result { let mut rows = conn .query( "SELECT anchor_json, owner_json, projection_generation FROM retrieval_anchors WHERE anchor_id = ?1", - params![anchor_id], + params![anchor.anchor_id.as_str()], ) .await?; let Some(row) = rows.next().await? else { - return Err(LcmError::SummarySourceNotOwnedBySession); + return Ok(false); }; - if row.get::(0)? != anchor_json - || row.get::(1)? != owner_json - || row.get::(2)? != PUBLICATION_ROUTE - { - return Err(LcmError::ImmutableSummaryConflict { - summary_id: conflict_id.to_string(), - }); - } - Ok(()) + Ok(row.get::(0)? == anchor.anchor_json + && row.get::(1)? == anchor.owner_json + && row.get::(2)? == PUBLICATION_ROUTE) } pub(super) async fn insert_payload_manifests( diff --git a/crates/tracedecay-session-temporal-store/src/operations/summary_projection.rs b/crates/tracedecay-session-temporal-store/src/operations/summary_projection.rs deleted file mode 100644 index 05ad893670..0000000000 --- a/crates/tracedecay-session-temporal-store/src/operations/summary_projection.rs +++ /dev/null @@ -1,58 +0,0 @@ -use tracedecay_runtime_core::db::engine::params; - -use tracedecay_lcm::contracts::{LcmError, LcmSourceRef}; - -use super::CanonicalPublicationManifest; - -/// Materializes canonical publication authority into the shipped LCM summary -/// tables used by retrieval, retention, and dashboard reads. -/// -/// Existing projection rows conflict at the database boundary; they are never -/// consulted for identity, replay, authorization, or publication decisions. -#[hotpath::measure(future = true, label = "session_temporal.publication.project_summary")] -pub(super) async fn project_canonical_summary( - conn: &impl crate::handle::SessionTemporalExec, - summary_id: &str, - manifest: &CanonicalPublicationManifest, - created_at: i64, -) -> Result<(), LcmError> { - conn.execute( - "INSERT INTO lcm_summary_nodes ( - node_id, provider, conversation_id, session_id, depth, summary_text, - summary_hash, summary_token_count, source_token_count, source_time_start, - source_time_end, expand_hint, metadata_json, created_at - ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14)", - params![ - summary_id, - manifest.provider.as_str(), - manifest.conversation_id.as_str(), - manifest.session_id.as_str(), - manifest.depth, - manifest.summary_text.as_str(), - manifest.summary_hash.as_str(), - manifest.summary_token_count, - manifest.source_token_count, - manifest.source_time_start, - manifest.source_time_end, - manifest.expand_hint.as_deref(), - manifest.metadata_json.as_deref(), - created_at, - ], - ) - .await?; - for (ordinal, source) in manifest.source_refs.iter().enumerate() { - let (kind, id) = match source { - LcmSourceRef::RawMessage { store_id } => ("raw_message", store_id.to_string()), - LcmSourceRef::SummaryNode { node_id } => ("summary_node", node_id.clone()), - }; - conn.execute( - "INSERT INTO lcm_summary_sources (node_id, source_kind, source_id, ordinal) - VALUES (?1, ?2, ?3, ?4)", - params![summary_id, kind, id.as_str(), ordinal as i64], - ) - .await?; - } - hotpath::gauge!("session_temporal.publication.summary_projection_rows") - .inc(1_u64.saturating_add(manifest.source_refs.len() as u64)); - Ok(()) -} diff --git a/crates/tracedecay-session-temporal-store/src/participant_freeze.rs b/crates/tracedecay-session-temporal-store/src/participant_freeze.rs index 18c85fde4a..77117ba116 100644 --- a/crates/tracedecay-session-temporal-store/src/participant_freeze.rs +++ b/crates/tracedecay-session-temporal-store/src/participant_freeze.rs @@ -8,10 +8,11 @@ use tracedecay_contracts::retrieval::{ }; use tracedecay_domain::{SessionId, SignedCursorKeyRefV1}; use tracedecay_runtime_core::db::engine::params; -use tracedecay_temporal_query::ports::{ - BindingDigest, MAX_TEMPORAL_PARTICIPANTS, TemporalAuthorizedRoot, - TemporalParticipantAuthorization, TemporalParticipantGeneration, TemporalParticipantManifest, - TemporalPreparedCandidateCohort, TemporalRetrievalScope, TemporalSourceAccess, +use tracedecay_temporal_query::execution::BindingDigest; +use tracedecay_temporal_query::ports::{TemporalAuthorizedRoot, TemporalRetrievalScope}; +use tracedecay_temporal_query::snapshot::{ + MAX_TEMPORAL_PARTICIPANTS, TemporalParticipantAuthorization, TemporalParticipantGeneration, + TemporalParticipantManifest, TemporalPreparedCandidateCohort, TemporalSourceAccess, TemporalWatermarks, }; @@ -523,11 +524,10 @@ mod tests { use tracedecay_runtime_core::db::engine::{Executor, TestConnection}; use tracedecay_temporal_query::candidates::CandidateChannel; use tracedecay_temporal_query::context::{ContextBudget, TokenPolicy, VersionedTokenEstimator}; - use tracedecay_temporal_query::ports::{ - ExecutionControl, ExecutionLimits, TemporalCandidatePopulationCount, - TemporalSnapshotRequest, - }; + use tracedecay_temporal_query::execution::{ExecutionControl, ExecutionLimits}; + use tracedecay_temporal_query::ports::TemporalSnapshotRequest; use tracedecay_temporal_query::ranking::DiversityLimits; + use tracedecay_temporal_query::snapshot::TemporalCandidatePopulationCount; fn root(project_id: Option<&str>) -> TemporalAuthorizedRoot { match project_id { @@ -746,7 +746,7 @@ mod tests { "root-fixture", ) .expect("resolution authorization"); - let anchor = tracedecay_store::build_observation_retrieval_anchor_v2( + let anchor = tracedecay_store::build_observation_retrieval_anchor( &observation, projection_generation, UtcMicros(1), @@ -916,8 +916,7 @@ mod tests { session_id, generation, occurrence_id, source_observation_id, source_provider, projection_output_ordinal, retrieval_anchor_id, message_id, turn_id, role, knowledge_at, valid_time_json, - evidence_json, sanitized_content_digest, sanitized_content_bytes, - snippet_text, index_text + evidence_json, sanitized_content_digest, sanitized_content_bytes, index_text ) VALUES (?1, 1, ?2, ?3, 'codex', 0, ?4, ?5, ?6, 'user', ?7, '{\"kind\":\"unknown\"}', '{\"authority\":\"provider_native\", @@ -928,7 +927,7 @@ mod tests { \"sanitizer_version\":\"root-sanitizer\" }}', '0000000000000000000000000000000000000000000000000000000000000000', - 14, ?8, ?8)", + 14, ?8)", params![ session_id.as_str(), occurrence_id.as_str(), @@ -1023,8 +1022,7 @@ mod tests { session_id, generation, occurrence_id, source_observation_id, source_provider, projection_output_ordinal, retrieval_anchor_id, message_id, turn_id, role, knowledge_at, valid_time_json, - evidence_json, sanitized_content_digest, sanitized_content_bytes, - snippet_text, index_text + evidence_json, sanitized_content_digest, sanitized_content_bytes, index_text ) VALUES ('session.000', 1, ?1, 'observation.000', 'codex', ?2, ?4, ?3, 'turn.000', 'user', ?2, '{\"kind\":\"unknown\"}', @@ -1036,7 +1034,7 @@ mod tests { \"sanitizer_version\":\"root-sanitizer\" }}', '0000000000000000000000000000000000000000000000000000000000000000', - 14, ?5, ?5)", + 14, ?5)", params![ occurrence_id.as_str(), i64::try_from(extra + 1).expect("member ordinal"), diff --git a/crates/tracedecay-session-temporal-store/src/projection.rs b/crates/tracedecay-session-temporal-store/src/projection.rs index 74a768f02c..1f272b5cef 100644 --- a/crates/tracedecay-session-temporal-store/src/projection.rs +++ b/crates/tracedecay-session-temporal-store/src/projection.rs @@ -7,7 +7,7 @@ use tracedecay_store::{ SessionRefreshBeginOrJoinRequestV1, SessionRefreshFrontierV1, SessionRefreshProgressV1, SessionStoreResult, SessionTemporalProjectionBatchReceiptV1, SessionTemporalProjectionBatchV1, }; -use tracedecay_temporal_query::ports::ExecutionControl; +use tracedecay_temporal_query::execution::ExecutionControl; use super::query::{PERSIST_OPERATION, storage}; use super::refresh::{SessionRefreshRecoveryV1, SessionRefreshRestartStateV1}; diff --git a/crates/tracedecay-session-temporal-store/src/projection/derived.rs b/crates/tracedecay-session-temporal-store/src/projection/derived.rs index df18db3bf5..6b0c0f3179 100644 --- a/crates/tracedecay-session-temporal-store/src/projection/derived.rs +++ b/crates/tracedecay-session-temporal-store/src/projection/derived.rs @@ -6,7 +6,7 @@ use tracedecay_domain::{ }; use tracedecay_runtime_core::db::engine::params; use tracedecay_store::{SessionStoreResult, SessionTemporalProjectionBatchV1}; -use tracedecay_temporal_query::ports::ExecutionControl; +use tracedecay_temporal_query::execution::ExecutionControl; use super::super::query::{PERSIST_OPERATION, generation_i64, storage, storage_message}; use super::super::rebuild::checkpoint_relation_rebuild_control; diff --git a/crates/tracedecay-session-temporal-store/src/projection/materialize.rs b/crates/tracedecay-session-temporal-store/src/projection/materialize.rs index 009f514b50..f325c8219a 100644 --- a/crates/tracedecay-session-temporal-store/src/projection/materialize.rs +++ b/crates/tracedecay-session-temporal-store/src/projection/materialize.rs @@ -12,9 +12,9 @@ use tracedecay_store::{ SessionRefreshProgressV1, SessionStoreError, SessionStoreResult, SessionTemporalProjectionBatchV1, }; -use tracedecay_temporal_query::ports::ExecutionControl; +use tracedecay_temporal_query::execution::ExecutionControl; -use crate::support::derive_projection; +use tracedecay_store::derive_canonical_projection; use super::super::query::{ PERSIST_OPERATION, frontier_i64, generation_i64, missing_observation, now_micros, @@ -324,8 +324,8 @@ pub(super) async fn materialize_effect_occurrences( .get(observation_id.as_str()) .ok_or_else(|| missing_observation(observation_id))?; // One derivation per observation, reused across all of its outputs. - let projection = - derive_projection(observation).map_err(|error| storage(MATERIALIZE_REFRESH, error))?; + let projection = derive_canonical_projection(observation) + .map_err(|error| storage(MATERIALIZE_REFRESH, error))?; let outputs = projection.messages().collect::>(); if outputs.len() != *output_count { return Err(storage_message( @@ -784,8 +784,8 @@ pub async fn canonical_parent_message_resolver( } else { None }; - let projection = - derive_projection(&observation).map_err(|error| storage(operation, error))?; + let projection = derive_canonical_projection(&observation) + .map_err(|error| storage(operation, error))?; for output in projection .messages() .filter(|output| output.session().session_id == session_id) diff --git a/crates/tracedecay-session-temporal-store/src/projection/persist.rs b/crates/tracedecay-session-temporal-store/src/projection/persist.rs index 1b76b212b9..b232ddbefb 100644 --- a/crates/tracedecay-session-temporal-store/src/projection/persist.rs +++ b/crates/tracedecay-session-temporal-store/src/projection/persist.rs @@ -3,7 +3,7 @@ use std::collections::{BTreeMap, BTreeSet}; use serde::Deserialize; use serde_json::{Value, json}; use tracedecay_domain::{ - AnchorProvenanceRelationV2, CanonicalObservationEnvelopeV1, CopyProofV1, DurableObservationV1, + AnchorProvenanceRelation, CanonicalObservationEnvelopeV1, CopyProofV1, DurableObservationV1, LogicalCopyRecordV1, MessageOccurrenceRecordV1, RetrievalAnchorRecord, SessionAuthorityClassV1, SessionId, TemporalAssertionKindV1, TemporalAssertionRecordV1, TemporalValidityV1, UtcMicros, derive_exact_observation_anchor_id, @@ -14,9 +14,9 @@ use tracedecay_store::{ SessionMessageProjection, SessionStoreError, SessionStoreResult, SessionTemporalProjectionBatchReceiptV1, SessionTemporalProjectionBatchV1, }; -use tracedecay_temporal_query::ports::ExecutionControl; +use tracedecay_temporal_query::execution::ExecutionControl; -use crate::support::derive_projection; +use tracedecay_store::derive_canonical_projection; use super::super::query::{ PERSIST_OPERATION, encode_watermarks, frontier_i64, generation_i64, now_micros, @@ -220,8 +220,8 @@ async fn canonical_occurrence_projection( .map_err(|error| storage(PERSIST_OPERATION, error))?; let output_count = usize::try_from(output_count).map_err(|error| storage(PERSIST_OPERATION, error))?; - let projection = - derive_projection(&observation).map_err(|error| storage(PERSIST_OPERATION, error))?; + let projection = derive_canonical_projection(&observation) + .map_err(|error| storage(PERSIST_OPERATION, error))?; let envelope = observation_envelope(&observation)?; let mut outputs = projection.messages().cloned().collect::>(); outputs.sort_unstable_by_key(SessionMessageProjection::output_ordinal); @@ -413,10 +413,10 @@ async fn persist_occurrence( thread_id, thread_grouping_json, turn_id, turn_grouping_json, message_id, agent_id, role, knowledge_at, valid_time_json, evidence_json, sanitized_content_digest, sanitized_content_bytes, - snippet_text, index_text + index_text ) VALUES ( ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, - ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18, ?19, ?20, ?21 + ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18, ?19, ?20 )", params![ batch.session_id().as_str(), @@ -451,7 +451,6 @@ async fn persist_occurrence( sanitized_content_digest, sanitized_content_bytes, sanitized_content, - sanitized_content, ], ) .await @@ -960,7 +959,7 @@ pub(super) async fn validate_copy_proof( && canonical_source.as_deref() == Some(copy.copied_from_occurrence_id.as_str()) && anchor.source_anchors().iter().any(|lineage| { - lineage.relation() == AnchorProvenanceRelationV2::CopiedFrom + lineage.relation() == AnchorProvenanceRelation::CopiedFrom && lineage.anchor_id() == assertion_anchor_id }) }) @@ -1225,20 +1224,20 @@ pub(super) async fn validate_assertion( } pub(super) const fn assertion_kind_for_relation( - relation: AnchorProvenanceRelationV2, + relation: AnchorProvenanceRelation, ) -> Option { match relation { - AnchorProvenanceRelationV2::Corrects => Some(TemporalAssertionKindV1::Corrects), - AnchorProvenanceRelationV2::Contradicts => Some(TemporalAssertionKindV1::Contradicts), - AnchorProvenanceRelationV2::Supersedes => Some(TemporalAssertionKindV1::Supersedes), - AnchorProvenanceRelationV2::Supports => Some(TemporalAssertionKindV1::Supports), - AnchorProvenanceRelationV2::CapturedFrom - | AnchorProvenanceRelationV2::Produced - | AnchorProvenanceRelationV2::Observed - | AnchorProvenanceRelationV2::ExecutedIn - | AnchorProvenanceRelationV2::Discussed - | AnchorProvenanceRelationV2::CopiedFrom - | AnchorProvenanceRelationV2::DerivedFrom => None, + AnchorProvenanceRelation::Corrects => Some(TemporalAssertionKindV1::Corrects), + AnchorProvenanceRelation::Contradicts => Some(TemporalAssertionKindV1::Contradicts), + AnchorProvenanceRelation::Supersedes => Some(TemporalAssertionKindV1::Supersedes), + AnchorProvenanceRelation::Supports => Some(TemporalAssertionKindV1::Supports), + AnchorProvenanceRelation::CapturedFrom + | AnchorProvenanceRelation::Produced + | AnchorProvenanceRelation::Observed + | AnchorProvenanceRelation::ExecutedIn + | AnchorProvenanceRelation::Discussed + | AnchorProvenanceRelation::CopiedFrom + | AnchorProvenanceRelation::DerivedFrom => None, } } diff --git a/crates/tracedecay-session-temporal-store/src/projection/receipts.rs b/crates/tracedecay-session-temporal-store/src/projection/receipts.rs index 07f34b3533..90903ed4fa 100644 --- a/crates/tracedecay-session-temporal-store/src/projection/receipts.rs +++ b/crates/tracedecay-session-temporal-store/src/projection/receipts.rs @@ -11,7 +11,7 @@ use tracedecay_store::{ SessionTemporalDigestV1, SessionTemporalProjectionBatchReceiptV1, SessionTemporalProjectionBatchV1, }; -use tracedecay_temporal_query::ports::ExecutionControl; +use tracedecay_temporal_query::execution::ExecutionControl; use super::super::query::{ PERSIST_OPERATION, encode_watermarks, frontier_i64, generation_i64, storage, storage_message, @@ -837,7 +837,7 @@ pub(super) async fn projection_coverage( .await?; let fts = digest_query_rows( conn, - "SELECT json_array(occurrence.occurrence_id, fts.index_text, fts.snippet_text) + "SELECT json_array(occurrence.occurrence_id, fts.index_text) FROM session_occurrences AS occurrence JOIN session_occurrences_fts AS fts ON fts.rowid = occurrence.rowid WHERE occurrence.session_id = ?1 AND occurrence.generation = ?2 diff --git a/crates/tracedecay-session-temporal-store/src/projection/tests.rs b/crates/tracedecay-session-temporal-store/src/projection/tests.rs index f3c3f9e74e..587102a602 100644 --- a/crates/tracedecay-session-temporal-store/src/projection/tests.rs +++ b/crates/tracedecay-session-temporal-store/src/projection/tests.rs @@ -3,16 +3,16 @@ use std::sync::Arc; use serde_json::{Value, json}; use tempfile::TempDir; use tracedecay_domain::{ - AnchorProvenanceRelationV2, CanonicalMessageRoleV1, CanonicalObservationEnvelopeV1, + AnchorProvenanceRelation, CanonicalMessageRoleV1, CanonicalObservationEnvelopeV1, CanonicalObservationEvidenceV1, CanonicalObservationFactV1, CanonicalObservationRelationsV1, CopyProofV1, DurableObservationV1, LogicalCopyRecordV1, MessageOccurrenceIdV1, ObservationId, ObservationIdentityMaterialV1, ObservationOrderingDomainV1, ObservationScopeV1, ObservationSourceCursorV1, ObservationSourceGenerationV1, ObservationSourceIdentityV1, ObservationSourceRangeV1, PayloadReferenceV1, ProjectionGenerationId, ProjectionOutputOrdinalV1, ProviderId, RetentionClass, RetrievalAnchorId, - RetrievalAnchorRecordV2, SanitizationReceiptId, SanitizationReceiptRefV1, - SanitizationReceiptV1, SanitizerDispositionV1, SensitivityV1, SessionId, - TemporalAssertionKindV1, TemporalValidityV1, UtcMicros, derive_exact_observation_anchor_id, + RetrievalAnchorRecord, SanitizationReceiptId, SanitizationReceiptRefV1, SanitizationReceiptV1, + SanitizerDispositionV1, SensitivityV1, SessionId, TemporalAssertionKindV1, TemporalValidityV1, + UtcMicros, derive_exact_observation_anchor_id, }; use tracedecay_graph_db::NeverCancelled; use tracedecay_store::{ @@ -22,15 +22,15 @@ use tracedecay_store::{ SessionRefreshFrontierV1, SessionRefreshProgressV1, SessionRefreshStore, SessionRefreshTerminalStateV1, SessionStoreError, SessionTemporalProjectionBatchV1, }; -use tracedecay_temporal_query::ports::ExecutionControl; +use tracedecay_temporal_query::execution::ExecutionControl; use super::super::refresh::SessionRefreshRestartStateV1; use super::materialize::*; use super::persist::persist_occurrences; use super::record_canonical_observation_effect; -use crate::SessionTemporalStore; use crate::handle::SessionTemporalRegisteredDb; use crate::test_support::QueryCountingConnection; +use crate::{SessionTemporalAccess, SessionTemporalStore}; use tracedecay_global_db::RegisteredGlobalDb; use tracedecay_global_db::tests::harness::{ HostAdmissionScope, HostAdmissionTestRuntimeV1, SessionTemporalFixtureCountV1, @@ -70,7 +70,7 @@ fn fixture_receipt(receipt_id: &str, payload: &Value) -> SanitizationReceiptV1 { fn fixture_observation( session_id: &SessionId, ordinal: u64, - lineage: Option<(AnchorProvenanceRelationV2, RetrievalAnchorId)>, + lineage: Option<(AnchorProvenanceRelation, RetrievalAnchorId)>, include_parent: bool, ) -> (DurableObservationV1, AnchoredObservationWrite) { let provider = ProviderId::new(format!("projector-test-{ordinal}")).unwrap(); @@ -104,7 +104,7 @@ fn fixture_observation( fn fixture_multi_output_observation( session_id: &SessionId, ordinal: u64, - lineage: Option<(AnchorProvenanceRelationV2, RetrievalAnchorId)>, + lineage: Option<(AnchorProvenanceRelation, RetrievalAnchorId)>, output_count: usize, ) -> (DurableObservationV1, AnchoredObservationWrite) { assert!(output_count > 1); @@ -155,7 +155,7 @@ fn fixture_observation_from_facts( record_id: ObservationId, relations: CanonicalObservationRelationsV1, facts: Vec, - lineage: Option<(AnchorProvenanceRelationV2, RetrievalAnchorId)>, + lineage: Option<(AnchorProvenanceRelation, RetrievalAnchorId)>, ) -> (DurableObservationV1, AnchoredObservationWrite) { let source = ObservationSourceIdentityV1::for_provider(provider.clone(), session_id.clone()).unwrap(); @@ -202,7 +202,7 @@ fn fixture_observation_from_facts( "projector-test", ) .unwrap(); - let anchor = tracedecay_store::build_observation_retrieval_anchor_v2( + let anchor = tracedecay_store::build_observation_retrieval_anchor( write.observation(), projection_generation.clone(), UtcMicros(1), @@ -217,7 +217,7 @@ fn fixture_observation_from_facts( "owner": write.observation().scope(), }]); } - let anchor: RetrievalAnchorRecordV2 = serde_json::from_value(anchor_json).unwrap(); + let anchor: RetrievalAnchorRecord = serde_json::from_value(anchor_json).unwrap(); let anchored = AnchoredObservationWrite::new(write, anchor, projection_generation).unwrap(); (observation, anchored) } @@ -266,7 +266,7 @@ fn fixture_goal_observation() -> (DurableObservationV1, AnchoredObservationWrite "projector-test", ) .unwrap(); - let anchor = tracedecay_store::build_observation_retrieval_anchor_v2( + let anchor = tracedecay_store::build_observation_retrieval_anchor( write.observation(), projection_generation.clone(), UtcMicros(1), @@ -422,7 +422,7 @@ async fn multi_output_projection_reuses_source_derivation_and_activates_shared_a let (multi, multi_write) = fixture_multi_output_observation( &session_id, 1, - Some((AnchorProvenanceRelationV2::Supersedes, first_anchor)), + Some((AnchorProvenanceRelation::Supersedes, first_anchor)), OUTPUT_COUNT, ); let multi_observation_id = multi.observation_id().clone(); @@ -565,7 +565,7 @@ async fn relation_batch_persists_restarts_and_completes_without_duplicates() { let (second, second_write) = fixture_observation( &session_id, 1, - Some((AnchorProvenanceRelationV2::Supersedes, first_anchor)), + Some((AnchorProvenanceRelation::Supersedes, first_anchor)), true, ); Box::pin(persist_fixture(&runtime, second, second_write)).await; @@ -1211,7 +1211,7 @@ async fn copied_from_lineage_is_not_auto_emitted_by_materializer() { let (second, second_write) = fixture_observation( &session_id, 1, - Some((AnchorProvenanceRelationV2::CopiedFrom, first_anchor)), + Some((AnchorProvenanceRelation::CopiedFrom, first_anchor)), false, ); Box::pin(persist_fixture(&runtime, second, second_write)).await; @@ -1255,7 +1255,7 @@ async fn relation_derivation_backs_off_to_the_total_batch_limit() { for ordinal in 0..501 { let lineage = previous_anchor .take() - .map(|anchor| (AnchorProvenanceRelationV2::Supersedes, anchor)); + .map(|anchor| (AnchorProvenanceRelation::Supersedes, anchor)); let (observation, write) = fixture_observation(&session_id, ordinal, lineage, ordinal > 0); previous_anchor = Some( derive_exact_observation_anchor_id(observation.scope(), observation.observation_id()) @@ -1580,7 +1580,7 @@ async fn explicit_copy_survives_reconstruction_in_the_native_relation_graph() { let (second, second_write) = fixture_observation( &session_id, 1, - Some((AnchorProvenanceRelationV2::CopiedFrom, first_anchor.clone())), + Some((AnchorProvenanceRelation::CopiedFrom, first_anchor.clone())), false, ); Box::pin(persist_fixture(&runtime, second, second_write)).await; @@ -1722,7 +1722,7 @@ async fn multi_batch_refresh_progress_survives_restart_under_guard() { let (observation, write) = fixture_observation( &session_id, ordinal, - Some((AnchorProvenanceRelationV2::Supersedes, first_anchor.clone())), + Some((AnchorProvenanceRelation::Supersedes, first_anchor.clone())), false, ); Box::pin(persist_fixture(&runtime, observation, write)).await; @@ -2087,14 +2087,16 @@ async fn explicit_discovery_visits_only_output_effects_past_frontier() { ); assert_eq!(filtered, 1, "one output-producing effect is pending"); - let pending = runtime - .registered_database(HostAdmissionScope::Profile) - .expect("profile registered database") - .pending_session_temporal_refresh_page_result(128, 1, None) - .await - .unwrap() - .into_parts() - .0; + let pending = SessionTemporalAccess::new( + runtime + .registered_database(HostAdmissionScope::Profile) + .expect("profile registered database"), + ) + .pending_session_temporal_refresh_page_result(128, 1, None) + .await + .unwrap() + .into_parts() + .0; assert_eq!(pending.len(), 1); assert_eq!(pending[0].session_id(), &session_id); @@ -2159,7 +2161,7 @@ async fn explicit_discovery_rediscovery_is_bounded_and_non_mutating() { let mut active_rows_scanned = 0usize; let mut pages = 0usize; loop { - let page = db + let page = SessionTemporalAccess::new(db) .pending_session_temporal_refresh_page_result(2, 1, cursor.as_ref()) .await .expect("discover missing native relation projection"); diff --git a/crates/tracedecay-session-temporal-store/src/rebuild.rs b/crates/tracedecay-session-temporal-store/src/rebuild.rs index e8deb218a0..328ab6f49e 100644 --- a/crates/tracedecay-session-temporal-store/src/rebuild.rs +++ b/crates/tracedecay-session-temporal-store/src/rebuild.rs @@ -7,7 +7,8 @@ use tracedecay_store::{ SessionGenerationRebuildReceiptV1, SessionGenerationRebuildRequestV1, SessionStoreError, SessionStoreResult, }; -use tracedecay_temporal_query::ports::{ExecutionControl, TemporalPortError}; +use tracedecay_temporal_query::execution::ExecutionControl; +use tracedecay_temporal_query::ports::TemporalPortError; use super::projection::{canonical_parent_message_resolver, validate_final_projection_receipt}; use super::query::{ diff --git a/crates/tracedecay-session-temporal-store/src/refresh.rs b/crates/tracedecay-session-temporal-store/src/refresh.rs index b901625472..bd0414e2a9 100644 --- a/crates/tracedecay-session-temporal-store/src/refresh.rs +++ b/crates/tracedecay-session-temporal-store/src/refresh.rs @@ -16,7 +16,7 @@ use tracedecay_store::{ SessionRefreshStateV1, SessionRefreshTerminalStateV1, SessionStoreError, SessionStoreResult, SessionTemporalProjectionBatchReceiptV1, SessionTemporalProjectionBatchV1, }; -use tracedecay_temporal_query::ports::ExecutionControl; +use tracedecay_temporal_query::execution::ExecutionControl; use super::cursor_keys::ensure_active_session_cursor_key_in_transaction; use super::projection::{ @@ -2489,33 +2489,6 @@ mod tests { ) } - #[test] - fn generation_copy_pages_keep_every_source_table() { - assert!(!GENERATION_COPY_STATEMENTS.is_empty()); - for statement in GENERATION_COPY_STATEMENTS { - let table = generation_copy_source_table(statement) - .expect("every generation copy names its source table"); - let insert = generation_copy_page_insert_sql(statement); - assert!( - insert.starts_with("INSERT OR IGNORE INTO"), - "{table}: a replayed page must not fail the primary key" - ); - assert!( - insert.contains("AND rowid > ?4 AND rowid <= ?5"), - "{table}: a page must be a bounded rowid range" - ); - let end_sql = generation_copy_page_end_sql(table); - assert!(end_sql.contains(&format!("FROM {table}"))); - assert!(end_sql.contains("LIMIT 32")); - let resume_sql = generation_copy_resume_sql(table); - assert!( - resume_sql.contains("OFFSET"), - "{table}: a later pass must skip rows already committed" - ); - assert!(resume_sql.contains(&format!("COUNT(*) FROM {table}"))); - } - } - #[test] fn progress_timestamp_uses_authoritative_validation_boundary() { let authoritative_validation_time = UtcMicros(1_000_000); diff --git a/crates/tracedecay-session-temporal-store/src/registered_lcm_render.rs b/crates/tracedecay-session-temporal-store/src/registered_lcm_render.rs index a1c09679a3..d0cd4acac5 100644 --- a/crates/tracedecay-session-temporal-store/src/registered_lcm_render.rs +++ b/crates/tracedecay-session-temporal-store/src/registered_lcm_render.rs @@ -14,6 +14,7 @@ use tracedecay_lcm::contracts::{ LcmSourceRef, LcmStorageKind, LcmSummaryNode, LcmSummaryNodeOverview, validate_payload_ref, }; use tracedecay_lcm::raw::{RAW_MESSAGE_METADATA_SELECT_COLUMNS, raw_message_metadata_from_row}; +use tracedecay_lcm::schema::SUMMARY_VISIBLE_SQL; use tracedecay_runtime_core::db::build_qmark_placeholders; use tracedecay_runtime_core::db::engine::{QueryExecutor, Row, Value, params, params_from_iter}; @@ -57,11 +58,13 @@ async fn session_summary_ids( ) -> Result, LcmError> { let mut rows = query( snapshot, - "SELECT node_id - FROM lcm_summary_nodes - WHERE provider = ?1 AND session_id = ?2 - ORDER BY depth, created_at, node_id - LIMIT 20", + &format!( + "SELECT n.summary_id + FROM session_summary_nodes n + WHERE n.provider = ?1 AND n.session_id = ?2 AND {SUMMARY_VISIBLE_SQL} + ORDER BY n.depth, n.created_at, n.summary_id + LIMIT 20" + ), params![provider, session_id], ) .await?; @@ -163,7 +166,6 @@ pub(super) async fn expand( summary_sources: Vec::new(), payload_ref: None, from_current_session: Some(true), - externalized_note: None, source_pagination: None, } } @@ -186,7 +188,6 @@ pub(super) async fn expand( summary_sources: Vec::new(), payload_ref, from_current_session: Some(from_current_session), - externalized_note: None, source_pagination: None, } } @@ -227,7 +228,6 @@ pub(super) async fn expand( summary_sources, payload_ref: None, from_current_session: None, - externalized_note: None, source_pagination: Some(source_pagination), } } @@ -249,7 +249,6 @@ pub(super) async fn expand( summary_sources: Vec::new(), payload_ref: Some(payload_ref), from_current_session: None, - externalized_note: None, source_pagination: None, } } @@ -273,17 +272,19 @@ async fn describe_counts( ) -> Result { let mut rows = query( snapshot, - "SELECT - (SELECT COUNT(*) FROM lcm_raw_messages - WHERE provider = ?1 AND session_id = ?2), - (SELECT COUNT(*) FROM lcm_summary_nodes - WHERE provider = ?1 AND session_id = ?2), - (SELECT COUNT(*) FROM lcm_external_payloads - WHERE provider = ?1 AND session_id = ?2), - (SELECT MIN(store_id) FROM lcm_raw_messages - WHERE provider = ?1 AND session_id = ?2), - (SELECT MAX(store_id) FROM lcm_raw_messages - WHERE provider = ?1 AND session_id = ?2)", + &format!( + "SELECT + (SELECT COUNT(*) FROM lcm_raw_messages + WHERE provider = ?1 AND session_id = ?2), + (SELECT COUNT(*) FROM session_summary_nodes n + WHERE n.provider = ?1 AND n.session_id = ?2 AND {SUMMARY_VISIBLE_SQL}), + (SELECT COUNT(*) FROM lcm_external_payloads + WHERE provider = ?1 AND session_id = ?2), + (SELECT MIN(store_id) FROM lcm_raw_messages + WHERE provider = ?1 AND session_id = ?2), + (SELECT MAX(store_id) FROM lcm_raw_messages + WHERE provider = ?1 AND session_id = ?2)" + ), params![provider, session_id], ) .await?; @@ -370,11 +371,13 @@ async fn summary_overviews( ) -> Result, LcmError> { let mut rows = query( snapshot, - "SELECT node_id, conversation_id, depth, summary_text, created_at - FROM lcm_summary_nodes - WHERE provider = ?1 AND session_id = ?2 - ORDER BY depth, created_at, node_id - LIMIT 20", + &format!( + "SELECT n.summary_id, n.conversation_id, n.depth, n.summary_text, n.created_at + FROM session_summary_nodes n + WHERE n.provider = ?1 AND n.session_id = ?2 AND {SUMMARY_VISIBLE_SQL} + ORDER BY n.depth, n.created_at, n.summary_id + LIMIT 20" + ), params![provider, session_id], ) .await?; @@ -406,11 +409,14 @@ async fn describe_summary_node( ) -> Result { let mut rows = query( snapshot, - "SELECT node_id, conversation_id, depth, summary_token_count, - source_token_count, source_time_start, source_time_end, - expand_hint, metadata_json, created_at - FROM lcm_summary_nodes - WHERE provider = ?1 AND session_id = ?2 AND node_id = ?3", + &format!( + "SELECT n.summary_id, n.conversation_id, n.depth, n.summary_token_count, + n.source_token_count, n.source_time_start, n.source_time_end, + n.expand_hint, n.metadata_json, n.created_at + FROM session_summary_nodes n + WHERE n.provider = ?1 AND n.session_id = ?2 AND n.summary_id = ?3 + AND {SUMMARY_VISIBLE_SQL}" + ), params![provider, session_id, node_id], ) .await?; @@ -629,12 +635,14 @@ async fn load_summary_node( ) -> Result { let mut rows = query( snapshot, - "SELECT node_id, provider, conversation_id, session_id, depth, - '' AS summary_text, summary_hash, summary_token_count, - source_token_count, source_time_start, source_time_end, - expand_hint, metadata_json, created_at - FROM lcm_summary_nodes - WHERE node_id = ?1", + &format!( + "SELECT n.summary_id, n.provider, n.conversation_id, n.session_id, n.depth, + '' AS summary_text, n.summary_hash, n.summary_token_count, + n.source_token_count, n.source_time_start, n.source_time_end, + n.expand_hint, n.metadata_json, n.created_at + FROM session_summary_nodes n + WHERE n.summary_id = ?1 AND {SUMMARY_VISIBLE_SQL}" + ), params![node_id], ) .await?; @@ -754,8 +762,8 @@ async fn anchor_store_id( /// Recovers the locator of a raw source whose row retention already dropped. /// /// Publication writes both lineage records from the same manifest source list: -/// the projected `lcm_summary_sources` row carries the `store_id` as text at the -/// source's ordinal (`operations::summary_projection`), and the relation graph +/// the `session_summary_sources` row carries the `store_id` as text at the +/// source's ordinal (`operations::publication`), and the relation graph /// carries the anchor at that same ordinal (`relations::build_graph` enumerates /// the same sequence). Retention drops the raw row but never the lineage, so the /// projected record still names the locator the anchor can no longer reach. @@ -774,8 +782,8 @@ async fn retention_dropped_store_id( let mut rows = query( snapshot, "SELECT source_id - FROM lcm_summary_sources - WHERE node_id = ?1 AND ordinal = ?2 AND source_kind = 'raw_message'", + FROM session_summary_sources + WHERE summary_id = ?1 AND ordinal = ?2 AND source_kind = 'raw_message'", params![summary_id, ordinal], ) .await?; @@ -934,13 +942,15 @@ async fn load_summary_nodes( .cloned() .map(Value::Text) .collect::>(); + // Children are the lineage of an already-visible parent, so they are read + // without the visibility rule; the parent's availability governs the page. let sql = format!( - "SELECT node_id, provider, conversation_id, session_id, depth, + "SELECT summary_id, provider, conversation_id, session_id, depth, '' AS summary_text, summary_hash, summary_token_count, source_token_count, source_time_start, source_time_end, expand_hint, metadata_json, created_at - FROM lcm_summary_nodes - WHERE node_id IN ({placeholders})" + FROM session_summary_nodes + WHERE summary_id IN ({placeholders})" ); let mut rows = query(snapshot, &sql, params_from_iter(values)).await?; let mut out = BTreeMap::new(); diff --git a/crates/tracedecay-session-temporal-store/src/registered_lcm_render/tests.rs b/crates/tracedecay-session-temporal-store/src/registered_lcm_render/tests.rs index 2dfcd7faec..1ba6722652 100644 --- a/crates/tracedecay-session-temporal-store/src/registered_lcm_render/tests.rs +++ b/crates/tracedecay-session-temporal-store/src/registered_lcm_render/tests.rs @@ -398,7 +398,10 @@ async fn registered_metadata_rows_do_not_fabricate_full_raw_messages() { async fn session_describe_reports_the_message_not_an_empty_stub() { let directory = tempdir().expect("temporary session store"); let runtime = seeded_render_fixture(directory.path()).await; - let content = "canonical raw message plus hidden tail"; + let content = format!( + "canonical raw {} hidden tail", + "x".repeat(tracedecay_lcm::MAX_DERIVED_SNIPPET_CHARS) + ); runtime .registered_database(HostAdmissionScope::Profile) .expect("registered session database") @@ -406,11 +409,11 @@ async fn session_describe_reports_the_message_not_an_empty_stub() { .expect("registered writer") .execute_batch(&format!( "UPDATE lcm_raw_messages - SET content = '{content}', snippet_text = 'canonical raw' + SET content = '{content}' WHERE message_id = 'message-a';" )) .await - .expect("shorten the stored preview without shortening the message"); + .expect("store a message longer than its derived preview"); let snapshot = runtime .registered_database(HostAdmissionScope::Profile) .expect("registered session database") @@ -435,7 +438,10 @@ async fn session_describe_reports_the_message_not_an_empty_stub() { .iter() .find(|message| message.message_id == "message-a") .expect("describe must list the captured message"); - assert_eq!(overview.content_preview, "canonical raw"); + assert_eq!( + overview.content_preview, + tracedecay_lcm::retrieval_content::derived_text_for_snippet(&content) + ); assert!(!overview.content_preview.contains("hidden tail")); assert_eq!( overview.content_range.total_chars, diff --git a/crates/tracedecay-session-temporal-store/src/relation_projection.rs b/crates/tracedecay-session-temporal-store/src/relation_projection.rs index ccedf01482..cf0cdb3fd7 100644 --- a/crates/tracedecay-session-temporal-store/src/relation_projection.rs +++ b/crates/tracedecay-session-temporal-store/src/relation_projection.rs @@ -2,7 +2,7 @@ use std::collections::BTreeMap; use std::sync::Arc; use tracedecay_domain::{ - AgentInstanceId, AnchorProvenanceRelationV2, CanonicalObservationEnvelopeV1, CopyProofV1, + AgentInstanceId, AnchorProvenanceRelation, CanonicalObservationEnvelopeV1, CopyProofV1, DurableObservationV1, MessageId, MessageOccurrenceIdV1, RetrievalAnchorId, RetrievalAnchorRecord, SessionId, SessionProjectionGenerationV1, TemporalValidityV1, ThreadId, UtcMicros, @@ -873,7 +873,7 @@ async fn reconstruct_occurrences( copied_from_anchor_ids: anchor .source_anchors() .iter() - .filter(|source| source.relation() == AnchorProvenanceRelationV2::CopiedFrom) + .filter(|source| source.relation() == AnchorProvenanceRelation::CopiedFrom) .map(|source| source.anchor_id().clone()) .collect(), thread_id: row diff --git a/crates/tracedecay-session-temporal-store/src/render.rs b/crates/tracedecay-session-temporal-store/src/render.rs index f8bf10893d..9bd4e83a63 100644 --- a/crates/tracedecay-session-temporal-store/src/render.rs +++ b/crates/tracedecay-session-temporal-store/src/render.rs @@ -148,8 +148,6 @@ mod tests { content_hash: "hash".to_string(), storage_kind: LcmStorageKind::Inline, payload_ref: None, - legacy_source: false, - legacy_truncated: false, metadata_json: None, }), raw_message_metadata: None, @@ -175,7 +173,6 @@ mod tests { summary_sources: vec![source(1), source(2), source(3), source(4)], payload_ref: None, from_current_session: None, - externalized_note: None, source_pagination: None, }; let hydration = vec![ @@ -269,7 +266,6 @@ mod tests { summary_sources: Vec::new(), payload_ref: None, from_current_session: None, - externalized_note: None, source_pagination: None, }; diff --git a/crates/tracedecay-session-temporal-store/src/retrieval.rs b/crates/tracedecay-session-temporal-store/src/retrieval.rs index ee2ee59bb1..5b155f10f6 100644 --- a/crates/tracedecay-session-temporal-store/src/retrieval.rs +++ b/crates/tracedecay-session-temporal-store/src/retrieval.rs @@ -21,16 +21,20 @@ use tracedecay_domain::{ #[cfg(test)] use tracedecay_runtime_core::db::engine; use tracedecay_temporal_query::candidates::{CandidateChannel, CandidatePlan}; +use tracedecay_temporal_query::execution::await_controlled; +use tracedecay_temporal_query::paging::{ + CANDIDATE_READ_BUDGET, CandidateFieldCaps, CandidatePageSink, CandidateReadState, PageLimits, + PageRequest, PageStatus, TemporalRecordPageSink, +}; use tracedecay_temporal_query::ports::{ - CANDIDATE_READ_BUDGET, CandidateFieldCaps, CandidatePageSink, CandidateReadState, - MeasuredTemporalValue, PageLimits, PageRequest, PageStatus, PortFuture, - TemporalCandidateFilterV1, TemporalCandidatePopulationCount, TemporalExecutionSnapshot, - TemporalMessageTypeFilterV1, TemporalPortError, TemporalPreparedCandidateCohort, - TemporalReadPort, TemporalRecordPageSink, TemporalRetrievalScope, TemporalSessionScopeFilterV1, - TemporalSnapshotRequest, await_controlled, begin_prepared_candidate_pull, - commit_prepared_candidate_pull, + MeasuredTemporalValue, PortFuture, TemporalCandidateFilterV1, TemporalMessageTypeFilterV1, + TemporalPortError, TemporalReadPort, TemporalRetrievalScope, TemporalSessionScopeFilterV1, + TemporalSnapshotRequest, begin_prepared_candidate_pull, commit_prepared_candidate_pull, }; use tracedecay_temporal_query::ranking::RankingCandidate; +use tracedecay_temporal_query::snapshot::{ + TemporalCandidatePopulationCount, TemporalExecutionSnapshot, TemporalPreparedCandidateCohort, +}; mod candidates; mod cursors; @@ -70,7 +74,7 @@ const ROOT_STRICT_POPULATION_COUNT_LIMIT: usize = 4_096; fn temporal_relation_error( error: SessionRelationError, - control: &tracedecay_temporal_query::ports::ExecutionControl, + control: &tracedecay_temporal_query::execution::ExecutionControl, resource: &'static str, ) -> TemporalPortError { if error == SessionRelationError::Cancelled @@ -837,15 +841,15 @@ impl<'a> SessionTemporalReadPort<'a> { let caps = request.candidate_field_caps(); let metadata_cap = caps.map_or( request.max_item_bytes(), - tracedecay_temporal_query::ports::CandidateFieldCaps::metadata_field_bytes, + tracedecay_temporal_query::paging::CandidateFieldCaps::metadata_field_bytes, ); let stable_cap = caps.map_or( request.max_item_bytes(), - tracedecay_temporal_query::ports::CandidateFieldCaps::stable_id_bytes, + tracedecay_temporal_query::paging::CandidateFieldCaps::stable_id_bytes, ); let anchor_cap = caps.map_or( request.max_item_bytes(), - tracedecay_temporal_query::ports::CandidateFieldCaps::anchor_id_bytes, + tracedecay_temporal_query::paging::CandidateFieldCaps::anchor_id_bytes, ); let provider = snapshot_request .provider_scope() @@ -1488,19 +1492,6 @@ impl<'a> SessionTemporalReadPort<'a> { } impl TemporalReadPort for SessionTemporalReadPort<'_> { - fn produce_candidate_page<'a>( - &'a self, - snapshot: &'a TemporalExecutionSnapshot, - plan: &'a CandidatePlan, - request: PageRequest, - sink: &'a mut CandidatePageSink<'_>, - ) -> PortFuture<'a, PageStatus> { - Box::pin(async move { - self.produce_candidates(snapshot.retrieval_scope(), snapshot, plan, &request, sink) - .await - }) - } - fn produce_candidate_page_for_scope<'a>( &'a self, scope: &'a TemporalRetrievalScope, @@ -1515,25 +1506,6 @@ impl TemporalReadPort for SessionTemporalReadPort<'_> { }) } - fn produce_temporal_record_page<'a>( - &'a self, - snapshot: &'a TemporalExecutionSnapshot, - candidates: &'a [RankingCandidate], - request: PageRequest, - sink: &'a mut TemporalRecordPageSink<'_>, - ) -> PortFuture<'a, PageStatus> { - Box::pin(async move { - self.produce_records( - snapshot.retrieval_scope(), - snapshot, - candidates, - &request, - sink, - ) - .await - }) - } - fn produce_temporal_record_page_for_scope<'a>( &'a self, scope: &'a TemporalRetrievalScope, diff --git a/crates/tracedecay-session-temporal-store/src/retrieval/candidates.rs b/crates/tracedecay-session-temporal-store/src/retrieval/candidates.rs index 1a03ff142c..c2a38a67eb 100644 --- a/crates/tracedecay-session-temporal-store/src/retrieval/candidates.rs +++ b/crates/tracedecay-session-temporal-store/src/retrieval/candidates.rs @@ -7,11 +7,12 @@ use tracedecay_runtime_core::db::engine::Value as SqlValue; use tracedecay_temporal_query::candidates::{ CandidateChannel, CandidateClause, is_fts_boolean_operator, }; +use tracedecay_temporal_query::paging::{CandidateFieldCaps, PageRequest}; use tracedecay_temporal_query::ports::{ - CandidateFieldCaps, PageRequest, ReadBudgetAccounting, TemporalExecutionSnapshot, - TemporalPortError, TemporalRetrievalScope, TemporalSnapshotRequest, + ReadBudgetAccounting, TemporalPortError, TemporalRetrievalScope, TemporalSnapshotRequest, }; use tracedecay_temporal_query::ranking::RankingCandidate; +use tracedecay_temporal_query::snapshot::TemporalExecutionSnapshot; use super::super::sql::{TemporalSqlRead, TemporalSqlRow, TemporalSqlRows}; use super::cursors::*; diff --git a/crates/tracedecay-session-temporal-store/src/retrieval/cursors.rs b/crates/tracedecay-session-temporal-store/src/retrieval/cursors.rs index ad9199e47f..ee775d5c65 100644 --- a/crates/tracedecay-session-temporal-store/src/retrieval/cursors.rs +++ b/crates/tracedecay-session-temporal-store/src/retrieval/cursors.rs @@ -3,7 +3,8 @@ use std::cmp; use serde::{Deserialize, Serialize, de::DeserializeOwned}; use tracedecay_domain::SignedCursorKeyRefV1; -use tracedecay_temporal_query::ports::{PageKey, PageRequest, TemporalPortError}; +use tracedecay_temporal_query::paging::{PageKey, PageRequest}; +use tracedecay_temporal_query::ports::TemporalPortError; use super::super::sql::TemporalSqlRow; use super::rows::*; diff --git a/crates/tracedecay-session-temporal-store/src/retrieval/graph_relation_tests.rs b/crates/tracedecay-session-temporal-store/src/retrieval/graph_relation_tests.rs index bd4f217013..67e7961221 100644 --- a/crates/tracedecay-session-temporal-store/src/retrieval/graph_relation_tests.rs +++ b/crates/tracedecay-session-temporal-store/src/retrieval/graph_relation_tests.rs @@ -9,13 +9,16 @@ use tracedecay_domain::{ }; use tracedecay_runtime_core::db::engine::{Executor, TestConnection}; use tracedecay_temporal_query::candidates::CandidateChannel; +use tracedecay_temporal_query::execution::{BindingDigest, ExecutionControl}; use tracedecay_temporal_query::ports::{ - BindingDigest, ExecutionControl, KernelVersions, TemporalCandidateFilterV1, - TemporalExecutionSnapshot, TemporalPortError, TemporalSessionScopeFilterV1, - TemporalSnapshotRequest, TemporalWatermarks, + TemporalCandidateFilterV1, TemporalPortError, TemporalSessionScopeFilterV1, + TemporalSnapshotRequest, }; use tracedecay_temporal_query::ranking::RankingCandidate; use tracedecay_temporal_query::resolution::ValidatedAuthorization; +use tracedecay_temporal_query::snapshot::{ + KernelVersions, TemporalExecutionSnapshot, TemporalWatermarks, +}; use super::SessionTemporalReadPort; use crate::relations::{ diff --git a/crates/tracedecay-session-temporal-store/src/retrieval/records.rs b/crates/tracedecay-session-temporal-store/src/retrieval/records.rs index 8d06bb7572..07d5eca4ac 100644 --- a/crates/tracedecay-session-temporal-store/src/retrieval/records.rs +++ b/crates/tracedecay-session-temporal-store/src/retrieval/records.rs @@ -1,10 +1,10 @@ use tracedecay_domain::TemporalModeV1; use tracedecay_runtime_core::db::engine::Value as SqlValue; -use tracedecay_temporal_query::ports::{ - PageRequest, TemporalExecutionSnapshot, TemporalPortError, TemporalRetrievalScope, -}; +use tracedecay_temporal_query::paging::PageRequest; +use tracedecay_temporal_query::ports::{TemporalPortError, TemporalRetrievalScope}; use tracedecay_temporal_query::ranking::RankingCandidate; +use tracedecay_temporal_query::snapshot::TemporalExecutionSnapshot; mod relations; diff --git a/crates/tracedecay-session-temporal-store/src/retrieval/records/relations.rs b/crates/tracedecay-session-temporal-store/src/retrieval/records/relations.rs index 09d0cf6b27..4c3a4b0113 100644 --- a/crates/tracedecay-session-temporal-store/src/retrieval/records/relations.rs +++ b/crates/tracedecay-session-temporal-store/src/retrieval/records/relations.rs @@ -4,11 +4,13 @@ use std::sync::Arc; use tracedecay_domain::{MessageOccurrenceIdV1, SessionId}; use tracedecay_graph_db::GraphCancellation; use tracedecay_temporal_query::candidates::CandidateChannel; +use tracedecay_temporal_query::execution::ExecutionControl; +use tracedecay_temporal_query::paging::PageRequest; use tracedecay_temporal_query::ports::{ - ExecutionControl, PageRequest, ReadBudgetAccounting, TemporalExecutionSnapshot, - TemporalPortError, TemporalRetrievalScope, + ReadBudgetAccounting, TemporalPortError, TemporalRetrievalScope, }; use tracedecay_temporal_query::ranking::RankingCandidate; +use tracedecay_temporal_query::snapshot::TemporalExecutionSnapshot; use super::super::super::relations::{ SessionRelationError, SessionRelationGraphStore, SessionRelationScope, SummarySourceRef, diff --git a/crates/tracedecay-session-temporal-store/src/retrieval/tests.rs b/crates/tracedecay-session-temporal-store/src/retrieval/tests.rs index a31eee6b9b..d6e3fdefe1 100644 --- a/crates/tracedecay-session-temporal-store/src/retrieval/tests.rs +++ b/crates/tracedecay-session-temporal-store/src/retrieval/tests.rs @@ -15,18 +15,26 @@ use tracedecay_runtime_core::db::{ engine::{Connection, Executor, TestConnection, Value as SqlValue}, }; use tracedecay_temporal_query::candidates::CandidateChannel; +use tracedecay_temporal_query::execution::{ + BindingDigest, ExecutionControl, ExecutionLimits, await_controlled, +}; +use tracedecay_temporal_query::paging::{ + CANDIDATE_READ_BUDGET, CandidateFieldCaps, CandidateReadState, PageLimits, PageRequest, + PageStatus, +}; use tracedecay_temporal_query::plan_temporal_candidates; use tracedecay_temporal_query::ports::{ - BindingDigest, CANDIDATE_READ_BUDGET, CandidateFieldCaps, CandidateReadState, ExecutionControl, - ExecutionLimits, KernelVersions, PageLimits, PageRequest, PageStatus, ReadBudgetAccounting, - TemporalAuthorizedRoot, TemporalExecutionSnapshot, TemporalParticipantAuthorization, - TemporalParticipantGeneration, TemporalParticipantManifest, TemporalPortError, - TemporalPreparedCandidateCohort, TemporalRecord, TemporalRetrievalScope, - TemporalSnapshotRequest, TemporalSourceAccess, TemporalWatermarks, await_controlled, - begin_prepared_candidate_pull, commit_prepared_candidate_pull, + ReadBudgetAccounting, TemporalAuthorizedRoot, TemporalPortError, TemporalRecord, + TemporalRetrievalScope, TemporalSnapshotRequest, begin_prepared_candidate_pull, + commit_prepared_candidate_pull, }; use tracedecay_temporal_query::ranking::RankingCandidate; use tracedecay_temporal_query::resolution::{SummarySourceState, ValidatedAuthorization}; +use tracedecay_temporal_query::snapshot::{ + KernelVersions, TemporalExecutionSnapshot, TemporalParticipantAuthorization, + TemporalParticipantGeneration, TemporalParticipantManifest, TemporalPreparedCandidateCohort, + TemporalSourceAccess, TemporalWatermarks, +}; mod relation_graph_tests; @@ -661,8 +669,7 @@ impl HostAdmissionRetrievalFixture for HostAdmissionTestRuntimeV1 { session_id, generation, occurrence_id, source_observation_id, source_provider, projection_output_ordinal, retrieval_anchor_id, message_id, turn_id, role, knowledge_at, valid_time_json, - evidence_json, sanitized_content_digest, sanitized_content_bytes, - snippet_text, index_text + evidence_json, sanitized_content_digest, sanitized_content_bytes, index_text ) VALUES ( 'session-plan-inside', 1, 'occurrence-plan-inside', @@ -670,7 +677,6 @@ impl HostAdmissionRetrievalFixture for HostAdmissionTestRuntimeV1 { 'message-plan-inside', 'turn-plan-inside', 'user', 20, '{\"kind\":\"unknown\"}', '{}', '0000000000000000000000000000000000000000000000000000000000000000', 38, - 'needle candidate derived needle inside', 'needle candidate derived needle inside' ), ( @@ -678,16 +684,14 @@ impl HostAdmissionRetrievalFixture for HostAdmissionTestRuntimeV1 { 'observation-plan-inside', 'claude', 1, 'anchor-plan-inside-old', 'message-plan-inside-old', 'turn-plan-inside-old', 'assistant', 10, '{\"kind\":\"unknown\"}', '{}', - '0000000000000000000000000000000000000000000000000000000000000000', 27, - 'derived needle older member', 'derived needle older member' + '0000000000000000000000000000000000000000000000000000000000000000', 27, 'derived needle older member' ), ( 'session-plan-inside', 1, 'occurrence-plan-inside-last', 'observation-plan-inside', 'claude', 2, 'anchor-plan-inside-last', 'message-plan-inside-last', 'turn-plan-inside-last', 'assistant', 5, '{\"kind\":\"unknown\"}', '{}', - '0000000000000000000000000000000000000000000000000000000000000000', 26, - 'derived needle last member', 'derived needle last member' + '0000000000000000000000000000000000000000000000000000000000000000', 26, 'derived needle last member' ), ( 'session-plan-outside', 1, 'occurrence-plan-outside', @@ -695,28 +699,29 @@ impl HostAdmissionRetrievalFixture for HostAdmissionTestRuntimeV1 { 'message-plan-outside', 'turn-plan-outside', 'user', 30, '{\"kind\":\"unknown\"}', '{}', '0000000000000000000000000000000000000000000000000000000000000000', 39, - 'needle candidate derived needle outside', 'needle candidate derived needle outside' ); INSERT INTO session_summary_nodes ( - summary_id, session_id, summary_anchor_id, summary_text, index_text, + summary_id, session_id, provider, conversation_id, depth, summary_anchor_id, + summary_text, summary_hash, summary_token_count, source_token_count, source_horizon_json, publication_json, created_at ) VALUES ( - 'summary-plan-inside', 'session-plan-inside', 'anchor-plan-summary', - 'needle summary inside newest', 'needle summary inside newest', '{}', + 'summary-plan-inside', 'session-plan-inside', 'claude', + 'session-plan-inside', 0, 'anchor-plan-summary', + 'needle summary inside newest', 'hash', 1, 1, '{}', '{\"provider\":\"claude\"}', 25 ), ( - 'summary-plan-inside-old', 'session-plan-inside', - 'anchor-plan-summary-old', - 'needle summary inside older', 'needle summary inside older', '{}', + 'summary-plan-inside-old', 'session-plan-inside', 'claude', + 'session-plan-inside', 0, 'anchor-plan-summary-old', + 'needle summary inside older', 'hash', 1, 1, '{}', '{\"provider\":\"claude\"}', 15 ), ( - 'summary-plan-outside', 'session-plan-outside', - 'anchor-plan-summary-outside', - 'needle summary outside', 'needle summary outside', '{}', + 'summary-plan-outside', 'session-plan-outside', 'claude', + 'session-plan-outside', 0, 'anchor-plan-summary-outside', + 'needle summary outside', 'hash', 1, 1, '{}', '{\"provider\":\"claude\"}', 35 ); INSERT INTO session_summary_availability ( @@ -907,26 +912,22 @@ impl HostAdmissionRetrievalFixture for HostAdmissionTestRuntimeV1 { session_id, generation, occurrence_id, source_observation_id, source_provider, projection_output_ordinal, retrieval_anchor_id, role, knowledge_at, valid_time_json, evidence_json, - sanitized_content_digest, sanitized_content_bytes, - snippet_text, index_text + sanitized_content_digest, sanitized_content_bytes, index_text ) VALUES ( 'session-a', 1, 'same-id', 'observation-shared', 'claude', 0, 'same-anchor', 'user', 5, '{\"kind\":\"unknown\"}', '{}', - '0000000000000000000000000000000000000000000000000000000000000000', 12, - 'same content', 'same content' + '0000000000000000000000000000000000000000000000000000000000000000', 12, 'same content' ), ( 'session-b', 1, 'same-id', 'observation-shared', 'claude', 0, 'same-anchor', 'user', 5, '{\"kind\":\"unknown\"}', '{}', - '0000000000000000000000000000000000000000000000000000000000000000', 12, - 'same content', 'same content' + '0000000000000000000000000000000000000000000000000000000000000000', 12, 'same content' ), ( 'session-b', 1, 'source-b', 'observation-shared', 'claude', 1, 'source-anchor-b', 'user', 4, '{\"kind\":\"unknown\"}', '{}', - '0000000000000000000000000000000000000000000000000000000000000000', 6, - 'source', 'source' + '0000000000000000000000000000000000000000000000000000000000000000', 6, 'source' ); INSERT INTO session_assertions ( session_id, generation, assertion_id, assertion_kind, @@ -976,7 +977,7 @@ impl HostAdmissionRetrievalFixture for HostAdmissionTestRuntimeV1 { \"sanitizer_version\":\"derived-sanitizer\" }}}}', '0000000000000000000000000000000000000000000000000000000000000000', 15, - 'member {index}', 'member {index}')", + 'member {index}')", occurrence_id(index), index + 5, ) @@ -1025,8 +1026,7 @@ impl HostAdmissionRetrievalFixture for HostAdmissionTestRuntimeV1 { session_id, generation, occurrence_id, source_observation_id, source_provider, projection_output_ordinal, retrieval_anchor_id, role, knowledge_at, valid_time_json, evidence_json, - sanitized_content_digest, sanitized_content_bytes, - snippet_text, index_text + sanitized_content_digest, sanitized_content_bytes, index_text ) VALUES {occurrences}; INSERT INTO session_derived_evidence ( session_id, generation, evidence_kind, evidence_id, @@ -1082,8 +1082,7 @@ impl HostAdmissionRetrievalFixture for HostAdmissionTestRuntimeV1 { session_id, generation, occurrence_id, source_observation_id, source_provider, projection_output_ordinal, retrieval_anchor_id, role, knowledge_at, valid_time_json, evidence_json, - sanitized_content_digest, sanitized_content_bytes, - snippet_text, index_text + sanitized_content_digest, sanitized_content_bytes, index_text ) VALUES ( 'session-snapshot', 1, 'sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd', @@ -1098,8 +1097,7 @@ impl HostAdmissionRetrievalFixture for HostAdmissionTestRuntimeV1 { \"sanitizer_version\":\"derived-sanitizer\" } }', - '0000000000000000000000000000000000000000000000000000000000000000', 15, - 'derived content', 'derived content' + '0000000000000000000000000000000000000000000000000000000000000000', 15, 'derived content' ); INSERT INTO session_derived_evidence ( session_id, generation, evidence_kind, evidence_id, @@ -1161,14 +1159,13 @@ impl HostAdmissionRetrievalFixture for HostAdmissionTestRuntimeV1 { session_id, generation, occurrence_id, source_observation_id, source_provider, projection_output_ordinal, retrieval_anchor_id, role, knowledge_at, valid_time_json, evidence_json, - sanitized_content_digest, sanitized_content_bytes, - snippet_text, index_text + sanitized_content_digest, sanitized_content_bytes, index_text ) VALUES ( 'session-snapshot', 1, 'occurrence-oversized', 'observation-1', 'claude', 0, 'anchor-evidence', 'user', 1, '{\"kind\":\"unknown\"}', ?1, '0000000000000000000000000000000000000000000000000000000000000000', - 5, 'snippet', 'index' + 5, 'index' )", [oversized_json.clone()], ) @@ -1206,8 +1203,7 @@ impl HostAdmissionRetrievalFixture for HostAdmissionTestRuntimeV1 { session_id, generation, occurrence_id, source_observation_id, source_provider, projection_output_ordinal, retrieval_anchor_id, role, knowledge_at, valid_time_json, evidence_json, - sanitized_content_digest, sanitized_content_bytes, - snippet_text, index_text + sanitized_content_digest, sanitized_content_bytes, index_text ) VALUES ( 'session-snapshot', 1, 'occurrence-claude', 'observation-claude', 'claude', 0, 'source-claude', 'user', 1, '{\"kind\":\"unknown\"}', @@ -1215,15 +1211,15 @@ impl HostAdmissionRetrievalFixture for HostAdmissionTestRuntimeV1 { \"source_anchor_id\":\"source-claude\", \"sanitization_receipt\":{\"receipt_id\":\"receipt-1\"}}', '0000000000000000000000000000000000000000000000000000000000000000', - 7, - 'snippet', 'index' + 7, 'index' ); INSERT INTO session_summary_nodes ( - summary_id, session_id, summary_anchor_id, summary_text, index_text, + summary_id, session_id, provider, conversation_id, depth, summary_anchor_id, + summary_text, summary_hash, summary_token_count, source_token_count, source_horizon_json, publication_json, created_at ) VALUES ( - 'summary-provider', 'session-snapshot', 'anchor-summary-provider', - 'summary', 'summary', + 'summary-provider', 'session-snapshot', 'test', 'session-snapshot', 0, + 'anchor-summary-provider', 'summary', 'hash', 1, 1, '{\"knowledge_through\":1,\"valid_through\":null}', NULL, 1 ); INSERT INTO session_summary_availability ( @@ -1268,24 +1264,21 @@ impl HostAdmissionRetrievalFixture for HostAdmissionTestRuntimeV1 { session_id, generation, occurrence_id, source_observation_id, source_provider, projection_output_ordinal, retrieval_anchor_id, role, knowledge_at, valid_time_json, evidence_json, - sanitized_content_digest, sanitized_content_bytes, - snippet_text, index_text + sanitized_content_digest, sanitized_content_bytes, index_text ) VALUES ( 'session-snapshot', 1, 'summary-source-at-5', 'summary-history-observation', 'claude', 0, 'shared-summary-source', 'user', 5, '{\"kind\":\"known\",\"valid_at\":5}', '{}', '0000000000000000000000000000000000000000000000000000000000000000', - 11, - 'source at 5', 'source at 5' + 11, 'source at 5' ), ( 'session-snapshot', 1, 'summary-source-at-10', 'summary-history-observation', 'claude', 1, 'shared-summary-source', 'user', 10, '{\"kind\":\"known\",\"valid_at\":10}', '{}', '0000000000000000000000000000000000000000000000000000000000000000', - 12, - 'source at 10', 'source at 10' + 12, 'source at 10' ); INSERT INTO session_current_entities ( session_id, generation, entity_kind, entity_id, @@ -1295,17 +1288,18 @@ impl HostAdmissionRetrievalFixture for HostAdmissionTestRuntimeV1 { NULL, 'summary-source-at-10', '{}' ); INSERT INTO session_summary_nodes ( - summary_id, session_id, summary_anchor_id, summary_text, index_text, + summary_id, session_id, provider, conversation_id, depth, summary_anchor_id, + summary_text, summary_hash, summary_token_count, source_token_count, source_horizon_json, publication_json, created_at ) VALUES ( - 'historical-summary', 'session-snapshot', 'historical-summary-anchor', - 'historical', 'historical', + 'historical-summary', 'session-snapshot', 'test', 'session-snapshot', 0, + 'historical-summary-anchor', 'historical', 'hash', 1, 1, '{\"knowledge_through\":5,\"valid_through\":5}', NULL, 5 ), ( - 'successor-summary', 'session-snapshot', 'successor-summary-anchor', - 'successor', 'successor', + 'successor-summary', 'session-snapshot', 'test', 'session-snapshot', 0, + 'successor-summary-anchor', 'successor', 'hash', 1, 1, '{\"knowledge_through\":10,\"valid_through\":10}', NULL, 10 ); INSERT INTO session_summary_availability ( diff --git a/crates/tracedecay-session-temporal-store/src/retrieval/tests/relation_graph_tests.rs b/crates/tracedecay-session-temporal-store/src/retrieval/tests/relation_graph_tests.rs index 8247d40568..fcd2e92aa0 100644 --- a/crates/tracedecay-session-temporal-store/src/retrieval/tests/relation_graph_tests.rs +++ b/crates/tracedecay-session-temporal-store/src/retrieval/tests/relation_graph_tests.rs @@ -315,8 +315,7 @@ async fn copy_lineage_comes_from_grafeo_without_a_sql_relation_table() { session_id, generation, occurrence_id, source_observation_id, source_provider, projection_output_ordinal, retrieval_anchor_id, role, knowledge_at, valid_time_json, evidence_json, - sanitized_content_digest, sanitized_content_bytes, - snippet_text, index_text + sanitized_content_digest, sanitized_content_bytes, index_text ) VALUES ('session-b', 1, '{target_value}', 'observation-shared', 'fixture-provider', 2, 'graph-target-anchor', 'user', 6, @@ -326,7 +325,7 @@ async fn copy_lineage_comes_from_grafeo_without_a_sql_relation_table() { \"sanitization_receipt\":{{ \"receipt_id\":\"receipt-1\",\"sanitizer_version\":\"fixture\" }}}}', - '{content_digest}', 6, 'target', 'target'), + '{content_digest}', 6, 'target'), ('session-b', 1, '{graph_source_value}', 'observation-shared', 'fixture-provider', 3, 'graph-source-anchor', 'user', 5, '{{\"kind\":\"unknown\"}}', @@ -335,7 +334,7 @@ async fn copy_lineage_comes_from_grafeo_without_a_sql_relation_table() { \"sanitization_receipt\":{{ \"receipt_id\":\"receipt-1\",\"sanitizer_version\":\"fixture\" }}}}', - '{content_digest}', 5, 'graph', 'graph');" + '{content_digest}', 5, 'graph');" ), ) .await diff --git a/crates/tracedecay-session-temporal-store/src/schema_constants.rs b/crates/tracedecay-session-temporal-store/src/schema_constants.rs index 633ae5a3ca..83b7dd192f 100644 --- a/crates/tracedecay-session-temporal-store/src/schema_constants.rs +++ b/crates/tracedecay-session-temporal-store/src/schema_constants.rs @@ -1,6 +1,11 @@ //! Session-temporal schema identity shared with registered-store admission. -pub const SESSION_TEMPORAL_SCHEMA_VERSION: i64 = 4; +/// Version 5 stores occurrence text once: `snippet_text` is a virtual alias +/// of `index_text`, and full-text search indexes only `index_text`. Version 6 +/// makes `session_summary_nodes` the one summary authority: the LCM summary +/// columns are real columns here, `session_summary_sources` carries the +/// lineage, and the summary FTS indexes `summary_text` alone. +pub const SESSION_TEMPORAL_SCHEMA_VERSION: i64 = 6; pub const TEMPORAL_TABLE_COLUMNS: &[(&str, &[&str])] = &[ ( @@ -12,14 +17,27 @@ pub const TEMPORAL_TABLE_COLUMNS: &[(&str, &[&str])] = &[ &[ "summary_id", "session_id", + "provider", + "conversation_id", + "depth", "summary_anchor_id", "summary_text", - "index_text", + "summary_hash", + "summary_token_count", + "source_token_count", + "source_time_start", + "source_time_end", + "expand_hint", + "metadata_json", "source_horizon_json", "publication_json", "created_at", ], ), + ( + "session_summary_sources", + &["summary_id", "ordinal", "source_kind", "source_id"], + ), ( "session_relation_receipts", &[ @@ -236,7 +254,6 @@ pub const TEMPORAL_TABLE_COLUMNS: &[(&str, &[&str])] = &[ "evidence_json", "sanitized_content_digest", "sanitized_content_bytes", - "snippet_text", "index_text", ], ), @@ -328,6 +345,6 @@ pub const TEMPORAL_TABLE_COLUMNS: &[(&str, &[&str])] = &[ "checked_at", ], ), - ("session_occurrences_fts", &["index_text", "snippet_text"]), - ("session_summary_nodes_fts", &["summary_text", "index_text"]), + ("session_occurrences_fts", &["index_text"]), + ("session_summary_nodes_fts", &["summary_text"]), ]; diff --git a/crates/tracedecay-session-temporal-store/src/sql.rs b/crates/tracedecay-session-temporal-store/src/sql.rs index 940cf6fba6..87a7654800 100644 --- a/crates/tracedecay-session-temporal-store/src/sql.rs +++ b/crates/tracedecay-session-temporal-store/src/sql.rs @@ -23,15 +23,13 @@ pub(super) const GENERATION_COPY_STATEMENTS: &[&str] = &[ source_provider, projection_output_ordinal, retrieval_anchor_id, thread_id, thread_grouping_json, turn_id, turn_grouping_json, message_id, agent_id, role, knowledge_at, valid_time_json, evidence_json, - sanitized_content_digest, sanitized_content_bytes, - snippet_text, index_text + sanitized_content_digest, sanitized_content_bytes, index_text ) SELECT session_id, ?2, occurrence_id, source_observation_id, source_provider, projection_output_ordinal, retrieval_anchor_id, thread_id, thread_grouping_json, turn_id, turn_grouping_json, message_id, agent_id, role, knowledge_at, valid_time_json, evidence_json, - sanitized_content_digest, sanitized_content_bytes, - snippet_text, index_text + sanitized_content_digest, sanitized_content_bytes, index_text FROM session_occurrences WHERE session_id = ?1 AND generation = ?3", "INSERT INTO session_turn_members ( session_id, generation, turn_id, occurrence_id, ordinal diff --git a/crates/tracedecay-session-temporal-store/src/store.rs b/crates/tracedecay-session-temporal-store/src/store.rs index d924a5a7d3..2d2beeb382 100644 --- a/crates/tracedecay-session-temporal-store/src/store.rs +++ b/crates/tracedecay-session-temporal-store/src/store.rs @@ -4,7 +4,7 @@ use std::{ }; use tracedecay_graph_db::GraphCancellation; -use tracedecay_temporal_query::ports::ExecutionControl; +use tracedecay_temporal_query::execution::ExecutionControl; use tracedecay_store::{ SessionGenerationActivatePermit, SessionGenerationActivationReceiptV1, @@ -388,7 +388,7 @@ mod tests { #[test] fn graph_cancellation_observes_the_callers_execution_control() { - let control = tracedecay_temporal_query::ports::ExecutionControl::default(); + let control = tracedecay_temporal_query::execution::ExecutionControl::default(); let cancellation = execution_control_graph_cancellation(&control); assert!(!cancellation.is_cancelled()); @@ -398,13 +398,13 @@ mod tests { #[test] fn graph_cancellation_observes_deadlines_and_work_budgets() { - let expired = tracedecay_temporal_query::ports::ExecutionControl::new(Some( + let expired = tracedecay_temporal_query::execution::ExecutionControl::new(Some( std::time::Instant::now(), )); assert!(execution_control_graph_cancellation(&expired).is_cancelled()); let budgeted = - tracedecay_temporal_query::ports::ExecutionControl::default().with_work_limit(1); + tracedecay_temporal_query::execution::ExecutionControl::default().with_work_limit(1); let cancellation = execution_control_graph_cancellation(&budgeted); assert!(!cancellation.is_cancelled()); assert!(cancellation.is_cancelled()); diff --git a/crates/tracedecay-session-temporal-store/src/support.rs b/crates/tracedecay-session-temporal-store/src/support.rs index d765ab9357..52b093afa9 100644 --- a/crates/tracedecay-session-temporal-store/src/support.rs +++ b/crates/tracedecay-session-temporal-store/src/support.rs @@ -1,14 +1,5 @@ //! Crate-local helpers that previously lived as `pub(crate)` global-db internals. -use tracedecay_domain::{CanonicalObservationEnvelopeV1, DurableObservationV1, ObservationScopeV1}; -use tracedecay_sessions::runtime::claude::{ - ClaudeRecordContext, ClaudeRecordDisposition, map_sanitized_claude_record, -}; -use tracedecay_store::{ - ObservationProjection, ProjectionSkipReason, ProjectionStoreResult, SessionRecord, - derive_canonical_projection, -}; - /// Millisecond-scale Unix timestamps are at least 13 digits. pub(crate) const UNIX_TIMESTAMP_MILLIS_THRESHOLD: i64 = 1_000_000_000_000; @@ -50,95 +41,3 @@ pub(crate) fn record_hydration_emitted_bytes(count: usize) { #[cfg(not(feature = "hotpath"))] let _ = count; } - -/// Same composition as the observation-projection derive path: canonical -/// envelopes go through store authority; legacy Claude records use the public -/// sessions mapper. Global-db calls this so the two projections cannot drift. -pub fn derive_projection( - observation: &DurableObservationV1, -) -> ProjectionStoreResult { - match observation.source().provider().as_str() { - "claude" if decode_canonical_envelope(observation.payload()).is_ok() => { - derive_canonical_projection(observation) - } - "claude" => derive_claude_projection(observation), - _ => derive_canonical_projection(observation), - } -} - -fn decode_canonical_envelope( - payload: &serde_json::Value, -) -> Result { - serde::Deserialize::deserialize(payload) -} - -fn derive_claude_projection( - observation: &DurableObservationV1, -) -> ProjectionStoreResult { - let session_id = observation.source().session_id().as_str(); - let payload = observation.payload(); - let durable_message_id = payload - .pointer("/message/id") - .and_then(serde_json::Value::as_str) - .or_else(|| payload.get("uuid").and_then(serde_json::Value::as_str)) - .filter(|id| !id.is_empty()); - let durable_tool_event_ids = payload - .pointer("/message/content") - .and_then(serde_json::Value::as_array) - .into_iter() - .flatten() - .filter_map(|item| { - item.get("id") - .or_else(|| item.get("tool_use_id")) - .and_then(serde_json::Value::as_str) - .filter(|id| !id.is_empty()) - .map(str::to_owned) - }) - .collect::>(); - let (project_key, project_path) = match observation.scope() { - ObservationScopeV1::Profile => ("user", "user"), - ObservationScopeV1::Project { project_id } => (project_id.as_str(), project_id.as_str()), - }; - let source_path = (observation.source().source_key() != observation.source().session_id()) - .then(|| observation.source().source_key().as_str()); - let context = ClaudeRecordContext { - session_id, - project_key, - project_path, - file_generation: observation.identity().generation().file_id(), - offset: observation.identity().position().start(), - session_cwd: None, - source_path, - raw_message_id: durable_message_id, - raw_tool_event_ids: &durable_tool_event_ids, - raw_hook_tool_use_id: None, - }; - - match map_sanitized_claude_record(payload, &context) { - ClaudeRecordDisposition::Message { draft, message } => { - let draft = *draft; - let message = *message; - let timestamp = message.timestamp; - let session = SessionRecord { - provider: "claude".to_string(), - session_id: draft.session_id, - project_key: draft.project_key, - project_path: draft.project_path, - title: draft.title, - started_at: timestamp, - ended_at: timestamp, - transcript_path: None, - metadata_json: draft.metadata_json, - parent_session_id: draft.parent_session_id, - is_subagent: draft.is_subagent, - agent_id: draft.agent_id, - parent_tool_use_id: draft.parent_tool_use_id, - }; - ObservationProjection::for_message(observation, session, message) - } - ClaudeRecordDisposition::NonConversational => ObservationProjection::for_skip( - observation, - ProjectionSkipReason::NonConversationalRecord, - ), - } -} diff --git a/crates/tracedecay-session-temporal-store/tests/hotpath_coverage.rs b/crates/tracedecay-session-temporal-store/tests/hotpath_coverage.rs index 1c0ccf5919..07f05db658 100644 --- a/crates/tracedecay-session-temporal-store/tests/hotpath_coverage.rs +++ b/crates/tracedecay-session-temporal-store/tests/hotpath_coverage.rs @@ -25,7 +25,6 @@ fn run_hydration_render_workload() -> usize { summary_sources: Vec::new(), payload_ref: None, from_current_session: None, - externalized_note: None, source_pagination: None, }; diff --git a/crates/tracedecay-sessions/benches/cursor_dispatch_model.rs b/crates/tracedecay-sessions/benches/cursor_dispatch_model.rs index 349a5f021b..dd16ef70a5 100644 --- a/crates/tracedecay-sessions/benches/cursor_dispatch_model.rs +++ b/crates/tracedecay-sessions/benches/cursor_dispatch_model.rs @@ -21,7 +21,7 @@ use std::time::Instant; use serde::Serialize; use serde_json::{Value, json}; use tempfile::TempDir; -use tracedecay_sessions::runtime::cursor::parent_dispatch_model_for_subagent_with_receipt; +use tracedecay_sessions::runtime::hosts::cursor::parent_dispatch_model_for_subagent_with_receipt; const PARENT_SESSION_ID: &str = "P"; const AGENT_ID: &str = "X"; diff --git a/crates/tracedecay-sessions/benches/git_evidence_bounded_reads.rs b/crates/tracedecay-sessions/benches/git_evidence_bounded_reads.rs index 0024856137..c883ae9692 100644 --- a/crates/tracedecay-sessions/benches/git_evidence_bounded_reads.rs +++ b/crates/tracedecay-sessions/benches/git_evidence_bounded_reads.rs @@ -287,9 +287,7 @@ fn main() { .expect("open graph view") { GitEvidenceGraphHead::Indexed(view) => view, - GitEvidenceGraphHead::Unpublished | GitEvidenceGraphHead::Legacy { .. } => { - panic!("seeded head must be indexed") - } + GitEvidenceGraphHead::Unpublished => panic!("seeded head must be indexed"), }; let mut reads = Vec::new(); diff --git a/crates/tracedecay-sessions/benches/host_transcript_io.rs b/crates/tracedecay-sessions/benches/host_transcript_io.rs index 87e35b477d..3e018924c1 100644 --- a/crates/tracedecay-sessions/benches/host_transcript_io.rs +++ b/crates/tracedecay-sessions/benches/host_transcript_io.rs @@ -10,8 +10,8 @@ use std::time::{Duration, SystemTime}; use criterion::{Criterion, criterion_group, criterion_main}; use tempfile::TempDir; +use tracedecay_sessions::runtime::hosts::vibe::VibeSource; use tracedecay_sessions::runtime::source::{TranscriptDiscoveryBounds, TranscriptSource}; -use tracedecay_sessions::runtime::vibe::VibeSource; fn write_vibe_session(root: &Path, name: &str, mtime_secs: u64) { let dir = root.join(name); diff --git a/crates/tracedecay-sessions/src/admission/mod.rs b/crates/tracedecay-sessions/src/admission/mod.rs index d761f5da53..cf98411eb8 100644 --- a/crates/tracedecay-sessions/src/admission/mod.rs +++ b/crates/tracedecay-sessions/src/admission/mod.rs @@ -22,8 +22,8 @@ use tracedecay_domain::{ CanonicalObservationIdV1, ObservationScopeV1, ObservationSourceCursorV1, ObservationSourceIdentityV1, SanitizationReceiptV1, }; +use tracedecay_store::ParseOffset; use tracedecay_store::observation::{CursorAdvanceOutcome, ObservationCursorAdvance}; -use tracedecay_store::{ObservationBatchFallbackCause, ParseOffset}; use crate::observation::{ CaptureObservationOutcome, CaptureObservationRequest, ObservationCancellation, @@ -80,7 +80,6 @@ pub struct HostAdmissionOutcome { #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum HostAdmissionRecovery { - BatchRequiresScalarFallback(ObservationBatchFallbackCause), DeterministicContentRefusal, } @@ -116,17 +115,6 @@ impl HostAdmissionOutcome { } } - #[hotpath::skip] - pub const fn batch_requires_scalar_fallback(cause: ObservationBatchFallbackCause) -> Self { - Self { - status: HostAdmissionStatus::Backpressured, - retryable: true, - reason_code: Some("batch_requires_scalar_fallback"), - recovery: Some(HostAdmissionRecovery::BatchRequiresScalarFallback(cause)), - storage_cause: None, - } - } - #[hotpath::skip] pub const fn deterministic_content_refusal(reason_code: &'static str) -> Self { Self { @@ -1039,9 +1027,6 @@ pub(crate) mod test_support { ObservationApplicationError::Cancelled => { HostAdmissionOutcome::retained_backpressured("admission_cancelled") } - ObservationApplicationError::Store( - ObservationStoreError::BatchRequiresScalarFallback { cause }, - ) => HostAdmissionOutcome::batch_requires_scalar_fallback(cause), _ => HostAdmissionOutcome::registered_authority_unavailable(), } } diff --git a/crates/tracedecay-sessions/src/host_ports.rs b/crates/tracedecay-sessions/src/host_ports.rs index 3c36a14cc8..36f730f120 100644 --- a/crates/tracedecay-sessions/src/host_ports.rs +++ b/crates/tracedecay-sessions/src/host_ports.rs @@ -6,44 +6,18 @@ //! spawner (root `src/hooks/`). Depending on either from here would point the //! session layer back at the composition root. //! -//! The Hermes pin resolver and the unregistered-admission factory are -//! process-global slots the composition root fills once during startup; each -//! reads as "unwired" until then. The session review scheduler is different: +//! The unregistered-admission factory is a process-global slot the +//! composition root fills once during startup; it reads as "unwired" until +//! then. The session review scheduler is different: //! it is an explicit [`session_review::SessionReviewPort`] value that a user //! ingest pass receives through its `SessionIngestAuthority`, so an unwired //! pass is a typed refusal rather than a silent skip. -//! -//! Root startup must call [`hermes_profile_pin::register`] before any -//! transcript ingest runs. use std::future::Future; use std::path::{Path, PathBuf}; use std::pin::Pin; use std::sync::OnceLock; -/// Reads the pinned TraceDecay project root out of a Hermes profile config. -/// -/// The parser is a host-bundle concern (`tracedecay-agent-hosts`); only the -/// answer matters here. An unregistered slot reports "no pin", which makes -/// legacy Hermes state stores skip rather than mis-attribute. -pub mod hermes_profile_pin { - use super::{OnceLock, Path}; - - pub type Resolver = fn(&Path) -> Option; - - static RESOLVER: OnceLock = OnceLock::new(); - - /// Installs the host-bundle resolver. First call wins. - pub fn register(resolver: Resolver) { - let _ = RESOLVER.set(resolver); - } - - /// Reads the pinned project root, or `None` when unwired. - pub fn resolve(config_path: &Path) -> Option { - RESOLVER.get().and_then(|resolver| resolver(config_path)) - } -} - /// Schedules the post-ingest user session review. /// /// The review hint is delivered through the root-owned daemon client. This diff --git a/crates/tracedecay-sessions/src/observation.rs b/crates/tracedecay-sessions/src/observation.rs index c7303ff0af..d95923ca4c 100644 --- a/crates/tracedecay-sessions/src/observation.rs +++ b/crates/tracedecay-sessions/src/observation.rs @@ -19,7 +19,7 @@ use tracedecay_store::{ ObservationCursorPort, ObservationPersistOutcome, ObservationProjectionStatus, ObservationReplayRequest, ObservationStore, ObservationStoreError, ObservationWrite, SESSION_MESSAGE_PROJECTOR_VERSION, StoredObservation, - build_observation_resolution_authorization_v1, build_observation_retrieval_anchor_v2, + build_observation_resolution_authorization_v1, build_observation_retrieval_anchor, }; use crate::repository_provenance::RepositoryProvenanceAdmissionContext; @@ -565,7 +565,7 @@ where ) }, ); - let retrieval_anchor = build_observation_retrieval_anchor_v2( + let retrieval_anchor = build_observation_retrieval_anchor( &observation, projection_generation.clone(), ingested_at, diff --git a/crates/tracedecay-sessions/src/observation_test.rs b/crates/tracedecay-sessions/src/observation_test.rs index 009a871024..4ee7ebc440 100644 --- a/crates/tracedecay-sessions/src/observation_test.rs +++ b/crates/tracedecay-sessions/src/observation_test.rs @@ -8,9 +8,9 @@ use serde_json::{Value, json}; use tempfile::TempDir; use tracedecay_domain::{ EvidenceAvailabilityV1, MAX_OBSERVATION_STRUCTURE_DEPTH, MAX_OBSERVATION_STRUCTURE_VALUES, - ObservationScopeV1, ObservationSourceGenerationV1, ObservationSourceIdentityV1, - ObservationSourceRangeV1, ProjectId, RepositoryId, RetrievalAnchorTargetV2, SessionId, - WorktreeId, + ObservationOrderingDomainV1, ObservationScopeV1, ObservationSourceGenerationV1, + ObservationSourceIdentityV1, ObservationSourceRangeV1, ProjectId, RepositoryId, + RetrievalAnchorTarget, SessionId, WorktreeId, }; use tracedecay_store::observation::{ CursorAdvanceOutcome, NonDurableFrameReason, ObservationCursorAdvance, @@ -20,7 +20,7 @@ use tracedecay_store::{ ObservationStoreResult, }; -use tracedecay_privacy::{ClaudeSanitizerPolicyV1, parse_claude_record_v1}; +use tracedecay_privacy::{ClaudeSanitizerPolicyV1, parse_observation_record_v1}; use super::*; @@ -312,9 +312,10 @@ fn request_at_for_session( ) -> CaptureClaudeObservationRequest { let encoded_frame = serde_json::to_vec(record).unwrap(); let end = start + u64::try_from(encoded_frame.len()).unwrap(); - let parsed_record = parse_claude_record_v1( + let parsed_record = parse_observation_record_v1( &encoded_frame, ObservationSourceRangeV1::new(start, end).unwrap(), + ObservationOrderingDomainV1::FileBytes, ) .unwrap(); let source = ObservationSourceIdentityV1::new(SessionId::new(session_id).unwrap()).unwrap(); @@ -323,7 +324,14 @@ fn request_at_for_session( }; let generation = ObservationSourceGenerationV1::new(1).unwrap(); let expected_cursor = (start != 0).then(|| { - ObservationSourceCursorV1::new(source.clone(), scope.clone(), generation, start).unwrap() + ObservationSourceCursorV1::for_ordering( + source.clone(), + scope.clone(), + generation, + ObservationOrderingDomainV1::FileBytes, + start, + ) + .unwrap() }); let identity = ObservationIdentityMaterialV1::new( source, @@ -556,8 +564,8 @@ async fn repository_provenance_is_bound_to_the_sanitized_observation_write() { Some(outcome.receipt().observation().observation_id()) ); assert!(matches!( - attachment.anchor().map(tracedecay_domain::RetrievalAnchorRecordV2::target), - Some(RetrievalAnchorTargetV2::RepositoryCapture { capture_id, .. }) + attachment.anchor().map(tracedecay_domain::RetrievalAnchorRecord::target), + Some(RetrievalAnchorTarget::RepositoryCapture { capture_id, .. }) if capture_id == provenance.capture_id() )); let encoded = serde_json::to_string(attachment).unwrap(); @@ -609,9 +617,10 @@ fn request_accepts_only_bounded_parser_evidence_for_the_identity_range() { let retention = || RetentionClass::new("retention.application-test").unwrap(); let raw = b"{}"; - let parsed = parse_claude_record_v1( + let parsed = parse_observation_record_v1( raw, ObservationSourceRangeV1::new(10, 10 + u64::try_from(raw.len()).unwrap()).unwrap(), + ObservationOrderingDomainV1::FileBytes, ) .unwrap(); assert!(matches!( @@ -1165,7 +1174,7 @@ async fn capture_observations_reports_cancellation_after_the_single_batch_commit #[tokio::test] async fn capture_observations_refuses_mixed_privacy_batch_before_persist() { - // `parse_claude_record_v1` (used by `request`/`request_at` below) enforces + // `parse_observation_record_v1` (used by `request`/`request_at` below) enforces // the parser's own 1 MiB ceiling before any record reaches the sanitizer, // so a record large enough to trip that ceiling can never reach // `sanitize_parsed`'s own `RecordSize` disposition through this path. To diff --git a/crates/tracedecay-sessions/src/repository_provenance.rs b/crates/tracedecay-sessions/src/repository_provenance.rs index fee8e0df07..ae0a419f88 100644 --- a/crates/tracedecay-sessions/src/repository_provenance.rs +++ b/crates/tracedecay-sessions/src/repository_provenance.rs @@ -14,13 +14,13 @@ use gix::bstr::ByteSlice; use sha2::{Digest, Sha256}; use tracedecay_domain::canonical_text::{encode_lowercase_hex, encode_tagged_lowercase_hex}; use tracedecay_domain::{ - AnchorDurabilityClass, AnchorSourceGenerationV2, CommitId, CoverageReportV1, + AnchorDurabilityClass, AnchorSourceGeneration, CommitId, CoverageReportV1, DurableObservationV1, EvidenceAvailabilityV1, EvidenceClass, GenerationBoundRepositoryProvenanceV1, PayloadAccessState, PrivacyDomainBoundLocatorDigest, ProjectId, ProjectionGenerationId, RefId, RepositoryDirtyStateV1, RepositoryEvidenceV1, RepositoryId, RepositoryProvenanceV1, RepositoryRemoteIdentityV1, ResolutionAuthorizationV1, - RetrievalAnchorRecordV2, RetrievalAnchorRecordV2Parts, RetrievalAnchorTargetV2, TreeId, - UtcMicros, VectorWatermark, WorktreeId, + RetrievalAnchorRecord, RetrievalAnchorRecordParts, RetrievalAnchorTarget, TreeId, UtcMicros, + VectorWatermark, WorktreeId, }; const MAX_REMOTE_IDENTITY_BYTES: usize = 8 * 1024; @@ -304,7 +304,7 @@ impl<'a> ObservationProjectId<'a> { #[derive(Clone, Debug, PartialEq, Eq)] pub struct PreparedRepositoryProvenanceV1 { availability: EvidenceAvailabilityV1, - anchor: Option, + anchor: Option, } impl PreparedRepositoryProvenanceV1 { @@ -320,7 +320,7 @@ impl PreparedRepositoryProvenanceV1 { &self.availability } - pub fn anchor(&self) -> Option<&RetrievalAnchorRecordV2> { + pub fn anchor(&self) -> Option<&RetrievalAnchorRecord> { self.anchor.as_ref() } } @@ -496,21 +496,19 @@ fn prepare_generation_binding( }; }; let capture = binding.capture(); - let target = RetrievalAnchorTargetV2::RepositoryCapture { + let target = RetrievalAnchorTarget::RepositoryCapture { repository_id: capture.repository_id().clone(), capture_id: binding.capture_id().clone(), receipt: observation.receipt().receipt().clone(), }; - let anchor = RetrievalAnchorRecordV2::new(RetrievalAnchorRecordV2Parts { + let anchor = RetrievalAnchorRecord::new(RetrievalAnchorRecordParts { target, owner: observation.scope().clone(), aliases: vec![], occurred_at: None, ingested_at, evidence_class: EvidenceClass::Observed, - source_generation: AnchorSourceGenerationV2::RepositoryCapture( - binding.capture_id().clone(), - ), + source_generation: AnchorSourceGeneration::RepositoryCapture(binding.capture_id().clone()), projection_generation: projection_generation.clone(), projection_watermark: VectorWatermark::default(), coverage: CoverageReportV1::default(), diff --git a/crates/tracedecay-sessions/src/runtime/git_correlation.rs b/crates/tracedecay-sessions/src/runtime/git_correlation.rs index 7ca2f6fcc1..6ca9268fed 100644 --- a/crates/tracedecay-sessions/src/runtime/git_correlation.rs +++ b/crates/tracedecay-sessions/src/runtime/git_correlation.rs @@ -40,12 +40,6 @@ pub const GIT_CORRELATION_SCHEMA_VERSION: i64 = 5; /// projection metadata) re-publishes an unchanged projection under a distinct /// generation instead of colliding with the previous shape's rows. pub const GIT_EVIDENCE_PROJECTOR_REVISION: &str = "session-git-evidence-projector.v2"; -/// The pre-index projector revision. A verified head that records no projector -/// revision was published under it. Its span and commit rows are identical to -/// the current shape, so full recovery still verifies and merges it, but it -/// carries no query index and answers bounded reads as unavailable until the -/// next publication re-projects it. -pub const GIT_EVIDENCE_LEGACY_PROJECTOR_REVISION_V1: &str = "session-git-evidence-projector.v1"; pub const DEFAULT_SPAN_MERGE_GAP_SECS: i64 = 30 * 60; pub const DEFAULT_SPAN_OBSERVATION_DEBOUNCE_SECS: i64 = 30; // The scope value type and session cap are owned by the LCM engine crate so @@ -1239,12 +1233,10 @@ pub use publication_outbox::{ enqueue_git_evidence_publication, pending_git_evidence_publication_count, replay_pending_git_evidence_publications, replay_pending_git_evidence_publications_outcome, }; -#[cfg(any(test, feature = "test-helpers"))] -pub use store::legacy_git_evidence_manifest_for_test; pub use store::{ AnalyticsSessionTimestamp, AnalyticsSessionTimestampSource, GitCorrelationSessionStore, GitCorrelationWriteTxn, GitEvidenceGraphHead, GitEvidenceGraphView, GitEvidenceProjectionStore, - GitEvidenceProjectorRevision, build_git_evidence_manifest_checked, git_evidence_generation_id, + build_git_evidence_manifest_checked, git_evidence_generation_id, git_evidence_projection_identity, open_git_evidence_graph_view, publish_git_evidence_projection, recover_git_evidence_projection, }; diff --git a/crates/tracedecay-sessions/src/runtime/git_correlation/backfill.rs b/crates/tracedecay-sessions/src/runtime/git_correlation/backfill.rs index f05657bb7f..57dda331f8 100644 --- a/crates/tracedecay-sessions/src/runtime/git_correlation/backfill.rs +++ b/crates/tracedecay-sessions/src/runtime/git_correlation/backfill.rs @@ -906,7 +906,7 @@ pub(super) async fn session_activity_rows( s.started_at, s.ended_at, MIN(m.timestamp), MAX(m.timestamp) FROM sessions s - LEFT JOIN session_messages m + LEFT JOIN lcm_raw_messages m ON m.provider = s.provider AND m.session_id = s.session_id GROUP BY s.provider, s.session_id ORDER BY COALESCE(MAX(m.timestamp), s.ended_at, s.started_at) DESC @@ -943,7 +943,7 @@ pub(super) async fn session_activity_page_after( s.rowid, COALESCE(MAX(m.timestamp), s.ended_at, s.started_at) FROM sessions s - LEFT JOIN session_messages m + LEFT JOIN lcm_raw_messages m ON m.provider = s.provider AND m.session_id = s.session_id GROUP BY s.rowid, s.provider, s.session_id HAVING COALESCE(MAX(m.timestamp), s.ended_at, s.started_at) > ?1 diff --git a/crates/tracedecay-sessions/src/runtime/git_correlation/backfill/bounded/tests.rs b/crates/tracedecay-sessions/src/runtime/git_correlation/backfill/bounded/tests.rs index bf79276504..48cbf0fa77 100644 --- a/crates/tracedecay-sessions/src/runtime/git_correlation/backfill/bounded/tests.rs +++ b/crates/tracedecay-sessions/src/runtime/git_correlation/backfill/bounded/tests.rs @@ -244,12 +244,12 @@ async fn prepare_store(path: &Path, project_path: &Path) -> TestStore { ended_at INTEGER, PRIMARY KEY(provider, session_id) ); - CREATE TABLE session_messages ( + CREATE TABLE lcm_raw_messages ( provider TEXT NOT NULL, message_id TEXT NOT NULL, session_id TEXT NOT NULL, timestamp INTEGER, - PRIMARY KEY(provider, message_id) + UNIQUE(provider, message_id) );", ) .await diff --git a/crates/tracedecay-sessions/src/runtime/git_correlation/graph_view_tests.rs b/crates/tracedecay-sessions/src/runtime/git_correlation/graph_view_tests.rs index 675698134e..52b44c86bc 100644 --- a/crates/tracedecay-sessions/src/runtime/git_correlation/graph_view_tests.rs +++ b/crates/tracedecay-sessions/src/runtime/git_correlation/graph_view_tests.rs @@ -175,9 +175,6 @@ fn open_indexed(runtime: &MemoryEvidenceGraphRuntime) -> GitEvidenceGraphView { match open_git_evidence_graph_view(runtime, &identity(), Arc::new(NeverCancelled)).unwrap() { GitEvidenceGraphHead::Indexed(view) => view, GitEvidenceGraphHead::Unpublished => panic!("projection was published"), - GitEvidenceGraphHead::Legacy { generation } => { - panic!("current publication reported legacy generation {generation}") - } } } @@ -509,94 +506,59 @@ fn single_selector_scope_resolution_stops_after_the_caller_bound() { assert_eq!(decodes, 101); } -#[test] -fn legacy_head_is_fully_recoverable_but_serves_no_bounded_reads() { - let projection = seeded_projection(24); - let runtime = MemoryEvidenceGraphRuntime::default(); - let manifest = legacy_git_evidence_manifest_for_test(identity(), &projection).unwrap(); - let generation = manifest.generation.clone(); - runtime - .publish_verified_manifest( - &manifest, - GraphIdempotencyKey::new("legacy-head").unwrap(), - never_cancelled(), - ) - .unwrap(); - - match open_git_evidence_graph_view(&runtime, &identity(), Arc::new(NeverCancelled)).unwrap() { - GitEvidenceGraphHead::Legacy { - generation: observed, - } => assert_eq!(observed, generation), - GitEvidenceGraphHead::Unpublished => panic!("legacy head was published"), - GitEvidenceGraphHead::Indexed(_) => panic!("legacy head carries no index"), - } - - let recovered = recover_git_evidence_projection(&runtime, &identity(), never_cancelled()) - .unwrap() - .expect("legacy rows recover in full"); - assert_eq!( - recovered.projector_revision(), - GitEvidenceProjectorRevision::LegacyV1 - ); - assert_eq!(recovered.projection(), &projection); - - // Re-projecting the same content publishes an indexed successor. - let republished = publish_git_evidence_projection( - &runtime, - identity(), - recovered.projection(), - &revision(), - GraphIdempotencyKey::new("legacy-head-reprojected").unwrap(), - never_cancelled(), - ) - .unwrap(); - assert_ne!(republished.verified_snapshot().generation(), &generation); - let view = open_indexed(&runtime); - assert_eq!( - view.verified_snapshot().generation(), - republished.verified_snapshot().generation() - ); - assert_eq!( - view.health(None).span_count, - projection.spans().len() as u64 - ); -} - -#[test] -fn unknown_recorded_projector_revision_is_corrupt_on_both_read_paths() { +fn publish_with_recorded_revision(recorded: Option<&str>) -> MemoryEvidenceGraphRuntime { let projection = seeded_projection(6); let mut manifest = build_git_evidence_manifest_checked(identity(), &projection, &revision(), &|| Ok(())) .unwrap(); + let revision_property = GraphPropertyName::new("projector-revision").unwrap(); for entity in &mut manifest.entities { if entity.identity.as_str() == "projection:session-git-evidence" { - entity.properties.insert( - GraphPropertyName::new("projector-revision").unwrap(), - GraphProperty::String("session-git-evidence-projector.v9".to_owned()), - ); + match recorded { + Some(recorded) => { + entity.properties.insert( + revision_property.clone(), + GraphProperty::String(recorded.to_owned()), + ); + } + None => { + entity.properties.remove(&revision_property); + } + } } } let runtime = MemoryEvidenceGraphRuntime::default(); runtime .publish_verified_manifest( &manifest, - GraphIdempotencyKey::new("future-revision").unwrap(), + GraphIdempotencyKey::new("foreign-revision").unwrap(), never_cancelled(), ) .unwrap(); + runtime +} - let view_error = - open_git_evidence_graph_view(&runtime, &identity(), Arc::new(NeverCancelled)).unwrap_err(); - assert!( - matches!(&view_error, GitCorrelationError::Corrupt(detail) if detail.contains("projector.v9")), - "{view_error}" - ); - let recovery_error = - recover_git_evidence_projection(&runtime, &identity(), never_cancelled()).unwrap_err(); - assert!( - matches!(&recovery_error, GitCorrelationError::Corrupt(detail) if detail.contains("projector.v9")), - "{recovery_error}" - ); +#[test] +fn unknown_or_missing_projector_revision_is_corrupt_on_both_read_paths() { + for (recorded, expected) in [ + (Some("session-git-evidence-projector.v9"), "projector.v9"), + (None, "no projector revision"), + ] { + let runtime = publish_with_recorded_revision(recorded); + let view_error = + open_git_evidence_graph_view(&runtime, &identity(), Arc::new(NeverCancelled)) + .unwrap_err(); + assert!( + matches!(&view_error, GitCorrelationError::Corrupt(detail) if detail.contains(expected)), + "{view_error}" + ); + let recovery_error = + recover_git_evidence_projection(&runtime, &identity(), never_cancelled()).unwrap_err(); + assert!( + matches!(&recovery_error, GitCorrelationError::Corrupt(detail) if detail.contains(expected)), + "{recovery_error}" + ); + } } #[test] diff --git a/crates/tracedecay-sessions/src/runtime/git_correlation/store.rs b/crates/tracedecay-sessions/src/runtime/git_correlation/store.rs index ae8e8b0cbd..2f81827ad8 100644 --- a/crates/tracedecay-sessions/src/runtime/git_correlation/store.rs +++ b/crates/tracedecay-sessions/src/runtime/git_correlation/store.rs @@ -20,9 +20,8 @@ use tracedecay_store::FactReadControl; use super::{ CommitRelationFilter, CommitSessionRecord, CorrelationIndexHealth, CorrelationIndexPresence, - GIT_EVIDENCE_LEGACY_PROJECTOR_REVISION_V1, GIT_EVIDENCE_PROJECTOR_REVISION, - GitCorrelationError, GitEvidenceProjectionV1, GitRefFilter, GitScopeFilter, - SessionGitCorrelationHit, SessionGitSpan, SessionsForQuery, SpanObservation, + GIT_EVIDENCE_PROJECTOR_REVISION, GitCorrelationError, GitEvidenceProjectionV1, GitRefFilter, + GitScopeFilter, SessionGitCorrelationHit, SessionGitSpan, SessionsForQuery, SpanObservation, canonical_provider_map, commit_hits, commit_identities_with_producer_fallback, commit_record_matches_query, commit_record_order, digest_bytes, scope_session_ids, sessions_for_limit, span_hits, span_matches_query, @@ -200,62 +199,6 @@ pub fn build_git_evidence_manifest_checked( .map_err(Into::into) } -/// The exact pre-index (`v1`) generation shape for `projection`: no projector -/// revision marker, no counts, no hubs or index relations, and the legacy -/// generation identity. Lets dependent crates exercise their legacy-head -/// handling against the shape a live store published before this projector. -#[cfg(any(test, feature = "test-helpers"))] -pub fn legacy_git_evidence_manifest_for_test( - identity: GraphProjectionIdentity, - projection: &GitEvidenceProjectionV1, -) -> Result { - let legacy_revision = - GraphProjectorRevision::try_from(GIT_EVIDENCE_LEGACY_PROJECTOR_REVISION_V1.to_owned())?; - let generation = git_evidence_generation_id(projection, &legacy_revision)?; - let providers = canonical_provider_map(projection.spans(), projection.commit_sessions())?; - let mut entities = vec![GraphEntity::new( - projection_entity_id()?, - BTreeSet::new(), - BTreeMap::from([( - GraphPropertyName::new(PROJECTION_RECORD_PROPERTY)?, - GraphProperty::String(projection.source_watermark().to_owned()), - )]), - )?]; - let mut relations = Vec::new(); - for (session_id, provider) in &providers { - entities.push(GraphEntity::new( - session_entity_id(session_id)?, - BTreeSet::from([GraphLabel::new(SESSION_LABEL)?]), - BTreeMap::from([( - GraphPropertyName::new(PROVIDER_PROPERTY)?, - GraphProperty::String(provider.to_owned()), - )]), - )?); - } - for span in projection.spans() { - entities.push(span_entity(span)?); - relations.push(session_span_relation(&identity, span)?); - } - let mut commits = BTreeSet::new(); - for record in projection.commit_sessions() { - if commits.insert(record.commit_sha.clone()) { - entities.push(commit_entity(&record.commit_sha)?); - } - relations.push(session_commit_relation(&identity, record)?); - } - GraphGenerationManifest::new_checked( - identity, - generation, - SourceGeneration::new(projection.source_watermark())?, - GraphWatermark::new(projection.source_watermark())?, - Vec::new(), - entities, - relations, - &|| Ok(()), - ) - .map_err(Into::into) -} - pub trait GitCorrelationWriteTxn: QueryExecutor + Executor + Sized + Send { fn commit(self) -> impl Future> + Send; } @@ -341,37 +284,23 @@ pub trait GitCorrelationSessionStore: Sync { fn graph_runtime(&self) -> Result<&dyn VerifiedGraphRuntimePortV1, GitCorrelationError>; } -/// Which projector revision published a verified Git-evidence head. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum GitEvidenceProjectorRevision { - /// [`GIT_EVIDENCE_PROJECTOR_REVISION`]: carries the bounded query index. - Current, - /// [`GIT_EVIDENCE_LEGACY_PROJECTOR_REVISION_V1`]: rows only, no index. - LegacyV1, -} - -impl GitEvidenceProjectorRevision { - /// Resolves the revision a head declares. Heads published before the - /// projector recorded its revision are the legacy shape; any recorded - /// revision other than the current one is a projector this build cannot - /// serve. - fn from_recorded(recorded: Option<&str>) -> Result { - match recorded { - None => Ok(Self::LegacyV1), - Some(GIT_EVIDENCE_PROJECTOR_REVISION) => Ok(Self::Current), - Some(other) => Err(GitCorrelationError::Corrupt(format!( - "verified Git evidence records unknown projector revision `{other}`" - ))), - } +/// Admits only heads published by the current indexed projector. A head +/// without a recorded revision, or with any other revision, is a projector +/// this build cannot serve. +fn require_current_projector_revision(recorded: Option<&str>) -> Result<(), GitCorrelationError> { + match recorded { + Some(GIT_EVIDENCE_PROJECTOR_REVISION) => Ok(()), + Some(other) => Err(GitCorrelationError::Corrupt(format!( + "verified Git evidence records unknown projector revision `{other}`" + ))), + None => Err(GitCorrelationError::Corrupt( + "verified Git evidence records no projector revision".to_owned(), + )), } +} - fn graph_revision(self) -> Result { - let revision = match self { - Self::Current => GIT_EVIDENCE_PROJECTOR_REVISION, - Self::LegacyV1 => GIT_EVIDENCE_LEGACY_PROJECTOR_REVISION_V1, - }; - GraphProjectorRevision::try_from(revision.to_owned()).map_err(Into::into) - } +fn current_projector_revision() -> Result { + GraphProjectorRevision::try_from(GIT_EVIDENCE_PROJECTOR_REVISION.to_owned()).map_err(Into::into) } /// Complete typed projection recovered from one verified graph generation. @@ -382,7 +311,6 @@ impl GitEvidenceProjectorRevision { pub struct GitEvidenceProjectionStore { snapshot: VerifiedGraphSnapshot, projection: GitEvidenceProjectionV1, - projector_revision: GitEvidenceProjectorRevision, } impl std::fmt::Debug for GitEvidenceProjectionStore { @@ -391,7 +319,6 @@ impl std::fmt::Debug for GitEvidenceProjectionStore { .debug_struct("GitEvidenceProjectionStore") .field("projection", self.snapshot.projection()) .field("generation", self.snapshot.generation()) - .field("projector_revision", &self.projector_revision) .field("span_count", &self.projection.spans().len()) .field("commit_count", &self.projection.commit_sessions().len()) .finish_non_exhaustive() @@ -414,7 +341,6 @@ impl GitEvidenceProjectionStore { let mut entities_done = false; let mut relations_done = false; let mut source_watermark = None; - let mut projector_revision = None; let mut spans = Vec::new(); let mut commit_sessions = Vec::new(); @@ -446,14 +372,12 @@ impl GitEvidenceProjectionStore { .to_owned(), )); } - projector_revision = GitEvidenceProjectorRevision::from_recorded(match entity - .properties - .get(&revision_property) - { - Some(GraphProperty::String(recorded)) => Some(recorded.as_str()), - _ => None, - }) - .map(Some)?; + require_current_projector_revision( + match entity.properties.get(&revision_property) { + Some(GraphProperty::String(recorded)) => Some(recorded.as_str()), + _ => None, + }, + )?; } if let Some(GraphProperty::Bytes(bytes)) = entity.properties.get(&span_property) { spans.push(serde_json::from_slice(bytes)?); @@ -470,23 +394,16 @@ impl GitEvidenceProjectionStore { entities_done = after_entity.is_none(); relations_done = after_relation.is_none(); } - let (Some(source_watermark), Some(projector_revision)) = - (source_watermark, projector_revision) - else { + let Some(source_watermark) = source_watermark else { return Err(GitCorrelationError::Corrupt( "verified Git evidence is missing projection metadata".to_owned(), )); }; let projection = GitEvidenceProjectionV1::new(source_watermark, spans, commit_sessions)?; - require_git_evidence_generation( - &snapshot, - &projection, - &projector_revision.graph_revision()?, - )?; + require_git_evidence_generation(&snapshot, &projection, ¤t_projector_revision()?)?; Ok(Self { snapshot, projection, - projector_revision, }) } @@ -498,10 +415,6 @@ impl GitEvidenceProjectionStore { &self.projection } - pub fn projector_revision(&self) -> GitEvidenceProjectorRevision { - self.projector_revision - } - pub fn sessions_for(&self, query: &SessionsForQuery) -> Vec { self.projection .sessions_for(query, CommitRelationFilter::Produced) @@ -569,7 +482,6 @@ pub fn publish_git_evidence_projection( Ok(GitEvidenceProjectionStore { snapshot, projection: projection.clone(), - projector_revision: GitEvidenceProjectorRevision::Current, }) } @@ -601,12 +513,6 @@ pub fn recover_git_evidence_projection( pub enum GitEvidenceGraphHead { /// No verified head has ever been published: the typed empty start. Unpublished, - /// The head was published before the projector carried a query index. Its - /// rows are recoverable in full, but no bounded read can be served until - /// the next publication re-projects it. - Legacy { - generation: GraphGenerationId, - }, Indexed(GitEvidenceGraphView), } @@ -668,15 +574,10 @@ pub fn open_git_evidence_graph_view( "verified Git evidence is missing projection metadata".to_owned(), ) })?; - let revision = GitEvidenceProjectorRevision::from_recorded(string_property( + require_current_projector_revision(string_property( &metadata.properties, PROJECTOR_REVISION_PROPERTY, )?)?; - if revision == GitEvidenceProjectorRevision::LegacyV1 { - return Ok(GitEvidenceGraphHead::Legacy { - generation: snapshot.generation().clone(), - }); - } let source_watermark = required_string_property( &metadata.properties, PROJECTION_RECORD_PROPERTY, diff --git a/crates/tracedecay-sessions/src/runtime/git_correlation/tests.rs b/crates/tracedecay-sessions/src/runtime/git_correlation/tests.rs index 9b12ca3da9..1e78d77989 100644 --- a/crates/tracedecay-sessions/src/runtime/git_correlation/tests.rs +++ b/crates/tracedecay-sessions/src/runtime/git_correlation/tests.rs @@ -242,10 +242,9 @@ fn manifest_encodes_sessions_spans_commits_and_evidence_relations() { fn manifest_rejects_a_foreign_projector_revision() { let identity = git_evidence_projection_identity(GraphNamespace::new("project").unwrap()).unwrap(); - let legacy = - GraphProjectorRevision::try_from(GIT_EVIDENCE_LEGACY_PROJECTOR_REVISION_V1.to_owned()) - .unwrap(); - let error = build_git_evidence_manifest_checked(identity, &projection(), &legacy, &|| Ok(())) + let foreign = + GraphProjectorRevision::try_from("session-git-evidence-projector.v1".to_owned()).unwrap(); + let error = build_git_evidence_manifest_checked(identity, &projection(), &foreign, &|| Ok(())) .unwrap_err(); assert!(matches!(error, GitCorrelationError::Contract(_)), "{error}"); } diff --git a/crates/tracedecay-sessions/src/runtime/hosts/claude.rs b/crates/tracedecay-sessions/src/runtime/hosts/claude.rs index a3b87cf432..4ead5ff7a8 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/claude.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/claude.rs @@ -6,66 +6,34 @@ //! (`"user"`/`"assistant"`/…), a `message` object (`role`, `content`, `model`, //! `id`), an ISO-8601 `timestamp`, the session `cwd`, and `sessionId`/`uuid`. //! -//! The accounting parser already reads these files for cost `turns`; this source -//! reuses the **same** append-only byte-offset machinery to also populate the -//! provider-neutral `session_messages` table. Files are scoped to the current -//! project by their recorded `cwd`, so a project only ingests its own sessions. -//! -//! Beyond `user`/`assistant` conversational turns, a handful of structured -//! record types carry high-signal telemetry that we surface as marker rows or -//! metadata (so `message_search`, git correlation, and LCM can find them): -//! `pr-link` records, `system` compaction boundaries, and model-fallback -//! records become dedicated marker rows; assistant attribution fields and -//! `toolUseResult` edited-file facts ride on the owning message row. See the -//! gate in `message_from_line` for the record types we deliberately drop. +//! This source discovers transcripts and filters their frames to the current +//! scope by recorded `cwd`, so a project only ingests its own sessions. Retained +//! frames are normalized to canonical envelopes and admitted through the +//! observation pipeline, whose store projector owns the session rows. +use std::io::Read; use std::path::{Path, PathBuf}; -use serde_json::Value; - -use crate::runtime::shared::{ - ProjectMembership, ProjectRootMatcherCache, StoredCursor, TranscriptLocationMetadataKeys, - TranscriptScopeMatcher, -}; -use crate::runtime::snapshot_observation::{ - MAX_SNAPSHOT_METADATA_BYTES, read_snapshot_text_bounded, -}; +use crate::runtime::shared::{ProjectMembership, ProjectRootMatcherCache, TranscriptScopeMatcher}; use crate::runtime::source::{ - FileDiscoveryLimit, FileDiscoveryReport, JsonlFrameDeferral, ParsedTranscript, - TranscriptCursorKey, TranscriptDiscoveryBounds, TranscriptSource, bound_path_list, - collect_files_with_ext_bounded, path_byte_len, + FileDiscoveryLimit, FileDiscoveryReport, JsonlFrameDeferral, TranscriptDiscoveryBounds, + bound_path_list, collect_files_with_ext_bounded, path_byte_len, }; use tracedecay_privacy::protect_sensitive_structural_id; -mod canonical_projection; mod cursor; mod frames; -mod parser; -mod record_metadata; mod source_records; -use cursor::{claude_cursor_key, claude_source_component}; +use cursor::claude_source_component; +#[cfg(test)] +pub use frames::scan_claude_source_frames; pub use frames::{ ClaudeFrameCoverage, ClaudeSkippedFrame, ClaudeSkippedFrameReason, ClaudeSourceFrame, ClaudeSourceFrameScan, identify_claude_source, try_scan_claude_source_frames_with_resume, }; -#[cfg(test)] -pub use frames::{scan_claude_source_frames, try_scan_claude_source_frames}; -#[cfg(test)] -use record_metadata::{SessionAccumulator, session_metadata}; -#[cfg(test)] -use source_records::reasoning_from_line; use source_records::record_cwd; pub use source_records::transcript_cwd; -pub use source_records::{ - ClaudeRecordContext, ClaudeRecordDisposition, map_sanitized_claude_record, -}; -#[cfg(test)] -use record_metadata::append_git_operation_metadata; -#[cfg(test)] -use serde_json::Map; -#[cfg(test)] -use source_records::message_from_line; #[cfg(test)] use tracedecay_capture::claude::{ encode_cursor_key as encode_claude_cursor_key, encode_source_id as encode_claude_source_id, @@ -73,33 +41,6 @@ use tracedecay_capture::claude::{ const PROVIDER: &str = "claude"; -/// Shared cross-source telemetry-row `kind` vocabulary. Cursor/Codex adapters -/// tag their structured marker rows with the same strings so `message_search` -/// and LCM can filter marker rows uniformly regardless of which agent produced -/// the transcript. -const KIND_PR_LINK: &str = "pr_link"; -const KIND_COMPACT_BOUNDARY: &str = "compact_boundary"; -const KIND_MODEL_FALLBACK: &str = "model_fallback"; -/// A separate reasoning row per assistant message, matching how Codex and Cursor -/// store the model's thinking as its own `kind="reasoning"` row instead of -/// leaving it buried inside the serialized assistant-message content blob. -const KIND_REASONING: &str = "reasoning"; - -/// Cap on the capped preview text carried on a marker row. -const MARKER_PREVIEW_BYTES: usize = 2000; - -const CLAUDE_SESSION_LOCATION_KEYS: TranscriptLocationMetadataKeys = - TranscriptLocationMetadataKeys::new( - "claude_session_cwd", - "claude_session_worktree", - "claude_session_location_provenance", - ); -const CLAUDE_MESSAGE_LOCATION_KEYS: TranscriptLocationMetadataKeys = - TranscriptLocationMetadataKeys::new( - "claude_message_cwd", - "claude_message_worktree", - "claude_message_location_provenance", - ); /// `~/.claude/projects//<…>.jsonl` is at most a few levels deep. /// Workflow-nested subagents add `subagents/workflows/wf_/` (three more /// components) so the scan must reach deeper than a top-level session. @@ -108,7 +49,7 @@ const MAX_SCAN_DEPTH: u8 = 9; /// `summary`/meta line without one. pub const CWD_PROBE_LINES: usize = 8; -/// Claude Code transcript locator + parser. +/// Claude Code transcript locator and scope filter. pub struct ClaudeSource { projects_dir: PathBuf, user_scope: Option, @@ -247,11 +188,26 @@ impl ClaudeSource { } scan.frames = retained; scan.skipped_frames.extend(excluded.iter().copied()); - scan.scope = Some(frames::ClaudeFrameScope { - project_root: project_root.to_path_buf(), - }); Some(excluded) } + + /// Bounded discovery of this source's transcripts: the scoped session + /// (and its subagents) for a live ingest, otherwise every project slug. + /// Frames are filtered by recorded `cwd` afterwards, so discovery need not + /// replicate Claude's slug-encoding scheme. + pub fn discover_transcript_paths( + &self, + bounds: TranscriptDiscoveryBounds, + ) -> FileDiscoveryReport { + if let Some(session_id) = self + .user_scope + .as_ref() + .and_then(|scope| scope.session_id.as_deref()) + { + return discover_claude_session_scoped_paths(&self.projects_dir, session_id, bounds); + } + collect_files_with_ext_bounded(&self.projects_dir, "jsonl", MAX_SCAN_DEPTH, bounds) + } } /// Profile ingestion through an already registered host-admission facade. @@ -261,7 +217,7 @@ pub async fn ingest_user_sessions_with_admission( registered_roots: Vec, admission: &dyn crate::admission::HostAdmission, ) -> crate::runtime::shared::TranscriptIngestStats { - match crate::runtime::claude_observation::ingest_user_sessions_with_admission( + match crate::runtime::hosts::claude_observation::ingest_user_sessions_with_admission( profile_root, session_id, registered_roots, @@ -375,89 +331,13 @@ fn discover_claude_session_scoped_paths( report } -impl TranscriptSource for ClaudeSource { - fn provider(&self) -> &'static str { - PROVIDER - } - - fn transcript_paths(&self, project_root: &Path) -> Vec { - self.discover_transcript_paths(project_root, TranscriptDiscoveryBounds::default_walk()) - .paths - } - - fn discover_transcript_paths( - &self, - _project_root: &Path, - bounds: TranscriptDiscoveryBounds, - ) -> FileDiscoveryReport { - if let Some(session_id) = self - .user_scope - .as_ref() - .and_then(|scope| scope.session_id.as_deref()) - { - return discover_claude_session_scoped_paths(&self.projects_dir, session_id, bounds); - } - // Scan every project slug; `parse_new` filters by recorded `cwd` so each - // project only ingests its own sessions without us having to replicate - // Claude's slug-encoding scheme. - collect_files_with_ext_bounded(&self.projects_dir, "jsonl", MAX_SCAN_DEPTH, bounds) - } - - fn cursor_key(&self, transcript_path: &Path) -> TranscriptCursorKey { - claude_cursor_key(transcript_path) - } - - fn parse_new( - &self, - path: &Path, - prev: StoredCursor, - project_root: &Path, - max_new_bytes: Option, - ) -> Option { - self.try_parse_new(path, prev, project_root, max_new_bytes) - .ok() - .flatten() - } - - fn try_parse_new( - &self, - path: &Path, - prev: StoredCursor, - project_root: &Path, - max_new_bytes: Option, - ) -> crate::runtime::source::TranscriptIngestResult> { - parser::try_parse_claude_transcript(self, path, prev, project_root, max_new_bytes) - } -} struct ClaudeSubagentInfo { parent_session_id: String, - agent_id: String, parent_transcript_path: PathBuf, - /// `agentType` from the sibling meta.json (e.g. "Explore", "general"). - agent_type: Option, - /// `description` from the sibling meta.json (the spawn prompt summary). - description: Option, - /// `toolUseId` from the sibling meta.json: the parent `tool_use` that - /// spawned this subagent. Maps to the `parent_tool_use_id` session column. - parent_tool_use_id: Option, - /// `spawnDepth` from the sibling meta.json (0 for a top-level subagent). - spawn_depth: Option, - /// The `wf_` run id when this subagent lives under - /// `subagents/workflows/wf_/`; `None` for a directly-spawned subagent. - workflow_run_id: Option, -} - -/// Facts folded from `agent-.meta.json` (all optional / fail-open). -#[derive(Default)] -struct ClaudeSubagentMeta { - agent_type: Option, - description: Option, - parent_tool_use_id: Option, - spawn_depth: Option, } /// Detect whether `path` is a subagent transcript and, if so, resolve its -/// identity, parent linkage, optional workflow-run id, and meta.json facts. +/// parent session linkage. /// /// A subagent transcript lives somewhere under a `subagents/` directory owned by /// its parent session: @@ -470,8 +350,6 @@ struct ClaudeSubagentMeta { /// immediate parent. That immediate-parent assumption was a bug: workflow-nested /// subagents failed it and were ingested as orphan standalone sessions. fn claude_subagent_identity(path: &Path) -> Option { - let session_id = claude_source_component(path.file_stem()?); - // Find the `subagents/` ancestor. `ancestors()` yields `path` first, so the // file itself can never match the directory name. let subagents_dir = path @@ -479,83 +357,61 @@ fn claude_subagent_identity(path: &Path) -> Option { .find(|anc| anc.file_name().and_then(|name| name.to_str()) == Some("subagents"))?; let parent_session_dir = subagents_dir.parent()?; let parent_session_id = claude_source_component(parent_session_dir.file_name()?); - - // Capture the workflow run id (`wf_`) when the subagent is nested under - // `subagents/workflows/wf_/`. - let workflow_run_id = path - .ancestors() - .filter_map(|anc| anc.file_name().and_then(|name| name.to_str())) - .find(|name| name.starts_with("wf_")) - .map(str::to_string); - - let agent_id = session_id - .strip_prefix("agent-") - .unwrap_or(&session_id) - .to_string(); // The parent transcript is the `.jsonl` sibling of the `` // directory that owns `subagents/`. let mut parent_filename = parent_session_dir.file_name()?.to_os_string(); parent_filename.push(".jsonl"); let parent_transcript_path = parent_session_dir.parent()?.join(parent_filename); - - let meta = read_subagent_meta(path); - let sanitize = tracedecay_privacy::sanitize_provider_metadata_text; - let retain_identifier = |value: Option| { - value.and_then(|value| { - // The structural pass may already have replaced a credential - // before this identifier-specific check. A redaction marker is - // safe display text, but it is not authoritative provider - // identity and must not become a durable relationship key. - (sanitize(&value).as_deref() == Some(value.as_str()) - && !value.contains("[TraceDecay redacted:")) - .then_some(value) - }) - }; - Some(ClaudeSubagentInfo { parent_session_id, - agent_id, parent_transcript_path, - agent_type: meta.agent_type.as_deref().and_then(sanitize), - description: meta.description.as_deref().and_then(sanitize), - parent_tool_use_id: retain_identifier(meta.parent_tool_use_id), - spawn_depth: meta.spawn_depth, - workflow_run_id: retain_identifier(workflow_run_id), }) } -/// Read the sibling `agent-.meta.json` next to a subagent transcript. Fail -/// open: a missing or malformed file yields empty facts rather than an error. -fn read_subagent_meta(transcript_path: &Path) -> ClaudeSubagentMeta { - let mut meta_filename = transcript_path - .file_stem() - .unwrap_or_default() - .to_os_string(); - meta_filename.push(".meta.json"); - let meta_path = transcript_path.with_file_name(meta_filename); - let Ok(Some(text)) = - read_snapshot_text_bounded(PROVIDER, &meta_path, MAX_SNAPSHOT_METADATA_BYTES) - else { - return ClaudeSubagentMeta::default(); - }; - let Some(value) = - tracedecay_privacy::sanitize_provider_metadata_json(&text, MAX_SNAPSHOT_METADATA_BYTES) - else { - return ClaudeSubagentMeta::default(); +/// Largest `agent-.meta.json` sidecar read; real ones are a few hundred +/// bytes. +const MAX_SUBAGENT_META_BYTES: u64 = 64 * 1024; + +/// The spawn a subagent transcript's sidecar `agent-.meta.json` records: +/// `toolUseId` is the parent's `tool_use` block id, and `parentAgentId` names +/// the spawning subagent when one spawned it (its transcript is +/// `agent-.jsonl`); otherwise the session owning `subagents/` +/// spawned it. A missing or unreadable sidecar keeps the directory parent and +/// no tool-use id. +fn claude_spawn_parent(path: &Path) -> Option<(String, Option)> { + let info = claude_subagent_identity(path)?; + let meta = read_subagent_meta(&path.with_extension("meta.json")); + let text = |key: &str| { + meta.as_ref() + .and_then(|meta| meta.get(key)) + .and_then(serde_json::Value::as_str) + .filter(|value| !value.is_empty()) }; - let string_field = |key: &str| { - value - .get(key) - .and_then(Value::as_str) - .filter(|text| !text.is_empty()) - .map(str::to_string) + let parent_session_id = + text("parentAgentId").map_or(info.parent_session_id, |agent| format!("agent-{agent}")); + let parent_session_id = protect_sensitive_structural_id(&parent_session_id).ok()?; + Some((parent_session_id, text("toolUseId").map(str::to_owned))) +} + +fn read_subagent_meta(path: &Path) -> Option { + let file = match std::fs::File::open(path) { + Ok(file) => file, + Err(error) => { + if error.kind() != std::io::ErrorKind::NotFound { + tracing::debug!(path = %path.display(), %error, "unreadable Claude subagent sidecar"); + } + return None; + } }; - ClaudeSubagentMeta { - agent_type: string_field("agentType"), - description: string_field("description"), - parent_tool_use_id: string_field("toolUseId"), - spawn_depth: value.get("spawnDepth").and_then(Value::as_i64), + let mut bytes = Vec::new(); + file.take(MAX_SUBAGENT_META_BYTES + 1) + .read_to_end(&mut bytes) + .ok()?; + if bytes.len() as u64 > MAX_SUBAGENT_META_BYTES { + tracing::debug!(path = %path.display(), "oversized Claude subagent sidecar"); + return None; } + serde_json::from_slice(&bytes).ok() } #[cfg(test)] diff --git a/crates/tracedecay-sessions/src/runtime/hosts/claude/canonical_projection.rs b/crates/tracedecay-sessions/src/runtime/hosts/claude/canonical_projection.rs deleted file mode 100644 index a5ab409eb3..0000000000 --- a/crates/tracedecay-sessions/src/runtime/hosts/claude/canonical_projection.rs +++ /dev/null @@ -1,540 +0,0 @@ -use std::path::{Path, PathBuf}; - -use serde::Serialize; -use serde_json::{Map, Value}; -use tracedecay_domain::{ - CanonicalBoundaryKindV1, CanonicalGitEvidenceKindV1, CanonicalObservationEnvelopeV1, - CanonicalObservationFactV1, CanonicalWorkflowEvidenceKindV1, -}; -use tracedecay_runtime_core::logging::log_daemon_event; - -use crate::runtime::SessionMessageRecord; -use crate::runtime::shared::{ - ProjectRootMatcherCache, TranscriptLocation, content_storage_text_and_tools, -}; -use crate::runtime::source::SessionDraft; - -use super::record_metadata::{ - SessionAccumulator, append_claude_location_metadata, append_git_operation_metadata, - model_fallback_row, pr_link_row, -}; -use super::source_records::{ - ClaudeRecordContext, ClaudeRecordDisposition, retain_unchanged_tool_event_ids, - system_hook_message_from_line, -}; -use super::{CLAUDE_MESSAGE_LOCATION_KEYS, PROVIDER}; - -pub(super) fn map_canonical_claude_record( - envelope: &CanonicalObservationEnvelopeV1, - context: &ClaudeRecordContext<'_>, - worktree_cache: Option<&ProjectRootMatcherCache>, -) -> ClaudeRecordDisposition { - if envelope.validate().is_err() { - return ClaudeRecordDisposition::NonConversational; - } - let Ok(offset) = i64::try_from(context.offset) else { - return ClaudeRecordDisposition::NonConversational; - }; - let source_path = context.source_path.map_or_else( - || PathBuf::from(format!("claude:{}", context.session_id)), - PathBuf::from, - ); - let draft = || SessionDraft { - session_id: context.session_id.to_owned(), - project_key: context.project_key.to_owned(), - project_path: context.project_path.to_owned(), - title: None, - metadata_json: None, - parent_session_id: None, - is_subagent: false, - agent_id: None, - parent_tool_use_id: None, - }; - let mut accumulator = SessionAccumulator::default(); - - if let Some(message) = - map_canonical_marker_row(envelope, context, &source_path, offset, &mut accumulator) - { - return ClaudeRecordDisposition::Message { - draft: Box::new(draft()), - message: Box::new(message), - }; - } - - let message_fact = envelope - .facts() - .iter() - .find(|fact| matches!(fact, CanonicalObservationFactV1::Message { .. })); - let Some(CanonicalObservationFactV1::Message { - role, - content, - model, - timestamp, - }) = message_fact - else { - let is_compact_boundary = envelope.facts().iter().any(|fact| { - matches!( - fact, - CanonicalObservationFactV1::Boundary { - boundary_kind: CanonicalBoundaryKindV1::CompactionBoundary, - } - ) - }); - if !is_compact_boundary { - return ClaudeRecordDisposition::NonConversational; - } - let compact_metadata = envelope.facts().iter().find_map(|fact| match fact { - CanonicalObservationFactV1::Compaction { - summary: Some(summary), - .. - } => Some(summary), - _ => None, - }); - let trigger = compact_metadata - .and_then(|metadata| metadata.get("trigger")) - .and_then(Value::as_str); - let pre_tokens = compact_metadata - .and_then(|metadata| metadata.get("preTokens")) - .and_then(Value::as_i64); - let logical_parent_uuid = envelope - .relations() - .parent_message_id() - .map(|parent| parent.as_str().to_owned()); - let mut metadata = Map::new(); - metadata.insert( - "source".to_owned(), - Value::String("claude_compact_boundary".to_owned()), - ); - if let Some(trigger) = trigger { - metadata.insert("trigger".to_owned(), Value::String(trigger.to_owned())); - } - if let Some(pre_tokens) = pre_tokens { - metadata.insert("pre_tokens".to_owned(), Value::from(pre_tokens)); - } - if let Some(logical_parent_uuid) = logical_parent_uuid { - metadata.insert( - "logical_parent_uuid".to_owned(), - Value::String(logical_parent_uuid), - ); - } - // LCM native-compaction recognition decodes this envelope to confirm - // the boundary's preservedSegment.anchorUuid still names the summary. - insert_canonical_envelope( - &mut metadata, - envelope, - context.session_id, - envelope.stable_record_id().as_str(), - ); - let message = SessionMessageRecord { - provider: PROVIDER.to_owned(), - message_id: format!( - "{}:{}", - super::KIND_COMPACT_BOUNDARY, - envelope.stable_record_id().as_str() - ), - session_id: context.session_id.to_owned(), - role: "system".to_owned(), - timestamp: envelope.evidence().native_timestamp(), - ordinal: offset, - text: "Claude compaction boundary".to_owned(), - kind: Some(super::KIND_COMPACT_BOUNDARY.to_owned()), - model: None, - tool_names: None, - source_path: Some(source_path.to_string_lossy().into_owned()), - source_offset: Some(offset), - metadata_json: serde_json::to_string(&metadata).ok(), - }; - return ClaudeRecordDisposition::Message { - draft: Box::new(draft()), - message: Box::new(message), - }; - }; - if envelope.native_record_kind() != "user" && envelope.native_record_kind() != "assistant" { - return ClaudeRecordDisposition::NonConversational; - } - let (text, _) = content_storage_text_and_tools(content, None); - if text.trim().is_empty() { - return ClaudeRecordDisposition::NonConversational; - } - let tool_names = envelope - .facts() - .iter() - .filter_map(|fact| match fact { - CanonicalObservationFactV1::ToolInvocation { name, .. } => Some(name.as_str()), - _ => None, - }) - .collect::>(); - let mut metadata = canonical_message_metadata_from_facts(envelope, context, worktree_cache); - metadata.insert( - "source_generation".to_string(), - Value::from(context.file_generation), - ); - retain_unchanged_tool_event_ids(&mut metadata, context.raw_tool_event_ids); - let message = SessionMessageRecord { - provider: PROVIDER.to_owned(), - message_id: envelope - .relations() - .message_id() - .unwrap_or_else(|| envelope.stable_record_id()) - .as_str() - .to_owned(), - session_id: context.session_id.to_owned(), - role: role.as_str().to_owned(), - timestamp: timestamp.or_else(|| envelope.evidence().native_timestamp()), - ordinal: offset, - text, - kind: Some("message".to_owned()), - model: model.clone(), - tool_names: (!tool_names.is_empty()).then(|| tool_names.join(",")), - source_path: Some(source_path.to_string_lossy().into_owned()), - source_offset: Some(offset), - metadata_json: serde_json::to_string(&metadata).ok(), - }; - ClaudeRecordDisposition::Message { - draft: Box::new(draft()), - message: Box::new(message), - } -} - -fn map_canonical_marker_row( - envelope: &CanonicalObservationEnvelopeV1, - context: &ClaudeRecordContext<'_>, - source_path: &Path, - offset: i64, - accumulator: &mut SessionAccumulator, -) -> Option { - for fact in envelope.facts() { - match fact { - CanonicalObservationFactV1::Git { - evidence_kind: CanonicalGitEvidenceKindV1::PullRequest, - content: Some(native), - .. - } => { - return pr_link_row( - native, - context.session_id, - context.file_generation, - source_path, - offset, - accumulator, - ); - } - CanonicalObservationFactV1::Workflow { - evidence_kind: CanonicalWorkflowEvidenceKindV1::ModelFallback, - content: Some(native), - .. - } => { - return model_fallback_row( - native, - context.session_id, - context.file_generation, - source_path, - offset, - ); - } - CanonicalObservationFactV1::Workflow { - evidence_kind: CanonicalWorkflowEvidenceKindV1::Unknown, - content: Some(native), - .. - } => { - let trusted = context - .raw_hook_tool_use_id - .filter(|raw| native.get("toolUseID").and_then(Value::as_str) == Some(*raw)); - if let Some(message) = - system_hook_message_from_line(native, source_path, context, trusted) - { - return Some(message); - } - } - _ => {} - } - } - None -} - -fn canonical_message_metadata_from_facts( - envelope: &CanonicalObservationEnvelopeV1, - context: &ClaudeRecordContext<'_>, - worktree_cache: Option<&ProjectRootMatcherCache>, -) -> Map { - let mut metadata = Map::new(); - metadata.insert( - "source".to_string(), - Value::String("claude_transcript".to_string()), - ); - metadata.insert( - "raw_type".to_string(), - Value::String(envelope.native_record_kind().to_owned()), - ); - - let location_cwd = envelope - .facts() - .iter() - .find_map(|fact| match fact { - CanonicalObservationFactV1::Session { - location_path: Some(path), - .. - } => Some(PathBuf::from(path)), - _ => None, - }) - .or_else(|| context.session_cwd.map(Path::to_path_buf)); - let location_provenance = if envelope.facts().iter().any(|fact| { - matches!( - fact, - CanonicalObservationFactV1::Session { - location_path: Some(_), - .. - } - ) - }) { - "transcript_record" - } else { - "transcript_session" - }; - append_claude_location_metadata( - &mut metadata, - CLAUDE_MESSAGE_LOCATION_KEYS, - TranscriptLocation::new(location_cwd.as_deref(), location_provenance), - worktree_cache, - ); - - let mut tool_events = Vec::new(); - for fact in envelope.facts() { - match fact { - CanonicalObservationFactV1::ToolInvocation { - invocation_id, - name, - arguments, - } => { - let mut event = Map::new(); - event.insert("type".to_string(), Value::String("tool_use".to_string())); - event.insert("tool_name".to_string(), Value::String(name.clone())); - event.insert( - "call_id".to_string(), - Value::String(invocation_id.as_str().to_owned()), - ); - event.insert( - "input_bytes".to_string(), - Value::from(arguments.to_string().len() as u64), - ); - tool_events.push(Value::Object(event)); - } - CanonicalObservationFactV1::ToolResult { - invocation_id, - content, - .. - } => { - let mut event = Map::new(); - event.insert("type".to_string(), Value::String("tool_result".to_string())); - if let Some(invocation_id) = invocation_id { - event.insert( - "call_id".to_string(), - Value::String(invocation_id.as_str().to_owned()), - ); - } - event.insert( - "output_bytes".to_string(), - Value::from(content.to_string().len() as u64), - ); - tool_events.push(Value::Object(event)); - } - _ => {} - } - } - if !tool_events.is_empty() { - metadata.insert("tool_events".to_string(), Value::Array(tool_events)); - } - - if envelope.native_record_kind() == "assistant" - && let Some(CanonicalObservationFactV1::Workflow { - evidence_kind: CanonicalWorkflowEvidenceKindV1::Attribution, - content: Some(Value::Object(attribution)), - .. - }) = envelope.facts().iter().find(|fact| { - matches!( - fact, - CanonicalObservationFactV1::Workflow { - evidence_kind: CanonicalWorkflowEvidenceKindV1::Attribution, - .. - } - ) - }) - { - for (key, value) in attribution { - metadata.insert(key.clone(), value.clone()); - } - } - - if envelope.native_record_kind() == "user" { - for fact in envelope.facts() { - match fact { - CanonicalObservationFactV1::Git { - evidence_kind: CanonicalGitEvidenceKindV1::FileEdit, - content: Some(native), - .. - } => { - append_edited_file_from_native(&mut metadata, native); - } - CanonicalObservationFactV1::Git { - evidence_kind: CanonicalGitEvidenceKindV1::Commit, - content: Some(native), - .. - } => { - append_git_operation_metadata(&mut metadata, native); - } - _ => {} - } - } - } - - if envelope - .facts() - .iter() - .any(|fact| matches!(fact, CanonicalObservationFactV1::Compaction { .. })) - { - // Compact-summary rows keep the canonical envelope so LCM can read the - // native flags and parent id without a second transcript pass. - insert_canonical_envelope( - &mut metadata, - envelope, - context.session_id, - envelope.stable_record_id().as_str(), - ); - } - - metadata -} - -fn append_edited_file_from_native(metadata: &mut Map, native: &Value) { - let Some(tool_use_result) = native - .get("toolUseResult") - .filter(|value| value.is_object()) - else { - return; - }; - let Some(file_path) = tool_use_result - .get("filePath") - .and_then(Value::as_str) - .filter(|path| !path.is_empty()) - else { - return; - }; - let change_type = tool_use_result - .get("type") - .and_then(Value::as_str) - .filter(|kind| !kind.is_empty()) - .unwrap_or("edit") - .to_string(); - let hunks = tool_use_result - .get("structuredPatch") - .and_then(Value::as_array) - .map_or(0, Vec::len); - let mut edited = Map::new(); - edited.insert("path".to_string(), Value::String(file_path.to_string())); - edited.insert("change_type".to_string(), Value::String(change_type)); - edited.insert("hunks".to_string(), Value::from(hunks as i64)); - metadata.insert("edited_file".to_string(), Value::Object(edited)); -} - -/// Serializer failure text for the operator log and the row marker: single -/// line and bounded, so a long or multi-line message can neither break the -/// logfmt record nor bloat the persisted metadata. -fn sanitized_serializer_reason(error: &serde_json::Error) -> String { - const MAX_REASON_CHARS: usize = 200; - error - .to_string() - .chars() - .map(|character| { - if character.is_control() { - ' ' - } else { - character - } - }) - .take(MAX_REASON_CHARS) - .collect() -} - -/// Attaches the canonical envelope LCM's pairing recognition reads, or -/// records why it is absent. -/// -/// Persisting `canonical_envelope: null` would fabricate evidence, but a -/// silently omitted key leaves the row byte-identical to one that never -/// carried pairing evidence, "no pairing evidence" and "never had any" must -/// stay distinguishable. So a serializer failure is both reported to the -/// operator log and marked on the row itself. -fn insert_canonical_envelope( - metadata: &mut Map, - envelope: &impl Serialize, - session_id: &str, - record_id: &str, -) { - match serde_json::to_value(envelope) { - Ok(envelope_value) => { - metadata.insert("canonical_envelope".to_owned(), envelope_value); - } - Err(error) => { - let reason = sanitized_serializer_reason(&error); - log_daemon_event( - "canonical_envelope_unavailable", - &[ - ("provider", PROVIDER.to_owned()), - ("session_id", session_id.to_owned()), - ("record_id", record_id.to_owned()), - ("reason", reason.clone()), - ], - ); - metadata.insert( - "canonical_envelope_unavailable".to_owned(), - Value::String(reason), - ); - } - } -} - -#[cfg(test)] -mod canonical_envelope_omission_tests { - use super::insert_canonical_envelope; - use serde::{Serialize, Serializer, ser::Error as SerError}; - use serde_json::{Map, Value}; - - struct RefusingEnvelope; - - impl Serialize for RefusingEnvelope { - fn serialize(&self, _serializer: S) -> Result { - Err(S::Error::custom("boundary\nfact\tcannot serialize")) - } - } - - #[test] - fn unserializable_envelope_is_marked_instead_of_omitted() { - let mut metadata = Map::new(); - - insert_canonical_envelope(&mut metadata, &RefusingEnvelope, "session-1", "record-1"); - - assert!( - !metadata.contains_key("canonical_envelope"), - "a failed serialization must never persist a fabricated envelope: {metadata:?}" - ); - let reason = metadata["canonical_envelope_unavailable"] - .as_str() - .expect("the omission must carry a typed reason"); - assert!( - reason.starts_with("boundary fact cannot serialize"), - "the reason must carry the sanitized serializer text: {reason:?}" - ); - } - - #[test] - fn serializable_envelope_keeps_the_pairing_evidence() { - let mut metadata = Map::new(); - - insert_canonical_envelope(&mut metadata, &"pairing", "session-1", "record-1"); - - assert_eq!( - metadata["canonical_envelope"], - Value::String("pairing".into()) - ); - assert!(!metadata.contains_key("canonical_envelope_unavailable")); - } -} diff --git a/crates/tracedecay-sessions/src/runtime/hosts/claude/frames.rs b/crates/tracedecay-sessions/src/runtime/hosts/claude/frames.rs index 07a034d4c4..e7319d9e99 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/claude/frames.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/claude/frames.rs @@ -1,7 +1,7 @@ use std::path::{Path, PathBuf}; use serde_json::Value; -use tracedecay_capture::claude::{normalize, stable_record_id}; +use tracedecay_capture::claude::{ClaudeSpawnParent, normalize_spawned, stable_record_id}; use tracedecay_domain::{ObservationOrderingDomainV1, ObservationSourceRangeV1}; use crate::runtime::shared::StoredCursor; @@ -11,7 +11,7 @@ use crate::runtime::source::{ try_stream_new_jsonl_raw_strict_with_resume, }; use tracedecay_privacy::{ - MAX_OBSERVATION_RECORD_BYTES, ParsedClaudeRecordV1, SanitizedClaudeRecordV1, + MAX_OBSERVATION_RECORD_BYTES, ParsedObservationRecordV1, parse_normalized_observation_record_v1, protect_sensitive_structural_id, }; @@ -28,10 +28,6 @@ pub struct ClaudeSourceScanIdentity { pub cursor_key: TranscriptCursorKey, } -pub(super) struct ClaudeFrameScope { - pub project_root: PathBuf, -} - /// Exact byte coverage achieved by one bounded scan. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ClaudeFrameCoverage { @@ -62,71 +58,24 @@ pub struct ClaudeSkippedFrame { pub reason: ClaudeSkippedFrameReason, } -enum ClaudeFramePayload { - Parsed(ParsedClaudeRecordV1), - Sanitized(SanitizedClaudeRecordV1), - Consumed, -} - /// One privacy-parsed Claude frame with its exact original source range. pub struct ClaudeSourceFrame { pub offset: u64, pub end_offset: u64, pub resume_fingerprint: u64, - raw_message_id: Option, - raw_tool_event_ids: Vec, - raw_hook_tool_use_id: Option, - raw_logical_parent_uuid: Option, scope_record: Value, - payload: ClaudeFramePayload, + parsed_record: Option, } impl ClaudeSourceFrame { - pub fn take_parsed_record(&mut self) -> Option { - match std::mem::replace(&mut self.payload, ClaudeFramePayload::Consumed) { - ClaudeFramePayload::Parsed(record) => Some(record), - other => { - self.payload = other; - None - } - } - } - - pub fn set_sanitized_record(&mut self, value: SanitizedClaudeRecordV1) -> bool { - if !matches!(self.payload, ClaudeFramePayload::Consumed) { - return false; - } - self.payload = ClaudeFramePayload::Sanitized(value); - true - } - - pub fn sanitized_record(&self) -> Option<&SanitizedClaudeRecordV1> { - match &self.payload { - ClaudeFramePayload::Sanitized(value) => Some(value), - ClaudeFramePayload::Parsed(_) | ClaudeFramePayload::Consumed => None, - } + pub fn take_parsed_record(&mut self) -> Option { + self.parsed_record.take() } #[hotpath::skip] pub(super) const fn scope_value(&self) -> &Value { &self.scope_record } - - pub(super) fn raw_message_id(&self) -> Option<&str> { - self.raw_message_id.as_deref() - } - - pub(super) fn raw_tool_event_ids(&self) -> &[String] { - &self.raw_tool_event_ids - } - - pub(super) fn raw_hook_tool_use_id(&self) -> Option<&str> { - self.raw_hook_tool_use_id.as_deref() - } - - pub(super) fn raw_logical_parent_uuid(&self) -> Option<&str> { - self.raw_logical_parent_uuid.as_deref() - } } /// Parsed Claude frames and the typed cursor transition they cover. @@ -142,7 +91,6 @@ pub struct ClaudeSourceFrameScan { pub frames: Vec, pub skipped_frames: Vec, pub coverage: ClaudeFrameCoverage, - pub(super) scope: Option, } /// Identify a Claude transcript before loading its durable cursor. @@ -170,7 +118,7 @@ pub fn scan_claude_source_frames( previous: StoredCursor, max_new_bytes: Option, ) -> Option { - match try_scan_claude_source_frames(identity, previous, max_new_bytes) { + match try_scan_claude_source_frames_with_resume(identity, previous, max_new_bytes, None) { Ok(scan) => scan, Err(error) => { tracing::debug!(error = %error, "skipping Claude transcript scan"); @@ -179,14 +127,6 @@ pub fn scan_claude_source_frames( } } -pub fn try_scan_claude_source_frames( - identity: ClaudeSourceScanIdentity, - previous: StoredCursor, - max_new_bytes: Option, -) -> TranscriptIngestResult> { - try_scan_claude_source_frames_with_resume(identity, previous, max_new_bytes, None) -} - #[hotpath::measure(label = "sessions.hosts.claude.scan_frames_resume")] pub fn try_scan_claude_source_frames_with_resume( identity: ClaudeSourceScanIdentity, @@ -225,56 +165,36 @@ pub fn try_scan_claude_source_frames_with_resume( }) .collect::>(); + let spawn = super::claude_spawn_parent(&identity.source_path); + let spawn = spawn + .as_ref() + .map(|(session_id, tool_use_id)| ClaudeSpawnParent { + session_id, + tool_use_id: tool_use_id.as_deref(), + }); for frame in raw.frames.drain(..) { let Ok(range) = ObservationSourceRangeV1::new(frame.offset, frame.end_offset) else { return Ok(None); }; - let mut raw_message_id = None; - let mut raw_tool_event_ids = Vec::new(); - let mut raw_hook_tool_use_id = None; - let mut raw_logical_parent_uuid = None; let mut scope_record = None; let Ok(record) = parse_normalized_observation_record_v1( &frame.bytes, range, ObservationOrderingDomainV1::FileBytes, |native| { - raw_message_id = native - .pointer("/message/id") - .and_then(Value::as_str) - .or_else(|| native.get("uuid").and_then(Value::as_str)) - .filter(|id| !id.is_empty()) - .map(str::to_owned); - raw_tool_event_ids = native - .pointer("/message/content") - .and_then(Value::as_array) - .into_iter() - .flatten() - .filter_map(|item| { - item.get("id") - .or_else(|| item.get("tool_use_id")) - .and_then(Value::as_str) - .filter(|id| !id.is_empty()) - .map(str::to_owned) - }) - .collect(); - raw_hook_tool_use_id = native - .get("toolUseID") - .and_then(Value::as_str) - .filter(|id| !id.is_empty()) - .map(str::to_owned); - raw_logical_parent_uuid = native - .get("logicalParentUuid") - .and_then(Value::as_str) - .filter(|id| !id.is_empty()) - .map(str::to_owned); scope_record = Some(serde_json::json!({ "type": native.get("type").cloned().unwrap_or(Value::Null), "cwd": native.get("cwd").cloned().unwrap_or(Value::Null), })); let stable_record_id = stable_record_id(&native, &identity.session_id, frame.offset)?; - normalize(&native, &identity.session_id, stable_record_id, range) + normalize_spawned( + &native, + &identity.session_id, + spawn, + stable_record_id, + range, + ) }, ) else { skipped_frames.push(ClaudeSkippedFrame { @@ -289,12 +209,8 @@ pub fn try_scan_claude_source_frames_with_resume( offset: frame.offset, end_offset: frame.end_offset, resume_fingerprint: frame.resume_fingerprint, - raw_message_id, - raw_tool_event_ids, - raw_hook_tool_use_id, - raw_logical_parent_uuid, scope_record: scope_record.unwrap_or(Value::Null), - payload: ClaudeFramePayload::Parsed(record), + parsed_record: Some(record), }); } @@ -326,7 +242,6 @@ pub fn try_scan_claude_source_frames_with_resume( frames, skipped_frames, coverage, - scope: None, })) } @@ -355,9 +270,14 @@ mod tests { std::fs::write(&path, format!("{native}\n")).unwrap(); let identity = identify_claude_source(&path).unwrap(); - let mut scan = try_scan_claude_source_frames(identity, StoredCursor::default(), None) - .unwrap() - .unwrap(); + let mut scan = try_scan_claude_source_frames_with_resume( + identity, + StoredCursor::default(), + None, + None, + ) + .unwrap() + .unwrap(); let parsed = scan.frames[0].take_parsed_record().unwrap(); let envelope = serde_json::from_value::(parsed.value().clone()) diff --git a/crates/tracedecay-sessions/src/runtime/hosts/claude/parser.rs b/crates/tracedecay-sessions/src/runtime/hosts/claude/parser.rs deleted file mode 100644 index 1f6a4dfc9c..0000000000 --- a/crates/tracedecay-sessions/src/runtime/hosts/claude/parser.rs +++ /dev/null @@ -1,308 +0,0 @@ -use std::borrow::Cow; -use std::path::Path; - -use serde_json::Value; -use tracedecay_domain::{ - ObservationIdentityMaterialV1, ObservationScopeV1, ObservationSourceGenerationV1, - ObservationSourceIdentityV1, RetentionClass, SessionId, -}; - -use crate::runtime::claude_observation::CLAUDE_TRANSCRIPT_RETENTION_CLASS; -use crate::runtime::shared::{StoredCursor, title_from_messages}; -use crate::runtime::source::{ - JsonlFrameDeferral, ParsedTranscript, SessionDraft, TranscriptIngestError, - TranscriptIngestResult, -}; -use tracedecay_privacy::{ClaudeRecordSanitizerV1, ClaudeSanitizationOutcomeV1}; - -use super::cursor::claude_cursor_key; -use super::frames::{ - ClaudeFrameCoverage, ClaudeSkippedFrameReason, ClaudeSourceFrame, ClaudeSourceFrameScan, - identify_claude_source, try_scan_claude_source_frames, -}; -use super::record_metadata::{SessionAccumulator, accumulate_session_facts, session_metadata}; -use super::source_records::{ - ClaudeRecordContext, ClaudeRecordDisposition, map_sanitized_claude_record_cached, - reasoning_from_line, record_cwd, structured_marker_from_line, system_hook_message_from_line, -}; -use super::{ClaudeSource, PROVIDER, claude_subagent_identity}; - -pub(super) fn fold_scanned_frames( - source: &ClaudeSource, - scan: &ClaudeSourceFrameScan, - project_root: &Path, -) -> Option { - scan.scope - .as_ref() - .filter(|scope| scope.project_root == project_root)?; - let subagent = claude_subagent_identity(&scan.identity.source_path); - let session_id = scan.identity.session_id.clone(); - let source_path = Path::new(&scan.identity.source_id); - let project = source.user_scope.as_ref().map_or_else( - || project_root.to_string_lossy().to_string(), - |_| "user".to_string(), - ); - let sanitized_session_cwd = scan - .frames - .iter() - .filter_map(ClaudeSourceFrame::sanitized_record) - .find_map(|record| record_cwd(record.payload())); - let mut accumulator = SessionAccumulator::default(); - let mut messages = Vec::new(); - - for frame in &scan.frames { - let record = frame.sanitized_record()?.payload(); - accumulate_session_facts(record, &mut accumulator); - let context = ClaudeRecordContext { - session_id: &session_id, - project_key: &project, - project_path: &project, - file_generation: scan.file_generation, - offset: frame.offset, - session_cwd: sanitized_session_cwd.as_deref(), - source_path: Some(scan.identity.source_id.as_str()), - raw_message_id: frame.raw_message_id(), - raw_tool_event_ids: frame.raw_tool_event_ids(), - raw_hook_tool_use_id: frame.raw_hook_tool_use_id(), - }; - let mut message = - match map_sanitized_claude_record_cached(record, &context, &source.project_matchers) { - ClaudeRecordDisposition::Message { message, .. } => Some(*message), - ClaudeRecordDisposition::NonConversational => { - let owned_native = envelope_native_content(record); - let native = owned_native.as_ref().unwrap_or(record); - system_hook_message_from_line( - native, - source_path, - &context, - frame.raw_hook_tool_use_id().filter(|raw| { - native.get("toolUseID").and_then(Value::as_str) == Some(*raw) - }), - ) - } - }; - if message.is_none() { - let owned_native = envelope_native_content(record); - let marker_source = owned_native.as_ref().unwrap_or(record); - let marker_record = if frame.raw_logical_parent_uuid() - != marker_source - .get("logicalParentUuid") - .and_then(Value::as_str) - || marker_source - .get("logicalParentUuid") - .and_then(Value::as_str) - .is_some_and(|id| id.starts_with("[TraceDecay redacted:")) - { - let mut record = marker_source.clone(); - if let Some(record) = record.as_object_mut() { - record.remove("logicalParentUuid"); - } - Cow::Owned(record) - } else { - Cow::Borrowed(marker_source) - }; - message = structured_marker_from_line( - marker_record.as_ref(), - source_path, - &context, - &mut accumulator, - ); - } - if let Some(reasoning) = reasoning_from_line( - record, - source_path, - &context, - message.as_ref().map(|message| message.message_id.as_str()), - ) { - messages.push(reasoning); - } - if let Some(message) = message { - messages.push(message); - } - } - - let draft = SessionDraft { - session_id, - project_key: project.clone(), - project_path: project, - title: title_from_messages(&messages), - metadata_json: serde_json::to_string(&session_metadata( - sanitized_session_cwd.as_deref(), - subagent.as_ref(), - &accumulator, - Some(&source.project_matchers), - )) - .ok(), - parent_session_id: subagent.as_ref().map(|info| info.parent_session_id.clone()), - is_subagent: subagent.is_some(), - agent_id: subagent.as_ref().map(|info| info.agent_id.clone()), - parent_tool_use_id: subagent - .as_ref() - .and_then(|info| info.parent_tool_use_id.clone()), - }; - Some(ParsedTranscript { - draft, - messages, - new_cursor: scan.next_cursor.state, - }) -} - -fn envelope_native_content(record: &Value) -> Option { - let envelope = - serde_json::from_value::(record.clone()) - .ok()?; - envelope.facts().iter().find_map(|fact| { - let (tracedecay_domain::CanonicalObservationFactV1::Git { - content: Some(content), - .. - } - | tracedecay_domain::CanonicalObservationFactV1::Workflow { - content: Some(content), - .. - } - | tracedecay_domain::CanonicalObservationFactV1::WorkflowLifecycle { - content: Some(content), - .. - } - | tracedecay_domain::CanonicalObservationFactV1::Compaction { - summary: Some(content), - .. - } - | tracedecay_domain::CanonicalObservationFactV1::Reasoning { - content: Some(content), - .. - } - | tracedecay_domain::CanonicalObservationFactV1::Message { content, .. } - | tracedecay_domain::CanonicalObservationFactV1::ToolResult { content, .. } - | tracedecay_domain::CanonicalObservationFactV1::ToolInvocation { - arguments: content, - .. - }) = fact - else { - return None; - }; - content.get("type").is_some().then(|| content.clone()) - }) -} - -#[hotpath::measure(label = "sessions.hosts.claude.parse")] -pub(super) fn try_parse_claude_transcript( - source_adapter: &ClaudeSource, - path: &Path, - prev: StoredCursor, - project_root: &Path, - max_new_bytes: Option, -) -> TranscriptIngestResult> { - let identity = identify_claude_source(path).ok_or_else(|| { - TranscriptIngestError::InvalidSourceIdentity { - provider: PROVIDER, - path: path.to_path_buf(), - } - })?; - let Some(mut scan) = try_scan_claude_source_frames(identity, prev, max_new_bytes)? else { - return Ok(None); - }; - if scan.previous_cursor.state != prev || scan.previous_cursor.key != claude_cursor_key(path) { - return Ok(None); - } - if let ClaudeFrameCoverage::Deferred { reason, .. } = scan.coverage { - tracing::debug!( - provider = PROVIDER, - line_offset = reason.offset(), - reason = reason.reason_code(), - "deferring transcript input at strict JSONL frame" - ); - } - let coverage = scan.coverage; - if let ClaudeFrameCoverage::Deferred { - start_offset, - covered_through, - reason: JsonlFrameDeferral::Backlog { .. }, - } = coverage - { - if start_offset == covered_through { - return Ok(None); - } - scan.coverage = ClaudeFrameCoverage::Complete { - start_offset, - end_offset: covered_through, - }; - } - let retained = source_adapter.retain_scoped_frames(&mut scan, project_root); - scan.coverage = coverage; - if retained.is_none() { - return Ok(None); - } - - // Selecting the frame and naming its reason in one total match keeps the - // non-durable reasons and the reported reason string from drifting apart; - // the previous split form had to assert the remaining variants away. - if let Some((skipped, reason)) = scan.skipped_frames.iter().find_map(|frame| { - let reason = match frame.reason { - ClaudeSkippedFrameReason::Malformed => "malformed", - ClaudeSkippedFrameReason::Oversized => "oversized", - ClaudeSkippedFrameReason::Whitespace | ClaudeSkippedFrameReason::OutOfScope => { - return None; - } - }; - Some((frame, reason)) - }) { - return Err(TranscriptIngestError::NonDurableRecord { - provider: PROVIDER, - offset: skipped.offset, - end_offset: skipped.end_offset, - reason, - }); - } - - // Legacy callers still use this trait path. They must receive the same - // sanitizer-issued payload as observation-first ingestion, never the - // parser's raw `Value` relabelled as sanitized. - let sanitizer = ClaudeRecordSanitizerV1::claude_v1()?; - let source = ObservationSourceIdentityV1::for_source( - SessionId::new(scan.identity.session_id.clone())?, - SessionId::new(scan.identity.source_id.clone())?, - )?; - let generation = ObservationSourceGenerationV1::new(scan.file_generation)?; - let retention_class = RetentionClass::new(CLAUDE_TRANSCRIPT_RETENTION_CLASS)?; - for frame in &mut scan.frames { - let parsed = frame - .take_parsed_record() - .ok_or(TranscriptIngestError::InvalidFrameState { provider: PROVIDER })?; - let range = *parsed.source_range(); - let identity = ObservationIdentityMaterialV1::new( - source.clone(), - ObservationScopeV1::Profile, - generation, - range, - )?; - let sanitized = - match sanitizer.sanitize_parsed(parsed, identity, retention_class.clone())? { - ClaudeSanitizationOutcomeV1::Durable { - sanitized_record, .. - } => sanitized_record, - ClaudeSanitizationOutcomeV1::Rejected { .. } => { - return Err(TranscriptIngestError::NonDurableRecord { - provider: PROVIDER, - offset: range.start(), - end_offset: range.end(), - reason: "sanitizer_rejected", - }); - } - ClaudeSanitizationOutcomeV1::Quarantined { .. } => { - return Err(TranscriptIngestError::NonDurableRecord { - provider: PROVIDER, - offset: range.start(), - end_offset: range.end(), - reason: "sanitizer_quarantined", - }); - } - }; - if !frame.set_sanitized_record(sanitized) { - return Err(TranscriptIngestError::InvalidFrameState { provider: PROVIDER }); - } - } - fold_scanned_frames(source_adapter, &scan, project_root) - .map(Some) - .ok_or(TranscriptIngestError::InvalidFrameState { provider: PROVIDER }) -} diff --git a/crates/tracedecay-sessions/src/runtime/hosts/claude/record_metadata.rs b/crates/tracedecay-sessions/src/runtime/hosts/claude/record_metadata.rs deleted file mode 100644 index c15737a268..0000000000 --- a/crates/tracedecay-sessions/src/runtime/hosts/claude/record_metadata.rs +++ /dev/null @@ -1,659 +0,0 @@ -use std::fmt::Write as _; -use std::path::Path; - -use serde_json::{Map, Value}; - -use crate::host_ports::parse_timestamp; -use crate::runtime::SessionMessageRecord; -use crate::runtime::shared::{ - ProjectRootMatcherCache, TranscriptLocation, TranscriptLocationMetadataKeys, - append_location_metadata, append_location_metadata_cached, append_tool_calls_metadata, - append_tool_event_metadata, append_usage_metadata, preview_truncated, -}; - -use super::source_records::record_cwd; -use super::{ - CLAUDE_MESSAGE_LOCATION_KEYS, CLAUDE_SESSION_LOCATION_KEYS, ClaudeSubagentInfo, - KIND_COMPACT_BOUNDARY, KIND_MODEL_FALLBACK, KIND_PR_LINK, MARKER_PREVIEW_BYTES, PROVIDER, -}; - -#[derive(Default)] -pub(super) struct SessionAccumulator { - /// Distinct PR links seen (`{pr_number, pr_url, pr_repository}`), deduped by - /// url+number so an append that re-reads a boundary line stays idempotent. - pr_links: Vec, - /// Distinct files edited (`{path, change_type, hunks}`), deduped by path. - edited_files: Vec, -} - -impl SessionAccumulator { - fn push_pr_link(&mut self, link: Value) { - let key = ( - link.get("pr_url") - .and_then(Value::as_str) - .map(str::to_string), - link.get("pr_number").cloned(), - ); - let exists = self.pr_links.iter().any(|existing| { - ( - existing - .get("pr_url") - .and_then(Value::as_str) - .map(str::to_string), - existing.get("pr_number").cloned(), - ) == key - }); - if !exists { - self.pr_links.push(link); - } - } - - fn push_edited_file(&mut self, path: &str, change_type: &str, hunks: usize) { - if self - .edited_files - .iter() - .any(|existing| existing.get("path").and_then(Value::as_str) == Some(path)) - { - return; - } - let mut entry = Map::new(); - entry.insert("path".to_string(), Value::String(path.to_string())); - entry.insert( - "change_type".to_string(), - Value::String(change_type.to_string()), - ); - entry.insert("hunks".to_string(), Value::from(hunks as i64)); - self.edited_files.push(Value::Object(entry)); - } -} - -pub(super) fn accumulate_session_facts(record: &Value, accumulator: &mut SessionAccumulator) { - append_edited_file_metadata(&mut Map::new(), record, accumulator); - if let Ok(envelope) = - serde_json::from_value::(record.clone()) - { - for fact in envelope.facts() { - match fact { - tracedecay_domain::CanonicalObservationFactV1::Git { - evidence_kind: tracedecay_domain::CanonicalGitEvidenceKindV1::FileEdit, - content: Some(native), - .. - } => append_edited_file_metadata(&mut Map::new(), native, accumulator), - tracedecay_domain::CanonicalObservationFactV1::Git { - evidence_kind: tracedecay_domain::CanonicalGitEvidenceKindV1::PullRequest, - content: Some(native), - .. - } => { - let mut link = Map::new(); - if let Some(number) = native.get("prNumber").filter(|value| !value.is_null()) { - link.insert("pr_number".to_string(), number.clone()); - } - if let Some(url) = native - .get("prUrl") - .and_then(Value::as_str) - .filter(|url| !url.is_empty()) - { - link.insert("pr_url".to_string(), Value::String(url.to_string())); - } - if let Some(repo) = native - .get("prRepository") - .and_then(Value::as_str) - .filter(|repo| !repo.is_empty()) - { - link.insert("pr_repository".to_string(), Value::String(repo.to_string())); - } - if !link.is_empty() { - accumulator.push_pr_link(Value::Object(link)); - } - } - _ => {} - } - } - } -} - -/// Read a record's optional wall-clock timestamp. -pub(super) fn record_timestamp(record: &Value) -> Option { - record - .get("timestamp") - .and_then(Value::as_str) - .and_then(parse_timestamp) - .map(|secs| secs as i64) -} - -/// Build a marker row for a `type=="pr-link"` record and fold the PR into the -/// session accumulator. Emits both so the git-correlation join has a per-turn -/// anchor (`message_search`) *and* a session-level `pr_links[]` summary. -pub(super) fn pr_link_row( - record: &Value, - session_id: &str, - file_generation: u64, - path: &Path, - offset: i64, - accumulator: &mut SessionAccumulator, -) -> Option { - let pr_number = record.get("prNumber").filter(|value| !value.is_null()); - let pr_url = record - .get("prUrl") - .and_then(Value::as_str) - .filter(|url| !url.is_empty()); - let pr_repository = record - .get("prRepository") - .and_then(Value::as_str) - .filter(|repo| !repo.is_empty()); - // A pr-link with no identifying fields is noise; drop it. - if pr_number.is_none() && pr_url.is_none() && pr_repository.is_none() { - return None; - } - - let number_display = pr_number.map(render_scalar).unwrap_or_default(); - let mut text = String::from("Claude PR link:"); - if let Some(repo) = pr_repository { - text.push(' '); - text.push_str(repo); - } - if !number_display.is_empty() { - text.push_str(" #"); - text.push_str(&number_display); - } - if let Some(url) = pr_url { - text.push(' '); - text.push_str(url); - } - - let mut link = Map::new(); - if let Some(number) = pr_number { - link.insert("pr_number".to_string(), number.clone()); - } - if let Some(url) = pr_url { - link.insert("pr_url".to_string(), Value::String(url.to_string())); - } - if let Some(repo) = pr_repository { - link.insert("pr_repository".to_string(), Value::String(repo.to_string())); - } - accumulator.push_pr_link(Value::Object(link.clone())); - - let mut metadata = Map::new(); - metadata.insert( - "source".to_string(), - Value::String("claude_pr_link".to_string()), - ); - for (key, value) in &link { - metadata.insert(key.clone(), value.clone()); - } - - let message_id = marker_message_id(record, session_id, file_generation, KIND_PR_LINK, offset); - Some(SessionMessageRecord { - provider: PROVIDER.to_string(), - message_id, - session_id: session_id.to_string(), - // Telemetry, not conversation: role "tool" keeps it out of LCM anchors. - role: "tool".to_string(), - timestamp: record_timestamp(record), - ordinal: offset, - text: preview_truncated(&text, MARKER_PREVIEW_BYTES), - kind: Some(KIND_PR_LINK.to_string()), - model: None, - tool_names: None, - source_path: Some(path.to_string_lossy().to_string()), - source_offset: Some(offset), - metadata_json: serde_json::to_string(&Value::Object(metadata)).ok(), - }) -} - -/// Build a `compact_boundary` marker row from a `system` record that carries -/// `compactMetadata` (a context-compaction boundary). LCM uses this to tell a -/// post-compaction summary apart from an original turn. -pub(super) fn compact_boundary_row( - record: &Value, - session_id: &str, - file_generation: u64, - path: &Path, - offset: i64, -) -> Option { - let subtype = record.get("subtype").and_then(Value::as_str); - let compact_metadata = record - .get("compactMetadata") - .filter(|value| value.is_object()); - if subtype != Some("compact_boundary") && compact_metadata.is_none() { - return None; - } - - let trigger = compact_metadata - .and_then(|meta| meta.get("trigger")) - .and_then(Value::as_str) - .or_else(|| record.get("trigger").and_then(Value::as_str)); - let pre_tokens = compact_metadata - .and_then(|meta| meta.get("preTokens")) - .and_then(Value::as_i64) - .or_else(|| record.get("preTokens").and_then(Value::as_i64)); - let logical_parent_uuid = record - .get("logicalParentUuid") - .and_then(Value::as_str) - .filter(|uuid| !uuid.is_empty()); - - let mut text = String::from("Claude compaction boundary"); - if let Some(trigger) = trigger { - let _ = write!(text, " (trigger: {trigger})"); - } - if let Some(pre_tokens) = pre_tokens { - let _ = write!(text, ", pre_tokens: {pre_tokens}"); - } - - let mut metadata = Map::new(); - metadata.insert( - "source".to_string(), - Value::String("claude_compact_boundary".to_string()), - ); - if let Some(trigger) = trigger { - metadata.insert("trigger".to_string(), Value::String(trigger.to_string())); - } - if let Some(pre_tokens) = pre_tokens { - metadata.insert("pre_tokens".to_string(), Value::from(pre_tokens)); - } - if let Some(logical_parent_uuid) = logical_parent_uuid { - metadata.insert( - "logical_parent_uuid".to_string(), - Value::String(logical_parent_uuid.to_string()), - ); - } - - let message_id = marker_message_id( - record, - session_id, - file_generation, - KIND_COMPACT_BOUNDARY, - offset, - ); - Some(SessionMessageRecord { - provider: PROVIDER.to_string(), - message_id, - session_id: session_id.to_string(), - // A compaction boundary is a genuine structural event LCM anchors on. - role: "system".to_string(), - timestamp: record_timestamp(record), - ordinal: offset, - text: preview_truncated(&text, MARKER_PREVIEW_BYTES), - kind: Some(KIND_COMPACT_BOUNDARY.to_string()), - model: None, - tool_names: None, - source_path: Some(path.to_string_lossy().to_string()), - source_offset: Some(offset), - metadata_json: serde_json::to_string(&Value::Object(metadata)).ok(), - }) -} - -/// Build a `model_fallback` marker row from a `system` model-refusal-fallback -/// record (Claude routed a refused request to a fallback model). -pub(super) fn model_fallback_row( - record: &Value, - session_id: &str, - file_generation: u64, - path: &Path, - offset: i64, -) -> Option { - let subtype = record.get("subtype").and_then(Value::as_str); - let original_model = record - .get("originalModel") - .and_then(Value::as_str) - .filter(|model| !model.is_empty()); - let fallback_model = record - .get("fallbackModel") - .and_then(Value::as_str) - .filter(|model| !model.is_empty()); - if subtype != Some("model_refusal_fallback") - && original_model.is_none() - && fallback_model.is_none() - { - return None; - } - - let trigger = record.get("trigger").and_then(Value::as_str); - let refusal_category = record - .get("apiRefusalCategory") - .and_then(Value::as_str) - .filter(|category| !category.is_empty()); - - let mut text = String::from("Claude model fallback"); - if let (Some(original), Some(fallback)) = (original_model, fallback_model) { - let _ = write!(text, ": {original} -> {fallback}"); - } else if let Some(fallback) = fallback_model { - let _ = write!(text, " -> {fallback}"); - } - if let Some(category) = refusal_category { - let _ = write!(text, " ({category})"); - } - - let mut metadata = Map::new(); - metadata.insert( - "source".to_string(), - Value::String("claude_model_fallback".to_string()), - ); - if let Some(original) = original_model { - metadata.insert( - "original_model".to_string(), - Value::String(original.to_string()), - ); - } - if let Some(fallback) = fallback_model { - metadata.insert( - "fallback_model".to_string(), - Value::String(fallback.to_string()), - ); - } - if let Some(trigger) = trigger { - metadata.insert("trigger".to_string(), Value::String(trigger.to_string())); - } - if let Some(category) = refusal_category { - metadata.insert( - "api_refusal_category".to_string(), - Value::String(category.to_string()), - ); - } - - let message_id = marker_message_id( - record, - session_id, - file_generation, - KIND_MODEL_FALLBACK, - offset, - ); - Some(SessionMessageRecord { - provider: PROVIDER.to_string(), - message_id, - session_id: session_id.to_string(), - role: "tool".to_string(), - timestamp: record_timestamp(record), - ordinal: offset, - text: preview_truncated(&text, MARKER_PREVIEW_BYTES), - kind: Some(KIND_MODEL_FALLBACK.to_string()), - model: fallback_model.map(str::to_string), - tool_names: None, - source_path: Some(path.to_string_lossy().to_string()), - source_offset: Some(offset), - metadata_json: serde_json::to_string(&Value::Object(metadata)).ok(), - }) -} - -/// Stable marker identity: prefer a usable record `uuid`, otherwise use the -/// source generation and offset shared by every sanitized Claude row. -fn marker_message_id( - record: &Value, - session_id: &str, - file_generation: u64, - kind: &str, - offset: i64, -) -> String { - record - .get("uuid") - .and_then(Value::as_str) - .filter(|uuid| !uuid.is_empty() && !is_redaction_marker(uuid)) - .map_or_else( - || source_position_message_id(session_id, file_generation, offset), - |uuid| format!("{kind}:{uuid}"), - ) -} - -pub(super) fn source_position_message_id( - session_id: &str, - file_generation: u64, - offset: i64, -) -> String { - format!("{session_id}:{file_generation}:{offset}") -} - -pub(super) fn is_redaction_marker(value: &str) -> bool { - value.starts_with("[TraceDecay redacted:") -} - -/// Render a JSON scalar (number/string/bool) as plain text for a marker preview. -fn render_scalar(value: &Value) -> String { - value - .as_str() - .map_or_else(|| value.to_string(), str::to_string) -} - -/// Dispatch location metadata through the source-lifetime worktree cache when -/// the caller has one (batch transcript parses), falling back to the uncached -/// per-call resolution for single-record paths (observation projection). -pub(super) fn append_claude_location_metadata( - map: &mut Map, - keys: TranscriptLocationMetadataKeys, - location: TranscriptLocation<'_>, - worktree_cache: Option<&ProjectRootMatcherCache>, -) { - match worktree_cache { - Some(cache) => append_location_metadata_cached(map, keys, location, cache), - None => append_location_metadata(map, keys, location), - } -} - -pub(super) fn session_metadata( - sanitized_session_cwd: Option<&Path>, - subagent: Option<&ClaudeSubagentInfo>, - accumulator: &SessionAccumulator, - worktree_cache: Option<&ProjectRootMatcherCache>, -) -> Value { - let mut metadata = Map::new(); - metadata.insert( - "source".to_string(), - Value::String("claude_transcript".to_string()), - ); - append_claude_location_metadata( - &mut metadata, - CLAUDE_SESSION_LOCATION_KEYS, - TranscriptLocation::new(sanitized_session_cwd, "transcript_session"), - worktree_cache, - ); - - // Subagent spawn provenance (from the sibling agent-.meta.json and the - // on-disk layout). `parent_tool_use_id` rides the dedicated session column; - // these richer facts have no column, so they land in metadata. - if let Some(subagent) = subagent { - if let Some(agent_type) = &subagent.agent_type { - metadata.insert("agent_type".to_string(), Value::String(agent_type.clone())); - } - if let Some(description) = &subagent.description { - metadata.insert( - "agent_description".to_string(), - Value::String(description.clone()), - ); - } - if let Some(spawn_depth) = subagent.spawn_depth { - metadata.insert("spawn_depth".to_string(), Value::from(spawn_depth)); - } - if let Some(workflow_run_id) = &subagent.workflow_run_id { - metadata.insert( - "workflow_run_id".to_string(), - Value::String(workflow_run_id.clone()), - ); - } - } - - // Session-level rollups: only emitted when the session actually produced - // them, so plain sessions keep byte-for-byte identical metadata. - if !accumulator.pr_links.is_empty() { - metadata.insert( - "pr_links".to_string(), - Value::Array(accumulator.pr_links.clone()), - ); - } - if !accumulator.edited_files.is_empty() { - metadata.insert( - "edited_files".to_string(), - Value::Array(accumulator.edited_files.clone()), - ); - } - - Value::Object(metadata) -} - -pub(super) fn message_metadata( - kind: &str, - record: &Value, - message: &Value, - content: &Value, - sanitized_session_cwd: Option<&Path>, - accumulator: &mut SessionAccumulator, - worktree_cache: Option<&ProjectRootMatcherCache>, -) -> Value { - let mut metadata = Map::new(); - metadata.insert( - "source".to_string(), - Value::String("claude_transcript".to_string()), - ); - metadata.insert("raw_type".to_string(), Value::String(kind.to_string())); - let record_cwd = record_cwd(record); - let (location_cwd, location_provenance) = if record_cwd.is_some() { - (record_cwd.as_deref(), "transcript_record") - } else { - (sanitized_session_cwd, "transcript_session") - }; - append_claude_location_metadata( - &mut metadata, - CLAUDE_MESSAGE_LOCATION_KEYS, - TranscriptLocation::new(location_cwd, location_provenance), - worktree_cache, - ); - if let Some(branch) = record - .get("gitBranch") - .and_then(Value::as_str) - .filter(|branch| !branch.is_empty()) - { - metadata.insert("git_branch".to_string(), Value::String(branch.to_string())); - } - append_tool_calls_metadata(&mut metadata, message); - append_tool_event_metadata(&mut metadata, content); - // Anthropic-style per-message counters: `message.usage.{input_tokens, - // output_tokens, cache_creation_input_tokens, cache_read_input_tokens}`. - append_usage_metadata(&mut metadata, &[message]); - // Per-turn adoption ground truth: which MCP server/tool/skill produced this - // assistant turn. The caller supplies the sanitizer-issued record, so these - // top-level fields have already crossed the mandatory privacy boundary. - if kind == "assistant" { - append_attribution_metadata(&mut metadata, record); - } - // Edit/Write tool results carry a top-level `toolUseResult` with the edited - // file path + structured patch. Record the file + hunk stats (never the - // patch bodies) and fold the file into the session summary. - if kind == "user" { - append_edited_file_metadata(&mut metadata, record, accumulator); - append_git_operation_metadata(&mut metadata, record); - } - Value::Object(metadata) -} - -/// Preserve Claude's structured git-operation event as direct commit evidence. -/// The abbreviated id is resolved against the repository before persistence; -/// raw stdout/stderr stays in the lossless transcript rather than metadata. -pub(super) fn append_git_operation_metadata(metadata: &mut Map, record: &Value) { - let Some(commit) = record - .pointer("/toolUseResult/gitOperation/commit") - .and_then(Value::as_object) - else { - return; - }; - let Some(sha) = commit.get("sha").and_then(Value::as_str).filter(|sha| { - (7..=64).contains(&sha.len()) && sha.chars().all(|ch| ch.is_ascii_hexdigit()) - }) else { - return; - }; - metadata.insert( - "produced_commit_candidates".to_string(), - Value::Array(vec![Value::String(sha.to_ascii_lowercase())]), - ); - metadata.insert( - "produced_commit_evidence".to_string(), - Value::String("host_event".to_string()), - ); - if let Some(kind) = commit - .get("kind") - .and_then(Value::as_str) - .filter(|kind| !kind.is_empty()) - { - metadata.insert( - "produced_commit_kind".to_string(), - Value::String(kind.to_string()), - ); - } - if let Some(branch) = record - .get("gitBranch") - .and_then(Value::as_str) - .filter(|branch| !branch.is_empty()) - { - metadata.insert("git_branch".to_string(), Value::String(branch.to_string())); - } -} - -/// Copy Claude's top-level attribution fields onto an assistant row's metadata. -fn append_attribution_metadata(metadata: &mut Map, record: &Value) { - for (source_key, dest_key) in [ - ("attributionMcpServer", "attribution_mcp_server"), - ("attributionMcpTool", "attribution_mcp_tool"), - ("attributionSkill", "attribution_skill"), - ("promptSource", "prompt_source"), - ] { - if let Some(value) = record - .get(source_key) - .and_then(Value::as_str) - .filter(|text| !text.is_empty()) - { - metadata.insert(dest_key.to_string(), Value::String(value.to_string())); - } - } - // `origin` only when it is a cheap scalar string; skip nested objects. - if let Some(origin) = record - .get("origin") - .and_then(Value::as_str) - .filter(|text| !text.is_empty()) - { - metadata.insert("origin".to_string(), Value::String(origin.to_string())); - } -} - -/// Record edited-file facts from a user `tool_result` record's top-level -/// `toolUseResult` (Edit/Write payloads), and fold the file into the session -/// accumulator. Stores only the path, change type, and hunk count, never the -/// patch bodies. -fn append_edited_file_metadata( - metadata: &mut Map, - record: &Value, - accumulator: &mut SessionAccumulator, -) { - let Some(tool_use_result) = record - .get("toolUseResult") - .filter(|value| value.is_object()) - else { - return; - }; - let Some(file_path) = tool_use_result - .get("filePath") - .and_then(Value::as_str) - .filter(|path| !path.is_empty()) - else { - return; - }; - // Write results carry an explicit `type` ("create"/"update"); Edit results - // do not, so an absent type means an in-place edit. - let change_type = tool_use_result - .get("type") - .and_then(Value::as_str) - .filter(|kind| !kind.is_empty()) - .unwrap_or("edit") - .to_string(); - let hunks = tool_use_result - .get("structuredPatch") - .and_then(Value::as_array) - .map_or(0, Vec::len); - - let mut edited = Map::new(); - edited.insert("path".to_string(), Value::String(file_path.to_string())); - edited.insert( - "change_type".to_string(), - Value::String(change_type.clone()), - ); - edited.insert("hunks".to_string(), Value::from(hunks as i64)); - metadata.insert("edited_file".to_string(), Value::Object(edited)); - - accumulator.push_edited_file(file_path, &change_type, hunks); -} diff --git a/crates/tracedecay-sessions/src/runtime/hosts/claude/source_records.rs b/crates/tracedecay-sessions/src/runtime/hosts/claude/source_records.rs index 3de4aa0802..034a00755c 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/claude/source_records.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/claude/source_records.rs @@ -1,168 +1,14 @@ use std::path::{Path, PathBuf}; -use serde_json::{Map, Value}; +use serde_json::Value; use tracedecay_domain::{ - CanonicalObservationEnvelopeV1, CanonicalObservationFactV1, CanonicalReasoningVisibilityV1, + CanonicalObservationEnvelopeV1, CanonicalObservationFactV1, ObservationOrderingDomainV1, }; -use crate::host_ports::parse_timestamp; -use crate::runtime::SessionMessageRecord; -use crate::runtime::shared::{ - ProjectRootMatcherCache, content_storage_text_and_tools, preview_truncated, -}; -use crate::runtime::source::{RawJsonlFrame, RawJsonlFrameReader, SessionDraft}; -use tracedecay_privacy::{MAX_OBSERVATION_RECORD_BYTES, parse_claude_record_v1}; - -use super::canonical_projection::map_canonical_claude_record; -use super::record_metadata::{ - SessionAccumulator, compact_boundary_row, is_redaction_marker, message_metadata, - model_fallback_row, pr_link_row, record_timestamp, source_position_message_id, -}; -use super::{CWD_PROBE_LINES, KIND_REASONING, MARKER_PREVIEW_BYTES, PROVIDER}; - -/// Durable context shared by V1 folding and the V2 observation projector. -pub struct ClaudeRecordContext<'a> { - pub session_id: &'a str, - pub project_key: &'a str, - pub project_path: &'a str, - pub file_generation: u64, - pub offset: u64, - pub session_cwd: Option<&'a Path>, - pub source_path: Option<&'a str>, - pub raw_message_id: Option<&'a str>, - pub raw_tool_event_ids: &'a [String], - pub raw_hook_tool_use_id: Option<&'a str>, -} - -/// Minimal canonical projection result. Rich reasoning and marker families remain -/// V1 enrichments until their explicit projection contracts. -pub enum ClaudeRecordDisposition { - Message { - draft: Box, - message: Box, - }, - NonConversational, -} - -/// Map one sanitized Claude record to the canonical conversational V1 row. -/// Pure: no I/O, cursor access, global state, or persistence. -pub fn map_sanitized_claude_record( - record: &Value, - context: &ClaudeRecordContext<'_>, -) -> ClaudeRecordDisposition { - map_sanitized_claude_record_with(record, context, None) -} - -/// [`map_sanitized_claude_record`] with cwd worktree resolution routed through -/// a source-lifetime cache, so batch transcript parses do not re-run git -/// discovery once per message row for the same cwd. -pub fn map_sanitized_claude_record_cached( - record: &Value, - context: &ClaudeRecordContext<'_>, - worktree_cache: &ProjectRootMatcherCache, -) -> ClaudeRecordDisposition { - map_sanitized_claude_record_with(record, context, Some(worktree_cache)) -} - -fn map_sanitized_claude_record_with( - record: &Value, - context: &ClaudeRecordContext<'_>, - worktree_cache: Option<&ProjectRootMatcherCache>, -) -> ClaudeRecordDisposition { - if let Ok(envelope) = serde_json::from_value::(record.clone()) { - return map_canonical_claude_record(&envelope, context, worktree_cache); - } - let Ok(offset) = i64::try_from(context.offset) else { - return ClaudeRecordDisposition::NonConversational; - }; - let mut accumulator = SessionAccumulator::default(); - let source_path = context.source_path.map_or_else( - || PathBuf::from(format!("claude:{}", context.session_id)), - PathBuf::from, - ); - let Some(mut message) = message_from_line( - record, - context.session_id, - &source_path, - offset, - context.session_cwd, - &mut accumulator, - worktree_cache, - ) else { - return ClaudeRecordDisposition::NonConversational; - }; - message.message_id = durable_record_message_id(record, context, offset); - let mut metadata = message - .metadata_json - .as_deref() - .and_then(|json| serde_json::from_str::>(json).ok()) - .unwrap_or_default(); - metadata.insert( - "source_generation".to_string(), - Value::from(context.file_generation), - ); - retain_unchanged_tool_event_ids(&mut metadata, context.raw_tool_event_ids); - message.metadata_json = serde_json::to_string(&metadata).ok(); - let draft = SessionDraft { - session_id: context.session_id.to_owned(), - project_key: context.project_key.to_owned(), - project_path: context.project_path.to_owned(), - title: None, - metadata_json: None, - parent_session_id: None, - is_subagent: false, - agent_id: None, - parent_tool_use_id: None, - }; - ClaudeRecordDisposition::Message { - draft: Box::new(draft), - message: Box::new(message), - } -} +use crate::runtime::source::{RawJsonlFrame, RawJsonlFrameReader}; +use tracedecay_privacy::{MAX_OBSERVATION_RECORD_BYTES, parse_observation_record_v1}; -fn durable_record_message_id( - record: &Value, - context: &ClaudeRecordContext<'_>, - offset: i64, -) -> String { - let sanitized_message_id = record - .pointer("/message/id") - .and_then(Value::as_str) - .or_else(|| record.get("uuid").and_then(Value::as_str)) - .filter(|id| !id.is_empty()); - if context.raw_message_id == sanitized_message_id - && sanitized_message_id.is_some_and(|id| !is_redaction_marker(id)) - { - return sanitized_message_id.unwrap_or_default().to_string(); - } - source_position_message_id(context.session_id, context.file_generation, offset) -} - -pub(super) fn retain_unchanged_tool_event_ids( - metadata: &mut Map, - raw_ids: &[String], -) { - let Some(events) = metadata - .get_mut("tool_events") - .and_then(Value::as_array_mut) - else { - return; - }; - let mut raw_ids = raw_ids.iter(); - for event in events { - let Some(event) = event.as_object_mut() else { - continue; - }; - let sanitized = event.get("call_id").and_then(Value::as_str); - if sanitized.is_none() { - continue; - } - let raw = raw_ids.next().map(String::as_str); - if raw != sanitized || sanitized.is_some_and(is_redaction_marker) { - event.remove("call_id"); - } - } -} +use super::CWD_PROBE_LINES; pub fn transcript_cwd(path: &Path) -> Option { let file = std::fs::File::open(path).ok()?; @@ -184,7 +30,8 @@ pub fn transcript_cwd(path: &Path) -> Option { continue; } let range = tracedecay_domain::ObservationSourceRangeV1::new(offset, end_offset).ok()?; - if let Ok(parsed) = parse_claude_record_v1(record, range) + if let Ok(parsed) = + parse_observation_record_v1(record, range, ObservationOrderingDomainV1::FileBytes) && let Some(cwd) = parsed.value().get("cwd").and_then(Value::as_str) && !cwd.is_empty() { @@ -195,455 +42,6 @@ pub fn transcript_cwd(path: &Path) -> Option { None } -/// Map one Claude transcript line to a provider-neutral message, or `None` for -/// lines that carry no conversational text (tool-result-only, meta lines, …). -/// -/// Gate: only `user`/`assistant` records become conversational rows here. Other -/// record types fall through to [`system_hook_message_from_line`] and -/// [`structured_marker_from_line`]. Two record families are deliberately dropped -/// with no row at all, because they are pure bloat/redundancy: -/// -/// * **hook attachments**, records that inject a hook's `hookAdditionalContext` -/// / attachment payload into the transcript. The signal we care about (hook -/// errors / prevented continuation) is already captured as a compact -/// `hook_event` row; the attachment body just duplicates content that lives on -/// the owning turn. -/// * **queue-operation records**, queued/removed user-turn bookkeeping. These -/// are ephemeral UI state; the actual user turn is ingested when it is sent. -pub(super) fn message_from_line( - record: &Value, - session_id: &str, - path: &Path, - offset: i64, - session_cwd: Option<&Path>, - accumulator: &mut SessionAccumulator, - worktree_cache: Option<&ProjectRootMatcherCache>, -) -> Option { - let kind = record.get("type").and_then(Value::as_str)?; - if kind != "user" && kind != "assistant" { - return None; - } - let message = record.get("message").unwrap_or(record); - let role = message - .get("role") - .and_then(Value::as_str) - .unwrap_or(kind) - .to_string(); - - let content = message.get("content").unwrap_or(message); - let indexed_content = if role == "assistant" { - content.as_array().map(|blocks| { - Value::Array( - blocks - .iter() - .filter(|block| { - !matches!( - block.get("type").and_then(Value::as_str), - Some("thinking" | "redacted_thinking") - ) - }) - .cloned() - .collect(), - ) - }) - } else { - None - }; - let content_for_index = indexed_content.as_ref().unwrap_or(content); - let (text, tool_names) = content_storage_text_and_tools( - content_for_index, - message - .get("tool_calls") - .or_else(|| record.get("tool_calls")), - ); - if text.trim().is_empty() { - return None; - } - - let message_id = conversational_message_id(message, record, session_id, offset); - let model = message - .get("model") - .and_then(Value::as_str) - .map(str::to_string); - let timestamp = record - .get("timestamp") - .and_then(Value::as_str) - .and_then(parse_timestamp) - .map(|secs| secs as i64); - - Some(SessionMessageRecord { - provider: PROVIDER.to_string(), - message_id, - session_id: session_id.to_string(), - role, - timestamp, - ordinal: offset, - text, - kind: Some("message".to_string()), - model, - tool_names: (!tool_names.is_empty()).then(|| tool_names.join(",")), - source_path: Some(path.to_string_lossy().to_string()), - source_offset: Some(offset), - metadata_json: serde_json::to_string(&message_metadata( - kind, - record, - message, - content, - session_cwd, - accumulator, - worktree_cache, - )) - .ok(), - }) -} - -/// Stable id for a conversational (`user`/`assistant`) row: the message `id`, -/// else the record `uuid`, else a synthesized `{session}:{offset}`. Shared by -/// the message row and the reasoning row so a reasoning row's -/// `{base}:thinking` id always links back to its owning assistant message. -fn conversational_message_id( - message: &Value, - record: &Value, - session_id: &str, - offset: i64, -) -> String { - message - .get("id") - .and_then(Value::as_str) - .or_else(|| record.get("uuid").and_then(Value::as_str)) - .filter(|id| !id.is_empty()) - .map_or_else(|| format!("{session_id}:{offset}"), ToString::to_string) -} - -/// Emit one searchable reasoning row for plaintext assistant `thinking` blocks. -/// Redacted-only blocks produce no row; the owning message remains unchanged. -pub(super) fn reasoning_from_line( - record: &Value, - path: &Path, - context: &ClaudeRecordContext<'_>, - owning_message_id: Option<&str>, -) -> Option { - if let Ok(envelope) = serde_json::from_value::(record.clone()) { - return reasoning_from_canonical_envelope(&envelope, path, context, owning_message_id); - } - let offset = i64::try_from(context.offset).ok()?; - if record.get("type").and_then(Value::as_str) != Some("assistant") { - return None; - } - let message = record.get("message").unwrap_or(record); - let blocks = message.get("content").and_then(Value::as_array)?; - - let mut thinking_parts = Vec::new(); - let mut redacted_blocks = 0usize; - for block in blocks { - match block.get("type").and_then(Value::as_str) { - Some("thinking") => { - if let Some(text) = block - .get("thinking") - .and_then(Value::as_str) - .filter(|text| !text.trim().is_empty()) - { - thinking_parts.push(text.to_string()); - } - } - Some("redacted_thinking") => redacted_blocks += 1, - _ => {} - } - } - if thinking_parts.is_empty() { - return None; - } - let base_id = owning_message_id.map_or_else( - || durable_record_message_id(record, context, offset), - str::to_string, - ); - let role = message - .get("role") - .and_then(Value::as_str) - .filter(|role| !role.is_empty()) - .unwrap_or("assistant") - .to_string(); - let model = message - .get("model") - .and_then(Value::as_str) - .map(str::to_string); - - build_reasoning_record( - path, - context, - ReasoningRecordParts { - base_id, - role, - model, - timestamp: record_timestamp(record), - thinking_parts, - redacted_blocks, - }, - ) -} - -fn reasoning_from_canonical_envelope( - envelope: &CanonicalObservationEnvelopeV1, - path: &Path, - context: &ClaudeRecordContext<'_>, - owning_message_id: Option<&str>, -) -> Option { - if envelope.native_record_kind() != "assistant" { - return None; - } - let mut thinking_parts = Vec::new(); - let mut redacted_blocks = 0usize; - let mut model = None; - let mut timestamp = None; - for fact in envelope.facts() { - match fact { - CanonicalObservationFactV1::Reasoning { - visibility: CanonicalReasoningVisibilityV1::Visible, - content: Some(content), - } => { - if let Some(text) = content.as_str().filter(|text| !text.trim().is_empty()) { - thinking_parts.push(text.to_owned()); - } - } - CanonicalObservationFactV1::Reasoning { - visibility: CanonicalReasoningVisibilityV1::Redacted, - .. - } => redacted_blocks = redacted_blocks.saturating_add(1), - CanonicalObservationFactV1::Message { - model: fact_model, - timestamp: fact_timestamp, - .. - } => { - model.clone_from(fact_model); - timestamp = *fact_timestamp; - } - _ => {} - } - } - if thinking_parts.is_empty() { - return None; - } - let base_id = owning_message_id.map_or_else( - || envelope.stable_record_id().as_str().to_owned(), - str::to_owned, - ); - build_reasoning_record( - path, - context, - ReasoningRecordParts { - base_id, - role: "assistant".to_owned(), - model, - timestamp, - thinking_parts, - redacted_blocks, - }, - ) -} - -struct ReasoningRecordParts { - base_id: String, - role: String, - model: Option, - timestamp: Option, - thinking_parts: Vec, - redacted_blocks: usize, -} - -fn build_reasoning_record( - path: &Path, - context: &ClaudeRecordContext<'_>, - parts: ReasoningRecordParts, -) -> Option { - let offset = i64::try_from(context.offset).ok()?; - let text = parts.thinking_parts.join("\n\n"); - let mut metadata = Map::new(); - metadata.insert( - "source".to_string(), - Value::String("claude_thinking".to_string()), - ); - // Parent linkage back to the assistant message row that owns this reasoning. - metadata.insert( - "parent_message_id".to_string(), - Value::String(parts.base_id.clone()), - ); - metadata.insert( - "thinking_blocks".to_string(), - Value::from(parts.thinking_parts.len() as i64), - ); - if parts.redacted_blocks > 0 { - metadata.insert( - "redacted_thinking_blocks".to_string(), - Value::from(parts.redacted_blocks as i64), - ); - } - - Some(SessionMessageRecord { - provider: PROVIDER.to_string(), - // `{base}:thinking` keeps re-ingest idempotent and can never collide - // with the owning message row's `{base}` id under the - // `(provider, message_id)` primary key. - message_id: format!("{}:thinking", parts.base_id), - session_id: context.session_id.to_string(), - role: parts.role, - timestamp: parts.timestamp, - ordinal: offset, - text, - kind: Some(KIND_REASONING.to_string()), - model: parts.model, - tool_names: None, - source_path: Some(path.to_string_lossy().to_string()), - source_offset: Some(offset), - metadata_json: serde_json::to_string(&Value::Object(metadata)).ok(), - }) -} - -/// Map a `type=="system"` hook-summary record to a compact, signal-only -/// `hook_event` row, or `None` for non-system records and routine hook -/// summaries that carry no error/interruption signal. -pub(super) fn system_hook_message_from_line( - record: &Value, - path: &Path, - context: &ClaudeRecordContext<'_>, - trusted_tool_use_id: Option<&str>, -) -> Option { - let offset = i64::try_from(context.offset).ok()?; - if record.get("type").and_then(Value::as_str) != Some("system") { - return None; - } - - let hook_errors: Vec<&Value> = record - .get("hookErrors") - .and_then(Value::as_array) - .map(|errors| errors.iter().collect()) - .unwrap_or_default(); - let stop_reason = record - .get("stopReason") - .and_then(Value::as_str) - .filter(|reason| !reason.is_empty()); - let prevented_continuation = record - .get("preventedContinuation") - .and_then(Value::as_bool) - .unwrap_or(false); - if hook_errors.is_empty() && stop_reason.is_none() && !prevented_continuation { - return None; - } - - let subtype = record.get("subtype").and_then(Value::as_str).unwrap_or(""); - let tool_use_id = trusted_tool_use_id; - - let mut lines = vec![format!("Claude hook event: {subtype}")]; - if let Some(tool_use_id) = tool_use_id { - lines.push(format!("tool_use_id: {tool_use_id}")); - } - if let Some(stop_reason) = stop_reason { - lines.push(format!("stop_reason: {stop_reason}")); - } - if prevented_continuation { - lines.push("prevented_continuation: true".to_string()); - } - if !hook_errors.is_empty() { - let joined = hook_errors - .iter() - .map(|error| { - error - .as_str() - .map_or_else(|| error.to_string(), str::to_string) - }) - .collect::>() - .join("; "); - lines.push(format!("hook_errors: {joined}")); - } - let joined = lines.join("\n"); - let text = preview_truncated(&joined, MARKER_PREVIEW_BYTES); - - let message_id = durable_record_message_id(record, context, offset); - let timestamp = record - .get("timestamp") - .and_then(Value::as_str) - .and_then(parse_timestamp) - .map(|secs| secs as i64); - - let mut metadata = Map::new(); - metadata.insert( - "source".to_string(), - Value::String("claude_system_record".to_string()), - ); - metadata.insert("subtype".to_string(), Value::String(subtype.to_string())); - if let Some(tool_use_id) = tool_use_id { - metadata.insert( - "tool_use_id".to_string(), - Value::String(tool_use_id.to_string()), - ); - } - if let Some(hook_count) = record.get("hookCount") { - metadata.insert("hook_count".to_string(), hook_count.clone()); - } - if let Some(level) = record.get("level").and_then(Value::as_str) { - metadata.insert("level".to_string(), Value::String(level.to_string())); - } - if prevented_continuation { - metadata.insert("prevented_continuation".to_string(), Value::Bool(true)); - } - - Some(SessionMessageRecord { - provider: PROVIDER.to_string(), - message_id, - session_id: context.session_id.to_string(), - // role "tool" keeps transient hook telemetry out of LCM policy anchors, which pin role system/developer. - role: "tool".to_string(), - timestamp, - ordinal: offset, - text, - kind: Some("hook_event".to_string()), - model: None, - tool_names: None, - source_path: Some(path.to_string_lossy().to_string()), - source_offset: Some(offset), - metadata_json: serde_json::to_string(&Value::Object(metadata)).ok(), - }) -} - -/// Map a structured, non-conversational Claude record to a marker row: -/// `pr-link` records, `system` compaction boundaries, and model-fallback -/// records. Returns `None` for every other record type (leaving the cursor to -/// advance without emitting a row). -pub(super) fn structured_marker_from_line( - record: &Value, - path: &Path, - context: &ClaudeRecordContext<'_>, - accumulator: &mut SessionAccumulator, -) -> Option { - let offset = i64::try_from(context.offset).ok()?; - match record.get("type").and_then(Value::as_str)? { - "pr-link" => pr_link_row( - record, - context.session_id, - context.file_generation, - path, - offset, - accumulator, - ), - "system" => compact_boundary_row( - record, - context.session_id, - context.file_generation, - path, - offset, - ) - .or_else(|| { - model_fallback_row( - record, - context.session_id, - context.file_generation, - path, - offset, - ) - }), - _ => None, - } -} - /// Read a record's `cwd`, falling back to the canonical envelope's session /// location fact. pub(super) fn record_cwd(record: &Value) -> Option { diff --git a/crates/tracedecay-sessions/src/runtime/hosts/claude/tests.rs b/crates/tracedecay-sessions/src/runtime/hosts/claude/tests.rs index 1bf77dd6c4..444b19784f 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/claude/tests.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/claude/tests.rs @@ -1,5 +1,5 @@ use super::*; -use crate::runtime::SessionMessageRecord; +use crate::runtime::shared::StoredCursor; use serde_json::json; use tracedecay_capture::claude as canonical; use tracedecay_runtime_core::git_discovery::{ @@ -112,213 +112,6 @@ fn bounded_scan_exposes_whitespace_ranges_without_parsing_them() { ); } -#[test] -fn compact_pair_projection_keeps_pairing_evidence() { - let fixtures = format!( - "{}/../../tests/fixtures/provider_normalization/claude", - env!("CARGO_MANIFEST_DIR") - ); - let context = ClaudeRecordContext { - session_id: "claude-compact-pair-session", - project_key: "project-1", - project_path: "/project-1", - file_generation: 1, - offset: 0, - session_cwd: Some(Path::new("/project-1")), - source_path: None, - raw_message_id: None, - raw_tool_event_ids: &[], - raw_hook_tool_use_id: None, - }; - let boundary = map_checked_in_claude_fixture( - &format!("{fixtures}/compact_summary_pair.boundary.input.json"), - &context, - ); - let summary = map_checked_in_claude_fixture( - &format!("{fixtures}/compact_summary_pair.summary.input.json"), - &context, - ); - assert_eq!( - boundary.message_id, - "compact_boundary:ffffffff-0000-1111-2222-333333333333" - ); - let boundary_metadata: Value = - serde_json::from_str(boundary.metadata_json.as_deref().unwrap()).unwrap(); - assert_eq!( - boundary_metadata["canonical_envelope"]["facts"] - .as_array() - .into_iter() - .flatten() - .find_map(|fact| fact.pointer("/summary/preservedSegment/anchorUuid")) - .and_then(Value::as_str), - Some("aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee") - ); - assert_eq!(summary.message_id, "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"); - let summary_metadata: Value = - serde_json::from_str(summary.metadata_json.as_deref().unwrap()).unwrap(); - assert_eq!( - summary_metadata["canonical_envelope"]["relations"]["parent_message_id"], - "ffffffff-0000-1111-2222-333333333333" - ); -} - -fn map_checked_in_claude_fixture( - path: &str, - context: &ClaudeRecordContext<'_>, -) -> SessionMessageRecord { - let bytes = std::fs::read(path).unwrap(); - let range = tracedecay_domain::ObservationSourceRangeV1::new(0, bytes.len() as u64).unwrap(); - let parsed = tracedecay_privacy::parse_normalized_observation_record_v1( - &bytes, - range, - tracedecay_domain::ObservationOrderingDomainV1::FileBytes, - |native| { - let stable = canonical::stable_record_id(&native, context.session_id, 0)?; - canonical::normalize(&native, context.session_id, stable, range) - }, - ) - .unwrap(); - let ClaudeRecordDisposition::Message { message, .. } = - map_sanitized_claude_record(parsed.value(), context) - else { - panic!("{} must map to a persisted row", path); - }; - *message -} - -#[test] -fn legacy_trait_parse_only_folds_sanitizer_issued_values() { - let dir = tempfile::tempdir().unwrap(); - let secret = "password = p@ssw0rd!"; - let project_root = dir.path().join(secret); - std::fs::create_dir_all(&project_root).unwrap(); - let transcript = dir.path().join("session-sanitized.jsonl"); - let record = json!({ - "type": "user", - "uuid": "user-sanitized", - "cwd": project_root, - "message": {"role": "user", "content": secret}, - }); - std::fs::write( - &transcript, - format!("{}\n", serde_json::to_string(&record).unwrap()), - ) - .unwrap(); - - let parsed = ClaudeSource::with_home(Path::new("/unused")) - .parse_new(&transcript, StoredCursor::default(), &project_root, None) - .expect("legacy trait parse"); - let mut durable = parsed.draft.metadata_json.clone().unwrap_or_default(); - for message in &parsed.messages { - durable.push_str(&message.text); - durable.push_str(message.metadata_json.as_deref().unwrap_or_default()); - } - - assert!(!durable.contains("p@ssw0rd!"), "{durable}"); - assert!(durable.contains("[TraceDecay redacted:"), "{durable}"); - assert_eq!(parsed.messages.len(), 1); -} - -#[test] -fn subagent_provider_metadata_is_sanitized_before_persistence() { - let dir = tempfile::tempdir().unwrap(); - let raw_secret = "abcdefghijklmnopqrstuvwxyz0123456789"; - let credential = format!("Bearer {raw_secret}"); - let workflow = format!("wf_ {credential}"); - let transcript = dir - .path() - .join("parent-session") - .join("subagents") - .join("workflows") - .join(&workflow) - .join("agent-child.jsonl"); - std::fs::create_dir_all(transcript.parent().unwrap()).unwrap(); - std::fs::write(&transcript, "").unwrap(); - std::fs::write( - transcript.with_file_name("agent-child.meta.json"), - serde_json::to_vec(&json!({ - "agentType": format!("Explore {credential}"), - "description": format!("Inspect {credential}"), - "toolUseId": format!("tool {credential}"), - "spawnDepth": 2, - })) - .unwrap(), - ) - .unwrap(); - - let info = claude_subagent_identity(&transcript).expect("subagent identity"); - let durable = serde_json::to_string(&session_metadata( - None, - Some(&info), - &SessionAccumulator::default(), - None, - )) - .unwrap(); - - assert!(!durable.contains(raw_secret), "{durable}"); - assert_eq!(info.parent_tool_use_id, None); - assert_eq!(info.workflow_run_id, None); - assert!(durable.contains("[TraceDecay redacted:")); - assert_eq!(info.spawn_depth, Some(2)); - - let other_secret = "0123456789abcdefghijklmnopqrstuvwxyz"; - let other_credential = format!("Bearer {other_secret}"); - let other_workflow = format!("wf_ {other_credential}"); - let other_transcript = dir - .path() - .join("parent-session") - .join("subagents") - .join("workflows") - .join(&other_workflow) - .join("agent-other.jsonl"); - std::fs::create_dir_all(other_transcript.parent().unwrap()).unwrap(); - std::fs::write(&other_transcript, "").unwrap(); - std::fs::write( - other_transcript.with_file_name("agent-other.meta.json"), - serde_json::to_vec(&json!({ - "toolUseId": format!("tool {other_credential}"), - })) - .unwrap(), - ) - .unwrap(); - - let other = claude_subagent_identity(&other_transcript).expect("other identity"); - assert_eq!(other.parent_tool_use_id, None); - assert_eq!(other.workflow_run_id, None); -} - -#[test] -fn subagent_metadata_exceeding_structural_limits_is_denied_as_a_whole() { - let dir = tempfile::tempdir().unwrap(); - let transcript = dir - .path() - .join("parent-session") - .join("subagents") - .join("agent-deep.jsonl"); - std::fs::create_dir_all(transcript.parent().unwrap()).unwrap(); - std::fs::write(&transcript, "").unwrap(); - - let mut nested = json!(true); - for _ in 0..=tracedecay_capture::ParseLimits::default_policy().depth { - nested = json!({"next": nested}); - } - std::fs::write( - transcript.with_file_name("agent-deep.meta.json"), - serde_json::to_vec(&json!({ - "agentType": "must-not-survive-partial-scan", - "nested": nested, - })) - .unwrap(), - ) - .unwrap(); - - let info = claude_subagent_identity(&transcript).expect("subagent identity"); - - assert_eq!(info.agent_type, None); - assert_eq!(info.description, None); - assert_eq!(info.spawn_depth, None); -} - #[test] fn cursor_key_round_trips_native_bytes_without_collisions() { let native_path: Vec = r"C:\Users\zack\.claude\projects\session.jsonl" @@ -362,10 +155,9 @@ fn non_utf8_paths_that_render_identically_have_distinct_cursor_keys() { let second = PathBuf::from(OsString::from_vec(b"session-\xfe.jsonl".to_vec())); assert_eq!(first.to_string_lossy(), second.to_string_lossy()); - let source = ClaudeSource::with_home(Path::new("/unused")); assert_ne!( - source.cursor_key(&first).durable_text(), - source.cursor_key(&second).durable_text() + cursor::claude_cursor_key(&first).durable_text(), + cursor::claude_cursor_key(&second).durable_text() ); let first_identity = identify_claude_source(&first).unwrap(); let second_identity = identify_claude_source(&second).unwrap(); @@ -417,239 +209,6 @@ fn observation_source_ids_are_private_and_follow_native_transcript_identity() { } } -#[test] -fn structured_git_operation_becomes_host_commit_evidence() { - let mut metadata = Map::new(); - append_git_operation_metadata( - &mut metadata, - &json!({ - "gitBranch": "feature/attribution", - "toolUseResult": { - "gitOperation": { - "commit": {"sha": "ABCDEF12", "kind": "commit"} - } - } - }), - ); - assert_eq!(metadata["produced_commit_candidates"], json!(["abcdef12"])); - assert_eq!(metadata["produced_commit_evidence"], "host_event"); - assert_eq!(metadata["git_branch"], "feature/attribution"); -} - -#[test] -fn unstructured_user_content_cannot_spoof_commit_evidence() { - let mut metadata = Map::new(); - append_git_operation_metadata( - &mut metadata, - &json!({"message": {"content": "gitOperation commit abcdef12"}}), - ); - assert!(metadata.is_empty()); -} - -fn assistant_record(content: &Value) -> Value { - json!({ - "type": "assistant", - "sessionId": "sess", - "uuid": "u-assistant", - "timestamp": "2026-01-01T00:00:05.000Z", - "message": { - "id": "msg_1", - "role": "assistant", - "model": "claude-opus-4-8", - "content": content.clone(), - } - }) -} - -fn record_context(raw_message_id: Option<&str>, offset: u64) -> ClaudeRecordContext<'_> { - ClaudeRecordContext { - session_id: "sess", - project_key: "project", - project_path: "/project", - file_generation: 7, - offset, - session_cwd: None, - source_path: None, - raw_message_id, - raw_tool_event_ids: &[], - raw_hook_tool_use_id: None, - } -} - -#[test] -fn thinking_blocks_are_split_from_the_visible_message_row() { - let record = assistant_record(&json!([ - {"type": "thinking", "thinking": "First I inspect the parser."}, - {"type": "thinking", "thinking": "Then I add the row."}, - {"type": "tool_use", "name": "Read", "input": {"file_path": "src/lib.rs"}}, - {"type": "text", "text": "Done."} - ])); - let path = Path::new("/tmp/sess.jsonl"); - - let mut accumulator = SessionAccumulator::default(); - let message = message_from_line(&record, "sess", path, 10, None, &mut accumulator, None) - .expect("assistant message row"); - assert_eq!(message.message_id, "msg_1"); - assert_eq!(message.kind.as_deref(), Some("message")); - assert!(!message.text.contains("First I inspect the parser")); - assert!(!message.text.contains("Then I add the row")); - assert!(message.text.contains("src/lib.rs")); - assert!(message.text.contains("Done.")); - assert_eq!(message.tool_names.as_deref(), Some("Read")); - - let context = record_context(Some("msg_1"), 10); - let reasoning = reasoning_from_line(&record, path, &context, Some(message.message_id.as_str())) - .expect("reasoning row for thinking"); - assert_eq!(reasoning.message_id, "msg_1:thinking"); - assert_eq!(reasoning.kind.as_deref(), Some("reasoning")); - assert_eq!(reasoning.role, "assistant"); - assert_eq!(reasoning.model.as_deref(), Some("claude-opus-4-8")); - assert_eq!(reasoning.ordinal, 10); - assert_eq!(reasoning.timestamp, Some(1_767_225_605)); - assert_eq!( - reasoning.text, - "First I inspect the parser.\n\nThen I add the row." - ); - let metadata: Value = serde_json::from_str(reasoning.metadata_json.as_deref().unwrap()) - .expect("reasoning metadata json"); - assert_eq!(metadata["source"], "claude_thinking"); - assert_eq!(metadata["parent_message_id"], "msg_1"); - assert_eq!(metadata["thinking_blocks"], 2); - assert!(metadata.get("redacted_thinking_blocks").is_none()); -} - -#[test] -fn redacted_only_thinking_records_no_reasoning_row() { - // Matches Codex's encrypted-reasoning convention: no plaintext, no row. - let record = assistant_record(&json!([ - {"type": "redacted_thinking", "data": "ENCRYPTED_SHOULD_NOT_INDEX"}, - {"type": "text", "text": "Answer."} - ])); - assert!( - reasoning_from_line( - &record, - Path::new("/tmp/sess.jsonl"), - &record_context(Some("msg_1"), 3), - None, - ) - .is_none() - ); -} - -#[test] -fn mixed_thinking_and_redacted_records_the_redacted_count_but_no_plaintext() { - let record = assistant_record(&json!([ - {"type": "thinking", "thinking": "Visible reasoning."}, - {"type": "redacted_thinking", "data": "ENCRYPTED_SHOULD_NOT_INDEX"} - ])); - let reasoning = reasoning_from_line( - &record, - Path::new("/tmp/sess.jsonl"), - &record_context(Some("msg_1"), 4), - Some("msg_1"), - ) - .expect("reasoning row for the plaintext block"); - assert_eq!(reasoning.text, "Visible reasoning."); - assert!(!reasoning.text.contains("ENCRYPTED")); - let metadata: Value = - serde_json::from_str(reasoning.metadata_json.as_deref().unwrap()).unwrap(); - assert_eq!(metadata["thinking_blocks"], 1); - assert_eq!(metadata["redacted_thinking_blocks"], 1); -} - -#[test] -fn reasoning_row_id_falls_back_to_record_uuid_when_message_id_is_absent() { - let record = json!({ - "type": "assistant", - "sessionId": "sess", - "uuid": "u-fallback", - "timestamp": "2026-01-01T00:00:05.000Z", - "message": { - "role": "assistant", - "content": [{"type": "thinking", "thinking": "Reasoning without a message id."}] - } - }); - let reasoning = reasoning_from_line( - &record, - Path::new("/tmp/sess.jsonl"), - &record_context(Some("u-fallback"), 9), - None, - ) - .expect("reasoning row"); - assert_eq!(reasoning.message_id, "u-fallback:thinking"); - let metadata: Value = - serde_json::from_str(reasoning.metadata_json.as_deref().unwrap()).unwrap(); - assert_eq!(metadata["parent_message_id"], "u-fallback"); -} - -#[test] -fn redacted_identity_uses_generation_offset_for_message_and_reasoning() { - let marker = "[TraceDecay redacted:credential]"; - let record = json!({ - "type": "assistant", - "uuid": marker, - "message": { - "id": marker, - "role": "assistant", - "content": [ - {"type": "thinking", "thinking": "private chain"}, - {"type": "text", "text": "answer"} - ] - } - }); - let context = record_context(Some("raw-sensitive-id"), 19); - let ClaudeRecordDisposition::Message { message, .. } = - map_sanitized_claude_record(&record, &context) - else { - panic!("assistant row must map"); - }; - assert_eq!(message.message_id, "sess:7:19"); - - let reasoning = reasoning_from_line( - &record, - Path::new("/tmp/sess.jsonl"), - &context, - Some(message.message_id.as_str()), - ) - .expect("reasoning row"); - let metadata: Value = - serde_json::from_str(reasoning.metadata_json.as_deref().unwrap()).unwrap(); - assert_eq!(reasoning.message_id, "sess:7:19:thinking"); - assert_eq!(metadata["parent_message_id"], "sess:7:19"); -} - -#[test] -fn redacted_marker_ids_do_not_collide() { - let record = json!({ - "type": "pr-link", - "uuid": "[TraceDecay redacted:credential]", - "prNumber": 5, - }); - let mut accumulator = SessionAccumulator::default(); - let first = record_metadata::pr_link_row( - &record, - "sess", - 7, - Path::new("/tmp/sess.jsonl"), - 10, - &mut accumulator, - ) - .unwrap(); - let second = record_metadata::pr_link_row( - &record, - "sess", - 7, - Path::new("/tmp/sess.jsonl"), - 20, - &mut accumulator, - ) - .unwrap(); - - assert_eq!(first.message_id, "sess:7:10"); - assert_eq!(second.message_id, "sess:7:20"); - assert_ne!(first.message_id, second.message_id); -} - #[test] fn claude_checked_in_assistant_fixture_crosses_the_canonical_boundary() { let path = format!( @@ -981,62 +540,6 @@ fn retrying_identity(path: &Path) -> GitRepositoryIdentityOutcome { }) } -#[test] -fn claude_message_metadata_reuses_worktree_for_repeated_cwd() { - let temp = tempfile::TempDir::new().expect("temp dir"); - let project_root = temp.path().join("repo"); - let nested_cwd = project_root.join("packages/app"); - std::fs::create_dir_all(&nested_cwd).expect("nested cwd"); - let status = std::process::Command::new("git") - .args(["init", "-q"]) - .current_dir(&project_root) - .status() - .expect("git init"); - assert!(status.success()); - - let cache = crate::runtime::shared::ProjectRootMatcherCache::default(); - let record = json!({ - "type": "user", - "sessionId": "sess", - "cwd": nested_cwd, - "message": {"role": "user", "content": "hello"} - }); - let path = Path::new("/tmp/sess.jsonl"); - - let mut accumulator = SessionAccumulator::default(); - let first = message_from_line( - &record, - "sess", - path, - 10, - Some(&nested_cwd), - &mut accumulator, - Some(&cache), - ) - .expect("first message"); - let first_metadata: serde_json::Value = - serde_json::from_str(first.metadata_json.as_deref().unwrap()).unwrap(); - let first_worktree = first_metadata["claude_message_worktree"].clone(); - assert!(first_worktree.is_string()); - - std::fs::rename(project_root.join(".git"), project_root.join(".git.hidden")) - .expect("hide git metadata after first lookup"); - - let second = message_from_line( - &record, - "sess", - path, - 20, - Some(&nested_cwd), - &mut accumulator, - Some(&cache), - ) - .expect("second message"); - let second_metadata: serde_json::Value = - serde_json::from_str(second.metadata_json.as_deref().unwrap()).unwrap(); - assert_eq!(second_metadata["claude_message_worktree"], first_worktree); -} - #[test] fn claude_unknown_membership_retries_without_advancing_cursor() { use std::sync::atomic::Ordering; @@ -1063,18 +566,30 @@ fn claude_unknown_membership_retries_without_advancing_cursor() { source.project_matchers = crate::runtime::shared::ProjectRootMatcherCache::with_identity_resolver(retrying_identity); - let previous = StoredCursor::default(); + let scan = || { + try_scan_claude_source_frames_with_resume( + identify_claude_source(&transcript).unwrap(), + StoredCursor::default(), + None, + None, + ) + .unwrap() + .unwrap() + }; + + let mut first = scan(); assert!( source - .parse_new(&transcript, previous, &project_root, None) + .retain_scoped_frames(&mut first, &project_root) .is_none(), - "unknown membership must abort before a new cursor can be persisted" + "unknown membership must defer the scan before a cursor or skip range can be persisted" ); - let retried = source - .parse_new(&transcript, previous, &project_root, None) + let mut retried = scan(); + let excluded = source + .retain_scoped_frames(&mut retried, &project_root) .expect("unknown membership must be resolved again on retry"); - assert_eq!(retried.messages.len(), 1); - assert!(retried.new_cursor.position > previous.position); + assert!(excluded.is_empty()); + assert_eq!(retried.frames.len(), 1); assert_eq!(UNKNOWN_PATH_ATTEMPTS.load(Ordering::SeqCst), 3); } diff --git a/crates/tracedecay-sessions/src/runtime/hosts/claude_observation.rs b/crates/tracedecay-sessions/src/runtime/hosts/claude_observation.rs index ae57c4b45b..a2f16bf335 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/claude_observation.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/claude_observation.rs @@ -10,9 +10,9 @@ use std::path::{Path, PathBuf}; use thiserror::Error; use tracedecay_domain::{ DomainError, ObservationContractError, ObservationId, ObservationIdentityMaterialV1, - ObservationScopeV1, ObservationSourceCursorV1, ObservationSourceGenerationV1, - ObservationSourceIdentityV1, ObservationSourceRangeV1, RetentionClass, SanitizationReceiptV1, - SessionId, + ObservationOrderingDomainV1, ObservationScopeV1, ObservationSourceCursorV1, + ObservationSourceGenerationV1, ObservationSourceIdentityV1, ObservationSourceRangeV1, + RetentionClass, SanitizationReceiptV1, SessionId, }; use tracedecay_store::observation::{ CursorAdvanceOutcome, NonDurableFrameReason, ObservationCursorAdvance, @@ -27,7 +27,7 @@ use crate::observation::{ CaptureClaudeObservationOutcome, CaptureClaudeObservationRequest, CaptureClaudeObservationRequestError, ObservationApplicationError, ObservationCancellation, }; -use crate::runtime::claude::{ +use crate::runtime::hosts::claude::{ ClaudeFrameCoverage, ClaudeSkippedFrame, ClaudeSkippedFrameReason, ClaudeSource, ClaudeSourceFrame, identify_claude_source, try_scan_claude_source_frames_with_resume, }; @@ -36,8 +36,7 @@ use crate::runtime::shared::{StoredCursor, TranscriptIngestStats}; use crate::runtime::snapshot_observation::host_admission_error; use crate::runtime::source::{ HostProviderCoverage, JsonlResumeState, STRICT_JSONL_BATCH_BYTES, TranscriptDiscoveryBounds, - TranscriptIngestError, TranscriptSource, persist_host_provider_coverage, - run_blocking_transcript_section, + TranscriptIngestError, persist_host_provider_coverage, run_blocking_transcript_section, }; use tracedecay_privacy::PrivacySanitizerError; @@ -290,7 +289,13 @@ fn cursor_at( offset: u64, resume_checkpoint: Option<(u64, u64)>, ) -> Result { - let cursor = ObservationSourceCursorV1::new(source.clone(), scope.clone(), generation, offset)?; + let cursor = ObservationSourceCursorV1::for_ordering( + source.clone(), + scope.clone(), + generation, + ObservationOrderingDomainV1::FileBytes, + offset, + )?; Ok( resume_checkpoint.map_or(cursor.clone(), |(file_identity, resume_fingerprint)| { cursor.with_resume_checkpoint(file_identity, resume_fingerprint) @@ -434,20 +439,9 @@ async fn capture_frame( } }; match captured { - CaptureClaudeObservationOutcome::Persisted { - outcome, - sanitized_record, - .. - } - | CaptureClaudeObservationOutcome::AcceptedForReplay { - outcome, - sanitized_record, - .. - } => { + CaptureClaudeObservationOutcome::Persisted { outcome, .. } + | CaptureClaudeObservationOutcome::AcceptedForReplay { outcome, .. } => { let receipt = outcome.receipt(); - if !frame.set_sanitized_record(*sanitized_record) { - return Err(ClaudeObservationIngestError::InvalidFrameState); - } Ok(FrameCaptureOutcome::Persisted(CapturedClaudeFrame { committed_cursor: receipt.committed_cursor().clone(), exact_duplicate: matches!( @@ -1208,13 +1202,11 @@ async fn scheduled_source_paths( admission: &A, scope: &ObservationScopeV1, source: &ClaudeSource, - project_root: &Path, ) -> Result<(Vec, usize), ClaudeObservationIngestError> { let discovery = hotpath::measure_block!( "sessions.hosts.claude.discover_blocking", run_blocking_transcript_section(|| { - source - .discover_transcript_paths(project_root, TranscriptDiscoveryBounds::default_walk()) + source.discover_transcript_paths(TranscriptDiscoveryBounds::default_walk()) }) ); let discovery_truncated = discovery.is_truncated(); @@ -1288,7 +1280,7 @@ where scope: &scope, cancellation: &cancellation, }; - let (paths, deferred) = scheduled_source_paths(admission, &scope, source, project_root).await?; + let (paths, deferred) = scheduled_source_paths(admission, &scope, source).await?; let scheduled_source_count = paths.len(); let mut stats = ClaudeObservationIngestStats { deferred_sources: u64::try_from(deferred).unwrap_or(u64::MAX), diff --git a/crates/tracedecay-sessions/src/runtime/hosts/claude_observation/tests.rs b/crates/tracedecay-sessions/src/runtime/hosts/claude_observation/tests.rs index 101fd61f25..2569745e8b 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/claude_observation/tests.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/claude_observation/tests.rs @@ -5,7 +5,7 @@ use tempfile::TempDir; use super::*; use crate::admission::test_support::MemoryHostAdmission; -use crate::runtime::claude::{scan_claude_source_frames, try_scan_claude_source_frames}; +use crate::runtime::hosts::claude::scan_claude_source_frames; #[path = "tests/projection.rs"] mod projection; @@ -213,24 +213,23 @@ async fn production_vertical_persists_only_sanitized_payload_and_searchable_v1_r ); let source = fixture.source("production-session"); assert_eq!( - source.transcript_paths(&fixture.profile), + source + .discover_transcript_paths(TranscriptDiscoveryBounds::default_walk()) + .paths, vec![fixture.transcript.clone()] ); - let (scheduled, deferred) = scheduled_source_paths( - &fixture.admission, - &ObservationScopeV1::Profile, - &source, - &fixture.profile, - ) - .await - .unwrap(); + let (scheduled, deferred) = + scheduled_source_paths(&fixture.admission, &ObservationScopeV1::Profile, &source) + .await + .unwrap(); assert_eq!(scheduled, vec![fixture.transcript.clone()]); assert_eq!(deferred, 0); let identity = identify_claude_source(&fixture.transcript).unwrap(); - let scan = try_scan_claude_source_frames( + let scan = try_scan_claude_source_frames_with_resume( identity, StoredCursor::default(), Some(STRICT_JSONL_BATCH_BYTES), + None, ) .unwrap() .unwrap(); @@ -352,7 +351,7 @@ async fn registered_claude_ingest_api_routes_through_observation_authority() { fixture.write_record("legacy API searchable", "legacy-api-secret"); let stats = crate::runtime::with_transcript_source_home( fixture.home.clone(), - crate::runtime::claude::ingest_user_sessions_with_admission( + crate::runtime::hosts::claude::ingest_user_sessions_with_admission( &fixture.profile, Some("legacy-api-session".to_string()), Vec::new(), diff --git a/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs b/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs index 195997fa0e..8223021f1d 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs @@ -34,7 +34,7 @@ mod goal_event_tests { use crate::admission::HostAdmission; use crate::admission::test_support::MemoryHostAdmission; use crate::observation::{CaptureObservationRequest, ObservationCancellation}; - use crate::runtime::codex::{ + use crate::runtime::hosts::codex::{ try_admit_codex_jsonl_observations_for_project_window, try_admit_codex_jsonl_observations_for_project_with_admission, }; @@ -1494,7 +1494,7 @@ mod recent_first_discovery_tests { use super::CodexSource; use crate::admission::test_support::MemoryHostAdmission; - use crate::runtime::codex::{ + use crate::runtime::hosts::codex::{ CodexCorpusEpoch, CodexDiscoveryDelivery, CodexDiscoveryFrontier, CodexDiscoveryHub, CodexDiscoverySourceKey, CodexDiscoveryState, CodexExactSessionPathAuthority, CodexIndexedPath, CodexReplayIndex, EXACT_HOOK_DISCOVERY_UNITS_PER_CALL, @@ -1881,7 +1881,6 @@ mod recent_first_discovery_tests { .expect("one source replay index"); assert_eq!(index.completed_enumerations, 1); assert_eq!(index.files_considered, 73); - assert!(!index._memory.is_empty()); } #[tokio::test] diff --git a/crates/tracedecay-sessions/src/runtime/hosts/codex_app_server.rs b/crates/tracedecay-sessions/src/runtime/hosts/codex_app_server.rs index 099e1558f8..fe0e78a1b6 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/codex_app_server.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/codex_app_server.rs @@ -169,22 +169,16 @@ impl CodexAppServerLaunchReceipt { } } -impl Default for CodexAppServerSummaryConfig { - fn default() -> Self { - Self { - codex_bin: "codex".to_string(), +impl CodexAppServerSummaryConfig { + /// Tuning for an executable the caller resolved through configuration + /// (`lcm.summarizer_executables.v1`). Only the model and timeout knobs + /// come from the environment; the binary is never looked up on `PATH`. + pub fn for_executable(codex_bin: &Path) -> Self { + let mut config = Self { + codex_bin: codex_bin.to_string_lossy().into_owned(), model: Some("gpt-5.6-sol".to_owned()), timeout: Duration::from_secs(90), - } - } -} - -impl CodexAppServerSummaryConfig { - pub fn from_env() -> Self { - let mut config = Self::default(); - if let Some(bin) = non_empty_env("TRACEDECAY_CODEX_BIN") { - config.codex_bin = bin; - } + }; if let Some(model) = non_empty_env("TRACEDECAY_CODEX_SUMMARY_MODEL") { config.model = Some(model); } diff --git a/crates/tracedecay-sessions/src/runtime/hosts/cursor.rs b/crates/tracedecay-sessions/src/runtime/hosts/cursor.rs index 8baa699f97..a2b34ba47e 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/cursor.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/cursor.rs @@ -1036,7 +1036,7 @@ const SLUG_DECODE_PROBE_BUDGET: u32 = 4096; pub struct CursorSweepSource { cursor_projects_dir: PathBuf, /// Session ids already owned by the richer composer store - /// ([`crate::runtime::cursor_composer`]). Transcript files whose stem is + /// ([`crate::runtime::hosts::cursor_composer`]). Transcript files whose stem is /// one of these are skipped so the two Cursor sources never double-ingest. skip_session_ids: std::collections::HashSet, user_registered_slugs: Option>, diff --git a/crates/tracedecay-sessions/src/runtime/hosts/cursor_composer.rs b/crates/tracedecay-sessions/src/runtime/hosts/cursor_composer.rs index 6746a49d9d..0f2dcc2887 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/cursor_composer.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/cursor_composer.rs @@ -2,7 +2,7 @@ //! //! Cursor's primary chat history does not live in the //! `~/.cursor/projects//agent-transcripts/**.jsonl` files that -//! [`crate::runtime::cursor`] sweeps, those cover only a slice of activity. +//! [`crate::runtime::hosts::cursor`] sweeps, those cover only a slice of activity. //! The bulk lives in two SQLite-backed stores this module reads **strictly //! read-only**: //! @@ -39,7 +39,7 @@ //! the snapshot generation and `SnapshotOrder`, so a sweep replays only //! uncovered positions. Because a composer session id equals the stem of its //! JSONL transcript for ~94% of sessions, the composer sweep runs *before* the -//! JSONL [`crate::runtime::cursor::CursorSweepSource`] and hands it the set of +//! JSONL [`crate::runtime::hosts::cursor::CursorSweepSource`] and hands it the set of //! composer-owned session ids to skip, so the richer composer rows win and no //! message row is ever double-ingested. diff --git a/crates/tracedecay-sessions/src/runtime/hosts/cursor_composer/ingest.rs b/crates/tracedecay-sessions/src/runtime/hosts/cursor_composer/ingest.rs index 1ce6a80c61..29b723b337 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/cursor_composer/ingest.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/cursor_composer/ingest.rs @@ -157,8 +157,8 @@ impl ComposerIngestContext<'_, '_> { async fn drain_composer_projection_queue( context: &ComposerIngestContext<'_, '_>, -) -> TranscriptIngestResult { - crate::runtime::cursor::projection::drain_cursor_observation_projections_with_sessions( +) -> TranscriptIngestResult { + crate::runtime::hosts::cursor::projection::drain_cursor_observation_projections_with_sessions( context.facade, &context.scope, context.cancellation, diff --git a/crates/tracedecay-sessions/src/runtime/hosts/cursor_composer/tests.rs b/crates/tracedecay-sessions/src/runtime/hosts/cursor_composer/tests.rs index 376d71f879..80e4f15780 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/cursor_composer/tests.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/cursor_composer/tests.rs @@ -772,10 +772,10 @@ fn composer_discovery_and_durable_id_batch_use_indexed_bounds() { connection .execute_batch( "CREATE TABLE cursorDiskKV (key TEXT PRIMARY KEY, value TEXT); - CREATE TABLE session_messages ( + CREATE TABLE lcm_raw_messages ( provider TEXT NOT NULL, message_id TEXT NOT NULL, - PRIMARY KEY(provider, message_id) + UNIQUE(provider, message_id) );", ) .unwrap(); @@ -809,11 +809,11 @@ fn composer_discovery_and_durable_id_batch_use_indexed_bounds() { message_plan .iter() .all(|detail| !detail.contains("SCAN messages")), - "durable message-id batches must not scan session_messages: {message_plan:?}" + "durable message-id batches must not scan lcm_raw_messages: {message_plan:?}" ); connection .execute_batch( - "INSERT INTO session_messages(provider, message_id) + "INSERT INTO lcm_raw_messages(provider, message_id) VALUES ('cursor', 'comp:b2'), ('codex', 'comp:b1');", ) .unwrap(); diff --git a/crates/tracedecay-sessions/src/runtime/hosts/cursor_composer/tests/projection.rs b/crates/tracedecay-sessions/src/runtime/hosts/cursor_composer/tests/projection.rs index c2e1ebff12..9c3e7d5835 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/cursor_composer/tests/projection.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/cursor_composer/tests/projection.rs @@ -116,7 +116,8 @@ fn write_composer_state( } fn write_cursor_jsonl(home: &std::path::Path, project: &std::path::Path, session_id: &str) { - let slug = crate::runtime::cursor::cursor_project_slug(project).expect("Cursor project slug"); + let slug = + crate::runtime::hosts::cursor::cursor_project_slug(project).expect("Cursor project slug"); let transcript_dir = home .join(".cursor") .join("projects") @@ -326,7 +327,7 @@ async fn queued_jsonl_projection_does_not_hide_new_message_from_the_same_session let jsonl = crate::runtime::with_transcript_source_home( home.path().to_path_buf(), - crate::runtime::cursor::try_ingest_cursor_project_sweep_capped( + crate::runtime::hosts::cursor::try_ingest_cursor_project_sweep_capped( project.path(), &admission, project_id, @@ -478,7 +479,7 @@ async fn capture_admission_failure_defers_owned_session_before_jsonl_handoff() { assert_eq!(outcome.messages_upserted, 0); let jsonl = crate::runtime::with_transcript_source_home( home.path().to_path_buf(), - crate::runtime::cursor::try_ingest_cursor_project_sweep_capped( + crate::runtime::hosts::cursor::try_ingest_cursor_project_sweep_capped( project.path(), &admission, project_id, diff --git a/crates/tracedecay-sessions/src/runtime/hosts/hermes.rs b/crates/tracedecay-sessions/src/runtime/hosts/hermes.rs index 3516e04fca..32b4df6dbb 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/hermes.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/hermes.rs @@ -30,6 +30,7 @@ mod state_db; #[cfg(test)] mod tests; +pub use ingest::ingest_user_sessions_capped_with_admission; pub use ingest::{ HermesSweepOutcome, ProjectIngestDestination, ingest_for_project, ingest_for_project_capped, ingest_for_project_capped_with_admission, @@ -37,7 +38,6 @@ pub use ingest::{ ingest_homes_capped, ingest_homes_capped_with_admission, ingest_homes_for_projects, ingest_user_homes, ingest_user_homes_capped, ingest_user_sessions_capped, }; -pub use ingest::{ingest_legacy_pinned_profile, ingest_user_sessions_capped_with_admission}; #[cfg(all(test, windows))] use coverage::sqlite_incarnation; diff --git a/crates/tracedecay-sessions/src/runtime/hosts/hermes/ingest.rs b/crates/tracedecay-sessions/src/runtime/hosts/hermes/ingest.rs index 3d2ca7642f..ecdb7d9191 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/hermes/ingest.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/hermes/ingest.rs @@ -6,10 +6,9 @@ use std::path::{Path, PathBuf}; use tracedecay_domain::{ObservationScopeV1, ProjectId}; use crate::admission::HostAdmission; -use crate::host_ports::hermes_profile_pin::resolve as read_config_pinned_project_root; use crate::observation::ObservationCancellation; use crate::runtime::ingest_byte_budget::IngestByteBudget; -use crate::runtime::shared::{TranscriptIngestStats, path_belongs_to_project}; +use crate::runtime::shared::TranscriptIngestStats; use crate::runtime::source::run_blocking_transcript_section; use super::DEFAULT_HERMES_SWEEP_BYTES; @@ -198,9 +197,7 @@ pub async fn ingest_homes_for_projects( run_blocking_transcript_section(|| { destinations .iter() - .filter(|destination| { - source_is_candidate_for_project(&source, destination.project_root) - }) + .filter(|destination| project_is_real(destination.project_root)) .cloned() .collect::>() }) @@ -472,77 +469,10 @@ async fn ingest_user_homes_capped_with_admission( outcome } -/// Strict one-time import for a legacy profile whose project pin was already -/// resolved by the migration layer. Unlike the normal catch-up sweep, any -/// open/query/write failure is returned so callers retain the pin and source. -#[hotpath::measure(label = "sessions.hosts.hermes.ingest_legacy", future = true)] -pub async fn ingest_legacy_pinned_profile( - admission: &dyn HostAdmission, - profile_dir: &Path, - project_root: &Path, - project_id: ProjectId, -) -> Result { - let source = hotpath::measure_block!( - "sessions.hosts.hermes.prepare_legacy_profile_blocking", - run_blocking_transcript_section(|| { - let state_db = profile_dir.join("state.db"); - if !state_db.is_file() { - return Ok::, String>(None); - } - let legacy_project_pin = - read_config_pinned_project_root(&profile_dir.join("config.yaml")) - .map(PathBuf::from) - .ok_or_else(|| { - format!( - "legacy Hermes state store '{}' has no project pin", - state_db.display() - ) - })?; - Ok(Some(HermesProfileSource { - state_db, - legacy_project_pin: Some(legacy_project_pin), - profile: profile_dir - .file_name() - .and_then(|name| name.to_str()) - .map(str::to_string), - })) - }) - )?; - let Some(source) = source else { - return Ok(TranscriptIngestStats::default()); - }; - let scope = ObservationScopeV1::Project { - project_id: project_id.clone(), - }; - let mut budget = new_sweep_budget(None); - let stats = try_ingest_state_db_bounded_with_admission( - &source, - project_root, - project_id, - admission, - &mut budget, - &ObservationCancellation::default(), - ) - .await?; - if budget.deferred() { - return Err(format!( - "legacy Hermes state store '{}' exceeded the bounded import sweep", - source.state_db.display() - )); - } - drain_hermes_projections_with_admission(admission, &scope).await?; - Ok(stats) -} - -/// Locates the `state.db` of every profile that maps to `project_root`. -/// -/// A legacy project pin may associate an entire profile. Otherwise the -/// profile is only a bounded candidate source and each session must carry a -/// matching code-project cwd. -/// +/// A profile `state.db` is only a bounded candidate source: each session must +/// carry a matching code-project cwd. pub(super) struct HermesProfileSource { pub state_db: PathBuf, - pub legacy_project_pin: Option, pub profile: Option, } @@ -563,14 +493,7 @@ fn all_profile_sources(hermes_homes: &[PathBuf]) -> Vec { for (profile_dir, profile) in profiles { let state_db = profile_dir.join("state.db"); if state_db.is_file() && seen.insert(state_db.clone()) { - out.push(HermesProfileSource { - state_db, - legacy_project_pin: read_config_pinned_project_root( - &profile_dir.join("config.yaml"), - ) - .map(PathBuf::from), - profile, - }); + out.push(HermesProfileSource { state_db, profile }); } } } @@ -580,9 +503,9 @@ fn all_profile_sources(hermes_homes: &[PathBuf]) -> Vec { fn candidate_state_dbs(hermes_homes: &[PathBuf], project_root: &Path) -> Vec { let mut out = Vec::new(); let mut seen = BTreeSet::new(); - let project_is_real = tracedecay_runtime_core::worktree::git_worktree_root(project_root) - .is_some() - || tracedecay_runtime_core::config::has_project_database(project_root); + if !project_is_real(project_root) { + return out; + } for home in hermes_homes { let mut candidates: Vec<(PathBuf, Option)> = vec![(home.clone(), None)]; if let Ok(entries) = std::fs::read_dir(home.join("profiles")) { @@ -602,40 +525,17 @@ fn candidate_state_dbs(hermes_homes: &[PathBuf], project_root: &Path) -> Vec bool { - if source - .legacy_project_pin - .as_deref() - .is_some_and(|pin| !path_belongs_to_project(pin, project_root)) - { - return false; - } - source.legacy_project_pin.is_some() - || tracedecay_runtime_core::worktree::git_worktree_root(project_root).is_some() - || tracedecay_runtime_core::config::has_project_database(project_root) +fn project_is_real(project_root: &Path) -> bool { + tracedecay_runtime_core::worktree::git_worktree_root(project_root).is_some() } #[cfg(test)] diff --git a/crates/tracedecay-sessions/src/runtime/hosts/hermes/observation.rs b/crates/tracedecay-sessions/src/runtime/hosts/hermes/observation.rs index f0d99b111c..1d9c9c7e41 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/hermes/observation.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/hermes/observation.rs @@ -47,7 +47,6 @@ pub(super) fn project_projection_metadata( location_provenance: &'static str, ) -> HermesProjectionMetadata { let presentation_path = match location_provenance { - "profile_pin" => source.legacy_project_pin.as_deref(), "session_cwd" => row.session_cwd.as_deref().map(Path::new), _ => None, } diff --git a/crates/tracedecay-sessions/src/runtime/hosts/hermes/routing.rs b/crates/tracedecay-sessions/src/runtime/hosts/hermes/routing.rs index 270b60c7ed..c8b8fd91be 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/hermes/routing.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/hermes/routing.rs @@ -21,10 +21,9 @@ pub(super) fn user_turn_locations( } let mut locations = HashSet::new(); for session_rows in by_session.into_values() { - let has_fallback = source.legacy_project_pin.is_some() - || session_rows - .iter() - .any(|row| Path::new(row.session_cwd.as_deref().unwrap_or_default()).is_absolute()) + let has_fallback = session_rows + .iter() + .any(|row| Path::new(row.session_cwd.as_deref().unwrap_or_default()).is_absolute()) || source.state_db.parent().is_some(); let mut turn = Vec::new(); for row in session_rows { @@ -55,7 +54,6 @@ fn assign_user_turn(rows: &[&HermesRow], has_fallback: bool, locations: &mut Has pub(super) fn turn_project_locations( rows: &[HermesRow], project_root: &Path, - source: &HermesProfileSource, ) -> HashMap { let mut by_session: HashMap<&str, Vec<&HermesRow>> = HashMap::new(); for row in rows { @@ -66,11 +64,8 @@ pub(super) fn turn_project_locations( for session_rows in by_session.into_values() { let has_fallback = session_rows .iter() - .any(|row| session_is_candidate_for_project(row, &project_matcher, source)); - let fallback_provenance = source - .legacy_project_pin - .as_ref() - .map_or("session_cwd", |_| "profile_pin"); + .any(|row| session_is_candidate_for_project(row, &project_matcher)); + let fallback_provenance = "session_cwd"; let mut turn = Vec::new(); for row in session_rows { if row.role == "user" && !turn.is_empty() { @@ -111,7 +106,6 @@ pub(super) enum DestinationRoutingError { pub(super) fn turn_project_locations_for_destinations( rows: &[HermesRow], destination_matchers: &[ProjectRootMatcher], - source: &HermesProfileSource, destination_routes: &mut HashMap>, ) -> Result, DestinationRoutingError> { let mut by_session: HashMap<&str, Vec<&HermesRow>> = HashMap::new(); @@ -124,22 +118,15 @@ pub(super) fn turn_project_locations_for_destinations( }) .collect::>(); for session_rows in by_session.into_values() { - let fallback_provenance = source - .legacy_project_pin - .as_ref() - .map_or("session_cwd", |_| "profile_pin"); - let fallback_candidates = if let Some(pin) = source.legacy_project_pin.as_ref() { - vec![pin.clone()] - } else { - let mut seen = BTreeSet::new(); - session_rows - .iter() - .filter_map(|row| { - let cwd = PathBuf::from(row.session_cwd.as_deref()?.trim()); - (cwd.is_absolute() && seen.insert(cwd.clone())).then_some(cwd) - }) - .collect::>() - }; + let fallback_provenance = "session_cwd"; + let mut seen = BTreeSet::new(); + let fallback_candidates = session_rows + .iter() + .filter_map(|row| { + let cwd = PathBuf::from(row.session_cwd.as_deref()?.trim()); + (cwd.is_absolute() && seen.insert(cwd.clone())).then_some(cwd) + }) + .collect::>(); let mut fallbacks = vec![false; destination_matchers.len()]; for cwd in fallback_candidates { for destination_index in @@ -308,14 +295,9 @@ fn structured_tool_project_paths(row: &HermesRow) -> Vec { paths } -fn session_is_candidate_for_project( - row: &HermesRow, - project_matcher: &ProjectRootMatcher, - source: &HermesProfileSource, -) -> bool { - source.legacy_project_pin.is_some() - || row.session_cwd.as_deref().is_some_and(|cwd| { - let cwd = Path::new(cwd.trim()); - cwd.is_absolute() && project_matcher.contains(cwd) - }) +fn session_is_candidate_for_project(row: &HermesRow, project_matcher: &ProjectRootMatcher) -> bool { + row.session_cwd.as_deref().is_some_and(|cwd| { + let cwd = Path::new(cwd.trim()); + cwd.is_absolute() && project_matcher.contains(cwd) + }) } diff --git a/crates/tracedecay-sessions/src/runtime/hosts/hermes/state_db.rs b/crates/tracedecay-sessions/src/runtime/hosts/hermes/state_db.rs index 74df0e4089..c1f3a72b9a 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/hermes/state_db.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/hermes/state_db.rs @@ -512,7 +512,7 @@ pub(super) async fn try_ingest_state_db_bounded_with_admission( resume_fingerprint, budget, |bounded| { - let locations = turn_project_locations(bounded, project_root, source); + let locations = turn_project_locations(bounded, project_root); move |row: &HermesRow| { locations.get(&row.id).copied().map(|provenance| { project_projection_metadata(row, source, project_root, provenance) @@ -574,7 +574,6 @@ pub(super) async fn try_ingest_state_db_for_projects( turn_project_locations_for_destinations( bounded, &destination_matchers, - source, &mut destination_routes, ) }) @@ -656,10 +655,7 @@ pub(super) async fn try_ingest_user_state_db_bounded_with_admission( |bounded| { let locations = user_turn_locations(bounded, source); let profile = source.profile.clone(); - let fallback_provenance = source - .legacy_project_pin - .as_ref() - .map_or("session_cwd", |_| "profile_pin"); + let fallback_provenance = "session_cwd"; move |row: &HermesRow| { locations .contains(&row.id) diff --git a/crates/tracedecay-sessions/src/runtime/hosts/hermes/tests.rs b/crates/tracedecay-sessions/src/runtime/hosts/hermes/tests.rs index d69d457c28..b85cf7e40a 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/hermes/tests.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/hermes/tests.rs @@ -204,7 +204,6 @@ async fn cancelled_hermes_sweep_stops_before_opening_the_host_database() { let dir = tempfile::tempdir().unwrap(); let source = HermesProfileSource { state_db: dir.path().join("state.db"), - legacy_project_pin: None, profile: Some("cancelled-fixture".to_string()), }; let cancellation = ObservationCancellation::default(); @@ -1550,7 +1549,6 @@ mod destination_routing_tests { GitDiscoveryUnknown, GitRepositoryIdentity, GitRepositoryIdentityOutcome, }; - use super::super::ingest::HermesProfileSource; use super::super::routing::turn_project_locations_for_destinations; use super::super::rows::HermesRow; @@ -1608,11 +1606,6 @@ mod destination_routing_tests { let cwd = project_root.join("packages/app"); std::fs::create_dir_all(&cwd).expect("cwd"); let rows = vec![row_with_cwd(&cwd)]; - let source = HermesProfileSource { - state_db: temp.path().join("state.db"), - legacy_project_pin: None, - profile: None, - }; let previous = StoredCursor::default(); let mut persisted = previous; let mut routes = HashMap::new(); @@ -1622,8 +1615,7 @@ mod destination_routing_tests { retrying_identity, )]; assert!( - turn_project_locations_for_destinations(&rows, &first_matchers, &source, &mut routes) - .is_err() + turn_project_locations_for_destinations(&rows, &first_matchers, &mut routes).is_err() ); assert_eq!(persisted, previous); assert!(routes.is_empty(), "unknown routes must not be cached"); @@ -1633,7 +1625,7 @@ mod destination_routing_tests { retrying_identity, )]; let locations = - turn_project_locations_for_destinations(&rows, &retry_matchers, &source, &mut routes) + turn_project_locations_for_destinations(&rows, &retry_matchers, &mut routes) .expect("the same source rows should route after identity recovers"); assert_eq!( locations[0].by_row_id.get(&1).copied(), @@ -1655,13 +1647,15 @@ async fn unreadable_state_db_is_a_counted_source_failure_not_a_clean_sweep() { initialize_owned_store_before_foreign_fixture(dir.path()).await; // The pin resolver is a root-registered port and stays unwired in this // crate's tests, so the project qualifies as a candidate destination by - // carrying an initialized project database instead of a config pin. + // being a git worktree instead of carrying a config pin. let project_root = dir.path().join("project"); - let project_db = project_root - .join(tracedecay_runtime_core::config::TRACEDECAY_DIR) - .join(tracedecay_runtime_core::config::DB_FILENAME); - std::fs::create_dir_all(project_db.parent().unwrap()).unwrap(); - std::fs::write(&project_db, b"").unwrap(); + std::fs::create_dir_all(&project_root).unwrap(); + let init = std::process::Command::new(tracedecay_runtime_core::git::try_git_program().unwrap()) + .args(["init", "-q"]) + .current_dir(&project_root) + .status() + .unwrap(); + assert!(init.success()); let home = dir.path().join("hermes-home"); std::fs::create_dir_all(&home).unwrap(); std::fs::write(home.join("state.db"), b"this is not a sqlite database").unwrap(); diff --git a/crates/tracedecay-sessions/src/runtime/hosts/mod.rs b/crates/tracedecay-sessions/src/runtime/hosts/mod.rs index db70fde861..44c93365c2 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/mod.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/mod.rs @@ -2,8 +2,7 @@ //! //! Each submodule owns one agent-host's on-disk transcript shape and the //! projection into provider-neutral session rows. Ingest dispatches through -//! these adapters; they are re-exported at `crate::runtime::{claude, …}` so -//! existing public paths stay stable. +//! these adapters. pub mod claude; pub mod claude_observation; diff --git a/crates/tracedecay-sessions/src/runtime/hosts/opencode.rs b/crates/tracedecay-sessions/src/runtime/hosts/opencode.rs index dfd422ba9a..203631762d 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/opencode.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/opencode.rs @@ -16,12 +16,12 @@ use tracedecay_store::ParseOffset; use crate::admission::HostAdmission; use crate::observation::{CaptureObservationRequest, ObservationCancellation}; use crate::runtime::host_scan::{HOST_SCAN_WINDOW, HostScanBudget, HostScanEvidence}; -use crate::runtime::opencode_frontier::{ +use crate::runtime::hosts::opencode_frontier::{ GENERATION_KEY as OPENCODE_GENERATION_FRONTIER_KEY, REWRITE_KEY as OPENCODE_REWRITE_FRONTIER_KEY, prepare_generation_rewrite, read as read_frontier, write as write_frontier, }; -use crate::runtime::opencode_snapshot::MAX_SNAPSHOT_DATABASE_IO_BYTES; +use crate::runtime::hosts::opencode_snapshot::MAX_SNAPSHOT_DATABASE_IO_BYTES; use crate::runtime::shared::TranscriptScopeMatcher; use crate::runtime::snapshot_observation::{ MAX_SNAPSHOT_CAPTURE_UNIT_BYTES, SnapshotAdmissionBatch, SnapshotAdmissionRecord, @@ -202,7 +202,8 @@ impl SnapshotAdmissionRecord for OpenCodeRecord { impl OpenCodeSource { pub fn new_for_project(project_root: &Path) -> Option { let home = crate::runtime::home_dir()?; - let snapshot_scratch_root = crate::runtime::opencode_snapshot::snapshot_scratch_root()?; + let snapshot_scratch_root = + crate::runtime::hosts::opencode_snapshot::snapshot_scratch_root()?; Some(Self::with_database_for_project_and_scratch( opencode_data_dir(&home).join("opencode.db"), snapshot_scratch_root, @@ -212,7 +213,8 @@ impl OpenCodeSource { pub fn new_for_user(roots: Vec) -> Option { let home = crate::runtime::home_dir()?; - let snapshot_scratch_root = crate::runtime::opencode_snapshot::snapshot_scratch_root()?; + let snapshot_scratch_root = + crate::runtime::hosts::opencode_snapshot::snapshot_scratch_root()?; Some(Self::with_database_for_user_and_scratch( opencode_data_dir(&home).join("opencode.db"), snapshot_scratch_root, @@ -293,7 +295,7 @@ pub(crate) async fn capture_opencode_observations( Instant::now() + HOST_SCAN_WINDOW, cancellation.clone(), ); - let snapshot_attempt = crate::runtime::opencode_snapshot::snapshot_database( + let snapshot_attempt = crate::runtime::hosts::opencode_snapshot::snapshot_database( source.database_path.clone(), source.snapshot_scratch_root.clone(), snapshot_budget, @@ -543,7 +545,9 @@ fn scan_reference_page( match scan_kind { OpenCodeScanKind::Messages => scan_message_reference_page(source, cursor, budget), OpenCodeScanKind::Parts => { - crate::runtime::opencode_part_scan::scan_part_reference_page(source, cursor, budget) + crate::runtime::hosts::opencode_part_scan::scan_part_reference_page( + source, cursor, budget, + ) } OpenCodeScanKind::Rewrite => scan_message_reference_page(source, cursor, budget), } diff --git a/crates/tracedecay-sessions/src/runtime/hosts/vibe/tests.rs b/crates/tracedecay-sessions/src/runtime/hosts/vibe/tests.rs index 43975756da..b7614f3002 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/vibe/tests.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/vibe/tests.rs @@ -100,8 +100,8 @@ fn pagination_completes_older_work_then_surfaces_finite_new_arrivals() { #[test] fn vibe_workflow_lookalike_stays_ordinary_message_without_goal_kind() { // Vibe has no DurableObservation / WorkflowLifecycle normalizer yet. - // Prove lookalike lifecycle bags are not promoted into session_messages - // kind=goal (the legacy goals surface) or metadata keys. + // Prove lookalike lifecycle bags are not promoted into session + // message kind=goal (the legacy goals surface) or metadata keys. let input: Value = serde_json::from_str(include_str!( "../../../../../../tests/fixtures/provider_normalization/vibe/workflow_lookalike.input.json" )) diff --git a/crates/tracedecay-sessions/src/runtime/ingest/failure.rs b/crates/tracedecay-sessions/src/runtime/ingest/failure.rs index 77a6a1e64c..ceab905352 100644 --- a/crates/tracedecay-sessions/src/runtime/ingest/failure.rs +++ b/crates/tracedecay-sessions/src/runtime/ingest/failure.rs @@ -7,7 +7,7 @@ use tracedecay_domain::ObservationSourceRangeV1; use crate::admission::HostAdmissionStatus; use crate::runtime::shared::TranscriptIngestStats; -use crate::runtime::{claude_observation, source}; +use crate::runtime::{hosts::claude_observation, source}; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct ClaudeObservationFailureClass { diff --git a/crates/tracedecay-sessions/src/runtime/ingest/project.rs b/crates/tracedecay-sessions/src/runtime/ingest/project.rs index 6fb919618b..044796880d 100644 --- a/crates/tracedecay-sessions/src/runtime/ingest/project.rs +++ b/crates/tracedecay-sessions/src/runtime/ingest/project.rs @@ -6,7 +6,7 @@ use super::authority::{IngestAdmissionBinding, SessionIngestAuthority}; use crate::observation::ObservationCancellation; use crate::repository_provenance::RepositoryProvenanceAdmissionContext; use crate::runtime::shared::TranscriptIngestStats; -use crate::runtime::{SessionProvider, claude_observation}; +use crate::runtime::{SessionProvider, hosts::claude_observation}; use tracedecay_domain::{BrainId, ObservationScopeV1, ProjectId, UserProfileId}; use tracedecay_store::StoreShardScopeV1; @@ -110,7 +110,7 @@ pub async fn ingest_project_sources_for_provider_with_cancellation_and_codex_sta provider: Option, include_hermes: bool, cancellation: &ObservationCancellation, - codex_discovery: &crate::runtime::codex::CodexDiscoveryHub, + codex_discovery: &crate::runtime::hosts::codex::CodexDiscoveryHub, codex_consumer: &str, ) -> TranscriptIngestOutcome { ingest_project_sources_for_provider_inner( @@ -190,7 +190,7 @@ async fn ingest_project_sources_for_provider_inner( provider: Option, include_hermes: bool, cancellation: &ObservationCancellation, - codex_discovery: Option<(&crate::runtime::codex::CodexDiscoveryHub, &str)>, + codex_discovery: Option<(&crate::runtime::hosts::codex::CodexDiscoveryHub, &str)>, ) -> TranscriptIngestOutcome { ingest_project_sources_for_provider_bounded_inner( registered, @@ -245,7 +245,7 @@ async fn ingest_project_sources_for_provider_bounded_inner, + codex_discovery: Option<(&crate::runtime::hosts::codex::CodexDiscoveryHub, &str)>, ) -> TranscriptIngestOutcome { let Some(canonical_project_id) = project_id else { return TranscriptIngestOutcome::new( diff --git a/crates/tracedecay-sessions/src/runtime/ingest/project_provider.rs b/crates/tracedecay-sessions/src/runtime/ingest/project_provider.rs index 24f02e9a60..de1dda5c27 100644 --- a/crates/tracedecay-sessions/src/runtime/ingest/project_provider.rs +++ b/crates/tracedecay-sessions/src/runtime/ingest/project_provider.rs @@ -15,8 +15,9 @@ use crate::runtime::source::{ run_blocking_transcript_section, }; use crate::runtime::{ - SessionProvider, claude, claude_observation, cline_like, codex, cursor, cursor_composer, - hermes, kimi, kiro, opencode, vibe, + SessionProvider, hosts::claude, hosts::claude_observation, hosts::cline_like, hosts::codex, + hosts::cursor, hosts::cursor_composer, hosts::hermes, hosts::kimi, hosts::kiro, + hosts::opencode, hosts::vibe, }; use super::failure::{ @@ -817,11 +818,11 @@ async fn ingest_project_claude_observations( mod tests { use std::collections::BTreeSet; - use crate::runtime::claude_observation::{ + use crate::runtime::hosts::claude_observation::{ ClaudeObservationIngestError, ClaudeObservationIngestStats, }; - use crate::runtime::cursor::{CursorSweepIngestOutcome, CursorTranscriptIngestStats}; - use crate::runtime::cursor_composer::CursorComposerSweepOutcome; + use crate::runtime::hosts::cursor::{CursorSweepIngestOutcome, CursorTranscriptIngestStats}; + use crate::runtime::hosts::cursor_composer::CursorComposerSweepOutcome; use crate::runtime::shared::TranscriptIngestStats; use crate::runtime::source::TranscriptIngestError; @@ -830,7 +831,7 @@ mod tests { codex_source_failure_saturates_pass, cursor_composer_run_outcome, hermes_run_outcome, merge_cursor_sweep_outcome, }; - use crate::runtime::hermes::HermesSweepOutcome; + use crate::runtime::hosts::hermes::HermesSweepOutcome; #[test] fn codex_source_failures_bound_each_provider_pass() { diff --git a/crates/tracedecay-sessions/src/runtime/ingest/scheduler.rs b/crates/tracedecay-sessions/src/runtime/ingest/scheduler.rs index 5f5022949c..6cad603d01 100644 --- a/crates/tracedecay-sessions/src/runtime/ingest/scheduler.rs +++ b/crates/tracedecay-sessions/src/runtime/ingest/scheduler.rs @@ -4,7 +4,7 @@ use tracedecay_store::ParseOffset; use crate::admission::DEFAULT_MAX_RECORDS; use crate::runtime::SessionProvider; -use crate::runtime::codex::CodexDiscoveryFrontier; +use crate::runtime::hosts::codex::CodexDiscoveryFrontier; use crate::runtime::snapshot_observation::MAX_SNAPSHOT_CAPTURE_UNIT_BYTES; use crate::runtime::source::{MAX_JSONL_RECORD_BYTES, TranscriptIngestResult}; use crate::runtime::store_port::TranscriptIngestStore; @@ -218,7 +218,7 @@ mod tests { use crate::runtime::git_correlation::{CommitSessionRecord, SpanObservation}; use crate::runtime::source::TranscriptDiscoveryBounds; use crate::runtime::store_port::TranscriptIngestStore; - use crate::runtime::{SessionRecord, codex}; + use crate::runtime::{SessionRecord, hosts::codex}; use super::{read_codex_discovery_frontier, write_codex_discovery_frontier}; diff --git a/crates/tracedecay-sessions/src/runtime/ingest/startup.rs b/crates/tracedecay-sessions/src/runtime/ingest/startup.rs index 40bf47963a..065dbced45 100644 --- a/crates/tracedecay-sessions/src/runtime/ingest/startup.rs +++ b/crates/tracedecay-sessions/src/runtime/ingest/startup.rs @@ -149,7 +149,7 @@ pub async fn ingest_user_global_sources_for_startup_with_db_and_codex_state< registry_db: &A, profile_root: &Path, cancellation: &ObservationCancellation, - codex_state: (&crate::runtime::codex::CodexDiscoveryHub, &str), + codex_state: (&crate::runtime::hosts::codex::CodexDiscoveryHub, &str), ) -> TranscriptIngestOutcome { ingest_user_global_sources_for_startup_inner( (brain_id, profile_id, registered), @@ -183,7 +183,7 @@ async fn ingest_user_global_sources_for_startup_inner registry_db: &A, profile_root: &Path, cancellation: &ObservationCancellation, - codex_discovery: Option<(&crate::runtime::codex::CodexDiscoveryHub, &str)>, + codex_discovery: Option<(&crate::runtime::hosts::codex::CodexDiscoveryHub, &str)>, ) -> TranscriptIngestOutcome { if cancellation.is_cancelled() { return TranscriptIngestOutcome::new( diff --git a/crates/tracedecay-sessions/src/runtime/ingest/tests.rs b/crates/tracedecay-sessions/src/runtime/ingest/tests.rs index 2483376232..05b21d308d 100644 --- a/crates/tracedecay-sessions/src/runtime/ingest/tests.rs +++ b/crates/tracedecay-sessions/src/runtime/ingest/tests.rs @@ -14,7 +14,9 @@ use tracedecay_store::{ use crate::observation::ObservationCancellation; use crate::runtime::shared::TranscriptIngestStats; -use crate::runtime::{SessionProvider, claude_observation, codex, git_correlation, source}; +use crate::runtime::{ + SessionProvider, git_correlation, hosts::claude_observation, hosts::codex, source, +}; use super::failure::{ IngestPassBounds, IngestPassCoverage, allocate_pass_byte_budgets, diff --git a/crates/tracedecay-sessions/src/runtime/ingest/user.rs b/crates/tracedecay-sessions/src/runtime/ingest/user.rs index f2e15c29b3..1708665019 100644 --- a/crates/tracedecay-sessions/src/runtime/ingest/user.rs +++ b/crates/tracedecay-sessions/src/runtime/ingest/user.rs @@ -6,7 +6,9 @@ use crate::admission::HostAdmission; use crate::observation::ObservationCancellation; use crate::runtime::shared::TranscriptIngestStats; use crate::runtime::source::{self, TranscriptDiscoveryBounds}; -use crate::runtime::{SessionProvider, claude_observation, codex, cursor, cursor_composer}; +use crate::runtime::{ + SessionProvider, hosts::claude_observation, hosts::codex, hosts::cursor, hosts::cursor_composer, +}; use tracedecay_domain::{BrainId, ObservationScopeV1, UserProfileId}; use tracedecay_store::StoreShardScopeV1; @@ -762,8 +764,8 @@ async fn ingest_user_global_sources_for_provider_with_roots_bounded_inner< mod cursor_tests { use std::collections::BTreeSet; - use crate::runtime::cursor::{CursorSweepIngestOutcome, CursorTranscriptIngestStats}; - use crate::runtime::cursor_composer::CursorComposerSweepOutcome; + use crate::runtime::hosts::cursor::{CursorSweepIngestOutcome, CursorTranscriptIngestStats}; + use crate::runtime::hosts::cursor_composer::CursorComposerSweepOutcome; use super::merge_user_cursor_sweep; diff --git a/crates/tracedecay-sessions/src/runtime/ingest/user_provider.rs b/crates/tracedecay-sessions/src/runtime/ingest/user_provider.rs index 8ef5967b9f..1253fbe4bf 100644 --- a/crates/tracedecay-sessions/src/runtime/ingest/user_provider.rs +++ b/crates/tracedecay-sessions/src/runtime/ingest/user_provider.rs @@ -11,7 +11,8 @@ use crate::runtime::source::{ }; use crate::runtime::store_port::TranscriptIngestStore; use crate::runtime::{ - SessionProvider, claude_observation, cline_like, hermes, kimi, kiro, opencode, vibe, + SessionProvider, hosts::claude_observation, hosts::cline_like, hosts::hermes, hosts::kimi, + hosts::kiro, hosts::opencode, hosts::vibe, }; use super::failure::{ @@ -105,7 +106,8 @@ pub(super) struct UserProviderUnit<'a, S> { pub(super) candidate: SessionProvider, pub(super) max_new_bytes: u64, pub(super) cancellation: &'a ObservationCancellation, - pub(super) codex_discovery: Option<(&'a crate::runtime::codex::CodexDiscoveryHub, &'a str)>, + pub(super) codex_discovery: + Option<(&'a crate::runtime::hosts::codex::CodexDiscoveryHub, &'a str)>, } impl UserProviderUnit<'_, S> { @@ -564,7 +566,7 @@ impl UserProviderUnit<'_, S> { #[cfg(test)] mod tests { - use crate::runtime::claude_observation::{ + use crate::runtime::hosts::claude_observation::{ ClaudeObservationIngestError, ClaudeObservationIngestStats, }; use crate::runtime::shared::TranscriptIngestStats; diff --git a/crates/tracedecay-sessions/src/runtime/mod.rs b/crates/tracedecay-sessions/src/runtime/mod.rs index 16899cad20..193ff4b821 100644 --- a/crates/tracedecay-sessions/src/runtime/mod.rs +++ b/crates/tracedecay-sessions/src/runtime/mod.rs @@ -4,14 +4,9 @@ pub use tracedecay_store::{SessionMessageRecord, SessionRecord}; // Runtime modules are public because the root composition crate mounts these // concrete provider and storage authorities directly. -mod hosts; -pub use hosts::{ - claude, claude_observation, cline_like, codex, codex_app_server, cursor, cursor_composer, - hermes, kimi, kiro, opencode, vibe, -}; -pub(in crate::runtime) use hosts::{opencode_frontier, opencode_part_scan, opencode_snapshot}; pub mod git_correlation; mod host_scan; +pub mod hosts; pub mod ingest; mod native_ingest_source; pub use native_ingest_source::native_ingest_source_identity; diff --git a/crates/tracedecay-sessions/src/runtime/native_ingest_source.rs b/crates/tracedecay-sessions/src/runtime/native_ingest_source.rs index 59f349ae76..1751a29bb8 100644 --- a/crates/tracedecay-sessions/src/runtime/native_ingest_source.rs +++ b/crates/tracedecay-sessions/src/runtime/native_ingest_source.rs @@ -22,7 +22,7 @@ pub fn native_ingest_source_identity( source_key: Option<&str>, ) -> TranscriptIngestResult { if provider == "codex" && source_key.is_none() { - return crate::runtime::codex::codex_observation_source_v2(session_id); + return crate::runtime::hosts::codex::codex_observation_source_v2(session_id); } if matches!(provider, "cline" | "roo-code" | "kilo") && source_key.is_none() { return Ok(ClineTranscriptStream::ApiHistory @@ -90,7 +90,7 @@ mod tests { #[test] fn codex_lookup_uses_the_v2_authority_not_the_legacy_session_source() { let written = - crate::runtime::codex::codex_observation_source_v2("codex-goal-dedupe").unwrap(); + crate::runtime::hosts::codex::codex_observation_source_v2("codex-goal-dedupe").unwrap(); let looked_up = native_ingest_source_identity("codex", "codex-goal-dedupe", None).unwrap(); let legacy = ObservationSourceIdentityV1::for_provider( ProviderId::new("codex").unwrap(), diff --git a/crates/tracedecay-sessions/src/runtime/observation/jsonl_observation_admission.rs b/crates/tracedecay-sessions/src/runtime/observation/jsonl_observation_admission.rs index 33036ce9ce..0ebb0ae08c 100644 --- a/crates/tracedecay-sessions/src/runtime/observation/jsonl_observation_admission.rs +++ b/crates/tracedecay-sessions/src/runtime/observation/jsonl_observation_admission.rs @@ -1775,10 +1775,11 @@ struct PendingAdmissionWindow<'window, State> { } enum CaptureWindowError { - ScalarFallback(#[allow(dead_code)] HostAdmissionRecovery), + /// A frame in the window was refused for its content. Replay one frame at + /// a time so the refusal settles on that frame alone. + ContentRefusal, /// The window compare-and-swap lost, and the durable cursor does not cover - /// the last frame. Replay one frame at a time; do not treat that as a - /// store-issued batch fallback. + /// the last frame. Replay one frame at a time. LostCursor, Ingest(TranscriptIngestError), } @@ -2227,13 +2228,8 @@ impl ActiveAdmission<'_> { if outcome.status == HostAdmissionStatus::Backpressured { hotpath::gauge!("jsonl_admission_backpressure_writer").inc(1.0); } - if let Some(recovery) = outcome.recovery { - match recovery { - HostAdmissionRecovery::BatchRequiresScalarFallback(_) - | HostAdmissionRecovery::DeterministicContentRefusal => { - return Err(CaptureWindowError::ScalarFallback(recovery)); - } - } + if let Some(HostAdmissionRecovery::DeterministicContentRefusal) = outcome.recovery { + return Err(CaptureWindowError::ContentRefusal); } // The batch is atomic: nothing in this window committed. When // the peer that won the CAS is already past the window's last @@ -2487,7 +2483,7 @@ pub(in crate::runtime) async fn admit_jsonl_observations( .await { Ok(()) => Ok(()), - Err(CaptureWindowError::ScalarFallback(_) | CaptureWindowError::LostCursor) => { + Err(CaptureWindowError::ContentRefusal | CaptureWindowError::LostCursor) => { for (checkpoint, range, bytes, prepared, hints) in backups { if active.cancellation.is_cancelled() { return Err(TranscriptIngestError::Cancelled { diff --git a/crates/tracedecay-sessions/src/runtime/observation/jsonl_observation_admission/tests.rs b/crates/tracedecay-sessions/src/runtime/observation/jsonl_observation_admission/tests.rs index 6c33688210..c247930493 100644 --- a/crates/tracedecay-sessions/src/runtime/observation/jsonl_observation_admission/tests.rs +++ b/crates/tracedecay-sessions/src/runtime/observation/jsonl_observation_admission/tests.rs @@ -27,11 +27,11 @@ use tracedecay_domain::{ ObservationSourceIdentityV1, ProjectId, ProviderId, RetentionClass, SessionId, }; use tracedecay_runtime_core::background_cpu::ProcessBackgroundCpuV1; +use tracedecay_store::ParseOffset; use tracedecay_store::observation::{ CursorAdvanceOutcome, ObservationCoverageReason, ObservationCursorAdvance, ObservationIdentityCollisionDispositionV1, }; -use tracedecay_store::{ObservationBatchFallbackCause, ParseOffset}; use crate::admission::test_support::MemoryHostAdmission; use crate::admission::{ @@ -40,7 +40,7 @@ use crate::admission::{ use crate::observation::{ CaptureObservationOutcome, CaptureObservationRequest, ObservationCancellation, }; -use crate::runtime::codex::{ +use crate::runtime::hosts::codex::{ try_admit_codex_jsonl_observations_for_profile_with_admission, try_admit_codex_jsonl_observations_for_project_with_admission, }; @@ -860,7 +860,7 @@ fn rollout_fixture() -> (tempfile::TempDir, PathBuf, u64) { } async fn stored_cursor(spy: &SeamSpyAdmission) -> Option { - let source = crate::runtime::codex::codex_observation_source_v2(SESSION_ID).unwrap(); + let source = crate::runtime::hosts::codex::codex_observation_source_v2(SESSION_ID).unwrap(); spy.get_source_cursor(&source, &ObservationScopeV1::Profile) .await .unwrap() @@ -1055,8 +1055,8 @@ async fn eligible_identity_collision_retries_once_with_normalizer_fallback() { let bytes = b"{\"role\":\"user\",\"content\":\"repeated\"}\n"; std::fs::write(&path, bytes).unwrap(); let spy = SeamSpyAdmission::default(); - spy.script_batch_error(HostAdmissionOutcome::batch_requires_scalar_fallback( - ObservationBatchFallbackCause::IntraBatchIdentityCollision, + spy.script_batch_error(HostAdmissionOutcome::deterministic_content_refusal( + "observation_identity_collision", )); spy.script_capture_error_once(HostAdmissionOutcome::deterministic_content_refusal( "observation_identity_collision", @@ -1173,8 +1173,8 @@ async fn exhausted_identity_collision_retry_uses_its_exact_terminal_coverage_rea let bytes = b"{\"role\":\"user\",\"content\":\"repeated\"}\n"; std::fs::write(&path, bytes).unwrap(); let spy = SeamSpyAdmission::default(); - spy.script_batch_error(HostAdmissionOutcome::batch_requires_scalar_fallback( - ObservationBatchFallbackCause::IntraBatchIdentityCollision, + spy.script_batch_error(HostAdmissionOutcome::deterministic_content_refusal( + "observation_identity_collision", )); spy.script_capture_error(HostAdmissionOutcome::deterministic_content_refusal( "observation_identity_collision", @@ -1332,7 +1332,7 @@ async fn codex_session_meta_prefix_is_decoded_once_across_consumers() { let (_temp, path, _) = rollout_fixture(); let first = SeamSpyAdmission::default(); let second = SeamSpyAdmission::default(); - let before = crate::runtime::codex::session_meta_read_count_for_test(&path); + let before = crate::runtime::hosts::codex::session_meta_read_count_for_test(&path); try_admit_codex_jsonl_observations_for_profile_with_admission(&path, None, &[], &first, None) .await @@ -1342,7 +1342,7 @@ async fn codex_session_meta_prefix_is_decoded_once_across_consumers() { .expect("second profile consumer"); assert_eq!( - crate::runtime::codex::session_meta_read_count_for_test(&path) - before, + crate::runtime::hosts::codex::session_meta_read_count_for_test(&path) - before, 1, "canonical path+native identity must share one bounded prefix decode" ); @@ -1382,8 +1382,8 @@ async fn batch_refusal_reuses_pre_context_switch_frames() { let contents = lines.join("\n") + "\n"; std::fs::write(&path, contents).unwrap(); let spy = SeamSpyAdmission::default(); - spy.script_batch_error(HostAdmissionOutcome::batch_requires_scalar_fallback( - ObservationBatchFallbackCause::IntraBatchIdentityCollision, + spy.script_batch_error(HostAdmissionOutcome::deterministic_content_refusal( + "observation_identity_collision", )); spy.script_capture_error(HostAdmissionOutcome::deterministic_content_refusal( "observation_identity_collision", diff --git a/crates/tracedecay-sessions/src/runtime/source.rs b/crates/tracedecay-sessions/src/runtime/source.rs index 5eeee4e344..77dad2cbff 100644 --- a/crates/tracedecay-sessions/src/runtime/source.rs +++ b/crates/tracedecay-sessions/src/runtime/source.rs @@ -107,7 +107,7 @@ pub(super) async fn read_host_provider_coverage( pub(super) async fn read_codex_history_frontier( admission: &dyn HostAdmission, scope: &ObservationScopeV1, -) -> TranscriptIngestResult { +) -> TranscriptIngestResult { let stored_frontier = admission .get_parse_offset(scope, CODEX_HISTORY_FRONTIER_KEY) .await @@ -122,14 +122,17 @@ pub(super) async fn read_codex_history_frontier( crate::runtime::snapshot_observation::host_admission_error("codex", outcome) })? .unwrap_or_default(); - crate::runtime::codex::CodexDiscoveryFrontier::from_parse_offsets(stored_frontier, stored_epoch) + crate::runtime::hosts::codex::CodexDiscoveryFrontier::from_parse_offsets( + stored_frontier, + stored_epoch, + ) } pub(super) async fn persist_codex_history_frontier( admission: &dyn HostAdmission, scope: &ObservationScopeV1, - expected: crate::runtime::codex::CodexDiscoveryFrontier, - frontier: crate::runtime::codex::CodexDiscoveryFrontier, + expected: crate::runtime::hosts::codex::CodexDiscoveryFrontier, + frontier: crate::runtime::hosts::codex::CodexDiscoveryFrontier, ) -> TranscriptIngestResult<()> { let (frontier_offset, epoch_offset) = frontier.into_parse_offsets(); let (expected_frontier, expected_epoch) = expected.into_parse_offsets(); diff --git a/crates/tracedecay-sessions/src/runtime/source/tests.rs b/crates/tracedecay-sessions/src/runtime/source/tests.rs index 8278ed6a94..adf1db9da0 100644 --- a/crates/tracedecay-sessions/src/runtime/source/tests.rs +++ b/crates/tracedecay-sessions/src/runtime/source/tests.rs @@ -247,7 +247,7 @@ impl tracedecay_store::TranscriptStore for CountingStore { #[tokio::test] async fn physical_transcript_locations_do_not_replace_opaque_checkpoint_identity() { let store = CountingStore::default(); - let source = crate::runtime::codex::CodexSource::with_home(Path::new("fixture-home")); + let source = crate::runtime::hosts::codex::CodexSource::with_home(Path::new("fixture-home")); let paths = vec![PathBuf::from("archived-rollout.jsonl")]; #[cfg(unix)] let paths = { diff --git a/crates/tracedecay-sessions/src/runtime/store_access/codex_goal_reconciliation.rs b/crates/tracedecay-sessions/src/runtime/store_access/codex_goal_reconciliation.rs index ee29499f0b..1ababe3dcf 100644 --- a/crates/tracedecay-sessions/src/runtime/store_access/codex_goal_reconciliation.rs +++ b/crates/tracedecay-sessions/src/runtime/store_access/codex_goal_reconciliation.rs @@ -27,7 +27,7 @@ pub async fn find_preceding_codex_goal_response( "SELECT message_id, kind, metadata_json FROM ( SELECT message_id, kind, metadata_json, ordinal - FROM session_messages + FROM lcm_raw_messages WHERE provider = ?1 AND session_id = ?2 AND ((?3 IS NULL AND source_path IS NULL) OR source_path = ?3) AND ordinal < ?4 diff --git a/crates/tracedecay-sessions/src/runtime/store_access/lcm.rs b/crates/tracedecay-sessions/src/runtime/store_access/lcm.rs index 9abf793d50..e9c8db4508 100644 --- a/crates/tracedecay-sessions/src/runtime/store_access/lcm.rs +++ b/crates/tracedecay-sessions/src/runtime/store_access/lcm.rs @@ -3,6 +3,7 @@ use std::path::Path; use tracedecay_runtime_core::db::DatabaseEngineReadSnapshot; use tracedecay_runtime_core::db::engine::{QueryExecutor, params}; +use tracedecay_lcm::raw::stored_message_record_select_columns; use tracedecay_lcm::{ LcmDescribeRequest, LcmDescribeResponse, LcmError, LcmExpandQueryRequest, LcmExpandQueryResponse, LcmExpandRequest, LcmExpandResponse, LcmGcConfig, LcmGcReport, @@ -37,22 +38,17 @@ async fn require_current_protection_input( conn: &(impl QueryExecutor + ?Sized), expected: &RawProtectionInput, ) -> Result<(), LcmError> { + let sql = format!( + "SELECT {}, + role, ordinal, timestamp, content_hash, storage_kind, payload_ref, metadata_json + FROM lcm_raw_messages AS raw + WHERE store_id = ?1 AND provider = ?2 AND session_id = ?3 AND message_id = ?4 + LIMIT 1", + stored_message_record_select_columns("raw") + ); let mut rows = conn .query( - "SELECT message.provider, message.message_id, message.session_id, - message.role, message.timestamp, message.ordinal, message.text, - message.kind, message.model, message.tool_names, message.source_path, - message.source_offset, message.metadata_json, - raw.role, raw.ordinal, raw.timestamp, raw.content_hash, - raw.storage_kind, raw.payload_ref, raw.metadata_json - FROM lcm_raw_messages AS raw - JOIN session_messages AS message - ON message.provider = raw.provider - AND message.message_id = raw.message_id - AND message.session_id = raw.session_id - WHERE raw.store_id = ?1 AND raw.provider = ?2 - AND raw.session_id = ?3 AND raw.message_id = ?4 - LIMIT 1", + &sql, params![ expected.store_id, expected.message.provider.as_str(), @@ -66,21 +62,7 @@ async fn require_current_protection_input( store_id: expected.store_id, }); }; - let actual_message = SessionMessageRecord { - provider: row.get(0)?, - message_id: row.get(1)?, - session_id: row.get(2)?, - role: row.get(3)?, - timestamp: row.get(4)?, - ordinal: row.get(5)?, - text: row.get(6)?, - kind: row.get(7)?, - model: row.get(8)?, - tool_names: row.get(9)?, - source_path: row.get(10)?, - source_offset: row.get(11)?, - metadata_json: row.get(12)?, - }; + let actual_message = message_record_from_row(&row)?; let actual_raw_revision = RawProtectionRevision { role: row.get(13)?, ordinal: row.get(14)?, @@ -98,6 +80,26 @@ async fn require_current_protection_input( Ok(()) } +fn message_record_from_row( + row: &tracedecay_runtime_core::db::engine::Row, +) -> Result { + Ok(SessionMessageRecord { + provider: row.get(0)?, + message_id: row.get(1)?, + session_id: row.get(2)?, + role: row.get(3)?, + timestamp: row.get(4)?, + ordinal: row.get(5)?, + text: row.get(6)?, + kind: row.get(7)?, + model: row.get(8)?, + tool_names: row.get(9)?, + source_path: row.get(10)?, + source_offset: row.get(11)?, + metadata_json: row.get(12)?, + }) +} + async fn require_current_raw_protection_revision( conn: &(impl QueryExecutor + ?Sized), store_id: i64, @@ -369,13 +371,10 @@ impl<'a, D: SessionRegisteredDb + Sync> SessionStoreAccess<'a, D> { /// ingest-protection shape before an LCM read or compression consumes /// them. /// - /// The observation projection lands `lcm_raw_messages` rows without a - /// sanitization receipt and deliberately preserves protected payloads on - /// replay, so this pass is the second phase of that design: each - /// unreceipted row is re-ingested from its canonical `session_messages` - /// projection through the privacy firewall, binding the receipt the - /// verified raw loads require. Already-protected rows are left untouched, - /// making the pass idempotent and bounded to one session. + /// Each unreceipted inline row is re-ingested from its own stored body + /// through the privacy firewall, binding the receipt the verified raw + /// loads require. Already-protected rows are left untouched, making the + /// pass idempotent and bounded to one session. #[hotpath::skip] pub async fn lcm_protect_session_raw_messages( &self, @@ -430,28 +429,24 @@ impl<'a, D: SessionRegisteredDb + Sync> SessionStoreAccess<'a, D> { .map(|row| row.get::(0)) .transpose()?; drop(generation_rows); + let sql = format!( + "SELECT {}, + role, ordinal, timestamp, content_hash, storage_kind, payload_ref, + metadata_json, store_id, + CASE WHEN json_extract( + metadata_json, + '$.ingest_protection.sanitization_receipt' + ) IS NULL THEN 1 ELSE 0 END, + COALESCE(length(CAST(content AS BLOB)), 0) + FROM lcm_raw_messages AS raw + WHERE provider = ?1 AND session_id = ?2 AND store_id > ?3 + ORDER BY store_id + LIMIT ?4", + stored_message_record_select_columns("raw") + ); let mut rows = QueryExecutor::query( &snapshot, - "SELECT raw.store_id, - CASE WHEN json_extract( - raw.metadata_json, - '$.ingest_protection.sanitization_receipt' - ) IS NULL THEN 1 ELSE 0 END, - COALESCE(length(CAST(message.text AS BLOB)), 0), - message.provider, message.message_id, message.session_id, message.role, - message.timestamp, message.ordinal, message.text, message.kind, - message.model, message.tool_names, message.source_path, - message.source_offset, message.metadata_json, - raw.role, raw.ordinal, raw.timestamp, raw.content_hash, - raw.storage_kind, raw.payload_ref, raw.metadata_json - FROM lcm_raw_messages AS raw - LEFT JOIN session_messages AS message - ON raw.provider = message.provider - AND raw.message_id = message.message_id - WHERE raw.provider = ?1 AND raw.session_id = ?2 - AND raw.store_id > ?3 - ORDER BY raw.store_id - LIMIT ?4", + &sql, params![ provider, session_id, @@ -469,9 +464,9 @@ impl<'a, D: SessionRegisteredDb + Sync> SessionStoreAccess<'a, D> { let mut frontier_store_id = after_store_id; let mut byte_limited = false; while let Some(row) = rows.next().await? { - let store_id: i64 = row.get(0)?; - let needs_protection = row.get::(1)? != 0; - let row_bytes = u64::try_from(row.get::(2)?).map_err(|error| { + let store_id: i64 = row.get(20)?; + let needs_protection = row.get::(21)? != 0; + let row_bytes = u64::try_from(row.get::(22)?).map_err(|error| { LcmError::Db(format!("invalid LCM protection row byte count: {error}")) })?; if bytes_scanned.saturating_add(row_bytes) > page_max_bytes { @@ -485,66 +480,30 @@ impl<'a, D: SessionRegisteredDb + Sync> SessionStoreAccess<'a, D> { bytes_scanned = bytes_scanned.saturating_add(row_bytes); frontier_store_id = store_id; let raw_revision = RawProtectionRevision { - role: row.get(16)?, - ordinal: row.get(17)?, - timestamp: row.get(18)?, - content_hash: row.get(19)?, - storage_kind: row.get(20)?, - payload_ref: row.get(21)?, - metadata_json: row.get(22)?, + role: row.get(13)?, + ordinal: row.get(14)?, + timestamp: row.get(15)?, + content_hash: row.get(16)?, + storage_kind: row.get(17)?, + payload_ref: row.get(18)?, + metadata_json: row.get(19)?, }; scanned_revisions.push((store_id, raw_revision.clone())); if !needs_protection { continue; } - let message = SessionMessageRecord { - provider: row.get::>(3)?.ok_or_else(|| { - LcmError::SummarySourceUnavailable { - source_id: store_id.to_string(), - reason: "canonical_session_message_missing".to_string(), - } - })?, - message_id: row.get::>(4)?.ok_or_else(|| { - LcmError::SummarySourceUnavailable { - source_id: store_id.to_string(), - reason: "canonical_session_message_missing".to_string(), - } - })?, - session_id: row.get::>(5)?.ok_or_else(|| { - LcmError::SummarySourceUnavailable { - source_id: store_id.to_string(), - reason: "canonical_session_message_missing".to_string(), - } - })?, - role: row.get::>(6)?.ok_or_else(|| { - LcmError::SummarySourceUnavailable { - source_id: store_id.to_string(), - reason: "canonical_session_message_missing".to_string(), - } - })?, - timestamp: row.get(7)?, - ordinal: row.get::>(8)?.ok_or_else(|| { - LcmError::SummarySourceUnavailable { - source_id: store_id.to_string(), - reason: "canonical_session_message_missing".to_string(), - } - })?, - text: row.get::>(9)?.ok_or_else(|| { - LcmError::SummarySourceUnavailable { - source_id: store_id.to_string(), - reason: "canonical_session_message_missing".to_string(), - } - })?, - kind: row.get(10)?, - model: row.get(11)?, - tool_names: row.get(12)?, - source_path: row.get(13)?, - source_offset: row.get(14)?, - metadata_json: row.get(15)?, - }; + // An unreceipted row is re-ingested from its own stored body. A + // body stored outside the row leaves only its placeholder here, + // which is not the message and must not be re-ingested as one. + if raw_revision.storage_kind != "inline" { + return Err(LcmError::SummarySourceUnavailable { + source_id: store_id.to_string(), + reason: "external_body_without_receipt".to_string(), + }); + } unprotected.push(RawProtectionInput { store_id, - message, + message: message_record_from_row(&row)?, raw_revision, }); } diff --git a/crates/tracedecay-sessions/src/runtime/store_access/sessions.rs b/crates/tracedecay-sessions/src/runtime/store_access/sessions.rs index 0aed0f6e9c..acf65d381f 100644 --- a/crates/tracedecay-sessions/src/runtime/store_access/sessions.rs +++ b/crates/tracedecay-sessions/src/runtime/store_access/sessions.rs @@ -9,7 +9,8 @@ use tracedecay_runtime_core::db::engine::{Error as EngineError, FromValue, Row, use tracedecay_store::{SESSION_MESSAGE_PROJECTOR_VERSION, SessionMessageRecord, SessionRecord}; use crate::runtime::SessionMessageSearchResult; -use crate::runtime::codex::codex_cursor_key; +use crate::runtime::hosts::codex::codex_cursor_key; +use tracedecay_lcm::raw::{message_body_record_select_columns, message_record_select_columns}; use tracedecay_lcm::retrieval_content::{ RelatedMessageCopyIdentity, dedupe_related_message_copies, rerank_fetch_limit, }; @@ -35,7 +36,7 @@ pub(crate) const SESSION_MESSAGE_ID_LOOKUP_MAX: usize = 256; /// provider ingest batch identity reads without duplicating session schema. pub(crate) const EXISTING_SESSION_MESSAGE_IDS_SQL: &str = "SELECT messages.message_id FROM json_each(?2) AS requested - CROSS JOIN session_messages AS messages + CROSS JOIN lcm_raw_messages AS messages WHERE requested.type = 'text' AND messages.provider = ?1 AND messages.message_id = requested.value"; @@ -90,7 +91,7 @@ fn descending_timestamp(left: Option, right: Option) -> std::cmp::Orde } } pub const SESSION_MESSAGES_AFTER_SQL: &str = "SELECT timestamp, ordinal, kind, tool_names, metadata_json \ - FROM session_messages \ + FROM lcm_raw_messages \ WHERE provider = ?1 AND session_id = ?2 \ AND timestamp IS NOT NULL AND timestamp >= ?3 \ ORDER BY timestamp, ordinal, message_id \ @@ -268,10 +269,13 @@ impl SessionStoreAccess<'_, D> { .iter() .flat_map(|(path, providers)| { providers.iter().map(move |provider| { + // `set_parse_offset` stores every cursor under its + // identity key, so a Windows location must be looked + // up the same way to find its checkpoint. let key = if provider == "codex" { codex_cursor_key(Path::new(path)).durable_text() } else { - path.clone() + path_identity_key(path) }; serde_json::json!({ "path": path, "key": key }) }) @@ -364,7 +368,7 @@ impl SessionStoreAccess<'_, D> { .read_connection() .query( "SELECT EXISTS( - SELECT 1 FROM session_messages + SELECT 1 FROM lcm_raw_messages WHERE provider = ?1 AND message_id = ?2 )", tracedecay_runtime_core::db::engine::params![provider, message_id], @@ -423,7 +427,7 @@ impl SessionStoreAccess<'_, D> { pub async fn session_message_count(&self) -> Result { let mut rows = self .read_connection() - .query("SELECT COUNT(*) FROM session_messages", ()) + .query("SELECT COUNT(*) FROM lcm_raw_messages", ()) .await .map_err(|error| format!("failed to count session messages: {error}"))?; let row = rows @@ -441,7 +445,7 @@ impl SessionStoreAccess<'_, D> { project_key: &str, ) -> Result { let mut sql = "SELECT COUNT(*) - FROM session_messages m + FROM lcm_raw_messages m JOIN sessions s ON s.provider = m.provider AND s.session_id = m.session_id WHERE 1 = 1" .to_owned(); @@ -532,14 +536,14 @@ impl SessionStoreAccess<'_, D> { .read_connection() .query( "WITH latest_seconds AS ( - SELECT timestamp FROM session_messages + SELECT timestamp FROM lcm_raw_messages WHERE timestamp IS NOT NULL AND timestamp < ?1 ORDER BY timestamp DESC LIMIT 1 ), latest_millis AS ( - SELECT timestamp FROM session_messages + SELECT timestamp FROM lcm_raw_messages WHERE timestamp >= ?1 ORDER BY timestamp DESC LIMIT 1 @@ -580,11 +584,14 @@ impl SessionStoreAccess<'_, D> { ) -> tracedecay_domain::errors::Result> { const OPERATION: &str = "read registered session message"; let snapshot = self.read_snapshot().await?; + let sql = format!( + "SELECT {} + FROM lcm_raw_messages AS message WHERE provider = ?1 AND message_id = ?2", + message_body_record_select_columns("message") + ); let mut rows = snapshot .query( - "SELECT provider, message_id, session_id, role, timestamp, ordinal, text, kind, - model, tool_names, source_path, source_offset, metadata_json - FROM session_messages WHERE provider = ?1 AND message_id = ?2", + &sql, tracedecay_runtime_core::db::engine::params![provider, message_id], ) .await @@ -626,18 +633,19 @@ impl SessionStoreAccess<'_, D> { let fetch_limit = rerank_fetch_limit(limit, SESSION_MESSAGE_SEARCH_MAX_FETCH); let snapshot = self.read_snapshot().await?; - let mut sql = "SELECT + let mut sql = format!( + "SELECT s.provider, s.session_id, s.project_key, s.project_path, s.title, s.started_at, s.ended_at, s.transcript_path, s.metadata_json, s.parent_session_id, s.is_subagent, s.agent_id, s.parent_tool_use_id, - m.provider, m.message_id, m.session_id, m.role, m.timestamp, m.ordinal, m.text, - m.kind, m.model, m.tool_names, m.source_path, m.source_offset, m.metadata_json, - bm25(session_messages_fts, 10.0, 2.0, 1.0, 1.0, 1.0) AS rank - FROM session_messages_fts - JOIN session_messages m ON session_messages_fts.rowid = m.rowid + {}, + bm25(lcm_raw_messages_fts, 10.0, 2.0, 1.0, 1.0, 1.0) AS rank + FROM lcm_raw_messages_fts + JOIN lcm_raw_messages m ON lcm_raw_messages_fts.rowid = m.store_id JOIN sessions s ON s.provider = m.provider AND s.session_id = m.session_id - WHERE session_messages_fts MATCH ?1" - .to_owned(); + WHERE lcm_raw_messages_fts MATCH ?1", + message_record_select_columns("m") + ); let mut query_params = vec![Value::Text(fts_query), Value::Text(provider.to_owned())]; let _ = write!(sql, " AND m.provider = ?{}", query_params.len()); if let Some(project_key) = project_key { @@ -647,7 +655,7 @@ impl SessionStoreAccess<'_, D> { query_params.push(Value::Text(term.clone())); let _ = write!( sql, - " AND instr(lower(m.text), ?{}) > 0", + " AND instr(lower(m.index_text), ?{}) > 0", query_params.len() ); } @@ -656,7 +664,7 @@ impl SessionStoreAccess<'_, D> { )); let _ = write!( sql, - " ORDER BY bm25(session_messages_fts, 10.0, 2.0, 1.0, 1.0, 1.0) + " ORDER BY bm25(lcm_raw_messages_fts, 10.0, 2.0, 1.0, 1.0, 1.0) LIMIT ?{}", query_params.len() ); @@ -775,58 +783,6 @@ impl SessionStoreAccess<'_, D> { }); } - let mut legacy_sql = "SELECT - s.provider, s.session_id, s.project_key, s.project_path, s.title, s.started_at, - s.ended_at, s.transcript_path, s.metadata_json, s.parent_session_id, - s.is_subagent, s.agent_id, s.parent_tool_use_id, - m.provider, m.message_id, m.session_id, m.role, m.timestamp, m.ordinal, m.text, - m.kind, m.model, m.tool_names, m.source_path, m.source_offset, m.metadata_json - FROM session_messages m - JOIN sessions s ON s.provider = m.provider AND s.session_id = m.session_id - WHERE m.kind = 'goal' - AND m.ordinal = ( - SELECT MAX(m2.ordinal) FROM session_messages m2 - WHERE m2.provider = m.provider - AND m2.session_id = m.session_id - AND m2.kind = 'goal' - ) - AND NOT EXISTS ( - SELECT 1 FROM observation_workflow_facts w - WHERE w.projector_version = ?1 - AND w.provider = m.provider - AND w.session_id = m.session_id - AND w.semantic_kind = 'goal' - )" - .to_owned(); - let mut legacy_params = vec![Value::Text(SESSION_MESSAGE_PROJECTOR_VERSION.to_owned())]; - if let Some(project_key) = project_key { - push_project_identity_predicate(&mut legacy_sql, &mut legacy_params, project_key); - } - legacy_params.push(Value::Integer(i64::try_from(limit).unwrap_or(i64::MAX))); - let _ = write!( - legacy_sql, - " ORDER BY (m.timestamp IS NULL) ASC, m.timestamp DESC, m.ordinal DESC LIMIT ?{}", - legacy_params.len() - ); - let mut rows = snapshot - .query(&legacy_sql, legacy_params) - .await - .map_err(|error| session_db_operation_error(OPERATION, error))?; - while let Some(row) = rows - .next() - .await - .map_err(|error| session_db_operation_error(OPERATION, error))? - { - let session = row_to_session(&row) - .map_err(|message| session_db_operation_message(OPERATION, message))?; - let message = row_to_message(&row, 13) - .map_err(|message| session_db_operation_message(OPERATION, message))?; - results.push(SessionMessageSearchResult { - session, - message, - score: 0.0, - }); - } results.sort_by(|left, right| { descending_timestamp(left.message.timestamp, right.message.timestamp) .then_with(|| right.message.ordinal.cmp(&left.message.ordinal)) @@ -1025,7 +981,8 @@ pub fn session_record_from_row(row: &Row) -> Result TranscriptGitEvidence<'a> { } } -const TRANSCRIPT_STATEMENT_WINDOW: usize = 64; - /// Prepares every privacy-protected raw-message write before the transaction /// acquires SQLite's single-writer lease. /// @@ -68,25 +66,6 @@ fn stage_full_transcript_messages( Ok(staged) } -#[hotpath::measure( - label = "sessions.store.transcript.flush_statement_window", - future = true -)] -async fn flush_transcript_statement_window( - conn: &impl Executor, - statements: &mut Vec, -) -> Result<(), TranscriptPersistenceError> { - if statements.is_empty() { - return Ok(()); - } - conn.execute_statements(std::mem::take(statements)) - .await - .map(|_| ()) - .map_err(|error| { - TranscriptPersistenceError::storage("upsert session message projections", error) - }) -} - async fn reconcile_codex_goal_response( conn: &impl Executor, current: &SessionMessageRecord, @@ -104,18 +83,8 @@ async fn reconcile_codex_goal_response( params![current.provider.as_str(), response_message_id.as_str()], ) .await - .map_err(|error| { - TranscriptPersistenceError::storage("remove paired Codex goal raw message", error) - })?; - conn.execute( - "DELETE FROM session_messages WHERE provider = ?1 AND message_id = ?2", - params![current.provider.as_str(), response_message_id.as_str()], - ) - .await .map(|_| ()) - .map_err(|error| { - TranscriptPersistenceError::storage("remove paired Codex goal projection", error) - }) + .map_err(|error| TranscriptPersistenceError::storage("remove paired Codex goal message", error)) } /// Reads one durable cursor by its canonical key. @@ -413,11 +382,10 @@ impl SessionStoreAccess<'_, D> { #[hotpath::skip] async fn upsert_session_message_in_existing_tx( - &self, conn: &impl Executor, message: &SessionMessageRecord, staged: raw::StagedRawMessageIngest, - ) -> Result { + ) -> Result<(), TranscriptPersistenceError> { // Clone-on-normalize: message text can be hundreds of kilobytes, and // most providers already emit in-range timestamps, so the full-record // copy is paid only when the timestamp actually changes. @@ -429,59 +397,10 @@ impl SessionStoreAccess<'_, D> { owned.timestamp = normalized_timestamp; std::borrow::Cow::Owned(owned) }; - let raw = raw::commit_staged_raw_message(conn, canonical_message.as_ref(), staged) + raw::commit_staged_raw_message(conn, canonical_message.as_ref(), staged) .await - .map_err(|error| { - TranscriptPersistenceError::storage("upsert LCM raw message", error) - })?; - Self::session_message_projection_statement( - canonical_message.as_ref(), - &raw.projection_text, - raw.projection_metadata_json.as_deref(), - ) - } - - fn session_message_projection_statement( - message: &SessionMessageRecord, - text: &str, - metadata_json: Option<&str>, - ) -> Result { - WriteStatement::new( - "INSERT INTO session_messages - (provider, message_id, session_id, role, timestamp, ordinal, text, kind, model, - tool_names, source_path, source_offset, metadata_json) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13) - ON CONFLICT(provider, message_id) DO UPDATE SET - session_id = excluded.session_id, - role = excluded.role, - timestamp = excluded.timestamp, - ordinal = excluded.ordinal, - text = excluded.text, - kind = excluded.kind, - model = excluded.model, - tool_names = excluded.tool_names, - source_path = excluded.source_path, - source_offset = excluded.source_offset, - metadata_json = excluded.metadata_json", - params![ - message.provider.clone(), - message.message_id.clone(), - message.session_id.clone(), - message.role.clone(), - message.timestamp, - message.ordinal, - text, - message.kind.clone(), - message.model.clone(), - message.tool_names.clone(), - message.source_path.clone(), - message.source_offset, - metadata_json, - ], - ) - .map_err(|error| { - TranscriptPersistenceError::storage("prepare session message projection", error) - }) + .map(|_| ()) + .map_err(|error| TranscriptPersistenceError::storage("upsert LCM raw message", error)) } /// Atomically upserts one transcript session + all parsed messages and then @@ -608,7 +527,6 @@ impl SessionStoreAccess<'_, D> { let transaction = self.begin_transcript_transaction().await?; let write_result: Result<(), TranscriptPersistenceError> = async { - let mut projection_statements = Vec::with_capacity(TRANSCRIPT_STATEMENT_WINDOW); // Full batches are one-winner compare-and-swap on the durable // parse cursor. `actual == next_offset` is not a retry grant: // a competing writer can share that destination while carrying @@ -631,10 +549,6 @@ impl SessionStoreAccess<'_, D> { correlation.source() == tracedecay_store::CodexGoalContextSource::ItemCompleted }) { - // Make any response written earlier in this batch - // visible to the bounded correlation query. - flush_transcript_statement_window(&transaction, &mut projection_statements) - .await?; reconcile_codex_goal_response(&transaction, message).await?; } let staged = staged_messages.next().ok_or_else(|| { @@ -643,17 +557,10 @@ impl SessionStoreAccess<'_, D> { "staged transcript message count did not match the write batch", ) })?; - projection_statements.push( - self.upsert_session_message_in_existing_tx(&transaction, message, staged) - .await?, - ); - if projection_statements.len() >= TRANSCRIPT_STATEMENT_WINDOW { - flush_transcript_statement_window(&transaction, &mut projection_statements) - .await?; - } + Self::upsert_session_message_in_existing_tx(&transaction, message, staged) + .await?; } } - flush_transcript_statement_window(&transaction, &mut projection_statements).await?; if staged_messages.next().is_some() { return Err(TranscriptPersistenceError::message( "upsert LCM raw message", @@ -837,64 +744,13 @@ async fn require_expected_pair_offset( #[cfg(test)] mod tests { - use std::sync::atomic::{AtomicUsize, Ordering}; - - use tracedecay_runtime_core::db::engine::{ - Executor, IntoParams, QueryExecutor, Rows, WriteStatement, params, - }; use tracedecay_store::{SessionMessageRecord, SessionRecord}; use super::{ PayloadFileRollback, TranscriptBatch, TranscriptPersistenceError, decode_u64_bits_value, - encode_u64_bits, flush_transcript_statement_window, stage_full_transcript_messages, + encode_u64_bits, stage_full_transcript_messages, }; - #[derive(Default)] - struct BatchCountingExecutor { - batch_submissions: AtomicUsize, - } - - impl QueryExecutor for BatchCountingExecutor { - async fn query

( - &self, - _sql: &str, - _params: P, - ) -> tracedecay_runtime_core::db::engine::Result - where - P: IntoParams, - { - panic!("statement-window test must not query") - } - } - - impl Executor for BatchCountingExecutor { - async fn execute

( - &self, - _sql: &str, - _params: P, - ) -> tracedecay_runtime_core::db::engine::Result - where - P: IntoParams, - { - panic!("statement-window test must not submit scalar writes") - } - - async fn execute_statements( - &self, - statements: Vec, - ) -> tracedecay_runtime_core::db::engine::Result> { - self.batch_submissions.fetch_add(1, Ordering::Relaxed); - Ok(vec![1; statements.len()]) - } - - async fn execute_batch( - &self, - _sql: &str, - ) -> tracedecay_runtime_core::db::engine::Result<()> { - panic!("statement-window test must not submit raw SQL batches") - } - } - /// Every `parse_offsets` column round-trips the whole `u64` domain: the /// Codex corpus epoch stores a 128-bit digest across `byte_offset` and /// `mtime`, so any half with its top bit set must persist losslessly and @@ -976,20 +832,4 @@ mod tests { other => panic!("expected attributed sanitization failure, got {other}"), } } - - #[tokio::test] - async fn transcript_statement_window_uses_one_batch_submission() { - let executor = BatchCountingExecutor::default(); - let mut statements = vec![ - WriteStatement::new("INSERT INTO example(value) VALUES (?1)", params![1_i64]).unwrap(), - WriteStatement::new("INSERT INTO example(value) VALUES (?1)", params![2_i64]).unwrap(), - ]; - - flush_transcript_statement_window(&executor, &mut statements) - .await - .unwrap(); - - assert!(statements.is_empty()); - assert_eq!(executor.batch_submissions.load(Ordering::Relaxed), 1); - } } diff --git a/crates/tracedecay-sessions/src/runtime/workflow/workflow_ingest.rs b/crates/tracedecay-sessions/src/runtime/workflow/workflow_ingest.rs index 44306684e8..04713043f9 100644 --- a/crates/tracedecay-sessions/src/runtime/workflow/workflow_ingest.rs +++ b/crates/tracedecay-sessions/src/runtime/workflow/workflow_ingest.rs @@ -329,13 +329,13 @@ fn run_cwd(run: &DiscoveredRun) -> Option { }); if let Some(cwd) = parent_transcript .as_deref() - .and_then(crate::runtime::claude::transcript_cwd) + .and_then(crate::runtime::hosts::claude::transcript_cwd) { return Some(cwd); } // Fall back to the first agent transcript that records a cwd. for path in agent_transcripts(&run.agents_dir) { - if let Some(cwd) = crate::runtime::claude::transcript_cwd(&path) { + if let Some(cwd) = crate::runtime::hosts::claude::transcript_cwd(&path) { return Some(cwd); } } diff --git a/crates/tracedecay-sessions/src/runtime/workflow/workflow_state.rs b/crates/tracedecay-sessions/src/runtime/workflow/workflow_state.rs index b8f7447784..f6bac7c9b5 100644 --- a/crates/tracedecay-sessions/src/runtime/workflow/workflow_state.rs +++ b/crates/tracedecay-sessions/src/runtime/workflow/workflow_state.rs @@ -46,7 +46,7 @@ pub async fn list_unfinished( ORDER BY COALESCE(raw.timestamp, 0) DESC, raw.store_id DESC LIMIT ?2", params![ - r#""session limit" OR blocked OR interrupted OR "runs 0""#, + r#"index_text : ("session limit" OR blocked OR interrupted OR "runs 0")"#, limit ], ) diff --git a/crates/tracedecay-source-edit/src/edits/test_support.rs b/crates/tracedecay-source-edit/src/edits/test_support.rs index 403db61798..da548e25fa 100644 --- a/crates/tracedecay-source-edit/src/edits/test_support.rs +++ b/crates/tracedecay-source-edit/src/edits/test_support.rs @@ -174,7 +174,7 @@ pub(super) fn fixture_symbol_code_graph( qualified_name: qualified_name.to_owned(), simple_name: simple_name.to_owned(), kind: "function".to_owned(), - visibility: "pub".to_owned(), + visibility: "public".to_owned(), branches: 0, loops: 0, max_nesting: 0, diff --git a/crates/tracedecay-source-edit/src/journal.rs b/crates/tracedecay-source-edit/src/journal.rs index 5bfc2a2d9b..51fa1602a8 100644 --- a/crates/tracedecay-source-edit/src/journal.rs +++ b/crates/tracedecay-source-edit/src/journal.rs @@ -1,4 +1,4 @@ -use std::fs::{self, File}; +use std::fs; use std::path::PathBuf; use serde::{Deserialize, Serialize}; @@ -8,6 +8,7 @@ use tracedecay_contracts::{ SourceEditVerificationV1, }; use tracedecay_domain::{ManifestDigest, UtcMicros, canonical_sha256}; +use tracedecay_private_fs::FileLease; use tracedecay_private_fs::framed_log::{DirectorySyncPolicy, sync_parent_directory}; use tracedecay_runtime_core::storage::try_acquire_sidecar_lock; @@ -42,7 +43,8 @@ pub(super) struct SourceEditJournalV1 { pub(super) effect_id: EffectId, pub(super) input_digest: ManifestDigest, pub(super) expected_state: ManifestDigest, - #[serde(default)] + /// `None` only on in-memory pre-effect records; every persisted journal + /// carries the exact previewed postimage digest. pub(super) predicted_state: Option, pub(super) candidate_files: Vec, #[serde(default)] @@ -54,7 +56,12 @@ pub(super) struct SourceEditJournalV1 { } impl SourceEditJournalV1 { - fn validate_recovery(&self) -> Result<()> { + fn validate_persisted(&self) -> Result<()> { + if self.predicted_state.is_none() { + return Err(config_error( + "unsupported source edit journal: it carries no predicted state", + )); + } match (&self.recovery_digest, self.recovery_files.is_empty()) { (None, true) => Ok(()), (Some(digest), false) @@ -175,7 +182,7 @@ impl SourceEditDurability { /// store behind one exclusive lock file. The lock is released when the /// returned handle drops; contention is a typed refusal, never a wait. #[hotpath::measure(label = "usecases.edit.lock")] - pub(super) fn lock(&self) -> Result { + pub(super) fn lock(&self) -> Result { let lock_path = self.root.join("source-edit.lock"); try_acquire_sidecar_lock(&lock_path)?.ok_or_else(|| TraceDecayError::SyncLock { message: format!( @@ -236,7 +243,7 @@ impl SourceEditDurability { &journal.request.authority, &journal.request.authority_proof, )?; - journal.validate_recovery()?; + journal.validate_persisted()?; } Ok(journal) } diff --git a/crates/tracedecay-source-edit/src/reconcile.rs b/crates/tracedecay-source-edit/src/reconcile.rs index faa55ab575..57d8ab1818 100644 --- a/crates/tracedecay-source-edit/src/reconcile.rs +++ b/crates/tracedecay-source-edit/src/reconcile.rs @@ -261,9 +261,7 @@ fn reconcile_prepared_source_edit_controlled( let (_outcome, record) = match request.disposition.clone() { SourceEditReconciliationDispositionV1::ConfirmCommitted { committed_state } => { let predicted_state = journal.predicted_state.as_ref().ok_or_else(|| { - config_error( - "source edit committed state cannot be proven from this legacy journal", - ) + config_error("source edit committed state cannot be proven without a prediction") })?; if &committed_state != predicted_state || observed_state != *predicted_state { return Err(config_error( @@ -379,9 +377,8 @@ pub(super) async fn recover_source_edit_transaction( // (i) Roll forward. The worktree already holds the exact previewed result, // so the write succeeded and only the bookkeeping was lost. Finalize the // commit and keep every byte; the client-timeout `ConfirmCommitted` - // disposition reaches the same durable record. `recovery_files` is only - // ever populated alongside `predicted_state` (see `execute.rs`), so a - // present predicted state is guaranteed here. + // disposition reaches the same durable record. Loading a persisted + // journal rejects one without a predicted state. if journal.predicted_state.as_ref() == Some(&observed_state) { hotpath::measure_block!( "usecases.edit.recover.commit", @@ -583,6 +580,23 @@ mod tests { assert!(durability.load_journal().is_err()); } + #[test] + fn journal_without_predicted_state_is_rejected() { + let directory = tempdir().unwrap(); + let durability = SourceEditDurability { + root: directory.path().to_path_buf(), + }; + let request = fixture_request(); + let mut journal = fixture_journal(&request, SourceEditJournalStateV1::Prepared); + durability.persist_journal(&journal).unwrap(); + assert!(durability.load_journal().unwrap().is_some()); + + journal.predicted_state = None; + durability.persist_journal(&journal).unwrap(); + let error = durability.load_journal().unwrap_err().to_string(); + assert!(error.contains("no predicted state"), "{error}"); + } + #[test] fn applied_restart_finalizes_original_receipt_and_clears_journal() { let directory = tempdir().unwrap(); diff --git a/crates/tracedecay-source-edit/src/test_support.rs b/crates/tracedecay-source-edit/src/test_support.rs index d3b7f04ee0..f4bcb17937 100644 --- a/crates/tracedecay-source-edit/src/test_support.rs +++ b/crates/tracedecay-source-edit/src/test_support.rs @@ -119,7 +119,7 @@ pub(super) fn fixture_journal( effect_id: effect_id(&request.idempotency_key, &input_digest).unwrap(), input_digest, expected_state: request.expected_state.clone(), - predicted_state: None, + predicted_state: Some(digest(SHA256_B)), candidate_files: vec!["src/lib.rs".to_owned()], recovery_files: Vec::new(), recovery_digest: None, diff --git a/crates/tracedecay-source-edit/src/verify.rs b/crates/tracedecay-source-edit/src/verify.rs index 7a71147a26..3fc6ad7fe3 100644 --- a/crates/tracedecay-source-edit/src/verify.rs +++ b/crates/tracedecay-source-edit/src/verify.rs @@ -172,19 +172,30 @@ mod tests { unavailable.state, SourceEditVerificationStateV1::Unavailable ); - assert!(unavailable.message.is_some()); + assert_eq!(unavailable.verdict, "unavailable"); + assert_message_retains(&unavailable, "diagnostics unavailable"); let cancelled = failed_edit_verification(TraceDecayError::Io(std::io::Error::new( std::io::ErrorKind::Interrupted, "diagnostics cancelled", ))); assert_eq!(cancelled.state, SourceEditVerificationStateV1::Cancelled); - assert!(cancelled.message.is_some()); + assert_eq!(cancelled.verdict, "cancelled"); + assert_message_retains(&cancelled, "diagnostics cancelled"); let failed = failed_edit_verification(TraceDecayError::Config { message: "diagnostics failed".to_owned(), }); assert_eq!(failed.state, SourceEditVerificationStateV1::Failed); - assert!(failed.message.is_some()); + assert_eq!(failed.verdict, "failed"); + assert_message_retains(&failed, "diagnostics failed"); + } + + fn assert_message_retains(verification: &SourceEditVerificationV1, cause: &str) { + let message = verification.message.as_deref().unwrap_or_default(); + assert!( + message.contains(cause), + "verification message {message:?} dropped the cause {cause:?}" + ); } } diff --git a/crates/tracedecay-store-runtime/Cargo.toml b/crates/tracedecay-store-runtime/Cargo.toml index d581c9bf45..b423677d16 100644 --- a/crates/tracedecay-store-runtime/Cargo.toml +++ b/crates/tracedecay-store-runtime/Cargo.toml @@ -37,7 +37,6 @@ thiserror = "2" tokio = { version = "1", features = ["full"] } tracing = "0.1" tracedecay-contracts = { path = "../tracedecay-contracts", version = "0.1.0" } -tracedecay-automation-runtime = { path = "../tracedecay-automation-runtime", version = "0.1.0" } tracedecay-code-index = { path = "../tracedecay-code-index", version = "0.1.0", default-features = false } tracedecay-code-index-retention = { path = "../tracedecay-code-index-retention", version = "0.1.0" } tracedecay-code-index-runtime = { path = "../tracedecay-code-index-runtime", version = "0.1.0", default-features = false } diff --git a/crates/tracedecay-store-runtime/src/lib.rs b/crates/tracedecay-store-runtime/src/lib.rs index 1eccd6c6ab..71caa1d4ee 100644 --- a/crates/tracedecay-store-runtime/src/lib.rs +++ b/crates/tracedecay-store-runtime/src/lib.rs @@ -49,8 +49,8 @@ pub use session_registry::maintenance::{ pub use session_registry::{ DaemonSessionRuntimeRegistryV1, MAX_RETAINED_GRAPH_DB_OWNERS, RemoteRecoveryAdmission, RemoteRecoveryProjectLifecycle, RemoteRecoveryQuiescence, - mark_process_long_lived_for_session_maintenance, open_user_memory_db, - process_runtime_generation, registry_open_error, release_process_allocator_memory, + mark_process_long_lived_for_session_maintenance, process_runtime_generation, + registry_open_error, release_process_allocator_memory, }; pub use standalone_session::join_standalone_session_registry; pub use store_shutdown::{ diff --git a/crates/tracedecay-store-runtime/src/remote_replay_transaction.rs b/crates/tracedecay-store-runtime/src/remote_replay_transaction.rs index 10b2535d47..e8bc10567c 100644 --- a/crates/tracedecay-store-runtime/src/remote_replay_transaction.rs +++ b/crates/tracedecay-store-runtime/src/remote_replay_transaction.rs @@ -21,7 +21,7 @@ use tracedecay_store::{ RuntimeRequestProbeV1, RuntimeSubmitOutcomeV1, RuntimeSubmitRequestV1, RuntimeTransactionIdV1, RuntimeTransactionScopeV1, StoreClientIdV1, StoreCommitReceiptV1, StoreIdempotencyKeyV1, StoreOperationIdV1, StoreOperationMetadataV1, StoreRuntimeBindingV1, VerifiedStoreLocatorV1, - build_observation_resolution_authorization_v1, build_observation_retrieval_anchor_v2, + build_observation_resolution_authorization_v1, build_observation_retrieval_anchor, }; const CHANNEL_CAPACITY: usize = 128; @@ -655,7 +655,7 @@ fn prepare_request( let authorization = build_observation_resolution_authorization_v1(&observation, PROJECTION_GENERATION) .map_err(|_| RemoteReplayTransactionErrorV1::CanonicalEffect)?; - let anchor = build_observation_retrieval_anchor_v2( + let anchor = build_observation_retrieval_anchor( &observation, projection_generation.clone(), committed_at, diff --git a/crates/tracedecay-store-runtime/src/retained_memory.rs b/crates/tracedecay-store-runtime/src/retained_memory.rs index 593595ae58..164a051a1d 100644 --- a/crates/tracedecay-store-runtime/src/retained_memory.rs +++ b/crates/tracedecay-store-runtime/src/retained_memory.rs @@ -42,7 +42,7 @@ use tracedecay_session_memory::memory_mutation::{ use tracedecay_session_memory::memory_tracking::{TrackedExplicitSearch, track_explicit_search}; use tracedecay_session_runtime::retained::map_execution_error; -use crate::session_registry::{DaemonSessionRuntimeRegistryV1, open_user_memory_db}; +use crate::session_registry::DaemonSessionRuntimeRegistryV1; macro_rules! execute_scoped_memory { ( @@ -80,7 +80,7 @@ macro_rules! execute_scoped_memory { memory_mapping::ensure_profile_request_scope($memory_scope, $selector)?; let (database, _) = bounded_memory_operation($context, async { hotpath::future!( - open_user_memory_db(registry), + registry.profile_memory(), label = "daemon.retained.memory.open_profile" ) .await diff --git a/crates/tracedecay-store-runtime/src/retained_memory/target.rs b/crates/tracedecay-store-runtime/src/retained_memory/target.rs index 07547a4cbf..145fc33dcf 100644 --- a/crates/tracedecay-store-runtime/src/retained_memory/target.rs +++ b/crates/tracedecay-store-runtime/src/retained_memory/target.rs @@ -15,7 +15,7 @@ use tracedecay_session_memory::fact_store::ProjectMemoryDbHandle; use tracedecay_session_runtime::retained::map_execution_error; use tracedecay_store::StoreShardScopeV1; -use crate::session_registry::{DaemonSessionRuntimeRegistryV1, open_user_memory_db}; +use crate::session_registry::DaemonSessionRuntimeRegistryV1; #[derive(Clone)] pub struct RetainedMemoryTargetAuthorityV1 { @@ -112,9 +112,13 @@ pub async fn open_project_retained_memory_target( if selector.is_some() { return denied(); } - let database = open_profile_memory(&authority.registry).await?; + let database = authority + .registry + .profile_memory() + .await + .map_err(map_execution_error)?; return Ok(RetainedMemoryTargetV1::new( - ProjectMemoryDbHandle::Owned(Box::new(database)), + ProjectMemoryDbHandle::Owned(Box::new(Arc::unwrap_or_clone(database))), FactOwnerV1::Profile, )); } @@ -165,14 +169,6 @@ pub async fn open_project_retained_memory_target( open_selected_project_read_only(authority, selected_project_id).await } -async fn open_profile_memory( - registry: &DaemonSessionRuntimeRegistryV1, -) -> Result { - open_user_memory_db(registry) - .await - .map_err(map_execution_error) -} - #[hotpath::measure(label = "daemon.retained.memory.open_selected", future = true)] async fn open_selected_project_read_only( authority: &RetainedMemoryTargetAuthorityV1, diff --git a/crates/tracedecay-store-runtime/src/session_registry.rs b/crates/tracedecay-store-runtime/src/session_registry.rs index 60c5c6bf58..00ff318b19 100644 --- a/crates/tracedecay-store-runtime/src/session_registry.rs +++ b/crates/tracedecay-store-runtime/src/session_registry.rs @@ -11,7 +11,6 @@ use std::sync::{Arc, Mutex as StdMutex, OnceLock, Weak}; use serde::{Deserialize, Serialize}; use tokio::sync::Mutex; -use tracedecay_automation_runtime::ports::project_runtime::{ProfileRuntime, RuntimeFuture}; use tracedecay_domain::BrainNodeId; use tracedecay_sessions::observation::ObservationCancellation; use tracedecay_store::{ @@ -50,7 +49,6 @@ mod code_reads; pub mod maintenance; mod memory_graph_reconciliation_tasks; mod mounts; -mod profile_memory; mod remote_recovery; mod retained_hook_tasks; mod terminal_tasks; @@ -63,8 +61,6 @@ use retained_hook_tasks::RetainedHookTasks; #[cfg(any(test, feature = "test-helpers"))] pub use mounts::SessionGraphPublicationTestGate; -pub use profile_memory::open_user_memory_db; - /// RAII hold for a root-owned remote-recovery writer admission. /// /// The concrete guard type stays in the composition root; this crate only @@ -3258,20 +3254,6 @@ impl DaemonSessionRuntimeRegistryV1 { } } -impl ProfileRuntime for DaemonSessionRuntimeRegistryV1 { - fn profile_id(&self) -> &tracedecay_domain::configuration::UserProfileId { - self.identity.profile_id() - } - - fn profile_sessions(&self) -> RuntimeFuture<'_, RegisteredGlobalDbLeaseV1> { - Box::pin(DaemonSessionRuntimeRegistryV1::profile_sessions(self)) - } - - fn open_user_memory_db(&self) -> RuntimeFuture<'_, Database> { - Box::pin(open_user_memory_db(self)) - } -} - #[hotpath::measure(label = "daemon.session_registry.runtime_incarnation")] fn runtime_incarnation(identity: &LocalProfileIdentityAuthorityV1) -> Result { let process_run_id = tracedecay_runtime_core::runtime_identity::process_run_id(); diff --git a/crates/tracedecay-store-runtime/src/session_registry/code_graph.rs b/crates/tracedecay-store-runtime/src/session_registry/code_graph.rs index c29aa51f93..b48989e94c 100644 --- a/crates/tracedecay-store-runtime/src/session_registry/code_graph.rs +++ b/crates/tracedecay-store-runtime/src/session_registry/code_graph.rs @@ -30,6 +30,9 @@ use tracedecay_store::{ }; use super::{DaemonSessionRuntimeRegistryV1, Result, session_registry_error}; +use tracedecay_code_index_retention::code_index_generations::{ + acquire_generation_segments_publication_lock, code_generation_segments_root, +}; use tracedecay_code_index_runtime::{ CodeGraphReplayBindingV1, CodeGraphSeatLeaseV1, CodeGraphSeatRuntimePortV1, }; @@ -40,6 +43,8 @@ pub(super) use memory_runtime::{ }; pub(super) mod graph_attachment; #[cfg(test)] +mod linked_bundle_tests; +#[cfg(test)] mod sealed_publication_tests; mod seals; use seals::{ @@ -265,7 +270,7 @@ impl GraphCancellation for ResidentMemoryGuardedGraphCancellationV1 { } } -struct MaintenanceGraphCancellationV1(tracedecay_session_memory::context::CancellationToken); +struct MaintenanceGraphCancellationV1(tracedecay_runtime_core::cancellation::CancellationToken); impl GraphCancellation for MaintenanceGraphCancellationV1 { fn is_cancelled(&self) -> bool { @@ -1225,10 +1230,23 @@ impl RetainedCodeGraphRuntimeV1 { pub fn sweep_aborted_read_bundle_temporaries(&self) -> std::result::Result<(), GraphDbError> { tracedecay_graph_db::sweep_aborted_sealed_read_bundle_temporaries( &self.generations_root, + &self.read_bundle_artifacts_root()?, &self.sealed_state_digest, ) } + fn code_store_root(&self) -> std::result::Result<&std::path::Path, GraphDbError> { + self.generations_root + .parent() + .ok_or_else(|| GraphDbError::unavailable("code generations root has no store root")) + } + + /// Bundle artifacts live beside the generation segments that every + /// worktree scope of the project shares, so identical graphs store one. + fn read_bundle_artifacts_root(&self) -> std::result::Result { + Ok(code_generation_segments_root(self.code_store_root()?)) + } + #[hotpath::measure(label = "daemon.session_registry.publish_snapshot")] pub fn publish_verified_snapshot( &self, @@ -2352,6 +2370,7 @@ impl RetainedCodeGraphRuntimeV1 { }; tracedecay_graph_db::load_sealed_read_bundle_artifact( &self.generations_root, + &self.read_bundle_artifacts_root()?, &self.sealed_state_digest, &identity, tracedecay_code_index::graph_projection::INTERACTIVE_CATALOG_ARTIFACT_NAME, @@ -2383,8 +2402,15 @@ impl RetainedCodeGraphRuntimeV1 { }; let stage = || { self.sweep_aborted_read_bundle_temporaries()?; + let artifacts_root = self.read_bundle_artifacts_root()?; + std::fs::create_dir_all(&artifacts_root).map_err(|error| { + GraphDbError::unavailable(format!( + "failed to create the sealed read bundle artifact root: {error}" + )) + })?; let mut writer = tracedecay_graph_db::SealedReadBundleWriterV1::create( &self.generations_root, + &artifacts_root, &self.sealed_state_digest, )?; writer.stage_artifact( @@ -2423,13 +2449,22 @@ impl RetainedCodeGraphRuntimeV1 { let Some(writer) = writer else { return; }; - match writer.commit(identity, &|| { - if self.lifecycle_cancelled.load(Ordering::Acquire) { - Err(GraphDbError::Cancelled) - } else { - Ok(()) - } - }) { + let cancelled = || self.lifecycle_cancelled.load(Ordering::Acquire); + let commit = || { + // A placed artifact is unreferenced until its manifest lands; the + // shared lock keeps the project's segment sweep out until then. + let _segments_lock = + acquire_generation_segments_publication_lock(self.code_store_root()?, &cancelled) + .map_err(|error| GraphDbError::unavailable(error.to_string()))?; + writer.commit(identity, &|| { + if cancelled() { + Err(GraphDbError::Cancelled) + } else { + Ok(()) + } + }) + }; + match commit() { Ok(manifest) => { tracing::info!( generation = %self.generation_id, @@ -2697,7 +2732,7 @@ impl DaemonSessionRuntimeRegistryV1 { &self, project_id: ProjectId, project_database: &tracedecay_runtime_core::db::Database, - cancellation: &tracedecay_session_memory::context::CancellationToken, + cancellation: &tracedecay_runtime_core::cancellation::CancellationToken, after: Option, ) -> std::result::Result, GraphDbError> { let project_shard = StoreShardIdV1::project( @@ -2811,7 +2846,7 @@ impl DaemonSessionRuntimeRegistryV1 { project_database: &tracedecay_runtime_core::db::Database, generation: &CodeGenerationId, generation_file: &str, - cancellation: &tracedecay_session_memory::context::CancellationToken, + cancellation: &tracedecay_runtime_core::cancellation::CancellationToken, ) -> std::result::Result { let sealed_digest = sealed_digest_from_generation_file(generation_file)?; let replay_root = project_database diff --git a/crates/tracedecay-store-runtime/src/session_registry/code_graph/linked_bundle_tests.rs b/crates/tracedecay-store-runtime/src/session_registry/code_graph/linked_bundle_tests.rs new file mode 100644 index 0000000000..ca235d5568 --- /dev/null +++ b/crates/tracedecay-store-runtime/src/session_registry/code_graph/linked_bundle_tests.rs @@ -0,0 +1,328 @@ +//! Linked worktrees of one project seal their read bundles into the project's +//! shared artifact root: identical trees store one catalog artifact, a +//! divergent tree never serves its catalog to a sibling, and retiring one +//! worktree keeps the artifact the other still names. + +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::sync::Arc; +use std::sync::atomic::AtomicBool; + +use sha2::{Digest, Sha256}; +use tracedecay_code_index_retention::code_index_generations::{ + CodeGenerationRetentionModeV1, DurablePublicationPointerV1, code_generation_segments_root, + run_code_generation_retention, +}; +use tracedecay_code_index_runtime::CodeGraphReplayBindingV1; +use tracedecay_code_index_runtime::code_index_scheduler::{ + CodeIndexWorktreeSchedulerV1, SharedCodeIndexBytePoolV1, scoped_code_index_store_root, +}; +use tracedecay_daemon_identity::profile_identity; +use tracedecay_domain::{ProjectId, UtcMicros, sha256_hex_suffix}; +use tracedecay_graph_db::{ + SealedGraphStateDigest, SealedReadBundleArtifactStateV1, retire_sealed_read_bundle, + sealed_read_bundle_artifact_file_digest, sealed_read_bundle_manifest_artifact_digests, +}; +use tracedecay_runtime_core::path_safety::canonical_root_identity; + +use super::super::DaemonSessionRuntimeRegistryV1; + +const SHARED_SOURCE: &str = "pub fn shared_alpha() -> usize { 1 }\n\ + pub fn shared_beta() -> usize { shared_alpha() + 1 }\n"; + +fn git(root: &Path, args: &[&str]) { + let output = Command::new("git") + .args(args) + .current_dir(root) + .output() + .expect("run git fixture command"); + assert!( + output.status.success(), + "git fixture command failed: {args:?}: {}", + String::from_utf8_lossy(&output.stderr) + ); +} + +struct LinkedBundleScopeV1 { + scope: PathBuf, + sealed: SealedGraphStateDigest, + /// The artifact digests this scope's own bundle manifest names. + artifacts: Vec, + /// The catalog bytes this scope's runtime loads through its manifest. + catalog: Vec, +} + +struct LinkedBundlesV1 { + _temporary: tempfile::TempDir, + first: LinkedBundleScopeV1, + linked: LinkedBundleScopeV1, +} + +/// Seals and publishes the primary checkout and one linked worktree of the +/// same project, each into its own scope of one `code-index-v1/`. The linked +/// worktree optionally commits a file the primary checkout does not have. +async fn publish_linked_worktree_bundles( + label: &str, + linked_only_source: Option<&str>, +) -> LinkedBundlesV1 { + let temporary = tempfile::tempdir().expect("temporary fixture parent"); + let root = canonical_root_identity(temporary.path()); + let profile_root = root.join("profile"); + let project_root = root.join("project"); + std::fs::create_dir_all(project_root.join("src")).expect("project source directory"); + git(&project_root, &["init", "-q", "-b", "main"]); + git(&project_root, &["config", "user.name", "TraceDecay Test"]); + git( + &project_root, + &["config", "user.email", "tracedecay@example.invalid"], + ); + std::fs::write(project_root.join("src/lib.rs"), SHARED_SOURCE).expect("project source"); + git(&project_root, &["add", "."]); + git(&project_root, &["commit", "-qm", "linked bundle fixture"]); + let project_id = ProjectId::new(format!("project.linked-bundle-{label}")).expect("project id"); + tracedecay_runtime_core::storage::pin_fixture_repository_identity( + &project_root, + project_id.as_str(), + ) + .expect("project enrollment"); + let linked_root = root.join("linked"); + git( + &project_root, + &[ + "worktree", + "add", + "-q", + "-b", + "linked", + linked_root.to_str().expect("UTF-8 linked root"), + "main", + ], + ); + if let Some(source) = linked_only_source { + std::fs::write(linked_root.join("src/only_linked.rs"), source).expect("linked source"); + git(&linked_root, &["add", "src/only_linked.rs"]); + git(&linked_root, &["commit", "-qm", "linked-only file"]); + } + let project_root = project_root.canonicalize().expect("canonical project root"); + let linked_root = linked_root.canonicalize().expect("canonical linked root"); + tracedecay_runtime_core::storage::pin_fixture_repository_identity( + &linked_root, + project_id.as_str(), + ) + .expect("linked worktree enrollment"); + + let code_index_root = root.join("code-index-store"); + let identity = profile_identity::load_or_create(&profile_root).expect("profile identity"); + let _database_scope = tracedecay_runtime_core::db::enter_daemon_database_scope( + &profile_root, + 44, + "linked worktree read bundles", + ) + .expect("daemon database scope"); + let registry = DaemonSessionRuntimeRegistryV1::open(identity) + .await + .expect("session runtime registry"); + let project_database = registry + .project_memory( + project_id.clone(), + [project_root.clone(), linked_root.clone()], + ) + .await + .expect("project graph database"); + + let mut scopes = Vec::with_capacity(2); + for worktree_root in [&project_root, &linked_root] { + let scope = scoped_code_index_store_root(&code_index_root, worktree_root); + let mut scheduler = CodeIndexWorktreeSchedulerV1::open( + project_id.clone(), + worktree_root, + scope.clone(), + Arc::new(SharedCodeIndexBytePoolV1::default()), + ) + .expect("open worktree scheduler"); + scheduler.reconcile_now().expect("seal the generation"); + let latest = scheduler.latest_complete().expect("complete generation"); + let worktree_id = scheduler.identity().worktree_id().clone(); + drop(scheduler); + let snapshot = latest.generation().snapshot(); + let pointer: DurablePublicationPointerV1 = serde_json::from_slice( + &std::fs::read(scope.join("active-code-generation-v1.json")) + .expect("active generation pointer"), + ) + .expect("decode active generation pointer"); + let generations_root = scope.join("code-generations-v1"); + let sealed = SealedGraphStateDigest::try_from(pointer.state_digest.clone()) + .expect("sealed state digest"); + let runtime = registry + .retain_code_graph_runtime( + project_id.clone(), + snapshot.repository.clone(), + worktree_id, + snapshot.reference.clone(), + latest.generation().manifest().generation_id.clone(), + Arc::clone(&project_database), + CodeGraphReplayBindingV1 { + generations_root: generations_root.clone(), + sealed_state_digest: sealed.clone(), + }, + None, + ) + .await + .expect("retain code graph runtime"); + runtime + .publish_verified_snapshot(latest.generation(), Arc::new(AtomicBool::new(false))) + .expect("seal the code graph"); + let manifest = generations_root.join(format!( + "read-bundle-{}.json", + sha256_hex_suffix(&pointer.state_digest).expect("sha256 state digest") + )); + let artifacts = sealed_read_bundle_manifest_artifact_digests(&manifest) + .expect("read bundle manifest") + .expect("the scope sealed a read bundle"); + let loaded = runtime + .load_sealed_read_bundle_catalog(&Arc::new(AtomicBool::new(false))) + .expect("load the bundle catalog"); + let SealedReadBundleArtifactStateV1::Loaded { bytes, .. } = loaded else { + panic!("a freshly sealed bundle must load, got {loaded:?}"); + }; + scopes.push(LinkedBundleScopeV1 { + scope, + sealed, + artifacts, + catalog: bytes, + }); + } + let linked = scopes.pop().expect("linked scope"); + let first = scopes.pop().expect("primary scope"); + LinkedBundlesV1 { + _temporary: temporary, + first, + linked, + } +} + +/// Every catalog artifact in the shared root of `scope`'s project, by digest. +fn shared_artifacts(scope: &Path) -> BTreeSet { + let Ok(entries) = std::fs::read_dir(code_generation_segments_root(scope)) else { + return BTreeSet::new(); + }; + entries + .map(|entry| entry.expect("shared artifact entry")) + .filter_map(|entry| sealed_read_bundle_artifact_file_digest(entry.file_name().to_str()?)) + .collect() +} + +fn contains(bytes: &[u8], needle: &[u8]) -> bool { + bytes.windows(needle.len()).any(|window| window == needle) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn linked_worktrees_with_identical_trees_share_one_read_bundle_artifact() { + let bundles = publish_linked_worktree_bundles("identical", None).await; + assert_ne!( + bundles.first.sealed, bundles.linked.sealed, + "each worktree seals its own generation" + ); + assert_eq!(bundles.first.artifacts, bundles.linked.artifacts); + assert_eq!(bundles.first.catalog, bundles.linked.catalog); + assert_eq!( + shared_artifacts(&bundles.first.scope), + bundles.first.artifacts.iter().cloned().collect(), + "the project stores the catalog both worktrees name exactly once" + ); + for scope in [&bundles.first.scope, &bundles.linked.scope] { + let local = std::fs::read_dir(scope.join("code-generations-v1")) + .expect("scope generations") + .map(|entry| entry.expect("generation entry").file_name()) + .filter(|name| name.to_string_lossy().ends_with(".bin")) + .collect::>(); + assert!( + local.is_empty(), + "a worktree scope keeps no artifact bytes: {local:?}" + ); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_linked_worktree_with_a_divergent_file_never_serves_its_catalog_to_the_other() { + let marker = "only_in_linked_worktree"; + let bundles = publish_linked_worktree_bundles( + "divergent", + Some(&format!("pub fn {marker}() -> usize {{ 7 }}\n")), + ) + .await; + assert_ne!(bundles.first.artifacts, bundles.linked.artifacts); + assert!( + contains(&bundles.linked.catalog, marker.as_bytes()), + "the linked worktree serves its own divergent symbol" + ); + assert!( + !contains(&bundles.first.catalog, marker.as_bytes()), + "the primary worktree must never be answered with the linked worktree's catalog" + ); + let mut both = bundles + .first + .artifacts + .iter() + .cloned() + .collect::>(); + both.extend(bundles.linked.artifacts.iter().cloned()); + assert_eq!(shared_artifacts(&bundles.first.scope), both); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn retiring_one_linked_worktree_keeps_the_read_bundle_artifact_its_sibling_names() { + let bundles = publish_linked_worktree_bundles("retire", None).await; + let shared = bundles + .first + .artifacts + .iter() + .cloned() + .collect::>(); + let sweep = |scope: &Path| { + run_code_generation_retention( + scope, + &BTreeSet::new(), + CodeGenerationRetentionModeV1::Apply, + UtcMicros(1), + None, + ) + .expect("project segment sweep") + }; + + // The linked worktree goes away: its bundle retires with its generation, + // then its whole scope is removed. + retire_sealed_read_bundle( + &bundles.linked.scope.join("code-generations-v1"), + &bundles.linked.sealed, + ) + .expect("retire the linked bundle"); + std::fs::remove_dir_all(&bundles.linked.scope).expect("remove the linked scope"); + sweep(&bundles.first.scope); + assert_eq!( + shared_artifacts(&bundles.first.scope), + shared, + "the primary worktree still names the shared catalog" + ); + let digest = shared.first().expect("one shared artifact"); + let path = code_generation_segments_root(&bundles.first.scope).join(format!( + "read-bundle-artifact-{}.bin", + sha256_hex_suffix(digest).expect("sha256 artifact digest") + )); + let bytes = std::fs::read(&path).expect("shared catalog artifact"); + assert_eq!(bytes, bundles.first.catalog); + assert_eq!( + format!("sha256:{}", hex::encode(Sha256::digest(&bytes))), + *digest + ); + + // Once no manifest names it, the sweep collects it. + retire_sealed_read_bundle( + &bundles.first.scope.join("code-generations-v1"), + &bundles.first.sealed, + ) + .expect("retire the primary bundle"); + sweep(&bundles.first.scope); + assert!(shared_artifacts(&bundles.first.scope).is_empty()); +} diff --git a/crates/tracedecay-store-runtime/src/session_registry/code_graph/sealed_publication_tests.rs b/crates/tracedecay-store-runtime/src/session_registry/code_graph/sealed_publication_tests.rs index 1ad88b89ea..89a9002531 100644 --- a/crates/tracedecay-store-runtime/src/session_registry/code_graph/sealed_publication_tests.rs +++ b/crates/tracedecay-store-runtime/src/session_registry/code_graph/sealed_publication_tests.rs @@ -17,7 +17,9 @@ use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; use std::sync::{Arc, Barrier}; use std::time::{Duration, Instant}; -use tracedecay_code_index_retention::code_index_generations::DurablePublicationPointerV1; +use tracedecay_code_index_retention::code_index_generations::{ + DurablePublicationPointerV1, code_generation_segments_root, +}; use tracedecay_domain::{ CodeGenerationId, ProjectId, RefId, RepositoryId, WorktreeId, canonical_sha256, sha256_hex_suffix, @@ -60,6 +62,21 @@ fn git(root: &Path, args: &[&str]) { ); } +/// The shared artifact files a scope's bundle manifest names. +fn read_bundle_artifact_paths(scope: &Path, manifest: &Path) -> Vec { + tracedecay_graph_db::sealed_read_bundle_manifest_artifact_digests(manifest) + .expect("read bundle manifest") + .expect("a bundle manifest path") + .iter() + .map(|digest| { + code_generation_segments_root(scope).join(format!( + "read-bundle-artifact-{}.bin", + sha256_hex_suffix(digest).expect("sha256 artifact digest") + )) + }) + .collect() +} + fn with_publication_context( label: &str, operation: impl FnOnce(&GraphPublicationOperationContextV1<'_>) -> T, @@ -351,7 +368,7 @@ async fn unreadable_pending_replay_is_discarded_before_fresh_publication() { .join("../tracedecay-code-index/tests/fixtures/partitioned_pre_paging"); let historical_digest = "6fece830a4b12904018853a467e404edc60ea76e2cab48d4645fbbb4132bd6af"; let generations_root = scoped_store.join("code-generations-v1"); - let segments_root = scoped_store.join("code-generation-segments-v1"); + let segments_root = code_generation_segments_root(&scoped_store); std::fs::create_dir_all(&segments_root).expect("historical segment root"); std::fs::copy( historical_fixture.join("manifest.json"), @@ -940,8 +957,6 @@ async fn sealed_read_bundle_serves_catalog_without_warm_and_degrades_typed() { .expect("sha256 state digest") .to_owned(); let bundle_manifest_path = generations_root.join(format!("read-bundle-{digest_hex}.json")); - let bundle_catalog_path = - generations_root.join(format!("read-bundle-{digest_hex}.interactive-catalog.bin")); let identity = profile_identity::load_or_create(&profile_root).expect("profile identity"); let _database_scope = tracedecay_runtime_core::db::enter_daemon_database_scope( @@ -987,6 +1002,9 @@ async fn sealed_read_bundle_serves_catalog_without_warm_and_degrades_typed() { bundle_manifest_path.is_file(), "sealing must write the read bundle manifest" ); + let bundle_catalog_path = read_bundle_artifact_paths(&scoped_store, &bundle_manifest_path) + .pop() + .expect("the bundle names its catalog artifact"); assert!( bundle_catalog_path.is_file(), "sealing must write the interactive-catalog artifact" @@ -1035,7 +1053,10 @@ async fn sealed_read_bundle_serves_catalog_without_warm_and_degrades_typed() { tracedecay_graph_db::retire_sealed_read_bundle(&generations_root, &sealed_state_digest) .expect("retire the read bundle"); assert!(!bundle_manifest_path.exists()); - assert!(!bundle_catalog_path.exists()); + assert!( + bundle_catalog_path.exists(), + "a shared artifact outlives its bundle until the project's sweep collects it" + ); let absent = runtime .load_sealed_read_bundle_catalog(&Arc::new(AtomicBool::new(false))) .expect("absent load is a typed state, not an error"); @@ -2130,7 +2151,7 @@ async fn concurrent_worktree_scopes_publish_with_one_corpus_build_and_bounded_rs sealed_source: scoped_store .join("code-generations-v1") .join(format!("generation-{digest}.json")), - segments_source_root: scoped_store.join("code-generation-segments-v1"), + segments_source_root: code_generation_segments_root(&scoped_store), sealed_state_digest: tracedecay_graph_db::SealedGraphStateDigest::try_from( pointer.state_digest, ) diff --git a/crates/tracedecay-store-runtime/src/session_registry/code_graph_manifest.rs b/crates/tracedecay-store-runtime/src/session_registry/code_graph_manifest.rs index 0147e688c4..22e6d8092b 100644 --- a/crates/tracedecay-store-runtime/src/session_registry/code_graph_manifest.rs +++ b/crates/tracedecay-store-runtime/src/session_registry/code_graph_manifest.rs @@ -6,11 +6,9 @@ use std::sync::{Arc, RwLock}; use sha2::{Digest, Sha256}; use tracedecay_code_index::graph_projection::CodeGraphProjectionError; -use tracedecay_code_index::production::{ - CodeIndexProductionErrorV1, UninterruptibleCodeIndexControlV1, -}; +use tracedecay_code_index::production::CodeIndexProductionErrorV1; use tracedecay_code_index_retention::code_index_generations::{ - CodeGenerationStoreLockV1, GRAPH_REPLAY_POOL_ACQUIRE_POLL, + CodeGenerationStoreLockV1, GRAPH_REPLAY_POOL_ACQUIRE_POLL, code_generation_segments_root, try_acquire_code_generation_store_lock, }; use tracedecay_domain::canonical_text::encode_lowercase_hex; @@ -435,26 +433,14 @@ fn decode_verified_seal_with_bundle_barrier( } })?; (check)()?; - let decoded = tracedecay_code_index::production::CodeIndexPublishedGenerationV1::decode_sealed_seek_reader( - &mut file, - admitted_len, - Some(&expected_digest), - &UninterruptibleCodeIndexControlV1, - ); #[cfg(feature = "hotpath")] hotpath::gauge!("session_registry.seal.decode.bytes_total").inc(admitted_len); - let monolithic = decoded - .map_err(|error| classify_sealed_generation_decode_error(error, &expected_digest))?; let mut lifetime_lock = Some(lifetime_lock); - let generation = if let Some(generation) = monolithic { - generation - } else { - file.seek(SeekFrom::Start(0)) - .map_err(|error| GraphDbError::Corrupt { - message: format!("sealed generation manifest seek failed: {error}"), - })?; + let generation = { let mut manifest = Vec::new(); - file.read_to_end(&mut manifest) + file.by_ref() + .take(admitted_len) + .read_to_end(&mut manifest) .map_err(|error| GraphDbError::Corrupt { message: format!("sealed generation manifest read failed: {error}"), })?; @@ -520,11 +506,7 @@ fn decode_verified_seal_with_bundle_barrier( if let Some(interruption) = interruption { return Err(interruption); } - decoded - .map_err(|error| classify_sealed_generation_decode_error(error, &expected_digest))? - .ok_or_else(|| GraphDbError::Corrupt { - message: "sealed code generation format revision is incompatible".to_owned(), - })? + decoded.map_err(|error| classify_sealed_generation_decode_error(error, &expected_digest))? }; (check)()?; let final_file_metadata = file.metadata().map_err(|error| GraphDbError::Corrupt { @@ -893,10 +875,11 @@ pub(super) fn verify_sealed_generation_source_from_roots( let digest = sha256_hex_suffix(sealed_state_digest.as_str()) .ok_or_else(|| GraphDbError::invalid("sealed state digest is not sha256"))?; let seal_file = format!("generation-{digest}.json"); - let segments_root = generations_root - .parent() - .ok_or_else(|| GraphDbError::invalid("generation root has no store parent"))? - .join("code-generation-segments-v1"); + let segments_root = code_generation_segments_root( + generations_root + .parent() + .ok_or_else(|| GraphDbError::invalid("generation root has no store parent"))?, + ); with_verified_seal_from_roots( &generations_root.join(&seal_file), &replay_root.join(&seal_file), @@ -1476,7 +1459,7 @@ impl GraphGenerationManifestProvider for DaemonCodeGraphManifestProviderV1 { route .generations_root .parent() - .map(|root| root.join("code-generation-segments-v1")) + .map(code_generation_segments_root) .ok_or_else(|| { GraphDbError::invalid( "canonical generation root has no store parent", @@ -1508,7 +1491,7 @@ impl GraphGenerationManifestProvider for DaemonCodeGraphManifestProviderV1 { } match decode_verified_seal( &canonical, - &store_root.join("code-generation-segments-v1"), + &code_generation_segments_root(store_root), digest, check, lock, @@ -1612,6 +1595,7 @@ fn classify_sealed_projection_build_error(error: CodeGraphProjectionError) -> Gr #[cfg(test)] mod tests { use std::collections::BTreeSet; + use std::fmt::Write as _; use std::io::{Seek, SeekFrom, Write}; use std::path::Path; use std::process::Command; @@ -1622,7 +1606,8 @@ mod tests { use tempfile::TempDir; use tracedecay_code_index_retention::code_index_generations::{ CodeGenerationRetentionModeV1, DurablePublicationPointerV1, - acquire_code_generation_store_lock, run_code_generation_retention, + acquire_code_generation_store_lock, code_generation_segments_root, + run_code_generation_retention, }; use tracedecay_domain::{ CodeGenerationId, ProjectId, RepositoryId, UtcMicros, sha256_hex_suffix, @@ -1966,6 +1951,7 @@ mod tests { struct PartitionedSealFixture { _temporary: TempDir, pool_manifest: std::path::PathBuf, + scope_root: std::path::PathBuf, segments_root: std::path::PathBuf, digest: String, project: ProjectId, @@ -1973,9 +1959,40 @@ mod tests { generation: CodeGenerationId, } - fn partitioned_seal_fixture(label: &str) -> PartitionedSealFixture { - use std::fmt::Write as _; + /// 1,600 functions named `{prefix}_{index}` whose bodies apply + /// `operator`. A clean generation's evidence is implied by its own + /// symbols and chunks and fits one page; a successor that changes every + /// body keeps one whole lineage row per function, which spans several. + fn multi_page_evidence_source(prefix: &str, operator: char) -> String { + let mut source = String::new(); + for index in 0..1_600 { + writeln!( + source, + "pub fn {prefix}_{index}(value: usize) -> usize {{ value {operator} {index} }}" + ) + .unwrap(); + } + source + } + + /// Publish a successor of the fixture's clean generation that changes + /// every function body, so the active generation's evidence spans pages. + fn publish_multi_page_evidence( + project_root: &Path, + prefix: &str, + scheduler: &mut CodeIndexWorktreeSchedulerV1, + ) { + scheduler.reconcile_now().unwrap(); + std::fs::write( + project_root.join("src/lib.rs"), + multi_page_evidence_source(prefix, '*'), + ) + .unwrap(); + git(project_root, &["commit", "-qam", "change every body"]); + scheduler.reconcile_now().unwrap(); + } + fn partitioned_seal_fixture(label: &str) -> PartitionedSealFixture { let temporary = TempDir::new().unwrap(); let root = temporary.path().canonicalize().unwrap(); let project_root = root.join("project"); @@ -1986,15 +2003,11 @@ mod tests { &project_root, &["config", "user.email", "tracedecay@example.invalid"], ); - let mut source = String::new(); - for index in 0..1_600 { - writeln!( - source, - "pub fn partitioned_fixture_{index}(value: usize) -> usize {{ value + {index} }}" - ) - .unwrap(); - } - std::fs::write(project_root.join("src/lib.rs"), source).unwrap(); + std::fs::write( + project_root.join("src/lib.rs"), + multi_page_evidence_source("partitioned_fixture", '+'), + ) + .unwrap(); git(&project_root, &["add", "."]); git(&project_root, &["commit", "-qm", "partitioned fixture"]); let project_id = ProjectId::new(format!("project.manifest-{label}")).unwrap(); @@ -2013,7 +2026,7 @@ mod tests { Arc::new(SharedCodeIndexBytePoolV1::default()), ) .unwrap(); - scheduler.reconcile_now().unwrap(); + publish_multi_page_evidence(&project_root, "partitioned_fixture", &mut scheduler); let latest = scheduler.latest_complete().unwrap(); let repository = latest.generation().snapshot().repository.clone(); let generation = latest.generation().manifest().generation_id.clone(); @@ -2026,7 +2039,7 @@ mod tests { let canonical_manifest = scoped_store .join("code-generations-v1") .join(pointer.generation_file); - let segments_root = scoped_store.join("code-generation-segments-v1"); + let segments_root = code_generation_segments_root(&scoped_store); let manifest: serde_json::Value = serde_json::from_slice(&std::fs::read(&canonical_manifest).unwrap()).unwrap(); assert!( @@ -2048,6 +2061,7 @@ mod tests { PartitionedSealFixture { _temporary: temporary, pool_manifest, + scope_root: scoped_store, segments_root, digest, project: project_id, @@ -2083,7 +2097,7 @@ mod tests { replay_root.clone(), ) .unwrap(); - let store = fixture.segments_root.parent().unwrap(); + let store = fixture.scope_root.as_path(); assert!(absent_store.as_path() < store); let route = provider .bind( @@ -2296,8 +2310,6 @@ mod tests { #[test] fn partitioned_replay_decode_pins_evidence_across_manifest_retirement() { - use std::fmt::Write as _; - let temporary = TempDir::new().unwrap(); let root = temporary.path().canonicalize().unwrap(); let project_root = root.join("project"); @@ -2308,15 +2320,11 @@ mod tests { &project_root, &["config", "user.email", "tracedecay@example.invalid"], ); - let mut source = String::new(); - for index in 0..1_600 { - writeln!( - source, - "pub fn pinned_evidence_{index}(value: usize) -> usize {{ value + {index} }}" - ) - .unwrap(); - } - std::fs::write(project_root.join("src/lib.rs"), source).unwrap(); + std::fs::write( + project_root.join("src/lib.rs"), + multi_page_evidence_source("pinned_evidence", '+'), + ) + .unwrap(); git(&project_root, &["add", "."]); git(&project_root, &["commit", "-qm", "pinned evidence fixture"]); let project_id = ProjectId::new("project.manifest-pinned-evidence").unwrap(); @@ -2335,7 +2343,7 @@ mod tests { Arc::new(SharedCodeIndexBytePoolV1::default()), ) .unwrap(); - scheduler.reconcile_now().unwrap(); + publish_multi_page_evidence(&project_root, "pinned_evidence", &mut scheduler); drop(scheduler); let pointer_path = scoped_store.join("active-code-generation-v1.json"); @@ -2353,12 +2361,15 @@ mod tests { .len() > 1 ); + let clean_parent = manifest["generation"]["manifest"]["parent_generation"] + .as_str() + .unwrap() + .to_owned(); let evidence_digest = manifest["generation"]["generation_evidence"]["segment_digest"] .as_str() .unwrap(); let evidence_digest = sha256_hex_suffix(evidence_digest).unwrap(); - let evidence_path = scoped_store - .join("code-generation-segments-v1") + let evidence_path = code_generation_segments_root(&scoped_store) .join(format!("segment-{evidence_digest}.json")); let replay_root = root.join("replay-pool"); @@ -2370,7 +2381,7 @@ mod tests { } std::fs::remove_file(pointer_path).unwrap(); - let segments_root = scoped_store.join("code-generation-segments-v1"); + let segments_root = code_generation_segments_root(&scoped_store); let decoded = decode_verified_seal_with_bundle_barrier( &staged_manifest, std::slice::from_ref(&segments_root), @@ -2387,7 +2398,13 @@ mod tests { Some(&replay_root), ) .unwrap(); - assert!(report.deleted_generations.is_empty()); + assert!( + report + .deleted_generations + .iter() + .all(|deleted| deleted.generation_id.as_str() == clean_parent), + "retention may retire only the fixture's clean parent generation" + ); assert!( !evidence_path.exists(), "retention must remove the pack pathname while decode owns its lifetime" diff --git a/crates/tracedecay-store-runtime/src/session_registry/graph_shutdown_contract_tests.rs b/crates/tracedecay-store-runtime/src/session_registry/graph_shutdown_contract_tests.rs index 58d1482198..37e3834430 100644 --- a/crates/tracedecay-store-runtime/src/session_registry/graph_shutdown_contract_tests.rs +++ b/crates/tracedecay-store-runtime/src/session_registry/graph_shutdown_contract_tests.rs @@ -169,6 +169,75 @@ async fn healthy_shutdown_joins_workers_then_closes_every_retained_graph_without .expect("shutdown close is idempotent after the drain"); } +fn wal_bytes(database: &Path) -> u64 { + let mut wal = database.as_os_str().to_owned(); + wal.push("-wal"); + std::fs::metadata(PathBuf::from(wal)).map_or(0, |metadata| metadata.len()) +} + +#[tokio::test] +async fn terminal_shutdown_truncates_every_released_session_store_wal() { + let temp = TempDir::new().expect("shutdown wal fixture root"); + let profile_root = temp.path().join("profile"); + let project_id = ProjectId::new("project.shutdown-wal-truncate").expect("project id"); + let project_root = enrolled_root(temp.path(), &project_id); + let _database_scope = tracedecay_runtime_core::db::enter_daemon_database_scope( + &profile_root, + 67, + "shutdown wal truncate", + ) + .expect("daemon database scope"); + let identity = profile_identity::load_or_create(&profile_root).expect("profile identity"); + let registry = DaemonSessionRuntimeRegistryV1::open(identity) + .await + .expect("daemon registry"); + let profile_sessions = registry + .profile_sessions() + .await + .expect("profile sessions authority"); + let project_sessions = registry + .project_sessions(project_id, [project_root]) + .await + .expect("project sessions authority"); + let stores = [ + profile_sessions.db_path().to_path_buf(), + project_sessions.db_path().to_path_buf(), + ]; + for store in &stores { + assert!( + wal_bytes(store) > 0, + "schema install leaves frames in {}", + store.display() + ); + } + + // The daemon's terminal owner order: join terminal and reconciliation + // tasks, then drain the owners and close. + registry + .shutdown_terminal_tasks() + .await + .expect("terminal tasks join"); + registry.cancel_memory_graph_reconciliation_tasks(); + registry + .shutdown_memory_graph_reconciliation_tasks() + .await + .expect("reconciliation workers join"); + drop((profile_sessions, project_sessions)); + registry + .close_retained_graph_runtimes_for_shutdown() + .await + .expect("terminal shutdown closes released runtimes"); + + for store in &stores { + assert_eq!( + wal_bytes(store), + 0, + "a released session store keeps no WAL after shutdown: {}", + store.display() + ); + } +} + #[tokio::test] async fn terminal_shutdown_refuses_an_in_flight_project_owner_transition() { let temp = TempDir::new().expect("shutdown transition fixture root"); diff --git a/crates/tracedecay-store-runtime/src/session_registry/maintenance.rs b/crates/tracedecay-store-runtime/src/session_registry/maintenance.rs index 8b4d7f450e..ca3f614ff3 100644 --- a/crates/tracedecay-store-runtime/src/session_registry/maintenance.rs +++ b/crates/tracedecay-store-runtime/src/session_registry/maintenance.rs @@ -78,21 +78,18 @@ enum SchemaConvergenceTarget { database: RegisteredGlobalDbLeaseV1, convergence: RegisteredSchemaConvergence, }, - RuntimeLedger(Database), } impl SchemaConvergenceTarget { fn binding(&self) -> &StoreRuntimeBindingV1 { match self { Self::Registered { database, .. } => database.binding(), - Self::RuntimeLedger(database) => database.registered_binding(), } } fn db_path(&self) -> &Path { match self { Self::Registered { database, .. } => database.db_path(), - Self::RuntimeLedger(database) => database.canonical_database_path(), } } @@ -102,16 +99,12 @@ impl SchemaConvergenceTarget { database, convergence, } => database.converge_schema(*convergence).await, - Self::RuntimeLedger(database) => { - tracedecay_global_db::schema_stages::converge_runtime_writer_ledger(database).await - } } } async fn release_connection_memory(&self) -> Result<()> { match self { Self::Registered { database, .. } => database.release_connection_memory().await, - Self::RuntimeLedger(database) => database.release_connection_memory().await, } } } @@ -279,20 +272,9 @@ impl RegisteredSchemaConvergenceMaintenance { }); } - pub(super) fn schedule_runtime_ledger(&self, database: Database) { - self.schedule_target(SchemaConvergenceTarget::RuntimeLedger(database)); - } - fn schedule_target(&self, target: SchemaConvergenceTarget) { let shard_id = target.binding().shard_id.clone(); - let stage = match &target { - SchemaConvergenceTarget::Registered { .. } => { - SchemaConvergenceStageV1::RegisteredSchema - } - SchemaConvergenceTarget::RuntimeLedger(_) => { - SchemaConvergenceStageV1::RuntimeWriterLedger - } - }; + let stage = SchemaConvergenceStageV1::RegisteredSchema; let mut tasks = self .tasks .lock() diff --git a/crates/tracedecay-store-runtime/src/session_registry/mounts.rs b/crates/tracedecay-store-runtime/src/session_registry/mounts.rs index f27f51261c..788827b4b4 100644 --- a/crates/tracedecay-store-runtime/src/session_registry/mounts.rs +++ b/crates/tracedecay-store-runtime/src/session_registry/mounts.rs @@ -576,10 +576,6 @@ impl DaemonSessionRuntimeRegistryV1 { label = "daemon.session_registry.mount.schema_migrate" ) .await?; - if self.long_lived_session_maintenance { - self.registered_schema_convergence - .schedule_runtime_ledger(database.clone()); - } let database_issuer = owner.weak_lease_issuer(); let graph = Arc::new(std::sync::Mutex::new( MemoryGraphAttachmentStateV1::Warming { @@ -846,10 +842,6 @@ impl DaemonSessionRuntimeRegistryV1 { format!("{error:?}"), ) })?; - if self.long_lived_session_maintenance { - self.registered_schema_convergence - .schedule_runtime_ledger(database.clone()); - } admission.publish(owner)?; (database, true, existed) } @@ -1032,6 +1024,11 @@ impl DaemonSessionRuntimeRegistryV1 { /// without a structural Conflict. Callers must have joined the /// reconciliation workers first; a graph client lease still held by a /// live consumer surfaces as a typed Conflict, not a hang. + /// + /// The dropped owners leave their `SQLite` runtimes mounted, so the + /// profile database owner and pin are released too and every store + /// runtime no longer held is then closed: its writer runs the shutdown + /// TRUNCATE checkpoint instead of leaving a retained WAL. #[hotpath::skip] pub async fn close_retained_graph_runtimes_for_shutdown(&self) -> Result<()> { let identities = self.drain_retained_graph_owners_for_shutdown()?; @@ -1048,6 +1045,21 @@ impl DaemonSessionRuntimeRegistryV1 { first_error = Some(error); } } + drop( + self.profile_database + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take(), + ); + drop(self.profile_pin.lock().await.take()); + if let Err(failure) = self.registry.close_idle_for_shutdown().await + && first_error.is_none() + { + first_error = Some(session_registry_error( + "close idle store runtimes for shutdown", + format!("{failure:?}"), + )); + } match first_error { Some(error) => Err(error), None => Ok(()), diff --git a/crates/tracedecay-store-runtime/src/session_registry/profile_memory.rs b/crates/tracedecay-store-runtime/src/session_registry/profile_memory.rs deleted file mode 100644 index 6f25d70072..0000000000 --- a/crates/tracedecay-store-runtime/src/session_registry/profile_memory.rs +++ /dev/null @@ -1,10 +0,0 @@ -use super::DaemonSessionRuntimeRegistryV1; -use tracedecay_domain::errors::Result; -use tracedecay_runtime_core::db::Database; - -pub async fn open_user_memory_db(registry: &DaemonSessionRuntimeRegistryV1) -> Result { - registry - .profile_memory() - .await - .map(|database| database.as_ref().clone()) -} diff --git a/crates/tracedecay-store/Cargo.toml b/crates/tracedecay-store/Cargo.toml index 7c989e6424..c2a3d4ee0b 100644 --- a/crates/tracedecay-store/Cargo.toml +++ b/crates/tracedecay-store/Cargo.toml @@ -9,6 +9,7 @@ repository = "https://github.com/ScriptedAlchemy/tracedecay" [dependencies] hotpath.workspace = true +schemars.workspace = true serde = { version = "1", features = ["derive"] } serde_json = "1" sha2 = "0.11" diff --git a/crates/tracedecay-store/src/canonical_projection.rs b/crates/tracedecay-store/src/canonical_projection.rs index 9751240815..3b644bbef9 100644 --- a/crates/tracedecay-store/src/canonical_projection.rs +++ b/crates/tracedecay-store/src/canonical_projection.rs @@ -57,13 +57,14 @@ pub fn derive_canonical_projection( derive_canonical_projection_for(observation, CanonicalRendering::Current) } -/// Whether `stored` is the message row a shipped release wrote for `observation`. +/// Whether the stored row is the message row a shipped release wrote for +/// `observation`, as judged by `stores` against each released message. /// /// A current-provenance row that still holds that rendering is an interrupted /// write. Any other body, including a derivation that does not complete, is not. pub fn stored_message_is_shipped_release_rendering( observation: &DurableObservationV1, - stored: &SessionMessageRecord, + stores: impl Fn(&SessionMessageRecord) -> bool, ) -> bool { let Ok(released) = derive_canonical_projection_for(observation, CanonicalRendering::ShippedRelease) @@ -72,7 +73,7 @@ pub fn stored_message_is_shipped_release_rendering( }; released .messages() - .any(|projection| projection.message() == stored) + .any(|projection| stores(projection.message())) } #[hotpath::measure(label = "store.projection.derive_canonical")] @@ -135,7 +136,17 @@ fn derive_canonical_projection_for( .and_then(|fields| fields.project_path.clone()) .unwrap_or(fallback_project_path); let session_metadata = canonical_session_metadata_map(&provider, session_fields.as_ref()); - let session_metadata_json = serialize_metadata_map(&session_metadata)?; + // The edit rollup is a session-level fact: it lands on the session row + // and stays out of the per-message metadata copy below. + let mut session_row_metadata = session_metadata.clone(); + let edited_files = canonical_edited_files(&envelope); + if !edited_files.is_empty() { + session_row_metadata.insert( + EDITED_FILES_KEY.to_owned(), + serde_json::Value::Array(edited_files), + ); + } + let session_metadata_json = serialize_metadata_map(&session_row_metadata)?; let session = SessionRecord { provider: provider.clone(), session_id: session_id.clone(), @@ -165,7 +176,13 @@ fn derive_canonical_projection_for( .relations() .agent_id() .map(|id| id.as_str().to_owned()), - parent_tool_use_id: None, + // A spawning call names a call in the parent session, so it is + // recorded only alongside that parent. + parent_tool_use_id: envelope + .relations() + .parent_session_id() + .and(envelope.relations().parent_tool_use_id()) + .map(|id| id.as_str().to_owned()), }; let ordinal = envelope .evidence() @@ -204,7 +221,7 @@ fn derive_canonical_projection_for( timestamp, ordinal, source_offset, - &metadata_json, + metadata_json.as_deref(), projected, ), )); @@ -221,7 +238,7 @@ fn derive_canonical_projection_for( derived.fields.timestamp.or(timestamp), ordinal, source_offset, - &metadata_json, + metadata_json.as_deref(), derived.fields, ), )); @@ -241,7 +258,7 @@ fn canonical_session_message_record( timestamp: Option, ordinal: i64, source_offset: Option, - metadata_json: &str, + metadata_json: Option<&str>, fields: CanonicalMessageFields, ) -> SessionMessageRecord { SessionMessageRecord { @@ -257,7 +274,7 @@ fn canonical_session_message_record( tool_names: fields.tool_names, source_path: None, source_offset, - metadata_json: Some(metadata_json.to_owned()), + metadata_json: metadata_json.map(str::to_owned), } } @@ -350,6 +367,76 @@ fn canonical_session_metadata_map( metadata } +/// `sessions.metadata_json` key of the provider-native edited-file rollup: +/// `[{path, edited_at_micros?, change_type?, hunks?}]`, one entry per file-edit +/// fact. The store reconciles the arrays of a session's records by union. +pub const EDITED_FILES_KEY: &str = "edited_files"; + +/// Message `metadata_json` key carrying the host's tool-use identifier for the +/// record's tool invocation (see [`host_tool_use_id`]). Absent when the host +/// recorded none. +pub const TOOL_USE_ID_KEY: &str = "tool_use_id"; + +/// One rollup entry per `Git { FileEdit }` fact that names its path. The time, +/// change type, and hunk count are copied only when the capture recorded them +/// from the host (`edited_at_micros`, `change_type`, `hunks` in the fact +/// content); nothing is derived from session bounds or neighbouring records. +fn canonical_edited_files(envelope: &CanonicalObservationEnvelopeV1) -> Vec { + envelope + .facts() + .iter() + .filter_map(|fact| match fact { + CanonicalObservationFactV1::Git { + evidence_kind: CanonicalGitEvidenceKindV1::FileEdit, + reference: Some(path), + content, + } if !path.is_empty() => { + let mut entry = serde_json::Map::new(); + entry.insert("path".to_owned(), serde_json::Value::String(path.clone())); + let content = content.as_ref().and_then(serde_json::Value::as_object); + for key in ["edited_at_micros", "change_type", "hunks"] { + if let Some(value) = content.and_then(|content| content.get(key)) { + entry.insert(key.to_owned(), value.clone()); + } + } + Some(serde_json::Value::Object(entry)) + } + _ => None, + }) + .collect() +} + +/// The host's own identifier of the record's tool invocation, for a fork or +/// tool result to bind to. Every capture falls back to the record's stable id +/// (or `{stable_id}:tool:{index}`) when the host wrote none, so an invocation +/// id rooted in the stable id is the capture's, not the host's, and is never +/// served. A subagent dispatch wins over other invocations on the same record +/// because that is the call a child session's `parent_tool_use_id` names. +fn host_tool_use_id(envelope: &CanonicalObservationEnvelopeV1) -> Option<&str> { + let stable_record_id = envelope.stable_record_id().as_str(); + let synthesized_prefix = format!("{stable_record_id}:tool:"); + let mut invocations = envelope.facts().iter().filter_map(|fact| match fact { + CanonicalObservationFactV1::ToolInvocation { + invocation_id, + name, + .. + } if invocation_id.as_str() != stable_record_id + && !invocation_id.as_str().starts_with(&synthesized_prefix) => + { + Some((invocation_id.as_str(), name.as_str())) + } + _ => None, + }); + let first = invocations.next()?; + Some( + std::iter::once(first) + .chain(invocations) + .find(|(_, name)| is_subagent_dispatch_tool(name)) + .unwrap_or(first) + .0, + ) +} + fn serialize_metadata_map( metadata: &serde_json::Map, ) -> ProjectionStoreResult> { @@ -374,30 +461,63 @@ fn canonical_session_metadata( fn canonical_message_metadata( envelope: &CanonicalObservationEnvelopeV1, session_metadata: Option<&serde_json::Map>, -) -> ProjectionStoreResult { +) -> ProjectionStoreResult> { canonical_message_metadata_for(CanonicalRendering::Current, envelope, session_metadata) } +/// Message metadata holds only what the envelope does not: session, tool, and +/// provider-semantics keys. The envelope itself stays in its `observations` +/// row; [`message_metadata_with_envelope`] merges the two for readers that +/// render the full record. fn canonical_message_metadata_for( rendering: CanonicalRendering, envelope: &CanonicalObservationEnvelopeV1, session_metadata: Option<&serde_json::Map>, -) -> ProjectionStoreResult { - let serde_json::Value::Object(mut metadata) = serde_json::to_value(envelope) - .map_err(|_| ProjectionStoreError::Contract(ObservationContractError::CanonicalEncoding))? - else { - return Err(ProjectionStoreError::Contract( - ObservationContractError::CanonicalEncoding, - )); +) -> ProjectionStoreResult> { + let mut metadata = match rendering { + // Released rows embedded the whole envelope. + CanonicalRendering::ShippedRelease => match serde_json::to_value(envelope) { + Ok(serde_json::Value::Object(envelope)) => envelope, + _ => { + return Err(ProjectionStoreError::Contract( + ObservationContractError::CanonicalEncoding, + )); + } + }, + CanonicalRendering::Current => serde_json::Map::new(), }; if let Some(session_metadata) = session_metadata { metadata.extend(session_metadata.clone()); } - if let Some(normalize) = - tool_metadata_normalizer(metadata.get("source").and_then(serde_json::Value::as_str)) - { + let source = metadata + .get("source") + .and_then(serde_json::Value::as_str) + .map(str::to_owned); + let normalizer = tool_metadata_normalizer(source.as_deref()); + if let Some(normalize) = normalizer { normalize(&mut metadata, envelope.facts())?; } + let tool_use_id = match rendering { + CanonicalRendering::Current => host_tool_use_id(envelope), + // Released rows carried the first subagent dispatch id of a Cursor + // transcript record, including the capture fallback. + CanonicalRendering::ShippedRelease => normalizer.and_then(|_| { + envelope.facts().iter().find_map(|fact| match fact { + CanonicalObservationFactV1::ToolInvocation { + invocation_id, + name, + .. + } if is_subagent_dispatch_tool(name) => Some(invocation_id.as_str()), + _ => None, + }) + }), + }; + if let Some(tool_use_id) = tool_use_id { + metadata.insert( + TOOL_USE_ID_KEY.to_owned(), + serde_json::Value::String(tool_use_id.to_owned()), + ); + } if let Some(CanonicalObservationFactV1::Message { role, content, .. }) = envelope .facts() .iter() @@ -413,8 +533,28 @@ fn canonical_message_metadata_for( { metadata.extend(semantics.metadata); } - serde_json::to_string(&metadata) - .map_err(|_| ProjectionStoreError::Contract(ObservationContractError::CanonicalEncoding)) + serialize_metadata_map(&metadata) +} + +/// The full message metadata: the observation `envelope` payload overlaid by +/// the row's stored keys, byte-identical to a row that embedded the envelope. +pub fn message_metadata_with_envelope( + stored: Option<&str>, + envelope: &serde_json::Value, +) -> ProjectionStoreResult { + let encoding = || ProjectionStoreError::Contract(ObservationContractError::CanonicalEncoding); + let serde_json::Value::Object(mut metadata) = envelope.clone() else { + return Err(encoding()); + }; + if let Some(stored) = stored { + let serde_json::Value::Object(stored) = + serde_json::from_str(stored).map_err(|_| encoding())? + else { + return Err(encoding()); + }; + metadata.extend(stored); + } + serde_json::to_string(&metadata).map_err(|_| encoding()) } fn canonical_workflow_facts( @@ -1243,7 +1383,9 @@ mod tests { canonical_session_metadata_map("cursor", Some(&cursor_transcript_session_fields())); let metadata: serde_json::Value = serde_json::from_str( - &canonical_message_metadata(&envelope, Some(&session_metadata)).unwrap(), + &canonical_message_metadata(&envelope, Some(&session_metadata)) + .unwrap() + .unwrap(), ) .unwrap(); assert_eq!(metadata["tool_calls"][0]["id"], "tool.dispatch"); @@ -1269,6 +1411,7 @@ mod tests { Some(&other_source), )), ) + .unwrap() .unwrap(), ) .unwrap(); @@ -1277,7 +1420,139 @@ mod tests { "tool-metadata normalization belongs to the cursor transcript source only" ); assert!(other_metadata.get("tool_events").is_none()); - assert!(other_metadata.get("tool_use_id").is_none()); + assert_eq!( + other_metadata["tool_use_id"], "tool.dispatch", + "the host tool-use id is provider-neutral" + ); + } + + #[test] + fn tool_use_id_is_the_host_id_never_the_capture_fallback() { + // A capture that found no host id falls back to the record's stable + // id (or `{stable}:tool:{index}`); neither is served as a tool-use id. + for synthesized in ["record.fixture", "record.fixture:tool:0"] { + let fallback = envelope(vec![CanonicalObservationFactV1::ToolInvocation { + invocation_id: ObservationId::new(synthesized).unwrap(), + name: "Read".to_owned(), + arguments: json!({}), + }]); + assert_eq!(host_tool_use_id(&fallback), None, "{synthesized}"); + assert!( + canonical_message_metadata(&fallback, None) + .unwrap() + .is_none() + ); + } + + // The subagent dispatch binds a fork even when it is not first. + let dispatching = envelope(vec![ + CanonicalObservationFactV1::ToolInvocation { + invocation_id: ObservationId::new("toolu_read").unwrap(), + name: "Read".to_owned(), + arguments: json!({}), + }, + CanonicalObservationFactV1::ToolInvocation { + invocation_id: ObservationId::new("toolu_task").unwrap(), + name: "Task".to_owned(), + arguments: json!({"prompt": "explore"}), + }, + ]); + assert_eq!(host_tool_use_id(&dispatching), Some("toolu_task")); + + let exec = envelope(vec![CanonicalObservationFactV1::ToolInvocation { + invocation_id: ObservationId::new("call_abc").unwrap(), + name: "exec".to_owned(), + arguments: json!({}), + }]); + let metadata: serde_json::Value = + serde_json::from_str(&canonical_message_metadata(&exec, None).unwrap().unwrap()) + .unwrap(); + assert_eq!(metadata, json!({"tool_use_id": "call_abc"})); + } + + #[test] + fn file_edit_facts_roll_up_on_the_session_row_only() { + let edits = envelope(vec![ + CanonicalObservationFactV1::Message { + role: CanonicalMessageRoleV1::User, + content: json!("edited"), + model: None, + timestamp: Some(42), + }, + CanonicalObservationFactV1::Git { + evidence_kind: CanonicalGitEvidenceKindV1::FileEdit, + reference: Some("/work/src/lib.rs".to_owned()), + content: Some(json!({ + "type": "FileChange", + "edited_at_micros": 1_700_000_000_123_456i64, + "change_type": "update", + "hunks": 2, + "unified_diff": "never copied" + })), + }, + CanonicalObservationFactV1::Git { + evidence_kind: CanonicalGitEvidenceKindV1::FileEdit, + reference: Some("/work/src/new.rs".to_owned()), + content: None, + }, + CanonicalObservationFactV1::Git { + evidence_kind: CanonicalGitEvidenceKindV1::Commit, + reference: Some("abc123".to_owned()), + content: None, + }, + ]); + assert_eq!( + serde_json::Value::Array(canonical_edited_files(&edits)), + json!([ + { + "path": "/work/src/lib.rs", + "edited_at_micros": 1_700_000_000_123_456i64, + "change_type": "update", + "hunks": 2 + }, + {"path": "/work/src/new.rs"} + ]), + "each recorded key is copied; absent keys stay absent" + ); + + let projection = derive_canonical_projection(&observation_without_native_record_id( + &provider_envelope("claude", edits.facts().to_vec()), + )) + .unwrap(); + let output = projection.messages().next().unwrap(); + let session_metadata: serde_json::Value = + serde_json::from_str(output.session().metadata_json.as_deref().unwrap()).unwrap(); + assert_eq!( + session_metadata["edited_files"][1]["path"], + "/work/src/new.rs" + ); + assert!( + output.message().metadata_json.is_none(), + "the rollup is session evidence, not message metadata: {:?}", + output.message().metadata_json + ); + + let no_edits = provider_envelope( + "claude", + vec![CanonicalObservationFactV1::Message { + role: CanonicalMessageRoleV1::Assistant, + content: json!("no edits"), + model: None, + timestamp: Some(43), + }], + ); + let projection = + derive_canonical_projection(&observation_without_native_record_id(&no_edits)).unwrap(); + assert!( + projection + .messages() + .next() + .unwrap() + .session() + .metadata_json + .is_none(), + "a record without edit facts records no edited_files array" + ); } #[test] @@ -1361,8 +1636,12 @@ mod tests { "Codex active goal: finish canonical projection" ); assert_eq!(fields.kind, "goal_context"); - let metadata: serde_json::Value = - serde_json::from_str(&canonical_message_metadata(&envelope, None).unwrap()).unwrap(); + let metadata: serde_json::Value = serde_json::from_str( + &canonical_message_metadata(&envelope, None) + .unwrap() + .unwrap(), + ) + .unwrap(); assert_eq!(metadata["source"], "codex_rollout"); assert_eq!(metadata["codex_internal_context"], "goal"); assert_eq!( @@ -1447,8 +1726,21 @@ mod tests { ); let session_metadata_map = canonical_session_metadata_map("codex", Some(&fields)); + let stored = canonical_message_metadata(&envelope, Some(&session_metadata_map)) + .unwrap() + .unwrap(); + let stored_metadata: serde_json::Value = serde_json::from_str(&stored).unwrap(); + assert!( + stored_metadata.get("stable_record_id").is_none() + && stored_metadata.get("facts").is_none(), + "the envelope is stored once, in its observation row: {stored}" + ); let message_metadata: serde_json::Value = serde_json::from_str( - &canonical_message_metadata(&envelope, Some(&session_metadata_map)).unwrap(), + &message_metadata_with_envelope( + Some(&stored), + &serde_json::to_value(&envelope).unwrap(), + ) + .unwrap(), ) .unwrap(); assert_eq!( diff --git a/crates/tracedecay-store/src/diagnostics/codec.rs b/crates/tracedecay-store/src/diagnostics/codec.rs index a374e212c3..e40df12f5b 100644 --- a/crates/tracedecay-store/src/diagnostics/codec.rs +++ b/crates/tracedecay-store/src/diagnostics/codec.rs @@ -20,17 +20,14 @@ use tracedecay_domain::{ /// Stored `record_state` text for a live record. pub const DIAGNOSTIC_STATE_CURRENT: &str = "current"; -/// Stored `record_state` text for a record replaced by a later generation. -pub const DIAGNOSTIC_STATE_SUPERSEDED: &str = "superseded"; /// Stored `record_state` text for a record cleared by a later generation. pub const DIAGNOSTIC_STATE_CLEARED: &str = "cleared"; /// The stored discriminant of `record_state`, decoupled from the -/// `state_generation` back-pointer that two of its three forms carry. +/// `state_generation` back-pointer that the cleared form carries. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum DiagnosticRecordStateKindV1 { Current, - Superseded, Cleared, } @@ -40,7 +37,6 @@ impl DiagnosticRecordStateKindV1 { pub fn parse(value: &str) -> Option { match value { DIAGNOSTIC_STATE_CURRENT => Some(Self::Current), - DIAGNOSTIC_STATE_SUPERSEDED => Some(Self::Superseded), DIAGNOSTIC_STATE_CLEARED => Some(Self::Cleared), _ => None, } @@ -51,7 +47,6 @@ impl DiagnosticRecordStateKindV1 { pub const fn as_str(self) -> &'static str { match self { Self::Current => DIAGNOSTIC_STATE_CURRENT, - Self::Superseded => DIAGNOSTIC_STATE_SUPERSEDED, Self::Cleared => DIAGNOSTIC_STATE_CLEARED, } } @@ -64,7 +59,6 @@ impl DiagnosticRecordStateKindV1 { pub const fn state_generation_field(self) -> Option<&'static str> { match self { Self::Current => None, - Self::Superseded => Some("successor_generation"), Self::Cleared => Some("cleared_in_generation"), } } @@ -80,11 +74,6 @@ impl DiagnosticRecordStateKindV1 { ) -> Option { match (self, state_generation) { (Self::Current, None) => Some(DiagnosticRecordStateV1::Current), - (Self::Superseded, Some(successor_generation)) => { - Some(DiagnosticRecordStateV1::Superseded { - successor_generation, - }) - } (Self::Cleared, Some(cleared_in_generation)) => { Some(DiagnosticRecordStateV1::Cleared { cleared_in_generation, @@ -101,12 +90,6 @@ impl DiagnosticRecordStateKindV1 { pub fn diagnostic_state_columns(state: &DiagnosticRecordStateV1) -> (&'static str, Option<&str>) { match state { DiagnosticRecordStateV1::Current => (DIAGNOSTIC_STATE_CURRENT, None), - DiagnosticRecordStateV1::Superseded { - successor_generation, - } => ( - DIAGNOSTIC_STATE_SUPERSEDED, - Some(successor_generation.as_str()), - ), DiagnosticRecordStateV1::Cleared { cleared_in_generation, } => ( @@ -201,9 +184,6 @@ mod tests { fn every_record_state_round_trips_through_its_columns() { let cases = [ DiagnosticRecordStateV1::Current, - DiagnosticRecordStateV1::Superseded { - successor_generation: generation("generation.successor"), - }, DiagnosticRecordStateV1::Cleared { cleared_in_generation: generation("generation.cleared"), }, @@ -226,12 +206,6 @@ mod tests { #[test] fn state_columns_and_back_pointer_must_agree() { - assert!( - DiagnosticRecordStateKindV1::Superseded - .into_state(None) - .is_none(), - "a superseded row without a successor is corrupt" - ); assert!( DiagnosticRecordStateKindV1::Cleared .into_state(None) @@ -245,6 +219,10 @@ mod tests { "a current row must not carry a state generation" ); assert!(DiagnosticRecordStateKindV1::parse("archived").is_none()); + assert!( + DiagnosticRecordStateKindV1::parse("superseded").is_none(), + "retired superseded rows must be refused, not reinterpreted" + ); } #[test] diff --git a/crates/tracedecay-store/src/diagnostics/mod.rs b/crates/tracedecay-store/src/diagnostics/mod.rs index 506904beeb..bff60b73e1 100644 --- a/crates/tracedecay-store/src/diagnostics/mod.rs +++ b/crates/tracedecay-store/src/diagnostics/mod.rs @@ -6,64 +6,13 @@ pub mod codec; mod ports; pub use codec::{ - DIAGNOSTIC_STATE_CLEARED, DIAGNOSTIC_STATE_CURRENT, DIAGNOSTIC_STATE_SUPERSEDED, - DiagnosticRecordStateKindV1, diagnostic_evidence_class_name, diagnostic_producer_kind_name, - diagnostic_severity_name, diagnostic_state_columns, parse_diagnostic_evidence_class, - parse_diagnostic_producer_kind, parse_diagnostic_severity, + DIAGNOSTIC_STATE_CLEARED, DIAGNOSTIC_STATE_CURRENT, DiagnosticRecordStateKindV1, + diagnostic_evidence_class_name, diagnostic_producer_kind_name, diagnostic_severity_name, + diagnostic_state_columns, parse_diagnostic_evidence_class, parse_diagnostic_producer_kind, + parse_diagnostic_severity, }; pub use ports::DiagnosticStore; -/// One admitted request to transition every current record of a prior -/// generation into the superseded state, back-pointing at its successor. -/// -/// Supersession is a distinct lane from clean publication: publication clears -/// the records a newer clean generation replaced, while supersession records -/// that a specific prior generation was superseded by a specific successor and -/// keeps the logical finding chain walkable. Validating the pair here means a -/// storage engine cannot be handed a self-supersession. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct DiagnosticGenerationSupersessionV1 { - prior_generation: CodeGenerationId, - successor_generation: CodeGenerationId, -} - -impl DiagnosticGenerationSupersessionV1 { - pub fn new( - prior_generation: CodeGenerationId, - successor_generation: CodeGenerationId, - ) -> DiagnosticStoreResult { - let request = Self { - prior_generation, - successor_generation, - }; - request.validate()?; - Ok(request) - } - - pub fn validate(&self) -> DiagnosticStoreResult<()> { - self.prior_generation - .validate() - .map_err(DiagnosticStoreError::Contract)?; - self.successor_generation - .validate() - .map_err(DiagnosticStoreError::Contract)?; - if self.prior_generation == self.successor_generation { - return Err(DiagnosticStoreError::SelfSupersession { - generation: self.prior_generation.clone(), - }); - } - Ok(()) - } - - pub fn prior_generation(&self) -> &CodeGenerationId { - &self.prior_generation - } - - pub fn successor_generation(&self) -> &CodeGenerationId { - &self.successor_generation - } -} - /// A complete durable diagnostic snapshot admitted from the normal sanitized /// clean-generation pipeline. /// @@ -125,68 +74,6 @@ impl SanitizedCleanDiagnosticSnapshotV1 { pub fn records(&self) -> &[GenerationDiagnosticV1] { &self.records } - - pub fn into_parts(self) -> (CodeGenerationId, Vec) { - (self.generation_id, self.records) - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum DiagnosticPublicationDispositionV1 { - Committed, - ExactReplay, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct DiagnosticPublicationReceiptV1 { - generation_id: CodeGenerationId, - publication_revision: u64, - inserted_records: u64, - cleared_records: u64, - disposition: DiagnosticPublicationDispositionV1, -} - -impl DiagnosticPublicationReceiptV1 { - pub fn new( - generation_id: CodeGenerationId, - publication_revision: u64, - inserted_records: u64, - cleared_records: u64, - disposition: DiagnosticPublicationDispositionV1, - ) -> Self { - Self { - generation_id, - publication_revision, - inserted_records, - cleared_records, - disposition, - } - } - - pub fn generation_id(&self) -> &CodeGenerationId { - &self.generation_id - } - - /// Store-issued monotone revision of this generation's diagnostic snapshot. - #[hotpath::skip] - pub const fn publication_revision(&self) -> u64 { - self.publication_revision - } - - #[hotpath::skip] - pub const fn inserted_records(&self) -> u64 { - self.inserted_records - } - - #[hotpath::skip] - pub const fn cleared_records(&self) -> u64 { - self.cleared_records - } - - #[hotpath::skip] - pub const fn disposition(&self) -> DiagnosticPublicationDispositionV1 { - self.disposition - } } #[derive(Debug, thiserror::Error)] @@ -204,8 +91,6 @@ pub enum DiagnosticStoreError { NonCurrentRecord { anchor: RetrievalAnchorId }, #[error("diagnostic anchor {anchor} occurs more than once in a clean snapshot")] DuplicateAnchor { anchor: RetrievalAnchorId }, - #[error("diagnostic generation {generation} cannot supersede itself")] - SelfSupersession { generation: CodeGenerationId }, #[error("diagnostic contract validation failed")] Contract(#[source] DomainError), #[error("diagnostic storage operation {operation} failed")] diff --git a/crates/tracedecay-store/src/diagnostics/ports.rs b/crates/tracedecay-store/src/diagnostics/ports.rs index ec5e4bad1f..1415cf7fc2 100644 --- a/crates/tracedecay-store/src/diagnostics/ports.rs +++ b/crates/tracedecay-store/src/diagnostics/ports.rs @@ -4,20 +4,10 @@ use tracedecay_domain::{ CodeGenerationId, FileOccurrenceId, GenerationDiagnosticV1, RetrievalAnchorId, }; -use super::{ - DiagnosticPublicationReceiptV1, DiagnosticStoreResult, SanitizedCleanDiagnosticSnapshotV1, -}; +use super::DiagnosticStoreResult; -/// Authoritative persistence boundary for generation-bound clean diagnostics. -/// -/// The write side accepts only [`SanitizedCleanDiagnosticSnapshotV1`], so live -/// analyzer sessions and dirty editor overlays cannot reach durable storage. +/// Authoritative read boundary for generation-bound clean diagnostics. pub trait DiagnosticStore: Send + Sync { - fn publish_clean_diagnostics( - &self, - snapshot: SanitizedCleanDiagnosticSnapshotV1, - ) -> impl Future> + Send; - fn current_diagnostic_generation( &self, ) -> impl Future>> + Send; @@ -27,12 +17,6 @@ pub trait DiagnosticStore: Send + Sync { generation: &CodeGenerationId, ) -> impl Future>> + Send; - fn diagnostics_for_publication( - &self, - generation: &CodeGenerationId, - publication_revision: u64, - ) -> impl Future>> + Send; - fn current_diagnostics( &self, generation: &CodeGenerationId, @@ -44,24 +28,8 @@ pub trait DiagnosticStore: Send + Sync { file_occurrence_id: &FileOccurrenceId, ) -> impl Future>> + Send; - fn stale_diagnostics( - &self, - generation: &CodeGenerationId, - ) -> impl Future>> + Send; - fn diagnostic_by_anchor( &self, anchor: &RetrievalAnchorId, ) -> impl Future>> + Send; - - fn diagnostic_supersession_chain( - &self, - anchor: &RetrievalAnchorId, - ) -> impl Future>> + Send; - - fn supersede_diagnostic_generation( - &self, - prior_generation: &CodeGenerationId, - successor_generation: &CodeGenerationId, - ) -> impl Future> + Send; } diff --git a/crates/tracedecay-store/src/evidence_assembly.rs b/crates/tracedecay-store/src/evidence_assembly.rs deleted file mode 100644 index 9d6a67ec05..0000000000 --- a/crates/tracedecay-store/src/evidence_assembly.rs +++ /dev/null @@ -1,2262 +0,0 @@ -//! Driver-neutral, payload-free evidence-assembly persistence contracts. - -use std::collections::{BTreeMap, BTreeSet}; - -use serde::{Deserialize, Serialize}; -use thiserror::Error; -use tracedecay_domain::canonical_text::{CANONICAL_TEXT_MAX_BYTES, is_canonical_text_within}; -use tracedecay_domain::{ - AnchorOwnerBindingV1, BlobId, CanonicalObservationIdV1, CanonicalSourceOccurrenceSetIdV1, - CapabilityId, ComponentVersion, CoverageReportV1, EvidenceAssemblyPublicationReceiptIdV1, - EvidenceSpanIdV1, EvidenceSpanProjectionReceiptIdV1, ManifestDigest, - ObservationOrderingDomainV1, ObservationScopeV1, ObservationSourceGenerationV1, - ObservationSourceIdentityV1, ObservationSourceRangeV1, PrivacyDomainBoundLocatorDigest, - PrivacyDomainId, ProjectionGenerationId, RepositoryCaptureId, RepositoryId, RetrievalAnchorId, - RetrievalAnchorRecordV3, RetrievalAnchorTargetV3, RetrieverContributionIdV1, - SanitizationReceiptRefV1, ScopeResolutionId, SourceOccurrenceId, TemporalModeV1, UseCaseId, - UtcMicros, VectorWatermark, canonical_sha256, -}; - -pub const MAX_EVIDENCE_ASSEMBLY_MEMBERS_V1: usize = 4_096; -const SOURCE_OCCURRENCE_ID_DOMAIN_V1: &str = "tracedecay.source-occurrence.identity.v1"; -const OCCURRENCE_SET_ID_DOMAIN_V1: &str = "tracedecay.source-occurrence-set.identity.v1"; -const EVIDENCE_SPAN_ID_DOMAIN_V1: &str = "tracedecay.evidence-span.identity.v1"; -const PROJECTION_RECEIPT_ID_DOMAIN_V1: &str = - "tracedecay.evidence-span-projection-receipt.identity.v1"; -const RETRIEVER_CONTRIBUTION_ID_DOMAIN_V1: &str = "tracedecay.retriever-contribution.identity.v1"; -const PUBLICATION_RECEIPT_ID_DOMAIN_V1: &str = - "tracedecay.evidence-assembly-publication.identity.v1"; - -#[derive(Clone, Debug, Error, PartialEq, Eq)] -pub enum EvidenceAssemblyStoreError { - #[error("evidence assembly store data is invalid: {0}")] - InvalidData(String), - #[error("evidence assembly replay conflicts with existing material")] - ReplayConflict, - #[error("evidence assembly target is unavailable")] - Unavailable, - #[error("evidence catalog binding does not match ordering proof")] - CatalogMismatch, - #[error("evidence integration manifest does not match ordering proof")] - IntegrationManifestMismatch, - #[error("evidence ordering proof is stale")] - StaleOrderingProof, - #[error("evidence occurrences do not share a comparable source order")] - IncomparableSourceOrder, - #[error("evidence consecutiveness was not verified")] - UnverifiedConsecutiveness, - #[error("evidence request digest does not match the owner privacy binding")] - RequestPrivacyBindingMismatch, - #[error("evidence sanitization receipt roles are incomplete or reused")] - ReceiptRoleMismatch, - #[error("evidence temporal horizon does not cover every member")] - HorizonMismatch, -} - -pub type EvidenceAssemblyStoreResult = Result; - -#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] -#[serde(deny_unknown_fields)] -pub struct SanitizedObservationByteRangeV1 { - pub start: u64, - pub end: u64, -} - -impl SanitizedObservationByteRangeV1 { - pub fn new(start: u64, end: u64) -> EvidenceAssemblyStoreResult { - if start >= end { - return Err(invalid("sanitized observation byte range")); - } - Ok(Self { start, end }) - } - - pub fn validate(&self) -> EvidenceAssemblyStoreResult<()> { - Self::new(self.start, self.end).map(|_| ()) - } -} - -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] -pub enum SourceOccurrenceCoordinateV1 { - ObservationProjection { - canonical_observation_id: CanonicalObservationIdV1, - source_range: ObservationSourceRangeV1, - projection_output_ordinal: u64, - sanitized_byte_range: SanitizedObservationByteRangeV1, - }, - ImmutableBlobSlice { - repository_id: RepositoryId, - blob_id: BlobId, - byte_start: u64, - byte_end: u64, - }, - CapturedWorktreeSlice { - repository_id: RepositoryId, - repository_capture_id: RepositoryCaptureId, - path_locator_digest: PrivacyDomainBoundLocatorDigest, - byte_start: u64, - byte_end: u64, - }, -} - -impl SourceOccurrenceCoordinateV1 { - pub fn validate(&self) -> EvidenceAssemblyStoreResult<()> { - match self { - Self::ObservationProjection { - canonical_observation_id, - source_range, - sanitized_byte_range, - .. - } => { - CanonicalObservationIdV1::new(canonical_observation_id.as_str()) - .map_err(invalid)?; - ObservationSourceRangeV1::new(source_range.start(), source_range.end()) - .map_err(invalid)?; - sanitized_byte_range.validate() - } - Self::ImmutableBlobSlice { - repository_id, - blob_id, - byte_start, - byte_end, - } => { - repository_id.validate().map_err(invalid)?; - blob_id.validate().map_err(invalid)?; - validate_half_open(*byte_start, *byte_end, "immutable blob byte range") - } - Self::CapturedWorktreeSlice { - repository_id, - repository_capture_id, - path_locator_digest, - byte_start, - byte_end, - } => { - repository_id.validate().map_err(invalid)?; - repository_capture_id.validate().map_err(invalid)?; - path_locator_digest.validate().map_err(invalid)?; - validate_half_open(*byte_start, *byte_end, "captured worktree byte range") - } - } - } - - #[hotpath::skip] - pub const fn is_code(&self) -> bool { - matches!( - self, - Self::ImmutableBlobSlice { .. } | Self::CapturedWorktreeSlice { .. } - ) - } - - pub fn source_order(&self) -> u64 { - match self { - Self::ObservationProjection { source_range, .. } => source_range.start(), - Self::ImmutableBlobSlice { byte_start, .. } - | Self::CapturedWorktreeSlice { byte_start, .. } => *byte_start, - } - } -} - -#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum SourceOccurrenceKindV1 { - Message, - ToolInvocation, - ToolResult, - CodeChunk, -} - -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] -pub enum SourceOccurrenceRelationV1 { - ToolResultFor { - invocation_occurrence_id: SourceOccurrenceId, - }, - DerivedFromOccurrence { - source_occurrence_id: SourceOccurrenceId, - }, -} - -impl SourceOccurrenceRelationV1 { - fn source_id(&self) -> &SourceOccurrenceId { - match self { - Self::ToolResultFor { - invocation_occurrence_id, - } => invocation_occurrence_id, - Self::DerivedFromOccurrence { - source_occurrence_id, - } => source_occurrence_id, - } - } - - fn validate(&self) -> EvidenceAssemblyStoreResult<()> { - self.source_id().validate().map_err(invalid) - } -} - -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct SourceOccurrenceSanitizationV1 { - pub capture: SanitizationReceiptRefV1, - pub projection: SanitizationReceiptRefV1, -} - -impl SourceOccurrenceSanitizationV1 { - pub fn new( - capture: SanitizationReceiptRefV1, - projection: SanitizationReceiptRefV1, - ) -> EvidenceAssemblyStoreResult { - let value = Self { - capture, - projection, - }; - value.validate()?; - Ok(value) - } - - pub fn validate(&self) -> EvidenceAssemblyStoreResult<()> { - self.capture.validate().map_err(invalid)?; - self.projection.validate().map_err(invalid)?; - if self.capture == self.projection { - return Err(EvidenceAssemblyStoreError::ReceiptRoleMismatch); - } - Ok(()) - } -} - -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct SourceTimelineKeyV1 { - pub source: ObservationSourceIdentityV1, - pub scope: ObservationScopeV1, - pub source_generation: ObservationSourceGenerationV1, - pub ordering_domain: ObservationOrderingDomainV1, -} - -impl SourceTimelineKeyV1 { - pub fn validate(&self) -> EvidenceAssemblyStoreResult<()> { - self.source.validate().map_err(invalid)?; - self.scope.validate().map_err(invalid)?; - ObservationSourceGenerationV1::new(self.source_generation.generation_id()) - .map_err(invalid)?; - Ok(()) - } - - pub fn digest(&self) -> EvidenceAssemblyStoreResult { - self.validate()?; - canonical_sha256(self).map_err(invalid) - } -} - -pub type EvidenceSourceTimelineV1 = SourceTimelineKeyV1; - -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(transparent)] -pub struct EvidenceAssemblyIdempotencyKeyV1(ManifestDigest); - -impl EvidenceAssemblyIdempotencyKeyV1 { - pub fn new(value: ManifestDigest) -> EvidenceAssemblyStoreResult { - value.validate().map_err(invalid)?; - Ok(Self(value)) - } - - pub fn as_digest(&self) -> &ManifestDigest { - &self.0 - } - - pub fn derive( - owner: &AnchorOwnerBindingV1, - key_epoch: u64, - privacy_key: &[u8], - raw_request_key: &[u8], - ) -> EvidenceAssemblyStoreResult { - owner.validate().map_err(invalid)?; - if key_epoch == 0 - || privacy_key.len() < 16 - || raw_request_key.is_empty() - || raw_request_key.len() > 4_096 - { - return Err(invalid("evidence assembly idempotency key material")); - } - Self::new(keyed_canonical_digest( - privacy_key, - &( - "tracedecay.evidence-assembly-idempotency.v1", - owner, - owner.privacy_domain_id(), - key_epoch, - raw_request_key, - ), - )?) - } -} - -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct EvidenceAssemblyOwnerV1 { - pub owner: AnchorOwnerBindingV1, - pub scope_digest: ManifestDigest, - pub key_epoch: u64, -} - -impl EvidenceAssemblyOwnerV1 { - pub fn validate(&self) -> EvidenceAssemblyStoreResult<()> { - self.owner.validate().map_err(invalid)?; - self.scope_digest.validate().map_err(invalid)?; - if self.key_epoch == 0 { - return Err(invalid("evidence assembly privacy key epoch")); - } - Ok(()) - } -} - -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct EvidenceSourceOccurrenceRecordV1 { - pub occurrence_id: SourceOccurrenceId, - pub owner: AnchorOwnerBindingV1, - pub timeline: EvidenceSourceTimelineV1, - pub exact_source_anchor: RetrievalAnchorId, - pub occurrence_anchor: RetrievalAnchorRecordV3, - pub source_order: u64, - pub coordinate: SourceOccurrenceCoordinateV1, - pub occurrence_kind: SourceOccurrenceKindV1, - pub relations: Vec, - pub projector_version: ComponentVersion, - pub sanitization: SourceOccurrenceSanitizationV1, - pub knowledge_time: UtcMicros, - pub valid_time: Option, -} - -/// Identity-bound material of one source occurrence, borrowed from wherever -/// the caller already owns it. -/// -/// Identity projections are transient serialization views: field names and -/// order are the canonical identity schema, so a borrowed field serializes to -/// exactly the bytes its owned counterpart would and existing durable ids do -/// not move. Never copy record vectors to build one. -#[derive(Clone, Copy, Debug, Serialize, PartialEq, Eq)] -pub struct SourceOccurrenceIdentityProjectionV1<'a> { - pub owner: &'a AnchorOwnerBindingV1, - pub timeline: &'a EvidenceSourceTimelineV1, - pub exact_source_anchor: &'a RetrievalAnchorId, - pub source_order: u64, - pub coordinate: &'a SourceOccurrenceCoordinateV1, - pub occurrence_kind: SourceOccurrenceKindV1, - pub relations: &'a [SourceOccurrenceRelationV1], - pub projector_version: &'a ComponentVersion, -} - -impl SourceOccurrenceIdentityProjectionV1<'_> { - pub fn validate(&self) -> EvidenceAssemblyStoreResult<()> { - self.owner.validate().map_err(invalid)?; - self.timeline.validate()?; - self.exact_source_anchor.validate().map_err(invalid)?; - self.coordinate.validate()?; - self.projector_version.validate().map_err(invalid)?; - let owner_scope_matches = match (self.owner.project_id(), &self.timeline.scope) { - (None, ObservationScopeV1::Profile) => true, - ( - Some(owner_project), - ObservationScopeV1::Project { - project_id: source_project, - }, - ) => owner_project == source_project, - _ => false, - }; - if !owner_scope_matches { - return Err(invalid("source occurrence timeline owner scope")); - } - if self.source_order != self.coordinate.source_order() - || (self.coordinate.is_code() - && self.timeline.ordering_domain != ObservationOrderingDomainV1::FileBytes) - { - return Err(EvidenceAssemblyStoreError::IncomparableSourceOrder); - } - if self.relations.len() > MAX_EVIDENCE_ASSEMBLY_MEMBERS_V1 { - return Err(invalid("source occurrence relation count")); - } - for relation in self.relations { - relation.validate()?; - } - ensure_unique( - self.relations - .iter() - .map(SourceOccurrenceRelationV1::source_id), - "source occurrence relations", - )?; - let tool_result_relations = self - .relations - .iter() - .filter(|relation| matches!(relation, SourceOccurrenceRelationV1::ToolResultFor { .. })) - .count(); - if (self.occurrence_kind == SourceOccurrenceKindV1::ToolResult - && tool_result_relations != 1) - || (self.occurrence_kind != SourceOccurrenceKindV1::ToolResult - && tool_result_relations != 0) - || (self.occurrence_kind == SourceOccurrenceKindV1::CodeChunk) - != self.coordinate.is_code() - { - return Err(invalid( - "source occurrence kind/coordinate/relation binding", - )); - } - Ok(()) - } -} - -pub fn derive_source_occurrence_id_v1( - projection: &SourceOccurrenceIdentityProjectionV1<'_>, -) -> EvidenceAssemblyStoreResult { - projection.validate()?; - let digest = canonical_identity_digest(SOURCE_OCCURRENCE_ID_DOMAIN_V1, projection)?; - SourceOccurrenceId::new(digest.as_str()).map_err(invalid) -} - -impl EvidenceSourceOccurrenceRecordV1 { - pub fn identity_projection(&self) -> SourceOccurrenceIdentityProjectionV1<'_> { - SourceOccurrenceIdentityProjectionV1 { - owner: &self.owner, - timeline: &self.timeline, - exact_source_anchor: &self.exact_source_anchor, - source_order: self.source_order, - coordinate: &self.coordinate, - occurrence_kind: self.occurrence_kind, - relations: &self.relations, - projector_version: &self.projector_version, - } - } - - pub fn validate(&self) -> EvidenceAssemblyStoreResult<()> { - self.occurrence_id.validate().map_err(invalid)?; - self.owner.validate().map_err(invalid)?; - self.timeline.validate()?; - self.exact_source_anchor.validate().map_err(invalid)?; - self.coordinate.validate()?; - self.sanitization.validate()?; - self.occurrence_anchor.validate().map_err(invalid)?; - match self.occurrence_anchor.target() { - RetrievalAnchorTargetV3::ExactSourceOccurrence(target) - if target == &self.occurrence_id => {} - _ => return Err(invalid("source occurrence anchor target")), - } - if self.occurrence_anchor.owner() != &self.owner { - return Err(invalid("source occurrence anchor owner")); - } - validate_derived_anchor_lineage( - &self.occurrence_anchor, - &self.owner, - std::slice::from_ref(&self.exact_source_anchor), - "source occurrence anchor lineage", - )?; - if self - .relations - .iter() - .any(|relation| relation.source_id() == &self.occurrence_id) - { - return Err(invalid("source occurrence self relation")); - } - if self.occurrence_id != derive_source_occurrence_id_v1(&self.identity_projection())? { - return Err(invalid("source occurrence identity")); - } - Ok(()) - } -} - -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct CanonicalSourceOccurrenceSetRecordV1 { - pub occurrence_set_id: CanonicalSourceOccurrenceSetIdV1, - pub owner: AnchorOwnerBindingV1, - /// Canonical set order, sorted by immutable occurrence identity. - pub members: Vec, -} - -#[derive(Clone, Copy, Debug, Serialize, PartialEq, Eq)] -pub struct CanonicalSourceOccurrenceSetIdentityProjectionV1<'a> { - pub owner: &'a AnchorOwnerBindingV1, - pub canonical_members: &'a [SourceOccurrenceId], -} - -impl CanonicalSourceOccurrenceSetIdentityProjectionV1<'_> { - pub fn validate(&self) -> EvidenceAssemblyStoreResult<()> { - self.owner.validate().map_err(invalid)?; - validate_member_count(self.canonical_members.len())?; - if self - .canonical_members - .windows(2) - .any(|pair| pair[0] >= pair[1]) - { - return Err(invalid("canonical occurrence set member order")); - } - Ok(()) - } -} - -pub fn derive_canonical_source_occurrence_set_id_v1( - projection: &CanonicalSourceOccurrenceSetIdentityProjectionV1<'_>, -) -> EvidenceAssemblyStoreResult { - projection.validate()?; - let digest = canonical_identity_digest(OCCURRENCE_SET_ID_DOMAIN_V1, projection)?; - CanonicalSourceOccurrenceSetIdV1::new(digest.as_str()).map_err(invalid) -} - -impl CanonicalSourceOccurrenceSetRecordV1 { - pub fn identity_projection(&self) -> CanonicalSourceOccurrenceSetIdentityProjectionV1<'_> { - CanonicalSourceOccurrenceSetIdentityProjectionV1 { - owner: &self.owner, - canonical_members: &self.members, - } - } - - pub fn validate(&self) -> EvidenceAssemblyStoreResult<()> { - self.occurrence_set_id.validate().map_err(invalid)?; - self.owner.validate().map_err(invalid)?; - validate_member_count(self.members.len())?; - if self.members.windows(2).any(|pair| pair[0] >= pair[1]) { - return Err(invalid("canonical occurrence set member order")); - } - if self.occurrence_set_id - != derive_canonical_source_occurrence_set_id_v1(&self.identity_projection())? - { - return Err(invalid("canonical occurrence set identity")); - } - Ok(()) - } -} - -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct SourceCapabilityCatalogBindingV1 { - pub connector_id: String, - pub root_id: String, - pub capability_id: CapabilityId, - pub catalog_digest: ManifestDigest, - pub integration_manifest_digest: ManifestDigest, - pub configuration_digest: ManifestDigest, - pub authorization_scope_digest: ManifestDigest, - pub projector_revision: ComponentVersion, - pub source_watermark: ManifestDigest, -} - -impl SourceCapabilityCatalogBindingV1 { - pub fn validate(&self) -> EvidenceAssemblyStoreResult<()> { - validate_label(&self.connector_id, "evidence source connector id")?; - validate_label(&self.root_id, "evidence source root id")?; - self.capability_id.validate().map_err(invalid)?; - self.catalog_digest.validate().map_err(invalid)?; - self.integration_manifest_digest - .validate() - .map_err(invalid)?; - self.configuration_digest.validate().map_err(invalid)?; - self.authorization_scope_digest - .validate() - .map_err(invalid)?; - self.projector_revision.validate().map_err(invalid)?; - self.source_watermark.validate().map_err(invalid) - } -} - -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] -// Boxing the large variant is wire-transparent but would change this public -// store-protocol API and ripple through construction/match sites. -#[allow(clippy::large_enum_variant)] -pub enum EvidenceSpanCatalogBindingV1 { - IntrinsicCanonicalOrdering, - SourceCapability { - binding: SourceCapabilityCatalogBindingV1, - }, -} - -impl EvidenceSpanCatalogBindingV1 { - pub fn validate(&self) -> EvidenceAssemblyStoreResult<()> { - match self { - Self::IntrinsicCanonicalOrdering => Ok(()), - Self::SourceCapability { binding } => binding.validate(), - } - } - - fn source_capability(&self) -> Option<&SourceCapabilityCatalogBindingV1> { - match self { - Self::IntrinsicCanonicalOrdering => None, - Self::SourceCapability { binding } => Some(binding), - } - } -} - -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct VerifiedSourceOrderingProofV1 { - pub timeline: SourceTimelineKeyV1, - pub catalog_binding: SourceCapabilityCatalogBindingV1, - pub ordered_occurrence_ids: Vec, - pub source_orders: Vec, -} - -impl VerifiedSourceOrderingProofV1 { - pub fn verify( - expected_timeline: SourceTimelineKeyV1, - expected_binding: SourceCapabilityCatalogBindingV1, - observed_binding: SourceCapabilityCatalogBindingV1, - ordered_occurrence_ids: Vec, - source_orders: Vec, - ) -> EvidenceAssemblyStoreResult { - expected_timeline.validate()?; - expected_binding.validate()?; - observed_binding.validate()?; - if expected_binding.catalog_digest != observed_binding.catalog_digest { - return Err(EvidenceAssemblyStoreError::CatalogMismatch); - } - if expected_binding.integration_manifest_digest - != observed_binding.integration_manifest_digest - { - return Err(EvidenceAssemblyStoreError::IntegrationManifestMismatch); - } - if expected_binding != observed_binding { - return Err(EvidenceAssemblyStoreError::StaleOrderingProof); - } - let proof = Self { - timeline: expected_timeline, - catalog_binding: expected_binding, - ordered_occurrence_ids, - source_orders, - }; - proof.validate()?; - Ok(proof) - } - - pub fn validate(&self) -> EvidenceAssemblyStoreResult<()> { - self.timeline.validate()?; - self.catalog_binding.validate()?; - validate_member_count(self.ordered_occurrence_ids.len())?; - if self.ordered_occurrence_ids.len() != self.source_orders.len() { - return Err(EvidenceAssemblyStoreError::IncomparableSourceOrder); - } - ensure_unique( - &self.ordered_occurrence_ids, - "verified ordering occurrence ids", - )?; - if self - .source_orders - .windows(2) - .any(|pair| pair[0].checked_add(1) != Some(pair[1])) - { - return Err(EvidenceAssemblyStoreError::UnverifiedConsecutiveness); - } - Ok(()) - } -} - -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct EvidenceSpanRunV1 { - pub assembly_ordinal: u64, - pub timeline: SourceTimelineKeyV1, - pub ordering_proof: VerifiedSourceOrderingProofV1, - pub timeline_digest: ManifestDigest, - pub first_source_order: u64, - pub last_source_order: u64, - pub occurrence_ids: Vec, -} - -impl EvidenceSpanRunV1 { - pub fn validate(&self) -> EvidenceAssemblyStoreResult<()> { - self.timeline.validate()?; - self.ordering_proof.validate()?; - self.timeline_digest.validate().map_err(invalid)?; - validate_member_count(self.occurrence_ids.len())?; - let expected_last = self - .first_source_order - .checked_add(u64::try_from(self.occurrence_ids.len() - 1).map_err(invalid)?) - .ok_or_else(|| invalid("evidence span source order overflow"))?; - if self.last_source_order != expected_last { - return Err(invalid("evidence span run adjacency")); - } - if self.timeline_digest != self.timeline.digest()? - || self.ordering_proof.timeline != self.timeline - || self.ordering_proof.ordered_occurrence_ids != self.occurrence_ids - || self.ordering_proof.source_orders.first().copied() != Some(self.first_source_order) - || self.ordering_proof.source_orders.last().copied() != Some(self.last_source_order) - { - return Err(EvidenceAssemblyStoreError::UnverifiedConsecutiveness); - } - ensure_unique(&self.occurrence_ids, "evidence span run occurrences") - } -} - -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct EvidenceSpanHorizonV1 { - pub knowledge_through: UtcMicros, - pub valid_through: Option, - pub contains_unknown_valid_time: bool, -} - -impl EvidenceSpanHorizonV1 { - pub fn validate_members( - &self, - members: &[EvidenceSourceOccurrenceRecordV1], - ) -> EvidenceAssemblyStoreResult<()> { - validate_member_count(members.len())?; - let max_knowledge = members - .iter() - .map(|member| member.knowledge_time) - .max_by_key(|time| time.0) - .ok_or(EvidenceAssemblyStoreError::HorizonMismatch)?; - let known_valid = members - .iter() - .filter_map(|member| member.valid_time) - .max_by_key(|time| time.0); - let has_unknown = members.iter().any(|member| member.valid_time.is_none()); - if self.knowledge_through.0 < max_knowledge.0 - || self.contains_unknown_valid_time != has_unknown - || match (self.valid_through, known_valid) { - (Some(bound), Some(maximum)) => bound.0 < maximum.0, - (None, Some(_)) => true, - _ => false, - } - { - return Err(EvidenceAssemblyStoreError::HorizonMismatch); - } - Ok(()) - } -} - -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct EvidenceSpanRecordV1 { - pub span_id: EvidenceSpanIdV1, - pub anchor: RetrievalAnchorRecordV3, - pub owner: AnchorOwnerBindingV1, - pub occurrence_set_id: CanonicalSourceOccurrenceSetIdV1, - pub runs: Vec, - pub exact_source_anchors: Vec, - pub projector_version: ComponentVersion, - pub horizon: EvidenceSpanHorizonV1, - pub catalog_binding: EvidenceSpanCatalogBindingV1, -} - -#[derive(Clone, Copy, Debug, Serialize, PartialEq, Eq)] -pub struct EvidenceSpanIdentityProjectionV1<'a> { - pub owner: &'a AnchorOwnerBindingV1, - pub occurrence_set_id: &'a CanonicalSourceOccurrenceSetIdV1, - pub ordered_runs: &'a [EvidenceSpanRunV1], - pub exact_source_anchors: &'a [RetrievalAnchorId], - pub projector_version: &'a ComponentVersion, - pub horizon: &'a EvidenceSpanHorizonV1, - pub catalog_binding: &'a EvidenceSpanCatalogBindingV1, -} - -impl EvidenceSpanIdentityProjectionV1<'_> { - pub fn validate(&self) -> EvidenceAssemblyStoreResult<()> { - self.owner.validate().map_err(invalid)?; - self.occurrence_set_id.validate().map_err(invalid)?; - self.projector_version.validate().map_err(invalid)?; - self.catalog_binding.validate()?; - validate_member_count(self.ordered_runs.len())?; - for (ordinal, run) in self.ordered_runs.iter().enumerate() { - run.validate()?; - if run.assembly_ordinal != u64::try_from(ordinal).map_err(invalid)? { - return Err(invalid("evidence span run order")); - } - } - let occurrence_count = self - .ordered_runs - .iter() - .map(|run| run.occurrence_ids.len()) - .sum::(); - validate_member_count(occurrence_count)?; - if self.exact_source_anchors.len() != occurrence_count { - return Err(invalid("evidence span exact source cardinality")); - } - for run in self.ordered_runs { - match ( - self.catalog_binding.source_capability(), - Some(&run.ordering_proof.catalog_binding), - ) { - (Some(expected), Some(observed)) if expected == observed => {} - (None, _) if run.occurrence_ids.len() == 1 => {} - (Some(_), _) => return Err(EvidenceAssemblyStoreError::CatalogMismatch), - (None, _) => return Err(EvidenceAssemblyStoreError::UnverifiedConsecutiveness), - } - } - Ok(()) - } -} - -pub fn derive_evidence_span_id_v1( - projection: &EvidenceSpanIdentityProjectionV1<'_>, -) -> EvidenceAssemblyStoreResult { - projection.validate()?; - let digest = canonical_identity_digest(EVIDENCE_SPAN_ID_DOMAIN_V1, projection)?; - EvidenceSpanIdV1::new(digest.as_str()).map_err(invalid) -} - -impl EvidenceSpanRecordV1 { - pub fn identity_projection(&self) -> EvidenceSpanIdentityProjectionV1<'_> { - EvidenceSpanIdentityProjectionV1 { - owner: &self.owner, - occurrence_set_id: &self.occurrence_set_id, - ordered_runs: &self.runs, - exact_source_anchors: &self.exact_source_anchors, - projector_version: &self.projector_version, - horizon: &self.horizon, - catalog_binding: &self.catalog_binding, - } - } - - pub fn ordered_occurrence_ids(&self) -> Vec { - self.runs - .iter() - .flat_map(|run| run.occurrence_ids.iter().cloned()) - .collect() - } - - pub fn validate(&self) -> EvidenceAssemblyStoreResult<()> { - self.owner.validate().map_err(invalid)?; - self.projector_version.validate().map_err(invalid)?; - self.catalog_binding.validate()?; - self.anchor.validate().map_err(invalid)?; - match self.anchor.target() { - RetrievalAnchorTargetV3::ExactEvidenceSpan(target) if target == &self.span_id => {} - _ => return Err(invalid("evidence span anchor target")), - } - if self.anchor.owner() != &self.owner { - return Err(invalid("evidence span anchor owner")); - } - validate_member_count(self.runs.len())?; - for (ordinal, run) in self.runs.iter().enumerate() { - run.validate()?; - if run.assembly_ordinal != u64::try_from(ordinal).map_err(invalid)? { - return Err(invalid("evidence span run order")); - } - } - let occurrence_count = self - .runs - .iter() - .map(|run| run.occurrence_ids.len()) - .sum::(); - validate_member_count(occurrence_count)?; - ensure_unique( - self.runs.iter().flat_map(|run| &run.occurrence_ids), - "evidence span occurrences", - )?; - if self.exact_source_anchors.len() != occurrence_count { - return Err(invalid("evidence span exact source cardinality")); - } - if self.span_id != derive_evidence_span_id_v1(&self.identity_projection())? { - return Err(invalid("evidence span identity")); - } - Ok(()) - } -} - -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct EvidenceSpanMemberReceiptBindingV1 { - pub occurrence_id: SourceOccurrenceId, - pub sanitization: SourceOccurrenceSanitizationV1, -} - -impl EvidenceSpanMemberReceiptBindingV1 { - pub fn validate(&self) -> EvidenceAssemblyStoreResult<()> { - self.occurrence_id.validate().map_err(invalid)?; - self.sanitization.validate() - } -} - -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct EvidenceSpanProjectionReceiptV1 { - pub projection_receipt_id: EvidenceSpanProjectionReceiptIdV1, - pub span_id: EvidenceSpanIdV1, - pub projector_snapshot: String, - pub projection_generation: ProjectionGenerationId, - pub projection_watermark: VectorWatermark, - pub source_watermark: ManifestDigest, - pub member_receipts: Vec, - pub ordered_occurrence_ids: Vec, - pub exact_source_anchors: Vec, -} - -#[derive(Clone, Copy, Debug, Serialize, PartialEq, Eq)] -pub struct EvidenceSpanProjectionReceiptIdentityProjectionV1<'a> { - pub span_id: &'a EvidenceSpanIdV1, - pub projector_snapshot: &'a str, - pub projection_generation: &'a ProjectionGenerationId, - pub projection_watermark: &'a VectorWatermark, - pub source_watermark: &'a ManifestDigest, - pub member_receipts: &'a [EvidenceSpanMemberReceiptBindingV1], - pub ordered_occurrence_ids: &'a [SourceOccurrenceId], - pub exact_source_anchors: &'a [RetrievalAnchorId], -} - -impl EvidenceSpanProjectionReceiptIdentityProjectionV1<'_> { - pub fn validate(&self) -> EvidenceAssemblyStoreResult<()> { - EvidenceSpanIdV1::new(self.span_id.as_str()).map_err(invalid)?; - validate_label(self.projector_snapshot, "evidence projector snapshot")?; - self.projection_generation.validate().map_err(invalid)?; - self.source_watermark.validate().map_err(invalid)?; - validate_member_count(self.ordered_occurrence_ids.len())?; - if self.ordered_occurrence_ids.len() != self.exact_source_anchors.len() - || self.member_receipts.len() != self.ordered_occurrence_ids.len() - { - return Err(invalid("evidence projection receipt cardinality")); - } - for binding in self.member_receipts { - binding.validate()?; - } - if self - .member_receipts - .iter() - .map(|binding| &binding.occurrence_id) - .ne(self.ordered_occurrence_ids.iter()) - { - return Err(EvidenceAssemblyStoreError::ReceiptRoleMismatch); - } - ensure_unique( - self.ordered_occurrence_ids, - "evidence projection receipt occurrences", - )?; - Ok(()) - } -} - -pub fn derive_evidence_span_projection_receipt_id_v1( - projection: &EvidenceSpanProjectionReceiptIdentityProjectionV1<'_>, -) -> EvidenceAssemblyStoreResult { - projection.validate()?; - let digest = canonical_identity_digest(PROJECTION_RECEIPT_ID_DOMAIN_V1, projection)?; - EvidenceSpanProjectionReceiptIdV1::new(digest.as_str()).map_err(invalid) -} - -impl EvidenceSpanProjectionReceiptV1 { - pub fn identity_projection(&self) -> EvidenceSpanProjectionReceiptIdentityProjectionV1<'_> { - EvidenceSpanProjectionReceiptIdentityProjectionV1 { - span_id: &self.span_id, - projector_snapshot: &self.projector_snapshot, - projection_generation: &self.projection_generation, - projection_watermark: &self.projection_watermark, - source_watermark: &self.source_watermark, - member_receipts: &self.member_receipts, - ordered_occurrence_ids: &self.ordered_occurrence_ids, - exact_source_anchors: &self.exact_source_anchors, - } - } - - pub fn validate(&self) -> EvidenceAssemblyStoreResult<()> { - self.projection_receipt_id.validate().map_err(invalid)?; - if self.projection_receipt_id - != derive_evidence_span_projection_receipt_id_v1(&self.identity_projection())? - { - return Err(invalid("evidence projection receipt identity")); - } - Ok(()) - } -} - -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct RetrieverIdentityV1 { - pub capability_id: CapabilityId, - pub component_version: ComponentVersion, -} - -impl RetrieverIdentityV1 { - pub fn validate(&self) -> EvidenceAssemblyStoreResult<()> { - self.capability_id.validate().map_err(invalid)?; - self.component_version.validate().map_err(invalid) - } -} - -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct PrivacyBoundRequestEnvelopeV1 { - pub use_case_id: UseCaseId, - pub scope_resolution_id: ScopeResolutionId, - pub temporal_mode: TemporalModeV1, - pub horizon: EvidenceSpanHorizonV1, - pub requested_capabilities: Vec, -} - -impl PrivacyBoundRequestEnvelopeV1 { - pub fn validate(&self) -> EvidenceAssemblyStoreResult<()> { - self.use_case_id.validate().map_err(invalid)?; - self.scope_resolution_id.validate().map_err(invalid)?; - if self.requested_capabilities.is_empty() - || self - .requested_capabilities - .windows(2) - .any(|pair| pair[0] >= pair[1]) - { - return Err(invalid("privacy-bound request capabilities")); - } - for capability in &self.requested_capabilities { - capability.validate().map_err(invalid)?; - } - Ok(()) - } -} - -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct PrivacyBoundRequestDigestV1 { - pub privacy_domain_id: PrivacyDomainId, - pub key_epoch: u64, - pub digest: ManifestDigest, -} - -impl PrivacyBoundRequestDigestV1 { - pub fn derive( - privacy_domain_id: PrivacyDomainId, - key_epoch: u64, - privacy_key: &[u8], - envelope: &PrivacyBoundRequestEnvelopeV1, - ) -> EvidenceAssemblyStoreResult { - privacy_domain_id.validate().map_err(invalid)?; - envelope.validate()?; - if key_epoch == 0 || privacy_key.len() < 16 { - return Err(EvidenceAssemblyStoreError::RequestPrivacyBindingMismatch); - } - let digest = keyed_canonical_digest( - privacy_key, - &( - "tracedecay.privacy-bound-request.v1", - privacy_domain_id.as_str(), - key_epoch, - envelope, - ), - )?; - Ok(Self { - privacy_domain_id, - key_epoch, - digest, - }) - } - - pub fn validate(&self) -> EvidenceAssemblyStoreResult<()> { - self.privacy_domain_id.validate().map_err(invalid)?; - self.digest.validate().map_err(invalid)?; - if self.key_epoch == 0 { - return Err(EvidenceAssemblyStoreError::RequestPrivacyBindingMismatch); - } - Ok(()) - } -} - -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct RetrieverWatermarkBindingV1 { - pub source_watermark: ManifestDigest, - pub projection_watermark: VectorWatermark, - pub index_watermark: Option, - pub summary_watermark: Option, -} - -impl RetrieverWatermarkBindingV1 { - pub fn validate(&self) -> EvidenceAssemblyStoreResult<()> { - self.source_watermark.validate().map_err(invalid)?; - if let Some(index) = &self.index_watermark { - index.validate().map_err(invalid)?; - } - if let Some(summary) = &self.summary_watermark { - summary.validate().map_err(invalid)?; - } - Ok(()) - } -} - -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct RetrieverContributionRecordV1 { - pub contribution_id: RetrieverContributionIdV1, - pub anchor: RetrievalAnchorRecordV3, - pub owner: EvidenceAssemblyOwnerV1, - pub retriever: RetrieverIdentityV1, - pub catalog_binding: SourceCapabilityCatalogBindingV1, - pub request_digest: PrivacyBoundRequestDigestV1, - pub scope_resolution_id: ScopeResolutionId, - pub temporal_mode: TemporalModeV1, - pub watermarks: RetrieverWatermarkBindingV1, - pub horizon: EvidenceSpanHorizonV1, - pub occurrence_set_id: CanonicalSourceOccurrenceSetIdV1, - pub span_id: EvidenceSpanIdV1, - pub span_anchor_id: RetrievalAnchorId, - pub exact_source_anchors: Vec, - pub coverage: CoverageReportV1, - pub created_at: UtcMicros, -} - -#[derive(Clone, Copy, Debug, Serialize, PartialEq, Eq)] -pub struct RetrieverContributionIdentityProjectionV1<'a> { - pub owner: &'a EvidenceAssemblyOwnerV1, - pub retriever: &'a RetrieverIdentityV1, - pub catalog_binding: &'a SourceCapabilityCatalogBindingV1, - pub request_digest: &'a PrivacyBoundRequestDigestV1, - pub scope_resolution_id: &'a ScopeResolutionId, - pub temporal_mode: TemporalModeV1, - pub watermarks: &'a RetrieverWatermarkBindingV1, - pub horizon: &'a EvidenceSpanHorizonV1, - pub occurrence_set_id: &'a CanonicalSourceOccurrenceSetIdV1, - pub span_id: &'a EvidenceSpanIdV1, - pub span_anchor_id: &'a RetrievalAnchorId, - pub exact_source_anchors: &'a [RetrievalAnchorId], - pub coverage: &'a CoverageReportV1, -} - -impl RetrieverContributionIdentityProjectionV1<'_> { - pub fn validate(&self) -> EvidenceAssemblyStoreResult<()> { - self.owner.validate()?; - self.retriever.validate()?; - self.catalog_binding.validate()?; - self.request_digest.validate()?; - self.scope_resolution_id.validate().map_err(invalid)?; - self.watermarks.validate()?; - self.coverage.validate().map_err(invalid)?; - if &self.request_digest.privacy_domain_id != self.owner.owner.privacy_domain_id() - || self.request_digest.key_epoch != self.owner.key_epoch - { - return Err(EvidenceAssemblyStoreError::RequestPrivacyBindingMismatch); - } - self.occurrence_set_id.validate().map_err(invalid)?; - EvidenceSpanIdV1::new(self.span_id.as_str()).map_err(invalid)?; - self.span_anchor_id.validate().map_err(invalid)?; - validate_member_count(self.exact_source_anchors.len())?; - Ok(()) - } -} - -pub fn derive_retriever_contribution_id_v1( - projection: &RetrieverContributionIdentityProjectionV1<'_>, -) -> EvidenceAssemblyStoreResult { - projection.validate()?; - let digest = canonical_identity_digest(RETRIEVER_CONTRIBUTION_ID_DOMAIN_V1, projection)?; - RetrieverContributionIdV1::new(digest.as_str()).map_err(invalid) -} - -impl RetrieverContributionRecordV1 { - pub fn identity_projection(&self) -> RetrieverContributionIdentityProjectionV1<'_> { - RetrieverContributionIdentityProjectionV1 { - owner: &self.owner, - retriever: &self.retriever, - catalog_binding: &self.catalog_binding, - request_digest: &self.request_digest, - scope_resolution_id: &self.scope_resolution_id, - temporal_mode: self.temporal_mode, - watermarks: &self.watermarks, - horizon: &self.horizon, - occurrence_set_id: &self.occurrence_set_id, - span_id: &self.span_id, - span_anchor_id: &self.span_anchor_id, - exact_source_anchors: &self.exact_source_anchors, - coverage: &self.coverage, - } - } - - pub fn validate(&self) -> EvidenceAssemblyStoreResult<()> { - self.contribution_id.validate().map_err(invalid)?; - self.owner.validate()?; - self.anchor.validate().map_err(invalid)?; - match self.anchor.target() { - RetrievalAnchorTargetV3::RetrieverContribution(target) - if target == &self.contribution_id => {} - _ => return Err(invalid("retriever contribution anchor target")), - } - if self.anchor.owner() != &self.owner.owner { - return Err(invalid("retriever contribution anchor owner")); - } - validate_derived_anchor_lineage( - &self.anchor, - &self.owner.owner, - std::slice::from_ref(&self.span_anchor_id), - "retriever contribution anchor lineage", - )?; - if self.contribution_id != derive_retriever_contribution_id_v1(&self.identity_projection())? - { - return Err(invalid("retriever contribution identity")); - } - Ok(()) - } -} - -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct EvidenceAssemblyPublicationReceiptV1 { - pub publication_receipt_id: EvidenceAssemblyPublicationReceiptIdV1, - pub owner: EvidenceAssemblyOwnerV1, - pub assembly_digest: ManifestDigest, - pub occurrence_set_id: CanonicalSourceOccurrenceSetIdV1, - pub span_id: EvidenceSpanIdV1, - pub span_anchor_id: RetrievalAnchorId, - pub contribution_id: RetrieverContributionIdV1, - pub contribution_anchor_id: RetrievalAnchorId, - pub projection_receipt_id: EvidenceSpanProjectionReceiptIdV1, - pub ordered_occurrence_ids: Vec, - pub exact_source_anchors: Vec, -} - -#[derive(Clone, Copy, Debug, Serialize, PartialEq, Eq)] -pub struct EvidenceAssemblyPublicationIdentityProjectionV1<'a> { - pub owner: &'a EvidenceAssemblyOwnerV1, - pub idempotency_key: &'a EvidenceAssemblyIdempotencyKeyV1, - pub assembly_digest: &'a ManifestDigest, - pub occurrence_set_id: &'a CanonicalSourceOccurrenceSetIdV1, - pub span_id: &'a EvidenceSpanIdV1, - pub span_anchor_id: &'a RetrievalAnchorId, - pub contribution_id: &'a RetrieverContributionIdV1, - pub contribution_anchor_id: &'a RetrievalAnchorId, - pub projection_receipt_id: &'a EvidenceSpanProjectionReceiptIdV1, - pub ordered_occurrence_ids: &'a [SourceOccurrenceId], - pub exact_source_anchors: &'a [RetrievalAnchorId], -} - -impl EvidenceAssemblyPublicationIdentityProjectionV1<'_> { - pub fn validate(&self) -> EvidenceAssemblyStoreResult<()> { - self.owner.validate()?; - self.idempotency_key - .as_digest() - .validate() - .map_err(invalid)?; - self.assembly_digest.validate().map_err(invalid)?; - self.occurrence_set_id.validate().map_err(invalid)?; - EvidenceSpanIdV1::new(self.span_id.as_str()).map_err(invalid)?; - self.span_anchor_id.validate().map_err(invalid)?; - self.contribution_id.validate().map_err(invalid)?; - self.contribution_anchor_id.validate().map_err(invalid)?; - self.projection_receipt_id.validate().map_err(invalid)?; - if self.ordered_occurrence_ids.is_empty() - || self.ordered_occurrence_ids.len() != self.exact_source_anchors.len() - { - return Err(invalid("evidence publication receipt cardinality")); - } - ensure_unique( - self.ordered_occurrence_ids, - "evidence publication receipt occurrences", - )?; - Ok(()) - } -} - -pub fn derive_evidence_assembly_publication_receipt_id_v1( - projection: &EvidenceAssemblyPublicationIdentityProjectionV1<'_>, -) -> EvidenceAssemblyStoreResult { - projection.validate()?; - let digest = canonical_identity_digest(PUBLICATION_RECEIPT_ID_DOMAIN_V1, projection)?; - EvidenceAssemblyPublicationReceiptIdV1::new(digest.as_str()).map_err(invalid) -} - -impl EvidenceAssemblyPublicationReceiptV1 { - pub fn validate(&self) -> EvidenceAssemblyStoreResult<()> { - self.publication_receipt_id.validate().map_err(invalid)?; - self.owner.validate()?; - self.assembly_digest.validate().map_err(invalid)?; - if self.ordered_occurrence_ids.is_empty() - || self.ordered_occurrence_ids.len() != self.exact_source_anchors.len() - { - return Err(invalid("evidence publication receipt cardinality")); - } - Ok(()) - } - - /// The idempotency key lives on the write, not the receipt, so the caller - /// lends it alongside the receipt's own fields. - pub fn identity_projection<'a>( - &'a self, - idempotency_key: &'a EvidenceAssemblyIdempotencyKeyV1, - ) -> EvidenceAssemblyPublicationIdentityProjectionV1<'a> { - EvidenceAssemblyPublicationIdentityProjectionV1 { - owner: &self.owner, - idempotency_key, - assembly_digest: &self.assembly_digest, - occurrence_set_id: &self.occurrence_set_id, - span_id: &self.span_id, - span_anchor_id: &self.span_anchor_id, - contribution_id: &self.contribution_id, - contribution_anchor_id: &self.contribution_anchor_id, - projection_receipt_id: &self.projection_receipt_id, - ordered_occurrence_ids: &self.ordered_occurrence_ids, - exact_source_anchors: &self.exact_source_anchors, - } - } -} - -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct EvidenceAssemblyWriteV1 { - pub owner: EvidenceAssemblyOwnerV1, - pub idempotency_key: EvidenceAssemblyIdempotencyKeyV1, - pub occurrences: Vec, - pub occurrence_set: CanonicalSourceOccurrenceSetRecordV1, - pub span: EvidenceSpanRecordV1, - pub projection_receipt: EvidenceSpanProjectionReceiptV1, - pub contribution: RetrieverContributionRecordV1, - pub receipt: EvidenceAssemblyPublicationReceiptV1, -} - -impl EvidenceAssemblyWriteV1 { - #[hotpath::measure(label = "store.evidence_assembly.validate_write")] - pub fn validate(&self) -> EvidenceAssemblyStoreResult<()> { - self.owner.validate()?; - validate_member_count(self.occurrences.len())?; - let mut occurrence_ids = Vec::with_capacity(self.occurrences.len()); - let mut source_anchors = Vec::with_capacity(self.occurrences.len()); - let mut occurrence_anchor_ids = Vec::with_capacity(self.occurrences.len()); - for occurrence in &self.occurrences { - occurrence.validate()?; - if occurrence.owner != self.owner.owner { - return Err(invalid("evidence occurrence owner")); - } - occurrence_ids.push(occurrence.occurrence_id.clone()); - source_anchors.push(occurrence.exact_source_anchor.clone()); - occurrence_anchor_ids.push(occurrence.occurrence_anchor.anchor_id().clone()); - } - ensure_unique(&occurrence_ids, "evidence assembly occurrences")?; - let by_id = self - .occurrences - .iter() - .map(|occurrence| (&occurrence.occurrence_id, occurrence)) - .collect::>(); - for occurrence in &self.occurrences { - let owner_scope_matches = - match (occurrence.owner.project_id(), &occurrence.timeline.scope) { - (None, ObservationScopeV1::Profile) => true, - ( - Some(owner_project), - ObservationScopeV1::Project { - project_id: source_project, - }, - ) => owner_project == source_project, - _ => false, - }; - if !owner_scope_matches { - return Err(invalid("source occurrence timeline owner scope")); - } - if let SourceOccurrenceCoordinateV1::ObservationProjection { source_range, .. } = - &occurrence.coordinate - && occurrence.source_order != source_range.start() - { - return Err(EvidenceAssemblyStoreError::IncomparableSourceOrder); - } - for relation in &occurrence.relations { - if let SourceOccurrenceRelationV1::ToolResultFor { - invocation_occurrence_id, - } = relation - { - let Some(invocation) = by_id.get(invocation_occurrence_id) else { - return Err(invalid("tool result invocation occurrence")); - }; - if invocation.occurrence_kind != SourceOccurrenceKindV1::ToolInvocation - || invocation.owner != occurrence.owner - || invocation.timeline != occurrence.timeline - { - return Err(invalid("tool result invocation binding")); - } - } - } - } - self.occurrence_set.validate()?; - self.span.validate()?; - self.span.horizon.validate_members(&self.occurrences)?; - for run in &self.span.runs { - for (ordinal, occurrence_id) in run.occurrence_ids.iter().enumerate() { - let Some(occurrence) = by_id.get(occurrence_id) else { - return Err(invalid("evidence run occurrence")); - }; - if occurrence.timeline != run.timeline - || run.ordering_proof.source_orders.get(ordinal).copied() - != Some(occurrence.source_order) - { - return Err(EvidenceAssemblyStoreError::IncomparableSourceOrder); - } - } - } - self.projection_receipt.validate()?; - self.contribution.validate()?; - self.receipt.validate()?; - validate_derived_anchor_lineage( - &self.span.anchor, - &self.owner.owner, - &occurrence_anchor_ids, - "evidence span anchor lineage", - )?; - let catalog_mismatch = self - .span - .catalog_binding - .source_capability() - .is_some_and(|binding| binding != &self.contribution.catalog_binding); - let mut canonical_occurrences = occurrence_ids.clone(); - canonical_occurrences.sort(); - let ordered_span_occurrences = self.span.ordered_occurrence_ids(); - if self.occurrence_set.owner != self.owner.owner - || self.span.owner != self.owner.owner - || self.contribution.owner != self.owner - || self.receipt.owner != self.owner - || self.occurrence_set.members != canonical_occurrences - || ordered_span_occurrences != occurrence_ids - || self.span.exact_source_anchors != source_anchors - || self.projection_receipt.span_id != self.span.span_id - || self.projection_receipt.ordered_occurrence_ids != occurrence_ids - || self.projection_receipt.exact_source_anchors != source_anchors - || self - .projection_receipt - .member_receipts - .iter() - .map(|binding| &binding.sanitization) - .ne(self - .occurrences - .iter() - .map(|occurrence| &occurrence.sanitization)) - || self.contribution.occurrence_set_id != self.occurrence_set.occurrence_set_id - || self.contribution.span_id != self.span.span_id - || self.contribution.span_anchor_id != *self.span.anchor.anchor_id() - || self.contribution.exact_source_anchors != source_anchors - || self.contribution.horizon != self.span.horizon - || catalog_mismatch - || self.receipt.occurrence_set_id != self.occurrence_set.occurrence_set_id - || self.receipt.span_id != self.span.span_id - || self.receipt.span_anchor_id != *self.span.anchor.anchor_id() - || self.receipt.contribution_id != self.contribution.contribution_id - || self.receipt.contribution_anchor_id != *self.contribution.anchor.anchor_id() - || self.receipt.projection_receipt_id != self.projection_receipt.projection_receipt_id - || self.receipt.ordered_occurrence_ids != occurrence_ids - || self.receipt.exact_source_anchors != source_anchors - { - return Err(invalid("evidence assembly cross-record binding")); - } - let expected_digest = self.compute_assembly_digest()?; - if self.receipt.assembly_digest != expected_digest { - return Err(invalid("evidence assembly digest")); - } - let expected_receipt_id = derive_evidence_assembly_publication_receipt_id_v1( - &self.receipt.identity_projection(&self.idempotency_key), - )?; - if self.receipt.publication_receipt_id != expected_receipt_id { - return Err(invalid("evidence assembly publication identity")); - } - Ok(()) - } - - pub fn compute_assembly_digest(&self) -> EvidenceAssemblyStoreResult { - canonical_sha256(&( - "tracedecay.evidence-assembly.write.v1", - &self.owner, - &self.idempotency_key, - &self.occurrences, - &self.occurrence_set, - &self.span, - &self.projection_receipt, - &self.contribution, - )) - .map_err(invalid) - } -} - -fn validate_derived_anchor_lineage( - anchor: &RetrievalAnchorRecordV3, - owner: &AnchorOwnerBindingV1, - expected_sources: &[RetrievalAnchorId], - field: &'static str, -) -> EvidenceAssemblyStoreResult<()> { - if anchor.source_anchors().len() != expected_sources.len() { - return Err(invalid(field)); - } - for (ordinal, (source, expected_id)) in anchor - .source_anchors() - .iter() - .zip(expected_sources) - .enumerate() - { - if source.source_ordinal() != u64::try_from(ordinal).map_err(invalid)? - || source.anchor_id() != expected_id - || source.owner() != owner - { - return Err(invalid(field)); - } - } - Ok(()) -} - -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct EvidenceAssemblyDrilldownPageV1 { - pub contribution: RetrieverContributionRecordV1, - pub span: EvidenceSpanRecordV1, - pub occurrence_set_id: CanonicalSourceOccurrenceSetIdV1, - pub occurrences: Vec, - pub next_ordinal: Option, -} - -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum EvidenceAssemblyReadOperationV1 { - PublicationByIdempotency { - owner: EvidenceAssemblyOwnerV1, - idempotency_key: EvidenceAssemblyIdempotencyKeyV1, - }, - ContributionPage { - owner: EvidenceAssemblyOwnerV1, - contribution_id: RetrieverContributionIdV1, - start_ordinal: u64, - page_size: u64, - }, -} - -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -// Boxing the large variant is wire-transparent but would change this public -// store-protocol API and ripple through construction/match sites. -#[allow(clippy::large_enum_variant)] -pub enum EvidenceAssemblyReadResultV1 { - Publication(Option), - ContributionPage(Option), -} - -fn validate_member_count(count: usize) -> EvidenceAssemblyStoreResult<()> { - if count == 0 || count > MAX_EVIDENCE_ASSEMBLY_MEMBERS_V1 { - return Err(invalid("evidence assembly member count")); - } - Ok(()) -} - -fn validate_half_open( - start: u64, - end: u64, - field: &'static str, -) -> EvidenceAssemblyStoreResult<()> { - if start >= end { - return Err(invalid(field)); - } - Ok(()) -} - -fn ensure_unique( - values: impl IntoIterator, - field: &'static str, -) -> EvidenceAssemblyStoreResult<()> { - let mut seen = BTreeSet::new(); - if values.into_iter().any(|value| !seen.insert(value)) { - return Err(invalid(field)); - } - Ok(()) -} - -fn validate_label(value: &str, field: &'static str) -> EvidenceAssemblyStoreResult<()> { - if !is_canonical_text_within(value, CANONICAL_TEXT_MAX_BYTES) { - return Err(invalid(field)); - } - Ok(()) -} - -fn invalid(error: impl std::fmt::Display) -> EvidenceAssemblyStoreError { - EvidenceAssemblyStoreError::InvalidData(error.to_string()) -} - -fn canonical_identity_digest( - domain: &'static str, - projection: &T, -) -> EvidenceAssemblyStoreResult { - hotpath::measure_block!( - "store.evidence_assembly.identity_digest", - canonical_sha256(&(domain, projection)).map_err(invalid) - ) -} - -fn keyed_canonical_digest( - key: &[u8], - material: &T, -) -> EvidenceAssemblyStoreResult { - canonical_sha256(&("tracedecay.privacy-keyed-digest.v1", key, material)).map_err(invalid) -} - -#[cfg(test)] -mod tests { - use super::*; - use tracedecay_domain::{ - AccessPolicyDigest, AnchorDurabilityClass, AnchorLineageRefV3, AnchorProvenanceRelationV2, - AnchorSourceGenerationV3, BlobId, EvidenceClass, PayloadAccessState, - PrivacyDomainBoundLocatorDigest, PrivacyDomainId, ProjectId, ProviderId, RepositoryId, - ResolutionAuthorizationV1, RetentionClass, SessionId, UserProfileId, - }; - - fn owner() -> EvidenceAssemblyOwnerV1 { - EvidenceAssemblyOwnerV1 { - owner: AnchorOwnerBindingV1::for_project( - UserProfileId::new("profile.fixture").unwrap(), - ProjectId::new("project.fixture").unwrap(), - PrivacyDomainId::new("privacy.fixture").unwrap(), - ) - .unwrap(), - scope_digest: ManifestDigest::new(format!("sha256:{}", "aa".repeat(32))).unwrap(), - key_epoch: 1, - } - } - - /// Owned identity material a test mutates before borrowing it as the - /// projection the derivation consumes. - struct OccurrenceIdentityFixture { - owner: AnchorOwnerBindingV1, - timeline: EvidenceSourceTimelineV1, - exact_source_anchor: RetrievalAnchorId, - source_order: u64, - coordinate: SourceOccurrenceCoordinateV1, - occurrence_kind: SourceOccurrenceKindV1, - relations: Vec, - projector_version: ComponentVersion, - } - - impl OccurrenceIdentityFixture { - fn projection(&self) -> SourceOccurrenceIdentityProjectionV1<'_> { - SourceOccurrenceIdentityProjectionV1 { - owner: &self.owner, - timeline: &self.timeline, - exact_source_anchor: &self.exact_source_anchor, - source_order: self.source_order, - coordinate: &self.coordinate, - occurrence_kind: self.occurrence_kind, - relations: &self.relations, - projector_version: &self.projector_version, - } - } - - fn id(&self) -> EvidenceAssemblyStoreResult { - derive_source_occurrence_id_v1(&self.projection()) - } - } - - fn occurrence_projection() -> OccurrenceIdentityFixture { - OccurrenceIdentityFixture { - owner: owner().owner, - timeline: EvidenceSourceTimelineV1 { - source: ObservationSourceIdentityV1::for_provider( - ProviderId::new("provider.fixture").unwrap(), - SessionId::new("session.fixture").unwrap(), - ) - .unwrap(), - scope: ObservationScopeV1::Project { - project_id: ProjectId::new("project.fixture").unwrap(), - }, - source_generation: ObservationSourceGenerationV1::new(1).unwrap(), - ordering_domain: ObservationOrderingDomainV1::DaemonSequence, - }, - exact_source_anchor: RetrievalAnchorId::new("retrieval.source.fixture").unwrap(), - source_order: 4, - coordinate: SourceOccurrenceCoordinateV1::ObservationProjection { - canonical_observation_id: CanonicalObservationIdV1::new(format!( - "sha256:{}", - "44".repeat(32) - )) - .unwrap(), - source_range: ObservationSourceRangeV1::new(4, 5).unwrap(), - projection_output_ordinal: 0, - sanitized_byte_range: SanitizedObservationByteRangeV1::new(0, 1).unwrap(), - }, - occurrence_kind: SourceOccurrenceKindV1::Message, - relations: Vec::new(), - projector_version: ComponentVersion::new("projector.v1").unwrap(), - } - } - - fn catalog_binding() -> SourceCapabilityCatalogBindingV1 { - let digest = ManifestDigest::new(format!("sha256:{}", "aa".repeat(32))).unwrap(); - SourceCapabilityCatalogBindingV1 { - connector_id: "connector.fixture".to_owned(), - root_id: "root.fixture".to_owned(), - capability_id: CapabilityId::new("capability.fixture").unwrap(), - catalog_digest: digest.clone(), - integration_manifest_digest: digest.clone(), - configuration_digest: digest.clone(), - authorization_scope_digest: digest.clone(), - projector_revision: ComponentVersion::new("projector.v1").unwrap(), - source_watermark: digest, - } - } - - fn retrieval_anchor( - target: RetrievalAnchorTargetV3, - sources: Vec, - ) -> RetrievalAnchorRecordV3 { - let owner = owner().owner; - RetrievalAnchorRecordV3::new(tracedecay_domain::RetrievalAnchorRecordV3Parts { - target, - owner: owner.clone(), - aliases: vec![], - occurred_at: None, - ingested_at: UtcMicros(1), - evidence_class: EvidenceClass::Observed, - source_generation: AnchorSourceGenerationV3::Unknown, - projection_generation: ProjectionGenerationId::new("projection.fixture").unwrap(), - projection_watermark: VectorWatermark::default(), - coverage: CoverageReportV1::default(), - source_observations: vec![], - source_anchors: sources - .into_iter() - .enumerate() - .map(|(ordinal, source)| { - AnchorLineageRefV3::new( - u64::try_from(ordinal).unwrap(), - AnchorProvenanceRelationV2::DerivedFrom, - source, - owner.clone(), - ) - .unwrap() - }) - .collect(), - authorization: ResolutionAuthorizationV1 { - resolved_scope_id: ScopeResolutionId::new("scope.fixture").unwrap(), - privacy_domain_id: PrivacyDomainId::new("privacy.fixture").unwrap(), - access_policy_digest: AccessPolicyDigest::new(format!( - "sha256:{}", - "aa".repeat(32) - )) - .unwrap(), - capability_id: CapabilityId::new("capability.fixture").unwrap(), - canonical_request_digest: PrivacyDomainBoundLocatorDigest::new(format!( - "sha256:{}", - "bb".repeat(32) - )) - .unwrap(), - }, - payload_access: PayloadAccessState::Eligible, - retention_class: RetentionClass::new("retention.fixture").unwrap(), - durability: AnchorDurabilityClass::DurableEvidence, - }) - .unwrap() - } - - #[test] - fn privacy_bound_digests_separate_domains_epochs_and_keys() { - let envelope = PrivacyBoundRequestEnvelopeV1 { - use_case_id: UseCaseId::new("use-case.fixture").unwrap(), - scope_resolution_id: ScopeResolutionId::new("scope.fixture").unwrap(), - temporal_mode: TemporalModeV1::Current, - horizon: EvidenceSpanHorizonV1 { - knowledge_through: UtcMicros(1), - valid_through: Some(UtcMicros(1)), - contains_unknown_valid_time: false, - }, - requested_capabilities: vec![CapabilityId::new("capability.fixture").unwrap()], - }; - let privacy_one = PrivacyDomainId::new("privacy.fixture").unwrap(); - let privacy_two = PrivacyDomainId::new("privacy.other").unwrap(); - let key_one = b"fixture-privacy-key-one"; - let key_two = b"fixture-privacy-key-two"; - let first = PrivacyBoundRequestDigestV1::derive(privacy_one.clone(), 1, key_one, &envelope) - .unwrap(); - assert_ne!( - first, - PrivacyBoundRequestDigestV1::derive(privacy_two.clone(), 1, key_one, &envelope) - .unwrap() - ); - assert_ne!( - first, - PrivacyBoundRequestDigestV1::derive(privacy_one.clone(), 2, key_one, &envelope) - .unwrap() - ); - assert_ne!( - first, - PrivacyBoundRequestDigestV1::derive(privacy_one, 1, key_two, &envelope).unwrap() - ); - - let owner_one = owner().owner; - let owner_two = AnchorOwnerBindingV1::for_project( - UserProfileId::new("profile.fixture").unwrap(), - ProjectId::new("project.fixture").unwrap(), - privacy_two, - ) - .unwrap(); - assert_ne!( - EvidenceAssemblyIdempotencyKeyV1::derive(&owner_one, 1, key_one, b"caller-key") - .unwrap(), - EvidenceAssemblyIdempotencyKeyV1::derive(&owner_two, 1, key_one, b"caller-key") - .unwrap() - ); - } - - #[test] - fn occurrence_identity_is_deterministic_and_rekeys_immutable_material() { - let projection = occurrence_projection(); - let replay = projection.id().unwrap(); - assert_eq!(replay, projection.id().unwrap()); - - let mut changed = projection; - changed.projector_version = ComponentVersion::new("projector.v2").unwrap(); - assert_ne!(replay, changed.id().unwrap()); - } - - #[test] - fn occurrence_anchor_binds_exact_lineage() { - let projection = occurrence_projection(); - let occurrence_id = projection.id().unwrap(); - let anchor = retrieval_anchor( - RetrievalAnchorTargetV3::ExactSourceOccurrence(occurrence_id), - vec![projection.exact_source_anchor.clone()], - ); - validate_derived_anchor_lineage( - &anchor, - &projection.owner, - std::slice::from_ref(&projection.exact_source_anchor), - "test occurrence lineage", - ) - .unwrap(); - assert!( - validate_derived_anchor_lineage( - &anchor, - &projection.owner, - &[RetrievalAnchorId::new("retrieval.other.fixture").unwrap()], - "test occurrence lineage", - ) - .is_err() - ); - } - - #[test] - fn occurrence_set_identity_requires_canonical_membership_order() { - let first = SourceOccurrenceId::new(format!("sha256:{}", "11".repeat(32))).unwrap(); - let second = SourceOccurrenceId::new(format!("sha256:{}", "22".repeat(32))).unwrap(); - let owner = owner().owner; - let canonical = CanonicalSourceOccurrenceSetIdentityProjectionV1 { - owner: &owner, - canonical_members: &[first.clone(), second.clone()], - }; - assert!( - derive_canonical_source_occurrence_set_id_v1(&canonical) - .unwrap() - .as_str() - .starts_with("sha256:") - ); - assert!(matches!( - derive_canonical_source_occurrence_set_id_v1( - &CanonicalSourceOccurrenceSetIdentityProjectionV1 { - owner: &owner, - canonical_members: &[second, first], - } - ), - Err(EvidenceAssemblyStoreError::InvalidData(_)) - )); - } - - #[test] - fn mixed_message_tool_and_code_runs_reject_order_and_kind_lookalikes() { - let message_projection = occurrence_projection(); - let message_id = message_projection.id().unwrap(); - - let mut invocation_projection = occurrence_projection(); - invocation_projection.source_order = 5; - invocation_projection.coordinate = observation_coordinate(5, "55"); - invocation_projection.occurrence_kind = SourceOccurrenceKindV1::ToolInvocation; - let invocation_id = invocation_projection.id().unwrap(); - - let mut result_projection = occurrence_projection(); - result_projection.source_order = 6; - result_projection.coordinate = observation_coordinate(6, "66"); - result_projection.occurrence_kind = SourceOccurrenceKindV1::ToolResult; - result_projection.relations = vec![SourceOccurrenceRelationV1::ToolResultFor { - invocation_occurrence_id: invocation_id.clone(), - }]; - let result_id = result_projection.id().unwrap(); - - let code_timeline = SourceTimelineKeyV1 { - source: ObservationSourceIdentityV1::for_provider( - ProviderId::new("git.fixture").unwrap(), - SessionId::new("capture.fixture").unwrap(), - ) - .unwrap(), - scope: ObservationScopeV1::Project { - project_id: ProjectId::new("project.fixture").unwrap(), - }, - source_generation: ObservationSourceGenerationV1::new(2).unwrap(), - ordering_domain: ObservationOrderingDomainV1::FileBytes, - }; - let code_projection = OccurrenceIdentityFixture { - owner: owner().owner, - timeline: code_timeline.clone(), - exact_source_anchor: RetrievalAnchorId::new("retrieval.code.fixture").unwrap(), - source_order: 0, - coordinate: SourceOccurrenceCoordinateV1::ImmutableBlobSlice { - repository_id: RepositoryId::new("repository.fixture").unwrap(), - blob_id: BlobId::new("blob.fixture").unwrap(), - byte_start: 0, - byte_end: 8, - }, - occurrence_kind: SourceOccurrenceKindV1::CodeChunk, - relations: Vec::new(), - projector_version: ComponentVersion::new("projector.v1").unwrap(), - }; - let code_id = code_projection.id().unwrap(); - - let observation_ids = vec![message_id.clone(), invocation_id.clone(), result_id.clone()]; - let observation_run = EvidenceSpanRunV1 { - assembly_ordinal: 0, - timeline: message_projection.timeline.clone(), - ordering_proof: VerifiedSourceOrderingProofV1::verify( - message_projection.timeline.clone(), - catalog_binding(), - catalog_binding(), - observation_ids.clone(), - vec![4, 5, 6], - ) - .unwrap(), - timeline_digest: message_projection.timeline.digest().unwrap(), - first_source_order: 4, - last_source_order: 6, - occurrence_ids: observation_ids, - }; - let code_run = EvidenceSpanRunV1 { - assembly_ordinal: 1, - timeline: code_timeline.clone(), - ordering_proof: VerifiedSourceOrderingProofV1::verify( - code_timeline.clone(), - catalog_binding(), - catalog_binding(), - vec![code_id.clone()], - vec![0], - ) - .unwrap(), - timeline_digest: code_timeline.digest().unwrap(), - first_source_order: 0, - last_source_order: 0, - occurrence_ids: vec![code_id.clone()], - }; - let owner = owner().owner; - let mut canonical_members = vec![ - message_id.clone(), - invocation_id.clone(), - result_id, - code_id, - ]; - canonical_members.sort(); - let occurrence_set_id = derive_canonical_source_occurrence_set_id_v1( - &CanonicalSourceOccurrenceSetIdentityProjectionV1 { - owner: &owner, - canonical_members: &canonical_members, - }, - ) - .unwrap(); - let observation_anchor = RetrievalAnchorId::new("retrieval.source.fixture").unwrap(); - let mut ordered_runs = vec![observation_run, code_run]; - let exact_source_anchors = [ - observation_anchor.clone(), - observation_anchor.clone(), - observation_anchor, - RetrievalAnchorId::new("retrieval.code.fixture").unwrap(), - ]; - let projector_version = ComponentVersion::new("projector.v1").unwrap(); - let horizon = EvidenceSpanHorizonV1 { - knowledge_through: UtcMicros(7), - valid_through: Some(UtcMicros(7)), - contains_unknown_valid_time: false, - }; - let span_catalog_binding = EvidenceSpanCatalogBindingV1::SourceCapability { - binding: catalog_binding(), - }; - let span_projection = |ordered_runs: &[EvidenceSpanRunV1]| { - derive_evidence_span_id_v1(&EvidenceSpanIdentityProjectionV1 { - owner: &owner, - occurrence_set_id: &occurrence_set_id, - ordered_runs, - exact_source_anchors: &exact_source_anchors, - projector_version: &projector_version, - horizon: &horizon, - catalog_binding: &span_catalog_binding, - }) - }; - let forward = span_projection(&ordered_runs).unwrap(); - ordered_runs.reverse(); - for (ordinal, run) in ordered_runs.iter_mut().enumerate() { - run.assembly_ordinal = u64::try_from(ordinal).unwrap(); - } - assert_ne!(forward, span_projection(&ordered_runs).unwrap()); - - let mut missing_pair = result_projection; - missing_pair.relations.clear(); - assert!(missing_pair.id().is_err()); - assert!(matches!( - VerifiedSourceOrderingProofV1::verify( - message_projection.timeline, - catalog_binding(), - catalog_binding(), - vec![message_id, invocation_id], - vec![4, 6], - ), - Err(EvidenceAssemblyStoreError::UnverifiedConsecutiveness) - )); - let mut code_lookalike = invocation_projection; - code_lookalike.occurrence_kind = SourceOccurrenceKindV1::CodeChunk; - assert!(code_lookalike.id().is_err()); - } - - /// Every durable identity minted from one deterministic assembly, in the - /// order the six record kinds are derived. - fn assembly_identities() -> Vec { - let message = occurrence_projection(); - let message_id = message.id().unwrap(); - let mut invocation = occurrence_projection(); - invocation.source_order = 5; - invocation.coordinate = observation_coordinate(5, "55"); - invocation.occurrence_kind = SourceOccurrenceKindV1::ToolInvocation; - let invocation_id = invocation.id().unwrap(); - let mut result = occurrence_projection(); - result.source_order = 6; - result.coordinate = observation_coordinate(6, "66"); - result.occurrence_kind = SourceOccurrenceKindV1::ToolResult; - result.relations = vec![SourceOccurrenceRelationV1::ToolResultFor { - invocation_occurrence_id: invocation_id.clone(), - }]; - let result_id = result.id().unwrap(); - let code_timeline = SourceTimelineKeyV1 { - source: ObservationSourceIdentityV1::for_provider( - ProviderId::new("git.fixture").unwrap(), - SessionId::new("capture.fixture").unwrap(), - ) - .unwrap(), - scope: ObservationScopeV1::Project { - project_id: ProjectId::new("project.fixture").unwrap(), - }, - source_generation: ObservationSourceGenerationV1::new(2).unwrap(), - ordering_domain: ObservationOrderingDomainV1::FileBytes, - }; - let code = OccurrenceIdentityFixture { - owner: owner().owner, - timeline: code_timeline.clone(), - exact_source_anchor: RetrievalAnchorId::new("retrieval.code.fixture").unwrap(), - source_order: 0, - coordinate: SourceOccurrenceCoordinateV1::CapturedWorktreeSlice { - repository_id: RepositoryId::new("repository.fixture").unwrap(), - repository_capture_id: tracedecay_domain::RepositoryCaptureId::new( - "capture.fixture", - ) - .unwrap(), - path_locator_digest: PrivacyDomainBoundLocatorDigest::new(format!( - "sha256:{}", - "77".repeat(32) - )) - .unwrap(), - byte_start: 0, - byte_end: 8, - }, - occurrence_kind: SourceOccurrenceKindV1::CodeChunk, - relations: Vec::new(), - projector_version: ComponentVersion::new("projector.v1").unwrap(), - }; - let code_id = code.id().unwrap(); - - let owner = owner(); - let mut canonical_members = vec![ - message_id.clone(), - invocation_id.clone(), - result_id.clone(), - code_id.clone(), - ]; - canonical_members.sort(); - let occurrence_set_id = derive_canonical_source_occurrence_set_id_v1( - &CanonicalSourceOccurrenceSetIdentityProjectionV1 { - owner: &owner.owner, - canonical_members: &canonical_members, - }, - ) - .unwrap(); - - let observation_ids = vec![message_id.clone(), invocation_id.clone(), result_id.clone()]; - let observation_run = EvidenceSpanRunV1 { - assembly_ordinal: 0, - timeline: message.timeline.clone(), - ordering_proof: VerifiedSourceOrderingProofV1::verify( - message.timeline.clone(), - catalog_binding(), - catalog_binding(), - observation_ids.clone(), - vec![4, 5, 6], - ) - .unwrap(), - timeline_digest: message.timeline.digest().unwrap(), - first_source_order: 4, - last_source_order: 6, - occurrence_ids: observation_ids, - }; - let code_run = EvidenceSpanRunV1 { - assembly_ordinal: 1, - timeline: code_timeline.clone(), - ordering_proof: VerifiedSourceOrderingProofV1::verify( - code_timeline.clone(), - catalog_binding(), - catalog_binding(), - vec![code_id.clone()], - vec![0], - ) - .unwrap(), - timeline_digest: code_timeline.digest().unwrap(), - first_source_order: 0, - last_source_order: 0, - occurrence_ids: vec![code_id.clone()], - }; - let observation_anchor = RetrievalAnchorId::new("retrieval.source.fixture").unwrap(); - let exact_source_anchors = vec![ - observation_anchor.clone(), - observation_anchor.clone(), - observation_anchor, - RetrievalAnchorId::new("retrieval.code.fixture").unwrap(), - ]; - let horizon = EvidenceSpanHorizonV1 { - knowledge_through: UtcMicros(7), - valid_through: None, - contains_unknown_valid_time: true, - }; - let projector_version = ComponentVersion::new("projector.v1").unwrap(); - let span_id = derive_evidence_span_id_v1(&EvidenceSpanIdentityProjectionV1 { - owner: &owner.owner, - occurrence_set_id: &occurrence_set_id, - ordered_runs: &[observation_run, code_run], - exact_source_anchors: &exact_source_anchors, - projector_version: &projector_version, - horizon: &horizon, - catalog_binding: &EvidenceSpanCatalogBindingV1::SourceCapability { - binding: catalog_binding(), - }, - }) - .unwrap(); - - let ordered_occurrence_ids = vec![ - message_id.clone(), - invocation_id.clone(), - result_id.clone(), - code_id.clone(), - ]; - let sanitization = SourceOccurrenceSanitizationV1::new( - SanitizationReceiptRefV1::new( - tracedecay_domain::SanitizationReceiptId::new("receipt.capture.fixture").unwrap(), - ComponentVersion::new("sanitizer.v1").unwrap(), - ) - .unwrap(), - SanitizationReceiptRefV1::new( - tracedecay_domain::SanitizationReceiptId::new("receipt.projection.fixture") - .unwrap(), - ComponentVersion::new("sanitizer.v1").unwrap(), - ) - .unwrap(), - ) - .unwrap(); - let member_receipts = ordered_occurrence_ids - .iter() - .map(|occurrence_id| EvidenceSpanMemberReceiptBindingV1 { - occurrence_id: occurrence_id.clone(), - sanitization: sanitization.clone(), - }) - .collect::>(); - let digest = ManifestDigest::new(format!("sha256:{}", "aa".repeat(32))).unwrap(); - let projection_receipt_id = derive_evidence_span_projection_receipt_id_v1( - &EvidenceSpanProjectionReceiptIdentityProjectionV1 { - span_id: &span_id, - projector_snapshot: "projector.snapshot.fixture", - projection_generation: &ProjectionGenerationId::new("projection.fixture").unwrap(), - projection_watermark: &VectorWatermark::default(), - source_watermark: &digest, - member_receipts: &member_receipts, - ordered_occurrence_ids: &ordered_occurrence_ids, - exact_source_anchors: &exact_source_anchors, - }, - ) - .unwrap(); - - let request_digest = PrivacyBoundRequestDigestV1 { - privacy_domain_id: owner.owner.privacy_domain_id().clone(), - key_epoch: owner.key_epoch, - digest: ManifestDigest::new(format!("sha256:{}", "bb".repeat(32))).unwrap(), - }; - let span_anchor_id = RetrievalAnchorId::new("retrieval.span.fixture").unwrap(); - let contribution_id = - derive_retriever_contribution_id_v1(&RetrieverContributionIdentityProjectionV1 { - owner: &owner, - retriever: &RetrieverIdentityV1 { - capability_id: CapabilityId::new("capability.fixture").unwrap(), - component_version: ComponentVersion::new("retriever.v1").unwrap(), - }, - catalog_binding: &catalog_binding(), - request_digest: &request_digest, - scope_resolution_id: &ScopeResolutionId::new("scope.fixture").unwrap(), - temporal_mode: TemporalModeV1::Current, - watermarks: &RetrieverWatermarkBindingV1 { - source_watermark: digest.clone(), - projection_watermark: VectorWatermark::default(), - index_watermark: Some(digest.clone()), - summary_watermark: None, - }, - horizon: &horizon, - occurrence_set_id: &occurrence_set_id, - span_id: &span_id, - span_anchor_id: &span_anchor_id, - exact_source_anchors: &exact_source_anchors, - coverage: &CoverageReportV1::default(), - }) - .unwrap(); - - let publication_receipt_id = derive_evidence_assembly_publication_receipt_id_v1( - &EvidenceAssemblyPublicationIdentityProjectionV1 { - owner: &owner, - idempotency_key: &EvidenceAssemblyIdempotencyKeyV1::new( - ManifestDigest::new(format!("sha256:{}", "cc".repeat(32))).unwrap(), - ) - .unwrap(), - assembly_digest: &digest, - occurrence_set_id: &occurrence_set_id, - span_id: &span_id, - span_anchor_id: &span_anchor_id, - contribution_id: &contribution_id, - contribution_anchor_id: &RetrievalAnchorId::new("retrieval.contribution.fixture") - .unwrap(), - projection_receipt_id: &projection_receipt_id, - ordered_occurrence_ids: &ordered_occurrence_ids, - exact_source_anchors: &exact_source_anchors, - }, - ) - .unwrap(); - - vec![ - message_id.as_str().to_owned(), - invocation_id.as_str().to_owned(), - result_id.as_str().to_owned(), - code_id.as_str().to_owned(), - occurrence_set_id.as_str().to_owned(), - span_id.as_str().to_owned(), - projection_receipt_id.as_str().to_owned(), - contribution_id.as_str().to_owned(), - publication_receipt_id.as_str().to_owned(), - ] - } - - /// Derived identities are durable: they are persisted and compared against - /// stored rows, so the canonical identity bytes of every record kind must - /// not move when the derivation code is restructured. - #[test] - fn derived_identities_match_pinned_durable_digests() { - assert_eq!( - assembly_identities(), - vec![ - "sha256:ad1c2aab1be1647c6b66b303ad6fbe6b9194cb61de56c2f6abe6aa89d38a0098", - "sha256:04e1147d2415b541df5a4d7403e2d99ce8c797f058f9827efa15192ad5fb0d2b", - "sha256:b0f5f25a66ee94ffe2ad6f53a25426bae8e23523985174fd72da687a182f327e", - "sha256:48ac18f08d86bcbb019b084b1392585c396a9d2172a75c02b9660edd2dbc1869", - "sha256:9c6721fbf9632074d473acc92d7cf66dfa71727f06dc6a5290631cdbc45e67a2", - "sha256:1669d8e418625abcce6bb56d4778136d76085b6a857471e15e6a761c72a8ffa4", - "sha256:8d5d809282672648dc914abe58c4240aa1ec189e88c90b624dbfc12253b9c465", - "sha256:23ac7f403ea112ff0eddae51c9187f93e01980b4ff30f278c167622ef01b0d80", - "sha256:bad60bf9c69d5aacfac96dbd6e56eda9236efa9b7e189cabe86b6f5963f3d583", - ] - ); - } - - fn observation_coordinate( - source_order: u64, - digest_byte: &str, - ) -> SourceOccurrenceCoordinateV1 { - SourceOccurrenceCoordinateV1::ObservationProjection { - canonical_observation_id: CanonicalObservationIdV1::new(format!( - "sha256:{}", - digest_byte.repeat(32) - )) - .unwrap(), - source_range: ObservationSourceRangeV1::new(source_order, source_order + 1).unwrap(), - projection_output_ordinal: 0, - sanitized_byte_range: SanitizedObservationByteRangeV1::new(0, 1).unwrap(), - } - } -} diff --git a/crates/tracedecay-store/src/external_source/mod.rs b/crates/tracedecay-store/src/external_source/mod.rs index a16216ca2b..7364ffda7d 100644 --- a/crates/tracedecay-store/src/external_source/mod.rs +++ b/crates/tracedecay-store/src/external_source/mod.rs @@ -878,6 +878,83 @@ impl SourceCommitReceiptV1 { } } +/// The durable identity of a committed receipt, retained after its frontiers +/// and payload were superseded. It answers idempotent replay: which request +/// the key committed, under which receipt, carrying which mutations. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct SourceCommitReceiptSummaryV1 { + binding: SourceBindingIdentityV1, + idempotency_key: ManifestDigest, + request_digest: ManifestDigest, + receipt_digest: ManifestDigest, + mutation_digests: Vec, +} + +impl SourceCommitReceiptSummaryV1 { + pub fn new( + binding: SourceBindingIdentityV1, + idempotency_key: ManifestDigest, + request_digest: ManifestDigest, + receipt_digest: ManifestDigest, + mutation_digests: Vec, + ) -> SourceStoreResult { + binding.validate()?; + for digest in [&idempotency_key, &request_digest, &receipt_digest] + .into_iter() + .chain(&mutation_digests) + { + digest.validate()?; + } + Ok(Self { + binding, + idempotency_key, + request_digest, + receipt_digest, + mutation_digests, + }) + } + + pub fn of(receipt: &SourceCommitReceiptV1) -> Self { + Self { + binding: receipt.source_frontier().binding().clone(), + idempotency_key: receipt.idempotency_key().clone(), + request_digest: receipt.request_digest().clone(), + receipt_digest: receipt.receipt_digest().clone(), + mutation_digests: receipt + .mutations() + .iter() + .map(|mutation| mutation.mutation_digest().clone()) + .collect(), + } + } + + pub fn binding(&self) -> &SourceBindingIdentityV1 { + &self.binding + } + + pub fn idempotency_key(&self) -> &ManifestDigest { + &self.idempotency_key + } + + pub fn request_digest(&self) -> &ManifestDigest { + &self.request_digest + } + + pub fn receipt_digest(&self) -> &ManifestDigest { + &self.receipt_digest + } + + pub fn mutation_digests(&self) -> &[ManifestDigest] { + &self.mutation_digests + } + + /// Whether this receipt committed exactly `mutation`. + pub fn committed(&self, mutation: &SourceObjectMutationV1) -> bool { + self.mutation_digests.contains(mutation.mutation_digest()) + } +} + #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct SourceAuthorityPublicationV1 { diff --git a/crates/tracedecay-store/src/lib.rs b/crates/tracedecay-store/src/lib.rs index 7d85f4f7c1..4e47109bd6 100644 --- a/crates/tracedecay-store/src/lib.rs +++ b/crates/tracedecay-store/src/lib.rs @@ -9,7 +9,6 @@ mod codex_goal_context; pub mod configuration; pub mod cursor_dispatch; pub mod diagnostics; -pub mod evidence_assembly; pub mod external_source; // The crash harness has to hold a live daemon inside a persistence boundary, so // it needs the filesystem and thread authority that these contracts refuse. @@ -33,7 +32,8 @@ pub mod session; pub mod transcript; pub use canonical_projection::{ - canonical_fact_text, derive_canonical_projection, stored_message_is_shipped_release_rendering, + EDITED_FILES_KEY, TOOL_USE_ID_KEY, canonical_fact_text, derive_canonical_projection, + message_metadata_with_envelope, stored_message_is_shipped_release_rendering, workflow_semantic_kind, }; pub use codex_goal_context::{ @@ -45,33 +45,12 @@ pub use configuration::{ ConfigurationRevisionStore, ConfigurationStoreError, ConfigurationStoreResult, }; pub use diagnostics::{ - DIAGNOSTIC_STATE_CLEARED, DIAGNOSTIC_STATE_CURRENT, DIAGNOSTIC_STATE_SUPERSEDED, - DiagnosticGenerationSupersessionV1, DiagnosticPublicationDispositionV1, - DiagnosticPublicationReceiptV1, DiagnosticRecordStateKindV1, DiagnosticStore, - DiagnosticStoreError, DiagnosticStoreResult, SanitizedCleanDiagnosticSnapshotV1, - diagnostic_evidence_class_name, diagnostic_producer_kind_name, diagnostic_severity_name, - diagnostic_snapshot_observation_eq, diagnostic_state_columns, parse_diagnostic_evidence_class, - parse_diagnostic_producer_kind, parse_diagnostic_severity, -}; -pub use evidence_assembly::{ - CanonicalSourceOccurrenceSetIdentityProjectionV1, CanonicalSourceOccurrenceSetRecordV1, - EvidenceAssemblyDrilldownPageV1, EvidenceAssemblyIdempotencyKeyV1, EvidenceAssemblyOwnerV1, - EvidenceAssemblyPublicationIdentityProjectionV1, EvidenceAssemblyPublicationReceiptV1, - EvidenceAssemblyReadOperationV1, EvidenceAssemblyReadResultV1, EvidenceAssemblyStoreError, - EvidenceAssemblyStoreResult, EvidenceAssemblyWriteV1, EvidenceSourceOccurrenceRecordV1, - EvidenceSourceTimelineV1, EvidenceSpanCatalogBindingV1, EvidenceSpanHorizonV1, - EvidenceSpanIdentityProjectionV1, EvidenceSpanMemberReceiptBindingV1, - EvidenceSpanProjectionReceiptIdentityProjectionV1, EvidenceSpanProjectionReceiptV1, - EvidenceSpanRecordV1, EvidenceSpanRunV1, MAX_EVIDENCE_ASSEMBLY_MEMBERS_V1, - PrivacyBoundRequestDigestV1, PrivacyBoundRequestEnvelopeV1, - RetrieverContributionIdentityProjectionV1, RetrieverContributionRecordV1, RetrieverIdentityV1, - RetrieverWatermarkBindingV1, SanitizedObservationByteRangeV1, SourceCapabilityCatalogBindingV1, - SourceOccurrenceCoordinateV1, SourceOccurrenceIdentityProjectionV1, SourceOccurrenceKindV1, - SourceOccurrenceRelationV1, SourceOccurrenceSanitizationV1, SourceTimelineKeyV1, - VerifiedSourceOrderingProofV1, derive_canonical_source_occurrence_set_id_v1, - derive_evidence_assembly_publication_receipt_id_v1, derive_evidence_span_id_v1, - derive_evidence_span_projection_receipt_id_v1, derive_retriever_contribution_id_v1, - derive_source_occurrence_id_v1, + DIAGNOSTIC_STATE_CLEARED, DIAGNOSTIC_STATE_CURRENT, DiagnosticRecordStateKindV1, + DiagnosticStore, DiagnosticStoreError, DiagnosticStoreResult, + SanitizedCleanDiagnosticSnapshotV1, diagnostic_evidence_class_name, + diagnostic_producer_kind_name, diagnostic_severity_name, diagnostic_snapshot_observation_eq, + diagnostic_state_columns, parse_diagnostic_evidence_class, parse_diagnostic_producer_kind, + parse_diagnostic_severity, }; pub use external_source::{ MAX_SOURCE_ACQUISITION_ATTEMPTS_V1, MAX_SOURCE_ACQUISITION_RECEIPTS_V1, @@ -79,13 +58,14 @@ pub use external_source::{ SourceAcquisitionQueueContractErrorV1, SourceAcquisitionQueueResultV1, SourceAcquisitionQueueStateV1, SourceAcquisitionRequestV1, SourceAuthorityPublicationApplyOutcomeV1, SourceAuthorityPublicationReceiptV1, - SourceAuthorityPublicationV1, SourceCommitApplyOutcomeV1, SourceCommitReceiptV1, - SourceCommitV1, SourceObjectLineageV1, SourceObjectMutationV1, SourceObjectTransitionV1, - SourceObservationEvidenceV1, SourcePendingProjectionV1, SourceProjectionApplyOutcomeV1, - SourceProjectionCommitV1, SourceProjectionEffectV1, SourceScheduledRefetchV1, - SourceStoreErrorV1, SourceStoreResult, SourceStoreStateV1, apply_source_authority_publication, - apply_source_authority_publication_owned, apply_source_commit, apply_source_commit_owned, - apply_source_projection, apply_source_projection_owned, build_source_projection, + SourceAuthorityPublicationV1, SourceCommitApplyOutcomeV1, SourceCommitReceiptSummaryV1, + SourceCommitReceiptV1, SourceCommitV1, SourceObjectLineageV1, SourceObjectMutationV1, + SourceObjectTransitionV1, SourceObservationEvidenceV1, SourcePendingProjectionV1, + SourceProjectionApplyOutcomeV1, SourceProjectionCommitV1, SourceProjectionEffectV1, + SourceScheduledRefetchV1, SourceStoreErrorV1, SourceStoreResult, SourceStoreStateV1, + apply_source_authority_publication, apply_source_authority_publication_owned, + apply_source_commit, apply_source_commit_owned, apply_source_projection, + apply_source_projection_owned, build_source_projection, }; pub use git_index_transactions::{ GitIndexPreviewInputReadV1, GitIndexTransactionBeginRequestV1, @@ -157,18 +137,17 @@ pub use observation::{ AnchoredObservationWrite, CursorAdvanceLedgerDisagreementV1, CursorAdvanceLedgerIdentityV1, CursorAdvanceLedgerOpaqueValueHashV1, CursorAdvanceLedgerReasonV1, CursorAdvanceLedgerReceiptIdV1, CursorAdvanceOutcome, OBSERVATION_CAPTURE_AUTHORITY_V1, - ObservationAdmissionPort, ObservationBatchFallbackCause, ObservationBatchPersistOutcome, - ObservationCaptureSink, ObservationCommitReceipt, ObservationCoverageReason, - ObservationCoverageV1, ObservationCursorAdvance, ObservationCursorPort, - ObservationPersistOutcome, ObservationProjectionStatus, ObservationReplayRequest, - ObservationStore, ObservationStoreError, ObservationStoreResult, ObservationWrite, - ObservedEvidenceAnchorResolution, RepositoryProvenanceAttachmentV1, StoredObservation, - build_observation_resolution_authorization_v1, build_observation_retrieval_anchor_v2, + ObservationAdmissionPort, ObservationBatchPersistOutcome, ObservationCaptureSink, + ObservationCommitReceipt, ObservationCoverageReason, ObservationCoverageV1, + ObservationCursorAdvance, ObservationCursorPort, ObservationPersistOutcome, + ObservationProjectionStatus, ObservationReplayRequest, ObservationStore, ObservationStoreError, + ObservationStoreResult, ObservationWrite, ObservedEvidenceAnchorResolution, + RepositoryProvenanceAttachmentV1, StoredObservation, + build_observation_resolution_authorization_v1, build_observation_retrieval_anchor, build_scope_resolution_authorization_v1, observation_capture_access_policy_digest_v1, }; pub use projection::{ - CLAUDE_SESSION_MESSAGE_PROJECTOR_VERSION, ClaudeObservationProjection, - ClaudeSessionMessageProjection, ObservationProjection, ObservationProjectionStore, + CLAUDE_SESSION_MESSAGE_PROJECTOR_VERSION, ObservationProjection, ObservationProjectionStore, PROVIDER_USAGE_PROJECTOR_VERSION, ProjectedObservation, ProjectionBatchItem, ProjectionCheckpoint, ProjectionDrainBatch, ProjectionPersistOutcome, ProjectionPredecessorConvergence, ProjectionProvenance, ProjectionRebuildOutcome, @@ -184,8 +163,8 @@ pub use remote::{RemoteObservationReplayWriteV1, RemoteWriterFenceInstallV1}; pub use retrieval_anchor::{ AnchorDerivativeKindV1, AnchorDispositionAppendOutcomeV1, AnchorDispositionReasonClassV1, AnchorDispositionStateV1, RetrievalAnchorDerivativeV1, RetrievalAnchorDispositionRecordV1, - RetrievalAnchorDispositionStore, RetrievalAnchorOwnerV1, RetrievalAnchorStoreError, - RetrievalAnchorStoreResult, RetrievalAnchorTombstoneV1, StoredRetrievalAnchorRecordV1, + RetrievalAnchorDispositionStore, RetrievalAnchorStoreError, RetrievalAnchorStoreResult, + RetrievalAnchorTombstoneV1, }; pub use runtime::{ AdmissionConfigV1, AuthorityEpoch, BACKGROUND_BATCH_MAX_BYTES, BACKGROUND_BATCH_MAX_OPERATIONS, @@ -232,8 +211,7 @@ pub use runtime::{ RuntimeReadResultV1, RuntimeRequestControlV1, RuntimeRequestProbeV1, RuntimeSubmitOutcomeV1, RuntimeSubmitRequestV1, RuntimeTransactionIdV1, RuntimeTransactionScopeV1, SaturationScopeV1, ScopeSetCasOutcomeV1, ShardWatermarkV1, SnapshotLeaseIdV1, SnapshotLeaseV1, - StorageRuntimeContractErrorV1, StorageRuntimeErrorV1, StorageRuntimePortErrorV1, - StorageRuntimePortFutureV1, StorageRuntimeReadPort, StoreAuthorityEpochV1, StoreClientIdV1, + StorageRuntimeContractErrorV1, StorageRuntimeErrorV1, StoreAuthorityEpochV1, StoreClientIdV1, StoreCommitReceiptV1, StoreEffectIdV1, StoreEffectOrderingKeyV1, StoreIdempotencyKeyV1, StoreIncarnationV1, StoreOperationIdV1, StoreOperationMetadataV1, StoreRuntimeBindingV1, StoreRuntimeRegistryPublicationV1, StoreShardIdV1, StoreShardScopeV1, StoreSnapshotIdV1, diff --git a/crates/tracedecay-store/src/memory/mod.rs b/crates/tracedecay-store/src/memory/mod.rs index ae218ce26f..8f692d054d 100644 --- a/crates/tracedecay-store/src/memory/mod.rs +++ b/crates/tracedecay-store/src/memory/mod.rs @@ -89,7 +89,7 @@ use queries::MAX_LINEAGE_LIMIT; #[cfg(test)] use tracedecay_domain::{ DomainError, FactAssertionV1, FactLineageEventKindV1, FactLineageEventV1, RetrievalAnchorId, - RetrievalAnchorRecordV2, + RetrievalAnchorRecord, }; #[cfg(test)] use write::{MAX_FACT_WRITE_BATCH_EVENTS, MAX_FACT_WRITE_BATCH_NEW_ANCHORS}; diff --git a/crates/tracedecay-store/src/memory/project_memory/automatic_facts.rs b/crates/tracedecay-store/src/memory/project_memory/automatic_facts.rs index bd6a97ba0c..eab4e0314e 100644 --- a/crates/tracedecay-store/src/memory/project_memory/automatic_facts.rs +++ b/crates/tracedecay-store/src/memory/project_memory/automatic_facts.rs @@ -1,3 +1,4 @@ +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use serde_json::Value; use tracedecay_domain::{ @@ -45,7 +46,7 @@ struct AutomaticFactDigestProjection<'a> { /// The only durable outcomes of an automatic fact apply. Candidate discovery /// and in-flight work are owned by the automation run receipt, never this /// terminal audit record. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "snake_case")] pub enum ProjectMemoryAutomaticFactStateV1 { Applied, diff --git a/crates/tracedecay-store/src/memory/project_memory/mod.rs b/crates/tracedecay-store/src/memory/project_memory/mod.rs index 9e802eb6c5..0ea58fc96a 100644 --- a/crates/tracedecay-store/src/memory/project_memory/mod.rs +++ b/crates/tracedecay-store/src/memory/project_memory/mod.rs @@ -3,7 +3,7 @@ use tracedecay_domain::canonical_text::is_canonical_text_within; use tracedecay_domain::{ Confidence, DomainError, FactAssertionId, FactCategoryV1, FactEventId, FactId, FactIdentityMaterialV1, FactIdentitySourceV1, FactLineageEventV1, FactOwnerV1, FactPayloadV1, - RetrievalAnchorId, RetrievalAnchorRecordV2, SanitizerDispositionV1, UtcMicros, + RetrievalAnchorId, RetrievalAnchorRecord, SanitizerDispositionV1, UtcMicros, }; use super::queries::{MAX_CURRENT_LIMIT, MAX_LINEAGE_LIMIT}; @@ -503,7 +503,7 @@ impl ProjectMemoryFactHistoryV1 { pub struct ProjectMemoryFactInspectionV1 { fact: ProjectMemoryFactV1, history: ProjectMemoryFactHistoryV1, - anchors: Vec, + anchors: Vec, status: ProjectMemoryFactStatusV1, } @@ -511,7 +511,7 @@ impl ProjectMemoryFactInspectionV1 { pub fn new( fact: ProjectMemoryFactV1, history: ProjectMemoryFactHistoryV1, - anchors: Vec, + anchors: Vec, status: ProjectMemoryFactStatusV1, ) -> FactStoreResult { history.validate_for_owner(fact.owner())?; @@ -566,7 +566,7 @@ impl ProjectMemoryFactInspectionV1 { pub fn history(&self) -> &ProjectMemoryFactHistoryV1 { &self.history } - pub fn anchors(&self) -> &[RetrievalAnchorRecordV2] { + pub fn anchors(&self) -> &[RetrievalAnchorRecord] { &self.anchors } pub fn status(&self) -> &ProjectMemoryFactStatusV1 { diff --git a/crates/tracedecay-store/src/memory/tests.rs b/crates/tracedecay-store/src/memory/tests.rs index bdb18c7e50..cb697373f0 100644 --- a/crates/tracedecay-store/src/memory/tests.rs +++ b/crates/tracedecay-store/src/memory/tests.rs @@ -1,15 +1,14 @@ use serde_json::json; use tracedecay_domain::{ - AccessPolicyDigest, ActorId, AnchorDurabilityClass, AnchorLineageRefV2, - AnchorProvenanceRelationV2, AnchorSourceGenerationV2, CapabilityId, ComponentVersion, - CoverageReportV1, EntityId, EntityKind, EntityRef, EvidenceClass, FactAssertionKindV1, - FactCategoryV1, FactCurationActionV1, FactEvidenceRefV1, FactEvidenceRelationV1, - FactIdentityMaterialV1, FactIdentitySourceV1, ObservationScopeV1, PayloadReferenceV1, - PrivacyDomainBoundLocatorDigest, PrivacyDomainId, ProjectionGenerationId, ProvenanceId, - ResolutionAuthorizationV1, RetentionClass, RetrievalAnchorRecordV2Parts, - RetrievalAnchorTargetV2, SanitizationReceiptId, SanitizationReceiptRefV1, - SanitizationReceiptV1, SanitizerDispositionV1, ScopeResolutionId, SensitivityV1, - VectorWatermark, + AccessPolicyDigest, ActorId, AnchorDurabilityClass, AnchorLineageRef, AnchorProvenanceRelation, + AnchorSourceGeneration, CapabilityId, ComponentVersion, CoverageReportV1, EntityId, EntityKind, + EntityRef, EvidenceClass, FactAssertionKindV1, FactCategoryV1, FactCurationActionV1, + FactEvidenceRefV1, FactEvidenceRelationV1, FactIdentityMaterialV1, FactIdentitySourceV1, + ObservationScopeV1, PayloadReferenceV1, PrivacyDomainBoundLocatorDigest, PrivacyDomainId, + ProjectionGenerationId, ProvenanceId, ResolutionAuthorizationV1, RetentionClass, + RetrievalAnchorRecordParts, RetrievalAnchorTarget, SanitizationReceiptId, + SanitizationReceiptRefV1, SanitizationReceiptV1, SanitizerDispositionV1, ScopeResolutionId, + SensitivityV1, VectorWatermark, }; use super::*; @@ -179,13 +178,13 @@ fn projected_fact( .unwrap() } -fn anchor(entity_id: &str, source_anchors: Vec) -> RetrievalAnchorRecordV2 { +fn anchor(entity_id: &str, source_anchors: Vec) -> RetrievalAnchorRecord { const DIGEST_A: &str = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; const DIGEST_B: &str = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; - RetrievalAnchorRecordV2::new(RetrievalAnchorRecordV2Parts { - target: RetrievalAnchorTargetV2::Entity(EntityRef { + RetrievalAnchorRecord::new(RetrievalAnchorRecordParts { + target: RetrievalAnchorTarget::Entity(EntityRef { id: EntityId::new(entity_id).unwrap(), kind: EntityKind::Document, }), @@ -194,7 +193,7 @@ fn anchor(entity_id: &str, source_anchors: Vec) -> Retrieval occurred_at: None, ingested_at: UtcMicros(1), evidence_class: EvidenceClass::Observed, - source_generation: AnchorSourceGenerationV2::Unknown, + source_generation: AnchorSourceGeneration::Unknown, projection_generation: ProjectionGenerationId::new("projection.fixture").unwrap(), projection_watermark: VectorWatermark::default(), coverage: CoverageReportV1::default(), @@ -214,9 +213,9 @@ fn anchor(entity_id: &str, source_anchors: Vec) -> Retrieval .unwrap() } -fn anchor_source(anchor_id: RetrievalAnchorId) -> AnchorLineageRefV2 { - AnchorLineageRefV2::new( - AnchorProvenanceRelationV2::DerivedFrom, +fn anchor_source(anchor_id: RetrievalAnchorId) -> AnchorLineageRef { + AnchorLineageRef::new( + AnchorProvenanceRelation::DerivedFrom, anchor_id, ObservationScopeV1::Profile, ) diff --git a/crates/tracedecay-store/src/memory/traits.rs b/crates/tracedecay-store/src/memory/traits.rs index 31952a35fb..58d3033f94 100644 --- a/crates/tracedecay-store/src/memory/traits.rs +++ b/crates/tracedecay-store/src/memory/traits.rs @@ -1,6 +1,6 @@ use std::future::Future; use tracedecay_domain::RunId; -use tracedecay_domain::{FactLineageEventV1, FactOwnerV1, ProvenanceId, RetrievalAnchorRecordV2}; +use tracedecay_domain::{FactLineageEventV1, FactOwnerV1, ProvenanceId, RetrievalAnchorRecord}; use super::ProjectMemoryAutomationRunReceiptsV1; use super::{ @@ -82,7 +82,7 @@ pub trait FactStore: Send + Sync { fn get_retrieval_anchor( &self, query: RetrievalAnchorQuery, - ) -> impl Future>> + Send; + ) -> impl Future>> + Send; } /// Single typed authority boundary for canonical project memory. diff --git a/crates/tracedecay-store/src/memory/write.rs b/crates/tracedecay-store/src/memory/write.rs index af48165a0e..2ee5826f1d 100644 --- a/crates/tracedecay-store/src/memory/write.rs +++ b/crates/tracedecay-store/src/memory/write.rs @@ -4,7 +4,7 @@ use std::sync::Arc; use tracedecay_domain::{ DomainError, FactAssertionId, FactAssertionKindV1, FactAssertionV1, FactCurationActionV1, FactEventId, FactId, FactIdentityMaterialV1, FactLineageEventKindV1, FactLineageEventV1, - FactOwnerV1, ManifestDigest, RetrievalAnchorId, RetrievalAnchorRecordV2, canonical_sha256, + FactOwnerV1, ManifestDigest, RetrievalAnchorId, RetrievalAnchorRecord, canonical_sha256, }; use super::{FactStoreError, FactStoreResult, validate_owned_fact_id}; @@ -52,7 +52,7 @@ pub struct FactWriteBatch { identity_material: Option, assertion: Option, events: Vec, - new_anchors: Vec, + new_anchors: Vec, referenced_anchor_ids: Vec, expected_last_event_id: Option, } @@ -64,7 +64,7 @@ impl FactWriteBatch { owner: FactOwnerV1, assertion: Option, events: Vec, - new_anchors: Vec, + new_anchors: Vec, referenced_anchor_ids: Vec, expected_last_event_id: Option, ) -> FactStoreResult { @@ -216,7 +216,7 @@ impl FactWriteBatch { &self.events } - pub fn new_anchors(&self) -> &[RetrievalAnchorRecordV2] { + pub fn new_anchors(&self) -> &[RetrievalAnchorRecord] { &self.new_anchors } @@ -237,7 +237,7 @@ impl FactWriteBatch { Option, Option, Vec, - Vec, + Vec, Vec, Option, ) { @@ -313,7 +313,7 @@ fn invalid_normalized_tag_batch() -> FactStoreError { } fn validate_anchor_lineage( - new_anchors: &[RetrievalAnchorRecordV2], + new_anchors: &[RetrievalAnchorRecord], referenced_anchor_ids: &[RetrievalAnchorId], ) -> FactStoreResult<()> { let referenced = referenced_anchor_ids.iter().collect::>(); diff --git a/crates/tracedecay-store/src/observation/anchored_write.rs b/crates/tracedecay-store/src/observation/anchored_write.rs index 7c8cf86e58..18aff117ba 100644 --- a/crates/tracedecay-store/src/observation/anchored_write.rs +++ b/crates/tracedecay-store/src/observation/anchored_write.rs @@ -1,7 +1,7 @@ use tracedecay_domain::{ - AnchorSourceGenerationV2, DurableObservationV1, EvidenceAvailabilityV1, + AnchorSourceGeneration, DurableObservationV1, EvidenceAvailabilityV1, GenerationBoundRepositoryProvenanceV1, ObservationScopeV1, ObservationSourceCursorV1, - ProjectionGenerationId, RetrievalAnchorId, RetrievalAnchorRecordV2, RetrievalAnchorTargetV2, + ProjectionGenerationId, RetrievalAnchorId, RetrievalAnchorRecord, RetrievalAnchorTarget, }; use super::{ObservationStoreError, ObservationStoreResult, ObservationWrite}; @@ -22,12 +22,12 @@ pub enum ObservationIdentityCollisionDispositionV1 { pub(super) fn validate_retrieval_anchor_binding( observation: &DurableObservationV1, - retrieval_anchor: &RetrievalAnchorRecordV2, + retrieval_anchor: &RetrievalAnchorRecord, projection_generation: &ProjectionGenerationId, ) -> ObservationStoreResult<()> { if !matches!( retrieval_anchor.target(), - RetrievalAnchorTargetV2::ExactObservation(observation_id) + RetrievalAnchorTarget::ExactObservation(observation_id) if observation_id == observation.observation_id() ) { return Err(ObservationStoreError::RetrievalAnchorObservationMismatch); @@ -36,7 +36,7 @@ pub(super) fn validate_retrieval_anchor_binding( return Err(ObservationStoreError::RetrievalAnchorOwnerMismatch); } if retrieval_anchor.source_generation() - != &AnchorSourceGenerationV2::Observation(observation.identity().generation()) + != &AnchorSourceGeneration::Observation(observation.identity().generation()) { return Err(ObservationStoreError::RetrievalAnchorSourceGenerationMismatch); } @@ -56,13 +56,13 @@ pub(super) fn validate_retrieval_anchor_binding( pub struct RepositoryProvenanceAttachmentV1 { availability: EvidenceAvailabilityV1, #[serde(default, skip_serializing_if = "Option::is_none")] - anchor: Option, + anchor: Option, } impl RepositoryProvenanceAttachmentV1 { pub fn new( availability: EvidenceAvailabilityV1, - anchor: Option, + anchor: Option, ) -> ObservationStoreResult { if availability.value().is_some() != anchor.is_some() { return Err(ObservationStoreError::RepositoryProvenanceAvailabilityMismatch); @@ -98,7 +98,7 @@ impl RepositoryProvenanceAttachmentV1 { self.availability.value() } - pub fn anchor(&self) -> Option<&RetrievalAnchorRecordV2> { + pub fn anchor(&self) -> Option<&RetrievalAnchorRecord> { self.anchor.as_ref() } @@ -135,12 +135,12 @@ impl RepositoryProvenanceAttachmentV1 { || anchor.source_observations() != [observation.observation_id().clone()] || !matches!( anchor.source_generation(), - AnchorSourceGenerationV2::RepositoryCapture(capture_id) + AnchorSourceGeneration::RepositoryCapture(capture_id) if capture_id == provenance.capture_id() ) || !matches!( anchor.target(), - RetrievalAnchorTargetV2::RepositoryCapture { + RetrievalAnchorTarget::RepositoryCapture { repository_id, capture_id, receipt, @@ -170,7 +170,7 @@ impl Default for RepositoryProvenanceAttachmentV1 { #[derive(Clone, Debug, PartialEq, Eq)] pub struct AnchoredObservationWrite { write: ObservationWrite, - retrieval_anchor: RetrievalAnchorRecordV2, + retrieval_anchor: RetrievalAnchorRecord, projection_generation: ProjectionGenerationId, repository_provenance: RepositoryProvenanceAttachmentV1, identity_collision_disposition: ObservationIdentityCollisionDispositionV1, @@ -179,7 +179,7 @@ pub struct AnchoredObservationWrite { impl AnchoredObservationWrite { pub fn new( write: ObservationWrite, - retrieval_anchor: RetrievalAnchorRecordV2, + retrieval_anchor: RetrievalAnchorRecord, projection_generation: ProjectionGenerationId, ) -> ObservationStoreResult { validate_retrieval_anchor_binding( @@ -213,7 +213,7 @@ impl AnchoredObservationWrite { pub fn with_repository_provenance_attachment( mut self, availability: EvidenceAvailabilityV1, - anchor: Option, + anchor: Option, ) -> ObservationStoreResult { let repository_provenance = RepositoryProvenanceAttachmentV1::new(availability, anchor)?; repository_provenance @@ -238,7 +238,7 @@ impl AnchoredObservationWrite { self.write.next_cursor() } - pub fn retrieval_anchor(&self) -> &RetrievalAnchorRecordV2 { + pub fn retrieval_anchor(&self) -> &RetrievalAnchorRecord { &self.retrieval_anchor } @@ -258,7 +258,7 @@ impl AnchoredObservationWrite { self, ) -> ( ObservationWrite, - RetrievalAnchorRecordV2, + RetrievalAnchorRecord, ProjectionGenerationId, RepositoryProvenanceAttachmentV1, ) { diff --git a/crates/tracedecay-store/src/observation/mod.rs b/crates/tracedecay-store/src/observation/mod.rs index 473a0488e3..04da546dc9 100644 --- a/crates/tracedecay-store/src/observation/mod.rs +++ b/crates/tracedecay-store/src/observation/mod.rs @@ -1,20 +1,18 @@ -use std::collections::HashMap; use std::error::Error; use std::future::Future; -use std::sync::{LazyLock, RwLock}; use sha2::{Digest, Sha256}; use tracedecay_domain::{ - AccessPolicyDigest, AnchorDurabilityClass, AnchorSourceGenerationV2, CanonicalObservationIdV1, - CapabilityId, CoverageReportV1, DomainError, DurableObservationV1, EvidenceClass, - NativeAliasKindV2, NativeAliasV2, ObservationCollisionOutcomeV1, ObservationContractError, + AccessPolicyDigest, AnchorDurabilityClass, AnchorSourceGeneration, CanonicalObservationIdV1, + CoverageReportV1, DomainError, DurableObservationV1, EvidenceClass, NativeAlias, + NativeAliasKind, ObservationCollisionOutcomeV1, ObservationContractError, ObservationOrderingDomainV1, ObservationScopeV1, ObservationSourceCursorV1, ObservationSourceGenerationV1, ObservationSourceIdentityV1, ObservationSourceRangeV1, PayloadAccessState, PayloadDigestV1, PayloadReferenceV1, PrivacyDomainBoundLocatorDigest, - PrivacyDomainId, ProjectionGenerationId, ResolutionAuthorizationV1, RetrievalAnchorId, - RetrievalAnchorRecordV2, RetrievalAnchorRecordV2Parts, RetrievalAnchorTargetV2, - SanitizationReceiptId, SanitizationReceiptV1, SanitizerDispositionV1, ScopeResolutionId, - UtcMicros, VectorWatermark, + ProjectionGenerationId, ResolutionAuthorizationV1, RetrievalAnchorId, RetrievalAnchorRecord, + RetrievalAnchorRecordParts, RetrievalAnchorTarget, SanitizationReceiptId, + SanitizationReceiptV1, SanitizerDispositionV1, UtcMicros, VectorWatermark, + authority_access_policy_digest, }; mod anchored_write; @@ -156,60 +154,11 @@ pub fn build_scope_resolution_authorization_v1( }) } -/// Upper bound on memoized access-policy digests. -/// -/// Authority namespaces are compile-time constants in production, so this only -/// exists so a caller passing unbounded namespaces cannot grow the memo without -/// limit; past the bound the digest is derived without being retained. -const MAX_MEMOIZED_ACCESS_POLICY_DIGESTS: usize = 64; - -/// Access-policy digests keyed by authority namespace. -/// -/// The digested value binds nothing but the authorization domain constant and -/// the namespace, so it is the same bytes for every resolution in that -/// namespace. Deriving it per resolution put a canonical-JSON encode plus a -/// SHA-256 compression on the anchor-resolution serving path for a value that -/// was born the first time the namespace was used. -/// -/// The lock type follows `hotpath::rw_lock!`: instrumented wrapper when the -/// `hotpath` feature is on, `std::sync::RwLock` when it is off. -static ACCESS_POLICY_DIGESTS: LazyLock>> = - LazyLock::new(|| { - hotpath::rw_lock!( - RwLock::new(HashMap::new()), - label = "store.observation.access_policy_digests" - ) - }); - -fn access_policy_digest_for(authority_namespace: &str) -> ObservationStoreResult { - hotpath::measure_block!("store.observation.access_policy_digest", { - if let Ok(memo) = ACCESS_POLICY_DIGESTS.read() - && let Some(digest) = memo.get(authority_namespace) - { - return Ok(digest.clone()); - } - let digest = PayloadReferenceV1::for_payload(&serde_json::json!({ - "domain": "tracedecay.observation-anchor.authorization.v1", - "authority": authority_namespace, - })) - .map_err(ObservationStoreError::Contract)? - .digest() - .as_str() - .to_owned(); - if let Ok(mut memo) = ACCESS_POLICY_DIGESTS.write() - && memo.len() < MAX_MEMOIZED_ACCESS_POLICY_DIGESTS - { - memo.insert(authority_namespace.to_owned(), digest.clone()); - } - Ok(digest) - }) -} - /// Returns the exact access-policy digest retained by production observation /// anchors so retrieval admission can bind to the same authority without /// duplicating its canonical digest construction. pub fn observation_capture_access_policy_digest_v1() -> ObservationStoreResult { - AccessPolicyDigest::new(access_policy_digest_for(OBSERVATION_CAPTURE_AUTHORITY_V1)?) + authority_access_policy_digest(OBSERVATION_CAPTURE_AUTHORITY_V1) .map_err(ObservationStoreError::RetrievalAnchorContract) } @@ -217,28 +166,22 @@ fn build_resolution_authorization_v1( authority_namespace: &str, canonical_request_digest: String, ) -> ObservationStoreResult { - let access_policy_digest = access_policy_digest_for(authority_namespace)?; - Ok(ResolutionAuthorizationV1 { - resolved_scope_id: ScopeResolutionId::new(format!("scope.{authority_namespace}")) - .map_err(ObservationStoreError::RetrievalAnchorContract)?, - privacy_domain_id: PrivacyDomainId::new(format!("privacy.{authority_namespace}")) - .map_err(ObservationStoreError::RetrievalAnchorContract)?, - access_policy_digest: AccessPolicyDigest::new(access_policy_digest) - .map_err(ObservationStoreError::RetrievalAnchorContract)?, - capability_id: CapabilityId::new(format!("capability.{authority_namespace}")) - .map_err(ObservationStoreError::RetrievalAnchorContract)?, - canonical_request_digest: PrivacyDomainBoundLocatorDigest::new(canonical_request_digest) - .map_err(ObservationStoreError::RetrievalAnchorContract)?, + hotpath::measure_block!("store.observation.access_policy_digest", { + PrivacyDomainBoundLocatorDigest::new(canonical_request_digest) + .and_then(|digest| { + ResolutionAuthorizationV1::for_authority(authority_namespace, digest) + }) + .map_err(ObservationStoreError::RetrievalAnchorContract) }) } /// Builds the canonical stable anchor for one retained sanitized observation. -pub fn build_observation_retrieval_anchor_v2( +pub fn build_observation_retrieval_anchor( observation: &DurableObservationV1, projection_generation: ProjectionGenerationId, ingested_at: UtcMicros, authorization: ResolutionAuthorizationV1, -) -> ObservationStoreResult { +) -> ObservationStoreResult { hotpath::measure_block!("store.observation.build_retrieval_anchor", { let aliases = observation .identity() @@ -257,20 +200,20 @@ pub fn build_observation_retrieval_anchor_v2( .to_owned(); let locator_digest = PrivacyDomainBoundLocatorDigest::new(digest) .map_err(ObservationStoreError::RetrievalAnchorContract)?; - NativeAliasV2::new(NativeAliasKindV2::ProviderRecord, locator_digest) + NativeAlias::new(NativeAliasKind::ProviderRecord, locator_digest) .map_err(ObservationStoreError::RetrievalAnchorContract) }) .transpose()? .into_iter() .collect(); - RetrievalAnchorRecordV2::new(RetrievalAnchorRecordV2Parts { - target: RetrievalAnchorTargetV2::ExactObservation(observation.observation_id().clone()), + RetrievalAnchorRecord::new(RetrievalAnchorRecordParts { + target: RetrievalAnchorTarget::ExactObservation(observation.observation_id().clone()), owner: observation.scope().clone(), aliases, occurred_at: None, ingested_at, evidence_class: EvidenceClass::Observed, - source_generation: AnchorSourceGenerationV2::Observation( + source_generation: AnchorSourceGeneration::Observation( observation.identity().generation(), ), projection_generation, @@ -298,7 +241,7 @@ pub enum ObservedEvidenceAnchorResolution { /// watermark is the store's current projection-stream position reported /// under exactly the shard keys the record's frozen watermark claims. Resolved { - record: Box, + record: Box, observed_watermark: VectorWatermark, }, /// No binding for the anchor exists in this authority. @@ -883,7 +826,7 @@ pub struct ObservationCommitReceipt { sequence: u64, observation: Box, committed_cursor: ObservationSourceCursorV1, - retrieval_anchor: Box, + retrieval_anchor: Box, projection_generation: ProjectionGenerationId, repository_provenance: RepositoryProvenanceAttachmentV1, } @@ -893,7 +836,7 @@ impl ObservationCommitReceipt { sequence: u64, observation: DurableObservationV1, committed_cursor: ObservationSourceCursorV1, - retrieval_anchor: RetrievalAnchorRecordV2, + retrieval_anchor: RetrievalAnchorRecord, projection_generation: ProjectionGenerationId, ) -> ObservationStoreResult { validate_retrieval_anchor_binding(&observation, &retrieval_anchor, &projection_generation)?; @@ -933,7 +876,7 @@ impl ObservationCommitReceipt { &self.committed_cursor } - pub fn retrieval_anchor(&self) -> &RetrievalAnchorRecordV2 { + pub fn retrieval_anchor(&self) -> &RetrievalAnchorRecord { self.retrieval_anchor.as_ref() } @@ -979,7 +922,7 @@ impl StoredObservation { sequence: u64, observation: DurableObservationV1, committed_cursor: ObservationSourceCursorV1, - retrieval_anchor: RetrievalAnchorRecordV2, + retrieval_anchor: RetrievalAnchorRecord, projection_generation: ProjectionGenerationId, projection_status: ObservationProjectionStatus, ) -> ObservationStoreResult { @@ -1029,7 +972,7 @@ impl StoredObservation { self.commit_receipt.repository_provenance_attachment() } - pub fn retrieval_anchor(&self) -> &RetrievalAnchorRecordV2 { + pub fn retrieval_anchor(&self) -> &RetrievalAnchorRecord { self.commit_receipt.retrieval_anchor() } @@ -1088,22 +1031,6 @@ pub enum ObservationProjectionStatus { NotQueued, } -/// Why one bounded observation batch must be retried as scalar operations. -/// -/// These causes describe only collisions between not-yet-durable members of -/// the current batch. Collisions against durable evidence remain terminal -/// store errors and must never be retried as scalar writes. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum ObservationBatchFallbackCause { - IntraBatchIdentityCollision, - IntraBatchSanitizationReceiptCollision, - IntraBatchRetrievalAnchorAliasCollision, - /// A collision path that must compare-and-set against the durable source - /// frontier, while an earlier member of this batch has not made that - /// frontier durable yet. - IntraBatchDurableFrontier, -} - #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct ObservationReplayRequest { after_sequence: u64, @@ -1136,10 +1063,6 @@ impl ObservationReplayRequest { #[derive(Debug, thiserror::Error)] #[non_exhaustive] pub enum ObservationStoreError { - #[error("observation batch requires scalar fallback: {cause:?}")] - BatchRequiresScalarFallback { - cause: ObservationBatchFallbackCause, - }, #[error("observation cursor does not match its source evidence")] CursorObservationMismatch, #[error("covered source evidence is not contiguous with the expected cursor")] @@ -1192,7 +1115,7 @@ pub enum ObservationStoreError { "retrieval anchor alias {alias:?} collided between existing anchor {existing_anchor_id:?} and candidate anchor {candidate_anchor_id:?}" )] RetrievalAnchorAliasCollision { - alias: Box, + alias: Box, existing_anchor_id: Box, candidate_anchor_id: Box, }, diff --git a/crates/tracedecay-store/src/observation/tests.rs b/crates/tracedecay-store/src/observation/tests.rs index 8b58c0547f..4bde3d2730 100644 --- a/crates/tracedecay-store/src/observation/tests.rs +++ b/crates/tracedecay-store/src/observation/tests.rs @@ -1,10 +1,10 @@ use serde_json::json; use tracedecay_domain::{ - AccessPolicyDigest, AnchorDurabilityClass, AnchorSourceGenerationV2, CapabilityId, + AccessPolicyDigest, AnchorDurabilityClass, AnchorSourceGeneration, CapabilityId, ComponentVersion, CoverageReportV1, EvidenceClass, ObservationId, ObservationIdentityMaterialV1, PayloadAccessState, PayloadReferenceV1, PrivacyDomainBoundLocatorDigest, PrivacyDomainId, ProjectId, ProviderId, - ResolutionAuthorizationV1, RetrievalAnchorRecordV2Parts, SanitizationReceiptId, + ResolutionAuthorizationV1, RetrievalAnchorRecordParts, SanitizationReceiptId, SanitizationReceiptRefV1, SanitizerDispositionV1, ScopeResolutionId, SensitivityV1, SessionId, UtcMicros, VectorWatermark, }; @@ -81,15 +81,15 @@ fn authorization() -> ResolutionAuthorizationV1 { fn anchor( observation: &DurableObservationV1, owner: ObservationScopeV1, - aliases: Vec, + aliases: Vec, ingested_at: i64, -) -> RetrievalAnchorRecordV2 { +) -> RetrievalAnchorRecord { anchor_with_provenance( observation, owner, aliases, ingested_at, - AnchorSourceGenerationV2::Observation(observation.identity().generation()), + AnchorSourceGeneration::Observation(observation.identity().generation()), vec![observation.observation_id().clone()], ) } @@ -97,13 +97,13 @@ fn anchor( fn anchor_with_provenance( observation: &DurableObservationV1, owner: ObservationScopeV1, - aliases: Vec, + aliases: Vec, ingested_at: i64, - source_generation: AnchorSourceGenerationV2, + source_generation: AnchorSourceGeneration, source_observations: Vec, -) -> RetrievalAnchorRecordV2 { - RetrievalAnchorRecordV2::new(RetrievalAnchorRecordV2Parts { - target: RetrievalAnchorTargetV2::ExactObservation(observation.observation_id().clone()), +) -> RetrievalAnchorRecord { + RetrievalAnchorRecord::new(RetrievalAnchorRecordParts { + target: RetrievalAnchorTarget::ExactObservation(observation.observation_id().clone()), owner, aliases, occurred_at: None, @@ -206,9 +206,7 @@ fn anchored_write_rejects_mismatched_source_generation_and_lineage() { ObservationScopeV1::Profile, vec![], 1, - AnchorSourceGenerationV2::Observation( - ObservationSourceGenerationV1::new(8).unwrap() - ), + AnchorSourceGeneration::Observation(ObservationSourceGenerationV1::new(8).unwrap()), vec![candidate.observation_id().clone()], ), projection_generation(), @@ -223,7 +221,7 @@ fn anchored_write_rejects_mismatched_source_generation_and_lineage() { ObservationScopeV1::Profile, vec![], 1, - AnchorSourceGenerationV2::Observation(candidate.identity().generation()), + AnchorSourceGeneration::Observation(candidate.identity().generation()), vec![ candidate.observation_id().clone(), other.observation_id().clone(), diff --git a/crates/tracedecay-store/src/projection.rs b/crates/tracedecay-store/src/projection.rs index 9e8c78337c..03bc04dc55 100644 --- a/crates/tracedecay-store/src/projection.rs +++ b/crates/tracedecay-store/src/projection.rs @@ -435,9 +435,6 @@ fn workflow_fact_output_digest( .clone()) } -pub type ClaudeObservationProjection = ObservationProjection; -pub type ClaudeSessionMessageProjection = SessionMessageProjection; - #[derive(Clone, Debug, PartialEq, Eq)] pub struct ProjectionCheckpoint { last_sequence: u64, diff --git a/crates/tracedecay-store/src/provider_descriptor.rs b/crates/tracedecay-store/src/provider_descriptor.rs index c3c5dbdd48..3a3499f645 100644 --- a/crates/tracedecay-store/src/provider_descriptor.rs +++ b/crates/tracedecay-store/src/provider_descriptor.rs @@ -23,7 +23,6 @@ use tracedecay_domain::{CanonicalObservationFactV1, ObservationContractError}; -use crate::cursor_dispatch::is_subagent_dispatch_tool; use crate::{ ProjectionStoreError, ProjectionStoreResult, codex_goal_context_from_text, codex_message_visible_text, @@ -124,14 +123,14 @@ pub fn tool_metadata_normalizer(source: Option<&str>) -> Option, facts: &[CanonicalObservationFactV1], ) -> ProjectionStoreResult<()> { let mut tool_calls = Vec::new(); let mut tool_events = Vec::new(); - let mut first_dispatch_id = None; for fact in facts { let CanonicalObservationFactV1::ToolInvocation { invocation_id, @@ -160,16 +159,10 @@ fn normalize_cursor_tool_metadata( "call_id": invocation_id.as_str(), "input_bytes": input_bytes, })); - if first_dispatch_id.is_none() && is_subagent_dispatch_tool(name) { - first_dispatch_id = Some(invocation_id.as_str()); - } } if !tool_calls.is_empty() { metadata.insert("tool_calls".to_owned(), tool_calls.into()); metadata.insert("tool_events".to_owned(), tool_events.into()); } - if let Some(tool_use_id) = first_dispatch_id { - metadata.insert("tool_use_id".to_owned(), tool_use_id.into()); - } Ok(()) } diff --git a/crates/tracedecay-store/src/retrieval_anchor.rs b/crates/tracedecay-store/src/retrieval_anchor.rs index 523f213086..53b69589be 100644 --- a/crates/tracedecay-store/src/retrieval_anchor.rs +++ b/crates/tracedecay-store/src/retrieval_anchor.rs @@ -10,10 +10,7 @@ use serde::{Deserialize, Serialize}; use thiserror::Error; use tracedecay_domain::canonical_text::{CANONICAL_TEXT_MAX_BYTES, is_canonical_text_within}; use tracedecay_domain::errors::TraceDecayError; -use tracedecay_domain::{ - AnchorOwnerBindingV1, FactOwnerV1, ProjectionGenerationId, RetrievalAnchorId, - RetrievalAnchorRecordV2, RetrievalAnchorRecordV3, UtcMicros, -}; +use tracedecay_domain::{FactOwnerV1, RetrievalAnchorId, UtcMicros}; #[derive(Clone, Debug, Error, PartialEq, Eq)] pub enum RetrievalAnchorStoreError { @@ -36,91 +33,6 @@ impl From for TraceDecayError { } } -/// Exact physical owner encoding for both byte-compatible V2 anchors and V3 -/// profile/privacy-bound anchors. Untagged serialization preserves the -/// canonical owner JSON embedded in existing anchor rows. -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(untagged)] -pub enum RetrievalAnchorOwnerV1 { - V3(AnchorOwnerBindingV1), - V2(FactOwnerV1), -} - -impl RetrievalAnchorOwnerV1 { - pub fn validate(&self) -> RetrievalAnchorStoreResult<()> { - match self { - Self::V3(owner) => owner.validate().map_err(domain), - Self::V2(owner) => owner.validate().map_err(domain), - } - } - - pub fn v3(&self) -> Option<&AnchorOwnerBindingV1> { - match self { - Self::V3(owner) => Some(owner), - Self::V2(_) => None, - } - } - - pub fn v2(&self) -> Option<&FactOwnerV1> { - match self { - Self::V3(_) => None, - Self::V2(owner) => Some(owner), - } - } -} - -impl From for RetrievalAnchorOwnerV1 { - fn from(owner: AnchorOwnerBindingV1) -> Self { - Self::V3(owner) - } -} - -impl From for RetrievalAnchorOwnerV1 { - fn from(owner: FactOwnerV1) -> Self { - Self::V2(owner) - } -} - -/// Byte-compatible persisted anchor record across the V2/V3 cutover. -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(untagged)] -pub enum StoredRetrievalAnchorRecordV1 { - V3(RetrievalAnchorRecordV3), - V2(RetrievalAnchorRecordV2), -} - -impl StoredRetrievalAnchorRecordV1 { - pub fn validate(&self) -> RetrievalAnchorStoreResult<()> { - match self { - Self::V3(record) => record.validate().map_err(domain), - Self::V2(record) => record.validate().map_err(domain), - } - } - - pub fn anchor_id(&self) -> &RetrievalAnchorId { - match self { - Self::V3(record) => record.anchor_id(), - Self::V2(record) => record.anchor_id(), - } - } - - pub fn owner(&self) -> RetrievalAnchorOwnerV1 { - match self { - Self::V3(record) => RetrievalAnchorOwnerV1::V3(record.owner().clone()), - Self::V2(record) => { - RetrievalAnchorOwnerV1::V2(FactOwnerV1::from(record.owner().clone())) - } - } - } - - pub fn projection_generation(&self) -> &ProjectionGenerationId { - match self { - Self::V3(record) => record.projection_generation(), - Self::V2(record) => record.projection_generation(), - } - } -} - #[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "snake_case")] pub enum AnchorDispositionStateV1 { @@ -268,7 +180,7 @@ impl AnchorDerivativeKindV1 { pub struct RetrievalAnchorDispositionRecordV1 { disposition_id: String, anchor_id: RetrievalAnchorId, - owner: RetrievalAnchorOwnerV1, + owner: FactOwnerV1, state: AnchorDispositionStateV1, superseded_by: Option, reason_class: AnchorDispositionReasonClassV1, @@ -280,7 +192,7 @@ impl RetrievalAnchorDispositionRecordV1 { pub fn new( disposition_id: impl Into, anchor_id: RetrievalAnchorId, - owner: impl Into, + owner: FactOwnerV1, state: AnchorDispositionStateV1, superseded_by: Option, reason_class: AnchorDispositionReasonClassV1, @@ -289,7 +201,7 @@ impl RetrievalAnchorDispositionRecordV1 { let record = Self { disposition_id: disposition_id.into(), anchor_id, - owner: owner.into(), + owner, state, superseded_by, reason_class, @@ -307,7 +219,7 @@ impl RetrievalAnchorDispositionRecordV1 { &self.anchor_id } - pub fn owner(&self) -> &RetrievalAnchorOwnerV1 { + pub fn owner(&self) -> &FactOwnerV1 { &self.owner } @@ -333,7 +245,7 @@ impl RetrievalAnchorDispositionRecordV1 { pub fn validate(&self) -> RetrievalAnchorStoreResult<()> { validate_label(&self.disposition_id, "disposition id")?; self.anchor_id.validate().map_err(domain)?; - self.owner.validate()?; + self.owner.validate().map_err(domain)?; if let Some(successor) = &self.superseded_by { successor.validate().map_err(domain)?; if successor == &self.anchor_id { @@ -353,7 +265,7 @@ impl RetrievalAnchorDispositionRecordV1 { #[serde(deny_unknown_fields)] pub struct RetrievalAnchorDerivativeV1 { source_anchor_id: RetrievalAnchorId, - owner: RetrievalAnchorOwnerV1, + owner: FactOwnerV1, kind: AnchorDerivativeKindV1, derivative_id: String, direct_evidence: bool, @@ -362,14 +274,14 @@ pub struct RetrievalAnchorDerivativeV1 { impl RetrievalAnchorDerivativeV1 { pub fn new( source_anchor_id: RetrievalAnchorId, - owner: impl Into, + owner: FactOwnerV1, kind: AnchorDerivativeKindV1, derivative_id: impl Into, direct_evidence: bool, ) -> RetrievalAnchorStoreResult { let derivative = Self { source_anchor_id, - owner: owner.into(), + owner, kind, derivative_id: derivative_id.into(), direct_evidence, @@ -382,7 +294,7 @@ impl RetrievalAnchorDerivativeV1 { &self.source_anchor_id } - pub fn owner(&self) -> &RetrievalAnchorOwnerV1 { + pub fn owner(&self) -> &FactOwnerV1 { &self.owner } @@ -402,7 +314,7 @@ impl RetrievalAnchorDerivativeV1 { pub fn validate(&self) -> RetrievalAnchorStoreResult<()> { self.source_anchor_id.validate().map_err(domain)?; - self.owner.validate()?; + self.owner.validate().map_err(domain)?; validate_label(&self.derivative_id, "anchor derivative id") } } @@ -413,7 +325,7 @@ impl RetrievalAnchorDerivativeV1 { #[serde(deny_unknown_fields)] pub struct RetrievalAnchorTombstoneV1 { anchor_id: RetrievalAnchorId, - owner: RetrievalAnchorOwnerV1, + owner: FactOwnerV1, terminal_state: AnchorDispositionStateV1, reason_class: AnchorDispositionReasonClassV1, effective_at: UtcMicros, @@ -422,14 +334,14 @@ pub struct RetrievalAnchorTombstoneV1 { impl RetrievalAnchorTombstoneV1 { pub fn new( anchor_id: RetrievalAnchorId, - owner: impl Into, + owner: FactOwnerV1, terminal_state: AnchorDispositionStateV1, reason_class: AnchorDispositionReasonClassV1, effective_at: UtcMicros, ) -> RetrievalAnchorStoreResult { let record = Self { anchor_id, - owner: owner.into(), + owner, terminal_state, reason_class, effective_at, @@ -450,14 +362,14 @@ impl RetrievalAnchorTombstoneV1 { return Err(invalid("retrieval anchor tombstone terminal state")); } self.anchor_id.validate().map_err(domain)?; - self.owner.validate() + self.owner.validate().map_err(domain) } pub fn anchor_id(&self) -> &RetrievalAnchorId { &self.anchor_id } - pub fn owner(&self) -> &RetrievalAnchorOwnerV1 { + pub fn owner(&self) -> &FactOwnerV1 { &self.owner } @@ -500,20 +412,20 @@ pub trait RetrievalAnchorDispositionStore: Send + Sync { fn current_disposition( &self, anchor_id: &RetrievalAnchorId, - owner: &RetrievalAnchorOwnerV1, + owner: &FactOwnerV1, ) -> impl Future>> + Send; fn tombstone( &self, anchor_id: &RetrievalAnchorId, - owner: &RetrievalAnchorOwnerV1, + owner: &FactOwnerV1, ) -> impl Future>> + Send; fn derivatives( &self, anchor_id: &RetrievalAnchorId, - owner: &RetrievalAnchorOwnerV1, + owner: &FactOwnerV1, ) -> impl Future>> + Send; } @@ -535,7 +447,7 @@ fn invalid(message: impl Into) -> RetrievalAnchorStoreError { #[cfg(test)] mod tests { use super::*; - use tracedecay_domain::{PrivacyDomainId, ProjectId, UserProfileId}; + use tracedecay_domain::ProjectId; fn owner() -> FactOwnerV1 { FactOwnerV1::Project { @@ -654,34 +566,4 @@ mod tests { Err(RetrievalAnchorStoreError::InvalidData(_)) )); } - - #[test] - fn authority_owner_preserves_v2_wire_and_admits_exact_v3_owner() { - let legacy = owner(); - let authority = RetrievalAnchorOwnerV1::from(legacy.clone()); - assert_eq!( - serde_json::to_value(&authority).unwrap(), - serde_json::to_value(&legacy).unwrap() - ); - assert_eq!( - serde_json::from_value::( - serde_json::to_value(&legacy).unwrap() - ) - .unwrap(), - authority - ); - - let v3 = AnchorOwnerBindingV1::for_project( - UserProfileId::new("profile.fixture").unwrap(), - ProjectId::new("project.fixture").unwrap(), - PrivacyDomainId::new("privacy.fixture").unwrap(), - ) - .unwrap(); - let authority = RetrievalAnchorOwnerV1::from(v3.clone()); - assert_eq!( - serde_json::to_value(&authority).unwrap(), - serde_json::to_value(&v3).unwrap() - ); - assert_eq!(authority.v3(), Some(&v3)); - } } diff --git a/crates/tracedecay-store/src/runtime/mod.rs b/crates/tracedecay-store/src/runtime/mod.rs index 9f842f900c..733297e004 100644 --- a/crates/tracedecay-store/src/runtime/mod.rs +++ b/crates/tracedecay-store/src/runtime/mod.rs @@ -88,8 +88,7 @@ pub use outbox::{ pub use ports::{ RuntimeInterruptionV1, RuntimeReadCoverageV1, RuntimeReadOperationV1, RuntimeReadOutcomeV1, RuntimeReadRequestV1, RuntimeReadResultV1, RuntimeRequestProbeV1, RuntimeSubmitOutcomeV1, - RuntimeSubmitRequestV1, StorageRuntimePortErrorV1, StorageRuntimePortFutureV1, - StorageRuntimePortResultV1, StorageRuntimeReadPort, single_shard_required_coverage_v1, + RuntimeSubmitRequestV1, single_shard_required_coverage_v1, }; pub use repository_read::{ CodeReadOperationV1, CodeReadResultV1, CodeRecoveryCandidatesPageV1, diff --git a/crates/tracedecay-store/src/runtime/operation.rs b/crates/tracedecay-store/src/runtime/operation.rs index e3d19000aa..6449e2f6ec 100644 --- a/crates/tracedecay-store/src/runtime/operation.rs +++ b/crates/tracedecay-store/src/runtime/operation.rs @@ -4,12 +4,11 @@ use serde::{Deserialize, Deserializer, Serialize, Serializer}; use tracedecay_domain::{ObservationScopeV1, UtcMicros}; use crate::{ - AnchoredObservationWrite, ConfigurationCommitV1, DiagnosticGenerationSupersessionV1, - EvidenceAssemblyWriteV1, FactWriteBatch, GitIndexTransactionRecordV1, ObservationCursorAdvance, - RemoteObservationReplayWriteV1, RemoteWriterFenceInstallV1, RetrievalAnchorDerivativeV1, - RetrievalAnchorDispositionRecordV1, SanitizedCleanDiagnosticSnapshotV1, - SourceAcquisitionQueueCasV1, SourceCommitV1, SourceProjectionCommitV1, - TransactionalInboxReceiptV1, TransactionalOutboxEntryV1, + AnchoredObservationWrite, ConfigurationCommitV1, FactWriteBatch, GitIndexTransactionRecordV1, + ObservationCursorAdvance, RemoteObservationReplayWriteV1, RemoteWriterFenceInstallV1, + RetrievalAnchorDerivativeV1, RetrievalAnchorDispositionRecordV1, + SanitizedCleanDiagnosticSnapshotV1, SourceAcquisitionQueueCasV1, SourceCommitV1, + SourceProjectionCommitV1, TransactionalInboxReceiptV1, TransactionalOutboxEntryV1, }; use super::identity::{canonical_id, validate_canonical_id}; @@ -776,8 +775,6 @@ pub enum RepositoryWritePayloadV1 { RemoteObservationReplay(Box), RemoteWriterFenceInstall(Box), Diagnostics(Box), - DiagnosticSupersession(Box), - EvidenceAssembly(Box), ExternalSource(Box), ExternalSourceBatch(Box<[SourceCommitV1]>), ExternalSourceProjection(Box), @@ -801,8 +798,6 @@ impl RepositoryWritePayloadV1 { Self::RemoteObservationReplay(_) => "replay remote observation", Self::RemoteWriterFenceInstall(_) => "install remote writer fence", Self::Diagnostics(_) => "publish diagnostics", - Self::DiagnosticSupersession(_) => "supersede diagnostic generation", - Self::EvidenceAssembly(_) => "publish evidence assembly", Self::ExternalSource(_) => "commit external source", Self::ExternalSourceBatch(_) => "commit external source batch", Self::ExternalSourceProjection(_) => "project external source", @@ -830,8 +825,6 @@ impl RepositoryWritePayloadV1 { | Self::RemoteWriterFenceInstall(_) => "observation", Self::Fact(_) | Self::Diagnostics(_) - | Self::DiagnosticSupersession(_) - | Self::EvidenceAssembly(_) | Self::RetrievalAnchorDisposition(_) | Self::RetrievalAnchorDerivative(_) => "project", Self::ExternalSource(_) @@ -869,15 +862,7 @@ impl RepositoryWritePayloadV1 { StoreShardScopeV1::ProfileMemory | StoreShardScopeV1::Project { .. } ) } - Self::Diagnostics(_) | Self::DiagnosticSupersession(_) => { - matches!(scope, StoreShardScopeV1::Project { .. }) - } - Self::EvidenceAssembly(_) => matches!( - scope, - StoreShardScopeV1::Project { .. } - | StoreShardScopeV1::ProjectSessions { .. } - | StoreShardScopeV1::ProfileSessions - ), + Self::Diagnostics(_) => matches!(scope, StoreShardScopeV1::Project { .. }), Self::ExternalSource(commit) => matches!( (&commit.binding().owner, scope), ( @@ -968,11 +953,6 @@ impl RepositoryWritePayloadV1 { } }) } - Self::EvidenceAssembly(write) => write.validate().map_err(|_| { - StorageRuntimeContractErrorV1::InvalidRepositoryPayload { - payload: self.name(), - } - }), Self::ExternalSource(commit) => commit.validate().map_err(|_| { StorageRuntimeContractErrorV1::InvalidRepositoryPayload { payload: self.name(), @@ -998,11 +978,6 @@ impl RepositoryWritePayloadV1 { payload: self.name(), } }), - Self::DiagnosticSupersession(request) => request.validate().map_err(|_| { - StorageRuntimeContractErrorV1::InvalidRepositoryPayload { - payload: self.name(), - } - }), Self::RemoteObservationReplay(write) => write.validate(), Self::RemoteWriterFenceInstall(install) => install.validate(), Self::ObservationBatch(writes) if writes.is_empty() => { @@ -1077,17 +1052,6 @@ impl RepositoryOperationEnvelopeV1 { shard_family: "memory", }); } - if let RepositoryWritePayloadV1::EvidenceAssembly(write) = &self.payload { - let exact_owner = write.owner.owner.profile_id() - == &self.metadata.shard_id.profile_id - && write.owner.owner.project_id() == self.metadata.shard_id.scope.project_id(); - if !exact_owner { - return Err(StorageRuntimeContractErrorV1::OperationScopeMismatch { - operation: self.payload.family_name(), - shard_family: "project", - }); - } - } if let RepositoryWritePayloadV1::ExternalSource(commit) = &self.payload { let exact_owner = match (&commit.binding().owner, &self.metadata.shard_id.scope) { ( @@ -1247,18 +1211,14 @@ fn fact_owner_matches_shard( } fn retrieval_anchor_owner_matches_shard( - owner: &crate::RetrievalAnchorOwnerV1, + owner: &tracedecay_domain::FactOwnerV1, shard_id: &StoreShardIdV1, ) -> bool { match owner { - crate::RetrievalAnchorOwnerV1::V3(owner) => { - owner.profile_id() == &shard_id.profile_id - && owner.project_id() == shard_id.scope.project_id() + tracedecay_domain::FactOwnerV1::Project { project_id } => { + shard_id.scope.project_id() == Some(project_id) } - crate::RetrievalAnchorOwnerV1::V2(tracedecay_domain::FactOwnerV1::Project { - project_id, - }) => shard_id.scope.project_id() == Some(project_id), - crate::RetrievalAnchorOwnerV1::V2(tracedecay_domain::FactOwnerV1::Profile) => { + tracedecay_domain::FactOwnerV1::Profile => { matches!(&shard_id.scope, StoreShardScopeV1::ProfileSessions) } } diff --git a/crates/tracedecay-store/src/runtime/ports.rs b/crates/tracedecay-store/src/runtime/ports.rs index 5e81f6651d..2f4b4a3427 100644 --- a/crates/tracedecay-store/src/runtime/ports.rs +++ b/crates/tracedecay-store/src/runtime/ports.rs @@ -1,21 +1,17 @@ -use std::future::Future; -use std::pin::Pin; - use super::{ CommitSequenceV1, ConsistencyModeV1, FrozenWatermarkCoverageV1, FrozenWatermarkVectorV1, GraphNodeV1, GraphSearchResultV1, GraphStatsV1, MaintenanceTelemetryV1, OperationPriorityV1, ReaderHealthLeaseIdV1, ReaderHealthLeaseV1, RuntimeCancellationIdentityV1, RuntimeCancellationStageV1, RuntimeDeadlineV1, RuntimeRequestControlV1, RuntimeTransactionScopeV1, SaturationScopeV1, ShardWatermarkV1, SnapshotLeaseIdV1, - SnapshotLeaseV1, StorageRuntimeContractErrorV1, StorageRuntimeErrorV1, StoreCommitReceiptV1, - StoreRuntimeBindingV1, UnavailableReasonV1, WatermarkCoverageStatusV1, + SnapshotLeaseV1, StorageRuntimeContractErrorV1, StoreCommitReceiptV1, StoreRuntimeBindingV1, + UnavailableReasonV1, WatermarkCoverageStatusV1, }; use super::{ RepositoryOperationEnvelopeV1, RepositoryReadOperationV1, RepositoryReadResultV1, StoreAuthorityEpochV1, StoreShardIdV1, }; use serde::{Deserialize, Deserializer, Serialize}; -use thiserror::Error; /// One caller-owned monotonic interruption decision. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -103,7 +99,7 @@ impl RuntimeSubmitRequestV1 { } /// Idempotent write outcomes, including expected admission and cancellation -/// states. Driver failures remain `StorageRuntimePortErrorV1`; these variants +/// states. Driver failures remain `StorageRuntimeErrorV1`; these variants /// are stable runtime decisions callers must handle explicitly. #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] @@ -820,98 +816,6 @@ fn binding_matches_watermark( && binding.authority_epoch == watermark.authority_epoch } -fn validate_probe( - control: &RuntimeRequestControlV1, - probe: &dyn RuntimeRequestProbeV1, -) -> Result<(), StorageRuntimeContractErrorV1> { - if probe.cancellation_identity() != &control.cancellation { - return Err(StorageRuntimeContractErrorV1::ReceiptBindingMismatch { - field: "runtime cancellation probe identity", - }); - } - if probe.deadline_identity() != &control.deadline { - return Err(StorageRuntimeContractErrorV1::ReceiptBindingMismatch { - field: "runtime deadline probe identity", - }); - } - Ok(()) -} - -fn read_interruption( - probe: &dyn RuntimeRequestProbeV1, -) -> Result, StorageRuntimeContractErrorV1> { - let reason = match probe.interruption() { - Some(RuntimeInterruptionV1::Cancelled) => UnavailableReasonV1::Cancelled, - Some(RuntimeInterruptionV1::DeadlineExceeded) => UnavailableReasonV1::DeadlineExceeded, - None => return Ok(None), - }; - RuntimeReadOutcomeV1::new( - None, - RuntimeReadCoverageV1::Unavailable { - coverage: None, - reason, - }, - ) - .map(Some) -} - -#[derive(Debug, Error)] -pub enum StorageRuntimePortErrorV1 { - #[error("invalid storage runtime request: {0}")] - InvalidRequest(StorageRuntimeContractErrorV1), - #[error("invalid storage runtime response: {0}")] - InvalidResponse(StorageRuntimeContractErrorV1), - #[error(transparent)] - Runtime(Box), -} - -impl From for StorageRuntimePortErrorV1 { - fn from(error: StorageRuntimeErrorV1) -> Self { - Self::Runtime(Box::new(error)) - } -} - -pub type StorageRuntimePortResultV1 = Result; -pub type StorageRuntimePortFutureV1<'a, T> = - Pin> + Send + 'a>>; - -/// Object-safe std-only asynchronous read boundary. -pub trait StorageRuntimeReadPort: Send + Sync { - fn dispatch_read<'a>( - &'a self, - request: RuntimeReadRequestV1, - probe: &'a dyn RuntimeRequestProbeV1, - ) -> StorageRuntimePortFutureV1<'a, RuntimeReadOutcomeV1>; - - fn read<'a>( - &'a self, - request: RuntimeReadRequestV1, - probe: &'a dyn RuntimeRequestProbeV1, - ) -> StorageRuntimePortFutureV1<'a, RuntimeReadOutcomeV1> { - Box::pin(async move { - request - .validate() - .and_then(|()| validate_probe(request.control(), probe)) - .map_err(StorageRuntimePortErrorV1::InvalidRequest)?; - if let Some(outcome) = - read_interruption(probe).map_err(StorageRuntimePortErrorV1::InvalidResponse)? - { - return Ok(outcome); - } - let outcome = self.dispatch_read(request.clone(), probe).await?; - outcome - .validate_for(&request) - .map_err(StorageRuntimePortErrorV1::InvalidResponse)?; - if let Some(interrupted) = - read_interruption(probe).map_err(StorageRuntimePortErrorV1::InvalidResponse)? - { - return Ok(interrupted); - } - Ok(outcome) - }) - } -} - // Keep the single-shard requirement helper explicit so adapter migrations do // not infer a vector from mutable ambient runtime state. pub fn single_shard_required_coverage_v1( diff --git a/crates/tracedecay-store/src/runtime/repository_read.rs b/crates/tracedecay-store/src/runtime/repository_read.rs index 4ba07bcb52..ad614f8a82 100644 --- a/crates/tracedecay-store/src/runtime/repository_read.rs +++ b/crates/tracedecay-store/src/runtime/repository_read.rs @@ -13,22 +13,20 @@ use serde::{Deserialize, Serialize}; use tracedecay_domain::{ CanonicalObservationIdV1, CodeGenerationId, ConfigurationRevisionId, DurableObservationV1, - FactLineageEventV1, FileOccurrenceId, GenerationDiagnosticV1, GitIndexIdempotencyKey, - GitIndexPreviewId, GitIndexPreviewV1, NativeAliasV2, ObservationScopeV1, + FactLineageEventV1, FactOwnerV1, FileOccurrenceId, GenerationDiagnosticV1, + GitIndexIdempotencyKey, GitIndexPreviewId, GitIndexPreviewV1, NativeAlias, ObservationScopeV1, ObservationSourceCursorV1, ObservationSourceIdentityV1, ProjectionGenerationId, RepositoryId, - RetrievalAnchorId, RetrievalAnchorRecordV2, SourceBindingIdentityV1, SourceBindingOwnerV1, + RetrievalAnchorId, RetrievalAnchorRecord, SourceBindingIdentityV1, SourceBindingOwnerV1, UtcMicros, }; use crate::{ - ConfigurationRevisionRecordV1, EvidenceAssemblyReadOperationV1, EvidenceAssemblyReadResultV1, - FactCurrentQuery, FactLineageQuery, GitIndexTransactionRecordV1, + ConfigurationRevisionRecordV1, FactCurrentQuery, FactLineageQuery, GitIndexTransactionRecordV1, RepositoryProvenanceAttachmentV1, RetrievalAnchorDerivativeV1, - RetrievalAnchorDispositionRecordV1, RetrievalAnchorOwnerV1, RetrievalAnchorTombstoneV1, - SourceAcquisitionQueueStateV1, SourceCommitReceiptV1, SourcePendingProjectionV1, - SourceStoreStateV1, StorageRuntimeContractErrorV1, StoreEffectIdV1, StoreRuntimeBindingV1, - StoreShardIdV1, StoreShardScopeV1, StoredFactV1, StoredRetrievalAnchorRecordV1, - TransactionalInboxReceiptV1, TransactionalOutboxEntryV1, + RetrievalAnchorDispositionRecordV1, RetrievalAnchorTombstoneV1, SourceAcquisitionQueueStateV1, + SourceCommitReceiptSummaryV1, SourcePendingProjectionV1, SourceStoreStateV1, + StorageRuntimeContractErrorV1, StoreEffectIdV1, StoreRuntimeBindingV1, StoreShardIdV1, + StoreShardScopeV1, StoredFactV1, TransactionalInboxReceiptV1, TransactionalOutboxEntryV1, }; /// One repository read operation, dispatched across the profile, project, @@ -66,9 +64,6 @@ impl RepositoryReadOperationV1 { Self::Project(ProjectReadOperationV1::Diagnostics(_)) => { matches!(&binding.shard_id.scope, StoreShardScopeV1::Project { .. }) } - Self::Project(ProjectReadOperationV1::EvidenceAssembly(operation)) => { - evidence_owner_matches_shard(evidence_read_owner(operation), &binding.shard_id) - } Self::ExternalSource(operation) => { external_source_read_matches_shard(operation, &binding.shard_id) } @@ -142,35 +137,6 @@ fn observation_read_matches_shard( } } -fn evidence_read_owner( - operation: &EvidenceAssemblyReadOperationV1, -) -> &crate::EvidenceAssemblyOwnerV1 { - match operation { - EvidenceAssemblyReadOperationV1::PublicationByIdempotency { owner, .. } - | EvidenceAssemblyReadOperationV1::ContributionPage { owner, .. } => owner, - } -} - -fn evidence_owner_matches_shard( - owner: &crate::EvidenceAssemblyOwnerV1, - shard: &StoreShardIdV1, -) -> bool { - owner.owner.profile_id() == &shard.profile_id - && match (&shard.scope, owner.owner.project_id()) { - ( - StoreShardScopeV1::Project { - project_id: shard_project, - } - | StoreShardScopeV1::ProjectSessions { - project_id: shard_project, - }, - Some(project_id), - ) => shard_project == project_id, - (StoreShardScopeV1::ProfileSessions, None) => true, - _ => false, - } -} - fn external_source_read_matches_shard( operation: &ExternalSourceReadOperationV1, shard: &StoreShardIdV1, @@ -205,7 +171,7 @@ fn external_source_read_matches_shard( } } -fn retrieval_read_owner(operation: &RetrievalAnchorReadOperationV1) -> &RetrievalAnchorOwnerV1 { +fn retrieval_read_owner(operation: &RetrievalAnchorReadOperationV1) -> &FactOwnerV1 { match operation { RetrievalAnchorReadOperationV1::AnchorById { owner, .. } | RetrievalAnchorReadOperationV1::CurrentDisposition { owner, .. } @@ -214,25 +180,9 @@ fn retrieval_read_owner(operation: &RetrievalAnchorReadOperationV1) -> &Retrieva } } -fn retrieval_owner_matches_shard(owner: &RetrievalAnchorOwnerV1, shard: &StoreShardIdV1) -> bool { +fn retrieval_owner_matches_shard(owner: &FactOwnerV1, shard: &StoreShardIdV1) -> bool { match owner { - RetrievalAnchorOwnerV1::V3(owner) => { - owner.profile_id() == &shard.profile_id - && match (&shard.scope, owner.project_id()) { - ( - StoreShardScopeV1::Project { - project_id: shard_project, - } - | StoreShardScopeV1::ProjectSessions { - project_id: shard_project, - }, - Some(project_id), - ) => shard_project == project_id, - (StoreShardScopeV1::ProfileSessions, None) => true, - _ => false, - } - } - RetrievalAnchorOwnerV1::V2(tracedecay_domain::FactOwnerV1::Project { project_id }) => { + FactOwnerV1::Project { project_id } => { matches!( &shard.scope, StoreShardScopeV1::Project { @@ -242,9 +192,7 @@ fn retrieval_owner_matches_shard(owner: &RetrievalAnchorOwnerV1, shard: &StoreSh } if shard_project == project_id ) } - RetrievalAnchorOwnerV1::V2(tracedecay_domain::FactOwnerV1::Profile) => { - matches!(&shard.scope, StoreShardScopeV1::ProfileSessions) - } + FactOwnerV1::Profile => matches!(&shard.scope, StoreShardScopeV1::ProfileSessions), } } @@ -314,7 +262,6 @@ pub enum ProjectReadOperationV1 { Fact(FactReadOperationV1), Observation(ObservationReadOperationV1), Diagnostics(DiagnosticReadOperationV1), - EvidenceAssembly(EvidenceAssemblyReadOperationV1), RetrievalAnchor(RetrievalAnchorReadOperationV1), } @@ -328,7 +275,6 @@ pub enum ProjectReadResultV1 { Fact(FactReadResultV1), Observation(ObservationReadResultV1), Diagnostics(DiagnosticReadResultV1), - EvidenceAssembly(EvidenceAssemblyReadResultV1), RetrievalAnchor(RetrievalAnchorReadResultV1), } @@ -360,7 +306,7 @@ pub enum ExternalSourceReadOperationV1 { #[serde(rename_all = "snake_case")] pub enum ExternalSourceReadResultV1 { State(Option>), - CommitReceipt(Option>), + CommitReceipt(Option>), PendingProjection(Option>), AcquisitionState(Option>), AcquisitionPendingCount(u64), @@ -373,19 +319,19 @@ pub enum ExternalSourceReadResultV1 { pub enum RetrievalAnchorReadOperationV1 { AnchorById { anchor_id: RetrievalAnchorId, - owner: RetrievalAnchorOwnerV1, + owner: FactOwnerV1, }, CurrentDisposition { anchor_id: RetrievalAnchorId, - owner: RetrievalAnchorOwnerV1, + owner: FactOwnerV1, }, Derivatives { anchor_id: RetrievalAnchorId, - owner: RetrievalAnchorOwnerV1, + owner: FactOwnerV1, }, Tombstone { anchor_id: RetrievalAnchorId, - owner: RetrievalAnchorOwnerV1, + owner: FactOwnerV1, }, } @@ -395,7 +341,7 @@ pub enum RetrievalAnchorReadOperationV1 { // store-protocol API and ripple through construction/match sites. #[allow(clippy::large_enum_variant)] pub enum RetrievalAnchorReadResultV1 { - Anchor(Option), + Anchor(Option), CurrentDisposition(Option), Derivatives(Vec), Tombstone(Option), @@ -430,7 +376,7 @@ pub enum ObservationReadOperationV1 { }, RetrievalAnchorByAlias { scope: ObservationScopeV1, - alias: NativeAliasV2, + alias: NativeAlias, }, Replay { after_sequence: u64, @@ -449,7 +395,7 @@ pub struct StoredObservationRowV1 { pub sequence: u64, pub observation: DurableObservationV1, pub committed_cursor: ObservationSourceCursorV1, - pub retrieval_anchor: RetrievalAnchorRecordV2, + pub retrieval_anchor: RetrievalAnchorRecord, pub projection_generation: ProjectionGenerationId, pub repository_provenance: RepositoryProvenanceAttachmentV1, pub projection_queued: bool, @@ -489,13 +435,6 @@ pub enum ObservationReadResultV1 { } /// Diagnostic-family read operations. -/// -/// The variant set covers the whole read surface of -/// [`DiagnosticStore`](crate::DiagnosticStore) so a storage cutover cannot -/// silently drop a lane: `Stale` answers `stale_diagnostics` and -/// `SupersessionChain` answers `diagnostic_supersession_chain`. Both are -/// history lanes, they read records that active publication excludes, and -/// neither may re-admit a stale record into the current set. #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "snake_case")] pub enum DiagnosticReadOperationV1 { @@ -510,10 +449,6 @@ pub enum DiagnosticReadOperationV1 { file_occurrence_id: FileOccurrenceId, }, ByAnchor(RetrievalAnchorId), - /// Superseded and cleared records bound to one generation. - Stale(CodeGenerationId), - /// The logical finding chain rooted at one diagnostic anchor, oldest first. - SupersessionChain(RetrievalAnchorId), } /// Diagnostic-family read results. @@ -656,15 +591,12 @@ pub enum EffectsReadResultV1 { #[cfg(test)] mod tests { use tracedecay_domain::{ - AnchorOwnerBindingV1, BrainId, FactOwnerV1, ManifestDigest, PrivacyDomainId, ProjectId, - RepositoryId, RetrievalAnchorId, SessionId, UserProfileId, WorktreeId, + BrainId, FactOwnerV1, ProjectId, RepositoryId, RetrievalAnchorId, SessionId, UserProfileId, + WorktreeId, }; use super::*; - use crate::{ - CodeShardScopeV1, EvidenceAssemblyIdempotencyKeyV1, EvidenceAssemblyOwnerV1, - StoreAuthorityEpochV1, StoreIncarnationV1, - }; + use crate::{CodeShardScopeV1, StoreAuthorityEpochV1, StoreIncarnationV1}; fn binding(profile: &str, scope: StoreShardScopeV1) -> StoreRuntimeBindingV1 { StoreRuntimeBindingV1::new( @@ -682,21 +614,6 @@ mod tests { ProjectId::new(value).unwrap() } - fn evidence_owner(profile: &str, project_id: Option) -> EvidenceAssemblyOwnerV1 { - let profile_id = UserProfileId::new(profile).unwrap(); - let privacy = PrivacyDomainId::new("privacy.fixture").unwrap(); - EvidenceAssemblyOwnerV1 { - owner: match project_id { - Some(project_id) => { - AnchorOwnerBindingV1::for_project(profile_id, project_id, privacy).unwrap() - } - None => AnchorOwnerBindingV1::for_profile(profile_id, privacy).unwrap(), - }, - scope_digest: ManifestDigest::new(format!("sha256:{}", "aa".repeat(32))).unwrap(), - key_epoch: 1, - } - } - #[test] fn repository_reads_require_exact_family_profile_and_project_binding() { let project_a = project("project.a"); @@ -720,7 +637,6 @@ mod tests { }, ); let profile_sessions_a = binding("profile.a", StoreShardScopeV1::ProfileSessions); - let profile_sessions_b = binding("profile.b", StoreShardScopeV1::ProfileSessions); let profile_memory_a = binding("profile.a", StoreShardScopeV1::ProfileMemory); let source_cursor = RepositoryReadOperationV1::Project( @@ -750,48 +666,12 @@ mod tests { .is_err() ); - let evidence = - RepositoryReadOperationV1::Project(ProjectReadOperationV1::EvidenceAssembly( - EvidenceAssemblyReadOperationV1::PublicationByIdempotency { - owner: evidence_owner("profile.a", Some(project_a.clone())), - idempotency_key: EvidenceAssemblyIdempotencyKeyV1::new( - ManifestDigest::new(format!("sha256:{}", "bb".repeat(32))).unwrap(), - ) - .unwrap(), - }, - )); - assert!(evidence.validate_for_binding(&project_sessions_a).is_ok()); - assert!(evidence.validate_for_binding(&project_sessions_b).is_err()); - assert!(evidence.validate_for_binding(&profile_sessions_a).is_err()); - - let profile_evidence = - RepositoryReadOperationV1::Project(ProjectReadOperationV1::EvidenceAssembly( - EvidenceAssemblyReadOperationV1::PublicationByIdempotency { - owner: evidence_owner("profile.a", None), - idempotency_key: EvidenceAssemblyIdempotencyKeyV1::new( - ManifestDigest::new(format!("sha256:{}", "cc".repeat(32))).unwrap(), - ) - .unwrap(), - }, - )); - assert!( - profile_evidence - .validate_for_binding(&profile_sessions_a) - .is_ok() - ); - assert!( - profile_evidence - .validate_for_binding(&profile_sessions_b) - .is_err() - ); - let retrieval = RepositoryReadOperationV1::Project( ProjectReadOperationV1::RetrievalAnchor(RetrievalAnchorReadOperationV1::AnchorById { anchor_id: RetrievalAnchorId::new("retrieval.fixture").unwrap(), owner: FactOwnerV1::Project { project_id: project_a.clone(), - } - .into(), + }, }), ); assert!(retrieval.validate_for_binding(&project_a_binding).is_ok()); diff --git a/crates/tracedecay-store/src/session/projection.rs b/crates/tracedecay-store/src/session/projection.rs index 7844053eee..2c83c4335f 100644 --- a/crates/tracedecay-store/src/session/projection.rs +++ b/crates/tracedecay-store/src/session/projection.rs @@ -5,7 +5,7 @@ use tracedecay_domain::{ LogicalCopyRecordV1, MessageOccurrenceRecordV1, SessionId, SessionProjectionGenerationV1, TemporalAssertionRecordV1, UtcMicros, }; -use tracedecay_temporal_query::ports::ExecutionControl; +use tracedecay_temporal_query::execution::ExecutionControl; use super::common::{ SessionFrozenWatermarksV1, SessionGenerationActivatePermit, diff --git a/crates/tracedecay-store/src/session/refresh.rs b/crates/tracedecay-store/src/session/refresh.rs index c6c8696dc9..54f525dfb3 100644 --- a/crates/tracedecay-store/src/session/refresh.rs +++ b/crates/tracedecay-store/src/session/refresh.rs @@ -7,7 +7,7 @@ use tracedecay_domain::{ SessionId, SessionRefreshKeyV1, SessionRefreshOperationIdV1, SessionSourceCoverageReceiptV1, SessionTemporalCoverageRequestV1, TemporalCoverageCountsV1, TemporalModeV1, UtcMicros, }; -use tracedecay_temporal_query::ports::ExecutionControl; +use tracedecay_temporal_query::execution::ExecutionControl; use super::common::{ SessionRefreshBeginOrJoinPermit, SessionRefreshCancelPermit, SessionRefreshCompletePermit, diff --git a/crates/tracedecay-store/src/session/retrieval.rs b/crates/tracedecay-store/src/session/retrieval.rs index 0af8ba37a5..6c1946804c 100644 --- a/crates/tracedecay-store/src/session/retrieval.rs +++ b/crates/tracedecay-store/src/session/retrieval.rs @@ -5,7 +5,7 @@ use tracedecay_domain::{ SessionId, SessionSummaryRecordV1, TemporalAssertionRecordV1, TemporalCoverageCountsV1, TemporalModeV1, }; -use tracedecay_temporal_query::ports::ExecutionControl; +use tracedecay_temporal_query::execution::ExecutionControl; use super::common::{ SessionSnapshotFreezePermit, SessionStoreError, SessionStoreResult, diff --git a/crates/tracedecay-store/tests/store_suite/diagnostics_contract.rs b/crates/tracedecay-store/tests/store_suite/diagnostics_contract.rs index 2342cd64a0..dab3df7dda 100644 --- a/crates/tracedecay-store/tests/store_suite/diagnostics_contract.rs +++ b/crates/tracedecay-store/tests/store_suite/diagnostics_contract.rs @@ -83,7 +83,7 @@ fn clean_snapshot_rejects_cross_snapshot_or_stale_records() { )); let stale = fixture_record(generation.as_str(), "anchor.diagnostic.stale") - .supersede(id("generation.clean.2")) + .clear(id("generation.clean.2")) .unwrap(); assert!(matches!( SanitizedCleanDiagnosticSnapshotV1::new(generation.clone(), vec![stale]), diff --git a/crates/tracedecay-store/tests/store_suite/session_contract/capabilities.rs b/crates/tracedecay-store/tests/store_suite/session_contract/capabilities.rs index e3d1d74499..81021714e9 100644 --- a/crates/tracedecay-store/tests/store_suite/session_contract/capabilities.rs +++ b/crates/tracedecay-store/tests/store_suite/session_contract/capabilities.rs @@ -1,6 +1,6 @@ use super::common::*; use super::*; -use tracedecay_temporal_query::ports::ExecutionControl; +use tracedecay_temporal_query::execution::ExecutionControl; struct CapabilityDeniedSessionPorts { capabilities: SessionTemporalCapabilitiesV1, diff --git a/crates/tracedecay-store/tests/store_suite/session_contract/projection.rs b/crates/tracedecay-store/tests/store_suite/session_contract/projection.rs index 6265325b7b..643e950323 100644 --- a/crates/tracedecay-store/tests/store_suite/session_contract/projection.rs +++ b/crates/tracedecay-store/tests/store_suite/session_contract/projection.rs @@ -1,6 +1,6 @@ use super::common::*; use super::*; -use tracedecay_temporal_query::ports::ExecutionControl; +use tracedecay_temporal_query::execution::ExecutionControl; #[test] fn rebuild_and_activation_validate_session_capability_and_generation_transition() { diff --git a/crates/tracedecay-store/tests/store_suite/session_contract/retrieval.rs b/crates/tracedecay-store/tests/store_suite/session_contract/retrieval.rs index 900e81d1dc..c3412469ed 100644 --- a/crates/tracedecay-store/tests/store_suite/session_contract/retrieval.rs +++ b/crates/tracedecay-store/tests/store_suite/session_contract/retrieval.rs @@ -1,6 +1,6 @@ use super::common::*; use super::*; -use tracedecay_temporal_query::ports::ExecutionControl; +use tracedecay_temporal_query::execution::ExecutionControl; #[test] fn frozen_snapshots_preserve_exact_session_and_reject_cross_session_reads() { diff --git a/crates/tracedecay-store/tests/store_suite/storage_runtime_contract.rs b/crates/tracedecay-store/tests/store_suite/storage_runtime_contract.rs index 240785f68c..f2a2fb307f 100644 --- a/crates/tracedecay-store/tests/store_suite/storage_runtime_contract.rs +++ b/crates/tracedecay-store/tests/store_suite/storage_runtime_contract.rs @@ -1,8 +1,5 @@ use std::fmt::Debug; -use std::future::Future; -use std::pin::pin; -use std::sync::atomic::{AtomicBool, AtomicU8, AtomicUsize, Ordering}; -use std::task::{Context, Poll, Waker}; +use std::sync::atomic::{AtomicBool, AtomicU8, Ordering}; use serde::Serialize; use serde::de::DeserializeOwned; @@ -148,18 +145,6 @@ fn submit_request(metadata: StoreOperationMetadataV1) -> RuntimeSubmitRequestV1 .unwrap() } -fn block_on(future: F) -> F::Output { - let waker = Waker::noop(); - let mut context = Context::from_waker(waker); - let mut future = pin!(future); - loop { - if let Poll::Ready(output) = future.as_mut().poll(&mut context) { - return output; - } - std::thread::yield_now(); - } -} - fn commit_receipt(metadata: &StoreOperationMetadataV1) -> StoreCommitReceiptV1 { StoreCommitReceiptV1 { operation_id: metadata.operation_id.clone(), @@ -594,37 +579,6 @@ fn runtime_commit_probe_grants_at_most_one_commit() { assert!(!cancelled.try_begin_commit()); } -struct FakeReadPort { - calls: AtomicUsize, -} - -impl StorageRuntimeReadPort for FakeReadPort { - fn dispatch_read<'a>( - &'a self, - request: RuntimeReadRequestV1, - _probe: &'a dyn RuntimeRequestProbeV1, - ) -> StorageRuntimePortFutureV1<'a, RuntimeReadOutcomeV1> { - self.calls.fetch_add(1, Ordering::SeqCst); - Box::pin(async move { - let observed = ShardWatermarkV1 { - shard_id: request.binding().shard_id.clone(), - incarnation: request.binding().incarnation, - authority_epoch: request.binding().authority_epoch, - commit_sequence: CommitSequenceV1(9), - }; - RuntimeReadOutcomeV1::new( - Some(RuntimeReadResultV1::CurrentWatermark { - watermark: observed.clone(), - }), - RuntimeReadCoverageV1::Latest { - observed: Some(observed), - }, - ) - .map_err(StorageRuntimePortErrorV1::InvalidResponse) - }) - } -} - fn read_request( binding: StoreRuntimeBindingV1, consistency: ConsistencyModeV1, @@ -682,51 +636,7 @@ fn runtime_submit_outcomes_validate_request_identity() { } #[test] -fn typed_async_reads_report_latest_exact_partial_stale_and_unavailable_coverage() { - let latest = read_request( - binding(project_shard("project.one")), - ConsistencyModeV1::LatestAvailable, - RuntimeReadOperationV1::CurrentWatermark, - ); - let probe = Probe::new(latest.control(), None); - let read_port = FakeReadPort { - calls: AtomicUsize::new(0), - }; - let object_safe_port: &dyn StorageRuntimeReadPort = &read_port; - assert!(matches!( - block_on(object_safe_port.read(latest, &probe)) - .unwrap() - .coverage(), - RuntimeReadCoverageV1::Latest { .. } - )); - - for (interruption, reason) in [ - ( - RuntimeInterruptionV1::Cancelled, - UnavailableReasonV1::Cancelled, - ), - ( - RuntimeInterruptionV1::DeadlineExceeded, - UnavailableReasonV1::DeadlineExceeded, - ), - ] { - let request = read_request( - binding(project_shard("project.one")), - ConsistencyModeV1::LatestAvailable, - RuntimeReadOperationV1::CurrentWatermark, - ); - let probe = Probe::new(request.control(), Some(interruption)); - let outcome = block_on(object_safe_port.read(request, &probe)).unwrap(); - assert!(matches!( - outcome.coverage(), - RuntimeReadCoverageV1::Unavailable { - coverage: None, - reason: actual, - } if *actual == reason - )); - } - assert_eq!(read_port.calls.load(Ordering::SeqCst), 1); - +fn read_outcomes_validate_exact_partial_stale_and_unavailable_coverage() { let at_least = read_request( binding(project_shard("project.one")), ConsistencyModeV1::AtLeast { diff --git a/crates/tracedecay-temporal-query/src/context/admission.rs b/crates/tracedecay-temporal-query/src/context/admission.rs index c88824be15..670641724b 100644 --- a/crates/tracedecay-temporal-query/src/context/admission.rs +++ b/crates/tracedecay-temporal-query/src/context/admission.rs @@ -6,7 +6,7 @@ use tracedecay_domain::{ ContextOmissionReasonV1, HydrationStateV1, RetrievalGrainV1, }; -use super::super::ports::ExecutionControl; +use super::super::execution::ExecutionControl; use super::super::resolution::summary::SummaryOmission; use super::assembly::{try_reserve, validate_bundle}; use super::wire::{ diff --git a/crates/tracedecay-temporal-query/src/context/assembly.rs b/crates/tracedecay-temporal-query/src/context/assembly.rs index 949f5f5133..86ffdfb2a3 100644 --- a/crates/tracedecay-temporal-query/src/context/assembly.rs +++ b/crates/tracedecay-temporal-query/src/context/assembly.rs @@ -6,8 +6,8 @@ use tracedecay_domain::{ CompactContextOmissionV1, ContextOmissionReasonV1, RetrievalAnchorId, RetrievalGrainV1, }; +use super::super::execution::ExecutionControl; use super::super::hydration::HydrationBatch; -use super::super::ports::ExecutionControl; use super::super::resolution::summary::{SummaryLineageRejection, SummaryOmission}; use super::admission::{ choose_admission, materialize_admission, measure_context, prepare_admission, render_exact, diff --git a/crates/tracedecay-temporal-query/src/context/estimation.rs b/crates/tracedecay-temporal-query/src/context/estimation.rs index 71fd577a33..2bdd8a3fc5 100644 --- a/crates/tracedecay-temporal-query/src/context/estimation.rs +++ b/crates/tracedecay-temporal-query/src/context/estimation.rs @@ -1,4 +1,4 @@ -use super::super::ports::ExecutionControl; +use super::super::execution::ExecutionControl; use super::{ContextError, TokenPolicy}; pub const TOKEN_SCAN_CHUNK_BYTES: usize = 4 * 1024; diff --git a/crates/tracedecay-temporal-query/src/context/tests.rs b/crates/tracedecay-temporal-query/src/context/tests.rs index 007925a5f5..29fb9e9703 100644 --- a/crates/tracedecay-temporal-query/src/context/tests.rs +++ b/crates/tracedecay-temporal-query/src/context/tests.rs @@ -16,7 +16,8 @@ use super::{ MAX_CONTEXT_FRAME_ITEMS, MAX_CONTEXT_OUTPUT_BYTES, OrderedTextContextAssembler, TemporalContextFrames, TokenPolicy, VersionedTokenEstimator, }; -use crate::ports::{ExecutionControl, ReadBudgetAccounting, TemporalPortError}; +use crate::execution::ExecutionControl; +use crate::ports::{ReadBudgetAccounting, TemporalPortError}; use crate::resolution::summary::{SummaryLineageRejection, SummaryOmission}; #[derive(Clone, Debug, PartialEq, Eq)] struct HydratedPayload { @@ -331,7 +332,7 @@ fn unavailable_hydration_states_have_explicit_metadata_only_reasons() { ContextOmissionReasonV1::Unavailable, ), ( - HydrationStateV1::UnverifiableLegacy, + HydrationStateV1::Unverifiable, ContextOmissionReasonV1::Unavailable, ), ]; diff --git a/crates/tracedecay-temporal-query/src/context/wire.rs b/crates/tracedecay-temporal-query/src/context/wire.rs index 1e5ecbe68a..d2769d7132 100644 --- a/crates/tracedecay-temporal-query/src/context/wire.rs +++ b/crates/tracedecay-temporal-query/src/context/wire.rs @@ -4,7 +4,8 @@ use serde::ser::{SerializeSeq, SerializeStruct}; use serde::{Serialize, Serializer}; use tracedecay_domain::{CompactContextBundleV1, ContextOmissionReasonV1, HydrationStateV1}; -use super::super::ports::{ExecutionControl, TemporalPortError}; +use super::super::execution::ExecutionControl; +use super::super::ports::TemporalPortError; use super::super::resolution::summary::SummaryOmission; use super::estimation::{TOKEN_SCAN_CHUNK_BYTES, TokenSummary}; use super::{ContextError, ContextPayload, MAX_CONTEXT_OUTPUT_BYTES, TokenPolicy}; @@ -269,6 +270,6 @@ pub const fn omission_reason(state: HydrationStateV1) -> ContextOmissionReasonV1 HydrationStateV1::Locked => ContextOmissionReasonV1::Locked, HydrationStateV1::Available | HydrationStateV1::RetainedButUnavailable - | HydrationStateV1::UnverifiableLegacy => ContextOmissionReasonV1::Unavailable, + | HydrationStateV1::Unverifiable => ContextOmissionReasonV1::Unavailable, } } diff --git a/crates/tracedecay-temporal-query/src/cursor.rs b/crates/tracedecay-temporal-query/src/cursor.rs index 5bf069e64f..614381a298 100644 --- a/crates/tracedecay-temporal-query/src/cursor.rs +++ b/crates/tracedecay-temporal-query/src/cursor.rs @@ -7,9 +7,9 @@ use tracedecay_domain::{ }; use super::ports::{ - CursorKeyError, CursorSignature, SessionCursorAuthenticator, TemporalExecutionSnapshot, - TemporalRetrievalScope, + CursorKeyError, CursorSignature, SessionCursorAuthenticator, TemporalRetrievalScope, }; +use super::snapshot::TemporalExecutionSnapshot; const CURSOR_FORMAT_VERSION: &str = "3"; const MAX_CURSOR_PAYLOAD_HEX_BYTES: usize = 2 * 65_536; @@ -470,14 +470,17 @@ mod tests { use super::*; use crate::candidates::CandidateChannel; + use crate::execution::BindingDigest; use crate::ports::{ - BindingDigest, CursorKeyError, CursorSignature, KernelVersions, MAX_TEMPORAL_PARTICIPANTS, - SessionCursorAuthenticator, TemporalExecutionSnapshot, TemporalParticipantAuthorization, - TemporalParticipantGeneration, TemporalParticipantManifest, - TemporalPreparedCandidateCohort, TemporalSnapshotRequest, TemporalSourceAccess, - TemporalWatermarks, + CursorKeyError, CursorSignature, SessionCursorAuthenticator, TemporalSnapshotRequest, }; use crate::ranking::RankingCandidate; + use crate::snapshot::{ + KernelVersions, MAX_TEMPORAL_PARTICIPANTS, TemporalExecutionSnapshot, + TemporalParticipantAuthorization, TemporalParticipantGeneration, + TemporalParticipantManifest, TemporalPreparedCandidateCohort, TemporalSourceAccess, + TemporalWatermarks, + }; const TEST_NOW_MICROS: i64 = 1_800_000_000_000_000; diff --git a/crates/tracedecay-temporal-query/src/ports/execution.rs b/crates/tracedecay-temporal-query/src/execution.rs similarity index 96% rename from crates/tracedecay-temporal-query/src/ports/execution.rs rename to crates/tracedecay-temporal-query/src/execution.rs index bc5b8471a3..8a06e2ae7d 100644 --- a/crates/tracedecay-temporal-query/src/ports/execution.rs +++ b/crates/tracedecay-temporal-query/src/execution.rs @@ -9,12 +9,13 @@ use std::time::Instant; use thiserror::Error; -use super::{ReadBudgetAccounting, TemporalPortError, over_ceiling}; +use crate::paging::over_ceiling; +use crate::ports::{ReadBudgetAccounting, TemporalPortError}; const SHA256_PREFIX: &str = "sha256:"; const SHA256_HEX_LEN: usize = 64; -pub(super) const MAX_READ_ITEMS: usize = 8_192; -pub(super) const MAX_READ_TOTAL_BYTES: usize = 64 * 1024 * 1024; +pub(crate) const MAX_READ_ITEMS: usize = 8_192; +pub(crate) const MAX_READ_TOTAL_BYTES: usize = 64 * 1024 * 1024; const MAX_READ_ITEM_BYTES: usize = 8 * 1024 * 1024; const MAX_CONTINUATION_KEY_BYTES: usize = 4_096; @@ -186,12 +187,12 @@ impl ExecutionLimits { #[derive(Clone)] pub struct ExecutionControl { - pub(super) cancellation: Arc, - pub(super) deadline: Option, - pub(super) remaining_work: Option>, + pub(crate) cancellation: Arc, + pub(crate) deadline: Option, + pub(crate) remaining_work: Option>, /// The ceiling `remaining_work` started from, so an exhausted checkpoint can /// report the budget it spent instead of only naming the resource. - pub(super) work_limit: Option, + pub(crate) work_limit: Option, } impl ExecutionControl { diff --git a/crates/tracedecay-temporal-query/src/hydration.rs b/crates/tracedecay-temporal-query/src/hydration.rs index 6e9c3bdb91..9b737e19e7 100644 --- a/crates/tracedecay-temporal-query/src/hydration.rs +++ b/crates/tracedecay-temporal-query/src/hydration.rs @@ -7,7 +7,9 @@ use thiserror::Error; use tracedecay_domain::{HydrationStateV1, RetrievalAnchorId}; use zeroize::Zeroizing; -use super::ports::{TemporalExecutionSnapshot, TemporalPortError, await_controlled}; +use super::execution::await_controlled; +use super::ports::TemporalPortError; +use super::snapshot::TemporalExecutionSnapshot; /// Fallible pre-allocation ceiling for a single authorized payload buffer. const MAX_HYDRATION_PREALLOC_BYTES: usize = 1024 * 1024; @@ -316,11 +318,10 @@ mod tests { use tracedecay_domain::{RetrievalAnchorId, RetrievalGrainV1, SessionId, TemporalModeV1}; use super::*; - use crate::ports::{ - BindingDigest, ExecutionControl, ExecutionLimits, KernelVersions, - TemporalExecutionSnapshot, TemporalPortError, TemporalSnapshotRequest, TemporalWatermarks, - }; + use crate::execution::{BindingDigest, ExecutionControl, ExecutionLimits}; + use crate::ports::{TemporalPortError, TemporalSnapshotRequest}; use crate::resolution::types::ValidatedAuthorization; + use crate::snapshot::{KernelVersions, TemporalExecutionSnapshot, TemporalWatermarks}; use crate::test_support::block_on; fn anchor(value: &str) -> RetrievalAnchorId { diff --git a/crates/tracedecay-temporal-query/src/lib.rs b/crates/tracedecay-temporal-query/src/lib.rs index 245084bfcd..91512d5f87 100644 --- a/crates/tracedecay-temporal-query/src/lib.rs +++ b/crates/tracedecay-temporal-query/src/lib.rs @@ -1,11 +1,14 @@ pub mod candidates; pub mod context; pub mod cursor; +pub mod execution; pub mod hydration; +pub mod paging; pub mod ports; pub mod ranking; pub mod resolution; mod retriever; +pub mod snapshot; pub use retriever::{hydrate_temporal_candidate_export, hydrate_temporal_candidate_selection}; @@ -27,12 +30,12 @@ use self::context::{ use self::cursor::{ CursorError, CursorPosition, StableSortKey, encode_cursor_position, verify_cursor_position, }; +use self::execution::ExecutionLimits; use self::hydration::{HydrationBatch, HydrationError, TemporalHydrationPort}; +use self::paging::{CandidateReadState, PageKey, PageLimits, PageStatus, TemporalRecordReadState}; use self::ports::{ - CandidateReadState, ExecutionLimits, PageKey, PageLimits, PageStatus, - SessionCursorAuthenticator, TemporalExecutionSnapshot, TemporalPortError, TemporalReadPort, - TemporalRecord, TemporalRecordBatch, TemporalRecordReadState, TemporalRetrievalScope, - pull_candidate_page, pull_temporal_record_page, + SessionCursorAuthenticator, TemporalPortError, TemporalReadPort, TemporalRecord, + TemporalRecordBatch, TemporalRetrievalScope, pull_candidate_page, pull_temporal_record_page, }; use self::ranking::{ DiversityLimits, RankedCandidate, RankingCandidate, RankingError, rank_candidates, @@ -45,6 +48,7 @@ use self::resolution::summary::{ use self::resolution::types::{ ResolutionLineageEdge, ResolutionLineageEdgeKind, ResolvedOccurrence, TemporalResolution, }; +use self::snapshot::TemporalExecutionSnapshot; #[derive(Clone, Debug, PartialEq, Eq)] pub struct TemporalKernelRequest { @@ -250,7 +254,7 @@ pub struct TemporalCandidateExport { resolution: TemporalResolution, summaries: Vec, summary_eligibility: SummaryLineageEligibility, - strict_population: Option, + strict_population: Option, } impl TemporalCandidateExport { @@ -426,7 +430,7 @@ pub async fn execute_temporal_candidate_export( .unwrap_or_default(); let strict_population = snapshot .prepared_candidate_cohort() - .and_then(ports::TemporalPreparedCandidateCohort::strict_population); + .and_then(snapshot::TemporalPreparedCandidateCohort::strict_population); let (candidates, next_window_keyset) = match snapshot.prepared_candidate_cohort() { Some(prepared) => (prepared.candidates().to_vec(), None), None => read_candidate_window(read_port, &snapshot, request, limits, &resume).await?, @@ -625,7 +629,7 @@ fn evaluate_summaries_for_scope( source_states: &BTreeMap, scope: &TemporalRetrievalScope, mode: tracedecay_domain::TemporalModeV1, - control: &ports::ExecutionControl, + control: &execution::ExecutionControl, ) -> Result { match scope { TemporalRetrievalScope::Session(session_id) => { @@ -879,7 +883,7 @@ fn increment_hydration_coverage(coverage: &mut TemporalCoverageCountsV1, state: | HydrationStateV1::RetentionExpired => CoverageClass::Redacted, HydrationStateV1::RetainedButUnavailable | HydrationStateV1::Locked - | HydrationStateV1::UnverifiableLegacy => CoverageClass::Unknown, + | HydrationStateV1::Unverifiable => CoverageClass::Unknown, HydrationStateV1::Available => return, }; increment_coverage(coverage, class); @@ -1022,8 +1026,9 @@ mod scope_tests { SummarySourceHorizonV1, TemporalModeV1, TemporalValidityV1, UtcMicros, }; + use super::execution::ExecutionControl; use super::hydration::HydrationBatch; - use super::ports::{ExecutionControl, TemporalRetrievalScope}; + use super::ports::TemporalRetrievalScope; use super::resolution::summary::{ SummaryLineageEligibility, SummaryLineageRejection, SummaryOmission, SummarySourceState, }; diff --git a/crates/tracedecay-temporal-query/src/ports/paging.rs b/crates/tracedecay-temporal-query/src/paging.rs similarity index 97% rename from crates/tracedecay-temporal-query/src/ports/paging.rs rename to crates/tracedecay-temporal-query/src/paging.rs index 35c4925833..0ac3f463f3 100644 --- a/crates/tracedecay-temporal-query/src/ports/paging.rs +++ b/crates/tracedecay-temporal-query/src/paging.rs @@ -1,17 +1,17 @@ use std::marker::PhantomData; -use super::{ - ExecutionControl, MeasuredTemporalValue, ReadBudgetAccounting, TemporalPortError, - TemporalRecord, +use crate::execution::ExecutionControl; +use crate::ports::{ + MeasuredTemporalValue, ReadBudgetAccounting, TemporalPortError, TemporalRecord, }; use crate::ranking::RankingCandidate; const MAX_READ_ITEMS: usize = 8_192; const MAX_READ_TOTAL_BYTES: usize = 64 * 1024 * 1024; const MAX_READ_ITEM_BYTES: usize = 8 * 1024 * 1024; -pub(super) const MAX_PAGE_ITEMS_CAP: usize = 1_024; +pub(crate) const MAX_PAGE_ITEMS_CAP: usize = 1_024; const MAX_CONTINUATION_KEY_BYTES: usize = 4_096; -pub(super) const MAX_BOUNDED_PAGE_PREALLOC: usize = 64; +pub(crate) const MAX_BOUNDED_PAGE_PREALLOC: usize = 64; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct PageLimits { @@ -24,7 +24,7 @@ pub struct PageLimits { /// Accounting for a request-shape ceiling check. A zero value is malformed /// rather than oversized, and reporting it as "requested 0" against a ceiling it /// never reached would be a fabricated number. -pub(super) fn over_ceiling(requested: usize, ceiling: usize) -> Option { +pub(crate) fn over_ceiling(requested: usize, ceiling: usize) -> Option { (requested > ceiling).then(|| ReadBudgetAccounting::requested(ceiling as u64, requested as u64)) } @@ -211,7 +211,7 @@ pub struct BoundedPage { items: Vec, encoded_bytes: usize, status: PageStatus, - pub(super) continuation: Option, + pub(crate) continuation: Option, } impl BoundedPage { @@ -408,7 +408,7 @@ pub const CANDIDATE_READ_BUDGET: ReadBudgetResources = ReadBudgetResources { total_bytes: "candidate total bytes", }; -pub(super) const RECORD_READ_BUDGET: ReadBudgetResources = ReadBudgetResources { +pub(crate) const RECORD_READ_BUDGET: ReadBudgetResources = ReadBudgetResources { item_count: "record item count", item_bytes: "record item bytes", total_bytes: "record total bytes", @@ -499,7 +499,7 @@ impl BoundedPageSink<'_, T> { } #[cfg(test)] - pub(super) fn preallocated_capacity(&self) -> usize { + pub(crate) fn preallocated_capacity(&self) -> usize { self.items.capacity() } diff --git a/crates/tracedecay-temporal-query/src/ports.rs b/crates/tracedecay-temporal-query/src/ports.rs index b642755e3b..cbccd8a048 100644 --- a/crates/tracedecay-temporal-query/src/ports.rs +++ b/crates/tracedecay-temporal-query/src/ports.rs @@ -1,16 +1,10 @@ mod contracts; mod cursor_authentication; -mod execution; -mod paging; mod request; -mod snapshot; pub use contracts::*; pub use cursor_authentication::*; -pub use execution::*; -pub use paging::*; pub use request::*; -pub use snapshot::*; use thiserror::Error; diff --git a/crates/tracedecay-temporal-query/src/ports/contracts.rs b/crates/tracedecay-temporal-query/src/ports/contracts.rs index dff9013e1d..3f5d755cd8 100644 --- a/crates/tracedecay-temporal-query/src/ports/contracts.rs +++ b/crates/tracedecay-temporal-query/src/ports/contracts.rs @@ -6,17 +6,18 @@ use std::pin::Pin; use serde::Serialize; use tracedecay_domain::{LogicalCopyRecordV1, SessionSummaryRecordV1}; -use super::{ +use super::{TemporalPortError, TemporalRetrievalScope, TemporalSnapshotRequest}; +use crate::candidates::{CandidateChannel, CandidatePlan}; +use crate::execution::{ExecutionLimits, await_controlled}; +use crate::paging::{ BoundedPage, CANDIDATE_READ_BUDGET, CandidateFieldCaps, CandidatePageSink, CandidateReadState, - ExecutionLimits, PageRequest, PageStatus, RECORD_READ_BUDGET, ReadBudgetResources, ReadState, - TemporalExecutionSnapshot, TemporalPortError, TemporalPreparedCandidateCohort, - TemporalRecordPageSink, TemporalRecordReadState, TemporalRetrievalScope, - TemporalSnapshotRequest, await_controlled, + PageRequest, PageStatus, RECORD_READ_BUDGET, ReadBudgetResources, ReadState, + TemporalRecordPageSink, TemporalRecordReadState, }; -use crate::candidates::{CandidateChannel, CandidatePlan}; use crate::ranking::RankingCandidate; use crate::resolution::summary::SummarySourceState; use crate::resolution::types::{ResolutionAssertion, ResolutionOccurrence}; +use crate::snapshot::TemporalExecutionSnapshot; const MAX_READ_ITEM_BYTES: usize = 8 * 1024 * 1024; @@ -48,14 +49,6 @@ pub type PortFuture<'a, T> = Pin> + Send + 'a>>; pub trait TemporalReadPort: Send + Sync { - fn produce_candidate_page<'a>( - &'a self, - snapshot: &'a TemporalExecutionSnapshot, - plan: &'a CandidatePlan, - request: PageRequest, - sink: &'a mut CandidatePageSink<'_>, - ) -> PortFuture<'a, PageStatus>; - fn produce_candidate_page_for_scope<'a>( &'a self, scope: &'a TemporalRetrievalScope, @@ -63,28 +56,6 @@ pub trait TemporalReadPort: Send + Sync { plan: &'a CandidatePlan, request: PageRequest, sink: &'a mut CandidatePageSink<'_>, - ) -> PortFuture<'a, PageStatus> { - match scope { - TemporalRetrievalScope::Session(_) => { - self.produce_candidate_page(snapshot, plan, request, sink) - } - TemporalRetrievalScope::AllSessionsInAuthorizedRoot => Box::pin(async { - Err(TemporalPortError::Read { - operation: "produce candidate page for scope", - message: - "root-wide retrieval requires an explicit scope-aware port implementation" - .to_string(), - }) - }), - } - } - - fn produce_temporal_record_page<'a>( - &'a self, - snapshot: &'a TemporalExecutionSnapshot, - candidates: &'a [RankingCandidate], - request: PageRequest, - sink: &'a mut TemporalRecordPageSink<'_>, ) -> PortFuture<'a, PageStatus>; fn produce_temporal_record_page_for_scope<'a>( @@ -94,93 +65,9 @@ pub trait TemporalReadPort: Send + Sync { candidates: &'a [RankingCandidate], request: PageRequest, sink: &'a mut TemporalRecordPageSink<'_>, - ) -> PortFuture<'a, PageStatus> { - match scope { - TemporalRetrievalScope::Session(_) => { - self.produce_temporal_record_page(snapshot, candidates, request, sink) - } - TemporalRetrievalScope::AllSessionsInAuthorizedRoot => Box::pin(async { - Err(TemporalPortError::Read { - operation: "produce temporal record page for scope", - message: - "root-wide retrieval requires an explicit scope-aware port implementation" - .to_string(), - }) - }), - } - } -} - -/// Bounded producer used before a root-wide participant manifest exists. -/// -/// The implementation is captured inside the already-authorized global DB -/// read snapshot; this port owns no authorization or persistence authority. -pub trait TemporalCandidatePreparationPort: Send + Sync { - fn produce_prepared_candidate_page<'a>( - &'a self, - request: PageRequest, - sink: &'a mut CandidatePageSink<'_>, ) -> PortFuture<'a, PageStatus>; } -#[hotpath::measure(future = true, label = "temporal_query.candidates.prepare")] -pub async fn prepare_temporal_candidate_cohort( - request: &TemporalSnapshotRequest, - port: &impl TemporalCandidatePreparationPort, -) -> Result { - request.execution_control().checkpoint()?; - let limits = request.limits(); - let candidate_page_items = limits.candidate_limit.min(64); - let candidate_limits = super::PageLimits::new( - limits.candidate_limit, - limits.candidate_total_bytes, - limits.candidate_item_bytes, - candidate_page_items, - )?; - let mut state = CandidateReadState::new(candidate_limits); - let mut candidates = Vec::with_capacity(limits.candidate_limit.min(256)); - loop { - let limits = begin_pull_request( - request, - &state, - |limits| { - ( - limits.candidate_limit, - limits.candidate_total_bytes, - limits.candidate_item_bytes, - ) - }, - CANDIDATE_READ_BUDGET, - )?; - let control = request.execution_control(); - let field_caps = CandidateFieldCaps::new( - limits.candidate_stable_id_bytes, - limits.candidate_anchor_id_bytes, - limits.candidate_metadata_field_bytes, - ); - let page_request = state.request(limits.candidate_key_bytes, Some(field_caps)); - let mut sink = state.begin_page( - control, - limits.candidate_key_bytes, - Some(field_caps), - CANDIDATE_READ_BUDGET, - ); - let status = await_controlled( - control, - port.produce_prepared_candidate_page(page_request, &mut sink), - ) - .await?; - let page = sink.finish(status)?; - let page = commit_pulled_page(&mut state, page, CANDIDATE_READ_BUDGET)?; - let status = page.status(); - candidates.extend(page.into_items()); - if status == PageStatus::Complete { - break; - } - } - TemporalPreparedCandidateCohort::new(candidates) -} - pub fn begin_prepared_candidate_pull( request: &TemporalSnapshotRequest, state: &mut CandidateReadState, diff --git a/crates/tracedecay-temporal-query/src/ports/request.rs b/crates/tracedecay-temporal-query/src/ports/request.rs index f1fa867c6c..d1abfa4bce 100644 --- a/crates/tracedecay-temporal-query/src/ports/request.rs +++ b/crates/tracedecay-temporal-query/src/ports/request.rs @@ -1,7 +1,8 @@ use serde::Serialize; use tracedecay_domain::{RetrievalGrainV1, SessionId, TemporalModeV1}; -use super::{BindingDigest, ExecutionControl, ExecutionLimits, TemporalPortError}; +use super::TemporalPortError; +use crate::execution::{BindingDigest, ExecutionControl, ExecutionLimits}; const PROFILE_ROOT_PROJECT_KEY: &str = "user"; @@ -108,7 +109,7 @@ impl TemporalAuthorizedRoot { } } -pub(super) fn validate_label(field: &'static str, value: &str) -> Result<(), TemporalPortError> { +pub(crate) fn validate_label(field: &'static str, value: &str) -> Result<(), TemporalPortError> { if value.is_empty() || value.trim() != value || value.len() > 512 diff --git a/crates/tracedecay-temporal-query/src/ports/tests.rs b/crates/tracedecay-temporal-query/src/ports/tests.rs index 82382e738f..2a28ae8a40 100644 --- a/crates/tracedecay-temporal-query/src/ports/tests.rs +++ b/crates/tracedecay-temporal-query/src/ports/tests.rs @@ -2,7 +2,7 @@ use std::sync::{ Arc, Mutex, atomic::{AtomicBool, AtomicUsize, Ordering}, }; -use std::time::{Duration, Instant}; +use std::time::Instant; use tracedecay_domain::{ RetrievalAnchorId, RetrievalGrainV1, SessionId, SessionSourceCoverageStateV1, @@ -10,13 +10,26 @@ use tracedecay_domain::{ }; use super::cursor_authentication::MAX_CURSOR_SECRET_BYTES; -use super::execution::{MAX_READ_ITEMS, MAX_READ_TOTAL_BYTES}; -use super::paging::{MAX_BOUNDED_PAGE_PREALLOC, MAX_PAGE_ITEMS_CAP}; use super::*; use crate::candidates::{CandidateChannel, CandidatePlan}; +use crate::execution::{ + BindingDigest, ExecutionControl, ExecutionLimitTighteningError, ExecutionLimits, + MAX_READ_ITEMS, MAX_READ_TOTAL_BYTES, +}; +use crate::paging::{ + CANDIDATE_READ_BUDGET, CandidatePageSink, CandidateReadState, MAX_BOUNDED_PAGE_PREALLOC, + MAX_PAGE_ITEMS_CAP, PageKey, PageLimits, PageRequest, PageStatus, TemporalRecordPageSink, + TemporalRecordReadState, +}; use crate::ranking::RankingCandidate; use crate::resolution::summary::SummarySourceState; use crate::resolution::types::ValidatedAuthorization; +use crate::snapshot::{ + KernelVersions, MAX_TEMPORAL_PARTICIPANT_MANIFEST_BYTES, MAX_TEMPORAL_PARTICIPANTS, + TemporalExecutionSnapshot, TemporalParticipantAuthorization, TemporalParticipantGeneration, + TemporalParticipantManifest, TemporalPreparedCandidateCohort, TemporalSourceAccess, + TemporalWatermarks, +}; use crate::test_support::block_on; fn session_id() -> SessionId { @@ -191,21 +204,6 @@ struct ScopeObservingPort { } impl TemporalReadPort for ScopeObservingPort { - fn produce_candidate_page<'a>( - &'a self, - _snapshot: &'a TemporalExecutionSnapshot, - _plan: &'a CandidatePlan, - _request: PageRequest, - _sink: &'a mut CandidatePageSink<'_>, - ) -> PortFuture<'a, PageStatus> { - Box::pin(async { - Err(TemporalPortError::Read { - operation: "legacy candidate entry point", - message: "scope-aware kernel must not call the legacy entry point".to_string(), - }) - }) - } - fn produce_candidate_page_for_scope<'a>( &'a self, scope: &'a TemporalRetrievalScope, @@ -223,21 +221,6 @@ impl TemporalReadPort for ScopeObservingPort { }) } - fn produce_temporal_record_page<'a>( - &'a self, - _snapshot: &'a TemporalExecutionSnapshot, - _candidates: &'a [RankingCandidate], - _request: PageRequest, - _sink: &'a mut TemporalRecordPageSink<'_>, - ) -> PortFuture<'a, PageStatus> { - Box::pin(async { - Err(TemporalPortError::Read { - operation: "legacy record entry point", - message: "scope-aware kernel must not call the legacy entry point".to_string(), - }) - }) - } - fn produce_temporal_record_page_for_scope<'a>( &'a self, scope: &'a TemporalRetrievalScope, @@ -472,8 +455,9 @@ struct PagingPort { } impl TemporalReadPort for PagingPort { - fn produce_candidate_page<'a>( + fn produce_candidate_page_for_scope<'a>( &'a self, + _scope: &'a TemporalRetrievalScope, _snapshot: &'a TemporalExecutionSnapshot, _plan: &'a CandidatePlan, request: PageRequest, @@ -497,8 +481,9 @@ impl TemporalReadPort for PagingPort { }) } - fn produce_temporal_record_page<'a>( + fn produce_temporal_record_page_for_scope<'a>( &'a self, + _scope: &'a TemporalRetrievalScope, _snapshot: &'a TemporalExecutionSnapshot, _candidates: &'a [RankingCandidate], _request: PageRequest, @@ -536,83 +521,12 @@ fn bounded_async_pull_streams_multiple_pages_without_preloaded_vecs() { }); } -struct OversizedPort; - -struct PreparationFromReadPort<'a> { - port: &'a dyn TemporalReadPort, - snapshot: &'a TemporalExecutionSnapshot, - plan: &'a CandidatePlan, -} - -impl TemporalCandidatePreparationPort for PreparationFromReadPort<'_> { - fn produce_prepared_candidate_page<'a>( - &'a self, - request: PageRequest, - sink: &'a mut CandidatePageSink<'_>, - ) -> PortFuture<'a, PageStatus> { - self.port - .produce_candidate_page(self.snapshot, self.plan, request, sink) - } -} - -impl TemporalReadPort for OversizedPort { - fn produce_candidate_page<'a>( - &'a self, - _snapshot: &'a TemporalExecutionSnapshot, - _plan: &'a CandidatePlan, - _request: PageRequest, - sink: &'a mut CandidatePageSink<'_>, - ) -> PortFuture<'a, PageStatus> { - Box::pin(async move { - sink.push(candidate("x".repeat(1024)))?; - Ok(PageStatus::Complete) - }) - } - - fn produce_temporal_record_page<'a>( - &'a self, - _snapshot: &'a TemporalExecutionSnapshot, - _candidates: &'a [RankingCandidate], - _request: PageRequest, - _sink: &'a mut TemporalRecordPageSink<'_>, - ) -> PortFuture<'a, PageStatus> { - Box::pin(async { Ok(PageStatus::Complete) }) - } -} - -#[test] -fn prepared_cohort_preserves_typed_candidate_byte_budget_failure() { - block_on(async { - let limits = ExecutionLimits { - candidate_limit: 1, - candidate_total_bytes: 128, - candidate_item_bytes: 128, - ..ExecutionLimits::default() - }; - let snapshot = snapshot_with_control(ExecutionControl::default()); - let request = snapshot.request().clone().with_limits(limits); - let plan = CandidatePlan::default(); - let port = PreparationFromReadPort { - port: &OversizedPort, - snapshot: &snapshot, - plan: &plan, - }; - - assert_eq!( - prepare_temporal_candidate_cohort(&request, &port).await, - Err(TemporalPortError::BudgetExceeded { - resource: "candidate item bytes", - accounting: Some(ReadBudgetAccounting::requested(128, 1_313)), - }) - ); - }); -} - struct OverproducingPort; impl TemporalReadPort for OverproducingPort { - fn produce_candidate_page<'a>( + fn produce_candidate_page_for_scope<'a>( &'a self, + _scope: &'a TemporalRetrievalScope, _snapshot: &'a TemporalExecutionSnapshot, _plan: &'a CandidatePlan, _request: PageRequest, @@ -625,8 +539,9 @@ impl TemporalReadPort for OverproducingPort { }) } - fn produce_temporal_record_page<'a>( + fn produce_temporal_record_page_for_scope<'a>( &'a self, + _scope: &'a TemporalRetrievalScope, _snapshot: &'a TemporalExecutionSnapshot, _candidates: &'a [RankingCandidate], _request: PageRequest, @@ -665,8 +580,9 @@ struct CancellingPort { } impl TemporalReadPort for CancellingPort { - fn produce_candidate_page<'a>( + fn produce_candidate_page_for_scope<'a>( &'a self, + _scope: &'a TemporalRetrievalScope, _snapshot: &'a TemporalExecutionSnapshot, _plan: &'a CandidatePlan, _request: PageRequest, @@ -681,8 +597,9 @@ impl TemporalReadPort for CancellingPort { }) } - fn produce_temporal_record_page<'a>( + fn produce_temporal_record_page_for_scope<'a>( &'a self, + _scope: &'a TemporalRetrievalScope, _snapshot: &'a TemporalExecutionSnapshot, _candidates: &'a [RankingCandidate], _request: PageRequest, @@ -713,89 +630,6 @@ fn async_pull_observes_live_cancellation_midstream() { }); } -#[test] -fn prepared_cohort_preserves_live_cancellation() { - block_on(async { - let control = ExecutionControl::default(); - let snapshot = snapshot_with_control(control.clone()); - let entered = Arc::new(AtomicBool::new(false)); - let producer = CancellingPort { - control, - entered: Arc::clone(&entered), - }; - let plan = CandidatePlan::default(); - let port = PreparationFromReadPort { - port: &producer, - snapshot: &snapshot, - plan: &plan, - }; - - let result = prepare_temporal_candidate_cohort(snapshot.request(), &port).await; - - assert!(entered.load(Ordering::Acquire)); - assert_eq!(result, Err(TemporalPortError::Cancelled)); - }); -} - -struct DeadlineCrossingPort { - deadline: Instant, - entered: Arc, -} - -impl TemporalReadPort for DeadlineCrossingPort { - fn produce_candidate_page<'a>( - &'a self, - _snapshot: &'a TemporalExecutionSnapshot, - _plan: &'a CandidatePlan, - _request: PageRequest, - _sink: &'a mut CandidatePageSink<'_>, - ) -> PortFuture<'a, PageStatus> { - let deadline = self.deadline; - let entered = Arc::clone(&self.entered); - Box::pin(async move { - entered.store(true, Ordering::Release); - while Instant::now() < deadline { - std::hint::spin_loop(); - } - Ok(PageStatus::Complete) - }) - } - - fn produce_temporal_record_page<'a>( - &'a self, - _snapshot: &'a TemporalExecutionSnapshot, - _candidates: &'a [RankingCandidate], - _request: PageRequest, - _sink: &'a mut TemporalRecordPageSink<'_>, - ) -> PortFuture<'a, PageStatus> { - Box::pin(async { Ok(PageStatus::Complete) }) - } -} - -#[test] -fn prepared_cohort_preserves_deadline_after_producer_work() { - block_on(async { - let deadline = Instant::now() + Duration::from_millis(100); - let snapshot = snapshot_with_control(ExecutionControl::new(Some(deadline))); - let entered = Arc::new(AtomicBool::new(false)); - let producer = DeadlineCrossingPort { - deadline, - entered: Arc::clone(&entered), - }; - let plan = CandidatePlan::default(); - let port = PreparationFromReadPort { - port: &producer, - snapshot: &snapshot, - plan: &plan, - }; - - let result = prepare_temporal_candidate_cohort(snapshot.request(), &port).await; - - assert!(entered.load(Ordering::Acquire)); - assert_eq!(result, Err(TemporalPortError::DeadlineExceeded)); - }); -} - fn summary_record(anchor_id: &str) -> TemporalRecord { TemporalRecord::SummarySource(SummarySourceRecord { anchor_id: anchor(anchor_id), @@ -820,8 +654,9 @@ impl AlwaysMorePort { } impl TemporalReadPort for AlwaysMorePort { - fn produce_candidate_page<'a>( + fn produce_candidate_page_for_scope<'a>( &'a self, + _scope: &'a TemporalRetrievalScope, _snapshot: &'a TemporalExecutionSnapshot, _plan: &'a CandidatePlan, request: PageRequest, @@ -839,8 +674,9 @@ impl TemporalReadPort for AlwaysMorePort { }) } - fn produce_temporal_record_page<'a>( + fn produce_temporal_record_page_for_scope<'a>( &'a self, + _scope: &'a TemporalRetrievalScope, _snapshot: &'a TemporalExecutionSnapshot, _candidates: &'a [RankingCandidate], request: PageRequest, @@ -865,8 +701,9 @@ struct ExactCompletePort { } impl TemporalReadPort for ExactCompletePort { - fn produce_candidate_page<'a>( + fn produce_candidate_page_for_scope<'a>( &'a self, + _scope: &'a TemporalRetrievalScope, _snapshot: &'a TemporalExecutionSnapshot, _plan: &'a CandidatePlan, request: PageRequest, @@ -889,8 +726,9 @@ impl TemporalReadPort for ExactCompletePort { }) } - fn produce_temporal_record_page<'a>( + fn produce_temporal_record_page_for_scope<'a>( &'a self, + _scope: &'a TemporalRetrievalScope, _snapshot: &'a TemporalExecutionSnapshot, _candidates: &'a [RankingCandidate], request: PageRequest, @@ -917,8 +755,9 @@ impl TemporalReadPort for ExactCompletePort { struct OversizedRecordPort; impl TemporalReadPort for OversizedRecordPort { - fn produce_candidate_page<'a>( + fn produce_candidate_page_for_scope<'a>( &'a self, + _scope: &'a TemporalRetrievalScope, _snapshot: &'a TemporalExecutionSnapshot, _plan: &'a CandidatePlan, _request: PageRequest, @@ -927,8 +766,9 @@ impl TemporalReadPort for OversizedRecordPort { Box::pin(async { Ok(PageStatus::Complete) }) } - fn produce_temporal_record_page<'a>( + fn produce_temporal_record_page_for_scope<'a>( &'a self, + _scope: &'a TemporalRetrievalScope, _snapshot: &'a TemporalExecutionSnapshot, _candidates: &'a [RankingCandidate], _request: PageRequest, @@ -1512,8 +1352,9 @@ struct StableIdPort { } impl TemporalReadPort for StableIdPort { - fn produce_candidate_page<'a>( + fn produce_candidate_page_for_scope<'a>( &'a self, + _scope: &'a TemporalRetrievalScope, _snapshot: &'a TemporalExecutionSnapshot, _plan: &'a CandidatePlan, _request: PageRequest, @@ -1525,8 +1366,9 @@ impl TemporalReadPort for StableIdPort { }) } - fn produce_temporal_record_page<'a>( + fn produce_temporal_record_page_for_scope<'a>( &'a self, + _scope: &'a TemporalRetrievalScope, _snapshot: &'a TemporalExecutionSnapshot, _candidates: &'a [RankingCandidate], _request: PageRequest, @@ -1567,8 +1409,9 @@ fn candidate_pull_observes_post_authorization_tightening() { struct UnreachableReadPort; impl TemporalReadPort for UnreachableReadPort { - fn produce_candidate_page<'a>( + fn produce_candidate_page_for_scope<'a>( &'a self, + _scope: &'a TemporalRetrievalScope, _snapshot: &'a TemporalExecutionSnapshot, _plan: &'a CandidatePlan, _request: PageRequest, @@ -1577,8 +1420,9 @@ impl TemporalReadPort for UnreachableReadPort { Box::pin(async { panic!("looser candidate read state reached the producer") }) } - fn produce_temporal_record_page<'a>( + fn produce_temporal_record_page_for_scope<'a>( &'a self, + _scope: &'a TemporalRetrievalScope, _snapshot: &'a TemporalExecutionSnapshot, _candidates: &'a [RankingCandidate], _request: PageRequest, @@ -1705,8 +1549,9 @@ fn continuation_key_enforces_exact_byte_cap() { key_len: usize, } impl TemporalReadPort for ContinuationPort { - fn produce_candidate_page<'a>( + fn produce_candidate_page_for_scope<'a>( &'a self, + _scope: &'a TemporalRetrievalScope, _snapshot: &'a TemporalExecutionSnapshot, _plan: &'a CandidatePlan, _request: PageRequest, @@ -1718,8 +1563,9 @@ fn continuation_key_enforces_exact_byte_cap() { Ok(PageStatus::More) }) } - fn produce_temporal_record_page<'a>( + fn produce_temporal_record_page_for_scope<'a>( &'a self, + _scope: &'a TemporalRetrievalScope, _snapshot: &'a TemporalExecutionSnapshot, _candidates: &'a [RankingCandidate], _request: PageRequest, @@ -1758,78 +1604,6 @@ fn continuation_key_enforces_exact_byte_cap() { }); } -#[test] -fn legacy_only_port_fails_closed_for_root_wide_scope() { - block_on(async { - struct LegacyOnlyPort; - impl TemporalReadPort for LegacyOnlyPort { - fn produce_candidate_page<'a>( - &'a self, - _snapshot: &'a TemporalExecutionSnapshot, - _plan: &'a CandidatePlan, - _request: PageRequest, - _sink: &'a mut CandidatePageSink<'_>, - ) -> PortFuture<'a, PageStatus> { - Box::pin(async { Ok(PageStatus::Complete) }) - } - fn produce_temporal_record_page<'a>( - &'a self, - _snapshot: &'a TemporalExecutionSnapshot, - _candidates: &'a [RankingCandidate], - _request: PageRequest, - _sink: &'a mut TemporalRecordPageSink<'_>, - ) -> PortFuture<'a, PageStatus> { - Box::pin(async { Ok(PageStatus::Complete) }) - } - } - let request = TemporalSnapshotRequest::new( - session_id(), - digest('0'), - digest('1'), - digest('2'), - TemporalModeV1::Current, - RetrievalGrainV1::LogicalMessage, - ) - .expect("valid request") - .with_retrieval_scope(TemporalRetrievalScope::AllSessionsInAuthorizedRoot); - let snapshot = TemporalExecutionSnapshot::new( - request, - TemporalWatermarks { - generation: 1, - source: 0, - projection: 0, - index: 0, - summary: 0, - }, - KernelVersions { - schema: 1, - ranking: 1, - configuration_digest: BindingDigest::new("configuration_digest", digest('3')) - .expect("valid digest"), - }, - None, - ) - .expect("valid snapshot"); - let mut candidate_state = - CandidateReadState::new(PageLimits::new(1, 1024, 1024, 1).expect("limits")); - let err = pull_candidate_page( - &LegacyOnlyPort, - &snapshot, - &CandidatePlan::default(), - &mut candidate_state, - ) - .await - .expect_err("root-wide must not use silent legacy default"); - assert!(matches!( - err, - TemporalPortError::Read { - operation: "produce candidate page for scope", - .. - } - )); - }); -} - #[test] fn participant_manifest_reports_mixed_source_freshness_from_real_frontiers() { let configuration = BindingDigest::new("configuration_digest", digest('3')).expect("digest"); @@ -1923,15 +1697,39 @@ fn authorized_lifecycle_states_do_not_become_snapshot_denials() { } #[test] -fn manifests_without_explicit_authorization_fail_closed() { - let participant = participant("session.stale", "claude", 1); - let mut wire = serde_json::to_value(participant).unwrap(); - wire.as_object_mut().unwrap().remove("q"); - let stale: TemporalParticipantGeneration = serde_json::from_value(wire).unwrap(); +fn manifests_without_explicit_authorization_are_rejected() { + let denied = TemporalParticipantGeneration::new( + SessionId::new("session.denied").unwrap(), + "claude", + TemporalWatermarks { + generation: 1, + source: 2, + projection: 3, + index: 4, + summary: 5, + }, + 6, + &BindingDigest::new("configuration", digest('7')).unwrap(), + &BindingDigest::new("authorization", digest('8')).unwrap(), + TemporalParticipantAuthorization::Denied, + TemporalSourceAccess::Available, + ) + .unwrap(); + let manifest = TemporalParticipantManifest::new(vec![denied]).unwrap(); + let mut wire = serde_json::to_value(&manifest).unwrap(); + let decoded: TemporalParticipantManifest = serde_json::from_value(wire.clone()).unwrap(); + decoded.validate().unwrap(); + assert_eq!(decoded, manifest); assert_eq!( - stale.authorization(), + decoded.entries()[0].authorization(), TemporalParticipantAuthorization::Denied ); - assert!(!stale.is_authorized_for_snapshot()); + + wire["p"][0].as_object_mut().unwrap().remove("q"); + let error = serde_json::from_value::(wire).unwrap_err(); + assert!( + error.to_string().contains("missing field `q`"), + "unexpected rejection: {error}" + ); } diff --git a/crates/tracedecay-temporal-query/src/resolution/resolver.rs b/crates/tracedecay-temporal-query/src/resolution/resolver.rs index 3a6a3ecc22..52349401a8 100644 --- a/crates/tracedecay-temporal-query/src/resolution/resolver.rs +++ b/crates/tracedecay-temporal-query/src/resolution/resolver.rs @@ -5,7 +5,8 @@ use tracedecay_domain::{ TemporalAssertionKindV1, TemporalModeV1, TemporalValidityV1, }; -use super::super::ports::{ExecutionControl, TemporalPortError}; +use super::super::execution::ExecutionControl; +use super::super::ports::TemporalPortError; use super::types::{ ResolutionAssertion, ResolutionCheckpoint, ResolutionLineageEdge, ResolutionLineageEdgeKind, ResolutionOccurrence, ResolvedOccurrence, TemporalResolution, diff --git a/crates/tracedecay-temporal-query/src/resolution/summary.rs b/crates/tracedecay-temporal-query/src/resolution/summary.rs index 2867a84e53..1bd1e3e8bf 100644 --- a/crates/tracedecay-temporal-query/src/resolution/summary.rs +++ b/crates/tracedecay-temporal-query/src/resolution/summary.rs @@ -6,7 +6,8 @@ use tracedecay_domain::{ TemporalValidityV1, UtcMicros, }; -use super::super::ports::{ExecutionControl, TemporalPortError}; +use super::super::execution::ExecutionControl; +use super::super::ports::TemporalPortError; #[derive(Clone, Copy, Debug, Serialize, PartialEq, Eq)] pub enum SummarySourceState { diff --git a/crates/tracedecay-temporal-query/src/resolution/tests.rs b/crates/tracedecay-temporal-query/src/resolution/tests.rs index 5f140c85af..0c04fa1af2 100644 --- a/crates/tracedecay-temporal-query/src/resolution/tests.rs +++ b/crates/tracedecay-temporal-query/src/resolution/tests.rs @@ -7,7 +7,8 @@ use tracedecay_domain::{ TemporalValidityV1, UtcMicros, }; -use super::super::ports::{ExecutionControl, ReadBudgetAccounting, TemporalPortError}; +use super::super::execution::ExecutionControl; +use super::super::ports::{ReadBudgetAccounting, TemporalPortError}; use super::resolver::{ resolve_temporal, resolve_temporal_controlled, resolve_temporal_with_checkpoints, }; diff --git a/crates/tracedecay-temporal-query/src/retriever.rs b/crates/tracedecay-temporal-query/src/retriever.rs index bb8a14ef8a..4caeab3121 100644 --- a/crates/tracedecay-temporal-query/src/retriever.rs +++ b/crates/tracedecay-temporal-query/src/retriever.rs @@ -17,7 +17,8 @@ use tracedecay_domain::{ use super::context::VersionedTokenEstimator; use super::context::assembly::assemble_context_with_frames_controlled; use super::hydration::{TemporalHydrationPort, hydrate_selected}; -use super::ports::{TemporalParticipantGeneration, TemporalPortError, TemporalSourceAccess}; +use super::ports::TemporalPortError; +use super::snapshot::{TemporalParticipantGeneration, TemporalSourceAccess}; use super::{ TemporalCandidateExport, TemporalHydratedResult, TemporalKernelError, TemporalKernelRequest, TemporalKernelResult, check_control, map_context_error, map_hydration_error, @@ -295,8 +296,7 @@ fn participant_freshness( | TemporalSourceAccess::Locked | TemporalSourceAccess::RetentionWithheld | TemporalSourceAccess::Deleted - | TemporalSourceAccess::Redacted - | TemporalSourceAccess::LegacyUnauthorized => FreshnessCompatibilityV1::Missing, + | TemporalSourceAccess::Redacted => FreshnessCompatibilityV1::Missing, }, policy_revision, } diff --git a/crates/tracedecay-temporal-query/src/ports/snapshot.rs b/crates/tracedecay-temporal-query/src/snapshot.rs similarity index 97% rename from crates/tracedecay-temporal-query/src/ports/snapshot.rs rename to crates/tracedecay-temporal-query/src/snapshot.rs index eb11e2247e..f1eb719f76 100644 --- a/crates/tracedecay-temporal-query/src/ports/snapshot.rs +++ b/crates/tracedecay-temporal-query/src/snapshot.rs @@ -9,12 +9,12 @@ use tracedecay_domain::{ SessionTemporalCoverageRequestV1, SignedCursorKeyRefV1, TemporalModeV1, }; -use super::request::validate_label; -use super::{ - BindingDigest, ExecutionLimitTighteningError, ExecutionLimits, MeasuredTemporalValue, - TemporalPortError, TemporalRetrievalScope, TemporalSnapshotRequest, -}; use crate::candidates::CandidateChannel; +use crate::execution::{BindingDigest, ExecutionLimitTighteningError, ExecutionLimits}; +use crate::ports::validate_label; +use crate::ports::{ + MeasuredTemporalValue, TemporalPortError, TemporalRetrievalScope, TemporalSnapshotRequest, +}; use crate::ranking::RankingCandidate; use crate::resolution::types::ValidatedAuthorization; @@ -162,11 +162,10 @@ pub struct KernelVersions { pub configuration_digest: BindingDigest, } -#[derive(Clone, Copy, Debug, Default, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] pub enum TemporalParticipantAuthorization { #[serde(rename = "a")] Authorized, - #[default] #[serde(rename = "n")] Denied, } @@ -185,8 +184,6 @@ pub enum TemporalSourceAccess { Deleted, #[serde(rename = "x")] Redacted, - #[serde(rename = "n")] - LegacyUnauthorized, } #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] @@ -195,7 +192,7 @@ pub struct TemporalParticipantGeneration { #[serde(rename = "s")] session_id: SessionId, #[serde(rename = "i")] - pub(super) source_id: String, + pub(crate) source_id: String, #[serde(rename = "g")] generation: u64, #[serde(rename = "w")] @@ -212,7 +209,7 @@ pub struct TemporalParticipantGeneration { configuration_digest: String, #[serde(rename = "a")] authorization_digest: String, - #[serde(default, rename = "q")] + #[serde(rename = "q")] authorization: TemporalParticipantAuthorization, #[serde(rename = "z")] access: TemporalSourceAccess, @@ -294,16 +291,12 @@ impl TemporalParticipantGeneration { } /// Snapshot authority is independent from per-source lifecycle state. - /// - /// The legacy unauthorized source wire state remains denied for old signed - /// manifests, while every newly built manifest uses the dedicated, - /// fail-closed authorization field. #[hotpath::skip] pub const fn is_authorized_for_snapshot(&self) -> bool { matches!( self.authorization, TemporalParticipantAuthorization::Authorized - ) && !matches!(self.access, TemporalSourceAccess::LegacyUnauthorized) + ) } #[hotpath::skip] @@ -443,8 +436,7 @@ impl TemporalParticipantManifest { SessionSourceCoverageStateV1::Redacted, SessionSourceCoverageReasonV1::Redacted, ), - TemporalSourceAccess::Unavailable - | TemporalSourceAccess::LegacyUnauthorized => ( + TemporalSourceAccess::Unavailable => ( SessionSourceCoverageStateV1::Unavailable, SessionSourceCoverageReasonV1::Unavailable, ), diff --git a/crates/tracedecay-temporal-query/src/tests.rs b/crates/tracedecay-temporal-query/src/tests.rs index 01ba6b2c06..32700e123f 100644 --- a/crates/tracedecay-temporal-query/src/tests.rs +++ b/crates/tracedecay-temporal-query/src/tests.rs @@ -17,22 +17,26 @@ use tracedecay_domain::{ use super::candidates::{CandidateChannel, CandidatePlan}; use super::context::{ContextBudget, TokenPolicy, VersionedTokenEstimator}; use super::cursor::{CursorError, verify_cursor}; +use super::execution::{BindingDigest, ExecutionLimits}; use super::hydration::{ HydrationAuthorization, HydrationDenial, HydrationFuture, HydrationGrant, HydrationSink, TemporalHydrationPort, }; +use super::paging::{CandidatePageSink, PageKey, PageRequest, PageStatus, TemporalRecordPageSink}; use super::ports::{ - BindingDigest, CandidatePageSink, ExecutionLimits, InMemoryCursorAuthenticator, KernelVersions, - PageKey, PageRequest, PageStatus, PortFuture, SummarySourceRecord, TemporalExecutionSnapshot, - TemporalParticipantAuthorization, TemporalParticipantGeneration, TemporalParticipantManifest, - TemporalPortError, TemporalPreparedCandidateCohort, TemporalReadPort, TemporalRecord, - TemporalRecordPageSink, TemporalSnapshotRequest, TemporalSourceAccess, TemporalWatermarks, + InMemoryCursorAuthenticator, PortFuture, SummarySourceRecord, TemporalPortError, + TemporalReadPort, TemporalRecord, TemporalSnapshotRequest, }; use super::ranking::{DiversityLimits, RankingCandidate, RankingError}; use super::resolution::summary::SummarySourceState; use super::resolution::types::{ ResolutionAssertion, ResolutionEvidence, ResolutionOccurrence, ValidatedAuthorization, }; +use super::snapshot::{ + KernelVersions, TemporalExecutionSnapshot, TemporalParticipantAuthorization, + TemporalParticipantGeneration, TemporalParticipantManifest, TemporalPreparedCandidateCohort, + TemporalSourceAccess, TemporalWatermarks, +}; use super::{ TemporalKernelError, TemporalKernelRequest, execute_temporal_candidate_export, execute_temporal_kernel, hydrate_temporal_candidate_selection, @@ -72,8 +76,9 @@ impl FakeReadPort { } impl TemporalReadPort for FakeReadPort { - fn produce_candidate_page<'a>( + fn produce_candidate_page_for_scope<'a>( &'a self, + _scope: &'a super::ports::TemporalRetrievalScope, snapshot: &'a TemporalExecutionSnapshot, _plan: &'a CandidatePlan, request: PageRequest, @@ -117,8 +122,9 @@ impl TemporalReadPort for FakeReadPort { }) } - fn produce_temporal_record_page<'a>( + fn produce_temporal_record_page_for_scope<'a>( &'a self, + _scope: &'a super::ports::TemporalRetrievalScope, _snapshot: &'a TemporalExecutionSnapshot, candidates: &'a [RankingCandidate], request: PageRequest, @@ -143,17 +149,6 @@ impl TemporalReadPort for FakeReadPort { }) }) } - - fn produce_temporal_record_page_for_scope<'a>( - &'a self, - _scope: &'a super::ports::TemporalRetrievalScope, - snapshot: &'a TemporalExecutionSnapshot, - candidates: &'a [RankingCandidate], - request: PageRequest, - sink: &'a mut TemporalRecordPageSink<'_>, - ) -> PortFuture<'a, PageStatus> { - self.produce_temporal_record_page(snapshot, candidates, request, sink) - } } fn page_start(request: &PageRequest) -> usize { diff --git a/crates/tracedecay-tool-catalog/Cargo.toml b/crates/tracedecay-tool-catalog/Cargo.toml index f5847a64f4..bf9dd232b7 100644 --- a/crates/tracedecay-tool-catalog/Cargo.toml +++ b/crates/tracedecay-tool-catalog/Cargo.toml @@ -9,7 +9,7 @@ repository = "https://github.com/ScriptedAlchemy/tracedecay" [dependencies] hotpath = { workspace = true, optional = true } -schemars = "1.2.1" +schemars.workspace = true serde = { version = "1", features = ["derive"] } serde_json = "1" sha2 = "0.11" diff --git a/crates/tracedecay-tool-catalog/src/binding.rs b/crates/tracedecay-tool-catalog/src/binding.rs index db2434ee74..d505c83f5b 100644 --- a/crates/tracedecay-tool-catalog/src/binding.rs +++ b/crates/tracedecay-tool-catalog/src/binding.rs @@ -103,36 +103,6 @@ impl ProtocolRevisionRange { } } -/// A bounded deprecation period for a formerly current surface spelling. -#[derive(Clone, Debug, PartialEq, Eq, Serialize)] -pub struct BindingDeprecation { - sunset_revision: u32, -} - -impl BindingDeprecation { - pub fn new(sunset_revision: u32) -> Result { - if sunset_revision == 0 { - return Err(CatalogValidationError::InvalidValue { - field: "binding deprecation sunset revision", - reason: "must be greater than zero", - }); - } - Ok(Self { sunset_revision }) - } - - pub const fn sunset_revision(&self) -> u32 { - self.sunset_revision - } -} - -/// Lifecycle state of a surface spelling. -#[derive(Clone, Debug, PartialEq, Eq, Serialize)] -#[serde(rename_all = "snake_case", tag = "status")] -pub enum BindingStatus { - Current, - Deprecated { deprecation: BindingDeprecation }, -} - /// Input used to construct an immutable surface binding. #[derive(Clone, Debug, PartialEq, Eq)] pub struct SurfaceBindingInputV1 { @@ -142,8 +112,6 @@ pub struct SurfaceBindingInputV1 { pub operation: SurfaceOperationName, pub protocol_revisions: ProtocolRevisionRange, pub required_features: Vec, - pub status: BindingStatus, - pub alias_of: Option, } /// A surface spelling pointing at exactly one capability. @@ -159,8 +127,6 @@ pub struct SurfaceBindingV1 { operation: SurfaceOperationName, protocol_revisions: ProtocolRevisionRange, required_features: Vec, - status: BindingStatus, - alias_of: Option, } impl SurfaceBindingV1 { @@ -174,8 +140,6 @@ impl SurfaceBindingV1 { operation: input.operation, protocol_revisions: input.protocol_revisions, required_features, - status: input.status, - alias_of: input.alias_of, }) } @@ -202,16 +166,4 @@ impl SurfaceBindingV1 { pub fn required_features(&self) -> &[FeatureId] { &self.required_features } - - pub fn status(&self) -> &BindingStatus { - &self.status - } - - pub fn alias_of(&self) -> Option<&BindingId> { - self.alias_of.as_ref() - } - - pub const fn is_alias(&self) -> bool { - self.alias_of.is_some() - } } diff --git a/crates/tracedecay-tool-catalog/src/lib.rs b/crates/tracedecay-tool-catalog/src/lib.rs index 42b659a0d5..20efe06ea0 100644 --- a/crates/tracedecay-tool-catalog/src/lib.rs +++ b/crates/tracedecay-tool-catalog/src/lib.rs @@ -19,8 +19,8 @@ mod snapshot; mod validation; pub use binding::{ - BindingDeprecation, BindingStatus, BindingSurface, ProtocolRevisionRange, - SurfaceBindingInputV1, SurfaceBindingV1, SurfaceOperationName, + BindingSurface, ProtocolRevisionRange, SurfaceBindingInputV1, SurfaceBindingV1, + SurfaceOperationName, }; pub use executable::{ BindingAvailabilityV1, BindingRegistryV1, ExecutableBindingAvailabilityV1, diff --git a/crates/tracedecay-tool-catalog/src/operation.rs b/crates/tracedecay-tool-catalog/src/operation.rs index f94a197b8d..8568b50a5a 100644 --- a/crates/tracedecay-tool-catalog/src/operation.rs +++ b/crates/tracedecay-tool-catalog/src/operation.rs @@ -70,8 +70,9 @@ macro_rules! application_surface_operations { /// MCP/CLI spelling for this canonical operation. /// - /// The diagnostics read keeps its established public tool spelling; - /// its catalog and HTTP/SDK identity remain `diagnostics_read`. + /// Diagnostics and the code-graph navigation reads keep their + /// established short public tool spellings; their catalog and + /// HTTP/SDK identities keep the canonical name. pub const fn mcp_operation_name(self) -> &'static str { match self { $( @@ -150,11 +151,11 @@ application_surface_operations! { CodeExactOccurrence => "code_exact_occurrence"; CodePhraseSearch => "code_phrase_search"; CodeSymbolSearch => "code_symbol_search"; - CodeSignatureSearch => "code_signature_search"; - CodeImplementations => "code_implementations"; - CodeTypeHierarchy => "code_type_hierarchy"; - CodeCallers => "code_callers"; - CodeCallees => "code_callees"; + CodeSignatureSearch => "code_signature_search", mcp: "signature_search"; + CodeImplementations => "code_implementations", mcp: "implementations"; + CodeTypeHierarchy => "code_type_hierarchy", mcp: "type_hierarchy"; + CodeCallers => "code_callers", mcp: "callers"; + CodeCallees => "code_callees", mcp: "callees"; CodeFacets => "code_facets"; CodeTimeline => "code_timeline"; CodeDeclaration => "code_declaration"; @@ -168,6 +169,15 @@ application_surface_operations! { SourceBody => "source_body"; SourceOutline => "source_outline"; ModuleApi => "module_api"; + Context => "context"; + Node => "node"; + Impact => "impact"; + Similar => "similar"; + Redundancy => "redundancy"; + RenamePreview => "rename_preview"; + PortStatus => "port_status"; + PortOrder => "port_order"; + Todos => "todos"; HealthRead => "health_read"; HealthDelta => "health_delta"; StorageStatus => "storage_status"; @@ -195,6 +205,63 @@ application_surface_operations! { ContextScoutClaim => "context_scout_claim"; ContextScoutDelivery => "context_scout_delivery"; ContextScoutFeedback => "context_scout_feedback"; + StrReplace => "str_replace"; + MultiStrReplace => "multi_str_replace"; + InsertAt => "insert_at"; + AstGrepRewrite => "ast_grep_rewrite"; + ReplaceSymbol => "replace_symbol"; + InsertAtSymbol => "insert_at_symbol"; + MoveSymbol => "move_symbol"; + RenameSymbol => "rename_symbol"; + SourceEditReconcile => "source_edit_reconcile"; + SourceEditRollback => "source_edit_rollback"; + FactStoreCurate => "fact_store_curate"; + FactStoreAdd => "fact_store_add"; + FactStoreSearch => "fact_store_search"; + FactStoreProbe => "fact_store_probe"; + FactStoreRelated => "fact_store_related"; + FactStoreReason => "fact_store_reason"; + FactStoreContradict => "fact_store_contradict"; + FactStoreGet => "fact_store_get"; + FactStoreUpdate => "fact_store_update"; + FactStoreRemove => "fact_store_remove"; + FactStoreSupersede => "fact_store_supersede"; + FactStoreList => "fact_store_list"; + FactFeedback => "fact_feedback"; + MemoryStatus => "memory_status"; + SessionRefreshStatus => "session_refresh_status"; + SessionRefreshCancel => "session_refresh_cancel"; + SessionRefreshBegin => "session_refresh_begin"; + MessageSearch => "message_search"; + SessionsFor => "sessions_for"; + Workflows => "workflows"; + LcmStatus => "lcm_status"; + LcmDoctor => "lcm_doctor"; + LcmLoadSession => "lcm_load_session"; + LcmGrep => "lcm_grep"; + LcmDescribe => "lcm_describe"; + LcmExpand => "lcm_expand"; + LcmExpandQuery => "lcm_expand_query"; +} + +impl ApplicationSurfaceOperation { + /// Graph and port reads answered by the project's graph-tool owner with + /// their typed catalog result in `ApplicationOutcome::Result`. + pub const GRAPH_TOOL_OPERATIONS: [Self; 9] = [ + Self::Context, + Self::Node, + Self::Impact, + Self::Similar, + Self::Redundancy, + Self::RenamePreview, + Self::PortStatus, + Self::PortOrder, + Self::Todos, + ]; + + pub fn is_graph_tool(self) -> bool { + Self::GRAPH_TOOL_OPERATIONS.contains(&self) + } } #[cfg(test)] @@ -273,6 +340,18 @@ mod tests { ApplicationSurfaceOperation::from_tool_name("tracedecay_diagnostics_read"), None ); + assert_eq!( + ApplicationSurfaceOperation::from_tool_name("tracedecay_callers"), + Some(ApplicationSurfaceOperation::CodeCallers) + ); + assert_eq!( + ApplicationSurfaceOperation::from_tool_name("tracedecay_code_callers"), + None + ); + assert_eq!( + ApplicationSurfaceOperation::CodeCallers.as_str(), + "code_callers" + ); assert_eq!( ApplicationSurfaceOperation::from_tool_name("tracedecay_not_an_operation"), None diff --git a/crates/tracedecay-tool-catalog/src/validation.rs b/crates/tracedecay-tool-catalog/src/validation.rs index 6a97f3259d..a7d5b8a51f 100644 --- a/crates/tracedecay-tool-catalog/src/validation.rs +++ b/crates/tracedecay-tool-catalog/src/validation.rs @@ -83,15 +83,6 @@ pub enum CatalogValidationError { binding_id: BindingId, capability_id: CapabilityId, }, - #[error("binding {binding_id} aliases missing binding {alias_of}")] - MissingAliasTarget { - binding_id: BindingId, - alias_of: BindingId, - }, - #[error("binding alias {binding_id} must target the same capability")] - AliasCapabilityMismatch { binding_id: BindingId }, - #[error("binding alias {binding_id} cannot target another alias")] - AliasTargetsAlias { binding_id: BindingId }, #[error("duplicate retrieval primitive capability ID {0}")] DuplicateRetrievalCapabilityId(CapabilityId), #[error("duplicate retriever ID {0}")] @@ -375,28 +366,6 @@ fn index_bindings<'a>( } } - for binding in bindings.values() { - let Some(alias_of) = binding.alias_of() else { - continue; - }; - let Some(canonical) = bindings.get(alias_of) else { - return Err(CatalogValidationError::MissingAliasTarget { - binding_id: binding.binding_id().clone(), - alias_of: alias_of.clone(), - }); - }; - if canonical.is_alias() { - return Err(CatalogValidationError::AliasTargetsAlias { - binding_id: binding.binding_id().clone(), - }); - } - if canonical.capability_id() != binding.capability_id() { - return Err(CatalogValidationError::AliasCapabilityMismatch { - binding_id: binding.binding_id().clone(), - }); - } - } - Ok(bindings) } diff --git a/crates/tracedecay-tool-catalog/tests/tool_catalog_suite/executable_binding_contract.rs b/crates/tracedecay-tool-catalog/tests/tool_catalog_suite/executable_binding_contract.rs index 97657bc174..84af086c52 100644 --- a/crates/tracedecay-tool-catalog/tests/tool_catalog_suite/executable_binding_contract.rs +++ b/crates/tracedecay-tool-catalog/tests/tool_catalog_suite/executable_binding_contract.rs @@ -72,12 +72,6 @@ fn schema_bodies_are_derived_from_rust_type_authority() { binding.result_schema().body()["properties"]["contents"]["type"], "string" ); - assert_eq!( - binding.request_schema().digest(), - typed_schema::(binding.request_schema().schema_ref().clone()) - .unwrap() - .digest() - ); } #[test] diff --git a/crates/tracedecay-tool-catalog/tests/tool_catalog_suite/profile_budget.rs b/crates/tracedecay-tool-catalog/tests/tool_catalog_suite/profile_budget.rs index 7a7d8d174a..83eced4476 100644 --- a/crates/tracedecay-tool-catalog/tests/tool_catalog_suite/profile_budget.rs +++ b/crates/tracedecay-tool-catalog/tests/tool_catalog_suite/profile_budget.rs @@ -3,7 +3,7 @@ use crate::common; use std::collections::BTreeSet; use tracedecay_tool_catalog::{ - BindingId, BindingStatus, BindingSurface, CatalogContributionInputV1, CatalogContributionV1, + BindingId, BindingSurface, CatalogContributionInputV1, CatalogContributionV1, CatalogSnapshotBuilderV1, CatalogValidationError, ContributionId, ProfileBudget, ProfileDefinition, ProfileDefinitionInputV1, ProfileKind, ProtocolRevisionRange, SurfaceBindingInputV1, SurfaceBindingV1, SurfaceOperationName, @@ -19,7 +19,7 @@ fn profile_budgets_reject_overflow_without_a_universal_tool_ceiling() { let profile_id = profile_id("profile.host-limited"); let capability_id = capability_id("capability.source.read"); let first_binding_id = BindingId::new("binding.source.read.cli").unwrap(); - let second_binding_id = BindingId::new("binding.source.read.alias").unwrap(); + let second_binding_id = BindingId::new("binding.source.get.cli").unwrap(); let manifest = read_manifest( capability_id.clone(), use_case_id("use-case.source.read"), @@ -35,8 +35,6 @@ fn profile_budgets_reject_overflow_without_a_universal_tool_ceiling() { operation: SurfaceOperationName::new("source read").unwrap(), protocol_revisions: ProtocolRevisionRange::new(1, 1).unwrap(), required_features: Vec::new(), - status: BindingStatus::Current, - alias_of: None, }) .unwrap(); let second_binding = SurfaceBindingV1::new(SurfaceBindingInputV1 { @@ -46,8 +44,6 @@ fn profile_budgets_reject_overflow_without_a_universal_tool_ceiling() { operation: SurfaceOperationName::new("source get").unwrap(), protocol_revisions: ProtocolRevisionRange::new(1, 1).unwrap(), required_features: Vec::new(), - status: BindingStatus::Current, - alias_of: Some(first_binding_id), }) .unwrap(); let profile = ProfileDefinition::new(ProfileDefinitionInputV1 { @@ -119,13 +115,21 @@ fn profile_absence_is_explicit_in_snapshot_discovery() { .add_contribution(contribution) .add_handler(handler_for(&manifest)) .add_profile(profile( - primary_profile_id, - vec![capability_id], + primary_profile_id.clone(), + vec![capability_id.clone()], ample_budget(), )) .add_profile(compact_profile); let snapshot = builder.build().unwrap(); + assert_eq!( + snapshot + .visible_capabilities(&primary_profile_id, &BTreeSet::new()) + .into_iter() + .map(|capability| capability.capability_id()) + .collect::>(), + vec![&capability_id] + ); assert!( snapshot .visible_capabilities(&compact_profile_id, &BTreeSet::new()) diff --git a/crates/tracedecay-tool-catalog/tests/tool_catalog_suite/retrieval_contract.rs b/crates/tracedecay-tool-catalog/tests/tool_catalog_suite/retrieval_contract.rs index 48204902f6..e7edb594ca 100644 --- a/crates/tracedecay-tool-catalog/tests/tool_catalog_suite/retrieval_contract.rs +++ b/crates/tracedecay-tool-catalog/tests/tool_catalog_suite/retrieval_contract.rs @@ -91,7 +91,11 @@ fn retrieval_primitives_canonicalize_temporal_and_cancellation_metadata() { ample_budget(), )); let snapshot = builder.build().unwrap(); - assert!(snapshot.retrieval_primitive(&capability_id).is_some()); + let registered = snapshot + .retrieval_primitive(&capability_id) + .expect("registered retrieval primitive"); + assert_eq!(registered.family(), RetrievalFamily::Source); + assert_eq!(registered.retriever_id().as_str(), "retriever.source.lines"); } #[test] diff --git a/crates/tracedecay-tool-catalog/tests/tool_catalog_suite/snapshot_contract.rs b/crates/tracedecay-tool-catalog/tests/tool_catalog_suite/snapshot_contract.rs index 9c444ed85d..471733f208 100644 --- a/crates/tracedecay-tool-catalog/tests/tool_catalog_suite/snapshot_contract.rs +++ b/crates/tracedecay-tool-catalog/tests/tool_catalog_suite/snapshot_contract.rs @@ -4,11 +4,10 @@ use std::collections::BTreeSet; use schemars::JsonSchema; use tracedecay_tool_catalog::{ - ApplicationHandlerDescriptorV1, BindingId, BindingStatus, BindingSurface, - CatalogContributionInputV1, CatalogContributionV1, CatalogSnapshotBuilderV1, - CatalogValidationError, ContributionId, ExecutableSchemaAuthority, ProfileDefinition, - ProfileDefinitionInputV1, ProfileKind, ProtocolRevisionRange, SurfaceBindingInputV1, - SurfaceBindingV1, SurfaceOperationName, + ApplicationHandlerDescriptorV1, BindingId, BindingSurface, CatalogContributionInputV1, + CatalogContributionV1, CatalogSnapshotBuilderV1, CatalogValidationError, ContributionId, + ExecutableSchemaAuthority, ProfileDefinition, ProfileDefinitionInputV1, ProfileKind, + ProtocolRevisionRange, SurfaceBindingInputV1, SurfaceBindingV1, SurfaceOperationName, }; use common::{ @@ -62,8 +61,6 @@ fn snapshot_digest_bytes_are_pinned_for_a_fixed_catalog() { operation: SurfaceOperationName::new("source read").unwrap(), protocol_revisions: ProtocolRevisionRange::new(1, 2).unwrap(), required_features: Vec::new(), - status: BindingStatus::Current, - alias_of: None, }) .unwrap(); let source_contribution_id = ContributionId::new("contribution.source").unwrap(); @@ -113,7 +110,7 @@ fn snapshot_digest_bytes_are_pinned_for_a_fixed_catalog() { assert_eq!( snapshot.digest().to_string(), - "sha256:1588d3f6cfa939feac2e00e953c0cd17bef681bb986515dcfefae9a53d41319a" + "sha256:e53f40e6f2e2260882a02a1c12b7efb682ab63c6652c2d7e27eb415f903c94a4" ); assert_eq!( snapshot diff --git a/crates/tracedecay/Cargo.toml b/crates/tracedecay/Cargo.toml index da35d4ac8b..726544503b 100644 --- a/crates/tracedecay/Cargo.toml +++ b/crates/tracedecay/Cargo.toml @@ -232,12 +232,10 @@ test-transport = [ "tracedecay-code-index-runtime/test-transport", ] -# The evaluator library is the only selector of `tracedecay-query/search-eval`, -# the eval-only in-memory lexical projection. Cargo rejects optional -# dev-dependencies, and an unconditional one unifies this feature into every -# test target, so the transport suites compile that projection. Journeys that -# compare the CLI receipt to the library enable this feature. `test-transport` -# and `production` must not. +# Links the search-quality evaluator library. Cargo rejects optional +# dev-dependencies, and an unconditional one links the evaluator into every +# test target. Journeys that compare the CLI receipt to the library enable +# this feature. `test-transport` and `production` must not. search-eval = ["dep:tracedecay-search-eval"] # The typed RMCP benchmark is the only consumer of the client-side RMCP @@ -341,8 +339,8 @@ tempfile = "3" futures-util = "0.3.33" rmcp = { version = "3.0.1", default-features = false, features = ["server"] } # Opt-in. Cargo rejects optional dev-dependencies, and an unconditional one -# unifies `tracedecay-query/search-eval` into every test target. Default and -# `production` builds leave it off, so the shipped CLI does not link it. +# links the evaluator into every test target. Default and `production` builds +# leave it off, so the shipped CLI does not link it. tracedecay-search-eval = { path = "../tracedecay-search-eval", version = "0.1.0", optional = true } # `kill(2)` for the daemon integration suites' physical-restart journeys @@ -443,13 +441,6 @@ required-features = ["test-helpers"] # `--list --format terse` probe used by `cargo test` and nextest. test = false -[[bench]] -name = "mcp_connection_pipeline" -path = "benches/mcp_connection_pipeline.rs" -harness = false -required-features = ["test-transport"] -test = false - [[bench]] name = "rmcp_connection_pipeline" path = "benches/rmcp_connection_pipeline.rs" diff --git a/crates/tracedecay/benches/mcp_connection_pipeline.rs b/crates/tracedecay/benches/mcp_connection_pipeline.rs deleted file mode 100644 index e25cc23c21..0000000000 --- a/crates/tracedecay/benches/mcp_connection_pipeline.rs +++ /dev/null @@ -1,172 +0,0 @@ -//! Hermetic one-connection JSON-RPC pipeline benchmark. -//! -//! Runs only against a temporary git project mounted by the production -//! composition harness. It never opens the operator daemon or profile store. - -#![allow(clippy::too_many_lines)] -use std::collections::HashMap; -use std::path::Path; -use std::process::Command; -use std::sync::Arc; -use std::time::Instant; - -use serde_json::{Value, json}; -use tracedecay::daemon::ProductionProjectCompositionHarnessV1; -use tracedecay::mcp::McpServer; -use tracedecay_mcp::transport::ChannelTransport; - -const READ_ROUNDS: usize = 16; -const MIXED_ROUNDS: usize = 8; - -fn git(root: &Path, args: &[&str]) { - let status = Command::new("git") - .args(args) - .current_dir(root) - .status() - .unwrap_or_else(|error| panic!("git {args:?} failed to start: {error}")); - assert!(status.success(), "git {args:?} failed"); -} - -fn request(id: u64, tool: &str, arguments: Value) -> String { - json!({ - "jsonrpc": "2.0", - "id": id, - "method": "tools/call", - "params": { - "name": tool, - "arguments": arguments, - } - }) - .to_string() -} - -fn percentile_95(mut samples: Vec) -> u64 { - samples.sort_unstable(); - let index = samples - .len() - .saturating_mul(95) - .div_ceil(100) - .saturating_sub(1); - samples.get(index).copied().unwrap_or_default() -} - -async fn run_workload(server: Arc) -> Value { - let (mut transport, sender, mut responses) = ChannelTransport::new(); - let serving = tokio::spawn(async move { server.run_connection(&mut transport).await }); - let mut sent_at = HashMap::new(); - let started = Instant::now(); - let mut next_id = 1_u64; - - for round in 0..READ_ROUNDS { - for (tool, arguments) in [ - ( - "tracedecay_search", - json!({"query": format!("pipeline symbol {round}"), "limit": 5}), - ), - ( - "tracedecay_status", - json!({"admission_only": true, "format": "json"}), - ), - ( - "tracedecay_fact_store_search", - json!({"query": format!("pipeline memory {round}"), "limit": 5}), - ), - ] { - sent_at.insert(next_id, Instant::now()); - sender - .send(request(next_id, tool, arguments)) - .expect("send independent benchmark read"); - next_id += 1; - } - } - - for round in 0..MIXED_ROUNDS { - for (tool, arguments) in [ - ( - "tracedecay_status", - json!({"admission_only": true, "format": "json"}), - ), - ( - "tracedecay_fact_store_add", - json!({ - "content": format!("pipeline effect {round}"), - "category": "project", - "trust": 0.9, - }), - ), - ( - "tracedecay_fact_store_search", - json!({"query": format!("pipeline effect {round}"), "limit": 5}), - ), - ] { - sent_at.insert(next_id, Instant::now()); - sender - .send(request(next_id, tool, arguments)) - .expect("send mixed benchmark request"); - next_id += 1; - } - } - let request_count = next_id - 1; - drop(sender); - - let mut queue_samples_us = Vec::with_capacity(request_count as usize); - let mut response_count = 0_u64; - while let Some(line) = responses.recv().await { - let response: Value = serde_json::from_str(line.trim()).expect("benchmark response JSON"); - let Some(id) = response.get("id").and_then(Value::as_u64) else { - continue; - }; - let sent = sent_at.remove(&id).expect("known benchmark response id"); - let total_us = u64::try_from(sent.elapsed().as_micros()).unwrap_or(u64::MAX); - let handler_us = response - .pointer("/result/_meta/duration_us") - .and_then(Value::as_u64) - .unwrap_or_default(); - queue_samples_us.push(total_us.saturating_sub(handler_us)); - response_count += 1; - } - serving - .await - .expect("join benchmark connection") - .expect("serve benchmark connection"); - assert_eq!(response_count, request_count); - let elapsed = started.elapsed(); - json!({ - "requests": request_count, - "elapsed_us": u64::try_from(elapsed.as_micros()).unwrap_or(u64::MAX), - "throughput_requests_per_second": request_count as f64 / elapsed.as_secs_f64(), - "p95_dispatch_queue_us": percentile_95(queue_samples_us), - "read_rounds": READ_ROUNDS, - "mixed_rounds": MIXED_ROUNDS, - }) -} - -#[tokio::main(flavor = "multi_thread", worker_threads = 8)] -async fn main() { - tracedecay::product_runtime::register_fixture_product_runtime(); - let sandbox = tempfile::TempDir::new().expect("pipeline benchmark sandbox"); - let project = sandbox.path().join("project"); - std::fs::create_dir_all(project.join("src")).expect("benchmark source directory"); - std::fs::write( - project.join("src/lib.rs"), - "pub fn pipeline_symbol() -> usize { 1 }\n", - ) - .expect("benchmark source"); - git(&project, &["init", "-q", "-b", "main"]); - git(&project, &["config", "user.email", "pipeline@test.invalid"]); - git(&project, &["config", "user.name", "Pipeline Benchmark"]); - git(&project, &["add", "."]); - git(&project, &["commit", "-q", "-m", "fixture"]); - - let harness = ProductionProjectCompositionHarnessV1::open(sandbox.path(), [project.clone()]) - .await - .expect("production benchmark composition"); - let server = harness.server(&project).expect("mounted benchmark server"); - server.set_timings_enabled(true); - let result = run_workload(server).await; - println!( - "{}", - serde_json::to_string_pretty(&result).expect("serialize benchmark result") - ); - harness.shutdown().await; -} diff --git a/crates/tracedecay/benches/queries.rs b/crates/tracedecay/benches/queries.rs index 8ae84b8283..129072856a 100644 --- a/crates/tracedecay/benches/queries.rs +++ b/crates/tracedecay/benches/queries.rs @@ -308,7 +308,7 @@ pub fn build_queries(ctx: &QueryContext) -> Vec { Query::read( "by_id", "tracedecay_callers", - json!({ "node_id": QueryContext::pick(&ctx.function_ids, i), "max_depth": 3 }), + json!({ "node_id": QueryContext::pick(&ctx.function_ids, i), "maximum_depth": 3 }), ) }), }); @@ -319,7 +319,7 @@ pub fn build_queries(ctx: &QueryContext) -> Vec { Query::read( "by_id", "tracedecay_callees", - json!({ "node_id": QueryContext::pick(&ctx.function_ids, i), "max_depth": 3 }), + json!({ "node_id": QueryContext::pick(&ctx.function_ids, i), "maximum_depth": 3 }), ) }), }); @@ -368,17 +368,6 @@ pub fn build_queries(ctx: &QueryContext) -> Vec { }), }); - groups.push(ToolGroup { - tool: "tracedecay_body", - queries: five(|i| { - Query::read( - "by_id", - "tracedecay_body", - json!({ "symbol": QueryContext::pick(&ctx.function_qnames, i) }), - ) - }), - }); - groups.push(ToolGroup { tool: "tracedecay_files", queries: five(|i| { diff --git a/crates/tracedecay/benches/rmcp/benchmark.rs b/crates/tracedecay/benches/rmcp/benchmark.rs index 9d430527d9..d90fb91739 100644 --- a/crates/tracedecay/benches/rmcp/benchmark.rs +++ b/crates/tracedecay/benches/rmcp/benchmark.rs @@ -27,10 +27,11 @@ use tracedecay_daemon_protocol::{ }; use tracedecay_domain::ProjectId; -use super::{BrokerStreamTransport, DaemonLifecycle, serve_routed_rmcp_connection}; +use super::{BrokerStreamTransport, serve_routed_rmcp_connection}; use crate::mcp::McpServer; -use crate::project::TraceDecayOpenOptions; -use crate::test_support::host_admission::HostAdmissionTestRuntimeV1; +use tracedecay_daemon_service::shutdown::DaemonLifecycle; +use tracedecay_project::project::TraceDecayOpenOptions; +use tracedecay_project::test_support::host_admission::HostAdmissionTestRuntimeV1; pub const PERSISTENT_WARMUP_REQUESTS: usize = 8; pub const PERSISTENT_MEASURED_REQUESTS: usize = 64; @@ -70,7 +71,7 @@ struct BenchmarkServerFixture { impl BenchmarkServerFixture { async fn open() -> Result { - crate::product_runtime::register_fixture_product_runtime(); + tracedecay_project::product_runtime::register_fixture_product_runtime(); let sandbox = tempfile::TempDir::new() .map_err(|error| format!("create benchmark sandbox: {error}"))?; let project = sandbox.path().join("project"); diff --git a/crates/tracedecay/benches/session_temporal/harness.rs b/crates/tracedecay/benches/session_temporal/harness.rs index e2feb14cc0..28b0683ff4 100644 --- a/crates/tracedecay/benches/session_temporal/harness.rs +++ b/crates/tracedecay/benches/session_temporal/harness.rs @@ -31,13 +31,14 @@ use tracedecay_tool_catalog::{CapabilityId, UseCaseId}; use tracedecay_global_db::{RegisteredGlobalDb, RegisteredGlobalDbLeaseV1}; use tracedecay_host_admission::{HostAdmissionAuthorities, HostAdmissionFacade}; +use tracedecay_runtime_core::cancellation::CancellationToken; use tracedecay_runtime_core::storage::{ read_repository_identity_marker, write_repository_identity_marker, }; use tracedecay_runtime_core::timeutil::nearest_rank; use tracedecay_session_memory::context::{ - BranchId, CancellationToken, CapabilityDigest, ConfigurationDigest, PolicyDigest, ProfileId, - RequestBudgets, ResolvedGitRoute, ResolvedSessionIdentity, SessionRootId, SessionStoreId, + BranchId, CapabilityDigest, ConfigurationDigest, PolicyDigest, ProfileId, RequestBudgets, + ResolvedGitRoute, ResolvedSessionIdentity, SessionRootId, SessionStoreId, application_observed_at, session_application_grant_digest, }; use tracedecay_session_memory::session::{ @@ -46,12 +47,14 @@ use tracedecay_session_memory::session::{ SessionRetrievalService, SessionScopeAuthorizationRequest, SessionScopeAuthorizer, SessionTemporalQuery, }; -use tracedecay_session_temporal_store::RegisteredGlobalDbSessionTemporalExecution; +use tracedecay_session_temporal_store::{ + RegisteredGlobalDbSessionTemporalExecution, SessionTemporalAccess, +}; use tracedecay_sessions::observation::ObservationCancellation; -use tracedecay_sessions::runtime::codex; +use tracedecay_sessions::runtime::hosts::codex; use tracedecay_store_runtime::DaemonSessionRuntimeRegistryV1; use tracedecay_temporal_query::context::{ContextBudget, TokenPolicy, VersionedTokenEstimator}; -use tracedecay_temporal_query::ports::ExecutionControl; +use tracedecay_temporal_query::execution::ExecutionControl; use tracedecay_temporal_query::ranking::DiversityLimits; mod root_relation_fixture; @@ -168,7 +171,7 @@ pub struct IsolatedBenchmarkEnv { impl IsolatedBenchmarkEnv { pub fn enter(prefix: &str) -> BenchResult { - let env_lock = crate::config::lock_user_data_dir_test_env(); + let env_lock = tracedecay_project::config::lock_user_data_dir_test_env(); let temp = tempfile::Builder::new() .prefix(prefix) .tempdir() @@ -718,7 +721,7 @@ fn measurement_result(source_identity: Value, measurement: Value) -> Value { /// benchmark-private handle. fn ensure_admission_resource_authorities() -> Arc { - crate::test_support::host_admission::ensure_process_background_cpu_authority() + tracedecay_project::test_support::host_admission::ensure_process_background_cpu_authority() .expect("install process capture authorities for the benchmark") } @@ -867,8 +870,7 @@ async fn run_one_repetition(repetition: usize) -> BenchResult Sandbox { .expect("bench main installed a sandbox HOME"); let project = tmp.path().join("project"); let profile = tmp.path().join("profile"); - std::fs::create_dir_all(project.join(".tracedecay")).unwrap(); - std::fs::write(project.join(".tracedecay/tracedecay.db"), "").unwrap(); + std::fs::create_dir_all(&project).unwrap(); let git = std::process::Command::new("git") .args(["init", "--quiet"]) .current_dir(&project) diff --git a/crates/tracedecay/examples/bench_extract.rs b/crates/tracedecay/examples/bench_extract.rs index 2409bd3c0e..1bcdbf4d6e 100644 --- a/crates/tracedecay/examples/bench_extract.rs +++ b/crates/tracedecay/examples/bench_extract.rs @@ -21,9 +21,9 @@ fn main() { || path.ends_with(".hpp") || path.ends_with(".hh") { - CppExtractor.extract(&path, &source) + CppExtractor.extract_artifact(&path, &source).result } else { - CExtractor.extract(&path, &source) + CExtractor.extract_artifact(&path, &source).result }; let elapsed = t0.elapsed(); eprintln!( diff --git a/crates/tracedecay/src/daemon.rs b/crates/tracedecay/src/daemon.rs index 5a35642655..15bfc36515 100644 --- a/crates/tracedecay/src/daemon.rs +++ b/crates/tracedecay/src/daemon.rs @@ -241,13 +241,13 @@ mod connection_serving; pub use connection_serving::rmcp_benchmark; #[cfg(unix)] use connection_serving::serve_authenticated_socket_client_with_class; -#[cfg(all(unix, test))] -use connection_serving::serve_socket_client; +#[cfg(any(test, feature = "test-transport"))] +pub(crate) use connection_serving::serve_routed_rmcp_connection; #[cfg(not(unix))] use connection_serving::serve_windows_broker_client_with_class_and_invocation; #[cfg(test)] use connection_serving::{ - await_project_owner_or_disconnect, serve_routed_rmcp_connection, serve_windows_broker_client, + await_project_owner_or_disconnect, serve_windows_broker_client, serve_windows_broker_client_with_class, }; mod core_admission; @@ -258,10 +258,6 @@ use engine::{ ensure_context_scout_owner_before_advertising, ensure_git_index_transactions_for_mutation_owners, }; -pub(crate) use tracedecay_daemon_service::automation_observation::{ - project_run_observation_producer as project_automation_observation_producer, - record_project_run as record_project_automation_run, -}; mod core_client; mod core_doctor; mod core_handshake; @@ -287,17 +283,6 @@ pub(crate) use core_doctor::*; pub use core_handshake::*; pub use core_hooks::*; pub use core_proxy::*; -// Daemon process lifecycle and logging live in `tracedecay-daemon-service`; -// the root's engine, bootstrap, and connection serving still read them by -// these names until they move. -#[cfg(unix)] -pub(crate) use tracedecay_daemon_service::logging::recent_watcher_events; -pub(crate) use tracedecay_daemon_service::logging::unavailable_error; -#[cfg(feature = "hotpath")] -pub use tracedecay_daemon_service::shutdown::install_hotpath_shutdown_finalizer; -pub(crate) use tracedecay_daemon_service::shutdown::{ - DAEMON_CLIENT_DRAIN_DEADLINE, DAEMON_TASK_ABORT_DEADLINE, DaemonLifecycle, ShutdownStatus, -}; mod github_credential_lifecycle; mod graph_resolution; use graph_resolution::retained_project_server_resolver; diff --git a/crates/tracedecay/src/daemon/bootstrap.rs b/crates/tracedecay/src/daemon/bootstrap.rs index 0a5ba18860..32ffe864a6 100644 --- a/crates/tracedecay/src/daemon/bootstrap.rs +++ b/crates/tracedecay/src/daemon/bootstrap.rs @@ -15,6 +15,9 @@ use tracedecay_domain::errors::{Result, TraceDecayError}; use tracedecay_runtime_core::DAEMON_SHUTDOWN_DEADLINE; use super::*; +use tracedecay_daemon_service::shutdown::DAEMON_CLIENT_DRAIN_DEADLINE; +#[cfg(not(unix))] +use tracedecay_daemon_service::shutdown::{DaemonLifecycle, ShutdownStatus}; use tracedecay_runtime_core::logging::log_daemon_event; /// Slice of the shutdown budget reserved for writing the terminal shutdown @@ -69,9 +72,10 @@ async fn run_foreground_loopback( remote_tls: Option, ) -> Result<()> { let bootstrap_started = Instant::now(); - let profile_root = crate::config::user_data_dir().ok_or_else(|| TraceDecayError::Config { - message: "could not determine TraceDecay user data directory".to_string(), - })?; + let profile_root = + tracedecay_project::config::user_data_dir().ok_or_else(|| TraceDecayError::Config { + message: "could not determine TraceDecay user data directory".to_string(), + })?; let catalog_prewarm = tokio::task::spawn_blocking(prewarm_static_daemon_bootstrap_catalog); let requested = default_loopback_endpoint(); let _lifecycle_lease = hotpath::measure_block!("daemon.bootstrap.lifecycle_lease", { @@ -163,7 +167,7 @@ async fn run_foreground_loopback( ); } let lifecycle = DaemonLifecycle::default(); - let sync_config = tracedecay_configuration::SyncConfig::default().with_env_overrides(); + let sync_config = tracedecay_configuration::SyncConfig::default(); let profile_database = store_administration.registered_profile_database().await?; let maintenance = maintenance::MaintenanceCoordinator::spawn( profile_root.clone(), @@ -173,7 +177,6 @@ async fn run_foreground_loopback( sync_config.retention.clone(), maintenance::BranchStoreGcCadenceV1 { branch_gc_days: sync_config.branch_gc_days, - orphan_db_gc_days: sync_config.orphan_db_gc_days, }, ) .await; @@ -504,9 +507,10 @@ async fn run_foreground_unix( remote_tls: Option, ) -> Result<()> { let bootstrap_started = Instant::now(); - let profile_root = crate::config::user_data_dir().ok_or_else(|| TraceDecayError::Config { - message: "could not determine TraceDecay user data directory".to_string(), - })?; + let profile_root = + tracedecay_project::config::user_data_dir().ok_or_else(|| TraceDecayError::Config { + message: "could not determine TraceDecay user data directory".to_string(), + })?; let catalog_prewarm = tokio::task::spawn_blocking(prewarm_static_daemon_bootstrap_catalog); let endpoint = DaemonEndpoint::Unix(socket_path); let _lifecycle = hotpath::measure_block!("daemon.bootstrap.lifecycle_lease", { @@ -633,20 +637,19 @@ async fn run_foreground_unix( &[("endpoint", format!("https://{endpoint}/remote/"))], ); } - let sync_config = tracedecay_configuration::SyncConfig::default().with_env_overrides(); + let sync_config = tracedecay_configuration::SyncConfig::default(); let profile_database = engine .store_administration .registered_profile_database() .await?; let maintenance = maintenance::MaintenanceCoordinator::spawn( profile_root.clone(), - profile_database.clone(), + profile_database, engine.store_administration.clone(), engine.invocation.code_index_schedulers.clone(), sync_config.retention.clone(), maintenance::BranchStoreGcCadenceV1 { branch_gc_days: sync_config.branch_gc_days, - orphan_db_gc_days: sync_config.orphan_db_gc_days, }, ) .await; @@ -751,7 +754,7 @@ async fn run_foreground_unix( // each of them; awaiting its receipt keeps this fence active until those // owners have either joined or reported a typed timeout. let _codex_shutdown = - tracedecay_sessions::runtime::codex_app_server::begin_codex_app_server_shutdown(); + tracedecay_sessions::runtime::hosts::codex_app_server::begin_codex_app_server_shutdown(); log_daemon_event( "daemon_shutdown", &[("socket", socket_path.display().to_string())], diff --git a/crates/tracedecay/src/daemon/bootstrap_route.rs b/crates/tracedecay/src/daemon/bootstrap_route.rs index 8d46575e20..4812593f7a 100644 --- a/crates/tracedecay/src/daemon/bootstrap_route.rs +++ b/crates/tracedecay/src/daemon/bootstrap_route.rs @@ -113,6 +113,12 @@ pub(super) fn daemon_bootstrap_response( response })), McpMethod::InitializedAck => Some(None), + McpMethod::TrivialAck => Some( + request + .id + .clone() + .map(|id| JsonRpcResponse::success(id, json!({}))), + ), McpMethod::ToolsList => Some(request.id.clone().map(|id| { let budget = project_node_count.map_or_else(|| explore_call_budget(0), explore_call_budget); diff --git a/crates/tracedecay/src/daemon/branch_add.rs b/crates/tracedecay/src/daemon/branch_add.rs index 0b7b5f17e2..1b9c55a8a2 100644 --- a/crates/tracedecay/src/daemon/branch_add.rs +++ b/crates/tracedecay/src/daemon/branch_add.rs @@ -19,6 +19,7 @@ use tracedecay_runtime_core::cancellation::CancellationToken; use tracedecay_runtime_core::logging::log_daemon_event; use super::{DaemonHandshake, StoreAdministration}; +use tracedecay_session_temporal_store::SessionTemporalAccess; const BRANCH_ADD_TOOL_NAME: &str = "tracedecay_admin_branch_add"; const CODE_INDEX_SCHEDULER_UNAVAILABLE: &str = "code_index_scheduler_unavailable"; @@ -144,7 +145,7 @@ pub(super) async fn branch_add_response( async fn activate_and_track_manual_branch( administration: &StoreAdministration, project_root: &Path, - graph: &Arc, + graph: &Arc, schedulers: &CodeIndexSchedulerRegistryV1, branch: &str, ) -> Result { @@ -312,7 +313,10 @@ async fn mount_published_branch_query_authority( let Some(session_db) = sessions.mounted_project_sessions(&project_id).await else { return; }; - let cursor_keys = match session_db.load_session_cursor_key_provider_result().await { + let cursor_keys = match SessionTemporalAccess::new(&*session_db) + .load_session_cursor_key_provider_result() + .await + { Ok(cursor_keys) => cursor_keys, Err(error) => { tracing::debug!( @@ -348,7 +352,7 @@ async fn mount_published_branch_query_authority( #[hotpath::measure(label = "daemon.branch_add.owner", future = true)] pub(super) async fn activate_and_track_manual_branch_owned( project_root: std::path::PathBuf, - graph: Arc, + graph: Arc, schedulers: CodeIndexSchedulerRegistryV1, branch: String, data_root: std::path::PathBuf, @@ -432,7 +436,7 @@ pub(super) async fn activate_and_track_manual_branch_owned( } pub(crate) fn branch_publication_context( - graph: &crate::project::TraceDecay, + graph: &tracedecay_project::project::TraceDecay, ) -> Result { BranchPublicationContextV1::new( graph.store_layout().identity.project_id.as_deref(), @@ -442,7 +446,7 @@ pub(crate) fn branch_publication_context( } fn graph_matches_project( - graph: &crate::project::TraceDecay, + graph: &tracedecay_project::project::TraceDecay, canonical_root: &std::path::Path, ) -> bool { graph.project_root() == canonical_root diff --git a/crates/tracedecay/src/daemon/branch_admin.rs b/crates/tracedecay/src/daemon/branch_admin.rs index 6e178505c1..83befb9e6d 100644 --- a/crates/tracedecay/src/daemon/branch_admin.rs +++ b/crates/tracedecay/src/daemon/branch_admin.rs @@ -1,4 +1,4 @@ -use std::collections::{HashMap, HashSet}; +use std::collections::HashMap; use std::future::Future; use std::path::{Path, PathBuf}; use std::sync::{ @@ -28,6 +28,7 @@ use tracedecay_agent_hosts::native_integration::DaemonNativeIntegrationServiceRe #[cfg(unix)] use tracedecay_automation_runtime::automation::maintenance_termination::MaintenanceTaskTermination; use tracedecay_code_index_runtime::git_transactions::DaemonGitIndexTransactionServiceRegistry; +use tracedecay_contracts::catalog_composition::build_application_catalog_snapshot; use tracedecay_daemon_identity::{authority, profile_identity}; use tracedecay_daemon_service::{ ProfileHostAdmissionBootstrapOperation, ProfileHostAdmissionBootstrapStatus, @@ -90,7 +91,7 @@ pub(super) fn owner_writer_scope(key: &ProjectServerKey) -> WriterScope { /// [`store_writer_scope`] for the store an open graph is serving. pub(super) fn graph_writer_scope( - cg: &crate::project::TraceDecay, + cg: &tracedecay_project::project::TraceDecay, class: StoreWriterClass, ) -> WriterScope { store_writer_scope(&cg.store_layout().data_root, class) @@ -597,9 +598,7 @@ impl Default for StoreAdministration { SessionTemporalRefreshSchedulerRegistry::default(), ), git_index_transaction_services: Arc::new( - DaemonGitIndexTransactionServiceRegistry::new( - crate::runtime_ports::compose_application_catalog_snapshot, - ), + DaemonGitIndexTransactionServiceRegistry::new(build_application_catalog_snapshot), ), native_integration_services: Arc::new(DaemonNativeIntegrationServiceRegistry::default()), remote_recovery_project_lifecycles: Arc::default(), @@ -998,7 +997,9 @@ impl StoreAdministration { } #[hotpath::measure(label = "daemon.branch_admin.mounted_project_graphs", future = true)] - pub(super) async fn mounted_project_graphs(&self) -> Vec> { + pub(super) async fn mounted_project_graphs( + &self, + ) -> Vec> { let servers = self.mounted_project_servers().await; let mut graphs = Vec::with_capacity(servers.len()); for server in &servers { @@ -1348,6 +1349,12 @@ impl StoreAdministration { service } + /// Drops the cached profile refresh services so their profile session + /// leases no longer keep the store runtime open at terminal close. + pub(super) async fn release_profile_session_refresh_services(&self) { + self.profile_session_refresh_services.lock().await.clear(); + } + pub(super) fn git_index_transaction_services( &self, ) -> &Arc { @@ -1757,27 +1764,28 @@ impl StoreAdministration { // only durable authority; it never consults legacy config input and a // genuinely unresolvable store still fails before any destructive store // action. - let config = crate::config::resolve_runtime_configuration_for_registered_database( - project_root, - &layout, - configuration_database, - ) - .await? - .into_config() - .sync; + let config = + tracedecay_project::config::resolve_runtime_configuration_for_registered_database( + project_root, + &layout, + configuration_database, + ) + .await? + .config() + .sync + .clone(); self.execute_branch_admin_in_layout( schedulers, project_root, &layout.data_root, action, config.branch_gc_days, - config.orphan_db_gc_days, ) .await } - /// Prepares, proves, and commits one destructive branch-store mutation under - /// the physical runtime registry's exact path reservation. + /// Prepares one branch-tracking mutation, retires the exact artifacts of + /// sealed manual branches, then CAS-publishes the metadata. #[hotpath::measure(label = "daemon.branch_admin.execute", future = true)] pub(super) async fn execute_branch_admin_in_layout( &self, @@ -1786,14 +1794,12 @@ impl StoreAdministration { data_root: &Path, action: tracedecay_runtime_core::branch::BranchAdminAction, branch_gc_days: u64, - orphan_db_gc_days: u64, ) -> Result { let prepared = tracedecay_runtime_core::branch::prepare_branch_admin_mutation( project_root, data_root, action, branch_gc_days, - orphan_db_gc_days, )?; let retirements = prepared .single_store_retirements() @@ -1808,73 +1814,15 @@ impl StoreAdministration { .cloned() .collect::>(); let lifecycle_leases = acquire_manual_branch_retirement_leases(data_root, &retirements)?; - let database_paths = canonical_branch_database_paths(prepared.database_paths())?; - if database_paths.is_empty() { - let lifecycle_leases = cleanup_manual_branch_retirements( - project_root, - data_root, - schedulers, - &retirements, - lifecycle_leases, - ) - .await?; - let report = prepared.finish_without_database_deletion()?; - drop(lifecycle_leases); - return Ok(report); - } - - { - let project_servers = self.project_servers.lock().await; - let refresh_scheduler_busy = self - .session_temporal_refresh_schedulers - .owns_project_database_paths(&database_paths) - .await; - #[cfg(unix)] - let scheduler_busy = cached_scheduler_owns_selected( - &*self.automation_schedulers.lock().await, - &database_paths, - ) || refresh_scheduler_busy; - #[cfg(not(unix))] - let scheduler_busy = refresh_scheduler_busy; - ensure_no_cached_store_owners(&project_servers, scheduler_busy, &database_paths)?; - } - - let mut canonical_paths = database_paths.iter().cloned().collect::>(); - canonical_paths.sort(); - let reservation = self - .session_runtime_registry() - .await? - .begin_destructive_code_maintenance(data_root, canonical_paths.iter().cloned()) - .await?; - let lifecycle_leases = match cleanup_manual_branch_retirements( + let lifecycle_leases = cleanup_manual_branch_retirements( project_root, data_root, schedulers, &retirements, lifecycle_leases, ) - .await - { - Ok(lifecycle_leases) => lifecycle_leases, - Err(error) => { - reservation - .abort_preserved() - .map_err(destructive_reservation_error)?; - return Err(error); - } - }; - let report = match prepared.commit_destructive() { - Ok(report) => report, - Err(error) => { - reservation - .abort_preserved() - .map_err(destructive_reservation_error)?; - return Err(error); - } - }; - reservation - .finish_deleted() - .map_err(destructive_reservation_error)?; + .await?; + let report = prepared.commit()?; drop(lifecycle_leases); Ok(report) } @@ -1964,57 +1912,6 @@ pub(super) fn parse_branch_admin_request( }) } -fn canonical_branch_database_paths(paths: &[PathBuf]) -> Result> { - paths - .iter() - .map(|path| authority::canonical_identity_path(path)) - .collect() -} - -fn branch_administration_busy(detail: impl Into) -> TraceDecayError { - TraceDecayError::project_route("branch_administration_busy", true, detail) -} - -#[cfg(any(unix, test))] -fn cached_scheduler_owns_selected( - automation_schedulers: &HashMap, - database_paths: &HashSet, -) -> bool { - automation_schedulers - .keys() - .any(|key| database_paths.contains(&key.owner.graph_db_path)) -} - -fn ensure_no_cached_store_owners( - project_servers: &DatabaseOwnerRegistry, - scheduler_busy: bool, - database_paths: &HashSet, -) -> Result<()> { - let server_busy = project_servers - .servers - .keys() - .any(|key| database_paths.contains(&key.owner.graph_db_path)); - if !server_busy && !scheduler_busy { - return Ok(()); - } - - let cached_as = match (server_busy, scheduler_busy) { - (true, true) => "a project server and a background scheduler", - (true, false) => "a project server", - (false, true) => "a background scheduler", - (false, false) => return Ok(()), - }; - let mut paths = database_paths - .iter() - .map(|path| path.display().to_string()) - .collect::>(); - paths.sort(); - Err(branch_administration_busy(format!( - "branch store administration is busy: selected database(s) {} are still cached by the daemon as {cached_as}; restart the TraceDecay daemon before retrying", - paths.join(", ") - ))) -} - fn destructive_reservation_error( error: tracedecay_runtime_core::shard_runtime::registry::StoreRuntimeRegistryFailure, ) -> TraceDecayError { @@ -2070,7 +1967,7 @@ pub(super) async fn write_branch_admin_response( #[cfg(test)] #[allow(clippy::expect_used)] mod tests { - use super::super::{AuthenticatedFirstRequest, ProjectRouteKey, StoreOwnerKey}; + use super::super::AuthenticatedFirstRequest; use super::*; use std::time::Duration; use tracedecay_daemon_service::BootstrapCompletion; @@ -2094,35 +1991,6 @@ mod tests { (meta_path, bytes) } - #[test] - fn branch_administration_busy_is_retryable_and_typed() { - let error = branch_administration_busy("another daemon writer is active"); - - assert_eq!( - error.project_route_context(), - Some(( - "branch_administration_busy", - true, - "another daemon writer is active", - )) - ); - - let response = branch_admin_error_response(json!(7), &error); - let response_error = response - .error - .expect("branch busy response must be an error"); - assert_eq!(response_error.code, ErrorCode::InternalError.as_i32()); - assert_eq!( - response_error.data, - Some(json!({ - "tool": BRANCH_ADMIN_TOOL_NAME, - "reason_code": "branch_administration_busy", - "retryable": true, - "detail": "another daemon writer is active", - })) - ); - } - /// Rank 1 regression: a git-watch sync of project A used to hold the one /// daemon-wide gate across a full `cg.sync()`, so the first request for /// project B parked behind it for as long as the sync ran. @@ -2377,159 +2245,6 @@ mod tests { administration.shutdown_host_admission_replay().await; } - fn owner(graph_db_path: &str) -> StoreOwnerKey { - StoreOwnerKey { - profile_root: PathBuf::from("/profile"), - global_db_path: PathBuf::from("/profile/global.db"), - project_id: Some("project".to_string()), - store_root: PathBuf::from("/profile/projects/project"), - graph_db_path: PathBuf::from(graph_db_path), - } - } - - fn server_key(graph_db_path: &str, scope_prefix: Option<&str>) -> ProjectServerKey { - ProjectServerKey { - owner: owner(graph_db_path), - project_root: PathBuf::from("/project"), - scope_prefix: scope_prefix.map(str::to_string), - } - } - - fn route(project_path: &str, scope_prefix: Option<&str>) -> ProjectRouteKey { - ProjectRouteKey { - profile_root: PathBuf::from("/profile"), - global_db_path: PathBuf::from("/profile/global.db"), - project_path: PathBuf::from(project_path), - scope_prefix: scope_prefix.map(str::to_string), - } - } - - #[test] - fn matching_cached_server_and_scheduler_fail_busy_without_mutation() { - let target_a = server_key("/profile/projects/project/branches/feature.db", None); - let target_b = server_key("/profile/projects/project/branches/feature.db", Some("src")); - let survivor = server_key("/profile/projects/project/tracedecay.db", None); - let target_route_a = route("/repo", None); - let target_route_b = route("/repo", Some("src")); - let survivor_route = route("/repo-main", None); - let target_server_a = Arc::new("target-a"); - let target_server_b = Arc::new("target-b"); - let survivor_server = Arc::new("survivor"); - let mut registry = DatabaseOwnerRegistry::default(); - registry.insert_route( - target_route_a.clone(), - target_a.clone(), - Arc::clone(&target_server_a), - ); - registry.insert_route( - target_route_b.clone(), - target_b.clone(), - Arc::clone(&target_server_b), - ); - registry.insert_route( - survivor_route.clone(), - survivor.clone(), - Arc::clone(&survivor_server), - ); - let scheduler = Arc::new("scheduler"); - let mut schedulers = HashMap::from([(target_b.clone(), Arc::clone(&scheduler))]); - let selected = HashSet::from([PathBuf::from( - "/profile/projects/project/branches/feature.db", - )]); - - let error = ensure_no_cached_store_owners( - ®istry, - cached_scheduler_owns_selected(&schedulers, &selected), - &selected, - ) - .expect_err("matching daemon owners must fail closed"); - - let message = error.to_string(); - assert!(message.contains("busy"), "{message}"); - assert!( - message.contains("restart the TraceDecay daemon"), - "{message}" - ); - ensure_no_cached_store_owners(®istry, false, &selected) - .expect_err("a matching project server alone must fail closed"); - let no_servers: DatabaseOwnerRegistry> = DatabaseOwnerRegistry::default(); - ensure_no_cached_store_owners( - &no_servers, - cached_scheduler_owns_selected(&schedulers, &selected), - &selected, - ) - .expect_err("a matching scheduler alone must fail closed"); - assert!(Arc::ptr_eq( - registry - .get_route(&target_route_a) - .expect("target a route") - .1, - &target_server_a - )); - assert!(Arc::ptr_eq( - registry - .get_route(&target_route_b) - .expect("target b route") - .1, - &target_server_b - )); - assert!(Arc::ptr_eq( - registry - .get_route(&survivor_route) - .expect("survivor route") - .1, - &survivor_server - )); - assert!(Arc::ptr_eq( - schedulers.get(&target_b).expect("scheduler entry"), - &scheduler - )); - assert_eq!(registry.servers.len(), 3); - assert_eq!(registry.aliases.len(), 3); - assert_eq!(schedulers.len(), 1); - - // Keep the maps mutable in this regression test so accidental eviction - // implementations cannot hide behind immutable test fixtures. - assert!(schedulers.remove(&survivor).is_none()); - } - - #[test] - fn unmatched_cached_owners_allow_administration_to_continue() { - let survivor = server_key("/profile/projects/project/tracedecay.db", None); - let survivor_route = route("/repo-main", None); - let survivor_server = Arc::new("survivor"); - let mut registry = DatabaseOwnerRegistry::default(); - registry.insert_route( - survivor_route.clone(), - survivor.clone(), - Arc::clone(&survivor_server), - ); - let scheduler = Arc::new("scheduler"); - let schedulers = HashMap::from([(survivor.clone(), Arc::clone(&scheduler))]); - let selected = HashSet::from([PathBuf::from( - "/profile/projects/project/branches/feature.db", - )]); - - ensure_no_cached_store_owners( - ®istry, - cached_scheduler_owns_selected(&schedulers, &selected), - &selected, - ) - .expect("unmatched owners must proceed to holder proof and commit"); - - assert!(Arc::ptr_eq( - registry - .get_route(&survivor_route) - .expect("survivor route") - .1, - &survivor_server - )); - assert!(Arc::ptr_eq( - schedulers.get(&survivor).expect("scheduler entry"), - &scheduler - )); - } - #[test] fn branch_admin_parser_accepts_only_the_hidden_destructive_tool() { let request = parsed_branch_admin_request( diff --git a/crates/tracedecay/src/daemon/branch_admin/project_retirement.rs b/crates/tracedecay/src/daemon/branch_admin/project_retirement.rs index 46fd8d9c5b..f584386e97 100644 --- a/crates/tracedecay/src/daemon/branch_admin/project_retirement.rs +++ b/crates/tracedecay/src/daemon/branch_admin/project_retirement.rs @@ -505,24 +505,25 @@ mod tests { project_root: &std::path::Path, project_id: &str, ) -> ( - crate::project::TraceDecay, - crate::test_support::host_admission::HostAdmissionTestRuntimeV1, + tracedecay_project::project::TraceDecay, + tracedecay_project::test_support::host_admission::HostAdmissionTestRuntimeV1, ) { std::fs::create_dir_all(profile_root).expect("isolated profile root"); std::fs::create_dir_all(project_root).expect("isolated project root"); let project_id = tracedecay_domain::ProjectId::new(project_id.to_owned()) .expect("typed project identity"); - let runtime = crate::test_support::host_admission::HostAdmissionTestRuntimeV1::project( - profile_root, - project_root, - project_id, - ) - .await - .expect("isolated host-admission runtime"); + let runtime = + tracedecay_project::test_support::host_admission::HostAdmissionTestRuntimeV1::project( + profile_root, + project_root, + project_id, + ) + .await + .expect("isolated host-admission runtime"); let graph = runtime .initialize_project_graph_for_test( project_root, - crate::project::TraceDecayOpenOptions { + tracedecay_project::project::TraceDecayOpenOptions { profile_root: Some(profile_root.to_path_buf()), global_db_path: None, }, @@ -533,13 +534,13 @@ mod tests { } async fn isolated_sibling_graph( - runtime: &crate::test_support::host_admission::HostAdmissionTestRuntimeV1, + runtime: &tracedecay_project::test_support::host_admission::HostAdmissionTestRuntimeV1, profile_root: &std::path::Path, project_root: &std::path::Path, project_id: &str, ) -> ( - crate::project::TraceDecay, - crate::test_support::host_admission::HostAdmissionTestRuntimeV1, + tracedecay_project::project::TraceDecay, + tracedecay_project::test_support::host_admission::HostAdmissionTestRuntimeV1, ) { std::fs::create_dir_all(project_root).expect("isolated sibling project root"); let project_id = tracedecay_domain::ProjectId::new(project_id.to_owned()) @@ -551,7 +552,7 @@ mod tests { let graph = sibling .initialize_project_graph_for_test( project_root, - crate::project::TraceDecayOpenOptions { + tracedecay_project::project::TraceDecayOpenOptions { profile_root: Some(profile_root.to_path_buf()), global_db_path: None, }, diff --git a/crates/tracedecay/src/daemon/branch_admin/remote_deletion_lifecycle.rs b/crates/tracedecay/src/daemon/branch_admin/remote_deletion_lifecycle.rs index 9874733aea..aeccadcd72 100644 --- a/crates/tracedecay/src/daemon/branch_admin/remote_deletion_lifecycle.rs +++ b/crates/tracedecay/src/daemon/branch_admin/remote_deletion_lifecycle.rs @@ -14,6 +14,7 @@ use super::super::remote_deletion::{ RemoteDeletionReceipt, RemoteDeletionReceiptTarget, }; use super::{StoreAdministration, destructive_reservation_error}; +use tracedecay_daemon_service::shutdown::DAEMON_TASK_ABORT_DEADLINE; struct RemoteDeletionCleanupError { code: RemoteDeletionFailureCode, @@ -399,7 +400,7 @@ impl StoreAdministration { if !open_tasks .shutdown_profile_with_deadline( &profile_root, - super::super::DAEMON_TASK_ABORT_DEADLINE, + DAEMON_TASK_ABORT_DEADLINE, ) .await { @@ -442,7 +443,7 @@ impl StoreAdministration { self.host_admission_brokers.lock().await.clear(); #[cfg(unix)] if !self - .settle_retirement_reapers(super::super::DAEMON_TASK_ABORT_DEADLINE) + .settle_retirement_reapers(DAEMON_TASK_ABORT_DEADLINE) .await { let cleanup = tracedecay_global_db::RemoteDeletionCleanupState::Settling { @@ -755,7 +756,7 @@ impl StoreAdministration { .settle_retirement_reapers_for_project( profile_root, project_id, - super::super::DAEMON_TASK_ABORT_DEADLINE, + DAEMON_TASK_ABORT_DEADLINE, ) .await { @@ -783,7 +784,7 @@ impl StoreAdministration { })?; super::retire_registered_context_scout_owner( &typed_project_id, - &data_root.join(crate::config::db_filename(&data_root)), + &data_root.join(tracedecay_project::config::db_filename(&data_root)), ); self.git_index_transaction_services .retire_project_database(&typed_project_id, &project_sessions_path) @@ -932,7 +933,7 @@ impl StoreAdministration { )); } let database_paths = [ - data_root.join(crate::config::db_filename(&data_root)), + data_root.join(tracedecay_project::config::db_filename(&data_root)), project_sessions_path.clone(), ] .into_iter() diff --git a/crates/tracedecay/src/daemon/branch_admin/remote_recovery_lifecycle.rs b/crates/tracedecay/src/daemon/branch_admin/remote_recovery_lifecycle.rs index 2eb6a8f10a..6a3b2e6bca 100644 --- a/crates/tracedecay/src/daemon/branch_admin/remote_recovery_lifecycle.rs +++ b/crates/tracedecay/src/daemon/branch_admin/remote_recovery_lifecycle.rs @@ -16,6 +16,7 @@ use super::{ }; use tracedecay_agent_hosts::native_integration::DaemonNativeIntegrationServiceRegistry; use tracedecay_daemon_identity::authority; +use tracedecay_daemon_service::shutdown::DAEMON_TASK_ABORT_DEADLINE; use tracedecay_domain::errors::{Result, TraceDecayError}; use tracedecay_maintenance::telemetry::StoreTelemetrySamplingRegistry; use tracedecay_store_runtime::WriterAdmissionGuard; @@ -263,7 +264,7 @@ impl RemoteRecoveryProjectLifecycleV1 { .await?; super::retire_registered_context_scout_owner( project_id, - &data_root.join(crate::config::db_filename(data_root)), + &data_root.join(tracedecay_project::config::db_filename(data_root)), ); self.git_index_transaction_services .retire_project_database(project_id, database.db_path()) @@ -335,7 +336,7 @@ impl RemoteRecoveryProjectLifecycleV1 { #[hotpath::skip] async fn settle_retained_runtime_retirement(&self, project_id: &ProjectId) -> Result<()> { - let deadline = tokio::time::Instant::now() + super::super::DAEMON_TASK_ABORT_DEADLINE; + let deadline = tokio::time::Instant::now() + DAEMON_TASK_ABORT_DEADLINE; let receipt = super::project_retirement::settle_project_retirements( &self.project_server_retirements, &self.profile_root, @@ -509,7 +510,7 @@ pub(super) async fn retire_runtime_work( ) .await; } - let deadline = tokio::time::Instant::now() + super::super::DAEMON_TASK_ABORT_DEADLINE; + let deadline = tokio::time::Instant::now() + DAEMON_TASK_ABORT_DEADLINE; let receipt = super::project_retirement::settle_project_retirements( tracked_retirements, profile_root, diff --git a/crates/tracedecay/src/daemon/broker_stream_transport_tests.rs b/crates/tracedecay/src/daemon/broker_stream_transport_tests.rs index 7d31cc1c80..11f55d5885 100644 --- a/crates/tracedecay/src/daemon/broker_stream_transport_tests.rs +++ b/crates/tracedecay/src/daemon/broker_stream_transport_tests.rs @@ -110,12 +110,31 @@ async fn settled_fanout( fanout } +/// A one-shot client half-closes while its accepted request is still owed a +/// response: the connection stays open for that response until the peer +/// fully closes. #[tokio::test] async fn rmcp_receive_waits_for_full_close_after_request_half_close() { let (server, client) = tokio::net::UnixStream::pair().expect("UnixStream pair"); let mut transport = BrokerStreamTransport::new(BrokerStream::Unix(server)); let (client_reader, mut client_writer) = client.into_split(); + client_writer + .write_all( + br#"{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"tracedecay_files","arguments":{}}} +"#, + ) + .await + .expect("request"); + client_writer.flush().await.expect("flush request"); + assert!( + >::receive( + &mut transport + ) + .await + .is_some(), + "the transport must accept the request" + ); client_writer .shutdown() .await @@ -124,10 +143,10 @@ async fn rmcp_receive_waits_for_full_close_after_request_half_close() { rmcp::RoleServer, >>::receive(&mut transport)); assert!( - tokio::time::timeout(std::time::Duration::from_millis(50), &mut receive) + tokio::time::timeout(std::time::Duration::from_millis(250), &mut receive) .await .is_err(), - "rmcp receive must not treat a request-half close as full peer loss" + "rmcp receive must not treat a request-half close as full peer loss while a response is owed" ); drop(client_writer); @@ -197,6 +216,47 @@ async fn rmcp_receive_closes_after_half_close_once_accepted_requests_settle() { drop(client_reader); } +/// A connection that carried only a notification owes nothing, so a +/// half-close ends it without waiting for the peer to drop its read half. +#[tokio::test] +async fn rmcp_receive_closes_a_request_less_connection_on_half_close() { + let (server, client) = tokio::net::UnixStream::pair().expect("UnixStream pair"); + let mut transport = BrokerStreamTransport::new(BrokerStream::Unix(server)); + let (client_reader, mut client_writer) = client.into_split(); + + client_writer + .write_all( + br#"{"jsonrpc":"2.0","method":"notifications/initialized"} +"#, + ) + .await + .expect("notification"); + client_writer + .shutdown() + .await + .expect("half-close client request side"); + assert!( + >::receive( + &mut transport + ) + .await + .is_some(), + "the transport must forward the notification" + ); + assert!( + tokio::time::timeout( + std::time::Duration::from_secs(2), + >::receive( + &mut transport, + ), + ) + .await + .expect("a request-less connection must close after request half-close") + .is_none() + ); + drop(client_reader); +} + #[tokio::test] async fn rmcp_selected_target_retirement_between_handler_and_send_suppresses_response() { let (server, client) = tokio::net::UnixStream::pair().expect("UnixStream pair"); diff --git a/crates/tracedecay/src/daemon/code_index_runtime_generation_census_tests.rs b/crates/tracedecay/src/daemon/code_index_runtime_generation_census_tests.rs index 89293482da..4e0ba4a320 100644 --- a/crates/tracedecay/src/daemon/code_index_runtime_generation_census_tests.rs +++ b/crates/tracedecay/src/daemon/code_index_runtime_generation_census_tests.rs @@ -10,10 +10,10 @@ use tempfile::TempDir; use crate::mcp::tools::handlers::{ ToolCallRegistryOptions, handle_tool_call_with_registry_options, }; -use crate::project::TraceDecay; use tracedecay_code_index_runtime::code_index_scheduler::CodeIndexSchedulerRegistryV1; use tracedecay_code_index_runtime::project_reads::project_code_index_generation_census_reader; use tracedecay_code_index_runtime::resolved_scope_for_project; +use tracedecay_project::project::TraceDecay; use tracedecay_runtime_core::config::PinnedUserDataDir; use tracedecay_runtime_core::path_safety::canonical_existing_identity; use tracedecay_runtime_core::runtime_telemetry::{ diff --git a/crates/tracedecay/src/daemon/code_index_runtime_graph_activation_tests.rs b/crates/tracedecay/src/daemon/code_index_runtime_graph_activation_tests.rs index 1e7dc739dc..117b2ec398 100644 --- a/crates/tracedecay/src/daemon/code_index_runtime_graph_activation_tests.rs +++ b/crates/tracedecay/src/daemon/code_index_runtime_graph_activation_tests.rs @@ -30,6 +30,7 @@ use tracedecay_graph_query::{ use tracedecay_runtime_core::runtime_telemetry::{ GenerationCensusServingFreshness, GenerationCensusSnapshot, GenerationCensusUnavailableReason, }; +use tracedecay_session_temporal_store::SessionTemporalAccess; use tracedecay_store_runtime::DaemonSessionRuntimeRegistryV1; use tracedecay_tool_catalog::{CapabilityId, UseCaseId}; @@ -363,7 +364,7 @@ async fn persistent_graph_activation_publishes_a_small_generation() { // Activation issues verified graph reads; the project graph runtime binds // asynchronously after `project_memory` returns, so an unawaited bind // races activation into "not ready for verified reads". - crate::test_support::host_admission::await_bound_graph_runtime( + tracedecay_project::test_support::host_admission::await_bound_graph_runtime( &project_database, "bind small persistent activation graph runtime", ) @@ -498,7 +499,7 @@ async fn persistent_callers_cursor_keeps_generation_a_without_repointing_generat .project_memory(project_id.clone(), [fixture.path().to_path_buf()]) .await .expect("project database"); - crate::test_support::host_admission::await_bound_graph_runtime( + tracedecay_project::test_support::host_admission::await_bound_graph_runtime( &project_database, "bind persistent historical cursor graph runtime", ) @@ -552,7 +553,7 @@ async fn persistent_callers_cursor_keeps_generation_a_without_repointing_generat .profile_sessions() .await .expect("profile session database"); - let cursor_keys = sessions + let cursor_keys = SessionTemporalAccess::new(&*sessions) .load_session_cursor_key_provider_result() .await .expect("cursor keys"); @@ -797,7 +798,7 @@ async fn restart_status_case(corrupt_graph: bool, dirty_before_restart: bool) { .project_memory(project_id.clone(), [fixture.path().to_path_buf()]) .await .expect("writable project database"); - crate::test_support::host_admission::await_bound_graph_runtime( + tracedecay_project::test_support::host_admission::await_bound_graph_runtime( &project_database, "bind stale graph status projection", ) @@ -874,7 +875,7 @@ async fn restart_status_case(corrupt_graph: bool, dirty_before_restart: bool) { .project_memory(project_id.clone(), [fixture.path().to_path_buf()]) .await .expect("restarted writable project database"); - crate::test_support::host_admission::await_bound_graph_runtime( + tracedecay_project::test_support::host_admission::await_bound_graph_runtime( &project_database, "bind restarted graph status projection", ) @@ -1307,7 +1308,7 @@ async fn restart_seats_the_retained_graph_while_its_text_owner_still_projects() .project_memory(project_id.clone(), [fixture.path().to_path_buf()]) .await .expect("writable project database"); - crate::test_support::host_admission::await_bound_graph_runtime( + tracedecay_project::test_support::host_admission::await_bound_graph_runtime( &project_database, "bind graph projection before restart", ) @@ -1355,7 +1356,7 @@ async fn restart_seats_the_retained_graph_while_its_text_owner_still_projects() .project_memory(project_id.clone(), [fixture.path().to_path_buf()]) .await .expect("restarted writable project database"); - crate::test_support::host_admission::await_bound_graph_runtime( + tracedecay_project::test_support::host_admission::await_bound_graph_runtime( &project_database, "bind restarted graph projection", ) diff --git a/crates/tracedecay/src/daemon/connection_serving.rs b/crates/tracedecay/src/daemon/connection_serving.rs index 18ad9346c3..9b4c747bc0 100644 --- a/crates/tracedecay/src/daemon/connection_serving.rs +++ b/crates/tracedecay/src/daemon/connection_serving.rs @@ -7,10 +7,11 @@ use super::*; use tracedecay_daemon_protocol::DaemonInvocationPayload; use tracedecay_daemon_service::ProfileHostAdmissionBootstrapStatus; +use tracedecay_daemon_service::shutdown::DaemonLifecycle; use tracedecay_daemon_service::{DaemonInvocationService, DaemonLspSessionAccess, Lease}; use tracedecay_mcp::BrokerSelectedResponseLease; +use tracedecay_runtime_core::cancellation::CancellationToken; use tracedecay_runtime_core::logging::log_daemon_event; -use tracedecay_session_memory::context::CancellationToken; /// Hermetic production-route benchmark support for the typed RMCP transport. /// @@ -85,20 +86,6 @@ fn report_profile_host_admission_bootstrap_status( } } -#[cfg(all(unix, test))] -pub(super) async fn serve_socket_client( - stream: tokio::net::UnixStream, - engine: DaemonEngine, -) -> Result<()> { - Box::pin(serve_broker_socket_client( - BrokerStream::Unix(stream), - engine, - None, - DaemonClientAdmissionClass::General, - )) - .await -} - #[cfg(unix)] pub(super) async fn serve_authenticated_socket_client_with_class( stream: BrokerStream, @@ -109,14 +96,14 @@ pub(super) async fn serve_authenticated_socket_client_with_class( Box::pin(serve_broker_socket_client( stream, engine, - Some(auth_token), + auth_token, admission_class, )) .await } #[hotpath::measure(label = "daemon.engine.transport.rmcp", future = true)] -pub(super) async fn serve_routed_rmcp_connection( +pub(crate) async fn serve_routed_rmcp_connection( server: Arc, transport: BrokerStreamTransport, first_request_line: String, @@ -170,12 +157,17 @@ fn serve_routed_rmcp_connection_inner( let transport = transport .with_rmcp_selected_project_responses(adapter.selected_project_responses()) .with_rmcp_work_delivery_settlement(adapter.work_delivery_settlement()); - let running = adapter - .serve(transport) - .await - .map_err(|error| TraceDecayError::Config { - message: format!("rmcp server initialization failed: {error}"), - })?; + let running = match adapter.serve(transport).await { + Ok(running) => running, + // The client left before a request settled the handshake; every + // frame it sent was already answered or refused on the wire. + Err(rmcp::service::ServerInitializeError::ConnectionClosed(_)) => return Ok(()), + Err(error) => { + return Err(TraceDecayError::Config { + message: format!("rmcp server initialization failed: {error}"), + }); + } + }; let cancellation = running.cancellation_token(); let waiting = running.waiting(); tokio::pin!(waiting); @@ -193,8 +185,32 @@ fn serve_routed_rmcp_connection_inner( }) } -fn is_mcp_initialize_request(request: Option<&JsonRpcRequest>) -> bool { - request.is_some_and(|request| request.method == "initialize") +fn opens_rmcp_session(request: Option<&JsonRpcRequest>) -> bool { + request.is_some_and(tracedecay_mcp::server::opens_rmcp_session) +} + +/// Answers a project-routed first request that neither initializes an MCP +/// session nor carries SEP-2575 per-request context. A notification gets no +/// frame; an unparseable line is answered with the null id. +async fn refuse_sessionless_request( + transport: &mut (impl McpTransport + Send), + request: &AuthenticatedFirstRequest, +) -> Result<()> { + if request.parsed().is_some_and(|request| request.id.is_none()) { + return Ok(()); + } + let request_id = request + .parsed() + .and_then(|request| request.id.clone()) + .unwrap_or(serde_json::Value::Null); + let response = JsonRpcResponse::error( + request_id, + ErrorCode::InvalidRequest, + "a daemon MCP connection must begin with initialize or carry SEP-2575 request _meta \ + (protocolVersion and clientCapabilities)" + .to_owned(), + ); + write_json_rpc_response(transport, &response).await } /// Answer an unparseable handshake with one typed refusal frame and drain @@ -787,7 +803,7 @@ where async fn serve_broker_socket_client( stream: BrokerStream, engine: DaemonEngine, - auth_token: Option, + auth_token: String, admission_class: DaemonClientAdmissionClass, ) -> Result<()> { serve_broker_socket_client_inner(stream, engine, auth_token, admission_class).await @@ -1036,7 +1052,7 @@ async fn serve_retained_invocation_connection( fn serve_broker_socket_client_inner( stream: BrokerStream, engine: DaemonEngine, - auth_token: Option, + auth_token: String, admission_class: DaemonClientAdmissionClass, ) -> std::pin::Pin> + Send + 'static>> { // Erase the deeply nested broker connection future before it reaches the @@ -1051,20 +1067,18 @@ fn serve_broker_socket_client_inner( _per_client_permit, )) = boxed_broker_connection_phase(async move { let mut transport = BrokerStreamTransport::new(stream); - if let Some(expected_token) = auth_token.as_deref() { - let preface_line = tokio::select! { - result = read_line_handling_wire_oversized(&mut transport) => result?, - () = engine.lifecycle.wait_for_draining() => return Ok(None), - }; - let Some(preface_line) = preface_line else { - return Ok(None); - }; - let authenticated = DaemonAuthPreface::from_line(&preface_line) - .is_ok_and(|preface| preface.authenticate(expected_token)); - if !authenticated { - refuse_unauthenticated_client(&mut transport, binary_version()?).await; - return Ok(None); - } + let preface_line = tokio::select! { + result = read_line_handling_wire_oversized(&mut transport) => result?, + () = engine.lifecycle.wait_for_draining() => return Ok(None), + }; + let Some(preface_line) = preface_line else { + return Ok(None); + }; + let authenticated = DaemonAuthPreface::from_line(&preface_line) + .is_ok_and(|preface| preface.authenticate(&auth_token)); + if !authenticated { + refuse_unauthenticated_client(&mut transport, binary_version()?).await; + return Ok(None); } let line = tokio::select! { result = read_line_handling_wire_oversized(&mut transport) => result?, @@ -1474,7 +1488,7 @@ fn serve_broker_socket_client_inner( return Err(error); } if let Some(server) = server { - if is_mcp_initialize_request(first_request.parsed()) { + if opens_rmcp_session(first_request.parsed()) { #[cfg(test)] tests::record_mcp_route( &handshake.client_instance_id, @@ -1496,27 +1510,7 @@ fn serve_broker_socket_client_inner( )) .await?; } else { - #[cfg(test)] - tests::record_mcp_route( - &handshake.client_instance_id, - tests::ObservedMcpRoute::Legacy, - ); - #[cfg(test)] - tests::record_first_request_replay( - &handshake.client_instance_id, - first_request.raw(), - ); - let mut transport = ReplayTransport::new(transport); - transport.push_replay(first_request.into_raw())?; - for line in pending_project_open_lines { - transport.push_replay(line)?; - } - Box::pin(server.run_daemon_connection_with_timings( - &mut transport, - handshake.timings, - &engine.lifecycle, - )) - .await?; + refuse_sessionless_request(&mut transport, &first_request).await?; } } else { let mut transport = ReplayTransport::new(transport); @@ -1958,7 +1952,7 @@ pub(super) async fn serve_windows_broker_client_with_class_and_invocation( }; drop(setup_activity); let (server, pending_lines) = server; - if is_mcp_initialize_request(first_request.parsed()) { + if opens_rmcp_session(first_request.parsed()) { #[cfg(test)] tests::record_mcp_route(&handshake.client_instance_id, tests::ObservedMcpRoute::Rmcp); #[cfg(test)] @@ -1974,24 +1968,7 @@ pub(super) async fn serve_windows_broker_client_with_class_and_invocation( )) .await?; } else { - #[cfg(test)] - tests::record_mcp_route( - &handshake.client_instance_id, - tests::ObservedMcpRoute::Legacy, - ); - #[cfg(test)] - tests::record_first_request_replay(&handshake.client_instance_id, first_request.raw()); - let mut transport = ReplayTransport::new(transport); - transport.push_replay(first_request.into_raw())?; - for line in pending_lines { - transport.push_replay(line)?; - } - Box::pin(server.run_daemon_connection_with_timings( - &mut transport, - handshake.timings, - lifecycle, - )) - .await?; + refuse_sessionless_request(&mut transport, &first_request).await?; } } else { drop(setup_activity); diff --git a/crates/tracedecay/src/daemon/core_client.rs b/crates/tracedecay/src/daemon/core_client.rs index df2d0052ca..e62a7e6084 100644 --- a/crates/tracedecay/src/daemon/core_client.rs +++ b/crates/tracedecay/src/daemon/core_client.rs @@ -12,7 +12,9 @@ use tokio::time::{Duration, Instant, timeout}; use tracedecay_daemon_control::default_socket_path; #[cfg(not(unix))] use tracedecay_daemon_identity::current_daemon_connection; -use tracedecay_daemon_identity::{ResolvedDaemonConnection, client_connection}; +use tracedecay_daemon_identity::{ + DAEMON_AUTHORITY_UNAVAILABLE, ResolvedDaemonConnection, client_connection, +}; pub(crate) use tracedecay_daemon_protocol::DAEMON_TOOL_LIVENESS_POLL_INTERVAL; pub(crate) use tracedecay_daemon_protocol::connection::{ DAEMON_RESTART_GRACE, DAEMON_RESTART_POLL_INTERVAL, daemon_connect_failure_advice, @@ -20,14 +22,15 @@ pub(crate) use tracedecay_daemon_protocol::connection::{ }; pub use tracedecay_daemon_protocol::daemon_tool_response_bound; use tracedecay_daemon_protocol::tool_request_deadline; +use tracedecay_mcp::server::attach_stateless_request_context; -#[cfg(unix)] -use super::unavailable_error; use super::{ BrokerStream, DaemonClientDeadline, DaemonHandshake, JsonRpcError, JsonRpcRequest, JsonRpcResponse, PROJECT_OPEN_RETRY_GRACE, PROJECT_OPEN_RETRY_INTERVAL, Result, TraceDecayError, error_is_project_open_retryable, tool_call_transport_error_is_retryable, }; +#[cfg(unix)] +use tracedecay_daemon_service::logging::unavailable_error; /// The caller's request deadline as an absolute wall-clock instant, for the /// wire. @@ -75,20 +78,20 @@ pub(crate) async fn ensure_daemon_connection_live( timeout( DAEMON_TOOL_HEALTH_CONNECT_TIMEOUT, - BrokerStream::connect(&connection.endpoint), + BrokerStream::connect(connection.endpoint()), ) .await .map_err(|_| TraceDecayError::Config { message: format!( "daemon health check timed out at '{}' while request '{request_label}' was awaiting a response; the request was already sent and was not retried", - connection.endpoint + connection.endpoint() ), })? .map(|_| ()) .map_err(|error| TraceDecayError::Config { message: format!( "daemon became unreachable at '{}' while request '{request_label}' was awaiting a response: {error}; the request was already sent and was not retried", - connection.endpoint + connection.endpoint() ), }) } @@ -119,7 +122,7 @@ pub(crate) async fn write_daemon_preamble( ) -> Result<()> { tracedecay_daemon_protocol::write_daemon_handshake_preamble( writer, - connection.auth_token.as_deref(), + connection.auth_token(), handshake, ) .await @@ -164,18 +167,22 @@ pub(crate) async fn connect_to_current_daemon_within( /// duplicated. Non-transient errors (e.g. permission denied) fail immediately. #[cfg(unix)] pub(crate) async fn connect_with_restart_grace( - connection: &ResolvedDaemonConnection, + socket_path: &Path, grace: Duration, poll_interval: Duration, ) -> Result { - let (_, stream) = - connect_with_restart_grace_resolving(|| Ok(connection.clone()), grace, poll_interval) - .await?; + let (_, stream) = connect_with_restart_grace_resolving( + || client_connection(socket_path), + grace, + poll_interval, + ) + .await?; Ok(stream) } /// Resolves endpoint authority on every retry because a daemon restart rotates -/// both its authority epoch and authentication token. +/// both its authority epoch and authentication token, and a daemon's first +/// start writes its record only moments before it binds. #[hotpath::measure(label = "daemon.core.connect_restart_grace", future = true)] async fn connect_with_restart_grace_resolving( mut resolve: impl FnMut() -> Result, @@ -184,21 +191,28 @@ async fn connect_with_restart_grace_resolving( ) -> Result<(ResolvedDaemonConnection, BrokerStream)> { let deadline = Instant::now() + grace; loop { - let connection = resolve()?; - match BrokerStream::connect(&connection.endpoint).await { + let connection = match resolve() { + Ok(connection) => connection, + Err(error) if authority_absent(&error) && Instant::now() < deadline => { + tokio::time::sleep(poll_interval).await; + continue; + } + Err(error) => return Err(error), + }; + match BrokerStream::connect(connection.endpoint()).await { Ok(stream) => return Ok((connection, stream)), Err(TraceDecayError::Io(err)) => { if !is_transient_daemon_connect_error(err.kind()) || Instant::now() >= deadline { return Err(if is_transient_daemon_connect_error(err.kind()) { tracedecay_daemon_protocol::daemon_connect_failure( - &connection.endpoint, + connection.endpoint(), &err, ) } else { TraceDecayError::Config { message: format!( "could not connect to TraceDecay daemon endpoint '{}': {err}. {}", - connection.endpoint, + connection.endpoint(), daemon_connect_failure_advice(err.kind()) ), } @@ -211,6 +225,12 @@ async fn connect_with_restart_grace_resolving( } } +fn authority_absent(error: &TraceDecayError) -> bool { + error + .project_route_context() + .is_some_and(|(code, retryable, _)| code == DAEMON_AUTHORITY_UNAVAILABLE && retryable) +} + #[hotpath::measure(label = "daemon.core.call_tool", future = true)] #[cfg_attr( not(feature = "hotpath"), @@ -261,12 +281,13 @@ pub(crate) async fn call_tool_with_liveness_poll( tracedecay_mcp::tool_call_deadline_meta(wire_request_deadline_micros(deadline)), ); } - let request = JsonRpcRequest { + let mut request = JsonRpcRequest { jsonrpc: "2.0".to_string(), id: Some(id.clone()), method: "tools/call".to_string(), params: Some(params), }; + attach_stateless_request_context(&mut request); let write = async { write_daemon_preamble(&mut writer, &connection, handshake).await?; diff --git a/crates/tracedecay/src/daemon/core_doctor.rs b/crates/tracedecay/src/daemon/core_doctor.rs index a31ff501eb..d4611fb661 100644 --- a/crates/tracedecay/src/daemon/core_doctor.rs +++ b/crates/tracedecay/src/daemon/core_doctor.rs @@ -10,6 +10,7 @@ use tracedecay_contracts::project_open::{ProjectOpenStatusStateV1, ProjectOpenSt use tracedecay_daemon_service::shutdown::DaemonActivity; use tracedecay_domain::errors::Result; use tracedecay_mcp::{JsonRpcRequest, JsonRpcResponse, McpTransport}; +use tracedecay_session_temporal_store::SessionTemporalAccess; #[path = "core_doctor_schema.rs"] mod schema; @@ -470,8 +471,12 @@ async fn doctor_runtime_value_inner( }); if let Some(db) = session_db.as_ref() { let health_budget = Duration::from_secs(8); + let session_temporal = SessionTemporalAccess::new(&**db); let (temporal, cursor_ingest, placeholder_paths) = tokio::join!( - Box::pin(timeout(health_budget, db.session_temporal_doctor_health())), + Box::pin(timeout( + health_budget, + session_temporal.session_temporal_doctor_health() + )), Box::pin(timeout(health_budget, db.cursor_session_ingest_health())), Box::pin(timeout( health_budget, @@ -523,7 +528,8 @@ async fn doctor_runtime_value_inner( pub(crate) async fn cold_doctor_runtime_value(handshake: &DaemonHandshake) -> serde_json::Value { // Owned stores are never path-opened as a fallback. Without the daemon's // retained runtime authority Doctor reports explicit unavailability. - let build_version = crate::product_runtime::register_fixture_product_runtime().build_version(); + let build_version = + tracedecay_project::product_runtime::register_fixture_product_runtime().build_version(); doctor_runtime_value_inner(handshake, None, false, build_version).await } @@ -536,7 +542,7 @@ pub(in crate::daemon) async fn write_doctor_runtime_response( request: DoctorRuntimeRequest, git_watcher_health: Option, ) -> Result<()> { - let build_version = crate::version::build_version()?; + let build_version = tracedecay_project::version::build_version()?; let mut value = Box::pin(doctor_runtime_value( handshake, store_administration, @@ -632,17 +638,16 @@ mod doctor_runtime_route_tests { CoreDoctorStatusV1, cold_doctor_runtime_value, core_status_request_id, doctor_runtime_coverage, doctor_runtime_request, serve_core_doctor_runtime_request, }; - use crate::daemon::{ - AuthenticatedFirstRequest, DaemonHandshake, DaemonLifecycle, StoreAdministration, - }; + use crate::daemon::{AuthenticatedFirstRequest, DaemonHandshake, StoreAdministration}; use crate::mcp::McpServer; use crate::mcp::server::McpServerConstructionContext; - use crate::project::{TraceDecay, TraceDecayOpenOptions}; use tracedecay_contracts::project_open::{ ProjectOpenStatusReasonV1, ProjectOpenStatusStateV1, ProjectOpenStatusV1, }; use tracedecay_daemon_protocol::DaemonClientIdentity; + use tracedecay_daemon_service::shutdown::DaemonLifecycle; use tracedecay_mcp::McpTransport; + use tracedecay_project::project::{TraceDecay, TraceDecayOpenOptions}; static REGISTERED_RUNTIME_NONCE: AtomicU64 = AtomicU64::new(1); @@ -720,7 +725,7 @@ mod doctor_runtime_route_tests { ) -> DaemonHandshake { // The doctor route serves the daemon's version from the product // runtime; route tests never pass through the binary's registration. - crate::product_runtime::register_fixture_product_runtime(); + tracedecay_project::product_runtime::register_fixture_product_runtime(); DaemonHandshake { project_path: Some(project_path), scope_prefix: None, @@ -735,7 +740,7 @@ mod doctor_runtime_route_tests { client_instance_id: "doctor-runtime-test".to_string(), tool_list_changed_capable: false, catalog_version: String::new(), - moved_store_adoption: crate::project::MovedStoreAdoption::Never, + moved_store_adoption: tracedecay_project::project::MovedStoreAdoption::Never, } } @@ -1067,7 +1072,7 @@ mod doctor_runtime_route_tests { .await .insert(key, server); let build_version = - crate::product_runtime::register_fixture_product_runtime().build_version(); + tracedecay_project::product_runtime::register_fixture_product_runtime().build_version(); let value = super::doctor_runtime_value( &handshake, &store_administration, @@ -1186,51 +1191,6 @@ mod doctor_runtime_route_tests { connection.execute("ROLLBACK", ()).unwrap(); } - #[tokio::test] - async fn doctor_store_paths_ignore_an_active_branch_database() { - let root = tempfile::TempDir::new().unwrap(); - let project = root.path().join("project"); - let profile = root.path().join("profile"); - std::fs::create_dir_all(&project).unwrap(); - std::fs::create_dir_all(&profile).unwrap(); - assert!( - std::process::Command::new("git") - .args(["init", "-b", "main"]) - .current_dir(&project) - .status() - .unwrap() - .success() - ); - let layout = initialize_test_project(&project, &profile).await; - let default_graph = layout.graph_db_path.clone(); - - let branch_relpath = "branches/feature_doctor.db"; - let branch_graph = layout.data_root.join(branch_relpath); - std::fs::create_dir_all(branch_graph.parent().unwrap()).unwrap(); - std::fs::copy(&default_graph, &branch_graph).unwrap(); - let mut meta = tracedecay_runtime_core::branch_meta::BranchMeta::new_for_dir( - &layout.data_root, - "main", - ); - meta.add_branch("feature/doctor", branch_relpath, "main"); - tracedecay_runtime_core::branch_meta::save_branch_meta(&layout.data_root, &meta).unwrap(); - assert!( - std::process::Command::new("git") - .args(["checkout", "-b", "feature/doctor"]) - .current_dir(&project) - .status() - .unwrap() - .success() - ); - - assert_eq!( - super::doctor_runtime_store_layout(&project, &profile) - .expect("resolve canonical Doctor store paths"), - (default_graph, layout.sessions_db_path), - "Doctor must not follow branch-specific database paths" - ); - } - #[tokio::test] async fn cold_uninitialized_sessions_store_reports_fixed_reason_without_artifacts() { let root = tempfile::TempDir::new().unwrap(); diff --git a/crates/tracedecay/src/daemon/core_doctor_truthful_tests.rs b/crates/tracedecay/src/daemon/core_doctor_truthful_tests.rs index 68f1b736f3..f139da63b1 100644 --- a/crates/tracedecay/src/daemon/core_doctor_truthful_tests.rs +++ b/crates/tracedecay/src/daemon/core_doctor_truthful_tests.rs @@ -5,8 +5,8 @@ use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use crate::daemon::{DaemonHandshake, StoreAdministration}; use crate::mcp::McpServer; use crate::mcp::server::McpServerConstructionContext; -use crate::project::{TraceDecay, TraceDecayOpenOptions}; use tracedecay_daemon_protocol::DaemonClientIdentity; +use tracedecay_project::project::{TraceDecay, TraceDecayOpenOptions}; static REGISTERED_RUNTIME_NONCE: AtomicU64 = AtomicU64::new(1); @@ -64,7 +64,7 @@ fn handshake( client_instance_id: "truthful-core-doctor-test".to_string(), tool_list_changed_capable: false, catalog_version: String::new(), - moved_store_adoption: crate::project::MovedStoreAdoption::Never, + moved_store_adoption: tracedecay_project::project::MovedStoreAdoption::Never, } } @@ -111,7 +111,8 @@ async fn live_runtime_snapshot_does_not_fabricate_store_metadata_after_observati std::fs::remove_file(&layout.graph_db_path) .expect("remove graph file after its retained route has gone live"); - let build_version = crate::product_runtime::register_fixture_product_runtime().build_version(); + let build_version = + tracedecay_project::product_runtime::register_fixture_product_runtime().build_version(); let value = super::doctor_runtime_value( &handshake, &store_administration, diff --git a/crates/tracedecay/src/daemon/core_handshake.rs b/crates/tracedecay/src/daemon/core_handshake.rs index a8f6ee53c4..e05a4c4820 100644 --- a/crates/tracedecay/src/daemon/core_handshake.rs +++ b/crates/tracedecay/src/daemon/core_handshake.rs @@ -28,8 +28,8 @@ pub fn handshake_for_current_client( pub fn handshake_open_options( handshake: &DaemonHandshake, -) -> crate::project::TraceDecayOpenOptions { - crate::project::TraceDecayOpenOptions { +) -> tracedecay_project::project::TraceDecayOpenOptions { + tracedecay_project::project::TraceDecayOpenOptions { profile_root: Some(handshake.client_identity.profile_root.clone()), global_db_path: Some(handshake.client_identity.global_db_path.clone()), } @@ -44,6 +44,6 @@ pub fn handshake_open_options( /// fallible because it reads the registered product runtime: a process whose /// entry point never registered one has no truthful version to advertise. pub(crate) fn binary_version() --> std::result::Result<&'static str, crate::product_runtime::ProductRuntimeError> { - crate::version::build_version() +-> std::result::Result<&'static str, tracedecay_project::product_runtime::ProductRuntimeError> { + tracedecay_project::version::build_version() } diff --git a/crates/tracedecay/src/daemon/core_hooks.rs b/crates/tracedecay/src/daemon/core_hooks.rs index 5d5408e6aa..66a30b02f8 100644 --- a/crates/tracedecay/src/daemon/core_hooks.rs +++ b/crates/tracedecay/src/daemon/core_hooks.rs @@ -1,4 +1,4 @@ -//! Host hook events: daemon notification over the broker connection. +//! Host hook events: one stateless daemon request over the broker connection. //! //! The wire metadata and event constructors are pure data and live in //! [`tracedecay_hooks::core_events`]. Only delivery, which needs the daemon @@ -10,11 +10,11 @@ use tokio::io::AsyncWriteExt; use tokio::time::{Duration, timeout}; use tracedecay_hooks::core_events::{DaemonHookEvent, HOOK_EVENT_METHOD, HookEventNotifyOutcomeV1}; -#[cfg(unix)] -use tracedecay_daemon_identity::connection_for_socket_path; -use tracedecay_daemon_identity::{ResolvedDaemonConnection, current_daemon_connection}; -#[cfg(unix)] +use tracedecay_daemon_identity::{ + ResolvedDaemonConnection, client_connection, current_daemon_connection, +}; use tracedecay_daemon_protocol::SOCKET_ENV; +use tracedecay_mcp::server::attach_stateless_request_context; use super::{BrokerStream, JsonRpcRequest, write_daemon_preamble}; @@ -25,19 +25,11 @@ pub async fn notify_hook_event( project_path: &Path, event: DaemonHookEvent, ) -> HookEventNotifyOutcomeV1 { - let connection = { - #[cfg(unix)] - { - std::env::var_os(SOCKET_ENV) - .filter(|path| !path.is_empty()) - .map(|path| connection_for_socket_path(Path::new(&path))) - .map_or_else(current_daemon_connection, Ok) - } - #[cfg(not(unix))] - { - current_daemon_connection() - } - }; + let connection = std::env::var_os(SOCKET_ENV) + .filter(|path| !path.is_empty()) + .map_or_else(current_daemon_connection, |path| { + client_connection(Path::new(&path)) + }); let Ok(connection) = connection else { return HookEventNotifyOutcomeV1::Unavailable; }; @@ -69,16 +61,20 @@ async fn notify_hook_event_to_connection( let Ok(params) = serde_json::to_value(event) else { return HookEventNotifyOutcomeV1::Malformed; }; - let request = JsonRpcRequest { + // A stateless request, not a notification: its own daemon connection has + // no `initialize` session for a notification to ride. The result is not + // awaited, so delivery stays fire-and-forget. + let mut request = JsonRpcRequest { jsonrpc: "2.0".to_string(), - id: None, + id: Some(serde_json::Value::from(1)), method: HOOK_EVENT_METHOD.to_string(), params: Some(params), }; + attach_stateless_request_context(&mut request); let Ok(line) = serde_json::to_string(&request) else { return HookEventNotifyOutcomeV1::Malformed; }; - let Ok(stream) = BrokerStream::connect(&connection.endpoint).await else { + let Ok(stream) = BrokerStream::connect(connection.endpoint()).await else { return HookEventNotifyOutcomeV1::Unavailable; }; let (_reader, mut writer) = stream.into_owned_split(); @@ -106,7 +102,6 @@ mod tests { use super::*; - #[cfg(unix)] #[tokio::test] async fn missing_hook_socket_returns_typed_unavailable_without_retry_delay() { // Delivery builds the client handshake first, and that handshake reads @@ -115,10 +110,11 @@ mod tests { // per-test process that does not would classify the socket outcome as // Malformed (no advertisable version) before it ever reaches the // connect this test covers. - crate::product_runtime::register_fixture_product_runtime(); + tracedecay_project::product_runtime::register_fixture_product_runtime(); let socket_dir = tempfile::tempdir().unwrap(); let missing_socket = socket_dir.path().join("missing.sock"); - let connection = connection_for_socket_path(&missing_socket); + let _authority = super::super::tests::seed_socket_authority(&missing_socket); + let connection = client_connection(&missing_socket).expect("seeded daemon authority"); let started = Instant::now(); let outcome = notify_hook_event_to_connection( diff --git a/crates/tracedecay/src/daemon/core_proxy.rs b/crates/tracedecay/src/daemon/core_proxy.rs index 025ef26848..359a756c92 100644 --- a/crates/tracedecay/src/daemon/core_proxy.rs +++ b/crates/tracedecay/src/daemon/core_proxy.rs @@ -1,6 +1,7 @@ //! Stdio MCP proxy: forwards host traffic to the daemon over the broker //! transport, tracking initialize-route and tool-catalog metadata. +use std::borrow::Cow; #[cfg(unix)] use std::collections::VecDeque; #[cfg(unix)] @@ -20,13 +21,12 @@ use super::{ #[cfg(unix)] use super::{binary_version, connect_with_restart_grace}; #[cfg(unix)] -use tracedecay_daemon_identity::connection_for_socket_path; -#[cfg(unix)] use tracedecay_daemon_protocol::{DAEMON_TOOL_RESPONSE_GRACE, version_skew_action}; use tracedecay_domain::errors::{Result, TraceDecayError}; use tracedecay_mcp::JsonRpcRequest; #[cfg(not(unix))] use tracedecay_mcp::McpTransport; +use tracedecay_mcp::server::attach_stateless_request_context; use tracedecay_mcp::transport::StdioTransport; #[cfg(unix)] use tracedecay_mcp::transport::{McpDuplexTransport, McpTransportReader, McpTransportWriter}; @@ -129,8 +129,7 @@ pub(crate) async fn should_proxy_serve_to_daemon_with( if installed_service_socket != Some(socket_path) { return false; } - let connection = connection_for_socket_path(socket_path); - connect_with_restart_grace(&connection, grace, poll_interval) + connect_with_restart_grace(socket_path, grace, poll_interval) .await .is_ok() } @@ -216,16 +215,26 @@ pub(crate) async fn proxy_transport_to_daemon_with_drain_bound( /// ([`tool_dispatch_ceiling`](tracedecay_mcp::tools::dispatch_ceiling::tool_dispatch_ceiling) /// with an empty name), not a named catalog tool's possibly shorter deadline. struct DaemonProxyRequest<'a> { - raw: &'a str, + raw: Cow<'a, str>, parsed: Option, } impl<'a> DaemonProxyRequest<'a> { + /// Every host request after `initialize` travels on its own daemon + /// connection, so it carries this proxy's SEP-2575 per-request context + /// instead of an `initialize` session. fn new(raw: &'a str) -> Self { - Self { - raw, - parsed: JsonRpcRequest::decode(raw.trim()).ok(), - } + let mut parsed = JsonRpcRequest::decode(raw.trim()).ok(); + let attached = parsed + .as_mut() + .is_some_and(attach_stateless_request_context); + let raw = match parsed.as_ref() { + Some(request) if attached => { + serde_json::to_string(request).map_or(Cow::Borrowed(raw), Cow::Owned) + } + _ => Cow::Borrowed(raw), + }; + Self { raw, parsed } } } @@ -508,7 +517,7 @@ pub(crate) async fn resolve_daemon_initialize_route( // to discover_project_root / Resolved admission. return Err(repository_discovery_deferred(&root, *reason)); } - if let Some(project_path) = crate::config::discover_project_root(&root) { + if let Some(project_path) = tracedecay_project::config::discover_project_root(&root) { return Ok(Some(InitializeRouteMetadata { project_path, allow_init: false, @@ -524,11 +533,12 @@ pub(crate) async fn resolve_daemon_initialize_route( // enabled), not fail-closed: treating a missing snapshot as // "disabled" contradicted the config default and left explicit // initialize-roots repos unable to open at all. - let allow_init = crate::config::cached_sync_config(&identity.worktree_root) - .map_or_else( - |_| tracedecay_configuration::SyncConfig::default().auto_init, - |config| config.auto_init, - ); + let allow_init = + tracedecay_project::config::cached_sync_config(&identity.worktree_root) + .map_or_else( + |_| tracedecay_configuration::SyncConfig::default().auto_init, + |config| config.auto_init, + ); return Ok(Some(InitializeRouteMetadata { project_path: identity.worktree_root, allow_init, diff --git a/crates/tracedecay/src/daemon/dashboard_automation.rs b/crates/tracedecay/src/daemon/dashboard_automation.rs index 0b49f41da4..0a7423b103 100644 --- a/crates/tracedecay/src/daemon/dashboard_automation.rs +++ b/crates/tracedecay/src/daemon/dashboard_automation.rs @@ -23,6 +23,9 @@ use tracedecay_contracts::now_micros; #[cfg(feature = "test-transport")] use tracedecay_daemon_identity::authority; use tracedecay_daemon_service::DaemonInvocationService; +use tracedecay_daemon_service::automation_observation::{ + project_run_observation_producer, record_project_run, +}; use tracedecay_dashboard_api::{ DashboardAutomationAuthorityErrorV1, DashboardAutomationAuthorityV1, DashboardAutomationObservationRecorderV1, DashboardAutomationRunOutcomeV1, @@ -32,9 +35,10 @@ use tracedecay_dashboard_api::{ }; use tracedecay_domain::configuration::UserProfileId; -use crate::mcp::server::{RetainedProjectGraphRequest, RetainedProjectServerResolver}; -use crate::project::TraceDecay; +use crate::mcp::server::RetainedProjectServerResolver; +use tracedecay_dashboard_api::project_graph::RetainedProjectGraphRequest; use tracedecay_domain::errors::{Result, TraceDecayError}; +use tracedecay_project::project::TraceDecay; type DashboardAutomationResult = std::result::Result; type DashboardAutomationProjectFuture = std::pin::Pin< @@ -51,10 +55,13 @@ struct DashboardAutomationRequestRuntime { } impl DashboardAutomationRequestRuntime { - fn new(configured: &AutomationConfig) -> Self { + fn new( + configured: &AutomationConfig, + codex: &tracedecay_domain::configuration::LcmSummarizerExecutableV1, + ) -> Self { let mut config = configured.clone(); config.timeout_secs = config.timeout_secs.min(USER_JOB_REQUEST_TIMEOUT_SECS); - let backend = CodexAppServerBackend::from_automation_config(&config); + let backend = CodexAppServerBackend::from_automation_config(&config, codex); Self { config, backend } } @@ -74,16 +81,13 @@ pub(crate) fn dashboard_automation_observation_port( Arc::new(move |project_root| { let invocation_service = invocation_service.clone(); Box::pin(async move { - let producer = crate::daemon::project_automation_observation_producer( - &invocation_service, - &project_root, - ) - .await - .ok_or_else(|| { - "dashboard automation observation authority is unavailable".to_owned() - })?; + let producer = project_run_observation_producer(&invocation_service, &project_root) + .await + .ok_or_else(|| { + "dashboard automation observation authority is unavailable".to_owned() + })?; Ok(Arc::new(move |record| { - crate::daemon::record_project_automation_run( + record_project_run( producer.as_ref(), &project_root, &record, @@ -354,14 +358,11 @@ async fn execute_dashboard_automation_run( _run_control: &AutomationRunControl, invocation_service: &DaemonInvocationService, ) -> DashboardAutomationResult { - let producer = crate::daemon::project_automation_observation_producer( - invocation_service, - cg.project_root(), - ) - .await - .ok_or_else(|| DashboardAutomationAuthorityErrorV1::Unavailable { - detail: "dashboard automation observation authority is unavailable".to_owned(), - })?; + let producer = project_run_observation_producer(invocation_service, cg.project_root()) + .await + .ok_or_else(|| DashboardAutomationAuthorityErrorV1::Unavailable { + detail: "dashboard automation observation authority is unavailable".to_owned(), + })?; let pinned = cg .configuration_runtime() .client() @@ -378,7 +379,8 @@ async fn execute_dashboard_automation_run( &pinned.snapshot().resolution_provenance_digest, ) .map_err(automation_failed)?; - let runtime = DashboardAutomationRequestRuntime::new(&config); + let runtime = + DashboardAutomationRequestRuntime::new(&config, &pinned.config().lcm_summarizers.codex); let (config, backend) = runtime.execution(); let run = match request { DashboardAutomationRunRequestV1::UserJob { job_id, run_id } => { @@ -638,7 +640,10 @@ mod tests { ..AutomationConfig::default() }; - let runtime = DashboardAutomationRequestRuntime::new(&configured); + let runtime = DashboardAutomationRequestRuntime::new( + &configured, + &tracedecay_domain::configuration::LcmSummarizerExecutableV1::Unconfigured, + ); assert_eq!(runtime.execution().0.timeout_secs, 120); } diff --git a/crates/tracedecay/src/daemon/engine.rs b/crates/tracedecay/src/daemon/engine.rs index 0c1c4bcda4..e35ad5685e 100644 --- a/crates/tracedecay/src/daemon/engine.rs +++ b/crates/tracedecay/src/daemon/engine.rs @@ -12,6 +12,8 @@ use tracedecay_code_index_runtime::{GitWatchSyncConfigV1, git_watch}; use tracedecay_daemon_identity::profile_identity; #[cfg(unix)] use tracedecay_daemon_protocol::{client_version_skew, version_skew_action}; +#[cfg(unix)] +use tracedecay_daemon_service::shutdown::{DAEMON_TASK_ABORT_DEADLINE, DaemonLifecycle}; use tracedecay_hooks::core_events::HOOK_EVENT_METHOD; #[cfg(unix)] @@ -171,19 +173,19 @@ fn ensure_git_index_transactions_for_mutation_owners_inner<'a>( #[hotpath::measure(label = "daemon.engine.context_scout.ensure_owner")] pub(super) fn ensure_context_scout_owner_before_advertising( - project: &crate::project::TraceDecay, + project: &tracedecay_project::project::TraceDecay, ) -> Result<()> { if project.store_layout().identity.project_id.is_none() { return Ok(()); } let owner = match project.context_scout_owner_lookup() { - crate::project::ContextScoutOwnerLookupV1::Ready(owner) => owner, - crate::project::ContextScoutOwnerLookupV1::ReadOnly => { + tracedecay_project::project::ContextScoutOwnerLookupV1::Ready(owner) => owner, + tracedecay_project::project::ContextScoutOwnerLookupV1::ReadOnly => { return Err(TraceDecayError::Config { message: "read-only project has no Context Scout owner".to_owned(), }); } - crate::project::ContextScoutOwnerLookupV1::Unregistered => { + tracedecay_project::project::ContextScoutOwnerLookupV1::Unregistered => { return Err(TraceDecayError::Config { message: "project Context Scout owner did not start".to_owned(), }); diff --git a/crates/tracedecay/src/daemon/engine/shutdown.rs b/crates/tracedecay/src/daemon/engine/shutdown.rs index 6993f5c425..e66f2dc527 100644 --- a/crates/tracedecay/src/daemon/engine/shutdown.rs +++ b/crates/tracedecay/src/daemon/engine/shutdown.rs @@ -2,8 +2,9 @@ //! //! `shutdown_owner_phases` names every retained background owner and hands it //! to the shutdown coordinator as (cancel, join) pairs in dependency order: -//! producers first, then the invocation registry that admits provider work, -//! then the store-settling reapers. The phase deadline bounds every join and +//! manual branch publications and project opens first, then the invocation +//! registry that holds the owners they register and admits provider work, +//! then the remaining producers and the store-settling reapers. The phase deadline bounds every join and //! reports a typed timeout under the owner's name. //! //! The `cancel` side is not decoration. `prepare_shutdown_owner_phases` runs @@ -11,9 +12,10 @@ //! an owner that only cancels inside its join future is not actually told to //! stop until its phase is reached, and if the coordinator aborts the drain //! runner first, it is never told at all and keeps running past the terminal -//! receipt. Owners with a cheap synchronous stop (`invocation`, `maintenance`, -//! `git_watcher`) therefore supply a real `cancel`; a `|| {}` cancel side is -//! only correct where no synchronous stop exists. +//! receipt. Owners with a cheap synchronous stop (`project_open`, +//! `invocation`, `maintenance`, `git_watcher`) therefore supply a real +//! `cancel`; a `|| {}` cancel side is only correct where no synchronous stop +//! exists. use std::sync::Arc; @@ -49,6 +51,7 @@ impl DaemonEngine { self.store_administration .session_temporal_refresh_schedulers(), ); + let refresh_services = self.store_administration.clone(); let automation_join = self.clone(); let replay_join = self.store_administration.clone(); @@ -82,6 +85,24 @@ impl DaemonEngine { } }, )], + // An admitted open registers its owners with the invocation + // registry, so it must settle before that registry drains: it + // either registers in time to be released or stops at a + // cancellation boundary before registering anything. + vec![ShutdownOwner::with_deadline_status( + "project_open", + { + let project_open_cancel = project_open.clone(); + move || project_open_cancel.cancel_all() + }, + move |_| async move { + if project_open.shutdown().await { + ShutdownStatus::Clean + } else { + ShutdownStatus::TimedOut + } + }, + )], vec![ShutdownOwner::with_deadline_status( "invocation", { @@ -91,17 +112,6 @@ impl DaemonEngine { move |_| async move { invocation_join.shutdown().await }, )], vec![ - ShutdownOwner::with_deadline_status( - "project_open", - || {}, - move |_| async move { - if project_open.shutdown().await { - ShutdownStatus::Clean - } else { - ShutdownStatus::TimedOut - } - }, - ), ShutdownOwner::new( "automation", { @@ -114,6 +124,9 @@ impl DaemonEngine { ), ShutdownOwner::new("session_temporal_refresh", || {}, async move { session_refresh.shutdown().await; + refresh_services + .release_profile_session_refresh_services() + .await; }), ShutdownOwner::new( "host_admission_replay", diff --git a/crates/tracedecay/src/daemon/graph_resolution.rs b/crates/tracedecay/src/daemon/graph_resolution.rs index 7246c09eb6..a828e64100 100644 --- a/crates/tracedecay/src/daemon/graph_resolution.rs +++ b/crates/tracedecay/src/daemon/graph_resolution.rs @@ -11,8 +11,11 @@ use super::*; use tracedecay_daemon_identity::authority; fn sole_mounted_server_matching( - servers: &[(Arc, Arc)], - predicate: impl Fn(&crate::project::TraceDecay) -> bool, + servers: &[( + Arc, + Arc, + )], + predicate: impl Fn(&tracedecay_project::project::TraceDecay) -> bool, ) -> std::result::Result>, ()> { let mut matches = servers .iter() @@ -116,13 +119,14 @@ pub(super) fn retained_project_server_resolver( }) }) .collect::>(); - let branch_matches = |graph: &crate::project::TraceDecay| { + let branch_matches = |graph: &tracedecay_project::project::TraceDecay| { request.requested_branch.as_deref().is_some_and(|branch| { graph.serving_branch() == Some(branch) || graph.active_branch() == Some(branch) }) }; - let root_matches = |graph: &crate::project::TraceDecay, root: &Path| { + let root_matches = |graph: &tracedecay_project::project::TraceDecay, + root: &Path| { authority::canonical_identity_path(graph.project_root()).ok() == Some(root.to_path_buf()) }; diff --git a/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs b/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs index cfb0b8e52b..482a753fd0 100644 --- a/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs +++ b/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs @@ -9,12 +9,13 @@ use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex as StdMutex, OnceLock, Weak}; use std::time::Duration; +use tracedecay_domain::NativeHostIdentityV1; use tracedecay_domain::UtcMicros; use tracedecay_domain::canonical_text::encode_lowercase_hex; use tracedecay_hooks::{ - HookHostV1, HookReplayAdmissionOutcomeV1, HookReplayPassReportV1, HookSpoolConfigV1, - HookSpoolV1, admit_replayed_envelope_with_authoritative_session, drain_host_spool_once, - hook_v2_spool_root, published_hook_scope_binding, + HookReplayAdmissionOutcomeV1, HookReplayPassReportV1, HookSpoolConfigV1, HookSpoolV1, + admit_replayed_envelope_with_authoritative_session, drain_host_spool_once, hook_v2_spool_root, + published_hook_scope_binding, }; use tracedecay_mcp::handlers::hook_runtime::{ @@ -41,7 +42,7 @@ fn replay_admission_outcome(outcome: HookV2AdmissionOutcomeV1) -> HookReplayAdmi #[hotpath::measure(label = "daemon.hook_replay.receipt_drain", future = true)] async fn drain_hook_delivery_receipts( data_root: &Path, - host: HookHostV1, + host: NativeHostIdentityV1, authority: &tracedecay_application::observability::DeliverySettlementAuthorityV1, ) { let root = tracedecay_hooks::hook_delivery_receipt_spool_root(data_root, host); @@ -109,7 +110,7 @@ impl Drop for HookReplaySweepObservation { #[hotpath::measure(label = "daemon.hook_replay.sweep", future = true)] async fn drain_all_hosts( - graph: &crate::project::TraceDecay, + graph: &tracedecay_project::project::TraceDecay, data_root: &Path, delivery_settlements: &tracedecay_application::observability::DeliverySettlementAuthorityV1, project_sessions: &tracedecay_global_db::RegisteredGlobalDb, @@ -181,11 +182,11 @@ async fn drain_all_hosts( } async fn drain_admitted_host_spool( - host: HookHostV1, + host: NativeHostIdentityV1, project_id: [u8; 16], worktree_id: [u8; 16], now: UtcMicros, - graph: &crate::project::TraceDecay, + graph: &tracedecay_project::project::TraceDecay, project_sessions: &tracedecay_global_db::RegisteredGlobalDb, background_cpu: &Arc, ) -> Option { @@ -254,7 +255,7 @@ fn hook_replay_now() -> UtcMicros { } struct RegisteredReplayConsumer { - graph: Weak, + graph: Weak, delivery_settlements: Weak, task: Option>, @@ -277,7 +278,7 @@ pub(crate) fn hook_v2_replay_consumer_registered(data_root: &Path) -> bool { /// Start the per-project replay consumer exactly once per hook data root. /// Returns `false` when one is already running for this root. pub(crate) fn register_hook_v2_replay_consumer( - graph: Arc, + graph: Arc, delivery_settlements: Arc, project_sessions: tracedecay_global_db::RegisteredGlobalDbLeaseV1, background_cpu: Arc, diff --git a/crates/tracedecay/src/daemon/http_application.rs b/crates/tracedecay/src/daemon/http_application.rs index cc910d0608..e66fa7e896 100644 --- a/crates/tracedecay/src/daemon/http_application.rs +++ b/crates/tracedecay/src/daemon/http_application.rs @@ -32,6 +32,7 @@ use tracedecay_contracts::{ APPLICATION_REQUEST_ID_HEADER, ApplicationProblem, RequestId, RetryDirective, SafeDiagnostic, }; use tracedecay_daemon_control::RemoteBrainTlsConfig; +use tracedecay_daemon_service::logging::unavailable_error; use tracedecay_daemon_service::remote_http_transport::RemoteBrainTlsListener; #[cfg(test)] use tracedecay_daemon_service::remote_http_transport::{ @@ -438,9 +439,7 @@ pub fn live_remote_operational_status() -> Result message: "TraceDecay daemon HTTP application endpoint is not published. Start or restart the daemon.".to_owned(), }); }; - let Some(auth_token) = connection.auth_token.as_deref() else { - return Err(missing_daemon_authority()); - }; + let auth_token = connection.auth_token(); let origin = format!("http://{endpoint}"); let url = format!("http://{endpoint}/remote-status"); let agent = ureq::Agent::config_builder() @@ -477,17 +476,9 @@ pub fn live_remote_operational_status() -> Result }) } -fn missing_daemon_authority() -> TraceDecayError { - TraceDecayError::Config { - message: - "TraceDecay daemon authority record is not available. Start or restart the daemon." - .to_owned(), - } -} - fn remote_status_daemon_unavailable() -> TraceDecayError { match tracedecay_daemon_control::default_socket_path() { - Ok(socket_path) => super::unavailable_error(&socket_path), + Ok(socket_path) => unavailable_error(&socket_path), Err(error) => error, } } diff --git a/crates/tracedecay/src/daemon/http_application_tests.rs b/crates/tracedecay/src/daemon/http_application_tests.rs index 5f4d6b4afc..079f43bdc7 100644 --- a/crates/tracedecay/src/daemon/http_application_tests.rs +++ b/crates/tracedecay/src/daemon/http_application_tests.rs @@ -263,7 +263,7 @@ async fn service_with_canonical_application( // The canonical handshake reports the client's build version from the // product runtime; this composition never passes through the binary's // registration. - crate::product_runtime::register_fixture_product_runtime(); + tracedecay_project::product_runtime::register_fixture_product_runtime(); let project = tempfile::tempdir().expect("canonical application project"); let broker = tokio::net::TcpListener::bind("127.0.0.1:0") .await @@ -284,7 +284,10 @@ async fn service_with_canonical_application( ) .expect("canonical application handshake"); let client = tracedecay_daemon_protocol::DaemonInvocationClient::new( - tracedecay_daemon_protocol::DaemonConnection::new(broker_endpoint, None), + tracedecay_daemon_protocol::DaemonConnection::new( + broker_endpoint, + "http-application-test-token".to_owned(), + ), handshake, ); let canonical = tracedecay_daemon_service::application_surface::http_application_router( diff --git a/crates/tracedecay/src/daemon/invocation_dispatch.rs b/crates/tracedecay/src/daemon/invocation_dispatch.rs index 45c9523d7c..1f1b4bc1d8 100644 --- a/crates/tracedecay/src/daemon/invocation_dispatch.rs +++ b/crates/tracedecay/src/daemon/invocation_dispatch.rs @@ -9,6 +9,8 @@ use super::*; use std::future::Future; use tracedecay_code_index_runtime::git_transactions; use tracedecay_contracts::SharedProfileStoreLocatorV1; +#[cfg(any(not(unix), test))] +use tracedecay_daemon_service::shutdown::DaemonLifecycle; use tracedecay_daemon_service::{ DaemonInvocationOperation, DaemonInvocationPayload, DaemonInvocationProblem, DaemonInvocationService, Lease, diff --git a/crates/tracedecay/src/daemon/invocation_executor.rs b/crates/tracedecay/src/daemon/invocation_executor.rs index 742f60f8e2..9c77cf64da 100644 --- a/crates/tracedecay/src/daemon/invocation_executor.rs +++ b/crates/tracedecay/src/daemon/invocation_executor.rs @@ -302,114 +302,19 @@ impl tracedecay_contracts::ApplicationInvocationExecutor for InProcessDaemonInvo > { Box::pin(async move { let (context, request) = invocation.into_parts(); - let (request_id, target, deadline, cancellation) = context.into_parts(); - if target.resolved().is_some_and(|scope| scope != &self.scope) { + if context + .target() + .resolved() + .is_some_and(|scope| scope != &self.scope) + { return Err(tracedecay_contracts::InvocationError::Denied); } - let target = match target { - tracedecay_contracts::InvocationTarget::CurrentProject => { - tracedecay_contracts::InvocationTarget::Resolved(self.scope.clone()) - } - target @ tracedecay_contracts::InvocationTarget::Resolved(_) => target, - }; match request { tracedecay_contracts::ApplicationRequest::Surface { binding, payload } => { - let (_binding_id, surface, operation, result_contract, _page) = - binding.into_parts(); - let operation = - ApplicationSurfaceOperation::from_surface_name(surface, operation.as_str()) - .ok_or(tracedecay_contracts::InvocationError::InvalidRequest)?; - let observed_at = tracedecay_daemon_protocol::invocation_now_micros(); - let cancellation_context = cancellation.context(); - let scope = match target { - tracedecay_contracts::InvocationTarget::CurrentProject => None, - tracedecay_contracts::InvocationTarget::Resolved(scope) => Some(scope), - }; - let policy = if matches!( - operation, - ApplicationSurfaceOperation::ConfigurationSet - | ApplicationSurfaceOperation::ConfigurationUnset - | ApplicationSurfaceOperation::ConfigurationBatch - ) { - tracedecay_daemon_protocol::InvocationCancellationPolicy::AuthoritativeEffect - } else { - tracedecay_daemon_protocol::InvocationCancellationPolicy::ReadOnly - }; - let request = match operation { - ApplicationSurfaceOperation::ConfigurationGet - | ApplicationSurfaceOperation::ConfigurationSet - | ApplicationSurfaceOperation::ConfigurationUnset - | ApplicationSurfaceOperation::ConfigurationBatch => { - let request = tracedecay_contracts::configuration_wire_request_from_invocation_payload( - operation.as_str(), - payload, - ) - .map_err(|_| { - tracedecay_contracts::InvocationError::InvalidRequest - })?; - DaemonInvocationRequest::configuration( - request_id.as_str(), - operation, - request, - observed_at, - deadline.clone(), - cancellation_context, - ) - .with_resolved_scope(scope) - .map_err(|_| { - tracedecay_contracts::InvocationError::InvalidRequest - })? - } - ApplicationSurfaceOperation::FeedbackGet => { - let typed = tracedecay_daemon_protocol::parse_application_surface_request( - operation, payload, - ) - .map_err(|_| { - tracedecay_contracts::InvocationError::InvalidRequest - })?; - let tracedecay_daemon_protocol::ApplicationSurfaceRequest::Feedback( - request, - ) = typed - else { - return Err( - tracedecay_contracts::InvocationError::InvalidRequest, - ); - }; - DaemonInvocationRequest::feedback( - request_id.as_str(), - operation, - request.request_handle, - observed_at, - deadline.clone(), - cancellation_context, - ) - .with_resolved_scope(scope) - .map_err(|_| { - tracedecay_contracts::InvocationError::InvalidRequest - })? - } - _ => { - return Err( - tracedecay_contracts::InvocationError::InvalidRequest, - ); - } - } - .with_delivery_route(tracedecay_daemon_protocol::application_delivery_route(surface)); - let response = - ::invoke_controlled( - self, - request, - deadline, - cancellation, - policy, - ) - .await - .map_err(tracedecay_daemon_protocol::map_invocation_error)?; - tracedecay_daemon_protocol::application_response( - request_id, - result_contract, - response.outcome, + tracedecay_daemon_protocol::invoke_application_surface( + self, context, binding, payload, ) + .await } tracedecay_contracts::ApplicationRequest::FeedbackObservation { configuration_digest, @@ -420,7 +325,7 @@ impl tracedecay_contracts::ApplicationInvocationExecutor for InProcessDaemonInvo .map_err(|_| tracedecay_contracts::InvocationError::InvalidRequest)?; let response = self .invoke_once(DaemonInvocationRequest::feedback_observation( - request_id.as_str(), + context.request_id().as_str(), configuration_digest, observed_at, event, @@ -440,11 +345,14 @@ impl tracedecay_contracts::ApplicationInvocationExecutor for InProcessDaemonInvo max_events, after_sequence, } => { + let (request_id, _, deadline, cancellation) = context.into_parts(); + let target = + tracedecay_contracts::InvocationTarget::Resolved(self.scope.clone()); let operation_id = tracedecay_application::operation_stream::OperationId::from_request( operation_id.clone(), ); - let observed_at = tracedecay_daemon_protocol::invocation_now_micros(); + let observed_at = tracedecay_contracts::now_micros(); let authority = self.invocation.service.operation_events(); let admitted = authority .resolve_invocation_context( @@ -528,11 +436,14 @@ impl tracedecay_contracts::ApplicationInvocationExecutor for InProcessDaemonInvo )) } tracedecay_contracts::ApplicationRequest::OperationCancel { operation_id } => { + let (request_id, _, deadline, cancellation) = context.into_parts(); + let target = + tracedecay_contracts::InvocationTarget::Resolved(self.scope.clone()); let operation_id = tracedecay_application::operation_stream::OperationId::from_request( operation_id.clone(), ); - let observed_at = tracedecay_daemon_protocol::invocation_now_micros(); + let observed_at = tracedecay_contracts::now_micros(); let authority = self.invocation.service.operation_events(); let admitted = authority .resolve_invocation_context( diff --git a/crates/tracedecay/src/daemon/invocation_state.rs b/crates/tracedecay/src/daemon/invocation_state.rs index 58e7ed4a97..959c34d018 100644 --- a/crates/tracedecay/src/daemon/invocation_state.rs +++ b/crates/tracedecay/src/daemon/invocation_state.rs @@ -30,6 +30,7 @@ use tracedecay_domain::errors::{Result, TraceDecayError}; use tracedecay_store_runtime::ShutdownStatus; use super::*; +use tracedecay_daemon_service::shutdown::DAEMON_TASK_ABORT_DEADLINE; use tracedecay_runtime_core::logging::log_daemon_event; mod project_invocation; @@ -105,10 +106,11 @@ impl DaemonInvocationState { database: tracedecay_global_db::RegisteredGlobalDbLeaseV1, profile_id: &tracedecay_domain::configuration::UserProfileId, ) -> Result { - let configured = crate::config::read_or_initialize_profile_code_index_worker_selection( - database, profile_id, - ) - .await?; + let configured = + tracedecay_project::config::read_or_initialize_profile_code_index_worker_selection( + database, profile_id, + ) + .await?; self.install_worker_selection(store_administration, configured) } @@ -1088,7 +1090,7 @@ impl DaemonInvocationState { // unwinding. The registry retains its worker until a retry joins it; // an incomplete sweep must keep the outer shutdown receipt unclean. let schedulers_timed_out = tokio::time::timeout( - super::DAEMON_TASK_ABORT_DEADLINE, + DAEMON_TASK_ABORT_DEADLINE, self.code_index_schedulers.shutdown(), ) .await diff --git a/crates/tracedecay/src/daemon/invocation_tests/configuration_registrars_tests.rs b/crates/tracedecay/src/daemon/invocation_tests/configuration_registrars_tests.rs index 50e5ef506e..e4fd154fea 100644 --- a/crates/tracedecay/src/daemon/invocation_tests/configuration_registrars_tests.rs +++ b/crates/tracedecay/src/daemon/invocation_tests/configuration_registrars_tests.rs @@ -15,17 +15,18 @@ use tracedecay_global_db::configuration::contracts::types::{ async fn read_only_project_configuration_requires_the_bootstrap_profile_plan() { let _pin = tracedecay_runtime_core::config::PinnedUserDataDir::new(); let project = tempfile::tempdir().expect("project root"); - let (graph, runtime) = crate::project::TraceDecay::init_test_fixture_with_registered_runtime( - project.path(), - "project.configuration.read-only-worker-plan", - ) - .await - .expect("registered graph"); + let (graph, runtime) = + tracedecay_project::project::TraceDecay::init_test_fixture_with_registered_runtime( + project.path(), + "project.configuration.read-only-worker-plan", + ) + .await + .expect("registered graph"); graph.close(); let read_only = runtime .open_project_graph_read_only_for_test( project.path(), - crate::project::TraceDecayOpenOptions { + tracedecay_project::project::TraceDecayOpenOptions { profile_root: Some( tracedecay_runtime_core::storage::default_profile_root() .expect("default profile root"), diff --git a/crates/tracedecay/src/daemon/invocation_tests/mod.rs b/crates/tracedecay/src/daemon/invocation_tests/mod.rs index 145b8d3ee7..e424461408 100644 --- a/crates/tracedecay/src/daemon/invocation_tests/mod.rs +++ b/crates/tracedecay/src/daemon/invocation_tests/mod.rs @@ -5,7 +5,7 @@ use std::sync::Arc; -use tracedecay_agent_hosts::agents::context_scout::ports::ContextScoutLifecycleAddressV1; +use tracedecay_agent_hosts::agents::context_scout::address_registry::ContextScoutLifecycleAddressV1; use tracedecay_application::feedback::observations::FeedbackObservationEmitterV1; use tracedecay_application::lsp_runtime::DaemonLspSessionFactory; use tracedecay_application::work::{ @@ -50,10 +50,10 @@ pub(super) fn empty_work_proposal_routing( // The runtime pin rejects a snapshot that omitted registry defaults // such as `index.include.v1`. Resolve through the same core registry // the daemon uses at project open, then overlay empty Work bindings. - let snapshot = crate::config::resolver::resolve_configuration( - &crate::config::registry::ConfigurationRegistry::core() + let snapshot = tracedecay_project::config::resolver::resolve_configuration( + &tracedecay_project::config::registry::ConfigurationRegistry::core() .expect("configuration registry defaults"), - &[crate::config::resolver::ConfigurationLayerV1 { + &[tracedecay_project::config::resolver::ConfigurationLayerV1 { layer: tracedecay_domain::configuration::ConfigurationLayerIdV1::Project { project_id: scope.project_id.clone(), }, @@ -202,7 +202,7 @@ fn hook_envelope(event: HookEventV2) -> HookEventEnvelopeV2 { HookEventEnvelopeV2 { schema_version: tracedecay_hooks::HOOK_EVENT_SCHEMA_VERSION, event_id: [1; 16], - producer: tracedecay_hooks::HookHostV1::Codex, + producer: tracedecay_domain::NativeHostIdentityV1::Codex, protected_session_id: [2; 32], project_id: [3; 16], repository_id: [4; 16], @@ -217,7 +217,7 @@ fn hook_envelope(event: HookEventV2) -> HookEventEnvelopeV2 { fn hook_binding() -> HookScopeBindingV1 { HookScopeBindingV1 { - host: tracedecay_hooks::HookHostV1::Codex, + host: tracedecay_domain::NativeHostIdentityV1::Codex, project_id: [3; 16], repository_id: [4; 16], worktree_id: [5; 16], @@ -234,7 +234,7 @@ fn hook_binding() -> HookScopeBindingV1 { .map(|family| tracedecay_hooks::HookCapabilityV1 { family, support: tracedecay_hooks::stock_event_support( - tracedecay_hooks::HookHostV1::Codex, + tracedecay_domain::NativeHostIdentityV1::Codex, family, ), }) diff --git a/crates/tracedecay/src/daemon/invocation_tests/types_tests.rs b/crates/tracedecay/src/daemon/invocation_tests/types_tests.rs index c0da89c571..ad74356ebc 100644 --- a/crates/tracedecay/src/daemon/invocation_tests/types_tests.rs +++ b/crates/tracedecay/src/daemon/invocation_tests/types_tests.rs @@ -999,17 +999,18 @@ async fn feedback_admission_conflicts_construct_zero_losing_producers() { let _pin = tracedecay_runtime_core::config::PinnedUserDataDir::new(); let project = tempfile::tempdir().expect("project root"); let project_id = ProjectId::new("project.feedback.atomic-publication").expect("project id"); - let host = crate::test_support::host_admission::HostAdmissionTestRuntimeV1::project( - tracedecay_runtime_core::storage::default_profile_root().expect("profile root"), - project.path(), - project_id.clone(), - ) - .await - .expect("registered project runtime"); + let host = + tracedecay_project::test_support::host_admission::HostAdmissionTestRuntimeV1::project( + tracedecay_runtime_core::storage::default_profile_root().expect("profile root"), + project.path(), + project_id.clone(), + ) + .await + .expect("registered project runtime"); let graph = host .initialize_project_graph_for_test( project.path(), - crate::project::TraceDecayOpenOptions::default(), + tracedecay_project::project::TraceDecayOpenOptions::default(), ) .await .expect("initialized project graph"); diff --git a/crates/tracedecay/src/daemon/invocation_tests/work_evidence_journey_tests.rs b/crates/tracedecay/src/daemon/invocation_tests/work_evidence_journey_tests.rs index 22cd39094d..29b9124bb7 100644 --- a/crates/tracedecay/src/daemon/invocation_tests/work_evidence_journey_tests.rs +++ b/crates/tracedecay/src/daemon/invocation_tests/work_evidence_journey_tests.rs @@ -8,8 +8,8 @@ use tracedecay_contracts::{ DisclosureClass, PrepareWorkProductMutationRequestV1, StartWorkAttemptCommand, WorkAttemptEvidenceRecordV1, WorkAttemptProviderOutcomeV1, WorkAttemptStoragePort, WorkEvidenceExpansionSelectorV1, WorkEvidenceRetrieveRequestV1, WorkEvidenceSourceV1, - WorkGraphReadRequestV1, WorkProductChangeDraftV1, WorkProductMutationRequestV1, - WorkProductSelectionScopeV1, WorkRelationScopeV1, + WorkGraphReadRequestV1, WorkProductAuthorizedRelationScopeV1, WorkProductChangeDraftV1, + WorkProductMutationRequestV1, WorkProductSelectionScopeV1, now_micros, }; use tracedecay_daemon_service::{DaemonInvocationService, *}; use tracedecay_domain::{ @@ -159,10 +159,10 @@ pub(super) fn configured_work_proposal_routing( .expect("configuration revision"); let key = SettingKey::new(WORK_EXECUTABLE_BINDINGS_SETTING_KEY) .expect("work executable bindings key"); - let snapshot = crate::config::resolver::resolve_configuration( - &crate::config::registry::ConfigurationRegistry::core() + let snapshot = tracedecay_project::config::resolver::resolve_configuration( + &tracedecay_project::config::registry::ConfigurationRegistry::core() .expect("configuration registry defaults"), - &[crate::config::resolver::ConfigurationLayerV1 { + &[tracedecay_project::config::resolver::ConfigurationLayerV1 { layer: ConfigurationLayerIdV1::Project { project_id: scope.project_id.clone(), }, @@ -201,7 +201,7 @@ fn seal_attempt( admitted: WorkAttemptV1, provider_session: ObservationSourceIdentityV1, ) { - let observed_at = current_micros(); + let observed_at = now_micros(); let running = admitted .transition( WorkAttemptStateV1::Running, @@ -269,7 +269,7 @@ async fn invoke_work( request_id: &str, request: WorkApplicationInvocationV1, ) -> DaemonInvocationOutcome { - let observed_at = current_micros(); + let observed_at = now_micros(); service .invoke( registry, @@ -296,7 +296,7 @@ async fn invoke_work_without_attempt_spawn( request_id: &str, request: WorkApplicationInvocationV1, ) -> DaemonInvocationOutcome { - let observed_at = current_micros(); + let observed_at = now_micros(); let canonical_root = project_root.canonicalize().expect("canonical project root"); let runtimes = service .project_runtimes @@ -327,13 +327,14 @@ async fn registered_work_evidence_hydrates_the_provider_qualified_task_session() let project_id = id::("project.work.evidence-journey"); let repository_id = id::("repository.work.evidence-journey"); let worktree_id = id::("worktree.work.evidence-journey"); - let host = crate::test_support::host_admission::HostAdmissionTestRuntimeV1::project( - profile.path(), - &project, - project_id.clone(), - ) - .await - .expect("registered project runtime"); + let host = + tracedecay_project::test_support::host_admission::HostAdmissionTestRuntimeV1::project( + profile.path(), + &project, + project_id.clone(), + ) + .await + .expect("registered project runtime"); let database = host .registered_database_arc(tracedecay_sessions::admission::HostAdmissionScope::Project) .expect("registered project database"); @@ -414,7 +415,7 @@ async fn registered_work_evidence_hydrates_the_provider_qualified_task_session() let actor = id::("actor.work.evidence-journey"); let grant_digest = digest('d'); - let journey_now = current_micros(); + let journey_now = now_micros(); let grant = CapabilityGrantSnapshot::new( id::("grant.work.evidence-journey"), 1, @@ -469,12 +470,13 @@ async fn registered_work_evidence_hydrates_the_provider_qualified_task_session() .await .expect("registered Work runtime"); let registry = Arc::new(Mutex::new(LspSessionRegistry::default())); - let selection = - WorkProductSelectionScopeV1::relations(BTreeSet::from([WorkRelationScopeV1::Repository { + let selection = WorkProductSelectionScopeV1::relations(BTreeSet::from([ + WorkProductAuthorizedRelationScopeV1::Repository { project_id: scope.project_id.clone(), repository_id: scope.repository_id.clone(), - }])) - .expect("repository Work selection"); + }, + ])) + .expect("repository Work selection"); let (initiative, plan, milestone, item) = product_task(task_id.clone()); let prepared = invoke_work( &service, @@ -651,7 +653,7 @@ async fn registered_work_evidence_hydrates_the_provider_qualified_task_session() commit: id::("0123456789abcdef0123456789abcdef01234567"), instructions: "Hydrate the exact provider session.".to_owned(), effect_state: WorkEffectStateV1::Observational, - occurred_at: current_micros(), + occurred_at: now_micros(), }), ) .await; @@ -677,7 +679,7 @@ async fn registered_work_evidence_hydrates_the_provider_qualified_task_session() "request.work.evidence-graph", WorkApplicationInvocationV1::Views(WorkGraphReadRequestV1::current( selection.clone(), - current_micros(), + now_micros(), )), ) .await; @@ -711,7 +713,7 @@ async fn registered_work_evidence_hydrates_the_provider_qualified_task_session() attempt: attempt.clone(), }), continuation: None, - observed_at: current_micros(), + observed_at: now_micros(), }), ) .await; diff --git a/crates/tracedecay/src/daemon/invocation_tests/work_tests.rs b/crates/tracedecay/src/daemon/invocation_tests/work_tests.rs index 41b1d66d13..9257a18697 100644 --- a/crates/tracedecay/src/daemon/invocation_tests/work_tests.rs +++ b/crates/tracedecay/src/daemon/invocation_tests/work_tests.rs @@ -6,8 +6,8 @@ use tokio::sync::Mutex; use tracedecay_contracts::{ ApplicationOutcome, CancellationContext, CapabilityGrantId, CapabilityGrantSnapshot, Deadline, DisclosureClass, GenerateProposalRequest, PrepareWorkProductMutationRequestV1, - WorkGraphReadRequestV1, WorkProductChangeDraftV1, WorkProductMutationRequestV1, - WorkProductSelectionScopeV1, WorkRelationScopeV1, + WorkGraphReadRequestV1, WorkProductAuthorizedRelationScopeV1, WorkProductChangeDraftV1, + WorkProductMutationRequestV1, WorkProductSelectionScopeV1, }; use tracedecay_daemon_service::{DaemonInvocationService, *}; use tracedecay_domain::{ @@ -73,13 +73,14 @@ async fn registered_work_services_dispatch_the_core_lifecycle() { let _pin = tracedecay_runtime_core::config::PinnedUserDataDir::new(); let project = tempfile::tempdir().expect("project root"); let project_id = ProjectId::new("project.work.core-invocation").expect("project id"); - let host = crate::test_support::host_admission::HostAdmissionTestRuntimeV1::project( - tracedecay_runtime_core::storage::default_profile_root().expect("profile root"), - project.path(), - project_id.clone(), - ) - .await - .expect("registered project runtime"); + let host = + tracedecay_project::test_support::host_admission::HostAdmissionTestRuntimeV1::project( + tracedecay_runtime_core::storage::default_profile_root().expect("profile root"), + project.path(), + project_id.clone(), + ) + .await + .expect("registered project runtime"); let database = host .registered_database_arc(tracedecay_sessions::admission::HostAdmissionScope::Project) .expect("registered project database"); @@ -214,7 +215,7 @@ async fn registered_work_services_dispatch_the_core_lifecycle() { } let product_selection = WorkProductSelectionScopeV1::relations( - [WorkRelationScopeV1::Repository { + [WorkProductAuthorizedRelationScopeV1::Repository { project_id: scope.project_id.clone(), repository_id: scope.repository_id.clone(), }] @@ -678,13 +679,14 @@ async fn committed_work_mutations_publish_task_activity_and_reads_do_not() { let _pin = tracedecay_runtime_core::config::PinnedUserDataDir::new(); let project = tempfile::tempdir().expect("project root"); let project_id = ProjectId::new("project.work.task-activity").expect("project id"); - let host = crate::test_support::host_admission::HostAdmissionTestRuntimeV1::project( - tracedecay_runtime_core::storage::default_profile_root().expect("profile root"), - project.path(), - project_id.clone(), - ) - .await - .expect("registered project runtime"); + let host = + tracedecay_project::test_support::host_admission::HostAdmissionTestRuntimeV1::project( + tracedecay_runtime_core::storage::default_profile_root().expect("profile root"), + project.path(), + project_id.clone(), + ) + .await + .expect("registered project runtime"); let database = host .registered_database_arc(tracedecay_sessions::admission::HostAdmissionScope::Project) .expect("registered project database"); @@ -778,7 +780,7 @@ async fn committed_work_mutations_publish_task_activity_and_reads_do_not() { } let product_selection = WorkProductSelectionScopeV1::relations( - [WorkRelationScopeV1::Repository { + [WorkProductAuthorizedRelationScopeV1::Repository { project_id: scope.project_id.clone(), repository_id: scope.repository_id.clone(), }] diff --git a/crates/tracedecay/src/daemon/lsp_sessions.rs b/crates/tracedecay/src/daemon/lsp_sessions.rs index 541b7f5198..4b89c9a275 100644 --- a/crates/tracedecay/src/daemon/lsp_sessions.rs +++ b/crates/tracedecay/src/daemon/lsp_sessions.rs @@ -4,9 +4,10 @@ //! when it goes away, and authorizes the workspace a request may reach. use tracedecay_daemon_service::{ - DaemonInvocationOutcome, DaemonInvocationPayload, DaemonInvocationService, - DaemonLspSessionAccess, + DaemonInvocationOutcome, DaemonInvocationPayload, DaemonInvocationProblem, + DaemonInvocationService, DaemonLspSessionAccess, }; +use tracedecay_runtime_core::logging::log_daemon_event; use super::*; @@ -121,31 +122,41 @@ async fn authorize_lsp_workspace_for_uris( if requested_uris.is_empty() || requested_uris.len() > tracedecay_daemon_protocol::MAX_LSP_WORKSPACE_ROOTS { - return None; + return lsp_workspace_refused("root_count_out_of_bounds", project_path); } // A single folder is only ever the active project: a lone sibling hint // must not silently reroute the session. A multi-folder workspace may span // registered roots, but the active project must be one of them so the // session stays anchored to the admitted route. let single_root = requested_uris.len() == 1; - let active_project_path = project_path.canonicalize().ok()?; + let Ok(active_project_path) = project_path.canonicalize() else { + return lsp_workspace_refused("active_project_unresolvable", project_path); + }; let graphs = store_administration.mounted_project_graphs().await; let mut selectors = Vec::with_capacity(requested_uris.len()); let mut canonical_uris = BTreeMap::new(); let mut admits_active_project = false; for requested_uri in requested_uris { - let uri = url::Url::parse(&requested_uri).ok()?; + let Ok(uri) = url::Url::parse(&requested_uri) else { + return lsp_workspace_refused("root_uri_unparseable", project_path); + }; if uri.scheme() != "file" || uri.query().is_some() || uri.fragment().is_some() { - return None; + return lsp_workspace_refused("root_uri_not_a_local_file", project_path); } - let requested_path = uri.to_file_path().ok()?.canonicalize().ok()?; + let Some(requested_path) = uri + .to_file_path() + .ok() + .and_then(|path| path.canonicalize().ok()) + else { + return lsp_workspace_refused("root_path_unresolvable", project_path); + }; if single_root && !tracedecay_runtime_core::path_safety::same_canonical_path( &requested_path, &active_project_path, ) { - return None; + return lsp_workspace_refused("single_root_is_not_the_active_project", &requested_path); } if tracedecay_runtime_core::path_safety::same_canonical_path( &requested_path, @@ -153,56 +164,99 @@ async fn authorize_lsp_workspace_for_uris( ) { admits_active_project = true; } - let mut candidates = Vec::new(); - for graph in &graphs { - if !tracedecay_runtime_core::path_safety::same_canonical_path( - graph.project_root(), - &requested_path, - ) { - continue; - } - let Some(raw_project_id) = graph.store_layout().identity.project_id.as_deref() else { - continue; - }; - let Ok(project_id) = tracedecay_domain::ProjectId::new(raw_project_id.to_owned()) - else { - continue; - }; - candidates.push(project_id); - } - candidates.sort(); - candidates.dedup(); - let [project_id] = candidates.as_slice() else { - return None; + let project_id = match mounted_project_for_root(&graphs, &requested_path) { + Ok(project_id) => project_id, + Err(reason_code) => return lsp_workspace_refused(reason_code, &requested_path), }; - selectors.push( - tracedecay_contracts::RegisteredRootSelectorV1::new( - project_id.clone(), - requested_path.clone(), - ) - .ok()?, - ); - let canonical_uri = url::Url::from_file_path(&requested_path).ok()?.to_string(); - canonical_uris.insert(requested_path, canonical_uri); + let Ok(selector) = + tracedecay_contracts::RegisteredRootSelectorV1::new(project_id, requested_path.clone()) + else { + return lsp_workspace_refused("root_selector_invalid", &requested_path); + }; + selectors.push(selector); + let Ok(canonical_uri) = url::Url::from_file_path(&requested_path) else { + return lsp_workspace_refused("root_uri_unrepresentable", &requested_path); + }; + canonical_uris.insert(requested_path, canonical_uri.to_string()); } if !admits_active_project { - return None; + return lsp_workspace_refused("active_project_not_requested", project_path); } - let resolved = super::invocation_dispatch::resolve_multi_root_projects( + let resolved = match super::invocation_dispatch::resolve_multi_root_projects( store_administration, service, &selectors, ) .await - .ok()?; - let resolved_roots = resolved - .into_iter() - .map(|(root, scope, locator)| { - let uri = canonical_uris.get(&root)?.clone(); - Some((root, uri, scope, locator)) - }) - .collect::>>()?; - service + { + Ok(resolved) => resolved, + Err(problem) => { + return lsp_workspace_refused( + if problem == DaemonInvocationProblem::Unavailable { + "registered_root_unavailable" + } else { + "registered_root_not_authorized" + }, + project_path, + ); + } + }; + let mut resolved_roots = Vec::with_capacity(resolved.len()); + for (root, scope, locator) in resolved { + let Some(uri) = canonical_uris.get(&root).cloned() else { + return lsp_workspace_refused("resolved_root_spelling_diverged", &root); + }; + resolved_roots.push((root, uri, scope, locator)); + } + let authorized = service .authorize_lsp_workspace(resolved_roots, tracedecay_contracts::clock::now_micros()) - .await + .await; + if authorized.is_none() { + return lsp_workspace_refused("workspace_authorization_refused", project_path); + } + authorized +} + +/// The one mounted project rooted at `requested_path`, or the refusal code +/// when none or several are. +fn mounted_project_for_root( + graphs: &[Arc], + requested_path: &Path, +) -> std::result::Result { + let mut candidates = Vec::new(); + for graph in graphs { + if !tracedecay_runtime_core::path_safety::same_canonical_path( + graph.project_root(), + requested_path, + ) { + continue; + } + let Some(raw_project_id) = graph.store_layout().identity.project_id.as_deref() else { + continue; + }; + let Ok(project_id) = tracedecay_domain::ProjectId::new(raw_project_id.to_owned()) else { + continue; + }; + candidates.push(project_id); + } + candidates.sort(); + candidates.dedup(); + match candidates.as_slice() { + [project_id] => Ok(project_id.clone()), + [] => Err("root_has_no_mounted_project"), + _ => Err("root_has_ambiguous_mounted_projects"), + } +} + +/// Every refusal above reaches the client as the same non-diagnostic +/// `Denied`, so the cause is recorded in the operator log instead. +fn lsp_workspace_refused(reason_code: &str, root: &Path) -> Option { + log_daemon_event( + "lsp_workspace_refused", + &[ + ("root", root.display().to_string()), + ("reason_code", reason_code.to_owned()), + ], + ); + None } diff --git a/crates/tracedecay/src/daemon/maintenance.rs b/crates/tracedecay/src/daemon/maintenance.rs index 60b54a3c57..60c1755182 100644 --- a/crates/tracedecay/src/daemon/maintenance.rs +++ b/crates/tracedecay/src/daemon/maintenance.rs @@ -16,6 +16,7 @@ use tracedecay_maintenance::tick::{ }; use super::branch_admin::StoreAdministration; +use tracedecay_daemon_service::shutdown::DAEMON_TASK_ABORT_DEADLINE; use tracedecay_runtime_core::logging::log_daemon_event; const MAINTENANCE_STORE_PAGE_LIMIT: usize = 8; @@ -25,7 +26,7 @@ async fn join_abandoned_maintenance_task(task: Option>, owner: &' return; }; task.abort(); - match tokio::time::timeout(super::DAEMON_TASK_ABORT_DEADLINE, task).await { + match tokio::time::timeout(DAEMON_TASK_ABORT_DEADLINE, task).await { Ok(Ok(()) | Err(_)) => {} Err(_) => { log_daemon_event( @@ -226,12 +227,11 @@ pub(super) struct MaintenanceMetricsV1 { pub(super) last_outcome: Option, } -/// Grace windows for the daily branch-store GC pass, taken from the pinned +/// Grace window for the daily branch-store GC pass, taken from the pinned /// sync configuration at daemon startup. #[derive(Clone, Copy, Debug)] pub(super) struct BranchStoreGcCadenceV1 { pub(super) branch_gc_days: u64, - pub(super) orphan_db_gc_days: u64, } /// Interval between branch-store GC passes across mounted projects. @@ -239,7 +239,7 @@ const BRANCH_STORE_GC_PERIOD: Duration = Duration::from_hours(24); #[derive(Clone)] pub(super) struct MaintenanceCoordinator { - cancellation: tracedecay_session_memory::context::CancellationToken, + cancellation: tracedecay_runtime_core::cancellation::CancellationToken, background_cpu: Option>, wake: Arc, task: Arc>>>, @@ -262,7 +262,7 @@ pub(super) struct MaintenanceCoordinator { impl Default for MaintenanceCoordinator { fn default() -> Self { Self { - cancellation: tracedecay_session_memory::context::CancellationToken::new(), + cancellation: tracedecay_runtime_core::cancellation::CancellationToken::new(), background_cpu: None, wake: Arc::new(MaintenanceWake::default()), task: Arc::new(Mutex::new(None)), @@ -282,7 +282,7 @@ impl Default for MaintenanceCoordinator { /// store stays alive for the duration of the writer-held critical section. enum MaintenanceStoreWork { Session(tracedecay_global_db::RegisteredGlobalDbLeaseV1), - Graph(Arc), + Graph(Arc), } impl MaintenanceStoreWork { @@ -295,7 +295,7 @@ impl MaintenanceStoreWork { } pub(crate) fn project_store_maintenance_lease( - graph: &crate::project::TraceDecay, + graph: &tracedecay_project::project::TraceDecay, ) -> ProjectStoreMaintenanceLeaseV1 { ProjectStoreMaintenanceLeaseV1::new( graph.project_root().to_path_buf(), @@ -684,7 +684,6 @@ impl MaintenanceCoordinator { profile_root, profile_database, retention.orphan_store_gc_days, - retention.incident_debris_retention_days, &self.cancellation, ) }) @@ -732,7 +731,6 @@ impl MaintenanceCoordinator { administration, code_index_schedulers, branch_gc.branch_gc_days, - branch_gc.orphan_db_gc_days, graph, ) .await; @@ -849,7 +847,7 @@ fn record_process_resident_memory_gauge(_log: &std::sync::Mutex; async fn run_resident_memory_sampler_loop( - cancellation: &tracedecay_session_memory::context::CancellationToken, + cancellation: &tracedecay_runtime_core::cancellation::CancellationToken, interval: Duration, sample: ResidentMemorySampleV1, ) { @@ -917,7 +915,6 @@ pub(super) fn retention_maintenance_enabled( retention.session_lcm.enabled || retention.observation.enabled || retention.orphan_store_gc_days.is_some() - || retention.incident_debris_retention_days.is_some() || retention.compaction.is_some() } @@ -1149,7 +1146,7 @@ mod tests { #[tokio::test(start_paused = true)] async fn repeated_wakes_do_not_move_the_maintenance_due_deadline() { let _lifecycle_isolation = MAINTENANCE_LOOP_LIFECYCLE.lock().await; - let cancellation = tracedecay_session_memory::context::CancellationToken::new(); + let cancellation = tracedecay_runtime_core::cancellation::CancellationToken::new(); let wake = Arc::new(MaintenanceWake::default()); let ticks = Arc::new(std::sync::atomic::AtomicUsize::new(0)); let baseline = maintenance_futures_active(); @@ -1211,7 +1208,7 @@ mod tests { #[tokio::test(start_paused = true)] async fn due_request_pulls_the_next_tick_forward_without_busy_looping() { let _lifecycle_isolation = MAINTENANCE_LOOP_LIFECYCLE.lock().await; - let cancellation = tracedecay_session_memory::context::CancellationToken::new(); + let cancellation = tracedecay_runtime_core::cancellation::CancellationToken::new(); let wake = Arc::new(MaintenanceWake::default()); let ticks = Arc::new(std::sync::atomic::AtomicUsize::new(0)); let task_cancellation = cancellation.clone(); @@ -1282,7 +1279,7 @@ mod tests { #[tokio::test(start_paused = true)] async fn progress_continuation_reenters_only_the_owning_phase() { let _lifecycle_isolation = MAINTENANCE_LOOP_LIFECYCLE.lock().await; - let cancellation = tracedecay_session_memory::context::CancellationToken::new(); + let cancellation = tracedecay_runtime_core::cancellation::CancellationToken::new(); let wake = Arc::new(MaintenanceWake::default()); let phases = Arc::new(std::sync::Mutex::new(Vec::new())); let task_cancellation = cancellation.clone(); @@ -1454,25 +1451,12 @@ mod tests { ); } - #[test] - fn debris_retention_enables_maintenance_without_orphan_gc() { - let mut retention = tracedecay_configuration::RetentionConfig::default(); - retention.session_lcm.enabled = false; - retention.observation.enabled = false; - retention.orphan_store_gc_days = None; - retention.incident_debris_retention_days = Some(30); - retention.compaction = None; - - assert!(super::retention_maintenance_enabled(&retention)); - } - #[test] fn soft_budget_alone_never_enables_destructive_maintenance() { let mut retention = tracedecay_configuration::RetentionConfig::default(); retention.session_lcm.enabled = false; retention.observation.enabled = false; retention.orphan_store_gc_days = None; - retention.incident_debris_retention_days = None; retention.compaction = None; retention .store_soft_budgets_bytes @@ -1509,7 +1493,7 @@ mod tests { #[tokio::test(start_paused = true)] async fn blocked_resident_memory_reclaimer_never_stalls_runtime_or_sampler_cancellation() { - let cancellation = tracedecay_session_memory::context::CancellationToken::new(); + let cancellation = tracedecay_runtime_core::cancellation::CancellationToken::new(); let started = Arc::new(Notify::new()); let calls = Arc::new(AtomicUsize::new(0)); let release = Arc::new((StdMutex::new(false), Condvar::new())); diff --git a/crates/tracedecay/src/daemon/pr_autotrack.rs b/crates/tracedecay/src/daemon/pr_autotrack.rs index fad49337c0..fb8e50f550 100644 --- a/crates/tracedecay/src/daemon/pr_autotrack.rs +++ b/crates/tracedecay/src/daemon/pr_autotrack.rs @@ -77,14 +77,14 @@ pub(super) use runtime::spawn_with_administration; #[derive(Clone, Copy)] struct PrStoreAdministration<'a> { schedulers: Option<&'a CodeIndexSchedulerRegistryV1>, - graph: Option<&'a Arc>, + graph: Option<&'a Arc>, command_control: &'a PrCommandControl, } impl<'a> PrStoreAdministration<'a> { fn with_control( schedulers: &'a CodeIndexSchedulerRegistryV1, - graph: &'a Arc, + graph: &'a Arc, command_control: &'a PrCommandControl, ) -> Self { Self { @@ -136,7 +136,7 @@ fn log_pr_skip(repo_root: &Path, branch_label: Option<&str>, pr: Option, re #[cfg(test)] pub(crate) async fn activate_manual_branch_head( repo_root: &Path, - graph: &Arc, + graph: &Arc, schedulers: Option<&CodeIndexSchedulerRegistryV1>, branch: &str, ) -> std::result::Result { @@ -160,7 +160,7 @@ pub(crate) async fn activate_manual_branch_head( #[hotpath::measure(label = "daemon.pr_autotrack.activate", future = true)] pub(crate) async fn activate_manual_branch_head_with_lifecycle( repo_root: &Path, - graph: &Arc, + graph: &Arc, schedulers: Option<&CodeIndexSchedulerRegistryV1>, branch: &str, lifecycle: &ManualBranchLifecycleLeaseV1, @@ -839,7 +839,7 @@ async fn track_pr( #[hotpath::measure(label = "daemon.pr_autotrack.activate_worktree", future = true)] async fn activate_linked_worktree( schedulers: &CodeIndexSchedulerRegistryV1, - graph: &crate::project::TraceDecay, + graph: &tracedecay_project::project::TraceDecay, worktree: &Path, ) -> std::result::Result<(), String> { let project_id = graph @@ -909,7 +909,6 @@ async fn remove_pr_store( fn pr_number_from_label(label: &str) -> Option { label .strip_prefix("tracedecay/autotrack/pr/") - .or_else(|| label.strip_prefix("pr/")) .and_then(|number| number.parse().ok()) } @@ -931,7 +930,6 @@ async fn cleanup_failed_track( data_root, pr, head_sha, - true, administration.command_control.clone(), ) .await @@ -957,14 +955,11 @@ async fn untrack_pr( managed: &ManagedPr, administration: PrStoreAdministration<'_>, ) -> std::result::Result<(), String> { - let expected_label = pr_label(managed.pr); - let legacy_label = format!("pr/{}", managed.pr); - let is_legacy = label == legacy_label; let expected_worktree = data_root .join("pr-worktrees") .join(format!("pr-{}", managed.pr)); let expected_ref = pr_tracking_ref(managed.pr); - if (label != expected_label && !is_legacy) + if label != pr_label(managed.pr) || managed.worktree != expected_worktree || managed.tracking_ref != expected_ref { @@ -976,7 +971,6 @@ async fn untrack_pr( data_root, managed.pr, &managed.head_sha, - !is_legacy, administration.command_control.clone(), ) .await @@ -1039,7 +1033,6 @@ async fn sweep_orphan_pr_worktrees( data_root, number, "", - true, administration.command_control.clone(), ) .await diff --git a/crates/tracedecay/src/daemon/pr_autotrack/runtime.rs b/crates/tracedecay/src/daemon/pr_autotrack/runtime.rs index e39846e224..8e35a72cb7 100644 --- a/crates/tracedecay/src/daemon/pr_autotrack/runtime.rs +++ b/crates/tracedecay/src/daemon/pr_autotrack/runtime.rs @@ -104,7 +104,7 @@ async fn tick( ) { let window = 14 * 86_400; let cap = 64; - let cutoff = crate::project::current_timestamp().saturating_sub(window); + let cutoff = tracedecay_runtime_core::tracedecay::current_timestamp().saturating_sub(window); let Ok(records) = database.list_code_projects(cap).await else { return; }; @@ -116,16 +116,17 @@ async fn tick( return; } let root = PathBuf::from(&record.canonical_root); - if !root.is_dir() || crate::config::is_ambient_project_root(&root) { + if !root.is_dir() || tracedecay_project::config::is_ambient_project_root(&root) { continue; } // A poll loop has no right to turn an arbitrary project path into // configuration authority. Missing/pending daemon snapshot means no // poll and, critically, no destructive disabled-state teardown. - let Ok(cfg) = - crate::config::cached_runtime_configuration_for_project_id(&root, &record.project_id) - .map(|configuration| configuration.into_config().sync) - else { + let Ok(cfg) = tracedecay_project::config::cached_runtime_configuration_for_project_id( + &root, + &record.project_id, + ) + .map(|configuration| configuration.config().sync.clone()) else { continue; }; let interval = Duration::from_secs(cfg.effective_auto_track_pr_poll_secs()); @@ -156,7 +157,7 @@ async fn tick( async fn retained_project_graph( administration: &StoreAdministration, project_root: &Path, -) -> Option> { +) -> Option> { let canonical = project_root .canonicalize() .unwrap_or_else(|_| project_root.to_path_buf()); diff --git a/crates/tracedecay/src/daemon/pr_autotrack/tests.rs b/crates/tracedecay/src/daemon/pr_autotrack/tests.rs index ab8335ec8a..0c89ae0d40 100644 --- a/crates/tracedecay/src/daemon/pr_autotrack/tests.rs +++ b/crates/tracedecay/src/daemon/pr_autotrack/tests.rs @@ -25,17 +25,12 @@ async fn reconcile_preserves_closed_pr_when_scheduler_retirement_is_unavailable( let repo_root = tempfile::tempdir().unwrap(); // not a git repo; git ops no-op let mut meta = BranchMeta::new("main"); - meta.add_branch("pr/5", "branches/pr_5.db", "main"); - std::fs::create_dir_all(data_root.path().join("branches")).unwrap(); - drop( - rusqlite::Connection::open(data_root.path().join("branches/pr_5.db")) - .expect("empty branch database"), - ); + meta.add_branch("tracedecay/autotrack/pr/5", "main"); save_branch_meta(data_root.path(), &meta).unwrap(); let mut state = PrAutotrackState::default(); state.managed.insert( - "pr/5".to_string(), + "tracedecay/autotrack/pr/5".to_string(), ManagedPr { pr: 5, head_branch: "feature-5".to_string(), @@ -86,11 +81,10 @@ async fn reconcile_preserves_closed_pr_when_scheduler_retirement_is_unavailable( load_state(data_root.path()) .expect("load managed PR state") .managed - .contains_key("pr/5") + .contains_key("tracedecay/autotrack/pr/5") ); let reloaded = load_branch_meta(data_root.path()).unwrap(); - assert!(reloaded.is_tracked("pr/5")); - assert!(data_root.path().join("branches/pr_5.db").exists()); + assert!(reloaded.is_tracked("tracedecay/autotrack/pr/5")); } #[tokio::test] @@ -309,9 +303,9 @@ async fn reconcile_activates_discovered_pr_head_when_scheduler_is_injected() { git(repo.path(), &["branch", "-q", "-D", "feature-11"]); let graph = Arc::new( - crate::project::TraceDecay::open_with_options_for_test( + tracedecay_project::project::TraceDecay::open_with_options_for_test( repo.path(), - crate::project::TraceDecayOpenOptions::default(), + tracedecay_project::project::TraceDecayOpenOptions::default(), ) .await .expect("open project graph"), @@ -441,14 +435,12 @@ async fn partial_discovery_suppresses_removals() { let repo_root = tempfile::tempdir().unwrap(); let mut meta = BranchMeta::new("main"); - meta.add_branch("pr/5", "branches/pr_5.db", "main"); - std::fs::create_dir_all(data_root.path().join("branches")).unwrap(); - std::fs::write(data_root.path().join("branches/pr_5.db"), b"db").unwrap(); + meta.add_branch("tracedecay/autotrack/pr/5", "main"); save_branch_meta(data_root.path(), &meta).unwrap(); let mut state = PrAutotrackState::default(); state.managed.insert( - "pr/5".to_string(), + "tracedecay/autotrack/pr/5".to_string(), ManagedPr { pr: 5, head_branch: "feature-5".to_string(), @@ -485,15 +477,14 @@ async fn partial_discovery_suppresses_removals() { load_state(data_root.path()) .expect("load managed PR state") .managed - .contains_key("pr/5"), + .contains_key("tracedecay/autotrack/pr/5"), "managed entry survives a partial discovery" ); assert!( load_branch_meta(data_root.path()) .unwrap() - .is_tracked("pr/5") + .is_tracked("tracedecay/autotrack/pr/5") ); - assert!(data_root.path().join("branches/pr_5.db").exists()); } fn init_manual_branch_repo(repo: &Path, branch: &str) { @@ -536,9 +527,9 @@ async fn manual_branch_activates_when_scheduler_is_injected() { init_manual_branch_repo(repo.path(), "feature-manual"); let graph = Arc::new( - crate::project::TraceDecay::open_with_options_for_test( + tracedecay_project::project::TraceDecay::open_with_options_for_test( repo.path(), - crate::project::TraceDecayOpenOptions::default(), + tracedecay_project::project::TraceDecayOpenOptions::default(), ) .await .expect("open project graph"), @@ -595,9 +586,9 @@ async fn retained_linked_worktree_honors_parent_native_graph_refusal() { let linked = linked_parent.path().join("linked"); init_manual_branch_repo(repo.path(), "feature-retained-refusal"); - let graph = crate::project::TraceDecay::open_with_options_for_test( + let graph = tracedecay_project::project::TraceDecay::open_with_options_for_test( repo.path(), - crate::project::TraceDecayOpenOptions::default(), + tracedecay_project::project::TraceDecayOpenOptions::default(), ) .await .expect("open writable parent graph"); @@ -653,9 +644,9 @@ async fn retained_linked_worktree_honors_parent_native_graph_refusal() { graph.close(); let graph = Arc::new( - crate::project::TraceDecay::open_read_only_with_options_for_test( + tracedecay_project::project::TraceDecay::open_read_only_with_options_for_test( repo.path(), - crate::project::TraceDecayOpenOptions::default(), + tracedecay_project::project::TraceDecayOpenOptions::default(), ) .await .expect("reopen parent graph from persisted configuration"), @@ -748,9 +739,9 @@ async fn manual_branch_identity_keeps_slashed_and_underscored_names_disjoint() { git(repo.path(), &["checkout", "-q", "main"]); let graph = Arc::new( - crate::project::TraceDecay::open_with_options_for_test( + tracedecay_project::project::TraceDecay::open_with_options_for_test( repo.path(), - crate::project::TraceDecayOpenOptions::default(), + tracedecay_project::project::TraceDecayOpenOptions::default(), ) .await .unwrap(), @@ -798,9 +789,9 @@ async fn manual_branch_stages_new_head_without_replacing_published_worktree() { let repo = tempfile::tempdir().unwrap(); init_manual_branch_repo(repo.path(), "feature/advance"); let graph = Arc::new( - crate::project::TraceDecay::open_with_options_for_test( + tracedecay_project::project::TraceDecay::open_with_options_for_test( repo.path(), - crate::project::TraceDecayOpenOptions::default(), + tracedecay_project::project::TraceDecayOpenOptions::default(), ) .await .unwrap(), @@ -969,9 +960,9 @@ async fn manual_branch_activation_refuses_exact_lifecycle_contention_before_muta let repo = tempfile::tempdir().unwrap(); init_manual_branch_repo(repo.path(), "feature/contended"); let graph = Arc::new( - crate::project::TraceDecay::open_with_options_for_test( + tracedecay_project::project::TraceDecay::open_with_options_for_test( repo.path(), - crate::project::TraceDecayOpenOptions::default(), + tracedecay_project::project::TraceDecayOpenOptions::default(), ) .await .unwrap(), @@ -1013,9 +1004,9 @@ async fn failed_manual_branch_sealing_retires_the_exact_mount_worktree_and_track let repo = tempfile::tempdir().unwrap(); init_manual_branch_repo(repo.path(), "feature/failure-cleanup"); let graph = Arc::new( - crate::project::TraceDecay::open_with_options_for_test( + tracedecay_project::project::TraceDecay::open_with_options_for_test( repo.path(), - crate::project::TraceDecayOpenOptions::default(), + tracedecay_project::project::TraceDecayOpenOptions::default(), ) .await .unwrap(), @@ -1075,9 +1066,9 @@ async fn manual_branch_fails_closed_without_scheduler_before_git_or_state_mutati init_manual_branch_repo(repo.path(), "feature-denied"); let graph = Arc::new( - crate::project::TraceDecay::open_with_options_for_test( + tracedecay_project::project::TraceDecay::open_with_options_for_test( repo.path(), - crate::project::TraceDecayOpenOptions::default(), + tracedecay_project::project::TraceDecayOpenOptions::default(), ) .await .expect("open project graph"), @@ -1115,9 +1106,9 @@ async fn manual_branch_missing_ref_is_typed_failure() { init_manual_branch_repo(repo.path(), "feature-present"); let graph = Arc::new( - crate::project::TraceDecay::open_with_options_for_test( + tracedecay_project::project::TraceDecay::open_with_options_for_test( repo.path(), - crate::project::TraceDecayOpenOptions::default(), + tracedecay_project::project::TraceDecayOpenOptions::default(), ) .await .expect("open project graph"), @@ -1167,9 +1158,9 @@ async fn cancelled_activation_keeps_its_lifecycle_owner_bounded_during_stalled_e let branch = "feature/stalled-exact-read"; init_manual_branch_repo(repo.path(), branch); let graph = Arc::new( - crate::project::TraceDecay::open_with_options_for_test( + tracedecay_project::project::TraceDecay::open_with_options_for_test( repo.path(), - crate::project::TraceDecayOpenOptions::default(), + tracedecay_project::project::TraceDecayOpenOptions::default(), ) .await .expect("open project graph"), diff --git a/crates/tracedecay/src/daemon/production_harness.rs b/crates/tracedecay/src/daemon/production_harness.rs index dd65b74e08..4f056b1c78 100644 --- a/crates/tracedecay/src/daemon/production_harness.rs +++ b/crates/tracedecay/src/daemon/production_harness.rs @@ -463,7 +463,7 @@ async fn mount_one_production_composition_project( allow_initialize_root_routing: false, tool_list_changed_capable: false, catalog_version: String::new(), - moved_store_adoption: crate::project::MovedStoreAdoption::Never, + moved_store_adoption: tracedecay_project::project::MovedStoreAdoption::Never, }; let (canonical_project_path, _) = project_route_for_handshake(&handshake)?; let composition = stores @@ -534,7 +534,8 @@ impl ProductionProjectCompositionHarnessV1 { isolation_root: impl AsRef, project_roots: impl IntoIterator, ) -> ProductionHarnessOpenFuture { - let live_profile_root = crate::config::user_data_dir().filter(|path| path.exists()); + let live_profile_root = + tracedecay_project::config::user_data_dir().filter(|path| path.exists()); Self::open_with_live_profile_root( isolation_root.as_ref().to_path_buf(), project_roots.into_iter().collect(), @@ -552,7 +553,8 @@ impl ProductionProjectCompositionHarnessV1 { isolation_root: impl AsRef, project_roots: impl IntoIterator, ) -> ProductionHarnessOpenFuture { - let live_profile_root = crate::config::user_data_dir().filter(|path| path.exists()); + let live_profile_root = + tracedecay_project::config::user_data_dir().filter(|path| path.exists()); Self::open_with_live_profile_root( isolation_root.as_ref().to_path_buf(), project_roots.into_iter().collect(), @@ -568,7 +570,8 @@ impl ProductionProjectCompositionHarnessV1 { project_roots: impl IntoIterator, scope_prefix: impl Into, ) -> ProductionHarnessOpenFuture { - let live_profile_root = crate::config::user_data_dir().filter(|path| path.exists()); + let live_profile_root = + tracedecay_project::config::user_data_dir().filter(|path| path.exists()); Self::open_with_live_profile_root( isolation_root.as_ref().to_path_buf(), project_roots.into_iter().collect(), @@ -598,7 +601,7 @@ impl ProductionProjectCompositionHarnessV1 { // product-runtime registration, so the canonical fixture is this // composition's provider; without it daemon bootstrap and version // reporting answer the typed missing-provider state. - crate::product_runtime::register_fixture_product_runtime(); + tracedecay_project::product_runtime::register_fixture_product_runtime(); let isolated = isolate_production_composition_roots( isolation_root, project_roots, diff --git a/crates/tracedecay/src/daemon/production_harness/configuration_idempotency_journey_test.rs b/crates/tracedecay/src/daemon/production_harness/configuration_idempotency_journey_test.rs index 76aec9b699..e424bcb678 100644 --- a/crates/tracedecay/src/daemon/production_harness/configuration_idempotency_journey_test.rs +++ b/crates/tracedecay/src/daemon/production_harness/configuration_idempotency_journey_test.rs @@ -94,7 +94,7 @@ async fn cli_configuration_set( maximum_millis, 15_000, "CLI configuration effects use the catalog-owned 15 second deadline" ); - let observed_at = tracedecay_daemon_protocol::invocation_now_micros(); + let observed_at = tracedecay_contracts::now_micros(); let deadline = Deadline::new(tracedecay_domain::UtcMicros( observed_at.0 + i64::try_from(maximum_millis).expect("deadline fits") * 1_000, )) @@ -188,7 +188,7 @@ async fn configuration_batch_via_surface( maximum_millis, 15_000, "user configuration effects use the catalog-owned 15 second deadline" ); - let observed_at = tracedecay_daemon_protocol::invocation_now_micros(); + let observed_at = tracedecay_contracts::now_micros(); let deadline = Deadline::new(tracedecay_domain::UtcMicros( observed_at.0 + i64::try_from(maximum_millis).expect("deadline fits") * 1_000, )) diff --git a/crates/tracedecay/src/daemon/production_harness/generation_retention_test.rs b/crates/tracedecay/src/daemon/production_harness/generation_retention_test.rs index 002911a44e..e21dfe3e03 100644 --- a/crates/tracedecay/src/daemon/production_harness/generation_retention_test.rs +++ b/crates/tracedecay/src/daemon/production_harness/generation_retention_test.rs @@ -12,7 +12,7 @@ use super::*; use crate::daemon::maintenance::project_store_maintenance_lease; use tracedecay_code_index_retention::code_index_generations::{ CodeGenerationRetentionErrorV1, MAX_CODE_GENERATION_RETENTION_BATCH_V1, - prepare_next_code_generation_retention_cancellable, + code_generation_segments_root, prepare_next_code_generation_retention_cancellable, }; use tracedecay_maintenance::tick::{MaintenanceContinuation, MaintenanceTickOutcome}; use tracedecay_runtime_core::path_safety::canonical_existing_identity; @@ -182,7 +182,7 @@ async fn mounted_code_generation_retention_continues_capped_segment_reclamation( assert!(first_source_file.is_file()); let observations = resources.store_administration.store_telemetry_sampling(); - let cancellation = tracedecay_session_memory::context::CancellationToken::new(); + let cancellation = tracedecay_runtime_core::cancellation::CancellationToken::new(); let findings_before = tracedecay_daemon_service::doctor_kernel::collect_code_generation_retention_findings( schedulers, @@ -231,7 +231,7 @@ async fn mounted_code_generation_retention_continues_capped_segment_reclamation( .expect("serving code generation survives retention"); assert_eq!(serving, latest); - let segment_root = code_store_root.join("code-generation-segments-v1"); + let segment_root = code_generation_segments_root(&code_store_root); let orphan_segments = (0..=MAX_CODE_GENERATION_RETENTION_BATCH_V1) .map(|index| { let bytes = format!("unreferenced production segment {index}"); diff --git a/crates/tracedecay/src/daemon/production_harness/lcm_preserved_profile_journey_test.rs b/crates/tracedecay/src/daemon/production_harness/lcm_preserved_profile_journey_test.rs index 8ad630e212..d4c4f81a0d 100644 --- a/crates/tracedecay/src/daemon/production_harness/lcm_preserved_profile_journey_test.rs +++ b/crates/tracedecay/src/daemon/production_harness/lcm_preserved_profile_journey_test.rs @@ -174,7 +174,7 @@ fn assert_admission_round(round: &AdmissionRound, session_elapsed: Duration, ses assert_under_budget("lexical admission", *lexical_elapsed, ADMISSION_BUDGET); assert_admitted_with_results("tracedecay_grep", lexical, "results"); assert_under_budget("graph admission", *graph_elapsed, ADMISSION_BUDGET); - assert_admitted_with_results("tracedecay_body", graph, "matches"); + assert_admitted_with_results("tracedecay_find_exact_symbol", graph, "matches"); assert_under_budget( "ordinary session admission", session_elapsed, @@ -655,7 +655,7 @@ async fn wait_for_preserved_discovery( #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn preserved_profile_lcm_discovery_converges_without_blocking_retrieval() { - let _profile = crate::config::PinnedUserDataDir::new(); + let _profile = tracedecay_project::config::PinnedUserDataDir::new(); let isolation = tempfile::TempDir::new().expect("isolated home/profile"); let project = isolation.path().join("project"); seed_project(&project); @@ -702,8 +702,8 @@ async fn preserved_profile_lcm_discovery_converges_without_blocking_retrieval() timed_raw( &harness, &project, - "tracedecay_body", - json!({"symbol": PROBE_SYMBOL, "format": "json"}), + "tracedecay_find_exact_symbol", + json!({"name": PROBE_SYMBOL, "format": "json"}), ), // This round proves admission, so one evidence result keeps // response-handle storage outside the concurrency assertion. diff --git a/crates/tracedecay/src/daemon/production_harness/project_server_capacity_journey_test.rs b/crates/tracedecay/src/daemon/production_harness/project_server_capacity_journey_test.rs index 859ebe30f2..0e8a13d58b 100644 --- a/crates/tracedecay/src/daemon/production_harness/project_server_capacity_journey_test.rs +++ b/crates/tracedecay/src/daemon/production_harness/project_server_capacity_journey_test.rs @@ -32,7 +32,7 @@ async fn open_project_composition( allow_initialize_root_routing: false, tool_list_changed_capable: false, catalog_version: String::new(), - moved_store_adoption: crate::project::MovedStoreAdoption::Never, + moved_store_adoption: tracedecay_project::project::MovedStoreAdoption::Never, }; let (canonical_project_path, _) = project_route_for_handshake(&handshake)?; resources diff --git a/crates/tracedecay/src/daemon/project_composition.rs b/crates/tracedecay/src/daemon/project_composition.rs index 26001bd62a..62d708fe4b 100644 --- a/crates/tracedecay/src/daemon/project_composition.rs +++ b/crates/tracedecay/src/daemon/project_composition.rs @@ -446,7 +446,7 @@ enum GraphOpen { /// The opened graph and the route-wide choices resolved from its configuration. struct OpenedProjectGraph { - cg: Arc, + cg: Arc, key: ProjectServerKey, runtime_configuration: tracedecay_configuration::config::PinnedRuntimeConfiguration, project_database_is_read_only: bool, @@ -497,7 +497,7 @@ impl ComposedCoreServer { fn publish_route_ports( &self, context: crate::mcp::server::McpServerConstructionContext, - cg: &Arc, + cg: &Arc, invocation: &DaemonInvocationState, ) -> crate::mcp::server::McpServerConstructionContext { let ports = &self.ports; @@ -1022,15 +1022,23 @@ impl ProjectOpenInputs<'_> { .await?, ) }; - // The source-edit owner is the core's only runtime component, so its - // registration is what creates the registry slot this publication - // attempt fences. A read-only database registers none. + // Graph reads are served from the core onward, read-only or not; the + // full server re-registers as their owner once it is swapped in. + resolved + .register_graph_tool_owner( + self.canonical_project_path, + core.ports.code_index.scope.clone(), + ) + .await?; + // The source-edit and graph-tool owners are the core's runtime + // components, so their registration creates the registry slot this + // publication attempt fences. let publication_attempt = self .invocation .service .project_runtimes .begin_publication(self.canonical_project_path); - if publication_attempt.is_none() && core_source_edit_mutation.is_some() { + if publication_attempt.is_none() { return Err(TraceDecayError::Config { message: "project runtime disappeared before its publication began".to_owned(), }); @@ -1059,7 +1067,7 @@ impl ProjectOpenInputs<'_> { #[hotpath::measure(label = "daemon.project.compose.admit_sessions", future = true)] async fn admit_session_databases( &self, - cg: &Arc, + cg: &Arc, project_id: &tracedecay_domain::ProjectId, project_database_is_read_only: bool, ) -> Result { @@ -1409,6 +1417,12 @@ impl ProjectOpenInputs<'_> { ) -> Result<()> { let full_setup_started = Instant::now(); project_open_cancellation_checkpoint(self.cancellation)?; + full_server + .register_graph_tool_owner( + self.canonical_project_path, + core.ports.code_index.scope.clone(), + ) + .await?; // The shared invocation registry admits one source-edit owner per // project root. Core publication already registered it; the full // upgrade reuses that owner and marks its mutation gate ready after @@ -1663,10 +1677,10 @@ struct ProjectCodeIndexAuthorities { generation_census_reader: tracedecay_runtime_core::runtime_telemetry::GenerationCensusReader, graph_read_admission_port: crate::mcp::server::CodeGraphReadAdmissionPort, search_authority: tracedecay_query::code_search::CodeIndexSearchAuthorityV1, - search_executor: crate::mcp::server::CodeIndexSearchExecutor, - similar_executor: crate::mcp::server::CodeIndexSimilarExecutor, - redundancy_executor: crate::mcp::server::CodeIndexRedundancyExecutor, - branch_diff_executor: crate::mcp::server::CodeIndexBranchDiffExecutor, + search_executor: tracedecay_query::code_search::CodeIndexSearchExecutor, + similar_executor: tracedecay_query::code_search::CodeIndexSimilarExecutor, + redundancy_executor: tracedecay_query::code_search::CodeIndexRedundancyExecutor, + branch_diff_executor: tracedecay_query::code_search::CodeIndexBranchDiffExecutor, } /// Resolve the project's search identity and bind every code-index read port to @@ -1674,7 +1688,7 @@ struct ProjectCodeIndexAuthorities { /// not the handshake path, so a relocated store still binds its own scope. fn project_code_index_authorities( invocation: &DaemonInvocationState, - cg: &Arc, + cg: &Arc, canonical_project_path: &Path, authoritative_project_id: &str, profile_identity: &profile_identity::LocalProfileIdentityAuthorityV1, @@ -1808,7 +1822,7 @@ fn project_dashboard_pr_autotrack_reader() /// never fatal: telemetry must not fail an otherwise healthy project open. fn register_route_store_telemetry( sampling: &tracedecay_maintenance::telemetry::StoreTelemetrySamplingRegistry, - cg: &Arc, + cg: &Arc, scope: &tracedecay_contracts::ResolvedScope, session_databases: [&tracedecay_global_db::RegisteredGlobalDb; 3], ) { diff --git a/crates/tracedecay/src/daemon/project_composition/code_index_activation.rs b/crates/tracedecay/src/daemon/project_composition/code_index_activation.rs index 8b58f4c51c..d521c861e4 100644 --- a/crates/tracedecay/src/daemon/project_composition/code_index_activation.rs +++ b/crates/tracedecay/src/daemon/project_composition/code_index_activation.rs @@ -9,6 +9,7 @@ use tracedecay_code_index_runtime::code_index_scheduler::{ CodeIndexDemandAdmissionV1, CodeIndexDemandV1, query_runtime::QueryRuntimeMountErrorV1, }; use tracedecay_runtime_core::logging::log_daemon_event; +use tracedecay_session_temporal_store::SessionTemporalAccess; /// Inputs the deferred mount closure re-clones on every activation attempt. /// Bundled so the builder keeps one argument list instead of ten positional @@ -233,7 +234,7 @@ async fn mount_core_query_authority_from_project_sessions( "project session database is not mounted".to_owned(), )); }; - let cursor_keys = session_db + let cursor_keys = SessionTemporalAccess::new(&*session_db) .load_session_cursor_key_provider_result() .await .map_err(|error| QueryRuntimeMountErrorV1::FallbackKeyUnavailable(error.to_string()))?; @@ -545,10 +546,10 @@ mod tests { assert!( sink( root.clone(), - crate::mcp::server::CodeIndexDemandV1::OperatorReconcile + tracedecay_code_index_runtime::code_index_scheduler::CodeIndexDemandV1::OperatorReconcile ) .await - == crate::mcp::server::CodeIndexDemandAdmissionV1::Queued, + == tracedecay_code_index_runtime::code_index_scheduler::CodeIndexDemandAdmissionV1::Queued, "a pre-mount reconcile request must be accepted, not dropped" ); @@ -622,12 +623,12 @@ mod tests { let hook_sink = code_index_hook_sink(Arc::clone(&activation)); assert_eq!( hook_sink(root.clone(), vec!["lib.rs".to_owned()]).await, - crate::mcp::server::CodeIndexDemandAdmissionV1::RefusedByPolicy + tracedecay_code_index_runtime::code_index_scheduler::CodeIndexDemandAdmissionV1::RefusedByPolicy ); let probe_sink = code_index_freshness_probe_sink(registry.clone(), Arc::clone(&activation)); assert_eq!( probe_sink(root.clone()).await, - crate::mcp::server::CodeIndexDemandAdmissionV1::RefusedByPolicy + tracedecay_code_index_runtime::code_index_scheduler::CodeIndexDemandAdmissionV1::RefusedByPolicy ); let sink = code_index_reconcile_sink(Arc::clone(&activation)); // The daemon's own whole-worktree demands, a `workspaceOpen` / @@ -639,10 +640,10 @@ mod tests { assert!( sink( root.clone(), - crate::mcp::server::CodeIndexDemandV1::Reconcile + tracedecay_code_index_runtime::code_index_scheduler::CodeIndexDemandV1::Reconcile ) .await - == crate::mcp::server::CodeIndexDemandAdmissionV1::RefusedByPolicy, + == tracedecay_code_index_runtime::code_index_scheduler::CodeIndexDemandAdmissionV1::RefusedByPolicy, "an automatic whole-worktree demand must honour the linked-worktree watch policy" ); for _ in 0..8 { @@ -656,10 +657,10 @@ mod tests { assert!( sink( root.clone(), - crate::mcp::server::CodeIndexDemandV1::OperatorReconcile + tracedecay_code_index_runtime::code_index_scheduler::CodeIndexDemandV1::OperatorReconcile ) .await - == crate::mcp::server::CodeIndexDemandAdmissionV1::Queued, + == tracedecay_code_index_runtime::code_index_scheduler::CodeIndexDemandAdmissionV1::Queued, "explicit reconcile demand must be accepted on a linked worktree" ); tokio::time::timeout(std::time::Duration::from_secs(5), async { diff --git a/crates/tracedecay/src/daemon/project_composition/future_size_tests.rs b/crates/tracedecay/src/daemon/project_composition/future_size_tests.rs index cc5760421a..13096b01dc 100644 --- a/crates/tracedecay/src/daemon/project_composition/future_size_tests.rs +++ b/crates/tracedecay/src/daemon/project_composition/future_size_tests.rs @@ -103,7 +103,7 @@ fn awaited_sizes() -> Vec<(&'static str, usize)> { ), ( "TraceDecay::register_project_store_in_global_registry", - future_size(crate::project::TraceDecay::register_project_store_in_global_registry), + future_size(tracedecay_project::project::TraceDecay::register_project_store_in_global_registry), ), ( "McpServer::new_with_context", diff --git a/crates/tracedecay/src/daemon/project_open_admission.rs b/crates/tracedecay/src/daemon/project_open_admission.rs index f487318c07..adc805e4fb 100644 --- a/crates/tracedecay/src/daemon/project_open_admission.rs +++ b/crates/tracedecay/src/daemon/project_open_admission.rs @@ -15,6 +15,7 @@ use tracedecay_contracts::project_open::{ ProjectOpenStatusReasonV1, ProjectOpenStatusStateV1, ProjectOpenStatusV1, }; use tracedecay_daemon_identity::authority; +use tracedecay_daemon_service::shutdown::DAEMON_TASK_ABORT_DEADLINE; #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub(super) struct ProjectServerKey { @@ -168,8 +169,7 @@ struct RefusedStoreFileIdentityV1 { } /// Every graph database the refused project store carried when the refusal -/// was recorded: the root graph DB plus the per-branch graph DBs under -/// `branches/`. Comparing the whole map catches deletions, replacements, and +/// was recorded. Comparing the whole map catches deletions, replacements, and /// newly recreated databases alike. #[derive(Clone, Debug, PartialEq, Eq)] struct RefusedStoreFingerprintV1 { @@ -206,16 +206,6 @@ fn refused_store_fingerprint(route: &ProjectRouteKey) -> Option Option (message.contains("identity cutover conflict") - || message.contains("ambiguous legacy profile stores") - || message.contains("enrollment marker did not resolve a profile store")) - .then_some(PROJECT_OPEN_FAILURE_RETRY_BACKOFF), // This audit's whole job is to read persisted rows and judge them, so // its verdict is a property of the stored data: a row rejected now is // rejected identically 250ms from now. Back off for the whole family @@ -919,6 +905,14 @@ impl ProjectOpenTasks { } } + /// Signals every admitted open to stop at its next cancellation boundary + /// without waiting; `shutdown` joins them. + pub(super) fn cancel_all(&self) { + for entry in self.lock_registry().routes.values() { + entry.cancellation.cancel(); + } + } + #[hotpath::skip] pub(super) async fn shutdown(&self) -> bool { self.shutdown_with_deadline(DAEMON_TASK_ABORT_DEADLINE, DAEMON_TASK_ABORT_DEADLINE) @@ -1204,7 +1198,7 @@ impl ProjectRouteKey { impl ProjectServerKey { pub(super) fn from_open_project( - cg: &crate::project::TraceDecay, + cg: &tracedecay_project::project::TraceDecay, handshake: &DaemonHandshake, ) -> Result { let layout = cg.store_layout(); @@ -1284,7 +1278,7 @@ mod refused_store_invalidation_tests { fn seed_refused_store(profile_root: &Path, project_root: &Path) -> PathBuf { let data_root = store_data_root(profile_root, project_root); std::fs::create_dir_all(&data_root).unwrap(); - let db_path = data_root.join(crate::config::db_filename(&data_root)); + let db_path = data_root.join(tracedecay_project::config::db_filename(&data_root)); std::fs::write(&db_path, b"refused-store-stand-in").unwrap(); db_path } @@ -1348,32 +1342,6 @@ mod refused_store_invalidation_tests { ProjectOpenTasks::wait_for_completion(state).await.unwrap(); } - /// Per-branch graph DBs are part of the refused store's fingerprint, so - /// a reset that removes only `branches/*.db` also clears the refusal. - #[tokio::test] - async fn branch_graph_db_reset_invalidates_the_cached_refusal() { - let temp = tempfile::TempDir::new().unwrap(); - let profile_root = temp.path().join("profile"); - let project_root = temp.path().join("project"); - std::fs::create_dir_all(&project_root).unwrap(); - seed_refused_store(&profile_root, &project_root); - let branches_dir = store_data_root(&profile_root, &project_root).join("branches"); - std::fs::create_dir_all(&branches_dir).unwrap(); - let branch_db = branches_dir.join("develop.db"); - std::fs::write(&branch_db, b"refused-branch-stand-in").unwrap(); - let route = route_for(&profile_root, &project_root); - let tasks = ProjectOpenTasks::default(); - record_reset_required_failure(&tasks, route.clone()).await; - assert!(tasks.cached_failure(&route).is_some()); - - std::fs::remove_file(&branch_db).unwrap(); - - assert!( - tasks.cached_failure(&route).is_none(), - "a branch graph DB reset must invalidate the cached refusal" - ); - } - /// The invalidation is scoped to typed `ResetRequired` refusals: other /// backed-off failures carry no store fingerprint and keep their plain /// time-based backoff even when store files change. diff --git a/crates/tracedecay/src/daemon/project_open_handshake.rs b/crates/tracedecay/src/daemon/project_open_handshake.rs index b71da29e1b..cfff98134b 100644 --- a/crates/tracedecay/src/daemon/project_open_handshake.rs +++ b/crates/tracedecay/src/daemon/project_open_handshake.rs @@ -18,11 +18,11 @@ pub(super) async fn open_project_for_handshake( project_path: &Path, handshake: &DaemonHandshake, store_administration: &StoreAdministration, -) -> Result { +) -> Result { let open_options = crate::daemon::handshake_open_options(handshake); let registry_database = store_administration.registered_profile_database().await?; let (store_layout, first_touch) = match Box::pin( - crate::project::TraceDecay::resolve_registered_configuration_layout( + tracedecay_project::project::TraceDecay::resolve_registered_configuration_layout( project_path, &open_options, registry_database.as_ref(), @@ -38,7 +38,7 @@ pub(super) async fn open_project_for_handshake( // fallback below bootstrap it. Err(err) if handshake.allow_init && is_unregistered_identity_error(&err) => ( Box::pin( - crate::project::TraceDecay::resolve_first_touch_configuration_layout_with_adoption( + tracedecay_project::project::TraceDecay::resolve_first_touch_configuration_layout_with_adoption( project_path, &open_options, registry_database.as_ref(), @@ -89,7 +89,7 @@ pub(super) async fn open_project_for_handshake( // and durable store authority; project composition schedules the maintained // bounded code-index owner after publication. let open_result = Box::pin( - crate::project::TraceDecay::open_with_registered_configuration( + tracedecay_project::project::TraceDecay::open_with_registered_configuration( project_path, open_options.clone(), store_layout.clone(), @@ -103,7 +103,7 @@ pub(super) async fn open_project_for_handshake( Ok(cg) => Ok(cg), Err(open_err) if is_readonly_database_error(&open_err) => { match Box::pin( - crate::project::TraceDecay::open_read_only_with_registered_configuration( + tracedecay_project::project::TraceDecay::open_read_only_with_registered_configuration( project_path, open_options, store_layout, @@ -127,7 +127,7 @@ pub(super) async fn open_project_for_handshake( // activation owner performs indexing after admission, so opening a // project never waits for a repository scan or rebuild. Box::pin( - crate::project::TraceDecay::init_with_registered_configuration( + tracedecay_project::project::TraceDecay::init_with_registered_configuration( project_path, open_options, store_layout, diff --git a/crates/tracedecay/src/daemon/project_open_orchestration.rs b/crates/tracedecay/src/daemon/project_open_orchestration.rs index a8609a7f75..5461147607 100644 --- a/crates/tracedecay/src/daemon/project_open_orchestration.rs +++ b/crates/tracedecay/src/daemon/project_open_orchestration.rs @@ -5,6 +5,7 @@ //! and a draining daemon never starts a new one. use super::*; +use tracedecay_daemon_service::shutdown::DaemonLifecycle; use tracedecay_runtime_core::logging::log_daemon_event; /// Bounds how long a foreground request waits for a route's background open. @@ -64,9 +65,18 @@ where // mid-statement. The lifecycle activity remains held until the task // reports its terminal outcome and shutdown explicitly joins it. let result = Box::pin(open_project_server(cancellation.clone())).await; + if cancellation.is_cancelled() { + log_daemon_event( + "project_server_warmup", + &[ + ("outcome", "cancelled".to_string()), + ("project", project_path.display().to_string()), + ], + ); + return Err(result.err().unwrap_or_else(project_open_cancellation_error)); + } match result { Ok(server) => { - project_open_cancellation_checkpoint(&cancellation)?; if let Some(initialize_request) = initialize_request { // Preserve the regular initialize side effect that records // the negotiated MCP client name on the real server. @@ -78,9 +88,6 @@ where Ok(()) } Err(error) => { - if cancellation.is_cancelled() { - return Err(error); - } log_daemon_event( "project_server_warmup", &[ @@ -271,7 +278,7 @@ fn remote_deleted_project_route_error(identity: &str) -> TraceDecayError { /// manufacturing a new identity. /// /// The profile registry is a *derived* index: the authoritative identity chain -/// in [`crate::project::TraceDecay::resolve_registered_configuration_layout`] +/// in [`tracedecay_project::project::TraceDecay::resolve_registered_configuration_layout`] /// consults the project's own enrollment marker (and the repository-identity /// marker) BEFORE it ever asks the registry, and a successful open republishes /// the registry rows via `register_project_store_in_global_registry`. A guard diff --git a/crates/tracedecay/src/daemon/project_open_owners.rs b/crates/tracedecay/src/daemon/project_open_owners.rs index c94b65cc5a..d2eee31f41 100644 --- a/crates/tracedecay/src/daemon/project_open_owners.rs +++ b/crates/tracedecay/src/daemon/project_open_owners.rs @@ -38,6 +38,7 @@ use tracedecay_daemon_service::{ use tracedecay_domain::errors::{Result, TraceDecayError}; use tracedecay_lsp::analyzer::broker::AdmittedLspProvider; use tracedecay_lsp::analyzer::client::LspRefreshTimeouts; +use tracedecay_session_temporal_store::SessionTemporalAccess; mod advisory_runtime; mod automation_effect_recovery; @@ -80,7 +81,7 @@ async fn install_project_open_source_edit_owners( pub(crate) async fn install_project_open_source_edit_preview_owner( server: &McpServer, - graph: Arc, + graph: Arc, code_graph: Arc, project_root: &Path, project_id: &str, @@ -130,6 +131,11 @@ pub(crate) async fn install_project_open_source_edit_owners_for_test( if server.daemon_invocation_service().is_none() { return Ok(false); } + if let Some(scope) = server.admitted_project_scope() { + server + .register_graph_tool_owner(graph.project_root(), scope) + .await?; + } let Some(code_graph) = server.code_graph_projection_read_port() else { // A directly constructed test server carries no production code-graph // projection port, so the daemon-owned source-edit authority cannot @@ -812,7 +818,10 @@ async fn register_project_query_authority( session_db: tracedecay_global_db::RegisteredGlobalDbLeaseV1, scope: ResolvedScope, ) { - let cursor_keys = match session_db.load_session_cursor_key_provider_result().await { + let cursor_keys = match SessionTemporalAccess::new(&*session_db) + .load_session_cursor_key_provider_result() + .await + { Ok(cursor_keys) => cursor_keys, Err(error) => { tracing::debug!( diff --git a/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime.rs b/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime.rs index afc9b90c12..0f6e299ff1 100644 --- a/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime.rs +++ b/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime.rs @@ -75,15 +75,15 @@ use tracedecay_lsp::{ DiagnosticTrigger, FeedbackCycleRequest, FeedbackCycleRuntimePort, LspRuntimeFailure, LspRuntimeFuture, }; -use tracedecay_session_memory::context::MonotonicDeadline; +use tracedecay_runtime_core::cancellation::MonotonicDeadline; use super::{DaemonInvocationState, POLICY_REVISION_V1, register_project_query_authority}; use crate::mcp::McpServer; -use tracedecay_agent_hosts::agents::context_scout::owner::ProjectContextScoutOwnerV1; -use tracedecay_agent_hosts::agents::context_scout::ports::{ +use tracedecay_agent_hosts::agents::context_scout::address_registry::{ ContextScoutAuthorityPinV1, ContextScoutCanonicalInputAssemblerV1, ContextScoutConfigurationPinV1, ProjectContextScoutAddressRegistryV1, }; +use tracedecay_agent_hosts::agents::context_scout::owner::ProjectContextScoutOwnerV1; use tracedecay_agent_hosts::agents::context_scout::{ ContextScoutDeliverySelectionInputV1, ContextScoutRuntimeOutcomeV1, ContextScoutServiceStateV1, ContextScoutTriggerV1, @@ -634,11 +634,11 @@ impl ProductionFeedbackCycleAuthorizationPort for ProjectOpenFeedbackCycleAuthor async fn install_project_open_context_scout_configuration( owner: &ProjectContextScoutOwnerV1, pin: ContextScoutConfigurationPinV1, - model_config: &tracedecay_automation_runtime::automation::config::AutomationConfig, + model_config: tracedecay_agent_hosts::agents::context_scout::model::ContextScoutModelConfig<'_>, ) -> Result<()> { let admitted_model_config = pin.control().model_path.and_then(|expected| { (tracedecay_agent_hosts::agents::context_scout::model::context_scout_backend_from_automation_config( - model_config, + model_config.automation, ) == expected) .then_some(model_config) }); @@ -655,7 +655,7 @@ async fn install_project_open_context_scout_configuration( /// exact current-generation authority that maps saved-edit hooks back to /// indexed documents. struct ProjectOpenScoutProducerV1 { - graph: Arc, + graph: Arc, scout_owner: Arc, scout_registry: Arc, feedback_cycle: tokio::sync::RwLock, @@ -998,7 +998,10 @@ async fn run_production_hook_cycle( if install_project_open_context_scout_configuration( producer.scout_owner.as_ref(), scout_configuration.clone(), - &model_config, + tracedecay_agent_hosts::agents::context_scout::model::ContextScoutModelConfig { + automation: &model_config, + codex: &pinned_configuration.config().lcm_summarizers.codex, + }, ) .await .is_err() @@ -1592,7 +1595,10 @@ async fn register_production_advisory_owner( install_project_open_context_scout_configuration( scout_owner.as_ref(), scout_configuration, - &model_config, + tracedecay_agent_hosts::agents::context_scout::model::ContextScoutModelConfig { + automation: &model_config, + codex: &configuration.config().lcm_summarizers.codex, + }, ) .await?; let scout_registry = invocation @@ -2002,7 +2008,7 @@ async fn register_project_proximity_read_authority( /// unavailable until that upgrade replaces this owner. #[derive(Clone)] struct ProjectOpenProximityReadOwnerV1 { - graph: Arc, + graph: Arc, scope: tracedecay_contracts::ResolvedScope, project_root: std::path::PathBuf, feedback_scope: FeedbackScopeV1, diff --git a/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/model.rs b/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/model.rs index f9eeeba590..a8fef0c750 100644 --- a/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/model.rs +++ b/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/model.rs @@ -5,7 +5,7 @@ use tracedecay_application::lsp_runtime::DaemonLspSessionFactory; use tracedecay_contracts::{ApplicationProblem, Deadline}; use tracedecay_domain::UtcMicros; use tracedecay_lsp::analyzer::broker::{DiagnosticBroker, MountedLspProvider}; -use tracedecay_session_memory::context::MonotonicDeadline; +use tracedecay_runtime_core::cancellation::MonotonicDeadline; /// State retained after independent owners publish and consumed only after the /// durable code-index generation has mounted. @@ -13,7 +13,7 @@ pub(crate) struct ProjectOpenDependentOwnerState { pub(in crate::daemon::project_open_owners) database: tracedecay_runtime_core::db::Database, pub(in crate::daemon::project_open_owners) session_db: tracedecay_global_db::RegisteredGlobalDbLeaseV1, - pub(in crate::daemon::project_open_owners) graph: Arc, + pub(in crate::daemon::project_open_owners) graph: Arc, pub(in crate::daemon::project_open_owners) code_graph: Arc, pub(in crate::daemon::project_open_owners) scope: tracedecay_contracts::ResolvedScope, diff --git a/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/scout_journey_tests.rs b/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/scout_journey_tests.rs index b366b21c85..66fe1ce821 100644 --- a/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/scout_journey_tests.rs +++ b/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/scout_journey_tests.rs @@ -254,9 +254,16 @@ async fn project_open_edit_stop_and_explicit_feedback_preserve_privacy_and_super let pin = configured_model_pin(); let control = pin.control(); let owner = test_scout_owner(&temporary).await; - install_project_open_context_scout_configuration(owner.as_ref(), pin, &model_config) - .await - .expect("install project-open Scout configuration"); + install_project_open_context_scout_configuration( + owner.as_ref(), + pin, + tracedecay_agent_hosts::agents::context_scout::model::ContextScoutModelConfig { + automation: &model_config, + codex: &tracedecay_domain::configuration::LcmSummarizerExecutableV1::Unconfigured, + }, + ) + .await + .expect("install project-open Scout configuration"); let now = UtcMicros( i64::try_from( std::time::SystemTime::now() @@ -508,9 +515,16 @@ async fn claim_refuses_work_from_a_displaced_configuration_revision() { let first_pin = configured_model_pin_with_timeout("revision.scout.claim.first", 30); let first_control = first_pin.control(); let owner = test_scout_owner(&temporary).await; - install_project_open_context_scout_configuration(owner.as_ref(), first_pin, &model_config) - .await - .expect("install first Scout configuration"); + install_project_open_context_scout_configuration( + owner.as_ref(), + first_pin, + tracedecay_agent_hosts::agents::context_scout::model::ContextScoutModelConfig { + automation: &model_config, + codex: &tracedecay_domain::configuration::LcmSummarizerExecutableV1::Unconfigured, + }, + ) + .await + .expect("install first Scout configuration"); let now = UtcMicros(1_000_000); let input = configured_model_input_at( first_control.configuration_revision, @@ -535,7 +549,10 @@ async fn claim_refuses_work_from_a_displaced_configuration_revision() { install_project_open_context_scout_configuration( owner.as_ref(), configured_model_pin_with_timeout("revision.scout.claim.second", 31), - &model_config, + tracedecay_agent_hosts::agents::context_scout::model::ContextScoutModelConfig { + automation: &model_config, + codex: &tracedecay_domain::configuration::LcmSummarizerExecutableV1::Unconfigured, + }, ) .await .expect("install replacement Scout configuration"); @@ -615,7 +632,11 @@ async fn stock_disabled_configuration_produces_nothing() { install_project_open_context_scout_configuration( owner.as_ref(), pin, - &tracedecay_automation_runtime::automation::config::AutomationConfig::default(), + tracedecay_agent_hosts::agents::context_scout::model::ContextScoutModelConfig { + automation: + &tracedecay_automation_runtime::automation::config::AutomationConfig::default(), + codex: &tracedecay_domain::configuration::LcmSummarizerExecutableV1::Unconfigured, + }, ) .await .expect("install disabled Scout configuration"); @@ -644,7 +665,7 @@ async fn stock_disabled_configuration_produces_nothing() { &tracedecay_hooks::HookEventEnvelopeV2 { schema_version: tracedecay_hooks::HOOK_EVENT_SCHEMA_VERSION, event_id: [64; 16], - producer: tracedecay_hooks::HookHostV1::Codex, + producer: tracedecay_domain::NativeHostIdentityV1::Codex, protected_session_id: input.address.protected_session_id, project_id: input.address.project_id, repository_id: [61; 16], diff --git a/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/tests.rs b/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/tests.rs index 4ec2de0aa2..65304cd712 100644 --- a/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/tests.rs +++ b/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/tests.rs @@ -52,7 +52,9 @@ fn hook_notice_registration_is_released_with_the_published_owner() { )); } -fn hook_binding(host: tracedecay_hooks::HookHostV1) -> tracedecay_hooks::HookScopeBindingV1 { +fn hook_binding( + host: tracedecay_domain::NativeHostIdentityV1, +) -> tracedecay_hooks::HookScopeBindingV1 { let capabilities = [ tracedecay_hooks::HookEventFamily::SessionBoundary, tracedecay_hooks::HookEventFamily::PromptBoundary, @@ -86,7 +88,7 @@ fn hook_notice_dispatch_requires_a_live_daemon_binding() { "an unpublished binding set must stay typed unbound" ); - let host = tracedecay_hooks::HookHostV1::ClaudeCode; + let host = tracedecay_domain::NativeHostIdentityV1::ClaudeCode; let expires_at = UtcMicros(published_at.0 + 60_000_000); tracedecay_hooks::HookConfigurationPublisherV1::new( tracedecay_hooks::HookConfigurationFileWriterV1::new(hook_configuration_path( diff --git a/crates/tracedecay/src/daemon/project_open_owners/automation_effect_recovery.rs b/crates/tracedecay/src/daemon/project_open_owners/automation_effect_recovery.rs index 5afb90d392..620fd21387 100644 --- a/crates/tracedecay/src/daemon/project_open_owners/automation_effect_recovery.rs +++ b/crates/tracedecay/src/daemon/project_open_owners/automation_effect_recovery.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use tracedecay_automation_runtime::automation::effect_recovery::recovery_report_fields; use tracedecay_contracts::CancellationSignal; -use crate::project::TraceDecay; +use tracedecay_project::project::TraceDecay; use tracedecay_runtime_core::logging::log_daemon_event; #[hotpath::measure(label = "daemon.project.automation_recovery", future = true)] diff --git a/crates/tracedecay/src/daemon/project_open_owners/git_catalog_tests.rs b/crates/tracedecay/src/daemon/project_open_owners/git_catalog_tests.rs index 3a83440ef1..db37d0d0dd 100644 --- a/crates/tracedecay/src/daemon/project_open_owners/git_catalog_tests.rs +++ b/crates/tracedecay/src/daemon/project_open_owners/git_catalog_tests.rs @@ -1,12 +1,15 @@ -use crate::runtime_ports::compose_application_catalog_snapshot; use crate::test_support::git::GIT_FIXTURE_CONFIG; use tracedecay_application::git_intelligence::NativeGitIntelligence; use tracedecay_code_index_runtime::git_transactions::DaemonGitIndexTransactionServiceRegistry; +use tracedecay_contracts::catalog_composition::{ + CatalogCompositionError, build_application_catalog_snapshot, +}; use tracedecay_contracts::git::GitIndexTransactionPortError; use tracedecay_contracts::{ - AuthorityReceipt, CancellationContext, CapabilityGrantId, CapabilityGrantSnapshot, Deadline, - DisclosureClass, GitIndexOperationBindingV1, GitIndexPreviewRequestV1, GitIndexTransactionPort, - IdempotencyKey, OperationTermination, PolicyDecisionRef, RequestContext, RequestId, + ApplicationContractError, AuthorityReceipt, CancellationContext, CapabilityGrantId, + CapabilityGrantSnapshot, Deadline, DisclosureClass, GitIndexOperationBindingV1, + GitIndexPreviewRequestV1, GitIndexTransactionPort, IdempotencyKey, OperationTermination, + PolicyDecisionRef, RequestContext, RequestId, }; use tracedecay_daemon_service::{GRANT_HORIZON, daemon_owned_project_source_access_at}; use tracedecay_domain::git::{ @@ -20,15 +23,13 @@ use tracedecay_domain::{ }; use tracedecay_domain::{ProjectId, UtcMicros}; -fn unavailable_catalog() -> Result< - tracedecay_tool_catalog::CatalogSnapshotV1, - tracedecay_code_index_runtime::ApplicationCatalogSnapshotErrorV1, -> { - Err( - tracedecay_code_index_runtime::ApplicationCatalogSnapshotErrorV1::new( - "catalog unavailable for this independently constructed owner", +fn unavailable_catalog() +-> Result { + Err(CatalogCompositionError::Application( + ApplicationContractError::Catalog( + "catalog unavailable for this independently constructed owner".to_owned(), ), - ) + )) } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -42,17 +43,18 @@ async fn git_owner_uses_explicit_canonical_catalog_and_rechecks_authorization() git(&project_root, &["add", "."]); git(&project_root, &["commit", "-m", "fixture"]); let project_id = ProjectId::new("project.git-catalog").unwrap(); - let fixture = crate::test_support::host_admission::HostAdmissionTestRuntimeV1::project( - &profile_root, - &project_root, - project_id.clone(), - ) - .await - .unwrap(); + let fixture = + tracedecay_project::test_support::host_admission::HostAdmissionTestRuntimeV1::project( + &profile_root, + &project_root, + project_id.clone(), + ) + .await + .unwrap(); let graph = fixture .initialize_project_graph_for_test( &project_root, - crate::project::TraceDecayOpenOptions { + tracedecay_project::project::TraceDecayOpenOptions { profile_root: Some(profile_root), global_db_path: None, }, @@ -81,7 +83,7 @@ async fn git_owner_uses_explicit_canonical_catalog_and_rechecks_authorization() .await .unwrap(); let registry = - DaemonGitIndexTransactionServiceRegistry::new(compose_application_catalog_snapshot); + DaemonGitIndexTransactionServiceRegistry::new(build_application_catalog_snapshot); registry .ensure( database.clone(), @@ -109,7 +111,7 @@ async fn git_owner_uses_explicit_canonical_catalog_and_rechecks_authorization() let initial = owner.current_authority(operation).unwrap(); assert_eq!( initial.catalog_digest.as_str(), - compose_application_catalog_snapshot() + build_application_catalog_snapshot() .unwrap() .digest() .to_string() diff --git a/crates/tracedecay/src/daemon/project_open_owners/query_authority_upgrade.rs b/crates/tracedecay/src/daemon/project_open_owners/query_authority_upgrade.rs index 55b1302eaf..859e4ce876 100644 --- a/crates/tracedecay/src/daemon/project_open_owners/query_authority_upgrade.rs +++ b/crates/tracedecay/src/daemon/project_open_owners/query_authority_upgrade.rs @@ -11,6 +11,7 @@ use tracedecay_code_index_runtime::code_index_scheduler::query_runtime::{ use tracedecay_contracts::ResolvedScope; use super::DaemonInvocationState; +use tracedecay_session_temporal_store::SessionTemporalAccess; /// Spawns the deferred query-authority waiter on the project owner. /// @@ -63,7 +64,10 @@ async fn try_deferred_mount( scope: &ResolvedScope, session_db: &tracedecay_global_db::RegisteredGlobalDbLeaseV1, ) -> DeferredMountAttemptV1 { - let cursor_keys = match session_db.load_session_cursor_key_provider_result().await { + let cursor_keys = match SessionTemporalAccess::new(&**session_db) + .load_session_cursor_key_provider_result() + .await + { Ok(cursor_keys) => cursor_keys, Err(error) => { tracing::warn!( diff --git a/crates/tracedecay/src/daemon/project_routing.rs b/crates/tracedecay/src/daemon/project_routing.rs index 17af494007..d97675c0d7 100644 --- a/crates/tracedecay/src/daemon/project_routing.rs +++ b/crates/tracedecay/src/daemon/project_routing.rs @@ -80,7 +80,7 @@ pub(super) fn project_route_for_handshake( }; let canonical_project_path = tracedecay_runtime_core::path_safety::canonical_root_identity(project_path); - if crate::config::is_ambient_project_root(&canonical_project_path) { + if tracedecay_project::config::is_ambient_project_root(&canonical_project_path) { return Err(TraceDecayError::Config { message: format!( "'{}' is an ambient user/filesystem root, not an active TraceDecay code project", @@ -210,12 +210,13 @@ pub(super) async fn resolved_project_server_key( return Ok(None); } let registry_database = store_administration.registered_profile_database().await?; - let Ok(layout) = crate::project::TraceDecay::resolve_registered_configuration_layout( - canonical_project_path, - &crate::daemon::handshake_open_options(handshake), - registry_database.as_ref(), - ) - .await + let Ok(layout) = + tracedecay_project::project::TraceDecay::resolve_registered_configuration_layout( + canonical_project_path, + &crate::daemon::handshake_open_options(handshake), + registry_database.as_ref(), + ) + .await else { // The canonical open remains responsible for typed identity errors and // any permitted repair; this is only a mounted-runtime reuse path. @@ -230,7 +231,7 @@ pub(super) async fn resolved_project_server_key( tracedecay_runtime_core::worktree::detached_worktree_graph_scope(&probe_path) }); let (graph_db_path, _, fallback_warning) = - crate::project::TraceDecay::resolve_db_for_branch( + tracedecay_project::project::TraceDecay::resolve_db_for_branch( &probe_path, &data_root, graph_scope.as_deref(), diff --git a/crates/tracedecay/src/daemon/project_server_lifecycle.rs b/crates/tracedecay/src/daemon/project_server_lifecycle.rs index 250e0d7087..889fa35584 100644 --- a/crates/tracedecay/src/daemon/project_server_lifecycle.rs +++ b/crates/tracedecay/src/daemon/project_server_lifecycle.rs @@ -6,6 +6,8 @@ use super::*; use std::collections::HashSet; +use tracedecay_agent_hosts::agents::context_scout::owner::unregister_registered_context_scout_owner; +use tracedecay_agent_hosts::hooks::hook_project_id_for_layout; use tracedecay_daemon_identity::authority; use tracedecay_daemon_service::ProfileHostAdmissionBootstrapStatus; use tracedecay_daemon_service::shutdown::ShutdownStatus; @@ -162,8 +164,17 @@ pub(super) async fn shutdown_detached_project_servers( &graph.hook_store_layout().data_root, ) .await; + let scout_owner = hook_project_id_for_layout(graph.hook_store_layout()) + .map(|project_id| (project_id, graph.db_path())); drop(graph); - server.shutdown_until(deadline).await + let status = server.shutdown_until(deadline).await; + // The process-global Context Scout owner holds the project + // graph `Database`; while registered, the store runtime cannot + // close and its writer never runs the shutdown checkpoint. + if let Some((project_id, graph_db_path)) = scout_owner { + unregister_registered_context_scout_owner(project_id, &graph_db_path); + } + status }) }), ) @@ -369,10 +380,10 @@ mod shutdown_owner_tests { #[tokio::test] async fn terminal_shutdown_failure_replays_without_retaining_server_owner() { - let _pin = crate::config::PinnedUserDataDir::new(); + let _pin = tracedecay_project::config::PinnedUserDataDir::new(); let project = tempfile::tempdir().expect("project root"); let (graph, _runtime) = - crate::project::TraceDecay::init_test_fixture_with_registered_runtime( + tracedecay_project::project::TraceDecay::init_test_fixture_with_registered_runtime( project.path(), "project.shutdown-owner", ) diff --git a/crates/tracedecay/src/daemon/projectless.rs b/crates/tracedecay/src/daemon/projectless.rs index 85757e6923..9c1e91a673 100644 --- a/crates/tracedecay/src/daemon/projectless.rs +++ b/crates/tracedecay/src/daemon/projectless.rs @@ -24,6 +24,7 @@ use tracedecay_sessions::serving::SessionRefreshWorkerPort; use tracedecay_store::StoreShardIdV1; use super::*; +use tracedecay_daemon_service::shutdown::DaemonLifecycle; type ProjectlessPhaseFutureV1<'a, T> = std::pin::Pin + Send + 'a>>; @@ -170,7 +171,7 @@ async fn projectless_response( ) -> Option { let id = request.id.clone()?; match request.method.as_str() { - "initialize" => Some(match crate::version::build_version() { + "initialize" => Some(match tracedecay_project::version::build_version() { Ok(version) => JsonRpcResponse::success( id, json!({ @@ -205,7 +206,7 @@ async fn projectless_response( ); Some(response) } - "ping" | "logging/setLevel" => Some(JsonRpcResponse::success(id, json!({}))), + "ping" => Some(JsonRpcResponse::success(id, json!({}))), _ => Some(JsonRpcResponse::error( id, ErrorCode::MethodNotFound, @@ -868,8 +869,8 @@ mod projectless_admission_tests { std::fs::set_permissions(&foreign_root, std::fs::Permissions::from_mode(0o700)) .expect("restrict foreign profile root"); } - crate::product_runtime::register_fixture_product_runtime(); - crate::test_support::host_admission::ensure_process_background_cpu_authority() + tracedecay_project::product_runtime::register_fixture_product_runtime(); + tracedecay_project::test_support::host_admission::ensure_process_background_cpu_authority() .expect("install fixture worker authority"); let identity = tracedecay_daemon_identity::profile_identity::load_or_create(&real_root) .expect("pin profile identity"); @@ -937,8 +938,8 @@ mod projectless_admission_tests { async fn removed_client_profile_symlink_keeps_retained_codex_path_pinned() { let temp = tempfile::tempdir().expect("tempdir"); let (real_root, linked_root) = linked_profile_root(temp.path()); - crate::product_runtime::register_fixture_product_runtime(); - crate::test_support::host_admission::ensure_process_background_cpu_authority() + tracedecay_project::product_runtime::register_fixture_product_runtime(); + tracedecay_project::test_support::host_admission::ensure_process_background_cpu_authority() .expect("install fixture worker authority"); let identity = tracedecay_daemon_identity::profile_identity::load_or_create(&real_root) .expect("pin profile identity"); diff --git a/crates/tracedecay/src/daemon/retained_test_support.rs b/crates/tracedecay/src/daemon/retained_test_support.rs index f25ef8f6cc..24a3ec1335 100644 --- a/crates/tracedecay/src/daemon/retained_test_support.rs +++ b/crates/tracedecay/src/daemon/retained_test_support.rs @@ -18,7 +18,7 @@ use tracedecay_lsp::LspSessionRegistry; use super::project_open_owners::project_open_retained_grant; use tracedecay_code_index_runtime::code_index_scheduler::CodeIndexSchedulerRegistryV1; use tracedecay_code_index_runtime::resolved_scope_for_project; -use tracedecay_daemon_protocol::invocation_now_micros; +use tracedecay_contracts::now_micros; use tracedecay_daemon_service::{ DaemonInvocationService, DaemonRetainedRuntimeRegistrar, daemon_owned_project_source_access_at, }; @@ -34,7 +34,7 @@ struct RetainedOwnerTestExecutor { impl tracedecay_contracts::ApplicationInvocationExecutor for RetainedOwnerTestExecutor { fn invoke( &self, - _invocation: tracedecay_contracts::ApplicationInvocation, + invocation: tracedecay_contracts::ApplicationInvocation, ) -> tracedecay_contracts::ApplicationInvocationFuture< '_, std::result::Result< @@ -42,7 +42,15 @@ impl tracedecay_contracts::ApplicationInvocationExecutor for RetainedOwnerTestEx tracedecay_contracts::InvocationError, >, > { - Box::pin(async { Err(tracedecay_contracts::InvocationError::Unavailable) }) + Box::pin(async move { + let (context, request) = invocation.into_parts(); + let tracedecay_contracts::ApplicationRequest::Surface { binding, payload } = request + else { + return Err(tracedecay_contracts::InvocationError::Unavailable); + }; + tracedecay_daemon_protocol::invoke_application_surface(self, context, binding, payload) + .await + }) } } @@ -157,7 +165,7 @@ pub(crate) async fn register_project_retained_owner_for_test( message: format!("retained test owner scope is invalid: {error}"), } })?; - let observed_at = invocation_now_micros(); + let observed_at = now_micros(); let configuration = graph .configuration_runtime() .client() diff --git a/crates/tracedecay/src/daemon/scheduler.rs b/crates/tracedecay/src/daemon/scheduler.rs index 3ccb8fb974..f1527c6fcc 100644 --- a/crates/tracedecay/src/daemon/scheduler.rs +++ b/crates/tracedecay/src/daemon/scheduler.rs @@ -4,19 +4,20 @@ use std::sync::Arc; use tokio::task::JoinHandle; use tokio::time::{Duration, timeout}; use tracedecay_automation_runtime::automation::AutomationRunControl; -use tracedecay_automation_runtime::automation::backend::AgentTaskKind; +use tracedecay_automation_runtime::automation::backend::{AgentTaskBackend, AgentTaskKind}; use tracedecay_automation_runtime::automation::maintenance_termination::MaintenanceTaskTermination; use tracedecay_automation_runtime::automation::scheduler_stop::AutomationSchedulerStop; -use crate::project::TraceDecay; use tracedecay_automation_runtime::automation::effect_runtime::settlement::{ AutomationEffectAdmission, AutomationEffectAuthority, RetainedAutomationSettlementOutcome, RetainedAutomationSettlementProjection, pinned_automation_configuration_digest, }; use tracedecay_domain::errors::{Result, TraceDecayError}; +use tracedecay_project::project::TraceDecay; use super::branch_admin::MaintenanceReaperKind; -use super::{DAEMON_TASK_ABORT_DEADLINE, DaemonEngine, DaemonHandshake, ProjectServerKey}; +use super::{DaemonEngine, DaemonHandshake, ProjectServerKey}; +use tracedecay_daemon_service::shutdown::DAEMON_TASK_ABORT_DEADLINE; use tracedecay_runtime_core::logging::log_daemon_event; mod combined_effect; @@ -395,7 +396,7 @@ impl DaemonEngine { key: ProjectServerKey, project_path: PathBuf, handshake: DaemonHandshake, - cg: Arc, + cg: Arc, ) { if !self.lifecycle.accepting() { return; @@ -994,7 +995,8 @@ impl DaemonEngine { .await .clear(); let _child_shutdown = - tracedecay_sessions::runtime::codex_app_server::begin_codex_app_server_shutdown(); + tracedecay_sessions::runtime::hosts::codex_app_server::begin_codex_app_server_shutdown( + ); let _ = timeout(DAEMON_TASK_ABORT_DEADLINE, async { for retirement in retirements { retirement.wait().await; @@ -1486,19 +1488,15 @@ fn finish_global_retention( fn global_table_retention_config( config: &tracedecay_configuration::RetentionConfig, ) -> tracedecay_maintenance::retention::RetentionConfig { - let (session_messages_days, lcm_raw_messages_days) = if config.session_lcm.enabled { - ( - config.session_lcm.dedupe_projected_after_days, - config.session_lcm.drop_after_days, - ) + let lcm_raw_messages_days = if config.session_lcm.enabled { + config.session_lcm.drop_after_days } else { - (None, None) + None }; tracedecay_maintenance::retention::RetentionConfig { // The root retention tree has no analytics-event window. Disabling // this legacy table is the only mapping that does not invent policy. analytics_events_days: None, - session_messages_days, lcm_raw_messages_days, } } @@ -1518,7 +1516,7 @@ async fn maybe_run_global_retention( ) else { return; }; - let now_secs = crate::project::current_timestamp(); + let now_secs = tracedecay_runtime_core::tracedecay::current_timestamp(); let global_config = global_table_retention_config(config); let Some(retention) = administration .try_with_writer(|| async { @@ -1645,18 +1643,23 @@ mod global_retention_tests { .execute_batch( "CREATE TABLE retention_delete_receipts (deleted_message_id TEXT NOT NULL); CREATE TRIGGER retention_delete_receipt - AFTER DELETE ON session_messages BEGIN + AFTER DELETE ON lcm_raw_messages BEGIN INSERT INTO retention_delete_receipts(deleted_message_id) VALUES (OLD.message_id); END; - INSERT INTO lcm_summary_nodes( - node_id, provider, conversation_id, session_id, depth, summary_text, - summary_hash, summary_token_count, source_token_count + INSERT INTO retrieval_anchors ( + anchor_id, anchor_json, owner_json, projection_generation + ) VALUES ('retention-summary-anchor', '{}', '{}', 'test'); + INSERT INTO session_summary_nodes( + summary_id, session_id, provider, conversation_id, depth, + summary_anchor_id, summary_text, summary_hash, summary_token_count, + source_token_count, source_horizon_json, created_at ) VALUES ( - 'retention-summary', 'claude', 'retention-session', 'retention-session', 0, - 'retention summary', 'retention-summary-hash', 1, 1 + 'retention-summary', 'retention-session', 'claude', 'retention-session', 0, + 'retention-summary-anchor', 'retention summary', 'retention-summary-hash', + 1, 1, '{}', 1 ); - INSERT INTO lcm_summary_sources(node_id, source_kind, source_id, ordinal) + INSERT INTO session_summary_sources(summary_id, source_kind, source_id, ordinal) SELECT 'retention-summary', 'raw_message', CAST(store_id AS TEXT), 0 FROM lcm_raw_messages WHERE provider = 'claude' AND message_id = 'retention-message';", @@ -1694,8 +1697,7 @@ mod global_retention_tests { fn global_retention_config() -> tracedecay_configuration::RetentionConfig { let mut config = tracedecay_configuration::RetentionConfig::default(); config.session_lcm.enabled = true; - config.session_lcm.dedupe_projected_after_days = Some(1); - config.session_lcm.drop_after_days = None; + config.session_lcm.drop_after_days = Some(1); config.session_lcm.offload_after_days = None; config } @@ -1841,7 +1843,7 @@ mod global_retention_tests { .expect("open registered writer for retention fault") .execute_batch( "CREATE TRIGGER fail_global_retention_prune - BEFORE DELETE ON session_messages + BEFORE DELETE ON lcm_raw_messages WHEN OLD.message_id = 'retention-message' BEGIN SELECT RAISE(ABORT, 'forced global retention prune failure'); @@ -1893,11 +1895,15 @@ struct PinnedAutomationConfiguration { configuration_revision_id: tracedecay_domain::configuration::ConfigurationRevisionId, configuration_digest: tracedecay_domain::ManifestDigest, settings: tracedecay_automation_runtime::automation::config::AutomationConfig, + /// The `codex` executable the same snapshot binds + /// (`lcm.summarizer_executables.v1`); the automation backend spawns only + /// this path. + codex_executable: tracedecay_domain::configuration::LcmSummarizerExecutableV1, } #[hotpath::measure(label = "daemon.scheduler.read_automation_config", future = true)] async fn effective_automation_config_for_project( - cg: &crate::project::TraceDecay, + cg: &tracedecay_project::project::TraceDecay, ) -> Result { let configuration = cg .configuration_runtime() @@ -1919,6 +1925,7 @@ async fn effective_automation_config_for_project( configuration_revision_id: configuration.revision_id().clone(), configuration_digest, settings, + codex_executable: configuration.config().lcm_summarizers.codex.clone(), }) } @@ -1960,7 +1967,7 @@ pub(super) fn automation_scheduler_configured( /// scheduled fixed task or a schedulable user-defined job. #[hotpath::measure(label = "daemon.scheduler.probe_scheduler_work", future = true)] async fn automation_scheduler_has_work( - cg: &crate::project::TraceDecay, + cg: &tracedecay_project::project::TraceDecay, config: &tracedecay_automation_runtime::automation::config::AutomationConfig, ) -> Result { use tracedecay_automation_runtime::automation::config::{ @@ -2002,7 +2009,7 @@ async fn run_user_jobs_scheduler_pass( project_id: &tracedecay_domain::ProjectId, project_path: &Path, profile_root: &Path, - cg: &crate::project::TraceDecay, + cg: &tracedecay_project::project::TraceDecay, configuration_digest: tracedecay_domain::ManifestDigest, config: &tracedecay_automation_runtime::automation::config::AutomationConfig, backend: &tracedecay_automation_runtime::automation::backend::CodexAppServerBackend, @@ -2047,6 +2054,7 @@ async fn run_user_jobs_scheduler_pass( match tracedecay_automation_runtime::automation::jobs::evaluate_and_record_scheduler_skip( &dashboard_root, config, + backend.executable(), job, &requested_run_id, occurrence_anchor_run_id.as_deref(), diff --git a/crates/tracedecay/src/daemon/scheduler/combined_effect.rs b/crates/tracedecay/src/daemon/scheduler/combined_effect.rs index 9f557cc39a..958aadde70 100644 --- a/crates/tracedecay/src/daemon/scheduler/combined_effect.rs +++ b/crates/tracedecay/src/daemon/scheduler/combined_effect.rs @@ -21,12 +21,12 @@ use super::scheduler_automation_effect; use crate::daemon::DaemonEngine; use tracedecay_automation_runtime::automation::effect_runtime::AutomationSettledTerminal; -use crate::project::TraceDecay; use tracedecay_automation_runtime::automation::effect_runtime::settlement::{ AutomationEffectAdmission, AutomationEffectAuthority, DeferredProblemSettlementRequest, DeferredRunSettlementRequest, DeferredSettlementOutcome, DeferredSettlementRequest, }; use tracedecay_domain::errors::Result; +use tracedecay_project::project::TraceDecay; use tracedecay_runtime_core::logging::log_daemon_event; pub(super) enum CombinedEffectAdmission { @@ -856,7 +856,7 @@ mod tests { let memory = Arc::new( TraceDecay::init_with_options_for_test( &project_root, - crate::project::TraceDecayOpenOptions { + tracedecay_project::project::TraceDecayOpenOptions { profile_root: Some(profile_root.clone()), global_db_path: Some(profile_root.join("global.db")), }, @@ -1058,6 +1058,10 @@ mod tests { self.calls.fetch_add(1, Ordering::SeqCst); panic!("a disabled scheduler run must not invoke its backend") } + + fn executable(&self) -> Option<&std::path::Path> { + None + } } struct RecordingEarlyGateRetrieval { diff --git a/crates/tracedecay/src/daemon/scheduler/effect_admission.rs b/crates/tracedecay/src/daemon/scheduler/effect_admission.rs index 7044e7226d..e514c27a0a 100644 --- a/crates/tracedecay/src/daemon/scheduler/effect_admission.rs +++ b/crates/tracedecay/src/daemon/scheduler/effect_admission.rs @@ -3,7 +3,7 @@ use std::path::Path; use std::pin::Pin; use tracedecay_automation_runtime::automation::AutomationRunControl; -use tracedecay_automation_runtime::automation::backend::AgentTaskKind; +use tracedecay_automation_runtime::automation::backend::{AgentTaskBackend, AgentTaskKind}; use super::super::{DaemonEngine, DaemonHandshake}; use super::{ @@ -12,7 +12,6 @@ use super::{ maybe_run_global_retention, run_user_jobs_scheduler_pass, scheduler_run_observer, settle_scheduler_retained_automation, }; -use crate::project::TraceDecay; use tracedecay_automation_runtime::automation::effect_runtime::settlement::{ AutomationEffectAdmission, AutomationEffectAuthority, }; @@ -20,6 +19,7 @@ use tracedecay_daemon_service::automation_effect::{ prepare as prepare_automation_effect, scheduler_automation_request_id, }; use tracedecay_domain::errors::{Result, TraceDecayError}; +use tracedecay_project::project::TraceDecay; use tracedecay_runtime_core::logging::log_daemon_event; pub(super) fn log_scheduler_pre_admission_problem( @@ -72,6 +72,7 @@ fn log_scheduler_schedule_skip( async fn fixed_task_schedule_decision( dashboard_root: &Path, config: &tracedecay_automation_runtime::automation::config::AutomationConfig, + executable: Option<&Path>, task: AgentTaskKind, activity: tracedecay_automation_runtime::automation::scheduler::SessionActivity, now_secs: i64, @@ -86,6 +87,7 @@ async fn fixed_task_schedule_decision( Ok( tracedecay_automation_runtime::automation::scheduler::schedule_decision( config, + executable, task, summary.records(), activity, @@ -101,7 +103,7 @@ async fn fixed_task_schedule_decision( )] pub(super) async fn scheduler_automation_effect( engine: &DaemonEngine, - memory: &crate::project::TraceDecay, + memory: &tracedecay_project::project::TraceDecay, run_control: &AutomationRunControl, project_path: &Path, dashboard_root: &Path, @@ -354,7 +356,8 @@ fn run_automation_scheduler_tick_inner<'a>( ) .await; } - let backend = CodexAppServerBackend::from_automation_config(config); + let backend = + CodexAppServerBackend::from_automation_config(config, &configuration.codex_executable); let session_database = engine .store_administration .registered_project_session_database( @@ -371,6 +374,7 @@ fn run_automation_scheduler_tick_inner<'a>( let memory_curator_decision = fixed_task_schedule_decision( &cg.store_layout().dashboard_root, config, + backend.executable(), AgentTaskKind::MemoryCurator, schedule_activity, schedule_now_secs, @@ -379,6 +383,7 @@ fn run_automation_scheduler_tick_inner<'a>( let session_reflector_decision = fixed_task_schedule_decision( &cg.store_layout().dashboard_root, config, + backend.executable(), AgentTaskKind::SessionReflector, schedule_activity, schedule_now_secs, @@ -387,6 +392,7 @@ fn run_automation_scheduler_tick_inner<'a>( let skill_writer_decision = fixed_task_schedule_decision( &cg.store_layout().dashboard_root, config, + backend.executable(), AgentTaskKind::SkillWriter, schedule_activity, schedule_now_secs, @@ -743,6 +749,7 @@ mod tests { let decision = fixed_task_schedule_decision( dashboard.path(), &config, + None, tracedecay_automation_runtime::automation::backend::AgentTaskKind::MemoryCurator, tracedecay_automation_runtime::automation::scheduler::SessionActivity::none(), 1, diff --git a/crates/tracedecay/src/daemon/scheduler/host_receipt_review.rs b/crates/tracedecay/src/daemon/scheduler/host_receipt_review.rs index 986e8641ea..91f3b34d38 100644 --- a/crates/tracedecay/src/daemon/scheduler/host_receipt_review.rs +++ b/crates/tracedecay/src/daemon/scheduler/host_receipt_review.rs @@ -4,8 +4,8 @@ use std::pin::Pin; use tracedecay_automation_runtime::automation::AutomationRunControl; -use crate::project::TraceDecay; use tracedecay_domain::errors::{Result, TraceDecayError}; +use tracedecay_project::project::TraceDecay; use super::{DaemonEngine, DaemonHandshake, effective_automation_config_for_project}; use tracedecay_runtime_core::logging::log_daemon_event; @@ -177,7 +177,8 @@ async fn run_one_host_receipt_review( automation_context.project_id(), ) .await?; - let backend = CodexAppServerBackend::from_automation_config(config); + let backend = + CodexAppServerBackend::from_automation_config(config, &configuration.codex_executable); let host_run_id = format!("host_receipt_{}", pending.generation); let combined_options = CombinedReviewAutomationOptions { session_reflector: SessionReflectorAutomationOptions { @@ -312,14 +313,16 @@ mod tests { std::fs::create_dir_all(project_root.join("src")).expect("project source directory"); std::fs::write(project_root.join("src/lib.rs"), "pub fn fixture() {}\n") .expect("project source"); - let options = crate::project::TraceDecayOpenOptions { + let options = tracedecay_project::project::TraceDecayOpenOptions { profile_root: Some(profile_root.clone()), global_db_path: Some(profile_root.join("global.db")), }; - let writable = - crate::project::TraceDecay::init_with_options_for_test(&project_root, options.clone()) - .await - .expect("initialize host receipt project"); + let writable = tracedecay_project::project::TraceDecay::init_with_options_for_test( + &project_root, + options.clone(), + ) + .await + .expect("initialize host receipt project"); let dashboard_root = writable.store_layout().dashboard_root.clone(); let route = Some(HookRouteMetadata { session_id: Some("session.context-failure".to_owned()), @@ -349,12 +352,13 @@ mod tests { .await .expect("mark host receipt ready"); writable.close(); - let read_only = crate::project::TraceDecay::open_read_only_with_options_for_test( - &project_root, - options, - ) - .await - .expect("open read-only host receipt project"); + let read_only = + tracedecay_project::project::TraceDecay::open_read_only_with_options_for_test( + &project_root, + options, + ) + .await + .expect("open read-only host receipt project"); let handshake = DaemonHandshake { project_path: Some(project_root.clone()), scope_prefix: None, diff --git a/crates/tracedecay/src/daemon/session_runtime_tests.rs b/crates/tracedecay/src/daemon/session_runtime_tests.rs index 537e1683cb..af1a1a491e 100644 --- a/crates/tracedecay/src/daemon/session_runtime_tests.rs +++ b/crates/tracedecay/src/daemon/session_runtime_tests.rs @@ -8,7 +8,7 @@ use tempfile::TempDir; use tracedecay_session_runtime::StoreOwnerKey; use tracedecay_sessions::admission::HostAdmissionScope; -use crate::test_support::host_admission::HostAdmissionTestRuntimeV1; +use tracedecay_project::test_support::host_admission::HostAdmissionTestRuntimeV1; #[tokio::test] async fn evicted_project_owner_releases_temporal_scheduler() { diff --git a/crates/tracedecay/src/daemon/store_maintenance/mod.rs b/crates/tracedecay/src/daemon/store_maintenance/mod.rs index 389b6b27cd..8b14ce29dc 100644 --- a/crates/tracedecay/src/daemon/store_maintenance/mod.rs +++ b/crates/tracedecay/src/daemon/store_maintenance/mod.rs @@ -8,7 +8,7 @@ use tracedecay_code_index_runtime::code_index_scheduler::CodeIndexSchedulerRegis use tracedecay_runtime_core::branch::BranchAdminAction; use super::branch_admin::StoreAdministration; -use crate::project::TraceDecay; +use tracedecay_project::project::TraceDecay; use tracedecay_runtime_core::logging::log_daemon_event; /// Runs branch-store GC for a project through the daemon administration @@ -20,7 +20,6 @@ pub(super) async fn run_gc( administration: &StoreAdministration, schedulers: &CodeIndexSchedulerRegistryV1, branch_gc_days: u64, - orphan_db_gc_days: u64, cg: &TraceDecay, ) -> bool { let root = cg.project_root(); @@ -35,7 +34,6 @@ pub(super) async fn run_gc( data_root, BranchAdminAction::Gc, branch_gc_days, - orphan_db_gc_days, ) .await; let report = match report { @@ -53,16 +51,12 @@ pub(super) async fn run_gc( } }; - if !report.removed_branches.is_empty() || !report.removed_orphan_dbs.is_empty() { + if !report.removed_branches.is_empty() { log_daemon_event( "retention_branch_gc", &[ ("project", root.display().to_string()), ("removed_tracked", report.removed_branches.len().to_string()), - ( - "removed_orphans", - report.removed_orphan_dbs.len().to_string(), - ), ], ); } diff --git a/crates/tracedecay/src/daemon/store_runtime_tests.rs b/crates/tracedecay/src/daemon/store_runtime_tests.rs index 5e67a6ec8c..0d69f66938 100644 --- a/crates/tracedecay/src/daemon/store_runtime_tests.rs +++ b/crates/tracedecay/src/daemon/store_runtime_tests.rs @@ -1,6 +1,6 @@ //! Composition-root integration tests for the store-runtime crate. //! -//! These tests reach `remote_protocol_tests` and `crate::config`, which compose +//! These tests reach `remote_protocol_tests` and `tracedecay_project::config`, which compose //! root daemon fixtures and therefore cannot live in //! `tracedecay-store-runtime` itself. @@ -10,8 +10,7 @@ use tracedecay_daemon_identity::profile_identity::LocalProfileIdentityAuthorityV use tracedecay_domain::errors::TraceDecayError; use tracedecay_domain::{ BrainNodeId, Confidence, FactCategoryV1, FactCurationActionV1, FactLineageEventKindV1, - FactOwnerV1, FactRelationKindV1, ObservationScopeV1, ObservationSourceGenerationV1, - ObservationSourceIdentityV1, ObservationSourceRangeV1, ProviderId, SessionId, + FactOwnerV1, FactRelationKindV1, }; use tracedecay_global_db::register_registered_schema_installer; use tracedecay_graph_db::{ @@ -29,8 +28,7 @@ use tracedecay_session_memory::memory::{ ProjectMemoryFactAddRequest, ProjectMemoryFactAddRequestOutcome, memory_application_for_db, }; use tracedecay_store::{ - CursorAdvanceOutcome, FactReadControl, FactWriteControl, ObservationCoverageReason, - ObservationCursorAdvance, ObservationStore, ProjectId, ProjectMemoryFactHistoryQueryV1, + FactReadControl, FactWriteControl, ProjectId, ProjectMemoryFactHistoryQueryV1, ProjectMemoryFactIdV1, ProjectMemoryFactProjectionV1, RetainedGraphStoreLeaseV1, StoreShardIdV1, }; @@ -193,10 +191,8 @@ async fn wait_for_schema_convergence( .expect("registered schema convergence must reach a terminal state") } -const LCM_STATUS_PERFORMANCE_INDEX_NAMES: [&str; 4] = [ - "idx_lcm_raw_legacy_truncated", +const LCM_STATUS_PERFORMANCE_INDEX_NAMES: [&str; 2] = [ "idx_lcm_raw_lossy_ingest", - "idx_lcm_summary_nodes_depth_tokens", "idx_lcm_external_payloads_owner_bytes", ]; const SUPERSEDED_LCM_PAYLOAD_OWNER_INDEX: &str = "idx_lcm_external_payloads_owner"; @@ -272,29 +268,6 @@ async fn lcm_migration_applied_at(connection: &(impl QueryExecutor + ?Sized)) -> .await } -fn runtime_cursor_advance( - project_id: &ProjectId, - marker: &str, - reason: ObservationCoverageReason, -) -> ObservationCursorAdvance { - ObservationCursorAdvance::new( - ObservationSourceIdentityV1::for_provider( - ProviderId::new("runtime-ledger-convergence").expect("cursor provider identity"), - SessionId::new(format!("session.runtime-ledger.{marker}")) - .expect("cursor session identity"), - ) - .expect("cursor source identity"), - ObservationScopeV1::Project { - project_id: project_id.clone(), - }, - ObservationSourceGenerationV1::new(1).expect("cursor source generation"), - None, - ObservationSourceRangeV1::new(0, 1).expect("cursor source range"), - reason, - ) - .expect("runtime cursor advance") -} - fn accepting_memory_write_control() -> FactWriteControl { FactWriteControl::new(Arc::new(|| false), Arc::new(|| true)) } @@ -810,18 +783,15 @@ async fn daemon_admission_remains_ready_while_lcm_indexes_converge_in_background VALUES ('cursor', 'deferred-index-session', 'project.schema-admission', '/deferred'); INSERT INTO lcm_raw_messages ( provider, message_id, session_id, role, ordinal, content, - content_hash, storage_kind, snippet_text, index_text, - legacy_truncated, metadata_json + content_hash, storage_kind, metadata_json ) VALUES ( 'cursor', 'deferred-index-message', 'deferred-index-session', 'assistant', 1, - 'deferred body', 'deferred-hash', 'inline', 'deferred', 'deferred', 1, NULL + 'deferred body', 'deferred-hash', 'inline', NULL ); UPDATE session_schema_migrations SET applied_at = 123 WHERE name = 'lcm'; - DROP INDEX idx_lcm_raw_legacy_truncated; DROP INDEX idx_lcm_raw_lossy_ingest; - DROP INDEX idx_lcm_summary_nodes_depth_tokens; DROP INDEX idx_lcm_external_payloads_owner_bytes; CREATE INDEX idx_lcm_external_payloads_owner ON lcm_external_payloads(provider, session_id);", @@ -920,35 +890,6 @@ async fn daemon_admission_remains_ready_while_lcm_indexes_converge_in_background ); } -/// The two activity indexes a store can carry: the blob-covering predecessor -/// and the replacement that leaves `metadata_json` in the table. -const SESSION_ACTIVITY_INDEX_NAMES: [&str; 2] = [ - "idx_session_messages_session_activity", - "idx_session_messages_session_activity_v2", -]; - -/// Rows still in the retired external-source table, or `None` once the -/// migration has dropped it. -async fn retired_external_source_rows(connection: &(impl QueryExecutor + ?Sized)) -> Option { - let present = scalar_i64( - connection, - "SELECT COUNT(*) FROM sqlite_master - WHERE type = 'table' AND name = 'external_source_objects_v1'", - ) - .await - > 0; - if !present { - return None; - } - Some( - scalar_i64( - connection, - "SELECT COUNT(*) FROM external_source_objects_v1", - ) - .await, - ) -} - async fn migrating_session_row_count(connection: &(impl QueryExecutor + ?Sized)) -> i64 { scalar_i64( connection, @@ -957,39 +898,24 @@ async fn migrating_session_row_count(connection: &(impl QueryExecutor + ?Sized)) .await } -/// The incident, end to end. Both of these migrations once ran inside the -/// open's leased schema transaction: on a large store each outran its -/// execution deadline, the batch was interrupted, and the daemon exited, so -/// systemd restarted it into the same failure forever. +/// Store-sized convergence runs after admission, never inside the open's +/// leased schema transaction, where a large store outran its execution +/// deadline and the daemon restarted into the same failure forever. /// -/// Admission must therefore leave both alone and complete, the store must -/// serve while they are outstanding, Doctor must report which store is -/// migrating rather than claiming health, and background convergence must be -/// what actually retires them. +/// Admission must therefore complete, the store must serve while +/// convergence is outstanding, Doctor must report which store is converging +/// rather than claiming health, and background convergence must settle it. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn store_sized_migrations_are_reported_and_converge_after_admission() { +async fn schema_convergence_is_reported_and_completes_after_admission() { let (_temporary, identity, project_id, project_root, sessions_path, _database_scope) = project_sessions_pending_convergence("project.schema-store-sized").await; let seed = TestConnection::open(&sessions_path); seed.execute_batch( r"INSERT INTO sessions(provider, session_id, project_key, project_path) - VALUES ('cursor', 'migrating-session', 'project.schema-store-sized', '/migrating'); - DROP INDEX IF EXISTS idx_session_messages_session_activity_v2; - CREATE INDEX idx_session_messages_session_activity - ON session_messages( - provider, session_id, timestamp, ordinal, message_id, kind, - tool_names, metadata_json - ); - CREATE TABLE external_source_objects_v1 ( - binding_id TEXT NOT NULL, native_object_digest TEXT NOT NULL, - partition_digest TEXT NOT NULL, mutation_digest TEXT NOT NULL, - mutation_json TEXT NOT NULL, - PRIMARY KEY (binding_id, native_object_digest)); - INSERT INTO external_source_objects_v1 VALUES - ('b', 'sha256:obj', 'sha256:part', 'sha256:mut', '{}');", + VALUES ('cursor', 'migrating-session', 'project.schema-store-sized', '/migrating');", ) .await - .expect("seed a store carrying both pre-migration shapes"); + .expect("seed a store row"); drop(seed); let shard_id = StoreShardIdV1::project_sessions( identity.brain_id().clone(), @@ -1012,16 +938,6 @@ async fn store_sized_migrations_are_reported_and_converge_after_admission() { .read_snapshot() .await .expect("ordinary read snapshot while migrations are outstanding"); - assert_eq!( - installed_index_names(&snapshot, &SESSION_ACTIVITY_INDEX_NAMES).await, - vec!["idx_session_messages_session_activity".to_owned()], - "admission must not rebuild the activity index inside its write lease" - ); - assert_eq!( - retired_external_source_rows(&snapshot).await, - Some(1), - "admission must not rewrite the retired external-source table" - ); assert_eq!( migrating_session_row_count(&snapshot).await, 1, @@ -1080,16 +996,6 @@ async fn store_sized_migrations_are_reported_and_converge_after_admission() { .read_snapshot() .await .expect("ordinary read snapshot after convergence"); - assert_eq!( - installed_index_names(&snapshot, &SESSION_ACTIVITY_INDEX_NAMES).await, - vec!["idx_session_messages_session_activity_v2".to_owned()], - "convergence must install the replacement and drop the blob-covering index" - ); - assert_eq!( - retired_external_source_rows(&snapshot).await, - None, - "convergence must retire the external-source predecessor once it is empty" - ); assert_eq!( migrating_session_row_count(&snapshot).await, 1, @@ -1097,237 +1003,6 @@ async fn store_sized_migrations_are_reported_and_converge_after_admission() { ); } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn retained_runtime_ledger_replays_during_bounded_background_convergence() { - let (_temporary, identity, project_id, project_root, sessions_path, _database_scope) = - project_sessions_current_convergence("project.runtime-ledger-convergence").await; - let shard_id = StoreShardIdV1::project_sessions( - identity.brain_id().clone(), - identity.profile_id().clone(), - project_id.clone(), - ); - - let seed_registry = DaemonSessionRuntimeRegistryV1::open(identity.clone()) - .await - .expect("seed session runtime registry"); - let seed_database = seed_registry - .project_sessions(project_id.clone(), [project_root.clone()]) - .await - .expect("seed registered project sessions"); - let retired_advance = runtime_cursor_advance( - &project_id, - "retired", - ObservationCoverageReason::OutOfScope, - ); - assert_eq!( - seed_database - .observation_store() - .advance_source_cursor(retired_advance.clone()) - .await - .expect("commit cursor before ledger migration"), - CursorAdvanceOutcome::Committed - ); - drop(seed_database); - drop(seed_registry); - - let seed = TestConnection::open(&sessions_path); - seed.execute_batch( - "CREATE TABLE td_runtime_writer_idempotency_v1 ( - shard_json TEXT NOT NULL, - incarnation INTEGER NOT NULL, - authority_epoch INTEGER NOT NULL, - idempotency_key TEXT NOT NULL, - request_digest TEXT NOT NULL, - original_receipt_json TEXT NOT NULL, - transaction_scope_json TEXT NOT NULL, - operation_id TEXT NOT NULL, - durability_json TEXT NOT NULL, - committed_at_micros INTEGER NOT NULL, - PRIMARY KEY (shard_json, incarnation, authority_epoch, idempotency_key) - ) WITHOUT ROWID; - INSERT INTO td_runtime_writer_idempotency_v1 - SELECT * FROM td_runtime_writer_idempotency_v2; - DROP TABLE td_runtime_writer_idempotency_v2", - ) - .await - .expect("restore released WITHOUT ROWID ledger name"); - let mut retired_rows = seed - .query( - "SELECT idempotency_key, authority_epoch, original_receipt_json - FROM td_runtime_writer_idempotency_v1 - WHERE idempotency_key LIKE 'cursor.%'", - (), - ) - .await - .expect("read retained cursor receipt identity"); - let retired_row = retired_rows - .next() - .await - .expect("read retained cursor receipt") - .expect("retained cursor receipt row"); - let retired_key = retired_row - .get::(0) - .expect("decode retained cursor key"); - let retired_epoch = retired_row - .get::(1) - .expect("decode retained cursor authority epoch"); - let retired_receipt_json = retired_row - .get::(2) - .expect("decode retained cursor receipt"); - assert!( - retired_rows - .next() - .await - .expect("check retained cursor receipt cardinality") - .is_none(), - "the released fixture carries one cursor receipt" - ); - drop(retired_rows); - drop(seed); - - let registry = DaemonSessionRuntimeRegistryV1::open_with_session_maintenance(identity, true) - .await - .expect("session runtime registry"); - let convergence_gate = registry.block_registered_schema_convergence_for_test(); - let database = registry - .project_sessions(project_id.clone(), [project_root]) - .await - .expect("admit retained runtime ledger"); - convergence_gate.wait_until_blocked().await; - - assert_eq!( - database - .observation_store() - .advance_source_cursor(retired_advance) - .await - .expect("replay retained cursor while convergence is pending"), - CursorAdvanceOutcome::ExactDuplicate - ); - // The same range under a different coverage reason finds the retained - // cursor already at its `next_cursor`, so it is a duplicate of the - // applied coverage rather than a collision (#1842). - let rereasoned_advance = runtime_cursor_advance( - &project_id, - "retired", - ObservationCoverageReason::BlankFrame, - ); - assert_eq!( - database - .observation_store() - .advance_source_cursor(rereasoned_advance) - .await - .expect("a later reason does not unseat the owned frontier"), - CursorAdvanceOutcome::ExactDuplicate - ); - - let fresh_advance = - runtime_cursor_advance(&project_id, "fresh", ObservationCoverageReason::OutOfScope); - assert_eq!( - database - .observation_store() - .advance_source_cursor(fresh_advance) - .await - .expect("ordinary cursor write while convergence is pending"), - CursorAdvanceOutcome::Committed - ); - - convergence_gate.release(); - assert_eq!( - wait_for_schema_convergence(®istry, &shard_id).await, - RegisteredSchemaConvergenceStatus::Complete - ); - let snapshot = database - .read_snapshot() - .await - .expect("runtime ledger convergence snapshot"); - let mut tables = snapshot - .query( - "SELECT COUNT(*) FROM sqlite_master - WHERE type = 'table' AND name = 'td_runtime_writer_idempotency_v1'", - (), - ) - .await - .expect("inspect retired runtime ledger table"); - assert_eq!( - tables - .next() - .await - .expect("read retired runtime ledger state") - .expect("retired runtime ledger state row") - .get::(0) - .expect("decode retired runtime ledger state"), - 0 - ); - let mut cursor_effects = snapshot - .query("SELECT COUNT(*) FROM source_cursors", ()) - .await - .expect("inspect committed cursor effects"); - assert_eq!( - cursor_effects - .next() - .await - .expect("read committed cursor effects") - .expect("committed cursor effect count row") - .get::(0) - .expect("decode committed cursor effect count"), - 2, - "the retained replay and the later reason must not create another cursor effect" - ); - let mut receipts = snapshot - .query( - "SELECT idempotency_key, authority_epoch, original_receipt_json - FROM td_runtime_writer_idempotency_v2 - WHERE idempotency_key LIKE 'cursor.%'", - (), - ) - .await - .expect("inspect converged runtime ledger receipts"); - let mut receipt_rows = Vec::new(); - while let Some(row) = receipts - .next() - .await - .expect("read converged runtime ledger receipt") - { - receipt_rows.push(( - row.get::(0).expect("decode converged cursor key"), - row.get::(1) - .expect("decode converged cursor authority epoch"), - row.get::(2) - .expect("decode converged cursor receipt"), - )); - } - let retired_identity_rows = receipt_rows - .iter() - .filter(|(key, _, _)| key == &retired_key) - .collect::>(); - assert_eq!( - retired_identity_rows.len(), - 2, - "the same logical cursor is receipted once per admitted authority epoch" - ); - assert!( - retired_identity_rows - .iter() - .any(|(_, epoch, receipt)| *epoch == retired_epoch - && receipt.as_str() == retired_receipt_json), - "bounded convergence must preserve the released receipt bytes" - ); - assert!( - retired_identity_rows - .iter() - .any(|(_, epoch, _)| *epoch != retired_epoch), - "the reopened authority records its own exact cursor acknowledgement" - ); - assert_eq!( - receipt_rows - .iter() - .filter(|(key, _, _)| key != &retired_key) - .count(), - 1, - "the fresh cursor has one runtime receipt" - ); -} - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn foreground_project_open_defers_historical_convergence_until_full_publication() { let (_temporary, identity, project_id, project_root, _sessions_path, _database_scope) = @@ -1710,7 +1385,8 @@ async fn worktree_graph_mount_does_not_require_git() { .await .expect("register non-git project authority"); let project_store_root = profile_root.join("projects/project.non-git-worktree"); - let database_path = project_store_root.join(crate::config::db_filename(&project_store_root)); + let database_path = + project_store_root.join(tracedecay_project::config::db_filename(&project_store_root)); std::fs::create_dir_all(database_path.parent().expect("database parent")) .expect("database directory"); let authority = DatabaseAuthority::for_runtime(&database_path, "non-git project graph mount") @@ -1990,7 +1666,8 @@ async fn corrupt_derived_graph_preserves_relational_owner_lifecycle() { ) .expect("daemon database scope"); let project_store_root = profile_root.join("projects/project.derived-graph-corrupt"); - let database_path = project_store_root.join(crate::config::db_filename(&project_store_root)); + let database_path = + project_store_root.join(tracedecay_project::config::db_filename(&project_store_root)); std::fs::create_dir_all(database_path.parent().expect("database parent")) .expect("database directory"); @@ -2278,7 +1955,8 @@ async fn read_only_project_graph_reuses_daemon_publication_without_write_authori .expect("register project authority"); let project_store_root = profile_root.join("projects/project.graph-publication"); std::fs::create_dir_all(&project_store_root).expect("project store directory"); - let main_path = project_store_root.join(crate::config::db_filename(&project_store_root)); + let main_path = + project_store_root.join(tracedecay_project::config::db_filename(&project_store_root)); let unpublished_path = project_store_root.join("unpublished.db"); rusqlite::Connection::open(&unpublished_path) .expect("seed unpublished branch database") diff --git a/crates/tracedecay/src/daemon/tests.rs b/crates/tracedecay/src/daemon/tests.rs index 09798889ae..f499649ba0 100644 --- a/crates/tracedecay/src/daemon/tests.rs +++ b/crates/tracedecay/src/daemon/tests.rs @@ -24,10 +24,11 @@ use super::explicit_git_state; #[cfg(unix)] use super::scheduler::{AutomationSchedulerExitBarrier, AutomationSchedulerLifecycle}; use super::{ - DaemonClientIdentity, DaemonHandshake, DaemonLifecycle, DatabaseOwnerRegistry, ProjectRouteKey, + DaemonClientIdentity, DaemonHandshake, DatabaseOwnerRegistry, ProjectRouteKey, ProjectServerKey, StoreAdministration, StoreOwnerKey, multi_root_family_allows, store_owner_key_from_paths, }; +use tracedecay_daemon_service::shutdown::DaemonLifecycle; mod bootstrap; mod code_index_hydration; @@ -66,7 +67,6 @@ fn git(root: &std::path::Path, args: &[&str]) { #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(super) enum ObservedMcpRoute { Rmcp, - Legacy, } fn observed_mcp_routes() @@ -268,7 +268,7 @@ fn test_daemon_engine_for_profile(profile_root: &std::path::Path) -> DaemonEngin // that drives the engine without ever building a handshake (for example // the unparseable-handshake refusals) would otherwise depend on some other // fixture in the same process registering it first. - crate::product_runtime::register_fixture_product_runtime(); + tracedecay_project::product_runtime::register_fixture_product_runtime(); prepare_test_profile_root(profile_root); let profile_identity = tracedecay_daemon_identity::profile_identity::load_or_create(profile_root) @@ -299,21 +299,6 @@ fn test_daemon_engine_for_profile(profile_root: &std::path::Path) -> DaemonEngin engine } -use crate::isolated_profile::EnvVarGuard; - -/// Pins the codex app-server launcher to a path that cannot exist so any -/// automation tick reached during the test fails with the typed spawn error -/// instead of invoking the operator's real `codex` binary. Without this, -/// a tick spawns a live external process whose runtime depends on a real -/// backend, and a harness kill (e.g. nextest SIGTERM) orphans that process -/// group because in-process cleanup never runs. -fn isolate_codex_app_server_binary(root: &std::path::Path) -> EnvVarGuard { - EnvVarGuard::set( - "TRACEDECAY_CODEX_BIN", - root.join("missing-codex-app-server-binary"), - ) -} - fn enter_test_daemon_database_scope( profile_root: &std::path::Path, label: &str, @@ -345,14 +330,16 @@ async fn initialize_test_project( // Heap-allocate the graph-init composition so every test awaiting this // fixture keeps a bounded resident frame (perf-profile layouts overflow // the test stack when the mega-future is inlined). - let project = Box::pin(crate::project::TraceDecay::init_with_exclusive_maintenance( - project_root, - crate::project::TraceDecayOpenOptions { - profile_root: Some(client_identity.profile_root.clone()), - global_db_path: Some(client_identity.global_db_path.clone()), - }, - &lifecycle, - )) + let project = Box::pin( + tracedecay_project::project::TraceDecay::init_with_exclusive_maintenance( + project_root, + tracedecay_project::project::TraceDecayOpenOptions { + profile_root: Some(client_identity.profile_root.clone()), + global_db_path: Some(client_identity.global_db_path.clone()), + }, + &lifecycle, + ), + ) .await .expect("initialize project"); let store_layout = project.store_layout().clone(); @@ -363,7 +350,8 @@ async fn initialize_test_project( fn test_handshake_defaults() -> DaemonHandshake { // Test processes only ever register the fixture product runtime, so every // handshake in the suite advertises one identical fixture version. - let build_version = crate::product_runtime::register_fixture_product_runtime().build_version(); + let build_version = + tracedecay_project::product_runtime::register_fixture_product_runtime().build_version(); DaemonHandshake { project_path: None, scope_prefix: None, @@ -375,10 +363,85 @@ fn test_handshake_defaults() -> DaemonHandshake { client_instance_id: tracedecay_runtime_core::runtime_identity::process_run_id().to_string(), tool_list_changed_capable: false, catalog_version: String::new(), - moved_store_adoption: crate::project::MovedStoreAdoption::Never, + moved_store_adoption: tracedecay_project::project::MovedStoreAdoption::Never, } } +/// Daemon credential the in-process socket fixtures serve and present. +#[cfg(unix)] +const TEST_AUTH_TOKEN: &str = "0123456789abcdef0123456789abcdef"; + +/// Serves one socket client through the production authenticated entry. +#[cfg(unix)] +async fn serve_authenticated_test_client( + stream: tokio::net::UnixStream, + engine: DaemonEngine, +) -> tracedecay_domain::errors::Result<()> { + Box::pin(super::serve_authenticated_socket_client_with_class( + tracedecay_daemon_protocol::BrokerStream::Unix(stream), + engine, + TEST_AUTH_TOKEN.to_owned(), + super::DaemonClientAdmissionClass::General, + )) + .await +} + +/// Writes the auth preface an authenticated client sends before its handshake. +#[cfg(unix)] +async fn write_test_auth_preface(writer: &mut (impl tokio::io::AsyncWrite + Unpin)) { + let preface = tracedecay_daemon_protocol::DaemonAuthPreface::new(TEST_AUTH_TOKEN) + .to_line() + .expect("test auth preface"); + writer + .write_all(preface.as_bytes()) + .await + .expect("write auth preface"); + writer.write_all(b"\n").await.expect("auth preface newline"); +} + +/// Publishes the authority record beside `socket` that clients resolve it +/// through. Hold the authority for as long as the socket should be served. +#[cfg(unix)] +pub(super) fn seed_socket_authority( + socket: &std::path::Path, +) -> tracedecay_daemon_identity::authority::DaemonAuthority { + tracedecay_daemon_identity::authority::DaemonAuthority::acquire( + socket.parent().expect("socket parent"), + &tracedecay_daemon_protocol::DaemonEndpoint::Unix(socket.to_path_buf()), + "test", + ) + .expect("seed daemon authority") +} + +/// Reads what a fake daemon receives first: the auth preface carrying +/// `token`, then the handshake. +#[cfg(unix)] +async fn read_authenticated_handshake( + lines: &mut tokio::io::Lines, + token: &str, +) -> DaemonHandshake +where + R: tokio::io::AsyncBufRead + Unpin, +{ + let preface = lines + .next_line() + .await + .expect("read auth preface") + .expect("auth preface line"); + let preface = tracedecay_daemon_protocol::DaemonAuthPreface::from_line(preface.trim()) + .expect("auth preface"); + assert!( + preface.authenticate(token), + "client must present the current daemon token" + ); + let handshake = lines + .next_line() + .await + .expect("read handshake") + .expect("handshake line"); + DaemonHandshake::from_line(&handshake).expect("parse handshake") +} + #[test] fn search_request_controls_distinguish_cancellation_and_timeout() { let cancellation = tracedecay_contracts::CancellationSignal::active("cancellation.search-test") @@ -631,7 +694,7 @@ async fn apply_project_setting_via_surface( .expect("configuration capability") .deadline() .maximum_millis(); - let observed_at = tracedecay_daemon_protocol::invocation_now_micros(); + let observed_at = tracedecay_contracts::now_micros(); let deadline = tracedecay_contracts::Deadline::new(tracedecay_domain::UtcMicros( observed_at.0 + i64::try_from(maximum_millis).expect("deadline fits") * 1_000, )) diff --git a/crates/tracedecay/src/daemon/tests/bootstrap.rs b/crates/tracedecay/src/daemon/tests/bootstrap.rs index bddff10781..98f3f3a09f 100644 --- a/crates/tracedecay/src/daemon/tests/bootstrap.rs +++ b/crates/tracedecay/src/daemon/tests/bootstrap.rs @@ -13,7 +13,7 @@ fn requirement_for(line: String) -> ProjectServerRequirement { } use tracedecay_mcp::JsonRpcResponse; #[cfg(unix)] -use tracedecay_session_memory::context::CancellationToken; +use tracedecay_runtime_core::cancellation::CancellationToken; static PRODUCTION_DASHBOARD_TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); @@ -186,7 +186,7 @@ fn daemon_project_route_rejects_the_user_profile_root() { // Portable production path: `project_route_for_handshake` is the Windows // and Unix authority. `DaemonEngine::project_route` is only a unix wrapper // around it and must not be referenced from this un-gated contract test. - let _profile = crate::config::PinnedUserDataDir::new(); + let _profile = tracedecay_project::config::PinnedUserDataDir::new(); let home = std::path::PathBuf::from(std::env::var_os("HOME").expect("pinned HOME")); let handshake = DaemonHandshake { project_path: Some(home), @@ -374,17 +374,18 @@ async fn orphaned_store_with_repository_identity_is_readopted_without_aliasing() .registered_profile_database() .await .expect("profile registry"); - let open_options = crate::project::TraceDecayOpenOptions { + let open_options = tracedecay_project::project::TraceDecayOpenOptions { profile_root: Some(profile_root.clone()), global_db_path: None, }; - let store_layout = crate::project::TraceDecay::resolve_registered_configuration_layout( - &project, - &open_options, - registry.as_ref(), - ) - .await - .expect("durable identity must resolve the registered layout"); + let store_layout = + tracedecay_project::project::TraceDecay::resolve_registered_configuration_layout( + &project, + &open_options, + registry.as_ref(), + ) + .await + .expect("durable identity must resolve the registered layout"); assert_eq!( store_layout.identity.project_id.as_deref(), Some(project_id), @@ -464,8 +465,8 @@ fn enroll_nongit_project_on_disk( #[cfg(unix)] fn moved_nongit_open_options( profile_root: &std::path::Path, -) -> crate::project::TraceDecayOpenOptions { - crate::project::TraceDecayOpenOptions { +) -> tracedecay_project::project::TraceDecayOpenOptions { + tracedecay_project::project::TraceDecayOpenOptions { profile_root: Some(profile_root.to_path_buf()), global_db_path: None, } @@ -508,11 +509,11 @@ async fn moved_nongit_project_is_readopted_only_when_confirmed() { // Ambient first-touch (`Never`) mints a fresh path-derived identity and // must not touch the moved project's registration. let ambient = - crate::project::TraceDecay::resolve_first_touch_configuration_layout_with_adoption( + tracedecay_project::project::TraceDecay::resolve_first_touch_configuration_layout_with_adoption( &moved, &moved_nongit_open_options(&profile_root), registry.as_ref(), - &crate::project::MovedStoreAdoption::Never, + &tracedecay_project::project::MovedStoreAdoption::Never, ) .await .expect("ambient first-touch mints fresh"); @@ -527,11 +528,11 @@ async fn moved_nongit_project_is_readopted_only_when_confirmed() { // Explicit init without adoption flags refuses with the candidate and // the explicit choices instead of silently remapping or silently // splitting identity. - let offer = crate::project::TraceDecay::resolve_first_touch_configuration_layout_with_adoption( + let offer = tracedecay_project::project::TraceDecay::resolve_first_touch_configuration_layout_with_adoption( &moved, &moved_nongit_open_options(&profile_root), registry.as_ref(), - &crate::project::MovedStoreAdoption::OfferCandidates, + &tracedecay_project::project::MovedStoreAdoption::OfferCandidates, ) .await .expect_err("explicit init without flags must refuse when a candidate exists"); @@ -555,11 +556,11 @@ async fn moved_nongit_project_is_readopted_only_when_confirmed() { // `init --yes` confirms adopting the unique candidate. let store_layout = - crate::project::TraceDecay::resolve_first_touch_configuration_layout_with_adoption( + tracedecay_project::project::TraceDecay::resolve_first_touch_configuration_layout_with_adoption( &moved, &moved_nongit_open_options(&profile_root), registry.as_ref(), - &crate::project::MovedStoreAdoption::AdoptUnique, + &tracedecay_project::project::MovedStoreAdoption::AdoptUnique, ) .await .expect("confirmed unique moved nongit project must be adopted"); @@ -627,11 +628,11 @@ async fn ambient_first_touch_never_adopts_a_moved_nongit_store() { let scratch_canonical = scratch.canonicalize().expect("canonical scratch root"); let layout = - crate::project::TraceDecay::resolve_first_touch_configuration_layout_with_adoption( + tracedecay_project::project::TraceDecay::resolve_first_touch_configuration_layout_with_adoption( &scratch, &moved_nongit_open_options(&profile_root), registry.as_ref(), - &crate::project::MovedStoreAdoption::Never, + &tracedecay_project::project::MovedStoreAdoption::Never, ) .await .expect("ambient first-touch on a fresh directory mints a fresh identity"); @@ -693,11 +694,11 @@ async fn moved_nongit_adoption_is_refused_when_ambiguous() { let target = root.join("nongit-new"); std::fs::create_dir_all(&target).expect("create adoption target"); - let error = crate::project::TraceDecay::resolve_first_touch_configuration_layout_with_adoption( + let error = tracedecay_project::project::TraceDecay::resolve_first_touch_configuration_layout_with_adoption( &target, &moved_nongit_open_options(&profile_root), registry.as_ref(), - &crate::project::MovedStoreAdoption::AdoptUnique, + &tracedecay_project::project::MovedStoreAdoption::AdoptUnique, ) .await .expect_err("ambiguous moved nongit adoption must refuse"); @@ -713,11 +714,11 @@ async fn moved_nongit_adoption_is_refused_when_ambiguous() { // The stale stores must not brick a genuinely new project: opting out of // adoption (`--fresh`, and every ambient first-touch) mints fresh. - let fresh = crate::project::TraceDecay::resolve_first_touch_configuration_layout_with_adoption( + let fresh = tracedecay_project::project::TraceDecay::resolve_first_touch_configuration_layout_with_adoption( &target, &moved_nongit_open_options(&profile_root), registry.as_ref(), - &crate::project::MovedStoreAdoption::Never, + &tracedecay_project::project::MovedStoreAdoption::Never, ) .await .expect("fresh init must stay possible with stale moved stores present"); @@ -771,11 +772,11 @@ async fn moved_nongit_adoption_honors_explicit_project_id() { std::fs::create_dir_all(&target).expect("create adoption target"); let store_layout = - crate::project::TraceDecay::resolve_first_touch_configuration_layout_with_adoption( + tracedecay_project::project::TraceDecay::resolve_first_touch_configuration_layout_with_adoption( &target, &moved_nongit_open_options(&profile_root), registry.as_ref(), - &crate::project::MovedStoreAdoption::AdoptNamed("proj_nongit_flag_a".to_owned()), + &tracedecay_project::project::MovedStoreAdoption::AdoptNamed("proj_nongit_flag_a".to_owned()), ) .await .expect("flagged adoption must select the named project"); @@ -831,11 +832,11 @@ async fn moved_nongit_adoption_refuses_conflicting_registered_root() { .expect("register moved project"); std::fs::rename(&original, root.join("nongit-moved")).expect("move conflicting project"); - let error = crate::project::TraceDecay::resolve_first_touch_configuration_layout_with_adoption( + let error = tracedecay_project::project::TraceDecay::resolve_first_touch_configuration_layout_with_adoption( &occupant, &moved_nongit_open_options(&profile_root), registry.as_ref(), - &crate::project::MovedStoreAdoption::AdoptNamed("proj_nongit_conflict".to_owned()), + &tracedecay_project::project::MovedStoreAdoption::AdoptNamed("proj_nongit_conflict".to_owned()), ) .await .expect_err("adoption onto another project's root must refuse"); @@ -893,11 +894,11 @@ async fn interrupted_moved_nongit_remap_resumes_on_next_explicit_init() { .expect("journal manifest write"); let resumed = - crate::project::TraceDecay::resolve_first_touch_configuration_layout_with_adoption( + tracedecay_project::project::TraceDecay::resolve_first_touch_configuration_layout_with_adoption( &moved, &moved_nongit_open_options(&profile_root), registry.as_ref(), - &crate::project::MovedStoreAdoption::OfferCandidates, + &tracedecay_project::project::MovedStoreAdoption::OfferCandidates, ) .await .expect("a torn remap must resume from its manifest journal record"); @@ -959,11 +960,11 @@ async fn unreadable_moved_store_evidence_is_a_typed_refusal() { .expect("profile-sharded layout carries a manifest path"); std::fs::write(manifest_path, b"not a manifest").expect("corrupt the manifest"); - let error = crate::project::TraceDecay::resolve_first_touch_configuration_layout_with_adoption( + let error = tracedecay_project::project::TraceDecay::resolve_first_touch_configuration_layout_with_adoption( &moved, &moved_nongit_open_options(&profile_root), registry.as_ref(), - &crate::project::MovedStoreAdoption::AdoptUnique, + &tracedecay_project::project::MovedStoreAdoption::AdoptUnique, ) .await .expect_err("unreadable evidence must be a typed error, not a silent non-match"); @@ -2189,9 +2190,11 @@ async fn route_open_backoff_retries_after_deadline_without_cross_route_blocking( let rejected_attempts = Arc::clone(&attempts); let rejected_state = match tasks.start(rejected.clone(), async move { rejected_attempts.fetch_add(1, std::sync::atomic::Ordering::Relaxed); - Err(tracedecay_domain::errors::TraceDecayError::Config { - message: "identity cutover conflict: strict route invariant".to_string(), - }) + Err(tracedecay_domain::errors::TraceDecayError::project_route( + crate::daemon::REPOSITORY_DISCOVERY_DEFERRED_REASON_CODE, + true, + "strict route invariant", + )) }) { super::super::ProjectOpenTaskClaim::InFlight(state) => state, super::super::ProjectOpenTaskClaim::Failed(_) => { @@ -4049,10 +4052,11 @@ async fn production_composition_harness_reads_retained_profile_analytics_authori .ledger_writes_settled() .await; - let second_owner = crate::test_support::host_admission::HostAdmissionTestRuntimeV1::profile( - harness.profile_root(), - ) - .await; + let second_owner = + tracedecay_project::test_support::host_admission::HostAdmissionTestRuntimeV1::profile( + harness.profile_root(), + ) + .await; let error = match second_owner { Ok(_) => panic!("parallel profile authority must remain rejected"), Err(error) => error, diff --git a/crates/tracedecay/src/daemon/tests/handshake.rs b/crates/tracedecay/src/daemon/tests/handshake.rs index b84185d0d1..5e87b7f26c 100644 --- a/crates/tracedecay/src/daemon/tests/handshake.rs +++ b/crates/tracedecay/src/daemon/tests/handshake.rs @@ -14,9 +14,14 @@ pub(super) async fn daemon_round_trip( let (server_stream, client_stream) = tokio::net::UnixStream::pair().expect("daemon socket pair"); let server = tokio::spawn(async move { - Box::pin(super::super::serve_socket_client(server_stream, engine)).await + Box::pin(super::serve_authenticated_test_client( + server_stream, + engine, + )) + .await }); let (reader, mut writer) = client_stream.into_split(); + super::write_test_auth_preface(&mut writer).await; writer .write_all(handshake.to_line().expect("handshake json").as_bytes()) .await @@ -74,7 +79,7 @@ fn daemon_handshake_defaults_missing_moved_store_adoption_to_never() { let decoded = DaemonHandshake::from_line(&encoded).expect("legacy handshake should decode"); assert_eq!( decoded.moved_store_adoption, - crate::project::MovedStoreAdoption::Never + tracedecay_project::project::MovedStoreAdoption::Never ); } @@ -274,7 +279,7 @@ fn missing_index_classifier_covers_every_auto_init_store_miss() { } let unrelated = tracedecay_domain::errors::TraceDecayError::Config { - message: "identity cutover conflict".to_string(), + message: "repository identity conflict".to_string(), }; assert!(!super::super::is_missing_index_error(&unrelated)); } @@ -310,9 +315,10 @@ async fn wire_drifted_handshake_reads_typed_refusal_then_clean_eof() { enter_test_daemon_database_scope(&client_identity.profile_root, "handshake-refusal-test"); let (client, server) = tokio::net::UnixStream::pair().expect("unix stream pair"); - let server_task = tokio::spawn(super::super::serve_socket_client(server, engine)); + let server_task = tokio::spawn(super::serve_authenticated_test_client(server, engine)); let (reader, mut writer) = client.into_split(); + super::write_test_auth_preface(&mut writer).await; // Valid JSON that is not this daemon's handshake shape, followed by the // pipelined first request a real client writes before it starts reading. writer @@ -372,9 +378,10 @@ async fn non_json_handshake_reads_invalid_handshake_refusal() { ); let (client, server) = tokio::net::UnixStream::pair().expect("unix stream pair"); - let server_task = tokio::spawn(super::super::serve_socket_client(server, engine)); + let server_task = tokio::spawn(super::serve_authenticated_test_client(server, engine)); let (reader, mut writer) = client.into_split(); + super::write_test_auth_preface(&mut writer).await; writer .write_all(b"GET / HTTP/1.1\n") .await diff --git a/crates/tracedecay/src/daemon/tests/lifecycle.rs b/crates/tracedecay/src/daemon/tests/lifecycle.rs index a9f0c5d17d..a9f99f4020 100644 --- a/crates/tracedecay/src/daemon/tests/lifecycle.rs +++ b/crates/tracedecay/src/daemon/tests/lifecycle.rs @@ -653,6 +653,7 @@ async fn tools_list_answers_under_general_saturation() { async fn one_shot_tool_call_receives_a_matching_saturation_response() { let temp = TempDir::new().expect("temp dir"); let socket = temp.path().join("daemon.sock"); + let _authority = seed_socket_authority(&socket); let listener = tokio::net::UnixListener::bind(&socket).expect("bind daemon socket"); let server = tokio::spawn(async move { let (stream, _) = listener.accept().await.expect("accept tool call"); @@ -889,6 +890,7 @@ async fn portable_broker_requests_reuse_one_authenticated_project_owner() { async fn one_shot_tool_call_aborts_when_daemon_liveness_fails_after_write() { let temp = TempDir::new().expect("temp dir"); let socket = temp.path().join("daemon.sock"); + let _authority = seed_socket_authority(&socket); let listener = tokio::net::UnixListener::bind(&socket).expect("bind daemon socket"); let server = tokio::spawn(async move { let (_stream, _) = listener.accept().await.expect("accept tool call"); @@ -926,6 +928,7 @@ async fn one_shot_tool_call_aborts_when_daemon_liveness_fails_after_write() { async fn proxied_request_uses_shared_liveness_boundary_after_write() { let temp = TempDir::new().expect("temp dir"); let socket = temp.path().join("daemon.sock"); + let _authority = seed_socket_authority(&socket); let listener = tokio::net::UnixListener::bind(&socket).expect("bind daemon socket"); let server = tokio::spawn(async move { let (_stream, _) = listener.accept().await.expect("accept proxied request"); @@ -967,16 +970,14 @@ async fn proxied_request_uses_shared_liveness_boundary_after_write() { async fn post_write_disconnect_reports_ambiguous_outcome_without_retry() { let temp = TempDir::new().expect("temp dir"); let socket = temp.path().join("daemon.sock"); + let authority = seed_socket_authority(&socket); + let token = authority.auth_token().to_string(); let listener = tokio::net::UnixListener::bind(&socket).expect("bind daemon socket"); let server = tokio::spawn(async move { let (stream, _) = listener.accept().await.expect("accept proxied request"); let (reader, _writer) = stream.into_split(); let mut lines = tokio::io::BufReader::new(reader).lines(); - lines - .next_line() - .await - .expect("read handshake") - .expect("handshake line"); + read_authenticated_handshake(&mut lines, &token).await; lines .next_line() .await @@ -1015,16 +1016,14 @@ async fn post_write_disconnect_reports_ambiguous_outcome_without_retry() { async fn one_shot_tool_call_allows_long_response_while_daemon_stays_live() { let temp = TempDir::new().expect("temp dir"); let socket = temp.path().join("daemon.sock"); + let authority = seed_socket_authority(&socket); + let token = authority.auth_token().to_string(); let listener = tokio::net::UnixListener::bind(&socket).expect("bind daemon socket"); let server = tokio::spawn(async move { let (stream, _) = listener.accept().await.expect("accept tool call"); let (reader, mut writer) = stream.into_split(); let mut lines = tokio::io::BufReader::new(reader).lines(); - lines - .next_line() - .await - .expect("read handshake") - .expect("handshake line"); + read_authenticated_handshake(&mut lines, &token).await; let request_line = lines .next_line() .await @@ -1077,16 +1076,14 @@ async fn one_shot_tool_call_allows_long_response_while_daemon_stays_live() { async fn one_shot_tool_call_preserves_response_split_across_liveness_poll() { let temp = TempDir::new().expect("temp dir"); let socket = temp.path().join("daemon.sock"); + let authority = seed_socket_authority(&socket); + let token = authority.auth_token().to_string(); let listener = tokio::net::UnixListener::bind(&socket).expect("bind daemon socket"); let server = tokio::spawn(async move { let (stream, _) = listener.accept().await.expect("accept tool call"); let (reader, mut writer) = stream.into_split(); let mut lines = tokio::io::BufReader::new(reader).lines(); - lines - .next_line() - .await - .expect("read handshake") - .expect("handshake line"); + read_authenticated_handshake(&mut lines, &token).await; let request_line = lines .next_line() .await @@ -1159,7 +1156,10 @@ async fn persistent_idle_client_closes_on_draining_without_timeout() { ) .await; - assert_eq!(receipt.clients, super::super::ShutdownStatus::Clean); + assert_eq!( + receipt.clients, + tracedecay_daemon_service::shutdown::ShutdownStatus::Clean + ); assert!(lifecycle.try_enter().is_none()); } diff --git a/crates/tracedecay/src/daemon/tests/multi_root_execute_mcp.rs b/crates/tracedecay/src/daemon/tests/multi_root_execute_mcp.rs index e3f8734989..80debfe8cc 100644 --- a/crates/tracedecay/src/daemon/tests/multi_root_execute_mcp.rs +++ b/crates/tracedecay/src/daemon/tests/multi_root_execute_mcp.rs @@ -22,9 +22,9 @@ use crate::daemon::{ DaemonHandshake, InProcessDaemonInvocationExecutor, execute_daemon_invocation, }; use crate::mcp::tools::{ToolCallRegistryOptions, handle_tool_call_with_registry_options}; -use crate::project::TraceDecay; use tracedecay_daemon_protocol::DaemonInvocationExecutor; use tracedecay_daemon_service::{DaemonInvocationOutcome, DaemonInvocationRequest}; +use tracedecay_project::project::TraceDecay; const SCOPE_SET_ID: &str = "scope-set.mcp-execute-proof"; const ALPHA_NAME: &str = "alpha_marker"; diff --git a/crates/tracedecay/src/daemon/tests/multi_root_scope_set_cas_mcp.rs b/crates/tracedecay/src/daemon/tests/multi_root_scope_set_cas_mcp.rs index c83f760e84..704f3d39c7 100644 --- a/crates/tracedecay/src/daemon/tests/multi_root_scope_set_cas_mcp.rs +++ b/crates/tracedecay/src/daemon/tests/multi_root_scope_set_cas_mcp.rs @@ -87,13 +87,14 @@ async fn run_scope_set_compare_and_swap() { tokio::net::UnixStream::pair().expect("scope-set socket pair"); let server_engine = engine.clone(); let server_task = tokio::spawn(async move { - Box::pin(super::super::serve_socket_client( + Box::pin(super::serve_authenticated_test_client( server_stream, server_engine, )) .await }); let (reader, mut writer) = client_stream.into_split(); + super::write_test_auth_preface(&mut writer).await; let mut reader = tokio::io::BufReader::new(reader); writer .write_all(handshake.to_line().expect("handshake").as_bytes()) diff --git a/crates/tracedecay/src/daemon/tests/ownership.rs b/crates/tracedecay/src/daemon/tests/ownership.rs index c24fd13046..a3885829ca 100644 --- a/crates/tracedecay/src/daemon/tests/ownership.rs +++ b/crates/tracedecay/src/daemon/tests/ownership.rs @@ -761,9 +761,9 @@ async fn failed_rekey_cancels_the_route_waiters_but_not_project_open() { let client_identity = test_client_identity_for(profile_root.clone()); initialize_test_project(&project, &client_identity).await; let graph = Arc::new( - crate::project::TraceDecay::open_with_options_for_test( + tracedecay_project::project::TraceDecay::open_with_options_for_test( &project, - crate::project::TraceDecayOpenOptions { + tracedecay_project::project::TraceDecayOpenOptions { profile_root: Some(profile_root.clone()), global_db_path: Some(client_identity.global_db_path.clone()), }, diff --git a/crates/tracedecay/src/daemon/tests/restart_proxy.rs b/crates/tracedecay/src/daemon/tests/restart_proxy.rs index 0e8602bf50..0560a3add1 100644 --- a/crates/tracedecay/src/daemon/tests/restart_proxy.rs +++ b/crates/tracedecay/src/daemon/tests/restart_proxy.rs @@ -10,41 +10,7 @@ async fn await_test_task(task: JoinHandle, label: &str) -> T { } #[cfg(unix)] -async fn answer_one_proxy_request(listener: tokio::net::UnixListener, generation: u64) { - let (stream, _addr) = listener.accept().await.expect("accept proxied client"); - let (reader, mut writer) = stream.into_split(); - let mut lines = tokio::io::BufReader::new(reader).lines(); - let handshake_line = lines - .next_line() - .await - .expect("read handshake") - .expect("handshake line"); - DaemonHandshake::from_line(&handshake_line).expect("parse handshake"); - let request_line = lines - .next_line() - .await - .expect("read request") - .expect("request line"); - let request: Value = serde_json::from_str(&request_line).expect("request json"); - let response = json!({ - "jsonrpc": "2.0", - "id": request["id"], - "result": { "generation": generation } - }); - writer - .write_all( - serde_json::to_string(&response) - .expect("response json") - .as_bytes(), - ) - .await - .expect("write response"); - writer.write_all(b"\n").await.expect("write newline"); - writer.shutdown().await.expect("shutdown fake daemon"); -} - -#[cfg(unix)] -async fn answer_one_authenticated_proxy_request( +async fn answer_one_proxy_request( listener: tokio::net::UnixListener, expected_token: &str, generation: u64, @@ -52,23 +18,7 @@ async fn answer_one_authenticated_proxy_request( let (stream, _addr) = listener.accept().await.expect("accept proxied client"); let (reader, mut writer) = stream.into_split(); let mut lines = tokio::io::BufReader::new(reader).lines(); - let auth_line = lines - .next_line() - .await - .expect("read auth preface") - .expect("auth preface line"); - let preface = - tracedecay_daemon_protocol::DaemonAuthPreface::from_line(auth_line.trim()).expect("auth"); - assert!( - preface.authenticate(expected_token), - "proxy must reload current daemon authority" - ); - let handshake_line = lines - .next_line() - .await - .expect("read handshake") - .expect("handshake line"); - DaemonHandshake::from_line(&handshake_line).expect("parse handshake"); + read_authenticated_handshake(&mut lines, expected_token).await; let request_line = lines .next_line() .await @@ -92,16 +42,12 @@ async fn answer_one_authenticated_proxy_request( #[cfg(unix)] async fn answer_initialize_route_proxy_request( stream: tokio::net::UnixStream, + expected_token: &str, daemon_target: &std::path::Path, ) -> Option { let (reader, mut writer) = stream.into_split(); let mut lines = tokio::io::BufReader::new(reader).lines(); - let handshake_line = lines - .next_line() - .await - .expect("read handshake") - .expect("handshake line"); - let handshake = DaemonHandshake::from_line(&handshake_line).expect("daemon handshake json"); + let handshake = read_authenticated_handshake(&mut lines, expected_token).await; let request_line = lines .next_line() .await @@ -243,6 +189,7 @@ fn read_deadline_classifier_accepts_typed_stalled() { async fn connect_with_restart_grace_reconnects_once_daemon_rebinds() { let dir = TempDir::new().expect("temp dir"); let socket = dir.path().join("daemon.sock"); + let _authority = seed_socket_authority(&socket); // Simulate the `tracedecay update` restart window: the socket is // missing for a while, then the new daemon binds the same path. @@ -253,7 +200,7 @@ async fn connect_with_restart_grace_reconnects_once_daemon_rebinds() { }); super::super::connect_with_restart_grace( - &tracedecay_daemon_identity::connection_for_socket_path(&socket), + &socket, std::time::Duration::from_secs(8), std::time::Duration::from_millis(50), ) @@ -266,18 +213,19 @@ async fn connect_with_restart_grace_reconnects_once_daemon_rebinds() { #[tokio::test(start_paused = true)] async fn connect_with_restart_grace_gives_up_with_restart_hint() { let dir = TempDir::new().expect("temp dir"); - let socket = dir.path().join("daemon.sock"); + let socket = dir + .path() + .canonicalize() + .expect("canonical temp dir") + .join("daemon.sock"); + let _authority = seed_socket_authority(&socket); let grace = std::time::Duration::from_millis(300); let poll = std::time::Duration::from_millis(50); let started = tokio::time::Instant::now(); - let err = super::super::connect_with_restart_grace( - &tracedecay_daemon_identity::connection_for_socket_path(&socket), - grace, - poll, - ) - .await - .expect_err("connect should fail when no daemon ever binds"); + let err = super::super::connect_with_restart_grace(&socket, grace, poll) + .await + .expect_err("connect should fail when no daemon ever binds"); let elapsed = started.elapsed(); assert!(elapsed >= grace, "restart grace must be fully observed"); @@ -334,6 +282,7 @@ async fn client_deadline_run_reports_typed_stalled() { async fn stalled_daemon_response_is_typed_within_deadline() { let dir = TempDir::new().expect("temp dir"); let socket = dir.path().join("daemon.sock"); + let _authority = seed_socket_authority(&socket); let listener = tokio::net::UnixListener::bind(&socket).expect("bind silent daemon"); let daemon = tokio::spawn(async move { let (stream, _) = listener.accept().await.expect("accept"); @@ -400,7 +349,7 @@ async fn long_lived_proxy_reloads_rotated_auth_after_daemon_restart() { let rebound_endpoint = endpoint.clone(); let (unbound_tx, unbound_rx) = tokio::sync::oneshot::channel(); let daemon = tokio::spawn(async move { - answer_one_authenticated_proxy_request(first_listener, &first_token, 1).await; + answer_one_proxy_request(first_listener, &first_token, 1).await; drop(first_authority); std::fs::remove_file(&rebound_socket).expect("unlink first socket"); unbound_tx.send(()).expect("notify daemon outage"); @@ -416,7 +365,7 @@ async fn long_lived_proxy_reloads_rotated_auth_after_daemon_restart() { .expect("second daemon authority"); let second_token = second_authority.auth_token().to_string(); assert_ne!(first_token, second_token, "daemon restart must rotate auth"); - answer_one_authenticated_proxy_request(second_listener, &second_token, 2).await; + answer_one_proxy_request(second_listener, &second_token, 2).await; }); let (mut transport, sender, mut receiver) = tracedecay_mcp::transport::ChannelTransport::new(); @@ -472,9 +421,11 @@ async fn initialize_root_routing_replaces_cached_project_and_scope() { let project_a = project_a.path().canonicalize().expect("project a path"); let project_b = project_b.path().canonicalize().expect("project b path"); let registry = - crate::test_support::host_admission::HostAdmissionTestRuntimeV1::profile(profile.path()) - .await - .expect("open retained profile runtime"); + tracedecay_project::test_support::host_admission::HostAdmissionTestRuntimeV1::profile( + profile.path(), + ) + .await + .expect("open retained profile runtime"); let global_db_path = profile.path().join("global.db"); registry .upsert_code_project("project-a", &project_a, None, None, None) @@ -574,9 +525,11 @@ async fn daemon_resolves_registry_only_initialize_root_alias() { let nested = alias.join("nested"); std::fs::create_dir_all(&nested).expect("nested alias path"); let registry = - crate::test_support::host_admission::HostAdmissionTestRuntimeV1::profile(profile.path()) - .await - .expect("open retained profile runtime"); + tracedecay_project::test_support::host_admission::HostAdmissionTestRuntimeV1::profile( + profile.path(), + ) + .await + .expect("open retained profile runtime"); let global_db_path = profile.path().join("global.db"); registry .upsert_code_project("project-registry-only", &canonical, None, None, None) @@ -653,14 +606,21 @@ async fn initialize_root_routing_fails_closed_without_pinned_configuration() { }) .to_string(); - let config = tracedecay_configuration::TraceDecayConfig { - root_dir: project.display().to_string(), - ..tracedecay_configuration::TraceDecayConfig::default() - }; - let config_path = tracedecay_configuration::get_config_path(&project); + let config_path = + tracedecay_runtime_core::storage::resolve_layout_for_current_profile(&project) + .map_or_else( + |_| tracedecay_runtime_core::config::get_tracedecay_dir(&project), + |layout| layout.data_root, + ) + .join("config.json"); std::fs::create_dir_all(config_path.parent().expect("legacy config parent")) .expect("create legacy config parent"); - let legacy_input = serde_json::to_string_pretty(&config).expect("serialize legacy config"); + let legacy_input = json!({ + "version": 1, + "root_dir": project.display().to_string(), + "sync": { "auto_init": false } + }) + .to_string(); std::fs::write(&config_path, &legacy_input).expect("write legacy config fixture"); let mut routed_handshake = base_handshake.clone(); @@ -755,6 +715,7 @@ async fn serve_stays_in_process_without_socket_or_installed_service() { async fn serve_waits_out_restart_window_when_service_owns_socket() { let dir = TempDir::new().expect("temp dir"); let socket = dir.path().join("daemon.sock"); + let _authority = seed_socket_authority(&socket); // Simulate the `tracedecay update` restart window: the service is // installed but the old daemon already unlinked the socket; the new @@ -778,6 +739,36 @@ async fn serve_waits_out_restart_window_when_service_owns_socket() { daemon.await.expect("daemon bind task"); } +#[cfg(unix)] +#[tokio::test(start_paused = true)] +async fn serve_attaches_during_first_service_start_once_the_record_appears() { + let dir = TempDir::new().expect("temp dir"); + let socket = dir.path().join("daemon.sock"); + + // A first-ever service start: neither the authority record nor the socket + // exists yet when serve starts; the daemon writes its record, then binds. + let bind_path = socket.clone(); + let daemon = tokio::spawn(async move { + tokio::time::sleep(std::time::Duration::from_millis(300)).await; + let authority = seed_socket_authority(&bind_path); + let listener = + tokio::net::UnixListener::bind(&bind_path).expect("bind first daemon socket"); + (authority, listener) + }); + + assert!( + super::super::should_proxy_serve_to_daemon_with( + &socket, + Some(&socket), + std::time::Duration::from_secs(8), + std::time::Duration::from_millis(50), + ) + .await, + "serve started before the first daemon wrote its record should still attach" + ); + daemon.await.expect("daemon start task"); +} + #[cfg(unix)] #[tokio::test(start_paused = true)] async fn serve_falls_back_when_installed_service_never_rebinds() { @@ -801,6 +792,8 @@ async fn serve_falls_back_when_installed_service_never_rebinds() { async fn proxied_request_survives_daemon_restart_window() { let dir = TempDir::new().expect("temp dir"); let socket = dir.path().join("daemon.sock"); + let authority = seed_socket_authority(&socket); + let token = authority.auth_token().to_string(); let bind_path = socket.clone(); let daemon = tokio::spawn(async move { @@ -810,12 +803,7 @@ async fn proxied_request_survives_daemon_restart_window() { let (stream, _addr) = listener.accept().await.expect("accept proxied client"); let (reader, mut writer) = stream.into_split(); let mut lines = tokio::io::BufReader::new(reader).lines(); - let handshake_line = lines - .next_line() - .await - .expect("read handshake") - .expect("handshake line"); - DaemonHandshake::from_line(&handshake_line).expect("parse handshake"); + read_authenticated_handshake(&mut lines, &token).await; let request_line = lines .next_line() .await @@ -862,18 +850,15 @@ async fn proxied_request_survives_daemon_restart_window() { async fn proxy_retries_bounded_project_warming_responses() { let dir = TempDir::new().expect("temp dir"); let socket = dir.path().join("daemon.sock"); + let authority = seed_socket_authority(&socket); + let token = authority.auth_token().to_string(); let listener = tokio::net::UnixListener::bind(&socket).expect("bind daemon socket"); let daemon = tokio::spawn(async move { for response_kind in 0..2 { let (stream, _addr) = listener.accept().await.expect("accept proxied client"); let (reader, mut writer) = stream.into_split(); let mut lines = tokio::io::BufReader::new(reader).lines(); - let handshake_line = lines - .next_line() - .await - .expect("read handshake") - .expect("handshake line"); - DaemonHandshake::from_line(&handshake_line).expect("parse handshake"); + read_authenticated_handshake(&mut lines, &token).await; let request_line = lines .next_line() .await @@ -951,68 +936,6 @@ async fn proxy_retries_bounded_project_warming_responses() { await_test_task(daemon, "warming retry daemon task").await; } -#[cfg(unix)] -#[tokio::test] -async fn long_lived_proxy_reconnects_after_daemon_socket_rebind() { - let dir = TempDir::new().expect("temp dir"); - let socket = dir.path().join("daemon.sock"); - let first_listener = tokio::net::UnixListener::bind(&socket).expect("bind first daemon socket"); - let rebound_socket = socket.clone(); - let (unbound_tx, unbound_rx) = tokio::sync::oneshot::channel(); - let daemon = tokio::spawn(async move { - answer_one_proxy_request(first_listener, 1).await; - std::fs::remove_file(&rebound_socket).expect("unlink first daemon socket"); - unbound_tx.send(()).expect("notify daemon outage"); - tokio::time::sleep(std::time::Duration::from_millis(100)).await; - let second_listener = - tokio::net::UnixListener::bind(&rebound_socket).expect("bind second daemon socket"); - answer_one_proxy_request(second_listener, 2).await; - }); - - let (mut transport, sender, mut receiver) = tracedecay_mcp::transport::ChannelTransport::new(); - let proxy_socket = socket.clone(); - let proxy = tokio::spawn(async move { - super::super::proxy_transport_to_daemon( - &proxy_socket, - &test_handshake_defaults(), - None, - &mut transport, - ) - .await - }); - - let request = |id| { - serde_json::to_string(&json!({ - "jsonrpc": "2.0", - "id": id, - "method": "tools/list" - })) - .expect("request json") - }; - sender.send(request(1)).expect("send first request"); - let first = tokio::time::timeout(std::time::Duration::from_secs(2), receiver.recv()) - .await - .expect("first response timed out") - .expect("first response"); - let first: Value = serde_json::from_str(first.trim()).expect("first response json"); - assert_eq!(first["result"]["generation"], json!(1)); - - unbound_rx.await.expect("first daemon should unlink socket"); - sender.send(request(2)).expect("send second request"); - let second = tokio::time::timeout(std::time::Duration::from_secs(2), receiver.recv()) - .await - .expect("second response timed out") - .expect("second response"); - let second: Value = serde_json::from_str(second.trim()).expect("second response json"); - assert_eq!(second["result"]["generation"], json!(2)); - - drop(sender); - await_test_task(proxy, "long-lived proxy task") - .await - .expect("proxy transport"); - await_test_task(daemon, "daemon rebind task").await; -} - #[cfg(unix)] #[tokio::test] async fn proxy_uses_daemon_initialize_route_without_registry_access() { @@ -1029,6 +952,8 @@ async fn proxy_uses_daemon_initialize_route_without_registry_access() { client_identity.global_db_path = temp_root.join("proxy-cannot-open-this-directory"); std::fs::create_dir_all(&client_identity.global_db_path).expect("non-database authority path"); + let authority = seed_socket_authority(&socket); + let token = authority.auth_token().to_string(); let listener = tokio::net::UnixListener::bind(&socket).expect("daemon socket"); let daemon_target = target.clone(); let accept_task = tokio::spawn(async move { @@ -1036,8 +961,9 @@ async fn proxy_uses_daemon_initialize_route_without_registry_access() { for _ in 0..4 { let (stream, _addr) = listener.accept().await.expect("accept daemon client"); let daemon_target = daemon_target.clone(); + let token = token.clone(); joins.push(tokio::spawn(async move { - answer_initialize_route_proxy_request(stream, &daemon_target).await + answer_initialize_route_proxy_request(stream, &token, &daemon_target).await })); } let mut projects = Vec::new(); @@ -1168,6 +1094,7 @@ async fn proxy_uses_daemon_initialize_route_without_registry_access() { async fn disconnected_client_does_not_outlive_a_daemon_that_never_answers() { let dir = TempDir::new().expect("temp dir"); let socket = dir.path().join("daemon.sock"); + let _authority = seed_socket_authority(&socket); let listener = tokio::net::UnixListener::bind(&socket).expect("bind fake daemon socket"); // A wedged daemon: it keeps accepting, so every liveness probe succeeds, // and it never answers the request it was handed. @@ -1240,8 +1167,10 @@ async fn disconnected_client_does_not_outlive_a_daemon_that_never_answers() { async fn batch_client_closing_stdin_immediately_still_receives_its_response() { let dir = TempDir::new().expect("temp dir"); let socket = dir.path().join("daemon.sock"); + let authority = seed_socket_authority(&socket); + let token = authority.auth_token().to_string(); let listener = tokio::net::UnixListener::bind(&socket).expect("bind fake daemon socket"); - let daemon = tokio::spawn(async move { answer_one_proxy_request(listener, 7).await }); + let daemon = tokio::spawn(async move { answer_one_proxy_request(listener, &token, 7).await }); let (mut transport, sender, mut receiver) = tracedecay_mcp::transport::ChannelTransport::new(); let proxy_socket = socket.clone(); diff --git a/crates/tracedecay/src/daemon/tests/rmcp_route.rs b/crates/tracedecay/src/daemon/tests/rmcp_route.rs index 82067bb8bb..9995204b2d 100644 --- a/crates/tracedecay/src/daemon/tests/rmcp_route.rs +++ b/crates/tracedecay/src/daemon/tests/rmcp_route.rs @@ -82,13 +82,23 @@ async fn rmcp_route_fixture_with_projects( #[cfg(not(unix))] let (store_administration, server) = { let store_administration = test_store_administration_for_profile(&profile_root); + // Daemon bootstrap installs the profile worker plan before any + // project opens; the portable route is driven directly here, so it + // reproduces that ordering or project open refuses. + let invocation = super::super::DaemonInvocationState::default(); + invocation + .install_worker_selection( + &store_administration, + tracedecay_domain::configuration::CodeIndexWorkerSelectionV1::default(), + ) + .expect("install portable route profile worker plan"); let server = Box::pin(super::super::portable_project_server_for_request( DaemonLifecycle::default(), store_administration.clone(), Arc::new(tokio::sync::Mutex::new( super::super::ProjectOpenGates::default(), )), - super::super::DaemonInvocationState::default(), + invocation, super::super::http_application::DaemonHttpApplicationRegistry::default(), &handshake, super::super::ProjectServerRequirement::Core, @@ -238,14 +248,6 @@ fn assert_delivered_cancellation(responses: &[Value], request_id: u64, context: ); } -fn assert_response_order(responses: &[Value], expected: &[u64], context: &str) { - let ids = responses - .iter() - .filter_map(|response| response["id"].as_u64()) - .collect::>(); - assert_eq!(ids, expected, "{context}: response order drifted"); -} - async fn assert_initialized_route_is_rmcp( mut reader: R, mut writer: W, @@ -327,16 +329,21 @@ async fn assert_initialized_route_is_rmcp( #[cfg(unix)] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn unix_production_route_selects_rmcp_only_after_initialize() { +async fn unix_production_route_serves_initialize_and_stateless_requests_over_rmcp() { let fixture = rmcp_route_fixture("unix-rmcp-production-route").await; let (server_stream, client_stream) = tokio::net::UnixStream::pair().expect("production route socket pair"); let engine = fixture.engine.clone(); let initialized_task = tokio::spawn(async move { - Box::pin(super::super::serve_socket_client(server_stream, engine)).await + Box::pin(super::serve_authenticated_test_client( + server_stream, + engine, + )) + .await }); - let (reader, writer) = client_stream.into_split(); + let (reader, mut writer) = client_stream.into_split(); + super::write_test_auth_preface(&mut writer).await; assert_initialized_route_is_rmcp( tokio::io::BufReader::new(reader), writer, @@ -347,82 +354,253 @@ async fn unix_production_route_selects_rmcp_only_after_initialize() { ) .await; - let fixture = rmcp_route_fixture("unix-legacy-production-route").await; - let response_lifecycle = fixture.server.project_server_response_lifecycle(); - let response_gate = Arc::clone(response_lifecycle.response_gate()); - let gate = response_gate.write().await; - let (server_stream, client_stream) = - tokio::net::UnixStream::pair().expect("legacy route socket pair"); + let fixture = rmcp_route_fixture("unix-stateless-production-route").await; + let client_instance_id = &fixture.handshake.client_instance_id; + let tool = unix_one_request(&fixture, &stateless_request(blocked_tool_request(4))).await; + assert_single_result(&tool, 4, "Unix stateless tools/call"); + let project = fixture + .handshake + .project_path + .clone() + .expect("fixture project"); + wait_for_host_ingest_publication(&fixture).await; + let hook = unix_one_request(&fixture, &stateless_request(hook_event_request(5, project))).await; + assert_single_result(&hook, 5, "Unix stateless hook event"); + assert_eq!(hook[0]["result"], json!({}), "{hook:?}"); + wait_for_mcp_routes( + client_instance_id, + &[ObservedMcpRoute::Rmcp, ObservedMcpRoute::Rmcp], + ) + .await; + + let refused = unix_one_request(&fixture, &blocked_tool_request(6)).await; + assert_sessionless_refusal(&refused, 6, "Unix sessionless tools/call"); + wait_for_mcp_routes( + client_instance_id, + &[ObservedMcpRoute::Rmcp, ObservedMcpRoute::Rmcp], + ) + .await; +} + +/// `tracedecay serve` opens one daemon connection per host line: the host's +/// `initialize` and `tools/call` are served by rmcp, and `initialized` and +/// `tools/list` by the static bootstrap. No line reaches another route. +#[cfg(unix)] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn serve_proxy_host_session_is_served_only_over_rmcp() { + let fixture = rmcp_route_fixture("serve-proxy-host-session").await; + let socket = fixture._temp.path().join("daemon.sock"); + let authority = super::seed_socket_authority(&socket); + let listener = tokio::net::UnixListener::bind(&socket).expect("bind proxy daemon socket"); let engine = fixture.engine.clone(); - let legacy_task = tokio::spawn(async move { - Box::pin(super::super::serve_socket_client(server_stream, engine)).await + let token = authority.auth_token().to_owned(); + let accepting = tokio::spawn(async move { + loop { + let (stream, _) = listener.accept().await.expect("accept proxy connection"); + let engine = engine.clone(); + let token = token.clone(); + tokio::spawn(async move { + Box::pin(super::super::serve_authenticated_socket_client_with_class( + tracedecay_daemon_protocol::BrokerStream::Unix(stream), + engine, + token, + super::super::DaemonClientAdmissionClass::General, + )) + .await + }); + } }); - let (reader, mut writer) = client_stream.into_split(); - let mut reader = tokio::io::BufReader::new(reader); - writer - .write_all( - fixture - .handshake - .to_line() - .expect("legacy handshake") - .as_bytes(), - ) - .await - .expect("write legacy handshake"); - writer.write_all(b"\n").await.expect("handshake newline"); - let first_request_line = format!(" \t{} ", blocked_tool_request(4)); - writer - .write_all(first_request_line.as_bytes()) - .await - .expect("write legacy first request"); - writer - .write_all(b"\n") - .await - .expect("legacy request newline"); - write_line(&mut writer, &ping_request(5)).await; + + let (mut host, host_lines, mut host_output) = + tracedecay_mcp::transport::ChannelTransport::new(); + for line in [ + initialize_request(), + json!({"jsonrpc": "2.0", "method": "notifications/initialized"}), + json!({"jsonrpc": "2.0", "id": 2, "method": "tools/list"}), + json!({ + "jsonrpc": "2.0", + "id": 3, + "method": "tools/call", + "params": { + "name": "tracedecay_status", + "arguments": {"admission_only": true, "format": "json"} + } + }), + ] { + host_lines.send(line.to_string()).expect("queue host line"); + } + drop(host_lines); + tokio::time::timeout( + PHASE_TIMEOUT, + super::super::proxy_transport_to_daemon(&socket, &fixture.handshake, None, &mut host), + ) + .await + .expect("proxied host session timed out") + .expect("proxied host session"); + accepting.abort(); + + let mut responses = Vec::new(); + while let Ok(line) = host_output.try_recv() { + if !line.trim().is_empty() { + responses.push(serde_json::from_str::(line.trim()).expect("host frame JSON")); + } + } + for id in 1..=3 { + let response = responses + .iter() + .find(|response| response["id"] == json!(id)) + .unwrap_or_else(|| panic!("host request {id} was not answered: {responses:?}")); + assert!(response.get("result").is_some(), "{response}"); + } wait_for_mcp_routes( &fixture.handshake.client_instance_id, - &[ObservedMcpRoute::Legacy], + &[ObservedMcpRoute::Rmcp, ObservedMcpRoute::Rmcp], ) .await; +} + +fn stateless_request(request: Value) -> Value { + let mut request: tracedecay_mcp::JsonRpcRequest = + serde_json::from_value(request).expect("JSON-RPC request fixture"); + assert!(tracedecay_mcp::server::attach_stateless_request_context( + &mut request + )); + serde_json::to_value(request).expect("stateless request") +} + +/// A hook event routes only to an owner that published registered host +/// ingest, which follows the core graph publication. +#[cfg(unix)] +async fn wait_for_host_ingest_publication(fixture: &RmcpRouteFixture) { + let project = fixture + .handshake + .project_path + .as_deref() + .expect("fixture project"); + let route = + ProjectRouteKey::from_handshake(project, &fixture.handshake).expect("fixture route"); + tokio::time::timeout(PHASE_TIMEOUT, async { + loop { + if fixture + .engine + .store_administration + .project_servers() + .lock() + .await + .get_route_and_touch_for( + &route, + super::super::ProjectServerRequirement::RegisteredHostIngest, + ) + .is_some() + { + return; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("fixture project never published registered host ingest"); +} + +#[cfg(unix)] +fn hook_event_request(id: u64, project: std::path::PathBuf) -> Value { + json!({ + "jsonrpc": "2.0", + "id": id, + "method": tracedecay_hooks::core_events::HOOK_EVENT_METHOD, + "params": tracedecay_hooks::core_events::DaemonHookEvent::cursor_after_shell_execution( + project, + ), + }) +} + +fn assert_single_result(responses: &[Value], id: u64, context: &str) { + assert_eq!(responses.len(), 1, "{context}: {responses:?}"); + assert_eq!(responses[0]["id"], json!(id), "{context}: {responses:?}"); + assert!( + responses[0].get("result").is_some(), + "{context}: {responses:?}" + ); +} + +fn assert_sessionless_refusal(responses: &[Value], id: u64, context: &str) { + assert_eq!(responses.len(), 1, "{context}: {responses:?}"); + assert_eq!(responses[0]["id"], json!(id), "{context}: {responses:?}"); assert_eq!( - first_request_replays(&fixture.handshake.client_instance_id), - vec![first_request_line], - "legacy transport must receive the bounded first request byte-for-byte" + responses[0]["error"]["code"], + json!(-32600), + "{context}: {responses:?}" ); - writer.shutdown().await.expect("shutdown legacy client"); - drop(gate); - let responses = tokio::time::timeout(PHASE_TIMEOUT, read_to_eof(&mut reader)) +} + +/// Sends one request on a fresh authenticated connection and returns every +/// frame the daemon wrote before closing it. +async fn one_request_responses( + mut reader: R, + mut writer: W, + handshake: &DaemonHandshake, + request: &Value, +) -> Vec +where + R: AsyncBufRead + Unpin, + W: AsyncWrite + Unpin, +{ + writer + .write_all(handshake.to_line().expect("handshake").as_bytes()) .await - .expect("legacy replay responses timed out"); - legacy_task + .expect("write handshake"); + writer.write_all(b"\n").await.expect("handshake newline"); + write_line(&mut writer, request).await; + writer + .shutdown() .await - .expect("join legacy route") - .expect("serve legacy route"); - assert_response_order( - &responses, - &[5, 4], - "Unix legacy transport must emit an independent ping before the blocked read", - ); + .expect("shutdown one-request client"); + tokio::time::timeout(PHASE_TIMEOUT, read_to_eof(&mut reader)) + .await + .expect("one-request responses timed out") } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn portable_production_route_selects_rmcp_after_initialize() { - let fixture = rmcp_route_fixture("portable-rmcp-production-route").await; +#[cfg(unix)] +async fn unix_one_request(fixture: &RmcpRouteFixture, request: &Value) -> Vec { + let (server_stream, client_stream) = + tokio::net::UnixStream::pair().expect("one-request socket pair"); + let engine = fixture.engine.clone(); + let task = tokio::spawn(async move { + Box::pin(super::serve_authenticated_test_client( + server_stream, + engine, + )) + .await + }); + let (reader, mut writer) = client_stream.into_split(); + super::write_test_auth_preface(&mut writer).await; + let responses = one_request_responses( + tokio::io::BufReader::new(reader), + writer, + &fixture.handshake, + request, + ) + .await; + task.await + .expect("join one-request connection") + .expect("serve one-request connection"); + responses +} + +async fn portable_one_request(fixture: &RmcpRouteFixture, request: &Value) -> Vec { let (listener, endpoint) = tracedecay_daemon_protocol::BrokerListener::bind( &tracedecay_daemon_protocol::default_loopback_endpoint(), ) .await - .expect("portable route listener"); + .expect("portable one-request listener"); let lifecycle = DaemonLifecycle::default(); - let server_lifecycle = lifecycle.clone(); let store_administration = fixture.store_administration.clone(); - let server_task = tokio::spawn(async move { + let task = tokio::spawn(async move { let stream = listener.accept().await.expect("accept portable client"); Box::pin(super::super::serve_windows_broker_client( stream, AUTH_TOKEN, - &server_lifecycle, + &lifecycle, store_administration, Arc::new(tokio::sync::Mutex::new( super::super::ProjectOpenGates::default(), @@ -443,33 +621,36 @@ async fn portable_production_route_selects_rmcp_after_initialize() { .await .expect("write auth preface"); writer.write_all(b"\n").await.expect("auth newline"); - assert_initialized_route_is_rmcp( + let responses = one_request_responses( tokio::io::BufReader::new(reader), writer, &fixture.handshake, - &fixture.server, - server_task, - Some(&lifecycle), + request, ) .await; + task.await + .expect("join portable one-request connection") + .expect("serve portable one-request connection"); + responses +} - let fixture = rmcp_route_fixture("portable-legacy-production-route").await; - let response_lifecycle = fixture.server.project_server_response_lifecycle(); - let response_gate = Arc::clone(response_lifecycle.response_gate()); - let gate = response_gate.write().await; +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn portable_production_route_serves_initialize_and_stateless_requests_over_rmcp() { + let fixture = rmcp_route_fixture("portable-rmcp-production-route").await; let (listener, endpoint) = tracedecay_daemon_protocol::BrokerListener::bind( &tracedecay_daemon_protocol::default_loopback_endpoint(), ) .await - .expect("portable legacy route listener"); + .expect("portable route listener"); let lifecycle = DaemonLifecycle::default(); + let server_lifecycle = lifecycle.clone(); let store_administration = fixture.store_administration.clone(); - let legacy_task = tokio::spawn(async move { - let stream = listener.accept().await.expect("accept legacy client"); + let server_task = tokio::spawn(async move { + let stream = listener.accept().await.expect("accept portable client"); Box::pin(super::super::serve_windows_broker_client( stream, AUTH_TOKEN, - &lifecycle, + &server_lifecycle, store_administration, Arc::new(tokio::sync::Mutex::new( super::super::ProjectOpenGates::default(), @@ -480,62 +661,42 @@ async fn portable_production_route_selects_rmcp_after_initialize() { }); let stream = tracedecay_daemon_protocol::BrokerStream::connect(&endpoint) .await - .expect("connect portable legacy client"); + .expect("connect portable client"); let (reader, mut writer) = stream.into_split(); - let mut reader = tokio::io::BufReader::new(reader); let preface = tracedecay_daemon_protocol::DaemonAuthPreface::new(AUTH_TOKEN) .to_line() - .expect("portable legacy auth preface"); + .expect("portable auth preface"); writer .write_all(preface.as_bytes()) .await - .expect("write legacy auth preface"); + .expect("write auth preface"); writer.write_all(b"\n").await.expect("auth newline"); - writer - .write_all( - fixture - .handshake - .to_line() - .expect("portable legacy handshake") - .as_bytes(), - ) - .await - .expect("write portable legacy handshake"); - writer.write_all(b"\n").await.expect("handshake newline"); - let first_request_line = format!(" \t{} ", blocked_tool_request(4)); - writer - .write_all(first_request_line.as_bytes()) - .await - .expect("write portable legacy first request"); - writer - .write_all(b"\n") - .await - .expect("legacy request newline"); - write_line(&mut writer, &ping_request(5)).await; + assert_initialized_route_is_rmcp( + tokio::io::BufReader::new(reader), + writer, + &fixture.handshake, + &fixture.server, + server_task, + Some(&lifecycle), + ) + .await; + + let fixture = rmcp_route_fixture("portable-stateless-production-route").await; + let tool = portable_one_request(&fixture, &stateless_request(blocked_tool_request(4))).await; + assert_single_result(&tool, 4, "portable stateless tools/call"); wait_for_mcp_routes( &fixture.handshake.client_instance_id, - &[ObservedMcpRoute::Legacy], + &[ObservedMcpRoute::Rmcp], + ) + .await; + + let refused = portable_one_request(&fixture, &blocked_tool_request(5)).await; + assert_sessionless_refusal(&refused, 5, "portable sessionless tools/call"); + wait_for_mcp_routes( + &fixture.handshake.client_instance_id, + &[ObservedMcpRoute::Rmcp], ) .await; - assert_eq!( - first_request_replays(&fixture.handshake.client_instance_id), - vec![first_request_line], - "portable legacy transport must receive the bounded first request byte-for-byte" - ); - writer.shutdown().await.expect("shutdown legacy client"); - drop(gate); - let responses = tokio::time::timeout(PHASE_TIMEOUT, read_to_eof(&mut reader)) - .await - .expect("portable legacy replay responses timed out"); - legacy_task - .await - .expect("join portable legacy route") - .expect("serve portable legacy route"); - assert_response_order( - &responses, - &[5, 4], - "portable legacy transport must emit an independent ping before the blocked read", - ); } #[cfg(unix)] @@ -718,9 +879,14 @@ async fn selected_target_rmcp_flushes_response_and_disconnect_cancels_selector_o tokio::net::UnixStream::pair().expect("selected target socket pair"); let engine = fixture.engine.clone(); let server_task = tokio::spawn(async move { - Box::pin(super::super::serve_socket_client(server_stream, engine)).await + Box::pin(super::serve_authenticated_test_client( + server_stream, + engine, + )) + .await }); let (reader, mut writer) = client_stream.into_split(); + super::write_test_auth_preface(&mut writer).await; let mut reader = tokio::io::BufReader::new(reader); writer .write_all( @@ -812,9 +978,14 @@ async fn selected_target_rmcp_flushes_response_and_disconnect_cancels_selector_o tokio::net::UnixStream::pair().expect("selector-owner cancellation socket pair"); let engine = fixture.engine.clone(); let server_task = tokio::spawn(async move { - Box::pin(super::super::serve_socket_client(server_stream, engine)).await + Box::pin(super::serve_authenticated_test_client( + server_stream, + engine, + )) + .await }); let (reader, mut writer) = client_stream.into_split(); + super::write_test_auth_preface(&mut writer).await; let mut reader = tokio::io::BufReader::new(reader); writer .write_all( @@ -945,9 +1116,14 @@ async fn production_rmcp_cancels_concurrent_requests_before_or_after_registratio tokio::net::UnixStream::pair().expect("cancellation socket pair"); let engine = fixture.engine.clone(); let server_task = tokio::spawn(async move { - Box::pin(super::super::serve_socket_client(server_stream, engine)).await + Box::pin(super::serve_authenticated_test_client( + server_stream, + engine, + )) + .await }); let (reader, mut writer) = client_stream.into_split(); + super::write_test_auth_preface(&mut writer).await; let mut reader = tokio::io::BufReader::new(reader); writer .write_all( diff --git a/crates/tracedecay/src/daemon/tests/runtime_identity.rs b/crates/tracedecay/src/daemon/tests/runtime_identity.rs index 9db70039d9..e0823015d0 100644 --- a/crates/tracedecay/src/daemon/tests/runtime_identity.rs +++ b/crates/tracedecay/src/daemon/tests/runtime_identity.rs @@ -238,22 +238,6 @@ async fn concurrent_same_identity_worktrees_keep_exact_server_and_scheduler_bind .exists(), "a stale worktree-local marker must never create or open a second project store" ); - let branch_store_exists = - std::fs::read_dir(primary_graph.store_layout().data_root.join("branches")) - .ok() - .into_iter() - .flatten() - .filter_map(std::result::Result::ok) - .any(|entry| { - entry - .path() - .extension() - .is_some_and(|extension| extension == "db") - }); - assert!( - !branch_store_exists, - "opening a linked worktree must not create a branch database" - ); // `f347a0a46` ("fix(index): require opt-in for linked worktree scopes") // gates project-open code-index activation for a linked worktree behind @@ -485,9 +469,9 @@ async fn concurrent_same_identity_worktrees_keep_exact_server_and_scheduler_bind | tracedecay_dashboard_api::AutomationSchedulerReconcileOutcome::Exiting )); assert!( - tracedecay_runtime_core::storage::read_legacy_enrollment_marker(&linked) + std::fs::read_to_string(linked.join(".tracedecay/enrollment.json")) .expect("read linked legacy marker") - .is_some_and(|marker| marker.project_id == stale_project_id), + .contains(stale_project_id), "routing must ignore, not rewrite or delete, a stale legacy worktree-local marker" ); // Every whole-worktree demand the daemon raised for the linked route on diff --git a/crates/tracedecay/src/daemon/tests/scheduler_config.rs b/crates/tracedecay/src/daemon/tests/scheduler_config.rs index d6f7532442..a2f0c83fbb 100644 --- a/crates/tracedecay/src/daemon/tests/scheduler_config.rs +++ b/crates/tracedecay/src/daemon/tests/scheduler_config.rs @@ -289,7 +289,6 @@ async fn daemon_scheduler_skips_stale_owner_key_after_rekey() { #[tokio::test] async fn disabled_finished_scheduler_reenables_with_a_fresh_owner() { let dir = TempDir::new().expect("temp dir"); - let _codex_bin = isolate_codex_app_server_binary(dir.path()); let project = dir.path().join("project"); let profile_root = dir.path().join("profile"); let client_identity = test_client_identity_for(profile_root); @@ -351,7 +350,6 @@ async fn disabled_finished_scheduler_reenables_with_a_fresh_owner() { #[tokio::test] async fn concurrent_reenable_creates_one_live_scheduler_owner() { let dir = TempDir::new().expect("temp dir"); - let _codex_bin = isolate_codex_app_server_binary(dir.path()); let project = dir.path().join("project"); let profile_root = dir.path().join("profile"); let client_identity = test_client_identity_for(profile_root); @@ -503,7 +501,6 @@ async fn unavailable_host_admission_spool_does_not_block_project_server_open() { #[tokio::test] async fn profile_reconcile_broadcasts_to_cached_projects_without_opening_uncached_projects() { let dir = TempDir::new().expect("temp dir"); - let _codex_bin = isolate_codex_app_server_binary(dir.path()); let profile_root = dir.path().join("profile"); let first_project = dir.path().join("first"); let second_project = dir.path().join("second"); @@ -645,7 +642,6 @@ async fn cached_project_reconciles_cli_enabled_automation_without_cache_probe() }; let dir = TempDir::new().expect("temp dir"); - let _codex_bin = isolate_codex_app_server_binary(dir.path()); let project = dir.path().canonicalize().expect("canonical temp dir"); let client_identity = test_client_identity_for(project.join("profile")); std::fs::create_dir_all(project.join("src")).expect("src dir"); @@ -803,7 +799,6 @@ async fn disabled_scheduler_reconcile_cannot_acknowledge_an_owner_that_then_exit use tracedecay_dashboard_api::AutomationSchedulerReconcileOutcome; let dir = TempDir::new().expect("temp dir"); - let _codex_bin = isolate_codex_app_server_binary(dir.path()); let project = dir.path().canonicalize().expect("canonical temp dir"); let client_identity = test_client_identity_for(project.join("profile")); std::fs::create_dir_all(project.join("src")).expect("src dir"); diff --git a/crates/tracedecay/src/daemon/tests/scheduler_config/paused_tick.rs b/crates/tracedecay/src/daemon/tests/scheduler_config/paused_tick.rs index 4ca26315be..9e97229611 100644 --- a/crates/tracedecay/src/daemon/tests/scheduler_config/paused_tick.rs +++ b/crates/tracedecay/src/daemon/tests/scheduler_config/paused_tick.rs @@ -11,8 +11,8 @@ use tracedecay_automation_runtime::automation::scheduler::{ use super::super::{ DaemonHandshake, apply_project_automation_patch_via_surface, enter_test_daemon_database_scope, - initialize_test_project, isolate_codex_app_server_binary, test_client_identity_for, - test_daemon_engine_for_profile, test_handshake_defaults, + initialize_test_project, test_client_identity_for, test_daemon_engine_for_profile, + test_handshake_defaults, }; #[tokio::test] @@ -46,7 +46,6 @@ fn automation_scheduler_tick_fits_the_daemon_worker_stack() { async fn paused_tick_scenario() { let dir = TempDir::new().expect("temp dir"); - let _codex_bin = isolate_codex_app_server_binary(dir.path()); let project = dir.path().canonicalize().expect("canonical temp dir"); let client_identity = test_client_identity_for(project.join("profile")); std::fs::create_dir_all(project.join("src")).expect("src dir"); diff --git a/crates/tracedecay/src/daemon/tests/socket.rs b/crates/tracedecay/src/daemon/tests/socket.rs index 03231c39fd..662d76c67a 100644 --- a/crates/tracedecay/src/daemon/tests/socket.rs +++ b/crates/tracedecay/src/daemon/tests/socket.rs @@ -17,6 +17,9 @@ use tracedecay_tool_catalog::ApplicationSurfaceOperation; /// broker tests use it too, so it is not gated on unix. const HALF_CLOSE_ROUND_TRIP_BOUND: std::time::Duration = std::time::Duration::from_secs(20); +#[cfg(unix)] +const LSP_TEST_TOKEN: &str = "lsp-test-token"; + #[cfg(unix)] fn future_lsp_deadline(after: std::time::Duration) -> tracedecay_contracts::Deadline { let now = std::time::SystemTime::now() @@ -59,10 +62,10 @@ fn lsp_test_invocation( client_instance_id: client_instance_id.to_owned(), tool_list_changed_capable: false, catalog_version: String::new(), - moved_store_adoption: crate::project::MovedStoreAdoption::Never, + moved_store_adoption: tracedecay_project::project::MovedStoreAdoption::Never, }; tracedecay_daemon_protocol::DaemonInvocationClient::new( - tracedecay_daemon_protocol::DaemonConnection::new(endpoint, None), + tracedecay_daemon_protocol::DaemonConnection::new(endpoint, LSP_TEST_TOKEN.to_owned()), handshake, ) } @@ -157,11 +160,7 @@ async fn dropping_lsp_client_closes_transport_without_spawning_detach() { let stream = listener.accept().await.expect("accept client"); let (reader, mut writer) = stream.into_split(); let mut lines = tokio::io::BufReader::new(reader).lines(); - lines - .next_line() - .await - .expect("read handshake") - .expect("handshake"); + super::read_authenticated_handshake(&mut lines, LSP_TEST_TOKEN).await; let open: Value = serde_json::from_str( &lines .next_line() @@ -230,11 +229,7 @@ async fn lsp_gateway_open_carries_control_and_returns_typed_deadline() { let stream = listener.accept().await.expect("accept client"); let (reader, _writer) = stream.into_split(); let mut lines = tokio::io::BufReader::new(reader).lines(); - lines - .next_line() - .await - .expect("read handshake") - .expect("handshake"); + super::read_authenticated_handshake(&mut lines, LSP_TEST_TOKEN).await; let open: Value = serde_json::from_str( &lines .next_line() @@ -286,11 +281,7 @@ async fn lsp_gateway_open_returns_typed_cancellation() { let stream = listener.accept().await.expect("accept client"); let (reader, _writer) = stream.into_split(); let mut lines = tokio::io::BufReader::new(reader).lines(); - lines - .next_line() - .await - .expect("read handshake") - .expect("handshake"); + super::read_authenticated_handshake(&mut lines, LSP_TEST_TOKEN).await; lines .next_line() .await @@ -342,11 +333,7 @@ async fn lsp_gateway_open_returns_typed_unavailable_when_daemon_disconnects() { let stream = listener.accept().await.expect("accept client"); let (reader, _writer) = stream.into_split(); let mut lines = tokio::io::BufReader::new(reader).lines(); - lines - .next_line() - .await - .expect("read handshake") - .expect("handshake"); + super::read_authenticated_handshake(&mut lines, LSP_TEST_TOKEN).await; lines .next_line() .await @@ -390,11 +377,7 @@ async fn stdio_bridge_session_reconnects_on_a_fresh_socket_and_resumes_frames() let first = listener.accept().await.expect("accept first connection"); let (reader, mut writer) = first.into_split(); let mut lines = tokio::io::BufReader::new(reader).lines(); - lines - .next_line() - .await - .expect("read first handshake") - .expect("first handshake"); + super::read_authenticated_handshake(&mut lines, LSP_TEST_TOKEN).await; let open: Value = serde_json::from_str( &lines .next_line() @@ -437,11 +420,7 @@ async fn stdio_bridge_session_reconnects_on_a_fresh_socket_and_resumes_frames() let second = listener.accept().await.expect("accept fresh connection"); let (reader, mut writer) = second.into_split(); let mut lines = tokio::io::BufReader::new(reader).lines(); - lines - .next_line() - .await - .expect("read second handshake") - .expect("second handshake"); + super::read_authenticated_handshake(&mut lines, LSP_TEST_TOKEN).await; let reconnect: Value = serde_json::from_str( &lines .next_line() @@ -570,10 +549,10 @@ async fn stdio_bridge_session_reconnects_on_a_fresh_socket_and_resumes_frames() client_instance_id: "client.reconnect-test".to_owned(), tool_list_changed_capable: false, catalog_version: String::new(), - moved_store_adoption: crate::project::MovedStoreAdoption::Never, + moved_store_adoption: tracedecay_project::project::MovedStoreAdoption::Never, }; let invocation = tracedecay_daemon_protocol::DaemonInvocationClient::new( - tracedecay_daemon_protocol::DaemonConnection::new(endpoint, None), + tracedecay_daemon_protocol::DaemonConnection::new(endpoint, LSP_TEST_TOKEN.to_owned()), handshake, ); let (deadline, cancellation) = active_lsp_control("cancel.lsp.reconnect-open"); @@ -670,9 +649,12 @@ async fn socket_client_requires_user_storage_scope_without_project() { prewarm_test_profile_runtime(&engine.store_administration).await; let (client, server) = tokio::net::UnixStream::pair().expect("unix stream pair"); - let server_task = tokio::spawn(Box::pin(super::super::serve_socket_client(server, engine))); + let server_task = tokio::spawn(Box::pin(super::serve_authenticated_test_client( + server, engine, + ))); let (reader, mut writer) = client.into_split(); + super::write_test_auth_preface(&mut writer).await; let handshake = DaemonHandshake { client_identity, ..test_handshake_defaults() @@ -744,8 +726,11 @@ async fn projectless_project_list_reads_the_empty_profile_registry() { ); let (client, server) = tokio::net::UnixStream::pair().expect("unix stream pair"); - let server_task = tokio::spawn(Box::pin(super::super::serve_socket_client(server, engine))); + let server_task = tokio::spawn(Box::pin(super::serve_authenticated_test_client( + server, engine, + ))); let (reader, mut writer) = client.into_split(); + super::write_test_auth_preface(&mut writer).await; let handshake = DaemonHandshake { client_identity, ..test_handshake_defaults() @@ -818,8 +803,11 @@ async fn projectless_tools_list_advertises_registry_tools() { ); let (client, server) = tokio::net::UnixStream::pair().expect("unix stream pair"); - let server_task = tokio::spawn(Box::pin(super::super::serve_socket_client(server, engine))); + let server_task = tokio::spawn(Box::pin(super::serve_authenticated_test_client( + server, engine, + ))); let (reader, mut writer) = client.into_split(); + super::write_test_auth_preface(&mut writer).await; let handshake = DaemonHandshake { client_identity, ..test_handshake_defaults() @@ -942,9 +930,12 @@ async fn user_session_read_bypasses_unregistered_project_route() { std::fs::create_dir_all(&unregistered_project).expect("unregistered project directory"); let (client, server) = tokio::net::UnixStream::pair().expect("unix stream pair"); - let server_task = tokio::spawn(Box::pin(super::super::serve_socket_client(server, engine))); + let server_task = tokio::spawn(Box::pin(super::serve_authenticated_test_client( + server, engine, + ))); let (reader, mut writer) = client.into_split(); + super::write_test_auth_preface(&mut writer).await; let handshake = DaemonHandshake { project_path: Some(unregistered_project), client_identity, @@ -1017,9 +1008,12 @@ async fn socket_client_routes_multiple_closed_invocations_without_falling_back_t ); prewarm_test_profile_runtime(&engine.store_administration).await; let (client, server) = tokio::net::UnixStream::pair().expect("unix stream pair"); - let server_task = tokio::spawn(Box::pin(super::super::serve_socket_client(server, engine))); + let server_task = tokio::spawn(Box::pin(super::serve_authenticated_test_client( + server, engine, + ))); let (reader, mut writer) = client.into_split(); + super::write_test_auth_preface(&mut writer).await; let handshake = DaemonHandshake { client_identity, ..test_handshake_defaults() @@ -1151,8 +1145,11 @@ async fn socket_git_preview_apply_replay_and_pre_admission_problems_are_canonica let (client, server) = tokio::net::UnixStream::pair().expect("unix stream pair"); let engine_for_test = engine.clone(); - let server_task = tokio::spawn(Box::pin(super::super::serve_socket_client(server, engine))); + let server_task = tokio::spawn(Box::pin(super::serve_authenticated_test_client( + server, engine, + ))); let (reader, mut writer) = client.into_split(); + super::write_test_auth_preface(&mut writer).await; writer .write_all(handshake.to_line().expect("handshake").as_bytes()) .await @@ -1583,12 +1580,12 @@ async fn daemon_linked_worktree_route_repairs_primary_identity_and_keeps_alias() .expect("linked project registry context present"); assert_eq!( context.project.canonical_root, - crate::test_support::host_admission::HostAdmissionTestRuntimeV1::canonical_project_key( + tracedecay_project::test_support::host_admission::HostAdmissionTestRuntimeV1::canonical_project_key( &primary ) ); assert!(context.aliases.iter().any(|alias| { alias.alias_path - == crate::test_support::host_admission::HostAdmissionTestRuntimeV1::canonical_project_key(&linked) + == tracedecay_project::test_support::host_admission::HostAdmissionTestRuntimeV1::canonical_project_key(&linked) })); } diff --git a/crates/tracedecay/src/daemon/wire_io.rs b/crates/tracedecay/src/daemon/wire_io.rs index 270ac86e12..563f69d2df 100644 --- a/crates/tracedecay/src/daemon/wire_io.rs +++ b/crates/tracedecay/src/daemon/wire_io.rs @@ -81,11 +81,11 @@ mod wire_bound_tests { use std::sync::Arc; use super::{ - BrokerStreamTransport, DaemonLifecycle, read_line_handling_wire_oversized, - serve_routed_rmcp_connection, + BrokerStreamTransport, read_line_handling_wire_oversized, serve_routed_rmcp_connection, }; use rmcp::transport::Transport; use tokio::io::{AsyncBufReadExt, AsyncWriteExt}; + use tracedecay_daemon_service::shutdown::DaemonLifecycle; use tracedecay_framing::{WIRE_RECORD_TOO_LARGE, is_wire_oversized_io_error}; use tracedecay_mcp::McpTransport; diff --git a/crates/tracedecay/src/daemon/work_evidence_retrieval_tests.rs b/crates/tracedecay/src/daemon/work_evidence_retrieval_tests.rs index ac2a3b9f54..ef8c50d7ec 100644 --- a/crates/tracedecay/src/daemon/work_evidence_retrieval_tests.rs +++ b/crates/tracedecay/src/daemon/work_evidence_retrieval_tests.rs @@ -31,13 +31,14 @@ async fn registered_project_session_hydrates_provider_qualified_task_evidence() let project_id = id::("project.work-task-session"); let repository_id = id::("repository.work-task-session"); let worktree_id = id::("worktree.work-task-session"); - let runtime = crate::test_support::host_admission::HostAdmissionTestRuntimeV1::project( - profile.path(), - &project, - project_id.clone(), - ) - .await - .expect("registered project session runtime"); + let runtime = + tracedecay_project::test_support::host_admission::HostAdmissionTestRuntimeV1::project( + profile.path(), + &project, + project_id.clone(), + ) + .await + .expect("registered project session runtime"); let database = runtime .registered_database_arc(tracedecay_sessions::admission::HostAdmissionScope::Project) .expect("registered project session database"); diff --git a/crates/tracedecay/src/daemon/work_evidence_retrieval_tests/continuation.rs b/crates/tracedecay/src/daemon/work_evidence_retrieval_tests/continuation.rs index 73c3b26c57..ff4be9ec52 100644 --- a/crates/tracedecay/src/daemon/work_evidence_retrieval_tests/continuation.rs +++ b/crates/tracedecay/src/daemon/work_evidence_retrieval_tests/continuation.rs @@ -25,13 +25,14 @@ async fn continuation_resumes_the_same_provider_session_without_repeating_eviden let project_id = id::("project.work-task-session-continuation"); let repository_id = id::("repository.work-task-session-continuation"); let worktree_id = id::("worktree.work-task-session-continuation"); - let runtime = crate::test_support::host_admission::HostAdmissionTestRuntimeV1::project( - profile.path(), - &project, - project_id.clone(), - ) - .await - .expect("registered project session runtime"); + let runtime = + tracedecay_project::test_support::host_admission::HostAdmissionTestRuntimeV1::project( + profile.path(), + &project, + project_id.clone(), + ) + .await + .expect("registered project session runtime"); let database = runtime .registered_database_arc(tracedecay_sessions::admission::HostAdmissionScope::Project) .expect("registered project session database"); diff --git a/crates/tracedecay/src/dashboard.rs b/crates/tracedecay/src/dashboard.rs index fea38c3726..fe27da90a9 100644 --- a/crates/tracedecay/src/dashboard.rs +++ b/crates/tracedecay/src/dashboard.rs @@ -1,4 +1,4 @@ -//! Root-side dashboard composition: the SPA-router seam plus the +//! Root-side dashboard composition: the graph-to-context mapping plus the //! daemon-coupled integration fixtures. //! //! The dashboard API, routes, read models, services and their tests, lives @@ -6,7 +6,7 @@ //! //! The embedded asset bundle is not generated here: the shipping binary crate //! embeds it and hands it to this library through the registered product -//! runtime ([`mod@crate::product_runtime`]). The canonical API crate owns the +//! runtime ([`mod@tracedecay_project::product_runtime`]). The canonical API crate owns the //! resulting HTTP router and transport policy. use tracedecay_dashboard_api::DashboardProjectContext; @@ -16,23 +16,15 @@ use tracedecay_daemon_service::DaemonInvocationService; #[cfg(feature = "test-transport")] use tracedecay_dashboard_api::{ DashboardApplicationRuntime, DashboardAutomationAuthorityV1, DashboardAutomationWriter, - DashboardGitCorrelationReadPortV1, DashboardLcmReadPortV1, - DashboardProfileCodeIndexWorkerSettingsPort, standalone_dashboard_automation_writer, + DashboardGitCorrelationReadPortV1, DashboardHostAdmissionTestAuthorityV1, + DashboardLcmReadPortV1, DashboardProfileCodeIndexWorkerSettingsPort, DashboardTestEndpointV1, + DashboardTestProjectGraphsV1, standalone_dashboard_automation_writer, }; #[cfg(feature = "test-transport")] use tracedecay_session_runtime::session_retrieval::{ DaemonSessionRetrievalRoot, DaemonSessionRetrievalService, SessionRetrievalServingIdentityV1, }; -#[cfg(feature = "test-transport")] -#[doc(hidden)] -pub use tracedecay_dashboard_api::contract_schema; -#[cfg(feature = "test-transport")] -#[doc(hidden)] -pub use tracedecay_dashboard_api::{ - DashboardHostAdmissionTestAuthorityV1, DashboardTestEndpointV1, -}; - /// Canonical observation-capture seeding for dashboard integration fixtures. #[cfg(any(test, feature = "test-transport"))] #[doc(hidden)] @@ -44,16 +36,6 @@ pub mod observation_seed; #[path = "dashboard_graph_test_runtime.rs"] pub mod dashboard_graph_test_runtime; -/// Embedded single-page-app routes shared by production and integration -/// servers. The caller supplies the registered product runtime's bundle; -/// `tracedecay-api` owns route matching, cache policy, and the API fallback -/// boundary. -#[doc(hidden)] -#[hotpath::measure(label = "dashboard.spa")] -pub fn spa_router(assets: tracedecay_api::StaticDashboardAssets) -> axum::Router { - tracedecay_api::static_dashboard_router(std::sync::Arc::new(assets)) -} - /// Installs the canonical root-owned registered schema port before dashboard /// integration fixtures open any database authority. #[cfg(feature = "test-transport")] @@ -63,8 +45,9 @@ pub fn register_test_schema_installer() { REGISTER.call_once(tracedecay_global_db::register_registered_schema_installer); } -pub(crate) fn dashboard_project_context( - graph: &crate::project::TraceDecay, +#[doc(hidden)] +pub fn dashboard_project_context( + graph: &tracedecay_project::project::TraceDecay, ) -> DashboardProjectContext { DashboardProjectContext { store_layout: graph.store_layout().clone(), @@ -76,26 +59,11 @@ pub(crate) fn dashboard_project_context( } } -#[cfg(feature = "test-transport")] -#[doc(hidden)] -#[derive(Clone, Default)] -pub struct DashboardTestProjectGraphsV1 { - contexts: tracedecay_dashboard_api::DashboardTestProjectGraphsV1, -} - -#[cfg(feature = "test-transport")] -impl DashboardTestProjectGraphsV1 { - pub fn register(&self, graph: std::sync::Arc) { - self.contexts - .register(std::sync::Arc::new(dashboard_project_context(&graph))); - } -} - #[cfg(feature = "test-transport")] #[doc(hidden)] #[allow(clippy::too_many_arguments)] pub async fn run_until_shutdown_for_tests_with_host_admission( - graph: std::sync::Arc, + graph: std::sync::Arc, authority: DashboardHostAdmissionTestAuthorityV1, project_graphs: DashboardTestProjectGraphsV1, endpoint: DashboardTestEndpointV1<'_>, @@ -109,7 +77,7 @@ where tracedecay_dashboard_api::run_until_shutdown_for_tests_with_host_admission( std::sync::Arc::new(dashboard_project_context(&graph)), authority, - project_graphs.contexts, + project_graphs, endpoint, build_version, spa_routes, @@ -129,7 +97,7 @@ where #[cfg(feature = "test-transport")] #[doc(hidden)] pub async fn dashboard_automation_authority_for_test( - cg: std::sync::Arc, + cg: std::sync::Arc, profile_root: impl AsRef, ) -> tracedecay_domain::errors::Result<(DashboardAutomationAuthorityV1, DashboardAutomationWriter)> { @@ -219,7 +187,7 @@ pub async fn dashboard_automation_authority_for_test( #[cfg(feature = "test-transport")] #[doc(hidden)] pub async fn dashboard_configuration_authorities_for_test( - cg: std::sync::Arc, + cg: std::sync::Arc, profile_database: tracedecay_global_db::RegisteredGlobalDbLeaseV1, ) -> tracedecay_domain::errors::Result<( std::sync::Arc, @@ -236,7 +204,7 @@ pub async fn dashboard_configuration_authorities_for_test( #[cfg(feature = "test-transport")] #[doc(hidden)] pub async fn dashboard_lcm_read_authority_for_test( - cg: &crate::project::TraceDecay, + cg: &tracedecay_project::project::TraceDecay, registry: &tracedecay_global_db::RegisteredGlobalDb, project_database: tracedecay_global_db::RegisteredGlobalDbLeaseV1, ) -> Option> { @@ -319,15 +287,17 @@ mod spa_router_tests { #[tokio::test] async fn unknown_api_paths_never_receive_the_single_page_app() { - let response = super::spa_router(crate::product_runtime::FIXTURE_DASHBOARD_ASSETS) - .oneshot( - Request::builder() - .uri("/api/not-a-real-route") - .body(Body::empty()) - .expect("request"), - ) - .await - .expect("SPA router response"); + let response = tracedecay_api::static_dashboard_router(std::sync::Arc::new( + tracedecay_project::product_runtime::FIXTURE_DASHBOARD_ASSETS, + )) + .oneshot( + Request::builder() + .uri("/api/not-a-real-route") + .body(Body::empty()) + .expect("request"), + ) + .await + .expect("SPA router response"); assert_eq!(response.status(), StatusCode::NOT_FOUND); } diff --git a/crates/tracedecay/src/dashboard/observation_seed.rs b/crates/tracedecay/src/dashboard/observation_seed.rs index b685880222..d3dac63d22 100644 --- a/crates/tracedecay/src/dashboard/observation_seed.rs +++ b/crates/tracedecay/src/dashboard/observation_seed.rs @@ -1,7 +1,7 @@ //! Canonical observation-capture seeding for dashboard integration fixtures. //! //! The daemon session-temporal projection discovers sessions exclusively from -//! `session_temporal_observation_effects`; raw `sessions`/`session_messages` +//! `session_temporal_observation_effects`; raw `sessions`/`lcm_raw_messages` //! upserts never reach it. Fixtures that want their sessions readable through //! the daemon LCM/explorer authorities must therefore seed messages through //! the same durable-observation persist/project route production ingest uses, @@ -19,9 +19,10 @@ use tracedecay_domain::{ SanitizationReceiptId, SanitizationReceiptRefV1, SanitizationReceiptV1, SanitizerDispositionV1, SensitivityV1, SessionId, UtcMicros, }; +use tracedecay_session_temporal_store::SessionTemporalAccess; use tracedecay_store::{ AnchoredObservationWrite, ObservationProjectionStore, ObservationStore, ObservationWrite, - build_observation_resolution_authorization_v1, build_observation_retrieval_anchor_v2, + build_observation_resolution_authorization_v1, build_observation_retrieval_anchor, }; use tracedecay_domain::errors::{Result, TraceDecayError}; @@ -171,7 +172,7 @@ pub async fn seed_session_message_observation_for_test( tracedecay_store::OBSERVATION_CAPTURE_AUTHORITY_V1, ) .map_err(|error| fixture_error("resolution authorization", error))?; - let anchor = build_observation_retrieval_anchor_v2( + let anchor = build_observation_retrieval_anchor( write.observation(), projection.clone(), UtcMicros(1), @@ -208,7 +209,7 @@ pub async fn materialize_session_temporal_refresh_for_test( .materialize_pending_session_refresh_for_test(&session_id) .await .map_err(|error| fixture_error("materialize session refresh", error))?; - project_database + SessionTemporalAccess::new(project_database) .apply_active_session_relation_projection( &session_id, std::sync::Arc::new(DashboardFixtureGraphCancellation), diff --git a/crates/tracedecay/src/dashboard_graph_test_runtime.rs b/crates/tracedecay/src/dashboard_graph_test_runtime.rs index 68f0a4ed6c..cb5a5042ed 100644 --- a/crates/tracedecay/src/dashboard_graph_test_runtime.rs +++ b/crates/tracedecay/src/dashboard_graph_test_runtime.rs @@ -95,11 +95,12 @@ impl DashboardGraphTestRuntimeV1 { label = "dashboard.graph.project_memory" ) .await?; - let graph_proxy = crate::test_support::host_admission::await_bound_graph_runtime( - &project_database, - "bind dashboard project graph", - ) - .await?; + let graph_proxy = + tracedecay_project::test_support::host_admission::await_bound_graph_runtime( + &project_database, + "bind dashboard project graph", + ) + .await?; // A lost set race means another caller already bound the same // weak proxy; the required postcondition holds either way. let _ = registered.bind_project_graph_runtime(graph_proxy); @@ -112,19 +113,19 @@ impl DashboardGraphTestRuntimeV1 { &self, project_root: &std::path::Path, project_id: tracedecay_domain::ProjectId, - ) -> tracedecay_domain::errors::Result { + ) -> tracedecay_domain::errors::Result { // Fixture identity is pinned in the sanctioned `.git/` repository // identity marker; nothing is written into the working tree. tracedecay_runtime_core::storage::pin_fixture_repository_identity( project_root, project_id.as_str(), )?; - let options = crate::project::TraceDecayOpenOptions { + let options = tracedecay_project::project::TraceDecayOpenOptions { profile_root: Some(self.profile_root.clone()), global_db_path: Some(self.profile_database.db_path().to_path_buf()), }; let layout = hotpath::future!( - crate::project::TraceDecay::resolve_registered_configuration_layout( + tracedecay_project::project::TraceDecay::resolve_registered_configuration_layout( project_root, &options, self.profile_database.as_ref(), @@ -139,7 +140,7 @@ impl DashboardGraphTestRuntimeV1 { } let project_database = self.project_sessions(project_root, project_id).await?; hotpath::future!( - crate::project::TraceDecay::init_with_registered_configuration( + tracedecay_project::project::TraceDecay::init_with_registered_configuration( project_root, options, layout, @@ -156,13 +157,13 @@ impl DashboardGraphTestRuntimeV1 { pub async fn reopen( &self, project_root: &std::path::Path, - ) -> tracedecay_domain::errors::Result { - let options = crate::project::TraceDecayOpenOptions { + ) -> tracedecay_domain::errors::Result { + let options = tracedecay_project::project::TraceDecayOpenOptions { profile_root: Some(self.profile_root.clone()), global_db_path: Some(self.profile_database.db_path().to_path_buf()), }; let layout = hotpath::future!( - crate::project::TraceDecay::resolve_registered_configuration_layout( + tracedecay_project::project::TraceDecay::resolve_registered_configuration_layout( project_root, &options, self.profile_database.as_ref(), @@ -186,7 +187,7 @@ impl DashboardGraphTestRuntimeV1 { })?; let project_database = self.project_sessions(project_root, project_id).await?; hotpath::future!( - crate::project::TraceDecay::open_with_registered_configuration( + tracedecay_project::project::TraceDecay::open_with_registered_configuration( project_root, options, layout, diff --git a/crates/tracedecay/src/doctor.rs b/crates/tracedecay/src/doctor.rs index e3413d8d77..d8e72b0fb0 100644 --- a/crates/tracedecay/src/doctor.rs +++ b/crates/tracedecay/src/doctor.rs @@ -25,6 +25,8 @@ use tracedecay_daemon_protocol::RequestedOutputFormat; use tracedecay_daemon_service::application_surface::{ execute_application_surface, resolve_application_surface_dispatch, }; +#[cfg(unix)] +use tracedecay_daemon_service::logging::recent_watcher_events; use tracedecay_runtime_core::text::format_token_count; /// Opens an isolated daemon-registered profile database so Doctor tests can @@ -103,7 +105,7 @@ pub async fn run_doctor( return Err(error); } }; - let build_version = crate::version::build_version()?; + let build_version = tracedecay_project::version::build_version()?; let mut dc = DoctorCounters::new(); eprintln!("\n\x1b[1mtracedecay doctor v{build_version}\x1b[0m\n"); @@ -559,8 +561,8 @@ fn fallback_database_path(project_path: &Path) -> Option { { return Some(layout.graph_db_path); } - let data_root = crate::config::get_tracedecay_dir(project_path); - let db_path = data_root.join(crate::config::db_filename(&data_root)); + let data_root = tracedecay_project::config::get_tracedecay_dir(project_path); + let db_path = data_root.join(tracedecay_project::config::db_filename(&data_root)); db_path.is_file().then_some(db_path) } @@ -571,7 +573,6 @@ fn database_recovery_guidance(db_path: &Path) -> String { let mut graph_dirty = db_path.as_os_str().to_os_string(); graph_dirty.push(".dirty"); let graph_dirty = PathBuf::from(graph_dirty); - let legacy_dirty = data_root.join("dirty"); let sessions_path = data_root.join(tracedecay_runtime_core::storage::SESSIONS_DB_FILENAME); format!( @@ -581,16 +582,14 @@ fn database_recovery_guidance(db_path: &Path) -> String { WAL: {}\n\ SHM: {}\n\ graph dirty sentinel: {}\n\ - legacy dirty sentinel (if present): {}\n\ `sessions.db` is separate and must not be removed: {}\n\ Facts are stored in the graph database; automatic default-store rebuild is intentionally blocked because it cannot preserve them generically.\n\ - Do not run `tracedecay init`, `tracedecay sync --force`, or `tracedecay wipe` until that recovery set is safely copied.\n\ + Do not run `tracedecay init`, `tracedecay sync`, or `tracedecay wipe` until that recovery set is safely copied.\n\ Report the preserved set at https://github.com/ScriptedAlchemy/tracedecay/issues for offline recovery.", db_path.display(), wal_path.display(), shm_path.display(), graph_dirty.display(), - legacy_dirty.display(), sessions_path.display(), ) } @@ -709,7 +708,7 @@ fn check_watcher(dc: &mut DoctorCounters) { #[cfg(unix)] { - let events = crate::daemon::recent_watcher_events(2000); + let events = recent_watcher_events(2000); if events.is_empty() { dc.info("Daemon running; no recent watcher telemetry in the log yet"); return; @@ -760,7 +759,8 @@ const DOMAIN_SYMBOL_RULES_FILENAME: &str = "domain-symbols.toml"; /// nodes, so Doctor is where the author finds out. `None` (the normal case) /// keeps Doctor silent about a file that is not there. fn domain_symbol_rules_warning(project_path: &Path) -> Option { - let rules = crate::config::get_tracedecay_dir(project_path).join(DOMAIN_SYMBOL_RULES_FILENAME); + let rules = tracedecay_project::config::get_tracedecay_dir(project_path) + .join(DOMAIN_SYMBOL_RULES_FILENAME); rules.is_file().then(|| { format!( "Domain symbol extraction is unavailable: no extractor reads {}, \ @@ -878,7 +878,6 @@ fn check_external_tools(dc: &mut DoctorCounters) { let diagnostics = tracedecay_mcp::ast_grep_diagnostics_json(); let installed = json_bool(&diagnostics, "installed"); let rewrite_available = json_bool(&diagnostics, "rewrite_available"); - let outline_available = json_bool(&diagnostics, "outline_available"); let version = diagnostics .get("version") .and_then(serde_json::Value::as_str) @@ -888,18 +887,12 @@ fn check_external_tools(dc: &mut DoctorCounters) { .and_then(serde_json::Value::as_str) .unwrap_or("ast-grep status unavailable"); - if outline_available { - dc.pass(&format!( - "ast-grep {version}: rewrite and outline support available" - )); + if rewrite_available { + dc.pass(&format!("ast-grep {version}: rewrite support available")); return; } - if rewrite_available { - dc.warn(&format!( - "ast-grep {version}: rewrite support available, but outline support is missing" - )); - } else if installed { + if installed { dc.warn(&format!( "ast-grep {version}: optional ast-grep-backed tools are unavailable" )); @@ -907,7 +900,7 @@ fn check_external_tools(dc: &mut DoctorCounters) { dc.warn("ast-grep not found on PATH; optional ast-grep-backed tools are hidden"); } dc.info(message); - dc.info("Install or update ast-grep to >= 0.44, then rerun `tracedecay install` or `tracedecay update-plugin` if your agent integration caches tool metadata."); + dc.info("Install or update ast-grep, then rerun `tracedecay install` or `tracedecay update-plugin` if your agent integration caches tool metadata."); } fn json_bool(value: &serde_json::Value, key: &str) -> bool { diff --git a/crates/tracedecay/src/doctor/tests.rs b/crates/tracedecay/src/doctor/tests.rs index 5f494d7690..462fe7704a 100644 --- a/crates/tracedecay/src/doctor/tests.rs +++ b/crates/tracedecay/src/doctor/tests.rs @@ -3,6 +3,7 @@ use std::time::SystemTime; use super::*; use tracedecay_agent_hosts::agents::AgentIntegration; +use tracedecay_session_temporal_store::SessionTemporalAccess; #[test] fn supported_optional_host_absences_reach_doctor_without_host_directories() { @@ -79,8 +80,10 @@ fn domain_symbol_rules_warning_is_silent_without_the_file() { let project = tempfile::tempdir().expect("temp project root"); assert_eq!(domain_symbol_rules_warning(project.path()), None); - std::fs::create_dir_all(crate::config::get_tracedecay_dir(project.path())) - .expect("create project marker dir"); + std::fs::create_dir_all(tracedecay_project::config::get_tracedecay_dir( + project.path(), + )) + .expect("create project marker dir"); assert_eq!( domain_symbol_rules_warning(project.path()), None, @@ -144,7 +147,9 @@ async fn temporal_health_adapter_is_read_only_and_clean_on_canonical_schema() { let before = std::fs::read(&db_path).unwrap(); let before_family = temporal_family_manifest(&db_path); - let report = db.session_temporal_doctor_health().await; + let report = SessionTemporalAccess::new(db) + .session_temporal_doctor_health() + .await; let encoded = serde_json::to_value(report).unwrap(); assert_eq!(encoded["status"], "complete"); @@ -207,7 +212,12 @@ async fn temporal_health_detects_index_and_column_migration_gaps() { ) .await .unwrap(); - let report = serde_json::to_value(db.session_temporal_doctor_health().await).unwrap(); + let report = serde_json::to_value( + SessionTemporalAccess::new(db) + .session_temporal_doctor_health() + .await, + ) + .unwrap(); assert_eq!(report["status"], "partial"); let findings = report["findings"].as_array().unwrap(); assert!( @@ -410,89 +420,6 @@ fn doctor_result_treats_unavailable_canonical_report_as_unknown() { .unwrap(); } -/// The canonical, plainly spelled identity of a fixture path. -/// -/// Canonicalizing on every host is what keeps the fixture and the production -/// resolver naming one directory; spelling the result plainly is what lets it -/// still be handed to `git`, which refuses the `\\?\` form `canonicalize` -/// returns on Windows. -fn canonical_temp_path(path: &std::path::Path) -> std::path::PathBuf { - tracedecay_runtime_core::path_safety::canonical_root_identity(path) -} - -#[tokio::test] -async fn store_layout_resolution_surfaces_split_identity_conflict() --> std::result::Result<(), Box> { - let dir = tempfile::TempDir::new()?; - let profile_root = dir.path().join("profile"); - let project_root = dir.path().join("repo"); - std::fs::create_dir_all(&project_root)?; - let project_root = canonical_temp_path(&project_root); - let status = std::process::Command::new("git") - .args(["init", "--quiet"]) - .current_dir(tracedecay_runtime_core::path_safety::plain_host_path( - &project_root, - )) - .status()?; - assert!(status.success()); - - for project_id in ["proj_doctor_selected", "proj_doctor_legacy"] { - let layout = tracedecay_runtime_core::storage::profile_sharded_layout( - &project_root, - &profile_root, - &tracedecay_runtime_core::storage::EnrollmentMarker { - project_id: project_id.to_string(), - storage_mode: tracedecay_runtime_core::storage::StorageMode::ProfileSharded, - }, - )?; - std::fs::create_dir_all(&layout.data_root)?; - std::fs::write(&layout.graph_db_path, b"graph")?; - tracedecay_runtime_core::storage::write_store_manifest(&layout)?; - } - tracedecay_runtime_core::storage::write_repository_identity_marker( - &project_root, - "proj_doctor_selected", - )?; - - let open_options = crate::project::TraceDecayOpenOptions { - profile_root: Some(profile_root.clone()), - global_db_path: Some(dir.path().join("global.db")), - }; - let selected_db = profile_root.join("projects/proj_doctor_selected/tracedecay.db"); - let legacy_db = profile_root.join("projects/proj_doctor_legacy/tracedecay.db"); - let selected_before = std::fs::read(&selected_db)?; - let legacy_before = std::fs::read(&legacy_db)?; - - let resolution = crate::project::TraceDecay::try_initialized_store_layout_with_options( - &project_root, - &open_options, - ) - .await; - let diagnostic = format!("{resolution:?}"); - assert!( - diagnostic.contains("identity cutover conflict"), - "{diagnostic}" - ); - assert!(diagnostic.contains("proj_doctor_selected"), "{diagnostic}"); - assert!(diagnostic.contains("proj_doctor_legacy"), "{diagnostic}"); - assert!( - diagnostic.contains("tracedecay migrate consolidate"), - "{diagnostic}" - ); - assert!( - diagnostic.contains("--source-project-id proj_doctor_legacy"), - "{diagnostic}" - ); - assert!( - diagnostic.contains("--target-project-id proj_doctor_selected"), - "{diagnostic}" - ); - assert!(diagnostic.contains("no files changed"), "{diagnostic}"); - assert_eq!(std::fs::read(selected_db)?, selected_before); - assert_eq!(std::fs::read(legacy_db)?, legacy_before); - Ok(()) -} - #[test] fn doctor_warns_for_intentionally_held_service_states_without_activation_advice() { use super::{DaemonServiceDoctorVerdict, daemon_service_doctor_verdict}; diff --git a/crates/tracedecay/src/host_admission_test.rs b/crates/tracedecay/src/host_admission_test.rs index 062e3bc9e9..a8aa0cf61a 100644 --- a/crates/tracedecay/src/host_admission_test.rs +++ b/crates/tracedecay/src/host_admission_test.rs @@ -13,7 +13,7 @@ use tracedecay_domain::{ ProjectId, ProviderId, RetentionClass, SessionId, UserProfileId, }; use tracedecay_global_db::{GlobalDbObservationStore, RegisteredGlobalDb}; -use tracedecay_privacy::{ClaudeRecordParseErrorV1, parse_normalized_observation_record_v1}; +use tracedecay_privacy::{ObservationRecordParseErrorV1, parse_normalized_observation_record_v1}; use tracedecay_runtime_core::background_cpu::ProcessBackgroundCpuV1; use tracedecay_sessions::admission::HostAdmissionStatus; use tracedecay_sessions::repository_provenance::RepositoryProvenanceAdmissionContext; @@ -85,7 +85,7 @@ fn host_capture_request(scope: ObservationScopeV1, record_id: &str) -> CaptureOb }], CanonicalObservationEvidenceV1::new(ordering_domain, range), ) - .map_err(|_| ClaudeRecordParseErrorV1::NormalizationFailed) + .map_err(|_| ObservationRecordParseErrorV1::NormalizationFailed) }) .unwrap(); CaptureObservationRequest::new( @@ -170,7 +170,7 @@ async fn host_ingress_binds_provenance_to_authoritative_project_and_replays_stab // CPU authority that plan installs and refuses with // `Unavailable/background_cpu_unavailable` when none is injected. let background_cpu = - crate::test_support::host_admission::ensure_process_background_cpu_authority() + tracedecay_project::test_support::host_admission::ensure_process_background_cpu_authority() .expect("install the process background CPU authority"); let root = TempDir::new().unwrap(); let repository_root = root.path().join("repository"); @@ -374,7 +374,7 @@ async fn registered_profile_runtime_is_required_and_mismatch_never_falls_back() // CPU authority that plan installs and refuses with // `Unavailable/background_cpu_unavailable` when none is injected. let background_cpu = - crate::test_support::host_admission::ensure_process_background_cpu_authority() + tracedecay_project::test_support::host_admission::ensure_process_background_cpu_authority() .expect("install the process background CPU authority"); let temporary = TempDir::new().unwrap(); let profile_root = temporary.path().join("profile"); @@ -508,7 +508,7 @@ async fn registered_project_runtime_is_exact_and_revocation_never_falls_back() { // CPU authority that plan installs and refuses with // `Unavailable/background_cpu_unavailable` when none is injected. let background_cpu = - crate::test_support::host_admission::ensure_process_background_cpu_authority() + tracedecay_project::test_support::host_admission::ensure_process_background_cpu_authority() .expect("install the process background CPU authority"); let temporary = TempDir::new().unwrap(); let profile_root = temporary.path().join("profile"); diff --git a/crates/tracedecay/src/lib.rs b/crates/tracedecay/src/lib.rs index a8cdd44416..bbb5ce8636 100644 --- a/crates/tracedecay/src/lib.rs +++ b/crates/tracedecay/src/lib.rs @@ -33,19 +33,11 @@ #![allow(clippy::missing_fields_in_debug)] #![allow(clippy::single_match_else)] -// The project handle, its configuration authority, and the product runtime -// live in `tracedecay-project`, below the MCP and daemon layers; the root -// keeps their historical paths. -pub use tracedecay_project::{config, product_runtime, project, version}; pub mod daemon; pub mod dashboard; pub mod doctor; // Fixture surface for integration tests, assembled by the composition root. // Gated so a default or `production` build carries none of it. -#[cfg(any(test, feature = "test-helpers"))] -#[allow(clippy::too_many_lines)] -pub mod test_support; -pub use tracedecay_application::git_query; mod hooks; #[cfg(test)] mod host_admission_test; @@ -55,11 +47,10 @@ mod host_admission_test; #[path = "../../../tests/support/isolated_profile.rs"] mod isolated_profile; pub mod mcp; -pub use tracedecay_project::product_runtime::{ - ProductRuntimeError, ProductRuntimeProvider, ProductSourceProvenance, product_runtime, - register_product_runtime, -}; mod runtime_ports; +#[cfg(any(test, feature = "test-helpers"))] +#[allow(clippy::too_many_lines)] +pub mod test_support; pub use runtime_ports::{hook_runtime, register_runtime_ports, session_review_port}; mod serve; // Session-temporal harness lives under `benches/`; the lib only paths it in diff --git a/crates/tracedecay/src/mcp/project_route.rs b/crates/tracedecay/src/mcp/project_route.rs index 11bcb2abdc..9c33564f29 100644 --- a/crates/tracedecay/src/mcp/project_route.rs +++ b/crates/tracedecay/src/mcp/project_route.rs @@ -71,7 +71,7 @@ pub(crate) async fn resolve_registered_project_route( let (requested_path, scope) = tracedecay_mcp::scope::resolve_query_scope(&context, requested_path) .map_err(|error| error.into_route_failure().into_error())?; - let request = crate::mcp::server::RetainedProjectGraphRequest::for_registered_project( + let request = tracedecay_dashboard_api::project_graph::RetainedProjectGraphRequest::for_registered_project( context.clone(), requested_path.clone(), ); @@ -467,19 +467,7 @@ fn route_identity_from_arguments(arguments: &Value, keys: &[&str]) -> Option &'static [&'static str] { - match tool_name { - "tracedecay_message_search" => &["project_path"], - _ => &[], - } -} - -pub(crate) fn arguments_have_project_selector(tool_name: &str, arguments: &Value) -> bool { - let semantic = semantic_route_argument_fields(tool_name); +pub(crate) fn arguments_have_project_selector(arguments: &Value) -> bool { [ "project_selector", "project_id", @@ -488,7 +476,7 @@ pub(crate) fn arguments_have_project_selector(tool_name: &str, arguments: &Value "root", ] .into_iter() - .any(|key| !semantic.contains(&key) && arguments.get(key).is_some()) + .any(|key| arguments.get(key).is_some()) } fn project_route_identity_matches( @@ -516,7 +504,8 @@ mod tests { }; use crate::daemon::ProductionProjectCompositionHarnessV1; use crate::mcp::server::McpServer; - use tracedecay_hooks::core_events::{HookAgent, HookRouteMetadata}; + use tracedecay_domain::HostIntegrationIdV1; + use tracedecay_hooks::core_events::HookRouteMetadata; use tracedecay_mcp::hook_events::{HookEvent, HookEventKind}; struct ResolvedRouteFixture { @@ -886,7 +875,7 @@ mod tests { fn hook_event(session_id: &str, thread_id: &str, cwd: &str) -> HookEvent { HookEvent { - agent: HookAgent::Claude, + agent: HostIntegrationIdV1::Claude, kind: HookEventKind::FileEdit, rel_paths: Vec::new(), had_command: false, diff --git a/crates/tracedecay/src/mcp/server.rs b/crates/tracedecay/src/mcp/server.rs index 463bf423d6..867b7ea0b4 100644 --- a/crates/tracedecay/src/mcp/server.rs +++ b/crates/tracedecay/src/mcp/server.rs @@ -12,8 +12,7 @@ use serde_json::{Value, json}; use crate::mcp::project_route::{ HookProjectRouteCache, SharedHookProjectRouteCache, mcp_analytics_session_id, }; -use crate::project::TraceDecay; -pub(crate) use tracedecay_code_index_runtime::code_index_scheduler::{ +use tracedecay_code_index_runtime::code_index_scheduler::{ CodeIndexDemandAdmissionV1, CodeIndexDemandUnavailableV1, CodeIndexDemandV1, }; use tracedecay_contracts::code_index_freshness::{ @@ -31,6 +30,11 @@ use tracedecay_mcp::response_handles::{ use tracedecay_mcp::tool_analytics::{ McpToolAnalyticsEvent, hook_route_analytics_event, mcp_tool_analytics_event, }; +use tracedecay_project::project::TraceDecay; +use tracedecay_query::code_search::{ + CodeIndexBranchDiffExecutor, CodeIndexRedundancyExecutor, CodeIndexSearchAuthorityV1, + CodeIndexSearchExecutor, CodeIndexSimilarExecutor, +}; use tracedecay_session_runtime::lcm_authority::{ MountedLcmAuthorityPort, mount_registered_lcm_authority, }; @@ -45,7 +49,8 @@ use tracedecay_sessions::runtime::git_correlation::{ }; use tracedecay_contracts::ProjectRegistryReadPort; -use tracedecay_mcp::hook_events::{self, HookAgent, HookEventPlan}; +use tracedecay_domain::HostIntegrationIdV1; +use tracedecay_mcp::hook_events::{self, HookEventPlan}; use tracedecay_mcp::tools::catalog_discovery::default_catalog_discovery_authority; use tracedecay_mcp::{ ErrorCode, JsonRpcRequest, JsonRpcResponse, ToolRegistryMode, explore_call_budget, @@ -55,13 +60,13 @@ use tracedecay_session_memory::session::SessionRefreshServicePort; mod connection; mod construction; +mod graph_tool_owner; mod hook_dispatch; mod hook_writes; mod ledger; mod lifecycle; mod project_open_access; mod requests; -pub use requests::TOKEN_ACCOUNTING_FOOTER_PREFIX; mod rmcp; mod routing; mod status_resource; @@ -100,9 +105,9 @@ pub(crate) const SERVER_INSTRUCTIONS: &str = concat!( pub(crate) fn initialize_result( instructions: &str, -) -> std::result::Result { +) -> std::result::Result { Ok(tracedecay_mcp::server::initialize_result( - crate::version::build_version()?, + tracedecay_project::version::build_version()?, instructions, )) } @@ -275,15 +280,6 @@ pub(crate) type CodeIndexPublicationIdentityResolver = Arc< + 'static, >; -/// Code-index search boundary contracts, owned by the query kernel. -/// -/// The whole `CodeIndexSearch*V1` family is pure request/outcome data with no -/// MCP coupling, so it lives in `tracedecay_query::code_search`. Re-exporting -/// it here keeps the historical `crate::mcp::server::CodeIndexSearch*` paths -/// resolving while the daemon depends on the query kernel instead of on -/// `crate::mcp`. -pub(crate) use tracedecay_query::code_search::*; - // Lock ordering: file_token_map -> method/resource/tool call counts (never nested) pub struct McpServer { /// The served code graph. Guarded so a mid-session `git checkout` can @@ -424,10 +420,9 @@ pub struct McpServer { retained_project_server_resolver: Option, #[cfg(any(test, feature = "test-transport"))] _host_admission_test_runtime: - Option>, + Option>, hook_project_routes: SharedHookProjectRouteCache, version_cache: std::sync::Mutex, - pending_notifications: std::sync::Mutex>, /// When the MCP server was started from a subdirectory of the project root, /// this holds the relative path prefix (e.g. `"src/mcp"`). Listing tools /// use it as the default path filter. `None` when cwd == project root. @@ -473,9 +468,9 @@ pub struct McpServer { /// entirely (the index is as fresh as auto-sync can make it). `Arc` so /// the retained refresh task can stamp it on completion. last_background_refresh_done_at: Arc, - /// The `[sync]` config resolved once at construction from the project - /// root (plus `TRACEDECAY_SYNC_*` env overrides). Cached so the read - /// hot path never re-reads the config file per `tools/call`. + /// The `[sync]` config resolved once at construction from the project's + /// pinned runtime configuration. Cached so the read hot path never + /// re-resolves configuration per `tools/call`. sync_config: tracedecay_configuration::SyncConfig, /// Savings-ledger recorder tasks spawned so far / finished so far, plus /// a notifier pinged on every completion. Production never awaits these @@ -636,7 +631,7 @@ impl McpServer { #[doc(hidden)] pub fn host_admission_test_runtime_for_test( &self, - ) -> Option<&crate::test_support::host_admission::HostAdmissionTestRuntimeV1> { + ) -> Option<&tracedecay_project::test_support::host_admission::HostAdmissionTestRuntimeV1> { self._host_admission_test_runtime.as_deref() } @@ -646,7 +641,7 @@ impl McpServer { pub async fn new_with_host_admission_test_runtime_for_test( cg: TraceDecay, scope_prefix: Option, - runtime: crate::test_support::host_admission::ProjectScopedTestRuntimeV1, + runtime: tracedecay_project::test_support::host_admission::ProjectScopedTestRuntimeV1, ) -> tracedecay_domain::errors::Result> { Self::new_with_retained_test_servers_for_test(cg, scope_prefix, runtime, Vec::new()).await } @@ -664,7 +659,7 @@ impl McpServer { pub async fn new_with_retained_test_servers_for_test( cg: TraceDecay, scope_prefix: Option, - runtime: crate::test_support::host_admission::ProjectScopedTestRuntimeV1, + runtime: tracedecay_project::test_support::host_admission::ProjectScopedTestRuntimeV1, retained_servers: Vec>, ) -> tracedecay_domain::errors::Result> { let runtime = runtime.into_runtime(); @@ -1141,7 +1136,6 @@ impl McpServer { checked_at: None, refreshing: false, }), - pending_notifications: std::sync::Mutex::new(Vec::new()), scope_prefix, shutdown: tracedecay_daemon_service::ShutdownCoordinatorV1::default(), timings_enabled: AtomicBool::new(telemetry_config.timings), @@ -1173,7 +1167,7 @@ impl McpServer { tokio::task::spawn_blocking(move || { let _ = cleanup_expired_response_handles( &response_handle_project_root, - crate::project::current_timestamp(), + tracedecay_runtime_core::tracedecay::current_timestamp(), ); }); if own_project_host_admission_replay diff --git a/crates/tracedecay/src/mcp/server/cancel_candidate_journey.rs b/crates/tracedecay/src/mcp/server/cancel_candidate_journey.rs index b667e4c556..814c99d3e1 100644 --- a/crates/tracedecay/src/mcp/server/cancel_candidate_journey.rs +++ b/crates/tracedecay/src/mcp/server/cancel_candidate_journey.rs @@ -1,17 +1,11 @@ -//! Large-candidate cancel journey on both MCP transports. +//! Large-candidate cancel journey on the production `rmcp` adapter. //! -//! Premise the earlier family-checkpoint fixes shared: a unit checkpoint inside -//! one scan finishes cancellation. Those checkpoints already stop lexical, -//! exact, redundancy, and similar work, and the search permit is released when -//! the scan observes the signal. What stayed open is the journey that starts -//! at `notifications/cancelled` on each transport and ends at that same -//! checkpoint: the request stops before the next candidate batch and the -//! single search permit is free for the next call. -//! -//! Both transports already register one [`tracedecay_mcp::server::RetainedDispatchAuthority`] -//! signal. This journey runs the production connection loop and the production -//! `rmcp` adapter against that signal, on a corpus larger than one candidate -//! batch, and checks the result the caller sees. +//! A unit checkpoint inside one scan finishes cancellation, and the search +//! permit is released when the scan observes the signal. This journey starts +//! at `notifications/cancelled` and ends at that same checkpoint: the request +//! stops before the next candidate batch and the single search permit is free +//! for the next call. It runs on a corpus larger than one candidate batch and +//! checks the result the caller sees. use std::collections::BTreeMap; use std::path::Path; @@ -43,8 +37,8 @@ use tracedecay_runtime_core::config::PinnedUserDataDir; use super::McpServer; use super::construction::McpServerConstructionContext; -use crate::project::TraceDecay; -use crate::test_support::host_admission::HostAdmissionTestRuntimeV1; +use tracedecay_project::project::TraceDecay; +use tracedecay_project::test_support::host_admission::HostAdmissionTestRuntimeV1; /// One candidate batch is 128. The fixture is larger so the fourth control /// observation, the same boundary the executor permit test uses, is the next @@ -171,24 +165,10 @@ struct MountedCorpus { _root: tempfile::TempDir, } -enum Transport { - Legacy, - Rmcp, -} - -impl Transport { - const fn name(&self) -> &'static str { - match self { - Self::Legacy => "legacy", - Self::Rmcp => "rmcp", - } - } -} - #[tokio::test(flavor = "current_thread")] -async fn cancelled_large_candidate_search_stops_on_legacy_and_rmcp() { +async fn cancelled_large_candidate_search_stops_before_the_next_batch() { let _profile = PinnedUserDataDir::new(); - crate::product_runtime::register_fixture_product_runtime(); + tracedecay_project::product_runtime::register_fixture_product_runtime(); let corpus = mount_candidate_corpus().await; let authority = CodeIndexSearchAuthorityV1 { principal: PrincipalId::new("principal.cancel-journey.fixture").expect("principal"), @@ -211,12 +191,8 @@ async fn cancelled_large_candidate_search_stops_on_legacy_and_rmcp() { admission .pause_at .store(NEXT_BATCH_CHECKPOINT, Ordering::SeqCst); - drive_cancel_journey(Transport::Legacy, &held.server, &admission, &resume_tx).await; - let seen = admission.scan_checkpoints.load(Ordering::SeqCst); - admission - .pause_at - .store(seen.saturating_add(NEXT_BATCH_CHECKPOINT), Ordering::SeqCst); - drive_cancel_journey(Transport::Rmcp, &held.server, &admission, &resume_tx).await; + let pause_at = admission.pause_at.load(Ordering::SeqCst); + drive_rmcp(&held.server, &admission, &resume_tx, pause_at).await; held.server.shutdown().await; corpus.registry.shutdown().await; @@ -344,56 +320,6 @@ async fn open_search_server( } } -async fn drive_cancel_journey( - transport: Transport, - server: &Arc, - admission: &PausingAdmission, - _resume_tx: &std::sync::mpsc::Sender<()>, -) { - let label = transport.name(); - let pause_at = admission.pause_at.load(Ordering::SeqCst); - match transport { - Transport::Legacy => { - let (mut wire, sender, mut responses) = - tracedecay_mcp::transport::ChannelTransport::new(); - let connection_server = Arc::clone(server); - let serving = tokio::spawn(async move { - connection_server - .run_connection(&mut wire) - .await - .expect("legacy connection"); - }); - sender - .send(search_call(11, 8).to_string()) - .expect("send legacy search"); - tokio::select! { - () = wait_for_batch_pause(admission, label) => {} - response = read_channel_response(&mut responses, label) => { - panic!("{label}: search settled before the batch checkpoint: {response}"); - } - } - sender - .send(cancel_notification(11).to_string()) - .expect("send legacy cancellation"); - let cancelled = read_channel_response(&mut responses, label).await; - assert_cancelled_before_next_batch(&cancelled, admission, pause_at, label); - sender - .send(search_call(12, 1).to_string()) - .expect("send legacy follow-up"); - let admitted = read_channel_response(&mut responses, label).await; - assert_next_search_admitted(&admitted, label); - drop(sender); - tokio::time::timeout(Duration::from_secs(10), serving) - .await - .expect("legacy connection did not close") - .expect("join legacy connection"); - } - Transport::Rmcp => { - drive_rmcp(server, admission, _resume_tx, pause_at).await; - } - } -} - async fn drive_rmcp( server: &Arc, admission: &PausingAdmission, @@ -583,29 +509,6 @@ async fn wait_for_batch_pause(admission: &PausingAdmission, label: &str) { }); } -fn assert_cancelled_before_next_batch( - response: &Value, - admission: &PausingAdmission, - pause_at: usize, - label: &str, -) { - assert!( - response.get("result").is_none(), - "{label}: a cancelled search must not return a result page: {response}" - ); - assert_eq!( - response["error"]["data"]["kind"], - json!("cancelled"), - "{label}: caller-visible cancellation: {response}" - ); - assert_eq!( - response["error"]["data"]["reason_code"], - json!("tool_dispatch_cancelled"), - "{label}: the admitted tool must settle as cancelled, not as a transport mystery: {response}" - ); - assert_scan_stopped(admission, pause_at, label); -} - fn assert_scan_stopped(admission: &PausingAdmission, pause_at: usize, label: &str) { assert_eq!( admission.scan_checkpoints.load(Ordering::SeqCst), @@ -615,10 +518,6 @@ fn assert_scan_stopped(admission: &PausingAdmission, pause_at: usize, label: &st ); } -fn assert_next_search_admitted(response: &Value, label: &str) { - assert_admitted_payload(&tool_payload(response, label), label); -} - fn assert_admitted_payload(payload: &Value, label: &str) { assert_ne!( payload["reason"], @@ -638,18 +537,6 @@ fn assert_admitted_payload(payload: &Value, label: &str) { ); } -fn tool_payload(response: &Value, label: &str) -> Value { - assert!( - response.get("error").is_none(), - "{label}: transport error instead of the tool result: {response}" - ); - let text = response["result"]["content"][0]["text"] - .as_str() - .unwrap_or_else(|| panic!("{label}: tool text missing: {response}")); - serde_json::from_str(text) - .unwrap_or_else(|error| panic!("{label}: tool JSON ({error}): {text}")) -} - fn search_call(id: u64, limit: u64) -> Value { json!({ "jsonrpc": "2.0", @@ -666,25 +553,6 @@ fn search_call(id: u64, limit: u64) -> Value { }) } -fn cancel_notification(id: u64) -> Value { - json!({ - "jsonrpc": "2.0", - "method": "notifications/cancelled", - "params": {"requestId": id, "reason": "stop before the next candidate batch"} - }) -} - -async fn read_channel_response( - responses: &mut tokio::sync::mpsc::UnboundedReceiver, - label: &str, -) -> Value { - let line = tokio::time::timeout(Duration::from_mins(1), responses.recv()) - .await - .unwrap_or_else(|_| panic!("{label}: response timed out")) - .unwrap_or_else(|| panic!("{label}: connection closed")); - serde_json::from_str(line.trim()).unwrap_or_else(|error| panic!("{label}: {error}: {line}")) -} - fn git(root: &Path, args: &[&str]) { let output = Command::new("git") .args(args) diff --git a/crates/tracedecay/src/mcp/server/connection.rs b/crates/tracedecay/src/mcp/server/connection.rs index 0898a6ff69..24c38f1188 100644 --- a/crates/tracedecay/src/mcp/server/connection.rs +++ b/crates/tracedecay/src/mcp/server/connection.rs @@ -1,7 +1,14 @@ -//! Connection lifecycle: the JSON-RPC read/write loop, shutdown -//! policy, and daemon-owned host-admission replay driving. +//! Connection context for the `rmcp` adapter, server shutdown, and +//! daemon-owned host-admission replay driving. use super::*; +#[cfg(any(test, feature = "test-transport"))] +use tokio::io::{AsyncBufReadExt, AsyncWriteExt}; +#[cfg(any(test, feature = "test-transport"))] +use tracedecay_daemon_protocol::BrokerStream; +#[cfg(any(test, feature = "test-transport"))] +use tracedecay_daemon_service::shutdown::DaemonLifecycle; +use tracedecay_daemon_service::shutdown::ShutdownStatus; pub(super) const MAX_CONCURRENT_CONNECTION_READS: usize = crate::daemon::MAX_CONCURRENT_REQUESTS_PER_DAEMON_CLIENT; @@ -36,7 +43,7 @@ impl tracedecay_mcp::server::McpConnectionContext for ProductionMcpConnectionCon } fn build_version(&self) -> Result<&'static str> { - crate::version::build_version().map_err(|error| TraceDecayError::Config { + tracedecay_project::version::build_version().map_err(|error| TraceDecayError::Config { message: error.to_string(), }) } @@ -50,16 +57,12 @@ impl tracedecay_mcp::server::McpConnectionContext for ProductionMcpConnectionCon .is_ok_and(tracedecay_tool_catalog::McpDispatchContractV1::read_only) } - fn tool_supports_live_cancellation(&self, tool_name: &str) -> bool { - super::requests::tool_supports_live_cancellation(tool_name) - } - fn dispatch<'a>( &'a self, request: tracedecay_mcp::server::McpDispatchRequest<'a>, timings_enabled: bool, connection: &'a mut Self::Connection, - cancellation: tracedecay_session_memory::context::CancellationToken, + cancellation: tracedecay_runtime_core::cancellation::CancellationToken, ) -> std::pin::Pin> + Send + 'a>> { Box::pin( @@ -77,12 +80,6 @@ impl tracedecay_mcp::server::McpConnectionContext for ProductionMcpConnectionCon self.server.dispatch_authority.cancellation_registered() } - fn take_pending_notifications(&self) -> Vec { - super::requests::recover_lock(&self.server.pending_notifications) - .drain(..) - .collect() - } - fn run_in_connection_admission<'a, T, F>( &'a self, future: F, @@ -96,36 +93,70 @@ impl tracedecay_mcp::server::McpConnectionContext for ProductionMcpConnectionCon future, )) } +} - fn shutdown( - self: Arc, - ) -> std::pin::Pin + Send>> { - Box::pin(async move { McpServer::shutdown(&self.server).await }) +/// A request line carrying the SEP-2575 per-request context the stdio proxy +/// stamps; `initialize`, notifications, and undecodable lines pass unchanged. +#[cfg(any(test, feature = "test-transport"))] +fn with_stateless_request_context(line: String) -> String { + let Ok(mut request) = tracedecay_mcp::JsonRpcRequest::decode(line.trim()) else { + return line; + }; + if !tracedecay_mcp::server::attach_stateless_request_context(&mut request) { + return line; } + match serde_json::to_string(&request) { + Ok(stamped) => stamped, + Err(_) => line, + } +} + +#[cfg(all(unix, any(test, feature = "test-transport")))] +fn connected_broker_pair() -> Result<(BrokerStream, tokio::net::UnixStream)> { + let (daemon, client) = tokio::net::UnixStream::pair()?; + Ok((BrokerStream::Unix(daemon), client)) +} + +#[cfg(all(not(unix), any(test, feature = "test-transport")))] +async fn connected_broker_pair() -> Result<(BrokerStream, tokio::net::TcpStream)> { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let client = tokio::net::TcpStream::connect(listener.local_addr()?).await?; + let (daemon, _) = listener.accept().await?; + Ok((BrokerStream::Tcp(daemon), client)) } +#[cfg(any(test, feature = "test-transport"))] impl McpServer { - fn connection_server( - self: &Arc, - ) -> Arc> { - tracedecay_mcp::server::McpConnectionServer::new(ProductionMcpConnectionContext::new( - Arc::clone(self), - )) + /// Serves one line-framed client through the daemon's production `rmcp` + /// route over a real socket. Requests carry the per-request context the + /// stdio proxy stamps, so a client that skips `initialize` is served the + /// way a proxied request is. + #[hotpath::skip] + pub async fn run_connection( + &self, + transport: &mut impl tracedecay_mcp::transport::McpTransport, + ) -> Result<()> { + self.run_connection_until(transport, &DaemonLifecycle::default()) + .await } + /// As [`Self::run_connection`], then shuts the server down. #[hotpath::skip] pub async fn run( self: &Arc, transport: &mut impl tracedecay_mcp::transport::McpTransport, ) -> Result<()> { - self.connection_server().run(transport).await + let served = self.run_connection(transport).await; + self.shutdown().await; + served } - #[cfg(any(test, feature = "test-transport"))] + /// As [`Self::run_connection`], draining when `lifecycle` drains. #[hotpath::skip] - pub async fn run_connection( + pub(crate) async fn run_connection_until( &self, transport: &mut impl tracedecay_mcp::transport::McpTransport, + lifecycle: &DaemonLifecycle, ) -> Result<()> { let server = self.dispatch_authority.server().upgrade().ok_or_else(|| { TraceDecayError::project_route( @@ -134,42 +165,70 @@ impl McpServer { "MCP server was released before connection dispatch", ) })?; - server.connection_server().run_connection(transport).await - } - - #[hotpath::skip] - pub(crate) async fn run_daemon_connection_with_timings( - self: &Arc, - transport: &mut impl tracedecay_mcp::transport::McpTransport, - timings_enabled: bool, - lifecycle: &dyn tracedecay_mcp::McpConnectionLifecyclePort, - ) -> Result<()> { - self.connection_server() - .run_daemon_connection_with_timings(transport, timings_enabled, lifecycle) - .await - } - - #[cfg(test)] - #[hotpath::skip] - pub(crate) async fn run_with_shutdown_policy( - self: &Arc, - transport: &mut impl tracedecay_mcp::transport::McpTransport, - shutdown_on_exit: bool, - listen_for_process_signals: bool, - timings_override: Option, - request_lifecycle: Option<&dyn tracedecay_mcp::McpConnectionLifecyclePort>, - ) -> Result<()> { - self.connection_server() - .run_with_shutdown_policy( - transport, - shutdown_on_exit, - listen_for_process_signals, - timings_override, - request_lifecycle, - ) - .await + let first_request = loop { + match transport.read_line().await? { + Some(line) if line.trim().is_empty() => {} + Some(line) => break with_stateless_request_context(line), + None => return Ok(()), + } + }; + #[cfg(unix)] + let (daemon_side, client_side) = connected_broker_pair()?; + #[cfg(not(unix))] + let (daemon_side, client_side) = connected_broker_pair().await?; + let serving = crate::daemon::serve_routed_rmcp_connection( + server, + tracedecay_mcp::BrokerStreamTransport::new(daemon_side), + first_request, + std::collections::VecDeque::new(), + None, + self.timings_enabled(), + lifecycle, + ); + let (reader, mut writer) = tokio::io::split(client_side); + let pump = async move { + let mut responses = tokio::io::BufReader::new(reader).lines(); + let mut input_open = true; + // Until the client's EOF this never resolves; after it, the + // client's full close drops both socket halves so the daemon side + // observes the same full close. + let mut peer_full_close: std::pin::Pin< + Box + Send>, + > = Box::pin(std::future::pending()); + loop { + tokio::select! { + biased; + incoming = transport.read_line(), if input_open => match incoming? { + Some(line) => { + writer + .write_all(with_stateless_request_context(line).as_bytes()) + .await?; + writer.write_all(b"\n").await?; + writer.flush().await?; + } + None => { + writer.shutdown().await?; + input_open = false; + peer_full_close = Box::pin(transport.peer_fully_closed_after_eof()); + } + }, + () = &mut peer_full_close => return Ok::<(), TraceDecayError>(()), + response = responses.next_line() => match response? { + Some(line) => { + transport.write_line(&format!("{line}\n")).await?; + transport.flush().await?; + } + None => return Ok::<(), TraceDecayError>(()), + }, + } + } + }; + let (served, pumped) = tokio::join!(serving, pump); + served.and(pumped) } +} +impl McpServer { /// Persists the tokens-saved counter, flushes pending tokens to the /// worldwide counter, checkpoints the WAL, and logs a session summary. /// @@ -190,17 +249,14 @@ impl McpServer { pub(crate) async fn shutdown_until( self: &Arc, deadline: tokio::time::Instant, - ) -> crate::daemon::ShutdownStatus { + ) -> ShutdownStatus { self.shutdown .coordinate_until(deadline, Arc::clone(self).run_shutdown(deadline)) .await } #[hotpath::skip] - async fn run_shutdown( - self: Arc, - deadline: tokio::time::Instant, - ) -> crate::daemon::ShutdownStatus { + async fn run_shutdown(self: Arc, deadline: tokio::time::Instant) -> ShutdownStatus { let mut failures = self.shutdown_background_tasks_until(deadline).await; let uptime = self.stats.started_at.elapsed(); @@ -248,7 +304,7 @@ impl McpServer { ) { config.pending_upload = 0; - let now = crate::project::current_timestamp(); + let now = tracedecay_runtime_core::tracedecay::current_timestamp(); config.last_upload_at = now; } if let Err(err) = config.save() { @@ -276,9 +332,9 @@ impl McpServer { uptime_secs = uptime.as_secs(), "MCP server shutdown complete" ); - crate::daemon::ShutdownStatus::Clean + ShutdownStatus::Clean } else { - crate::daemon::ShutdownStatus::Failed(failures.join("; ")) + ShutdownStatus::Failed(failures.join("; ")) } } @@ -524,7 +580,7 @@ mod cancellable_queue_tests { impl DelayedRouteFixture { async fn new() -> Self { let fixture_guard = DELAYED_ROUTE_FIXTURE_LOCK.lock().await; - crate::product_runtime::register_fixture_product_runtime(); + tracedecay_project::product_runtime::register_fixture_product_runtime(); let isolation = tempfile::TempDir::new().expect("route concurrency isolation"); let active_root = isolation.path().join("active"); let target_root = isolation.path().join("target"); @@ -620,63 +676,6 @@ mod cancellable_queue_tests { reads: Arc, } - #[derive(Clone, Default)] - struct TestConnectionLifecycle { - accepting: Arc, - active: Arc, - draining: Arc, - } - - struct TestRequestActivity(Arc); - - impl Drop for TestRequestActivity { - fn drop(&mut self) { - self.0.fetch_sub(1, Ordering::AcqRel); - } - } - - impl TestConnectionLifecycle { - fn accepting() -> Self { - Self { - accepting: Arc::new(AtomicBool::new(true)), - ..Self::default() - } - } - - fn begin_draining(&self) { - self.accepting.store(false, Ordering::Release); - self.draining.notify_waiters(); - } - } - - impl tracedecay_mcp::McpConnectionLifecyclePort for TestConnectionLifecycle { - fn accepting(&self) -> bool { - self.accepting.load(Ordering::Acquire) - } - - fn try_enter(&self) -> Option { - if !self.accepting() { - return None; - } - self.active.fetch_add(1, Ordering::AcqRel); - if self.accepting() { - return Some(tracedecay_mcp::McpRequestActivity::retain( - TestRequestActivity(Arc::clone(&self.active)), - )); - } - self.active.fetch_sub(1, Ordering::AcqRel); - None - } - - fn wait_for_draining(&self) -> tracedecay_mcp::McpLifecycleDrainFuture<'_> { - Box::pin(async move { - while self.accepting() { - self.draining.notified().await; - } - }) - } - } - impl tracedecay_mcp::transport::McpTransport for ObservedTransport { async fn read_line(&mut self) -> std::io::Result> { let line = self.inner.read_line().await?; @@ -799,11 +798,10 @@ mod cancellable_queue_tests { } #[tokio::test] - async fn ordinary_connection_read_has_one_connection_task_owner() { + async fn ordinary_connection_read_has_one_retained_task_owner() { let fixture = DelayedRouteFixture::new().await; let registry = fixture.caller.dispatch_authority.registry(); let retained_before = registry.retained_spawn_count_for_test(); - let connection_owned_before = registry.connection_owned_count_for_test(); let (mut transport, sender, mut responses) = tracedecay_mcp::transport::ChannelTransport::new(); let serving = tokio::spawn({ @@ -829,13 +827,8 @@ mod cancellable_queue_tests { assert_eq!( registry.retained_spawn_count_for_test(), - retained_before, - "the connection's active-read task must be the sole task owner" - ); - assert_eq!( - registry.connection_owned_count_for_test(), - connection_owned_before + 1, - "one inline registry lease must cover the ordinary read" + retained_before + 1, + "the retained dispatch registry must own exactly the one ordinary read" ); drop(sender); @@ -1023,84 +1016,10 @@ mod cancellable_queue_tests { fixture.harness.shutdown().await; } - #[tokio::test] - async fn notification_is_an_ordering_barrier_for_later_reads() { - let fixture = DelayedRouteFixture::new().await; - let (mut transport, sender, mut responses) = - tracedecay_mcp::transport::ChannelTransport::new(); - let serving = tokio::spawn({ - let caller = Arc::clone(&fixture.caller); - async move { caller.run_connection(&mut transport).await } - }); - - sender - .send( - serde_json::json!({ - "jsonrpc": "2.0", - "id": 20, - "method": "tools/call", - "params": { - "name": "tracedecay_grep", - "arguments": { - "pattern": "route_fixture", - "fixed_strings": true, - "project_selector": { - "project_id": fixture.target_project_id.clone() - }, - "format": "json" - } - } - }) - .to_string(), - ) - .expect("send read before notification"); - fixture.wait_for_routes(1).await; - sender - .send( - serde_json::json!({ - "jsonrpc": "2.0", - "method": "notifications/initialized" - }) - .to_string(), - ) - .expect("send ordering notification"); - sender - .send( - serde_json::json!({ - "jsonrpc": "2.0", - "id": 21, - "method": "tools/call", - "params": { - "name": "tracedecay_status", - "arguments": {"admission_only": true} - } - }) - .to_string(), - ) - .expect("send read after notification"); - - assert!( - tokio::time::timeout(Duration::from_millis(50), responses.recv()) - .await - .is_err(), - "the later read must not overtake an ordered notification" - ); - fixture.route_release.add_permits(1); - assert_eq!(receive_response(&mut responses).await["id"], json!(20)); - assert_eq!(receive_response(&mut responses).await["id"], json!(21)); - - drop(sender); - serving - .await - .expect("join notification barrier connection") - .expect("serve notification barrier connection"); - fixture.harness.shutdown().await; - } - #[tokio::test] async fn daemon_drain_cancels_and_joins_concurrent_reads() { let fixture = DelayedRouteFixture::new().await; - let lifecycle = TestConnectionLifecycle::accepting(); + let lifecycle = DaemonLifecycle::default(); let (mut transport, sender, _responses) = tracedecay_mcp::transport::ChannelTransport::new(); let serving = tokio::spawn({ @@ -1108,7 +1027,7 @@ mod cancellable_queue_tests { let lifecycle = lifecycle.clone(); async move { caller - .run_with_shutdown_policy(&mut transport, false, false, None, Some(&lifecycle)) + .run_connection_until(&mut transport, &lifecycle) .await } }); @@ -1135,21 +1054,16 @@ mod cancellable_queue_tests { ) .expect("send read held across drain"); fixture.wait_for_routes(1).await; - assert_eq!(lifecycle.active.load(Ordering::Acquire), 1); lifecycle.begin_draining(); - tokio::time::timeout(Duration::from_secs(5), serving) + tokio::time::timeout(Duration::from_secs(10), serving) .await - .expect("draining connection did not join active reads") + .expect("draining connection did not join its in-flight read") .expect("join draining connection") .expect("serve draining connection"); - assert_eq!( - lifecycle.active.load(Ordering::Acquire), - 0, - "shutdown drain must release every admitted request activity" - ); drop(sender); + fixture.route_release.add_permits(1); fixture.harness.shutdown().await; } diff --git a/crates/tracedecay/src/mcp/server/construction.rs b/crates/tracedecay/src/mcp/server/construction.rs index a81315fe67..1cf0e9e674 100644 --- a/crates/tracedecay/src/mcp/server/construction.rs +++ b/crates/tracedecay/src/mcp/server/construction.rs @@ -8,12 +8,13 @@ use std::pin::Pin; use std::sync::Arc; use std::sync::atomic::AtomicBool; -use crate::project::TraceDecay; use tracedecay_contracts::{ ProfileIdentityReadPort, remote::status::RemoteOperationalStatusReaderV1, }; use tracedecay_daemon_identity::profile_identity::LocalProfileIdentityAuthorityV1; +use tracedecay_dashboard_api::project_graph::RetainedProjectGraphRequest; use tracedecay_global_db::RegisteredGlobalDbLeaseV1; +use tracedecay_project::project::TraceDecay; use tracedecay_runtime_core::background_cpu::ProcessBackgroundCpuV1; use tracedecay_session_memory::session::SessionRefreshServicePort; use tracedecay_sessions::serving::{SessionProjectionServingStatusPort, SessionRefreshWorkerPort}; @@ -57,7 +58,6 @@ pub(crate) type CodeIndexIgnoredDependencyAdmissionPort = Arc< /// Concrete route bridge to a project server already mounted by the daemon. /// Routed handlers retain the whole server so its graph, query ports, session /// stores, application executor, and lifecycle remain one authority. -pub(crate) use tracedecay_dashboard_api::project_graph::RetainedProjectGraphRequest; pub(crate) type RetainedProjectServerFuture = Pin< Box< dyn Future>>> @@ -160,17 +160,22 @@ pub(crate) struct McpServerConstructionContext { pub(crate) code_index_reconcile_sink: Option, pub(crate) code_index_freshness_probe_sink: Option, pub(crate) code_index_publication_identity: Option, - pub(crate) code_index_search_executor: Option, - pub(crate) code_index_similar_executor: Option, - pub(crate) code_index_redundancy_executor: Option, - pub(crate) code_index_branch_diff_executor: Option, + pub(crate) code_index_search_executor: + Option, + pub(crate) code_index_similar_executor: + Option, + pub(crate) code_index_redundancy_executor: + Option, + pub(crate) code_index_branch_diff_executor: + Option, pub(crate) code_graph_projection_read_port: Option, pub(crate) code_graph_read_admission_port: Option, pub(crate) verified_graph_query_port: Option>, pub(crate) code_index_ignored_dependency_admission: Option, - pub(crate) code_index_search_authority: Option, + pub(crate) code_index_search_authority: + Option, /// The one checkout this server answers for, resolved once by project open /// through the daemon code-index authority. `None` on a direct server and /// on the core server that answers before project-open publication. @@ -188,7 +193,7 @@ pub(crate) struct McpServerConstructionContext { pub(crate) project_server_live: Option>, #[cfg(any(test, feature = "test-transport"))] pub(crate) host_admission_test_runtime: - Option>, + Option>, } pub(crate) struct McpServerWriters { @@ -515,7 +520,7 @@ impl McpServerConstructionContext { pub(crate) fn with_code_index_search_executor( mut self, - executor: super::CodeIndexSearchExecutor, + executor: tracedecay_query::code_search::CodeIndexSearchExecutor, ) -> Self { self.code_index_search_executor = Some(executor); self @@ -523,7 +528,7 @@ impl McpServerConstructionContext { pub(crate) fn with_code_index_similar_executor( mut self, - executor: super::CodeIndexSimilarExecutor, + executor: tracedecay_query::code_search::CodeIndexSimilarExecutor, ) -> Self { self.code_index_similar_executor = Some(executor); self @@ -531,7 +536,7 @@ impl McpServerConstructionContext { pub(crate) fn with_code_index_redundancy_executor( mut self, - executor: super::CodeIndexRedundancyExecutor, + executor: tracedecay_query::code_search::CodeIndexRedundancyExecutor, ) -> Self { self.code_index_redundancy_executor = Some(executor); self @@ -539,7 +544,7 @@ impl McpServerConstructionContext { pub(crate) fn with_code_index_branch_diff_executor( mut self, - executor: super::CodeIndexBranchDiffExecutor, + executor: tracedecay_query::code_search::CodeIndexBranchDiffExecutor, ) -> Self { self.code_index_branch_diff_executor = Some(executor); self @@ -579,7 +584,7 @@ impl McpServerConstructionContext { pub(crate) fn with_code_index_search_authority( mut self, - authority: super::CodeIndexSearchAuthorityV1, + authority: tracedecay_query::code_search::CodeIndexSearchAuthorityV1, ) -> Self { self.code_index_search_authority = Some(authority); self @@ -721,7 +726,7 @@ mod tests { #[tokio::test] async fn direct_context_installs_only_explicit_code_index_executors() { - let _pin = crate::config::PinnedUserDataDir::new(); + let _pin = tracedecay_project::config::PinnedUserDataDir::new(); let project = tempfile::tempdir().expect("project"); let git_init = Command::new("git") .args(["init", "--quiet"]) @@ -739,32 +744,31 @@ mod tests { ) .await .expect("registered graph"); - let executor: crate::mcp::server::CodeIndexSearchExecutor = Arc::new(|_| { + let executor: tracedecay_query::code_search::CodeIndexSearchExecutor = Arc::new(|_| { Box::pin(async { - crate::mcp::server::CodeIndexSearchOutcomeV1::Unavailable( - crate::mcp::server::CodeIndexSearchUnavailableV1 { + tracedecay_query::code_search::CodeIndexSearchOutcomeV1::Unavailable( + tracedecay_query::code_search::CodeIndexSearchUnavailableV1 { code_generation: None, - reason: crate::mcp::server::CodeIndexSearchUnavailableReasonV1::AuthorityUnavailable, - coverage: crate::mcp::server::CodeIndexSearchCoverageV1::unavailable( + reason: tracedecay_query::code_search::CodeIndexSearchUnavailableReasonV1::AuthorityUnavailable, + coverage: tracedecay_query::code_search::CodeIndexSearchCoverageV1::unavailable( "authority_unavailable", ), }, ) }) }); - let branch_diff_executor: crate::mcp::server::CodeIndexBranchDiffExecutor = Arc::new( - |_| { + let branch_diff_executor: tracedecay_query::code_search::CodeIndexBranchDiffExecutor = + Arc::new(|_| { Box::pin(async { - crate::mcp::server::CodeIndexBranchDiffOutcomeV1::Unavailable( - crate::mcp::server::CodeIndexBranchDiffUnavailableV1 { + tracedecay_query::code_search::CodeIndexBranchDiffOutcomeV1::Unavailable( + tracedecay_query::code_search::CodeIndexBranchDiffUnavailableV1 { base_generation: None, head_generation: None, - reason: crate::mcp::server::CodeIndexSearchUnavailableReasonV1::AuthorityUnavailable, + reason: tracedecay_query::code_search::CodeIndexSearchUnavailableReasonV1::AuthorityUnavailable, }, ) }) - }, - ); + }); let context = McpServerConstructionContext::direct(cg, None) .with_code_index_search_executor(executor) .with_code_index_branch_diff_executor(branch_diff_executor); diff --git a/crates/tracedecay/src/mcp/server/freshness_tests.rs b/crates/tracedecay/src/mcp/server/freshness_tests.rs index d02f7948f0..3fd9ffebbe 100644 --- a/crates/tracedecay/src/mcp/server/freshness_tests.rs +++ b/crates/tracedecay/src/mcp/server/freshness_tests.rs @@ -1,12 +1,12 @@ use super::{DatabaseOwnerReconciler, McpServer, McpServerConstructionContext}; -use crate::config::PinnedUserDataDir; -use crate::project::TraceDecay; use std::sync::atomic::Ordering; use std::sync::{Arc, Mutex}; use std::time::Duration; use tempfile::TempDir; use tracedecay_global_db::RegisteredGlobalDbLeaseV1; use tracedecay_mcp::tool_error_response; +use tracedecay_project::config::PinnedUserDataDir; +use tracedecay_project::project::TraceDecay; use tracedecay_store_runtime::DaemonSessionRuntimeRegistryV1; struct FreshnessRuntime { @@ -60,7 +60,7 @@ fn git(root: &std::path::Path, args: &[&str]) { struct FreshnessFixtureAuthority { _pin: PinnedUserDataDir, - _runtime: Arc, + _runtime: Arc, } async fn init_indexed_repo() -> (TraceDecay, TempDir, FreshnessFixtureAuthority) { @@ -101,7 +101,7 @@ async fn branch_drift_serves_the_old_snapshot_until_the_swap_lands() { drop(cg); let mut meta = tracedecay_runtime_core::branch_meta::BranchMeta::new("main"); - meta.add_branch("feature", "branches/feature.db", "main"); + meta.add_branch("feature", "main"); tracedecay_runtime_core::branch_meta::save_branch_meta(&layout.data_root, &meta).unwrap(); // `add_branch` only admits the branch; until its exact graph source is // published the branch is still indexing and a reopen legitimately keeps @@ -128,18 +128,15 @@ async fn branch_drift_serves_the_old_snapshot_until_the_swap_lands() { ), "the feature branch must be query-eligible before the drift" ); - std::fs::create_dir_all(layout.data_root.join("branches")).unwrap(); - std::fs::copy( - &layout.graph_db_path, - layout.data_root.join("branches/feature.db"), - ) - .unwrap(); git(root, &["checkout", "-q", "-b", "feature"]); git(root, &["checkout", "-q", "main"]); let main = fixture_authority ._runtime - .open_project_graph_for_test(root, crate::project::TraceDecayOpenOptions::default()) + .open_project_graph_for_test( + root, + tracedecay_project::project::TraceDecayOpenOptions::default(), + ) .await .unwrap(); let observed = Arc::new(Mutex::new(Vec::new())); @@ -256,7 +253,8 @@ async fn a_cancelled_machine_reads_as_settled_and_refuses_further_phases() { #[tokio::test] async fn direct_server_keeps_configured_profile_root_with_overridden_registry_db() { let (cg, dir, _pin) = init_indexed_repo().await; - let profile_root = crate::config::user_data_dir().expect("configured profile root"); + let profile_root = + tracedecay_project::config::user_data_dir().expect("configured profile root"); let override_root = dir.path().join("registry-override"); let runtime = FreshnessRuntime::open(&override_root).await; let registry = runtime.profile_database().await; diff --git a/crates/tracedecay/src/mcp/server/graph_tool_owner.rs b/crates/tracedecay/src/mcp/server/graph_tool_owner.rs new file mode 100644 index 0000000000..422cdb7ff1 --- /dev/null +++ b/crates/tracedecay/src/mcp/server/graph_tool_owner.rs @@ -0,0 +1,97 @@ +//! The project's graph-tool owner: the daemon invocation service computes +//! graph and port reads through the serving MCP server's admitted +//! authorities. + +use std::path::Path; +use std::sync::{Arc, Weak}; + +use tracedecay_contracts::ResolvedScope; +use tracedecay_daemon_service::{ + GraphToolFuture, GraphToolInvocationV1, ProjectGraphToolPortV1, RegisteredGraphToolOwnerV1, +}; +use tracedecay_domain::errors::{Result, TraceDecayError}; + +use super::McpServer; +use crate::mcp::tools::{ + ToolCallRegistryOptions, compute_graph_tool_for_owner, graph_tool_error_problem, +}; + +struct McpGraphToolPort { + server: Weak, +} + +impl ProjectGraphToolPortV1 for McpGraphToolPort { + fn execute(&self, invocation: GraphToolInvocationV1) -> GraphToolFuture<'_> { + Box::pin(async move { + let Some(server) = self.server.upgrade() else { + return Err(graph_tool_error_problem(&TraceDecayError::project_route( + "tool_dispatch_shutdown", + true, + "the MCP server was released before the graph read was admitted", + ))); + }; + server + .compute_graph_tool(invocation) + .await + .map_err(|error| graph_tool_error_problem(&error)) + }) + } +} + +impl McpServer { + async fn compute_graph_tool( + &self, + invocation: GraphToolInvocationV1, + ) -> Result { + let (cg, _live_branch) = self.reopen_if_branch_drifted_memoized().await; + let options = ToolCallRegistryOptions { + registered_project_session_db: self.project_session_db.clone(), + application_request_id: Some(invocation.request_id), + application_deadline: Some(invocation.deadline), + application_cancellation: Some(invocation.cancellation), + code_index_search_executor: self.code_index_search_executor.clone(), + code_index_similar_executor: self.code_index_similar_executor.clone(), + code_index_redundancy_executor: self.code_index_redundancy_executor.clone(), + code_index_branch_diff_executor: self.code_index_branch_diff_executor.clone(), + code_index_search_authority: self.code_index_search_authority.clone(), + admitted_project_scope: self.admitted_project_scope.clone(), + verified_graph_query_port: self.verified_graph_query_port.clone(), + code_index_freshness_reader: self.dashboard_code_index_freshness_reader.clone(), + ..ToolCallRegistryOptions::default() + }; + compute_graph_tool_for_owner( + cg.as_ref(), + invocation.operation, + serde_json::Value::Object(invocation.arguments), + self.scope_prefix(), + options, + ) + .await + } + + /// Registers this server as its project's graph-tool owner, replacing an + /// earlier server for the same authorized scope. + pub(crate) async fn register_graph_tool_owner( + &self, + project_root: &Path, + scope: ResolvedScope, + ) -> Result<()> { + let Some(service) = self.daemon_invocation_service() else { + return Err(TraceDecayError::Config { + message: "the graph-tool owner requires the daemon invocation service".to_owned(), + }); + }; + let port: Arc = Arc::new(McpGraphToolPort { + server: self.dispatch_authority.server(), + }); + service + .register_graph_tool_owner( + project_root.to_path_buf(), + RegisteredGraphToolOwnerV1::new(scope, port), + ) + .await + .map_err(|error| TraceDecayError::Config { + message: format!("the graph-tool owner failed to register: {error}"), + }) + } +} diff --git a/crates/tracedecay/src/mcp/server/hook_boundary_failure_matrix_tests.rs b/crates/tracedecay/src/mcp/server/hook_boundary_failure_matrix_tests.rs index 3371fce0bf..4cfc9fc030 100644 --- a/crates/tracedecay/src/mcp/server/hook_boundary_failure_matrix_tests.rs +++ b/crates/tracedecay/src/mcp/server/hook_boundary_failure_matrix_tests.rs @@ -17,18 +17,23 @@ use super::writer_test_support::{ }; use super::{CodeIndexReconcileSink, McpServer}; use crate::mcp::project_route::HookProjectRouteCache; -use tracedecay_hooks::core_events::{DaemonHookEvent, HookAgent}; +use tracedecay_domain::HostIntegrationIdV1; +use tracedecay_hooks::core_events::DaemonHookEvent; use tracedecay_host_admission::{ HostAdmissionBroker, HostAdmissionRuntime, SharedHostAdmissionBroker, SpoolBounds, }; use tracedecay_sessions::admission::HostAdmissionStatus; fn session_start(root: PathBuf) -> Value { - serde_json::to_value(DaemonHookEvent::session_start(HookAgent::Codex, root)).unwrap() + serde_json::to_value(DaemonHookEvent::session_start( + HostIntegrationIdV1::Codex, + root, + )) + .unwrap() } async fn server_with_broker( - cg: crate::project::TraceDecay, + cg: tracedecay_project::project::TraceDecay, authority: &WriterTestFixtureAuthority, broker: SharedHostAdmissionBroker, reconcile_sink: CodeIndexReconcileSink, @@ -42,7 +47,7 @@ async fn server_with_broker( } async fn server_without_broker( - cg: crate::project::TraceDecay, + cg: tracedecay_project::project::TraceDecay, authority: &WriterTestFixtureAuthority, reconcile_sink: CodeIndexReconcileSink, ) -> Arc { @@ -357,7 +362,7 @@ async fn after_edit_hook_delivers_touched_paths_to_code_index_sink() { .expect("registered test server"); let mut routes = HookProjectRouteCache::default(); let event = serde_json::to_value(DaemonHookEvent::post_tool_use_edit( - HookAgent::Codex, + HostIntegrationIdV1::Codex, vec!["src/lib.rs".to_owned()], project.path().to_path_buf(), )) diff --git a/crates/tracedecay/src/mcp/server/hook_dispatch.rs b/crates/tracedecay/src/mcp/server/hook_dispatch.rs index 82cb801d7e..aa4a9a7d29 100644 --- a/crates/tracedecay/src/mcp/server/hook_dispatch.rs +++ b/crates/tracedecay/src/mcp/server/hook_dispatch.rs @@ -181,7 +181,7 @@ impl McpServer { pub(crate) async fn run_hook_incremental_sync( &self, cg: Arc, - agent: HookAgent, + agent: HostIntegrationIdV1, ) -> HostAdmissionOutcome { match self.accept_debounced_code_index_reconcile(&cg, agent).await { Ok(changed) => HostAdmissionOutcome::replay_completed(changed, !changed), @@ -193,10 +193,10 @@ impl McpServer { async fn accept_debounced_code_index_reconcile( &self, cg: &TraceDecay, - agent: HookAgent, + agent: HostIntegrationIdV1, ) -> std::result::Result { let marker = hook_events::sync_marker_path(&cg.store_layout().data_root, agent); - let now = crate::project::current_timestamp(); + let now = tracedecay_runtime_core::tracedecay::current_timestamp(); if !hook_events::should_run_sync(&marker, now, 3) { return Ok(false); } diff --git a/crates/tracedecay/src/mcp/server/hook_writes.rs b/crates/tracedecay/src/mcp/server/hook_writes.rs index 1c706904b8..e4c2df5e1c 100644 --- a/crates/tracedecay/src/mcp/server/hook_writes.rs +++ b/crates/tracedecay/src/mcp/server/hook_writes.rs @@ -7,9 +7,9 @@ use std::path::PathBuf; use std::pin::Pin; use std::sync::Arc; -use crate::project::TraceDecay; use tracedecay_code_index_runtime::code_index_scheduler::CodeIndexDemandAdmissionV1; use tracedecay_domain::errors::{Result, TraceDecayError}; +use tracedecay_project::project::TraceDecay; /// Complete detached reconciliation admission requested by the MCP server. #[derive(Clone, Copy, Debug, Eq, PartialEq)] diff --git a/crates/tracedecay/src/mcp/server/host_admission_tests.rs b/crates/tracedecay/src/mcp/server/host_admission_tests.rs index c1e9e44856..4ccd05fc48 100644 --- a/crates/tracedecay/src/mcp/server/host_admission_tests.rs +++ b/crates/tracedecay/src/mcp/server/host_admission_tests.rs @@ -12,13 +12,12 @@ use super::writer_test_support::{ }; use super::{CodeIndexReconcileSink, McpServer, McpServerConstructionContext}; use crate::mcp::project_route::HookProjectRouteCache; -use crate::test_support::host_admission::HostAdmissionTestRuntimeV1; -use tracedecay_hooks::core_events::{ - DaemonHookEvent, HookAgent, HookRouteMetadata, HookTerminalReceipt, -}; +use tracedecay_domain::HostIntegrationIdV1; +use tracedecay_hooks::core_events::{DaemonHookEvent, HookRouteMetadata, HookTerminalReceipt}; use tracedecay_host_admission::{ HostAdmissionBroker, HostAdmissionRuntime, SharedHostAdmissionBroker, SpoolBounds, }; +use tracedecay_project::test_support::host_admission::HostAdmissionTestRuntimeV1; use tracedecay_sessions::admission::{ HostAdmissionOutcome, HostAdmissionScope, HostAdmissionStatus, }; @@ -28,7 +27,11 @@ use tracedecay_sessions::runtime::git_correlation::{ use tracedecay_sessions::runtime::{SessionMessageRecord, SessionRecord}; fn session_start(root: PathBuf) -> Value { - serde_json::to_value(DaemonHookEvent::session_start(HookAgent::Codex, root)).unwrap() + serde_json::to_value(DaemonHookEvent::session_start( + HostIntegrationIdV1::Codex, + root, + )) + .unwrap() } /// Builds the terminal-receipt wire event the Hermes plugin sends. Production @@ -39,7 +42,7 @@ fn hermes_terminal_receipt_event( receipt: HookTerminalReceipt, ) -> DaemonHookEvent { DaemonHookEvent { - agent: HookAgent::Hermes.as_wire().to_string(), + agent: HostIntegrationIdV1::Hermes.as_wire().to_string(), event: "terminalReceipt".to_string(), rel_paths: Vec::new(), command: None, @@ -71,7 +74,7 @@ fn terminal_receipt(root: PathBuf) -> Value { } async fn server_with_broker( - cg: crate::project::TraceDecay, + cg: tracedecay_project::project::TraceDecay, authority: &WriterTestFixtureAuthority, broker: SharedHostAdmissionBroker, reconcile_sink: CodeIndexReconcileSink, @@ -83,7 +86,7 @@ async fn server_with_broker( } async fn server_with_owned_project_replay_worker( - cg: crate::project::TraceDecay, + cg: tracedecay_project::project::TraceDecay, authority: &WriterTestFixtureAuthority, broker: SharedHostAdmissionBroker, reconcile_sink: CodeIndexReconcileSink, @@ -120,7 +123,7 @@ fn sync_current_branch_payload(branch: &str) -> Vec { tracedecay_mcp::hook_events::encode_durable_hook_event_plan( &tracedecay_mcp::hook_events::HookEventPlan::SyncCurrentBranch { branch: branch.to_string(), - agent: HookAgent::Codex, + agent: HostIntegrationIdV1::Codex, }, ) .expect("sync_current_branch plan should encode") @@ -148,7 +151,7 @@ async fn hook_watch_policy_refusal_is_not_scheduler_unavailable() { tracedecay_mcp::hook_events::HookEventPlan::SyncFiles(vec!["src/a.rs".to_owned()]), tracedecay_mcp::hook_events::HookEventPlan::SyncCurrentBranch { branch: cg.active_branch().unwrap().to_owned(), - agent: HookAgent::Codex, + agent: HostIntegrationIdV1::Codex, }, ] { let outcome = server @@ -610,7 +613,7 @@ fn add_branch_at_payload(root: PathBuf, branch: &str) -> Vec { &tracedecay_mcp::hook_events::HookEventPlan::AddBranchAt { root, branch: branch.to_string(), - agent: HookAgent::Codex, + agent: HostIntegrationIdV1::Codex, }, ) .expect("add_branch_at plan should encode") @@ -1068,7 +1071,7 @@ async fn add_branch_at_restart_replay_rejects_symlink_swap() { fn session_start_with_route(root: PathBuf) -> Value { serde_json::to_value( - DaemonHookEvent::session_start(HookAgent::Codex, root.clone()).with_route(Some( + DaemonHookEvent::session_start(HostIntegrationIdV1::Codex, root.clone()).with_route(Some( HookRouteMetadata { session_id: Some("session-admission-test".to_string()), thread_id: Some("thread-admission-test".to_string()), @@ -1082,7 +1085,7 @@ fn session_start_with_route(root: PathBuf) -> Value { } async fn server_with_broker_and_runtime( - cg: crate::project::TraceDecay, + cg: tracedecay_project::project::TraceDecay, broker: SharedHostAdmissionBroker, reconcile_sink: CodeIndexReconcileSink, runtime: Arc, @@ -1196,15 +1199,14 @@ async fn durable_route_survives_unavailable_effect_for_same_connection_retry() { server_with_broker_and_runtime(cg, Arc::clone(&broker), reconcile_sink, test_runtime).await; let raw_session = ["AKIA", "SYNTHETIC", "CANARY", "3"].concat(); let event = serde_json::to_value( - DaemonHookEvent::session_start(HookAgent::Codex, project.path().to_path_buf()).with_route( - Some(HookRouteMetadata { + DaemonHookEvent::session_start(HostIntegrationIdV1::Codex, project.path().to_path_buf()) + .with_route(Some(HookRouteMetadata { session_id: Some(raw_session.clone()), thread_id: None, cwd: Some(project.path().to_path_buf()), worktree: Some(project.path().to_path_buf()), branch: Some("main".to_string()), - }), - ), + })), ) .unwrap(); let mut routes = HookProjectRouteCache::default(); @@ -1362,7 +1364,7 @@ async fn committed_admissions_emit_post_commit_private_route_analytics() { async fn credential_canary_receipt_analytics_and_git_span_survive_database_reopen() { let (cg, project, authority) = init_indexed_repo().await; let dashboard_root = cg.store_layout().dashboard_root.clone(); - let profile_root = crate::config::user_data_dir().expect("isolated profile root"); + let profile_root = tracedecay_project::config::user_data_dir().expect("isolated profile root"); let project_id = tracedecay_domain::ProjectId::new( cg.store_layout() .identity diff --git a/crates/tracedecay/src/mcp/server/lcm_claude_recall_tests.rs b/crates/tracedecay/src/mcp/server/lcm_claude_recall_tests.rs index e1faf7cf53..68b0023be6 100644 --- a/crates/tracedecay/src/mcp/server/lcm_claude_recall_tests.rs +++ b/crates/tracedecay/src/mcp/server/lcm_claude_recall_tests.rs @@ -15,12 +15,12 @@ use tempfile::TempDir; use tracedecay_domain::{ObservationScopeV1, ProjectId, SessionId}; use super::McpServer; -use crate::project::TraceDecayOpenOptions; -use crate::test_support::host_admission::HostAdmissionTestRuntimeV1; use tracedecay_mcp::transport::JsonRpcRequest; +use tracedecay_project::project::TraceDecayOpenOptions; +use tracedecay_project::test_support::host_admission::HostAdmissionTestRuntimeV1; use tracedecay_sessions::admission::HostAdmissionScope; use tracedecay_sessions::observation::ObservationCancellation; -use tracedecay_sessions::runtime::claude::ClaudeSource; +use tracedecay_sessions::runtime::hosts::claude::ClaudeSource; const PROJECT_ID: &str = "project.claude-recall"; const SESSION: &str = "claude-recall-session"; @@ -41,8 +41,12 @@ fn git(root: &std::path::Path, args: &[&str]) { assert!(status.success(), "git {args:?} failed"); } -async fn server_with_authorities() -> (Arc, TempDir, crate::config::PinnedUserDataDir) { - let pin = crate::config::PinnedUserDataDir::new(); +async fn server_with_authorities() -> ( + Arc, + TempDir, + tracedecay_project::config::PinnedUserDataDir, +) { + let pin = tracedecay_project::config::PinnedUserDataDir::new(); let dir = TempDir::new().expect("temp project"); git(dir.path(), &["init", "-q", "-b", "main"]); git(dir.path(), &["config", "user.email", "test@example.com"]); @@ -57,7 +61,7 @@ async fn server_with_authorities() -> (Arc, TempDir, crate::config::P git(dir.path(), &["add", "."]); git(dir.path(), &["commit", "-q", "-m", "initial"]); let runtime = HostAdmissionTestRuntimeV1::project( - crate::config::user_data_dir().expect("isolated profile root"), + tracedecay_project::config::user_data_dir().expect("isolated profile root"), dir.path(), ProjectId::new(PROJECT_ID).expect("typed project identity"), ) @@ -209,7 +213,7 @@ async fn ingest_and_project( project_id: ProjectId::new(PROJECT_ID).expect("typed project identity"), }; let stats = - tracedecay_sessions::runtime::claude_observation::ingest_source_with_observations_with_admission( + tracedecay_sessions::runtime::hosts::claude_observation::ingest_source_with_observations_with_admission( &source, project, scope, @@ -239,7 +243,7 @@ async fn ingested_server() -> ( Arc, TempDir, TempDir, - crate::config::PinnedUserDataDir, + tracedecay_project::config::PinnedUserDataDir, ) { let (server, dir, pin) = server_with_authorities().await; let home = TempDir::new().expect("temp home"); diff --git a/crates/tracedecay/src/mcp/server/ledger.rs b/crates/tracedecay/src/mcp/server/ledger.rs index 3d753e8e68..4cfaf9ca91 100644 --- a/crates/tracedecay/src/mcp/server/ledger.rs +++ b/crates/tracedecay/src/mcp/server/ledger.rs @@ -313,7 +313,7 @@ impl McpServer { /// never await configuration or cloud I/O and shutdown still drains it. #[hotpath::measure(label = "mcp.ledger.flush_worldwide")] pub(crate) fn maybe_flush_worldwide(self: &Arc) { - let now = crate::project::current_timestamp(); + let now = tracedecay_runtime_core::tracedecay::current_timestamp(); let last = self.last_flush_at.load(Ordering::Relaxed); if now - last < 30 { return; @@ -399,7 +399,7 @@ impl McpServer { response_tokens: 0, net_saved_tokens: 0, duration_us, - timestamp: crate::project::current_timestamp(), + timestamp: tracedecay_runtime_core::tracedecay::current_timestamp(), request_id, arguments, internal_analytics: None, @@ -432,7 +432,7 @@ impl McpServer { project_root, event, current_branch, - crate::project::current_timestamp(), + tracedecay_runtime_core::tracedecay::current_timestamp(), admission_seq, ) else { return; @@ -494,7 +494,7 @@ impl McpServer { }; let thread_id = bounded_span_identifier(route.thread_id.as_deref()) .and_then(|value| tracedecay_privacy::protect_sensitive_structural_id(&value).ok()); - let ts = crate::project::current_timestamp(); + let ts = tracedecay_runtime_core::tracedecay::current_timestamp(); // Session-only pre-debounce: the full key needs branch/worktree, which // cost gix/git discovery. A burst for one session almost always shares // those, so reject here before paying for derivation. Mid-session @@ -590,7 +590,7 @@ fn persist_worldwide_delta(delta: u64, upload_enabled: bool) -> bool { && tracedecay_dashboard_api::cloud::flush_pending(config.pending_upload).is_some() { config.pending_upload = 0; - config.last_upload_at = crate::project::current_timestamp(); + config.last_upload_at = tracedecay_runtime_core::tracedecay::current_timestamp(); } match config.save() { Ok(()) => true, @@ -618,9 +618,9 @@ mod tests { use super::*; fn desired_configuration() -> ConfigurationSnapshotV1 { - let registry = - crate::config::registry::ConfigurationRegistry::core().expect("configuration registry"); - crate::config::resolver::resolve_configuration(®istry, &[]) + let registry = tracedecay_project::config::registry::ConfigurationRegistry::core() + .expect("configuration registry"); + tracedecay_project::config::resolver::resolve_configuration(®istry, &[]) .expect("default desired configuration") .snapshot } @@ -695,7 +695,7 @@ mod tests { #[test] fn disabled_upload_records_each_delta_once_after_durable_save() { - let _profile = crate::config::PinnedUserDataDir::new(); + let _profile = tracedecay_project::config::PinnedUserDataDir::new(); let mut config = tracedecay_session_memory::user_config::UserConfig::load(); config.pending_upload = 0; config.save().expect("initialize isolated user config"); diff --git a/crates/tracedecay/src/mcp/server/lifecycle.rs b/crates/tracedecay/src/mcp/server/lifecycle.rs index 272759e6de..36429a259f 100644 --- a/crates/tracedecay/src/mcp/server/lifecycle.rs +++ b/crates/tracedecay/src/mcp/server/lifecycle.rs @@ -283,7 +283,7 @@ impl McpServer { return; } } - let now = crate::project::current_timestamp(); + let now = tracedecay_runtime_core::tracedecay::current_timestamp(); self.last_staleness_check_at.store(now, Ordering::Release); self.startup_catch_up.settle(); @@ -333,7 +333,7 @@ impl McpServer { return; } let cg = self.cg_snapshot().await; - let now = crate::project::current_timestamp(); + let now = tracedecay_runtime_core::tracedecay::current_timestamp(); let previous = self.last_staleness_check_at.load(Ordering::Acquire); if previous != 0 && now.saturating_sub(previous) < 30 { return; @@ -406,7 +406,7 @@ impl McpServer { return; } - let now = crate::project::current_timestamp(); + let now = tracedecay_runtime_core::tracedecay::current_timestamp(); let cooldown = self.sync_config.read_cooldown_secs as i64; let previous = self.last_background_refresh_at.load(Ordering::Acquire); if previous != 0 && now.saturating_sub(previous) < cooldown { @@ -476,7 +476,10 @@ impl McpServer { ); } } - done_at.store(crate::project::current_timestamp(), Ordering::Release); + done_at.store( + tracedecay_runtime_core::tracedecay::current_timestamp(), + Ordering::Release, + ); }); } diff --git a/crates/tracedecay/src/mcp/server/requests.rs b/crates/tracedecay/src/mcp/server/requests.rs index 521cbb4694..7a60562ed3 100644 --- a/crates/tracedecay/src/mcp/server/requests.rs +++ b/crates/tracedecay/src/mcp/server/requests.rs @@ -9,6 +9,9 @@ use tracedecay_mcp::server::{ ApplicationCancellationRegistration, DispatchControl, DispatchControlRequest, DispatchSettlement, DispatchToolPolicy, PreparedDispatchControl, dispatch_cancelled_error, }; +use tracedecay_mcp::tools::response_trailers::{ + ToolTokenAccounting, record_token_accounting, response_token_count, +}; use tracedecay_mcp::{ ToolResult, mark_semantic_tool_error, semantic_failure_reason, server::resources_list_result, tool_error_response, tool_result_has_semantic_error, @@ -16,11 +19,6 @@ use tracedecay_mcp::{ use tracedecay_runtime_core::db::migrations::render_expected_final_schema_markdown; use tracedecay_tool_catalog::ApplicationSurfaceOperation; -/// Prefix of the out-of-band token-accounting block appended after a tool's -/// payload. `tracedecay tool` routes blocks carrying it to stderr so a JSON -/// payload on stdout stays a single document for scripts and hosts. -pub const TOKEN_ACCOUNTING_FOOTER_PREFIX: &str = "tracedecay_metrics:"; - mod tool_dispatch; struct PreparedToolCall { @@ -54,12 +52,6 @@ impl Drop for ToolActivityPublishRunning { } } -struct ToolTokenAccounting { - raw_file_tokens: u64, - response_tokens: u64, - net_saved_tokens: u64, -} - pub(super) fn invocation_target_for_route( route: Option<&crate::mcp::project_route::ResolvedProjectRoute>, ) -> tracedecay_contracts::InvocationTarget { @@ -274,11 +266,11 @@ impl McpServer { .map(|id| tool_error_response(id, &request.method, &error)); } }; - Box::pin(self.handle_request_for_connection( - request, + Box::pin(self.dispatch_envelope( + McpDispatchRequest::raw(request), self.timings_enabled(), &mut connection, - false, + tracedecay_runtime_core::cancellation::CancellationToken::new(), )) .await } @@ -313,33 +305,6 @@ impl McpServer { } } - /// Dispatches a request parsed off the legacy line-oriented JSON-RPC - /// transport. - /// - /// A thin adapter onto [`Self::dispatch_envelope`]: the raw params are - /// borrowed from the parsed request exactly as before, so this transport's - /// behavior and wire bytes are unchanged by the typed envelope. - #[hotpath::skip] - pub(crate) async fn handle_request_for_connection( - &self, - request: &JsonRpcRequest, - timings_enabled: bool, - connection: &mut ConnectionRouteState, - pre_cancelled: bool, - ) -> Option { - let cancellation = tracedecay_session_memory::context::CancellationToken::new(); - if pre_cancelled { - cancellation.cancel(); - } - Box::pin(self.dispatch_envelope( - McpDispatchRequest::from_legacy(request), - timings_enabled, - connection, - cancellation, - )) - .await - } - /// The single dispatch authority behind every MCP transport. /// /// Reads the request only through [`McpDispatchRequest`] accessors, so a @@ -352,7 +317,7 @@ impl McpServer { request: McpDispatchRequest<'_>, timings_enabled: bool, connection: &mut ConnectionRouteState, - cancellation: tracedecay_session_memory::context::CancellationToken, + cancellation: tracedecay_runtime_core::cancellation::CancellationToken, ) -> Option { // A response lease belongs to exactly one request. Production // transports take it before writing; direct callers drop it with this @@ -973,21 +938,6 @@ impl McpServer { .or_insert_with(|| json!(elapsed_us)); } - fn response_token_count(result: &ToolResult) -> u64 { - result - .value - .get("content") - .and_then(|content| content.as_array()) - .map_or(0, |content| { - let total_chars: usize = content - .iter() - .filter_map(|item| item.get("text").and_then(|text| text.as_str())) - .map(str::len) - .sum(); - (total_chars / 4) as u64 - }) - } - /// Resolves the raw-read counterfactual from the retained cache, falling /// back to bounded metadata reads for files owned by the current response. /// @@ -1043,42 +993,43 @@ impl McpServer { cg: &TraceDecay, tool_name: &str, result: &mut ToolResult, + ) -> ToolTokenAccounting { + // A result the shared renderer already accounted carries its figures + // and footer; only persist them. + let accounting = match result.token_accounting() { + Some(accounting) => accounting, + None => self.account_unrendered_result(cg, result).await, + }; + self.spawn_token_accounting_persist( + cg.project_root(), + tool_name, + accounting.net_saved_tokens(), + accounting.raw_file_tokens, + ); + self.maybe_flush_worldwide(); + accounting + } + + async fn account_unrendered_result( + &self, + cg: &TraceDecay, + result: &mut ToolResult, ) -> ToolTokenAccounting { // Estimate approximate token count of the graph response // ("after"), before any banners/metrics lines are appended. - let response_tokens = Self::response_token_count(result); + let response_tokens = response_token_count(result); // "Before" counterfactual: reading every referenced file raw, // in full. Counters credit only the net saving per call, // before minus what this response actually delivered. let raw_file_tokens = self .raw_file_tokens(cg.project_root(), &result.touched_files) .await; - let net_saved_tokens = raw_file_tokens.saturating_sub(response_tokens); - self.spawn_token_accounting_persist( - cg.project_root(), - tool_name, - net_saved_tokens, - raw_file_tokens, - ); - self.maybe_flush_worldwide(); - - // Append per-call token savings to the response content. - if raw_file_tokens > 0 - && let Some(content) = result - .value - .get_mut("content") - .and_then(|c| c.as_array_mut()) - { - content.push(json!({"type": "text", "text": format!( - "\n{TOKEN_ACCOUNTING_FOOTER_PREFIX} before={raw_file_tokens} after={response_tokens}" - )})); - } - - ToolTokenAccounting { + let accounting = ToolTokenAccounting { raw_file_tokens, response_tokens, - net_saved_tokens, - } + }; + record_token_accounting(result, accounting); + accounting } #[allow(clippy::too_many_arguments)] @@ -1109,15 +1060,15 @@ impl McpServer { let savings_db = self.accounting_db.clone(); let analytics_db = self.global_db.clone(); if savings_db.is_some() || analytics_db.is_some() { + let net_saved_tokens = accounting.net_saved_tokens(); let ToolTokenAccounting { raw_file_tokens, response_tokens, - net_saved_tokens, } = accounting; let project_path_str = RegisteredGlobalDb::canonical_project_key(accounting_project_root); let tool_name_owned = tool_name.to_string(); - let ts = crate::project::current_timestamp(); + let ts = tracedecay_runtime_core::tracedecay::current_timestamp(); let failure_reason = (analytics_outcome == "error") .then(|| semantic_failure_reason(result)) .flatten(); @@ -1192,31 +1143,16 @@ impl McpServer { } #[hotpath::measure(label = "mcp.server.tools_call.complete.version_check")] - fn append_version_notice( - &self, - result: &mut ToolResult, - connection_notifications: &std::sync::Mutex>, - ) { - // Prepend the version-update warning and queue the corresponding - // protocol notification. The check serves the cached answer and - // refreshes in the background, so completion never awaits the fetch. - if let Some(warning) = self.check_version_update() { - if let Some(content) = result + fn append_version_notice(&self, result: &mut ToolResult) { + // The check serves the cached answer and refreshes in the background, + // so completion never awaits the fetch. + if let Some(warning) = self.check_version_update() + && let Some(content) = result .value .get_mut("content") .and_then(|c| c.as_array_mut()) - { - content.insert(0, json!({"type": "text", "text": &warning})); - } - recover_lock(connection_notifications).push(json!({ - "jsonrpc": "2.0", - "method": "notifications/message", - "params": { - "level": "warning", - "logger": "tracedecay", - "data": warning - } - })); + { + content.insert(0, json!({"type": "text", "text": warning})); } } @@ -1255,7 +1191,6 @@ impl McpServer { let client_name = connection_server.client_name(); let connection_client_name = client_name.as_deref(); let connection_instance_id = connection_server.connection_identity.instance_id(); - let connection_notifications = &connection_server.pending_notifications; let DispatchedToolCall { cg, selected_owner, @@ -1318,7 +1253,7 @@ impl McpServer { ) .await; } - self.append_version_notice(&mut result, connection_notifications); + self.append_version_notice(&mut result); self.prepend_index_warnings(selected_owner.is_none(), &mut result); hotpath::measure_block!( "mcp.server.tools_call.complete.response", @@ -1380,7 +1315,7 @@ impl McpServer { fn message_search_worker_is_unavailable(&self, tool_name: &str, arguments: &Value) -> bool { if tool_name != "tracedecay_message_search" - || arguments.get("catch_up").and_then(Value::as_bool) != Some(true) + || arguments.get("require_fresh").and_then(Value::as_bool) != Some(true) { return false; } @@ -1461,7 +1396,7 @@ impl McpServer { params: ToolCallParams<'_>, timings_enabled: bool, connection: &mut ConnectionRouteState, - cancellation: tracedecay_session_memory::context::CancellationToken, + cancellation: tracedecay_runtime_core::cancellation::CancellationToken, ) -> JsonRpcResponse { let started = timings_enabled.then(std::time::Instant::now); let mut response = Box::pin(self.handle_tools_call_inner( @@ -1490,7 +1425,7 @@ impl McpServer { params: ToolCallParams<'_>, timings_enabled: bool, connection: &mut ConnectionRouteState, - cancellation: tracedecay_session_memory::context::CancellationToken, + cancellation: tracedecay_runtime_core::cancellation::CancellationToken, ) -> JsonRpcResponse { let PreparedToolCall { tool_name, @@ -1683,17 +1618,9 @@ impl McpServer { ) .await) }; - let dispatch_outcome = if connection.connection_owns_dispatch() - && control.permits_connection_owned_execution() - { - control - .run_connection_owned(dispatch_server.dispatch_authority.registry(), worker) - .await - } else { - control - .run_retained(dispatch_server.dispatch_authority.registry(), worker) - .await - }; + let dispatch_outcome = control + .run_retained(dispatch_server.dispatch_authority.registry(), worker) + .await; // Safety: each guard is dropped exactly once, here, after the worker // has settled, and neither is used again. unsafe { @@ -1886,7 +1813,6 @@ mod git_read_control_tests { "{tool_name} must carry the caller cancellation signal into the verified graph" ); } - assert!(!tool_supports_live_cancellation("tracedecay_outline")); for tool_name in [ "tracedecay_git_status", "tracedecay_git_diff", @@ -2034,8 +1960,6 @@ mod git_read_control_tests { #[test] fn non_git_reads_stay_outside_the_controlled_read_horizon() { for tool_name in [ - "tracedecay_outline", - "tracedecay_body", "tracedecay_dead_code", "tracedecay_health", "tracedecay_context", diff --git a/crates/tracedecay/src/mcp/server/requests/tool_dispatch.rs b/crates/tracedecay/src/mcp/server/requests/tool_dispatch.rs index 07db64633f..d5bf316164 100644 --- a/crates/tracedecay/src/mcp/server/requests/tool_dispatch.rs +++ b/crates/tracedecay/src/mcp/server/requests/tool_dispatch.rs @@ -1,6 +1,7 @@ //! Project-route selection, tool-dispatch assembly, and identical-read sharing. use super::*; +use crate::mcp::tools::handlers::ServedCodeGraphSlot; use crate::mcp::tools::{ToolCallRegistryOptions, handle_tool_call_with_registry_options}; use tracedecay_mcp::server::{ReadFlightClaim, tool_allows_identical_read_coalescing}; @@ -36,7 +37,6 @@ impl McpServer { let routed_project = match private_route { Some(_) if crate::mcp::project_route::arguments_have_project_selector( - tool_name, &handler_arguments, ) => { @@ -346,7 +346,7 @@ impl McpServer { generation_census_reader: self.generation_census_reader(), retained_project_server_resolver: self.retained_project_server_resolver.clone(), session_sync_service: session_sync_service.as_deref(), - served_stale_graph_generation: std::sync::Arc::new(std::sync::OnceLock::new()), + served_code_graph: ServedCodeGraphSlot::default(), session_authorities: tracedecay_mcp::handlers::SessionAuthorities::new( self.project_session_db.as_ref(), self.profile_session_db.as_ref(), diff --git a/crates/tracedecay/src/mcp/server/rmcp.rs b/crates/tracedecay/src/mcp/server/rmcp.rs index ea8338c6a4..c7f09453e8 100644 --- a/crates/tracedecay/src/mcp/server/rmcp.rs +++ b/crates/tracedecay/src/mcp/server/rmcp.rs @@ -143,7 +143,7 @@ mod tests { impl RmcpWireFixture { async fn start() -> Self { - crate::product_runtime::register_fixture_product_runtime(); + tracedecay_project::product_runtime::register_fixture_product_runtime(); let (cg, repo, authority) = crate::mcp::server::writer_test_support::init_indexed_repo().await; let context = @@ -201,7 +201,7 @@ mod tests { .rev() .find(|message| message.get("id") == Some(&response_id)) .expect("recorded client request for response"); - serde_json::from_value(request.clone()).expect("legacy request shape") + serde_json::from_value(request.clone()).expect("raw request shape") } fn last_response(&self) -> Value { @@ -213,22 +213,22 @@ mod tests { .clone() } - async fn assert_last_response_matches_legacy(&self, decorate_initialize: bool) { + async fn assert_last_response_matches_raw_dispatch(&self, decorate_initialize: bool) { let request = self.last_request(); let mut expected = self .server .handle_request(&request) .await - .expect("legacy response"); + .expect("raw response"); if decorate_initialize { - expected.result.as_mut().expect("legacy initialize result")["_meta"]["tracedecayInitializeRoute"] = json!({ + expected.result.as_mut().expect("raw initialize result")["_meta"]["tracedecayInitializeRoute"] = json!({ "projectPath": "/wire/oracle", "allowInit": false, }); } assert_eq!( self.last_response(), - serde_json::to_value(expected).expect("serialize legacy response"), + serde_json::to_value(expected).expect("serialize raw response"), ); } @@ -243,7 +243,7 @@ mod tests { async fn malformed_initialize_is_refused_typed_and_a_corrected_handshake_still_serves() { use tokio::io::{AsyncBufReadExt, AsyncWriteExt}; - crate::product_runtime::register_fixture_product_runtime(); + tracedecay_project::product_runtime::register_fixture_product_runtime(); let (cg, repo, authority) = crate::mcp::server::writer_test_support::init_indexed_repo().await; let context = crate::mcp::server::writer_test_support::registered_context(cg, &authority); @@ -281,6 +281,10 @@ mod tests { ); line.clear(); + client_write + .write_all(b"{\"jsonrpc\":\"2.0\",\"method\":\"notifications/initialized\"}\n") + .await + .expect("send the pipelined initialized notification"); client_write .write_all( br#"{"jsonrpc":"2.0","id":2,"method":"initialize","params":{"protocolVersion":"2025-06-18","capabilities":{},"clientInfo":{"name":"handshake-retry","version":"0"}}} @@ -307,8 +311,93 @@ mod tests { drop(repo); } + /// MCP lets either party ping at any time; a stateless SEP-2575 connection + /// must still answer one after it has served another request. + #[tokio::test] + async fn stateless_connection_answers_ping_after_an_earlier_request() { + use tokio::io::{AsyncBufReadExt, AsyncWriteExt}; + + tracedecay_project::product_runtime::register_fixture_product_runtime(); + let (cg, repo, authority) = + crate::mcp::server::writer_test_support::init_indexed_repo().await; + let context = crate::mcp::server::writer_test_support::registered_context(cg, &authority); + let server = McpServer::new_with_registered_test_context(context, Vec::new()) + .await + .expect("registered RMCP stateless server"); + let adapter = production_adapter(&server, None); + let (server_io, client_io) = tokio::io::duplex(2 * 1024 * 1024); + let serving = tokio::spawn(async move { + let running = adapter + .serve(IntoTransport::::into_transport(server_io)) + .await + .expect("a stateless first request opens the RMCP session"); + let _ = running.waiting().await; + }); + let (client_read, mut client_write) = tokio::io::split(client_io); + let mut client_read = tokio::io::BufReader::new(client_read); + let stateless_line = |request: Value| { + let mut request: JsonRpcRequest = + serde_json::from_value(request).expect("JSON-RPC request fixture"); + assert!(tracedecay_mcp::server::attach_stateless_request_context( + &mut request + )); + format!( + "{}\n", + serde_json::to_string(&request).expect("stateless request") + ) + }; + + client_write + .write_all( + stateless_line(json!({"jsonrpc": "2.0", "id": 1, "method": "tools/list"})) + .as_bytes(), + ) + .await + .expect("send stateless tools/list"); + let mut line = String::new(); + client_read + .read_line(&mut line) + .await + .expect("read tools/list response"); + let listed: Value = serde_json::from_str(&line).expect("tools/list frame"); + assert_eq!(listed["id"], json!(1), "{line}"); + assert!(listed["result"]["tools"].is_array(), "{line}"); + + for (request, id) in [ + ( + stateless_line(json!({"jsonrpc": "2.0", "id": 2, "method": "ping"})), + 2, + ), + ( + r#"{"jsonrpc":"2.0","id":3,"method":"ping"}"#.to_owned() + "\n", + 3, + ), + ] { + client_write + .write_all(request.as_bytes()) + .await + .expect("send ping"); + line.clear(); + client_read + .read_line(&mut line) + .await + .expect("read ping response"); + assert_eq!( + line.trim_end(), + format!(r#"{{"jsonrpc":"2.0","id":{id},"result":{{}}}}"#), + "a later ping on a stateless connection must get an empty result: {request}", + ); + } + + drop(client_write); + drop(client_read); + serving.await.expect("join RMCP server"); + server.shutdown().await; + drop(repo); + } + #[tokio::test] - async fn rmcp_wire_matrix_matches_legacy_initialize_tools_and_resources() { + async fn rmcp_wire_matrix_matches_raw_dispatch_initialize_tools_and_resources() { let fixture = RmcpWireFixture::start().await; let initialize_id = fixture.last_response()["id"].clone(); let mut initialize_result = @@ -323,7 +412,7 @@ mod tests { fixture.last_response(), serde_json::to_value(JsonRpcResponse::success(initialize_id, initialize_result)) .expect("serialize initialize oracle"), - "rmcp negotiates the client protocol version while preserving the legacy payload", + "rmcp negotiates the client protocol version while preserving the raw payload", ); assert_eq!( fixture.last_response()["result"]["_meta"]["tracedecayInitializeRoute"], @@ -336,7 +425,9 @@ mod tests { .list_tools(None) .await .expect("RMCP tools/list"); - fixture.assert_last_response_matches_legacy(false).await; + fixture + .assert_last_response_matches_raw_dispatch(false) + .await; fixture .client @@ -369,7 +460,9 @@ mod tests { ) .await .expect_err("unknown tool must be a JSON-RPC error"); - fixture.assert_last_response_matches_legacy(false).await; + fixture + .assert_last_response_matches_raw_dispatch(false) + .await; assert_eq!( fixture.last_response()["error"]["code"], json!(-32603), @@ -403,14 +496,18 @@ mod tests { .list_resources(None) .await .expect("RMCP resources/list"); - fixture.assert_last_response_matches_legacy(false).await; + fixture + .assert_last_response_matches_raw_dispatch(false) + .await; fixture .client .read_resource(ReadResourceRequestParams::new("tracedecay://schema")) .await .expect("RMCP resources/read"); - fixture.assert_last_response_matches_legacy(false).await; + fixture + .assert_last_response_matches_raw_dispatch(false) + .await; let unknown_resource = fixture .client @@ -419,11 +516,13 @@ mod tests { )) .await .expect_err("an unknown resource URI must be a JSON-RPC error"); - fixture.assert_last_response_matches_legacy(false).await; + fixture + .assert_last_response_matches_raw_dispatch(false) + .await; assert_eq!( fixture.last_response()["error"]["code"], json!(-32602), - "the typed resources/read refusal keeps the legacy invalid-params code", + "the typed resources/read refusal keeps the raw invalid-params code", ); assert!( unknown_resource @@ -509,20 +608,54 @@ mod tests { fixture.shutdown().await; } - /// The legacy raw JSON-RPC transport must stay byte-for-byte what it was + /// `rmcp` moves wire `params._meta` into the request context; the caller + /// deadline it carries must still reach dispatch as on the raw path. + #[tokio::test] + async fn rmcp_tool_call_honours_the_request_meta_caller_deadline() { + let fixture = RmcpWireFixture::start().await; + let mut params = CallToolRequestParams::new("tracedecay_status").with_arguments( + json!({"admission_only": true, "format": "json"}) + .as_object() + .cloned() + .expect("object arguments"), + ); + params.meta = Some(rmcp::model::RequestMetaObject::from( + rmcp::model::MetaObject( + tracedecay_mcp::tool_call_deadline_meta(tracedecay_domain::UtcMicros(1)) + .as_object() + .cloned() + .expect("deadline meta object"), + ), + )); + let elapsed = fixture.client.call_tool(params).await; + fixture + .assert_last_response_matches_raw_dispatch(false) + .await; + assert!( + elapsed.is_err() + || elapsed + .as_ref() + .is_ok_and(|result| result.is_error == Some(true)), + "an elapsed caller deadline must not complete as a success: {:?}", + fixture.last_response() + ); + fixture.shutdown().await; + } + + /// Raw JSON-RPC dispatch frames must stay byte-for-byte what they were /// before the typed envelope: the envelope is an internal representation, /// never a wire change. These are the shapes a host actually parses, /// method refusals, param refusals, the trivial ack, and a resource body, /// pinned as exact serialized frames rather than as structural matches. #[tokio::test] - async fn legacy_json_rpc_wire_frames_are_unchanged_by_the_typed_envelope() { - crate::product_runtime::register_fixture_product_runtime(); + async fn raw_json_rpc_wire_frames_are_unchanged_by_the_typed_envelope() { + tracedecay_project::product_runtime::register_fixture_product_runtime(); let (cg, _repo, authority) = crate::mcp::server::writer_test_support::init_indexed_repo().await; let context = crate::mcp::server::writer_test_support::registered_context(cg, &authority); let server = McpServer::new_with_registered_test_context(context, Vec::new()) .await - .expect("registered legacy wire server"); + .expect("registered raw wire server"); for (request_line, expected) in [ ( @@ -551,15 +684,12 @@ mod tests { ), ] { let request: JsonRpcRequest = - serde_json::from_str(request_line).expect("legacy request line"); - let response = server - .handle_request(&request) - .await - .expect("legacy response"); + serde_json::from_str(request_line).expect("raw request line"); + let response = server.handle_request(&request).await.expect("raw response"); assert_eq!( - serde_json::to_string(&response).expect("serialize legacy response"), + serde_json::to_string(&response).expect("serialize raw response"), expected, - "legacy wire frame changed for {request_line}", + "raw wire frame changed for {request_line}", ); } @@ -568,19 +698,19 @@ mod tests { let schema_request: JsonRpcRequest = serde_json::from_str( r#"{"jsonrpc":"2.0","id":6,"method":"resources/read","params":{"uri":"tracedecay://schema"}}"#, ) - .expect("legacy request line"); + .expect("raw request line"); let schema = serde_json::to_string( &server .handle_request(&schema_request) .await - .expect("legacy response"), + .expect("raw response"), ) - .expect("serialize legacy response"); + .expect("serialize raw response"); assert!( schema.starts_with( r#"{"jsonrpc":"2.0","id":6,"result":{"contents":[{"mimeType":"text/markdown","text":"# ) && schema.ends_with(r#","uri":"tracedecay://schema"}]}}"#), - "legacy resources/read frame shape changed: {schema}", + "raw resources/read frame shape changed: {schema}", ); let parsed: Value = serde_json::from_str(&schema).expect("schema frame"); let text = parsed["result"]["contents"][0]["text"] @@ -594,16 +724,16 @@ mod tests { "schema resource must be the migration inventory, not a second document" ); - // Notifications stay responseless on the legacy transport. + // Notifications stay responseless on raw dispatch. for notification in [ r#"{"jsonrpc":"2.0","method":"notifications/initialized"}"#, r#"{"jsonrpc":"2.0","method":"notifications/cancelled","params":{"requestId":1}}"#, ] { let request: JsonRpcRequest = - serde_json::from_str(notification).expect("legacy notification line"); + serde_json::from_str(notification).expect("raw notification line"); assert!( server.handle_request(&request).await.is_none(), - "legacy notification produced a response: {notification}", + "raw notification produced a response: {notification}", ); } server.shutdown().await; @@ -657,14 +787,14 @@ mod tests { } #[test] - fn adapter_accepts_the_legacy_initialize_response_shape() { - crate::product_runtime::register_fixture_product_runtime(); + fn adapter_accepts_the_dispatch_initialize_response_shape() { + tracedecay_project::product_runtime::register_fixture_product_runtime(); let initialized: InitializeResult = rmcp_response_result(JsonRpcResponse::success( json!(1), crate::mcp::server::initialize_result("TraceDecay instructions") .expect("fixture product runtime registered"), )) - .expect("rmcp must preserve legacy MCP initialization compatibility"); + .expect("rmcp must accept the server's own initialize result shape"); assert_eq!( serde_json::to_value(&initialized).expect("serialize initialized response")["protocolVersion"], diff --git a/crates/tracedecay/src/mcp/server/routing.rs b/crates/tracedecay/src/mcp/server/routing.rs index f5f439e84b..6ac2330cf6 100644 --- a/crates/tracedecay/src/mcp/server/routing.rs +++ b/crates/tracedecay/src/mcp/server/routing.rs @@ -21,7 +21,7 @@ use tracedecay_global_db::RegisteredGlobalDb; /// wire contract or a second routing identity. pub(crate) struct SelectedProjectResponseLease { _guard: tokio::sync::OwnedRwLockReadGuard<()>, - revoked: tracedecay_session_memory::context::CancellationToken, + revoked: tracedecay_runtime_core::cancellation::CancellationToken, _active: ResponseLeaseGaugeGuard, } @@ -43,7 +43,7 @@ impl Drop for ResponseLeaseGaugeGuard { impl SelectedProjectResponseLease { pub(crate) fn new( guard: tokio::sync::OwnedRwLockReadGuard<()>, - revoked: tracedecay_session_memory::context::CancellationToken, + revoked: tracedecay_runtime_core::cancellation::CancellationToken, ) -> Self { Self { _guard: guard, @@ -52,20 +52,20 @@ impl SelectedProjectResponseLease { } } - pub(crate) fn revoked(&self) -> &tracedecay_session_memory::context::CancellationToken { + pub(crate) fn revoked(&self) -> &tracedecay_runtime_core::cancellation::CancellationToken { &self.revoked } } impl McpResponseLease for SelectedProjectResponseLease { - fn revoked(&self) -> &tracedecay_session_memory::context::CancellationToken { + fn revoked(&self) -> &tracedecay_runtime_core::cancellation::CancellationToken { &self.revoked } } /// Per-connection routing and identity context, constructed once per client /// connection (or per initialize-replay dispatch) and threaded through -/// [`McpServer::handle_request_for_connection`]. Bundling these values keeps +/// [`McpServer::dispatch_envelope`]. Bundling these values keeps /// persisted application request correlation and cancellation scoped to the /// exact client connection. pub(crate) struct ConnectionRouteState { @@ -78,7 +78,6 @@ pub(crate) struct ConnectionRouteState { pub(crate) route_cache: HookProjectRouteCache, selected_response_lease: Option, selected_request_server: Option>, - connection_owns_dispatch: bool, } impl ConnectionRouteState { @@ -89,7 +88,6 @@ impl ConnectionRouteState { route_cache, selected_response_lease: None, selected_request_server: None, - connection_owns_dispatch: false, } } @@ -125,20 +123,9 @@ impl ConnectionRouteState { route_cache: self.route_cache.clone(), selected_response_lease: None, selected_request_server: None, - connection_owns_dispatch: false, } } - pub(crate) fn fork_for_connection_owned_read(&self) -> Self { - let mut fork = self.fork_for_independent_read(); - fork.connection_owns_dispatch = true; - fork - } - - pub(crate) fn connection_owns_dispatch(&self) -> bool { - self.connection_owns_dispatch - } - pub(crate) fn install_selected_response_lease(&mut self, lease: SelectedProjectResponseLease) { self.selected_response_lease = Some(lease); } @@ -180,10 +167,6 @@ impl McpConnectionState for ConnectionRouteState { ConnectionRouteState::fork_for_independent_read(self) } - fn fork_for_connection_owned_read(&self) -> Self { - ConnectionRouteState::fork_for_connection_owned_read(self) - } - fn take_selected_response_lease(&mut self) -> Option { ConnectionRouteState::take_selected_response_lease(self) } @@ -492,7 +475,7 @@ mod tests { resolve_initialize_roots_project_path, resolve_initialize_roots_project_route, select_initialize_project_path, }; - use crate::test_support::host_admission::HostAdmissionTestRuntimeV1; + use tracedecay_project::test_support::host_admission::HostAdmissionTestRuntimeV1; use tracedecay_sessions::admission::HostAdmissionScope; fn run_git(root: &Path, args: &[&str]) { diff --git a/crates/tracedecay/src/mcp/server/status_resource.rs b/crates/tracedecay/src/mcp/server/status_resource.rs index 9690c455b3..da808dc300 100644 --- a/crates/tracedecay/src/mcp/server/status_resource.rs +++ b/crates/tracedecay/src/mcp/server/status_resource.rs @@ -1,6 +1,7 @@ //! Typed status resource rendering. use serde_json::{Value, json}; +use tracedecay_mcp::handlers::info::graph_statistics_value; use super::{ErrorCode, JsonRpcResponse, McpServer}; @@ -9,11 +10,11 @@ impl McpServer { #[hotpath::skip] pub(crate) async fn read_resource_status(&self, id: Value) -> JsonRpcResponse { let cg = self.reopen_if_branch_drifted().await; - let graph_statistics = match crate::mcp::tools::handlers::info::graph_statistics_value( - self.generation_census_reader().as_ref(), - ) - .await - { + let census = match self.generation_census_reader() { + Some(reader) => Some(reader().await), + None => None, + }; + let graph_statistics = match graph_statistics_value(census.as_ref()) { Ok(value) => value, Err(error) => { return JsonRpcResponse::error( diff --git a/crates/tracedecay/src/mcp/server/writer_test_support.rs b/crates/tracedecay/src/mcp/server/writer_test_support.rs index a3fff9aab7..6b928685f9 100644 --- a/crates/tracedecay/src/mcp/server/writer_test_support.rs +++ b/crates/tracedecay/src/mcp/server/writer_test_support.rs @@ -4,10 +4,10 @@ use std::sync::Arc; use tempfile::TempDir; use tracedecay_runtime_core::path_safety::{plain_git_args, plain_host_path}; -use crate::config::PinnedUserDataDir; use crate::mcp::server::McpServerConstructionContext; -use crate::project::TraceDecay; -use crate::test_support::host_admission::HostAdmissionTestRuntimeV1; +use tracedecay_project::config::PinnedUserDataDir; +use tracedecay_project::project::TraceDecay; +use tracedecay_project::test_support::host_admission::HostAdmissionTestRuntimeV1; pub(super) fn git(root: &Path, args: &[&str]) { let output = std::process::Command::new( @@ -35,7 +35,7 @@ impl WriterTestFixtureAuthority { self.runtime .open_project_graph_for_test( project_root, - crate::project::TraceDecayOpenOptions { + tracedecay_project::project::TraceDecayOpenOptions { profile_root: Some(self.runtime.profile_root_for_test().to_path_buf()), global_db_path: None, }, diff --git a/crates/tracedecay/src/mcp/tools/handlers/application_surface.rs b/crates/tracedecay/src/mcp/tools/handlers/application_surface.rs index 9efd8ab5eb..7bc3802e8c 100644 --- a/crates/tracedecay/src/mcp/tools/handlers/application_surface.rs +++ b/crates/tracedecay/src/mcp/tools/handlers/application_surface.rs @@ -6,8 +6,8 @@ use tracedecay_contracts::{ use tracedecay_domain::UtcMicros; use tracedecay_tool_catalog::{ApplicationSurfaceOperation, BindingId}; -use crate::project::TraceDecay; use tracedecay_contracts::request_identity::{GlobalRequestSurface, mint_global_request_id}; +use tracedecay_contracts::retrieval::ServedCodeGraphGenerationV1; use tracedecay_daemon_protocol::{ ApplicationSurfaceInvocationResult, ApplicationToolRequest, parse_application_surface_request, }; @@ -18,6 +18,8 @@ use tracedecay_mcp::tools::dispatch::{ resolve_mcp_application_surface_for_target, resolve_mcp_application_surface_with_controls_for_target, }; +use tracedecay_mcp::tools::response_trailers::append_code_graph_freshness; +use tracedecay_project::project::TraceDecay; pub(super) fn request_id() -> Result { mint_global_request_id(GlobalRequestSurface::McpFallback).map_err(|_| TraceDecayError::Config { @@ -170,30 +172,30 @@ pub(super) async fn handle_application_surface( } .map_err(application_surface_dispatch_error)?; - let served_stale = served_stale_code_graph_read(&result); + let served = served_code_graph_read(&result); let mut rendered = render_result(cg, result)?; - if let Some(served) = served_stale.as_ref() { - super::append_code_graph_freshness(&mut rendered, served); + if let Some(served) = served.as_ref() { + append_code_graph_freshness(&mut rendered, served); } Ok(rendered) } -fn served_stale_code_graph_read( +fn served_code_graph_read( result: &ApplicationSurfaceInvocationResult, -) -> Option { +) -> Option { let Ok(envelope) = &result.result else { return None; }; let ApplicationOutcome::Evidence(evidence) = &envelope.outcome else { return None; }; - served_stale_code_graph_temporal(result.operation, &evidence.temporal) + served_code_graph_temporal(result.operation, &evidence.temporal) } -fn served_stale_code_graph_temporal( +fn served_code_graph_temporal( operation: ApplicationSurfaceOperation, temporal: &tracedecay_contracts::TemporalState, -) -> Option { +) -> Option { if !matches!( operation, ApplicationSurfaceOperation::CodeSymbolSearch @@ -205,18 +207,9 @@ fn served_stale_code_graph_temporal( ) { return None; } - let generation = temporal.source_generation.as_ref()?; - let Some(tracedecay_graph_query::CodeGraphReadFreshnessV1::LastCompleteStale { - sealed_at, - rebuild_in_flight, - }) = temporal.code_graph_freshness - else { - return None; - }; - Some(super::ServedStaleCodeGraphReadV1 { - generation: generation.as_str().to_owned(), - sealed_at, - rebuild_in_flight, + Some(ServedCodeGraphGenerationV1 { + generation: temporal.source_generation.as_ref()?.as_str().to_owned(), + freshness: temporal.code_graph_freshness?, }) } @@ -325,6 +318,306 @@ fn render_result_parts( }) } +/// A settled retained tool call, before rendering. +pub struct RetainedSurfaceExecution { + pub operation: ApplicationSurfaceOperation, + pub binding_id: BindingId, + pub requested_format: RequestedOutputFormat, + pub result: ApplicationResult, +} + +/// Run one retained memory, session, or workflow tool on `surface` and render +/// its tool result exactly as the retained tools always have. +#[allow(clippy::too_many_arguments)] +#[hotpath::measure(future = true, label = "mcp.retained.total")] +pub async fn run_retained_surface_tool( + project_root: Option<&std::path::Path>, + surface: tracedecay_tool_catalog::BindingSurface, + operation: ApplicationSurfaceOperation, + args: Value, + executor: Option<&dyn DaemonInvocationExecutor>, + protocol_request_id: Option, + deadline: Option, + cancellation: Option, +) -> Result { + let execution = execute_retained_surface_tool( + surface, + operation, + args, + executor, + protocol_request_id, + deadline, + cancellation, + ) + .await?; + render_retained_execution(project_root, &execution) +} + +/// Render a settled retained tool call. +pub fn render_retained_execution( + project_root: Option<&std::path::Path>, + execution: &RetainedSurfaceExecution, +) -> Result { + hotpath::measure_block!( + "mcp.retained.render", + render_result_parts( + project_root, + execution.operation.as_str(), + &execution.binding_id, + &execution.result, + execution.requested_format, + ) + ) +} + +/// Decode, dispatch, and settle one retained tool call. `Err` is an argument +/// or transport failure the caller reports as-is. +#[allow(clippy::too_many_arguments)] +pub async fn execute_retained_surface_tool( + surface: tracedecay_tool_catalog::BindingSurface, + operation: ApplicationSurfaceOperation, + args: Value, + executor: Option<&dyn DaemonInvocationExecutor>, + protocol_request_id: Option, + deadline: Option, + cancellation: Option, +) -> Result { + let tool_name = operation.mcp_tool_name(); + let retained = RetainedSurfaceOperation::from_application(operation) + .ok_or_else(|| super::unknown_tool_error(tool_name))?; + let normalized = + tracedecay_daemon_protocol::separate_application_tool_request(args).map_err(|error| { + TraceDecayError::Config { + message: error.to_string(), + } + })?; + let requested_format = normalized.requested_format; + let request = hotpath::measure_block!( + "mcp.retained.decode", + tracedecay_daemon_protocol::decode_retained_request(retained, normalized.request) + ) + .map_err(|error| TraceDecayError::Config { + message: format!("invalid retained application request for {tool_name}: {error}"), + })?; + let request_id = match protocol_request_id { + Some(request_id) => request_id, + None => self::request_id()?, + }; + let (deadline, cancellation) = + complete_retained_protocol_controls(retained, &request_id, deadline, cancellation)? + .ok_or_else(|| { + TraceDecayError::project_route( + "retained_application_controls_unavailable", + true, + "retained application protocol controls are unavailable", + ) + })?; + // The executor belongs to the selected project's server, and the retained + // daemon payload carries no resolved scope, so the target stays current. + let dispatched = tracedecay_daemon_service::application_surface::resolve_application_surface_dispatch_with_controls( + surface, + operation, + request_id.clone(), + tracedecay_daemon_protocol::ApplicationSurfaceRequest::Retained(request), + tracedecay_contracts::PageRequest::first(10).map_err(|error| TraceDecayError::Config { + message: error.to_string(), + })?, + Some(deadline), + cancellation, + requested_format, + ) + .map_err(application_surface_dispatch_error)?; + let binding_id = dispatched.invocation.binding_id.clone(); + let result_contract = + tracedecay_contracts::ResultContractRef::from_schema(&dispatched.invocation.result_schema); + let unavailable = |code: String, message: String| { + tracedecay_contracts::ApplicationProblemEnvelope::new( + result_contract.clone(), + request_id.clone(), + tracedecay_contracts::ApplicationProblem::unavailable( + tracedecay_contracts::SafeDiagnostic { code, message }, + ), + ) + .map_err(|error| TraceDecayError::Config { + message: format!("invalid retained application problem envelope: {error}"), + }) + }; + let result = match executor { + None => Err(unavailable( + "application.transport.unavailable".to_owned(), + "The daemon retained application transport is unavailable".to_owned(), + )?), + Some(executor) => match hotpath::future!( + tracedecay_daemon_service::application_surface::execute_application_surface( + operation, + dispatched, + Some(executor), + ), + label = "mcp.retained.invoke" + ) + .await + { + Ok(result) => result.result, + Err( + tracedecay_daemon_protocol::ApplicationSurfaceAdapterError::DaemonUnreachable { + reason_code, + detail, + }, + ) => Err(unavailable(reason_code, detail)?), + Err(error) => return Err(application_surface_dispatch_error(error)), + }, + }; + Ok(RetainedSurfaceExecution { + operation, + binding_id, + requested_format, + result, + }) +} + +/// Invoke one graph-tool operation through the project's graph-tool owner and +/// return its typed result. A refusal comes back as the handler's own error +/// kind, so every surface reports the failure it always reported. +#[allow(clippy::too_many_arguments)] +#[hotpath::measure(future = true, label = "mcp.graph_tool.total")] +pub async fn execute_graph_tool_surface( + surface: tracedecay_tool_catalog::BindingSurface, + operation: ApplicationSurfaceOperation, + args: Value, + executor: Option<&dyn DaemonInvocationExecutor>, + protocol_request_id: Option, + deadline: Option, + cancellation: Option, +) -> Result { + let request = parse_application_surface_request(operation, args).map_err(|error| { + TraceDecayError::Config { + message: match error { + tracedecay_daemon_protocol::ApplicationSurfaceAdapterError::InvalidSurfaceRequest { + detail, + } => detail, + error => error.to_string(), + }, + } + })?; + let request_id = match protocol_request_id { + Some(request_id) => request_id, + None => self::request_id()?, + }; + let (deadline, cancellation) = + complete_protocol_controls(operation, &request_id, deadline, cancellation)?.ok_or_else( + || { + TraceDecayError::project_route( + "application_surface_controls_unavailable", + true, + "graph-tool protocol controls are unavailable", + ) + }, + )?; + let dispatched = tracedecay_daemon_service::application_surface::resolve_application_surface_dispatch_with_controls( + surface, + operation, + request_id, + request, + tracedecay_contracts::PageRequest::first(10).map_err(|error| TraceDecayError::Config { + message: error.to_string(), + })?, + Some(deadline), + cancellation, + RequestedOutputFormat::Json, + ) + .map_err(application_surface_dispatch_error)?; + let result = tracedecay_daemon_service::application_surface::execute_application_surface( + operation, dispatched, executor, + ) + .await + .map_err(application_surface_dispatch_error)? + .result; + let envelope = result.map_err(|problem| graph_tool_problem_error(&problem.problem))?; + let ApplicationOutcome::Result(value) = envelope.outcome else { + return Err(TraceDecayError::project_route( + "application_surface_invalid_response", + false, + format!( + "{} returned a non-result outcome", + operation.mcp_tool_name() + ), + )); + }; + let result = + tracedecay_contracts::graph_tool::GraphToolResultV1::from_result_value(operation, value) + .map_err(|error| { + TraceDecayError::project_route( + "application_surface_invalid_response", + false, + format!( + "{} returned an invalid result: {error}", + operation.mcp_tool_name() + ), + ) + })?; + Ok(tracedecay_contracts::graph_tool::GraphToolCompletionV1 { + result, + touched_files: envelope.touched_files, + code_graph: envelope.code_graph, + analytics: envelope.analytics, + }) +} + +/// The graph-tool owner reports handler argument errors as invalid requests +/// and every other refusal under its own reason code. +fn graph_tool_problem_error( + problem: &tracedecay_contracts::ApplicationProblemRecord, +) -> TraceDecayError { + let message = problem.diagnostic.as_ref().map_or_else( + || problem.message.clone(), + |diagnostic| diagnostic.message.clone(), + ); + match problem.kind { + ApplicationProblemKind::InvalidRequest => TraceDecayError::Config { message }, + _ => TraceDecayError::project_route(problem.code.clone(), problem.retryable, message), + } +} + +/// The owner-side counterpart of [`graph_tool_problem_error`]. +pub(crate) fn graph_tool_error_problem( + error: &TraceDecayError, +) -> tracedecay_contracts::ApplicationProblem { + match error { + TraceDecayError::Config { message } => { + tracedecay_contracts::ApplicationProblem::invalid_request_without_action( + "application.surface.invalid_request", + message.clone(), + ) + } + TraceDecayError::ProjectRoute { + reason_code, + retryable, + detail, + } => graph_tool_unavailable(reason_code, *retryable, detail), + error => graph_tool_unavailable("graph_tool.failed", false, &error.to_string()), + } +} + +fn graph_tool_unavailable( + code: &str, + retryable: bool, + message: &str, +) -> tracedecay_contracts::ApplicationProblem { + let diagnostic = tracedecay_contracts::SafeDiagnostic { + code: code.to_owned(), + message: message.to_owned(), + }; + if retryable { + return tracedecay_contracts::ApplicationProblem::unavailable(diagnostic); + } + tracedecay_contracts::ApplicationProblem::Unavailable { + classification: tracedecay_contracts::ApplicationUnavailableClassV1::Authority, + diagnostic, + retry: tracedecay_contracts::RetryDirective::Never, + legal_actions: Vec::new(), + } +} + pub(super) fn render_retained_result( project_root: Option<&std::path::Path>, operation: RetainedSurfaceOperation, @@ -362,7 +655,7 @@ mod tests { use tracedecay_contracts::{CancellationSignal, Deadline, RequestId, TemporalState}; use tracedecay_domain::{CodeGenerationId, UtcMicros}; - use super::{complete_protocol_controls, served_stale_code_graph_temporal}; + use super::{complete_protocol_controls, served_code_graph_temporal}; use tracedecay_tool_catalog::ApplicationSurfaceOperation; #[test] @@ -440,13 +733,11 @@ mod tests { rebuild_in_flight: true, }, ); - let served = served_stale_code_graph_temporal( - ApplicationSurfaceOperation::CodeSymbolSearch, - &temporal, - ) - .expect("stale page metadata"); + let served = + served_code_graph_temporal(ApplicationSurfaceOperation::CodeSymbolSearch, &temporal) + .expect("stale page metadata"); let mut rendered = super::super::text_tool_result("{}"); - super::super::append_code_graph_freshness(&mut rendered, &served); + super::append_code_graph_freshness(&mut rendered, &served); let trailer = rendered .value .pointer("/content/1/text") diff --git a/crates/tracedecay/src/mcp/tools/handlers/configuration_dispatch_tests.rs b/crates/tracedecay/src/mcp/tools/handlers/configuration_dispatch_tests.rs index cd90c1d4f3..9ae83dc7ae 100644 --- a/crates/tracedecay/src/mcp/tools/handlers/configuration_dispatch_tests.rs +++ b/crates/tracedecay/src/mcp/tools/handlers/configuration_dispatch_tests.rs @@ -10,8 +10,8 @@ use tracedecay_tool_catalog::ApplicationSurfaceOperation; use super::dispatch_test_support::*; use super::*; -use crate::config::lock_user_data_dir_test_env; -use crate::project::TraceDecay; +use tracedecay_project::config::lock_user_data_dir_test_env; +use tracedecay_project::project::TraceDecay; #[derive(Default)] struct UnavailableEffectExecutor { @@ -47,7 +47,15 @@ impl tracedecay_contracts::ApplicationInvocationExecutor for UnavailableEffectEx .unwrap() .push((binding.operation().as_str().to_owned(), payload.clone())); } - Box::pin(async { Err(tracedecay_contracts::InvocationError::Unavailable) }) + Box::pin(async move { + let (context, request) = invocation.into_parts(); + let tracedecay_contracts::ApplicationRequest::Surface { binding, payload } = request + else { + return Err(tracedecay_contracts::InvocationError::Unavailable); + }; + tracedecay_daemon_protocol::invoke_application_surface(self, context, binding, payload) + .await + }) } } @@ -65,7 +73,6 @@ impl tracedecay_daemon_protocol::DaemonInvocationExecutor for UnavailableEffectE tracedecay_daemon_protocol::DaemonInvocationError, >, > { - self.invocations.fetch_add(1, Ordering::SeqCst); if let tracedecay_daemon_protocol::DaemonInvocationPayload::Configuration { surface_operation, request, @@ -264,10 +271,9 @@ async fn every_other_configuration_effect_reaches_the_authoritative_daemon_execu } let application_invocations = executor.application_surface_invocations.lock().unwrap(); - let migrated_effects = &effects[..2]; - assert_eq!(application_invocations.len(), migrated_effects.len()); + assert_eq!(application_invocations.len(), effects.len()); for ((actual_operation, request), (_, expected_operation, idempotency_key, _)) in - application_invocations.iter().zip(migrated_effects) + application_invocations.iter().zip(&effects) { assert_eq!(actual_operation, expected_operation.as_str()); assert_eq!(request["idempotency_key"], *idempotency_key); @@ -275,10 +281,9 @@ async fn every_other_configuration_effect_reaches_the_authoritative_daemon_execu drop(application_invocations); let invocations = executor.configuration_invocations.lock().unwrap(); - let daemon_effects = &effects[2..]; - assert_eq!(invocations.len(), daemon_effects.len()); + assert_eq!(invocations.len(), effects.len()); for ((actual_operation, request, policy), (_, expected_operation, idempotency_key, _)) in - invocations.iter().zip(daemon_effects) + invocations.iter().zip(&effects) { assert_eq!(actual_operation, expected_operation); assert_eq!( @@ -371,23 +376,24 @@ async fn every_configuration_read_and_preview_reaches_its_canonical_daemon_handl } let application_invocations = executor.application_surface_invocations.lock().unwrap(); - assert_eq!(application_invocations.len(), 1); - assert_eq!(application_invocations[0].0, "configuration_get"); + assert_eq!(application_invocations.len(), reads.len()); + for ((actual_operation, _), (_, expected_operation, _)) in + application_invocations.iter().zip(&reads) + { + assert_eq!(actual_operation, expected_operation.as_str()); + } assert_eq!( - application_invocations[0].1, + application_invocations[1].1, json!({"key": "mcp.tool_timings"}) ); drop(application_invocations); let invocations = executor.configuration_invocations.lock().unwrap(); - assert_eq!(invocations.len(), reads.len() - 1); - let expected_operations = reads.iter().filter_map(|(_, operation, _)| { - (*operation != ApplicationSurfaceOperation::ConfigurationGet).then_some(*operation) - }); - for ((actual_operation, request, policy), expected_operation) in - invocations.iter().zip(expected_operations) + assert_eq!(invocations.len(), reads.len()); + for ((actual_operation, request, policy), (_, expected_operation, _)) in + invocations.iter().zip(&reads) { - assert_eq!(actual_operation, &expected_operation); + assert_eq!(actual_operation, expected_operation); assert_eq!( policy, &tracedecay_daemon_protocol::InvocationCancellationPolicy::ReadOnly diff --git a/crates/tracedecay/src/mcp/tools/handlers/context_scout_control_dispatch_tests.rs b/crates/tracedecay/src/mcp/tools/handlers/context_scout_control_dispatch_tests.rs index 693e819379..99b8db2617 100644 --- a/crates/tracedecay/src/mcp/tools/handlers/context_scout_control_dispatch_tests.rs +++ b/crates/tracedecay/src/mcp/tools/handlers/context_scout_control_dispatch_tests.rs @@ -6,8 +6,8 @@ use tempfile::TempDir; use super::dispatch_test_support::*; use super::*; -use crate::config::lock_user_data_dir_test_env; -use crate::project::TraceDecay; +use tracedecay_project::config::lock_user_data_dir_test_env; +use tracedecay_project::project::TraceDecay; #[derive(Default)] struct RecordingUnavailableExecutor { @@ -23,7 +23,7 @@ struct RecordingUnavailableExecutor { impl tracedecay_contracts::ApplicationInvocationExecutor for RecordingUnavailableExecutor { fn invoke( &self, - _invocation: tracedecay_contracts::ApplicationInvocation, + invocation: tracedecay_contracts::ApplicationInvocation, ) -> tracedecay_contracts::ApplicationInvocationFuture< '_, std::result::Result< @@ -31,7 +31,15 @@ impl tracedecay_contracts::ApplicationInvocationExecutor for RecordingUnavailabl tracedecay_contracts::InvocationError, >, > { - Box::pin(async { Err(tracedecay_contracts::InvocationError::Unavailable) }) + Box::pin(async move { + let (context, request) = invocation.into_parts(); + let tracedecay_contracts::ApplicationRequest::Surface { binding, payload } = request + else { + return Err(tracedecay_contracts::InvocationError::Unavailable); + }; + tracedecay_daemon_protocol::invoke_application_surface(self, context, binding, payload) + .await + }) } } diff --git a/crates/tracedecay/src/mcp/tools/handlers/dashboard.rs b/crates/tracedecay/src/mcp/tools/handlers/dashboard.rs index b0372ecef9..971b97d9b7 100644 --- a/crates/tracedecay/src/mcp/tools/handlers/dashboard.rs +++ b/crates/tracedecay/src/mcp/tools/handlers/dashboard.rs @@ -26,9 +26,9 @@ use tracedecay_domain::configuration::{ use tracedecay_global_db::configuration::contracts::types::DirectConfigurationMutation; use tracedecay_tool_catalog::ApplicationSurfaceOperation; -use crate::project::TraceDecay; use tracedecay_domain::errors::{Result, TraceDecayError}; use tracedecay_global_db::RegisteredGlobalDbLeaseV1; +use tracedecay_project::project::TraceDecay; use tracedecay_mcp::ToolResult; use tracedecay_mcp::handlers::dashboard_lcm::DashboardLcmReadAdapter; @@ -92,7 +92,7 @@ impl DashboardProfileCodeIndexWorkerSettingsPort let database = self.database.clone(); let profile_id = self.profile_id.clone(); Box::pin(async move { - crate::config::read_or_initialize_profile_code_index_worker_configuration( + tracedecay_project::config::read_or_initialize_profile_code_index_worker_configuration( database, &profile_id, ) @@ -134,7 +134,7 @@ impl DashboardProfileCodeIndexWorkerSettingsPort }), Err(_) => { let current = - crate::config::read_or_initialize_profile_code_index_worker_configuration( + tracedecay_project::config::read_or_initialize_profile_code_index_worker_configuration( database, &profile_id, ) @@ -805,7 +805,7 @@ pub(super) async fn handle_dashboard( } })?; let retained_server = retained_server_resolver( - crate::mcp::server::RetainedProjectGraphRequest::for_mounted_root( + tracedecay_dashboard_api::project_graph::RetainedProjectGraphRequest::for_mounted_root( cg.project_root().to_path_buf(), ), ) @@ -949,7 +949,7 @@ pub(super) async fn handle_dashboard( crate::hooks::install_dashboard_hook_readiness_projection()?; // One fetch covers the served bundle and the advertised build // version; both come from the registered product runtime. - let product_runtime = crate::product_runtime::product_runtime()?; + let product_runtime = tracedecay_project::product_runtime::product_runtime()?; let state = build_state_with_automation_reconciler( retained_cg.clone(), DashboardStateCompositionV1 { @@ -983,7 +983,7 @@ pub(super) async fn handle_dashboard( let app = router( retained_cg.as_ref(), state, - crate::dashboard::spa_router(product_runtime.dashboard()), + tracedecay_api::static_dashboard_router(Arc::new(product_runtime.dashboard())), ) .await?; let (listener, addr) = bind_dashboard(&host, port).await?; diff --git a/crates/tracedecay/src/mcp/tools/handlers/dispatch_controls.rs b/crates/tracedecay/src/mcp/tools/handlers/dispatch_controls.rs index 26175c66fa..b27db10f95 100644 --- a/crates/tracedecay/src/mcp/tools/handlers/dispatch_controls.rs +++ b/crates/tracedecay/src/mcp/tools/handlers/dispatch_controls.rs @@ -3,9 +3,9 @@ use serde_json::Value; use tracedecay_contracts::{CancellationSignal, Deadline}; -use crate::project::TraceDecay; use tracedecay_domain::errors::{Result, TraceDecayError}; use tracedecay_global_db::RegisteredGlobalDbLeaseV1; +use tracedecay_project::project::TraceDecay; use super::ToolCallRegistryOptions; use tracedecay_mcp::ToolResult; diff --git a/crates/tracedecay/src/mcp/tools/handlers/dispatch_groups.rs b/crates/tracedecay/src/mcp/tools/handlers/dispatch_groups.rs index b8a8b5a704..2466028c8e 100644 --- a/crates/tracedecay/src/mcp/tools/handlers/dispatch_groups.rs +++ b/crates/tracedecay/src/mcp/tools/handlers/dispatch_groups.rs @@ -1,15 +1,11 @@ use serde_json::Value; -use tracedecay_contracts::{ - ApplicationOperation, ApplicationProblem, ResultContractRef, RetainedSurfaceOperation, -}; +use tracedecay_contracts::{ApplicationOperation, RetainedSurfaceOperation}; use tracedecay_graph_query::VerifiedGraphQueryRequest; use tracedecay_tool_catalog::{ApplicationSurfaceOperation, BindingSurface}; -use crate::project::TraceDecay; -use tracedecay_daemon_protocol::InvocationCancellationPolicy; -use tracedecay_daemon_service::application_surface::resolve_catalog_tool_binding; use tracedecay_domain::errors::{Result, TraceDecayError}; use tracedecay_global_db::RegisteredGlobalDbLeaseV1; +use tracedecay_project::project::TraceDecay; use tracedecay_contracts::code_index_freshness::CodeIndexFreshnessReader; use tracedecay_dashboard_api::AdmittedDoctorReportV1; @@ -20,6 +16,8 @@ use tracedecay_mcp::handlers::info as portable_info; use tracedecay_mcp::handlers::{ VerifiedGraphOpenFuture, unknown_tool_error, verified_read_operation, }; +use tracedecay_mcp::tools::binding::tool_dispatches_registered_project_reader; +use tracedecay_mcp::tools::dispatch_ceiling::{tool_dispatch_budget, tool_dispatch_deadline_error}; use tracedecay_mcp::{ AdmittedCodeIndex, McpAdmittedProjectV1, McpDoctorReportV1, McpProjectIdentityV1, McpRequestAuthoritiesV1, McpToolBinding, McpToolContext, RequestControls, ToolResult, @@ -35,9 +33,6 @@ use tracedecay_mcp::handlers::{ mod health_dispatch; pub(super) use health_dispatch::dispatch_health_tools; -use tracedecay_mcp::{ - retained_problem_envelope, retained_safe_diagnostic, validated_retained_response, -}; fn graph_read_unavailable(detail: &str) -> TraceDecayError { TraceDecayError::ProjectRoute { @@ -57,7 +52,7 @@ async fn admitted_graph_query( /// Lends the root's verified-graph admission funnel to a portable dispatch /// table for one tool call. The borrow of `options` is the whole lifetime of /// the table's dispatch, so every lazy open it issues reports back through -/// the same `served_stale_graph_generation` slot. +/// the same `served_code_graph` slot. fn verified_graph_open<'o>( options: &'o ToolCallRegistryOptions<'_>, ) -> impl Fn(ApplicationOperation) -> VerifiedGraphOpenFuture<'o> + Sync + 'o { @@ -101,22 +96,15 @@ async fn admitted_graph_query_for_operation( label = "mcp.dispatch.graph_query_admission" ) .await?; - if let tracedecay_graph_query::CodeGraphReadFreshnessV1::LastCompleteStale { - sealed_at, - rebuild_in_flight, - } = query.freshness() - { - // Every graph-backed tool funnels through this open, so this is the - // single point that reports serve-old-while-rebuilding back to the - // dispatch boundary for the typed response trailer. - let _ = options - .served_stale_graph_generation - .set(super::ServedStaleCodeGraphReadV1 { - generation: query.generation().as_str().to_owned(), - sealed_at, - rebuild_in_flight, - }); - } + // Every graph-backed tool funnels through this open, so this is the + // single point that reports the served generation, and a + // serve-old-while-rebuilding seat, back to the dispatch boundary. + options.served_code_graph.record( + tracedecay_contracts::retrieval::ServedCodeGraphGenerationV1 { + generation: query.generation().as_str().to_owned(), + freshness: query.freshness(), + }, + ); Ok(query) } @@ -168,7 +156,7 @@ fn dispatch_graph_tools_inner<'a>( } /// Dispatch project-info, registry, and file-inspection tools -/// (`tracedecay_status`, `tracedecay_project_list`, `tracedecay_read`, ...). +/// (`tracedecay_status`, `tracedecay_project_list`, `tracedecay_files`, ...). #[allow(clippy::too_many_arguments)] #[hotpath::measure(future = true, label = "mcp.dispatch.info")] pub(super) async fn dispatch_info_tools( @@ -255,7 +243,7 @@ fn dispatch_info_tools_inner<'a>( .await } "tracedecay_admin_sync" => { - info::handle_admin_sync(cg, args, options.code_index_reconcile_sink.as_ref()).await + info::handle_admin_sync(cg, options.code_index_reconcile_sink.as_ref()).await } _ => { portable_info::dispatch_tool( @@ -358,7 +346,7 @@ pub(super) async fn dispatch_application_surface_tools( fn dispatch_application_surface_tools_inner<'a>( tool_name: &'a str, cg: &'a TraceDecay, - args: Value, + mut args: Value, options: ToolCallRegistryOptions<'a>, ) -> std::pin::Pin> + Send + 'a>> { // Erase the deeply nested application-surface future before it reaches @@ -367,6 +355,72 @@ fn dispatch_application_surface_tools_inner<'a>( let Some(operation) = ApplicationSurfaceOperation::from_tool_name(tool_name) else { return Err(unknown_tool_error(tool_name)); }; + let retained = RetainedSurfaceOperation::from_application(operation).is_some(); + let source_edit = tracedecay_daemon_protocol::is_source_edit_operation(operation); + let graph_tool = operation.is_graph_tool(); + // An already-elapsed carried deadline is refused before these tools + // dispatch, exactly as their retained handlers always refused it. + if (retained || source_edit || graph_tool) + && tool_dispatch_budget(tool_name, options.application_deadline.as_ref()).is_none() + { + return Err(tool_dispatch_deadline_error( + tool_name, + std::time::Duration::ZERO, + )); + } + if retained { + return application_surface::run_retained_surface_tool( + Some(cg.project_root()), + BindingSurface::Mcp, + operation, + args, + options.application_invocation_executor, + options.application_request_id.clone(), + options.application_deadline.clone(), + options.application_cancellation.clone(), + ) + .await; + } + if graph_tool { + let execution = application_surface::execute_graph_tool_surface( + BindingSurface::Mcp, + operation, + args.clone(), + options.application_invocation_executor, + options.application_request_id.clone(), + options.application_deadline.clone(), + options.application_cancellation.clone(), + ) + .await?; + return tracedecay_mcp::handlers::graph_tool::render_graph_tool( + Some(cg.project_root()), + &args, + execution, + ); + } + if source_edit { + return edit::source_edit_tool( + Some(cg.project_root()), + BindingSurface::Mcp, + operation, + args, + edit::SourceEditInvocationContext { + executor: options.application_invocation_executor, + target: options.application_invocation_target, + request_id: options.application_request_id.clone(), + deadline: options.application_deadline.clone(), + cancellation: options.application_cancellation.clone(), + }, + ) + .await; + } + // Routing resolved the registered-project selector into the invocation + // target before dispatch; the canonical request schema does not carry it. + if tool_dispatches_registered_project_reader(tool_name) + && let Some(arguments) = args.as_object_mut() + { + arguments.remove("project_selector"); + } let normalized_args = match tracedecay_daemon_protocol::adapt_application_tool_request(tool_name, args) { Ok(args) => args, @@ -392,6 +446,53 @@ fn dispatch_application_surface_tools_inner<'a>( }) } +/// Computes one graph-tool operation for the project's graph-tool owner, +/// under the same admitted authorities and dispatch ceiling as every other +/// graph read. +pub(crate) fn compute_graph_tool_for_owner<'a>( + cg: &'a TraceDecay, + operation: ApplicationSurfaceOperation, + args: Value, + scope_prefix: Option<&'a str>, + options: ToolCallRegistryOptions<'a>, +) -> std::pin::Pin< + Box< + dyn std::future::Future< + Output = Result, + > + Send + + 'a, + >, +> { + Box::pin(async move { + let tool_name = operation.mcp_tool_name(); + let Some(budget) = tool_dispatch_budget(tool_name, options.application_deadline.as_ref()) + else { + return Err(tool_dispatch_deadline_error( + tool_name, + std::time::Duration::ZERO, + )); + }; + let project = admitted_project_authorities(cg, &options)?; + let snapshots = AdmittedRequestSnapshotsV1::default(); + let freshness = graph_freshness_reader(tool_name, &options); + let ctx = admitted_tool_context(&options, &project, &snapshots, freshness)?; + let open = verified_graph_open(&options); + let computed = tracedecay_mcp::handlers::graph_tool::compute_graph_tool( + &ctx, + &open, + operation, + args, + scope_prefix, + ); + let mut completion = match tokio::time::timeout(budget, computed).await { + Ok(result) => result?, + Err(_elapsed) => return Err(tool_dispatch_deadline_error(tool_name, budget)), + }; + completion.code_graph = options.served_code_graph.served(); + Ok(completion) + }) +} + /// Dispatch static-analysis report tools such as `tracedecay_dead_code` and /// `tracedecay_complexity`. #[hotpath::measure(future = true, label = "mcp.dispatch.analysis")] @@ -648,225 +749,6 @@ fn admitted_tool_context<'a>( Ok(McpToolContext::bind(McpToolBinding { project, request })?) } -/// Dispatch source-editing tools (`tracedecay_str_replace`, -/// `tracedecay_move_symbol`, ...). -#[hotpath::measure(future = true, label = "mcp.dispatch.edit")] -pub(super) async fn dispatch_edit_tools( - tool_name: &str, - cg: &TraceDecay, - args: Value, - options: ToolCallRegistryOptions<'_>, -) -> Result { - dispatch_edit_tools_inner(tool_name, cg, args, options).await -} - -fn dispatch_edit_tools_inner<'a>( - tool_name: &'a str, - cg: &'a TraceDecay, - args: Value, - options: ToolCallRegistryOptions<'a>, -) -> std::pin::Pin> + Send + 'a>> { - // Erase the deeply nested match-arm futures before they reach the - // measured wrapper so every profiling feature can compute its layout. - Box::pin(async move { - let invocation = edit::SourceEditInvocationContext { - executor: options.application_invocation_executor, - request_id: options.application_request_id.clone(), - deadline: options.application_deadline.clone(), - cancellation: options.application_cancellation.clone(), - }; - match tool_name { - "tracedecay_str_replace" => { - edit::handle_str_replace(cg, args, invocation.clone()).await - } - "tracedecay_multi_str_replace" => { - edit::handle_multi_str_replace(cg, args, invocation.clone()).await - } - "tracedecay_insert_at" => edit::handle_insert_at(cg, args, invocation.clone()).await, - "tracedecay_ast_grep_rewrite" => { - edit::handle_ast_grep_rewrite(cg, args, invocation.clone()).await - } - "tracedecay_replace_symbol" => { - edit::handle_replace_symbol(cg, args, invocation.clone()).await - } - "tracedecay_insert_at_symbol" => { - edit::handle_insert_at_symbol(cg, args, invocation.clone()).await - } - "tracedecay_move_symbol" => { - edit::handle_move_symbol(cg, args, invocation.clone()).await - } - "tracedecay_rename_symbol" => { - edit::handle_rename_symbol(cg, args, invocation.clone()).await - } - "tracedecay_source_edit_rollback" => { - edit::handle_source_edit_rollback(cg, args, invocation.clone()).await - } - "tracedecay_source_edit_reconcile" => { - edit::handle_source_edit_reconcile(cg, args, invocation).await - } - _ => Err(unknown_tool_error(tool_name)), - } - }) -} - -/// Dispatch retained memory, session, and workflow operations only after the -/// application-owned catalog has resolved their stable operation identity. -#[hotpath::measure(future = true, label = "mcp.dispatch.retained_application")] -pub(super) async fn dispatch_retained_application_tools( - tool_name: &str, - cg: &TraceDecay, - args: Value, - _scope_prefix: Option<&str>, - _active_project_session_db: Option<&RegisteredGlobalDbLeaseV1>, - options: ToolCallRegistryOptions<'_>, -) -> Result { - dispatch_retained_application_tools_inner( - tool_name, - cg, - args, - _scope_prefix, - _active_project_session_db, - options, - ) - .await -} - -#[expect( - clippy::too_many_lines, - reason = "Retained-application dispatch is one name match onto the application surface." -)] -fn dispatch_retained_application_tools_inner<'a>( - tool_name: &'a str, - cg: &'a TraceDecay, - args: Value, - _scope_prefix: Option<&'a str>, - _active_project_session_db: Option<&'a RegisteredGlobalDbLeaseV1>, - options: ToolCallRegistryOptions<'a>, -) -> std::pin::Pin> + Send + 'a>> { - // Erase the deeply nested retained-application future before it reaches - // the measured wrapper so every profiling feature can compute its layout. - Box::pin(async move { - let retained_operation = RetainedSurfaceOperation::from_tool_name(tool_name) - .ok_or_else(|| unknown_tool_error(tool_name))?; - let binding = resolve_catalog_tool_binding(BindingSurface::Mcp, tool_name) - .map_err(|error| TraceDecayError::Config { - message: error.to_string(), - })? - .ok_or_else(|| unknown_tool_error(tool_name))?; - let normalized = tracedecay_daemon_protocol::separate_application_tool_request(args) - .map_err(|error| TraceDecayError::Config { - message: error.to_string(), - })?; - let requested_format = normalized.requested_format; - let request = hotpath::measure_block!( - "mcp.retained.decode", - tracedecay_daemon_service::application_surface::retained::decode_request( - retained_operation, - normalized.request, - ) - ) - .map_err(|error| TraceDecayError::Config { - message: format!("invalid retained application request for {tool_name}: {error}"), - })?; - if request.operation() != retained_operation { - return Err(TraceDecayError::Config { - message: format!("retained application request does not match {tool_name}"), - }); - } - let request_id = match options.application_request_id { - Some(request_id) => request_id, - None => application_surface::request_id()?, - }; - let result_contract = ResultContractRef::from_schema(&binding.result_schema); - let result = match options.application_invocation_executor { - Some(executor) => { - let (deadline, cancellation) = - application_surface::complete_retained_protocol_controls( - retained_operation, - &request_id, - options.application_deadline, - options.application_cancellation, - )? - .ok_or_else(|| { - TraceDecayError::project_route( - "retained_application_controls_unavailable", - true, - "retained application protocol controls are unavailable", - ) - })?; - let invocation = - tracedecay_daemon_protocol::DaemonInvocationRequest::retained_application( - request_id.as_str(), - request, - tracedecay_contracts::now_micros(), - deadline.clone(), - cancellation.context(), - ); - let policy = - if tracedecay_contracts::retained_surfaces::retained_surface_operation_is_effect( - retained_operation, - ) { - InvocationCancellationPolicy::AuthoritativeEffect - } else { - InvocationCancellationPolicy::ReadOnly - }; - match hotpath::future!( - executor.invoke_controlled(invocation, deadline, cancellation, policy), - label = "mcp.retained.invoke" - ) - .await - { - Ok(response) - if response.protocol - == tracedecay_daemon_protocol::DAEMON_INVOCATION_PROTOCOL - && response.revision - == tracedecay_daemon_protocol::DAEMON_INVOCATION_REVISION - && response.request_id == request_id.as_str() => - { - validated_retained_response( - response.outcome, - retained_operation, - &request_id, - &result_contract, - )? - } - Ok(_) => Err(retained_problem_envelope( - result_contract.clone(), - request_id.clone(), - ApplicationProblem::unavailable(retained_safe_diagnostic( - "application.surface.invalid_response", - "The daemon returned an invalid retained application envelope", - )?), - )?), - Err(error) => Err(retained_problem_envelope( - result_contract.clone(), - request_id.clone(), - error.into_application_problem(), - )?), - } - } - None => Err(retained_problem_envelope( - result_contract, - request_id, - ApplicationProblem::unavailable(retained_safe_diagnostic( - "application.transport.unavailable", - "The daemon retained application transport is unavailable", - )?), - )?), - }; - hotpath::measure_block!( - "mcp.retained.render", - application_surface::render_retained_result( - Some(cg.project_root()), - retained_operation, - &binding.binding_id, - result, - requested_format, - ) - ) - }) -} - /// Dispatch memory, skill, and analytics tools (`tracedecay_fact_store_add`, /// `tracedecay_skill_list`, `tracedecay_analytics`, ...). #[hotpath::measure(future = true, label = "mcp.dispatch.memory")] diff --git a/crates/tracedecay/src/mcp/tools/handlers/dispatch_groups/health_dispatch.rs b/crates/tracedecay/src/mcp/tools/handlers/dispatch_groups/health_dispatch.rs index 53580e9006..43a51854e6 100644 --- a/crates/tracedecay/src/mcp/tools/handlers/dispatch_groups/health_dispatch.rs +++ b/crates/tracedecay/src/mcp/tools/handlers/dispatch_groups/health_dispatch.rs @@ -2,9 +2,9 @@ use serde_json::Value; -use crate::project::TraceDecay; use tracedecay_domain::errors::Result; use tracedecay_global_db::RegisteredGlobalDbLeaseV1; +use tracedecay_project::project::TraceDecay; use super::super::ToolCallRegistryOptions; use super::verified_graph_open; @@ -38,7 +38,7 @@ pub(in crate::mcp::tools::handlers) async fn dispatch_health_tools( &ctx, args, options.global_db.map(RegisteredGlobalDbLeaseV1::as_ref), - crate::version::build_version()?, + tracedecay_project::version::build_version()?, ) .await } diff --git a/crates/tracedecay/src/mcp/tools/handlers/dispatch_test_support.rs b/crates/tracedecay/src/mcp/tools/handlers/dispatch_test_support.rs index 56ad2cc904..a7cdca16b0 100644 --- a/crates/tracedecay/src/mcp/tools/handlers/dispatch_test_support.rs +++ b/crates/tracedecay/src/mcp/tools/handlers/dispatch_test_support.rs @@ -3,7 +3,7 @@ use std::path::Path; use std::sync::Arc; use super::*; -use crate::config::USER_DATA_DIR_ENV; +use tracedecay_project::config::USER_DATA_DIR_ENV; #[derive(Clone)] struct FixtureCodeGraphProjection { @@ -276,12 +276,12 @@ pub(super) fn verified_graph_error_options<'a>( /// runtime's daemon session registry instead of constructing another runtime /// on the same profile. pub(super) async fn init_sibling_registered_fixture( - runtime: &crate::test_support::host_admission::HostAdmissionTestRuntimeV1, + runtime: &tracedecay_project::test_support::host_admission::HostAdmissionTestRuntimeV1, project_root: &Path, project_id: &str, ) -> ( TraceDecay, - Arc, + Arc, ) { let profile_root = tracedecay_runtime_core::storage::default_profile_root().expect("sibling profile root"); @@ -296,7 +296,7 @@ pub(super) async fn init_sibling_registered_fixture( let graph = sibling .initialize_project_graph_for_test( project_root, - crate::project::TraceDecayOpenOptions { + tracedecay_project::project::TraceDecayOpenOptions { profile_root: Some(profile_root), global_db_path: None, }, @@ -350,7 +350,6 @@ pub(super) async fn concrete_dispatch_group_accepts( }; match group { McpToolDispatchGroup::ApplicationSurface - | McpToolDispatchGroup::RetainedApplication | McpToolDispatchGroup::Work | McpToolDispatchGroup::Workflow => false, McpToolDispatchGroup::MultiRoot => { @@ -371,9 +370,6 @@ pub(super) async fn concrete_dispatch_group_accepts( McpToolDispatchGroup::Git => { owned(dispatch_git_tools(tool_name, cg, invalid_args, options).await) } - McpToolDispatchGroup::Edit => { - owned(dispatch_edit_tools(tool_name, cg, invalid_args, options).await) - } McpToolDispatchGroup::Health => { owned(dispatch_health_tools(tool_name, cg, invalid_args, None, None, options).await) } diff --git a/crates/tracedecay/src/mcp/tools/handlers/dispatch_tests.rs b/crates/tracedecay/src/mcp/tools/handlers/dispatch_tests.rs index 219efd1d46..0bd4a7e73d 100644 --- a/crates/tracedecay/src/mcp/tools/handlers/dispatch_tests.rs +++ b/crates/tracedecay/src/mcp/tools/handlers/dispatch_tests.rs @@ -9,7 +9,7 @@ use tempfile::TempDir; use super::super::get_tool_definitions; use super::dispatch_test_support::*; use super::*; -use crate::config::lock_user_data_dir_test_env; +use tracedecay_project::config::lock_user_data_dir_test_env; /// Records the daemon operation every multi-root tool routes to, then refuses /// it. The refusal is the point: it proves the MCP name reached the closed @@ -266,7 +266,7 @@ fn git_dispatch_family_is_visible_to_the_server_horizon() { "{tool_name} dispatches through the git family", ); } - assert!(!tool_dispatches_git_reads("tracedecay_outline")); + assert!(!tool_dispatches_git_reads("tracedecay_files")); assert!(!tool_dispatches_git_reads("tracedecay_diagnostics")); } @@ -329,44 +329,6 @@ async fn advertised_tools_resolve_one_concrete_dispatch_entry() { "{} has no canonical Workflow operation entry", definition.name ), - McpToolDispatchGroup::RetainedApplication => { - let composition = retained_mcp_composition().unwrap_or_else(|error| { - panic!("{} catalog composition failed: {error}", definition.name) - }); - let profile = ProfileId::new(APPLICATION_DEFAULT_PROFILE_ID).unwrap(); - let operation = RetainedSurfaceOperation::from_tool_name(&definition.name) - .unwrap_or_else(|| { - panic!("{} has no retained-surface handler entry", definition.name) - }); - let operation_name = SurfaceOperationName::new(operation.as_str()).unwrap(); - let capability = composition - .snapshot() - .resolve_binding( - &profile, - BindingSurface::Mcp, - &operation_name, - 1, - &BTreeSet::new(), - ) - .unwrap_or_else(|| { - panic!( - "{} action {} catalog binding is not callable", - definition.name, - operation.as_str() - ) - }); - let expected = retained_surface_application_operation(operation).unwrap(); - assert_eq!(capability.capability_id(), expected.capability_id()); - assert_eq!(capability.use_case_id(), expected.use_case_id()); - assert!( - composition - .bind_handler(capability.use_case_id(), &()) - .is_some(), - "{} action {} application handler is not registered", - definition.name, - operation.as_str() - ); - } group => { assert_eq!( dispatch_group_for_tool(&definition.name), @@ -417,7 +379,7 @@ async fn advertised_tools_resolve_one_concrete_dispatch_entry() { } #[test] -fn graph_reader_selector_dispatch_policy_is_allowlisted() { +fn registered_project_selector_dispatch_policy_matches_tool_schemas() { for tool in get_tool_definitions().expect("tool definitions") { let properties = &tool.input_schema["properties"]; let schema_has_registered_project_selector = properties.get("project_selector").is_some(); @@ -456,37 +418,6 @@ fn graph_reader_selector_dispatch_policy_is_allowlisted() { ); } } - - for tool_name in [ - // `tracedecay_search` resolves a daemon-owned code-index search - // authority that is bound to the active project, so a selector - // would run the active authority against a different graph. - "tracedecay_search", - "tracedecay_str_replace", - "tracedecay_run_affected_tests", - "tracedecay_status", - "tracedecay_health", - "tracedecay_dead_code", - ] { - assert!( - !tool_accepts_registered_project_selector(tool_name), - "{tool_name} should not be routed by the pure graph-reader selector policy" - ); - } - - // Pure graph reads that need nothing but the selected project's graph - // must accept a selector. - for tool_name in [ - "tracedecay_type_hierarchy", - "tracedecay_outline", - "tracedecay_read", - "tracedecay_body", - ] { - assert!( - tool_accepts_registered_project_selector(tool_name), - "{tool_name} should route through the graph-reader selector policy" - ); - } } #[tokio::test] @@ -651,7 +582,10 @@ async fn status_serving_branch_reports_the_lane_serving_truth() { let meta = tracedecay_runtime_core::branch_meta::BranchMeta::new("main"); tracedecay_runtime_core::branch_meta::save_branch_meta(&layout.data_root, &meta).unwrap(); let cg = runtime - .open_project_graph_for_test(&project, crate::project::TraceDecayOpenOptions::default()) + .open_project_graph_for_test( + &project, + tracedecay_project::project::TraceDecayOpenOptions::default(), + ) .await .unwrap(); assert_eq!( @@ -871,11 +805,7 @@ async fn status_serving_branch_reports_the_lane_serving_truth() { // tracking ref rather than the user-visible branch name. let mut branch_meta = tracedecay_runtime_core::branch_meta::load_branch_meta(&layout.data_root) .expect("main branch metadata"); - branch_meta.add_branch( - "feature", - tracedecay_runtime_core::config::DB_FILENAME, - "main", - ); + branch_meta.add_branch("feature", "main"); tracedecay_runtime_core::branch_meta::save_branch_meta(&layout.data_root, &branch_meta) .unwrap(); run_git_in(&project, &["checkout", "-b", "feature"]); @@ -1199,16 +1129,20 @@ async fn selected_project_retrieve_finds_selected_project_response_handle() { let target_server = crate::mcp::McpServer::new_with_host_admission_test_runtime_for_test( target, None, - crate::test_support::host_admission::ProjectScopedTestRuntimeV1::new(target_runtime) - .expect("target project-scoped runtime"), + tracedecay_project::test_support::host_admission::ProjectScopedTestRuntimeV1::new( + target_runtime, + ) + .expect("target project-scoped runtime"), ) .await .expect("target retained server"); let server = crate::mcp::McpServer::new_with_retained_test_servers_for_test( active, None, - crate::test_support::host_admission::ProjectScopedTestRuntimeV1::new(active_runtime) - .expect("active project-scoped runtime"), + tracedecay_project::test_support::host_admission::ProjectScopedTestRuntimeV1::new( + active_runtime, + ) + .expect("active project-scoped runtime"), vec![target_server], ) .await @@ -1633,16 +1567,14 @@ async fn graph_tools_reject_blank_node_ids_and_zero_depth_with_typed_errors() { TraceDecay::init_test_fixture_with_registered_runtime(&project, "project.blank-node-id") .await .unwrap(); - for tool_name in [ - "tracedecay_impact", - "tracedecay_callers", - "tracedecay_callees", - "tracedecay_node", + for (tool_name, operation) in [ + ("tracedecay_impact", ApplicationSurfaceOperation::Impact), + ("tracedecay_node", ApplicationSurfaceOperation::Node), ] { for blank in ["", " "] { - let error = dispatch_graph_tools( - tool_name, + let error = super::compute_graph_tool_for_owner( &cg, + operation, json!({"node_id": blank}), None, verified_graph_options(&cg, ToolCallRegistryOptions::default()), @@ -1660,14 +1592,10 @@ async fn graph_tools_reject_blank_node_ids_and_zero_depth_with_typed_errors() { // Handlers clamp depth with `min(max)`, which leaves an explicit zero // intact, so a valid node id still reaches the guard from this side. let node_id = "symbol.blank-probe"; - for tool_name in [ - "tracedecay_impact", - "tracedecay_callers", - "tracedecay_callees", - ] { - let error = dispatch_graph_tools( - tool_name, + for (tool_name, operation) in [("tracedecay_impact", ApplicationSurfaceOperation::Impact)] { + let error = super::compute_graph_tool_for_owner( &cg, + operation, json!({"node_id": node_id, "max_depth": 0}), None, verified_graph_options(&cg, ToolCallRegistryOptions::default()), @@ -1942,12 +1870,93 @@ async fn a_stale_served_graph_read_carries_the_typed_freshness_trailer() { cg.close(); } +/// The graph-tool owner reports the generation it served on the completion, +/// so the envelope carries the stale seat and every surface renders the same +/// trailer from it. +#[tokio::test] +async fn graph_tool_owner_reports_the_served_generation_for_the_trailer() { + let _env_lock = lock_user_data_dir_test_env(); + let dir = TempDir::new().unwrap(); + let _env = SelectorEnv::new(dir.path()); + let project = dir.path().join("graph-tool-trailer"); + fs::create_dir_all(project.join("src")).unwrap(); + fs::write(project.join("src/lib.rs"), "pub fn probe() {}\n").unwrap(); + let (cg, _runtime) = TraceDecay::init_test_fixture_with_registered_runtime( + &project, + "project.graph-tool-trailer", + ) + .await + .unwrap(); + + let stale = super::compute_graph_tool_for_owner( + &cg, + ApplicationSurfaceOperation::Todos, + json!({}), + None, + verified_graph_stale_options(&cg, ToolCallRegistryOptions::default()), + ) + .await + .expect("a stale-served graph tool still answers"); + let served = stale.code_graph.clone().expect("served generation"); + assert_eq!(served.generation, "generation.mcp-verified-graph-fixture.1"); + assert!(served.freshness.is_stale()); + let rendered = tracedecay_mcp::handlers::graph_tool::render_graph_tool( + Some(cg.project_root()), + &json!({}), + stale, + ) + .unwrap(); + let rendered = serde_json::to_string(&rendered.value).unwrap(); + assert!( + rendered.contains( + "code_graph_freshness: stale, serving the last complete generation \ + generation.mcp-verified-graph-fixture.1 (sealed 1m ago) while the code index rebuilds" + ), + "{rendered}" + ); + + let current = super::compute_graph_tool_for_owner( + &cg, + ApplicationSurfaceOperation::Todos, + json!({}), + None, + verified_graph_options(&cg, ToolCallRegistryOptions::default()), + ) + .await + .expect("a current graph tool answers"); + assert!( + !current + .code_graph + .as_ref() + .expect("served generation") + .freshness + .is_stale() + ); + let rendered = tracedecay_mcp::handlers::graph_tool::render_graph_tool( + Some(cg.project_root()), + &json!({}), + current, + ) + .unwrap(); + assert!( + !serde_json::to_string(&rendered.value) + .unwrap() + .contains("code_graph_freshness") + ); + + cg.close(); +} + #[test] -fn unavailable_effect_contract_fails_before_handler_dispatch() { - assert!(super::ensure_mcp_dispatch_available("tracedecay_lcm_doctor").is_ok()); - assert!(super::ensure_mcp_dispatch_available("tracedecay_lcm_compress").is_err()); - assert!(super::ensure_mcp_dispatch_available("tracedecay_dashboard").is_ok()); - assert!(super::ensure_mcp_dispatch_available("tracedecay_search").is_ok()); +fn uncataloged_tool_fails_before_handler_dispatch() { + let error = super::ensure_mcp_dispatch_available("tracedecay_lcm_compress").unwrap_err(); + let TraceDecayError::Config { message } = error else { + panic!("a tool with no dispatch contract must be a typed Config error: {error:?}"); + }; + assert_eq!( + message, + "advertised MCP tool 'tracedecay_lcm_compress' has no dispatch contract" + ); } #[tokio::test] @@ -2262,7 +2271,7 @@ async fn admin_sync_reports_terminal_publication_corruption_without_queueing() { let reconcile_sink: crate::mcp::server::CodeIndexReconcileSink = std::sync::Arc::new( move |_, _| { Box::pin(async move { - crate::mcp::server::CodeIndexDemandAdmissionV1::Terminal( + tracedecay_code_index_runtime::code_index_scheduler::CodeIndexDemandAdmissionV1::Terminal( tracedecay_contracts::code_index_freshness::CodeIndexConvergenceParkedV1 { reason: "the publication authority is corrupt and requires an index reset: injected sync refusal".to_owned(), blocked_reason: Some( diff --git a/crates/tracedecay/src/mcp/tools/handlers/graph_search_dispatch_tests.rs b/crates/tracedecay/src/mcp/tools/handlers/graph_search_dispatch_tests.rs index 0d20b4c8fa..2ee087d40a 100644 --- a/crates/tracedecay/src/mcp/tools/handlers/graph_search_dispatch_tests.rs +++ b/crates/tracedecay/src/mcp/tools/handlers/graph_search_dispatch_tests.rs @@ -9,7 +9,7 @@ use tracedecay_domain::ExactClass; use tracedecay_mcp::ToolResult; use tracedecay_query::retrieval::lexical::LexicalRoutingV1; -use crate::project::TraceDecay; +use tracedecay_project::project::TraceDecay; fn completed_sparse_search() -> tracedecay_query::code_search::CodeIndexSearchOutcomeV1 { completed_sparse_search_for_generation("generation.mcp-verified-graph-fixture.1") @@ -112,7 +112,7 @@ fn search_test_options<'a>( } fn run_with_locked_user_data_dir(test: impl Future) { - let _env_lock = crate::config::lock_user_data_dir_test_env(); + let _env_lock = tracedecay_project::config::lock_user_data_dir_test_env(); tokio::runtime::Builder::new_current_thread() .enable_all() .build() diff --git a/crates/tracedecay/src/mcp/tools/handlers/hook_runtime_behavior_tests.rs b/crates/tracedecay/src/mcp/tools/handlers/hook_runtime_behavior_tests.rs index 2c79c6714f..912510eaea 100644 --- a/crates/tracedecay/src/mcp/tools/handlers/hook_runtime_behavior_tests.rs +++ b/crates/tracedecay/src/mcp/tools/handlers/hook_runtime_behavior_tests.rs @@ -13,9 +13,9 @@ use tempfile::TempDir; use tracedecay_mcp::{JsonRpcRequest, JsonRpcResponse}; use super::dispatch_test_support::SelectorEnv; -use crate::config::lock_user_data_dir_test_env; use crate::mcp::McpServer; -use crate::project::TraceDecay; +use tracedecay_project::config::lock_user_data_dir_test_env; +use tracedecay_project::project::TraceDecay; async fn open_server() -> (TempDir, SelectorEnv, Arc) { let dir = TempDir::new().expect("temp dir"); @@ -31,8 +31,9 @@ async fn open_server() -> (TempDir, SelectorEnv, Arc) { .expect("enrolled project"); cg.add_local_counter(41).await.expect("seed local counter"); cg.set_tokens_saved(12).await.expect("seed saved tokens"); - let scoped = crate::test_support::host_admission::ProjectScopedTestRuntimeV1::new(runtime) - .expect("project-scoped runtime"); + let scoped = + tracedecay_project::test_support::host_admission::ProjectScopedTestRuntimeV1::new(runtime) + .expect("project-scoped runtime"); let server = McpServer::new_with_host_admission_test_runtime_for_test(cg, None, scoped) .await .expect("project MCP server"); diff --git a/crates/tracedecay/src/mcp/tools/handlers/info/mod.rs b/crates/tracedecay/src/mcp/tools/handlers/info/mod.rs index 5b89c5ea42..52de249752 100644 --- a/crates/tracedecay/src/mcp/tools/handlers/info/mod.rs +++ b/crates/tracedecay/src/mcp/tools/handlers/info/mod.rs @@ -16,28 +16,8 @@ mod status; pub(super) use status::handle_admin_sync; -/// Snapshot + serialize the generation census for the `tracedecay://status` -/// resource. Handler families read the already-computed snapshot from -/// [`tracedecay_mcp::McpToolContext`]; this wrapper keeps the resource on -/// the composition-root reader until that surface moves. -pub(crate) async fn graph_statistics_value( - generation_census_reader: Option< - &tracedecay_runtime_core::runtime_telemetry::GenerationCensusReader, - >, -) -> tracedecay_domain::errors::Result { - let census = match generation_census_reader { - Some(reader) => reader().await, - None => { - tracedecay_runtime_core::runtime_telemetry::GenerationCensusSnapshot::Unavailable { - reason: tracedecay_runtime_core::runtime_telemetry::GenerationCensusUnavailableReason::AuthorityUnavailable, - } - } - }; - tracedecay_mcp::handlers::info::graph_statistics_value(Some(&census)) -} +pub(super) use serde_json::json; -pub(super) use serde_json::{Value, json}; - -pub(super) use crate::project::TraceDecay; pub(super) use tracedecay_domain::errors::{Result, TraceDecayError}; pub(super) use tracedecay_mcp::ToolResult; +pub(super) use tracedecay_project::project::TraceDecay; diff --git a/crates/tracedecay/src/mcp/tools/handlers/info/remote_status_dispatch_tests.rs b/crates/tracedecay/src/mcp/tools/handlers/info/remote_status_dispatch_tests.rs index 889282a017..9e96200566 100644 --- a/crates/tracedecay/src/mcp/tools/handlers/info/remote_status_dispatch_tests.rs +++ b/crates/tracedecay/src/mcp/tools/handlers/info/remote_status_dispatch_tests.rs @@ -14,13 +14,13 @@ use tracedecay_contracts::remote::status::{ use tracedecay_contracts::{DoctorCoverageCompletenessV1, RemoteListenerReadV1}; use tracedecay_domain::{CurrentRemoteAuthorityStateV1, UtcMicros}; -use crate::config::lock_user_data_dir_test_env; use crate::mcp::tools::handlers::dispatch_test_support::SelectorEnv; use crate::mcp::tools::handlers::{ ToolCallRegistryOptions, handle_tool_call_with_registry_options, }; -use crate::project::TraceDecay; use tracedecay_mcp::ToolResult; +use tracedecay_project::config::lock_user_data_dir_test_env; +use tracedecay_project::project::TraceDecay; fn available_authority() -> CurrentRemoteAuthorityStateV1 { serde_json::from_value(json!({ diff --git a/crates/tracedecay/src/mcp/tools/handlers/info/status.rs b/crates/tracedecay/src/mcp/tools/handlers/info/status.rs index e2e5ade791..2b2d11d263 100644 --- a/crates/tracedecay/src/mcp/tools/handlers/info/status.rs +++ b/crates/tracedecay/src/mcp/tools/handlers/info/status.rs @@ -11,10 +11,8 @@ use tracedecay_code_index_runtime::code_index_scheduler::{ #[hotpath::measure(label = "mcp.info.admin_sync.total")] pub(crate) async fn handle_admin_sync( cg: &TraceDecay, - args: Value, reconcile_sink: Option<&crate::mcp::server::CodeIndexReconcileSink>, ) -> Result { - let force = args.get("force").and_then(Value::as_bool).unwrap_or(false); let project_root = cg.project_root().to_path_buf(); let reconcile_sink = reconcile_sink.ok_or_else(|| { TraceDecayError::project_route( @@ -44,7 +42,6 @@ pub(crate) async fn handle_admin_sync( } }; let output = json!({ - "requested_mode": if force { "force" } else { "refresh" }, "reconcile_scope": "authoritative_project", "status": status, "project_root": cg.project_root(), diff --git a/crates/tracedecay/src/mcp/tools/handlers/mod.rs b/crates/tracedecay/src/mcp/tools/handlers/mod.rs index 9c9c1c276e..4980c32fd5 100644 --- a/crates/tracedecay/src/mcp/tools/handlers/mod.rs +++ b/crates/tracedecay/src/mcp/tools/handlers/mod.rs @@ -5,6 +5,12 @@ //! formats the result. mod application_surface; +pub(crate) use application_surface::graph_tool_error_problem; +pub use application_surface::{ + RetainedSurfaceExecution, execute_graph_tool_surface, execute_retained_surface_tool, + render_retained_execution, +}; +pub(crate) use dispatch_groups::compute_graph_tool_for_owner; #[cfg(test)] #[allow( clippy::unwrap_used, @@ -153,26 +159,17 @@ use std::sync::Arc; pub(crate) use tool_call_support::resolve_registered_project_route_for_tool; pub(super) use tool_call_support::text_tool_result; -use serde_json::{Value, json}; +use serde_json::Value; use tracedecay_contracts::RetainedSurfaceOperation; -#[cfg(test)] -use tracedecay_contracts::{ - APPLICATION_DEFAULT_PROFILE_ID, retained_surface_application_operation, -}; +use tracedecay_contracts::retrieval::ServedCodeGraphGenerationV1; use tracedecay_tool_catalog::{ApplicationSurfaceOperation, BindingSurface}; -#[cfg(test)] -use tracedecay_tool_catalog::{ProfileId, SurfaceOperationName}; use super::LegacyToolCompatibilityOwner; -use crate::project::TraceDecay; use dispatch_groups::{ dispatch_admin_tools, dispatch_analysis_tools, dispatch_application_surface_tools, - dispatch_edit_tools, dispatch_git_tools, dispatch_graph_tools, dispatch_health_tools, - dispatch_info_tools, dispatch_memory_tools, dispatch_retained_application_tools, - dispatch_session_workflow_tools, + dispatch_git_tools, dispatch_graph_tools, dispatch_health_tools, dispatch_info_tools, + dispatch_memory_tools, dispatch_session_workflow_tools, }; -#[cfg(test)] -use retained_catalog::retained_mcp_composition; use retained_catalog::{ dispatch_profile_retained_application_tool, session_refresh_profile_scope_requested, }; @@ -191,7 +188,9 @@ use tracedecay_mcp::tools::binding::{ tool_dispatches_registered_project_reader, tool_requires_canonical_effect_settlement, }; use tracedecay_mcp::tools::dispatch_ceiling::{tool_dispatch_budget, tool_dispatch_deadline_error}; +use tracedecay_mcp::tools::response_trailers::append_code_graph_freshness; use tracedecay_mcp::{handle_multi_root, handle_work, handle_workflow}; +use tracedecay_project::project::TraceDecay; use tracedecay_runtime_core::storage::registered_project_id; /// Dispatches a tool call to the appropriate handler. @@ -258,19 +257,32 @@ pub(crate) fn opened_project_scope(cg: &TraceDecay) -> Result

  • No matches
    • +
  • No matches
    • ) : ( filtered.map((entry, i) => (
  • {entry.label} - + {entry.hint} {i === activeIndex ? : null} diff --git a/dashboard/src/app/shell/NavRail.tsx b/dashboard/src/app/shell/NavRail.tsx index 3825e3c942..7c7b8db93d 100644 --- a/dashboard/src/app/shell/NavRail.tsx +++ b/dashboard/src/app/shell/NavRail.tsx @@ -163,7 +163,7 @@ function RailLink({ {channelNumber(path)} - {label} + {label} {health ? : null} )} @@ -210,7 +210,7 @@ function useDoctorHealth(): DoctorHealth { const findings = useStorageFindings(); const result = findings.data; if (!result || result.outcome === 'transport') return 'unknown'; - const statuses = result.envelope.payload.kind_statuses; + const statuses = result.envelope.payload.storage_kind_statuses; // A report naming no producers has established nothing about this store. if (statuses.length === 0) return 'unknown'; const readings = statuses.map(kindHealth); diff --git a/dashboard/src/app/shell/ScopeBar.tsx b/dashboard/src/app/shell/ScopeBar.tsx index ed876feec0..b6ec111114 100644 --- a/dashboard/src/app/shell/ScopeBar.tsx +++ b/dashboard/src/app/shell/ScopeBar.tsx @@ -67,11 +67,13 @@ export function ScopeBar({ // Pinned to exactly 52 the bar could not take them, and a clip would cut // off the project name and its `unverified`/`not in registry` caveat at // precisely the zoom level someone would be using to read them. -
    + // Below sm the scope and channel cannot share one row with the controls, + // so the register stacks on every page: scope first, controls beneath. +
    {/* `min-w-0` without `overflow-hidden`: the horizontal containment comes * from `truncate` on the label itself, which shortens the name and * leaves the caveat beside it readable. */} -
    +
    {scope.kind === 'project' ? (
    ); } diff --git a/dashboard/src/ui/EvidencePattern.tsx b/dashboard/src/ui/EvidencePattern.tsx index de0b16083c..e8fa7e0363 100644 --- a/dashboard/src/ui/EvidencePattern.tsx +++ b/dashboard/src/ui/EvidencePattern.tsx @@ -20,10 +20,10 @@ export function EvidencePattern({ className?: string; }) { return ( - + ) : ( - {scoreKind} + {scoreKind} ) ) : null}
    diff --git a/dashboard/src/ui/LegacyStates.tsx b/dashboard/src/ui/LegacyStates.tsx deleted file mode 100644 index ad721883a7..0000000000 --- a/dashboard/src/ui/LegacyStates.tsx +++ /dev/null @@ -1,65 +0,0 @@ -import type { ReactNode } from 'react'; -import { Readout } from './instrument.tsx'; - -/** A read that failed, said where its reading would have gone. - * - * The counterpart to `CenteredState` for a read whose failure is local to one - * plate: the surrounding surface still has readings to show, so this is a line - * in place of the missing one rather than a panel over the whole channel. - * - * `band` is the page-width form for the rows that sit outside a card, directly - * under the readout whose figures the failure explains; the default is the - * inline form a card body carries. The two are a class list rather than a `cn` - * merge because they disagree on text size, and `cn` is a plain joiner with no - * conflict resolution. */ -export function ReadFailure({ - label, - detail, - band = false, -}: { - label: string; - detail?: string | null | undefined; - band?: boolean; -}) { - return ( -

    - {label} - {detail ? `: ${detail}` : '.'} -

    - ); -} - -/** Compact readout tile. Kept as a named export because a dozen workspaces - * call it; the presentation is now the instrument readout, engraved legend, - * monospaced tabular value, quiet annotation, inside a hairline cell. */ -export function StatTile({ - label, - value, - hint, - dense, -}: { - label: string; - value: ReactNode; - /** Widened from `string` so a tile can annotate its value with the shared - * evidence-class marker, which `Readout`'s own `note` already accepts. */ - hint?: ReactNode; - /** Narrow-rail variant: smaller numerals that never clip. */ - dense?: boolean; -}) { - // A readout is something you look AT, so the tile sits on the raised plane - // rather than flush with the panel behind it. The values these carry are - // small counts, not brain-scale magnitudes, so they stay off the display - // tier -- a repository count set at 34px would be shouting a three. - return ( -
    - -
    - ); -} diff --git a/dashboard/src/ui/MetricPlate.tsx b/dashboard/src/ui/MetricPlate.tsx index 997dd66762..7826413c9a 100644 --- a/dashboard/src/ui/MetricPlate.tsx +++ b/dashboard/src/ui/MetricPlate.tsx @@ -63,7 +63,7 @@ export function MetricPlate({ - {annotation ?
    {annotation}
    : null} + {annotation ?
    {annotation}
    : null} {/* The reason is the reading when there is no figure, so it sits where a * value would and carries the state chip rather than hiding in a @@ -71,43 +71,43 @@ export function MetricPlate({ {!presentation.available ? (
    - + {presentation.unavailableReason ?? 'the daemon reported no reason'}
    ) : null} -
    +
    -
    denominator
    +
    denominator
    {presentation.denominator}
    -
    coverage
    +
    coverage
    {presentation.coverage}
    {presentation.interval ? (
    -
    interval
    +
    interval
    {presentation.interval}
    ) : null} {presentation.delta ? (
    -
    delta
    +
    delta
    {presentation.delta}
    ) : null} {presentation.calibration ? (
    -
    calibration
    +
    calibration
    {presentation.calibration}
    ) : null}
    -
    source
    +
    source
    {presentation.provenance}
    @@ -135,7 +135,7 @@ export function MetricGroups({ const groups = groupBySource(metrics); if (groups.length === 0) { return ( -

    +

    {emptyLabel}

    ); @@ -160,7 +160,7 @@ function MetricSourceGroup({ group }: { group: MetricGroup }) {

    {group.label}

    - + {available} of {group.metrics.length} measured
    diff --git a/dashboard/src/ui/ReadFailure.tsx b/dashboard/src/ui/ReadFailure.tsx new file mode 100644 index 0000000000..f7fbb0b115 --- /dev/null +++ b/dashboard/src/ui/ReadFailure.tsx @@ -0,0 +1,34 @@ +/** A read that failed, said where its reading would have gone. + * + * The counterpart to `CenteredState` for a read whose failure is local to one + * plate: the surrounding surface still has readings to show, so this is a line + * in place of the missing one rather than a panel over the whole channel. + * + * `band` is the page-width form for the rows that sit outside a card, directly + * under the readout whose figures the failure explains; the default is the + * inline form a card body carries. The two are a class list rather than a `cn` + * merge because they disagree on text size, and `cn` is a plain joiner with no + * conflict resolution. */ +export function ReadFailure({ + label, + detail, + band = false, +}: { + label: string; + detail?: string | null | undefined; + band?: boolean; +}) { + return ( +

    + {label} + {detail ? `: ${detail}` : '.'} +

    + ); +} diff --git a/dashboard/src/ui/ReadSection.tsx b/dashboard/src/ui/ReadSection.tsx index 8a5cdb4e8c..8686b5166b 100644 --- a/dashboard/src/ui/ReadSection.tsx +++ b/dashboard/src/ui/ReadSection.tsx @@ -208,7 +208,7 @@ export function ReadSection({ return (

    {title}

    - {blurb ?

    {blurb}

    : null} + {blurb ?

    {blurb}

    : null} {state.kind === 'ready' ? ( children(state.value) ) : ( @@ -385,13 +385,13 @@ export function CenteredState({
    -

    +

    {title}

    {guidance ? ( -

    +

    {guidance.sentence}{' '} {guidance.action}

    diff --git a/dashboard/src/ui/StateChip.dom.test.tsx b/dashboard/src/ui/StateChip.dom.test.tsx index e18edc94b4..6ff6d2953c 100644 --- a/dashboard/src/ui/StateChip.dom.test.tsx +++ b/dashboard/src/ui/StateChip.dom.test.tsx @@ -2,37 +2,6 @@ import { describe, it, expect } from 'vitest'; import { render, screen, cleanup } from '@testing-library/react'; import { StateChip, type DomainStateKind } from './StateChip'; -/** - * Every domain state renders a - * non-color-alone chip, an icon *and* a text label. The `Record` type below is - * the compile-time exhaustiveness gate: if the taxonomy in StateChip.tsx gains - * or drops a state, tsc fails here until this table is updated, so the "all - * states" claim can never silently rot. - */ -const EXPECTED_LABELS: Record = { - loading: 'Loading', - complete_zero_findings: 'Complete · zero findings', - ready: 'Ready', - partial: 'Partial', - rate_limited: 'Rate limited', - stale: 'Stale', - locked: 'Locked', - denied: 'Denied', - unauthorized: 'Unauthorized', - redacted: 'Redacted', - conflicting: 'Conflicting', - unavailable: 'Source unavailable', - offline: 'Offline', - unknown: 'Unknown', - cancelled: 'Cancelled', - timed_out: 'Timed out', - error: 'Error', - unsupported: 'Unsupported', - unsupported_schema: 'Unsupported schema', -}; - -const ENTRIES = Object.entries(EXPECTED_LABELS) as [DomainStateKind, string][]; - function chipVisual(kind: DomainStateKind) { const { container } = render(); const chip = container.querySelector(`[data-state="${kind}"]`); @@ -45,21 +14,6 @@ function chipVisual(kind: DomainStateKind) { } describe('StateChip', () => { - it.each(ENTRIES)('renders icon + label for "%s"', (kind, label) => { - const { container } = render(); - - const chip = container.querySelector(`[data-state="${kind}"]`); - expect(chip, `chip for ${kind}`).not.toBeNull(); - - // Icon: lucide renders an inline (aria-hidden), never color alone. - expect(chip!.querySelector('svg'), `icon for ${kind}`).not.toBeNull(); - - // Label: the human-readable text is present and exact. - expect(screen.getByText(label)).toBeTruthy(); - - cleanup(); - }); - /** * The two near-neighbours a reader must never confuse: a reachable authority * reporting that one source cannot answer, and nothing being reachable at @@ -99,8 +53,9 @@ describe('StateChip', () => { }); it('renders an optional detail suffix alongside the label', () => { + expect(chipVisual('stale').label).toBe('Stale'); + cleanup(); render(); - expect(screen.getByText('Stale')).toBeTruthy(); - expect(screen.getByText(/12m ago/)).toBeTruthy(); + expect(screen.getByText('Stale').parentElement?.textContent).toBe('Stale· 12m ago'); }); }); diff --git a/dashboard/src/ui/archetypes/ExplorerSplit.tsx b/dashboard/src/ui/archetypes/ExplorerSplit.tsx index ce99e08864..45020065aa 100644 --- a/dashboard/src/ui/archetypes/ExplorerSplit.tsx +++ b/dashboard/src/ui/archetypes/ExplorerSplit.tsx @@ -246,7 +246,7 @@ export function DataRow({ aria-pressed={selected ?? false} style={{ height: height != null ? `${height}px` : 'var(--row-height-data)' }} className={cn( - 'relative flex w-full gap-3 border-b border-edge-subtle pl-3 pr-3 text-left text-xs', + 'relative flex w-full gap-3 border-b border-edge-subtle pl-3 pr-3 text-left text-sm', align === 'start' ? 'items-start pt-2' : 'items-center', 'hover:bg-surface-1 focus-visible:bg-surface-1', // Lists in this archetype pin a `ListCaption` at `top-0`, so a row @@ -285,7 +285,7 @@ export function ListCaption({ return (

    @@ -401,7 +401,7 @@ export function KeyValueTree({ value, depth = 0 }: { value: unknown; depth?: num if (typeof value !== 'object') { const text = String(value); return ( - + {withPathBreaks(text)} ); @@ -419,13 +419,13 @@ export function KeyValueTree({ value, depth = 0 }: { value: unknown; depth?: num {value.slice(0, 60).map((v, i) => ( {v === null || v === undefined ? '—' : withPathBreaks(String(v))} ))} {value.length > 60 ? ( - … {value.length - 60} more + … {value.length - 60} more ) : null}

    ); @@ -447,7 +447,7 @@ export function KeyValueTree({ value, depth = 0 }: { value: unknown; depth?: num // line. Capping the reservation at one track holds however deep the // payload nests and however narrow the container is. className={cn( - 'grid gap-x-2 gap-y-0.5 border-b border-edge-subtle/60 py-1 text-2xs last:border-b-0', + 'grid gap-x-2 gap-y-0.5 border-b border-edge-subtle/60 py-1 text-sm last:border-b-0', depth === 0 ? 'grid-cols-1 sm:grid-cols-[minmax(5rem,9rem)_1fr] sm:gap-y-0' : 'grid-cols-1', @@ -462,7 +462,7 @@ export function KeyValueTree({ value, depth = 0 }: { value: unknown; depth?: num
    ))} {entries.length > 60 ? ( - … {entries.length - 60} more + … {entries.length - 60} more ) : null}
    ); diff --git a/dashboard/src/ui/instrument.tsx b/dashboard/src/ui/instrument.tsx index 9c3497c1d0..222f476ba5 100644 --- a/dashboard/src/ui/instrument.tsx +++ b/dashboard/src/ui/instrument.tsx @@ -167,7 +167,7 @@ export type ReadoutSize = 'sm' | 'md' | 'lg' | 'xl' | 'display'; * weight for large monospaced figures. The two tiers are different kinds of * object, not different sizes of the same one. */ const VALUE_SIZE: Record = { - sm: 'td-value text-xs', + sm: 'td-value text-sm', md: 'td-value text-base font-medium', lg: 'td-value text-xl font-medium', xl: 'td-display text-2xl', @@ -232,7 +232,7 @@ export function Readout({ ) : null} - {note ? {note} : null} + {note ? {note} : null} ); } @@ -335,7 +335,7 @@ export function MeterRow({ className?: string; }) { return ( -
    +
    {leading} {label} @@ -348,7 +348,7 @@ export function MeterRow({ /> - + {value} {unit ? {unit} : null} @@ -410,7 +410,7 @@ export function FigureRail({ export function Field({ label, children }: { label: string; children: ReactNode }) { return (
    -
    {label}
    +
    {label}
    {children}
    ); @@ -436,8 +436,8 @@ export function Fact({
    {value} @@ -545,7 +545,7 @@ export function WorkspaceHeader({ {channelNumber(path)} -

    +

    {title}

    {/* Withdrawn below `sm`, where the line has no width to spend on filler: @@ -553,7 +553,7 @@ export function WorkspaceHeader({ * header overflow at 320 CSS px. Above `sm` it earns its width. */} {note ? ( - + {note} ) : null} diff --git a/dashboard/src/ui/search/Highlight.tsx b/dashboard/src/ui/search/Highlight.tsx index 0877ea9d51..38afa2e034 100644 --- a/dashboard/src/ui/search/Highlight.tsx +++ b/dashboard/src/ui/search/Highlight.tsx @@ -23,7 +23,7 @@ export function Highlight({ segment.hit ? ( {segment.text} diff --git a/dashboard/src/ui/search/SearchField.tsx b/dashboard/src/ui/search/SearchField.tsx index b9f16dc1ad..9caea17421 100644 --- a/dashboard/src/ui/search/SearchField.tsx +++ b/dashboard/src/ui/search/SearchField.tsx @@ -92,7 +92,7 @@ export function SearchField({ spellCheck={false} autoComplete="off" className={cn( - 'min-w-0 flex-1 self-stretch bg-transparent text-sm text-text-primary outline-none', + 'min-w-0 flex-1 self-stretch bg-transparent text-body text-text-primary outline-none', 'placeholder:text-text-muted', )} /> @@ -115,7 +115,7 @@ export function SearchField({
    {hint ? ( -

    +

    {hint}

    ) : null} diff --git a/dashboard/src/viz/chart/Chart.dom.test.tsx b/dashboard/src/viz/chart/Chart.dom.test.tsx index d33e277c68..12eafc7b15 100644 --- a/dashboard/src/viz/chart/Chart.dom.test.tsx +++ b/dashboard/src/viz/chart/Chart.dom.test.tsx @@ -188,6 +188,26 @@ describe('Chart registered-series guard', () => { await waitFor(() => expect(applied.length).toBe(0)); }); + it('draws a scatter series, which the memory projection plots', async () => { + const { getByRole, queryByText } = render( + , + ); + + const option = await lastOption(); + expect(option.series).toEqual([{ type: 'scatter', data: [[0.2, 0.7]] }]); + expect(getByRole('img').getAttribute('aria-label')).toBe('memory projection'); + expect(queryByText(/cannot draw/i)).toBeNull(); + }); + it('reports every unregistered series once, and still refuses a mixed option', () => { const { getByText } = render( ({ - graph: undefined as Graph | undefined, - layers: {} as Record, - constructCount: 0, - killCount: 0, -})); - -vi.mock('./emergentLayout.ts', () => ({ - settleEmergentOffThread: async () => true, -})); - -vi.mock('sigma', () => { - /** A layer that answers honestly about the context it holds, the way a real - * canvas does: one that already has a 2d context returns null for every - * WebGL id, which is the rule the renderer's probe rests on. */ - const layer = (context: 'webgl' | '2d'): HTMLCanvasElement => { - const canvas = document.createElement('canvas'); - canvas.dataset['layerContext'] = context; - return canvas; - }; - return { - default: class MockSigma { - constructor(graph: Graph, container: HTMLElement) { - // Sigma's real stack, in its real order: the bodies and relations are - // drawn in WebGL, while labels, hover decoration and pointer capture - // ride 2d layers that no context loss can reach. - sigmaState.layers = { - edges: layer('webgl'), - edgeLabels: layer('2d'), - nodes: layer('webgl'), - labels: layer('2d'), - hovers: layer('2d'), - hoverNodes: layer('webgl'), - mouse: layer('2d'), - }; - for (const canvas of Object.values(sigmaState.layers)) { - container.appendChild(canvas); - } - sigmaState.graph = graph; - sigmaState.constructCount += 1; - } - - setCustomBBox() {} - - getCanvases() { - return sigmaState.layers; - } - - resize() {} - on() {} - refresh() {} - setSetting() {} - - /** Real `kill` detaches every layer and forgets the map, which is why the - * watch has to hold the canvases it captured: the restore is dispatched - * at the canvas of the renderer that died. */ - kill() { - for (const canvas of Object.values(sigmaState.layers)) canvas.remove(); - sigmaState.killCount += 1; - } - }, - }; -}); - -/** A manually pumped animation clock, so "the loop is asleep" is observed - * rather than inferred from timing. */ -const frames: { id: number; run: FrameRequestCallback }[] = []; -let nextFrameId = 1; - -const NODES = [ - { id: 'repo:r', label: 'r', kind: 'repository', degree: 2 }, - { id: 'p1', label: 'p1', kind: 'checkout', degree: 1 }, -]; -const EDGES = [{ source: 'repo:r', target: 'p1', kind: 'checkout' }]; - -/** The WebGL layers a real loss would reach, named so a regression that watches - * only the layer the bodies sit on is visible. */ -const WEBGL_LAYERS = ['edges', 'nodes', 'hoverNodes']; - -function loseContext(canvas: HTMLCanvasElement): Event { - const event = new Event('webglcontextlost', { cancelable: true }); - canvas.dispatchEvent(event); - return event; -} - -describe('GraphCanvas WebGL context loss', () => { - beforeEach(() => { - sigmaState.graph = undefined; - sigmaState.layers = {}; - sigmaState.constructCount = 0; - sigmaState.killCount = 0; - frames.length = 0; - nextFrameId = 1; - Object.defineProperty(HTMLCanvasElement.prototype, 'getContext', { - configurable: true, - value: function getContext(this: HTMLCanvasElement, kind: string) { - const held = this.dataset['layerContext'] ?? 'webgl'; - if (held === '2d') return kind === '2d' ? ({} as RenderingContext) : null; - return kind.startsWith('webgl') ? ({} as RenderingContext) : null; - }, - }); - Object.defineProperties(HTMLElement.prototype, { - clientWidth: { configurable: true, get: () => 640 }, - clientHeight: { configurable: true, get: () => 320 }, - offsetWidth: { configurable: true, get: () => 640 }, - offsetHeight: { configurable: true, get: () => 320 }, - }); - Object.defineProperty(window, 'matchMedia', { - configurable: true, - value: vi.fn().mockReturnValue({ matches: false }), - }); - vi.stubGlobal('requestAnimationFrame', (callback: FrameRequestCallback) => { - const id = nextFrameId++; - frames.push({ id, run: callback }); - return id; - }); - vi.stubGlobal('cancelAnimationFrame', (id: number) => { - const at = frames.findIndex((frame) => frame.id === id); - if (at >= 0) frames.splice(at, 1); - }); - }); - - afterEach(() => { - vi.unstubAllGlobals(); - }); - - it('states the lost renderer instead of leaving a dead canvas on screen', async () => { - const field = new ActivationField({ halfLifeMs: 4200 }); - render(); - await waitFor(() => expect(sigmaState.graph).toBeDefined()); - // The field really was drawn: the canvas region is on screen and the - // activation loop is running against live heat. - expect(screen.queryByRole('img')).not.toBeNull(); - field.strike(['p1'], 0.9); - expect(frames).toHaveLength(1); - - const lost = await act(async () => loseContext(sigmaState.layers['nodes']!)); - - // Without this the browser abandons the context for good and no restore is - // ever attempted, so it is the first thing the handler does. - expect(lost.defaultPrevented).toBe(true); - // The reader is told the renderer was lost, not that the neighbourhood is - // empty, which is the sentence an undrawn field would otherwise imply, and - // is pointed at the same equivalent the no-context path names. - const stated = screen.getByText(/lost its WebGL context/i); - expect(stated.textContent).toMatch(/no longer being drawn/i); - expect(stated.textContent).toMatch(/read the field description below/i); - expect(document.querySelector('[role="status"][aria-live="polite"]')).not.toBeNull(); - expect(screen.queryByText(/no graph neighborhood to draw/i)).toBeNull(); - expect(document.querySelector('[data-state="unavailable"]')).not.toBeNull(); - // Nothing is drawn in its place: no canvas region, so no blank field that - // could be read as a graph with nothing in it. - expect(screen.queryByRole('img')).toBeNull(); - expect(document.querySelector('canvas')).toBeNull(); - - // The loop stopped, and the field is still warm, so it stopped because the - // renderer died, not because there was nothing left to animate. - expect(frames).toHaveLength(0); - expect(field.warm).toBe(true); - expect(sigmaState.killCount).toBe(1); - }); - - it('brings the field back when the browser restores the context', async () => { - render(); - await waitFor(() => expect(sigmaState.constructCount).toBe(1)); - // Captured before the loss: killing the renderer detaches every layer, and - // the restore arrives at the canvas that lost the context. - const canvas = sigmaState.layers['nodes']!; - - await act(async () => loseContext(canvas)); - expect(screen.getByText(/lost its WebGL context/i)).toBeTruthy(); - - await act(async () => { - canvas.dispatchEvent(new Event('webglcontextrestored')); - }); - - // Composed again from the same nodes and edges the caller last handed over, - // with no action of its own. - await waitFor(() => expect(sigmaState.constructCount).toBe(2)); - expect(screen.queryByText(/lost its WebGL context/i)).toBeNull(); - expect(screen.queryByRole('img')).not.toBeNull(); - - // ...and the field that came back is watched as well, on its own layers: a - // renderer rebuilt after one loss is exactly as exposed to the next. - await act(async () => loseContext(sigmaState.layers['nodes']!)); - expect(screen.getByText(/lost its WebGL context/i)).toBeTruthy(); - }); - - it.each(WEBGL_LAYERS)('reports a context lost on the %s layer', async (id) => { - render(); - await waitFor(() => expect(sigmaState.constructCount).toBe(1)); - - const lost = await act(async () => loseContext(sigmaState.layers[id]!)); - - expect(lost.defaultPrevented).toBe(true); - expect(screen.getByText(/lost its WebGL context/i)).toBeTruthy(); - }); - - it('leaves the 2d layers alone, because their context cannot be lost', async () => { - render(); - await waitFor(() => expect(sigmaState.constructCount).toBe(1)); - - const lost = await act(async () => loseContext(sigmaState.layers['mouse']!)); - - expect(lost.defaultPrevented).toBe(false); - expect(screen.queryByText(/lost its WebGL context/i)).toBeNull(); - expect(screen.queryByRole('img')).not.toBeNull(); - }); - - it('releases the watch with the canvas, so a dead layer cannot rebuild it', async () => { - const { unmount } = render(); - await waitFor(() => expect(sigmaState.constructCount).toBe(1)); - const canvas = sigmaState.layers['nodes']!; - - unmount(); - - const lost = await act(async () => loseContext(canvas)); - await act(async () => { - canvas.dispatchEvent(new Event('webglcontextrestored')); - }); - - expect(lost.defaultPrevented).toBe(false); - expect(sigmaState.constructCount).toBe(1); - }); -}); diff --git a/dashboard/src/viz/graph/GraphCanvas.dom.test.tsx b/dashboard/src/viz/graph/GraphCanvas.dom.test.tsx deleted file mode 100644 index b690f2fc0f..0000000000 --- a/dashboard/src/viz/graph/GraphCanvas.dom.test.tsx +++ /dev/null @@ -1,360 +0,0 @@ -import { fireEvent, render, waitFor } from '@testing-library/react'; -import type Graph from 'graphology'; -import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { GraphCanvas } from './GraphCanvas.tsx'; - -type NodeAttributes = Record; -type NodeReducer = (node: string, data: NodeAttributes) => NodeAttributes; - -const sigmaState = vi.hoisted(() => ({ - graph: undefined as Graph | undefined, - nodeReducer: undefined as NodeReducer | undefined, - drawNodeHover: undefined as unknown, - refreshCount: 0, - constructCount: 0, - killCount: 0, - resizeCount: 0, - strikeListeners: new Set<() => void>(), - handlers: new Map void>(), - cameraActions: [] as string[], -})); - -vi.mock('./activation.ts', () => ({ - ActivationField: class MockActivationField { - heatOf() { - return 0; - } - - get warm() { - return false; - } - - tick() { - return false; - } - - subscribe() { - return () => {}; - } - }, - cssColorToRgb: () => [128, 128, 128], - lerpRgb: () => 'rgb(128, 128, 128)', - lerpRgbTuple: () => [128, 128, 128], - restingNodeTint: () => [128, 128, 128], - approach: (_current: number, target: number) => target, - settled: () => true, -})); - -vi.mock('./emergentLayout.ts', () => ({ - settleEmergentOffThread: async () => true, -})); - -/** Mirrors the one behaviour of the real renderer this file is about: Sigma - * measures `container.offsetWidth` on construction AND on every render, and - * throws rather than drawing when the answer is zero. A mock that quietly - * tolerated a collapsed container could not have caught the bug. */ -vi.mock('sigma', () => ({ - default: class MockSigma { - private readonly container: HTMLElement; - /** Sigma stacks several canvas layers over the container and hands them - * back by id; the renderer asks for that map to find the ones whose WebGL - * context can be lost. `GraphCanvas.context.dom.test.tsx` is where losing - * one is exercised. */ - private readonly layers: Record = { - edges: document.createElement('canvas'), - nodes: document.createElement('canvas'), - hoverNodes: document.createElement('canvas'), - }; - - constructor( - graph: Graph, - container: HTMLElement, - settings: { nodeReducer?: NodeReducer; defaultDrawNodeHover?: unknown }, - ) { - this.container = container; - this.measure(); - sigmaState.graph = graph; - sigmaState.nodeReducer = settings.nodeReducer; - sigmaState.drawNodeHover = settings.defaultDrawNodeHover; - sigmaState.constructCount += 1; - } - - private measure() { - if (this.container.offsetWidth === 0) { - throw new Error('Sigma: Container has no width.'); - } - } - - setCustomBBox() {} - - getCanvases() { - return this.layers; - } - - /** Real Sigma exposes `resize(force?: boolean): this`; a live renderer - * absorbs a container resize through it rather than being rebuilt. */ - resize() { - sigmaState.resizeCount += 1; - return this; - } - getCamera() { - return { - ratio: 1, - getBoundedRatio: (ratio: number) => ratio, - setState: () => sigmaState.cameraActions.push('set'), - animatedZoom: () => { - sigmaState.cameraActions.push('in'); - return Promise.resolve(); - }, - animatedUnzoom: () => { - sigmaState.cameraActions.push('out'); - return Promise.resolve(); - }, - animatedReset: () => { - sigmaState.cameraActions.push('fit'); - return Promise.resolve(); - }, - }; - } - on(name: string, handler: (event?: { node: string }) => void) { - sigmaState.handlers.set(name, handler); - } - refresh() { - this.measure(); - sigmaState.refreshCount += 1; - } - setSetting() {} - kill() { - sigmaState.killCount += 1; - } - }, -})); - -/** jsdom has no WebGL, so the canvas would take its no-context fallback. - * Simulate a WebGL-capable browser for the rendering tests, and the absence of - * one where that is the case under test. */ -function stubWebGl(available: boolean) { - Object.defineProperty(HTMLCanvasElement.prototype, 'getContext', { - configurable: true, - value: (kind: string) => - available && kind.startsWith('webgl') ? ({} as unknown as RenderingContext) : null, - }); -} - -/** Give anything the canvas started asynchronously, an emergent field loads - * its layout engine on demand, its turn to run, so an assertion that nothing - * was built means never rather than not yet. */ -async function flushPendingWork(): Promise { - for (let tick = 0; tick < 3; tick += 1) { - await new Promise((resolve) => setTimeout(resolve, 0)); - } -} - -/** The measured box every element reports, mutable so a test can take it away - * the way navigating off a workspace does. */ -const box = { width: 640, height: 320 }; -/** Every live ResizeObserver callback, so a test can deliver a measurement. */ -const observerCallbacks = new Set<() => void>(); - -describe('GraphCanvas', () => { - beforeEach(() => { - sigmaState.graph = undefined; - sigmaState.nodeReducer = undefined; - sigmaState.drawNodeHover = undefined; - sigmaState.refreshCount = 0; - sigmaState.constructCount = 0; - sigmaState.killCount = 0; - sigmaState.resizeCount = 0; - sigmaState.strikeListeners.clear(); - sigmaState.handlers.clear(); - sigmaState.cameraActions.length = 0; - box.width = 640; - box.height = 320; - observerCallbacks.clear(); - stubWebGl(true); - vi.stubGlobal( - 'ResizeObserver', - class MockResizeObserver { - private readonly callback: () => void; - constructor(callback: () => void) { - this.callback = callback; - observerCallbacks.add(callback); - } - observe() {} - disconnect() { - observerCallbacks.delete(this.callback); - } - unobserve() {} - }, - ); - Object.defineProperties(HTMLElement.prototype, { - clientWidth: { configurable: true, get: () => box.width }, - clientHeight: { configurable: true, get: () => box.height }, - // Sigma measures `offsetWidth`, and so does the guard that decides - // whether a renderer may exist, so the fixture has to answer that name. - offsetWidth: { configurable: true, get: () => box.width }, - offsetHeight: { configurable: true, get: () => box.height }, - }); - Object.defineProperty(window, 'matchMedia', { - configurable: true, - value: vi.fn().mockReturnValue({ matches: false }), - }); - }); - - describe('renderer lifetime against the container box', () => { - const NODES = [{ id: 'node', label: 'Node', kind: 'function', degree: 1 }]; - - /** Deliver a measurement the way the browser does after a layout change. */ - function deliverMeasurement() { - for (const callback of [...observerCallbacks]) callback(); - } - - it('does not build a renderer for a container that has no box', async () => { - box.width = 0; - box.height = 0; - - expect(() => render()).not.toThrow(); - expect(sigmaState.constructCount).toBe(0); - // An emergent field now fetches its layout engine before composing, so - // "no renderer" has to outlast that boundary as well: the guard is on - // whether a box exists, never on whether the engine has answered yet. - await flushPendingWork(); - expect(sigmaState.constructCount).toBe(0); - }); - - it('builds one once the container is measured, without a mount retry', async () => { - box.width = 0; - box.height = 0; - render(); - expect(sigmaState.constructCount).toBe(0); - - box.width = 640; - box.height = 320; - deliverMeasurement(); - - await waitFor(() => expect(sigmaState.constructCount).toBe(1)); - }); - - // The regression: leaving a workspace collapses the container while the - // renderer is still alive, and Sigma's next frame -- including ones it - // schedules itself from a window resize -- measures zero and throws. The - // renderer has to be gone by then, not merely told to skip a frame. - it('kills the renderer when the container loses its box', async () => { - const { rerender } = render( - , - ); - await waitFor(() => expect(sigmaState.constructCount).toBe(1)); - expect(sigmaState.killCount).toBe(0); - - box.width = 0; - box.height = 0; - deliverMeasurement(); - - expect(sigmaState.killCount).toBe(1); - // A repaint request arriving after the collapse must find nothing to - // repaint rather than reaching a renderer that would measure zero. - expect(() => - rerender(), - ).not.toThrow(); - }); - - it('rebuilds when the box comes back', async () => { - render(); - await waitFor(() => expect(sigmaState.constructCount).toBe(1)); - - box.width = 0; - box.height = 0; - deliverMeasurement(); - expect(sigmaState.killCount).toBe(1); - - box.width = 800; - box.height = 400; - deliverMeasurement(); - - await waitFor(() => expect(sigmaState.constructCount).toBe(2)); - }); - - // The counterpart to the three tests above, and the reason the renderer's - // lifetime is keyed on whether a box exists rather than on how big it is. - // Keying it on the dimensions made every drag of a window edge or opening - // of a side panel kill the renderer, rebuild the graphology graph and re-run - // the 200-iteration ForceAtlas2 settle -- a full layout per resize frame. - it('resizes the live renderer instead of rebuilding it', async () => { - render(); - await waitFor(() => expect(sigmaState.constructCount).toBe(1)); - const refreshesBeforeResize = sigmaState.refreshCount; - - box.width = 900; - box.height = 500; - deliverMeasurement(); - box.width = 1200; - box.height = 640; - deliverMeasurement(); - - await waitFor(() => expect(sigmaState.resizeCount).toBeGreaterThan(0)); - expect(sigmaState.refreshCount).toBeGreaterThan(refreshesBeforeResize); - expect(sigmaState.constructCount).toBe(1); - expect(sigmaState.killCount).toBe(0); - }); - }); - - it('shares focus with the accessible list and exposes text camera controls', async () => { - const nodes = [{ id: 'node', label: 'Node', kind: 'project', degree: 1 }]; - const onInspect = vi.fn(); - const view = render( - , - ); - await waitFor(() => expect(sigmaState.constructCount).toBe(1)); - - sigmaState.handlers.get('enterNode')?.({ node: 'node' }); - expect(onInspect).toHaveBeenLastCalledWith('node'); - sigmaState.handlers.get('leaveNode')?.(); - expect(onInspect).toHaveBeenLastCalledWith(null); - - view.rerender( - , - ); - await waitFor(() => { - const attrs = sigmaState.graph!.getNodeAttributes('node'); - expect(sigmaState.nodeReducer?.('node', attrs)['zIndex']).toBe(3); - }); - - fireEvent.click(view.getByRole('button', { name: 'Zoom in graph' })); - fireEvent.click(view.getByRole('button', { name: 'Zoom out graph' })); - fireEvent.click(view.getByRole('button', { name: 'Fit' })); - expect(sigmaState.cameraActions).toEqual(['in', 'out', 'fit']); - }); - - it('states the missing WebGL context and the caller-supplied text alternative', async () => { - stubWebGl(false); - const { getByText } = render( - , - ); - expect(getByText(/no WebGL context/i)).toBeTruthy(); - expect(getByText(/project registry remains available/i)).toBeTruthy(); - expect(document.querySelector('[role="status"][aria-live="polite"]')).not.toBeNull(); - // Never constructed: Sigma throws without a context, and that exception - // would take the whole workspace route down through the error boundary. - // Held across the layout engine's async boundary too, a renderer that - // merely arrives late is still a renderer that must not exist. - expect(sigmaState.nodeReducer).toBeUndefined(); - await flushPendingWork(); - expect(sigmaState.nodeReducer).toBeUndefined(); - }); -}); diff --git a/dashboard/src/viz/graph/GraphCanvas.measured.dom.test.tsx b/dashboard/src/viz/graph/GraphCanvas.measured.dom.test.tsx deleted file mode 100644 index 46a0bc8490..0000000000 --- a/dashboard/src/viz/graph/GraphCanvas.measured.dom.test.tsx +++ /dev/null @@ -1,151 +0,0 @@ -import { act, render, waitFor } from '@testing-library/react'; -import type Graph from 'graphology'; -import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { GraphCanvas } from './GraphCanvas.tsx'; - -/** Measured coordinates bypass the worker; emergent positions are unavailable - * until a bounded background layout completes. */ - -const forceState = vi.hoisted(() => ({ - requested: false, - signal: null as AbortSignal | null, - pending: null as Promise | null, -})); - -vi.mock('./emergentLayout.ts', () => ({ - settleEmergentOffThread: async (_prepared: unknown, signal: AbortSignal) => { - forceState.requested = true; - forceState.signal = signal; - return forceState.pending ?? true; - }, -})); - -const sigmaState = vi.hoisted(() => ({ - graph: undefined as Graph | undefined, - bbox: undefined as { x: [number, number]; y: [number, number] } | undefined, -})); - -vi.mock('sigma', () => ({ - default: class MockSigma { - /** The layer map the renderer reads to find the canvases whose WebGL - * context it must watch. */ - private readonly layers = { nodes: document.createElement('canvas') }; - - constructor(graph: Graph) { - sigmaState.graph = graph; - } - - setCustomBBox(bbox: { x: [number, number]; y: [number, number] }) { - sigmaState.bbox = bbox; - } - - getCanvases() { - return this.layers; - } - - resize() {} - on() {} - refresh() {} - setSetting() {} - kill() {} - }, -})); - -/** Anything the canvas started asynchronously has had its turn by the time - * this resolves, so "never requested" means never rather than not yet. */ -async function flushPendingWork(): Promise { - for (let tick = 0; tick < 3; tick += 1) { - await new Promise((resolve) => setTimeout(resolve, 0)); - } -} - -const PLACED = [ - { id: 'a', label: 'A', kind: 'project', degree: 2, x: -1, y: -1 }, - { id: 'b', label: 'B', kind: 'project', degree: 1, x: 1, y: 1 }, -]; -const EMERGENT = PLACED.map(({ x: _x, y: _y, ...node }) => node); -const EDGES = [{ source: 'a', target: 'b' }]; - -describe('GraphCanvas layout engine loading', () => { - beforeEach(() => { - forceState.requested = false; - forceState.signal = null; - forceState.pending = null; - sigmaState.graph = undefined; - sigmaState.bbox = undefined; - Object.defineProperty(HTMLCanvasElement.prototype, 'getContext', { - configurable: true, - value: (kind: string) => - kind.startsWith('webgl') ? ({} as unknown as RenderingContext) : null, - }); - Object.defineProperties(HTMLElement.prototype, { - clientWidth: { configurable: true, get: () => 640 }, - clientHeight: { configurable: true, get: () => 320 }, - offsetWidth: { configurable: true, get: () => 640 }, - offsetHeight: { configurable: true, get: () => 320 }, - }); - Object.defineProperty(window, 'matchMedia', { - configurable: true, - value: vi.fn().mockReturnValue({ matches: false }), - }); - }); - - it('never asks for the force layout when every node was measured', async () => { - render( - , - ); - await waitFor(() => expect(sigmaState.graph).toBeDefined()); - await flushPendingWork(); - - expect(forceState.requested).toBe(false); - // ...and the field really was drawn, from the caller's own coordinates, - // framed by the axis it named rather than by the bodies that occupy it. - expect(sigmaState.graph!.getNodeAttribute('a', 'x')).toBe(-1); - expect(sigmaState.graph!.getNodeAttribute('b', 'y')).toBe(1); - expect(sigmaState.bbox).toEqual({ x: [-4, 4], y: [-4, 4] }); - }); - - it('asks for it once a field has no measured positions of its own', async () => { - render(); - - await waitFor(() => expect(forceState.requested).toBe(true)); - // Nothing is drawn until the engine has answered: a seed circle on screen - // would be a composition the reader would read meaning into. - await waitFor(() => expect(sigmaState.graph).toBeDefined()); - }); - - it('keeps positions explicitly pending and aborts layout on unmount', async () => { - let resolve!: (result: boolean) => void; - forceState.pending = new Promise((done) => { resolve = done; }); - const view = render(); - expect(view.getByRole('status').textContent).toContain('Calculating graph positions'); - expect((view.getByRole('button', { name: 'Zoom in graph' }) as HTMLButtonElement).disabled).toBe(true); - const signal = forceState.signal; - view.unmount(); - expect(signal?.aborted).toBe(true); - await act(async () => resolve(true)); - expect(sigmaState.graph).toBeUndefined(); - forceState.pending = null; - }); - - it('does not install a late layout over a newer measured topology', async () => { - let resolve!: (result: boolean) => void; - forceState.pending = new Promise((done) => { resolve = done; }); - const view = render(); - const signal = forceState.signal; - view.rerender(); - const measuredGraph = sigmaState.graph; - expect(signal?.aborted).toBe(true); - expect(measuredGraph?.getNodeAttribute('a', 'x')).toBe(-1); - await act(async () => resolve(true)); - expect(sigmaState.graph).toBe(measuredGraph); - }); - - it('reports a failed layout instead of drawing the initial seed', async () => { - forceState.pending = Promise.reject(new Error('Worker unavailable')); - const view = render(); - await waitFor(() => expect(view.getByRole('status').textContent).toContain('could not be completed')); - expect(view.container.querySelector('[role="img"]')).toBeNull(); - forceState.pending = null; - }); -}); diff --git a/dashboard/src/viz/graph/GraphCanvas.propagation.dom.test.tsx b/dashboard/src/viz/graph/GraphCanvas.propagation.dom.test.tsx deleted file mode 100644 index 3e24dd7dbe..0000000000 --- a/dashboard/src/viz/graph/GraphCanvas.propagation.dom.test.tsx +++ /dev/null @@ -1,325 +0,0 @@ -import { render, waitFor } from '@testing-library/react'; -import type Graph from 'graphology'; -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { GraphCanvas } from './GraphCanvas.tsx'; -import { ActivationField } from './activation.ts'; -import { setMotionPreference } from '../trace/reducedMotion.ts'; - -/** - * End-to-end evidence for travelling activation. - * - * Unlike `GraphCanvas.dom.test.tsx`, this file deliberately does NOT mock - * `activation.ts`: the whole question is whether a strike delivered from - * outside the renderer, the way the Brain's SSE effect delivers one, reaches - * the drawn graph and travels the real edge. Sigma is mocked only far enough - * to hand back the graphology instance it was given, so every assertion below - * is made against the geometry the renderer actually composed. - */ - -type Reducer = (id: string, data: Record) => Record; - -const sigmaState = vi.hoisted(() => ({ - graph: undefined as Graph | undefined, - edgeReducer: undefined as Reducer | undefined, - refreshes: 0, - handlers: new Map void>(), -})); - -vi.mock('./emergentLayout.ts', () => ({ - settleEmergentOffThread: async () => true, -})); - -vi.mock('sigma', () => ({ - default: class MockSigma { - /** The layer map the renderer reads to find the canvases whose WebGL - * context it must watch. */ - private readonly layers = { nodes: document.createElement('canvas') }; - - constructor( - graph: Graph, - _container: unknown, - settings: { edgeReducer?: Reducer }, - ) { - sigmaState.graph = graph; - sigmaState.edgeReducer = settings.edgeReducer; - } - - setCustomBBox() {} - - getCanvases() { - return this.layers; - } - - resize() {} - on(name: string, handler: (event: { node: string }) => void) { - sigmaState.handlers.set(name, handler); - } - refresh() { - sigmaState.refreshes += 1; - } - setSetting() {} - kill() {} - }, -})); - -/** A manually pumped animation clock, so "the loop is asleep" and "the loop - * ran a frame" are both directly observable rather than inferred from timing. - * Handles are stable and cancellation genuinely dequeues, because "stopped" is - * one of the states under test and a no-op cancel cannot distinguish it. */ -const frames: { id: number; run: FrameRequestCallback }[] = []; -let nextFrameId = 1; - -function pump(now: number) { - const due = frames.splice(0, frames.length); - for (const frame of due) frame.run(now); -} - -/** The Brain's own shape: one repository hub wired to one checkout. */ -const NODES = [ - { id: 'repo:r', label: 'r', kind: 'repository', degree: 2 }, - { id: 'p1', label: 'p1', kind: 'checkout', degree: 1 }, -]; -const EDGES = [{ source: 'repo:r', target: 'p1', kind: 'checkout' }]; - -function pulseNodes(graph: Graph): string[] { - return graph.nodes().filter((node) => node.startsWith('__pulse__')); -} - -function positionOf(graph: Graph, id: string): [number, number] { - return [ - graph.getNodeAttribute(id, 'x') as number, - graph.getNodeAttribute(id, 'y') as number, - ]; -} - -describe('GraphCanvas travelling activation', () => { - beforeEach(() => { - sigmaState.graph = undefined; - sigmaState.edgeReducer = undefined; - sigmaState.refreshes = 0; - sigmaState.handlers.clear(); - frames.length = 0; - nextFrameId = 1; - Object.defineProperty(HTMLCanvasElement.prototype, 'getContext', { - configurable: true, - value: (kind: string) => (kind.startsWith('webgl') ? ({} as RenderingContext) : null), - }); - Object.defineProperties(HTMLElement.prototype, { - clientWidth: { configurable: true, get: () => 640 }, - clientHeight: { configurable: true, get: () => 320 }, - // Sigma measures `offsetWidth`, and so does the guard that decides - // whether a renderer may exist, so the fixture has to answer that name. - offsetWidth: { configurable: true, get: () => 640 }, - offsetHeight: { configurable: true, get: () => 320 }, - }); - Object.defineProperty(window, 'matchMedia', { - configurable: true, - value: vi.fn().mockReturnValue({ matches: false }), - }); - vi.stubGlobal('requestAnimationFrame', (callback: FrameRequestCallback) => { - const id = nextFrameId++; - frames.push({ id, run: callback }); - return id; - }); - vi.stubGlobal('cancelAnimationFrame', (id: number) => { - const at = frames.findIndex((frame) => frame.id === id); - if (at >= 0) frames.splice(at, 1); - }); - }); - - afterEach(() => { - vi.useRealTimers(); - vi.unstubAllGlobals(); - localStorage.removeItem('td.motion-preference'); - }); - - it('selects and hovers measured projects and repository hubs without creating activity', async () => { - const field = new ActivationField({ halfLifeMs: 4200 }); - const onSelect = vi.fn(); - const nodes = [ - ...NODES, - { id: 'p2', label: 'sibling checkout', kind: 'checkout', degree: 1 }, - ].map((node, index) => ({ ...node, x: index * 100, y: index * 20 })); - const edges = [...EDGES, { source: 'repo:r', target: 'p2', kind: 'checkout' }]; - render(); - await waitFor(() => expect(sigmaState.graph).toBeDefined()); - vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); - const graph = sigmaState.graph!; - const click = sigmaState.handlers.get('clickNode')!; - const hover = sigmaState.handlers.get('enterNode')!; - expect(click).toBeTypeOf('function'); - expect(hover).toBeTypeOf('function'); - - let now = 0; - for (const node of ['p1', 'repo:r']) { - const refreshes = sigmaState.refreshes; - hover({ node }); - pump(now += 200); - expect(sigmaState.refreshes).toBeGreaterThan(refreshes); - click({ node }); - expect(onSelect).toHaveBeenLastCalledWith(node); - vi.advanceTimersByTime(200); - pump(now += 200); - expect(field.warm).toBe(false); - expect(nodes.map(({ id }) => field.heatOf(id))).toEqual([0, 0, 0]); - expect(pulseNodes(graph)).toEqual([]); - } - }); - - it('carries an externally delivered strike along the real edge and then sleeps', async () => { - const field = new ActivationField({ halfLifeMs: 4200 }); - render(); - await waitFor(() => expect(sigmaState.graph).toBeDefined()); - const graph = sigmaState.graph!; - - // A cold field leaves the loop asleep: the canvas composed one static - // frame and requested nothing. - expect(frames).toHaveLength(0); - expect(pulseNodes(graph)).toEqual([]); - - // Exactly what BrainPage does when one SSE event lands: fire the neuron - // its scope names, then one hop along the drawn edge at a third the - // energy. Neither call knows the render loop exists. - field.strike(['p1'], 0.9); - field.strike(['repo:r'], 0.3); - - // The strike alone woke the loop, nothing else could have, the field has - // no clock of its own. - expect(frames).toHaveLength(1); - - pump(0); - const travellers = pulseNodes(graph); - expect(travellers).toEqual(['__pulse__0']); - - // The edge between the two struck neurons conducts, because both of its - // ends are warm. - const lit = sigmaState.edgeReducer?.('e', { - srcReal: 'repo:r', - dstReal: 'p1', - size: 1, - }); - const cold = sigmaState.edgeReducer?.('e', { - srcReal: 'repo:r', - dstReal: 'absent', - size: 1, - }); - expect(lit?.['color']).not.toEqual(cold?.['color']); - expect(lit?.['size']).toBeGreaterThan(1); - - // The light leaves from the end where the event actually happened. `p1` is - // the hotter node because it is the one the event's own scope named; the - // hub was only reached by the hop. So the traveller starts on `p1` and - // runs toward the hub, not the other way round. - const hub = positionOf(graph, 'repo:r'); - const origin = positionOf(graph, 'p1'); - expect(positionOf(graph, '__pulse__0')).toEqual(origin); - - const distanceTo = ([x, y]: [number, number], [tx, ty]: [number, number]) => - Math.hypot(x - tx, y - ty); - pump(500); - const midway = positionOf(graph, '__pulse__0'); - expect(midway).not.toEqual(origin); - expect(distanceTo(midway, hub)).toBeLessThan(distanceTo(origin, hub)); - expect(distanceTo(midway, origin)).toBeGreaterThan(0); - - // ...and when the field finally goes cold the traveller is removed and the - // loop stops asking for frames. An idle dashboard costs nothing. - // `pump` drains the queue, so an empty queue afterwards means this frame - // declined to request a successor rather than that one was discarded. - pump(400_000); - expect(field.warm).toBe(false); - expect(pulseNodes(graph)).toEqual([]); - expect(frames).toHaveLength(0); - }); - - // The canvas used to read `prefers-reduced-motion` directly, which meant the - // app's own persisted control, the one a reader actually sets, and the only - // way to ask for stillness on an OS that reports no preference, had no effect - // on the single most motion-heavy surface in the product. These two cover both - // directions of that pin, because a control that can only agree with the OS is - // not a control. - it('honours a pinned "reduced" preference on an OS that reports no preference', async () => { - localStorage.setItem('td.motion-preference', 'reduced'); - const field = new ActivationField({ halfLifeMs: 4200 }); - render(); - await waitFor(() => expect(sigmaState.graph).toBeDefined()); - const graph = sigmaState.graph!; - - field.strike(['p1'], 0.9); - field.strike(['repo:r'], 0.3); - - expect(frames).toHaveLength(0); - expect(pulseNodes(graph)).toEqual([]); - // The reading still arrives, statically. - expect(graph.hasNode('__halo__p1')).toBe(true); - }); - - it('honours a pinned "full" preference on an OS that asks for reduced motion', async () => { - Object.defineProperty(window, 'matchMedia', { - configurable: true, - value: vi.fn().mockReturnValue({ matches: true }), - }); - localStorage.setItem('td.motion-preference', 'full'); - const field = new ActivationField({ halfLifeMs: 4200 }); - render(); - await waitFor(() => expect(sigmaState.graph).toBeDefined()); - const graph = sigmaState.graph!; - - field.strike(['p1'], 0.9); - field.strike(['repo:r'], 0.3); - - expect(frames).toHaveLength(1); - pump(0); - expect(pulseNodes(graph)).toEqual(['__pulse__0']); - }); - - it('stops a running loop the moment motion is turned off mid-flight', async () => { - const field = new ActivationField({ halfLifeMs: 4200 }); - const { rerender } = render( - , - ); - await waitFor(() => expect(sigmaState.graph).toBeDefined()); - const graph = sigmaState.graph!; - - field.strike(['p1'], 0.9); - field.strike(['repo:r'], 0.3); - pump(0); - expect(pulseNodes(graph)).toEqual(['__pulse__0']); - - // Setting the preference notifies every `useReducedMotion` subscriber; React - // re-renders and the canvas settles. A shortened animation would leave the - // traveller parked somewhere on the curve, a genuine no-motion path has - // nowhere for it to be, so it is gone. - setMotionPreference('reduced'); - rerender(); - await waitFor(() => expect(pulseNodes(graph)).toEqual([])); - expect(frames).toHaveLength(0); - expect(field.warm).toBe(true); - }); - - it('does not rebuild the renderer when only the select handler identity changes', async () => { - const field = new ActivationField(); - const { rerender } = render( - {}} />, - ); - await waitFor(() => expect(sigmaState.graph).toBeDefined()); - const first = sigmaState.graph; - - rerender( - {}} />, - ); - rerender( - {}} - />, - ); - - // Same renderer, same laid-out graph: a live event arriving every second - // must not cost a teardown and a fresh force layout. - expect(sigmaState.graph).toBe(first); - }); -}); diff --git a/dashboard/src/viz/graph/GraphCanvas.tsx b/dashboard/src/viz/graph/GraphCanvas.tsx deleted file mode 100644 index 4d3781d833..0000000000 --- a/dashboard/src/viz/graph/GraphCanvas.tsx +++ /dev/null @@ -1,661 +0,0 @@ -import { useCallback, useEffect, useRef, useState, type ReactNode } from 'react'; -import { ActivationField } from './activation.ts'; -import { isMeasuredField } from './layout.ts'; -import { hasWebGl, watchWebGlContext } from './renderer.ts'; -import { - buildEmergentScene, - buildMeasuredScene, - type GraphScene, - type SceneRequest, -} from './scene.ts'; -import { - DEFAULT_ENCODING, - type FieldExtent, - type GraphCanvasEdge, - type GraphCanvasEncoding, - type GraphCanvasNode, -} from './types.ts'; -import { useReducedMotion } from '../trace/reducedMotion.ts'; -import { EvidencePattern } from '../../ui/EvidencePattern'; -import { cn } from '../../ui/cn'; - -export type { GraphCanvasEdge, GraphCanvasEncoding, GraphCanvasNode } from './types.ts'; - -/** Sigma over Graphology, default connected-graph renderer. - * - * Deterministic ForceAtlas2 settle (laid out once, never animated), nodes - * sized by degree and lit by their real vitality, relations drawn as curved - * connective tissue rather than chords. Everything that moves is a response to - * a real event: an activation strike from the live stream. Pointer focus only - * isolates the neighborhood. At rest the field is completely still and the render loop is - * asleep. The synchronized list next to the canvas remains the accessible - * surface. - * - * This component owns the React side only, the props, the container's - * measured box, and when a scene may exist. Preparing the layout, drawing it - * and animating it live in `layout.ts`, `renderer.ts` and - * `activationOverlay.ts`; `scene.ts` composes the three into one thing with - * one lifetime. */ -export function GraphCanvas({ - nodes, - edges, - selectedId, - onSelect, - inspectedId, - onInspect, - cameraControls = false, - height = 320, - fill = false, - activation, - canvasClassName, - caption, - encoding = DEFAULT_ENCODING, - ariaLabel, - fallbackDescription, - extent, - overlay, -}: { - nodes: GraphCanvasNode[]; - edges: GraphCanvasEdge[]; - selectedId?: string | null; - onSelect?: (id: string | null) => void; - /** Focus shared with the caller's accessible list or inspector. */ - inspectedId?: string | null; - /** Receives pointer focus from the canvas. Focus never creates activity. */ - onInspect?: (id: string | null) => void; - /** Text camera controls for fields whose spatial exploration is meaningful. */ - cameraControls?: boolean; - height?: number; - /** Occupy the parent's full height instead of a fixed one. The parent must - * establish the height (e.g. `flex-1 min-h-0`). */ - fill?: boolean; - /** External synapse field; when omitted the canvas owns an idle local one. */ - activation?: ActivationField; - /** Extra classes merged onto the canvas element itself (not the figure) -- - * for a caller that needs to guarantee a minimum rendered height on a - * breakpoint where its own flex ancestors would otherwise squeeze a `fill` - * canvas toward zero. */ - canvasClassName?: string; - /** What this particular field means. The default sentence describes a - * force-laid symbol graph; any caller composing a different field MUST - * replace it, because the caption is the only place the reader is told what - * position, size and brightness encode, leaving the default on a measured - * layout would state something untrue about the picture. */ - caption?: ReactNode; - /** Compact visible key for the canvas's four visual channels. Callers with - * measured placement or mass must name those meanings explicitly. */ - encoding?: GraphCanvasEncoding; - /** Accessible description of the canvas, for the same reason. */ - ariaLabel?: string; - /** The caller's actual non-canvas continuation. Different graph views expose - * different text alternatives, so a fallback cannot truthfully name one - * generic "symbol list". */ - fallbackDescription?: string; - /** The frame a measured field is drawn in, in the caller's own coordinates. - * Only meaningful alongside placed nodes. Without it the camera frames the - * bodies that happen to exist, so a field with an empty region, no dormant - * projects, say, silently loses that region and the reader is never shown - * the absence. With it, an empty part of the axis stays empty on screen, - * which is the finding. */ - extent?: FieldExtent; - /** HUD drawn over the canvas box and nothing else, legends, scale, the - * rule that chose the slice. Pointer-transparent so it never steals a drag; - * a child that must be operable re-enables its own pointer events. Rendered - * only while a field is drawn, so a failure state is never decorated. */ - overlay?: ReactNode; -}) { - const unknownDegreeCount = nodes.filter((node) => node.degree == null).length; - const containerRef = useRef(null); - /** The live scene, or nothing. Every out-of-band poke at the renderer, - * a selection repaint, a resize, a theme flip, a strike, goes through this, - * so none of them can reach a renderer that has already been killed. */ - const sceneRef = useRef(null); - /** - * Tears the live scene down. Held in a ref so the size observer can call - * it synchronously, ahead of any frame Sigma has scheduled for itself. - */ - const teardownRef = useRef<(() => void) | null>(null); - const resizeObserverRef = useRef(null); - /** - * The container's last measured box. - * - * Sigma's render path calls `resize()`, which THROWS on a zero-width - * container, and one of the callers of that path is a `window` resize - * listener installed inside Sigma that no guard on our side can reach. So - * the renderer's lifetime is bound to a real measured box rather than - * merely started once one appears: it is built when the container has been - * measured non-zero, and torn down the moment the measurement says it has - * none. That is why this is observed state and not a mount-time retry, a - * retry answers "has it arrived yet", and the error we were getting came - * from the other direction, a container that had a box and then lost it as - * its workspace was navigated away from. - */ - const [box, setBox] = useState<{ width: number; height: number }>({ width: 0, height: 0 }); - /** - * Whether the container has a box at all, the only distinction the renderer's - * lifetime turns on. - * - * Sigma reads `offsetWidth` in `resize()` and throws on a 0×0 or detached - * container, so a renderer may exist exactly while this is true. How large the - * box is does not affect that, which is why the mount effect depends on this - * boolean and the dimensions drive `resize()` instead. - */ - const hasBox = box.width > 0 && box.height > 0; - /** Bumped whenever a collapse killed a live renderer, so the mount effect can - * rebuild even when the box it measures never appeared to change. */ - const [teardownGeneration, setTeardownGeneration] = useState(0); - const [layoutPendingFor, setLayoutPendingFor] = useState(null); - /** - * The topology whose layout engine failed to load, if one did. - * - * An emergent field fetches ForceAtlas2 on demand, so for the first time - * this canvas has a way to fail that is neither "no context" nor "too - * large". Drawing the seed circle instead would be a lie, a ring of nodes - * is a composition, and the reader would read meaning into it, so the - * failure is stated. Held as the topology it happened to rather than a - * boolean, so a caller handing over different nodes gets a fresh attempt - * without any reset of its own. - */ - const [engineFailedFor, setEngineFailedFor] = useState( - null, - ); - /** - * The topology whose GPU context was lost after it had been drawn, if one - * was. - * - * The only failure on this canvas that arrives after a successful frame, and - * the one with no symptom of its own: a lost context leaves the last drawn - * pixels frozen, or clears them to nothing, and either reading is false. So - * it is stated, and held as the topology it happened to for the same reason - * the engine failure is, different nodes are a different attempt. - */ - const [contextLostFor, setContextLostFor] = useState( - null, - ); - /** - * Releases the watch on the WebGL layers of the renderer that is, or was, - * live. - * - * Held for as long as this canvas is mounted rather than for one scene's - * lifetime: a lost context has to take its renderer down with it, and the - * restore that brings the field back is dispatched at the canvas of the - * renderer that died. A watch released with the scene could report the loss - * but never the recovery. - */ - const contextWatchRef = useRef<(() => void) | null>(null); - - /** - * Attach the observer as the container mounts rather than in an effect: an - * effect would need the container in its own dependency list to notice it - * appearing, and the element is behind three early returns. - */ - const attachContainer = useCallback((node: HTMLDivElement | null) => { - containerRef.current = node; - resizeObserverRef.current?.disconnect(); - resizeObserverRef.current = null; - if (!node) { - setBox({ width: 0, height: 0 }); - return; - } - const measure = (): void => { - // `offsetWidth`, matching what Sigma itself reads in `resize()`. A - // display:none ancestor and a detached node both report 0 here, which - // are exactly the two states that make Sigma throw. - const width = node.offsetWidth; - const height = node.offsetHeight; - if (width === 0 || height === 0) { - // Synchronous, before React re-renders: a scheduled Sigma frame would - // otherwise reach `resize()` first and throw. Killing here also - // removes Sigma's own window-resize listener. - const teardown = teardownRef.current; - teardown?.(); - // This teardown is imperative, so the mount effect cannot infer it from - // the box alone: a collapse and a re-expansion that land in one commit - // leave the measured box non-zero at both ends, and the effect would - // see no change to react to and never rebuild the renderer it no longer - // has. The generation makes the teardown itself observable. - if (teardown) setTeardownGeneration((generation) => generation + 1); - } - setBox((previous) => - previous.width === width && previous.height === height - ? previous - : { width, height }, - ); - }; - measure(); - // Same guard the other observing surfaces use. Without a ResizeObserver - // the one measurement above still lets a sized container mount; what is - // lost is the teardown on collapse, which is the honest degradation. - if (typeof ResizeObserver !== 'function') return; - const observer = new ResizeObserver(measure); - observer.observe(node); - resizeObserverRef.current = observer; - }, []); - const webglRef = useRef(null); - if (webglRef.current === null) webglRef.current = hasWebGl(); - const fieldRef = useRef(null); - if (activation) fieldRef.current = activation; - else if (!fieldRef.current) fieldRef.current = new ActivationField(); - const field = fieldRef.current; - // Selection and the select handler are read through refs rather than closed - // over, so they can change without re-running the mount effect. They used to - // sit in its dependency list, and `onSelect` is an inline arrow at every call - // site: every parent render, including one per live SSE pulse, tore the - // renderer down and re-ran a 200-iteration ForceAtlas2 layout. That both - // burned a layout per event and hid the sleeping render loop behind a - // remount. The effect now depends on topology alone. - const selectedIdRef = useRef(selectedId); - selectedIdRef.current = selectedId; - const onSelectRef = useRef<((id: string | null) => void) | undefined>(onSelect); - onSelectRef.current = onSelect; - const inspectedIdRef = useRef(inspectedId); - inspectedIdRef.current = inspectedId; - const onInspectRef = useRef<((id: string | null) => void) | undefined>(onInspect); - onInspectRef.current = onInspect; - // The app's persisted three-state motion control, not the bare OS query this - // used to read: pinning "Reduced" had no effect on the field, which is the one - // surface in the product where motion is actually the point. Held in a ref for - // the same reason selection is, the renderer costs a 200-iteration - // ForceAtlas2 layout to build, so a preference flip must reach the live render - // loop without tearing the field down and re-laying it out. - const { reduced } = useReducedMotion(); - const reducedRef = useRef(reduced); - reducedRef.current = reduced; - - // Selection is a static repaint, not an animation: recolour once and leave - // the loop asleep. `sceneRef` is cleared the moment the container loses its - // box, so this cannot repaint into a renderer that has nothing to measure. - useEffect(() => { - sceneRef.current?.repaint(); - }, [selectedId]); - - // The accessible list and the WebGL field are two views of the same focus. - // Applying list focus only repaints isolation; it never strikes the field. - useEffect(() => { - sceneRef.current?.focusNode(inspectedId ?? null); - }, [inspectedId]); - - // A caller-owned field is struck from entirely outside this component: the - // Brain's SSE effect calls `field.strike(...)` when a real event lands, with - // no knowledge of any render loop. If the loop is asleep (which, correctly, - // it is whenever the field is cold) that heat would sit undrawn and - // undecayed forever. Subscribing turns every real strike, wherever it - // originates, into exactly one wake, and nothing else can produce one, - // because the field has no clock. The subscription belongs to the FIELD, not - // to any one scene: a strike that arrives while no scene exists finds - // nothing to wake, which is the same nothing the unsubscribed version did. - useEffect(() => field.subscribe(() => sceneRef.current?.wake()), [field]); - - // A theme flip is a property of the document, not of any one renderer, so - // the observer lives as long as this canvas does and re-samples whichever - // scene is live at the time, or nothing, if none is. - useEffect(() => { - const observer = new MutationObserver(() => sceneRef.current?.retheme()); - observer.observe(document.documentElement, { - attributes: true, - attributeFilter: ['data-theme'], - }); - return () => observer.disconnect(); - }, []); - - // The context watch is the one piece of wiring that outlives the scene it was - // armed for, so releasing it is this canvas's own last act. The effect below - // cannot: it has already unwound, along with the container it owned, by the - // time a restore can arrive. - useEffect(() => () => contextWatchRef.current?.(), []); - - // The scene's own lifetime. The dependency list is deliberately exactly the - // set that invalidates a composed field: its topology, the axis it is framed - // in, whether the container has a box at all, and any teardown that happened - // out of band. Everything else the scene needs, selection, the select - // handler, the motion preference, the activation field, is read through a - // ref precisely so it can change without costing a layout. - useEffect(() => { - const container = containerRef.current; - if (!container || nodes.length === 0 || !webglRef.current) return; - // Not a retry: `hasBox` is derived from the observed measurement, so this - // effect re-runs by itself once the container has one, and unwinds again if - // it loses it. - if (!hasBox) return; - - let cancelled = false; - const layoutAbort = new AbortController(); - let detach: (() => void) | null = null; - const request: SceneRequest = { - container, - nodes, - edges, - extent, - // Read through the ref for the same reason selection is: swapping the - // field a caller owns must not cost a teardown and a fresh layout. - field: fieldRef.current ?? field, - selectedId: () => selectedIdRef.current, - inspectedId: () => inspectedIdRef.current, - onSelect: (id) => onSelectRef.current?.(id), - onInspect: (id) => onInspectRef.current?.(id), - isReduced: () => reducedRef.current, - }; - const install = (scene: GraphScene): void => { - sceneRef.current = scene; - detach = () => { - scene.teardown(); - if (sceneRef.current === scene) sceneRef.current = null; - if (teardownRef.current === detach) teardownRef.current = null; - }; - teardownRef.current = detach; - // Sigma keeps no watch of its own, so a GPU context dropped after this - // field was drawn would leave a frozen or blank canvas on screen while - // everything around it went on presenting a drawn graph. - contextWatchRef.current?.(); - contextWatchRef.current = watchWebGlContext(scene.webGlCanvases, { - onLost: () => { - if (cancelled) return; - // Synchronously, ahead of any frame the overlay has already asked - // for: the renderer died with its context, and this is the same - // one-way latch the size observer tears a live scene down through, - // so the animation loop stops and the pointer wiring goes with it. - detach?.(); - setContextLostFor(nodes); - }, - onRestored: () => { - // Deliberately NOT gated on `cancelled`: by the time a restore lands - // this effect has unwound along with the container it owned, which is - // precisely why nothing is installed from here. Clearing the state - // re-renders the container and the generation makes the rebuild - // observable to the effect, which then owns whatever it measures. - setContextLostFor(null); - setTeardownGeneration((generation) => generation + 1); - }, - }); - }; - - if (isMeasuredField(nodes)) { - // Synchronous, and reaching no layout engine at all: the coordinates are - // already the caller's measurement. - install(buildMeasuredScene(request)); - } else { - // Cancel the bounded worker when this topology loses its container; - // no late result may install a scene into a newer or collapsed field. - setLayoutPendingFor(nodes); - const cancelLayout = (): void => layoutAbort.abort(); - teardownRef.current = cancelLayout; - void buildEmergentScene(request, layoutAbort.signal).then( - (scene) => { - if (cancelled || layoutAbort.signal.aborted) { - scene?.teardown(); - return; - } - setLayoutPendingFor(null); - if (scene) install(scene); - }, - () => { - if (!cancelled && !layoutAbort.signal.aborted) { - setLayoutPendingFor(null); - setEngineFailedFor(nodes); - } - }, - ); - } - - return () => { - cancelled = true; - layoutAbort.abort(); - if (!detach) teardownRef.current = null; - detach?.(); - }; - }, [nodes, edges, extent, hasBox, teardownGeneration]); - - /** - * A resize of a container that still has a box is a resize, not a remount. - * - * The renderer's lifetime is bound to `hasBox` above rather than to the - * measured numbers, because depending on the numbers made every drag of a - * window edge or opening of a side panel kill the renderer, rebuild the whole - * graphology graph and re-run the 200-iteration ForceAtlas2 settle, a - * layout per resize frame. Only the zero/non-zero transition changes what - * Sigma can legally do; every other change is something Sigma resizes itself - * into. `resize()` is also what Sigma's own window listener would call. - */ - useEffect(() => { - if (!hasBox) return; - sceneRef.current?.resize(); - }, [hasBox, box.width, box.height]); - - // Turning motion off has to take effect on the field the reader is looking at, - // not merely on the next one they open: a loop already running keeps running - // until something stops it. Turning it back on needs no counterpart, the next - // real event wakes the loop through `wake`. - useEffect(() => { - if (reduced) sceneRef.current?.settle(); - }, [reduced]); - - if (nodes.length === 0) { - return ( -

    - no graph neighborhood to draw -

    - ); - } - // Sigma is WebGL-only and throws during construction without a context, - // which React Router's error boundary turns into a dead workspace. Browsers - // with WebGL disabled or blocklisted get the truthful state instead. - if (!webglRef.current) { - return ( - - this browser has no WebGL context, so the {nodes.length.toLocaleString()}-symbol - graph canvas cannot draw, {fallbackDescription ?? 'read the field description below'} - - ); - } - // This canvas admits at most 5,000 nodes. Larger returned sets retain their - // DOM evidence; no alternate renderer is mounted for them. - if (nodes.length > 5_000) { - return ( - - {nodes.length.toLocaleString()} symbols exceeds this canvas's 5,000-symbol limit. - Narrow the neighborhood to draw it here. - - ); - } - if (engineFailedFor === nodes) { - return ( - - the force layout could not be completed, so the{' '} - {nodes.length.toLocaleString()}-symbol graph canvas has no positions to - draw, {fallbackDescription ?? 'read the field description below'} - - ); - } - // The one failure that happens to a field that WAS drawn. The frozen last - // frame is the trap: it is a picture of a graph, so it reads as a live one, - // and a cleared context reads as an empty one. Neither is a reading, so the - // canvas is taken down and this is said in its place. - if (contextLostFor === nodes) { - return ( - - the graph canvas lost its WebGL context, so the{' '} - {nodes.length.toLocaleString()}-symbol field is no longer being drawn, - {fallbackDescription ?? 'read the field description below'}, and the field - returns if the browser restores the context - - ); - } - return ( -
    - {/* The canvas box and its HUD share one frame so the overlay is measured - * against the field alone and never drifts down over the caption. The - * frame carries the fill geometry; the container inside it keeps the - * box Sigma measures, exactly as before. */} -
    -
    - {overlay ? ( -
    - {overlay} -
    - ) : null} -
    - {layoutPendingFor === nodes ? ( -

    - Calculating graph positions. The symbol list remains available. -

    - ) : null} - {cameraControls ? ( -
    - - - -
    - ) : null} -
    - - {unknownDegreeCount > 0 ? ( - // Provenance is carried by the shared evidence PATTERN axis, not by - // prose alone: the dashed `unknown` swatch says "this quantity was - // never measured" in the same visual language the rest of the app - // uses, and it survives monochrome and forced-colors. The sentence - // stays, because the pattern says which class of evidence this is and - // only the sentence says what was missing. -

    - - - Connectedness is absent for {unknownDegreeCount}{' '} - {unknownDegreeCount === 1 ? 'symbol' : 'symbols'}; each uses the - minimum marker, not zero. - -

    - ) : null} -
    - {caption ?? ( - <> - {nodes.length} symbols · {edges.length} relations · hover isolates - a neighbourhood · activity glow follows supplied events - - )} -
    -
    -
    - ); -} - -/** A field that is NOT being drawn, no WebGL context, no layout engine, a - * graph past this renderer's tier, or a context the GPU took back after the - * field had been composed. - * - * Distinct from an empty field on purpose, and the distinction has to be visible - * rather than only readable: "nothing is here" and "this could not be rendered" - * are different claims, and a quiet line of muted prose reads as the first. - * Wearing the dashed `unknown` evidence pattern states in the app's own visual - * language that no measurement backs this region, so it can never be mistaken - * for a drawn graph that happens to be sparse. Deliberately NOT given the - * atmospheric graph field: the aperture treatment is what a rendered field - * looks like, and lending it to a failure is exactly the kind of beautiful - * smoothing-over that would make a failure look like data. */ -export function GraphUnavailable({ children }: { children: ReactNode }) { - return ( -
    - -

    {children}

    - -
    - ); -} - -function GraphEncodingKey({ encoding }: { encoding: GraphCanvasEncoding }) { - const items = [ - { label: 'disc', value: encoding.body }, - { label: 'size', value: encoding.size }, - { label: 'hue', value: encoding.hue }, - { label: 'glow', value: encoding.signal }, - { label: 'line', value: encoding.relation }, - ]; - return ( -
    - {items.map((item, index) => ( - - - {index === 0 ? ( - - ) : null} - {item.label} - - - · - - - {item.value} - - - ))} -
    - ); -} diff --git a/dashboard/src/viz/graph/activation.test.ts b/dashboard/src/viz/graph/activation.test.ts index 00585adb55..9c43511435 100644 --- a/dashboard/src/viz/graph/activation.test.ts +++ b/dashboard/src/viz/graph/activation.test.ts @@ -1,35 +1,46 @@ import { describe, expect, it, vi } from 'vitest'; -import { ActivationField, lerpRgbTuple, luma, restingNodeTint } from './activation.ts'; +import { ActivationField, luma, restingNodeTint } from './activation.ts'; describe('ActivationField subscription', () => { - it('stays silent when a strike carries no ids, nothing real happened', () => { + it('notifies on a strike that lands heat, and stays silent on one that carries no ids', () => { const field = new ActivationField(); const listener = vi.fn(); field.subscribe(listener); field.strike([], 1); - expect(listener).not.toHaveBeenCalled(); + expect(listener).toHaveBeenCalledTimes(0); + expect(field.warm).toBe(false); + + field.strike(['a'], 0.5); + expect(listener).toHaveBeenCalledTimes(1); + expect(field.warm).toBe(true); + expect(field.heatOf('a')).toBe(0.5); }); it('never fires on its own: decay is not an event', () => { // The field has no clock. `tick` is decay bookkeeping driven by whoever is // already drawing; if it notified, a renderer would wake itself forever. const field = new ActivationField({ halfLifeMs: 100 }); - field.strike(['a'], 1); const listener = vi.fn(); field.subscribe(listener); - field.tick(0); - field.tick(1_000); - field.tick(2_000); - expect(listener).not.toHaveBeenCalled(); + field.strike(['a'], 1); + expect(listener).toHaveBeenCalledTimes(1); + expect(field.tick(1_000)).toBe(true); + expect(field.tick(2_000)).toBe(false); + expect(listener).toHaveBeenCalledTimes(1); expect(field.warm).toBe(false); }); - it('stops notifying once unsubscribed', () => { + it('stops notifying once unsubscribed, while strikes still land', () => { const field = new ActivationField(); const listener = vi.fn(); - field.subscribe(listener)(); + const unsubscribe = field.subscribe(listener); field.strike(['a'], 1); - expect(listener).not.toHaveBeenCalled(); + expect(listener).toHaveBeenCalledTimes(1); + + unsubscribe(); + field.strike(['b'], 1); + expect(listener).toHaveBeenCalledTimes(1); + expect(field.heatOf('b')).toBe(1); }); }); @@ -48,25 +59,20 @@ const DARK_KIND: [number, number, number] = [110, 205, 215]; const LIGHT_KIND: [number, number, number] = [150, 170, 175]; describe('restingNodeTint', () => { - it('leaves the dark theme unchanged: headroom already clears the floor', () => { - for (const vitality of [0, 0.25, 0.6, 1]) { - const mix = 0.34 + 0.66 * vitality; - const raw = lerpRgbTuple(DARK_SUBSTRATE, DARK_KIND, mix); - expect(restingNodeTint(DARK_SUBSTRATE, DARK_KIND, vitality, false)).toEqual(raw); - } + it('leaves the dark theme on the plain substrate-to-kind mix: headroom already clears the floor', () => { + expect(restingNodeTint(DARK_SUBSTRATE, DARK_KIND, 0, false)).toEqual([56, 90, 97]); + expect(restingNodeTint(DARK_SUBSTRATE, DARK_KIND, 0.25, false)).toEqual([69, 118, 126]); + expect(restingNodeTint(DARK_SUBSTRATE, DARK_KIND, 0.6, false)).toEqual([88, 159, 168]); + expect(restingNodeTint(DARK_SUBSTRATE, DARK_KIND, 1, false)).toEqual([110, 205, 215]); }); it('keeps a fully-dormant light-theme node from washing into the substrate', () => { + // The un-nudged 0.34 mix of these fixtures is [213, 220, 223], about 28 + // luma units under the paper; the nudge darkens it to clear the floor, up + // to integer-channel rounding. const tint = restingNodeTint(LIGHT_SUBSTRATE, LIGHT_KIND, 0, true); - const rawMix = lerpRgbTuple(LIGHT_SUBSTRATE, LIGHT_KIND, 0.34); - const rawOffset = luma(LIGHT_SUBSTRATE) - luma(rawMix); - const nudgedOffset = luma(LIGHT_SUBSTRATE) - luma(tint); - // The un-nudged 0.34 mix of these fixtures clears well under the floor; - // the nudge must close nearly all of that gap (integer-channel rounding - // accounts for the last fraction of a luma unit). - expect(rawOffset).toBeLessThan(30); - expect(nudgedOffset).toBeGreaterThanOrEqual(41); - expect(nudgedOffset).toBeGreaterThan(rawOffset); + expect(tint).toEqual([199, 206, 208]); + expect(luma(LIGHT_SUBSTRATE) - luma(tint)).toBeGreaterThanOrEqual(41); }); }); diff --git a/dashboard/src/viz/graph/activationOverlay.ts b/dashboard/src/viz/graph/activationOverlay.ts deleted file mode 100644 index edafd19196..0000000000 --- a/dashboard/src/viz/graph/activationOverlay.ts +++ /dev/null @@ -1,247 +0,0 @@ -import type Graph from 'graphology'; -import { - approach, - lerpRgbTuple, - restingNodeTint, - settled, - type ActivationField, -} from './activation.ts'; -import { BLOOM, HALO, PULSE, RING, upsert, type Strand } from './managed.ts'; -import { rgba, type ThemeBox } from './palette.ts'; -import type { FocusState } from './renderer.ts'; - -/** - * The activation overlay: everything on the field that is a response to a real - * event rather than a fact about the graph. - * - * It owns the glow companions (a body's corona and bloom), the light that - * travels a warm dendrite, and the render loop that resolves both. The loop - * runs only while something real is unresolved, a warm activation field, or a - * hover isolation still easing into place, and stops itself the moment both - * settle, so an idle dashboard costs nothing. It also advances the hover - * easing, because that easing shares the same frames; the renderer owns where - * the hover points, this owns how fast it gets there. - */ - -export interface ActivationOverlayOptions { - graph: Graph; - /** The caller's own nodes; companions are gated on this count, never on the - * graph's, which the dendrite pass has already inflated with waypoints. */ - realNodes: readonly string[]; - strands: readonly Strand[]; - field: ActivationField; - theme: ThemeBox; - focus: FocusState; - /** Repaint the renderer. A no-op once the scene is gone. */ - paint: () => void; - /** Read per call, never captured: the reader can change this while the field - * is on screen and every decision below must see the new answer. */ - isReduced: () => boolean; -} - -export interface ActivationOverlay { - /** Start (or keep) the loop, or compose statically under reduced motion. */ - wake(): void; - /** The no-motion composition: jump every eased quantity to its destination, - * remove the travelling light entirely, and paint once. */ - settle(): void; - /** One static composition of the resting field. */ - repaintResting(): void; - stop(): void; -} - -export function createActivationOverlay({ - graph, - realNodes, - strands, - field, - theme, - focus, - paint, - isReduced, -}: ActivationOverlayOptions): ActivationOverlay { - // ---- glow companions ------------------------------------------------ - // Every point is a body with falloff, not a flat disc: a tight corona in - // the node's own hue plus a wide, very faint bloom give depth without a - // shader. Both ride real signal, vitality at rest, heat when struck, so - // a quiet graph is genuinely quieter, not merely smaller. - const restingGlow = realNodes.length <= 400; - const syncGlow = (): void => { - const colors = theme.colors; - for (const node of realNodes) { - const heat = field.heatOf(node); - const haloId = HALO + node; - const bloomId = BLOOM + node; - const ringId = RING + node; - if (heat > 0.1 || restingGlow) { - const attrs = graph.getNodeAttributes(node); - const size = attrs['size'] as number; - const vitality = (attrs['vitality'] as number | undefined) ?? 0.6; - const [kr, kg, kb] = - (attrs['kindRgb'] as [number, number, number] | undefined) ?? colors.hot; - const resting = restingNodeTint( - colors.substrate, - [kr, kg, kb], - vitality, - colors.light, - ); - const lit = heat > 0 - ? lerpRgbTuple(resting, colors.hot, Math.min(1, heat)) - : resting; - const shared = { x: attrs['x'], y: attrs['y'], label: null, type: 'glow', owner: node }; - const haloAlpha = 0.045 + 0.07 * vitality + 0.22 * heat; - const bloomAlpha = 0.012 + 0.022 * vitality + 0.08 * heat; - // The companion program fades these extents radially. Their size and - // brightness still derive only from measured vitality and real heat. - upsert(graph, haloId, { - ...shared, - // Tight enough to read as a luminous rim, not a second donut body. - size: size * (1.11 + 0.55 * heat), - color: rgba(lit, haloAlpha), - glowRgb: lit, - glowAlpha: haloAlpha, - zIndex: 1, - }); - upsert(graph, bloomId, { - ...shared, - size: size * (1.68 + 1.3 * heat), - color: rgba(lit, bloomAlpha), - glowRgb: lit, - glowAlpha: bloomAlpha, - zIndex: 0, - }); - // Impact flare: a wide, faint ring pops on strike and expands as the - // bloom settles, so a firing is legible even in peripheral vision. - if (heat > 0.5) { - const ringAlpha = 0.075 * heat; - upsert(graph, ringId, { - ...shared, - size: size * (2.1 + 2.2 * (1 - heat)), - color: rgba(colors.hot, ringAlpha), - glowRgb: colors.hot, - glowAlpha: ringAlpha, - zIndex: 1, - }); - } else if (graph.hasNode(ringId)) { - graph.dropNode(ringId); - } - } else { - for (const id of [haloId, bloomId, ringId]) { - if (graph.hasNode(id)) graph.dropNode(id); - } - } - } - }; - - // ---- travelling light ------------------------------------------------ - // While a relation is warm, one bright point runs the dendrite from the - // hotter end to the cooler one, following the curve rather than the chord. - // Pulses exist only while the field is warm, and the frozen bbox keeps - // them from ever rescaling the camera. - const syncPulses = (now: number): void => { - const period = 1100; - const phase = (now % period) / period; - for (let index = 0; index < strands.length; index += 1) { - const strand = strands[index]!; - const heatFrom = field.heatOf(strand.from); - const heatTo = field.heatOf(strand.to); - const travel = Math.max(heatFrom, heatTo); - const pulseId = `${PULSE}${index}`; - if (travel > 0.18 && !isReduced()) { - const forward = heatFrom >= heatTo; - const points = strand.points; - const spans = points.length - 1; - const walked = (forward ? phase : 1 - phase) * spans; - const span = Math.max(0, Math.min(spans - 1, Math.floor(walked))); - const local = walked - span; - const a = points[span]!; - const b = points[span + 1]!; - upsert(graph, pulseId, { - x: a[0] + (b[0] - a[0]) * local, - y: a[1] + (b[1] - a[1]) * local, - size: 1.4 + 2 * travel, - color: rgba(theme.colors.hot, 0.9 * travel), - label: '', - zIndex: 4, - }); - } else if (graph.hasNode(pulseId)) { - graph.dropNode(pulseId); - } - } - }; - - const dropPulses = (): void => { - for (const node of [...graph.nodes()]) { - if (node.startsWith(PULSE)) graph.dropNode(node); - } - }; - - let stopped = false; - let lastFrame = 0; - let raf = 0; - - // Reduced motion never starts the loop: state is applied in one static - // refresh instead. - const step = (now: number): void => { - const delta = lastFrame === 0 ? 16 : now - lastFrame; - lastFrame = now; - const warm = field.tick(now); - focus.t = approach(focus.t, focus.target, delta, 90); - const focusSettled = settled(focus.t, focus.target); - if (focusSettled) { - focus.t = focus.target; - if (focus.target === 0) focus.node = null; - } - syncGlow(); - syncPulses(now); - if (!warm) dropPulses(); - paint(); - const keepGoing = !stopped && (warm || !focusSettled); - raf = keepGoing && !isReduced() ? requestAnimationFrame(step) : 0; - if (!keepGoing) lastFrame = 0; - }; - - /** Nothing here is "faster", the intermediate frames do not exist. */ - const settle = (): void => { - if (raf) { - cancelAnimationFrame(raf); - raf = 0; - } - lastFrame = 0; - field.tick(performance.now()); - focus.t = focus.target; - if (focus.target === 0) focus.node = null; - // A pulse is pure travel, so under reduced motion it has no resting form - // to snap to; it is dropped rather than parked somewhere along its curve. - dropPulses(); - syncGlow(); - paint(); - }; - - const wake = (): void => { - if (isReduced()) { - settle(); - return; - } - if (!raf) { - lastFrame = 0; - raf = requestAnimationFrame(step); - } - }; - - return { - wake, - settle, - repaintResting: () => { - syncGlow(); - paint(); - }, - stop: () => { - stopped = true; - if (raf) { - cancelAnimationFrame(raf); - raf = 0; - } - }, - }; -} diff --git a/dashboard/src/viz/graph/adjacency.test.ts b/dashboard/src/viz/graph/adjacency.test.ts deleted file mode 100644 index 70e47b7bee..0000000000 --- a/dashboard/src/viz/graph/adjacency.test.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { buildAdjacency, neighborsOf } from './adjacency.ts'; - -describe('buildAdjacency', () => { - it('makes a drawn relation conduct in both directions', () => { - const adjacency = buildAdjacency([{ source: 'repo:r', target: 'p1' }]); - expect(neighborsOf(adjacency, 'p1')).toEqual(['repo:r']); - expect(neighborsOf(adjacency, 'repo:r')).toEqual(['p1']); - }); - - it('stops one hop short of siblings that did nothing', () => { - // The Brain's shape: two checkouts of one repository. An event in `p1` - // reaches the hub and must stop there, `p2` is two hops away and nothing - // happened in it. - const adjacency = buildAdjacency([ - { source: 'repo:r', target: 'p1' }, - { source: 'repo:r', target: 'p2' }, - ]); - expect(neighborsOf(adjacency, 'p1')).toEqual(['repo:r']); - expect(neighborsOf(adjacency, 'p1')).not.toContain('p2'); - }); - - it('records a node reached by several relations once', () => { - const adjacency = buildAdjacency([ - { source: 'a', target: 'b' }, - { source: 'b', target: 'a' }, - { source: 'a', target: 'b' }, - ]); - expect(neighborsOf(adjacency, 'a')).toEqual(['b']); - }); - - it('ignores a self-relation rather than letting a node hop to itself', () => { - const adjacency = buildAdjacency([{ source: 'a', target: 'a' }]); - expect(neighborsOf(adjacency, 'a')).toEqual([]); - }); - - it('propagates nowhere from a node with no drawn relation', () => { - expect(neighborsOf(buildAdjacency([]), 'lonely')).toEqual([]); - }); -}); diff --git a/dashboard/src/viz/graph/adjacency.ts b/dashboard/src/viz/graph/adjacency.ts deleted file mode 100644 index ee8bb40c36..0000000000 --- a/dashboard/src/viz/graph/adjacency.ts +++ /dev/null @@ -1,48 +0,0 @@ -import type { GraphCanvasEdge } from './GraphCanvas.tsx'; - -/** - * Undirected one-hop adjacency over the SAME edge list the canvas draws. - * - * Travelling activation is only honest if it travels along relations that - * actually exist. Building the map from the rendered edge set, rather than - * re-deriving "who is next to whom" from whatever data happened to shape the - * graph, means a strike can never light a neighbour the viewer cannot also - * see a line to. If an edge is not on screen, it cannot conduct. - * - * One hop only, deliberately. Each additional hop is a further claim about - * what happened, and the claim gets weaker with distance: on the Brain graph - * one hop from a checkout reaches its repository (true, activity in a - * checkout is activity in that repository), while two hops would reach its - * sibling checkouts and assert something that did not happen at all. - */ -export function buildAdjacency( - edges: readonly GraphCanvasEdge[], -): ReadonlyMap { - const adjacency = new Map(); - const link = (from: string, to: string) => { - const existing = adjacency.get(from); - if (existing) { - if (!existing.includes(to)) existing.push(to); - } else { - adjacency.set(from, [to]); - } - }; - for (const edge of edges) { - if (edge.source === edge.target) continue; - link(edge.source, edge.target); - link(edge.target, edge.source); - } - return adjacency; -} - -/** The neighbours of `id`, or an empty list when the node has none. Returning - * an empty list (never `undefined`) keeps a caller from having to decide what - * "no adjacency recorded" means: an isolated node simply propagates nowhere. */ -export function neighborsOf( - adjacency: ReadonlyMap, - id: string, -): readonly string[] { - return adjacency.get(id) ?? EMPTY; -} - -const EMPTY: readonly string[] = []; diff --git a/dashboard/src/viz/graph/fieldRenderers/emergentPositions.ts b/dashboard/src/viz/graph/fieldRenderers/emergentPositions.ts new file mode 100644 index 0000000000..6bf664080a --- /dev/null +++ b/dashboard/src/viz/graph/fieldRenderers/emergentPositions.ts @@ -0,0 +1,52 @@ +import { useEffect, useState } from 'react'; +import { prepareField } from '../layout.ts'; +import { settleEmergentOffThread } from '../emergentLayout.ts'; +import type { GraphCanvasEdge, GraphCanvasNode } from '../types.ts'; + +export type EmergentPositions = + | { state: 'pending' } + | { state: 'failed'; reason: string } + | { state: 'ready'; positions: ReadonlyMap }; + +/** + * Force-settled coordinates for a returned graph, computed by the bounded + * emergent-layout worker. `null` input skips layout for renderers that do not + * place symbols spatially. + */ +export function useEmergentPositions( + nodes: readonly GraphCanvasNode[] | null, + edges: readonly GraphCanvasEdge[], +): EmergentPositions { + const [result, setResult] = useState({ state: 'pending' }); + useEffect(() => { + if (!nodes) return; + setResult({ state: 'pending' }); + const abort = new AbortController(); + const prepared = prepareField({ + nodes, + edges, + viewport: { width: 1200, height: 800 }, + kindRgb: () => [0, 0, 0], + }); + settleEmergentOffThread(prepared, abort.signal).then( + (done) => { + if (!done || abort.signal.aborted) return; + const positions = new Map(); + for (const id of prepared.realNodes) { + positions.set(id, [ + prepared.graph.getNodeAttribute(id, 'x') as number, + prepared.graph.getNodeAttribute(id, 'y') as number, + ]); + } + setResult({ state: 'ready', positions }); + }, + (error: unknown) => { + if (!abort.signal.aborted) { + setResult({ state: 'failed', reason: error instanceof Error ? error.message : 'layout failed' }); + } + }, + ); + return () => abort.abort(); + }, [nodes, edges]); + return result; +} diff --git a/dashboard/src/viz/graph/fieldRenderers/fieldRenderers.test.ts b/dashboard/src/viz/graph/fieldRenderers/fieldRenderers.test.ts new file mode 100644 index 0000000000..3123bae267 --- /dev/null +++ b/dashboard/src/viz/graph/fieldRenderers/fieldRenderers.test.ts @@ -0,0 +1,120 @@ +import { describe, expect, it } from 'vitest'; +import { MAX_POINTS, createPicker, pointCloud, unitsPerPoint } from './pointField.ts'; +import { + bodyScreenRadius, + createSpatialIndex, + fitCamera, + resolveZoom, + toScreen, + toWorld, + zoomAt, + type FieldScene, + type SceneBody, + type SceneCluster, +} from './scene.ts'; + +function body(id: string, x: number, y: number, extra: Partial = {}): SceneBody { + return { + id, + label: id, + role: 'body', + kind: 'primary', + x, + y, + radius: 0.1, + mass: 10, + units: { stores: 3, artifacts: 7 }, + vitality: 1, + detail: ['stores 3', 'artifacts 7', 'mass 10', 'seen 1h ago', 'branch main', 'repo r'], + group: null, + cluster: null, + ...extra, + }; +} + +function registry(bodies: SceneBody[], clusters: SceneCluster[] = []): FieldScene { + return { + bodies, + paths: [], + clusters, + extent: { x: [-0.5, 4.5], y: [0, 3] }, + columns: ['today', 'week', 'month', 'quarter', 'dormant'].map((label) => ({ label, bound: label, count: 0 })), + neighbors: new Map(), + }; +} + +const NO_PAD = { top: 0, right: 0, bottom: 0, left: 0 }; + +describe('field camera', () => { + it('fits the extent inside the padded viewport, centred', () => { + const camera = fitCamera({ x: [0, 10], y: [0, 5] }, 1000, 600, NO_PAD); + expect(camera.scale).toBe(100); + expect(toScreen(camera, 0, 5)).toEqual([0, 50]); + expect(toScreen(camera, 10, 0)).toEqual([1000, 550]); + }); + + it('zooms around the pointer, keeping the world point under it fixed', () => { + const camera = fitCamera({ x: [0, 10], y: [0, 5] }, 1000, 600, NO_PAD); + const before = toWorld(camera, 250, 300); + const zoomed = zoomAt(camera, 250, 300, 2); + expect(zoomed.scale).toBe(200); + expect(toWorld(zoomed, 250, 300)).toEqual(before); + }); + + it('grows bodies slower than positions so packed members separate', () => { + expect(bodyScreenRadius(0.1, 100, 100)).toBe(10); + expect(bodyScreenRadius(0.1, 800, 100)).toBeCloseTo(16.82, 2); + expect(resolveZoom(0.1, 0.5)).toBe(1); + expect(resolveZoom(0.24, 0.115)).toBeCloseTo(6.72, 2); + }); +}); + +describe('point field', () => { + it('draws one point per indexed unit, stores first, inside the unit disc', () => { + const cloud = pointCloud({ stores: 3, artifacts: 7 }, 1); + expect(cloud.offsets.length).toBe(20); + expect(cloud.stores).toBe(3); + for (let index = 0; index < 10; index += 1) { + expect(Math.hypot(cloud.offsets[index * 2]!, cloud.offsets[index * 2 + 1]!)).toBeLessThanOrEqual(1); + } + }); + + it('shares one units-per-point ratio once a registry exceeds the point budget', () => { + expect(unitsPerPoint(registry([body('a', 0, 1)]))).toBe(1); + const heavy = registry([body('a', 0, 1, { units: { stores: 1, artifacts: MAX_POINTS * 3 } })]); + expect(unitsPerPoint(heavy)).toBe(4); + expect(pointCloud({ stores: 1, artifacts: MAX_POINTS * 3 }, 4).offsets.length / 2).toBe(1 + MAX_POINTS * 0.75); + }); + + it('picks the body under the pointer and nothing in empty space', () => { + const scene = registry([body('a', 0, 1), body('b', 3, 2)]); + const camera = fitCamera(scene.extent, 1000, 600, NO_PAD); + const picker = createPicker(scene); + const [ax, ay] = toScreen(camera, 0, 1); + expect(picker.pick(camera, camera.scale, ax + 3, ay - 2)).toEqual({ kind: 'body', body: scene.bodies[0] }); + expect(picker.pick(camera, camera.scale, ax + 200, ay)).toBeNull(); + }); + + it('picks an unresolved cell as a whole, and its member once zoomed past the resolve zoom', () => { + const cell: SceneCluster = { id: 'cell:0:0', members: ['a', 'b'], x: 0, y: 1, width: 0.8, height: 0.25, resolveZoom: 4, mass: 20, spacing: 0.1 }; + const scene = registry([body('a', -0.05, 1, { cluster: cell.id }), body('b', 0.05, 1, { cluster: cell.id })], [cell]); + const fit = fitCamera(scene.extent, 1000, 600, NO_PAD); + const picker = createPicker(scene); + const [cx, cy] = toScreen(fit, -0.05, 1); + expect(picker.pick(fit, fit.scale, cx, cy)).toEqual({ kind: 'cluster', cluster: cell }); + const close = zoomAt(fit, cx, cy, 5); + expect(picker.pick(close, fit.scale, cx, cy)).toEqual({ kind: 'body', body: scene.bodies[0] }); + }); + + it('answers a pick from a few index buckets among thousands of bodies', () => { + const bodies = Array.from({ length: 20_000 }, (_, index) => body(`p${index}`, (index % 200) * 0.3, Math.floor(index / 200) * 0.3)); + const index = createSpatialIndex(bodies, 0.25); + const near = index.near(3, 3, 0.2); + expect(near.map((item) => item.id).sort()).toEqual(['p2010']); + const scene = { ...registry(bodies), extent: { x: [0, 60] as [number, number], y: [0, 30] as [number, number] } }; + const camera = fitCamera(scene.extent, 1200, 600, NO_PAD); + const [x, y] = toScreen(camera, 3, 3); + const picked = createPicker(scene).pick(camera, camera.scale, x, y); + expect(picked?.kind === 'body' ? picked.body.id : null).toBe('p2010'); + }); +}); diff --git a/dashboard/src/viz/graph/fieldRenderers/pointField.ts b/dashboard/src/viz/graph/fieldRenderers/pointField.ts new file mode 100644 index 0000000000..429601f6a8 --- /dev/null +++ b/dashboard/src/viz/graph/fieldRenderers/pointField.ts @@ -0,0 +1,631 @@ +import { cssColorToRgb, lerpRgbTuple } from '../activation.ts'; +import { kindColor } from '../kindColor.ts'; +import { + EasedCamera, + SYNAPSE_TRAVEL_MS, + attachCameraGestures, + bodyScreenRadius, + bow, + createFrameLoop, + createSpatialIndex, + createSpriteCache, + drawColumns, + drawGraticule, + drawMassAxis, + fitCamera, + focusCamera, + mountCanvas, + rgbaString, + toScreen, + toWorld, + type CameraState, + type FieldPalette, + type FieldRendererFactory, + type FieldScene, + type FieldView, + type SceneBody, + type SceneCluster, + type Synapse, +} from './scene.ts'; + +/** + * The Brain point field: one additive point per indexed unit. + * + * A project body is a Vogel disc of its own stores (ice, at the core) and + * artifacts (kind hue), so its area is its holdings and its luminance is the + * density of what TraceDecay actually holds. Past {@link MAX_POINTS} units a + * point stands for several, and the legend prints the ratio. A packed cell + * is drawn as one counted frame until the camera is close enough for its + * members to separate. Hand-written Canvas2D: no WebGL context, frames only + * while something real is unresolved. + */ +export const MAX_POINTS = 24_000; +/** Camera zoom bounds, relative to the fitted field. */ +export const MIN_ZOOM = 0.5; +export const MAX_ZOOM = 48; + +const GOLDEN = Math.PI * (3 - Math.sqrt(5)); +const PAD = { top: 44, right: 150, bottom: 28, left: 56 }; +/** Spatial index cell, in field units; about one body diameter. */ +const INDEX_CELL = 0.25; + +export interface PointCloud { + /** Unit offsets inside a unit disc, interleaved x,y. */ + offsets: Float32Array; + /** How many leading points are stores. */ + stores: number; +} + +/** Units drawn per point for a whole scene, so every body shares one ratio. */ +export function unitsPerPoint(scene: FieldScene): number { + const total = scene.bodies.reduce( + (sum, body) => sum + (body.units ? body.units.stores + body.units.artifacts : 0), + 0, + ); + return Math.max(1, Math.ceil(total / MAX_POINTS)); +} + +/** A deterministic Vogel spiral, stores first, drawn slightly denser toward + * the core so the stores read as a nucleus. The envelope, not the spread, + * states the body's area. */ +export function pointCloud(units: { stores: number; artifacts: number }, ratio: number): PointCloud { + const stores = Math.ceil(units.stores / ratio); + const count = stores + Math.ceil(units.artifacts / ratio); + const offsets = new Float32Array(count * 2); + for (let index = 0; index < count; index += 1) { + const r = ((index + 0.5) / count) ** 0.7; + const theta = index * GOLDEN; + offsets[index * 2] = r * Math.cos(theta); + offsets[index * 2 + 1] = r * Math.sin(theta); + } + return { offsets, stores }; +} + +export type Picked = { kind: 'cluster'; cluster: SceneCluster } | { kind: 'body'; body: SceneBody } | null; + +/** Picking over a spatial index: an unresolved cluster frame first, then the + * nearest drawn body within reach. Cost is the bodies near the pointer. */ +export function createPicker(scene: FieldScene) { + const index = createSpatialIndex(scene.bodies, INDEX_CELL); + const maxRadius = scene.bodies.reduce((max, body) => Math.max(max, body.radius), 0); + const clusterOf = new Map(scene.clusters.map((cluster) => [cluster.id, cluster])); + const unresolved = (cluster: SceneCluster | undefined, zoom: number): boolean => + cluster != null && zoom < cluster.resolveZoom; + return { + unresolved, + clusterOf, + pick(camera: CameraState, fitScale: number, sx: number, sy: number): Picked { + const zoom = camera.scale / fitScale; + const [wx, wy] = toWorld(camera, sx, sy); + for (const cluster of scene.clusters) { + if ( + unresolved(cluster, zoom) && + Math.abs(wx - cluster.x) <= cluster.width / 2 && + Math.abs(wy - cluster.y) <= cluster.height / 2 + ) { + return { kind: 'cluster', cluster }; + } + } + let best: SceneBody | null = null; + let bestDistance = Infinity; + for (const body of index.near(wx, wy, maxRadius + 10 / camera.scale)) { + if (body.cluster && unresolved(clusterOf.get(body.cluster), zoom)) continue; + const [x, y] = toScreen(camera, body.x, body.y); + const reach = body.role === 'hub' ? 9 : bodyScreenRadius(body.radius, camera.scale, fitScale) + 5; + const distance = Math.hypot(sx - x, sy - y); + if (distance <= reach && distance < bestDistance) { + best = body; + bestDistance = distance; + } + } + return best ? { kind: 'body', body: best } : null; + }, + }; +} + +function pointFieldCamera(scene: FieldScene, width: number, height: number): CameraState { + return fitCamera(scene.extent, width, height, scene.columns ? PAD : { top: 24, right: 24, bottom: 24, left: 24 }); +} + +export const createPointField: FieldRendererFactory = ({ + container, + scene, + field, + palette, + isReduced, + onHover, + onSelect, +}) => { + const mounted = mountCanvas(container); + if (!mounted) throw new Error('This browser has no 2D canvas context.'); + const { canvas, context, size, fitToContainer } = mounted; + let colors: FieldPalette = palette; + let view: FieldView = { inspected: null, focus: null }; + let hovered: string | null = null; + let hoveredCluster: string | null = null; + const synapses: Synapse[] = []; + const ratio = unitsPerPoint(scene); + const clouds = new Map(); + for (const body of scene.bodies) if (body.units) clouds.set(body.id, pointCloud(body.units, ratio)); + const kindRgb = new Map(); + const hue = (kind: string): [number, number, number] => { + let value = kindRgb.get(kind); + if (!value) { + value = cssColorToRgb(kindColor(kind, false)); + kindRgb.set(kind, value); + } + return value; + }; + const byId = new Map(scene.bodies.map((body) => [body.id, body])); + const picker = createPicker(scene); + const sprites = createSpriteCache(); + const fitState = (): CameraState => pointFieldCamera(scene, size().width, size().height); + /** The whole field, or the camera focus when one is set. */ + const framed = (): CameraState => { + const members = view.focus ? scene.bodies.filter((body) => view.focus!.has(body.id)) : []; + return focusCamera(members, size().width, size().height, 90) ?? fitState(); + }; + const camera = new EasedCamera(fitState()); + let fitScale = camera.current.scale; + const limit = (scale: number): number => Math.min(fitScale * MAX_ZOOM, Math.max(fitScale * MIN_ZOOM, scale)); + const zoomNow = (): number => camera.current.scale / fitScale; + /** The cluster a body is hidden inside at the current zoom, if any. */ + const hiddenIn = (body: SceneBody): SceneCluster | undefined => { + const cluster = body.cluster ? picker.clusterOf.get(body.cluster) : undefined; + return picker.unresolved(cluster, zoomNow()) ? cluster : undefined; + }; + /** Drawn radius: the shared mass scale, except that a member of an + * unresolved cell is held inside its packed slot, so the cell reads as a + * counted bin of holdings rather than a smear across its neighbours. */ + const radiusOf = (body: SceneBody, cam: CameraState): number => { + const radius = bodyScreenRadius(body.radius, cam.scale, fitScale); + const cell = hiddenIn(body); + return cell ? Math.min(radius, (cell.spacing * cam.scale) / 2) : radius; + }; + + const lit = (id: string): boolean => { + if (view.focus?.has(id)) return true; + const anchor = hovered ?? view.inspected; + if (anchor == null) return true; + return id === anchor || (scene.neighbors.get(anchor)?.includes(id) ?? false); + }; + const receded = (id: string): boolean => view.focus != null && !view.focus.has(id); + + /** + * The resting field is cached in an offscreen layer and redrawn only when + * the camera, the reader's view, the theme or the set of warm bodies + * changes. A decay frame blits it and draws the heat alone, so frame cost + * does not grow with the number of units on the field. + */ + const layer = document.createElement('canvas'); + const layerContext = layer.getContext('2d'); + if (!layerContext) throw new Error('This browser has no 2D canvas context.'); + let dirty = true; + let warmKey = ''; + const fitLayer = (): void => { + layer.width = canvas.width; + layer.height = canvas.height; + const scale = canvas.width / Math.max(size().width, 1); + layerContext.setTransform(scale, 0, 0, scale, 0, 0); + dirty = true; + }; + fitLayer(); + + const clusterRect = (cluster: SceneCluster, cam: CameraState): [number, number, number, number] => { + const [x0, y0] = toScreen(cam, cluster.x - cluster.width / 2, cluster.y + cluster.height / 2); + const [x1, y1] = toScreen(cam, cluster.x + cluster.width / 2, cluster.y - cluster.height / 2); + return [x0, y0, x1 - x0, y1 - y0]; + }; + + const renderStatic = (g: CanvasRenderingContext2D, cam: CameraState, zoom: number): void => { + const { width, height } = size(); + g.globalCompositeOperation = 'source-over'; + drawGraticule(g, width, height, cam, colors); + if (scene.columns) { + drawColumns(g, width, height, cam, scene.columns, colors); + drawMassAxis(g, cam, scene.extent, height, colors); + } + // Relations: hairline luminance, solid for exact, dashed for inferred. + for (const path of scene.paths) { + const a = byId.get(path.source); + const b = byId.get(path.target); + if (!a || !b) continue; + const [ax, ay] = toScreen(cam, a.x, a.y); + const [bx, by] = toScreen(cam, b.x, b.y); + const dim = (lit(a.id) && lit(b.id) ? 1 : 0.25) * (receded(a.id) && receded(b.id) ? 0.2 : 1); + g.setLineDash(path.grade === 'EXACT' ? [] : [3, 3]); + g.lineWidth = 1; + g.strokeStyle = rgbaString(colors.ink, (scene.columns ? 0.42 : 0.16) * dim); + const [cx, cy] = bow(ax, ay, bx, by, path.relation); + g.beginPath(); + g.moveTo(ax, ay); + g.quadraticCurveTo(cx, cy, bx, by); + g.stroke(); + } + g.setLineDash([]); + // Envelopes: one hairline frame per measured body, stating its area. + for (const body of scene.bodies) { + if (body.role !== 'body' || !body.units || hiddenIn(body)) continue; + const [x, y] = toScreen(cam, body.x, body.y); + g.strokeStyle = rgbaString(colors.edgeStrong, (receded(body.id) ? 0.08 : 0.3) * (lit(body.id) ? 1 : 0.4)); + g.lineWidth = 1; + g.beginPath(); + g.arc(x, y, radiusOf(body, cam) + 3, 0, Math.PI * 2); + g.stroke(); + } + // Units, additive, so luminance is density of real holdings. Members of an + // unresolved cell still draw every unit: the cell's glow is their sum. + g.globalCompositeOperation = 'lighter'; + for (const body of scene.bodies) { + if (body.role !== 'body') continue; + const alpha = (0.35 + 0.65 * (body.vitality ?? 0.6)) * (lit(body.id) ? 1 : 0.22) * (receded(body.id) ? 0.14 : 1); + drawUnits(g, cam, zoom, body, alpha, null); + } + g.globalAlpha = 1; + g.globalCompositeOperation = 'source-over'; + // Unresolved cells: one counted hairline frame each. + for (const cluster of scene.clusters) { + if (!picker.unresolved(cluster, zoom)) continue; + const [x, y, w, h] = clusterRect(cluster, cam); + g.strokeStyle = rgbaString(cluster.id === hoveredCluster ? colors.ink : colors.edgeStrong, cluster.id === hoveredCluster ? 0.95 : 0.7); + g.lineWidth = 1; + g.strokeRect(Math.round(x) + 0.5, Math.round(y) + 0.5, Math.round(w), Math.round(h)); + } + // Hubs: hollow massless junctions. + for (const body of scene.bodies) { + if (body.role !== 'hub') continue; + drawHub(g, cam, body, colors.ink, (lit(body.id) ? 0.85 : 0.3) * (receded(body.id) ? 0.2 : 1)); + } + drawLabels(g, cam, zoom); + }; + + /** One body's units; `tint` re-draws them in amber for a warm body. */ + const drawUnits = ( + g: CanvasRenderingContext2D, + cam: CameraState, + zoom: number, + body: SceneBody, + alpha: number, + tint: [number, number, number] | null, + ): void => { + const { width, height } = size(); + const [x, y] = toScreen(cam, body.x, body.y); + const r = radiusOf(body, cam); + if (x < -r || x > width + r || y < -r || y > height + r) return; + const cloud = clouds.get(body.id); + if (!cloud) { + // A symbol body: one point sized by connectedness. + g.globalAlpha = 0.9 * alpha; + const d = Math.max(3, r * 2.6); + g.drawImage(sprites.get(tint ?? hue(body.kind)), x - d / 2, y - d / 2, d, d); + return; + } + const count = cloud.offsets.length / 2; + const spacing = Math.sqrt((Math.PI * r * r) / Math.max(count, 1)); + const dot = Math.max(2.4, Math.min(7, spacing * 1.25 * Math.min(zoom, 2.5) ** 0.3)); + const half = dot / 2; + let sprite = sprites.get(tint ?? colors.ice); + g.globalAlpha = Math.min(1, alpha * 1.15); + for (let index = 0; index < count; index += 1) { + if (index === cloud.stores) { + sprite = sprites.get(tint ?? hue(body.kind)); + g.globalAlpha = alpha * 0.8; + } + g.drawImage(sprite, x + cloud.offsets[index * 2]! * r - half, y - cloud.offsets[index * 2 + 1]! * r - half, dot, dot); + } + }; + + const drawHub = (g: CanvasRenderingContext2D, cam: CameraState, body: SceneBody, rgb: [number, number, number], alpha: number): void => { + const [x, y] = toScreen(cam, body.x, body.y); + g.fillStyle = rgbaString(colors.substrate, 1); + g.strokeStyle = rgbaString(rgb, alpha); + g.lineWidth = 1.5; + g.beginPath(); + g.arc(x, y, 5, 0, Math.PI * 2); + g.fill(); + g.stroke(); + g.beginPath(); + g.arc(x, y, 1.5, 0, Math.PI * 2); + g.fillStyle = g.strokeStyle; + g.fill(); + }; + + const draw = (now: number, deltaMs: number): boolean => { + const warm = field.tick(now); + const moving = camera.step(deltaMs); + const { width, height } = size(); + const cam = camera.current; + const zoom = cam.scale / fitScale; + const nextWarmKey = scene.bodies.filter((body) => field.heatOf(body.id) > 0.02).map((body) => body.id).join('\u0000'); + if (moving || nextWarmKey !== warmKey) dirty = true; + warmKey = nextWarmKey; + if (dirty) { + renderStatic(layerContext, cam, zoom); + dirty = false; + } + context.globalCompositeOperation = 'copy'; + context.drawImage(layer, 0, 0, width, height); + context.globalCompositeOperation = 'source-over'; + + // Admitted activity: the exact touched identity blooms amber, and a + // relation conducts only while both of its ends are warm. + for (const path of scene.paths) { + const heat = Math.min(field.heatOf(path.source), field.heatOf(path.target)); + const a = byId.get(path.source); + const b = byId.get(path.target); + if (heat <= 0.05 || !a || !b) continue; + const [ax, ay] = toScreen(cam, a.x, a.y); + const [bx, by] = toScreen(cam, b.x, b.y); + const [cx, cy] = bow(ax, ay, bx, by, path.relation); + context.lineWidth = 1 + heat; + context.strokeStyle = rgbaString(lerpRgbTuple(colors.ink, colors.alert, Math.min(1, heat * 1.4)), 0.5 + 0.5 * heat); + context.beginPath(); + context.moveTo(ax, ay); + context.quadraticCurveTo(cx, cy, bx, by); + context.stroke(); + } + context.globalCompositeOperation = 'lighter'; + for (const body of scene.bodies) { + const heat = field.heatOf(body.id); + if (heat <= 0) continue; + const [x, y] = toScreen(cam, body.x, body.y); + const reach = Math.max(radiusOf(body, cam), 8) * (1.5 + heat); + const gradient = context.createRadialGradient(x, y, 0, x, y, reach); + gradient.addColorStop(0, rgbaString(colors.alert, 0.5 * heat)); + gradient.addColorStop(1, rgbaString(colors.alert, 0)); + context.fillStyle = gradient; + context.beginPath(); + context.arc(x, y, reach, 0, Math.PI * 2); + context.fill(); + if (body.role === 'body') drawUnits(context, cam, zoom, body, 0.8 * heat, colors.alert); + } + context.globalAlpha = 1; + let travelling = false; + if (!isReduced()) { + for (const synapse of synapses) { + const age = now - synapse.at; + const a = byId.get(synapse.from); + const b = synapse.to != null ? byId.get(synapse.to) : undefined; + if (age < 0 || age > SYNAPSE_TRAVEL_MS || !a || !b) continue; + travelling = true; + const t = age / SYNAPSE_TRAVEL_MS; + const [ax, ay] = toScreen(cam, a.x, a.y); + const [bx, by] = toScreen(cam, b.x, b.y); + const [cx, cy] = bow(ax, ay, bx, by, 'checkout'); + const u = 1 - t; + context.fillStyle = rgbaString(colors.alert, 0.95); + context.beginPath(); + context.arc(u * u * ax + 2 * u * t * cx + t * t * bx, u * u * ay + 2 * u * t * cy + t * t * by, 3, 0, Math.PI * 2); + context.fill(); + } + } + context.globalCompositeOperation = 'source-over'; + for (const body of scene.bodies) { + const heat = field.heatOf(body.id); + if (body.role === 'hub' && heat > 0) drawHub(context, cam, body, lerpRgbTuple(colors.ink, colors.alert, Math.min(1, heat * 1.4)), 1); + } + // Keyboard inspection: a 2px cyan ring on the exact body, and on its + // cell's frame while that cell is unresolved. Never a glow. The pointer's + // own body gets no mark; hover only dims the unrelated. + const inspected = view.inspected != null && view.inspected !== hovered ? byId.get(view.inspected) : undefined; + if (inspected) { + const [x, y] = toScreen(cam, inspected.x, inspected.y); + context.strokeStyle = rgbaString(colors.hot, 1); + context.lineWidth = 2; + context.beginPath(); + context.arc(x, y, inspected.role === 'hub' ? 9 : radiusOf(inspected, cam) + 6, 0, Math.PI * 2); + context.stroke(); + const cell = hiddenIn(inspected); + if (cell) { + const [rx, ry, rw, rh] = clusterRect(cell, cam); + context.strokeRect(rx - 2, ry - 2, rw + 4, rh + 4); + } + } + return warm || moving || travelling; + }; + + const drawLabels = (g: CanvasRenderingContext2D, cam: CameraState, zoom: number): void => { + const occupied: Array<[number, number, number, number]> = []; + const free = (x: number, y: number, w: number, h: number): boolean => + !occupied.some(([ox, oy, ow, oh]) => x < ox + ow && x + w > ox && y < oy + oh && y + h > oy); + const anchor = hovered ?? view.inspected; + const struck = new Set(synapses.filter((synapse) => field.heatOf(synapse.from) > 0.02).map((synapse) => synapse.from)); + const lastStrike = (id: string): Synapse | undefined => + struck.has(id) ? [...synapses].reverse().find((synapse) => synapse.from === id) : undefined; + const { width: fieldWidth, height: fieldHeight } = size(); + const put = (lines: string[], lx: number, ly: number, force: boolean, colour: (index: number) => string, alpha: number): void => { + // Mono glyphs have one advance, so width needs no measurement per frame. + const width = Math.max(...lines.map((line) => line.length)) * 6.7; + const height = 13 * lines.length; + if (!force && (lx + width > fieldWidth - 4 || !free(lx - 2, ly - 2, width + 6, height + 4))) return; + occupied.push([lx - 2, ly - 2, width + 6, height + 4]); + g.fillStyle = rgbaString(colors.substrate, 0.62 * alpha); + g.fillRect(lx - 3, ly, width + 6, height + 2); + lines.forEach((line, index) => { + g.font = index === 0 ? `600 11px ${colors.labelFont}` : `400 10px ${colors.labelFont}`; + g.fillStyle = colour(index); + g.fillText(line, lx, ly + 10 + index * 13); + }); + }; + g.textAlign = 'left'; + + // Unresolved cells speak first: their exact count, and any struck member. + for (const cluster of scene.clusters) { + if (!picker.unresolved(cluster, zoom)) continue; + const [x, y, w] = clusterRect(cluster, cam); + if (x > fieldWidth || x + w < 0 || y > fieldHeight) continue; + const lines = [ + `${cluster.members.length.toLocaleString()} projects`, + `mass ${cluster.mass.toLocaleString()}`, + `zoom ×${Math.ceil(cluster.resolveZoom)}`, + ]; + let strikeLine = -1; + for (const member of cluster.members) { + const strike = lastStrike(member); + if (strike) { + strikeLine = lines.length; + lines.push(`${byId.get(member)?.label ?? member} · ${strike.label} · ${strike.time}`); + } + } + const inspectedHere = view.inspected != null && cluster.members.includes(view.inspected); + if (inspectedHere) lines.push(`inspecting ${byId.get(view.inspected!)?.label ?? view.inspected}`); + // Above the frame, and always drawn: a cell's exact count is never culled. + put(lines, x, y - 4 - 13 * lines.length, true, (index) => + index === strikeLine + ? rgbaString(colors.alert, 0.95) + : index === 0 + ? rgbaString(colors.ink, 0.94) + : inspectedHere && index === lines.length - 1 + ? rgbaString(colors.hot, 0.95) + : rgbaString(colors.inkMuted, 0.92), 1); + } + + const priority = (body: SceneBody): number => + (body.id === anchor ? 1e9 : 0) + + (struck.has(body.id) ? 1e8 : 0) + + (view.focus?.has(body.id) ? 1e7 : 0) + + (body.role === 'hub' ? 0.5 : (body.mass ?? 0) + 1); + const ranked = [...scene.bodies].sort((a, b) => priority(b) - priority(a)); + const detailed = zoom >= 1.6 || view.focus != null; + for (const body of ranked) { + if ((receded(body.id) && body.id !== anchor) || hiddenIn(body)) continue; + const [x, y] = toScreen(cam, body.x, body.y); + if (x < -40 || x > fieldWidth || y < -20 || y > fieldHeight + 20) continue; + const r = body.role === 'hub' ? 6 : radiusOf(body, cam) * 1.12 + 4; + const lines = [body.role === 'hub' ? `repo:${body.label}` : body.label]; + if (body.role === 'hub') lines.push('hub · massless'); + else if (detailed || body.id === anchor) lines.push(...body.detail.slice(0, 3)); + else if (scene.columns) lines.push(body.detail[2] ?? ''); + const strike = lastStrike(body.id); + if (strike) lines.push(`${strike.label} · ${strike.time}`); + const width = Math.max(...lines.map((line) => line.length)) * 6.7; + const alpha = lit(body.id) ? 1 : 0.4; + put( + lines, + body.role === 'hub' ? x - width / 2 : x + r, + body.role === 'hub' ? y + 12 : y - 6, + body.id === anchor, + (index) => + strike && index === lines.length - 1 + ? rgbaString(colors.alert, 0.95) + : index === 0 + ? rgbaString(body.id === view.inspected && body.id !== hovered ? colors.hot : colors.ink, 0.94 * alpha) + : rgbaString(colors.inkMuted, 0.92 * alpha), + alpha, + ); + } + }; + + const loop = createFrameLoop(draw, isReduced); + const repaint = (): void => { + draw(performance.now(), 0); + loop.wake(); + }; + const unsubscribe = field.subscribe(() => loop.wake()); + const releaseGestures = attachCameraGestures( + canvas, + camera, + () => { + dirty = true; + loop.wake(); + }, + limit, + ); + let downAt: { x: number; y: number } | null = null; + const local = (event: MouseEvent): [number, number] => { + const rect = canvas.getBoundingClientRect(); + return [event.clientX - rect.left, event.clientY - rect.top]; + }; + const pointerDown = (event: PointerEvent): void => { + downAt = { x: event.clientX, y: event.clientY }; + }; + const pointerMove = (event: PointerEvent): void => { + if (event.buttons !== 0) return; + const picked = picker.pick(camera.current, fitScale, ...local(event)); + const body = picked?.kind === 'body' ? picked.body : null; + const cluster = picked?.kind === 'cluster' ? picked.cluster.id : null; + canvas.style.cursor = picked != null && (picked.kind === 'cluster' || picked.body.role === 'body') ? 'pointer' : 'default'; + if ((body?.id ?? null) === hovered && cluster === hoveredCluster) return; + hovered = body?.id ?? null; + hoveredCluster = cluster; + dirty = true; + if (body) onHover(body.id); + repaint(); + }; + const pointerLeave = (): void => { + if (hovered == null && hoveredCluster == null) return; + hovered = null; + hoveredCluster = null; + dirty = true; + repaint(); + }; + const click = (event: MouseEvent): void => { + if (downAt && Math.hypot(event.clientX - downAt.x, event.clientY - downAt.y) >= 4) return; + const picked = picker.pick(camera.current, fitScale, ...local(event)); + if (picked?.kind === 'body' && picked.body.role === 'body') onSelect(picked.body.id); + if (picked?.kind === 'cluster') { + // Zoom the cell open around its own centre, just past its resolve zoom. + const { width, height } = size(); + const scale = limit(fitScale * picked.cluster.resolveZoom * 1.15); + camera.set({ scale, tx: width / 2 - picked.cluster.x * scale, ty: height / 2 + picked.cluster.y * scale }, isReduced()); + dirty = true; + repaint(); + } + }; + canvas.addEventListener('pointerdown', pointerDown); + canvas.addEventListener('pointermove', pointerMove); + canvas.addEventListener('pointerleave', pointerLeave); + canvas.addEventListener('click', click); + repaint(); + + return { + setView(next) { + const focusChanged = next.focus !== view.focus; + view = next; + dirty = true; + if (focusChanged) camera.set(framed(), isReduced()); + repaint(); + }, + synapse(synapse) { + synapses.push(synapse); + if (synapses.length > 32) synapses.shift(); + loop.wake(); + }, + resize() { + fitToContainer(); + fitLayer(); + fitScale = fitState().scale; + camera.set(framed(), true); + repaint(); + }, + zoom(factor) { + const { width, height } = size(); + const t = camera.target; + const [wx, wy] = [(width / 2 - t.tx) / t.scale, (t.ty - height / 2) / t.scale]; + const scale = limit(t.scale * factor); + camera.set({ scale, tx: width / 2 - wx * scale, ty: height / 2 + wy * scale }, isReduced()); + dirty = true; + repaint(); + }, + fit() { + camera.set(fitState(), isReduced()); + dirty = true; + repaint(); + }, + retheme(next) { + colors = next; + dirty = true; + repaint(); + }, + destroy() { + loop.stop(); + unsubscribe(); + releaseGestures(); + canvas.removeEventListener('pointerdown', pointerDown); + canvas.removeEventListener('pointermove', pointerMove); + canvas.removeEventListener('pointerleave', pointerLeave); + canvas.removeEventListener('click', click); + canvas.remove(); + }, + }; +}; diff --git a/dashboard/src/viz/graph/fieldRenderers/scene.ts b/dashboard/src/viz/graph/fieldRenderers/scene.ts new file mode 100644 index 0000000000..6c255e17ab --- /dev/null +++ b/dashboard/src/viz/graph/fieldRenderers/scene.ts @@ -0,0 +1,566 @@ +import { approach, cssColorToRgb, settled, type ActivationField } from '../activation.ts'; +import { palette, type GraphPalette } from '../palette.ts'; + +/** + * The renderer-neutral field the Brain draws. + * + * A renderer receives positioned bodies and drawn relations whose + * geometry was already decided by a measured or emergent layout, plus an + * activation field it may only sample. It never decides what a position, + * size or relation means; the scene builders do, and the host's legend + * states it. + */ + +export interface SceneBody { + id: string; + label: string; + /** `body` carries a holdings measure; `hub` is a massless relation junction + * drawn at a fixed categorical size. */ + role: 'body' | 'hub'; + kind: string; + x: number; + y: number; + /** Radius in field units on the one shared mass scale; hubs are fixed. */ + radius: number; + /** The named holdings measure, or null when the source did not measure it. */ + mass: number | null; + /** Indexed units by class, for renderers that draw one mark per unit. */ + units: { stores: number; artifacts: number } | null; + /** Recency 0..1 relative to the field's horizon, or null when unmeasured. */ + vitality: number | null; + /** Exact printed readings, in order. `absent` stays printed. */ + detail: readonly string[]; + /** The repository identity this body belongs to, when recorded. */ + group: string | null; + /** The packed cell this body sits in, when it was crowded into one. */ + cluster: string | null; +} + +/** A packed cell of bodies. Drawn as one counted frame until the camera is + * close enough for its members to stop overlapping. */ +export interface SceneCluster { + id: string; + members: readonly string[]; + x: number; + y: number; + width: number; + height: number; + /** Zoom (camera scale over fit scale) at which members separate. */ + resolveZoom: number; + /** Summed member holdings, printed on the frame with the exact count. */ + mass: number; + /** Packed distance between neighbouring members, in field units. */ + spacing: number; +} + +export type RelationGrade = 'EXACT' | 'INFERRED'; + +export interface ScenePath { + source: string; + target: string; + relation: string; + grade: RelationGrade; +} + +export interface SceneColumn { + label: string; + bound: string; + count: number; +} + +export interface FieldScene { + bodies: readonly SceneBody[]; + paths: readonly ScenePath[]; + clusters: readonly SceneCluster[]; + /** Camera frame in field units, larger y is up. */ + extent: { x: [number, number]; y: [number, number] }; + /** Recency columns centred on x = 0..n-1, when the field is measured. */ + columns: readonly SceneColumn[] | null; + /** Drawn adjacency; activation may travel only along it. */ + neighbors: ReadonlyMap; +} + +/** Reader state: inspection and camera focus. None of it is activity. */ +export interface FieldView { + inspected: string | null; + /** Camera focus (repository zoom or focused plate). Bodies outside recede. */ + focus: ReadonlySet | null; +} + +/** One admitted event that reached a drawn body, with its evidenced hop. */ +export interface Synapse { + from: string; + to: string | null; + /** `performance.now()` at admission, the clock the frame loop runs on. */ + at: number; + label: string; + /** Wall-clock receipt time, printed beside the heat. */ + time: string; +} + +export interface FieldPalette extends GraphPalette { + /** Stores and other identity-neutral holdings. */ + ice: [number, number, number]; + grid: [number, number, number]; + gridMajor: [number, number, number]; + ink: [number, number, number]; + inkMuted: [number, number, number]; + face: [number, number, number]; + edgeStrong: [number, number, number]; + displayFont: string; +} + +export function sampleFieldPalette(element: HTMLElement): FieldPalette { + const style = getComputedStyle(element); + const token = (name: string, fallback: string): [number, number, number] => + cssColorToRgb(style.getPropertyValue(name).trim() || fallback); + return { + ...palette(element), + ice: token('--raw-graph-ice', '#d4ecf7'), + grid: token('--raw-grid-minor', '#23262c'), + gridMajor: token('--raw-grid', '#2c3036'), + ink: token('--ink-primary', '#eef0f3'), + inkMuted: token('--ink-muted', '#9ea3aa'), + face: token('--night-face', '#16181b'), + edgeStrong: token('--edge-strong', '#5c6168'), + displayFont: style.getPropertyValue('--font-display').trim() || 'sans-serif', + }; +} + +export interface FieldRendererOptions { + container: HTMLElement; + scene: FieldScene; + field: ActivationField; + palette: FieldPalette; + isReduced: () => boolean; + onHover: (id: string | null) => void; + onSelect: (id: string) => void; +} + +export interface FieldRenderer { + setView(view: FieldView): void; + synapse(synapse: Synapse): void; + resize(): void; + zoom(factor: number): void; + fit(): void; + retheme(palette: FieldPalette): void; + destroy(): void; +} + +export type FieldRendererFactory = (options: FieldRendererOptions) => FieldRenderer; + +/** + * How a body's drawn radius grows with zoom: as zoom^0.25 rather than + * linearly, so zooming separates positions faster than bodies swell and a + * packed cell resolves into its members. + */ +export const BODY_ZOOM_GROWTH = 0.25; + +export function bodyScreenRadius(radius: number, scale: number, fitScale: number): number { + return radius * fitScale * (scale / fitScale) ** BODY_ZOOM_GROWTH; +} + +/** The zoom at which bodies of `maxRadius` packed `spacing` apart stop + * overlapping under {@link bodyScreenRadius}. Never below 1. */ +export function resolveZoom(maxRadius: number, spacing: number): number { + return Math.max(1, ((2 * maxRadius) / Math.max(spacing, 1e-9)) ** (1 / (1 - BODY_ZOOM_GROWTH))); +} + +/** + * A uniform grid over static world positions, so pointer picking touches the + * handful of bodies near the pointer instead of every body on the field. + */ +export function createSpatialIndex(items: readonly T[], cell: number) { + const buckets = new Map(); + const key = (cx: number, cy: number): string => `${cx}:${cy}`; + for (const item of items) { + const k = key(Math.floor(item.x / cell), Math.floor(item.y / cell)); + const bucket = buckets.get(k); + if (bucket) bucket.push(item); + else buckets.set(k, [item]); + } + return { + /** Items whose position lies within `radius` of (x, y), plus bucket slack. */ + near(x: number, y: number, radius: number): T[] { + const found: T[] = []; + const x0 = Math.floor((x - radius) / cell); + const x1 = Math.floor((x + radius) / cell); + const y0 = Math.floor((y - radius) / cell); + const y1 = Math.floor((y + radius) / cell); + for (let cx = x0; cx <= x1; cx += 1) { + for (let cy = y0; cy <= y1; cy += 1) { + const bucket = buckets.get(key(cx, cy)); + if (bucket) found.push(...bucket); + } + } + return found; + }, + }; +} + +/** How long a travelling synapse light takes to cross its one hop. */ +export const SYNAPSE_TRAVEL_MS = 700; +/** Reduced motion repaints decaying heat in discrete steps, never per frame. */ +export const REDUCED_REPAINT_MS = 1000; + +/** World (field units, y up) to screen (CSS px, y down), fit with padding and + * a pointer-centred zoom on top. Pure arithmetic so it can be tested and so a + * capture harness can locate a body without a renderer. */ +export interface CameraState { + scale: number; + tx: number; + ty: number; +} + +export function fitCamera( + extent: FieldScene['extent'], + width: number, + height: number, + pad: { top: number; right: number; bottom: number; left: number }, +): CameraState { + const spanX = Math.max(extent.x[1] - extent.x[0], 1e-6); + const spanY = Math.max(extent.y[1] - extent.y[0], 1e-6); + const innerW = Math.max(width - pad.left - pad.right, 1); + const innerH = Math.max(height - pad.top - pad.bottom, 1); + const scale = Math.min(innerW / spanX, innerH / spanY); + const tx = pad.left + (innerW - spanX * scale) / 2 - extent.x[0] * scale; + const ty = pad.top + (innerH - spanY * scale) / 2 + extent.y[1] * scale; + return { scale, tx, ty }; +} + +export function toScreen(camera: CameraState, x: number, y: number): [number, number] { + return [camera.tx + x * camera.scale, camera.ty - y * camera.scale]; +} + +export function toWorld(camera: CameraState, sx: number, sy: number): [number, number] { + return [(sx - camera.tx) / camera.scale, (camera.ty - sy) / camera.scale]; +} + +/** Zoom by `factor` keeping the world point under (sx, sy) fixed. */ +export function zoomAt(camera: CameraState, sx: number, sy: number, factor: number): CameraState { + const [wx, wy] = toWorld(camera, sx, sy); + const scale = camera.scale * factor; + return { scale, tx: sx - wx * scale, ty: sy + wy * scale }; +} + +/** Frame a set of bodies, used by repository zoom and plate focus. */ +export function focusCamera( + bodies: readonly SceneBody[], + width: number, + height: number, + margin: number, +): CameraState | null { + if (bodies.length === 0) return null; + let minX = Infinity; + let maxX = -Infinity; + let minY = Infinity; + let maxY = -Infinity; + for (const body of bodies) { + minX = Math.min(minX, body.x - body.radius); + maxX = Math.max(maxX, body.x + body.radius); + minY = Math.min(minY, body.y - body.radius); + maxY = Math.max(maxY, body.y + body.radius); + } + const padX = Math.max((maxX - minX) * 0.35, 0.35); + const padY = Math.max((maxY - minY) * 0.35, 0.35); + return fitCamera( + { x: [minX - padX, maxX + padX], y: [minY - padY, maxY + padY] }, + width, + height, + { top: margin, right: margin, bottom: margin, left: margin }, + ); +} + +/** A camera that eases toward its target unless motion is reduced. */ +export class EasedCamera { + current: CameraState; + target: CameraState; + + constructor(initial: CameraState) { + this.current = { ...initial }; + this.target = { ...initial }; + } + + set(next: CameraState, reduced: boolean): void { + this.target = { ...next }; + if (reduced) this.current = { ...next }; + } + + /** Advance toward the target; returns true while still moving. */ + step(deltaMs: number): boolean { + const c = this.current; + const t = this.target; + c.scale = approach(c.scale, t.scale, deltaMs, 90); + c.tx = approach(c.tx, t.tx, deltaMs, 90); + c.ty = approach(c.ty, t.ty, deltaMs, 90); + const done = + settled(c.scale / t.scale, 1, 0.001) && settled(c.tx, t.tx, 0.4) && settled(c.ty, t.ty, 0.4); + if (done) this.current = { ...t }; + return !done; + } +} + +/** + * The single paint scheduler for a field canvas. Frames run only while + * something real is unresolved (warm heat, a travelling synapse, an easing + * camera); an idle field schedules nothing. Under reduced motion heat is + * repainted in one-second steps and nothing travels. + */ +export function createFrameLoop(draw: (now: number, deltaMs: number) => boolean, isReduced: () => boolean) { + let frame = 0; + let timer: ReturnType | null = null; + let last = 0; + let alive = true; + const run = (now: number): void => { + frame = 0; + if (!alive) return; + const delta = last === 0 ? 16 : Math.min(now - last, 100); + last = now; + const more = draw(now, delta); + if (!more) { + last = 0; + return; + } + if (isReduced()) { + timer = setTimeout(() => { + timer = null; + run(performance.now()); + }, REDUCED_REPAINT_MS); + } else { + frame = requestAnimationFrame(run); + } + }; + return { + wake(): void { + if (!alive || frame !== 0 || timer !== null) return; + frame = requestAnimationFrame(run); + }, + stop(): void { + alive = false; + if (frame !== 0) cancelAnimationFrame(frame); + if (timer !== null) clearTimeout(timer); + }, + }; +} + +/** A device-pixel-ratio aware 2D canvas filling its container. */ +export function mountCanvas(container: HTMLElement): { + canvas: HTMLCanvasElement; + context: CanvasRenderingContext2D; + size: () => { width: number; height: number }; + fitToContainer: () => void; +} | null { + const canvas = document.createElement('canvas'); + canvas.style.position = 'absolute'; + canvas.style.inset = '0'; + canvas.style.width = '100%'; + canvas.style.height = '100%'; + const context = canvas.getContext('2d'); + if (!context) return null; + container.appendChild(canvas); + let width = 0; + let height = 0; + const fitToContainer = (): void => { + width = container.clientWidth; + height = container.clientHeight; + const ratio = window.devicePixelRatio || 1; + canvas.width = Math.max(1, Math.round(width * ratio)); + canvas.height = Math.max(1, Math.round(height * ratio)); + context.setTransform(ratio, 0, 0, ratio, 0, 0); + }; + fitToContainer(); + return { canvas, context, size: () => ({ width, height }), fitToContainer }; +} + +/** Wheel zoom around the pointer, bounded by `limit`, and drag pan. */ +export function attachCameraGestures( + canvas: HTMLCanvasElement, + camera: EasedCamera, + changed: () => void, + limit: (scale: number) => number, +): () => void { + let drag: { x: number; y: number; moved: boolean } | null = null; + const wheel = (event: WheelEvent): void => { + event.preventDefault(); + const rect = canvas.getBoundingClientRect(); + const t = camera.target; + const factor = limit(t.scale * Math.exp(-event.deltaY * 0.0015)) / t.scale; + camera.set(zoomAt(camera.target, event.clientX - rect.left, event.clientY - rect.top, factor), true); + changed(); + }; + const down = (event: PointerEvent): void => { + drag = { x: event.clientX, y: event.clientY, moved: false }; + }; + const move = (event: PointerEvent): void => { + if (!drag || event.buttons === 0) return; + const dx = event.clientX - drag.x; + const dy = event.clientY - drag.y; + if (!drag.moved && Math.hypot(dx, dy) < 4) return; + drag.moved = true; + drag.x = event.clientX; + drag.y = event.clientY; + const t = camera.target; + camera.set({ scale: t.scale, tx: t.tx + dx, ty: t.ty + dy }, true); + changed(); + }; + const up = (): void => { + drag = null; + }; + canvas.addEventListener('wheel', wheel, { passive: false }); + canvas.addEventListener('pointerdown', down); + window.addEventListener('pointermove', move); + window.addEventListener('pointerup', up); + return () => { + canvas.removeEventListener('wheel', wheel); + canvas.removeEventListener('pointerdown', down); + window.removeEventListener('pointermove', move); + window.removeEventListener('pointerup', up); + }; +} + +/** Antialiased point sprites per colour: a solid core with a one-pixel soft + * edge, so thousands of `drawImage` calls read as points rather than squares. */ +export function createSpriteCache(): { get(rgb: [number, number, number]): HTMLCanvasElement } { + const cache = new Map(); + return { + get(rgb) { + const quantized = rgb.map((channel) => Math.round(channel / 6) * 6) as [number, number, number]; + const key = quantized.join(','); + let sprite = cache.get(key); + if (sprite) return sprite; + sprite = document.createElement('canvas'); + sprite.width = sprite.height = 16; + const context = sprite.getContext('2d'); + if (context) { + const gradient = context.createRadialGradient(8, 8, 0, 8, 8, 8); + gradient.addColorStop(0, rgbaString(quantized, 1)); + gradient.addColorStop(0.4, rgbaString(quantized, 0.85)); + gradient.addColorStop(0.75, rgbaString(quantized, 0.22)); + gradient.addColorStop(1, rgbaString(quantized, 0)); + context.fillStyle = gradient; + context.fillRect(0, 0, 16, 16); + } + cache.set(key, sprite); + return sprite; + }, + }; +} + +export function rgbaString([r, g, b]: [number, number, number], alpha: number): string { + return `rgba(${r}, ${g}, ${b}, ${Math.max(0, Math.min(1, alpha)).toFixed(3)})`; +} + +/** The substrate with its 32 px minor and 128 px major graticule, anchored + * to the field origin so it pans with the camera. */ +export function drawGraticule( + context: CanvasRenderingContext2D, + width: number, + height: number, + camera: CameraState, + colors: FieldPalette, +): void { + context.fillStyle = rgbaString(colors.substrate, 1); + context.fillRect(0, 0, width, height); + context.lineWidth = 1; + const [ox, oy] = toScreen(camera, 0, 0); + for (const [step, color, alpha] of [ + [32, colors.grid, 0.55], + [128, colors.gridMajor, 0.5], + ] as const) { + context.strokeStyle = rgbaString(color, alpha); + context.beginPath(); + for (let x = ((ox % step) + step) % step; x < width; x += step) { + context.moveTo(Math.round(x) + 0.5, 0); + context.lineTo(Math.round(x) + 0.5, height); + } + for (let y = ((oy % step) + step) % step; y < height; y += step) { + context.moveTo(0, Math.round(y) + 0.5); + context.lineTo(width, Math.round(y) + 0.5); + } + context.stroke(); + } +} + +export function drawColumns( + context: CanvasRenderingContext2D, + width: number, + height: number, + camera: CameraState, + columns: readonly SceneColumn[], + colors: FieldPalette, +): void { + context.save(); + context.font = `500 10px ${colors.labelFont}`; + context.textAlign = 'center'; + for (let index = 0; index <= columns.length; index += 1) { + const [x] = toScreen(camera, index - 0.5, 0); + if (x < -2 || x > width + 2) continue; + context.strokeStyle = rgbaString(colors.edgeStrong, index === 0 || index === columns.length ? 0.35 : 0.22); + context.setLineDash(index === 0 || index === columns.length ? [] : [2, 4]); + context.beginPath(); + context.moveTo(Math.round(x) + 0.5, 34); + context.lineTo(Math.round(x) + 0.5, height - 8); + context.stroke(); + } + context.setLineDash([]); + // Narrow columns keep the bound and the count; the column name is the + // first thing dropped. Columns too narrow for their bounds print no header + // at all rather than overprinting; the registry readout beside the field + // prints the same bounds and counts. + const spacing = camera.scale; + context.letterSpacing = '0.16em'; + const widest = Math.max(...columns.map((column) => context.measureText(column.bound.toUpperCase()).width)); + context.letterSpacing = '0px'; + if (spacing < widest + 8) { + context.restore(); + return; + } + columns.forEach((column, index) => { + const [x] = toScreen(camera, index, 0); + if (x < -80 || x > width + 80) return; + context.fillStyle = rgbaString(colors.ink, 0.86); + engraved(context, column.bound.toUpperCase(), x, 16); + context.fillStyle = rgbaString(colors.inkMuted, 0.9); + context.fillText(spacing < 120 ? String(column.count) : `${column.label} · ${column.count}`, x, 29); + }); + context.restore(); +} + +export function drawMassAxis( + context: CanvasRenderingContext2D, + camera: CameraState, + extent: FieldScene['extent'], + height: number, + colors: FieldPalette, +): void { + const [, top] = toScreen(camera, 0, extent.y[1]); + const [, bottom] = toScreen(camera, 0, extent.y[0]); + const x = 22; + context.strokeStyle = rgbaString(colors.edgeStrong, 0.55); + context.lineWidth = 1; + context.beginPath(); + context.moveTo(x + 0.5, Math.max(48, top + 12)); + context.lineTo(x + 0.5, Math.min(height - 16, bottom - 12)); + context.stroke(); + context.save(); + context.font = `500 9px ${colors.labelFont}`; + context.fillStyle = rgbaString(colors.inkMuted, 0.9); + context.translate(x - 7, (top + bottom) / 2); + context.rotate(-Math.PI / 2); + context.textAlign = 'center'; + context.fillText('INDEXED MASS · LOG ↑', 0, 0); + context.restore(); +} + +/** Quadratic control point; curvature is a property of the relation kind. */ +export function bow(ax: number, ay: number, bx: number, by: number, relation: string): [number, number] { + const k = relation === 'checkout' ? 0.16 : relation === 'contains' ? 0 : 0.08; + return [(ax + bx) / 2 - (by - ay) * k, (ay + by) / 2 + (bx - ax) * k]; +} + +/** An engraved legend: uppercase at the design system's 0.16em tracking. */ +export function engraved(context: CanvasRenderingContext2D, text: string, x: number, y: number): void { + context.letterSpacing = '0.16em'; + context.fillText(text, x, y); + context.letterSpacing = '0px'; +} diff --git a/dashboard/src/viz/graph/glowProgram.ts b/dashboard/src/viz/graph/glowProgram.ts deleted file mode 100644 index 2d03710e54..0000000000 --- a/dashboard/src/viz/graph/glowProgram.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { NodeCircleProgram } from 'sigma/rendering'; - -/** Reuse Sigma's geometry, camera uniforms and lifecycle. Only decorative - * companions have radial falloff; project bodies retain their exact hit area. */ -export class GlowProgram extends NodeCircleProgram { - override getDefinition() { - return { - ...super.getDefinition(), - FRAGMENT_SHADER_SOURCE: ` -precision highp float; -varying vec4 v_color; -varying vec2 v_diffVector; -varying float v_radius; -void main(void) { - // Decoration must never intercept a body's pointer target. - #ifdef PICKING_MODE - discard; - #else - float radius = length(v_diffVector) / max(v_radius, 0.0001); - float falloff = exp(-4.0 * radius * radius) * (1.0 - smoothstep(0.7, 1.0, radius)); - gl_FragColor = v_color * falloff; - #endif -}`, - }; - } -} diff --git a/dashboard/src/viz/graph/kindColor.test.ts b/dashboard/src/viz/graph/kindColor.test.ts new file mode 100644 index 0000000000..c9342a7265 --- /dev/null +++ b/dashboard/src/viz/graph/kindColor.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, it } from 'vitest'; + +import { kindColor, kindColorVars } from './kindColor.ts'; + +function parseOklch(color: string): { l: number; c: number; h: number } { + const match = /^oklch\(([\d.]+) ([\d.]+) ([\d.]+)\)$/.exec(color); + if (!match) throw new Error(`not an oklch colour: ${color}`); + return { l: Number(match[1]), c: Number(match[2]), h: Number(match[3]) }; +} + +describe('kindColor', () => { + it('resolves a named kind case-insensitively, one side per medium', () => { + expect(kindColor('Function', false)).toBe('oklch(0.8 0.12 202)'); + expect(kindColor('FUNCTION', true)).toBe('oklch(0.49 0.135 205)'); + }); + + it('hashes other names onto the same ramp, stably', () => { + expect(kindColor('repository', false)).toBe('oklch(0.78 0.11 185)'); + expect(kindColor('worktree', false)).toBe('oklch(0.82 0.08 244)'); + expect(kindColor('project', true)).toBe('oklch(0.58 0.145 250)'); + expect(kindColor('codex', false)).toBe('oklch(0.9 0.045 222)'); + }); + + it('never leaves the cool identity band for amber, violet, red or ready green', () => { + const names = ['function', 'macro', 'src/viz', 'claude', 'run-7f3a', 'unknown', 'x', '']; + for (let index = 0; index < 200; index += 1) names.push(`kind-${index}`); + for (const name of names) { + for (const light of [false, true]) { + const { c, h } = parseOklch(kindColor(name, light)); + expect(h).toBeGreaterThanOrEqual(165); + expect(h).toBeLessThanOrEqual(250); + expect(c).toBeLessThan(0.15); + } + } + }); + + it('keeps the two ramps in step: the brightest dark slot is the darkest ink', () => { + const kinds = ['function', 'method', 'struct', 'trait', 'module', 'enum', 'field', 'impl']; + const byDark = [...kinds].sort( + (a, b) => parseOklch(kindColor(b, false)).l - parseOklch(kindColor(a, false)).l, + ); + const byInk = [...kinds].sort( + (a, b) => parseOklch(kindColor(a, true)).l - parseOklch(kindColor(b, true)).l, + ); + expect(byDark).toEqual(['trait', 'field', 'impl', 'function', 'struct', 'module', 'method', 'enum']); + expect(byInk).toEqual(byDark); + }); + + it('hands DOM marks both sides as custom properties', () => { + expect(kindColorVars('method')).toEqual({ + '--kind-dark': 'oklch(0.68 0.13 248)', + '--kind-light': 'oklch(0.58 0.145 250)', + }); + }); +}); diff --git a/dashboard/src/viz/graph/kindColor.ts b/dashboard/src/viz/graph/kindColor.ts index 156bca940f..b77706ad66 100644 --- a/dashboard/src/viz/graph/kindColor.ts +++ b/dashboard/src/viz/graph/kindColor.ts @@ -1,22 +1,72 @@ import type { CSSProperties } from 'react'; /** - * Deep-space plasma palette: a node's kind picks a hue on the cyan → violet - * arc at fixed lightness. One rule instead of a hardcoded map, so every graph - * in the app harmonizes, repositories and checkouts in Brain, symbol kinds in - * Code, and an unseen kind still lands somewhere deliberate rather than - * defaulting to grey. The arc is bounded so colour never wanders into muddy - * yellows that read as "warning" against the dark field; chroma varies a - * little across the arc so neighbouring hues stay tellable apart. + * Kind palette: a fixed ordinal ramp of eight slots inside the cool band the + * design system leaves free for identity (teal h 168 through blue h 248), + * separated by lightness as much as by hue. Every graph in the app draws from + * it: repositories and checkouts in Brain, symbol kinds in Code, providers and + * runs elsewhere. The band is closed on purpose. Amber means measured activity, + * violet means restricted, red means refused and a saturated h 155 green means + * ready, so a kind that landed on any of them would read as a state it is not. + * Chroma stays under the signal cyan's 0.15 so a kind never passes for focus. * - * This lives outside `GraphCanvas` because the canvas is no longer the only + * The common symbol kinds hold named slots, so the kinds a field shows most are + * the ones kept furthest apart; any other name hashes onto the same ramp, so an + * unseen kind is still stable across reloads and never falls back to grey. + * + * This lives outside `GraphCanvas` because the canvas is not the only * consumer: the Code workspace's connectivity spine tints its marks by the * same rule, which is what makes the spine and the field above it read as one - * instrument rather than two views that happen to share a dataset. Two copies - * of this arithmetic would silently drift; one copy cannot. + * instrument rather than two views that happen to share a dataset. + */ + +/** + * Each slot is one hue drawn against both media. A body is lit against its + * medium, so which side of the substrate it sits on flips with the theme: on + * the dark field a slot is light and moderately saturated; on paper it is ink, + * darker than the medium with a little more chroma to hold its hue at the lower + * lightness. The two ramps stay in step: the slot that stands furthest from + * the dark field (ice, L 0.90) is also the darkest ink on paper (L 0.41), so + * relative prominence survives a theme flip. Chroma is what survives overlap: + * forty translucent pastel discs accumulate into an undifferentiated pale mass, + * saturated bodies further down the range stay tellable apart when they pile up. */ +interface KindSlot { + dark: string; + light: string; +} + +const CYAN_SLOT: KindSlot = { dark: 'oklch(0.8 0.12 202)', light: 'oklch(0.49 0.135 205)' }; + +const KIND_RAMP: readonly KindSlot[] = [ + CYAN_SLOT, + { dark: 'oklch(0.68 0.13 248)', light: 'oklch(0.58 0.145 250)' }, // blue + { dark: 'oklch(0.78 0.11 185)', light: 'oklch(0.5 0.125 186)' }, // teal + { dark: 'oklch(0.9 0.045 222)', light: 'oklch(0.41 0.06 225)' }, // ice + { dark: 'oklch(0.7 0.12 225)', light: 'oklch(0.57 0.135 228)' }, // sky + { dark: 'oklch(0.62 0.1 186)', light: 'oklch(0.63 0.115 188)' }, // deep teal + { dark: 'oklch(0.86 0.075 168)', light: 'oklch(0.44 0.09 170)' }, // seafoam + { dark: 'oklch(0.82 0.08 244)', light: 'oklch(0.47 0.095 246)' }, // slate blue +]; -/** Stable per-kind hash, the only input the palette has. */ +const NAMED_SLOTS: Readonly> = { + function: 0, + method: 1, + struct: 2, + class: 2, + trait: 3, + interface: 3, + module: 4, + file: 4, + enum: 5, + type: 5, + field: 6, + constant: 6, + variable: 6, + impl: 7, +}; + +/** Stable per-name hash for kinds without a named slot. */ function hashKind(kind: string): number { let hash = 0; for (let index = 0; index < kind.length; index += 1) { @@ -25,26 +75,15 @@ function hashKind(kind: string): number { return hash; } -/** - * @param light whether the kind is being drawn against a light medium. - * - * A body is lit against its medium, so which side of the substrate it sits on - * has to flip with the theme. Pinned at L 0.78 the kind hues were tuned for a - * dark field; on the light field they landed ABOVE the background and forty - * overlapping translucent discs accumulated into a white cloud with no - * structure in it at all. On paper a node is saturated ink: darker than its - * medium, with a little more chroma to hold its hue at the lower lightness. - * Chroma is what survives overlap. At the old 0.112 the dark hues were pastels - * sitting near the top of the lightness range, so a dense cluster of them - * accumulated into an undifferentiated pale mass, the graph lost its colour - * exactly where it had the most structure to show. Saturated bodies a little - * further down the range stay tellable apart when they pile up. - */ +function kindSlot(kind: string): KindSlot { + const slot = NAMED_SLOTS[kind.toLowerCase()] ?? hashKind(kind) % KIND_RAMP.length; + return KIND_RAMP[slot] ?? CYAN_SLOT; +} + +/** @param light whether the kind is being drawn against a light medium. */ export function kindColor(kind: string, light: boolean): string { - const hash = hashKind(kind); - const chroma = (light ? 0.135 : 0.152) + ((hash >>> 9) % 6) * 0.012; - const lightness = light ? 0.55 : 0.72; - return `oklch(${lightness} ${chroma.toFixed(3)} ${186 + (hash % 148)})`; + const slot = kindSlot(kind); + return light ? slot.light : slot.dark; } /** diff --git a/dashboard/src/viz/graph/measuredField.ts b/dashboard/src/viz/graph/measuredField.ts deleted file mode 100644 index 736ff15731..0000000000 --- a/dashboard/src/viz/graph/measuredField.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { axisFrame, nodeHullFrame, type FieldFrame, type PreparedField } from './layout.ts'; -import type { FieldExtent } from './types.ts'; - -/** - * The measured path: a field whose coordinates are the caller's own - * measurement. - * - * There is no layout step here at all, and that is the point, running a force - * pass over placed coordinates, or re-centering their components onto a ring, - * would destroy the very measurement the positions were carrying. Nothing on - * this path loads a layout engine, because nothing on it has anything to lay - * out. - */ -export function frameMeasuredField( - prepared: PreparedField, - extent: FieldExtent | undefined, -): FieldFrame { - // A measured field is framed by its AXIS, not by its occupants. Framing - // the occupants would rescale the picture every time a body enters or - // leaves a region, and would quietly delete an empty region, which on - // this kind of field is itself a reading. - if (extent) return axisFrame(extent); - // Without a stated axis there is nothing to frame but the bodies, so a - // measured field with no extent falls back to the same hull an emergent one - // uses. The caller has said where each body is but not what the frame means. - return nodeHullFrame(prepared.graph, prepared.realNodes); -} diff --git a/dashboard/src/viz/graph/nodeHover.test.ts b/dashboard/src/viz/graph/nodeHover.test.ts deleted file mode 100644 index e6277bbe98..0000000000 --- a/dashboard/src/viz/graph/nodeHover.test.ts +++ /dev/null @@ -1,116 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { createNodeHoverDrawer } from './nodeHover.ts'; -import { rgb, rgba, type GraphPalette, type ThemeBox } from './palette.ts'; -import type { Settings } from 'sigma/settings'; - -/** - * The hover pass, drawn against both mediums. - * - * The regression this guards: Sigma's default hover drawer paints an opaque - * white shadowed disc and a white label backdrop, which read as the hovered - * body "going white" and growing a blob on the dark field. Every paint this - * drawer makes must come from the theme box it was handed, and from the box's - * CURRENT colors, so a theme flip re-lights hovers without a rebuild. - */ - -const DARK: GraphPalette = { - hot: [93, 231, 255], - alert: [224, 182, 74], - edge: [55, 83, 114], - label: [196, 212, 232], - labelFont: 'ui-monospace, monospace', - substrate: [7, 11, 22], - dim: [38, 55, 76], - light: false, -}; - -const LIGHT: GraphPalette = { - hot: [11, 116, 145], - alert: [160, 110, 20], - edge: [148, 166, 188], - label: [32, 42, 56], - labelFont: 'ui-monospace, monospace', - substrate: [244, 246, 250], - dim: [210, 218, 228], - light: true, -}; - -interface PaintOp { - op: 'stroke' | 'fillRect' | 'fillText'; - style: string; -} - -/** A recording 2d context: every stroke and fill is captured with the style it - * was painted in, which is the whole question this file asks. */ -function recordingContext(): { context: CanvasRenderingContext2D; paints: PaintOp[] } { - const paints: PaintOp[] = []; - const context = { - fillStyle: '', - strokeStyle: '', - lineWidth: 0, - font: '', - beginPath: () => undefined, - arc: () => undefined, - stroke() { - paints.push({ op: 'stroke', style: String(this.strokeStyle) }); - }, - measureText: (text: string) => ({ width: text.length * 6 }), - fillRect() { - paints.push({ op: 'fillRect', style: String(this.fillStyle) }); - }, - fillText() { - paints.push({ op: 'fillText', style: String(this.fillStyle) }); - }, - }; - return { context: context as unknown as CanvasRenderingContext2D, paints }; -} - -const SETTINGS = { - labelSize: 11, - labelWeight: 'normal', - labelFont: 'ui-monospace, monospace', -} as Settings; - -const NODE = { x: 10, y: 20, size: 6, label: 'alpha::beta', color: 'rgb(93, 231, 255)' }; - -/** Styles that would reproduce the white-blob regression. */ -const WHITES = [/rgb\(255,\s*255,\s*255\)/, /#fff/i, /\bwhite\b/]; - -describe('createNodeHoverDrawer', () => { - it('paints only theme colors on the dark field: accent ring, substrate backdrop, label ink', () => { - const theme: ThemeBox = { colors: DARK }; - const { context, paints } = recordingContext(); - createNodeHoverDrawer(theme)(context, NODE, SETTINGS); - - expect(paints).toEqual([ - { op: 'stroke', style: rgba(DARK.hot, 0.9) }, - { op: 'fillRect', style: rgba(DARK.substrate, 0.85) }, - { op: 'fillText', style: rgb(DARK.label) }, - ]); - for (const paint of paints) { - for (const white of WHITES) expect(paint.style).not.toMatch(white); - } - }); - - it('flips with the theme box without a rebuild: the same drawer re-lit from the light palette', () => { - const theme: ThemeBox = { colors: DARK }; - const drawer = createNodeHoverDrawer(theme); - theme.colors = LIGHT; - - const { context, paints } = recordingContext(); - drawer(context, NODE, SETTINGS); - - expect(paints).toEqual([ - { op: 'stroke', style: rgba(LIGHT.hot, 0.9) }, - { op: 'fillRect', style: rgba(LIGHT.substrate, 0.85) }, - { op: 'fillText', style: rgb(LIGHT.label) }, - ]); - }); - - it('draws only the ring for a body whose label the reducer withheld', () => { - const { context, paints } = recordingContext(); - createNodeHoverDrawer({ colors: DARK })(context, { ...NODE, label: '' }, SETTINGS); - - expect(paints).toEqual([{ op: 'stroke', style: rgba(DARK.hot, 0.9) }]); - }); -}); diff --git a/dashboard/src/viz/graph/nodeHover.ts b/dashboard/src/viz/graph/nodeHover.ts deleted file mode 100644 index 7eb1b42128..0000000000 --- a/dashboard/src/viz/graph/nodeHover.ts +++ /dev/null @@ -1,38 +0,0 @@ -import type { NodeHoverDrawingFunction } from 'sigma/rendering'; -import { rgb, rgba, type ThemeBox } from './palette.ts'; - -/** - * The hover pass, drawn in the field's own palette. - * - * Sigma's default (`drawDiscNodeHover`) paints an opaque white shadowed disc - * fused with a white label backdrop over the hovered body, on the dark - * instrument field that read as the node "going white" and growing a blob, - * and on the light field it erased the body against the paper. The reducers - * already carry the hover response (the body recolours to the hot accent and - * its neighbourhood isolates), so this 2d layer only adds a thin accent ring - * and a substrate-backed label that stay in the theme. - * - * Reads the {@link ThemeBox} per draw rather than capturing its colors, so a - * theme flip re-lights hovers without rebuilding the renderer, the same - * contract every other drawing pass on this canvas holds. - */ -export function createNodeHoverDrawer(theme: ThemeBox): NodeHoverDrawingFunction { - return (context, data, settings) => { - const colors = theme.colors; - context.beginPath(); - context.arc(data.x, data.y, data.size + 3, 0, Math.PI * 2); - context.strokeStyle = rgba(colors.hot, 0.9); - context.lineWidth = 1.5; - context.stroke(); - if (!data.label) return; - const size = settings.labelSize; - context.font = `${settings.labelWeight} ${size}px ${settings.labelFont}`; - const width = context.measureText(data.label).width; - const x = data.x + data.size + 6; - const y = data.y + size / 3; - context.fillStyle = rgba(colors.substrate, 0.85); - context.fillRect(x - 3, y - size, width + 6, size + 5); - context.fillStyle = rgb(colors.label); - context.fillText(data.label, x, y); - }; -} diff --git a/dashboard/src/viz/graph/renderer.test.ts b/dashboard/src/viz/graph/renderer.test.ts deleted file mode 100644 index a664abd185..0000000000 --- a/dashboard/src/viz/graph/renderer.test.ts +++ /dev/null @@ -1,260 +0,0 @@ -import Graph from 'graphology'; -import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { ActivationField, lerpRgbTuple, restingNodeTint, luma } from './activation.ts'; -import { createFieldRenderer, createFocusState, type FocusState } from './renderer.ts'; -import { rgb, rgba, type GraphPalette, type ThemeBox } from './palette.ts'; -import type { Settings } from 'sigma/settings'; - -/** - * The renderer's interaction states, asserted as the colors they actually - * paint, on both mediums. - * - * Every state on the canvas is a response to something real: rest is measured - * vitality, hover recolours the body to the hot accent and dims everything - * outside the neighbourhood, a strike lerps toward the accent and swells, and - * selection holds the accent. The regressions this file exists for are the - * theme-blind ones, a body or hover pass that paints white (or any color not - * derived from the theme box) reads as data on one medium and vanishes on the - * other. - */ - -type NodeAttributes = Record; -type NodeReducer = (node: string, data: NodeAttributes) => NodeAttributes; -type HoverDrawer = ( - context: CanvasRenderingContext2D, - data: { x: number; y: number; size: number; label: string; color: string }, - settings: Settings, -) => void; - -const captured = vi.hoisted(() => ({ - settings: undefined as - | { nodeReducer?: NodeReducer; defaultDrawNodeHover?: HoverDrawer } - | undefined, - handlers: new Map void>(), -})); - -vi.mock('sigma', () => ({ - default: class MockSigma { - constructor(_graph: Graph, _container: HTMLElement, settings: Record) { - captured.settings = settings; - } - setCustomBBox() {} - getCanvases(): Record { - return {}; - } - on(event: string, handler: (payload: never) => void) { - captured.handlers.set(event, handler); - } - refresh() {} - resize() { - return this; - } - setSetting() {} - kill() {} - }, -})); - -// The renderer reducer is exercised without a GPU; shader compilation belongs -// to the real-browser pass, just as Sigma construction does. -vi.mock('./glowProgram.ts', () => ({ GlowProgram: class {} })); - -const DARK: GraphPalette = { - hot: [93, 231, 255], - alert: [224, 182, 74], - edge: [55, 83, 114], - label: [196, 212, 232], - labelFont: 'ui-monospace, monospace', - substrate: [7, 11, 22], - dim: [38, 55, 76], - light: false, -}; - -const LIGHT: GraphPalette = { - hot: [11, 116, 145], - alert: [160, 110, 20], - edge: [148, 166, 188], - label: [32, 42, 56], - labelFont: 'ui-monospace, monospace', - substrate: [244, 246, 250], - dim: [210, 218, 228], - light: true, -}; - -const KIND_RGB: [number, number, number] = [120, 180, 240]; - -interface Harness { - reducer: NodeReducer; - hoverDrawer: HoverDrawer; - focus: FocusState; - field: ActivationField; - enterNode: (node: string) => void; - leaveNode: () => void; - attrs: (node: string) => NodeAttributes; - setSelected: (node: string | null) => void; -} - -function buildField(colors: GraphPalette): Harness { - const graph = new Graph({ multi: true, type: 'mixed' }); - for (const node of ['a', 'b', 'c']) { - graph.addNode(node, { - x: 0, - y: 0, - size: 5, - label: `symbol ${node}`, - vitality: 0.6, - kindRgb: KIND_RGB, - }); - } - graph.addEdgeWithKey('a->b', 'a', 'b', { srcReal: 'a', dstReal: 'b', size: 1 }); - const focus = createFocusState(); - const field = new ActivationField(); - let selected: string | null = null; - const theme: ThemeBox = { colors }; - createFieldRenderer({ - graph, - container: {} as HTMLElement, - theme, - focus, - field, - neighborsOf: new Map([ - ['a', ['b']], - ['b', ['a']], - ['c', []], - ]), - nodeCount: 3, - denseField: false, - roominess: 1, - frame: { x: [0, 1], y: [0, 1] }, - selectedId: () => selected, - onNodeClick: () => undefined, - onStageClick: () => undefined, - onInspect: () => undefined, - onFocusChange: () => undefined, - }); - const settings = captured.settings; - if (!settings?.nodeReducer || !settings.defaultDrawNodeHover) { - throw new Error('renderer registered no reducer or hover drawer'); - } - return { - reducer: settings.nodeReducer, - hoverDrawer: settings.defaultDrawNodeHover, - focus, - field, - enterNode: (node) => - captured.handlers.get('enterNode')?.({ node } as never), - leaveNode: () => captured.handlers.get('leaveNode')?.(undefined as never), - attrs: (node) => graph.getNodeAttributes(node), - setSelected: (node) => { - selected = node; - }, - }; -} - -function paintedColor(harness: Harness, node: string): string { - return String(harness.reducer(node, harness.attrs(node))['color']); -} - -beforeEach(() => { - captured.settings = undefined; - captured.handlers.clear(); -}); - -describe('field renderer interaction states', () => { - it('rests every body in its vitality tint, lit against the correct side of each medium', () => { - for (const colors of [DARK, LIGHT]) { - const harness = buildField(colors); - const painted = paintedColor(harness, 'a'); - expect(painted).toBe(rgb(restingNodeTint(colors.substrate, KIND_RGB, 0.6, colors.light))); - // The ember rule: a resting body never dims past its own background, - // lighter than a dark substrate, darker than a light one. - const match = /rgb\((\d+), (\d+), (\d+)\)/.exec(painted); - const paintedLuma = luma([ - Number(match?.[1]), - Number(match?.[2]), - Number(match?.[3]), - ]); - if (colors.light) expect(paintedLuma).toBeLessThan(luma(colors.substrate)); - else expect(paintedLuma).toBeGreaterThan(luma(colors.substrate)); - } - }); - - it('recolours the hovered body to the hot accent and dims only outside its neighbourhood', () => { - for (const colors of [DARK, LIGHT]) { - const harness = buildField(colors); - harness.enterNode('a'); - expect(harness.focus.target).toBe(1); - // The overlay owns the easing; the reducer reads wherever it has got to. - harness.focus.t = 1; - - expect(paintedColor(harness, 'a')).toBe(rgb(colors.hot)); - // The neighbour keeps its resting tint; the stranger dims to the token. - const resting = restingNodeTint(colors.substrate, KIND_RGB, 0.6, colors.light); - expect(paintedColor(harness, 'b')).toBe(rgb(resting)); - expect(paintedColor(harness, 'c')).toBe( - rgb(lerpRgbTuple(resting, colors.dim, 1)), - ); - - harness.leaveNode(); - expect(harness.focus.target).toBe(0); - } - }); - - it('dims glow companions with their owner instead of leaving bright orphan rings', () => { - const harness = buildField(DARK); - harness.enterNode('a'); - harness.focus.t = 1; - const companion = { - owner: 'c', - glowRgb: KIND_RGB, - glowAlpha: 0.1, - color: rgba(KIND_RGB, 0.1), - }; - - expect(harness.reducer('__halo__c', companion)['color']).toBe(rgba(DARK.dim, 0.008)); - expect(harness.reducer('__halo__b', { ...companion, owner: 'b' })['color']).toBe( - companion.color, - ); - }); - - it('lerps a struck body toward the accent and swells it with its heat', () => { - const harness = buildField(DARK); - harness.field.strike(['b'], 0.8); - const resting = restingNodeTint(DARK.substrate, KIND_RGB, 0.6, DARK.light); - const reduced = harness.reducer('b', harness.attrs('b')); - expect(reduced['color']).toBe(rgb(lerpRgbTuple(resting, DARK.hot, 0.8))); - expect(reduced['size']).toBeCloseTo(5 * (1 + 0.5 * 0.8)); - }); - - it('holds the selected body at the accent above the field', () => { - const harness = buildField(DARK); - harness.setSelected('c'); - const reduced = harness.reducer('c', harness.attrs('c')); - expect(reduced['color']).toBe(rgb(DARK.hot)); - expect(reduced['zIndex']).toBe(3); - }); - - it('wires the theme hover drawer in place of sigma default white disc', () => { - const harness = buildField(DARK); - const strokes: string[] = []; - const context = { - strokeStyle: '', - fillStyle: '', - lineWidth: 0, - font: '', - beginPath: () => undefined, - arc: () => undefined, - stroke() { - strokes.push(String(this.strokeStyle)); - }, - measureText: () => ({ width: 30 }), - fillRect: () => undefined, - fillText: () => undefined, - }; - harness.hoverDrawer( - context as unknown as CanvasRenderingContext2D, - { x: 0, y: 0, size: 5, label: 'symbol a', color: rgb(DARK.hot) }, - { labelSize: 11, labelWeight: 'normal', labelFont: 'monospace' } as Settings, - ); - expect(strokes).toEqual([rgba(DARK.hot, 0.9)]); - }); -}); diff --git a/dashboard/src/viz/graph/renderer.ts b/dashboard/src/viz/graph/renderer.ts deleted file mode 100644 index 947bd394e4..0000000000 --- a/dashboard/src/viz/graph/renderer.ts +++ /dev/null @@ -1,404 +0,0 @@ -import type Graph from 'graphology'; -import Sigma from 'sigma'; -import { - cssColorToRgb, - lerpRgb, - lerpRgbTuple, - restingNodeTint, - type ActivationField, -} from './activation.ts'; -import { kindColor } from './kindColor.ts'; -import { isManaged } from './managed.ts'; -import { createNodeHoverDrawer } from './nodeHover.ts'; -import { GlowProgram } from './glowProgram.ts'; -import { palette, rgb, rgba, type ThemeBox } from './palette.ts'; -import type { FieldFrame } from './layout.ts'; - -/** - * Sigma instance lifecycle: construct the renderer over a prepared graph, wire - * its pointer events, absorb a resize, re-sample the theme, and kill it. - * - * Everything here is about drawing. What is drawn was already decided by the - * layout pass; what MOVES is decided by the activation overlay, which drives - * this renderer's repaints. The reducers below are the seam between the two: - * they read the live heat and hover state on every frame the overlay paints. - */ - -/** - * How isolated the hovered neighbourhood currently is. `t` is eased rather - * than switched so focus propagates outward instead of blinking; the overlay's - * loop advances it toward `target`. - */ -export interface FocusState { - node: string | null; - t: number; - target: number; -} - -export function createFocusState(): FocusState { - return { node: null, t: 0, target: 0 }; -} - -export interface FieldRendererOptions { - graph: Graph; - container: HTMLElement; - theme: ThemeBox; - focus: FocusState; - field: ActivationField; - neighborsOf: ReadonlyMap; - nodeCount: number; - denseField: boolean; - roominess: number; - frame: FieldFrame; - /** Read per call, never captured: selection changes without a rebuild. */ - selectedId: () => string | null | undefined; - onNodeClick: (node: string) => void; - onStageClick: () => void; - /** Pointer focus leaves the canvas through this bridge; DOM focus enters - * through {@link FieldRenderer.focusNode}. Neither path is activity. */ - onInspect: (node: string | null) => void; - /** The hover state just changed; the overlay has an easing to run. */ - onFocusChange: () => void; -} - -export interface FieldRenderer { - refresh(): void; - resize(): void; - focusNode(node: string | null): void; - zoomIn(reduced: boolean): void; - zoomOut(reduced: boolean): void; - fit(reduced: boolean): void; - /** Re-sample the theme tokens and re-derive anything baked from them. */ - retheme(): void; - /** - * The layers this renderer draws WebGL into, so the one thing that can stop - * a drawn field from being a field, the GPU dropping its context, can be - * watched where it actually happens. - * - * Sigma stacks several canvases over the container and only some of them are - * WebGL (labels, hover decoration and pointer capture ride 2d), so the set is - * taken from the renderer's own layer map and each layer is asked which - * context it holds, no layer name is assumed and nothing reads the - * container's children. Captured at construction because {@link kill} empties - * that map, and a restore is dispatched at the canvas of the renderer that - * died. - */ - readonly webGlCanvases: readonly HTMLCanvasElement[]; - kill(): void; -} - -export function createFieldRenderer({ - graph, - container, - theme, - focus, - field, - neighborsOf, - nodeCount, - denseField, - roominess, - frame, - selectedId, - onNodeClick, - onStageClick, - onInspect, - onFocusChange, -}: FieldRendererOptions): FieldRenderer { - const roomyDenseField = denseField && roominess >= 0.8; - - /** Neighbourhood of the current hover, rebuilt only when focus changes. - * The node reducer used to walk each body's adjacency array with - * `includes` on every frame. */ - let focusedNeighborhood: Set | null = null; - let focusedNeighborhoodOf: string | null = null; - const neighborhoodOf = (hovered: string | null): Set | null => { - if (hovered == null) { - focusedNeighborhood = null; - focusedNeighborhoodOf = null; - return null; - } - if (hovered === focusedNeighborhoodOf && focusedNeighborhood != null) { - return focusedNeighborhood; - } - const next = new Set(neighborsOf.get(hovered) ?? []); - next.add(hovered); - focusedNeighborhood = next; - focusedNeighborhoodOf = hovered; - return next; - }; - - const sigma = new Sigma(graph, container, { - nodeProgramClasses: { glow: GlowProgram }, - // Sigma's default hover pass paints an opaque white shadowed disc over - // the hovered body; ours stays in the field's palette. See nodeHover.ts. - defaultDrawNodeHover: createNodeHoverDrawer(theme), - renderLabels: true, - labelRenderedSizeThreshold: roomyDenseField - ? 0 - : denseField - ? 6.5 - : nodeCount <= 60 - ? 4.5 - : 8, - labelDensity: 1, - labelGridCellSize: roomyDenseField ? 90 : 100, - labelFont: theme.colors.labelFont, - labelSize: roomyDenseField ? 12 : 11, - labelColor: { color: rgb(theme.colors.label) }, - defaultEdgeColor: rgba(theme.colors.edge, 0.9), - // Every reducer below hands back a `zIndex` (bloom=0, halo=1, body=2, - // selected/hot=3, travelling pulse=4) on the assumption that draw order - // honours it. Sigma does not, unless told to: this setting is off by - // default, so without it every node paints in graph insertion order - // regardless of its zIndex attribute. The glow companions are inserted - // AFTER the real body (`syncGlow` runs once the body already exists), - // so they were silently painting OVER it -- their own faint colour, - // stacked as halo then bloom on top of an opaque body, was what read as - // a plain white disc on the light theme's near-white field (worst on - // the largest, highest-degree bodies, which carry the largest glow). - // Enabling real z-ordering restores what the reducers already intended: - // bloom, then halo, then the body on top, then anything hot. - zIndex: true, - nodeReducer: (node, data) => { - const colors = theme.colors; - if (isManaged(node)) { - const owner = data['owner']; - const glowRgb = data['glowRgb']; - const glowAlpha = data['glowAlpha']; - if (typeof owner !== 'string' || !Array.isArray(glowRgb) || typeof glowAlpha !== 'number') { - return data; - } - const dim = - focus.node != null && neighborhoodOf(focus.node)?.has(owner) !== true ? focus.t : 0; - if (dim === 0) return data; - return { - ...data, - color: rgba( - lerpRgbTuple(glowRgb as [number, number, number], colors.dim, dim), - glowAlpha * (1 - 0.92 * dim), - ), - }; - } - const hovered = focus.node; - const isSelected = node === selectedId(); - const isHovered = node === hovered; - const isNeighbor = neighborhoodOf(hovered)?.has(node) === true; - const dim = hovered != null && !isNeighbor ? focus.t : 0; - const heat = field.heatOf(node); - const vitality = (data['vitality'] as number | undefined) ?? 0.6; - // Two axes, both real. Vitality is the slow one: a node's resting - // luminance is how alive the caller measured it to be, so a dormant - // corner of the graph literally recedes into the substrate while a - // live one holds its hue. Heat is the fast one: a strike blooms the - // node toward the accent and swells it, then decays on the field's - // exponential half-life. Hover isolation is a third, transient mix - // toward the neutral dim token so an isolated neighbourhood is - // unambiguous. - const [kr, kg, kb] = (data['kindRgb'] as [number, number, number] | undefined) ?? [ - 149, 152, 157, - ]; - let tint = restingNodeTint(colors.substrate, [kr, kg, kb], vitality, colors.light); - if (dim > 0) tint = lerpRgbTuple(tint, colors.dim, dim); - const color = - isSelected || isHovered - ? rgb(colors.hot) - : heat > 0 - ? lerpRgb(tint, colors.hot, Math.min(1, heat)) - : rgb(tint); - return { - ...data, - color, - size: (data['size'] as number) * (1 + 0.5 * heat), - zIndex: isSelected || isHovered || heat > 0.4 ? 3 : 2, - label: - isSelected || isHovered || heat > 0.5 || data['isHub'] || nodeCount <= 60 - ? data['label'] - : '', - }; - }, - edgeReducer: (edge, data) => { - const colors = theme.colors; - const hovered = focus.node; - const from = (data['srcReal'] as string | undefined) ?? ''; - const to = (data['dstReal'] as string | undefined) ?? ''; - const dim = hovered != null && from !== hovered && to !== hovered ? focus.t : 0; - // A relation conducts only when both of its ends are warm, that is - // what makes it a synapse rather than a wire. Vitality sets how - // present the tissue is at rest. - const edgeHeat = Math.min(field.heatOf(from), field.heatOf(to)); - const restVitality = - (((graph.hasNode(from) ? (graph.getNodeAttribute(from, 'vitality') as number) : 0.6) ?? - 0.6) + - ((graph.hasNode(to) ? (graph.getNodeAttribute(to, 'vitality') as number) : 0.6) ?? - 0.6)) / - 2; - const alpha = (0.36 + 0.5 * restVitality) * (1 - 0.92 * dim); - const color = - edgeHeat > 0.05 - ? rgba( - lerpRgbTuple(colors.edge, colors.hot, Math.min(1, edgeHeat)), - Math.min(1, alpha + 0.4 * edgeHeat), - ) - : rgba(colors.edge, alpha); - return { ...data, color, size: edgeHeat > 0.05 ? 1 + 2 * edgeHeat : data['size'] }; - }, - }); - - sigma.setCustomBBox({ x: frame.x, y: frame.y }); - - sigma.on('enterNode', ({ node }) => { - if (isManaged(node)) return; - focus.node = node; - focus.target = 1; - onFocusChange(); - onInspect(node); - }); - sigma.on('leaveNode', () => { - focus.target = 0; - onFocusChange(); - onInspect(null); - }); - sigma.on('clickNode', ({ node }) => { - if (isManaged(node)) return; - onNodeClick(node); - }); - sigma.on('clickStage', () => onStageClick()); - - const webGlCanvases = Object.values(sigma.getCanvases()).filter( - (canvas) => webGlContextOf(canvas) !== null, - ); - - return { - webGlCanvases, - refresh: () => { - sigma.refresh(); - }, - resize: () => { - sigma.resize(); - // Sigma clears its WebGL buffers while resizing. The resting field has - // no animation loop to repaint them, so a resize must restore one frame. - sigma.refresh(); - }, - focusNode: (node) => { - const next = node != null && graph.hasNode(node) && !isManaged(node) ? node : null; - if (next == null) focus.target = 0; - else { - focus.node = next; - focus.target = 1; - } - onFocusChange(); - }, - zoomIn: (reduced) => { - const camera = sigma.getCamera(); - if (reduced) camera.setState({ ratio: camera.getBoundedRatio(camera.ratio / 1.5) }); - else void camera.animatedZoom({ factor: 1.5, duration: 180 }); - }, - zoomOut: (reduced) => { - const camera = sigma.getCamera(); - if (reduced) camera.setState({ ratio: camera.getBoundedRatio(camera.ratio * 1.5) }); - else void camera.animatedUnzoom({ factor: 1.5, duration: 180 }); - }, - fit: (reduced) => { - const camera = sigma.getCamera(); - if (reduced) camera.setState({ x: 0.5, y: 0.5, ratio: 1, angle: 0 }); - else void camera.animatedReset({ duration: 180 }); - }, - retheme: () => { - const wasLight = theme.colors.light; - theme.colors = palette(container); - sigma.setSetting('defaultEdgeColor', rgba(theme.colors.edge, 0.9)); - sigma.setSetting('labelColor', { color: rgb(theme.colors.label) }); - // kindRgb is baked once at construction, so a theme flip used to leave - // every node wearing the other theme's lightness, the hues only looked - // right on whichever theme happened to be active at mount. Re-derive - // them when, and only when, the medium actually changed sides. - if (theme.colors.light !== wasLight) { - for (const node of graph.nodes()) { - if (isManaged(node)) continue; - const kind = graph.getNodeAttribute(node, 'kind') as string | undefined; - if (kind == null) continue; - graph.setNodeAttribute( - node, - 'kindRgb', - cssColorToRgb(kindColor(kind, theme.colors.light)), - ); - } - } - }, - kill: () => { - sigma.kill(); - }, - }; -} - -/** Whether this browser can give Sigma a WebGL context at all. Probed once per - * canvas mount against a throwaway element: a blocklisted or disabled GPU - * stack returns null here rather than throwing inside the renderer. */ -export function hasWebGl(): boolean { - if (typeof document === 'undefined') return false; - return webGlContextOf(document.createElement('canvas')) !== null; -} - -/** - * The WebGL context a canvas holds, or null when it holds another kind. - * - * A question rather than a mutation on any canvas Sigma has already made: - * `getContext` hands back the existing context for a matching id and null for a - * mismatched one, so the 2d layers answer null without being disturbed. One - * rule for both readings of it, whether this browser can draw at all, and - * whether this is a layer that draws. - */ -function webGlContextOf(canvas: HTMLCanvasElement): RenderingContext | null { - try { - return ( - canvas.getContext('webgl2') ?? - canvas.getContext('webgl') ?? - canvas.getContext('experimental-webgl') - ); - } catch { - return null; - } -} - -/** What the GPU did to a drawn field, told to whoever can state it. */ -export interface WebGlContextHandlers { - /** The context is gone; nothing on these canvases is a reading any more. */ - onLost: () => void; - /** The browser gave the context back; the field can be composed again. */ - onRestored: () => void; -} - -/** - * Watch a renderer's WebGL layers for a context lost, or restored, by the - * GPU stack, which is the one failure that arrives AFTER a successful draw. - * - * The release is deliberately not the renderer's `kill`. A lost context has to - * take the renderer with it (Sigma's own window listener would otherwise - * measure a container that has gone), yet the restore that brings the field - * back is dispatched at the canvas of the renderer that died, so this watch - * outlives it and belongs to whoever owns the surface, not to one composition. - */ -export function watchWebGlContext( - canvases: readonly HTMLCanvasElement[], - handlers: WebGlContextHandlers, -): () => void { - const lost = (event: Event): void => { - // Unconditional, and before anything else: the default action of - // `webglcontextlost` is to abandon the context for good, so without this - // the browser never attempts a restore and `webglcontextrestored` can - // never arrive. - event.preventDefault(); - handlers.onLost(); - }; - const restored = (): void => handlers.onRestored(); - for (const canvas of canvases) { - canvas.addEventListener('webglcontextlost', lost); - canvas.addEventListener('webglcontextrestored', restored); - } - return () => { - for (const canvas of canvases) { - canvas.removeEventListener('webglcontextlost', lost); - canvas.removeEventListener('webglcontextrestored', restored); - } - }; -} diff --git a/dashboard/src/viz/graph/scene.ts b/dashboard/src/viz/graph/scene.ts deleted file mode 100644 index 20d03b3380..0000000000 --- a/dashboard/src/viz/graph/scene.ts +++ /dev/null @@ -1,192 +0,0 @@ -import { cssColorToRgb, type ActivationField } from './activation.ts'; -import { createActivationOverlay } from './activationOverlay.ts'; -import { frameEmergentField } from './emergentField.ts'; -import { settleEmergentOffThread } from './emergentLayout.ts'; -import { kindColor } from './kindColor.ts'; -import { buildDendrites, prepareField, type FieldFrame, type PreparedField } from './layout.ts'; -import { frameMeasuredField } from './measuredField.ts'; -import { palette, type ThemeBox } from './palette.ts'; -import { createFieldRenderer, createFocusState } from './renderer.ts'; -import type { FieldExtent, GraphCanvasEdge, GraphCanvasNode } from './types.ts'; - -/** - * A live field: one prepared layout, one Sigma renderer, one activation - * overlay, and the single latch that kills all three together. - * - * The two builders below are the field's two paths, and they differ in exactly - * one thing: whether the coordinates were measured by the caller or have to be - * discovered. A measured field is composed synchronously and never reaches the - * layout engine at all; an emergent one waits for that engine before a single - * pixel is drawn, so the reader never sees the seed circle it starts from. - */ -export interface GraphScene { - /** Repaint the current composition. Static: no loop is started. */ - repaint(): void; - resize(): void; - focusNode(node: string | null): void; - zoomIn(): void; - zoomOut(): void; - fit(): void; - settle(): void; - wake(): void; - retheme(): void; - /** The layers whose WebGL context this whole composition depends on, for a - * surface that has to state a context lost after the field was drawn. They - * outlive the scene, because a restore is dispatched at the canvas of the - * renderer that died. */ - readonly webGlCanvases: readonly HTMLCanvasElement[]; - /** Idempotent: React's cleanup, the size observer and a lost context all - * call it, and on an ordinary unmount two of them fire. */ - teardown(): void; -} - -export interface SceneRequest { - container: HTMLElement; - nodes: readonly GraphCanvasNode[]; - edges: readonly GraphCanvasEdge[]; - extent: FieldExtent | undefined; - field: ActivationField; - selectedId: () => string | null | undefined; - inspectedId: () => string | null | undefined; - onSelect: (id: string | null) => void; - onInspect: (id: string | null) => void; - isReduced: () => boolean; -} - -/** Build the scene for a field whose coordinates are the caller's own - * measurement. Synchronous end to end, there is no layout to wait for. */ -export function buildMeasuredScene(request: SceneRequest): GraphScene { - const { theme, prepared } = prepare(request); - return compose(request, theme, prepared, frameMeasuredField(prepared, request.extent)); -} - -/** - * Build the scene for a field whose shape is the finding. - * - * The graph is constructed first, then settled in a bounded worker, and only - * then is anything drawn: Sigma is not constructed until the coordinates are - * final, so there is no frame in which the seed circle is on screen. Resolves - * to `null` when the caller cancels the worker job. Termination stops the - * calculation, and no result reaches a component that is gone. - */ -export async function buildEmergentScene( - request: SceneRequest, - signal: AbortSignal, -): Promise { - const { theme, prepared } = prepare(request); - if (!await settleEmergentOffThread(prepared, signal) || signal.aborted) return null; - return compose(request, theme, prepared, frameEmergentField(prepared)); -} - -function prepare(request: SceneRequest): { theme: ThemeBox; prepared: PreparedField } { - const { container, nodes, edges } = request; - const theme: ThemeBox = { colors: palette(container) }; - const prepared = prepareField({ - nodes, - edges, - viewport: { width: container.clientWidth, height: container.clientHeight }, - kindRgb: (kind) => cssColorToRgb(kindColor(kind, theme.colors.light)), - }); - return { theme, prepared }; -} - -function compose( - request: SceneRequest, - theme: ThemeBox, - prepared: PreparedField, - frame: FieldFrame, -): GraphScene { - const { container, edges, field, inspectedId, isReduced, onInspect, onSelect, selectedId } = - request; - const { graph, realNodes, neighborsOf, nodeCount, denseField, roominess } = prepared; - const strands = buildDendrites(graph, edges.length); - const focus = createFocusState(); - - /** - * One-way latch guarding every repaint below. Once the container has lost - * its box there is no such thing as a correct frame, so the loop is not - * slowed or deferred, it stops, and `paint` becomes a no-op for whatever - * is still holding a closure over this renderer. - */ - let alive = true; - let renderer: ReturnType | null = null; - const paint = (): void => { - if (alive) renderer?.refresh(); - }; - - const overlay = createActivationOverlay({ - graph, - realNodes, - strands, - field, - theme, - focus, - paint, - isReduced, - }); - - renderer = createFieldRenderer({ - graph, - container, - theme, - focus, - field, - neighborsOf, - nodeCount, - denseField, - roominess, - frame, - selectedId, - // Selection is reader intent, never an admitted activity event. - onNodeClick: onSelect, - onStageClick: () => onSelect(null), - onInspect, - onFocusChange: () => overlay.wake(), - }); - const initialInspection = inspectedId(); - if (initialInspection != null) renderer.focusNode(initialInspection); - - // One static composition of the resting field, so the graph is fully - // rendered before anything ever fires. - overlay.repaintResting(); - // Heat that landed while the layout engine was loading is still real; the - // field has no clock of its own, so nothing else would ever draw or decay it. - if (field.warm) overlay.wake(); - - return { - webGlCanvases: renderer.webGlCanvases, - repaint: paint, - resize: () => { - if (alive) renderer?.resize(); - }, - focusNode: (node) => { - if (alive) renderer?.focusNode(node); - }, - zoomIn: () => { - if (alive) renderer?.zoomIn(isReduced()); - }, - zoomOut: () => { - if (alive) renderer?.zoomOut(isReduced()); - }, - fit: () => { - if (alive) renderer?.fit(isReduced()); - }, - settle: () => { - if (alive) overlay.settle(); - }, - wake: () => { - if (alive) overlay.wake(); - }, - retheme: () => { - if (!alive) return; - renderer?.retheme(); - overlay.repaintResting(); - }, - teardown: () => { - if (!alive) return; - alive = false; - overlay.stop(); - renderer?.kill(); - }, - }; -} diff --git a/dashboard/src/viz/temporal/TemporalScene.dom.test.tsx b/dashboard/src/viz/temporal/TemporalScene.dom.test.tsx index 6989bd5d80..f7765db5a2 100644 --- a/dashboard/src/viz/temporal/TemporalScene.dom.test.tsx +++ b/dashboard/src/viz/temporal/TemporalScene.dom.test.tsx @@ -90,7 +90,7 @@ function fixture(): TemporalSceneModel { }, cursor: { x: 320, laneId: ROOT, xBasis: 'time' }, counts: { lanesTotal: 7, lanesVisible: 3, lanesCollapsed: 4, eventsTotal: 12, eventsDrawn: 6, eventsCulled: 0, eventsWithheld: 4, eventsFiltered: 0, eventsFolded: 2, relationsTotal: 2, relationsDrawn: 1, relationsWithheld: 1 }, - denseDefault: false, + denseDepth: null, }; } @@ -133,7 +133,6 @@ function renderScene(overrides: Partial = {}) { { expect(container.querySelector('[data-scene-layer="canvas"]')).toBeTruthy(); expect(container.querySelector('[data-scene-layer="unavailable"]')).toBeNull(); expect(calls).toContain('stroke'); - expect(calls).toContain('bezierCurveTo'); + expect(calls).toContain('arcTo'); expect(screen.queryByRole('status')).toBeNull(); }); @@ -179,6 +178,15 @@ describe('TemporalScene', () => { }); }); + describe('cursor and tail', () => { + it('draws the reveal cursor at the model x', () => { + const { container } = renderScene(); + const cursor = container.querySelector('[data-cursor]')!; + expect(cursor.getAttribute('x1')).toBe('320'); + expect(container.querySelector('[data-cursor-mark]')?.getAttribute('data-cursor-basis')).toBe('time'); + }); + }); + describe('events', () => { it('renders one button per node with a Select label', () => { const { container, model } = renderScene(); @@ -199,6 +207,29 @@ describe('TemporalScene', () => { expect(onSelectEvent).toHaveBeenCalledTimes(2); }); + it('walks the glyphs from one scene tab stop with arrows, Home, End and Enter', () => { + const { container, model, onSelectEvent } = renderScene(); + const overlay = container.querySelector('svg[data-scene-layer="overlay"]')!; + expect(overlay.getAttribute('tabindex')).toBe('0'); + for (const glyph of container.querySelectorAll('[data-event]')) expect(glyph.getAttribute('tabindex')).toBe('-1'); + const position = () => container.querySelector('[data-event-position]')?.textContent; + const current = () => container.querySelector('[data-event-current]')!; + const total = model.nodes.length; + expect(position()).toBe(`event 1 of ${total}`); + fireEvent.keyDown(overlay, { key: 'ArrowRight' }); + expect(position()).toBe(`event 2 of ${total}`); + expect(overlay.getAttribute('aria-activedescendant')).toBe(current().id); + fireEvent.keyDown(overlay, { key: 'End' }); + expect(position()).toBe(`event ${total} of ${total}`); + fireEvent.keyDown(overlay, { key: 'ArrowRight' }); + expect(position()).toBe(`event ${total} of ${total}`); + fireEvent.keyDown(overlay, { key: 'Home' }); + expect(position()).toBe(`event 1 of ${total}`); + const first = current().getAttribute('data-event'); + fireEvent.keyDown(overlay, { key: 'Enter' }); + expect(onSelectEvent).toHaveBeenLastCalledWith(first); + }); + it('declares a sequence-placed node as recorded order', () => { const { container } = renderScene(); const sequenced = container.querySelector('[data-event="n-msg"]')!; @@ -214,6 +245,8 @@ describe('TemporalScene', () => { expect(onInspect).toHaveBeenLastCalledWith(model.nodes.find((entry) => entry.id === 'n-tool')); const otherLane = container.querySelector(`[data-lane-group='${CHILD}']`) as SVGGElement; expect(otherLane.style.opacity).toBe('0.55'); + // Hover dims the unrelated and draws nothing on the hovered mark. + expect(tool.querySelector('rect[fill="none"][stroke="var(--raw-graph-accent)"], line[stroke="var(--raw-graph-accent)"]')).toBeNull(); fireEvent.mouseOut(tool); expect(onInspect).toHaveBeenLastCalledWith(null); expect(otherLane.style.opacity).toBe('1'); @@ -260,15 +293,6 @@ describe('TemporalScene', () => { }); }); - describe('cursor and tail', () => { - it('draws the reveal cursor at the model x and the tail marker label', () => { - const { container } = renderScene(); - const cursor = container.querySelector('[data-cursor]')!; - expect(cursor.getAttribute('x1')).toBe('320'); - expect(screen.getByText('LOADED END')).toBeTruthy(); - }); - }); - describe('time window', () => { it('zooms in to the middle half around the centre', () => { const { onWindowChange } = renderScene(); diff --git a/dashboard/src/viz/temporal/TemporalScene.tsx b/dashboard/src/viz/temporal/TemporalScene.tsx index 0c773400d2..c91c6e4b86 100644 --- a/dashboard/src/viz/temporal/TemporalScene.tsx +++ b/dashboard/src/viz/temporal/TemporalScene.tsx @@ -1,12 +1,11 @@ /** * Renderer for a laid-out `TemporalSceneModel`. * - * Two layers over one coordinate space. A Canvas2D substrate carries the - * atmosphere, rails, grid, cluster bodies, glowing threads, the unrevealed - * hatch, and is painted once per model or palette change; nothing here - * animates. A crisp SVG overlay carries every selectable mark, every label and - * every title, so the surface stays complete when the canvas is missing and - * every pointer action has a keyboard path. + * Two layers over one coordinate space. A Canvas2D substrate (`scene/paint`), + * painted once per model, density or palette change; nothing there animates. + * A crisp SVG overlay carries every selectable mark, every label and every + * title, so the surface stays complete when the canvas is missing and every + * pointer action has a keyboard path. * * This component draws. It never lays out, never grades, and never invents a * quantity: every coordinate comes from the model. Hover only inspects. @@ -14,22 +13,18 @@ import type { JSX, KeyboardEvent, PointerEvent as ReactPointerEvent } from 'react'; import { useEffect, useId, useMemo, useRef, useState } from 'react'; import { formatMoment } from '../../workspaces/loom/tracks.ts'; -import { EventGlyph, glyphLabel, TemporalLegend } from './glyphs.tsx'; -import { - gradeColorVar, - gradeDashArray, - gradeStroke, - resolveTemporalPalette, - type TemporalPalette, -} from './palette.ts'; +import type { SceneDensity } from './density.ts'; +import { glyphLabel, TemporalLegend } from './glyphs.tsx'; +import { resolveTemporalPalette, type TemporalPalette } from './palette.ts'; +import { focusAlpha, type SceneFrame } from './scene/frame.ts'; +import { ClusterMark, FieldOverlay, LegendEncodings, laneDetail, NODE_CLASS, NodeMark } from './scene/marks.tsx'; +import { paintScene } from './scene/paint.ts'; import type { - FocusTreatment, SceneCluster, SceneGap, SceneInterval, SceneLane, SceneNode, - ScenePath, SceneWindow, TemporalSceneModel, } from './types.ts'; @@ -44,13 +39,13 @@ export interface TemporalSceneProps { onWindowChange: (window: SceneWindow) => void; /** Pixel width the caller should lay out for; reported when the host resizes. */ onMeasure?: (width: number) => void; - /** Newest loaded record label for the right marker, e.g. 'LOADED END' or 'NOW'. */ - tailLabel: string; reducedMotion: boolean; onInspect?: (node: SceneNode | null) => void; className?: string; /** The whole projection extent, so Fit has somewhere to return to. */ fullWindow?: SceneWindow; + /** Per-lane density over the same window: summaries, totals, recency, NOW. */ + density?: SceneDensity | null; } /** Height of the time ruler strip across the top of the field. */ @@ -72,22 +67,6 @@ interface DragState { /* ---- shared encodings ---------------------------------------------------- */ -function focusAlpha(focus: FocusTreatment): number { - switch (focus) { - case 'context': - return 0.35; - case 'path': - return 0.8; - case 'selected': - case 'neutral': - return 1; - default: { - const exhaustive: never = focus; - throw new Error(`unknown focus treatment: ${String(exhaustive)}`); - } - } -} - function proximityTone(tone: SceneInterval['tone']): string { switch (tone) { case 'candidate': @@ -122,184 +101,6 @@ function sameWindow(a: SceneWindow, b: SceneWindow, tolerance: number): boolean return Math.abs(a.start - b.start) <= tolerance && Math.abs(a.end - b.end) <= tolerance; } -/* ---- canvas substrate ---------------------------------------------------- */ - -function tracePath(ctx: CanvasRenderingContext2D, controls: readonly number[]): void { - if (controls.length >= 8) { - ctx.moveTo(controls[0]!, controls[1]!); - ctx.bezierCurveTo(controls[2]!, controls[3]!, controls[4]!, controls[5]!, controls[6]!, controls[7]!); - } else if (controls.length >= 4) { - ctx.moveTo(controls[0]!, controls[1]!); - ctx.lineTo(controls[2]!, controls[3]!); - } -} - -function strokePath( - ctx: CanvasRenderingContext2D, - path: ScenePath, - palette: TemporalPalette, - glowWidth: number, - coreWidth: number, -): void { - const stroke = gradeStroke(path.grade, palette); - const alpha = focusAlpha(path.focus); - ctx.lineCap = 'round'; - ctx.lineJoin = 'round'; - - ctx.save(); - if (!palette.light) ctx.globalCompositeOperation = 'lighter'; - ctx.globalAlpha = 0.1 * alpha; - ctx.strokeStyle = stroke.color; - ctx.lineWidth = glowWidth; - ctx.setLineDash([]); - ctx.beginPath(); - tracePath(ctx, path.controls); - ctx.stroke(); - ctx.restore(); - - ctx.globalAlpha = alpha; - ctx.strokeStyle = stroke.color; - ctx.lineWidth = coreWidth; - ctx.setLineDash([...stroke.dash]); - ctx.beginPath(); - tracePath(ctx, path.controls); - ctx.stroke(); - ctx.setLineDash([]); -} - -function drawScene(ctx: CanvasRenderingContext2D, model: TemporalSceneModel, palette: TemporalPalette): void { - const { viewport, height } = model; - const fieldX0 = viewport.left; - const fieldX1 = viewport.width - viewport.right; - const fieldWidth = Math.max(1, fieldX1 - fieldX0); - - ctx.save(); - ctx.beginPath(); - ctx.rect(fieldX0, 0, fieldWidth, height); - ctx.clip(); - - for (const rail of model.rails) { - ctx.globalAlpha = palette.light ? 0.05 : 0.07; - ctx.fillStyle = palette.text; - ctx.fillRect(fieldX0, rail.y0, fieldWidth, rail.y1 - rail.y0); - ctx.globalAlpha = 0.4; - ctx.fillStyle = palette.edge; - ctx.fillRect(fieldX0, rail.y0, fieldWidth, 1); - } - - ctx.globalAlpha = 0.35; - ctx.strokeStyle = palette.grid; - ctx.lineWidth = 1; - ctx.setLineDash([]); - for (const tick of model.ticks) { - const x = Math.round(tick.x) + 0.5; - ctx.beginPath(); - ctx.moveTo(x, RULER); - ctx.lineTo(x, height); - ctx.stroke(); - } - - // Records stop at a dated cursor. The layout already withholds later ones; - // this clip keeps a curve's easing from reaching into the unrevealed band. - const recordX1 = model.cursor && model.cursor.xBasis === 'time' ? Math.min(fieldX1, model.cursor.x) : fieldX1; - ctx.save(); - ctx.beginPath(); - ctx.rect(fieldX0, 0, Math.max(0, recordX1 - fieldX0), height); - ctx.clip(); - - for (const cluster of model.clusters) { - const alpha = focusAlpha(cluster.focus); - const x1 = Math.max(cluster.x1, cluster.x0 + 2); - const body = ctx.createLinearGradient(cluster.x0, 0, x1, 0); - body.addColorStop(0, 'transparent'); - body.addColorStop(0.18, palette.signal); - body.addColorStop(0.82, palette.signal); - body.addColorStop(1, 'transparent'); - ctx.globalAlpha = 0.18 * alpha; - ctx.fillStyle = body; - ctx.fillRect(cluster.x0, cluster.y - cluster.height / 2, x1 - cluster.x0, cluster.height); - ctx.globalAlpha = 0.7 * alpha; - ctx.strokeStyle = palette.signalHot; - ctx.lineWidth = 1.5; - ctx.beginPath(); - ctx.moveTo(cluster.x0, cluster.y); - ctx.lineTo(x1, cluster.y); - ctx.stroke(); - } - - for (const path of model.paths) { - switch (path.kind) { - case 'lane': { - const weight = path.weight ?? 0; - const stroke = gradeStroke(path.grade, palette); - strokePath(ctx, path, palette, 6 + weight * 10, stroke.width + weight * 1.4); - break; - } - case 'spawn': - case 'handoff': - case 'rejoin': - case 'result': { - const stroke = gradeStroke(path.grade, palette); - strokePath(ctx, path, palette, 5, stroke.width); - break; - } - case 'sequence': { - ctx.globalAlpha = 0.55 * focusAlpha(path.focus); - ctx.strokeStyle = palette.text; - ctx.lineWidth = 1; - ctx.setLineDash([2, 3]); - ctx.beginPath(); - tracePath(ctx, path.controls); - ctx.stroke(); - ctx.setLineDash([]); - break; - } - default: { - const exhaustive: never = path.kind; - throw new Error(`unknown path kind: ${String(exhaustive)}`); - } - } - } - - for (const node of model.nodes) { - const stroke = gradeStroke(node.grade, palette); - const radius = node.selected ? 22 : 13; - const halo = ctx.createRadialGradient(node.x, node.y, 1, node.x, node.y, radius); - halo.addColorStop(0, stroke.color); - halo.addColorStop(1, 'transparent'); - ctx.save(); - if (!palette.light) ctx.globalCompositeOperation = 'lighter'; - ctx.globalAlpha = (node.selected ? 0.42 : 0.26) * focusAlpha(node.focus); - ctx.fillStyle = halo; - ctx.beginPath(); - ctx.arc(node.x, node.y, radius, 0, Math.PI * 2); - ctx.fill(); - ctx.restore(); - } - ctx.restore(); - - if (model.cursor && model.cursor.x < fieldX1) { - const x0 = Math.max(fieldX0, model.cursor.x); - const bandHeight = height - RULER; - ctx.save(); - ctx.beginPath(); - ctx.rect(x0, RULER, fieldX1 - x0, bandHeight); - ctx.clip(); - ctx.globalAlpha = 0.05; - ctx.strokeStyle = palette.text; - ctx.lineWidth = 1; - ctx.beginPath(); - for (let x = x0 - bandHeight; x < fieldX1; x += 8) { - ctx.moveTo(x, height); - ctx.lineTo(x + bandHeight, RULER); - } - ctx.stroke(); - ctx.restore(); - } - - ctx.restore(); -} - /* ---- component ----------------------------------------------------------- */ export function TemporalScene(props: TemporalSceneProps): JSX.Element { @@ -312,11 +113,11 @@ export function TemporalScene(props: TemporalSceneProps): JSX.Element { onSelectEncounter, onWindowChange, onMeasure, - tailLabel, reducedMotion, onInspect, className, fullWindow, + density = null, } = props; const clipId = useId(); const hostRef = useRef(null); @@ -326,6 +127,7 @@ export function TemporalScene(props: TemporalSceneProps): JSX.Element { const [palette, setPalette] = useState(null); const [layer, setLayer] = useState('canvas'); const [hover, setHover] = useState(null); + const [cursorId, setCursorId] = useState(null); const { viewport, height } = model; const width = viewport.width; @@ -334,6 +136,10 @@ export function TemporalScene(props: TemporalSceneProps): JSX.Element { const fieldX0 = viewport.left; const fieldX1 = width - viewport.right; const fieldWidth = Math.max(1, fieldX1 - fieldX0); + const frame: SceneFrame = useMemo( + () => ({ model, density, fieldX0, fieldX1, top: RULER, height }), + [model, density, fieldX0, fieldX1, height], + ); const lanes = useMemo(() => [...model.lanes].sort((a, b) => a.row - b.row), [model.lanes]); const clusterByLane = useMemo( @@ -381,6 +187,22 @@ export function TemporalScene(props: TemporalSceneProps): JSX.Element { return counts; }, [lanes]); const hoverLaneId = hover === null ? null : model.nodes.find((node) => node.id === hover)?.laneId ?? null; + /** Keyboard order: lanes top to bottom, then time within a lane. */ + const eventOrder = useMemo(() => { + const rowOf = new Map(lanes.map((lane) => [lane.id, lane.row])); + return [...model.nodes].sort( + (a, b) => (rowOf.get(a.laneId) ?? 0) - (rowOf.get(b.laneId) ?? 0) || a.x - b.x || a.id.localeCompare(b.id), + ); + }, [model.nodes, lanes]); + const cursorIndex = (() => { + const at = cursorId === null ? -1 : eventOrder.findIndex((node) => node.id === cursorId); + if (at >= 0) return at; + const selected = eventOrder.findIndex((node) => node.selected); + return selected >= 0 ? selected : eventOrder.length > 0 ? 0 : -1; + })(); + const cursorNode = cursorIndex >= 0 ? eventOrder[cursorIndex]! : null; + const eventPosition = useMemo(() => new Map(eventOrder.map((node, index) => [node.id, index])), [eventOrder]); + const eventDomId = (node: SceneNode): string => `${clipId}-event-${eventPosition.get(node.id) ?? 0}`; useEffect(() => { const host = hostRef.current; @@ -409,8 +231,12 @@ export function TemporalScene(props: TemporalSceneProps): JSX.Element { const canvas = canvasRef.current; if (!canvas || !palette) return; const dpr = globalThis.devicePixelRatio || 1; - canvas.width = Math.max(1, Math.round(width * dpr)); - canvas.height = Math.max(1, Math.round(height * dpr)); + // Assigning a size reallocates the bitmap even when it is unchanged; a + // window change repaints into the one it already has. + const pixelWidth = Math.max(1, Math.round(width * dpr)); + const pixelHeight = Math.max(1, Math.round(height * dpr)); + if (canvas.width !== pixelWidth) canvas.width = pixelWidth; + if (canvas.height !== pixelHeight) canvas.height = pixelHeight; const ctx = canvas.getContext('2d'); if (!ctx) { setLayer('unavailable'); @@ -419,8 +245,8 @@ export function TemporalScene(props: TemporalSceneProps): JSX.Element { setLayer('canvas'); ctx.setTransform(dpr, 0, 0, dpr, 0, 0); ctx.clearRect(0, 0, width, height); - drawScene(ctx, model, palette); - }, [model, palette, width, height]); + paintScene(ctx, frame, palette); + }, [frame, palette, width, height]); /* ---- window arithmetic ---- */ @@ -527,6 +353,55 @@ export function TemporalScene(props: TemporalSceneProps): JSX.Element { setHover(null); onInspect?.(null); }; + + /** The nearest event in the next or previous lane that has any, by x. */ + const acrossLanes = (from: SceneNode, direction: 1 | -1): SceneNode | null => { + const rows = [...new Set(eventOrder.map((node) => node.laneId))]; + const target = rows[rows.indexOf(from.laneId) + direction]; + if (target === undefined) return null; + const candidates = eventOrder.filter((node) => node.laneId === target); + return candidates.reduce((best, node) => (Math.abs(node.x - from.x) < Math.abs(best.x - from.x) ? node : best)); + }; + const onOverlayKey = (event: KeyboardEvent): void => { + if (cursorNode === null) return; + let next: SceneNode | null = null; + switch (event.key) { + case 'ArrowRight': + next = eventOrder[Math.min(eventOrder.length - 1, cursorIndex + 1)]!; + break; + case 'ArrowLeft': + next = eventOrder[Math.max(0, cursorIndex - 1)]!; + break; + case 'ArrowDown': + next = acrossLanes(cursorNode, 1); + break; + case 'ArrowUp': + next = acrossLanes(cursorNode, -1); + break; + case 'Home': + next = eventOrder[0]!; + break; + case 'End': + next = eventOrder[eventOrder.length - 1]!; + break; + case 'Enter': + case ' ': + event.preventDefault(); + onSelectEvent(cursorNode.id); + return; + default: + return; + } + event.preventDefault(); + if (next === null) return; + setCursorId(next.id); + enterNode(next); + // Focus follows the cursor when it already sits on a glyph; from the + // overlay itself, aria-activedescendant carries it. + if (event.target !== event.currentTarget) { + (document.getElementById(eventDomId(next)) as SVGGElement | null)?.focus(); + } + }; const laneGroupStyle = (laneId: string) => ({ opacity: hoverLaneId !== null && hoverLaneId !== laneId ? 0.55 : 1, transition: reducedMotion ? 'none' : 'opacity 120ms', @@ -665,21 +540,14 @@ export function TemporalScene(props: TemporalSceneProps): JSX.Element { const renderCluster = (cluster: SceneCluster, lane: SceneLane): JSX.Element => { const x1 = Math.max(cluster.x1, cluster.x0 + 2); const top = cluster.y - cluster.height / 2; - const bracket = 6; const label = `Expand branch ${lane.label} · ${cluster.counts.sessions} sessions · ${cluster.counts.subagents} subagents · ${cluster.counts.messages} messages`; - const outline = [ - `M ${cluster.x0} ${top + bracket} V ${top} H ${cluster.x0 + bracket}`, - `M ${x1 - bracket} ${top} H ${x1} V ${top + bracket}`, - `M ${x1} ${top + cluster.height - bracket} V ${top + cluster.height} H ${x1 - bracket}`, - `M ${cluster.x0 + bracket} ${top + cluster.height} H ${cluster.x0} V ${top + cluster.height - bracket}`, - ].join(' '); return ( onToggleBranch(cluster.laneId)} onKeyDown={(event) => { @@ -690,75 +558,62 @@ export function TemporalScene(props: TemporalSceneProps): JSX.Element { }} > {label} - - - - - {cluster.counts.sessions} sessions - + + ); }; const renderNode = (node: SceneNode): JSX.Element => { - const hovered = hover === node.id; - const color = gradeColorVar(node.grade); - const haloRadius = node.selected ? 13 : hovered ? 11 : 0; const halfHit = Math.max(1, node.halfHit); const title = `${glyphLabel(node.kind)} · ${node.label}${node.detail ? ` · ${node.detail}` : ''} · ${node.grade}${node.xBasis === 'sequence' ? ' · recorded order, timestamp unrecorded' : ''}`; - const select = (): void => onSelectEvent(node.id); + const select = (): void => { + setCursorId(node.id); + onSelectEvent(node.id); + }; return ( { if (isActivation(event)) { event.preventDefault(); + event.stopPropagation(); select(); } }} onMouseEnter={() => enterNode(node)} - onFocus={() => enterNode(node)} + onFocus={() => { + setCursorId(node.id); + enterNode(node); + }} onMouseLeave={leaveNode} onBlur={leaveNode} > {title} - - - - - + ); }; @@ -770,7 +625,9 @@ export function TemporalScene(props: TemporalSceneProps): JSX.Element { const twoLine = lane.height >= 28; const toggle = branchToggleFor(lane); const bundleCount = clusterByLane.get(lane.id)?.counts.sessions ?? lane.collapsedDescendants; - const detailLine = lane.kind === 'bundle' ? `${lane.provider} · bundle · ${bundleCount} sessions` : lane.provider; + const detailLine = + laneDetail(lane, frame) ?? + (lane.kind === 'bundle' ? `${lane.provider} · bundle · ${bundleCount} sessions` : lane.provider); // Lane rows and toggles are pointer affordances for the same actions the // branch navigator table offers as 44px DOM controls; they stay out of the // tab order so a dense page does not become hundreds of stops. @@ -838,7 +695,11 @@ export function TemporalScene(props: TemporalSceneProps): JSX.Element { }; return ( -
    +
    @@ -858,6 +719,11 @@ export function TemporalScene(props: TemporalSceneProps): JSX.Element { {formatMoment(window.start)} – {formatMoment(window.end)} + {cursorNode ? ( + + event {cursorIndex + 1} of {eventOrder.length} + + ) : null}
    0 ? 0 : -1} data-scene-layer="overlay" - className="relative block touch-none select-none" + onKeyDown={onOverlayKey} + onBlur={(event) => { + const next = event.relatedTarget; + if (!(next instanceof Node) || !event.currentTarget.contains(next)) leaveNode(); + }} + className="relative block touch-none select-none outline-none [&:focus-visible_[data-event-current]>.td-focus-ring]:opacity-100" width="100%" viewBox={`0 0 ${width} ${height}`} > @@ -896,10 +769,6 @@ export function TemporalScene(props: TemporalSceneProps): JSX.Element { ))} - - - {tailLabel} - ))} - {model.cursor && ( - - - - - )} +
    - + + +
    ); } diff --git a/dashboard/src/viz/temporal/density.test.ts b/dashboard/src/viz/temporal/density.test.ts new file mode 100644 index 0000000000..cd3259582c --- /dev/null +++ b/dashboard/src/viz/temporal/density.test.ts @@ -0,0 +1,139 @@ +import { describe, expect, it } from 'vitest'; +import { densityIndex, layoutDensity, membershipKey, newestLoadedTime } from './density.ts'; +import { DEFAULT_DENSE_LANE_THRESHOLD, layoutTemporalScene } from './layout.ts'; +import type { JourneyEvent, JourneyLane, JourneyProjection, LayoutOptions } from './types.ts'; + +const T0 = 1_784_700_000; + +function lane(over: Partial & { id: string }): JourneyLane { + return { + sessionId: over.id, provider: 'cursor', label: over.id, agent: null, start: T0, end: null, endSource: null, + parentId: null, depth: 0, isSubagent: false, messages: 0, editedFilesRecorded: false, editedFileCount: 0, models: [], + ...over, + }; +} + +function event(over: Partial & { id: string; laneId: string; time: number | null }): JourneyEvent { + return { kind: 'session_start', sequence: 0, grade: 'exact', source: 'session', label: 'start', detail: null, ref: over.id, ...over }; +} + +const A = lane({ id: 'A', start: T0 + 50, end: T0 + 450, endSource: 'session_end', messages: 30 }); +const B = lane({ id: 'B', parentId: 'A', depth: 1, isSubagent: true, start: T0 + 150, messages: 5 }); + +const PROJECTION: JourneyProjection = { + lanes: [A, B], + events: [ + event({ id: 'start:A', laneId: 'A', time: T0 + 50 }), + event({ id: 'spawn:B', laneId: 'A', kind: 'spawn', time: T0 + 150, source: 'parentage', ref: 'B', sequence: 1 }), + event({ id: 'commit:A', laneId: 'A', kind: 'commit', time: T0 + 320, source: 'commit', grade: 'inferred', sequence: 2 }), + event({ id: 'end:A', laneId: 'A', kind: 'session_end', time: T0 + 450, sequence: 3 }), + event({ id: 'msg:A:0', laneId: 'A', kind: 'message_user', time: null, source: 'transcript', sequence: 0 }), + event({ id: 'start:B', laneId: 'B', time: T0 + 150 }), + ], + relations: [{ id: 'rel:spawn:B', kind: 'spawn', fromLaneId: 'A', toLaneId: 'B', time: T0 + 150, grade: 'exact', basis: 'parent_session_id' }], + gaps: [], + intervals: [], + extent: { start: T0 + 50, end: T0 + 3650 }, + stats: { lanes: 2, roots: 1, subagents: 1, messages: 35, openEnded: 1, hollow: 0, undated: 0, providers: [{ id: 'cursor', lanes: 2, messages: 35 }] }, +}; + +/** One pixel per second: a 1000px field over a 1000s window, 100px bins. */ +function options(over: Partial = {}): LayoutOptions { + return { + viewport: { width: 1220, left: 200, right: 20, window: { start: T0, end: T0 + 1000 } }, + zoom: 'agent', + branches: { collapsed: new Set(), expanded: new Set() }, + selectedLaneId: null, + selectedEventId: null, + reveal: null, + hiddenKinds: new Set(), + denseLaneThreshold: DEFAULT_DENSE_LANE_THRESHOLD, + ...over, + }; +} + +function density(over: Partial = {}) { + const opts = options(over); + const model = layoutTemporalScene(PROJECTION, opts); + return layoutDensity(densityIndex(PROJECTION, model, { reveal: opts.reveal, hiddenKinds: opts.hiddenKinds }), model, 100); +} + +describe('layoutDensity', () => { + it('bins measured extent as active and an unrecorded end as open, never active', () => { + const result = density(); + const a = result.lanes.get('A')!; + const b = result.lanes.get('B')!; + expect(a.bins.map((bin) => bin.active)).toEqual([1, 1, 1, 1, 1, 0, 0, 0, 0, 0]); + expect(a.bins.map((bin) => bin.events)).toEqual([1, 1, 0, 1, 1, 0, 0, 0, 0, 0]); + expect(a.bins.map((bin) => bin.starts)).toEqual([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + expect(b.bins.map((bin) => bin.active)).toEqual([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + expect(b.bins.map((bin) => bin.open)).toEqual([0, 1, 1, 1, 1, 1, 1, 1, 1, 1]); + expect(a.bins[0]).toMatchObject({ x0: 200, x1: 300 }); + }); + + it('counts an undated turn in the totals and in no bin', () => { + const a = density().lanes.get('A')!; + expect(a.totals).toEqual({ sessions: 1, messages: 30, commits: 1, events: 5, undated: 1, openEnded: 0 }); + expect(a.bins.reduce((sum, bin) => sum + bin.events, 0)).toBe(4); + }); + + it('measures the tightest gap between drawn marks on a row', () => { + expect(density().lanes.get('A')!.minGap).toBe(100); + expect(density().lanes.get('B')!.minGap).toBe(Infinity); + }); + + it('measures a crowded undated gutter as its own row', () => { + const turns = Array.from({ length: 5 }, (_, sequence) => + event({ id: `msg:A:${sequence}`, laneId: 'A', kind: 'message_assistant', time: null, source: 'transcript', sequence }), + ); + const projection = { ...PROJECTION, events: [...PROJECTION.events.filter((e) => e.source !== 'transcript'), ...turns] }; + const opts = options({ zoom: 'event', selectedLaneId: 'A' }); + const model = layoutTemporalScene(projection, opts); + const a = layoutDensity(densityIndex(projection, model, { reveal: null, hiddenKinds: new Set() }), model, 100).lanes.get('A')!; + // Five turns spread over the 400px lane extent sit 400/6 apart, closer than the row's 100px. + expect(a.minGap).toBeCloseTo(400 / 6, 6); + expect(a.totals.undated).toBe(5); + }); + + it('folds a collapsed subtree into its bundle with exact member totals', () => { + const bundle = density({ branches: { collapsed: new Set(['A']), expanded: new Set() } }).lanes.get('A')!; + expect(bundle.totals).toEqual({ sessions: 2, messages: 35, commits: 1, events: 6, undated: 1, openEnded: 1 }); + expect(bundle.bins.map((bin) => bin.open)).toEqual([0, 1, 1, 1, 1, 1, 1, 1, 1, 1]); + expect(bundle.bins.map((bin) => bin.starts)).toEqual([1, 1, 0, 0, 0, 0, 0, 0, 0, 0]); + }); + + it('bins nothing past a dated playback cursor', () => { + const result = density({ reveal: { time: T0 + 300, laneId: 'A', sequence: 0 } }); + const a = result.lanes.get('A')!; + expect(a.bins.map((bin) => bin.active)).toEqual([1, 1, 1, 1, 0, 0, 0, 0, 0, 0]); + expect(a.bins.map((bin) => bin.events)).toEqual([1, 1, 0, 0, 0, 0, 0, 0, 0, 0]); + expect(result.lanes.get('B')!.bins.map((bin) => bin.open)).toEqual([0, 1, 1, 1, 0, 0, 0, 0, 0, 0]); + }); + + it('names the newest loaded record as the tail, not the padded extent', () => { + expect(newestLoadedTime(PROJECTION)).toBe(T0 + 450); + const result = density(); + expect(result.tailTime).toBe(T0 + 450); + expect(result.headTime).toBe(T0 + 50); + expect(result.tailX).toBe(650); + const outside = density({ viewport: { width: 1220, left: 200, right: 20, window: { start: T0 + 500, end: T0 + 1500 } } }); + expect(outside.tailX).toBeNull(); + }); + + it('keeps one index across window changes and rebuilds it when a branch closes', () => { + const wide = layoutTemporalScene(PROJECTION, options()); + const narrow = layoutTemporalScene( + PROJECTION, + options({ viewport: { width: 1220, left: 200, right: 20, window: { start: T0, end: T0 + 500 } } }), + ); + const collapsed = layoutTemporalScene(PROJECTION, options({ branches: { collapsed: new Set(['A']), expanded: new Set() } })); + expect(membershipKey(narrow)).toBe(membershipKey(wide)); + expect(membershipKey(collapsed)).not.toBe(membershipKey(wide)); + // The index built for the wide window bins the narrow one exactly as a fresh index does. + const reused = layoutDensity(densityIndex(PROJECTION, wide, { reveal: null, hiddenKinds: new Set() }), narrow, 100); + const fresh = layoutDensity(densityIndex(PROJECTION, narrow, { reveal: null, hiddenKinds: new Set() }), narrow, 100); + expect(reused).toEqual(fresh); + // Half the span: A's measured extent 50..450 now covers bins 1 through 9. + expect(reused.lanes.get('A')!.bins.map((bin) => bin.active)).toEqual([0, 1, 1, 1, 1, 1, 1, 1, 1, 1]); + }); +}); diff --git a/dashboard/src/viz/temporal/density.ts b/dashboard/src/viz/temporal/density.ts new file mode 100644 index 0000000000..e956693e84 --- /dev/null +++ b/dashboard/src/viz/temporal/density.ts @@ -0,0 +1,238 @@ +/** + * Per-lane density summaries over the current window: the aggregation layer + * a renderer draws when one pixel holds more records than a glyph can carry. + * + * Pure arithmetic over the projection and the laid-out model, so every + * renderer summarizes the same counts. A bin counts only what a record says: + * a member session whose measured extent covers the bin is `active`; one that + * began with no recorded end is `open` (begun, extent unknown), never active; + * `events` are dated records in the bin. Undated records have no bin and are + * counted only in the totals. Nothing after a dated playback cursor is binned. + */ +import { timeToX } from './layout.ts'; +import type { + JourneyEvent, + JourneyEventKind, + JourneyLane, + JourneyProjection, + RevealBoundary, + TemporalSceneModel, +} from './types.ts'; + +export interface DensityBin { + readonly x0: number; + readonly x1: number; + readonly active: number; + readonly open: number; + readonly starts: number; + readonly events: number; +} + +export interface LaneDensity { + readonly laneId: string; + readonly bins: readonly DensityBin[]; + readonly peak: { readonly active: number; readonly open: number; readonly events: number }; + /** Exact counts over every member session in the loaded page, not the window. */ + readonly totals: { + readonly sessions: number; + readonly messages: number; + readonly commits: number; + readonly events: number; + readonly undated: number; + readonly openEnded: number; + }; + /** Smallest pixel gap between two drawn marks sharing a row of the lane + * (its line or its undated gutter); `Infinity` when no row holds two. */ + readonly minGap: number; +} + +export interface SceneDensity { + readonly binPx: number; + readonly lanes: ReadonlyMap; + /** The oldest session start in the loaded page. */ + readonly headTime: number | null; + /** The newest recorded time in the loaded page: what `NOW` names. */ + readonly tailTime: number | null; + /** `tailTime` on the axis, or null when it lies outside the window. */ + readonly tailX: number | null; +} + +export interface DensityOptions { + readonly reveal: RevealBoundary | null; + readonly hiddenKinds: ReadonlySet; +} + +const DEFAULT_BIN_PX = 6; + +/** Newest dated record in the page: a session start or end, a dated event, or + * an interval end. Null when the page holds no dated record at all. */ +export function newestLoadedTime(projection: JourneyProjection): number | null { + let newest = -Infinity; + for (const lane of projection.lanes) { + newest = Math.max(newest, lane.start, lane.end ?? -Infinity); + } + for (const event of projection.events) { + if (event.time !== null) newest = Math.max(newest, event.time); + } + for (const interval of projection.intervals) newest = Math.max(newest, interval.end); + return Number.isFinite(newest) ? newest : null; +} + +interface LaneIndex { + /** Revealed member extents; `end` null when unrecorded. */ + readonly members: readonly { readonly start: number; readonly end: number | null }[]; + /** Revealed dated event times, unsorted. */ + readonly eventTimes: readonly number[]; + readonly totals: LaneDensity['totals']; +} + +/** The window-independent half of the density summary: who belongs to each + * scene lane and which of their records are revealed. Rebuilt only when the + * loaded page, the bundle membership, the filters or the cursor change. */ +export interface DensityIndex { + readonly lanes: ReadonlyMap; + readonly revealTime: number | null; + readonly headTime: number | null; + readonly tailTime: number | null; +} + +/** Identity of the scene's lane and bundle membership; equal across window + * changes, different once a branch opens or closes. */ +export function membershipKey(model: TemporalSceneModel): string { + return `${model.lanes.map((lane) => lane.id).join('\n')}\u0000${model.clusters.map((cluster) => cluster.laneId).join('\n')}`; +} + +export function densityIndex(projection: JourneyProjection, model: TemporalSceneModel, options: DensityOptions): DensityIndex { + const { reveal, hiddenKinds } = options; + const revealTime = reveal !== null ? reveal.time : null; + const laneById = new Map(projection.lanes.map((lane) => [lane.id, lane] as const)); + const eventsByLane = new Map(); + for (const event of projection.events) { + const bucket = eventsByLane.get(event.laneId); + if (bucket) bucket.push(event); + else eventsByLane.set(event.laneId, [event]); + } + const membersOf = new Map(); + for (const cluster of model.clusters) membersOf.set(cluster.laneId, cluster.memberLaneIds); + const withheld = (event: JourneyEvent): boolean => { + if (reveal === null) return false; + if (event.time !== null && revealTime !== null && event.time > revealTime) return true; + return event.laneId === reveal.laneId && event.source === 'transcript' && event.sequence > reveal.sequence; + }; + const lanes = new Map(); + for (const sceneLane of model.lanes) { + const memberIds = [sceneLane.id, ...(membersOf.get(sceneLane.id) ?? [])]; + const members = memberIds.map((id) => laneById.get(id)).filter((lane): lane is JourneyLane => lane !== undefined); + const extents: { start: number; end: number | null }[] = []; + const eventTimes: number[] = []; + let messages = 0; + let commits = 0; + let events = 0; + let undated = 0; + let openEnded = 0; + for (const member of members) { + messages += member.messages; + if (member.end === null) openEnded += 1; + for (const event of eventsByLane.get(member.id) ?? []) { + if (hiddenKinds.has(event.kind)) continue; + events += 1; + if (event.kind === 'commit') commits += 1; + if (event.time === null) undated += 1; + else if (!withheld(event)) eventTimes.push(event.time); + } + if (revealTime === null || member.start <= revealTime) extents.push({ start: member.start, end: member.end }); + } + lanes.set(sceneLane.id, { + members: extents, + eventTimes, + totals: { sessions: members.length, messages, commits, events, undated, openEnded }, + }); + } + const headTime = projection.lanes.reduce( + (oldest, lane) => (oldest === null || lane.start < oldest ? lane.start : oldest), + null, + ); + return { lanes, revealTime, headTime, tailTime: newestLoadedTime(projection) }; +} + +/** Bins an index over the model's window, and measures mark spacing. */ +export function layoutDensity(index: DensityIndex, model: TemporalSceneModel, binPx = DEFAULT_BIN_PX): SceneDensity { + const { viewport } = model; + const pitch = Math.max(1, binPx); + const fieldX0 = viewport.left; + const fieldX1 = viewport.width - viewport.right; + const binCount = Math.max(1, Math.ceil((fieldX1 - fieldX0) / pitch)); + const { revealTime } = index; + /** Bin index containing `time`, clamped; null when outside the window. */ + const binOf = (time: number): number | null => { + const x = timeToX(viewport, time); + if (x < fieldX0 || x > fieldX1) return null; + return Math.min(binCount - 1, Math.floor((x - fieldX0) / pitch)); + }; + const revealBin = revealTime === null ? binCount - 1 : binOf(revealTime) ?? (revealTime < viewport.window.start ? -1 : binCount - 1); + // Marks collide only within one row: the lane's own line or its undated gutter. + const rowXs = new Map>(); + for (const node of model.nodes) { + let rows = rowXs.get(node.laneId); + if (!rows) rowXs.set(node.laneId, (rows = new Map())); + const bucket = rows.get(node.y); + if (bucket) bucket.push(node.x); + else rows.set(node.y, [node.x]); + } + + const lanes = new Map(); + for (const sceneLane of model.lanes) { + const laneIndex = index.lanes.get(sceneLane.id); + if (!laneIndex) continue; + const active = new Array(binCount).fill(0); + const open = new Array(binCount).fill(0); + const starts = new Array(binCount).fill(0); + const events = new Array(binCount).fill(0); + for (const time of laneIndex.eventTimes) { + const bin = binOf(time); + if (bin !== null && bin <= revealBin) events[bin] = (events[bin] ?? 0) + 1; + } + for (const member of laneIndex.members) { + const startBin = binOf(member.start); + if (startBin !== null && startBin <= revealBin) starts[startBin] = (starts[startBin] ?? 0) + 1; + const first = member.start < viewport.window.start ? 0 : startBin; + if (first === null) continue; + if (member.end === null) { + for (let bin = first; bin <= revealBin; bin += 1) open[bin] = (open[bin] ?? 0) + 1; + continue; + } + const last = member.end > viewport.window.end ? binCount - 1 : binOf(member.end); + if (last === null) continue; + for (let bin = first; bin <= Math.min(last, revealBin); bin += 1) active[bin] = (active[bin] ?? 0) + 1; + } + const bins: DensityBin[] = active.map((count, bin) => ({ + x0: fieldX0 + bin * pitch, + x1: Math.min(fieldX1, fieldX0 + (bin + 1) * pitch), + active: count, + open: open[bin] ?? 0, + starts: starts[bin] ?? 0, + events: events[bin] ?? 0, + })); + let minGap = Infinity; + for (const row of rowXs.get(sceneLane.id)?.values() ?? []) { + const xs = [...row].sort((a, b) => a - b); + for (let i = 1; i < xs.length; i += 1) minGap = Math.min(minGap, xs[i]! - xs[i - 1]!); + } + lanes.set(sceneLane.id, { + laneId: sceneLane.id, + bins, + peak: { active: Math.max(0, ...active), open: Math.max(0, ...open), events: Math.max(0, ...events) }, + totals: laneIndex.totals, + minGap, + }); + } + + const tailX = index.tailTime === null ? null : timeToX(viewport, index.tailTime); + return { + binPx: pitch, + lanes, + headTime: index.headTime, + tailTime: index.tailTime, + tailX: tailX !== null && tailX >= fieldX0 && tailX <= fieldX1 ? tailX : null, + }; +} diff --git a/dashboard/src/viz/temporal/glyphs.tsx b/dashboard/src/viz/temporal/glyphs.tsx index 58974739a1..63f761a76d 100644 --- a/dashboard/src/viz/temporal/glyphs.tsx +++ b/dashboard/src/viz/temporal/glyphs.tsx @@ -6,7 +6,7 @@ * forced-colors. Glyphs are drawn in a 16x16 box centred at the origin so the * scene can translate and scale them without knowing what they are. */ -import type { JSX } from 'react'; +import type { JSX, ReactNode } from 'react'; import { gradeColorVar, gradeDashArray } from './palette.ts'; import { JOURNEY_EVENT_KINDS, type EvidenceGrade, type JourneyEventKind, type SceneGap } from './types.ts'; @@ -65,6 +65,13 @@ function glyphShape(kind: JourneyEventKind): JSX.Element { ); + case 'file_edit': + return ( + <> + + + + ); default: { const exhaustive: never = kind; throw new Error(`unknown event kind: ${String(exhaustive)}`); @@ -105,6 +112,8 @@ export function glyphLabel(kind: JourneyEventKind): string { return 'spawn'; case 'commit': return 'commit'; + case 'file_edit': + return 'file edit'; default: { const exhaustive: never = kind; throw new Error(`unknown event kind: ${String(exhaustive)}`); @@ -112,7 +121,30 @@ export function glyphLabel(kind: JourneyEventKind): string { } } -export function TemporalLegend({ gaps }: { gaps: readonly SceneGap[] }): JSX.Element { +function LineSwatch({ grade }: { grade: EvidenceGrade }): JSX.Element { + return ( + + ); +} + +export function TemporalLegend({ + gaps, + children, +}: { + gaps: readonly SceneGap[]; + /** The field's own encodings, printed beside the grade ladder. */ + children?: ReactNode; +}): JSX.Element { const pageWide = gaps.filter((gap) => gap.laneId === null); return (
    @@ -120,21 +152,12 @@ export function TemporalLegend({ gaps }: { gaps: readonly SceneGap[] }): JSX.Ele Legend {GRADES.map((grade) => ( - + {grade.toUpperCase()} ))}
    + {children}
    {JOURNEY_EVENT_KINDS.map((kind) => ( diff --git a/dashboard/src/viz/temporal/journey.test.ts b/dashboard/src/viz/temporal/journey.test.ts index 09f9eb46d1..2a3402551e 100644 --- a/dashboard/src/viz/temporal/journey.test.ts +++ b/dashboard/src/viz/temporal/journey.test.ts @@ -9,6 +9,7 @@ import type { LoomSessionRowV1, LoomTemporalPayloadV1, } from '../../contracts/generated.ts'; +import { resolveFixture } from '../../../stories/fixtures/data.ts'; import { laneIdOf, orderMessages, projectJourney, type JourneySources } from './journey.ts'; /** @@ -319,6 +320,16 @@ describe('projectJourney lanes', () => { editedFileCount: 2, agent: 'lead', }); + // The rollup carries no edit time, so the edits are a typed absence, not marks. + expect(projection.gaps.filter((gap) => gap.kind === 'edit_time_unrecorded')).toEqual([ + { + id: `gap:edit_time_unrecorded:${ROOT}`, + laneId: ROOT, + kind: 'edit_time_unrecorded', + grade: 'unavailable', + detail: '2 edited files recorded · no edit time in this read', + }, + ]); }); it('projects identically regardless of wire order', () => { @@ -375,7 +386,7 @@ describe('projectJourney lanes', () => { }); describe('projectJourney parentage', () => { - it('emits an exact spawn relation and parent-lane event only for a linked pair in the page', () => { + it('emits an inferred fork at the child start and a parent-lane event only for a linked pair in the page', () => { const projection = projectJourney(parentChild()); expect(projection.relations).toHaveLength(1); const relation = projection.relations[0]; @@ -384,17 +395,20 @@ describe('projectJourney parentage', () => { fromLaneId: ROOT, toLaneId: CHILD, time: T0 + 60, - grade: 'exact', + grade: 'inferred', }); - expect(relation?.basis).toContain('toolu_01'); - expect(relation?.basis).toContain('parent_session_id'); + // No parent transcript is loaded, so the fork cannot sit on the parent's + // tool call and says so. + expect(relation?.basis).toBe( + 'subagent tree · parent_session_id · parent_tool_use_id toolu_01 · fork placed at the child start: the parent transcript is not loaded', + ); const child = projection.lanes.find((lane) => lane.id === CHILD); expect(child).toMatchObject({ parentId: ROOT, depth: 1, agent: 'explorer' }); const spawn = projection.events.find((event) => event.kind === 'spawn'); expect(spawn).toMatchObject({ laneId: ROOT, time: T0 + 60, - grade: 'exact', + grade: 'inferred', source: 'parentage', label: 'child', ref: CHILD, @@ -403,6 +417,226 @@ describe('projectJourney parentage', () => { expect(projection.stats).toMatchObject({ lanes: 2, roots: 1, subagents: 1 }); }); + describe('parentage from the session row and the subagent tree', () => { + const rows = (child: Partial, extra: LoomSessionRowV1[] = []) => + temporal({ + sessions: [ + session({ ended_at: T0 + 3600 }), + session({ session_id: 'other', started_at: T0 + 10 }), + session({ session_id: 'child', started_at: T0 + 60, is_subagent: true, ...child }), + ...extra, + ], + }); + const linked = (parent: string, tool: string | null) => + node({ session_id: 'child', link: 'linked', parent_session_id: parent, parent_tool_use_id: tool, is_subagent: true, depth: 1 }); + + it('forks from the row column alone when the tree is silent', () => { + const projection = projectJourney(sources({ temporal: rows({ parent_session_id: 'root', parent_tool_use_id: 'toolu_09' }) })); + expect(projection.relations.filter((r) => r.kind === 'spawn')).toEqual([ + { + id: `rel:spawn:${CHILD}`, + kind: 'spawn', + fromLaneId: ROOT, + toLaneId: CHILD, + time: T0 + 60, + grade: 'inferred', + basis: 'sessions row · parent_session_id · parent_tool_use_id toolu_09 · fork placed at the child start: the parent transcript is not loaded', + }, + ]); + expect(projection.lanes.find((lane) => lane.id === CHILD)).toMatchObject({ parentId: ROOT, depth: 1 }); + }); + + const parentPage = (messages: LcmMessageV1[]) => ({ laneId: ROOT, messages }); + const taskCall = message({ message_id: 'root:b-task', ordinal: 3, role: 'assistant', tool_name: 'Task', tool_use_id: 'toolu_09', timestamp: T0 + 55 }); + + it('forks exactly from the loaded tool call whose id the child recorded', () => { + const projection = projectJourney( + sources({ + temporal: rows({ parent_session_id: 'root', parent_tool_use_id: 'toolu_09' }), + selected: parentPage([message({ message_id: 'root:b-read', ordinal: 2, tool_name: 'Read', tool_use_id: 'toolu_08', timestamp: T0 + 50 }), taskCall]), + }), + ); + expect(projection.relations.filter((r) => r.kind === 'spawn')).toEqual([ + { + id: `rel:spawn:${CHILD}`, + kind: 'spawn', + fromLaneId: ROOT, + toLaneId: CHILD, + time: T0 + 55, + grade: 'exact', + basis: 'sessions row · parent_session_id · parent_tool_use_id toolu_09 · fork placed on the spawning tool call Task', + fromEventId: `msg:${ROOT}:root:b-task`, + }, + ]); + // The tool-call glyph is the fork's mark: no second spawn mark. + expect(projection.events.filter((event) => event.kind === 'spawn')).toEqual([]); + expect(projection.events.find((event) => event.id === `msg:${ROOT}:root:b-task`)).toMatchObject({ kind: 'tool_call', time: T0 + 55 }); + }); + + it('keeps the fork inferred at the child start when no loaded tool call carries the id', () => { + const projection = projectJourney( + sources({ + temporal: rows({ parent_session_id: 'root', parent_tool_use_id: 'toolu_09' }), + selected: parentPage([message({ message_id: 'root:b-read', tool_name: 'Read', tool_use_id: 'toolu_08', timestamp: T0 + 50 })]), + }), + ); + const [fork] = projection.relations.filter((r) => r.kind === 'spawn'); + expect(fork).toMatchObject({ time: T0 + 60, grade: 'inferred' }); + expect(fork?.fromEventId).toBeUndefined(); + expect(fork?.basis).toBe( + 'sessions row · parent_session_id · parent_tool_use_id toolu_09 · fork placed at the child start: no loaded parent tool call carries toolu_09', + ); + expect(projection.events.find((event) => event.kind === 'spawn')).toMatchObject({ laneId: ROOT, time: T0 + 60, grade: 'inferred' }); + + const unrecorded = projectJourney(sources({ temporal: rows({ parent_session_id: 'root' }), selected: parentPage([taskCall]) })); + expect(unrecorded.relations.filter((r) => r.kind === 'spawn').map((r) => [r.time, r.grade, r.basis])).toEqual([ + [T0 + 60, 'inferred', 'sessions row · parent_session_id · parent_tool_use_id unrecorded · fork placed at the child start: no parent tool-use id recorded'], + ]); + }); + + it('keeps a matched fork ambiguous when the subagent tree names another tool use', () => { + const projection = projectJourney( + sources({ + temporal: rows({ parent_session_id: 'root', parent_tool_use_id: 'toolu_09' }), + hierarchy: tree([node(), linked('root', 'toolu_77')]), + selected: parentPage([taskCall]), + }), + ); + expect(projection.relations.filter((r) => r.kind === 'spawn').map((r) => [r.time, r.grade, r.fromEventId])).toEqual([ + [T0 + 55, 'ambiguous', `msg:${ROOT}:root:b-task`], + ]); + expect(projection.gaps.find((gap) => gap.kind === 'parentage_conflict')?.detail).toBe( + 'sessions row names tool use toolu_09; subagent tree names toolu_77', + ); + }); + + it('draws one fork when the two sources agree', () => { + const projection = projectJourney( + sources({ temporal: rows({ parent_session_id: 'root', parent_tool_use_id: 'toolu_09' }), hierarchy: tree([node(), linked('root', 'toolu_09')]) }), + ); + const spawns = projection.relations.filter((r) => r.kind === 'spawn'); + expect(spawns).toHaveLength(1); + expect(spawns[0]?.basis.startsWith('sessions row and subagent tree agree · ')).toBe(true); + expect(projection.gaps.some((gap) => gap.kind === 'parentage_conflict')).toBe(false); + }); + + it('draws both candidates as ambiguous when the sources name different parents', () => { + const projection = projectJourney( + sources({ temporal: rows({ parent_session_id: 'root' }), hierarchy: tree([node(), linked('other', null)]) }), + ); + const spawns = projection.relations.filter((r) => r.kind === 'spawn'); + expect(spawns.map((r) => [r.id, r.fromLaneId, r.grade])).toEqual([ + [`rel:spawn:${CHILD}:sessions row`, ROOT, 'ambiguous'], + [`rel:spawn:${CHILD}:subagent tree`, laneIdOf('cursor', 'other'), 'ambiguous'], + ]); + // Placement follows the session row; the disagreement is a gap, not a merge. + expect(projection.lanes.find((lane) => lane.id === CHILD)?.parentId).toBe(ROOT); + expect(projection.gaps.find((gap) => gap.kind === 'parentage_conflict')).toEqual({ + id: `gap:parentage_conflict:${CHILD}`, + laneId: CHILD, + kind: 'parentage_conflict', + grade: 'ambiguous', + detail: 'sessions row names parent root; subagent tree names other', + }); + }); + + it('grades an agreed parent ambiguous when the sources name different tool uses', () => { + const projection = projectJourney( + sources({ temporal: rows({ parent_session_id: 'root', parent_tool_use_id: 'toolu_a' }), hierarchy: tree([node(), linked('root', 'toolu_b')]) }), + ); + expect(projection.relations.filter((r) => r.kind === 'spawn').map((r) => r.grade)).toEqual(['ambiguous']); + expect(projection.gaps.find((gap) => gap.kind === 'parentage_conflict')?.detail).toBe( + 'sessions row names tool use toolu_a; subagent tree names toolu_b', + ); + }); + + it('infers a join only where the child ends inside the parent measured extent', () => { + const ended = projectJourney(sources({ temporal: rows({ parent_session_id: 'root', ended_at: T0 + 900 }) })); + expect(ended.relations.filter((r) => r.kind === 'rejoin')).toEqual([ + { + id: `rel:rejoin:${CHILD}`, + kind: 'rejoin', + fromLaneId: CHILD, + toLaneId: ROOT, + time: T0 + 900, + grade: 'inferred', + basis: "child recorded end inside the parent's measured extent · no result or handoff record in this read", + }, + ]); + const open = projectJourney(sources({ temporal: rows({ parent_session_id: 'root' }) })); + expect(open.relations.some((r) => r.kind === 'rejoin')).toBe(false); + const outlives = projectJourney(sources({ temporal: rows({ parent_session_id: 'root', ended_at: T0 + 7200 }) })); + expect(outlives.relations.some((r) => r.kind === 'rejoin')).toBe(false); + }); + }); + + describe('edited files', () => { + it('links a timed edit to the one loaded tool call recorded in its second', () => { + const editAt = (T0 + 120) * 1_000_000 + 994_000; + const project = (messages: LcmMessageV1[]) => + projectJourney( + sources({ + temporal: temporal({ + sessions: [session({ edited_files_recorded: true })], + edited_files: [{ path: 'src/auth/mod.rs', provider: 'cursor', session_id: 'root', change_type: null, hunks: null, edited_at_micros: editAt }], + }), + selected: { laneId: ROOT, messages }, + }), + ).events.find((event) => event.kind === 'file_edit'); + const edit = message({ message_id: 'root:b-edit', ordinal: 1, tool_name: 'edit_file_v2', tool_use_id: 'call_2jug3QnkUS9kwSbiI4oSDy5a', timestamp: T0 + 120 }); + expect(project([message({ message_id: 'root:b-read', ordinal: 0, tool_name: 'read_file_v2', timestamp: T0 + 120 }), edit])).toEqual({ + id: `edit:${ROOT}:src/auth/mod.rs:${editAt}`, + laneId: ROOT, + kind: 'file_edit', + time: editAt / 1_000_000, + sequence: 1, + grade: 'exact', + source: 'file_rollup', + label: 'mod.rs', + detail: 'src/auth/mod.rs · tool call edit_file_v2 call_2jug3QnkUS9kwSbiI4oSDy5a', + ref: 'src/auth/mod.rs', + linkedEventId: `msg:${ROOT}:root:b-edit`, + }); + // Two tool calls in that second name no single call. + const twin = message({ message_id: 'root:b-edit-2', ordinal: 2, tool_name: 'edit_file_v2', tool_use_id: 'call_other', timestamp: T0 + 120 }); + expect(project([edit, twin])?.linkedEventId).toBeUndefined(); + // A different second is not a coincidence. + expect(project([{ ...edit, timestamp: T0 + 121 }])?.linkedEventId).toBeUndefined(); + }); + + it('places a timed edit at its recorded time and keeps untimed edits as a gap', () => { + const projection = projectJourney( + sources({ + temporal: temporal({ + sessions: [session({ edited_files_recorded: true })], + edited_files: [ + { path: 'src/a.ts', provider: 'cursor', session_id: 'root', change_type: 'modified', hunks: 2, edited_at_micros: (T0 + 120) * 1_000_000 }, + { path: 'src/b.ts', provider: 'cursor', session_id: 'root', change_type: null, hunks: null, edited_at_micros: null }, + { path: 'src/c.ts', provider: 'cursor', session_id: 'root', change_type: 'added', hunks: 1 }, + ], + }), + }), + ); + expect(projection.events.filter((event) => event.kind === 'file_edit')).toEqual([ + { + id: `edit:${ROOT}:src/a.ts:${(T0 + 120) * 1_000_000}`, + laneId: ROOT, + kind: 'file_edit', + time: T0 + 120, + sequence: 1, + grade: 'exact', + source: 'file_rollup', + label: 'a.ts', + detail: 'modified · 2 hunks · src/a.ts', + ref: 'src/a.ts', + }, + ]); + expect(projection.gaps.find((gap) => gap.kind === 'edit_time_unrecorded')?.detail).toBe( + '2 edited files recorded · no edit time in this read', + ); + }); + }); + it('grades the relation ambiguous when the child starts before its parent', () => { const projection = projectJourney(parentChild(T0 - 30)); expect(projection.relations[0]).toMatchObject({ grade: 'ambiguous' }); @@ -425,7 +659,7 @@ describe('projectJourney parentage', () => { laneId: ROOT, kind: 'parent_outside_page', grade: 'unavailable', - detail: 'recorded parent elsewhere is outside this loaded page', + detail: 'subagent tree names parent elsewhere, outside this loaded page', }), ); }); @@ -730,3 +964,58 @@ describe('projectJourney intervals and extent', () => { }); }); }); + +describe('the Loom fixture', () => { + const payload = (path: string, search = ''): T => (resolveFixture(path, search) as { payload: T }).payload; + const temporalPage = payload('/api/loom/temporal', 'limit=200'); + const subagents = payload('/api/plugins/analytics/subagent-tree'); + const forksWith = (provider: string, sessionId: string) => + projectJourney({ + temporal: temporalPage, + hierarchy: subagents, + hierarchyState: 'loaded', + selected: { + laneId: laneIdOf(provider, sessionId), + messages: payload<{ messages: LcmMessageV1[] }>(`/api/plugins/hermes-lcm/session/${sessionId}`).messages, + }, + encounters: [], + }).relations.filter((relation) => relation.kind === 'spawn'); + + it('forks exactly on the spawning Task call of the loaded parent transcript', () => { + const root = laneIdOf('codex', 'session.codex.root'); + const child = laneIdOf('codex', 'session.codex.child'); + const forks = new Map(forksWith('codex', 'session.codex.root').map((relation) => [relation.toLaneId, relation])); + expect(forks.get(child)).toEqual({ + id: `rel:spawn:${child}`, + kind: 'spawn', + fromLaneId: root, + toLaneId: child, + time: null, + grade: 'exact', + basis: 'sessions row and subagent tree agree · parent_session_id · parent_tool_use_id toolu_codex_01 · fork placed on the spawning tool call Task', + fromEventId: `msg:${root}:session.codex.root:0007`, + }); + // The grandchild's parent transcript is not the loaded one. + expect(forks.get(laneIdOf('codex', 'session.codex.grandchild'))).toMatchObject({ + fromLaneId: child, + grade: 'inferred', + basis: 'sessions row and subagent tree agree · parent_session_id · parent_tool_use_id toolu_codex_02 · fork placed at the child start: the parent transcript is not loaded', + }); + }); + + it('forks a row-parented child exactly only where the loaded parent carries its call', () => { + const parent = '02bc8f3c-d4e6-4176-afea-000000770509'; + const exact = forksWith('cursor', parent).find((relation) => relation.fromLaneId === laneIdOf('cursor', parent)); + expect([exact?.grade, exact?.fromEventId, exact?.basis]).toEqual([ + 'exact', + `msg:${laneIdOf('cursor', parent)}:${parent}:0019`, + 'sessions row · parent_session_id · parent_tool_use_id toolu_loom_5 · fork placed on the spawning tool call Task', + ]); + const other = '037c8f3c-d4e6-4176-afea-000000770521'; + const inferred = forksWith('cursor', other).find((relation) => relation.fromLaneId === laneIdOf('cursor', other)); + expect([inferred?.grade, inferred?.basis]).toEqual([ + 'inferred', + 'sessions row · parent_session_id · parent_tool_use_id toolu_loom_17 · fork placed at the child start: no loaded parent tool call carries toolu_loom_17', + ]); + }); +}); diff --git a/dashboard/src/viz/temporal/journey.ts b/dashboard/src/viz/temporal/journey.ts index 504920e439..669e2a0045 100644 --- a/dashboard/src/viz/temporal/journey.ts +++ b/dashboard/src/viz/temporal/journey.ts @@ -152,6 +152,19 @@ function proximityGrade(relation: FeedbackProximityRelationV1): EvidenceGrade { } } +interface ToolCallAnchor { + readonly eventId: string; + readonly toolUseId: string; + readonly label: string; + readonly time: number | null; +} + +interface ParentClaim { + readonly parentSessionId: string; + readonly toolUseId: string | null; + readonly source: 'sessions row' | 'subagent tree'; +} + function byStartThenId(a: LaneDraft, b: LaneDraft): number { return a.start - b.start || compareStrings(a.id, b.id); } @@ -271,7 +284,44 @@ export function projectJourney(sources: JourneySources): JourneyProjection { const gaps: JourneyGap[] = []; const spawnEvents: JourneyEvent[] = []; + // Tool calls the selected transcript page carries, by the host's own + // tool-use id. A fork or an edit binds to one only through that identity + // (or, for an edit, its recorded second); the first recorded call wins. + const toolCalls = new Map(); + const toolCallsBySecond = new Map(); + if (selected && drafts.has(selected.laneId)) { + for (const message of orderMessages(selected.messages)) { + const toolUseId = message.tool_use_id?.trim(); + if (!toolUseId || transcriptKind(message) !== 'tool_call') continue; + const key = JSON.stringify([selected.laneId, toolUseId]); + if (toolCalls.has(key)) continue; + const time = isFinitePositive(message.timestamp) ? message.timestamp : null; + const anchor = { eventId: `msg:${selected.laneId}:${message.message_id}`, toolUseId, label: transcriptLabel(message), time }; + toolCalls.set(key, anchor); + if (time === null) continue; + const secondKey = JSON.stringify([selected.laneId, time]); + const bucket = toolCallsBySecond.get(secondKey); + if (bucket) bucket.push(anchor); + else toolCallsBySecond.set(secondKey, [anchor]); + } + } + // --- parentage ----------------------------------------------------------- + // Two sources name a parent: the session row's own `parent_session_id` + // column, and the subagent tree. Where both speak they must agree; a + // disagreement is drawn as AMBIGUOUS with both candidates, never merged. + const claims = new Map(); + const claimFor = (laneId: string) => { + let entry = claims.get(laneId); + if (!entry) claims.set(laneId, (entry = { row: null, tree: null })); + return entry; + }; + for (const row of temporal.sessions) { + const parent = row.parent_session_id?.trim(); + const laneId = keyOf(row.provider, row.session_id); + if (!parent || !drafts.has(laneId)) continue; + claimFor(laneId).row = { parentSessionId: parent, toolUseId: row.parent_tool_use_id?.trim() || null, source: 'sessions row' }; + } const nodeByLane = new Map(); for (const node of hierarchy?.nodes ?? []) { nodeByLane.set(keyOf(node.provider, node.session_id), node); @@ -283,49 +333,11 @@ export function projectJourney(sources: JourneySources): JourneyProjection { switch (node.link) { case 'root': break; - case 'linked': { - if (node.parent_session_id === null) break; - const parentId = keyOf(node.provider, node.parent_session_id); - const parent = drafts.get(parentId); - if (!parent) { - gaps.push({ - id: `gap:parent_outside_page:${child.id}`, - laneId: child.id, - kind: 'parent_outside_page', - grade: 'unavailable', - detail: `recorded parent ${node.parent_session_id} is outside this loaded page`, - }); - break; + case 'linked': + if (node.parent_session_id !== null) { + claimFor(laneId).tree = { parentSessionId: node.parent_session_id, toolUseId: node.parent_tool_use_id, source: 'subagent tree' }; } - child.parentId = parent.id; - const precedes = child.start < parent.start; - const grade: EvidenceGrade = precedes ? 'ambiguous' : 'exact'; - const basis = - `parent_session_id · parent_tool_use_id ${node.parent_tool_use_id ?? 'unrecorded'}` + - (precedes ? ' · child start precedes parent start' : ''); - relations.push({ - id: `rel:spawn:${child.id}`, - kind: 'spawn', - fromLaneId: parent.id, - toLaneId: child.id, - time: child.start, - grade, - basis, - }); - spawnEvents.push({ - id: `spawn:${child.id}`, - laneId: parent.id, - kind: 'spawn', - time: child.start, - sequence: 0, - grade, - source: 'parentage', - label: child.label, - detail: basis, - ref: child.id, - }); break; - } case 'missing_parent': gaps.push({ id: `gap:parent_outside_page:${child.id}`, @@ -350,6 +362,106 @@ export function projectJourney(sources: JourneySources): JourneyProjection { } } } + for (const [laneId, { row, tree }] of claims) { + const child = drafts.get(laneId); + const primary = row ?? tree; + if (!child || !primary) continue; + const both = row !== null && tree !== null ? { row, tree } : null; + const parentsDiffer = both !== null && both.row.parentSessionId !== both.tree.parentSessionId; + const toolsDiffer = + both !== null && + !parentsDiffer && + both.row.toolUseId !== null && + both.tree.toolUseId !== null && + both.row.toolUseId !== both.tree.toolUseId; + if (both !== null && (parentsDiffer || toolsDiffer)) { + gaps.push({ + id: `gap:parentage_conflict:${child.id}`, + laneId: child.id, + kind: 'parentage_conflict', + grade: 'ambiguous', + detail: parentsDiffer + ? `sessions row names parent ${both.row.parentSessionId}; subagent tree names ${both.tree.parentSessionId}` + : `sessions row names tool use ${both.row.toolUseId}; subagent tree names ${both.tree.toolUseId}`, + }); + } + const candidates = both !== null && parentsDiffer ? [both.row, both.tree] : [primary]; + for (const claim of candidates) { + const parent = drafts.get(keyOf(child.provider, claim.parentSessionId)); + if (!parent) { + gaps.push({ + id: `gap:parent_outside_page:${child.id}:${claim.source}`, + laneId: child.id, + kind: 'parent_outside_page', + grade: 'unavailable', + detail: `${claim.source} names parent ${claim.parentSessionId}, outside this loaded page`, + }); + continue; + } + if (child.parentId === null) child.parentId = parent.id; + const precedes = child.start < parent.start; + const agreed = both !== null && !parentsDiffer; + // The fork sits on the parent's tool call only when a loaded parent + // message carries the recorded tool-use id; otherwise at the child's + // start, saying why. + const anchor = + claim.toolUseId === null ? undefined : toolCalls.get(JSON.stringify([parent.id, claim.toolUseId])); + const placement = anchor + ? `fork placed on the spawning tool call ${anchor.label}` + : claim.toolUseId === null + ? 'fork placed at the child start: no parent tool-use id recorded' + : selected?.laneId !== parent.id + ? 'fork placed at the child start: the parent transcript is not loaded' + : `fork placed at the child start: no loaded parent tool call carries ${claim.toolUseId}`; + const grade: EvidenceGrade = + parentsDiffer || toolsDiffer || precedes ? 'ambiguous' : anchor ? 'exact' : 'inferred'; + const basis = [ + agreed ? 'sessions row and subagent tree agree' : claim.source, + `parent_session_id · parent_tool_use_id ${claim.toolUseId ?? 'unrecorded'}`, + placement, + parentsDiffer ? 'the other source names a different parent' : null, + toolsDiffer ? 'the sources name different tool uses' : null, + precedes ? 'child start precedes parent start' : null, + ] + .filter((part): part is string => part !== null) + .join(' · '); + const suffix = parentsDiffer ? `:${claim.source}` : ''; + if (anchor) { + // The tool-call glyph is the fork's mark; no second spawn mark. + relations.push({ id: `rel:spawn:${child.id}${suffix}`, kind: 'spawn', fromLaneId: parent.id, toLaneId: child.id, time: anchor.time, grade, basis, fromEventId: anchor.eventId }); + continue; + } + relations.push({ id: `rel:spawn:${child.id}${suffix}`, kind: 'spawn', fromLaneId: parent.id, toLaneId: child.id, time: child.start, grade, basis }); + spawnEvents.push({ + id: `spawn:${child.id}${suffix}`, + laneId: parent.id, + kind: 'spawn', + time: child.start, + sequence: 0, + grade, + source: 'parentage', + label: child.label, + detail: basis, + ref: child.id, + }); + } + } + // A join only where the page shows it: the child's measured end inside its + // parent's measured extent. No result record backs it, so it is inferred. + for (const child of drafts.values()) { + const parent = child.parentId === null ? undefined : drafts.get(child.parentId); + if (!parent || child.end === null || parent.end === null) continue; + if (child.end < parent.start || child.end > parent.end) continue; + relations.push({ + id: `rel:rejoin:${child.id}`, + kind: 'rejoin', + fromLaneId: child.id, + toLaneId: parent.id, + time: child.end, + grade: 'inferred', + basis: `child ${child.endSource === 'session_end' ? 'recorded end' : 'last message'} inside the parent's measured extent · no result or handoff record in this read`, + }); + } if (hierarchy === null) { gaps.push({ @@ -383,7 +495,7 @@ export function projectJourney(sources: JourneySources): JourneyProjection { kind: 'handoff_unavailable', grade: 'unavailable', detail: - 'no handoff, result or rejoin authority is bound to session identity in this read; branches end at their recorded extent', + 'no handoff or result authority is bound to session identity in this read; a join is drawn only where a child ends inside its parent\'s measured extent, graded inferred', }); // --- depth (guarded against a cycle the authority did not flag) ---------- @@ -399,10 +511,30 @@ export function projectJourney(sources: JourneySources): JourneyProjection { draft.depth = depth; } + const untimedEditPaths = new Map>(); + for (const file of temporal.edited_files) { + if (file.edited_at_micros != null) continue; + const laneId = keyOf(file.provider, file.session_id); + const bucket = untimedEditPaths.get(laneId); + if (bucket) bucket.add(file.path); + else untimedEditPaths.set(laneId, new Set([file.path])); + } const ordered = orderLanes([...drafts.values()]); const laneIndex = new Map(ordered.map((lane, index) => [lane.id, index] as const)); for (const lane of ordered) { + // An edit the rollup recorded without an integer time has no honest x; + // the lane says how many of its edited files it cannot place. + const untimed = untimedEditPaths.get(lane.id)?.size ?? 0; + if (untimed > 0) { + gaps.push({ + id: `gap:edit_time_unrecorded:${lane.id}`, + laneId: lane.id, + kind: 'edit_time_unrecorded', + grade: 'unavailable', + detail: `${untimed} edited ${untimed === 1 ? 'file' : 'files'} recorded · no edit time in this read`, + }); + } if (lane.end === null) { gaps.push({ id: `gap:extent_unknown:${lane.id}`, @@ -469,6 +601,33 @@ export function projectJourney(sources: JourneySources): JourneyProjection { ref: commit.commit_sha, }); } + for (const file of temporal.edited_files) { + const laneId = keyOf(file.provider, file.session_id); + if (!laneIndex.has(laneId) || file.edited_at_micros == null) continue; + // An edit binds to the one loaded tool call recorded in its second; two + // calls in that second name no single call, so neither is linked. + const sameSecond = toolCallsBySecond.get(JSON.stringify([laneId, Math.floor(file.edited_at_micros / 1_000_000)])); + const call = sameSecond?.length === 1 ? sameSecond[0] : undefined; + const parts = [ + file.change_type, + file.hunks === null ? null : `${file.hunks} ${file.hunks === 1 ? 'hunk' : 'hunks'}`, + file.path, + call ? `tool call ${call.label} ${call.toolUseId}` : null, + ]; + pushRecorded({ + id: `edit:${laneId}:${file.path}:${file.edited_at_micros}`, + laneId, + kind: 'file_edit', + time: file.edited_at_micros / 1_000_000, + sequence: 0, + grade: 'exact', + source: 'file_rollup', + label: file.path.split('/').pop() || file.path, + detail: parts.filter((part): part is string => part !== null).join(' · '), + ref: file.path, + ...(call ? { linkedEventId: call.eventId } : {}), + }); + } for (const event of spawnEvents) pushRecorded(event); const transcriptByLane = new Map(); diff --git a/dashboard/src/viz/temporal/layout.test.ts b/dashboard/src/viz/temporal/layout.test.ts index ce851e44cd..0458739908 100644 --- a/dashboard/src/viz/temporal/layout.test.ts +++ b/dashboard/src/viz/temporal/layout.test.ts @@ -357,7 +357,7 @@ describe('layoutTemporalScene', () => { } const proj = projection({ lanes }); const dense = layoutTemporalScene(proj, optionsFor(proj, { denseLaneThreshold: 10 })); - expect(dense.denseDefault).toBe(true); + expect(dense.denseDepth).toBe(0); expect(dense.lanes.map((l) => l.id)).toEqual(['r0', 'r1', 'r2', 'r3', 'r4', 'r5']); expect(dense.lanes.every((l) => l.kind === 'bundle')).toBe(true); expect(dense.counts.lanesCollapsed).toBe(6); @@ -374,10 +374,34 @@ describe('layoutTemporalScene', () => { expect(reopened.counts.lanesCollapsed).toBe(5); const sparse = layoutTemporalScene(proj, optionsFor(proj, { denseLaneThreshold: 12 })); - expect(sparse.denseDefault).toBe(false); + expect(sparse.denseDepth).toBeNull(); expect(sparse.lanes).toHaveLength(12); }); + it('bundles a lone orchestrator at its workstream leads, not into one bundle', () => { + const lanes: JourneyLane[] = [lane({ id: 'orch', start: T0, end: T0 + 900, endSource: 'session_end' })]; + for (let l = 0; l < 3; l += 1) { + lanes.push(lane({ id: `lead${l}`, parentId: 'orch', depth: 1, start: T0 + 10 + l })); + for (let w = 0; w < 4; w += 1) { + lanes.push(lane({ id: `w${l}.${w}`, parentId: `lead${l}`, depth: 2, start: T0 + 20 + l * 10 + w })); + } + } + const proj = projection({ lanes }); + // 16 lanes over a threshold of 10: depth 1 keeps 4 visible, depth 2 would keep 16. + const dense = layoutTemporalScene(proj, optionsFor(proj, { denseLaneThreshold: 10 })); + expect(dense.denseDepth).toBe(1); + expect(dense.lanes.map((l) => [l.id, l.kind])).toEqual([ + ['orch', 'session'], + ['lead0', 'bundle'], + ['lead1', 'bundle'], + ['lead2', 'bundle'], + ]); + expect(dense.clusters.map((c) => c.counts.sessions)).toEqual([4, 4, 4]); + // Workstream zoom bundles where more than one session first delegates. + const workstream = layoutTemporalScene(proj, optionsFor(proj, { zoom: 'workstream' })); + expect(workstream.lanes.map((l) => [l.id, l.kind])).toEqual(dense.lanes.map((l) => [l.id, l.kind])); + }); + it('collapses every parent at workstream zoom', () => { const proj = treeProjection(); const model = layoutTemporalScene( @@ -670,6 +694,50 @@ describe('layoutTemporalScene', () => { expect(path).toMatchObject({ fromId: 'A', grade: 'exact', basis: spawn(A, B).basis, weight: null }); }); + it('leaves a fork from its spawning tool-call glyph and links a same-second edit to it', () => { + const task = event({ id: 'msg:A:task', laneId: 'A', kind: 'tool_call', time: T0 + 540, source: 'transcript', label: 'Task' }); + const undatedTask = event({ id: 'msg:A:undated', laneId: 'A', kind: 'tool_call', time: null, sequence: 1, source: 'transcript', label: 'Task' }); + const edit = event({ id: 'edit:A:src/a.ts', laneId: 'A', kind: 'file_edit', time: T0 + 540.25, source: 'file_rollup', label: 'a.ts', linkedEventId: 'msg:A:task' }); + const proj = projection({ + lanes: [A, B, C], + events: [...laneEvents(A, [edit]), task, undatedTask, ...laneEvents(B), ...laneEvents(C)], + relations: [ + { ...spawn(A, B), time: T0 + 540, fromEventId: 'msg:A:task' }, + { ...spawn(A, C), time: null, fromEventId: 'msg:A:undated' }, + ], + }); + const expanded = layoutTemporalScene(proj, optionsFor(proj, { zoom: 'event', selectedLaneId: 'A' })); + const glyph = expanded.nodes.find((n) => n.id === 'msg:A:task'); + const undatedGlyph = expanded.nodes.find((n) => n.id === 'msg:A:undated'); + const b = sceneLane(expanded, 'B'); + const c = sceneLane(expanded, 'C'); + expect(glyph).toMatchObject({ x: timeToX(expanded.viewport, T0 + 540), xBasis: 'time' }); + const forks = new Map(expanded.paths.filter((p) => p.kind === 'spawn').map((p) => [p.toId, p.controls])); + const kOf = (y0: number, y1: number) => Math.min(28, Math.max(8, Math.abs(y1 - y0) * 0.35)); + const kb = kOf(glyph!.y, b.y); + expect(forks.get('B')).toEqual([glyph!.x - kb, glyph!.y, glyph!.x + kb * 0.2, glyph!.y, glyph!.x - kb * 0.2, b.y, glyph!.x + kb, b.y]); + // An undated tool call still anchors the fork, in the recorded-order gutter. + expect(undatedGlyph?.xBasis).toBe('sequence'); + expect(forks.get('C')?.slice(0, 2)).toEqual([undatedGlyph!.x - kOf(undatedGlyph!.y, c.y), undatedGlyph!.y]); + const editNode = expanded.nodes.find((n) => n.id === 'edit:A:src/a.ts'); + expect(expanded.paths.find((p) => p.kind === 'edit_link')).toMatchObject({ + fromId: 'edit:A:src/a.ts', + toId: 'msg:A:task', + grade: 'exact', + controls: [editNode!.x, editNode!.y, glyph!.x, glyph!.y], + }); + + // Folded transcript: the dated fork falls back to its recorded time on the + // parent row, the undated one has no x at all, and the edit links nothing. + const folded = layoutTemporalScene(proj, optionsFor(proj)); + const a = sceneLane(folded, 'A'); + const x = timeToX(folded.viewport, T0 + 540); + expect(folded.paths.filter((p) => p.kind === 'spawn').map((p) => [p.toId, p.controls[0], p.controls[1]])).toEqual([ + ['B', x - kOf(a.y, sceneLane(folded, 'B').y), a.y], + ]); + expect(folded.paths.some((p) => p.kind === 'edit_link')).toBe(false); + }); + it('places intervals on their lane rows, clamped to the window', () => { const proj = treeProjection(); const model = layoutTemporalScene(proj, optionsFor(proj)); diff --git a/dashboard/src/viz/temporal/layout.ts b/dashboard/src/viz/temporal/layout.ts index 860099cc2e..a33eb3a843 100644 --- a/dashboard/src/viz/temporal/layout.ts +++ b/dashboard/src/viz/temporal/layout.ts @@ -233,11 +233,32 @@ export function layoutTemporalScene( // --- collapse & visibility ------------------------------------------------- const denseDefault = lanes.length > options.denseLaneThreshold; + // A dense page bundles at the deepest hierarchy level whose lanes still fit + // the threshold: a lone orchestrator opens onto its workstreams, not into + // one bundle of everything. + let denseDepth: number | null = null; + if (denseDefault) { + denseDepth = 0; + for (let depth = 1; lanes.filter((lane) => lane.depth <= depth).length <= options.denseLaneThreshold; depth += 1) { + if (!lanes.some((lane) => lane.depth > depth)) break; + denseDepth = depth; + } + } + // Workstream zoom bundles where the work first fans out: the shallowest + // level at which more than one session delegates. + let workstreamDepth = 0; + const maxDepth = lanes.reduce((max, lane) => Math.max(max, lane.depth), 0); + for (let depth = 0; depth <= maxDepth; depth += 1) { + if (lanes.filter((lane) => lane.depth === depth && (childrenOf.get(lane.id)?.length ?? 0) > 0).length > 1) { + workstreamDepth = depth; + break; + } + } const isCollapsed = (lane: JourneyLane): boolean => { if ((childrenOf.get(lane.id)?.length ?? 0) === 0) return false; switch (zoom) { case 'workstream': - return true; + return lane.depth === workstreamDepth; case 'agent': case 'event': break; @@ -247,7 +268,7 @@ export function layoutTemporalScene( } } if (branches.collapsed.has(lane.id)) return true; - return denseDefault && lane.depth === 0 && !branches.expanded.has(lane.id); + return lane.depth === denseDepth && !branches.expanded.has(lane.id); }; const collapsedIds = new Set(lanes.filter(isCollapsed).map((lane) => lane.id)); // The bundle a lane is hidden under is its outermost collapsed ancestor. @@ -425,6 +446,7 @@ export function layoutTemporalScene( // --- nodes ----------------------------------------------------------------- const nodes: SceneNode[] = []; const nodeById = new Map(); + const linkedEventIds = new Map(); const gutterYOf = (lane: SceneLane): number => lane.y + lane.height / 2 - SEQUENCE_GUTTER_RISE_PX; const undatedByLane = new Map(); @@ -467,6 +489,7 @@ export function layoutTemporalScene( }; nodes.push(scene); nodeById.set(scene.id, scene); + if (node.event.linkedEventId !== undefined) linkedEventIds.set(scene.id, node.event.linkedEventId); if (scene.xBasis === 'sequence') laneUndated.push(scene); }); if (laneUndated.length > 0) undatedByLane.set(sceneLane.id, laneUndated); @@ -502,14 +525,16 @@ export function layoutTemporalScene( relationsWithheld += 1; continue; } - if (relation.time === null) continue; + // A fork bound to a drawn tool-call glyph leaves from that glyph. + const origin = relation.fromEventId === undefined ? undefined : nodeById.get(relation.fromEventId); + const px = origin ? origin.x : relation.time === null ? null : x(relation.time); + if (px === null) continue; const targetId = standInFor(relation.toLaneId); if (targetId === null || targetId === parent.id) continue; const target = sceneLaneById.get(targetId); if (!target) continue; - const px = x(relation.time); if (!inWindow(px)) continue; - const py = parent.y; + const py = origin ? origin.y : parent.y; const ty = target.y; const k = Math.min(28, Math.max(8, Math.abs(ty - py) * 0.35)); const childFocus = focusFor(relation.toLaneId); @@ -547,6 +572,23 @@ export function layoutTemporalScene( } } + for (const node of nodes) { + const linkedId = linkedEventIds.get(node.id); + const call = linkedId === undefined ? undefined : nodeById.get(linkedId); + if (!call) continue; + paths.push({ + id: `edit_link:${node.id}→${call.id}`, + kind: 'edit_link', + fromId: node.id, + toId: call.id, + grade: 'exact', + basis: 'edit recorded in the same second as the tool call', + focus: node.focus, + controls: [node.x, node.y, call.x, call.y], + weight: null, + }); + } + // --- clusters -------------------------------------------------------------- const clusters: SceneCluster[] = []; for (const sceneLane of sceneLanes) { @@ -641,9 +683,12 @@ export function layoutTemporalScene( return { ...base, x: sceneLane.x1, y: sceneLane.y }; case 'parent_outside_page': case 'parent_cycle': + case 'parentage_conflict': case 'parentage_unavailable': case 'handoff_unavailable': return { ...base, x: sceneLane.x0, y: sceneLane.y }; + case 'edit_time_unrecorded': + return { ...base, x: (sceneLane.x0 + sceneLane.x1) / 2, y: sceneLane.y + Math.min(10, sceneLane.height * 0.3) }; case 'undated_events': return { ...base, @@ -789,6 +834,6 @@ export function layoutTemporalScene( relationsDrawn, relationsWithheld, }, - denseDefault, + denseDepth, }; } diff --git a/dashboard/src/viz/temporal/palette.ts b/dashboard/src/viz/temporal/palette.ts index ee90a7c704..b8dd6c4c5a 100644 --- a/dashboard/src/viz/temporal/palette.ts +++ b/dashboard/src/viz/temporal/palette.ts @@ -67,13 +67,15 @@ export function resolveTemporalPalette(element: HTMLElement): TemporalPalette { }; } +/** The one stroke grammar for the grade ladder on every drawn relation: + * solid, dash-dot, dashed, dotted, short dash, sparse dot. */ const GRADE_DASH: Readonly> = { exact: [], - explicit: [], - inferred: [5, 3], - ambiguous: [3, 3], - stale: [7, 2, 1, 2], - unavailable: [1, 4], + explicit: [6, 2, 1, 2], + inferred: [4, 3], + ambiguous: [1, 3], + stale: [2, 2], + unavailable: [1, 5], }; export function gradeStroke( diff --git a/dashboard/src/viz/temporal/scene/frame.ts b/dashboard/src/viz/temporal/scene/frame.ts new file mode 100644 index 0000000000..158bca2f77 --- /dev/null +++ b/dashboard/src/viz/temporal/scene/frame.ts @@ -0,0 +1,98 @@ +/** + * What the scene paints from, and the encodings every mark shares: focus, + * recency within the loaded page, and whether a lane resolves to glyphs. + */ +import { timeToX } from '../layout.ts'; +import type { SceneDensity } from '../density.ts'; +import type { FocusTreatment, SceneLane, TemporalSceneModel } from '../types.ts'; + +export interface SceneFrame { + readonly model: TemporalSceneModel; + readonly density: SceneDensity | null; + readonly fieldX0: number; + readonly fieldX1: number; + /** Top of the field, below the time ruler. */ + readonly top: number; + readonly height: number; +} + +/** Below this row height a lane cannot hold a glyph plate and aggregates. */ +export const LEGIBLE_PITCH_PX = 22; +/** Below this gap between neighbouring marks a lane aggregates. */ +export const COLLIDE_PX = 9; +/** The oldest record in the page keeps this share of full luminance. */ +export const RECENCY_FLOOR = 0.5; + +export function focusAlpha(focus: FocusTreatment): number { + switch (focus) { + case 'context': + return 0.35; + case 'path': + return 0.8; + case 'selected': + case 'neutral': + return 1; + default: { + const exhaustive: never = focus; + throw new Error(`unknown focus treatment: ${String(exhaustive)}`); + } + } +} + +export function isLifted(focus: FocusTreatment): boolean { + return focus === 'selected' || focus === 'path'; +} + +/** Where a spawn-family path leaves its parent and lands on its child. The + * layout centres the curve on the relation instant, so that x is the ends' + * midpoint. */ +export function pathEnds(controls: readonly number[]): { x0: number; y0: number; x1: number; y1: number } { + const n = controls.length; + return { x0: controls[0] ?? 0, y0: controls[1] ?? 0, x1: controls[n - 2] ?? 0, y1: controls[n - 1] ?? 0 }; +} + +/** The page's recency axis on screen: the oldest session start and NOW. */ +export function recencySpan(frame: SceneFrame): { headX: number; tailX: number } | null { + const head = frame.density?.headTime; + const tail = frame.density?.tailTime; + if (head == null || tail == null || tail <= head) return null; + return { headX: timeToX(frame.model.viewport, head), tailX: timeToX(frame.model.viewport, tail) }; +} + +/** Luminance for a mark at `x`: full at NOW, `RECENCY_FLOOR` at the oldest start. */ +export function recencyAlpha(frame: SceneFrame, x: number): number { + const span = recencySpan(frame); + if (!span) return 1; + const r = Math.min(1, Math.max(0, (x - span.headX) / (span.tailX - span.headX))); + return RECENCY_FLOOR + (1 - RECENCY_FLOOR) * r; +} + +const resolvedByFrame = new WeakMap>(); + +/** Lanes drawn as rails and glyphs; every other lane draws its density + * summary. A lane resolves when its row can hold a plate and its marks sit + * apart, or when it is the expanded session. */ +export function resolvedLanes(frame: SceneFrame): ReadonlySet { + const cached = resolvedByFrame.get(frame); + if (cached) return cached; + const resolves = (lane: SceneLane): boolean => { + if (lane.expanded) return true; + if (frame.model.zoom === 'workstream' || lane.height < LEGIBLE_PITCH_PX) return false; + const gap = frame.density?.lanes.get(lane.id)?.minGap ?? Infinity; + return gap >= COLLIDE_PX; + }; + const ids = new Set(frame.model.lanes.filter(resolves).map((lane) => lane.id)); + resolvedByFrame.set(frame, ids); + return ids; +} + +/** Hatch rows across a band, clipped by the caller. */ +export function hatch(ctx: CanvasRenderingContext2D, x0: number, y0: number, x1: number, y1: number, step: number): void { + const h = y1 - y0; + ctx.beginPath(); + for (let x = x0 - h; x < x1 + h; x += step) { + ctx.moveTo(x, y1); + ctx.lineTo(x + h, y0); + } + ctx.stroke(); +} diff --git a/dashboard/src/viz/temporal/scene/marks.tsx b/dashboard/src/viz/temporal/scene/marks.tsx new file mode 100644 index 0000000000..b1fc96fcdd --- /dev/null +++ b/dashboard/src/viz/temporal/scene/marks.tsx @@ -0,0 +1,230 @@ +/** + * The SVG marks inside the host's accessible buttons, and the + * non-interactive overlay: grade tags on causal links, engraved rail legends, + * the selection gutter, the playback cursor and `NOW`. + * + * Loaded-page language only: `NOW` names the newest record in this page and + * never a live connection. A recorded-order cursor spans only its own lane, + * because an x in the undated gutter is not a time. + */ +import type { CSSProperties, JSX } from 'react'; +import { formatMoment } from '../../../workspaces/loom/tracks.ts'; +import { EventGlyph } from '../glyphs.tsx'; +import { xToTime } from '../layout.ts'; +import { gradeColorVar, gradeDashArray } from '../palette.ts'; +import type { EvidenceGrade, SceneCluster, SceneLane, SceneNode, ScenePath } from '../types.ts'; +import { focusAlpha, isLifted, pathEnds, recencyAlpha, resolvedLanes, type SceneFrame } from './frame.ts'; + +/** Focus is a stable 2px cyan ring, shown only for keyboard focus. */ +export const NODE_CLASS = 'cursor-pointer outline-none [&:focus-visible>.td-focus-ring]:opacity-100'; + +/** Above this many drawn links, non-EXACT tags print only on the lifted chain. */ +const TAGGED_LINKS_MAX = 16; + +const ENGRAVED: CSSProperties = { fontFamily: 'var(--font-display)', fontStretch: '112%' }; +const MONO: CSSProperties = { fontFamily: 'var(--font-mono)' }; + +const GRADE_TAG: Readonly> = { + exact: 'EXACT', + explicit: 'EXPLICIT', + inferred: 'INFERRED', + ambiguous: 'AMBIGUOUS', + stale: 'STALE', + unavailable: 'UNAVAILABLE', +}; + +function isLink(path: ScenePath): boolean { + return path.kind === 'spawn' || path.kind === 'handoff' || path.kind === 'rejoin' || path.kind === 'result'; +} + +function FocusRing({ x, y, r }: { x: number; y: number; r: number }): JSX.Element { + return ; +} + +export function NodeMark({ node, frame }: { node: SceneNode; frame: SceneFrame }): JSX.Element { + const color = gradeColorVar(node.grade); + const luminance = isLifted(node.focus) ? 1 : recencyAlpha(frame, node.x); + if (!resolvedLanes(frame).has(node.laneId) && !node.selected) { + return ; + } + return ( + <> + {node.selected && } + {node.selected && ( + + )} + + + + + + + + + ); +} + +export function ClusterMark({ cluster }: { cluster: SceneCluster }): JSX.Element { + const x1 = Math.max(cluster.x1, cluster.x0 + 2); + const top = cluster.y - cluster.height / 2 + 1; + const h = cluster.height - 3; + return ( + + ); +} + +function CursorAndTail({ frame }: { frame: SceneFrame }): JSX.Element { + const { model, fieldX0, fieldX1, top, height, density } = frame; + const cursor = model.cursor; + const cursorLane = cursor === null ? undefined : model.lanes.find((lane) => lane.id === cursor.laneId); + const tailX = density?.tailX ?? null; + const tailTime = density?.tailTime ?? null; + const tailLater = tailTime !== null && tailTime > model.viewport.window.end; + const cursorText = + cursor === null + ? null + : cursor.xBasis === 'time' + ? `CURSOR ${formatMoment(xToTime(model.viewport, cursor.x))}` + : 'CURSOR · RECORDED ORDER'; + const nearRight = cursor !== null && cursor.x > fieldX1 - 180; + return ( + + {tailX !== null && ( + + {`NOW = newest record in this loaded page${tailTime === null ? '' : ` · ${formatMoment(tailTime)}`} · not a live stream`} + + + fieldX1 - 30 ? 'end' : 'middle'} fill="var(--raw-graph-accent)" style={ENGRAVED}> + NOW + + + )} + {tailX === null && tailLater && ( + + NOW lies after this window + + NOW → + + + )} + {cursor && cursor.x >= fieldX0 && cursor.x <= fieldX1 && ( + + {cursor.xBasis === 'time' || !cursorLane ? ( + <> + + + + ) : ( + + )} + {cursorText && ( + + {cursorText} + + )} + + )} + + ); +} + +export function FieldOverlay({ frame }: { frame: SceneFrame }): JSX.Element { + const { model, fieldX1 } = frame; + const links = model.paths.filter(isLink); + const tagAll = links.length <= TAGGED_LINKS_MAX; + const gutterX = fieldX1 + (model.viewport.right + 8) / 2; + // Every drawn link's grade is printed: per link while they fit, and always + // as a tally in the ruler so a dense page never hides one. + const tally = new Map(); + for (const path of links) { + const key = `${path.kind === 'rejoin' ? 'JOINS' : 'FORKS'} ${GRADE_TAG[path.grade]}`; + tally.set(key, (tally.get(key) ?? 0) + 1); + } + const tallyText = [...tally.entries()].map(([key, count]) => `${count} ${key}`).join(' · '); + return ( + + {tallyText && ( + + {tallyText} + + )} + {model.rails.map((rail) => { + const text = `${rail.label.toUpperCase()} · ${rail.lanes}`; + if (rail.y1 - rail.y0 < text.length * 7 + 8) return null; + const cy = (rail.y0 + rail.y1) / 2; + return ( + + + + {text} + + + ); + })} + {model.lanes + .filter((lane) => lane.focus === 'selected') + .map((lane) => ( + + ))} + {links.map((path) => { + const { x0, y0, x1, y1 } = pathEnds(path.controls); + if (Math.abs(y1 - y0) < 20) return null; + // Solid EXACT is the default reading; every other grade is printed, + // and the lifted chain prints all of its grades. + if (!isLifted(path.focus) && (path.grade === 'exact' || !tagAll)) return null; + return ( + + {GRADE_TAG[path.grade]} + + ); + })} + + + ); +} + +const COUNT_FORMAT = new Intl.NumberFormat(); + +/** The lane column's second line: exact totals over the page, never the window. */ +export function laneDetail(lane: SceneLane, frame: SceneFrame): string | null { + const density = frame.density?.lanes.get(lane.id); + if (!density) return null; + const { totals, peak } = density; + if (lane.kind === 'bundle') { + return `1+${totals.sessions - 1} sess · ${COUNT_FORMAT.format(totals.messages)} msg · peak ${peak.active}${peak.open > 0 ? ` · ${peak.open} open` : ''}`; + } + const commits = totals.commits > 0 ? ` · ${totals.commits} ${totals.commits === 1 ? 'commit' : 'commits'}` : ''; + return `${lane.provider} · ${COUNT_FORMAT.format(totals.messages)} msg${commits}`; +} + +export function LegendEncodings(): JSX.Element { + return ( +

    + brightness = recency within the loaded page, dimmest at the oldest start and full at NOW · rail weight = messages (log) · bundle bars = member sessions with a measured extent (√, one scale for every bundle) · dots = begun, extent unknown · floor ticks = dated events per bin where glyphs would not fit · link tags print every non-EXACT grade · the selected chain lifts over the recency veil +

    + ); +} diff --git a/dashboard/src/viz/temporal/scene/paint.ts b/dashboard/src/viz/temporal/scene/paint.ts new file mode 100644 index 0000000000..3ca9111c8b --- /dev/null +++ b/dashboard/src/viz/temporal/scene/paint.ts @@ -0,0 +1,315 @@ +/** + * The Canvas2D substrate of the temporal field. + * + * A measuring instrument: 32px graticule hairlines with the labelled ticks as + * majors, one banded rail per session, and causal links routed orthogonally + * in their grade's line style. Luminance is spent only on real quantities: + * recency within the loaded page (dimmest at the oldest start, full at NOW) + * and the selected causal chain, which lifts over the recency veil with a + * restrained cyan halo. Additive blending applies to rails and links so that + * only where two real marks cross does the field brighten. + * + * Semantic zoom is one rule: a lane that cannot hold legible glyphs draws its + * density summary instead. A collapsed branch is always a summary, a + * shared-scale histogram of member sessions with a measured extent, with + * begun-but-unmeasured sessions dotted above it; a session lane whose marks + * collide draws its extent band and a floor rug of dated events. Zooming the + * window or expanding a branch resolves the same lanes to rails and glyphs. + */ +import type { DensityBin, LaneDensity } from '../density.ts'; +import { gradeStroke, type TemporalPalette } from '../palette.ts'; +import type { SceneLane, ScenePath } from '../types.ts'; +import { focusAlpha, hatch, isLifted, pathEnds, recencySpan, resolvedLanes, type SceneFrame } from './frame.ts'; + +const GRATICULE_PX = 32; +const CORNER_PX = 4; +const HALO_PX = 7; +const HALO_ALPHA = 0.16; +const BAND_INSET = 3; + +function isLink(path: ScenePath): boolean { + return path.kind === 'spawn' || path.kind === 'handoff' || path.kind === 'rejoin' || path.kind === 'result'; +} + +/** Down the relation instant, a small corner, then along the child rail. */ +function traceRoute(ctx: CanvasRenderingContext2D, path: ScenePath): void { + const { x0, y0, x1, y1 } = pathEnds(path.controls); + const x = (x0 + x1) / 2; + const dir = y1 >= y0 ? 1 : -1; + const r = Math.min(CORNER_PX, Math.abs(y1 - y0) / 2); + ctx.moveTo(x, y0); + ctx.lineTo(x, y1 - dir * r); + ctx.arcTo(x, y1, x + r, y1, r); + ctx.lineTo(x1, y1); +} + +function traceRail(ctx: CanvasRenderingContext2D, lane: SceneLane): void { + const y = Math.round(lane.y) + 0.5; + ctx.moveTo(lane.x0, y); + ctx.lineTo(Math.max(lane.x1, lane.x0 + 1), y); +} + +/** One path per layer: a dense page is hundreds of bins per lane. */ +function fillBins( + ctx: CanvasRenderingContext2D, + bins: readonly DensityBin[], + color: string, + alpha: number, + rect: (bin: DensityBin, w: number) => readonly [number, number, number, number] | null, +): void { + ctx.beginPath(); + for (const bin of bins) { + const r = rect(bin, Math.max(1, bin.x1 - bin.x0 - 1)); + if (r) ctx.rect(r[0], r[1], r[2], r[3]); + } + ctx.globalAlpha = alpha; + ctx.fillStyle = color; + ctx.fill(); +} + +/** Dots through every rect in one clip: begun, extent unknown. */ +function dotRects(ctx: CanvasRenderingContext2D, rects: readonly (readonly [number, number, number, number])[], color: string, alpha: number): void { + if (rects.length === 0) return; + ctx.save(); + ctx.beginPath(); + let x0 = Infinity; + let x1 = -Infinity; + let y0 = Infinity; + let y1 = -Infinity; + for (const [x, y, w, h] of rects) { + ctx.rect(x, y, w, h); + x0 = Math.min(x0, x); + x1 = Math.max(x1, x + w); + y0 = Math.min(y0, y); + y1 = Math.max(y1, y + h); + } + ctx.clip(); + ctx.beginPath(); + for (let x = x0 + 1; x < x1; x += 3) for (let y = y0 + 1; y < y1; y += 3) ctx.rect(x, y, 1, 1); + ctx.globalAlpha = alpha; + ctx.fillStyle = color; + ctx.fill(); + ctx.restore(); +} + +function paintBundle(ctx: CanvasRenderingContext2D, lane: SceneLane, density: LaneDensity, peak: number, palette: TemporalPalette, alpha: number): void { + const floor = lane.y + lane.height / 2 - BAND_INSET; + const room = Math.max(4, lane.height - BAND_INSET * 2 - 2); + const h = (count: number): number => Math.sqrt(count / peak) * room; + fillBins(ctx, density.bins, palette.signal, 0.55 * alpha, (bin, w) => (bin.active > 0 ? [bin.x0, floor - h(bin.active), w, h(bin.active)] : null)); + const open = density.bins.flatMap((bin) => { + const top = h(bin.active + bin.open); + const base = h(bin.active); + return top - base > 0.5 ? [[bin.x0, floor - top, Math.max(1, bin.x1 - bin.x0 - 1), top - base] as const] : []; + }); + dotRects(ctx, open, palette.text, 0.55 * alpha); +} + +function paintRug(ctx: CanvasRenderingContext2D, lane: SceneLane, density: LaneDensity, palette: TemporalPalette, alpha: number): void { + const floor = lane.y + lane.height / 2 - 1; + fillBins(ctx, density.bins, palette.text, 0.75 * alpha, (bin, w) => { + if (bin.events === 0) return null; + const tick = Math.min(BAND_INSET + 2, 1 + Math.log2(1 + bin.events)); + return [bin.x0 + w / 2 - 0.5, floor - tick, 1, tick]; + }); +} + +export function paintScene(ctx: CanvasRenderingContext2D, frame: SceneFrame, palette: TemporalPalette): void { + const { model, fieldX0, fieldX1, top, height, density } = frame; + const width = Math.max(1, fieldX1 - fieldX0); + const resolved = resolvedLanes(frame); + const additive = palette.light ? 'source-over' : 'lighter'; + ctx.save(); + ctx.beginPath(); + ctx.rect(fieldX0, top, width, height - top); + ctx.clip(); + ctx.lineWidth = 1; + ctx.setLineDash([]); + + // Graticule: 32px minor hairlines as texture, the labelled ticks as majors. + ctx.strokeStyle = palette.grid; + ctx.globalAlpha = palette.light ? 0.35 : 0.3; + ctx.beginPath(); + for (let x = fieldX0 + GRATICULE_PX; x < fieldX1; x += GRATICULE_PX) { + ctx.moveTo(Math.round(x) + 0.5, top); + ctx.lineTo(Math.round(x) + 0.5, height); + } + ctx.stroke(); + ctx.strokeStyle = palette.edge; + ctx.globalAlpha = 0.35; + ctx.beginPath(); + for (const tick of model.ticks) { + ctx.moveTo(Math.round(tick.x) + 0.5, top); + ctx.lineTo(Math.round(tick.x) + 0.5, height); + } + ctx.stroke(); + + // Lane bands with hairline separators; the selected band carries the tint. + for (const lane of model.lanes) { + const y0 = lane.y - lane.height / 2; + if (isLifted(lane.focus)) { + ctx.globalAlpha = lane.focus === 'selected' ? 0.08 : 0.035; + ctx.fillStyle = palette.signal; + ctx.fillRect(fieldX0, y0, width, lane.height); + } else if (lane.row % 2 === 1) { + ctx.globalAlpha = palette.light ? 0.03 : 0.022; + ctx.fillStyle = palette.text; + ctx.fillRect(fieldX0, y0, width, lane.height); + } + ctx.globalAlpha = 0.7; + ctx.fillStyle = palette.grid; + ctx.fillRect(fieldX0, Math.round(y0 + lane.height) - 1, width, 1); + } + for (const rail of model.rails) { + ctx.globalAlpha = 0.9; + ctx.fillStyle = palette.edge; + ctx.fillRect(fieldX0, Math.round(rail.y0), width, 1); + } + + // Summaries where lanes cannot hold glyphs, on one scale for every bundle. + let bundlePeak = 1; + for (const lane of model.lanes) { + const peak = lane.kind === 'bundle' ? density?.lanes.get(lane.id)?.peak : undefined; + if (peak) bundlePeak = Math.max(bundlePeak, peak.active + peak.open); + } + for (const lane of model.lanes) { + const laneDensity = density?.lanes.get(lane.id); + if (!laneDensity || !lane.revealed) continue; + const alpha = focusAlpha(lane.focus); + if (lane.kind === 'bundle') paintBundle(ctx, lane, laneDensity, bundlePeak, palette, alpha); + if (!resolved.has(lane.id)) paintRug(ctx, lane, laneDensity, palette, alpha); + } + + ctx.globalCompositeOperation = additive; + const laneById = new Map(model.lanes.map((lane) => [lane.id, lane] as const)); + for (const path of model.paths) { + if (path.kind !== 'lane') continue; + const lane = laneById.get(path.fromId); + if (!lane || lane.kind === 'bundle') continue; + const stroke = gradeStroke(path.grade, palette); + const resolves = resolved.has(lane.id); + if (!resolves) { + // An aggregated session is its extent band; the grade rides on the + // hairline through it, since a dashed band reads as a barcode. + ctx.globalAlpha = 0.28 * focusAlpha(path.focus); + ctx.strokeStyle = stroke.color; + ctx.lineWidth = Math.max(2, lane.height - BAND_INSET * 4); + ctx.setLineDash([]); + ctx.beginPath(); + traceRail(ctx, lane); + ctx.stroke(); + } + ctx.globalAlpha = focusAlpha(path.focus); + ctx.strokeStyle = stroke.color; + ctx.lineWidth = 1 + (path.weight ?? 0) * 1.5; + ctx.setLineDash([...stroke.dash]); + ctx.beginPath(); + traceRail(ctx, lane); + ctx.stroke(); + } + const links = model.paths.filter(isLink); + for (const path of links) { + const stroke = gradeStroke(path.grade, palette); + // A join is an inference from two extents; it recedes behind the forks. + ctx.globalAlpha = focusAlpha(path.focus) * (path.kind === 'rejoin' ? 0.6 : 1); + ctx.strokeStyle = stroke.color; + ctx.lineWidth = 1; + ctx.setLineDash([...stroke.dash]); + ctx.beginPath(); + traceRoute(ctx, path); + ctx.stroke(); + } + ctx.globalCompositeOperation = 'source-over'; + ctx.setLineDash([1, 3]); + ctx.strokeStyle = palette.text; + for (const path of model.paths) { + if (path.kind !== 'sequence' && path.kind !== 'edit_link') continue; + ctx.globalAlpha = 0.45 * focusAlpha(path.focus); + ctx.beginPath(); + ctx.moveTo(path.controls[0]!, path.controls[1]!); + ctx.lineTo(path.controls[2]!, path.controls[3]!); + ctx.stroke(); + } + ctx.setLineDash([]); + + // A lane with no measured extent ends in a hatched stub. + ctx.strokeStyle = palette.dim; + for (const lane of model.lanes) { + if (lane.endSource !== null || lane.offscreen || !lane.revealed || lane.kind === 'bundle') continue; + ctx.save(); + ctx.beginPath(); + ctx.rect(lane.x0, lane.y - 3, 18, 6); + ctx.clip(); + ctx.globalAlpha = 0.8 * focusAlpha(lane.focus); + hatch(ctx, lane.x0, lane.y - 3, lane.x0 + 18, lane.y + 3, 3); + ctx.restore(); + } + + // Recency: a veil from the oldest start in the page to NOW. + const span = recencySpan(frame); + if (span) { + const veil = ctx.createLinearGradient(span.headX, 0, span.tailX, 0); + veil.addColorStop(0, palette.substrate); + veil.addColorStop(1, 'transparent'); + ctx.globalAlpha = 0.5; + ctx.fillStyle = veil; + ctx.fillRect(fieldX0, top, width, height - top); + } + + // The selected causal chain lifts over the veil: a restrained halo, then + // the crisp stroke in its own grade style. + ctx.lineCap = 'round'; + const lifted: [ScenePath, (c: CanvasRenderingContext2D) => void][] = []; + for (const path of model.paths) { + if (!isLifted(path.focus)) continue; + if (path.kind === 'lane') { + const lane = laneById.get(path.fromId); + if (lane && lane.kind !== 'bundle' && resolved.has(lane.id)) lifted.push([path, (c) => traceRail(c, lane)]); + } else if (isLink(path)) { + lifted.push([path, (c) => traceRoute(c, path)]); + } + } + for (const [, trace] of lifted) { + ctx.globalAlpha = HALO_ALPHA; + ctx.strokeStyle = palette.signal; + ctx.lineWidth = HALO_PX; + ctx.setLineDash([]); + ctx.beginPath(); + trace(ctx); + ctx.stroke(); + } + ctx.lineCap = 'butt'; + for (const [path, trace] of lifted) { + const stroke = gradeStroke(path.grade, palette); + ctx.globalAlpha = 1; + ctx.strokeStyle = path.focus === 'selected' && path.kind === 'lane' ? palette.signalHot : stroke.color; + ctx.lineWidth = path.kind === 'lane' ? 1.5 + (path.weight ?? 0) * 1.5 : 1.5; + ctx.setLineDash([...stroke.dash]); + ctx.beginPath(); + trace(ctx); + ctx.stroke(); + } + ctx.setLineDash([]); + ctx.restore(); + paintUnrevealed(ctx, frame, palette); +} + +/** Veils and hatches the band after a dated cursor: those records are + * withheld, so the band reads as not yet revealed rather than empty. */ +function paintUnrevealed(ctx: CanvasRenderingContext2D, frame: SceneFrame, palette: TemporalPalette): void { + const { model, fieldX1, top, height } = frame; + const cursor = model.cursor; + if (!cursor || cursor.xBasis !== 'time' || cursor.x >= fieldX1) return; + const x0 = Math.max(frame.fieldX0, cursor.x); + ctx.save(); + ctx.beginPath(); + ctx.rect(x0, top, fieldX1 - x0, height - top); + ctx.clip(); + ctx.globalAlpha = 0.5; + ctx.fillStyle = palette.substrate; + ctx.fillRect(x0, top, fieldX1 - x0, height - top); + ctx.globalAlpha = 0.06; + ctx.strokeStyle = palette.text; + hatch(ctx, x0, top, fieldX1, height, 9); + ctx.restore(); +} diff --git a/dashboard/src/viz/temporal/scene/scene.dom.test.tsx b/dashboard/src/viz/temporal/scene/scene.dom.test.tsx new file mode 100644 index 0000000000..f9705c1e5d --- /dev/null +++ b/dashboard/src/viz/temporal/scene/scene.dom.test.tsx @@ -0,0 +1,308 @@ +import { render, screen } from '@testing-library/react'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { DensityBin, LaneDensity, SceneDensity } from '../density.ts'; +import { TemporalScene } from '../TemporalScene.tsx'; +import type { SceneNode, SceneWindow, TemporalSceneModel } from '../types.ts'; + +/** + * What the field paints, observed through the canvas calls it makes and the + * overlay marks it emits: semantic zoom between density summaries and glyphs, + * recency luminance, the lifted causal chain, grade tags and loaded-page + * markers. The shared overlay contract is in `TemporalScene.dom.test`. + */ + +const WINDOW: SceneWindow = { start: 1_784_700_000, end: 1_784_707_200 }; +const ROOT = JSON.stringify(['cursor', 'root']); +const CHILD = JSON.stringify(['cursor', 'child']); +const BUNDLE = JSON.stringify(['codex', 'bundle']); + +function node(overrides: Partial & Pick): SceneNode { + return { laneId: ROOT, y: 80, xBasis: 'time', grade: 'exact', source: 'session', label: overrides.id, detail: null, ref: overrides.id, selected: false, focus: 'neutral', halfHit: 12, ...overrides }; +} + +function model(over: Partial = {}): TemporalSceneModel { + return { + viewport: { width: 960, left: 200, right: 28, window: WINDOW }, + zoom: 'agent', + height: 240, + lanes: [ + { id: ROOT, kind: 'session', label: 'root session', provider: 'cursor', depth: 0, y: 80, height: 48, x0: 200, x1: 810, endSource: 'session_end', focus: 'neutral', expanded: false, offscreen: false, revealed: true, collapsedDescendants: 0, row: 0 }, + { id: CHILD, kind: 'session', label: 'child agent', provider: 'cursor', depth: 1, y: 130, height: 48, x0: 444, x1: 688, endSource: null, focus: 'neutral', expanded: false, offscreen: false, revealed: true, collapsedDescendants: 0, row: 1 }, + { id: BUNDLE, kind: 'bundle', label: 'codex fan-out', provider: 'codex', depth: 0, y: 190, height: 48, x0: 300, x1: 700, endSource: 'last_message', focus: 'neutral', expanded: false, offscreen: false, revealed: true, collapsedDescendants: 3, row: 2 }, + ], + nodes: [ + node({ id: 'n-start', kind: 'session_start', x: 200 }), + node({ id: 'n-spawn', kind: 'spawn', x: 444, source: 'parentage', ref: CHILD }), + node({ id: 'n-commit', kind: 'commit', x: 450, grade: 'inferred', source: 'commit', label: '9f3c2ab' }), + ], + paths: [ + { id: 'p-root', kind: 'lane', fromId: ROOT, toId: ROOT, grade: 'exact', basis: null, focus: 'neutral', controls: [200, 80, 810, 80], weight: 0.7 }, + { id: 'p-child', kind: 'lane', fromId: CHILD, toId: CHILD, grade: 'unavailable', basis: null, focus: 'neutral', controls: [444, 130, 462, 130], weight: 0.2 }, + { id: 'p-bundle', kind: 'lane', fromId: BUNDLE, toId: BUNDLE, grade: 'exact', basis: null, focus: 'neutral', controls: [300, 190, 700, 190], weight: 0.5 }, + { id: 'p-spawn', kind: 'spawn', fromId: ROOT, toId: CHILD, grade: 'ambiguous', basis: 'child start precedes parent start', focus: 'neutral', controls: [428, 80, 447, 80, 441, 130, 460, 130], weight: null }, + { id: 'p-spawn-exact', kind: 'spawn', fromId: ROOT, toId: BUNDLE, grade: 'exact', basis: 'parent_session_id', focus: 'neutral', controls: [572, 80, 591, 80, 585, 190, 604, 190], weight: null }, + ], + clusters: [ + { id: 'c-bundle', laneId: BUNDLE, memberLaneIds: ['a', 'b', 'c'], x0: 300, x1: 700, y: 190, height: 48, counts: { sessions: 3, subagents: 3, messages: 57, commits: 0, openEnded: 1 }, grades: { exact: 3 }, focus: 'neutral' }, + ], + intervals: [], + gaps: [], + rails: [{ id: 'rail-cursor', kind: 'provider', label: 'cursor', y0: 56, y1: 154, lanes: 2 }], + ticks: [0, 1, 2, 3].map((step) => ({ x: 200 + step * 183, time: WINDOW.start + step * 1800, label: `09:${40 + step * 5}` })), + labels: [], + minimap: { bins: [], lanes: [], window: { x0: 0, x1: 960 }, width: 960, height: 48 }, + cursor: null, + counts: { lanesTotal: 6, lanesVisible: 3, lanesCollapsed: 1, eventsTotal: 3, eventsDrawn: 3, eventsCulled: 0, eventsWithheld: 0, eventsFiltered: 0, eventsFolded: 0, relationsTotal: 2, relationsDrawn: 2, relationsWithheld: 0 }, + denseDepth: null, + ...over, + }; +} + +function laneDensity(laneId: string, minGap: number, bins: DensityBin[] = []): LaneDensity { + return { + laneId, + bins, + peak: { active: Math.max(0, ...bins.map((bin) => bin.active)), open: Math.max(0, ...bins.map((bin) => bin.open)), events: Math.max(0, ...bins.map((bin) => bin.events)) }, + totals: { sessions: laneId === BUNDLE ? 4 : 1, messages: laneId === BUNDLE ? 91 : 12, commits: laneId === ROOT ? 1 : 0, events: 3, undated: 0, openEnded: 1 }, + minGap, + }; +} + +function density(over: Partial = {}, rootGap = 244): SceneDensity { + const bundleBins: DensityBin[] = [ + { x0: 300, x1: 306, active: 2, open: 1, starts: 1, events: 2 }, + { x0: 306, x1: 312, active: 1, open: 1, starts: 0, events: 0 }, + ]; + return { + binPx: 6, + lanes: new Map([ + [ROOT, laneDensity(ROOT, rootGap)], + [CHILD, laneDensity(CHILD, Infinity)], + [BUNDLE, laneDensity(BUNDLE, Infinity, bundleBins)], + ]), + headTime: WINDOW.start, + tailTime: WINDOW.start + 5400, + tailX: 749, + ...over, + }; +} + +interface Recorded { + calls: string[]; + rects: number[][]; + lineWidths: number[]; + composites: string[]; +} + +const originalGetContext = HTMLCanvasElement.prototype.getContext; + +function stubCanvas(): Recorded { + const recorded: Recorded = { calls: [], rects: [], lineWidths: [], composites: [] }; + const context = new Proxy({} as Record, { + get(_target, property) { + if (typeof property !== 'string') return undefined; + return (...args: number[]): unknown => { + recorded.calls.push(property); + if (property === 'rect') recorded.rects.push(args); + if (property === 'stroke') recorded.lineWidths.push(currentLineWidth); + return property.startsWith('create') ? { addColorStop(): void {} } : undefined; + }; + }, + set(_target, property, value) { + if (property === 'lineWidth') currentLineWidth = Number(value); + if (property === 'globalCompositeOperation') recorded.composites.push(String(value)); + return true; + }, + }); + let currentLineWidth = 1; + Object.defineProperty(HTMLCanvasElement.prototype, 'getContext', { configurable: true, value: () => context }); + return recorded; +} + +function renderWith(over: { model?: TemporalSceneModel; density?: SceneDensity | null } = {}) { + return render( + , + ); +} + +const withFocus = (laneIds: Record, pathIds: Record = {}) => { + const base = model(); + return model({ + lanes: base.lanes.map((lane) => ({ ...lane, focus: laneIds[lane.id] ?? 'context' })), + paths: base.paths.map((path) => ({ ...path, focus: pathIds[path.id] ?? 'context' })), + }); +}; + +describe('temporal field paint', () => { + let recorded: Recorded; + beforeEach(() => { + recorded = stubCanvas(); + }); + afterEach(() => { + Object.defineProperty(HTMLCanvasElement.prototype, 'getContext', { configurable: true, value: originalGetContext }); + }); + + describe('semantic zoom', () => { + const glyphs = (container: HTMLElement) => container.querySelectorAll('[data-event] [data-glyph]').length; + + it('draws glyphs where marks sit apart and every event keeps its button', () => { + const { container } = renderWith(); + expect(glyphs(container)).toBe(3); + expect(container.querySelectorAll('[data-event]').length).toBe(3); + }); + + it('aggregates a lane whose marks collide into its event rug', () => { + const { container } = renderWith({ density: density({}, 8) }); + expect(glyphs(container)).toBe(0); + expect(container.querySelectorAll('[data-event]').length).toBe(3); + }); + + it('aggregates every lane at workstream zoom and below the legible pitch', () => { + expect(glyphs(renderWith({ model: model({ zoom: 'workstream' }) }).container)).toBe(0); + const short = model(); + const { container } = renderWith({ model: model({ lanes: short.lanes.map((lane) => ({ ...lane, height: 16 })) }) }); + expect(glyphs(container)).toBe(0); + }); + + it('resolves the expanded session even when its marks are crowded', () => { + const base = model(); + const expanded = model({ lanes: base.lanes.map((lane) => (lane.id === ROOT ? { ...lane, expanded: true } : lane)) }); + expect(glyphs(renderWith({ model: expanded, density: density({}, 4) }).container)).toBe(3); + }); + + it('bars a bundle by measured member sessions on a square-root scale', () => { + renderWith(); + // Peak active+open is 3; two active of a 40px room is sqrt(2/3)*40 tall, + // standing on the floor at 190 + 24 - 3. + const bar = recorded.rects.find((rect) => rect[0] === 300 && rect[2] === 5 && rect[1]! < 211)!; + expect(bar[1]).toBeCloseTo(211 - Math.sqrt(2 / 3) * 40, 6); + expect(bar[3]).toBeCloseTo(Math.sqrt(2 / 3) * 40, 6); + expect(recorded.calls).toContain('clip'); + }); + }); + + describe('luminance', () => { + it('dims an older mark toward half luminance and keeps NOW at full', () => { + const { container } = renderWith(); + // Head at x=200, NOW at 200 + 5400/7200 * 732 = 749. + const commit = container.querySelector('[data-event="n-commit"] g[opacity]')!; + expect(Number(commit.getAttribute('opacity'))).toBeCloseTo(0.5 + 0.5 * (250 / 549), 6); + const start = container.querySelector('[data-event="n-start"] g[opacity]')!; + expect(Number(start.getAttribute('opacity'))).toBe(0.5); + }); + + it('keeps luminance flat when the page has no dated extent', () => { + const { container } = renderWith({ density: density({ headTime: null, tailTime: null, tailX: null }) }); + expect(container.querySelector('[data-event="n-commit"] g[opacity]')?.getAttribute('opacity')).toBe('1'); + }); + + it('lifts the selected chain with a 7px halo, and paints no halo without a selection', () => { + renderWith(); + expect(recorded.lineWidths).not.toContain(7); + recorded.lineWidths.length = 0; + renderWith({ model: withFocus({ [CHILD]: 'selected', [ROOT]: 'path' }, { 'p-root': 'path', 'p-spawn': 'selected' }) }); + expect(recorded.lineWidths.filter((width) => width === 7)).toHaveLength(2); + }); + + it('holds a lifted mark at full luminance and halos the selected event', () => { + const base = model(); + const selected = model({ nodes: base.nodes.map((entry) => (entry.id === 'n-start' ? { ...entry, selected: true, focus: 'selected' } : entry)) }); + const { container } = renderWith({ model: selected }); + expect(container.querySelector('[data-event="n-start"] g[opacity]')?.getAttribute('opacity')).toBe('1'); + expect(container.querySelector('[data-event="n-start"] circle[opacity="0.14"]')).toBeTruthy(); + }); + + it('blends rails and links additively in the dark theme and restores normal paint', () => { + renderWith(); + expect(recorded.composites).toEqual(['lighter', 'source-over']); + }); + }); + + describe('causal links', () => { + it('routes links orthogonally', () => { + renderWith(); + expect(recorded.calls).toContain('arcTo'); + expect(recorded.calls).not.toContain('bezierCurveTo'); + }); + + it('tags every non-EXACT link with its grade', () => { + const { container } = renderWith(); + expect([...container.querySelectorAll('[data-link-grade]')].map((tag) => tag.textContent)).toEqual(['AMBIGUOUS']); + }); + + it('tallies every drawn link grade in the ruler, joins apart from forks', () => { + const base = model(); + const withJoin = model({ + paths: [...base.paths, { id: 'p-join', kind: 'rejoin', fromId: CHILD, toId: ROOT, grade: 'inferred', basis: 'child recorded end inside the parent', focus: 'neutral', controls: [672, 130, 691, 130, 685, 80, 704, 80], weight: null }], + }); + const { container } = renderWith({ model: withJoin }); + expect(container.querySelector('[data-link-tally]')?.textContent).toBe('1 FORKS AMBIGUOUS · 1 FORKS EXACT · 1 JOINS INFERRED'); + }); + + it('tags an EXACT link once it is on the lifted chain', () => { + const { container } = renderWith({ model: withFocus({ [BUNDLE]: 'selected' }, { 'p-spawn-exact': 'selected' }) }); + expect([...container.querySelectorAll('[data-link-grade]')].map((tag) => tag.textContent)).toEqual(['AMBIGUOUS', 'EXACT']); + }); + }); + + describe('loaded-page markers', () => { + it('marks NOW at the newest loaded record, not at the window edge', () => { + const { container } = renderWith(); + const tail = container.querySelector('[data-tail-marker]')!; + expect(tail.getAttribute('data-tail-x')).toBe('749'); + expect(tail.querySelector('title')?.textContent).toContain('NOW = newest record in this loaded page'); + expect(tail.querySelector('title')?.textContent).toContain('not a live stream'); + }); + + it('points past the window when NOW lies after it', () => { + const { container } = renderWith({ density: density({ tailX: null, tailTime: WINDOW.end + 600 }) }); + expect(container.querySelector('[data-tail-marker]')?.getAttribute('data-tail-x')).toBe('later'); + expect(screen.getByText('NOW →')).toBeTruthy(); + }); + + it('keeps a recorded-order cursor inside its own lane', () => { + const { container } = renderWith({ model: model({ cursor: { x: 520, laneId: CHILD, xBasis: 'sequence' } }) }); + const cursor = container.querySelector('[data-cursor]')!; + expect(cursor.getAttribute('y1')).toBe('106'); + expect(cursor.getAttribute('y2')).toBe('154'); + expect(screen.getByText('CURSOR · RECORDED ORDER')).toBeTruthy(); + }); + + it('spans the whole field with a dated cursor', () => { + const { container } = renderWith({ model: model({ cursor: { x: 520, laneId: ROOT, xBasis: 'time' } }) }); + expect(container.querySelector('[data-cursor]')?.getAttribute('y2')).toBe('240'); + }); + + it('engraves the provider rail legend in the gutter', () => { + const { container } = renderWith(); + expect(container.querySelector('[data-rail-legend="cursor"] text')?.textContent).toBe('CURSOR · 2'); + }); + }); + + describe('lane column', () => { + const laneTitle = (container: HTMLElement, laneId: string) => + container.querySelector(`[data-lane-row='${laneId}'] title`)?.textContent; + + it('prints reconciled totals: root plus delegated sessions, and what was not measured', () => { + const { container } = renderWith(); + expect(laneTitle(container, BUNDLE)).toBe('codex fan-out · 1+3 sess · 91 msg · peak 2 · 1 open'); + expect(laneTitle(container, ROOT)).toBe('root session · cursor · 12 msg · 1 commit'); + }); + + it('marks persistent selection with a 2px cyan gutter', () => { + const { container } = renderWith({ model: withFocus({ [CHILD]: 'selected' }) }); + const gutter = container.querySelector('[data-selected-gutter]')!; + expect(gutter.getAttribute('y')).toBe('106'); + expect(gutter.getAttribute('width')).toBe('2'); + }); + }); +}); diff --git a/dashboard/src/viz/temporal/types.ts b/dashboard/src/viz/temporal/types.ts index f5fb64d3c8..d3d35d5e04 100644 --- a/dashboard/src/viz/temporal/types.ts +++ b/dashboard/src/viz/temporal/types.ts @@ -81,7 +81,8 @@ export type JourneyEventKind = | 'message_other' | 'tool_call' | 'spawn' - | 'commit'; + | 'commit' + | 'file_edit'; export const JOURNEY_EVENT_KINDS: readonly JourneyEventKind[] = [ 'session_start', @@ -92,6 +93,7 @@ export const JOURNEY_EVENT_KINDS: readonly JourneyEventKind[] = [ 'tool_call', 'spawn', 'commit', + 'file_edit', ]; /** One drawable record on a lane. */ @@ -113,6 +115,8 @@ export interface JourneyEvent { readonly detail: string | null; /** The source record's own identifier: message id, commit SHA, child lane id. */ readonly ref: string; + /** For a file edit, the loaded tool-call event recorded in the same second. */ + readonly linkedEventId?: string; } export type JourneyRelationKind = 'spawn' | 'handoff' | 'rejoin' | 'result'; @@ -128,14 +132,19 @@ export interface JourneyRelation { readonly grade: EvidenceGrade; /** The stated basis, e.g. `parent_session_id · parent_tool_use_id toolu_01`. */ readonly basis: string; + /** The parent-lane event the relation leaves from: the loaded tool call + * whose tool-use id the child recorded. */ + readonly fromEventId?: string; } export type JourneyGapKind = | 'parent_outside_page' | 'parent_cycle' + | 'parentage_conflict' | 'parentage_unavailable' | 'extent_unknown' | 'undated_events' + | 'edit_time_unrecorded' | 'handoff_unavailable'; /** Something Loom cannot prove, kept spatially visible and selectable. */ @@ -315,7 +324,8 @@ export type ScenePathKind = | 'handoff' | 'rejoin' | 'result' - | 'sequence'; + | 'sequence' + | 'edit_link'; /** A curve. `controls` is `[x0,y0,cx0,cy0,cx1,cy1,x1,y1]` for a cubic, or * `[x0,y0,x1,y1]` for a straight segment. */ @@ -459,6 +469,7 @@ export interface TemporalSceneModel { readonly minimap: SceneMinimap; readonly cursor: SceneCursor | null; readonly counts: SceneCounts; - /** True when the dense threshold made roots start collapsed. */ - readonly denseDefault: boolean; + /** On a dense page, the hierarchy depth whose branches start bundled; + * null when the page is under the dense threshold. */ + readonly denseDepth: number | null; } diff --git a/dashboard/src/viz/trace/PlateField.tsx b/dashboard/src/viz/trace/PlateField.tsx new file mode 100644 index 0000000000..1d20c150c1 --- /dev/null +++ b/dashboard/src/viz/trace/PlateField.tsx @@ -0,0 +1,375 @@ +/** + * The TRACE anatomy plate. Geometry comes from `plate.ts`; this file draws it + * and wires the inspect state. Nothing here decides a number. + * + * Connectors are faint hairlines at one fixed opacity. Where a corridor + * carries several, they overlap exactly, so a trunk's brightness is the + * compositing of its links, a count, not a styling choice. The inspected + * route is redrawn on top in cyan and every other connector steps back. + */ +import { useMemo } from 'react'; + +import { kindColorVars } from '../graph/kindColor.ts'; +import { cn } from '../../ui/cn'; +import type { NeighborsPayload, UndrawnNeighbour } from './model.ts'; +import { + kindShape, + layoutPlate, + PLATE_FIELD_H, + PLATE_HEAD_H, + PLATE_ROW, + type PlateFocusMeta, + type PlateLayout, +} from './plate.ts'; +import type { TraceModel } from './types.ts'; +import { clip, clipStart, plateDescription } from './inspect.ts'; +import { + DIM, + InspectReadout, + KIND_FILL, + KindGlyph, + LegendEntry, + SymbolTarget, + useHostWidth, + useInspect, + type Inspect, + type PinnedSymbol, +} from './fieldKit.tsx'; + +const SEGMENT_GAP = 1.5; + +/** Tick step for the call-site ruler: the smallest round step at least 28px apart. */ +function niceStep(pxPerCall: number): number { + for (const step of [1, 2, 5, 10, 20, 50, 100]) if (step * pxPerCall >= 28) return step; + return 200; +} + +/** One connector's opacity at rest; overlapping ones composite into a trunk. */ +const CONNECTOR_ALPHA = 0.2; + +function Ruler({ layout, x0, x1, grow }: { layout: PlateLayout; x0: number; x1: number; grow: 1 | -1 }) { + const { pxPerCall, maxCalls, y } = layout.scale; + const anchor = grow === 1 ? x0 : x1; + const step = niceStep(pxPerCall); + const ticks: number[] = []; + for (let n = 0; n <= maxCalls; n += step) ticks.push(n); + return ( + + + {ticks.map((n) => ( + + + + {n} + + + ))} + + ); +} + +function Plate({ + layout, + model, + inspect, +}: { + layout: PlateLayout; + model: TraceModel; + inspect: Inspect; +}) { + const focus = model.nodes.find((node) => node.id === model.focusId)!; + const { x, y, width, height } = layout.plate; + const chars = Math.floor((width - 28) / 6.6); + return ( + + + + {/* The selection gutter: this plate IS the selected symbol. */} + + + {clip(focus.name, Math.floor((width - 28) / 9))} + + + {/* Selection said in words and position, never by hue alone. */} + + SELECTED + + + {focus.kind.toUpperCase()} + + + {layout.fields.map((field, i) => { + const fy = y + PLATE_HEAD_H + 14 + i * PLATE_FIELD_H; + const value = field.label === 'file' ? clipStart(field.value, chars) : clip(field.value, chars); + return ( + + + {field.label.toUpperCase()} + + + {value} + + + ); + })} + + ); +} + +export function PlateField({ + model, + root, + meta, + undrawn, + reduced, + onPin, +}: { + model: TraceModel; + root: NeighborsPayload; + meta: PlateFocusMeta; + undrawn: readonly UndrawnNeighbour[]; + reduced: boolean; + onPin?: ((node: PinnedSymbol) => void) | undefined; +}) { + const { ref, width } = useHostWidth(); + const inspect = useInspect(model); + const { signature, endLine } = meta; + const layout = useMemo( + () => (width > 0 ? layoutPlate(model, root, { signature, endLine }, undrawn, width) : null), + [model, root, signature, endLine, undrawn, width], + ); + const fade = reduced ? '' : 'motion-safe:transition-opacity motion-safe:duration-150'; + + return ( +
    +
    + {layout === null ? null : ( + + {layout.columns.map((column) => ( + + + {column.title.toUpperCase()} + + {column.notes.map((note) => ( + + {note.text} + + ))} + + ))} + + + {layout.connectors.map((connector) => ( + + ))} + {inspect.id === null + ? null + : layout.connectors + .filter((connector) => inspect.litChannel(connector.key)) + .map((connector) => ( + + ))} + + + + {layout.throughPorts.map((port) => ( + + ))} + + {layout.rows.map((row) => { + const textX = row.anchorX; + const anchor = row.grow === -1 ? 'end' : 'start'; + let cursor = row.anchorX; + return ( + + + + + {row.name} + + {row.segments.map((calls, i) => { + const length = calls * layout.scale.pxPerCall; + const x = row.grow === 1 ? cursor : cursor - length; + cursor += row.grow * (length + SEGMENT_GAP); + return ( + + ); + })} + + {row.segments.length === 0 ? ( + no drawn channel inward · + ) : ( + {`${row.segments.join('+')} · `} + )} + {row.meta} + + + + ); + })} + + {layout.columns + .filter((column) => column.hop === 1 || layout.stacked) + .slice(0, layout.stacked ? 1 : 2) + .map((column) => ( + + ))} + + ONE SCALE · CALL SITES PER CHANNEL + + {layout.crossLinks > 0 ? ( + + {`${layout.crossLinks} of ${model.channels.length} links run between neighbours: connector only, no bar`} + + ) : null} + + )} +
    + +
    + } + /> + + + + + } + /> + + + + + } + /> + + + + + + } + /> + } + /> + } + /> +
    +
    + ); +} diff --git a/dashboard/src/viz/trace/fieldKit.tsx b/dashboard/src/viz/trace/fieldKit.tsx new file mode 100644 index 0000000000..8377c02940 --- /dev/null +++ b/dashboard/src/viz/trace/fieldKit.tsx @@ -0,0 +1,221 @@ +/** + * The Trace plate's DOM interaction: the measured width it lays out against, + * the inspect state a hover or keyboard focus sets, and the focusable symbol + * target. + * + * Hover and keyboard focus both INSPECT (light the symbol's drawn route to the + * focus); only click or Enter re-centres the trace, which is the same + * production path the list's re-centre control takes. The 2px cyan focus mark + * is drawn only for keyboard focus, so a pointer never fakes it. + */ +import { + useCallback, + useEffect, + useRef, + useState, + type CSSProperties, + type KeyboardEvent, + type ReactNode, +} from 'react'; + +import type { KindShape } from './plate.ts'; +import type { TraceModel, TraceNode } from './types.ts'; +import { inspectLine, inspectPath, type InspectedPath } from './inspect.ts'; + +/** Kind hue as an SVG fill or stroke, through the same vars the list uses. */ +export const KIND_FILL = 'fill-[var(--kind-dark)] [[data-theme=light]_&]:fill-[var(--kind-light)]'; + +/** The pin target a renderer hands back: what the trace knows of a symbol. */ +export interface PinnedSymbol { + id: string; + kind: string; + name: string; + file_path: string | null; + start_line: number | null; + degree: number | null; +} + +function pinnedFrom(node: TraceNode): PinnedSymbol { + return { + id: node.id, + kind: node.kind, + name: node.name, + file_path: node.filePath, + start_line: node.startLine, + degree: node.degree, + }; +} + +export interface Inspect { + /** The symbol being inspected, by hover first, then keyboard focus. */ + readonly id: string | null; + /** The symbol holding keyboard focus with a visible focus ring. */ + readonly focusVisible: string | null; + readonly path: InspectedPath; + /** Whether a mark is on the inspected route (true when nothing is inspected). */ + lit(id: string): boolean; + litChannel(key: string): boolean; + hover(id: string | null): void; + focus(id: string | null, visible: boolean): void; +} + +export function useInspect(model: TraceModel): Inspect { + const [hovered, setHovered] = useState(null); + const [focused, setFocused] = useState<{ id: string; visible: boolean } | null>(null); + const id = hovered ?? focused?.id ?? null; + const path = inspectPath(model, id); + return { + id, + focusVisible: focused?.visible ? focused.id : null, + path, + lit: (node) => id === null || path.nodes.has(node), + litChannel: (key) => id === null || path.channels.has(key), + hover: setHovered, + focus: (next, visible) => setFocused(next === null ? null : { id: next, visible }), + }; +} + +/** A focusable symbol mark. `box` is the focus ring, in the SVG's own units. */ +export function SymbolTarget({ + model, + node, + inspect, + onPin, + box, + children, +}: { + model: TraceModel; + node: TraceNode; + inspect: Inspect; + onPin?: ((node: PinnedSymbol) => void) | undefined; + box: { x: number; y: number; width: number; height: number }; + children: ReactNode; +}) { + const pin = useCallback(() => { + if (node.id !== model.focusId) onPin?.(pinnedFrom(node)); + }, [model.focusId, node, onPin]); + const onKeyDown = (event: KeyboardEvent) => { + if (event.key === 'Enter' || event.key === ' ') { + event.preventDefault(); + pin(); + } + }; + return ( + inspect.hover(node.id)} + onPointerLeave={() => inspect.hover(null)} + onFocus={(event) => inspect.focus(node.id, event.currentTarget.matches(':focus-visible'))} + onBlur={() => inspect.focus(null, false)} + onClick={pin} + onKeyDown={onKeyDown} + > + {/* Transparent hit area, so the whole labelled mark answers the pointer. */} + + {children} + {inspect.focusVisible === node.id ? ( + + ) : null} + + ); +} + +/** The one line under a candidate field that says what is being inspected. */ +export function InspectReadout({ model, inspect }: { model: TraceModel; inspect: Inspect }) { + return ( +

    + {inspect.id === null ? ( + 'Hover or focus a symbol to light its drawn route to the focus. Click or Enter re-centres the trace on it.' + ) : ( + {inspectLine(model, inspect.id)} + )} +

    + ); +} + +/** A kind's shape cue, centred on (x, y); the hue comes from the caller. */ +export function KindGlyph({ + shape, + x, + y, + size = 7, + className, + style, +}: { + shape: KindShape; + x: number; + y: number; + size?: number; + className?: string; + style?: CSSProperties; +}) { + const h = size / 2; + const common = { className, style }; + switch (shape) { + case 'circle': + return ; + case 'square': + return ; + case 'diamond': + return ; + case 'triangle': + return ; + case 'bar': + return ; + default: { + const exhaustive: never = shape; + return exhaustive; + } + } +} + +/** A legend entry: a small drawn sample and what it encodes. */ +export function LegendEntry({ sample, label }: { sample: ReactNode; label: string }) { + return ( +
    + + {sample} + + {label} +
    + ); +} + +/** Opacity for a mark off the inspected route. */ +export const DIM = 0.18; + +/** A host ref and its width in CSS px; 0 until laid out. */ +export function useHostWidth() { + const ref = useRef(null); + const [width, setWidth] = useState(0); + useEffect(() => { + const host = ref.current; + if (!host) return; + const measure = () => setWidth(Math.round(host.getBoundingClientRect().width)); + measure(); + if (typeof ResizeObserver !== 'function') return; + const observer = new ResizeObserver(measure); + observer.observe(host); + return () => observer.disconnect(); + }, []); + return { ref, width }; +} diff --git a/dashboard/src/viz/trace/inspect.ts b/dashboard/src/viz/trace/inspect.ts new file mode 100644 index 0000000000..8418401fc3 --- /dev/null +++ b/dashboard/src/viz/trace/inspect.ts @@ -0,0 +1,113 @@ +/** + * Geometry-free rules the Trace plate and its interaction share: channel + * identity, the route a hover or keyboard focus lights, and the one line the + * inspect readout prints. Nothing here reads a payload. + */ +import { coverageCaption } from './model.ts'; +import type { TraceChannel, TraceModel } from './types.ts'; + +export function channelKey(channel: TraceChannel): string { + return `${channel.a}\0${channel.b}`; +} + +/** The drawn channel between two symbols, in either direction. */ +export function channelBetween( + model: TraceModel, + a: string, + b: string, +): TraceChannel | undefined { + return model.channels.find( + (channel) => (channel.a === a && channel.b === b) || (channel.a === b && channel.b === a), + ); +} + +export interface InspectedPath { + readonly nodes: ReadonlySet; + readonly channels: ReadonlySet; +} + +/** + * What a hover or keyboard focus on `id` lights: every drawn route from it to + * the focus that steps strictly inward one hop at a time, plus the channels + * incident on it. Only drawn channels are walked, so a lit path is a path the + * reader can see. + */ +export function inspectPath(model: TraceModel, id: string | null): InspectedPath { + const nodes = new Set(); + const channels = new Set(); + if (id === null) return { nodes, channels }; + const ringOf = new Map(model.nodes.map((node) => [node.id, Math.abs(node.ring)])); + if (!ringOf.has(id)) return { nodes, channels }; + nodes.add(id); + for (const channel of model.channels) { + if (channel.a === id || channel.b === id) { + channels.add(channelKey(channel)); + nodes.add(channel.a === id ? channel.b : channel.a); + } + } + let frontier = [id]; + while (frontier.length > 0) { + const next: string[] = []; + for (const current of frontier) { + const hop = ringOf.get(current)!; + for (const channel of model.channels) { + const other = channel.a === current ? channel.b : channel.b === current ? channel.a : null; + if (other === null || ringOf.get(other) !== hop - 1) continue; + channels.add(channelKey(channel)); + if (!next.includes(other)) next.push(other); + nodes.add(other); + } + } + frontier = next; + } + return { nodes, channels }; +} + +/** One line the inspect readout prints for a symbol; `absent` is printed. */ +export function inspectLine(model: TraceModel, id: string): string { + const node = model.nodes.find((candidate) => candidate.id === id); + if (!node) return ''; + const hop = + node.ring === 0 + ? 'focus' + : `${Math.abs(node.ring)} ${Math.abs(node.ring) === 1 ? 'hop' : 'hops'} ${node.ring < 0 ? 'up (caller side)' : 'down (callee side)'}`; + const drawn = model.channels + .filter((channel) => channel.a === id || channel.b === id) + .reduce((sum, channel) => sum + channel.calls, 0); + const place = + node.filePath === null + ? 'file absent' + : `${node.filePath}${node.startLine === null ? '' : `:${node.startLine}`}`; + return [ + node.name, + node.kind, + hop, + `${drawn} drawn call ${drawn === 1 ? 'site' : 'sites'}`, + node.degree === null ? 'degree absent' : `degree ${node.degree}`, + node.undrawnEdges === null ? 'undrawn edges absent' : `${node.undrawnEdges} edges not drawn`, + place, + ].join(' · '); +} + +/** The plate's accessible description: what is drawn, and the coverage caption. */ +export function plateDescription(model: TraceModel): string { + const focus = model.nodes.find((node) => node.id === model.focusId); + const up = model.nodes.filter((node) => node.ring < 0).length; + const down = model.nodes.filter((node) => node.ring > 0).length; + const sites = model.channels.reduce((sum, channel) => sum + channel.calls, 0); + return ( + `Call neighbourhood of ${focus?.name ?? model.focusId} as an anatomy plate, callers left and callees right on one call-site scale. ` + + `${up} calling and ${down} called symbols, joined by ${model.channels.length} channels carrying ${sites} call sites. ` + + `${coverageCaption(model)}. Each symbol is a focusable control; the ranked list below carries the same symbols as text.` + ); +} + +/** Truncate from the end with an ellipsis, for a fixed label budget. */ +export function clip(text: string, max: number): string { + return text.length <= max ? text : `${text.slice(0, Math.max(1, max - 1))}…`; +} + +/** Truncate from the start, for paths whose tail is the identifying part. */ +export function clipStart(text: string, max: number): string { + return text.length <= max ? text : `…${text.slice(text.length - Math.max(1, max - 1))}`; +} diff --git a/dashboard/src/viz/trace/model.test.ts b/dashboard/src/viz/trace/model.test.ts index 8c9e3c8383..c1e1ea0be0 100644 --- a/dashboard/src/viz/trace/model.test.ts +++ b/dashboard/src/viz/trace/model.test.ts @@ -14,8 +14,7 @@ import { DashboardEnvelopeV1Schema, GraphNeighborsPayloadV1Schema, } from '../../contracts/generated.ts'; -import { TRACE_BUDGET, buildSimSpec, buildTraceModel, type NeighborsPayload } from './model.ts'; -import { ringLabel } from './render.ts'; +import { TRACE_BUDGET, buildTraceModel, type NeighborsPayload } from './model.ts'; function neighbors(id: string): NeighborsPayload { return DashboardEnvelopeV1Schema(GraphNeighborsPayloadV1Schema).parse( @@ -61,12 +60,9 @@ describe('buildTraceModel', () => { // that gives "first discovery wins" its meaning has broken. if (ring === 2) expect(hop1.has(node.id)).toBe(false); } - // Every drawn ring has a row, and the rows read top-to-bottom by ring. - const ys = [...built.rows].sort((a, b) => a[0] - b[0]).map(([, y]) => y); - expect(ys).toEqual([...ys].sort((a, b) => a - b)); - expect(ringLabel(-2)).toBe('2 hops up'); - expect(ringLabel(1)).toBe('1 hop down'); - expect(ringLabel(0)).toBe('focus'); + // Nodes read nearest hop first, the order the plate and list both use. + const hops = built.nodes.map((node) => Math.abs(node.ring)); + expect(hops).toEqual([...hops].sort((a, b) => a - b)); }); it('draws only channels whose BOTH ends are drawn, and counts the rest', () => { @@ -85,15 +81,18 @@ describe('buildTraceModel', () => { }); it('reports recursion as a self-call rather than as a channel', () => { - // A self-loop spring has no second body, so a naive builder either crashes - // or drops the row. Neither is acceptable: recursion is measured. + // A self-loop couples no two symbols, so a naive builder either draws a + // degenerate channel or drops the row. Neither is acceptable. const built = model(); const recursive = built.nodes.filter((node) => node.selfCalls > 0); - expect(recursive.length).toBeGreaterThan(0); + expect(recursive.map((node) => [node.id, node.selfCalls])).toEqual([ + ['sym-24', 4], + ['sym-19', 2], + ['sym-8', 7], + ]); for (const node of recursive) { expect(built.channels.some((c) => c.a === node.id && c.b === node.id)).toBe(false); } - expect(buildSimSpec(built).springs.every((s) => s.a !== s.b)).toBe(true); }); it('derives membranes from contains rows only, and says so when there are none', () => { @@ -150,18 +149,12 @@ describe('buildTraceModel', () => { expect(built.nodes.some((node) => (node.undrawnEdges ?? 0) > 0)).toBe(true); }); - it('keeps the field inside the drawing budget and inside the world box', () => { + it('keeps the field inside the drawing budget', () => { const built = model(); expect(built.nodes.length).toBeLessThanOrEqual( 1 + TRACE_BUDGET.hop1PerSide * 2 + TRACE_BUDGET.hop2PerSide * 2, ); - for (const node of built.nodes) { - expect(node.x0).toBeGreaterThanOrEqual(0); - expect(node.x0).toBeLessThanOrEqual(built.world.width); - expect(node.y0).toBeGreaterThanOrEqual(0); - expect(node.y0).toBeLessThanOrEqual(built.world.height); - } - // Node ids are unique: a duplicate would crash the simulation at build. + // Node ids are unique: a duplicate would draw one symbol twice. expect(new Set(built.nodes.map((n) => n.id)).size).toBe(built.nodes.length); }); diff --git a/dashboard/src/viz/trace/model.ts b/dashboard/src/viz/trace/model.ts index 612bcd5439..a3a1e7d7ae 100644 --- a/dashboard/src/viz/trace/model.ts +++ b/dashboard/src/viz/trace/model.ts @@ -1,5 +1,5 @@ /** - * Measurement → layout for the TRACE surface. + * Measurement → model for the TRACE surface. * * This module turns what `GET /api/plugins/graph/node/{id}/neighbors` actually * returns into a `TraceModel`. It is pure, DOM-free and deterministic, and it @@ -13,16 +13,14 @@ * - `callers` / `callees` are `calls` edges ONLY, one ROW PER EDGE. A caller * with three call sites appears three times with different `edge_line`, so * the call-site count of a pair is the number of its rows. That count is the - * channel's width AND its spring stiffness, the drawn channel and the felt - * channel are the same number by construction. + * length of the channel's bar on the plate's one call-site scale. * - `degree` is the node's total (in + out) edge count over ALL edge kinds. - * Subtracting the call sites this frame draws gives the edges it does not, - * which is what a dashed mouth reports. + * Subtracting the call sites this frame draws gives the edges it does not. * - `edges` carries every edge kind incident on the focus, including * `contains`. Membranes are derived from those rows and from nothing else, * no shared-file-path guessing. When the payload carries no `contains` rows, - * `coverage.membranesAvailable` is false, the field draws no enclosures, and - * the caption says the wire did not carry them. + * `coverage.membranesAvailable` is false and the readout says the wire did + * not carry them. * - Both lists are truncated at `limit` (max 200). A list that comes back * exactly at `limit` is a prefix, and `coverage.capped` records it. * @@ -36,7 +34,6 @@ import type { GraphNodeV1, } from '../../contracts/generated.ts'; import type { - SensoryChannel, TraceChannel, TraceChannelDirection, TraceCoverage, @@ -44,26 +41,6 @@ import type { TraceModel, TraceNode, } from './types.ts'; -import type { SimSpec } from './sim.ts'; - -/** - * The world layout anchors live in. - * - * Narrower than the static sheet's 1440x1160, and deliberately: the drill-in - * occupies the workspace's list column, not a full-bleed page, so a 1440-wide - * world was being scaled to roughly half size and every label with it. Sizing - * the world near the column's real width keeps the scale factor close to 1, - * which is what makes the type legible without the renderer having to fight its - * own transform. - */ -export const TRACE_WORLD = Object.freeze({ width: 1200, height: 1040 }); - -/** Vertical band the hop rings are spread across. */ -const ROW_TOP = 128; -const ROW_BOTTOM = 940; -/** Horizontal band nodes are placed in, leaving room for ring labels. */ -const COL_LEFT = 172; -const COL_RIGHT = 1092; /** * Drawing budget. The plan caps a readable subgraph at 80–250 nodes; this @@ -140,20 +117,6 @@ function callSites(list: NeighborRow[]): Map(items: readonly T[]): T[] { - const out: T[] = []; - items.forEach((item, i) => { - if (i % 2 === 0) out.push(item); - else out.unshift(item); - }); - return out; -} - /* ---- the build ---------------------------------------------------------- */ interface Draft { @@ -191,11 +154,6 @@ export function buildTraceModel(input: TraceModelInput): TraceModel { const pairCalls = new Map(); /** Every symbol any fetched list named, drawn or not. */ const named = new Set([focus.id]); - /** - * Field names actually observed on neighbour rows. Read, never assumed, - * this is what decides which sensory channels this field may drive. - */ - const rowFields = new Set(); function recordPair(from: string, to: string, calls: number): void { // NUL separator, written as an escape rather than as a raw byte: a literal @@ -212,9 +170,6 @@ export function buildTraceModel(input: TraceModelInput): TraceModel { } function absorb(payload: NeighborsPayload, ownerId: string): void { - for (const row of [...rows(payload.callers), ...rows(payload.callees)]) { - for (const key of Object.keys(row)) rowFields.add(key); - } for (const [id, entry] of callSites(rows(payload.callers))) { named.add(id); recordPair(id, ownerId, entry.calls); @@ -299,9 +254,9 @@ export function buildTraceModel(input: TraceModelInput): TraceModel { const b = drafts.get(to); if (!a || !b) continue; if (from === to) { - // Recursion. A real `calls` row, and a self-loop spring is undefined - // (zero length, no second body), so it is counted on the node and - // printed there rather than drawn as a channel or quietly discarded. + // Recursion. A real `calls` row that couples no two symbols, so it is + // counted on the node and printed there rather than drawn as a channel + // or quietly discarded. selfCallsOn.set(from, (selfCallsOn.get(from) ?? 0) + calls); callSitesOn.set(from, (callSitesOn.get(from) ?? 0) + calls); continue; @@ -338,9 +293,8 @@ export function buildTraceModel(input: TraceModelInput): TraceModel { } } const membranes: TraceMembrane[] = [...byContainer] - // A one-member enclosure is a true `contains` edge but not an enclosure a - // reader can see the flow enter and leave, so it is not drawn as one. It - // remains a counted `contains` edge on its member's mouth. + // A one-member enclosure is a true `contains` edge but encloses nothing + // else on this frame, so it is not counted as a type the calls enter. .filter(([, entry]) => entry.of.length >= 2) .map(([id, entry]) => ({ id, label: entry.label, of: entry.of })) .sort((a, b) => a.id.localeCompare(b.id)); @@ -350,94 +304,27 @@ export function buildTraceModel(input: TraceModelInput): TraceModel { for (const member of membrane.of) membraneOf.set(member, membrane.id); } - /* ---- layout ----------------------------------------------------------- */ - const ringsPresent = [...new Set([...drafts.values()].map((d) => d.ring))].sort((a, b) => a - b); - const rowY = new Map(); - ringsPresent.forEach((ring, i) => { - const span = ringsPresent.length > 1 ? (ROW_BOTTOM - ROW_TOP) / (ringsPresent.length - 1) : 0; - rowY.set(ring, ROW_TOP + span * i); - }); - - // Channel adjacency, weighted by call sites, for the barycentre pass below. - const neighboursOf = new Map>(); - for (const channel of drawnChannels) { - if (!neighboursOf.has(channel.a)) neighboursOf.set(channel.a, []); - if (!neighboursOf.has(channel.b)) neighboursOf.set(channel.b, []); - neighboursOf.get(channel.a)!.push({ other: channel.b, calls: channel.calls }); - neighboursOf.get(channel.b)!.push({ other: channel.a, calls: channel.calls }); - } - - const nodes: TraceNode[] = []; - const placedX = new Map(); - const centre = (COL_LEFT + COL_RIGHT) / 2; - - /** - * Rings are laid out from the focus outward, and each ring is ordered by the - * call-site-weighted mean x of the neighbours already placed on the ring - * inside it, a one-pass barycentre ordering. - * - * Without it, ordering a ring by raw strength puts a symbol nowhere near the - * symbols it actually calls, and every channel has to cross the field to - * reach its partner. The resulting picture is a legible watershed instead of - * a hairball, and no measurement is touched: barycentre decides only which of - * several equally-valid x slots a node occupies within the row its HOP - * DISTANCE already assigned it. - */ - const byDistance = [...ringsPresent].sort((a, b) => Math.abs(a) - Math.abs(b) || a - b); - for (const ring of byDistance) { - const inRing = [...drafts.values()].filter((d) => d.ring === ring); - const keyOf = (id: string): number => { - let weight = 0; - let sum = 0; - for (const { other, calls } of neighboursOf.get(id) ?? []) { - const x = placedX.get(other); - if (x === undefined) continue; - weight += calls; - sum += x * calls; - } - return weight > 0 ? sum / weight : centre; - }; - const keys = new Map(inRing.map((draft) => [draft.id, keyOf(draft.id)])); - // Membrane siblings share their group's mean key, so a type's members stay - // adjacent and its enclosure is a compact box rather than a band spanning - // the whole field. - for (const membrane of membranes) { - const members = membrane.of.filter((id) => keys.has(id)); - if (members.length < 2) continue; - const mean = members.reduce((sum, id) => sum + keys.get(id)!, 0) / members.length; - for (const id of members) keys.set(id, mean); - } - const ordered = inRing.sort( + /* ---- nodes: nearest hop first, then call sites, then id ------------- */ + const nodes: TraceNode[] = [...drafts.values()] + .sort( (a, b) => - keys.get(a.id)! - keys.get(b.id)! || + Math.abs(a.ring) - Math.abs(b.ring) || + a.ring - b.ring || (callSitesOn.get(b.id) ?? 0) - (callSitesOn.get(a.id) ?? 0) || a.id.localeCompare(b.id), - ); - // The focus is the one node whose slot is not negotiable: it is the basin - // the whole field drains toward, so it sits dead centre. - const placed = ring === 0 ? centreOut(ordered) : ordered; - const y = rowY.get(ring)!; - placed.forEach((draft, i) => { - const span = placed.length > 1 ? (COL_RIGHT - COL_LEFT) / (placed.length - 1) : 0; - const x = placed.length > 1 ? COL_LEFT + span * i : centre; - const finalX = draft.id === focus.id ? centre : x; - placedX.set(draft.id, finalX); - const drawn = callSitesOn.get(draft.id) ?? 0; - nodes.push({ - id: draft.id, - name: draft.name, - kind: draft.kind, - degree: draft.degree, - filePath: draft.filePath, - startLine: draft.startLine, - ring: draft.ring, - x0: finalX, - y0: y, - undrawnEdges: draft.degree == null ? null : Math.max(0, draft.degree - drawn), - selfCalls: selfCallsOn.get(draft.id) ?? 0, - }); - }); - } + ) + .map((draft) => ({ + id: draft.id, + name: draft.name, + kind: draft.kind, + degree: draft.degree, + filePath: draft.filePath, + startLine: draft.startLine, + ring: draft.ring, + undrawnEdges: + draft.degree == null ? null : Math.max(0, draft.degree - (callSitesOn.get(draft.id) ?? 0)), + selfCalls: selfCallsOn.get(draft.id) ?? 0, + })); /* ---- channel direction refinement: same membrane is a lateral move ---- */ const channels: TraceChannel[] = drawnChannels.map((channel) => { @@ -466,13 +353,10 @@ export function buildTraceModel(input: TraceModelInput): TraceModel { cappedAt: limit, capped, membranesAvailable: containsSeen > 0 || containsRows.length > 0, - rowFields: [...rowFields].sort(), }; return { focusId: focus.id, - world: TRACE_WORLD, - rows: rowY, nodes, channels, membranes, @@ -491,111 +375,6 @@ function directionOf(ringA: number, ringB: number): TraceChannelDirection { return outer < 0 ? 'up' : 'down'; } -/** - * Field names that would carry each unbound sensory measurement. - * - * These are candidate names, matched against `coverage.rowFields`, the fields - * the payload actually delivered. The point of matching rather than asserting is - * that a producer which starts serving one of these makes the channel go live - * on its own; nothing here has to be re-edited, and the surface cannot end up - * understating coverage it has been given. Matching is on the field's presence, - * not on a host or provider name, because a capability is a property of the - * response and not of who produced it. - */ -export const SENSORY_FIELD_CANDIDATES = Object.freeze({ - /** Cyclomatic complexity, for the texture/grain channel. */ - complexity: Object.freeze([ - 'complexity', - 'cyclomatic', - 'cyclomatic_complexity', - ] as const), - /** Churn recency, for the warmth channel. */ - churn: Object.freeze([ - 'churn', - 'churn_recency', - 'last_modified', - 'last_modified_at', - 'last_commit_at', - ] as const), - /** Symbol- or path-scoped live activity, for the pulse channel. */ - activity: Object.freeze([ - 'activity', - 'last_strike_at', - 'activity_path', - ] as const), -}); - -function served(coverage: TraceCoverage, candidates: readonly string[]): string | null { - return candidates.find((name) => coverage.rowFields.includes(name)) ?? null; -} - -/** - * The five sensory channels, each resolved against the payload in hand. - * - * The sensory contract is app-wide and fixed, weight is always connectedness, - * tension is always coupling, but which channels a given field can actually - * DRIVE depends on what arrived. This returns all five either way, so the - * surface can show a channel as inert instead of omitting it, and a reader - * learns the same mapping everywhere even where a measurement is missing. - */ -export function sensoryChannels(model: TraceModel): readonly SensoryChannel[] { - const c = model.coverage; - const anyDegree = model.nodes.some((node) => node.degree != null); - const callSiteTotal = model.channels.reduce((sum, channel) => sum + channel.calls, 0); - const complexityField = served(c, SENSORY_FIELD_CANDIDATES.complexity); - const churnField = served(c, SENSORY_FIELD_CANDIDATES.churn); - const activityField = served(c, SENSORY_FIELD_CANDIDATES.activity); - - return [ - { - feel: 'weight / inertia', - measurement: 'connectedness (degree)', - state: anyDegree ? 'measured' : 'not-on-this-wire', - staticEquivalent: 'sill width', - note: anyDegree - ? 'degree sets each body\'s mass, so hover latency, bloom depth and settle time all scale with it' - : 'no row on this payload carried a degree, so every body is at the mass floor and weight reads nothing', - }, - { - feel: 'tension / deformation', - measurement: 'coupling strength (call sites on one edge)', - state: callSiteTotal > 0 ? 'measured' : 'not-on-this-wire', - staticEquivalent: 'channel thickness', - note: - callSiteTotal > 0 - ? `each channel is a spring stiffened by its own call-site count (${callSiteTotal} across ${model.channels.length} channels), so dragging deforms the neighbourhood in proportion to coupling` - : 'no calls rows arrived, so no channel carries a spring', - }, - { - feel: 'texture / grain', - measurement: 'cyclomatic complexity', - state: complexityField ? 'measured' : 'not-on-this-wire', - staticEquivalent: 'contour tightness', - note: complexityField - ? `driven by the payload's ${complexityField} field` - : 'this route\'s rows carry no complexity field, so the channel is inert, the symbols are not being claimed to be simple', - }, - { - feel: 'warmth', - measurement: 'churn recency', - state: churnField ? 'measured' : 'not-on-this-wire', - staticEquivalent: 'heat tint held at its current value', - note: churnField - ? `driven by the payload's ${churnField} field` - : 'this route\'s rows carry no churn or last-modified field, so nothing is tinted, untinted here means unmeasured, not cold', - }, - { - feel: 'pulse', - measurement: 'live activity', - state: activityField ? 'measured' : 'coarser-scope', - staticEquivalent: 'pinned-lit', - note: activityField - ? `driven by the payload's ${activityField} field` - : 'the live activity stream is project-scoped and carries no path, so no strike can be attributed to a symbol on this field', - }, - ]; -} - /** * Everything the field is NOT showing, counted from rows in hand. * @@ -625,50 +404,52 @@ export function coverageCaption(model: TraceModel): string { } parts.push( c.membranesAvailable - ? `${model.membranes.length} type ${model.membranes.length === 1 ? 'membrane' : 'membranes'} from contains edges` - : 'the payload carried no contains edges, so no type membranes are drawn, this says nothing about whether these symbols have types', + ? `${model.membranes.length} type ${model.membranes.length === 1 ? 'enclosure' : 'enclosures'} from contains edges` + : 'the payload carried no contains edges, which says nothing about whether these symbols have types', ); return parts.join(' · '); } -/** The `role="img"` description. Says what is drawn and what is left out. */ -export function fieldDescription(model: TraceModel): string { - const focus = model.nodes.find((node) => node.id === model.focusId); - const up = model.nodes.filter((node) => node.ring < 0).length; - const down = model.nodes.filter((node) => node.ring > 0).length; - const callSites = model.channels.reduce((sum, channel) => sum + channel.calls, 0); - return ( - `Call topography of ${focus?.name ?? model.focusId}. ` + - `${up} calling symbols are drawn above it as tributaries and ${down} called symbols below it as a delta, ` + - `joined by ${model.channels.length} channels carrying ${callSites} call sites in total. ` + - `${coverageCaption(model)}. ` + - 'The ranked list below carries the same symbols as text.' - ); +/** A symbol a fetched list named that the field does not draw. */ +export interface UndrawnNeighbour { + readonly id: string; + readonly filePath: string | null; + /** 1 when the focus's own list named it, 2 when an expanded neighbour's did. */ + readonly hop: 1 | 2; + /** Side of the drawn symbol whose list named it first. */ + readonly side: 'up' | 'down'; } /** - * Translate the model into the simulation's vocabulary: mass IS degree, - * stiffness IS the call-site count, and the anchor IS the layout position. No - * shaping and no normalisation that would launder the measurement, the - * simulation's own parameters do the scaling, in one place, where they can be - * read off a table. - * - * A node whose `degree` the payload omitted enters at the parameter floor - * (`minMass`), because a body with no inertia is a numerical singularity, not - * an honest zero. Its sill is drawn hollow by the renderer, so absence stays - * visible. + * The symbols behind `coverage.namedButNotDrawn`, with the file each row + * carried, so a renderer can print the omission where it happens instead of + * only as one total. Read from the same payloads `buildTraceModel` absorbed. */ -export function buildSimSpec(model: TraceModel, seed = 20260725): SimSpec { - return { - seed, - nodes: model.nodes.map((node) => ({ - id: node.id, - mass: node.degree ?? 0, - x0: node.x0, - y0: node.y0, - })), - springs: model.channels - .filter((channel) => channel.calls > 0) - .map((channel) => ({ a: channel.a, b: channel.b, stiffness: channel.calls })), +export function undrawnNeighbours( + input: TraceModelInput, + model: TraceModel, +): readonly UndrawnNeighbour[] { + const drawn = new Map(model.nodes.map((node) => [node.id, node.ring])); + const out = new Map(); + const visit = (payload: NeighborsPayload, hop: 1 | 2, ownerRing: number) => { + for (const side of ['callers', 'callees'] as const) { + for (const row of rows(payload[side])) { + if (drawn.has(row.id) || out.has(row.id)) continue; + const up = hop === 1 ? side === 'callers' : ownerRing < 0; + out.set(row.id, { + id: row.id, + filePath: row.file_path ?? null, + hop, + side: up ? 'up' : 'down', + }); + } + } }; + visit(input.root, 1, 0); + for (const [seed, payload] of input.expanded) { + const ring = drawn.get(seed); + if (ring === undefined || Math.abs(ring) !== 1) continue; + visit(payload, 2, ring); + } + return [...out.values()]; } diff --git a/dashboard/src/viz/trace/palette.ts b/dashboard/src/viz/trace/palette.ts deleted file mode 100644 index 7774487356..0000000000 --- a/dashboard/src/viz/trace/palette.ts +++ /dev/null @@ -1,52 +0,0 @@ -/** - * Token sampling for the TRACE canvas. - * - * Canvas2D cannot read CSS custom properties, so the resolved strings have to - * be handed to the renderer. Sampling happens here, once at mount and once per - * theme flip, rather than inside the draw loop, which keeps `tokens.css` the - * single source of the instrument's colour at 60 Hz. - */ -import type { TracePalette } from './types.ts'; - -/** Fallbacks are only reached if a token is missing; they mirror the dark set. */ -const FALLBACK: Record = { - '--raw-surface-0': '#141619', - '--raw-surface-1': '#1c2029', - '--raw-text-primary': '#eef0f4', - '--raw-text-muted': '#9aa1b0', - '--raw-edge-subtle': '#333a46', - '--raw-edge-strong': '#4c5464', - '--raw-grid': '#2a2f38', - '--raw-accent': '#5fd0e0', - '--raw-state-partial': '#e0b45f', - '--raw-state-unknown': '#8d919b', -}; - -export function resolveTracePalette(element: HTMLElement): TracePalette { - const style = getComputedStyle(element); - const token = (name: string): string => - style.getPropertyValue(name).trim() || FALLBACK[name] || '#888888'; - - // Which medium the field is suspended in, measured rather than assumed, so a - // future theme that is neither of the two shipped ones still resolves. - const light = document.documentElement.dataset['theme'] === 'light'; - return { - surface0: token('--raw-surface-0'), - surface1: token('--raw-surface-1'), - textPrimary: token('--raw-text-primary'), - textMuted: token('--raw-text-muted'), - edgeSubtle: token('--raw-edge-subtle'), - edgeStrong: token('--raw-edge-strong'), - grid: token('--raw-grid'), - accent: token('--raw-accent'), - // Upstream and downstream are the accent and the muted ink rather than two - // invented hues: kind colour already owns the hue channel on this field, - // and a third palette competing with it made the ribbons unreadable. - upstream: token('--raw-accent'), - downstream: token('--raw-edge-strong'), - statePartial: token('--raw-state-partial'), - stateUnknown: token('--raw-state-unknown'), - membraneFill: token('--raw-surface-1'), - light, - }; -} diff --git a/dashboard/src/viz/trace/plate.test.ts b/dashboard/src/viz/trace/plate.test.ts new file mode 100644 index 0000000000..2b3861f1bd --- /dev/null +++ b/dashboard/src/viz/trace/plate.test.ts @@ -0,0 +1,190 @@ +/** + * The Trace anatomy plate against the wire-true neighbors fixture. Every + * expectation is a value observed on that fixture, so a layout that starts + * inventing, dropping or pooling a count fails here. + */ +import { describe, expect, it } from 'vitest'; + +import { resolveFixture } from '../../../stories/fixtures/data.ts'; +import { + DashboardEnvelopeV1Schema, + GraphNeighborsPayloadV1Schema, +} from '../../contracts/generated.ts'; +import { + TRACE_BUDGET, + buildTraceModel, + undrawnNeighbours, + type NeighborsPayload, + type TraceModelInput, +} from './model.ts'; +import { elbowPath, kindShape, layoutPlate } from './plate.ts'; +import type { TraceModel } from './types.ts'; +import { channelKey, inspectPath, plateDescription } from './inspect.ts'; + +function neighbors(id: string, limit = 200): NeighborsPayload { + return DashboardEnvelopeV1Schema(GraphNeighborsPayloadV1Schema).parse( + resolveFixture(`/api/plugins/graph/node/${id}/neighbors`, `?limit=${limit}`), + ).payload; +} + +function traceInput(focusId = 'sym-0', limit = 200): TraceModelInput { + const root = neighbors(focusId, limit); + const ids = [...new Set([...(root.callers ?? []), ...(root.callees ?? [])].map((row) => row.id))] + .filter((id) => id !== focusId) + .slice(0, TRACE_BUDGET.expand); + return { + focus: { + id: focusId, + kind: 'function', + name: 'subgraph_payload', + file_path: 'src/dashboard/graph_service.rs', + start_line: 40, + degree: 16, + }, + root, + expanded: new Map(ids.map((id) => [id, neighbors(id, limit)] as const)), + }; +} + +function built(focusId = 'sym-0', limit = 200): { input: TraceModelInput; model: TraceModel } { + const input = traceInput(focusId, limit); + return { input, model: buildTraceModel(input) }; +} + +describe('undrawnNeighbours', () => { + it('itemises exactly the symbols the coverage total counts', () => { + const { input, model } = built(); + const undrawn = undrawnNeighbours(input, model); + expect(model.coverage.namedButNotDrawn).toBe(9); + expect(undrawn).toHaveLength(9); + expect(undrawn.every((entry) => entry.hop === 2 && entry.side === 'up')).toBe(true); + const drawn = new Set(model.nodes.map((node) => node.id)); + expect(undrawn.some((entry) => drawn.has(entry.id))).toBe(false); + }); +}); + +describe('inspectPath', () => { + it('lights a hop-2 symbol back to the focus through the drawn hop-1 route only', () => { + const { model } = built(); + const path = inspectPath(model, 'sym-33'); + const names = [...path.nodes].map((id) => model.nodes.find((node) => node.id === id)!.name); + expect(names).toEqual(['validate_grant', 'search_payload', 'subgraph_payload']); + expect(path.channels.size).toBe(2); + expect(inspectPath(model, null).nodes.size).toBe(0); + expect(inspectPath(model, 'not-drawn').nodes.size).toBe(0); + }); + + it('describes the plate without the spring field vocabulary', () => { + const { model } = built(); + const text = plateDescription(model); + expect(text).toContain( + 'Call neighbourhood of subgraph_payload as an anatomy plate, callers left and callees right on one call-site scale.', + ); + expect(text).toContain('16 calling and 13 called symbols, joined by 90 channels'); + expect(text).not.toMatch(/tributar|delta/); + }); +}); + +describe('layoutPlate', () => { + it('prints only measured plate fields and says absent for the rest', () => { + const { input, model } = built(); + const layout = layoutPlate(model, input.root, { signature: null, endLine: 52 }, undrawnNeighbours(input, model), 926); + const fields = Object.fromEntries(layout.fields.map((field) => [field.label, field.value])); + expect(fields).toEqual({ + file: 'src/dashboard/graph_service.rs', + lines: '40–52', + signature: 'absent', + degree: '16 edges, all kinds', + callers: '7 symbols · 45 sites', + callees: '4 symbols · 21 sites', + 'self calls': '0', + enclosure: 'impl RetrievalService', + 'edges by kind': 'calls 66 · contains 1 · references 1', + drawn: '7 of 7 callers · 4 of 4 callees', + }); + expect(layout.fields.find((field) => field.label === 'signature')!.absent).toBe(true); + }); + + it('holds callers and callees to one call-site scale and counts what it leaves off', () => { + const { input, model } = built(); + const layout = layoutPlate(model, input.root, { signature: null, endLine: null }, undrawnNeighbours(input, model), 926); + expect(layout.stacked).toBe(false); + expect(layout.scale.maxCalls).toBe(22); + // One px-per-call for every bar on both sides. + const lengths = layout.rows.map((row) => row.segments.reduce((a, b) => a + b, 0) * layout.scale.pxPerCall); + expect(Math.max(...lengths)).toBeCloseTo(22 * layout.scale.pxPerCall); + expect(layout.rows).toHaveLength(29); + expect(layout.crossLinks).toBe(52); + // Readouts wrap to their column; the words are all there. + expect(layout.columns.find((c) => c.side === 'up' && c.hop === 2)!.notes.map((n) => n.text).join(' ')).toBe( + '+9 named, not drawn', + ); + const scheduler = layout.rows.find((row) => row.node.name === 'scheduler_tick')!; + expect(scheduler.segments).toEqual([11, 10, 1]); + }); + + it('stacks callers above and callees below in a narrow column', () => { + const { input, model } = built(); + const layout = layoutPlate(model, input.root, { signature: null, endLine: null }, undrawnNeighbours(input, model), 289); + expect(layout.stacked).toBe(true); + // One corridor down the gutter carries every link. + expect(new Set(layout.connectors.flatMap((c) => c.corridors))).toEqual(new Set([12])); + expect(layout.connectors).toHaveLength(model.channels.length); + expect(layout.rows.every((row) => row.grow === 1)).toBe(true); + const order = layout.columns.map((column) => `${column.side}${column.hop}`); + expect(order).toEqual(['up2', 'up1', 'down1', 'down2']); + }); + + it('marks a list that came back at the endpoint limit as a prefix', () => { + const { input, model } = built('sym-0', 6); + const layout = layoutPlate(model, input.root, { signature: null, endLine: null }, undrawnNeighbours(input, model), 926); + expect(layout.fields.find((field) => field.label === 'callers')!.value).toBe( + '1 symbol · 6 sites · prefix', + ); + expect(layout.columns.find((c) => c.side === 'up' && c.hop === 1)!.notes.map((n) => n.text).join(' ')).toBe( + 'a list hit the row limit: prefix only', + ); + }); +}); + +describe('plate connectors', () => { + it('draws every drawn call link, bars or not, as one connector', () => { + const { input, model } = built(); + const layout = layoutPlate(model, input.root, { signature: null, endLine: null }, undrawnNeighbours(input, model), 926); + expect(model.channels).toHaveLength(90); + expect(layout.connectors.map((c) => c.key).sort()).toEqual(model.channels.map(channelKey).sort()); + expect(layout.connectors.every((c) => c.d.startsWith('M'))).toBe(true); + // Both plate edges carry a through port on this neighbourhood. + expect(layout.throughPorts.map((p) => p.x)).toEqual([layout.plate.x, layout.plate.x + layout.plate.width]); + }); + + it('bundles a column into one corridor, so trunks carry counted links', () => { + const { input, model } = built(); + const layout = layoutPlate(model, input.root, { signature: null, endLine: null }, undrawnNeighbours(input, model), 926); + const corridor = new Map(); + for (const c of layout.connectors) for (const x of c.corridors) corridor.set(x, (corridor.get(x) ?? 0) + 1); + // Four corridors between five slots, nothing else. + expect(corridor.size).toBe(4); + const leftOfPlate = Math.round((layout.columns.find((c) => c.side === 'up' && c.hop === 1)!.x1 + layout.plate.x) / 2); + const upToFocus = model.channels.filter( + (c) => c.b === model.focusId && model.nodes.find((n) => n.id === c.a)!.ring === -1, + ); + expect(upToFocus).toHaveLength(7); + for (const channel of upToFocus) { + expect(layout.connectors.find((c) => c.key === channelKey(channel))!.corridors).toEqual([leftOfPlate]); + } + expect([...corridor.values()].reduce((a, b) => a + b, 0)).toBeGreaterThan(model.channels.length); + }); + + it('rounds each elbow softly and never draws a zero-length corner', () => { + expect(elbowPath([[0, 0], [10, 0], [10, 10]])).toBe('M0,0 L5,0 Q10,0 10,5 L10,10'); + expect(elbowPath([[0, 0], [20, 0], [20, 0], [20, 30]])).toBe('M0,0 L14,0 Q20,0 20,6 L20,30'); + expect(elbowPath([[0, 0]])).toBe(''); + }); + + it('gives a named kind its shape and hashes any other kind onto the same five', () => { + expect(kindShape('method')).toBe('diamond'); + expect(kindShape('enum')).toBe('square'); + expect(kindShape('Method')).toBe('bar'); + }); +}); diff --git a/dashboard/src/viz/trace/plate.ts b/dashboard/src/viz/trace/plate.ts new file mode 100644 index 0000000000..d6070fa1b7 --- /dev/null +++ b/dashboard/src/viz/trace/plate.ts @@ -0,0 +1,485 @@ +/** + * Layout for the TRACE anatomy plate (plan 11b Surface 1). + * + * The focus is a machined plate carrying only measured fields, `absent` + * printed where the wire was silent. Callers stand left and callees right as + * bars on ONE call-site scale, so a 10-site caller and a 10-site callee are the + * same length. Hop 2 stands in a second column per side, each bar split into + * one segment per drawn channel into hop 1, and every omission is printed at + * the foot of the column it happened in. + * + * Every drawn call link is also a connector: from its caller's row port to + * its callee's, down the corridor beside each column and, across sides, + * behind the plate. All links in one corridor share one x, so parallel runs + * are one trunk, and a trunk's brightness is the number of links it carries. + * + * Pure and DOM-free: numbers in, geometry out. + */ +import type { NeighborsPayload, UndrawnNeighbour } from './model.ts'; +import type { TraceModel, TraceNode } from './types.ts'; +import { channelBetween, channelKey, clip } from './inspect.ts'; + +export interface PlateFocusMeta { + readonly signature: string | null; + readonly endLine: number | null; +} + +export interface PlateField { + readonly label: string; + readonly value: string; + readonly absent: boolean; +} + +export interface PlateColumn { + readonly side: 'up' | 'down'; + readonly hop: 1 | 2; + readonly x0: number; + readonly x1: number; + readonly title: string; + readonly titleY: number; + /** Titles sit on the plate side for hop 1 and the outer edge for hop 2. */ + readonly titleX: number; + readonly titleAnchor: 'start' | 'end'; + readonly notes: readonly { y: number; text: string }[]; +} + +export interface PlateRow { + readonly node: TraceNode; + readonly column: PlateColumn; + readonly y: number; + /** Call sites per drawn channel into the hop inside, strongest first. */ + readonly segments: readonly number[]; + readonly name: string; + readonly meta: string; + /** x the bar grows from, and the direction it grows in (+1 right, -1 left). */ + readonly anchorX: number; + readonly grow: 1 | -1; +} + +/** One call link, caller row port to callee row port. */ +export interface PlateConnector { + readonly key: string; + readonly d: string; + /** x of each vertical run: the corridors this link shares with others. */ + readonly corridors: readonly number[]; +} + +export type KindShape = 'circle' | 'diamond' | 'square' | 'triangle' | 'bar'; + +export interface PlateLayout { + readonly width: number; + readonly height: number; + readonly stacked: boolean; + readonly plate: { x: number; y: number; width: number; height: number }; + readonly fields: readonly PlateField[]; + readonly columns: readonly PlateColumn[]; + readonly rows: readonly PlateRow[]; + readonly connectors: readonly PlateConnector[]; + /** Where connectors pass behind the plate, one per plate edge in use. */ + readonly throughPorts: readonly { x: number; y: number }[]; + readonly scale: { readonly pxPerCall: number; readonly maxCalls: number; readonly y: number }; + /** Drawn links no bar segment encodes: lateral and cross-side, connector only. */ + readonly crossLinks: number; +} + +/** Below this width the columns stack above and below the plate. */ +export const PLATE_STACK_BELOW = 700; +export const PLATE_ROW = 44; +const MARGIN = 12; +const TITLE_H = 30; +export const PLATE_FIELD_H = 29; +export const PLATE_HEAD_H = 48; +const NOTE_H = 14; +/** Stacked form: the gutter the one shared corridor runs down. */ +const GUTTER = 12; +/** Corner radius of a connector's soft elbow. */ +const ELBOW = 6; +/** Monospace advance at 11px, used to clip names to their column. */ +const CHAR_PX = 6.7; + +function count(n: number, unit: string): string { + return `${n} ${unit}${n === 1 ? '' : 's'}`; +} + +function distinctRows(list: NeighborsPayload['callers']): { distinct: number; sites: number } { + const rows = (list ?? []).filter((row) => row.id.length > 0); + return { distinct: new Set(rows.map((row) => row.id)).size, sites: rows.length }; +} + +export function plateFields( + model: TraceModel, + root: NeighborsPayload, + meta: PlateFocusMeta, +): readonly PlateField[] { + const focus = model.nodes.find((node) => node.id === model.focusId)!; + const limit = typeof root.limit === 'number' ? root.limit : null; + const callers = distinctRows(root.callers); + const callees = distinctRows(root.callees); + const prefix = (sites: number) => (limit !== null && sites >= limit ? ' · prefix' : ''); + const enclosure = (root.edges ?? []).find( + (edge) => edge.kind === 'contains' && edge.target === model.focusId, + ); + const kinds = (root.edges_by_kind ?? []).map((entry) => `${entry.kind} ${entry.count}`); + const drawnUp = model.nodes.filter((node) => node.ring === -1).length; + const drawnDown = model.nodes.filter((node) => node.ring === 1).length; + const field = (label: string, value: string | null): PlateField => ({ + label, + value: value ?? 'absent', + absent: value === null, + }); + return [ + field('file', focus.filePath), + field( + 'lines', + focus.startLine === null + ? null + : `${focus.startLine}–${meta.endLine === null ? 'end absent' : meta.endLine}`, + ), + field('signature', meta.signature), + field('degree', focus.degree === null ? null : `${focus.degree} edges, all kinds`), + field('callers', `${count(callers.distinct, 'symbol')} · ${count(callers.sites, 'site')}${prefix(callers.sites)}`), + field('callees', `${count(callees.distinct, 'symbol')} · ${count(callees.sites, 'site')}${prefix(callees.sites)}`), + field('self calls', String(focus.selfCalls)), + field('enclosure', enclosure ? (enclosure.source_name ?? enclosure.source) : null), + field('edges by kind', kinds.length ? kinds.join(' · ') : null), + field('drawn', `${drawnUp} of ${callers.distinct} callers · ${drawnDown} of ${callees.distinct} callees`), + ]; +} + +function wrap(text: string, width: number): string[] { + const lines: string[] = []; + let line = ''; + for (const word of text.split(' ')) { + if (line && line.length + 1 + word.length > width) { + lines.push(line); + line = word; + } else line = line ? `${line} ${word}` : word; + } + if (line) lines.push(line); + return lines; +} + +const NAMED_SHAPES: Readonly> = { + function: 'circle', + method: 'diamond', + struct: 'square', + class: 'square', + trait: 'triangle', + interface: 'triangle', + module: 'bar', + file: 'bar', +}; +const SHAPES: readonly KindShape[] = ['circle', 'diamond', 'square', 'triangle', 'bar']; + +/** A kind's shape cue, so kind never rests on hue alone. Unknown kinds hash + * onto the same five shapes, stable across reloads. */ +export function kindShape(kind: string): KindShape { + const named = NAMED_SHAPES[kind]; + if (named) return named; + let hash = 0; + for (let i = 0; i < kind.length; i += 1) hash = (hash * 31 + kind.charCodeAt(i)) >>> 0; + return SHAPES[hash % SHAPES.length]!; +} + +/** A polyline as a path with soft elbows: each corner rounded by `ELBOW`. */ +export function elbowPath(points: readonly (readonly [number, number])[]): string { + const f = (n: number) => Math.round(n * 10) / 10; + const pts = points.filter( + (p, i) => i === 0 || p[0] !== points[i - 1]![0] || p[1] !== points[i - 1]![1], + ); + if (pts.length < 2) return ''; + let d = `M${f(pts[0]![0])},${f(pts[0]![1])}`; + for (let i = 1; i < pts.length - 1; i += 1) { + const [px, py] = pts[i - 1]!; + const [x, y] = pts[i]!; + const [nx, ny] = pts[i + 1]!; + const inLen = Math.hypot(x - px, y - py); + const outLen = Math.hypot(nx - x, ny - y); + const r = Math.min(ELBOW, inLen / 2, outLen / 2); + const ax = x - ((x - px) / (inLen || 1)) * r; + const ay = y - ((y - py) / (inLen || 1)) * r; + const bx = x + ((nx - x) / (outLen || 1)) * r; + const by = y + ((ny - y) / (outLen || 1)) * r; + d += ` L${f(ax)},${f(ay)} Q${f(x)},${f(y)} ${f(bx)},${f(by)}`; + } + const last = pts[pts.length - 1]!; + return `${d} L${f(last[0])},${f(last[1])}`; +} + +function columnTitle(side: 'up' | 'down', hop: 1 | 2): string { + if (side === 'up') return hop === 1 ? 'callers · 1 hop' : '2 hops · via callers'; + return hop === 1 ? 'callees · 1 hop' : '2 hops · via callees'; +} + +export function layoutPlate( + model: TraceModel, + root: NeighborsPayload, + meta: PlateFocusMeta, + undrawn: readonly UndrawnNeighbour[], + width: number, +): PlateLayout { + const W = Math.max(240, width); + const stacked = W < PLATE_STACK_BELOW; + const fields = plateFields(model, root, meta); + const byRing = (ring: number) => model.nodes.filter((node) => node.ring === ring); + + // Segments: call sites on each drawn channel into the hop one step inward. + const segmentsOf = (node: TraceNode): { segments: number[]; parents: TraceNode[] } => { + const inward = Math.abs(node.ring) - 1; + const parents = model.nodes.filter( + (other) => + Math.abs(other.ring) === inward && + (inward === 0 || Math.sign(other.ring) === Math.sign(node.ring)) && + channelBetween(model, node.id, other.id) !== undefined, + ); + const segments = parents + .map((parent) => channelBetween(model, node.id, parent.id)!.calls) + .sort((a, b) => b - a); + return { segments, parents }; + }; + const total = (node: TraceNode) => segmentsOf(node).segments.reduce((sum, n) => sum + n, 0); + + const ordered = (ring: number, parentOrder: ReadonlyMap | null): TraceNode[] => + byRing(ring).sort((a, b) => { + if (parentOrder) { + const pa = Math.min(...segmentsOf(a).parents.map((p) => parentOrder.get(p.id) ?? 99), 99); + const pb = Math.min(...segmentsOf(b).parents.map((p) => parentOrder.get(p.id) ?? 99), 99); + if (pa !== pb) return pa - pb; + } + return total(b) - total(a) || a.name.localeCompare(b.name); + }); + + const up1 = ordered(-1, null); + const down1 = ordered(1, null); + const up2 = ordered(-2, new Map(up1.map((node, i) => [node.id, i]))); + const down2 = ordered(2, new Map(down1.map((node, i) => [node.id, i]))); + + const notesFor = (side: 'up' | 'down', hop: 1 | 2): string[] => { + const out: string[] = []; + const hidden = undrawn.filter((entry) => entry.side === side && entry.hop === hop).length; + if (hidden > 0) out.push(`+${hidden} named, not drawn`); + if (hop === 2) { + const seeds = (side === 'up' ? up1 : down1).length; + if (model.coverage.hopsFetched === 1 && seeds > 0) out.push('hop 2 not fetched'); + } + if (hop === 1 && model.coverage.capped) out.push('a list hit the row limit: prefix only'); + return out; + }; + + // The widest bar is a row's whole stack of segments, not its largest one. + // Readouts wrap to their column rather than truncate: an omission count is + // never the part of a line that gets cut. + const noteLines = (frame: { side: 'up' | 'down'; hop: 1 | 2; x0: number; x1: number }): string[] => + notesFor(frame.side, frame.hop).flatMap((text) => wrap(text, Math.max(12, Math.floor((frame.x1 - frame.x0) / 6.2)))); + + const maxCalls = Math.max(1, ...model.nodes.filter((n) => n.ring !== 0).map(total)); + + /* ---- column frames ---------------------------------------------------- */ + let plate: PlateLayout['plate']; + const frames: Array<{ side: 'up' | 'down'; hop: 1 | 2; x0: number; x1: number; nodes: TraceNode[]; y0: number }> = []; + const plateH = PLATE_HEAD_H + fields.length * PLATE_FIELD_H + 14; + + if (!stacked) { + const plateW = Math.min(320, Math.max(250, Math.round(W * 0.28))); + const plateX = Math.round((W - plateW) / 2); + const sideW = plateX - MARGIN - 34; + const place = (side: 'up' | 'down', inner: TraceNode[], outer: TraceNode[]) => { + const hasOuter = outer.length > 0; + const innerW = hasOuter ? Math.round(sideW * 0.54) : sideW; + const outerW = hasOuter ? sideW - innerW - 30 : 0; + if (side === 'up') { + const innerX1 = plateX - 34; + frames.push({ side, hop: 1, x0: innerX1 - innerW, x1: innerX1, nodes: inner, y0: TITLE_H }); + if (hasOuter) frames.push({ side, hop: 2, x0: MARGIN, x1: MARGIN + outerW, nodes: outer, y0: TITLE_H }); + } else { + const innerX0 = plateX + plateW + 34; + frames.push({ side, hop: 1, x0: innerX0, x1: innerX0 + innerW, nodes: inner, y0: TITLE_H }); + if (hasOuter) frames.push({ side, hop: 2, x0: W - MARGIN - outerW, x1: W - MARGIN, nodes: outer, y0: TITLE_H }); + } + }; + place('up', up1, up2); + place('down', down1, down2); + const hop1Rows = Math.max(up1.length, down1.length, 1); + const plateY = Math.max(TITLE_H, TITLE_H + (hop1Rows * PLATE_ROW - plateH) / 2); + plate = { x: plateX, y: Math.round(plateY), width: plateW, height: plateH }; + } else { + // Stacked: flow reads top to bottom, callers of callers first. + let y = 0; + const stack = (side: 'up' | 'down', hop: 1 | 2, nodes: TraceNode[]) => { + if (nodes.length === 0 && notesFor(side, hop).length === 0) return; + frames.push({ side, hop, x0: MARGIN + GUTTER, x1: W - MARGIN, nodes, y0: y + TITLE_H }); + y += TITLE_H + nodes.length * PLATE_ROW + noteLines({ side, hop, x0: MARGIN + GUTTER, x1: W - MARGIN }).length * NOTE_H + 12; + }; + stack('up', 2, up2); + stack('up', 1, up1); + plate = { x: MARGIN + GUTTER, y: y + 6, width: W - MARGIN * 2 - GUTTER, height: plateH }; + y += plateH + 18; + stack('down', 1, down1); + stack('down', 2, down2); + } + + // Channels a bar segment encodes; anything else incident on a row is + // counted on that row, so an omission is attributed rather than pooled. + const encoded = new Set(); + for (const node of model.nodes) { + for (const parent of segmentsOf(node).parents) encoded.add(channelKey(channelBetween(model, node.id, parent.id)!)); + } + const crossOn = (id: string) => + model.channels.filter((c) => (c.a === id || c.b === id) && !encoded.has(channelKey(c))).length; + + const columns: PlateColumn[] = []; + const rows: PlateRow[] = []; + for (const frame of frames) { + const notes = noteLines(frame).map((text, i) => ({ + y: frame.y0 + frame.nodes.length * PLATE_ROW + 12 + i * NOTE_H, + text, + })); + const column: PlateColumn = { + side: frame.side, + hop: frame.hop, + x0: frame.x0, + x1: frame.x1, + title: columnTitle(frame.side, frame.hop), + titleY: frame.y0 - 12, + titleX: stacked || (frame.side === 'up') === (frame.hop === 2) ? frame.x0 : frame.x1, + titleAnchor: stacked || (frame.side === 'up') === (frame.hop === 2) ? 'start' : 'end', + notes, + }; + columns.push(column); + // Left columns grow their bars leftward from the inner edge; right columns + // and every stacked column grow rightward, so the scale reads one way. + const growLeft = !stacked && frame.side === 'up'; + // The kind glyph takes the first 14px on the name line. + const chars = Math.max(8, Math.floor((frame.x1 - frame.x0 - 14) / CHAR_PX)); + frame.nodes.forEach((node, i) => { + const y = frame.y0 + i * PLATE_ROW; + const { segments, parents } = segmentsOf(node); + const bits = [node.kind, node.degree === null ? 'degree absent' : `deg ${node.degree}`]; + if (node.undrawnEdges === null) bits.push('undrawn edges absent'); + else if (node.undrawnEdges > 0) bits.push(`+${node.undrawnEdges} edges not drawn`); + if (node.selfCalls > 0) bits.push(`↻ ${node.selfCalls} self`); + const cross = crossOn(node.id); + if (cross > 0) bits.push(`⇄ ${cross} cross-link${cross === 1 ? '' : 's'}`); + if (stacked && Math.abs(node.ring) === 2 && parents.length > 0) { + bits.push(`via ${parents.map((p) => p.name).join(', ')}`); + } + const anchorX = growLeft ? frame.x1 : frame.x0; + rows.push({ + node, + column, + y, + segments, + name: clip(node.name, chars), + meta: clip(bits.join(' · '), Math.max(4, chars - segments.join('+').length - 3)), + anchorX, + grow: growLeft ? -1 : 1, + }); + }); + } + + const narrowest = Math.min(...columns.map((c) => c.x1 - c.x0 - 6), 280); + const pxPerCall = Math.max(2, narrowest / maxCalls); + + /* ---- connectors ----------------------------------------------------- */ + const rowOf = new Map(rows.map((row) => [row.node.id, row])); + const throughY = Math.round(plate.y + plate.height / 2); + const connectors: PlateConnector[] = []; + const throughUsed = new Set(); + type Pt = readonly [number, number]; + if (stacked) { + // One corridor down the gutter; the plate's port is its left edge. + const bus = MARGIN; + const port = (id: string): Pt => + id === model.focusId ? [plate.x, throughY] : [rowOf.get(id)!.column.x0, rowOf.get(id)!.y + 21]; + for (const channel of model.channels) { + if (!rowOf.has(channel.a) && channel.a !== model.focusId) continue; + if (!rowOf.has(channel.b) && channel.b !== model.focusId) continue; + const [ax, ay] = port(channel.a); + const [bx, by] = port(channel.b); + if (channel.a === model.focusId || channel.b === model.focusId) throughUsed.add(plate.x); + connectors.push({ + key: channelKey(channel), + d: elbowPath([[ax, ay], [bus, ay], [bus, by], [bx, by]]), + corridors: [bus], + }); + } + } else { + // Slots left to right: the four columns and the plate between them. + const slots = [ + ...columns.map((column) => ({ x0: column.x0, x1: column.x1, column })), + { x0: plate.x, x1: plate.x + plate.width, column: null }, + ].sort((p, q) => p.x0 - q.x0); + const slotOf = (id: string) => + id === model.focusId + ? slots.findIndex((slot) => slot.column === null) + : slots.findIndex((slot) => slot.column === rowOf.get(id)!.column); + const plateSlot = slotOf(model.focusId); + const gapX = (left: number) => Math.round((slots[left]!.x1 + slots[left + 1]!.x0) / 2); + const portY = (id: string) => (id === model.focusId ? throughY : rowOf.get(id)!.y + 21); + const edge = (id: string, facing: 'left' | 'right'): number => { + const slot = slots[slotOf(id)]!; + return facing === 'left' ? slot.x0 : slot.x1; + }; + for (const channel of model.channels) { + const ends = [channel.a, channel.b]; + if (ends.some((id) => id !== model.focusId && !rowOf.has(id))) continue; + const [l, r] = [...ends].sort((p, q) => slotOf(p) - slotOf(q)) as [string, string]; + const i = slotOf(l); + const j = slotOf(r); + let points: Pt[]; + let corridors: number[]; + if (i === j) { + // Same column: out through the corridor on the plate side and back. + const toward = i < plateSlot ? i : i - 1; + const facing = i < plateSlot ? 'right' : 'left'; + const bus = gapX(toward); + points = [[edge(l, facing), portY(l)], [bus, portY(l)], [bus, portY(r)], [edge(r, facing), portY(r)]]; + corridors = [bus]; + } else if (j === i + 1) { + const bus = gapX(i); + points = [[edge(l, 'right'), portY(l)], [bus, portY(l)], [bus, portY(r)], [edge(r, 'left'), portY(r)]]; + corridors = [bus]; + } else { + // Across the plate: both corridors meet the plate's through line, and + // the run between them is hidden behind the plate face. + const first = gapX(i); + const last = gapX(j - 1); + points = [ + [edge(l, 'right'), portY(l)], + [first, portY(l)], + [first, throughY], + [last, throughY], + [last, portY(r)], + [edge(r, 'left'), portY(r)], + ]; + corridors = [first, last]; + } + if (i < plateSlot && j >= plateSlot) throughUsed.add(plate.x); + if (j > plateSlot && i <= plateSlot) throughUsed.add(plate.x + plate.width); + connectors.push({ key: channelKey(channel), d: elbowPath(points), corridors }); + } + } + const throughPorts = [...throughUsed].sort((a, b) => a - b).map((x) => ({ x, y: throughY })); + + const contentBottom = Math.max( + plate.y + plate.height, + ...frames.map( + (frame) => + frame.y0 + frame.nodes.length * PLATE_ROW + noteLines(frame).length * NOTE_H, + ), + ); + const scaleY = contentBottom + 20; + return { + width: W, + height: scaleY + 40, + stacked, + plate, + fields, + columns, + rows, + connectors, + throughPorts, + scale: { pxPerCall, maxCalls, y: scaleY }, + crossLinks: model.channels.length - encoded.size, + }; +} diff --git a/dashboard/src/viz/trace/readout.test.ts b/dashboard/src/viz/trace/readout.test.ts index dd41e09ed7..0065e08bf2 100644 --- a/dashboard/src/viz/trace/readout.test.ts +++ b/dashboard/src/viz/trace/readout.test.ts @@ -1,10 +1,9 @@ /** - * The instrument plate must not become a second source of truth. + * The readout strip must not become a second source of truth. * - * Two failure modes are worth a test each. The first is DRIFT: the legend says - * "channel width is call sites" and prints a range that the drawn channels do - * not actually span, because someone typed the range once and the payload - * moved. The second is SILENT ABSENCE: a measurement the wire never sent gets + * Two failure modes are worth a test each. The first is DRIFT: a cell prints a + * count the drawn rows do not actually carry, because someone typed it once + * and the payload moved. The second is SILENT ABSENCE: a measurement the wire never sent gets * rendered as a blank cell or a plausible zero, which reads as "none" when the * truth is "not asked, not answered". * @@ -18,7 +17,7 @@ import { GraphNeighborsPayloadV1Schema, } from '../../contracts/generated.ts'; import { TRACE_BUDGET, buildTraceModel, type NeighborsPayload } from './model.ts'; -import { legendPanels, readoutCells, type ReadoutValue } from './readout.ts'; +import { readoutCells, type ReadoutValue } from './readout.ts'; import type { TraceModel, TraceNode } from './types.ts'; function neighbors(id: string): NeighborsPayload { @@ -51,8 +50,6 @@ function node(over: Partial & { id: string }): TraceNode { filePath: 'crates/retrieval/src/lib.rs', startLine: 1, ring: 0, - x0: 0, - y0: 0, undrawnEdges: 0, selfCalls: 0, ...over, @@ -69,8 +66,6 @@ function synthetic(over: { const nodes = over.nodes ?? [node({ id: 'focus' })]; return { focusId: 'focus', - world: { width: 1200, height: 1040 }, - rows: new Map([[0, 520]]), nodes, channels: over.channels ?? [], membranes: over.membranes ?? [], @@ -82,7 +77,6 @@ function synthetic(over: { cappedAt: null, capped: false, membranesAvailable: true, - rowFields: ['degree', 'id', 'kind', 'name'], ...over.coverage, }, }; @@ -166,40 +160,3 @@ describe('the header readout strip', () => { expect(readoutCells(model)[6]!.value.kind).toBe('absent'); }); }); - -describe('the legend row', () => { - it('prints the call-site range the drawn channels actually span', () => { - const model = fixtureModel(); - const calls = model.channels.map((c) => c.calls); - const low = Math.min(...calls); - const high = Math.max(...calls); - const panel = legendPanels(model)[0]!; - expect(panel.reading).toMatchObject({ - value: low === high ? String(low) : `${low}–${high}`, - unit: `across ${model.channels.length} channels`, - }); - }); - - it('reports the membrane panel as absent when the wire carried no contains edges', () => { - const panel = legendPanels(synthetic({ coverage: { membranesAvailable: false } }))[4]!; - expect(panel.reading.kind).toBe('absent'); - expect(panel.qualifier).toContain('no enclosure is drawn'); - }); - - it('counts dashed mouths and the edges behind them from the drawn nodes', () => { - const model = synthetic({ - nodes: [ - node({ id: 'focus', undrawnEdges: 5 }), - node({ id: 'b', undrawnEdges: 0 }), - node({ id: 'c', undrawnEdges: 2 }), - ], - }); - expect(legendPanels(model)[5]!.reading).toMatchObject({ value: '7', unit: 'at 2 symbols' }); - }); - - it('distinguishes "no mouth drawn" from a measured zero', () => { - const panel = legendPanels(synthetic({ nodes: [node({ id: 'focus', undrawnEdges: null })] }))[5]!; - expect(panel.reading.kind).toBe('absent'); - }); - -}); diff --git a/dashboard/src/viz/trace/readout.ts b/dashboard/src/viz/trace/readout.ts index a77f787766..e78c5d9c96 100644 --- a/dashboard/src/viz/trace/readout.ts +++ b/dashboard/src/viz/trace/readout.ts @@ -1,18 +1,7 @@ /** - * The instrument plate for the TRACE field: the header readout strip and the - * legend row, both derived from the one `TraceModel` the field is drawn from. - * - * Why this is a module and not JSX - * -------------------------------- - * Sheet 02 of the approved design carries two plates around its field, a - * seven-cell readout strip above and a six-panel key below. A legend is the - * one place in an instrument where a *second* source of truth can grow: the - * picture is drawn from the payload, the legend is typed by hand, and the day - * the payload changes only one of them moves. The approved sheet avoids this - * by printing counts it computed from its own dataset, and this module is how - * that property survives the port: every number on both plates is counted here - * from `model`, the same record `render.ts` draws. Nothing on either plate is - * a literal that a payload change could falsify. + * The header readout strip for the TRACE surface, counted from the one + * `TraceModel` the anatomy plate is drawn from, so no cell is a literal a + * payload change could falsify. * * The house rule from the design note, "every position, size, elevation and * width encodes a stated measurement", has a corollary this file exists to @@ -21,9 +10,7 @@ * that reason. A caller cannot render a cell without having decided what it * says when the wire was silent. * - * Pure by construction: types only, no DOM, no colour, no clock. `sim.ts` - * imports nothing at all and keeps that stronger boundary; this module sits - * beside it on the same side of the honesty line. + * Pure by construction: types only, no DOM, no colour, no clock. */ import type { TraceModel, TraceNode } from './types.ts'; @@ -51,23 +38,6 @@ export interface ReadoutCell { readonly qualifier: string | null; } -/** One panel of the legend row. */ -export interface LegendPanel { - readonly label: string; - /** The sensory contract in one clause: what this channel means. */ - readonly teach: string; - /** What that channel is actually carrying on THIS frame. */ - readonly reading: ReadoutValue; - readonly qualifier: string | null; - /** - * Which sample the row should draw beside the panel. A closed set so the - * component cannot invent a swatch for a channel the field does not draw. - */ - readonly sample: LegendSample; -} - -export type LegendSample = 'channel' | 'sill' | 'rows' | 'hue' | 'membrane' | 'mouth'; - /* ---- small shared counting helpers -------------------------------------- */ function measured(value: string, unit: string | null = null): ReadoutValue { @@ -82,22 +52,6 @@ function plural(n: number, one: string, many: string): string { return n === 1 ? one : many; } -/** - * Inclusive range of a set of numbers as the legend prints it. Returns null - * for an empty set so the caller must decide what absence reads as, rather - * than receiving a plausible `0–0`. - */ -function range(values: readonly number[]): string | null { - if (values.length === 0) return null; - let low = values[0] as number; - let high = low; - for (const value of values) { - if (value < low) low = value; - if (value > high) high = value; - } - return low === high ? String(low) : `${low}–${high}`; -} - /** * The module a symbol belongs to: the directory of its `file_path`. * @@ -237,99 +191,3 @@ function countCrossings(model: TraceModel): number { } return crossings; } - -/* ---- the legend row ------------------------------------------------------ */ - -/** - * The six channels this field actually draws, each with what it is carrying - * right now. - * - * The approved sheet's sixth panel is `Underlay`, sheet 01's module relief, - * dimmed behind the flow. This surface draws no relief, so that panel is not - * here: a legend panel for a channel the renderer does not paint would be the - * exact drift this module exists to prevent. Its slot goes to `Sill`, which - * the field does draw and which the sheet folds into its `Width` caption. - */ -export function legendPanels(model: TraceModel): readonly LegendPanel[] { - const capped = cappedQualifier(model); - - const callSites = model.channels.map((channel) => channel.calls); - const degrees = model.nodes - .map((node) => node.degree) - .filter((degree): degree is number => degree !== null); - const unmeasuredDegrees = model.nodes.length - degrees.length; - - const up = model.nodes.filter((node) => node.ring < 0).length; - const down = model.nodes.filter((node) => node.ring > 0).length; - const kinds = new Set(model.nodes.map((node) => node.kind)); - - const mouths = model.nodes.filter((node) => (node.undrawnEdges ?? 0) > 0); - const undrawn = mouths.reduce((sum, node) => sum + (node.undrawnEdges ?? 0), 0); - - const callRange = range(callSites); - const degreeRange = range(degrees); - - return [ - { - label: 'Channel width', - teach: 'call sites on that one edge', - reading: - callRange === null - ? absent('no calls edge was drawn on this frame') - : measured(callRange, `across ${model.channels.length} ${plural(model.channels.length, 'channel', 'channels')}`), - qualifier: capped, - sample: 'channel', - }, - { - label: 'Sill width', - teach: "the symbol's degree, straight off the payload", - reading: - degreeRange === null - ? absent('no drawn row carried a degree') - : measured(degreeRange, `over ${degrees.length} ${plural(degrees.length, 'symbol', 'symbols')}`), - qualifier: - unmeasuredDegrees > 0 - ? `${unmeasuredDegrees} without a degree, hollow sill at the floor width` - : null, - sample: 'sill', - }, - { - label: 'Row', - // Named in the sheet's own words. Sheet 01 spends height on dependency - // depth; this one does not, and says so rather than borrowing that axis. - teach: 'hop distance from the focus, not elevation, not importance', - reading: measured(`${up} ↑ / ${down} ↓`, `${model.coverage.drawn} drawn`), - qualifier: null, - sample: 'rows', - }, - { - label: 'Hue', - teach: 'symbol kind, off the same arc as the connectivity spine', - reading: measured(String(kinds.size), plural(kinds.size, 'kind', 'kinds')), - qualifier: null, - sample: 'hue', - }, - { - label: 'Membrane', - teach: 'one type enclosure, from contains edges', - reading: model.coverage.membranesAvailable - ? measured( - String(model.membranes.length), - plural(model.membranes.length, 'enclosure', 'enclosures'), - ) - : absent('the payload carried no contains edges'), - qualifier: model.coverage.membranesAvailable ? null : 'no enclosure is drawn on this frame', - sample: 'membrane', - }, - { - label: 'Dashed mouth', - teach: 'edges this frame does not draw', - reading: - mouths.length === 0 - ? absent('every drawn symbol had all its edges drawn, or none carried a degree') - : measured(String(undrawn), `at ${mouths.length} ${plural(mouths.length, 'symbol', 'symbols')}`), - qualifier: null, - sample: 'mouth', - }, - ]; -} diff --git a/dashboard/src/viz/trace/render.test.ts b/dashboard/src/viz/trace/render.test.ts deleted file mode 100644 index 31e164d818..0000000000 --- a/dashboard/src/viz/trace/render.test.ts +++ /dev/null @@ -1,50 +0,0 @@ -/** - * The measurement→mark laws in `render.ts`, tested without a canvas. - * - * These are the functions that turn a counted quantity into pixels, so they - * are exactly where a "nicer looking" tweak can quietly stop telling the - * truth. The taper is the live example: it is a shape choice, and a shape - * choice is only allowed here if the measured widths survive it intact. - */ -import { describe, expect, it } from 'vitest'; - -import { channelWidth, sillWidth, taperAt } from './render.ts'; - -describe('channel and sill widths', () => { - it('puts a square root between a call-site count and a width', () => { - // Magnitude reaches the eye through area, not radius, the same rule the - // Code spine's mark diameter follows. - expect(channelWidth(4) - channelWidth(1)).toBeCloseTo(1.15, 5); - expect(channelWidth(16) - channelWidth(9)).toBeCloseTo(1.15, 5); - }); - - it('draws an unmeasured degree at the floor rather than as a measured zero', () => { - expect(sillWidth(null)).toBe(sillWidth(0)); - expect(sillWidth(null)).toBeGreaterThan(0); - }); -}); - -describe('the hydrological taper', () => { - it('lands exactly on both measured widths', () => { - // The whole licence for shaping the run is that the ends are untouched. - // If either endpoint drifts, the taper has started editing a measurement. - expect(taperAt(0, 0.55)).toBeCloseTo(0.55, 12); - expect(taperAt(1, 0.55)).toBe(1); - expect(taperAt(0, 0.3)).toBeCloseTo(0.3, 12); - expect(taperAt(1, 0.3)).toBe(1); - }); - - it('widens monotonically from head to mouth', () => { - let previous = -Infinity; - for (let i = 0; i <= 40; i += 1) { - const width = taperAt(i / 40); - expect(width).toBeGreaterThan(previous); - previous = width; - } - }); - - it('clamps outside the run instead of extrapolating a width', () => { - expect(taperAt(-3)).toBe(taperAt(0)); - expect(taperAt(9)).toBe(taperAt(1)); - }); -}); diff --git a/dashboard/src/viz/trace/render.ts b/dashboard/src/viz/trace/render.ts deleted file mode 100644 index df4cf6c9c1..0000000000 --- a/dashboard/src/viz/trace/render.ts +++ /dev/null @@ -1,918 +0,0 @@ -/** - * Canvas2D renderer for the TRACE surface. - * - * Lifted from the round-two prototype's `render.js` and reduced to what the - * neighbors endpoint can actually feed. This module draws and does nothing - * else: it is handed positions, per-channel stretch, per-node bloom and a - * resolved palette, and it paints one frame. It never integrates, never decides - * how a body responds to a gesture, and never reads the clock, if a mark - * moves, it is because the simulation moved it. - * - * Dropped from the prototype, deliberately: the dimmed cortex relief underlay. - * Those region blobs are module rollups from a separate aggregation the - * neighbors endpoint does not serve, and a shoreline drawn from guessed - * membership would make cross-module calls look measured when they are not. - * The field carries no underlay until the cortex endpoint exists. - * - * Canvas cannot read CSS custom properties, so the composing component resolves - * the token block once per theme flip and hands the resolved strings in. That - * keeps `tokens.css` the single source of the instrument's colour without a - * `getComputedStyle` call inside the draw loop. - */ -import { kindColor } from '../graph/kindColor.ts'; -import type { - TraceChannelDirection, - TraceFrame, - TraceModel, - TracePalette, -} from './types.ts'; - -/* ---- measurement → mark ------------------------------------------------- */ - -/** Channel width in px: the call-site count on that one edge. */ -export function channelWidth(calls: number): number { - return 2.2 + Math.sqrt(Math.max(0, calls)) * 1.15; -} - -/** Node sill width in px: the symbol's degree, straight off the payload. */ -export function sillWidth(degree: number | null): number { - // An unmeasured degree gets the floor width and a hollow sill (see - // `drawNodes`), absence is drawn, never rendered as a measured zero. - return 16 + Math.max(0, degree ?? 0) * 0.62; -} - -/** - * How wide a channel is at its head, as a fraction of its width at the mouth. - * - * The approved sheet tapers 0.78 → 1.0 and calls it hydrological; at that - * depth, over a run this short, the two edges are within a pixel of parallel - * and the ribbon reads as a machined bar with a hue on it. Direction is - * supposed to be said twice, by hue and by taper, and only one of them was - * audible. 0.55 makes the second one legible without letting the head fall - * under the 2.2 px floor `channelWidth` sets for a single call site. - */ -export const CHANNEL_HEAD_FRACTION = 0.55; - -/** - * Width along a channel as a fraction of its width at the mouth, at `t` = 0 - * (head) through 1 (mouth). - * - * A straight line between two widths is a wedge, and a wedge is a machined - * shape. Water is not: a watercourse gains width as the square root of the - * flow it has accumulated, which is the standing exponent in hydraulic - * geometry, and it is also, not coincidentally, the same square root - * `channelWidth` above already puts between a call-site count and a width, and - * that `markDiameter` puts between a symbol count and a mark on the Code - * spine. So the taper is not a new law invented for this curve. It is the law - * the field already uses for magnitude, applied along the run instead of - * across it: accumulate flow linearly down the channel, then take its root. - * - * The consequence is a slightly convex edge, fuller at mid-run than a line - * would be, easing as it nears the mouth, which is the profile of a - * watercourse rather than a funnel. - * - * Both endpoints stay exact: `t = 0` returns `headFraction` and `t = 1` - * returns 1, so the measured width at the mouth is drawn at the mouth and the - * shaping happens strictly between two measurements, never at one. - */ -export function taperAt(t: number, headFraction: number = CHANNEL_HEAD_FRACTION): number { - const clamped = t < 0 ? 0 : t > 1 ? 1 : t; - // Flow at the head, back-derived so that √(flow) lands exactly on the head - // fraction, the inverse of the width law, so the two cannot disagree. - const headFlow = headFraction * headFraction; - return Math.sqrt(headFlow + (1 - headFlow) * clamped); -} - -/** Below this stretch a channel is at rest and gets no tension rail. */ -export const TENSION_FLOOR_PX = 5; -/** - * Stretch at which the reduced-motion tension rail reaches full thickness. - * The rail saturates because a 172 px stretch drawn at true scale is a 23 px - * slab that swamps the field; the figure is printed alongside so two different - * stretches drawing the same thickness can still be told apart. - */ -export const TENSION_SATURATION_PX = 60; - -/** Ring label. Rings are hop DISTANCE from the focus, never elevation. */ -export function ringLabel(ring: number): string { - if (ring === 0) return 'focus'; - const hops = Math.abs(ring); - const unit = hops === 1 ? 'hop' : 'hops'; - return `${hops} ${unit} ${ring < 0 ? 'up' : 'down'}`; -} - -/* ---- geometry ----------------------------------------------------------- */ - -type Point = readonly [number, number]; -type ScratchPoint = [number, number]; - -function mulberry32(seed: number): () => number { - let a = seed >>> 0; - return function next(): number { - a = (a + 0x6d2b79f5) >>> 0; - let t = a; - t = Math.imul(t ^ (t >>> 15), t | 1); - t ^= t + Math.imul(t ^ (t >>> 7), t | 61); - return ((t ^ (t >>> 14)) >>> 0) / 4294967296; - }; -} - -/** A closed irregular outline, used for the focus basin. */ -function blob( - ctx: CanvasRenderingContext2D, - cx: number, - cy: number, - radiusX: number, - radiusY: number, - seed: number, - rough = 0.14, - samples = 72, -): void { - const random = mulberry32(seed); - const harmonics = [ - { k: 2 + Math.floor(random() * 2), a: rough * (0.55 + random() * 0.5), p: random() * 6.283 }, - { k: 3 + Math.floor(random() * 3), a: rough * (0.34 + random() * 0.4), p: random() * 6.283 }, - { k: 6 + Math.floor(random() * 4), a: rough * (0.14 + random() * 0.2), p: random() * 6.283 }, - ]; - ctx.beginPath(); - for (let i = 0; i <= samples; i += 1) { - const t = (i / samples) * Math.PI * 2; - let m = 1; - for (const h of harmonics) m += h.a * Math.sin(h.k * t + h.p); - const px = cx + Math.cos(t) * radiusX * m; - const py = cy + Math.sin(t) * radiusY * m; - if (i === 0) ctx.moveTo(px, py); - else ctx.lineTo(px, py); - } - ctx.closePath(); -} - -/** - * Catmull–Rom through the waypoints, emitted as cubics. This is what makes - * several channels sharing a trunk read as one braided river rather than as - * parallel arrows. - */ -function curvePoint(points: readonly Point[], count: number, index: number): Point { - if (index <= 0) return points[0]!; - if (index >= count + 1) return points[count - 1]!; - return points[index - 1]!; -} - -function traceCurve( - ctx: CanvasRenderingContext2D, - points: readonly Point[], - move: boolean, - count = points.length, -): void { - if (count === 0) return; - const start = points[0]!; - if (move) ctx.moveTo(start[0], start[1]); - else ctx.lineTo(start[0], start[1]); - for (let i = 1; i < count; i += 1) { - const [x0, y0] = curvePoint(points, count, i - 1); - const [x1, y1] = curvePoint(points, count, i); - const [x2, y2] = curvePoint(points, count, i + 1); - const [x3, y3] = curvePoint(points, count, i + 2); - ctx.bezierCurveTo( - x1 + (x2 - x0) / 6, - y1 + (y2 - y0) / 6, - x2 - (x3 - x1) / 6, - y2 - (y3 - y1) / 6, - x2, - y2, - ); - } -} - -/** - * A tapered ribbon: width is a measured quantity at each end, and the run - * between them follows the hydrological profile in `taperAt`. - * - * The wider end is the mouth whichever end it is on, so an upstream tributary - * converging on the focus and a downstream distributary fanning away from it - * are the same curve read in opposite directions rather than two shapes. - */ -/** Per-draw scratch. Draw is synchronous on the main thread, so one ribbon at - * a time reuses these instead of allocating three arrays per channel. */ -const ribbonNormals: ScratchPoint[] = []; -const ribbonUpper: ScratchPoint[] = []; -const ribbonLower: ScratchPoint[] = []; -const channelPathIn: ScratchPoint[] = [ - [0, 0], - [0, 0], - [0, 0], -]; -const channelPathRun: ScratchPoint[] = [ - [0, 0], - [0, 0], - [0, 0], - [0, 0], -]; - -function growScratch(buffer: ScratchPoint[], n: number): void { - while (buffer.length < n) buffer.push([0, 0]); -} - -function setPoint(point: ScratchPoint, x: number, y: number): void { - point[0] = x; - point[1] = y; -} - -function ribbon( - ctx: CanvasRenderingContext2D, - points: readonly Point[], - widthStart: number, - widthEnd: number, -): void { - const n = points.length; - if (n < 2) return; - growScratch(ribbonNormals, n); - growScratch(ribbonUpper, n); - growScratch(ribbonLower, n); - for (let i = 0; i < n; i += 1) { - const a = points[Math.max(0, i - 1)]!; - const b = points[Math.min(n - 1, i + 1)]!; - const dx = b[0] - a[0]; - const dy = b[1] - a[1]; - const len = Math.hypot(dx, dy) || 1; - setPoint(ribbonNormals[i]!, -dy / len, dx / len); - } - const mouthWidth = Math.max(widthStart, widthEnd); - const headWidth = Math.min(widthStart, widthEnd); - // Which end the mouth is on. `t` always runs head → mouth so the profile is - // written once and read forwards or backwards. - const mouthAtEnd = widthEnd >= widthStart; - const headFraction = mouthWidth === 0 ? 1 : headWidth / mouthWidth; - const half = (i: number): number => { - const along = i / (n - 1); - return (mouthWidth * taperAt(mouthAtEnd ? along : 1 - along, headFraction)) / 2; - }; - for (let i = 0; i < n; i += 1) { - const pt = points[i]!; - const nx = ribbonNormals[i]![0]; - const ny = ribbonNormals[i]![1]; - const offset = half(i); - setPoint(ribbonUpper[i]!, pt[0] + nx * offset, pt[1] + ny * offset); - setPoint(ribbonLower[n - 1 - i]!, pt[0] - nx * offset, pt[1] - ny * offset); - } - ctx.beginPath(); - traceCurve(ctx, ribbonUpper, true, n); - traceCurve(ctx, ribbonLower, false, n); - ctx.closePath(); -} - -function roundRect( - ctx: CanvasRenderingContext2D, - x: number, - y: number, - w: number, - h: number, - r: number, -): void { - const radius = Math.min(r, w / 2, h / 2); - ctx.beginPath(); - ctx.moveTo(x + radius, y); - ctx.arcTo(x + w, y, x + w, y + h, radius); - ctx.arcTo(x + w, y + h, x, y + h, radius); - ctx.arcTo(x, y + h, x, y, radius); - ctx.arcTo(x, y, x + w, y, radius); - ctx.closePath(); -} - -/* ---- the renderer ------------------------------------------------------- */ - -export interface TraceViewport { - width: number; - height: number; - dpr: number; - scale: number; - offsetX: number; - offsetY: number; -} - -export interface TraceRenderer { - setPalette(palette: TracePalette): void; - setViewport(box: { width: number; height: number; dpr: number }): TraceViewport; - readonly viewport: TraceViewport; - toScreen(worldX: number, worldY: number): { x: number; y: number }; - toWorld(screenX: number, screenY: number): { x: number; y: number }; - /** Nearest node to a world point within `radius`, or null. */ - hitTest(positions: Float64Array, worldX: number, worldY: number, radius?: number): string | null; - draw(frame: TraceFrame): void; -} - -export function createRenderer( - canvas: HTMLCanvasElement, - model: TraceModel, -): TraceRenderer { - const context = canvas.getContext('2d', { alpha: true }); - if (!context) throw new Error('Canvas2D unavailable'); - // Bound to a fresh, explicitly typed const: the guard above narrows - // `context`, but the draw helpers below are hoisted function declarations and - // TypeScript will not carry a narrowing into them. - const ctx: CanvasRenderingContext2D = context; - - const nodes = model.nodes; - const indexOfId = new Map(nodes.map((node, i) => [node.id, i])); - - /** - * Label rank within a ring: 0 for the nearest row of names, 1 for the row - * pushed further out. Computed once from the LAYOUT anchors, not from live - * positions, so a drag does not make the labels reshuffle underneath the - * reader's hand. - */ - const staggerIndex = new Map(); - for (const ring of new Set(nodes.map((node) => node.ring))) { - [...nodes] - .filter((node) => node.ring === ring) - .sort((a, b) => a.x0 - b.x0) - .forEach((node, i) => staggerIndex.set(node.id, i % 2)); - } - - let palette: TracePalette | null = null; - let view: TraceViewport = { - width: 0, - height: 0, - dpr: 1, - scale: 1, - offsetX: 0, - offsetY: 0, - }; - - /** Cheap per-frame scratch so a 60 Hz loop allocates nothing. */ - const point = { x: 0, y: 0 }; - function readPosition(positions: Float64Array, i: number): { x: number; y: number } { - point.x = positions[i * 2] ?? 0; - point.y = positions[i * 2 + 1] ?? 0; - return point; - } - - interface LabelOptions { - size?: number; - color?: string; - align?: CanvasTextAlign; - tracking?: number; - halo?: boolean; - upper?: boolean; - } - - const NO_LABEL_OPTIONS: LabelOptions = {}; - const labelWidths: number[] = []; - - /** - * Type is specified in CSS pixels and divided back out of the world - * transform, so a label is the same physical size whether the field is - * fitted into a 1440 px page or a 700 px column. - * - * Without this the world scale silently shrank every label with the picture: - * at the widths this drill-in actually gets, a 9 px world label rendered at - * roughly 4.5 device pixels, which is not small type, it is no type. The - * compensation is capped so a very narrow column blows the labels up until - * they swamp the marks instead. - */ - function typeScale(): number { - return Math.min(2.2, 1 / Math.max(0.05, view.scale)); - } - - function label(text: string, x: number, y: number, options: LabelOptions = NO_LABEL_OPTIONS): void { - const pal = palette!; - const { size = 9, color, align = 'left', tracking = 0, halo = true, upper = false } = options; - const body = upper ? text.toUpperCase() : text; - ctx.font = `${(size * typeScale()).toFixed(2)}px ui-monospace, "SFMono-Regular", "JetBrains Mono", monospace`; - ctx.textAlign = tracking > 0 ? 'left' : align; - ctx.textBaseline = 'alphabetic'; - // A label on a moving field has to survive every crossing, so it is set - // with a substrate-coloured halo exactly as the static sheets do it. - const haloWidth = 3 * typeScale(); - if (tracking > 0) { - const track = tracking * typeScale(); - const glyphs = [...body]; - // One measurement per glyph, reused for both the alignment total and the - // cursor advance, this runs per label per frame, and `measureText` is - // the expensive call in it. Widths live in renderer scratch so the map - // and reduce do not allocate a fresh array every tracked label. - while (labelWidths.length < glyphs.length) labelWidths.push(0); - let total = -track; - for (let i = 0; i < glyphs.length; i += 1) { - const width = ctx.measureText(glyphs[i]!).width; - labelWidths[i] = width; - total += width + track; - } - let cursor = align === 'right' ? x - total : align === 'center' ? x - total / 2 : x; - for (let i = 0; i < glyphs.length; i += 1) { - const glyph = glyphs[i]!; - if (halo) { - ctx.strokeStyle = pal.surface0; - ctx.lineWidth = haloWidth; - ctx.strokeText(glyph, cursor, y); - } - ctx.fillStyle = color ?? pal.textMuted; - ctx.fillText(glyph, cursor, y); - cursor += labelWidths[i]! + track; - } - return; - } - if (halo) { - ctx.strokeStyle = pal.surface0; - ctx.lineWidth = haloWidth; - ctx.lineJoin = 'round'; - ctx.strokeText(body, x, y); - } - ctx.fillStyle = color ?? pal.textMuted; - ctx.fillText(body, x, y); - } - - /* ---- 1. hop rings ------------------------------------------------------ */ - function drawRings(): void { - const pal = palette!; - const t = typeScale(); - ctx.save(); - for (const [ring, y] of model.rows) { - ctx.beginPath(); - ctx.moveTo(6, y); - ctx.lineTo(model.world.width - 8, y); - ctx.strokeStyle = pal.grid; - ctx.lineWidth = 1; - ctx.setLineDash(ring === 0 ? [] : [1, 6]); - ctx.stroke(); - ctx.setLineDash([]); - // Set flush left and ABOVE its own rule. Right-aligning these into the - // margin clipped every one of them the moment the labels started - // compensating for the world scale, the margin is a fixed number of - // world units and the type no longer is. - label(ringLabel(ring), 6, y - 5 * t, { - align: 'left', - color: ring === 0 ? pal.accent : pal.textMuted, - tracking: 1.1, - upper: true, - }); - } - ctx.restore(); - label('hop ring, distance from the focus, not elevation', 6, 24 * t, { - align: 'left', - tracking: 1.1, - upper: true, - }); - } - - /* ---- 2. membranes: the types the flow passes through ------------------- */ - interface Box { - x0: number; - y0: number; - x1: number; - y1: number; - } - const boxes = new Map(); - - function drawMembranes(positions: Float64Array): void { - const pal = palette!; - const t = typeScale(); - boxes.clear(); - model.membranes.forEach((membrane, m) => { - let x0 = Infinity; - let x1 = -Infinity; - let y0 = Infinity; - let y1 = -Infinity; - for (const id of membrane.of) { - const i = indexOfId.get(id); - if (i === undefined) continue; - const p = readPosition(positions, i); - x0 = Math.min(x0, p.x); - x1 = Math.max(x1, p.x); - y0 = Math.min(y0, p.y); - y1 = Math.max(y1, p.y); - } - if (!Number.isFinite(x0)) return; - x0 -= 44; - x1 += 44; - y0 -= 34; - y1 += 34; - ctx.save(); - roundRect(ctx, x0, y0, x1 - x0, y1 - y0, Math.min((y1 - y0) / 2, 60)); - ctx.fillStyle = pal.membraneFill; - ctx.fill(); - ctx.strokeStyle = pal.edgeStrong; - ctx.lineWidth = 1; - ctx.globalAlpha = 0.7; - ctx.stroke(); - ctx.restore(); - // Enclosures nest and overlap freely, two types can each hold symbols on - // the same ring, so their names are stacked rather than all set on the - // box's own top edge, where they overprinted each other into mush. - label(membrane.label, x0 + 10, y0 - (7 + (m % 3) * 15) * t, { - color: pal.textMuted, - tracking: 1.1, - upper: true, - }); - boxes.set(membrane.id, { x0, y0, x1, y1 }); - }); - } - - /** Weakest-first paint order, computed once from the immutable model. */ - const drawOrder = model.channels - .map((channel, e) => ({ channel, e })) - .sort((a, b) => a.channel.calls - b.channel.calls); - - /* ---- 3. channels: width is call sites, brightness is live tension ------ */ - interface Reading { - x: number; - y: number; - text: string; - hue: string; - } - - function channelPath( - ax: number, - ay: number, - bx: number, - by: number, - dir: TraceChannelDirection, - focusX: number, - ): readonly Point[] { - switch (dir) { - case 'in': - // A call that entered a type and moves between its methods before - // leaving. Drawn, not implied, it is the sheet's whole argument. - setPoint(channelPathIn[0]!, ax, ay); - setPoint(channelPathIn[1]!, (ax + bx) / 2, Math.min(ay, by) - 46); - setPoint(channelPathIn[2]!, bx, by); - return channelPathIn; - case 'up': - case 'down': - case 'lost': { - const mid = (ay + by) / 2; - const pull = dir === 'up' ? 0.42 : 0.34; - setPoint(channelPathRun[0]!, ax, ay); - setPoint(channelPathRun[1]!, ax + (focusX - ax) * pull * 0.5, mid - (by - ay) * 0.18); - setPoint(channelPathRun[2]!, bx + (focusX - bx) * pull * 0.12, mid + (by - ay) * 0.2); - setPoint(channelPathRun[3]!, bx, by); - return channelPathRun; - } - default: { - const unhandled: never = dir; - return unhandled; - } - } - } - - function drawChannels( - positions: Float64Array, - stretches: Float64Array, - reducedMotion: boolean, - ): Reading[] { - const pal = palette!; - const focusIndex = indexOfId.get(model.focusId) ?? 0; - const focusX = positions[focusIndex * 2] ?? model.world.width / 2; - const readings: Reading[] = []; - // Weakest first, so a one-call-site hairline can never be laid over the - // 40-call-site trunk it crosses. Painted order is the only depth cue a 2D - // field has, and it should agree with the measurement everything else here - // encodes. - for (const { channel, e } of drawOrder) { - const ai = indexOfId.get(channel.a); - const bi = indexOfId.get(channel.b); - if (ai === undefined || bi === undefined) continue; - const ax = positions[ai * 2] ?? 0; - const ay = positions[ai * 2 + 1] ?? 0; - const bx = positions[bi * 2] ?? 0; - const by = positions[bi * 2 + 1] ?? 0; - const w = channelWidth(channel.calls); - const points = channelPath(ax, ay, bx, by, channel.dir, focusX); - const upstream = channel.dir === 'up' || channel.dir === 'in'; - const hue = upstream ? pal.upstream : pal.downstream; - // A channel that leaves the graph keeps FULL width to its dashed mouth. - // The design note's absence beat is "full width, then it stops", the - // flow it carried was measured, and narrowing it toward the end would - // draw the lost traffic as dwindling when what is unknown is only where - // it went. - const lost = channel.dir === 'lost'; - const head = lost ? w : w * CHANNEL_HEAD_FRACTION; - const widthStart = channel.dir === 'up' ? head : w; - const widthEnd = channel.dir === 'up' ? w : head; - const stretch = Math.abs(stretches[e] ?? 0); - // Tension is the SAME measurement as width (call sites); under load the - // channel reports how far it has been pulled off its rest length, so the - // felt channel and the drawn channel cannot disagree. - const load = Math.min(1, stretch / 26); - ctx.save(); - ribbon(ctx, points, widthStart, widthEnd); - ctx.fillStyle = hue; - // Base fill is deliberately low: a dense neighbourhood stacks dozens of - // translucent ribbons, and at 0.42 they accumulated into one solid slab - // in which no individual channel, and therefore no call-site width, - // could be read at all. - ctx.globalAlpha = 0.26 + load * 0.34; - ctx.fill(); - ctx.globalAlpha = 0.62 + load * 0.38; - ctx.strokeStyle = hue; - ctx.lineWidth = 0.8 + load * 1.4; - ctx.stroke(); - ctx.restore(); - - if (reducedMotion && stretch > TENSION_FLOOR_PX) { - // Reduced motion: tension becomes literal thickness on a core rail, - // because a reader who cannot see the deformation must still be able - // to measure it. It wears the PARTIAL state hue rather than the accent, - // because an accent rail is indistinguishable from the upstream - // channel it sits inside. - const railWidth = 1.5 + Math.min(1, stretch / TENSION_SATURATION_PX) * 6.5; - ctx.save(); - ribbon(ctx, points, railWidth, railWidth); - ctx.fillStyle = pal.statePartial; - ctx.globalAlpha = 0.72; - ctx.fill(); - ctx.restore(); - readings.push({ - x: (ax + bx) / 2, - y: (ay + by) / 2, - text: `${stretch.toFixed(0)} px`, - hue: pal.statePartial, - }); - } else if (channel.calls >= 4) { - readings.push({ - x: (ax + bx) / 2, - y: (ay + by) / 2 + (channel.dir === 'in' ? -30 : 0), - text: String(channel.calls), - hue, - }); - } - } - return readings; - } - - /* ---- 4. membrane ports: where flow crosses a type boundary ------------- */ - function drawPorts(positions: Float64Array): void { - const pal = palette!; - ctx.save(); - ctx.strokeStyle = pal.edgeStrong; - ctx.lineWidth = 2.4; - for (const membrane of model.membranes) { - const box = boxes.get(membrane.id); - if (!box) continue; - for (const id of membrane.of) { - const i = indexOfId.get(id); - if (i === undefined) continue; - const p = readPosition(positions, i); - for (const edgeY of [box.y0, box.y1]) { - ctx.beginPath(); - ctx.moveTo(p.x - 9, edgeY); - ctx.lineTo(p.x + 9, edgeY); - ctx.stroke(); - } - } - } - ctx.restore(); - } - - /* ---- 5. dashed mouths: the edges this frame does NOT draw -------------- */ - function drawMouths(positions: Float64Array): void { - const pal = palette!; - ctx.save(); - ctx.setLineDash([4, 4]); - ctx.lineWidth = 1.4; - ctx.strokeStyle = pal.stateUnknown; - ctx.globalAlpha = 0.75; - nodes.forEach((node, i) => { - if (!node.undrawnEdges) return; - const p = readPosition(positions, i); - // The mouth points away from the focus, so it reads as flow leaving the - // frame rather than as another channel inside it. - const away = node.ring < 0 ? -1 : 1; - const reach = 14 + Math.min(34, Math.sqrt(node.undrawnEdges) * 6); - ctx.beginPath(); - ctx.moveTo(p.x, p.y + away * 7); - ctx.lineTo(p.x, p.y + away * (7 + reach)); - ctx.stroke(); - }); - ctx.restore(); - } - - /* ---- 6. hover bloom: depth and latency are the node's degree ----------- */ - function drawBloom(positions: Float64Array, bloom: Float64Array, light: boolean): void { - const pal = palette!; - nodes.forEach((node, i) => { - const value = bloom[i] ?? 0; - if (value < 0.004) return; - const p = readPosition(positions, i); - const hue = node.degree == null ? pal.stateUnknown : kindColor(node.kind, light); - // Bloom DEPTH scales with degree: a hub opens a wide slow well, a leaf a - // tight flick. Same number that sets its inertia. - const reach = (14 + Math.max(3, node.degree ?? 0) * 0.9) * value; - ctx.save(); - for (let ring = 3; ring >= 1; ring -= 1) { - ctx.beginPath(); - ctx.ellipse( - p.x, - p.y, - sillWidth(node.degree) / 2 + reach * ring * 0.55, - 7 + reach * ring * 0.34, - 0, - 0, - Math.PI * 2, - ); - ctx.strokeStyle = hue; - ctx.globalAlpha = value * (0.42 - ring * 0.09); - ctx.lineWidth = ring === 1 ? 1.6 : 0.9; - ctx.stroke(); - } - ctx.restore(); - }); - } - - /* ---- 7. nodes: a sill whose WIDTH is the symbol's degree --------------- */ - function drawNodes(positions: Float64Array, light: boolean, draggingId: string | null): void { - const pal = palette!; - nodes.forEach((node, i) => { - if (node.id === model.focusId) return; - const p = readPosition(positions, i); - const w = sillWidth(node.degree); - if (node.degree == null) { - // Unmeasured degree: a hollow dashed sill at the floor width. Absence - // is drawn as absence, never as a measured zero. - ctx.save(); - roundRect(ctx, p.x - w / 2, p.y - 5, w, 10, 5); - ctx.strokeStyle = pal.textMuted; - ctx.lineWidth = 1; - ctx.globalAlpha = 0.55; - ctx.setLineDash([3, 4]); - ctx.stroke(); - ctx.restore(); - return; - } - roundRect(ctx, p.x - w / 2, p.y - 5, w, 10, 5); - ctx.save(); - ctx.fillStyle = kindColor(node.kind, light); - ctx.globalAlpha = 0.9; - ctx.fill(); - ctx.restore(); - ctx.strokeStyle = node.id === draggingId ? pal.accent : pal.surface0; - ctx.lineWidth = node.id === draggingId ? 1.8 : 1; - ctx.stroke(); - }); - } - - /* ---- 8. the focus basin ------------------------------------------------ */ - function drawFocus(positions: Float64Array): void { - const pal = palette!; - const i = indexOfId.get(model.focusId); - if (i === undefined) return; - const p = readPosition(positions, i); - ctx.save(); - for (let r = 4; r >= 1; r -= 1) { - blob(ctx, p.x, p.y, 20 + r * 15, 12 + r * 8, 77 + r * 13, 0.07); - if (r === 1) { - ctx.fillStyle = pal.accent; - ctx.globalAlpha = 0.85; - ctx.fill(); - } - ctx.strokeStyle = pal.accent; - ctx.globalAlpha = 0.25 + (5 - r) * 0.16; - ctx.lineWidth = r === 1 ? 1.6 : 0.9; - ctx.stroke(); - } - ctx.restore(); - } - - /* ---- 9. labels, set last ---------------------------------------------- */ - function drawLabels(positions: Float64Array, readings: Reading[]): void { - const pal = palette!; - const t = typeScale(); - nodes.forEach((node, i) => { - if (node.id === model.focusId) return; - const p = readPosition(positions, i); - const above = node.ring < 0; - // Every offset scales with the type, or a compensated label lands on top - // of the sill it is naming. The stagger is the second half of that: a row - // of seven names at one height collides at any column width the workspace - // offers, and alternating rows is cheaper to read than truncating. - const stagger = staggerIndex.get(node.id) ?? 0; - const dy = (above ? -14 : 22) * t + (above ? -1 : 1) * stagger * 26 * t; - label(node.name, p.x, p.y + dy, { - size: 11, - color: pal.textPrimary, - align: 'center', - }); - const degreeText = node.degree == null ? 'degree absent' : `deg ${node.degree}`; - label(degreeText, p.x, p.y + dy + (above ? -11 : 11) * t, { - color: node.degree == null ? pal.stateUnknown : pal.textMuted, - align: 'center', - }); - const notes: string[] = []; - if (node.selfCalls) notes.push(`↻ ${node.selfCalls} self`); - if (node.undrawnEdges) notes.push(`+${node.undrawnEdges} not drawn`); - if (notes.length) { - label(notes.join(' · '), p.x, p.y + dy + (above ? -22 : 22) * t, { - color: pal.stateUnknown, - align: 'center', - }); - } - }); - - const fi = indexOfId.get(model.focusId); - if (fi !== undefined) { - const f = readPosition(positions, fi); - const focusNode = nodes[fi]!; - label(focusNode.name, f.x, f.y + 4 * t, { - size: 15, - color: pal.textPrimary, - align: 'center', - }); - const stats = - focusNode.degree == null - ? 'degree absent from payload' - : `degree ${focusNode.degree}`; - label(stats, f.x, f.y + 26 * t, { color: pal.textMuted, align: 'center' }); - if (focusNode.filePath) { - label( - focusNode.startLine == null - ? focusNode.filePath - : `${focusNode.filePath}:${focusNode.startLine}`, - f.x, - f.y + 40 * t, - { color: pal.accent, align: 'center', tracking: 1.1, upper: true }, - ); - } - } - - for (const reading of readings) { - label(reading.text, reading.x, reading.y + 3 * t, { color: reading.hue, align: 'center' }); - } - } - - return { - setPalette(next: TracePalette) { - palette = next; - }, - - /** - * Fit the world into the canvas box. The composing component uses the same - * transform to map a pointer back into world space, so a gesture lands on - * the mark the reader aimed at. - */ - setViewport({ width, height, dpr }) { - const scale = Math.min(width / model.world.width, height / model.world.height); - view = { - width, - height, - dpr, - scale, - offsetX: (width - model.world.width * scale) / 2, - offsetY: (height - model.world.height * scale) / 2, - }; - canvas.width = Math.round(width * dpr); - canvas.height = Math.round(height * dpr); - canvas.style.width = `${width}px`; - canvas.style.height = `${height}px`; - return { ...view }; - }, - - get viewport() { - return { ...view }; - }, - - toScreen(worldX, worldY) { - return { x: view.offsetX + worldX * view.scale, y: view.offsetY + worldY * view.scale }; - }, - toWorld(screenX, screenY) { - return { x: (screenX - view.offsetX) / view.scale, y: (screenY - view.offsetY) / view.scale }; - }, - - hitTest(positions, worldX, worldY, radius = 46) { - let best: string | null = null; - let bestDistance = radius; - nodes.forEach((node, i) => { - const dx = (positions[i * 2] ?? 0) - worldX; - const dy = (positions[i * 2 + 1] ?? 0) - worldY; - // Sills are wide and short, so the hit region is scaled to match the - // mark rather than being a circle over a lozenge. - const distance = Math.hypot(dx / Math.max(1, sillWidth(node.degree) / 26), dy); - if (distance < bestDistance) { - bestDistance = distance; - best = node.id; - } - }); - return best; - }, - - draw(frame: TraceFrame) { - if (!palette) throw new Error('setPalette must be called before draw'); - const { positions, stretches, bloom, draggingId, reducedMotion } = frame; - ctx.setTransform(view.dpr, 0, 0, view.dpr, 0, 0); - ctx.clearRect(0, 0, view.width, view.height); - ctx.translate(view.offsetX, view.offsetY); - ctx.scale(view.scale, view.scale); - ctx.lineJoin = 'round'; - ctx.lineCap = 'round'; - - drawRings(); - drawMembranes(positions); - const readings = drawChannels(positions, stretches, reducedMotion); - drawPorts(positions); - drawMouths(positions); - drawBloom(positions, bloom, palette.light); - drawNodes(positions, palette.light, draggingId); - drawFocus(positions); - drawLabels(positions, readings); - }, - }; -} diff --git a/dashboard/src/viz/trace/sim.test.ts b/dashboard/src/viz/trace/sim.test.ts deleted file mode 100644 index c70a920e8c..0000000000 --- a/dashboard/src/viz/trace/sim.test.ts +++ /dev/null @@ -1,404 +0,0 @@ -/** - * Numeric contract for the TRACE simulation. - * - * Ported from the round-two prototype's `sim.test.mjs` - * (`mockups/code-topography/prototype/`, branch - * `worktree-agent-af882d6565fbab159`). The prototype ran these under - * `node --test` against its hand-authored 26-node sheet; here they run under - * vitest against a field built by `model.ts` from the wire-true neighbors - * fixture, so the physics contract and the real payload shape are held to the - * same assertions in one place. - * - * Tolerances are named and justified where they appear. Where a claim can be - * exact, determinism, reduced-motion equivalence, it is asserted EXACTLY, by - * array comparison, because a tolerance there would be hiding drift. - */ -import { describe, expect, it } from 'vitest'; - -import { resolveFixture } from '../../../stories/fixtures/data.ts'; -import { - DashboardEnvelopeV1Schema, - GraphNeighborsPayloadV1Schema, -} from '../../contracts/generated.ts'; -import { - bloomStep, - createSimulation, - hopDistances, - type SimParams, - type Simulation, -} from './sim.ts'; -import { buildSimSpec, buildTraceModel, type NeighborsPayload } from './model.ts'; -import type { TraceModel } from './types.ts'; - -const DT = 1 / 60; - -function neighbors(id: string): NeighborsPayload { - return DashboardEnvelopeV1Schema(GraphNeighborsPayloadV1Schema).parse( - resolveFixture(`/api/plugins/graph/node/${id}/neighbors`), - ).payload; -} - -/** The field the drill-in actually draws for `sym-0`, hop 2, from fixtures. */ -function fixtureModel(): TraceModel { - const root = neighbors('sym-0'); - const hop1 = new Set(); - for (const row of [...(root.callers ?? []), ...(root.callees ?? [])]) { - if (typeof row.id === 'string') hop1.add(row.id); - } - const expanded = new Map(); - for (const id of [...hop1].slice(0, 12)) expanded.set(id, neighbors(id)); - return buildTraceModel({ - focus: { id: 'sym-0', kind: 'function', name: 'sym_0', degree: 24 }, - root, - expanded, - }); -} - -const MODEL = fixtureModel(); - -function fieldSim(seed: number): Simulation { - return createSimulation(buildSimSpec(MODEL, seed)); -} - -/** - * A single anchored body with no channels at all, the weight channel in - * isolation, so a mass claim cannot be contaminated by a neighbour's pull. - */ -function loneBody(mass: number, params?: Partial): Simulation { - return createSimulation({ - seed: 7, - params: { ...params, jitter: 0 }, - nodes: [{ id: 'a', mass, x0: 0, y0: 0 }], - springs: [], - }); -} - -/** Two bodies joined by one channel of a chosen stiffness. */ -function pair( - stiffness: number, - { mass = 20, separation = 200 }: { mass?: number; separation?: number } = {}, -): Simulation { - return createSimulation({ - seed: 11, - params: { jitter: 0 }, - nodes: [ - { id: 'a', mass, x0: 0, y0: 0 }, - { id: 'b', mass, x0: separation, y0: 0 }, - ], - springs: [{ a: 'a', b: 'b', stiffness }], - }); -} - -/** - * Settling time of a released body, in frames, by the classic envelope - * definition: the last frame at which the body is still further than `tol` of - * its pull from home. - * - * A speed threshold cannot be used here. An underdamped body passes through - * zero speed at every turning point, so "first frame under 0.6 px/s" samples - * WHERE IN THE SWING the frame happened to land and is not monotone in mass. - * The envelope is what a reader actually perceives as "it has stopped". - */ -function framesToRest(sim: Simulation, id: string, offsetX: number, tol = 0.02): number { - const anchor = sim.anchorOf(id); - sim.applyDrag(id, anchor.x + offsetX, anchor.y); - sim.step(DT); - sim.release(); - const threshold = Math.abs(offsetX) * tol; - let last = 0; - for (let frame = 1; frame <= 4000; frame += 1) { - sim.step(DT); - if (Math.abs(sim.positionOf(id).x - anchor.x) >= threshold) last = frame; - } - return last; -} - -describe('TRACE simulation', () => { - it('rejects malformed measurements rather than guessing', () => { - expect(() => createSimulation({ nodes: [], springs: [] })).toThrow(/at least one node/); - expect(() => - createSimulation({ - nodes: [ - { id: 'a', mass: 1, x0: 0, y0: 0 }, - { id: 'a', mass: 1, x0: 1, y0: 1 }, - ], - springs: [], - }), - ).toThrow(/duplicate node id a/); - expect(() => - createSimulation({ - nodes: [{ id: 'a', mass: 1, x0: 0, y0: 0 }], - springs: [{ a: 'a', b: 'ghost', stiffness: 1 }], - }), - ).toThrow(/unknown node ghost/); - expect(() => - createSimulation({ - nodes: [{ id: 'a', mass: 1, x0: Number.NaN, y0: 0 }], - springs: [], - }), - ).toThrow(/nodes\[0\]\.x0 must be a finite number/); - const sim = createSimulation({ nodes: [{ id: 'a', mass: 1, x0: 0, y0: 0 }], springs: [] }); - expect(() => sim.step(0)).toThrow(/dt must be > 0/); - expect(() => sim.positionOf('ghost')).toThrow(/unknown node id ghost/); - }); - - it('makes the layout an exact equilibrium: rest length is anchor separation', () => { - const sim = pair(30); - // With jitter off and every spring at rest length, nothing should move. - for (let i = 0; i < 240; i += 1) sim.step(DT); - expect(sim.maxSpeed()).toBe(0); - expect(Array.from(sim.positions())).toEqual([0, 0, 200, 0]); - expect(sim.energy().total).toBe(0); - }); - - it('WEIGHT: settle time increases monotonically with mass', () => { - const masses = [4, 9, 20, 40, 63]; - const settle = masses.map((mass) => framesToRest(loneBody(mass), 'a', 160)); - for (let i = 1; i < settle.length; i += 1) { - expect( - settle[i]! > settle[i - 1]!, - `mass ${masses[i]} settled in ${settle[i]} frames, not slower than mass ${masses[i - 1]} at ${settle[i - 1]}`, - ).toBe(true); - } - // The spread is the feel budget: enough to be felt, not enough to stall. - const spread = settle[settle.length - 1]! / settle[0]!; - expect(spread).toBeGreaterThan(1.5); - expect(spread).toBeLessThan(4); - }); - - it('WEIGHT: hover bloom latency increases with mass, and both extremes arrive', () => { - const halfBloomSeconds = (mass: number): number => { - let value = 0; - let seconds = 0; - while (value < 0.5 && seconds < 10) { - value = bloomStep(value, 1, mass, DT); - seconds += DT; - } - return seconds; - }; - const leaf = halfBloomSeconds(4); - const hub = halfBloomSeconds(63); - expect(hub).toBeGreaterThan(leaf * 2); - // A leaf must flick; a hub must still arrive. - expect(leaf).toBeLessThan(0.2); - expect(hub).toBeLessThan(0.6); - // Release is slower than attack, so a bloom lingers rather than snapping off. - expect(bloomStep(1, 0, 20, DT)).toBeGreaterThan(1 - (1 - bloomStep(0, 1, 20, DT))); - }); - - it('TENSION: a stiff channel propagates displacement, a weak one does not', () => { - const PULL = 200; - function follow(calls: number): number { - const sim = pair(calls); - const start = sim.positionOf('b').x; - // Hold 'a' displaced long enough for the neighbourhood to reach the - // deformed equilibrium, this measures coupling, not the transient. - for (let i = 0; i < 600; i += 1) { - sim.applyDrag('a', -PULL, 0); - sim.step(DT); - } - return Math.abs(sim.positionOf('b').x - start) / PULL; - } - const stiff = follow(58); - const weak = follow(3); - expect(stiff).toBeGreaterThanOrEqual(0.25); - expect(weak).toBeLessThan(0.06); - expect(stiff / weak).toBeGreaterThan(8); - }); - - it('TENSION: on the fixture subgraph, deformation falls off with hop distance', () => { - const sim = fieldSim(3); - sim.settle({ dt: DT }); - const before = sim.positions(); - const ids = sim.nodeIds; - const hops = hopDistances(sim, MODEL.focusId); - const anchor = sim.anchorOf(MODEL.focusId); - for (let i = 0; i < 600; i += 1) { - sim.applyDrag(MODEL.focusId, anchor.x - 180, anchor.y + 90); - sim.step(DT); - } - const after = sim.positions(); - const moved = new Map( - ids.map((id, i) => [ - id, - Math.hypot(after[i * 2]! - before[i * 2]!, after[i * 2 + 1]! - before[i * 2 + 1]!), - ]), - ); - const byHop = new Map(); - for (const [id, hop] of hops) { - if (hop === 0) continue; - byHop.set(hop, Math.max(byHop.get(hop) ?? 0, moved.get(id) ?? 0)); - } - const oneHop = byHop.get(1) ?? 0; - const threeHop = byHop.get(3) ?? 0; - expect(oneHop, 'one-hop neighbours barely moved, coupling is not being felt').toBeGreaterThan( - 12, - ); - expect(threeHop).toBeLessThan(oneHop * 0.35); - }); - - it('RELEASE: total energy decays monotonically and nearly to nothing', () => { - const sim = fieldSim(5); - sim.settle({ dt: DT }); - const anchor = sim.anchorOf(MODEL.focusId); - for (let i = 0; i < 120; i += 1) { - sim.applyDrag(MODEL.focusId, anchor.x - 220, anchor.y); - sim.step(DT); - } - sim.release(); - - const first = sim.energy().total; - expect(first, 'nothing was stored to decay').toBeGreaterThan(1000); - let previous = first; - let peak = first; - for (let i = 0; i < 900; i += 1) { - sim.step(DT); - const total = sim.energy().total; - // Tolerance is RELATIVE and tiny: it covers the discretisation error of - // a symplectic step, not a physical energy gain. A real blow-up fails - // this by orders of magnitude. - expect(total, `energy rose at frame ${i}`).toBeLessThanOrEqual(previous * (1 + 1e-6) + 1e-9); - previous = total; - peak = Math.max(peak, total); - } - expect(peak, 'the post-release peak must be the release itself').toBe(first); - expect(previous).toBeLessThan(first * 1e-4); - expect(sim.isSettled(), 'the field never came to rest after release').toBe(true); - }); - - it('RELEASE: the swing decays fast, one small overshoot, then nothing', () => { - const PULL = 200; - const sim = loneBody(63); - const anchor = sim.anchorOf('a'); - sim.applyDrag('a', anchor.x + PULL, anchor.y); - sim.step(DT); - sim.release(); - - // A damped sinusoid crosses zero forever, so counting crossings measures - // float noise, not feel. What a reader perceives is the sequence of swing - // amplitudes, so that is what is asserted. - const swings: number[] = []; - let previous = sim.positionOf('a').x - anchor.x; - let extreme = previous; - for (let i = 0; i < 3000; i += 1) { - sim.step(DT); - const offset = sim.positionOf('a').x - anchor.x; - if (Math.sign(offset) !== Math.sign(previous) && Math.sign(offset) !== 0) { - swings.push(Math.abs(extreme)); - extreme = offset; - } else if (Math.abs(offset) > Math.abs(extreme)) { - extreme = offset; - } - previous = offset; - if (swings.length >= 3) break; - } - expect(swings.length, 'the body never swung back').toBeGreaterThanOrEqual(2); - expect(swings[0]).toBeGreaterThanOrEqual(PULL * 0.9); - expect(swings[1]!, 'overshoot is more than 12 % of the pull').toBeLessThan(PULL * 0.12); - if (swings[2] !== undefined) { - expect(swings[2], 'second swing is still visible, this is ringing').toBeLessThan( - PULL * 0.02, - ); - } - }); - - it('DETERMINISM: same seed and same gesture script give an identical trajectory', () => { - const focus = MODEL.focusId; - const other = MODEL.nodes.find((node) => node.id !== focus)!.id; - const script = [ - { frames: 30, drag: null }, - { frames: 45, drag: { id: focus, dx: -170, dy: 60 } }, - { frames: 12, drag: { id: other, dx: 90, dy: -40 } }, - { frames: 120, drag: null }, - ] as const; - function run(): number[][] { - const sim = fieldSim(20260725); - const trajectory: number[][] = []; - for (const phase of script) { - if (!phase.drag) sim.release(); - for (let i = 0; i < phase.frames; i += 1) { - if (phase.drag) { - const anchor = sim.anchorOf(phase.drag.id); - sim.applyDrag(phase.drag.id, anchor.x + phase.drag.dx, anchor.y + phase.drag.dy); - } - sim.step(DT); - trajectory.push(Array.from(sim.positions())); - } - } - return trajectory; - } - const a = run(); - const b = run(); - expect(a.length).toBe(207); - expect(a, 'two runs of the same script diverged').toEqual(b); - // A different seed must actually change the trajectory, or the seed is a lie. - const seeded = fieldSim(999); - seeded.step(DT); - const same = fieldSim(20260725); - same.step(DT); - expect(Array.from(seeded.positions())).not.toEqual(Array.from(same.positions())); - }); - - it('REDUCED MOTION: settling in one shot lands on identical positions', () => { - // Reduced motion is not an approximation of the animated path, it is the - // same `step()` sequence with the paints removed. So the arrays must match - // EXACTLY, and this catches any renderer-side shortcut that broke it. - const animated = fieldSim(42); - const frames = animated.settle({ dt: DT }); - expect(frames).toBeGreaterThan(5); - expect(frames).toBeLessThan(1200); - - const reduced = fieldSim(42); - for (let i = 0; i < frames; i += 1) reduced.step(DT); - expect(Array.from(reduced.positions())).toEqual(Array.from(animated.positions())); - - // And with a gesture in the middle: hold, settle, release, settle. - const held = MODEL.nodes.find((node) => node.id !== MODEL.focusId)!.id; - const anchor = animated.anchorOf(held); - for (const sim of [animated, reduced]) { - for (let i = 0; i < 90; i += 1) { - sim.applyDrag(held, anchor.x - 140, anchor.y + 70); - sim.step(DT); - } - sim.release(); - } - const settledFrames = animated.settle({ dt: DT }); - for (let i = 0; i < settledFrames; i += 1) reduced.step(DT); - expect(Array.from(reduced.positions())).toEqual(Array.from(animated.positions())); - // Final positions must be the layout again: release returns the field home. - for (const id of animated.nodeIds) { - const home = animated.anchorOf(id); - const now = animated.positionOf(id); - expect( - Math.hypot(now.x - home.x, now.y - home.y), - `${id} settled away from its layout anchor`, - ).toBeLessThan(1); - } - }); - - it('exposes readback surfaces as copies, not live views into the integrator', () => { - const sim = fieldSim(1); - const snapshot = sim.positions(); - snapshot[0] = 1e9; - sim.step(DT); - expect(sim.positions()[0]).not.toBe(1e9); - expect(sim.positionOf(MODEL.focusId).x).not.toBe(1e9); - const params = sim.params; - expect(() => { - (params as { anchorBase: number }).anchorBase = 1; - }).toThrow(); - }); - - it('treats substepping as a refinement, not a different simulation', () => { - // One 1/60 step must equal four 1/240 steps taken through the public API, - // which is what lets the surface choose its frame budget without changing - // feel. - const coarse = fieldSim(8); - const fine = fieldSim(8); - coarse.step(1 / 60); - for (let i = 0; i < 4; i += 1) fine.step(1 / 240); - expect(Array.from(coarse.positions())).toEqual(Array.from(fine.positions())); - expect(coarse.substepCount).toBe(fine.substepCount); - }); - -}); diff --git a/dashboard/src/viz/trace/sim.ts b/dashboard/src/viz/trace/sim.ts deleted file mode 100644 index a91bcad46c..0000000000 --- a/dashboard/src/viz/trace/sim.ts +++ /dev/null @@ -1,534 +0,0 @@ -/** - * Pure spring simulation for the TRACE surface. - * - * Lifted essentially verbatim from the round-two live prototype - * (`mockups/code-topography/prototype/sim.js`, branch - * `worktree-agent-af882d6565fbab159`) and typed. The physics, the tuning - * defaults and the numeric contract are unchanged, because they are what the - * owner reviewed; the diff against the prototype is types, not behaviour. - * - * This module is the honesty boundary. Every felt quantity, weight, latency, - * deformation, settle time, is computed HERE from a stated measurement, and - * the renderer only draws what comes out. Nothing in this file touches the DOM, - * a canvas, `Date.now`, `performance.now` or `Math.random`, so the same seed - * and the same gesture script produce a bit-identical trajectory on every - * machine and inside vitest. - * - * Physics: hand-rolled position Verlet (Störmer–Verlet) with per-node velocity - * damping, integrated at a fixed substep. Two force families: - * - * anchor every node is held to its watershed layout position by a spring of - * stiffness `anchorBase * mass^anchorMassExponent`. Because the - * anchor is the ONLY force at rest (see `restLength` below), the - * layout is an exact equilibrium: the live field at rest is the - * static sheet, pixel for pixel. - * channel every drawn `calls` edge is a spring whose stiffness is its - * call-site count. Rest length is the distance between the two - * layout anchors, so a channel stores zero energy at rest and only - * pulls once something has been displaced. - * - * Mass is the symbol's degree. With a uniform damping RATIO (not a uniform - * damping coefficient) the closed form of the anchored oscillator gives a - * settle time monotone in mass, which is the "hubs are slow and deep, leaves - * flick" clause of the sensory contract, and `sim.test.ts` asserts the - * monotonicity rather than trusting the algebra. - */ - -/** - * Tuning defaults. Every one of these is a feel knob; the prototype README - * carries the table with the reasoning and the measured consequence of each, - * and the values below are that table's committed column. - */ -export interface SimParams { - anchorBase: number; - anchorMassExponent: number; - edgeStiffnessScale: number; - dampingRatio: number; - substep: number; - restSpeed: number; - minMass: number; - jitter: number; - bloomAttack: number; - bloomRelease: number; - bloomMassExponent: number; -} - -export const DEFAULT_PARAMS: Readonly = Object.freeze({ - /** Anchor stiffness at unit mass, in force units per px. */ - anchorBase: 90, - /** - * Exponent applied to mass when scaling anchor stiffness. 0 would make every - * node oscillate at its own natural frequency and a 63-degree hub would take - * ~4.6x as long as a 3-degree leaf to settle, true to the measurement but - * unusable as an interface. 1 would cancel mass out entirely and destroy the - * weight channel. 0.5 keeps latency strictly monotone in degree with a ~2.1x - * spread, which reads as weight without stalling. - */ - anchorMassExponent: 0.5, - /** Channel stiffness per call site, in force units per px. */ - edgeStiffnessScale: 6, - /** - * Damping ratio of the anchored oscillator. Below 1 is underdamped. 0.72 - * gives one small overshoot, flesh, not jelly, and no ringing. - */ - dampingRatio: 0.72, - /** Integrator substep, in seconds. `step(dt)` subdivides down to this. */ - substep: 1 / 240, - /** Speed below which a node counts as at rest, in px/s. */ - restSpeed: 0.6, - /** Degree floor, so an unresolved (degree 0) node still has inertia. */ - minMass: 3, - /** Amplitude of the seeded startup displacement, in px. */ - jitter: 6, - /** Hover-bloom approach rate at unit mass, in 1/s, growing. */ - bloomAttack: 9.5, - /** Hover-bloom approach rate at unit mass, in 1/s, decaying. */ - bloomRelease: 5, - /** Exponent applied to mass when slowing the bloom approach. */ - bloomMassExponent: 0.42, -}); - -export interface SimNodeSpec { - readonly id: string; - /** The symbol's degree. */ - readonly mass: number; - /** Layout anchor, world coordinates. */ - readonly x0: number; - readonly y0: number; -} - -export interface SimSpringSpec { - readonly a: string; - readonly b: string; - /** The edge's call-site count. */ - readonly stiffness: number; - /** Defaults to the distance between the two anchors. */ - readonly restLength?: number; -} - -export interface SimSpec { - readonly nodes: readonly SimNodeSpec[]; - readonly springs: readonly SimSpringSpec[]; - /** Seed for the startup displacement only. */ - readonly seed?: number; - readonly params?: Partial; -} - -export interface SimEnergy { - readonly kinetic: number; - readonly anchor: number; - readonly channel: number; - readonly total: number; -} - -export interface Simulation { - /** Node ids in readback order. */ - readonly nodeIds: string[]; - readonly nodeCount: number; - readonly springCount: number; - /** Frozen copy of the parameters actually in force. */ - readonly params: Readonly; - readonly stepCount: number; - readonly substepCount: number; - indexOf(nodeId: string): number; - massOf(nodeId: string): number; - anchorOf(nodeId: string): { x: number; y: number }; - /** Channel stiffnesses incident on a node, keyed by the other endpoint. */ - springsOf(nodeId: string): Array<{ other: string; stiffness: number; restLength: number }>; - step(dt: number): Simulation; - applyDrag(nodeId: string, targetX: number, targetY: number): Simulation; - release(): Simulation; - pinnedIds(): string[]; - positions(): Float64Array; - positionOf(nodeId: string): { x: number; y: number }; - velocities(): Float64Array; - maxSpeed(): number; - energy(): SimEnergy; - stretches(): Float64Array; - isSettled(restSpeed?: number): boolean; - settle(options?: { dt?: number; maxFrames?: number; restSpeed?: number }): number; -} - -/** - * Deterministic PRNG (mulberry32). Used ONCE, at construction, to break the - * perfect symmetry of the layout so the field visibly breathes into place. - * Never called during stepping, that is what makes replay exact. - */ -function mulberry32(seed: number): () => number { - let a = seed >>> 0; - return function next(): number { - a = (a + 0x6d2b79f5) >>> 0; - let t = a; - t = Math.imul(t ^ (t >>> 15), t | 1); - t ^= t + Math.imul(t ^ (t >>> 7), t | 61); - return ((t ^ (t >>> 14)) >>> 0) / 4294967296; - }; -} - -function requireFinite(value: unknown, what: string): number { - if (typeof value !== 'number' || !Number.isFinite(value)) { - throw new TypeError(`${what} must be a finite number, got ${String(value)}`); - } - return value; -} - -/** Build a simulation over a measured subgraph. */ -export function createSimulation(spec: SimSpec): Simulation { - if (!spec || !Array.isArray(spec.nodes) || !Array.isArray(spec.springs)) { - throw new TypeError('createSimulation needs { nodes: [], springs: [] }'); - } - const params: SimParams = { ...DEFAULT_PARAMS, ...(spec.params ?? {}) }; - requireFinite(params.substep, 'params.substep'); - if (params.substep <= 0) throw new RangeError('params.substep must be > 0'); - - const count = spec.nodes.length; - if (count === 0) throw new RangeError('createSimulation needs at least one node'); - - const ids: string[] = new Array(count); - const indexOfId = new Map(); - const mass = new Float64Array(count); - const invMass = new Float64Array(count); - const anchorX = new Float64Array(count); - const anchorY = new Float64Array(count); - const anchorK = new Float64Array(count); - const dampPerSubstep = new Float64Array(count); - const x = new Float64Array(count); - const y = new Float64Array(count); - const prevX = new Float64Array(count); - const prevY = new Float64Array(count); - const forceX = new Float64Array(count); - const forceY = new Float64Array(count); - const pinned = new Uint8Array(count); - const pinX = new Float64Array(count); - const pinY = new Float64Array(count); - - const random = mulberry32(requireFinite(spec.seed ?? 1, 'spec.seed')); - - spec.nodes.forEach((node, i) => { - if (!node || typeof node.id !== 'string' || node.id.length === 0) { - throw new TypeError(`nodes[${i}].id must be a non-empty string`); - } - if (indexOfId.has(node.id)) throw new TypeError(`duplicate node id ${node.id}`); - indexOfId.set(node.id, i); - ids[i] = node.id; - const m = Math.max(params.minMass, requireFinite(node.mass, `nodes[${i}].mass`)); - mass[i] = m; - invMass[i] = 1 / m; - anchorX[i] = requireFinite(node.x0, `nodes[${i}].x0`); - anchorY[i] = requireFinite(node.y0, `nodes[${i}].y0`); - anchorK[i] = params.anchorBase * Math.pow(m, params.anchorMassExponent); - // c = 2 ζ sqrt(k m) is the damping coefficient that realises the target - // ratio against this node's own anchor spring. Applying it as a - // multiplicative velocity factor per substep (exp of the decay rate) means - // kinetic energy can only ever go DOWN in the damping half of the step, - // which is what keeps the post-release energy curve monotone. - const c = 2 * params.dampingRatio * Math.sqrt(anchorK[i]! * m); - dampPerSubstep[i] = Math.exp((-c / m) * params.substep); - }); - - // Seeded startup displacement, applied as an offset with zero initial - // velocity (prev === current), so the field falls into the layout. - for (let i = 0; i < count; i += 1) { - const angle = random() * Math.PI * 2; - const radius = params.jitter * (0.35 + random() * 0.65); - x[i] = anchorX[i]! + Math.cos(angle) * radius; - y[i] = anchorY[i]! + Math.sin(angle) * radius; - prevX[i] = x[i]!; - prevY[i] = y[i]!; - } - - const springCount = spec.springs.length; - const springA = new Int32Array(springCount); - const springB = new Int32Array(springCount); - const springK = new Float64Array(springCount); - const springRest = new Float64Array(springCount); - const adjacency = new Map>(); - ids.forEach((id) => adjacency.set(id, [])); - - spec.springs.forEach((spring, i) => { - const a = indexOfId.get(spring?.a); - const b = indexOfId.get(spring?.b); - if (a === undefined) { - throw new TypeError(`springs[${i}].a references unknown node ${String(spring?.a)}`); - } - if (b === undefined) { - throw new TypeError(`springs[${i}].b references unknown node ${String(spring?.b)}`); - } - if (a === b) throw new TypeError(`springs[${i}] is a self-loop on ${ids[a]}`); - const k = requireFinite(spring.stiffness, `springs[${i}].stiffness`); - if (k <= 0) throw new RangeError(`springs[${i}].stiffness must be > 0`); - springA[i] = a; - springB[i] = b; - springK[i] = k * params.edgeStiffnessScale; - springRest[i] = - spring.restLength === undefined - ? Math.hypot(anchorX[b]! - anchorX[a]!, anchorY[b]! - anchorY[a]!) - : requireFinite(spring.restLength, `springs[${i}].restLength`); - adjacency.get(ids[a]!)!.push({ spring: i, other: ids[b]! }); - adjacency.get(ids[b]!)!.push({ spring: i, other: ids[a]! }); - }); - - let stepCount = 0; - let substepCount = 0; - - function accumulateForces(): void { - for (let i = 0; i < count; i += 1) { - forceX[i] = -anchorK[i]! * (x[i]! - anchorX[i]!); - forceY[i] = -anchorK[i]! * (y[i]! - anchorY[i]!); - } - for (let s = 0; s < springCount; s += 1) { - const a = springA[s]!; - const b = springB[s]!; - const dx = x[b]! - x[a]!; - const dy = y[b]! - y[a]!; - const length = Math.hypot(dx, dy); - if (length === 0) continue; - const magnitude = springK[s]! * (length - springRest[s]!); - const fx = (dx / length) * magnitude; - const fy = (dy / length) * magnitude; - forceX[a] = forceX[a]! + fx; - forceY[a] = forceY[a]! + fy; - forceX[b] = forceX[b]! - fx; - forceY[b] = forceY[b]! - fy; - } - } - - function integrate(dt: number): void { - accumulateForces(); - const dt2 = dt * dt; - for (let i = 0; i < count; i += 1) { - if (pinned[i]) { - // A pinned node is the pointer, not a body: it goes exactly where the - // gesture says and carries no velocity of its own. Its neighbours feel - // it only through the channel springs. - x[i] = pinX[i]!; - y[i] = pinY[i]!; - prevX[i] = pinX[i]!; - prevY[i] = pinY[i]!; - continue; - } - const damp = dampPerSubstep[i]!; - const stepX = (x[i]! - prevX[i]!) * damp + forceX[i]! * invMass[i]! * dt2; - const stepY = (y[i]! - prevY[i]!) * damp + forceY[i]! * invMass[i]! * dt2; - prevX[i] = x[i]!; - prevY[i] = y[i]!; - x[i] = x[i]! + stepX; - y[i] = y[i]! + stepY; - } - substepCount += 1; - } - - function resolveIndex(nodeId: string): number { - const i = indexOfId.get(nodeId); - if (i === undefined) throw new TypeError(`unknown node id ${String(nodeId)}`); - return i; - } - - const sim: Simulation = { - get nodeIds() { - return ids.slice(); - }, - get nodeCount() { - return count; - }, - get springCount() { - return springCount; - }, - get params() { - return Object.freeze({ ...params }); - }, - get stepCount() { - return stepCount; - }, - get substepCount() { - return substepCount; - }, - - indexOf: resolveIndex, - massOf(nodeId) { - return mass[resolveIndex(nodeId)]!; - }, - anchorOf(nodeId) { - const i = resolveIndex(nodeId); - return { x: anchorX[i]!, y: anchorY[i]! }; - }, - springsOf(nodeId) { - resolveIndex(nodeId); - return adjacency.get(nodeId)!.map((entry) => ({ - other: entry.other, - stiffness: springK[entry.spring]!, - restLength: springRest[entry.spring]!, - })); - }, - - /** - * Advance by `dt` seconds, subdivided into equal substeps no larger than - * `params.substep`. Callers MUST pass a fixed `dt` (the surface drives one - * fixed step per animation frame), wall-clock jitter never reaches the - * integrator, which is the whole reason a run can be replayed. - */ - step(dt) { - requireFinite(dt, 'dt'); - if (dt <= 0) throw new RangeError('dt must be > 0'); - const substeps = Math.max(1, Math.ceil(dt / params.substep - 1e-9)); - const h = dt / substeps; - for (let s = 0; s < substeps; s += 1) integrate(h); - stepCount += 1; - return sim; - }, - - /** Pin a node to a gesture position. Idempotent per frame. */ - applyDrag(nodeId, targetX, targetY) { - const i = resolveIndex(nodeId); - pinned[i] = 1; - pinX[i] = requireFinite(targetX, 'targetX'); - pinY[i] = requireFinite(targetY, 'targetY'); - return sim; - }, - - /** Release every pinned node. Velocity stays zero, no fling. */ - release() { - pinned.fill(0); - return sim; - }, - - pinnedIds() { - const out: string[] = []; - for (let i = 0; i < count; i += 1) if (pinned[i]) out.push(ids[i]!); - return out; - }, - - /** Flat `[x0,y0,x1,y1,…]` copy in `nodeIds` order. */ - positions() { - const out = new Float64Array(count * 2); - for (let i = 0; i < count; i += 1) { - out[i * 2] = x[i]!; - out[i * 2 + 1] = y[i]!; - } - return out; - }, - positionOf(nodeId) { - const i = resolveIndex(nodeId); - return { x: x[i]!, y: y[i]! }; - }, - /** Flat `[vx0,vy0,…]` in px/s, derived from the Verlet position history. */ - velocities() { - const out = new Float64Array(count * 2); - const inv = 1 / params.substep; - for (let i = 0; i < count; i += 1) { - out[i * 2] = (x[i]! - prevX[i]!) * inv; - out[i * 2 + 1] = (y[i]! - prevY[i]!) * inv; - } - return out; - }, - maxSpeed() { - const inv = 1 / params.substep; - let worst = 0; - for (let i = 0; i < count; i += 1) { - if (pinned[i]) continue; - const speed = Math.hypot(x[i]! - prevX[i]!, y[i]! - prevY[i]!) * inv; - if (speed > worst) worst = speed; - } - return worst; - }, - /** - * Kinetic + anchor + channel energy. With every node damped and no node - * pinned, this can only fall; `sim.test.ts` asserts exactly that. - */ - energy() { - const inv = 1 / params.substep; - let kinetic = 0; - let anchor = 0; - for (let i = 0; i < count; i += 1) { - const vx = (x[i]! - prevX[i]!) * inv; - const vy = (y[i]! - prevY[i]!) * inv; - kinetic += 0.5 * mass[i]! * (vx * vx + vy * vy); - const dx = x[i]! - anchorX[i]!; - const dy = y[i]! - anchorY[i]!; - anchor += 0.5 * anchorK[i]! * (dx * dx + dy * dy); - } - let channel = 0; - for (let s = 0; s < springCount; s += 1) { - const stretch = - Math.hypot(x[springB[s]!]! - x[springA[s]!]!, y[springB[s]!]! - y[springA[s]!]!) - - springRest[s]!; - channel += 0.5 * springK[s]! * stretch * stretch; - } - return { kinetic, anchor, channel, total: kinetic + anchor + channel }; - }, - /** Signed stretch per spring, in px: what the renderer draws as tension. */ - stretches() { - const out = new Float64Array(springCount); - for (let s = 0; s < springCount; s += 1) { - out[s] = - Math.hypot(x[springB[s]!]! - x[springA[s]!]!, y[springB[s]!]! - y[springA[s]!]!) - - springRest[s]!; - } - return out; - }, - isSettled(restSpeed = params.restSpeed) { - return sim.maxSpeed() < restSpeed; - }, - /** - * Run the SAME `step()` the animated loop runs until the field is at rest. - * The reduced-motion mode is exactly this: identical arithmetic, drawn once - * instead of sixty times a second. That identity is the a11y guarantee, and - * it is asserted in `sim.test.ts`, not asserted in prose. - */ - settle({ dt = 1 / 60, maxFrames = 1200, restSpeed = params.restSpeed } = {}) { - let frames = 0; - while (frames < maxFrames) { - sim.step(dt); - frames += 1; - if (sim.maxSpeed() < restSpeed) break; - } - return frames; - }, - }; - - return sim; -} - -/** - * Hop distance over the channel graph, ignoring direction. - * - * @returns id → hops, unreachable ids omitted. - */ -export function hopDistances(sim: Simulation, sourceId: string): Map { - const seen = new Map([[sourceId, 0]]); - let frontier = [sourceId]; - while (frontier.length) { - const next: string[] = []; - for (const id of frontier) { - const hops = seen.get(id)! + 1; - for (const { other } of sim.springsOf(id)) { - if (seen.has(other)) continue; - seen.set(other, hops); - next.push(other); - } - } - frontier = next; - } - return seen; -} - -/** - * One exponential-approach step of the hover bloom. - * - * Bloom is not physics, it is the hover channel of the sensory contract, and - * it lives here so it is testable and lifts with the simulation rather than - * with a renderer. The approach RATE is divided by mass, so a leaf snaps and a - * hub arrives late and keeps arriving: "hover-response latency scales with - * degree", drawn from the same number that sets the node's inertia. - */ -export function bloomStep( - current: number, - target: number, - mass: number, - dt: number, - params: SimParams = DEFAULT_PARAMS, -): number { - const base = target > current ? params.bloomAttack : params.bloomRelease; - const rate = base / Math.pow(Math.max(params.minMass, mass), params.bloomMassExponent); - return current + (target - current) * (1 - Math.exp(-rate * dt)); -} diff --git a/dashboard/src/viz/trace/types.ts b/dashboard/src/viz/trace/types.ts index 0c734c7cad..c75b4f1322 100644 --- a/dashboard/src/viz/trace/types.ts +++ b/dashboard/src/viz/trace/types.ts @@ -1,25 +1,20 @@ /** - * Vocabulary for the TRACE surface, the drill-in that floods a selected - * symbol's call topography. + * Vocabulary for the TRACE surface, a selected symbol's call neighbourhood. * - * Three modules share these types and nothing else: `model.ts` turns the - * neighbors wire payload into a `TraceModel` (measurement → layout), `sim.ts` - * turns that model into forces (measurement → sensation), and `render.ts` - * draws whatever the other two produced and decides nothing. The split is the - * honesty boundary named in the plan's "Rendering strategy": every felt or - * drawn quantity has to be traceable to a field on one of these records. + * `model.ts` turns the neighbors wire payload into a `TraceModel`; `plate.ts` + * lays that model out as the anatomy plate and decides nothing about the + * data. Every drawn quantity has to be traceable to a field on these records. * * The one rule that governs every field below: if the wire did not carry it, * it is absent here and the surface says so in a caption. Nothing in this file * has a plausible default. */ -/** Which side of the focus a drawn channel lies on. Drawing direction only, - * the simulation treats all four as the same undirected spring. */ +/** Which side of the focus a drawn channel lies on. */ export type TraceChannelDirection = - /** Caller side: a tributary flowing into the focus. */ + /** Caller side. */ | 'up' - /** Callee side: the delta fanning out of it. */ + /** Callee side. */ | 'down' /** A lateral move between two members of the same membrane. */ | 'in' @@ -35,9 +30,8 @@ export interface TraceNode { readonly kind: string; /** * Total (in + out) edge count, as the neighbors endpoint reports it in - * `degree`. This is the node's MASS in the simulation and the width of its - * sill in the renderer. `null` when the payload omitted it, an unmeasured - * degree is never coerced to zero. + * `degree`. `null` when the payload omitted it, an unmeasured degree is + * never coerced to zero. */ readonly degree: number | null; /** `file_path` from the payload, or null when the row carried none. */ @@ -47,22 +41,19 @@ export interface TraceNode { /** * Signed hop ring: negative on the caller side, positive on the callee side, * 0 for the focus. This is the hop at which the symbol was FETCHED, which is - * exactly what the row position encodes, not elevation, not importance. + * exactly what the plate column encodes, not elevation, not importance. */ readonly ring: number; - /** Layout anchor in world coordinates. The simulation holds the node here. */ - readonly x0: number; - readonly y0: number; /** * Edges incident on this node that this frame does NOT draw, derived as - * `degree - drawn incident channels`. Drawn as a dashed mouth. `null` when + * `degree - drawn incident call sites`. `null` when * `degree` is absent, because an unmeasured degree cannot be differenced. */ readonly undrawnEdges: number | null; /** * Call sites where this symbol calls itself. A self-call is a real `calls` - * row and is reported, but it is not a channel: it couples no two bodies, so - * it carries no spring and no ribbon. Printed on the node instead. + * row and is reported, but it is not a channel: it couples no two symbols. + * Printed on the row instead. */ readonly selfCalls: number; } @@ -73,8 +64,7 @@ export interface TraceChannel { readonly b: string; /** * Call sites on this one edge: the number of `calls` rows the endpoint - * returned for this ordered pair. This is the channel's WIDTH and its spring - * STIFFNESS, the felt channel and the drawn channel are the same number. + * returned for this ordered pair. This is the length of its plate bar. */ readonly calls: number; readonly dir: TraceChannelDirection; @@ -84,8 +74,8 @@ export interface TraceChannel { * A type enclosure derived from `contains` edges in the neighbors payload. * * Only emitted when the payload actually carried `contains` edges whose - * container encloses at least two drawn members; otherwise the surface omits - * membranes entirely and says so, rather than inventing an enclosure from + * container encloses at least two drawn members; otherwise the surface counts + * none and says so, rather than inventing an enclosure from * shared file paths (wire-honesty: do not invent an enclosure). */ export interface TraceMembrane { @@ -130,108 +120,13 @@ export interface TraceCoverage { * carry them, it does not imply the code has no types. */ readonly membranesAvailable: boolean; - /** - * Every distinct field name observed on the neighbour rows this model was - * built from, sorted. - * - * Recorded because the sensory contract has five channels and this route - * serves the measurement behind two of them. Which two is a property of the - * PAYLOAD, not of a capability list someone typed here: the schemas are - * passthrough, so the day a producer starts sending a complexity or churn - * field, that field appears in this array and the corresponding channel goes - * live without a copy edit. Understating what the wire carries would be as - * false as overstating it, so neither is asserted, both are read. - */ - readonly rowFields: readonly string[]; -} - -/** Whether a sensory channel can be driven by the payload actually in hand. */ -export type SensoryChannelState = - /** The measurement arrived on this payload; the channel is live. */ - | 'measured' - /** - * No field on this payload carries the measurement. Absence of a field is - * not absence of the property, this says the wire was silent, nothing more. - */ - | 'not-on-this-wire' - /** - * The measurement exists but only at a coarser scope than this field draws, - * so binding it to a symbol here would be a fabricated join. - */ - | 'coarser-scope'; - -/** - * One channel of the app-wide sensory contract as it stands on THIS field. - * - * The contract is "sensation encodes a stated measurement". A channel with no - * measurement behind it must therefore be inert AND said out loud, because a - * surface that quietly animates four channels and drives two is claiming two - * measurements it does not have. - */ -export interface SensoryChannel { - /** The felt quantity, in the contract's own words. */ - readonly feel: string; - /** The measurement it is bound to, in the contract's own words. */ - readonly measurement: string; - readonly state: SensoryChannelState; - /** How this channel reads when motion is reduced. */ - readonly staticEquivalent: string; - /** - * The mechanism when measured, or why it is inert when not. Never empty: - * an unexplained inert channel is indistinguishable from a broken one. - */ - readonly note: string; } /** The complete drawable field: pure data, no DOM, no colour, no clock. */ export interface TraceModel { readonly focusId: string; - readonly world: { readonly width: number; readonly height: number }; - /** Ring → world y. Keys are the signed rings present in `nodes`. */ - readonly rows: ReadonlyMap; readonly nodes: readonly TraceNode[]; readonly channels: readonly TraceChannel[]; readonly membranes: readonly TraceMembrane[]; readonly coverage: TraceCoverage; } - -/** - * Resolved theme tokens. Canvas2D cannot read CSS custom properties, so the - * composing component samples `tokens.css` once per theme flip and hands the - * resolved strings in. That keeps the stylesheet the single source of the - * instrument's colour without a `getComputedStyle` call inside the draw loop. - */ -export interface TracePalette { - readonly surface0: string; - readonly surface1: string; - readonly textPrimary: string; - readonly textMuted: string; - readonly edgeSubtle: string; - readonly edgeStrong: string; - readonly grid: string; - readonly accent: string; - readonly upstream: string; - readonly downstream: string; - readonly statePartial: string; - readonly stateUnknown: string; - readonly membraneFill: string; - /** Whether the field is drawn against a light medium. */ - readonly light: boolean; -} - -/** One painted frame's worth of state. Everything here comes from the sim. */ -export interface TraceFrame { - /** Flat `[x0,y0,x1,y1,…]` in `model.nodes` order. */ - readonly positions: Float64Array; - /** Signed px off rest length, in `model.channels` order. */ - readonly stretches: Float64Array; - /** Hover bloom per node in `[0,1]`, in `model.nodes` order. */ - readonly bloom: Float64Array; - readonly draggingId: string | null; - readonly hoveredId: string | null; - /** - * Static-equivalent mode: the same settled positions, with tension drawn as - * thickness instead of as motion. A rendering mode, not a degradation. - */ - readonly reducedMotion: boolean; -} diff --git a/dashboard/src/workspaces/agents/AgentFailureContext.dom.test.tsx b/dashboard/src/workspaces/agents/AgentFailureContext.dom.test.tsx index c9b48c4235..4f75ff57a4 100644 --- a/dashboard/src/workspaces/agents/AgentFailureContext.dom.test.tsx +++ b/dashboard/src/workspaces/agents/AgentFailureContext.dom.test.tsx @@ -27,7 +27,7 @@ const OUTCOMES = [ const EVENTS = [ { timestamp: SECOND, tool_name: 'tracedecay_grep', event_kind: 'mcp_tool_call', outcome: 'success' }, - { timestamp: SECOND - 30, tool_name: 'tracedecay_read', event_kind: 'mcp_tool_call', outcome: 'error' }, + { timestamp: SECOND - 30, tool_name: 'tracedecay_source_lines', event_kind: 'mcp_tool_call', outcome: 'error' }, { timestamp: SECOND - 90, tool_name: 'Bash', event_kind: 'tool_call', outcome: 'timed_out' }, ]; @@ -87,7 +87,7 @@ describe('AgentFailureContext', () => { it('reads the failures off the served tape and says what the tape is', () => { renderContext(readAttemptFailures(attempts())); const tape = document.querySelector('[data-agent-failure-tape="2"]')!; - expect(within(tape as HTMLElement).getByText('tracedecay_read')).toBeTruthy(); + expect(within(tape as HTMLElement).getByText('tracedecay_source_lines')).toBeTruthy(); expect(within(tape as HTMLElement).getByText('Bash')).toBeTruthy(); expect(within(tape as HTMLElement).queryByText('tracedecay_grep')).toBeNull(); expect(screen.getByText(/nothing here explains why any of them failed/)).toBeTruthy(); diff --git a/dashboard/src/workspaces/agents/AgentHandoffTokens.tsx b/dashboard/src/workspaces/agents/AgentHandoffTokens.tsx index f47b7ab6e6..5174d92872 100644 --- a/dashboard/src/workspaces/agents/AgentHandoffTokens.tsx +++ b/dashboard/src/workspaces/agents/AgentHandoffTokens.tsx @@ -1,5 +1,5 @@ import type { ListedTaskHandoffV1 } from '../../contracts/generated.ts'; -import { ReadFailure } from '../../ui/LegacyStates.tsx'; +import { ReadFailure } from '../../ui/ReadFailure.tsx'; import { formatMicrosUtc } from '../../ui/format.ts'; import { handoffTargetLabel, type HandoffTokenReading } from './handoffTokens.ts'; diff --git a/dashboard/src/workspaces/agents/AgentInspector.tsx b/dashboard/src/workspaces/agents/AgentInspector.tsx index ba7c1783ab..bba1fbb0f8 100644 --- a/dashboard/src/workspaces/agents/AgentInspector.tsx +++ b/dashboard/src/workspaces/agents/AgentInspector.tsx @@ -1,5 +1,5 @@ import type { ReactNode } from 'react'; -import type { AnalyticsRecentHookV1 } from '../../contracts/generated.ts'; +import type { AnalyticsRecentHookV1, AnalyticsSubagentNodeV1 } from '../../contracts/generated.ts'; import { EvidenceGrade } from '../../ui/EvidenceGrade.tsx'; import { StateChip } from '../../ui/StateChip.tsx'; import { cn } from '../../ui/cn'; @@ -15,6 +15,13 @@ import type { DelegationTopologyModel } from './delegationTopology.ts'; import type { AttemptFailureReading } from './failure.ts'; import type { AgentHandoffReading } from './handoff.ts'; import { handoffTargetLabel, type HandoffTokenReading } from './handoffTokens.ts'; +import { + PartialMark, + coverageLabel, + sessionUsage, + tokenCount, + type UsageCoverage, +} from './sessionUsage.tsx'; import { subagentElapsedSeconds } from './subagentTree.ts'; /** @@ -276,6 +283,8 @@ export function AgentInspector({ />
    + +
    }> +

    + tokens absent · {coverageLabel(coverage)} +

    +
    + ); + } + return ( +
    : + } + > +
    + [label, tokenCount(value)] as const), + ['events', usage.events.toLocaleString()], + ]} + /> +
    + {usage.state === 'partial' ? ( +

    + the provider marked this aggregate incomplete · counts are a floor +

    + ) : null} +
    + ); +} + function TokenFrontier({ tokens, mode, diff --git a/dashboard/src/workspaces/agents/AgentTelemetryRegister.tsx b/dashboard/src/workspaces/agents/AgentTelemetryRegister.tsx index 1fd1161895..ed21e09e90 100644 --- a/dashboard/src/workspaces/agents/AgentTelemetryRegister.tsx +++ b/dashboard/src/workspaces/agents/AgentTelemetryRegister.tsx @@ -1,5 +1,5 @@ import { OverviewCard, OverviewGrid } from '../../ui/archetypes/OverviewGrid'; -import { ReadFailure } from '../../ui/LegacyStates.tsx'; +import { ReadFailure } from '../../ui/ReadFailure.tsx'; import { ReadSection, envelopeReadState } from '../../ui/ReadSection.tsx'; import { MeterRow, ReadoutBar } from '../../ui/instrument.tsx'; import { cn } from '../../ui/cn'; diff --git a/dashboard/src/workspaces/agents/AgentToolActivity.dom.test.tsx b/dashboard/src/workspaces/agents/AgentToolActivity.dom.test.tsx index 2a606ea49b..25d30d4527 100644 --- a/dashboard/src/workspaces/agents/AgentToolActivity.dom.test.tsx +++ b/dashboard/src/workspaces/agents/AgentToolActivity.dom.test.tsx @@ -30,7 +30,7 @@ function read(overrides: Partial = {}): ToolActivityRead { { agent: 'Codex', tool_name: 'tracedecay_grep', session_id: 's1' }, { agent: 'Codex', tool_name: 'tracedecay_grep', session_id: 's1' }, { agent: 'Codex', tool_name: 'Bash', session_id: 's2' }, - { agent: 'Claude', tool_name: 'tracedecay_body', session_id: 's3' }, + { agent: 'Claude', tool_name: 'tracedecay_source_body', session_id: 's3' }, ], hook_window: { truncated: true }, ...overrides, diff --git a/dashboard/src/workspaces/agents/AgentsPage.dom.test.tsx b/dashboard/src/workspaces/agents/AgentsPage.dom.test.tsx index d936864a17..ec46e7efb2 100644 --- a/dashboard/src/workspaces/agents/AgentsPage.dom.test.tsx +++ b/dashboard/src/workspaces/agents/AgentsPage.dom.test.tsx @@ -1,3 +1,4 @@ +import { MemoryRouter } from 'react-router'; import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; import { fireEvent, render, screen, within } from '@testing-library/react'; import { afterEach, describe, expect, it, vi } from 'vitest'; @@ -177,7 +178,7 @@ describe('AgentsPage read coverage', () => { recent_events: [ { timestamp: 1_700_000_000, - tool_name: 'tracedecay_read', + tool_name: 'tracedecay_source_lines', outcome: 'error', event_kind: 'post_tool_use', hook_name: 'post_tool_use', @@ -543,7 +544,9 @@ function renderAgents() { }); return render( - + + + , ); } diff --git a/dashboard/src/workspaces/agents/AgentsPage.topology.dom.test.tsx b/dashboard/src/workspaces/agents/AgentsPage.topology.dom.test.tsx index a43081f77b..a338dd440b 100644 --- a/dashboard/src/workspaces/agents/AgentsPage.topology.dom.test.tsx +++ b/dashboard/src/workspaces/agents/AgentsPage.topology.dom.test.tsx @@ -1,3 +1,4 @@ +import { MemoryRouter } from 'react-router'; import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; import { fireEvent, render, screen, waitFor, within } from '@testing-library/react'; import { http, HttpResponse, type JsonBodyType } from 'msw'; @@ -24,7 +25,9 @@ function renderAgents() { }); return render( - + + + , ); } @@ -33,8 +36,12 @@ const inspector = () => document.querySelector('[data-agent-inspector]')!; const mark = (id: string) => document.querySelector(`[data-topology-control="session"][data-topology-id="${id}"]`)!; +/** Settled once the population line reconciles every session it read as + * drawn, whatever the fixture's population is. */ async function settled() { - await screen.findByText('5 sessions · 5 drawn · 3 generations'); + const line = await screen.findByText(/^\d+ sessions · \d+ drawn · \d+ generations$/); + const [, read, drawn] = /^(\d+) sessions · (\d+) drawn/.exec(line.textContent!)!; + expect(drawn).toBe(read); await waitFor(() => expect(inspector().getAttribute('data-agent-inspector-mode')).toBe('default'), ); @@ -45,18 +52,13 @@ describe('AgentsPage delegation topology', () => { renderAgents(); await settled(); - // Every drawn session is a real button in column order; the fixture's five - // sessions are all drawn, so nothing is folded and the counts reconcile. - const controls = [...document.querySelectorAll('[data-topology-control="session"]')].map( - (node) => node.getAttribute('data-topology-id'), - ); - expect(controls).toEqual([ - 'codex:session.codex.root', - 'claude:session.claude.orphan', - 'cursor:session.cursor.solo', - 'codex:session.codex.child', - 'codex:session.codex.grandchild', - ]); + // Every session the reading holds is a real button, and the buttons run in + // column order: generation never decreases along the tab order. + const served = (resolveFixture('/api/plugins/analytics/subagent-tree') as { payload: { nodes: unknown[] } }).payload; + const controls = [...document.querySelectorAll('[data-topology-control="session"]')]; + expect(controls).toHaveLength(served.nodes.length); + const generations = controls.map((node) => Number(/generation (\d+)/.exec(node.getAttribute('aria-label')!)![1])); + expect(generations).toEqual([...generations].sort((a, b) => a - b)); expect(document.querySelectorAll('[data-topology-control="bundle"]')).toHaveLength(0); // The cut edge is drawn as a typed stub, not as a root. expect(document.querySelector('[data-topology-stub="missing_parent"]')).toBeTruthy(); diff --git a/dashboard/src/workspaces/agents/AgentsPage.transport.dom.test.tsx b/dashboard/src/workspaces/agents/AgentsPage.transport.dom.test.tsx index 92b5b9e6ba..c34f166f8f 100644 --- a/dashboard/src/workspaces/agents/AgentsPage.transport.dom.test.tsx +++ b/dashboard/src/workspaces/agents/AgentsPage.transport.dom.test.tsx @@ -1,3 +1,4 @@ +import { MemoryRouter } from 'react-router'; import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; import { render, screen } from '@testing-library/react'; import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest'; @@ -26,7 +27,7 @@ beforeAll(() => server.listen({ onUnhandledRequest: 'error' })); afterEach(() => server.resetHandlers()); afterAll(() => server.close()); -/** The sentence `LegacyStates` prints for each state it can reach. Distinct +/** The sentence `ReadFailure` prints for each state it can reach. Distinct * wording per state is the point: a reader has to be able to tell "the daemon * is down" from "the daemon answered wrong" without opening a console. */ const GUIDANCE = { @@ -178,7 +179,9 @@ function renderAgents() { }); return render( - + + + , ); } diff --git a/dashboard/src/workspaces/agents/AgentsPage.tsx b/dashboard/src/workspaces/agents/AgentsPage.tsx index 28e414d9c9..afbed4bfae 100644 --- a/dashboard/src/workspaces/agents/AgentsPage.tsx +++ b/dashboard/src/workspaces/agents/AgentsPage.tsx @@ -1,6 +1,6 @@ import { useCallback, useMemo, useState } from 'react'; import { OverviewCard, OverviewGrid } from '../../ui/archetypes/OverviewGrid'; -import { ReadFailure } from '../../ui/LegacyStates.tsx'; +import { ReadFailure } from '../../ui/ReadFailure.tsx'; import { ReadSection, envelopeReadState } from '../../ui/ReadSection.tsx'; import { StateChip } from '../../ui/StateChip.tsx'; import { MeterRow, Panel, WorkspaceHeader } from '../../ui/instrument.tsx'; @@ -22,7 +22,8 @@ import { AgentHandoffs } from './AgentHandoffs.tsx'; import { AgentHandoffTokens } from './AgentHandoffTokens.tsx'; import { AgentInspector, type DiagnosticsForInspector } from './AgentInspector.tsx'; import { AgentTelemetryRegister } from './AgentTelemetryRegister.tsx'; -import { DelegationTopology } from './DelegationTopology.tsx'; +import { DelegationTimeline } from './DelegationTimeline.tsx'; +import { DelegationTopology, type TopologyInteraction } from './DelegationTopology.tsx'; import { SubagentTree } from './SubagentTree.tsx'; import { resolveSubject } from './agentInspector.ts'; import { useAgentWorkGraph } from './agentWorkQuery.ts'; @@ -33,11 +34,12 @@ import { usageAuthority, workAuthority, } from './authorityRegister.ts'; -import { fitDelegationTopology, markId } from './delegationTopology.ts'; +import { fitDelegationTopology, markId, type FittedTopology } from './delegationTopology.ts'; import { readAttemptFailures } from './failure.ts'; import { readHandoffFrontier } from './handoff.ts'; import { newestTreeSession, useAgentHandoffTokens } from './handoffTokenQuery.ts'; import { readHandoffTokens } from './handoffTokens.ts'; +import { AgentsViewSwitcher, agentsViewNote, useAgentsView, type AgentsView } from './agentsView.tsx'; const BASE = '/api/plugins/analytics'; @@ -106,6 +108,7 @@ export function AgentsPage() { const [inspectedId, setInspectedId] = useState(null); const [selectedId, setSelectedId] = useState(null); const [expanded, setExpanded] = useState>(() => new Set()); + const [view, setView] = useAgentsView(); const toggleExpanded = useCallback((id: string) => { setExpanded((current) => { const next = new Set(current); @@ -211,8 +214,11 @@ export function AgentsPage() { +
    + +
    @@ -222,7 +228,7 @@ export function AgentsPage() {
    {fit !== null && fit.model.marks.length > 0 ? ( - + <> + + ) : null}
    ; + case 'timeline': + return ; + default: { + const unhandled: never = view; + return unhandled; + } + } +} + /** The model the inspector is handed before the tree has been read. */ const EMPTY_MODEL = fitDelegationTopology({ available: true, diff --git a/dashboard/src/workspaces/agents/AgentsPage.usage.dom.test.tsx b/dashboard/src/workspaces/agents/AgentsPage.usage.dom.test.tsx new file mode 100644 index 0000000000..a9a388e07d --- /dev/null +++ b/dashboard/src/workspaces/agents/AgentsPage.usage.dom.test.tsx @@ -0,0 +1,133 @@ +import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; +import { fireEvent, render, screen, waitFor } from '@testing-library/react'; +import { http, HttpResponse, type JsonBodyType } from 'msw'; +import { MemoryRouter } from 'react-router'; +import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest'; +import { resolveFixture } from '../../../stories/fixtures/data.ts'; +import { fixtureServer } from '../../../stories/fixtures/handlers.ts'; +import { AgentsPage } from './AgentsPage.tsx'; + +/** + * Per-session provider usage on both views. The fixture tree holds all three + * node states: measured (root, orphan), partial (child: `complete: false`), + * and absent (grandchild, solo), under a tree coverage of `partial`. + */ +const server = fixtureServer(); +beforeAll(() => server.listen({ onUnhandledRequest: 'error' })); +afterEach(() => server.resetHandlers()); +afterAll(() => server.close()); + +async function renderAgents(entry = '/agents') { + const client = new QueryClient({ defaultOptions: { queries: { retry: false, gcTime: 0 } } }); + render( + + + + + , + ); + await screen.findByText(/^\d+ sessions · \d+ drawn · \d+ generations$/); +} + +const control = (id: string) => + document.querySelector(`[data-topology-control="session"][data-topology-id="${id}"]`)!; + +/** Every session control's usage label, as rendered, in tab order. */ +function usageLabels(scope: string) { + return [...document.querySelectorAll(`${scope} [data-topology-control="session"]`)].map((node) => { + const label = node.querySelector('[data-usage-state]')!; + return [ + node.getAttribute('data-topology-id'), + label.getAttribute('data-usage-state'), + label.querySelector('[data-usage-total]')!.textContent, + label.querySelector('[data-usage-partial]')?.textContent ?? null, + ]; + }); +} + +const inspectorUsage = () => document.querySelector('[data-agent-inspector-usage]')!; + +describe('agents per-session provider usage', () => { + it('rings print each total with its unit, the partial marker, or absent', async () => { + await renderAgents(); + expect(usageLabels('[data-delegation-topology]')).toEqual([ + ['codex:session.codex.root', 'measured', '1,515,630 tokens', null], + ['claude:session.claude.orphan', 'measured', '108,400 tokens', null], + ['cursor:session.cursor.solo', 'absent', 'tokens absent', null], + ['codex:session.codex.child', 'partial', '357,250 tokens', 'partial'], + ['codex:session.codex.grandchild', 'absent', 'tokens absent', null], + ]); + const legend = document.querySelector('[data-usage-coverage]')!; + expect(legend.getAttribute('data-usage-coverage')).toBe('partial'); + expect(legend.textContent).toBe('partialtokens · usage read partial · absent may be unread'); + }); + + it('the inspector splits a measured session by counter', async () => { + await renderAgents(); + fireEvent.click(control('codex:session.codex.root')); + await waitFor(() => expect(inspectorUsage().getAttribute('data-agent-inspector-usage')).toBe('measured')); + expect(inspectorUsage().textContent).toBe( + 'total1,515,630 tokensinput182,400 tokensoutput24,310 tokenscache read1,204,800 tokens' + + 'cache write96,000 tokensreasoning8,120 tokensevents64', + ); + }); + + it('a partial aggregate keeps its counters, marks unreported ones, and says it is a floor', async () => { + await renderAgents(); + fireEvent.click(control('codex:session.codex.child')); + await waitFor(() => expect(inspectorUsage().getAttribute('data-agent-inspector-usage')).toBe('partial')); + expect(inspectorUsage().textContent).toBe( + 'total357,250 tokensinput41,200 tokensoutput6,050 tokenscache read310,000 tokens' + + 'cache writeunreportedreasoningunreportedevents11', + ); + const section = screen.getByRole('region', { name: 'Provider usage' }); + expect(section.querySelector('[data-usage-partial]')!.textContent).toBe('partial'); + expect(section.textContent).toContain('the provider marked this aggregate incomplete · counts are a floor'); + }); + + it('an absent session reads against the tree coverage', async () => { + await renderAgents(); + fireEvent.click(control('codex:session.codex.grandchild')); + await waitFor(() => expect(inspectorUsage().getAttribute('data-agent-inspector-usage')).toBe('absent')); + expect(inspectorUsage().textContent).toBe('tokens absent · usage read partial · absent may be unread'); + }); + + it('the timeline rows carry the same labels', async () => { + await renderAgents('/agents?view=timeline'); + expect(usageLabels('[data-delegation-timeline]').map(([id, state, total]) => [id, state, total])).toEqual([ + ['codex:session.codex.root', 'measured', '1,515,630 tokens'], + ['codex:session.codex.child', 'partial', '357,250 tokens'], + ['codex:session.codex.grandchild', 'absent', 'tokens absent'], + ['claude:session.claude.orphan', 'measured', '108,400 tokens'], + ['cursor:session.cursor.solo', 'absent', 'tokens absent'], + ]); + expect(document.querySelector('[data-usage-coverage]')!.getAttribute('data-usage-coverage')).toBe('partial'); + }); + + it('a failed usage read is distinguishable from none recorded', async () => { + const served = resolveFixture('/api/plugins/analytics/subagent-tree') as { + payload: { nodes: Record[] }; + }; + const payload = { + ...served.payload, + usage_coverage: 'unavailable', + nodes: served.payload.nodes.map(({ usage: _usage, ...node }) => node), + }; + server.use( + http.get('*/api/plugins/analytics/subagent-tree', () => + HttpResponse.json({ ...served, payload } as JsonBodyType), + ), + ); + await renderAgents(); + expect(new Set(usageLabels('[data-delegation-topology]').map(([, state, total]) => `${state}:${total}`))).toEqual( + new Set(['absent:tokens absent']), + ); + expect(document.querySelector('[data-usage-coverage]')!.textContent).toBe( + 'tokens · usage read unavailable · every session absent', + ); + fireEvent.click(control('codex:session.codex.root')); + await waitFor(() => + expect(inspectorUsage().textContent).toBe('tokens absent · usage read unavailable · every session absent'), + ); + }); +}); diff --git a/dashboard/src/workspaces/agents/AgentsPage.views.dom.test.tsx b/dashboard/src/workspaces/agents/AgentsPage.views.dom.test.tsx new file mode 100644 index 0000000000..fcb62f9e93 --- /dev/null +++ b/dashboard/src/workspaces/agents/AgentsPage.views.dom.test.tsx @@ -0,0 +1,115 @@ +import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; +import { fireEvent, render, screen, waitFor } from '@testing-library/react'; +import { MemoryRouter, useLocation } from 'react-router'; +import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest'; +import { fixtureServer } from '../../../stories/fixtures/handlers.ts'; +import { AgentsPage } from './AgentsPage.tsx'; + +/** + * The two Agents views over the fixture reading. Topology is the default and + * draws hollow rings; Timeline is a real view chosen from the workspace's view + * bar and kept in `?view`. Both hand the page the same acts: hover inspects + * without selecting, click selects and lifts the selection's subtree. + */ +const server = fixtureServer(); +beforeAll(() => server.listen({ onUnhandledRequest: 'error' })); +afterEach(() => server.resetHandlers()); +afterAll(() => server.close()); + +const address = { search: '' }; +function AddressProbe() { + address.search = useLocation().search; + return null; +} + +const inspector = () => document.querySelector('[data-agent-inspector]')!; +const control = (id: string) => + document.querySelector(`[data-topology-control="session"][data-topology-id="${id}"]`)!; + +/** Settled once the population line reconciles: every session read is drawn. */ +async function renderAgents(entry = '/agents') { + const client = new QueryClient({ defaultOptions: { queries: { retry: false, gcTime: 0 } } }); + render( + + + + + + , + ); + const line = await screen.findByText(/^\d+ sessions · \d+ drawn · \d+ generations$/); + const [, read, drawn] = /^(\d+) sessions · (\d+) drawn/.exec(line.textContent!)!; + expect(drawn).toBe(read); +} + +describe('agents views', () => { + it('topology is the default: rings named by kind, cores lit by sessions beneath', async () => { + await renderAgents(); + expect(document.querySelector('[data-agents-view]')!.getAttribute('data-agents-view')).toBe('topology'); + const rings = [...document.querySelectorAll('[data-topology-ring]')].map((node) => [ + node.getAttribute('data-topology-ring'), + node.getAttribute('data-topology-core'), + ]); + expect(rings).toEqual([ + ['origin', '0.36'], + ['cut', '0.06'], + ['origin', '0.06'], + ['delegate', '0.249'], + ['delegate', '0.06'], + ]); + fireEvent.mouseEnter(control('codex:session.codex.child')); + const card = document.querySelector('[data-topology-hovercard="codex:session.codex.child"]')!; + expect(card.textContent).toContain('beneath1'); + expect(card.textContent).toContain('span1,400 s'); + expect(card.textContent).toContain('tokens357,250 tokenspartial'); + expect(inspector().getAttribute('data-agent-inspector-mode')).toBe('inspecting'); + expect(control('codex:session.codex.child').getAttribute('aria-pressed')).toBe('false'); + }); + + it('a selection lifts its subtree with a halo while the rest dims', async () => { + await renderAgents(); + fireEvent.click(control('codex:session.codex.child')); + fireEvent.mouseLeave(screen.getByRole('group', { name: 'Delegation topology field' })); + await waitFor(() => expect(inspector().getAttribute('data-agent-inspector-mode')).toBe('selected')); + expect(document.querySelectorAll('[data-topology-selected]')).toHaveLength(1); + expect(document.querySelectorAll('[data-topology-halo]')).toHaveLength(1); + expect(document.querySelectorAll('[data-topology-lifted]')).toHaveLength(1); + expect(control('codex:session.codex.root').className).toContain('opacity-40'); + expect(control('codex:session.codex.grandchild').className).not.toContain('opacity-40'); + }); + + it('timeline is a view kept in the address, with open ends, inferred joins and a recency ramp', async () => { + await renderAgents(); + fireEvent.click(document.querySelector('[data-agents-view-option="timeline"]')!); + await waitFor(() => expect(address.search).toBe('?view=timeline')); + expect(screen.getByText('Delegation timeline · read-only')).toBeTruthy(); + expect( + [...document.querySelectorAll('[data-delegation-timeline] [data-topology-control="session"]')].map((node) => + node.getAttribute('data-topology-id'), + ), + ).toEqual([ + 'codex:session.codex.root', + 'codex:session.codex.child', + 'codex:session.codex.grandchild', + 'claude:session.claude.orphan', + 'cursor:session.cursor.solo', + ]); + expect(document.querySelectorAll('[data-timeline-bracket="spawn"]')).toHaveLength(2); + expect(document.querySelectorAll('[data-timeline-bracket="join"]')).toHaveLength(1); + expect(document.querySelectorAll('[data-timeline-bar="open"]')).toHaveLength(1); + expect(document.querySelector('[data-timeline-lane="0"]')!.getAttribute('data-timeline-peak')).toBe('2'); + expect( + [...document.querySelectorAll('[data-timeline-recency]')].map((node) => node.getAttribute('data-timeline-recency')), + ).toEqual(['0.4', '0.267', '0.4', '0.225', '0.308']); + + fireEvent.click(control('codex:session.codex.child')); + await waitFor(() => expect(inspector().getAttribute('data-agent-inspector-mode')).toBe('selected')); + expect(control('codex:session.codex.child').getAttribute('aria-pressed')).toBe('true'); + }); + + it('opens on the view the address names', async () => { + await renderAgents('/agents?view=timeline'); + expect(document.querySelector('[data-agents-view]')!.getAttribute('data-agents-view')).toBe('timeline'); + expect(document.querySelector('[data-delegation-timeline]')).not.toBeNull(); + }); +}); diff --git a/dashboard/src/workspaces/agents/DelegationTimeline.tsx b/dashboard/src/workspaces/agents/DelegationTimeline.tsx new file mode 100644 index 0000000000..440b30ec87 --- /dev/null +++ b/dashboard/src/workspaces/agents/DelegationTimeline.tsx @@ -0,0 +1,390 @@ +import { useId, useMemo, useRef } from 'react'; +import { cn } from '../../ui/cn'; +import { + OpenedStrip, + Swatch, + TopologyPopulation, + useApertureWidth, + type TopologyInteraction, +} from './DelegationTopology.tsx'; +import { neighbourhood, subtree, type FittedTopology, type TopologyMark } from './delegationTopology.ts'; +import { + layoutDelegationTimeline, + timelineTickLabel, + timelineTicks, + type TimelineRow, +} from './delegationTimeline.ts'; +import { subagentElapsedSeconds } from './subagentTree.ts'; +import { markHandlers } from './agentsView.tsx'; +import { UsageCoverageLegend, UsageLabel } from './sessionUsage.tsx'; + +/** + * The delegation timeline: recorded time across, the delegation hierarchy + * down. Each row is one drawn mark in pre-order and is itself the 44px + * control, so the field keeps the disc field's operability without a second + * tab order. Bars, brackets and density are SVG decoration of those rows. + */ + +const GUTTER = 232; +const PAD_RIGHT = 40; +const ROW = 44; +const LANE_HEADER = 24; +const LANE_GAP = 10; +const AXIS = 30; +const INDENT = 12; +const MIN_WIDTH = 640; + +export function DelegationTimeline({ + fit, + interaction, +}: { + fit: FittedTopology; + interaction: TopologyInteraction; +}) { + const { model } = fit; + const { inspectedId, selectedId } = interaction; + const apertureRef = useRef(null); + const width = Math.max(MIN_WIDTH, useApertureWidth(apertureRef) ?? 960); + const hatchId = useId(); + const timeline = useMemo(() => layoutDelegationTimeline(model), [model]); + const keep = useMemo( + () => + inspectedId !== null && model.marks.some((mark) => mark.id === inspectedId) + ? neighbourhood(model, inspectedId) + : null, + [model, inspectedId], + ); + const lifted = useMemo( + () => (keep === null && selectedId !== null ? subtree(model, selectedId) : null), + [keep, model, selectedId], + ); + const isDim = (id: string) => (keep !== null ? !keep.has(id) : lifted !== null && !lifted.has(id)); + const linked = (a: string, b: string) => + keep !== null ? keep.has(a) && keep.has(b) : lifted !== null && lifted.has(a) && lifted.has(b); + + const laneTop: number[] = []; + let cursor = AXIS; + for (const lane of timeline.lanes) { + laneTop.push(cursor); + cursor += LANE_HEADER + lane.rows * ROW + LANE_GAP; + } + const height = cursor; + const rowY = (row: TimelineRow) => { + const lane = timeline.lanes[row.lane]!; + return laneTop[row.lane]! + LANE_HEADER + (row.index - lane.firstRow) * ROW + ROW / 2; + }; + const plot = width - GUTTER - PAD_RIGHT; + const domain = timeline.domain; + const x = (at: number) => + domain === null ? GUTTER : GUTTER + ((at - domain.start) / (domain.end - domain.start)) * plot; + const ticks = domain === null ? null : timelineTicks(domain, plot); + const peak = timeline.lanes.reduce((max, lane) => Math.max(max, lane.peak), 0); + + return ( +
    +
    interaction.onInspect(null)} + onBlur={(event) => { + const next = event.relatedTarget; + if (!(next instanceof Node) || !event.currentTarget.contains(next)) interaction.onInspect(null); + }} + > +
    + + + + + + + {ticks?.ticks.map((at) => ( + + + + {timelineTickLabel(at, ticks.step)} + + + ))} + + UTC · RECORDED START → END + + {timeline.lanes.map((lane) => { + const top = laneTop[lane.index]!; + const bandHeight = LANE_HEADER - 8; + const points = + domain === null || peak === 0 + ? '' + : lane.density + .map((step, index) => { + const next = lane.density[index + 1]; + const y = top + 4 + bandHeight - (step.count / peak) * bandHeight; + return `L${x(step.at)},${y} L${next ? x(next.at) : x(step.at)},${y}`; + }) + .join(' '); + return ( + + + + {points === '' ? null : ( + + )} + + ); + })} + {timeline.brackets.map((bracket) => { + const parent = timeline.rows[bracket.parentRow]!; + const child = timeline.rows[bracket.childRow]!; + const lit = linked(parent.mark.id, child.mark.id); + const dim = (keep !== null || lifted !== null) && !lit; + const at = x(bracket.at); + const y1 = rowY(parent); + const y2 = rowY(child); + return bracket.kind === 'spawn' ? ( + + ) : ( + + ); + })} + {timeline.rows.map((row) => ( + + ))} + + {timeline.lanes.map((lane) => ( + + lane {lane.index + 1} · peak {lane.peak} + {lane.unmeasured > 0 ? ` · ${lane.unmeasured} unmeasured` : ''} + + ))} +
      + {timeline.rows.map((row) => ( +
    • + +
      • + ))} +
    +
    +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + {timeline.unplaced} unplaced · start absent + + {timeline.open} open · end absent + +
    + +
    + ); +} + +/** Bar fill from recency within the loaded page: the latest-ending bar reads + * brightest, the earliest faintest. Relative to this reading's own extent, + * so it says "later in what was loaded", never "live". */ +export function timelineRecency(row: TimelineRow, domain: { start: number; end: number }): number { + const at = row.end ?? domain.end; + const fraction = Math.max(0, Math.min(1, (at - domain.start) / (domain.end - domain.start))); + return Math.round((0.1 + 0.3 * fraction) * 1000) / 1000; +} + +function barTone(mark: TopologyMark): { stroke: string; fill: string } { + if (mark.kind === 'bundle') return { stroke: 'var(--raw-graph-text)', fill: 'hatch' }; + switch (mark.node.link) { + case 'missing_parent': + return { stroke: 'var(--raw-graph-alert)', fill: 'var(--raw-graph-alert)' }; + case 'cycle': + return { stroke: 'var(--raw-state-conflicting)', fill: 'var(--raw-state-conflicting)' }; + case 'root': + case 'linked': + return { stroke: 'var(--raw-graph-accent)', fill: 'var(--raw-graph-accent)' }; + default: { + const unhandled: never = mark.node.link; + return unhandled; + } + } +} + +function TimelineBar({ + row, + y, + x, + domain, + hatchId, + selected, + lifted, + dim, +}: { + row: TimelineRow; + y: number; + x: (at: number) => number; + domain: { start: number; end: number } | null; + hatchId: string; + selected: boolean; + lifted: boolean; + dim: boolean; +}) { + const domainEnd = domain?.end ?? null; + const tone = barTone(row.mark); + const stub = row.mark.kind === 'session' && row.mark.parentId === null && row.mark.node.link !== 'root'; + if (row.start === null || domainEnd === null) { + return ( + + start absent · not placed on time + + ); + } + const x1 = x(row.start); + const open = row.end === null; + const x2 = Math.max(x1 + 3, x(row.end ?? domainEnd)); + const recency = timelineRecency(row, domain!); + return ( + + {lifted ? : null} + {selected ? : null} + {stub ? ( + + ) : null} + + {open ? ( + + open + + ) : null} + + ); +} + +function rowDetail(mark: TopologyMark): string { + if (mark.kind === 'bundle') { + return `${mark.sessions} sessions${mark.descendants > 0 ? ` · ${mark.descendants} beneath` : ''} · open bundle`; + } + const elapsed = subagentElapsedSeconds(mark.node); + return [ + mark.node.provider, + mark.node.descendants > 0 ? `${mark.node.descendants} beneath` : null, + mark.foldedDescendants > 0 ? `${mark.foldedDescendants} folded` : null, + elapsed === null ? 'span absent' : `${elapsed.toLocaleString()}s`, + ] + .filter(Boolean) + .join(' · '); +} + +function TimelineRowControl({ + row, + top, + width, + interaction, + dim, +}: { + row: TimelineRow; + top: number; + width: number; + interaction: TopologyInteraction; + dim: boolean; +}) { + const selected = row.mark.id === interaction.selectedId; + return ( + + ); +} diff --git a/dashboard/src/workspaces/agents/DelegationTopology.tsx b/dashboard/src/workspaces/agents/DelegationTopology.tsx index c8400f1cb1..2871d96a60 100644 --- a/dashboard/src/workspaces/agents/DelegationTopology.tsx +++ b/dashboard/src/workspaces/agents/DelegationTopology.tsx @@ -1,14 +1,16 @@ import { useEffect, useId, useMemo, useRef, useState, type ReactNode } from 'react'; import { cn } from '../../ui/cn'; import { subagentElapsedSeconds } from './subagentTree.ts'; +import { RingGlyph, RingHoverCard, ringCountLine, ringRadius } from './delegationRings.tsx'; +import { UsageCoverageLegend, UsageLabel } from './sessionUsage.tsx'; import { TOPOLOGY_GEOMETRY, columnPitchFor, edgePath, fieldSize, markPosition, - markRadius, neighbourhood, + subtree, type DelegationTopologyModel, type FittedTopology, type TopologyBundleMark, @@ -54,7 +56,7 @@ export interface TopologyInteraction { /** The aperture's own width, so columns can stretch to fill it. `null` until * measured, which under jsdom is forever, the default pitch then holds. */ -function useApertureWidth(ref: React.RefObject): number | null { +export function useApertureWidth(ref: React.RefObject): number | null { const [width, setWidth] = useState(null); useEffect(() => { const node = ref.current; @@ -76,6 +78,7 @@ export function DelegationTopology({ interaction: TopologyInteraction; }) { const { model } = fit; + const radiusOf = (mark: TopologyMark) => ringRadius(mark, model.maxDescendants); const { inspectedId, selectedId } = interaction; const apertureRef = useRef(null); const apertureWidth = useApertureWidth(apertureRef); @@ -90,7 +93,13 @@ export function DelegationTopology({ () => (inspectedId !== null && byId.has(inspectedId) ? neighbourhood(model, inspectedId) : null), [model, byId, inspectedId], ); - const isDim = (id: string) => keep !== null && !keep.has(id); + // A selection lifts its subtree while nothing is being inspected; hover + // answers its own question first and the lift returns when it ends. + const lifted = useMemo( + () => (keep === null && selectedId !== null && byId.has(selectedId) ? subtree(model, selectedId) : null), + [keep, model, byId, selectedId], + ); + const isDim = (id: string) => (keep !== null ? !keep.has(id) : lifted !== null && !lifted.has(id)); return (
    @@ -155,25 +164,28 @@ export function DelegationTopology({ const from = byId.get(edge.from); const to = byId.get(edge.to); if (!from || !to) return null; - const lit = keep !== null && keep.has(edge.from) && keep.has(edge.to); - const dim = keep !== null && !lit; + const isolated = keep !== null && keep.has(edge.from) && keep.has(edge.to); + const inLift = lifted !== null && lifted.has(edge.from) && lifted.has(edge.to); + const lit = isolated || inLift; + const dim = (keep !== null || lifted !== null) && !lit; return ( ); })} @@ -181,7 +193,7 @@ export function DelegationTopology({ const to = byId.get(stub.to); if (!to) return null; const at = markPosition(to, geometry); - const radius = markRadius(to, model.maxDescendants); + const radius = radiusOf(to); const dim = isDim(stub.to); if (stub.kind === 'missing_parent') { return ( @@ -222,14 +234,15 @@ export function DelegationTopology({ ); })} {model.marks.map((mark) => ( - ))} @@ -243,17 +256,27 @@ export function DelegationTopology({
    • ))} + {inspectedId !== null && byId.has(inspectedId) ? ( +
    + +
    + ) : null}
    - +
    ); @@ -265,7 +288,7 @@ export function DelegationTopology({ * bundle has no mark of its own any more, and the inspector's subject moves * with the pointer; this strip is reachable whatever is being inspected. */ -function OpenedStrip({ +export function OpenedStrip({ model, onToggleExpanded, }: { @@ -384,140 +407,26 @@ function GenerationHeaders({ ); } -/** The drawn body of one mark: session disc, bundle hatch, source ring, - * selection ring, inspection halo and the folded tail. Purely visual. */ -function MarkGlyph({ - mark, - model, - geometry, - hatchId, - inspected, - selected, - dim, -}: { - mark: TopologyMark; - model: DelegationTopologyModel; - geometry: TopologyGeometry; - hatchId: string; - inspected: boolean; - selected: boolean; - dim: boolean; -}) { - const at = markPosition(mark, geometry); - const radius = markRadius(mark, model.maxDescendants); - const source = - mark.kind === 'session' && - mark.generation === 0 && - mark.node.link === 'root' && - (mark.drawnChildren > 0 || mark.foldedDescendants > 0); - const fill = - mark.kind === 'bundle' - ? `url(#${hatchId})` - : mark.node.link === 'missing_parent' - ? 'var(--raw-graph-alert)' - : mark.node.link === 'cycle' - ? `url(#${hatchId})` - : 'var(--raw-graph-accent)'; - const stroke = - mark.kind === 'bundle' - ? 'var(--raw-graph-text)' - : mark.node.link === 'cycle' - ? 'var(--raw-state-conflicting)' - : mark.node.link === 'missing_parent' - ? 'var(--raw-graph-alert)' - : 'var(--raw-graph-accent)'; - return ( - - {source ? ( - - ) : null} - {selected ? ( - - ) : null} - {inspected && !selected ? ( - - ) : null} - - {mark.kind === 'session' && mark.foldedDescendants > 0 ? ( - - - - - ) : null} - - ); -} - /** The operable mark: a 44px hit area over the disc, the label to its right, * and, for a folded session, the tail control that opens the generation * beneath. Hover and focus inspect; click and Enter select or open. */ function MarkControl({ mark, - model, + radius, geometry, interaction, dim, + countLine, }: { mark: TopologyMark; - model: DelegationTopologyModel; + radius: number; geometry: TopologyGeometry; interaction: TopologyInteraction; dim: boolean; + /** The exact count the ring is sized by. */ + countLine: string; }) { const at = markPosition(mark, geometry); - const radius = markRadius(mark, model.maxDescendants); const selected = mark.id === interaction.selectedId; const inspect = () => interaction.onInspect(mark.id); const commonStyle = { top: at.y - HIT / 2, left: at.x - HIT / 2 } as const; @@ -550,7 +459,10 @@ function MarkControl({ > {bundleTitle(mark)} - {mark.descendants > 0 ? `${mark.descendants} beneath · ` : ''}open + {countLine} · open + + + @@ -559,12 +471,9 @@ function MarkControl({ const elapsed = subagentElapsedSeconds(mark.node); const detail = [ + countLine, + mark.foldedDescendants > 0 ? `${mark.foldedDescendants} folded` : null, mark.node.provider, - mark.drawnChildren > 0 - ? `${mark.node.descendants} beneath` - : mark.foldedDescendants > 0 - ? `${mark.foldedDescendants} folded` - : null, elapsed != null ? `${elapsed.toLocaleString()}s` : 'span unrecorded', ] .filter(Boolean) @@ -601,7 +510,10 @@ function MarkControl({ {mark.label} - {detail} + + + · {detail} + {mark.foldedDescendants > 0 ? ( @@ -635,53 +547,48 @@ function bundleTitle(mark: TopologyBundleMark): string { /** The field's key and its population, reconciled: drawn plus folded equals * the reading, printed so the sum can be checked rather than trusted. */ -function TopologyLegend({ model, fit }: { model: DelegationTopologyModel; fit: FittedTopology }) { - const hatchId = useId(); +export function TopologyPopulation({ model, fit }: { model: DelegationTopologyModel; fit: FittedTopology }) { const foldedGenerations = fit.maxDepth + 1 - model.columns; + return ( + + {model.totalSessions.toLocaleString()} sessions · {model.drawnSessions.toLocaleString()} drawn + {model.bundledSessions > 0 ? ` · ${model.bundledSessions.toLocaleString()} folded` : ''} ·{' '} + {model.columns} {model.columns === 1 ? 'generation' : 'generations'} + {foldedGenerations > 0 ? ` of ${fit.maxDepth + 1}` : ''} + + ); +} + +/** Legend for the ring marks: what the ring, core and inner glyph mean. */ +function RingLegend({ model, fit }: { model: DelegationTopologyModel; fit: FittedTopology }) { return (
    - - {model.totalSessions.toLocaleString()} sessions · {model.drawnSessions.toLocaleString()} drawn - {model.bundledSessions > 0 ? ` · ${model.bundledSessions.toLocaleString()} folded` : ''} ·{' '} - {model.columns} {model.columns === 1 ? 'generation' : 'generations'} - {foldedGenerations > 0 ? ` of ${fit.maxDepth + 1}` : ''} - - - - + + + + + + + + - - + - - - - - - - - - - - - - - - + - - - + + + - size = sessions beneath, log band - hover inspects · click selects · Escape clears + + hover shows exact counts · click selects
    ); } -function Swatch({ label, children }: { label: string; children: ReactNode }) { +export function Swatch({ label, children }: { label: string; children: ReactNode }) { return ( diff --git a/dashboard/src/workspaces/agents/agentsView.tsx b/dashboard/src/workspaces/agents/agentsView.tsx new file mode 100644 index 0000000000..6e7df727cd --- /dev/null +++ b/dashboard/src/workspaces/agents/agentsView.tsx @@ -0,0 +1,115 @@ +import { useCallback } from 'react'; +import { useSearchParams } from 'react-router'; +import { cn } from '../../ui/cn'; +import type { TopologyMark } from './delegationTopology.ts'; +import type { TopologyInteraction } from './DelegationTopology.tsx'; + +/** + * The Agents views. Both read the same fitted topology and hand the page the + * same inspect/select acts; they differ in which dimension carries the + * reading. Topology lays generations across and sizes rings by sessions + * beneath; Timeline lays the store's recorded start and end across and keeps + * the hierarchy down. The view lives in `?view` the way Code's lens and + * Work's camera do, replaced rather than pushed. + */ +export type AgentsView = 'topology' | 'timeline'; + +export const AGENTS_VIEW_PARAM = 'view'; + +const AGENTS_VIEWS: readonly { readonly value: AgentsView; readonly label: string; readonly note: string }[] = [ + { value: 'topology', label: 'Topology', note: 'generations across · rings sized by sessions beneath' }, + { value: 'timeline', label: 'Timeline', note: 'recorded start → end across · hierarchy down' }, +]; + +export function agentsViewNote(view: AgentsView): string { + return AGENTS_VIEWS.find((candidate) => candidate.value === view)!.note; +} + +export function useAgentsView(): [AgentsView, (next: AgentsView) => void] { + const [params, setParams] = useSearchParams(); + const active: AgentsView = params.get(AGENTS_VIEW_PARAM) === 'timeline' ? 'timeline' : 'topology'; + const select = useCallback( + (next: AgentsView) => { + const updated = new URLSearchParams(params); + if (next === 'topology') updated.delete(AGENTS_VIEW_PARAM); + else updated.set(AGENTS_VIEW_PARAM, next); + setParams(updated, { replace: true }); + }, + [params, setParams], + ); + return [active, select]; +} + +/** The view tabs, in the workspace's control bar under the header: engraved, + * the active one framed in signal cyan with a position bar. */ +export function AgentsViewSwitcher({ + active, + onSelect, +}: { + active: AgentsView; + onSelect: (view: AgentsView) => void; +}) { + return ( +
      + {AGENTS_VIEWS.map((view) => { + const selected = view.value === active; + return ( +
    1. + +
      2. + ); + })} +
    + + ); +} + +/** One accessible name for a mark, whichever renderer draws it. */ +export function markAccessibleName(mark: TopologyMark, selected: boolean): string { + if (mark.kind === 'bundle') { + const title = mark.basis === 'remainder' ? mark.label : `${mark.sessions} × ${mark.label}`; + return `${title}: ${mark.sessions} sessions in generation ${mark.generation}${mark.descendants > 0 ? `, ${mark.descendants} beneath them` : ''}. Open this bundle.`; + } + const linkWord = + mark.node.link === 'missing_parent' + ? ', parent not in this reading' + : mark.node.link === 'cycle' + ? ', on a parent cycle' + : ''; + return `${mark.label}, ${mark.node.provider} session ${mark.node.session_id}, generation ${mark.generation}${linkWord}. ${selected ? 'Selected.' : 'Select to read its token frontier.'}`; +} + +/** The pointer and keyboard wiring every renderer's mark control shares: + * hover and focus inspect; click selects a session and opens a bundle. */ +export function markHandlers(mark: TopologyMark, interaction: TopologyInteraction) { + const inspect = () => interaction.onInspect(mark.id); + return { + onMouseEnter: inspect, + onFocus: inspect, + onClick: () => + mark.kind === 'bundle' ? interaction.onToggleExpanded(mark.id) : interaction.onSelect(mark.id), + 'aria-label': markAccessibleName(mark, mark.id === interaction.selectedId), + 'data-topology-control': mark.kind, + 'data-topology-id': mark.id, + ...(mark.kind === 'bundle' + ? { 'aria-expanded': false } + : { 'aria-pressed': mark.id === interaction.selectedId, 'data-topology-link': mark.node.link }), + } as const; +} diff --git a/dashboard/src/workspaces/agents/delegationRenderers.test.ts b/dashboard/src/workspaces/agents/delegationRenderers.test.ts new file mode 100644 index 0000000000..36cd7041f4 --- /dev/null +++ b/dashboard/src/workspaces/agents/delegationRenderers.test.ts @@ -0,0 +1,142 @@ +import { describe, expect, it } from 'vitest'; +import type { + AnalyticsSubagentNodeV1, + AnalyticsSubagentTreePayloadV1, +} from '../../contracts/generated.ts'; +import { ringCoreAlpha, ringKind, ringRadius } from './delegationRings.tsx'; +import { timelineRecency } from './DelegationTimeline.tsx'; +import { layoutDelegationTimeline, timelineTickLabel, timelineTicks } from './delegationTimeline.ts'; +import { fitDelegationTopology, subtree, type TopologySessionMark } from './delegationTopology.ts'; + +/** + * The ring and timeline views re-read one fitted topology. The + * reading is the story fixture's shape: a three-level Codex tree whose + * grandchild never recorded an end, a Claude session whose parent was never + * ingested, and one flat Cursor session. + */ + +const T0 = 1_760_000_000; + +function node(overrides: Partial & { session_id: string; depth: number }): AnalyticsSubagentNodeV1 { + return { + provider: 'codex', + parent_session_id: null, + agent: 'Codex', + title: null, + started_at: T0, + ended_at: T0 + 100, + is_subagent: overrides.depth > 0, + parent_tool_use_id: null, + descendants: 0, + link: overrides.depth > 0 ? 'linked' : 'root', + ...overrides, + }; +} + +const READING: AnalyticsSubagentTreePayloadV1 = { + available: true, + source: 'sessions', + error: null, + nodes: [ + node({ session_id: 'root', depth: 0, descendants: 2, started_at: T0, ended_at: T0 + 3_600 }), + node({ session_id: 'child', depth: 1, descendants: 1, parent_session_id: 'root', started_at: T0 + 600, ended_at: T0 + 2_000 }), + node({ session_id: 'grandchild', depth: 2, parent_session_id: 'child', started_at: T0 + 900, ended_at: null }), + node({ + session_id: 'orphan', + depth: 0, + provider: 'claude', + agent: 'Claude', + is_subagent: true, + link: 'missing_parent', + parent_session_id: 'never-ingested', + started_at: T0 + 1_000, + ended_at: T0 + 1_500, + }), + node({ session_id: 'solo', depth: 0, provider: 'cursor', agent: 'Cursor', started_at: T0 + 2_400, ended_at: T0 + 2_500 }), + ], + sessions_read: 5, + root_count: 2, + edge_count: 2, + max_depth: 2, + missing_parent_count: 1, + cycle_count: 0, + truncated: false, +}; + +const { model } = fitDelegationTopology(READING); +const mark = (id: string) => model.marks.find((candidate) => candidate.id.endsWith(`:${id}`))!; + +describe('ring marks', () => { + it('size by sessions beneath on a log band and name the reading kind', () => { + expect(ringRadius(mark('root'), model.maxDescendants)).toBe(20); + expect(ringRadius(mark('child'), model.maxDescendants)).toBeCloseTo(14.833, 3); + expect(ringRadius(mark('grandchild'), model.maxDescendants)).toBe(6); + expect(['root', 'child', 'grandchild', 'orphan', 'solo'].map((id) => ringKind(mark(id)))).toEqual([ + 'origin', + 'delegate', + 'delegate', + 'cut', + 'origin', + ]); + }); + + it('brighten the core with sessions beneath, and a leaf stays a whisper', () => { + expect(ringCoreAlpha(mark('root'), model.maxDescendants)).toBe(0.36); + expect(ringCoreAlpha(mark('child'), model.maxDescendants)).toBe(0.249); + expect(ringCoreAlpha(mark('grandchild'), model.maxDescendants)).toBe(0.06); + }); + + it('lift a selection with its drawn subtree and nothing else', () => { + const lifted = [...subtree(model, mark('child').id)].map((id) => id.split(':')[1]); + expect(lifted).toEqual(['child', 'grandchild']); + expect(subtree(model, 'codex:not-drawn').size).toBe(0); + }); +}); + +describe('layoutDelegationTimeline', () => { + const timeline = layoutDelegationTimeline(model); + + it('reads rows in pre-order, one lane per top', () => { + expect(timeline.rows.map((row) => [(row.mark as TopologySessionMark).node.session_id, row.lane])).toEqual([ + ['root', 0], + ['child', 0], + ['grandchild', 0], + ['orphan', 1], + ['solo', 2], + ]); + expect(timeline.domain).toEqual({ start: T0, end: T0 + 3_600 }); + expect(timeline.open).toBe(1); + expect(timeline.unplaced).toBe(0); + }); + + it('brackets spawns at the child start and joins only where an end was recorded', () => { + expect(timeline.brackets).toEqual([ + { kind: 'spawn', parentRow: 0, childRow: 1, at: T0 + 600 }, + { kind: 'join', parentRow: 0, childRow: 1, at: T0 + 2_000 }, + { kind: 'spawn', parentRow: 1, childRow: 2, at: T0 + 900 }, + ]); + }); + + it('counts lane concurrency over sessions with both ends, and says what it left out', () => { + const lane = timeline.lanes[0]!; + expect(lane.density).toEqual([ + { at: T0, count: 1 }, + { at: T0 + 600, count: 2 }, + { at: T0 + 2_000, count: 1 }, + { at: T0 + 3_600, count: 0 }, + ]); + expect([lane.peak, lane.measured, lane.unmeasured]).toEqual([2, 2, 1]); + }); + + it('ramps bar fill by recency within the loaded page', () => { + const domain = timeline.domain!; + expect(timeline.rows.map((row) => timelineRecency(row, domain))).toEqual([0.4, 0.267, 0.4, 0.225, 0.308]); + }); + + it('spaces ticks by label width and prints them in UTC', () => { + const ticks = timelineTicks({ start: T0, end: T0 + 3_600 }, 600); + expect(ticks.step).toBe(600); + expect(ticks.ticks).toHaveLength(6); + expect(timelineTickLabel(ticks.ticks[0]!, ticks.step)).toBe('09:00'); + }); +}); diff --git a/dashboard/src/workspaces/agents/delegationRings.tsx b/dashboard/src/workspaces/agents/delegationRings.tsx new file mode 100644 index 0000000000..2a74188e61 --- /dev/null +++ b/dashboard/src/workspaces/agents/delegationRings.tsx @@ -0,0 +1,219 @@ +import type { ReactNode } from 'react'; +import { subagentElapsedSeconds } from './subagentTree.ts'; +import type { TopologyMark } from './delegationTopology.ts'; +import { UsageLabel } from './sessionUsage.tsx'; + +/** + * The ring mark: the plate's hollow, descendant-scaled ring with a faint + * core, drawn over the same generation-column layout as the disc field. + * + * Size is the one measured quantity the hierarchy authority serves per node, + * sessions beneath it. Token counts would read better on a ring, and the plate + * prints them, but `AnalyticsSubagentNodeV1` carries none, so the ring never + * claims one. The glyph inside the ring names what the reading says the mark + * is, not what it did. + */ + +/** Core fill opacity from sessions beneath, on the same log band as the + * radius: a leaf's core is a whisper, the widest subtree's reads as filled. + * Measured, never decorative, so a ring with nothing beneath never glows. */ +export function ringCoreAlpha(mark: TopologyMark, maxDescendants: number): number { + const beneath = mark.kind === 'bundle' ? mark.sessions + mark.descendants : mark.node.descendants; + const ceiling = Math.max(maxDescendants, beneath); + const fraction = ceiling <= 0 || beneath <= 0 ? 0 : Math.log1p(beneath) / Math.log1p(ceiling); + return Math.round((0.06 + 0.3 * fraction) * 1000) / 1000; +} + +export const RING_GEOMETRY = { + minRadius: 6, + maxRadius: 20, +} as const; + +/** Radius from sessions beneath, on a log band against the widest drawn. A + * bundle is sized by everything it hides, so a closed group is never smaller + * than the sessions folded into it. */ +export function ringRadius(mark: TopologyMark, maxDescendants: number): number { + const { minRadius, maxRadius } = RING_GEOMETRY; + const beneath = mark.kind === 'bundle' ? mark.sessions + mark.descendants : mark.node.descendants; + const ceiling = Math.max(maxDescendants, beneath); + if (ceiling <= 0 || beneath <= 0) return minRadius; + return minRadius + (maxRadius - minRadius) * (Math.log1p(beneath) / Math.log1p(ceiling)); +} + +/** What the glyph inside a ring says. `origin` is a session the store records + * as no one's subagent; `delegate` is a linked child; the other three are the + * reading's own typed states. */ +export type RingKind = 'origin' | 'delegate' | 'bundle' | 'cut' | 'cycle'; + +export function ringKind(mark: TopologyMark): RingKind { + if (mark.kind === 'bundle') return 'bundle'; + switch (mark.node.link) { + case 'missing_parent': + return 'cut'; + case 'cycle': + return 'cycle'; + case 'root': + case 'linked': + return mark.node.is_subagent ? 'delegate' : 'origin'; + default: { + const unhandled: never = mark.node.link; + return unhandled; + } + } +} + +function ringTone(kind: RingKind): { stroke: string; dash: string | undefined } { + switch (kind) { + case 'origin': + case 'delegate': + return { stroke: 'var(--raw-graph-accent)', dash: undefined }; + case 'bundle': + return { stroke: 'var(--raw-graph-text)', dash: '3 2' }; + case 'cut': + return { stroke: 'var(--raw-graph-alert)', dash: '3 2' }; + case 'cycle': + return { stroke: 'var(--raw-state-conflicting)', dash: '2 2' }; + default: { + const unhandled: never = kind; + return unhandled; + } + } +} + +/** The monoline glyph inside the ring, 6px tall whatever the ring's size. */ +function KindGlyph({ kind, x, y, stroke }: { kind: RingKind; x: number; y: number; stroke: string }) { + switch (kind) { + case 'origin': + return ; + case 'delegate': + return ; + case 'bundle': + return ( + + ); + case 'cut': + return ; + case 'cycle': + return ( + + ); + default: { + const unhandled: never = kind; + return unhandled; + } + } +} + +export function RingGlyph({ + mark, + at, + radius, + maxDescendants, + hatchId, + selected, + lifted, + dim, +}: { + mark: TopologyMark; + at: { x: number; y: number }; + radius: number; + maxDescendants: number; + hatchId: string; + selected: boolean; + /** Beneath the selection: a restrained cyan halo, no glow. */ + lifted: boolean; + dim: boolean; +}) { + const core = ringCoreAlpha(mark, maxDescendants); + const kind = ringKind(mark); + const tone = ringTone(kind); + return ( + + {selected ? ( + + ) : null} + {lifted ? ( + + ) : null} + + + {kind === 'bundle' ? ( + + ) : null} + + {mark.kind === 'session' && mark.foldedDescendants > 0 ? ( + + + + + ) : null} + + ); +} + +/** The ring label's second line: the exact count the ring is sized by. */ +export function ringCountLine(mark: TopologyMark): string { + if (mark.kind === 'bundle') { + return `${mark.sessions + mark.descendants} sessions folded`; + } + if (mark.node.descendants > 0) return `${mark.node.descendants.toLocaleString()} beneath`; + return 'leaf'; +} + +/** Hover inspects: the exact counts behind the ring, beside the mark. */ +export function RingHoverCard({ mark, at, radius }: { mark: TopologyMark; at: { x: number; y: number }; radius: number }) { + const rows: [string, ReactNode][] = + mark.kind === 'bundle' + ? [ + ['sessions', mark.sessions.toLocaleString()], + ['beneath them', mark.descendants.toLocaleString()], + ['basis', mark.basis], + ['tokens', ], + ] + : [ + ['beneath', mark.node.descendants.toLocaleString()], + ['drawn children', mark.drawnChildren.toLocaleString()], + ['folded', mark.foldedDescendants.toLocaleString()], + ['span', (() => { + const elapsed = subagentElapsedSeconds(mark.node); + return elapsed === null ? 'absent' : `${elapsed.toLocaleString()} s`; + })()], + ['tokens', ], + ]; + return ( +
    + {rows.map(([label, value]) => ( + + {label} + {value} + + ))} +
    + ); +} diff --git a/dashboard/src/workspaces/agents/delegationTimeline.ts b/dashboard/src/workspaces/agents/delegationTimeline.ts new file mode 100644 index 0000000000..470b5055bb --- /dev/null +++ b/dashboard/src/workspaces/agents/delegationTimeline.ts @@ -0,0 +1,204 @@ +import type { AnalyticsSubagentNodeV1 } from '../../contracts/generated.ts'; +import type { DelegationTopologyModel, TopologyMark } from './delegationTopology.ts'; + +/** + * The delegation timeline: the fitted topology re-read with recorded time on + * x and the hierarchy on y. + * + * Rows are the tree in pre-order, a parent directly above its children, so y + * is the hierarchy and nothing else. Each top opens a lane. X is the store's + * own `started_at`/`ended_at` in Unix seconds and nothing is interpolated: + * + * - a session with no recorded start has no x and is counted as unplaced; + * - a session with a start and no end is open, its bar runs to the reading's + * last recorded instant and is marked open rather than given an end; + * - a spawn bracket joins a drawn parent to a child at the child's recorded + * start. The parent/child relation is the daemon's; the instant is the + * child's own. + * - a join bracket marks the child's recorded end on the parent's row. The + * store records that the child ended, not that it reported back, so the + * join is an inference from the end time and is drawn as one. + * + * Lane density is concurrency over sessions whose start and end are both + * recorded; open and unplaced sessions are counted beside it, never guessed + * into it. + */ + +export interface TimelineRow { + readonly mark: TopologyMark; + readonly index: number; + readonly lane: number; + /** Seconds, or null when the reading recorded no start. */ + readonly start: number | null; + /** Seconds, or null when no end is recorded. */ + readonly end: number | null; +} + +export interface TimelineBracket { + readonly kind: 'spawn' | 'join'; + readonly parentRow: number; + readonly childRow: number; + readonly at: number; +} + +export interface TimelineDensityStep { + readonly at: number; + readonly count: number; +} + +export interface TimelineLane { + readonly index: number; + readonly topId: string; + readonly firstRow: number; + readonly rows: number; + /** Concurrency steps: from `at` until the next step, `count` sessions ran. */ + readonly density: readonly TimelineDensityStep[]; + readonly peak: number; + /** Sessions in the lane with both ends recorded, the population `density` counts. */ + readonly measured: number; + /** Sessions in the lane left out of `density` because an end is missing. */ + readonly unmeasured: number; +} + +export interface DelegationTimelineModel { + readonly rows: readonly TimelineRow[]; + readonly brackets: readonly TimelineBracket[]; + readonly lanes: readonly TimelineLane[]; + /** The recorded extent in seconds, or null when nothing drawn has a start. */ + readonly domain: { readonly start: number; readonly end: number } | null; + readonly unplaced: number; + readonly open: number; +} + +function span(nodes: readonly AnalyticsSubagentNodeV1[]): { start: number | null; end: number | null } { + let start: number | null = null; + let end: number | null = null; + let allEnded = true; + for (const node of nodes) { + if (node.started_at != null) start = start === null ? node.started_at : Math.min(start, node.started_at); + if (node.ended_at == null) allEnded = false; + else end = end === null ? node.ended_at : Math.max(end, node.ended_at); + } + return { start, end: allEnded ? end : null }; +} + +function markNodes(mark: TopologyMark): readonly AnalyticsSubagentNodeV1[] { + return mark.kind === 'bundle' ? mark.members : [mark.node]; +} + +function density(intervals: readonly (readonly [number, number])[]): TimelineDensityStep[] { + const events: [number, number][] = []; + for (const [start, end] of intervals) { + events.push([start, 1], [end, -1]); + } + // Ends before starts at one instant: a hand-over is not an overlap. + events.sort((a, b) => a[0] - b[0] || a[1] - b[1]); + const steps: TimelineDensityStep[] = []; + let count = 0; + for (const [at, delta] of events) { + count += delta; + const last = steps[steps.length - 1]; + if (last !== undefined && last.at === at) steps[steps.length - 1] = { at, count }; + else steps.push({ at, count }); + } + return steps; +} + +export function layoutDelegationTimeline(model: DelegationTopologyModel): DelegationTimelineModel { + const children = new Map(); + for (const mark of model.marks) { + const bucket = children.get(mark.parentId); + if (bucket) bucket.push(mark); + else children.set(mark.parentId, [mark]); + } + for (const bucket of children.values()) bucket.sort((a, b) => a.row - b.row); + + const rows: TimelineRow[] = []; + const rowOf = new Map(); + const lanes: TimelineLane[] = []; + const visit = (mark: TopologyMark, lane: number) => { + const { start, end } = span(markNodes(mark)); + const index = rows.length; + rows.push({ mark, index, lane, start, end }); + rowOf.set(mark.id, index); + for (const child of children.get(mark.id) ?? []) visit(child, lane); + }; + for (const top of children.get(null) ?? []) { + const firstRow = rows.length; + visit(top, lanes.length); + const laneRows = rows.slice(firstRow); + const intervals: [number, number][] = []; + let unmeasured = 0; + for (const row of laneRows) { + for (const node of markNodes(row.mark)) { + if (node.started_at != null && node.ended_at != null && node.ended_at >= node.started_at) { + intervals.push([node.started_at, node.ended_at]); + } else { + unmeasured += 1; + } + } + } + const steps = density(intervals); + lanes.push({ + index: lanes.length, + topId: top.id, + firstRow, + rows: laneRows.length, + density: steps, + peak: steps.reduce((max, step) => Math.max(max, step.count), 0), + measured: intervals.length, + unmeasured, + }); + } + + const brackets: TimelineBracket[] = []; + for (const row of rows) { + if (row.mark.kind !== 'session' || row.mark.parentId === null) continue; + const parentRow = rowOf.get(row.mark.parentId); + if (parentRow === undefined) continue; + if (row.start !== null) brackets.push({ kind: 'spawn', parentRow, childRow: row.index, at: row.start }); + if (row.end !== null) brackets.push({ kind: 'join', parentRow, childRow: row.index, at: row.end }); + } + + let start = Number.POSITIVE_INFINITY; + let end = Number.NEGATIVE_INFINITY; + for (const row of rows) { + if (row.start === null) continue; + start = Math.min(start, row.start); + end = Math.max(end, row.end ?? row.start); + } + return { + rows, + brackets, + lanes, + domain: Number.isFinite(start) ? { start, end: Math.max(end, start + 1) } : null, + unplaced: rows.filter((row) => row.start === null).length, + open: rows.filter((row) => row.start !== null && row.end === null).length, + }; +} + +const TICK_STEPS = [1, 5, 10, 15, 30, 60, 300, 600, 900, 1_800, 3_600, 7_200, 10_800, 21_600, 43_200, 86_400, 172_800, 604_800]; + +/** Tick instants for a domain drawn `width` pixels wide, at least `minPitch` + * apart, the pitch the widest label this step prints needs. */ +export function timelineTicks( + domain: { start: number; end: number }, + width: number, + minPitch = 96, +): { step: number; ticks: number[] } { + const seconds = domain.end - domain.start; + const step = + TICK_STEPS.find((candidate) => (candidate / seconds) * width >= minPitch) ?? + TICK_STEPS[TICK_STEPS.length - 1]!; + const ticks: number[] = []; + for (let at = Math.ceil(domain.start / step) * step; at <= domain.end; at += step) ticks.push(at); + return { step, ticks }; +} + +/** A tick label in UTC: the time of day under a day, the date from a day up. */ +export function timelineTickLabel(at: number, step: number): string { + const iso = new Date(at * 1000).toISOString(); + if (step >= 86_400) return iso.slice(5, 10); + if (step < 60) return iso.slice(11, 19); + return iso.slice(11, 16); +} diff --git a/dashboard/src/workspaces/agents/delegationTopology.test.ts b/dashboard/src/workspaces/agents/delegationTopology.test.ts index 152f2dfcd3..20a14f33fa 100644 --- a/dashboard/src/workspaces/agents/delegationTopology.test.ts +++ b/dashboard/src/workspaces/agents/delegationTopology.test.ts @@ -10,11 +10,11 @@ import { fitDelegationTopology, layoutDelegationTopology, markPosition, - markRadius, neighbourhood, type TopologyBundleMark, type TopologySessionMark, } from './delegationTopology.ts'; +import { ringRadius } from './delegationRings.tsx'; function node(overrides: Partial & { session_id: string; depth: number }): AnalyticsSubagentNodeV1 { return { @@ -123,13 +123,6 @@ describe('layoutDelegationTopology', () => { ]); }); - it('lays out identically for identical readings', () => { - const first = layoutDelegationTopology(fixtureTree()); - const second = layoutDelegationTopology(fixtureTree()); - expect(second).toEqual(first); - expect(fieldSize(second)).toEqual(fieldSize(first)); - }); - it('bundles fan-out past the limit by agent and reconciles the counts', () => { const children = Array.from({ length: 12 }, (_, index) => node({ @@ -403,19 +396,16 @@ describe('fitDelegationTopology', () => { }); describe('topology geometry', () => { - it('scales radii from measured descendants and never below the floor', () => { + it('scales ring radii from measured descendants and never below the floor', () => { const model = layoutDelegationTopology(fixtureTree()); const byId = new Map(model.marks.map((mark) => [mark.id, mark])); - const root = byId.get('codex:root')!; - const leaf = byId.get('codex:grandchild')!; - const childA = byId.get('codex:child-a')!; - expect(markRadius(root, model.maxDescendants)).toBe(16); - expect(markRadius(leaf, model.maxDescendants)).toBe(5); - const mid = markRadius(childA, model.maxDescendants); - expect(mid).toBeGreaterThan(5); - expect(mid).toBeLessThan(12); + expect(ringRadius(byId.get('codex:root')!, model.maxDescendants)).toBe(20); + expect(ringRadius(byId.get('codex:grandchild')!, model.maxDescendants)).toBe(6); // A solo root with nothing beneath is a leaf, not a source. - expect(markRadius(byId.get('cursor:solo')!, model.maxDescendants)).toBe(5); + expect(ringRadius(byId.get('cursor:solo')!, model.maxDescendants)).toBe(6); + const mid = ringRadius(byId.get('codex:child-a')!, model.maxDescendants); + expect(mid).toBeGreaterThan(6); + expect(mid).toBeLessThan(20); }); it('draws an edge from trailing edge to leading edge through the mid column', () => { diff --git a/dashboard/src/workspaces/agents/delegationTopology.ts b/dashboard/src/workspaces/agents/delegationTopology.ts index 5406f80c1f..f79de26d2c 100644 --- a/dashboard/src/workspaces/agents/delegationTopology.ts +++ b/dashboard/src/workspaces/agents/delegationTopology.ts @@ -2,6 +2,7 @@ import type { AnalyticsSubagentNodeV1, AnalyticsSubagentTreePayloadV1, } from '../../contracts/generated.ts'; +import { usageCoverage, type UsageCoverage } from './sessionUsage.tsx'; import { subagentLabel } from './subagentTree.ts'; /** @@ -134,6 +135,8 @@ export interface DelegationTopologyModel { readonly maxDescendants: number; /** Bundles of top sessions the reader opened; they hang off no parent mark. */ readonly openedTopBundles: readonly OpenedBundle[]; + /** Whether the provider-usage read behind every node's `usage` completed. */ + readonly usageCoverage: UsageCoverage; } export function markId(node: AnalyticsSubagentNodeV1): string { @@ -455,6 +458,7 @@ export function layoutDelegationTopology( bundledSessions, maxDescendants, openedTopBundles, + usageCoverage: usageCoverage(payload), }; } @@ -508,9 +512,6 @@ export const TOPOLOGY_GEOMETRY = { rowPitch: 40, padX: 88, padY: 36, - minRadius: 5, - maxRadius: 12, - sourceRadius: 16, /** Room the last column's labels need to the right of their marks. */ labelRoom: 168, /** Widest a column may stretch when the aperture has width to spare. */ @@ -557,29 +558,6 @@ export function fieldSize( }; } -/** - * Mark radius from what the reading measured: sessions beneath it, on a log - * band against the widest fan-out drawn. A generation-0 root reads as the - * source and takes the source radius; a bundle is sized by the sessions it - * folds so an unopened group is never smaller than the sessions it hides. - */ -export function markRadius(mark: TopologyMark, maxDescendants: number): number { - const { minRadius, maxRadius, sourceRadius } = TOPOLOGY_GEOMETRY; - if (mark.kind === 'bundle') { - return Math.min(maxRadius, minRadius + Math.log1p(mark.sessions) * 2); - } - if ( - mark.generation === 0 && - mark.node.link === 'root' && - (mark.drawnChildren > 0 || mark.foldedDescendants > 0) - ) { - return sourceRadius; - } - if (maxDescendants <= 0 || mark.node.descendants <= 0) return minRadius; - const fraction = Math.log1p(mark.node.descendants) / Math.log1p(maxDescendants); - return minRadius + (maxRadius - minRadius) * Math.max(0, Math.min(1, fraction)); -} - /** Cubic path from one mark's trailing edge to the next mark's leading edge, * bending at the midpoint column so parallel delegations read as a fan. */ export function edgePath( @@ -612,3 +590,21 @@ export function neighbourhood( } return keep; } + +/** A mark and every drawn mark beneath it: what a selection lifts. */ +export function subtree(model: DelegationTopologyModel, id: string): ReadonlySet { + const children = new Map(); + for (const mark of model.marks) { + if (mark.parentId === null) continue; + const bucket = children.get(mark.parentId); + if (bucket) bucket.push(mark.id); + else children.set(mark.parentId, [mark.id]); + } + const keep = new Set(); + const queue = model.marks.some((mark) => mark.id === id) ? [id] : []; + for (let cursor = queue.shift(); cursor !== undefined; cursor = queue.shift()) { + keep.add(cursor); + queue.push(...(children.get(cursor) ?? [])); + } + return keep; +} diff --git a/dashboard/src/workspaces/agents/handoffTokens.test.ts b/dashboard/src/workspaces/agents/handoffTokens.test.ts index dcc1a6b1d1..457def888f 100644 --- a/dashboard/src/workspaces/agents/handoffTokens.test.ts +++ b/dashboard/src/workspaces/agents/handoffTokens.test.ts @@ -46,11 +46,6 @@ function landed(handoffs: Record[]): WorkResult { - it('names the operation and path the daemon actually mounts', () => { - expect(HANDOFF_LIST_TASK_ROUTE.operation).toBe('operation.handoff.list_task_handoffs'); - expect(HANDOFF_LIST_TASK_ROUTE.path).toBe('/api/application/handoff/list-task'); - }); - it('accepts a bare session id and refuses a request carrying a bearer', () => { expect(HANDOFF_LIST_TASK_ROUTE.request.safeParse({ session_id: 's' }).success).toBe(true); // The contract is `.strict()`: a client cannot smuggle a token onto this diff --git a/dashboard/src/workspaces/agents/sessionUsage.tsx b/dashboard/src/workspaces/agents/sessionUsage.tsx new file mode 100644 index 0000000000..1415e62537 --- /dev/null +++ b/dashboard/src/workspaces/agents/sessionUsage.tsx @@ -0,0 +1,151 @@ +import type { + AnalyticsSubagentNodeV1, + AnalyticsSubagentTreePayloadV1, + ProviderUsageCoverageV1, +} from '../../contracts/generated.ts'; +import { cn } from '../../ui/cn'; +import type { TopologyMark } from './delegationTopology.ts'; + +/** + * Provider-reported usage per session, as the subagent tree carries it. + * + * Three states a reader must be able to tell apart: `measured` (the provider + * reported counters for this session), `partial` (it reported some, but the + * aggregate says it is incomplete), and `absent` (no usage row joins this + * session). Whether `absent` means "none recorded" or "the read failed" is the + * tree's `usage_coverage`, printed once in the legend, never guessed per node. + */ +export type SessionUsage = + | { readonly state: 'absent' } + | { + readonly state: 'measured' | 'partial'; + readonly total: number | null; + readonly events: number; + readonly split: ReadonlyArray; + }; + +/** The tree-level coverage, or `unreported` when the reading predates it. */ +export type UsageCoverage = ProviderUsageCoverageV1 | 'unreported'; + +export function sessionUsage(node: AnalyticsSubagentNodeV1): SessionUsage { + const usage = node.usage; + if (usage == null) return { state: 'absent' }; + const { counters } = usage; + return { + state: usage.complete ? 'measured' : 'partial', + total: counters.total_tokens, + events: usage.usage_events, + split: [ + ['input', counters.input_tokens], + ['output', counters.output_tokens], + ['cache read', counters.cache_read_tokens], + ['cache write', counters.cache_write_tokens], + ['reasoning', counters.reasoning_tokens], + ], + }; +} + +export function usageCoverage(payload: AnalyticsSubagentTreePayloadV1): UsageCoverage { + return payload.usage_coverage ?? 'unreported'; +} + +export function tokenCount(value: number | null): string { + return value === null ? 'unreported' : `${value.toLocaleString()} tokens`; +} + +/** The mark-level label: the total with its unit, or the typed gap. */ +export function usageTotalLabel(usage: SessionUsage): string { + return usage.state === 'absent' ? 'tokens absent' : tokenCount(usage.total); +} + +/** Sum of the measured members' totals across a bundle, with how many joined. */ +export function bundleUsage(members: readonly AnalyticsSubagentNodeV1[]): { + readonly total: number; + readonly measured: number; + readonly partial: boolean; +} { + let total = 0; + let measured = 0; + let partial = false; + for (const member of members) { + const usage = sessionUsage(member); + if (usage.state === 'absent') continue; + measured += 1; + total += usage.total ?? 0; + partial ||= usage.state === 'partial' || usage.total === null; + } + return { total, measured, partial }; +} + +export function coverageLabel(coverage: UsageCoverage): string { + switch (coverage) { + case 'complete': + return 'usage read complete · absent means none recorded'; + case 'partial': + return 'usage read partial · absent may be unread'; + case 'unavailable': + return 'usage read unavailable · every session absent'; + case 'unreported': + return 'usage coverage not reported by this reading'; + default: { + const unhandled: never = coverage; + return unhandled; + } + } +} + +/** The typed-state partial marker: amber, hatched, and the word. */ +export function PartialMark({ className }: { className?: string }) { + return ( + + partial + + ); +} + +/** Printed once per view beside the mark legend. */ +export function UsageCoverageLegend({ coverage }: { coverage: UsageCoverage }) { + return ( + + {coverage === 'partial' ? : null} + tokens · {coverageLabel(coverage)} + + ); +} + +/** A mark's usage in place: the total with its unit (a bundle sums its + * measured members and says how many joined), then the partial marker. */ +export function UsageLabel({ mark }: { mark: TopologyMark }) { + let text: string; + let state: SessionUsage['state']; + if (mark.kind === 'bundle') { + const summed = bundleUsage(mark.members); + state = summed.measured === 0 ? 'absent' : summed.partial ? 'partial' : 'measured'; + text = + summed.measured === 0 + ? 'tokens absent' + : `${summed.total.toLocaleString()} tokens · ${summed.measured} of ${mark.members.length}`; + } else { + const usage = sessionUsage(mark.node); + state = usage.state; + text = usageTotalLabel(usage); + } + return ( + + + {text} + + {state === 'partial' ? : null} + + ); +} diff --git a/dashboard/src/workspaces/automations/AutomationLedgers.tsx b/dashboard/src/workspaces/automations/AutomationLedgers.tsx index a10ade2728..8c4559274a 100644 --- a/dashboard/src/workspaces/automations/AutomationLedgers.tsx +++ b/dashboard/src/workspaces/automations/AutomationLedgers.tsx @@ -1,11 +1,5 @@ -import { - tallied, - talliedFactReceipts, - type AutomaticFactReceipt, - type JobRow, - type RunRow, - type SkillRow, -} from '../../data/query/automation.ts'; +import type { AutomaticFactReceipt, AutomationJob, AutomationRunRowV1, ManagedSkill } from '../../contracts/generated.ts'; +import { tallied, talliedFactReceipts } from '../../data/query/automation.ts'; import { Panel } from '../../ui/instrument.tsx'; import { Absent, Cell, InspectRow, LedgerTable, ToneWord } from './LedgerTable.tsx'; import { @@ -37,14 +31,14 @@ interface InspectProps { function PartialNotice({ reason }: { reason: string }) { return ( -

    +

    Showing a partial list: {reason}.

    ); } function EmptyNotice({ children }: { children: string }) { - return

    {children}

    ; + return

    {children}

    ; } /* ---- user jobs ---------------------------------------------------------- */ @@ -55,10 +49,10 @@ export function UserJobsLedger({ runs, ...inspect }: { - jobs: readonly JobRow[]; + jobs: readonly AutomationJob[]; count: number; /** Loaded ledger rows, or null while that read is blocked. */ - runs: readonly RunRow[] | null; + runs: readonly AutomationRunRowV1[] | null; } & InspectProps) { const reading = tallied(jobs, count, 'jobs'); return ( @@ -86,8 +80,8 @@ export function UserJobsLedger({ onSelect={() => inspect.onSelect(identity)} identity={ - {job.name} - {jobTaskKey(job.id)} + {job.name} + {jobTaskKey(job.id)} } > @@ -126,7 +120,7 @@ function Stamp({ stamp }: { stamp: string }) { /* ---- managed skills ----------------------------------------------------- */ -export function SkillsLedger({ skills, count }: { skills: readonly SkillRow[]; count: number }) { +export function SkillsLedger({ skills, count }: { skills: readonly ManagedSkill[]; count: number }) { const reading = tallied(skills, count, 'managed skills'); return ( @@ -141,10 +135,10 @@ export function SkillsLedger({ skills, count }: { skills: readonly SkillRow[]; c - {meta.title} - + {meta.title} + {meta.id} - {meta.category ? ` · ${meta.category}` : ''} + {` · ${meta.category}`} @@ -152,23 +146,19 @@ export function SkillsLedger({ skills, count }: { skills: readonly SkillRow[]; c - {meta.provenance ? ( - - {meta.provenance.source.replaceAll('_', ' ')} - - {meta.provenance.actor} - {meta.provenance.run_id ? ` · ${meta.provenance.run_id}` : ''} - + + {meta.provenance.source.replaceAll('_', ' ')} + + {meta.provenance.actor} + {meta.provenance.run_id ? ` · ${meta.provenance.run_id}` : ''} - ) : ( - not served - )} + - {meta.targets && meta.targets.length > 0 ? ( - {meta.targets.join(' · ')} + {meta.targets.length > 0 ? ( + {meta.targets.join(' · ')} ) : ( - not served + no install targets )} @@ -218,10 +208,10 @@ export function FactOutcomesLedger({ onSelect={() => inspect.onSelect(identity)} identity={ - + {formatUtc(Math.floor(receipt.recorded_at_micros / 1_000_000))} - {receipt.apply_id} + {receipt.apply_id} } > @@ -233,17 +223,17 @@ export function FactOutcomesLedger({ {content !== undefined ? ( - {content} + {content} ) : ( receipt carries no fact text )} {receipt.quarantine_reason ? ( - quarantine: {receipt.quarantine_reason} + quarantine: {receipt.quarantine_reason} ) : null} {receipt.evidence_hash ? ( - {receipt.evidence_hash.slice(0, 16)} + {receipt.evidence_hash.slice(0, 16)} ) : ( none )} diff --git a/dashboard/src/workspaces/automations/AutomationsPage.dom.test.tsx b/dashboard/src/workspaces/automations/AutomationsPage.dom.test.tsx index 92f3c972db..35d28cb818 100644 --- a/dashboard/src/workspaces/automations/AutomationsPage.dom.test.tsx +++ b/dashboard/src/workspaces/automations/AutomationsPage.dom.test.tsx @@ -140,18 +140,16 @@ describe("AutomationsPage ledgers", () => { expect(within(digest).getByText("disabled")).toBeTruthy(); }); - it("prints skill state and provenance, and a typed absence where provenance is not served", async () => { + it("prints skill state, provenance, and an empty target set as such", async () => { stubAutomation({ skills: skillsBody([ - { - metadata: { - id: "code-slop", - title: "Code Slop Cleanup", - state: "active", - provenance: { source: "automation_run", actor: "skill_writer", run_id: "run-sw-1" }, - }, - }, - { metadata: { id: "bare", title: "Bare Skill", state: "disabled" } }, + skill({ + id: "code-slop", + title: "Code Slop Cleanup", + state: "active", + provenance: { source: "automation_run", actor: "skill_writer", run_id: "run-sw-1" }, + }), + skill({ id: "bare", title: "Bare Skill", state: "disabled", targets: [] }), ]), }); renderAutomations(); @@ -161,7 +159,8 @@ describe("AutomationsPage ledgers", () => { expect(within(slop).getByText(/skill_writer · run-sw-1/)).toBeTruthy(); const bare = within(skills).getByTestId("skill-row-bare"); expect(within(bare).getByText("disabled")).toBeTruthy(); - expect(within(bare).getAllByText(/not served/).length).toBe(2); + expect(within(bare).getByText("user")).toBeTruthy(); + expect(within(bare).getByText("no install targets")).toBeTruthy(); }); it("renders terminal fact receipts with their state and files them under their run", async () => { @@ -313,7 +312,6 @@ describe("AutomationsPage inspector", () => { run_id: "run-mc-1", artifact: artifacts.artifacts[0], payload: { applied_ops: [{ op: "normalize_tags", fact_id: "fact.v1.test" }] }, - error: "", }, }); renderAutomations(); @@ -424,21 +422,42 @@ function jobsBody(jobs: unknown[], count = jobs.length) { return { jobs, count }; } +function job(overrides: Record & { id: string; name: string }) { + return { + prompt: "Summarize the day's changes.", + enabled: true, + delivery: { mode: "file" }, + created_at: NOW - 86_400, + updated_at: NOW - 3600, + ...overrides, + }; +} + function skillsBody(skills: unknown[], count = skills.length) { + return { count, skills }; +} + +function skill(metadata: Record & { id: string; title: string; state: string }) { return { - profile_root: "/home/x/.tracedecay", - skills_root: "/home/x/.tracedecay/managed-skills", - count, - skills, - skill_metadata: [], - usage_summaries: [], - stale_recommendations: [], - improvement_recommendations: [], + metadata: { + summary: "Managed skill fixture.", + routing_description: "Use when the fixture applies.", + category: "workflow", + targets: ["cursor"], + pinned: false, + checksum: "sha256:fixture", + created_at: NOW - 86_400, + updated_at: NOW - 3600, + provenance: { source: "user", actor: "dashboard-test", run_id: null }, + ...metadata, + }, + body_markdown: "Body.", + support_files: [], }; } function receiptsBody(receipts: unknown[], count = receipts.length) { - return { receipts, count, limit: 50, error: "" }; + return { receipts, count, limit: 50 }; } function receipt(id: string, state: "applied" | "quarantined" = "applied") { @@ -448,7 +467,15 @@ function receipt(id: string, state: "applied" | "quarantined" = "applied") { run_id: "run-mc-1", state, evidence_hash: `evidence.${id}`, - add_fact_request: { content: "A recorded project fact.", category: "preference" }, + add_fact_request: { + content: "A recorded project fact.", + category: "user_pref", + source_label: null, + tags: [], + entities: [], + trust: null, + metadata: {}, + }, quarantine_reason: state === "quarantined" ? "validation failed" : undefined, validation: { disposition: state === "applied" ? "accepted" : "rejected", policy: "automatic-memory-v1" }, applied_fact_id: state === "applied" ? `fact.${id}` : undefined, @@ -473,6 +500,7 @@ function run( const task = options.task ?? "memory_curator"; const started = options.startedAt ?? NOW - 2 * 86_400; return { + schema_version: 2, run_id: id, task, task_key: options.taskKey === undefined ? task : options.taskKey, @@ -495,13 +523,15 @@ function run( } function runsBody(rows: unknown[]) { - return { runs: rows, count: rows.length, limit: 50, has_more: false, malformed_row_count: 0, completeness: "known", error: "" }; + return { runs: rows, count: rows.length, limit: 50, has_more: false, malformed_row_count: 0, completeness: "known" }; } function artifactsBody(runId: string, integrity: string) { return { run_id: runId, - artifacts: [{ kind: "traces", path: `runs/${runId}/traces.json`, sha256: "a".repeat(64), created_at: String(NOW) }], + artifacts: [ + { schema_version: 1, kind: "traces", path: `runs/${runId}/traces.json`, sha256: "a".repeat(64), created_at: String(NOW) }, + ], artifact_chain: { expected_kinds: ["traces", "feedback"], present_kinds: ["traces"], @@ -510,7 +540,6 @@ function artifactsBody(runId: string, integrity: string) { integrity_status: integrity, }, count: 1, - error: "", }; } @@ -527,21 +556,17 @@ function stubAutomation(overrides: Record = {}) { const fallbacks: Record = { "scheduler/status": scheduler(), jobs: jobsBody([ - { + job({ id: "nightly-sweep", name: "Nightly sweep", schedule: "0 3 * * *", - enabled: true, interval_secs: null, cooldown_secs: 1800, skill_ids: ["code-slop"], - delivery: { mode: "file" }, - created_at: NOW - 86_400, - updated_at: NOW - 3600, - }, - { id: "pr-digest", name: "PR digest", schedule: null, enabled: false, interval_secs: 3600 }, + }), + job({ id: "pr-digest", name: "PR digest", schedule: null, enabled: false, interval_secs: 3600 }), ]), - skills: skillsBody([{ metadata: { id: "code-slop", title: "Code Slop Cleanup", state: "active" } }]), + skills: skillsBody([skill({ id: "code-slop", title: "Code Slop Cleanup", state: "active" })]), "automatic-fact-receipts": receiptsBody([receipt("apply-1"), receipt("apply-2", "quarantined")]), runs: runsBody([ run("run-nightly-1", { diff --git a/dashboard/src/workspaces/automations/LedgerTable.tsx b/dashboard/src/workspaces/automations/LedgerTable.tsx index d446a6bd00..d0ce31ad74 100644 --- a/dashboard/src/workspaces/automations/LedgerTable.tsx +++ b/dashboard/src/workspaces/automations/LedgerTable.tsx @@ -75,7 +75,7 @@ export function LedgerTable({ ref={tableRef} onKeyDown={onKeyDown} onPointerLeave={leave} - className="w-full min-w-0 border-collapse text-2xs" + className="w-full min-w-0 border-collapse text-sm" > {caption} @@ -173,7 +173,7 @@ export function Cell({ return ( {children} @@ -252,7 +252,7 @@ export function Term({ return (
    {label}
    -
    +
    {children}
    diff --git a/dashboard/src/workspaces/automations/RunHistory.dom.test.tsx b/dashboard/src/workspaces/automations/RunHistory.dom.test.tsx index 61caa1dd21..bbf853f99f 100644 --- a/dashboard/src/workspaces/automations/RunHistory.dom.test.tsx +++ b/dashboard/src/workspaces/automations/RunHistory.dom.test.tsx @@ -36,7 +36,7 @@ describe("RunHistory", () => { reviewed: 4, }), run("run-2", { - task: "skill_writing", + task: "skill_writer", status: "failed", error: "backend refused", }), @@ -54,16 +54,16 @@ describe("RunHistory", () => { it("preserves the daemon's newest-first run order", async () => { stubRuns({ runs: runsBody([ - run("run-newest", { task: "newest_run", status: "succeeded" }), - run("run-older", { task: "older_run", status: "succeeded" }), + run("run-newest", { task: "skill_writer", status: "succeeded" }), + run("run-older", { task: "session_reflector", status: "succeeded" }), ]), }); renderRunHistory(); - await screen.findByText("newest_run"); + await screen.findByText("skill_writer"); const rows = screen.getAllByRole("button"); - expect(rows[0]?.textContent).toContain("newest_run"); - expect(rows[1]?.textContent).toContain("older_run"); + expect(rows[0]?.textContent).toContain("skill_writer"); + expect(rows[1]?.textContent).toContain("session_reflector"); }); it("fetches artifacts only when a run is opened, and prints the daemon integrity verdict", async () => { @@ -120,7 +120,6 @@ describe("RunHistory", () => { validation_report: { decision: "automatic" }, }, }, - error: "", }, }); renderRunHistory(); @@ -149,12 +148,12 @@ describe("RunHistory", () => { it("says when an opened run recorded no artifacts instead of issuing a read", async () => { const fetchMock = stubRuns({ runs: runsBody([ - run("run-1", { task: "session_reflection", status: "completed" }), + run("run-1", { task: "session_reflector", status: "succeeded" }), ]), }); renderRunHistory(); await userEvent.click( - await screen.findByRole("button", { name: /session_reflection/ }), + await screen.findByRole("button", { name: /session_reflector/ }), ); expect(await screen.findByText(/recorded no artifacts/i)).toBeTruthy(); @@ -203,7 +202,6 @@ function runsBody(rows: unknown[]) { has_more: false, malformed_row_count: 0, completeness: "known", - error: "", }; } @@ -219,6 +217,7 @@ function run( }, ) { return { + schema_version: 2, run_id: id, task: options.task, task_key: options.task, @@ -245,6 +244,7 @@ function artifactsBody(runId: string, integrity: string) { run_id: runId, artifacts: [ { + schema_version: 1, kind: "traces", path: `runs/${runId}/traces.json`, sha256: "a".repeat(64), @@ -259,7 +259,6 @@ function artifactsBody(runId: string, integrity: string) { integrity_status: integrity, }, count: 1, - error: "", }; } diff --git a/dashboard/src/workspaces/automations/RunHistory.tsx b/dashboard/src/workspaces/automations/RunHistory.tsx index 83d31ca989..a9b1354f30 100644 --- a/dashboard/src/workspaces/automations/RunHistory.tsx +++ b/dashboard/src/workspaces/automations/RunHistory.tsx @@ -3,15 +3,8 @@ import { ChevronDown, ChevronRight } from "lucide-react"; import { PayloadBoundary } from "../../ui/ReadSection.tsx"; import { relativeAge } from "../../ui/time.ts"; import { cn } from "../../ui/cn"; -import { - automationRunsReading, - useAutomationRunArtifactPayload, - useAutomationRunArtifacts, - useAutomationRuns, - type RunArtifactRow, - type RunArtifactsPayload, - type RunRow, -} from "../../data/query/automation.ts"; +import type { AutomationRunArtifact, AutomationRunArtifactsPayloadV1, AutomationRunRowV1 } from "../../contracts/generated.ts"; +import { automationRunsReading, useAutomationRunArtifactPayload, useAutomationRunArtifacts, useAutomationRuns } from "../../data/query/automation.ts"; import { artifactPayloadBelongsTo, missingArtifactKinds } from "./ledger.ts"; /** @@ -41,13 +34,13 @@ export function RunHistory() { // The ledger route answers an absent ledger file with an empty list, // which is the truthful reading: no run has ever been recorded here. return reading.complete ? ( -

    +

    no automation runs are recorded in this project's ledger

    ) : (

    Showing a partial list: {reading.reason}.

    @@ -58,7 +51,7 @@ export function RunHistory() { {reading.complete ? null : (

    Showing a partial list: {reading.reason}.

    @@ -77,7 +70,7 @@ export function RunHistory() { /** One run: a disclosure row whose panel holds the artifact reading. The * artifact request is issued only when the row first opens. */ -function RunLine({ run }: { run: RunRow }) { +function RunLine({ run }: { run: AutomationRunRowV1 }) { const [open, setOpen] = useState(false); const started = Number(run.started_at); const age = Number.isFinite(started) @@ -107,23 +100,23 @@ function RunLine({ run }: { run: RunRow }) { {run.task} {run.status} - + {run.accepted_count} accepted · {run.rejected_count} rejected {/* The record's timestamp verbatim when it does not parse as epoch * seconds: a raw string is a truthful oddity, a blank is a lie. */} - + {age ?? run.started_at} {run.error ? ( -

    +

    {run.error}

    ) : null} @@ -147,7 +140,7 @@ function RunArtifacts({ return (
    {recordedKinds.length === 0 ? ( -

    +

    this run recorded no artifacts in its ledger entry

    ) : ( @@ -163,7 +156,7 @@ function RunArtifacts({ ); } -function ArtifactList({ data }: { data: RunArtifactsPayload }) { +function ArtifactList({ data }: { data: AutomationRunArtifactsPayloadV1 }) { const chain = data.artifact_chain; const missing = missingArtifactKinds(chain); return ( @@ -172,7 +165,7 @@ function ArtifactList({ data }: { data: RunArtifactsPayload }) { * matches the published chain. Its words, not a green summary. */}

    setOpen((value) => !value)} className="flex flex-wrap items-baseline gap-x-2 gap-y-0.5 text-left" > - + {open ? "Hide" : "Inspect"} {artifact.kind.replaceAll("_", " ")} {artifact.summary ? ( - + {artifact.summary} ) : null} - + {artifact.sha256.slice(0, 12)} @@ -223,12 +216,12 @@ function ArtifactLine({ artifactPayloadBelongsTo(data, runId, artifact) ? ( 

                     {JSON.stringify(data.payload, null, 2)}
    ) : ( -

    +

    the artifact payload does not belong to this run and kind

    ) diff --git a/dashboard/src/workspaces/automations/RunInspector.tsx b/dashboard/src/workspaces/automations/RunInspector.tsx index 79ac981be4..4f9061e608 100644 --- a/dashboard/src/workspaces/automations/RunInspector.tsx +++ b/dashboard/src/workspaces/automations/RunInspector.tsx @@ -1,15 +1,15 @@ import { useState, type ReactNode } from 'react'; -import type { AutomationTaskStatusV1 } from '../../contracts/generated.ts'; -import { - useAutomationRunArtifactPayload, - useAutomationRunArtifacts, - type AutomaticFactReceipt, - type JobRow, - type RunArtifactRow, - type RunArtifactsPayload, - type RunRow, -} from '../../data/query/automation.ts'; +import type { + AutomaticFactReceipt, + AutomationJob, + AutomationRunArtifact, + AutomationRunArtifactsPayloadV1, + AutomationRunLedgerRecord, + AutomationRunRowV1, + AutomationTaskStatusV1, +} from '../../contracts/generated.ts'; +import { useAutomationRunArtifactPayload, useAutomationRunArtifacts } from '../../data/query/automation.ts'; import { cn } from '../../ui/cn'; import { Corners } from '../../ui/instrument.tsx'; import { PayloadBoundary } from '../../ui/ReadSection.tsx'; @@ -25,7 +25,6 @@ import { jobTaskKey, latestRunForKey, missingArtifactKinds, - readLastSchedulerRun, receiptStateTone, runStatusTone, runTiming, @@ -57,9 +56,9 @@ export function RunInspector({ pinned: boolean; /** Each source is null while its read is blocked, so the inspector can say * "not readable" rather than "not found". */ - runs: readonly RunRow[] | null; + runs: readonly AutomationRunRowV1[] | null; tasks: readonly AutomationTaskStatusV1[] | null; - jobs: readonly JobRow[] | null; + jobs: readonly AutomationJob[] | null; receipts: readonly AutomaticFactReceipt[] | null; receiptsByRun: ReadonlyMap | null; onSelect: (next: Inspected) => void; @@ -93,7 +92,7 @@ export function RunInspector({ /> )}
    -